mirror of
https://github.com/lancedb/lancedb.git
synced 2026-09-08 22:39:05 +00:00
Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d49ac46dc5 |
+1
-1
@@ -1,5 +1,5 @@
|
||||
[tool.bumpversion]
|
||||
current_version = "0.39.0-beta.6"
|
||||
current_version = "0.39.0-beta.1"
|
||||
parse = """(?x)
|
||||
(?P<major>0|[1-9]\\d*)\\.
|
||||
(?P<minor>0|[1-9]\\d*)\\.
|
||||
|
||||
@@ -232,10 +232,7 @@ jobs:
|
||||
ALL_FEATURES=`cargo metadata --format-version=1 --no-deps \
|
||||
| jq -r '.packages[] | .features | keys | .[]' \
|
||||
| grep -v s3-test | sort | uniq | paste -s -d "," -`
|
||||
# Run doctests before test binaries fill the runner disk. Examples are
|
||||
# already built by the Linux job, so avoid retaining them here.
|
||||
cargo test --profile ci --features $ALL_FEATURES --locked --doc
|
||||
cargo test --profile ci --features $ALL_FEATURES --locked --lib --tests
|
||||
cargo test --profile ci --features $ALL_FEATURES --locked
|
||||
|
||||
windows:
|
||||
strategy:
|
||||
|
||||
Generated
+50
-137
@@ -332,34 +332,6 @@ dependencies = [
|
||||
"num-traits",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "arrow-flight"
|
||||
version = "58.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b2dbe34824c639e43136af8f106992792ab456540d54b880bc320a3192502d2e"
|
||||
dependencies = [
|
||||
"arrow-arith",
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
"arrow-cast",
|
||||
"arrow-data",
|
||||
"arrow-ipc",
|
||||
"arrow-ord",
|
||||
"arrow-row",
|
||||
"arrow-schema",
|
||||
"arrow-select",
|
||||
"arrow-string",
|
||||
"base64 0.22.1",
|
||||
"bytes",
|
||||
"futures",
|
||||
"once_cell",
|
||||
"paste",
|
||||
"prost",
|
||||
"prost-types",
|
||||
"tonic",
|
||||
"tonic-prost",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "arrow-ipc"
|
||||
version = "58.4.0"
|
||||
@@ -1157,7 +1129,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"axum-core 0.4.5",
|
||||
"axum-core",
|
||||
"bytes",
|
||||
"futures-util",
|
||||
"http 1.5.0",
|
||||
@@ -1166,7 +1138,7 @@ dependencies = [
|
||||
"hyper 1.9.0",
|
||||
"hyper-util",
|
||||
"itoa",
|
||||
"matchit 0.7.3",
|
||||
"matchit",
|
||||
"memchr",
|
||||
"mime",
|
||||
"percent-encoding",
|
||||
@@ -1184,31 +1156,6 @@ dependencies = [
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "axum"
|
||||
version = "0.8.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90"
|
||||
dependencies = [
|
||||
"axum-core 0.5.6",
|
||||
"bytes",
|
||||
"futures-util",
|
||||
"http 1.5.0",
|
||||
"http-body 1.1.0",
|
||||
"http-body-util",
|
||||
"itoa",
|
||||
"matchit 0.8.4",
|
||||
"memchr",
|
||||
"mime",
|
||||
"percent-encoding",
|
||||
"pin-project-lite",
|
||||
"serde_core",
|
||||
"sync_wrapper",
|
||||
"tower",
|
||||
"tower-layer",
|
||||
"tower-service",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "axum-core"
|
||||
version = "0.4.5"
|
||||
@@ -1230,24 +1177,6 @@ dependencies = [
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "axum-core"
|
||||
version = "0.5.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"futures-core",
|
||||
"http 1.5.0",
|
||||
"http-body 1.1.0",
|
||||
"http-body-util",
|
||||
"mime",
|
||||
"pin-project-lite",
|
||||
"sync_wrapper",
|
||||
"tower-layer",
|
||||
"tower-service",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "backoff"
|
||||
version = "0.4.0"
|
||||
@@ -3526,8 +3455,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
|
||||
|
||||
[[package]]
|
||||
name = "fsst"
|
||||
version = "12.0.0-beta.15"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
|
||||
version = "12.0.0-beta.11"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"rand 0.9.5",
|
||||
@@ -4886,8 +4815,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"
|
||||
|
||||
[[package]]
|
||||
name = "lance"
|
||||
version = "12.0.0-beta.15"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
|
||||
version = "12.0.0-beta.11"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"arrow",
|
||||
@@ -4959,8 +4888,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-arrow"
|
||||
version = "12.0.0-beta.15"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
|
||||
version = "12.0.0-beta.11"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -4982,7 +4911,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "lance-arrow-scalar"
|
||||
version = "58.0.0"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -4996,7 +4925,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "lance-arrow-stats"
|
||||
version = "58.0.0"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-schema",
|
||||
@@ -5005,8 +4934,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-bitpacking"
|
||||
version = "12.0.0-beta.15"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
|
||||
version = "12.0.0-beta.11"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
|
||||
dependencies = [
|
||||
"arrayref",
|
||||
"crunchy",
|
||||
@@ -5016,8 +4945,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-core"
|
||||
version = "12.0.0-beta.15"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
|
||||
version = "12.0.0-beta.11"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -5054,8 +4983,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-datafusion"
|
||||
version = "12.0.0-beta.15"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
|
||||
version = "12.0.0-beta.11"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -5085,8 +5014,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-datagen"
|
||||
version = "12.0.0-beta.15"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
|
||||
version = "12.0.0-beta.11"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -5103,8 +5032,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-derive"
|
||||
version = "12.0.0-beta.15"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
|
||||
version = "12.0.0-beta.11"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -5113,8 +5042,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-encoding"
|
||||
version = "12.0.0-beta.15"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
|
||||
version = "12.0.0-beta.11"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
|
||||
dependencies = [
|
||||
"arrow-arith",
|
||||
"arrow-array",
|
||||
@@ -5147,8 +5076,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-file"
|
||||
version = "12.0.0-beta.15"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
|
||||
version = "12.0.0-beta.11"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
|
||||
dependencies = [
|
||||
"arrow-arith",
|
||||
"arrow-array",
|
||||
@@ -5179,8 +5108,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-index"
|
||||
version = "12.0.0-beta.15"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
|
||||
version = "12.0.0-beta.11"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"arrow",
|
||||
@@ -5244,8 +5173,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-index-core"
|
||||
version = "12.0.0-beta.15"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
|
||||
version = "12.0.0-beta.11"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-schema",
|
||||
@@ -5267,8 +5196,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-io"
|
||||
version = "12.0.0-beta.15"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
|
||||
version = "12.0.0-beta.11"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -5308,8 +5237,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-linalg"
|
||||
version = "12.0.0-beta.15"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
|
||||
version = "12.0.0-beta.11"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-schema",
|
||||
@@ -5323,29 +5252,27 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-namespace"
|
||||
version = "12.0.0-beta.15"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
|
||||
version = "12.0.0-beta.11"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"async-trait",
|
||||
"bytes",
|
||||
"lance-core",
|
||||
"lance-namespace-reqwest-client",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"snafu 0.9.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lance-namespace-impls"
|
||||
version = "12.0.0-beta.15"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
|
||||
version = "12.0.0-beta.11"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-ipc",
|
||||
"arrow-schema",
|
||||
"async-trait",
|
||||
"axum 0.7.9",
|
||||
"axum",
|
||||
"base64 0.22.1",
|
||||
"bytes",
|
||||
"chrono",
|
||||
@@ -5378,9 +5305,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-namespace-reqwest-client"
|
||||
version = "0.12.0"
|
||||
version = "0.11.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d8d23e54b1634d5bbb434f8dd33dc3c05f6e58d876a9a27b3b4aef58ddbe11af"
|
||||
checksum = "1d06b1fbb5d41f93bc652b61e2872af92e8a6c5f6b4ce8839a8ecfa05365d359"
|
||||
dependencies = [
|
||||
"reqwest 0.12.28",
|
||||
"serde",
|
||||
@@ -5392,8 +5319,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-select"
|
||||
version = "12.0.0-beta.15"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
|
||||
version = "12.0.0-beta.11"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -5407,8 +5334,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-table"
|
||||
version = "12.0.0-beta.15"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
|
||||
version = "12.0.0-beta.11"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -5448,8 +5375,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-testing"
|
||||
version = "12.0.0-beta.15"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
|
||||
version = "12.0.0-beta.11"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-schema",
|
||||
@@ -5462,8 +5389,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-tokenizer"
|
||||
version = "12.0.0-beta.15"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
|
||||
version = "12.0.0-beta.11"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
|
||||
dependencies = [
|
||||
"frostem",
|
||||
"icu_segmenter",
|
||||
@@ -5476,7 +5403,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lancedb"
|
||||
version = "0.39.0-beta.5"
|
||||
version = "0.39.0-beta.0"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"anyhow",
|
||||
@@ -5485,7 +5412,6 @@ dependencies = [
|
||||
"arrow-buffer",
|
||||
"arrow-cast",
|
||||
"arrow-data",
|
||||
"arrow-flight",
|
||||
"arrow-ipc",
|
||||
"arrow-ord",
|
||||
"arrow-schema",
|
||||
@@ -5541,7 +5467,6 @@ dependencies = [
|
||||
"polars",
|
||||
"polars-arrow",
|
||||
"pprof 0.14.1",
|
||||
"prost",
|
||||
"rand 0.9.5",
|
||||
"random_word",
|
||||
"regex",
|
||||
@@ -5558,7 +5483,6 @@ dependencies = [
|
||||
"test-log",
|
||||
"tokenizers",
|
||||
"tokio",
|
||||
"tonic",
|
||||
"url",
|
||||
"urlencoding",
|
||||
"uuid",
|
||||
@@ -5567,7 +5491,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lancedb-nodejs"
|
||||
version = "0.39.0-beta.5"
|
||||
version = "0.39.0-beta.0"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -5592,7 +5516,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lancedb-python"
|
||||
version = "0.39.0-beta.5"
|
||||
version = "0.39.0-beta.0"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"async-trait",
|
||||
@@ -5616,7 +5540,6 @@ dependencies = [
|
||||
"serde_json",
|
||||
"snafu 0.8.9",
|
||||
"tokio",
|
||||
"uuid",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -5937,12 +5860,6 @@ version = "0.7.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94"
|
||||
|
||||
[[package]]
|
||||
name = "matchit"
|
||||
version = "0.8.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3"
|
||||
|
||||
[[package]]
|
||||
name = "matrixmultiply"
|
||||
version = "0.3.10"
|
||||
@@ -7818,7 +7735,6 @@ dependencies = [
|
||||
"pyo3-build-config",
|
||||
"pyo3-ffi",
|
||||
"pyo3-macros",
|
||||
"uuid",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -10171,7 +10087,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ac2a5518c70fa84342385732db33fb3f44bc4cc748936eb5833d2df34d6445ef"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"axum 0.8.9",
|
||||
"base64 0.22.1",
|
||||
"bytes",
|
||||
"h2 0.4.16",
|
||||
@@ -10183,11 +10098,9 @@ dependencies = [
|
||||
"hyper-util",
|
||||
"percent-encoding",
|
||||
"pin-project",
|
||||
"rustls-native-certs",
|
||||
"socket2 0.6.3",
|
||||
"sync_wrapper",
|
||||
"tokio",
|
||||
"tokio-rustls 0.26.4",
|
||||
"tokio-stream",
|
||||
"tower",
|
||||
"tower-layer",
|
||||
|
||||
+15
-17
@@ -13,20 +13,20 @@ categories = ["database-implementations"]
|
||||
rust-version = "1.91.0"
|
||||
|
||||
[workspace.dependencies]
|
||||
lance = { "version" = "=12.0.0-beta.15", default-features = false, "tag" = "v12.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-core = { "version" = "=12.0.0-beta.15", "tag" = "v12.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datagen = { "version" = "=12.0.0-beta.15", "tag" = "v12.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-file = { "version" = "=12.0.0-beta.15", "tag" = "v12.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-io = { "version" = "=12.0.0-beta.15", default-features = false, "tag" = "v12.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-index = { "version" = "=12.0.0-beta.15", "tag" = "v12.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-linalg = { "version" = "=12.0.0-beta.15", "tag" = "v12.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace = { "version" = "=12.0.0-beta.15", "tag" = "v12.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace-impls = { "version" = "=12.0.0-beta.15", default-features = false, "tag" = "v12.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-table = { "version" = "=12.0.0-beta.15", "tag" = "v12.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-testing = { "version" = "=12.0.0-beta.15", "tag" = "v12.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datafusion = { "version" = "=12.0.0-beta.15", "tag" = "v12.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-encoding = { "version" = "=12.0.0-beta.15", "tag" = "v12.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-arrow = { "version" = "=12.0.0-beta.15", "tag" = "v12.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance = { "version" = "=12.0.0-beta.11", default-features = false, "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-core = { "version" = "=12.0.0-beta.11", "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datagen = { "version" = "=12.0.0-beta.11", "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-file = { "version" = "=12.0.0-beta.11", "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-io = { "version" = "=12.0.0-beta.11", default-features = false, "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-index = { "version" = "=12.0.0-beta.11", "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-linalg = { "version" = "=12.0.0-beta.11", "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace = { "version" = "=12.0.0-beta.11", "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace-impls = { "version" = "=12.0.0-beta.11", default-features = false, "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-table = { "version" = "=12.0.0-beta.11", "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-testing = { "version" = "=12.0.0-beta.11", "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datafusion = { "version" = "=12.0.0-beta.11", "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-encoding = { "version" = "=12.0.0-beta.11", "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-arrow = { "version" = "=12.0.0-beta.11", "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lancedb = { path = "rust/lancedb", default-features = false }
|
||||
ahash = "0.8"
|
||||
# Note that this one does not include pyarrow
|
||||
@@ -39,7 +39,6 @@ arrow-ord = "58.0.0"
|
||||
arrow-schema = "58.0.0"
|
||||
arrow-select = "58.0.0"
|
||||
arrow-cast = "58.0.0"
|
||||
arrow-flight = { version = "58.0.0", features = ["flight-sql-experimental"] }
|
||||
async-trait = "0"
|
||||
bytes = "1"
|
||||
datafusion = { version = "54.0.0", default-features = false }
|
||||
@@ -72,8 +71,7 @@ serde = "1"
|
||||
serde_json = "1"
|
||||
tempfile = "3.5.0"
|
||||
tokio = { version = "1.23", features = ["rt-multi-thread", "sync"] }
|
||||
tonic = { version = "0.14", features = ["tls-native-roots", "tls-ring"] }
|
||||
uuid = { version = "1.7.0", features = ["v4", "v7"] }
|
||||
uuid = { version = "1.7.0", features = ["v4"] }
|
||||
chrono = { version = "0.4", default-features = false, features = ["clock"] }
|
||||
|
||||
[profile.ci]
|
||||
|
||||
@@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`:
|
||||
<dependency>
|
||||
<groupId>com.lancedb</groupId>
|
||||
<artifactId>lancedb-core</artifactId>
|
||||
<version>0.39.0-beta.6</version>
|
||||
<version>0.39.0-beta.1</version>
|
||||
</dependency>
|
||||
```
|
||||
|
||||
|
||||
@@ -448,6 +448,26 @@ on the returned job to know when cleanup has finished.
|
||||
|
||||
***
|
||||
|
||||
### getJob()
|
||||
|
||||
```ts
|
||||
abstract getJob(jobId): Promise<null | JobDescription>
|
||||
```
|
||||
|
||||
Describe a single server-side job by id.
|
||||
|
||||
Resolves to `null` when the server has no such job.
|
||||
|
||||
#### Parameters
|
||||
|
||||
* **jobId**: `string`
|
||||
|
||||
#### Returns
|
||||
|
||||
`Promise`<`null` \| [`JobDescription`](../interfaces/JobDescription.md)>
|
||||
|
||||
***
|
||||
|
||||
### isOpen()
|
||||
|
||||
```ts
|
||||
@@ -462,6 +482,48 @@ Return true if the connection has not been closed
|
||||
|
||||
***
|
||||
|
||||
### job()
|
||||
|
||||
```ts
|
||||
abstract job(jobId): Job
|
||||
```
|
||||
|
||||
A [Job](Job.md) handle for a server-side job by id.
|
||||
|
||||
The handle is constructed without a server round trip; an unknown id
|
||||
surfaces when the handle is used. Dropping the handle has no effect on
|
||||
the job itself.
|
||||
|
||||
#### Parameters
|
||||
|
||||
* **jobId**: `string`
|
||||
|
||||
#### Returns
|
||||
|
||||
[`Job`](Job.md)
|
||||
|
||||
***
|
||||
|
||||
### jobHistory()
|
||||
|
||||
```ts
|
||||
abstract jobHistory(jobId?): Promise<Table<any>>
|
||||
```
|
||||
|
||||
The lifecycle event history of a server-side job, as an Arrow table.
|
||||
|
||||
Lists history across all jobs when `jobId` is omitted.
|
||||
|
||||
#### Parameters
|
||||
|
||||
* **jobId?**: `string`
|
||||
|
||||
#### Returns
|
||||
|
||||
`Promise`<`Table`<`any`>>
|
||||
|
||||
***
|
||||
|
||||
### listJobs()
|
||||
|
||||
```ts
|
||||
@@ -586,30 +648,6 @@ A page of table names and an
|
||||
|
||||
***
|
||||
|
||||
### openJob()
|
||||
|
||||
```ts
|
||||
abstract openJob(jobId): Promise<Job>
|
||||
```
|
||||
|
||||
Open a server-side job by id, returning a handle with its record already
|
||||
populated. Rejects when the server has no such job, the way
|
||||
[Connection.openTable](Connection.md#opentable) does for a missing table.
|
||||
|
||||
The returned [Job](Job.md) answers for its own state, specification,
|
||||
result, failure and event history, so there is no separate
|
||||
connection-level call for any of them.
|
||||
|
||||
#### Parameters
|
||||
|
||||
* **jobId**: `string`
|
||||
|
||||
#### Returns
|
||||
|
||||
`Promise`<[`Job`](Job.md)>
|
||||
|
||||
***
|
||||
|
||||
### openMaterializedView()
|
||||
|
||||
```ts
|
||||
|
||||
+16
-163
@@ -8,116 +8,28 @@
|
||||
|
||||
A handle to an operation that may still be running.
|
||||
|
||||
The operation may already be complete when the handle is created.
|
||||
## Constructors
|
||||
|
||||
The detail getters read what the handle last observed. Submitting an
|
||||
operation returns only a job id, so populating them eagerly would cost an
|
||||
extra round trip on every call:
|
||||
### new Job()
|
||||
|
||||
- [Job.refresh](Job.md#refresh) and [Job.status](Job.md#status) fetch the whole record.
|
||||
- [Job.wait](Job.md#wait) records the terminal state it establishes, but not the
|
||||
rest of the record.
|
||||
- Everything is null until one of those runs.
|
||||
```ts
|
||||
new Job(): Job
|
||||
```
|
||||
|
||||
#### Returns
|
||||
|
||||
[`Job`](Job.md)
|
||||
|
||||
## Accessors
|
||||
|
||||
### creationMs
|
||||
|
||||
```ts
|
||||
get creationMs(): null | number
|
||||
```
|
||||
|
||||
When the job was created, in milliseconds since the epoch.
|
||||
|
||||
#### Returns
|
||||
|
||||
`null` \| `number`
|
||||
|
||||
***
|
||||
|
||||
### failure
|
||||
|
||||
```ts
|
||||
get failure(): null | JobFailureInfo
|
||||
```
|
||||
|
||||
Why the job failed, when it failed and the server reports a reason.
|
||||
|
||||
#### Returns
|
||||
|
||||
`null` \| [`JobFailureInfo`](../interfaces/JobFailureInfo.md)
|
||||
|
||||
***
|
||||
|
||||
### id
|
||||
|
||||
```ts
|
||||
get id(): null | string
|
||||
```
|
||||
|
||||
Identifies the operation on the server that is running it.
|
||||
|
||||
Operations that run in this process have no server id. The value is
|
||||
opaque: parsing it or storing it to resume the job later is not supported.
|
||||
|
||||
#### Returns
|
||||
|
||||
`null` \| `string`
|
||||
|
||||
***
|
||||
|
||||
### jobType
|
||||
|
||||
```ts
|
||||
get jobType(): null | string
|
||||
```
|
||||
|
||||
The job's type, as the server names it. Null for an in-process job, which
|
||||
has no server-side record.
|
||||
|
||||
#### Returns
|
||||
|
||||
`null` \| `string`
|
||||
|
||||
***
|
||||
|
||||
### result
|
||||
|
||||
```ts
|
||||
get result(): any
|
||||
```
|
||||
|
||||
The job-type-specific terminal result. Null until the job succeeds, so a
|
||||
job that never terminates reports its progress through [Job.events](Job.md#events)
|
||||
instead.
|
||||
|
||||
#### Returns
|
||||
|
||||
`any`
|
||||
|
||||
***
|
||||
|
||||
### spec
|
||||
|
||||
```ts
|
||||
get spec(): any
|
||||
```
|
||||
|
||||
The job-type-specific specification it was submitted with.
|
||||
|
||||
#### Returns
|
||||
|
||||
`any`
|
||||
|
||||
***
|
||||
|
||||
### state
|
||||
|
||||
```ts
|
||||
get state(): null | string
|
||||
```
|
||||
|
||||
The last observed lifecycle state, without contacting the backend.
|
||||
Identifies the operation on the server that is running it. Operations
|
||||
that run in this process have no server id. The value is opaque.
|
||||
|
||||
#### Returns
|
||||
|
||||
@@ -139,61 +51,18 @@ Request cancellation. Cancelling a finished operation is a no-op.
|
||||
|
||||
***
|
||||
|
||||
### events()
|
||||
|
||||
```ts
|
||||
events(options?): Promise<Table<any>>
|
||||
```
|
||||
|
||||
This job's recorded lifecycle events.
|
||||
|
||||
Where the getters above report a terminal result only once the job reaches
|
||||
one, events are written as the job runs and outlive the workers that
|
||||
produced them. A distributed job records a `claim`/`claim_complete` pair
|
||||
per unit of work, each carrying `rows_processed`, so a job that never
|
||||
finishes still accounts for what it did.
|
||||
|
||||
The server caps results at 1000 rows by default and 10,000 at most, and
|
||||
truncates without saying so, so pass `limit` for a job that emits an event
|
||||
per fragment. `filter` is a SQL-like expression over the `state`,
|
||||
`updated_by`, `emitted_from`, `emitted_by`, and `claim_entity` columns.
|
||||
|
||||
#### Parameters
|
||||
|
||||
* **options?**: [`JobEventsOptions`](../interfaces/JobEventsOptions.md)
|
||||
|
||||
#### Returns
|
||||
|
||||
`Promise`<`Table`<`any`>>
|
||||
|
||||
***
|
||||
|
||||
### refresh()
|
||||
|
||||
```ts
|
||||
refresh(): Promise<void>
|
||||
```
|
||||
|
||||
Ask the backend for this job's current state, and for a server-side job
|
||||
its full record, then cache it for the getters above.
|
||||
|
||||
#### Returns
|
||||
|
||||
`Promise`<`void`>
|
||||
|
||||
***
|
||||
|
||||
### status()
|
||||
|
||||
```ts
|
||||
status(): Promise<string>
|
||||
```
|
||||
|
||||
The operation's current lifecycle state: "running", "finished", "failed",
|
||||
or "cancelled".
|
||||
The operation's current lifecycle state: "running", "finished",
|
||||
"failed", or "cancelled".
|
||||
|
||||
A point snapshot; unlike [Job.wait](Job.md#wait) it does not block or reject on a
|
||||
terminal failure state. Also refreshes the getters above.
|
||||
A point snapshot; unlike [Job.wait](Job.md#wait) it does not block or reject
|
||||
on a terminal failure state. States a newer server reports that this
|
||||
client version does not know pass through as-is.
|
||||
|
||||
#### Returns
|
||||
|
||||
@@ -201,22 +70,6 @@ terminal failure state. Also refreshes the getters above.
|
||||
|
||||
***
|
||||
|
||||
### toString()
|
||||
|
||||
```ts
|
||||
toString(): string
|
||||
```
|
||||
|
||||
Every field the handle currently knows, one per line, with the JSON
|
||||
payloads indented -- a refresh job's spec and result are the point of
|
||||
printing it.
|
||||
|
||||
#### Returns
|
||||
|
||||
`string`
|
||||
|
||||
***
|
||||
|
||||
### wait()
|
||||
|
||||
```ts
|
||||
|
||||
@@ -96,7 +96,7 @@
|
||||
- [IvfFlatOptions](interfaces/IvfFlatOptions.md)
|
||||
- [IvfPqOptions](interfaces/IvfPqOptions.md)
|
||||
- [IvfRqOptions](interfaces/IvfRqOptions.md)
|
||||
- [JobEventsOptions](interfaces/JobEventsOptions.md)
|
||||
- [JobDescription](interfaces/JobDescription.md)
|
||||
- [JobFailureInfo](interfaces/JobFailureInfo.md)
|
||||
- [JobInfo](interfaces/JobInfo.md)
|
||||
- [ListNamespacesOptions](interfaces/ListNamespacesOptions.md)
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
[**@lancedb/lancedb**](../README.md) • **Docs**
|
||||
|
||||
***
|
||||
|
||||
[@lancedb/lancedb](../globals.md) / JobDescription
|
||||
|
||||
# Interface: JobDescription
|
||||
|
||||
A described job from `Connection.getJob`.
|
||||
|
||||
## Properties
|
||||
|
||||
### creationMs
|
||||
|
||||
```ts
|
||||
creationMs: number;
|
||||
```
|
||||
|
||||
When the job was created, in milliseconds since the epoch.
|
||||
|
||||
***
|
||||
|
||||
### failure?
|
||||
|
||||
```ts
|
||||
optional failure: JobFailureInfo;
|
||||
```
|
||||
|
||||
Why the job failed, when the job is failed and the server reports a
|
||||
reason.
|
||||
|
||||
***
|
||||
|
||||
### jobId
|
||||
|
||||
```ts
|
||||
jobId: string;
|
||||
```
|
||||
|
||||
***
|
||||
|
||||
### jobType
|
||||
|
||||
```ts
|
||||
jobType: string;
|
||||
```
|
||||
|
||||
***
|
||||
|
||||
### specJson?
|
||||
|
||||
```ts
|
||||
optional specJson: string;
|
||||
```
|
||||
|
||||
The job-type-specific specification as a JSON string, when present.
|
||||
|
||||
***
|
||||
|
||||
### state
|
||||
|
||||
```ts
|
||||
state: string;
|
||||
```
|
||||
|
||||
Lifecycle state: "running", "finished", "failed", or "cancelled".
|
||||
@@ -1,29 +0,0 @@
|
||||
[**@lancedb/lancedb**](../README.md) • **Docs**
|
||||
|
||||
***
|
||||
|
||||
[@lancedb/lancedb](../globals.md) / JobEventsOptions
|
||||
|
||||
# Interface: JobEventsOptions
|
||||
|
||||
Which of a job's events [Job.events](../classes/Job.md#events) returns.
|
||||
|
||||
## Properties
|
||||
|
||||
### filter?
|
||||
|
||||
```ts
|
||||
optional filter: string;
|
||||
```
|
||||
|
||||
SQL-like filter over the event columns.
|
||||
|
||||
***
|
||||
|
||||
### limit?
|
||||
|
||||
```ts
|
||||
optional limit: number;
|
||||
```
|
||||
|
||||
Maximum event rows to return, up to the server maximum of 10,000.
|
||||
@@ -26,7 +26,7 @@ When the job was created, in milliseconds since the epoch.
|
||||
jobId: string;
|
||||
```
|
||||
|
||||
The job id -- what `Connection.openJob` and `Connection.cancelJob`
|
||||
The job id -- what `Connection.getJob` and `Connection.cancelJob`
|
||||
accept.
|
||||
|
||||
***
|
||||
|
||||
@@ -28,59 +28,6 @@ is also an [asynchronous API client](#connections-asynchronous).
|
||||
|
||||
::: lancedb.Session
|
||||
|
||||
## Remote SQL
|
||||
|
||||
Submit SQL against a remote LanceDB database through the connection.
|
||||
The connected database and `default_namespace_path=["public"]` are used for
|
||||
unqualified tables. Fully qualified references can still query other databases
|
||||
and namespaces available to the same deployment. `execute_query` returns a
|
||||
reader as soon as its initial result stream is available. `execute_query_async`
|
||||
returns a query handle immediately; use it to inspect progress, open a reader,
|
||||
or cancel the query. The SQL client is initialized by the first query and
|
||||
retained for the lifetime of the remote connection. Query ids are random,
|
||||
connection-scoped references rather than encoded SQL or durable resume tokens:
|
||||
|
||||
```python
|
||||
import lancedb
|
||||
|
||||
db = lancedb.connect(
|
||||
"db://analytics",
|
||||
api_key="ldb_...",
|
||||
host_override="https://api.example.com",
|
||||
sql_host_override="grpc+tls://sql.example.com:10026",
|
||||
)
|
||||
reader = db.execute_query(
|
||||
"""
|
||||
SELECT events.id, accounts.name
|
||||
FROM analytics.public.events AS events
|
||||
JOIN users.public.accounts AS accounts ON events.user_id = accounts.id
|
||||
""",
|
||||
default_namespace_path=["public"],
|
||||
)
|
||||
for batch in reader:
|
||||
print(batch.num_rows)
|
||||
|
||||
query = db.execute_query_async("SELECT * FROM events")
|
||||
print(query.id)
|
||||
print(query.describe().status)
|
||||
for batch in query.reader():
|
||||
print(batch.num_rows)
|
||||
|
||||
# The async connection exposes the same lifecycle without blocking:
|
||||
# async_db = await lancedb.connect_async(
|
||||
# "db://analytics",
|
||||
# api_key="ldb_...",
|
||||
# host_override="https://api.example.com",
|
||||
# sql_host_override="grpc+tls://sql.example.com:10026",
|
||||
# )
|
||||
# reader = await async_db.execute_query("SELECT * FROM events")
|
||||
# query = await async_db.execute_query_async("SELECT * FROM events")
|
||||
# description = await async_db.describe_query(query.id)
|
||||
# async for batch in await query.reader():
|
||||
# print(batch.num_rows)
|
||||
# await query.cancel()
|
||||
```
|
||||
|
||||
## Namespaces (Synchronous)
|
||||
|
||||
A namespace-backed connection resolves tables through a
|
||||
@@ -147,8 +94,6 @@ listing a storage directory.
|
||||
|
||||
::: lancedb.functions.OutputMapping
|
||||
|
||||
::: lancedb.functions.AssignmentMapping
|
||||
|
||||
::: lancedb.functions.FunctionBinding
|
||||
|
||||
::: lancedb.functions.RefreshColumnResult
|
||||
@@ -157,18 +102,6 @@ listing a storage directory.
|
||||
|
||||
::: lancedb.job.AsyncJob
|
||||
|
||||
::: lancedb.job.JobInfo
|
||||
|
||||
::: lancedb.job.JobDescription
|
||||
|
||||
::: lancedb.job.JobFailureInfo
|
||||
|
||||
::: lancedb.sql.Query
|
||||
|
||||
::: lancedb.sql.AsyncQuery
|
||||
|
||||
::: lancedb.sql.QueryDescription
|
||||
|
||||
## Materialized Views (Synchronous)
|
||||
|
||||
::: lancedb.materialized_view.MaterializedView
|
||||
@@ -316,12 +249,6 @@ still work. Queries return descriptors. Call
|
||||
|
||||
::: lancedb.exceptions.MissingColumnError
|
||||
|
||||
::: lancedb.exceptions.JobNotFoundError
|
||||
|
||||
::: lancedb.exceptions.JobFailedError
|
||||
|
||||
::: lancedb.exceptions.JobCancelledError
|
||||
|
||||
## Integrations
|
||||
|
||||
## Pydantic
|
||||
|
||||
@@ -8,7 +8,7 @@
|
||||
<parent>
|
||||
<groupId>com.lancedb</groupId>
|
||||
<artifactId>lancedb-parent</artifactId>
|
||||
<version>0.39.0-beta.6</version>
|
||||
<version>0.39.0-beta.1</version>
|
||||
<relativePath>../pom.xml</relativePath>
|
||||
</parent>
|
||||
|
||||
|
||||
+2
-2
@@ -6,7 +6,7 @@
|
||||
|
||||
<groupId>com.lancedb</groupId>
|
||||
<artifactId>lancedb-parent</artifactId>
|
||||
<version>0.39.0-beta.6</version>
|
||||
<version>0.39.0-beta.1</version>
|
||||
<packaging>pom</packaging>
|
||||
<name>${project.artifactId}</name>
|
||||
<description>LanceDB Java SDK Parent POM</description>
|
||||
@@ -28,7 +28,7 @@
|
||||
<properties>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<arrow.version>15.0.0</arrow.version>
|
||||
<lance-core.version>12.0.0-beta.15</lance-core.version>
|
||||
<lance-core.version>12.0.0-beta.11</lance-core.version>
|
||||
<spotless.skip>false</spotless.skip>
|
||||
<spotless.version>2.30.0</spotless.version>
|
||||
<spotless.java.googlejavaformat.version>1.7</spotless.java.googlejavaformat.version>
|
||||
|
||||
+1
-1
@@ -1,7 +1,7 @@
|
||||
[package]
|
||||
name = "lancedb-nodejs"
|
||||
edition.workspace = true
|
||||
version = "0.39.0-beta.6"
|
||||
version = "0.39.0-beta.1"
|
||||
publish = false
|
||||
license.workspace = true
|
||||
description.workspace = true
|
||||
|
||||
@@ -939,7 +939,6 @@ describe("remote connection jobs surface", () => {
|
||||
const { tableFromArrays, tableToIPC } = await import("apache-arrow");
|
||||
const eventsTable = tableFromArrays({ state: ["created", "succeeded"] });
|
||||
const eventsBody = Buffer.from(tableToIPC(eventsTable, "stream"));
|
||||
const queryEventsPayloads: Record<string, unknown>[] = [];
|
||||
|
||||
await withMockDatabase(
|
||||
(req, res) => {
|
||||
@@ -968,16 +967,6 @@ describe("remote connection jobs surface", () => {
|
||||
);
|
||||
}
|
||||
} else if (req.url === "/v1/jobs/describe") {
|
||||
if (payload["job_id"] === "job-2") {
|
||||
res
|
||||
.writeHead(200, { "Content-Type": "application/json" })
|
||||
.end(
|
||||
'{"job_id": "job-2", "job_type": "refresh_column", ' +
|
||||
'"job_state": "DONE", "creation_ms": 2000, ' +
|
||||
'"result": {"rows_assigned": 1000000}}',
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (payload["job_id"] !== "job-1") {
|
||||
res.writeHead(404).end("no such job");
|
||||
return;
|
||||
@@ -999,7 +988,6 @@ describe("remote connection jobs surface", () => {
|
||||
.writeHead(200, { "Content-Type": "application/json" })
|
||||
.end('{"job_id": "job-1"}');
|
||||
} else if (req.url === "/v1/jobs/query_events") {
|
||||
queryEventsPayloads.push(payload);
|
||||
res
|
||||
.writeHead(200, {
|
||||
"Content-Type": "application/vnd.apache.arrow.stream",
|
||||
@@ -1016,65 +1004,22 @@ describe("remote connection jobs surface", () => {
|
||||
expect(jobs[0].state).toEqual("running");
|
||||
expect(jobs[1].state).toEqual("finished");
|
||||
|
||||
const description = await db.getJob("job-1");
|
||||
expect(description?.state).toEqual("failed");
|
||||
expect(JSON.parse(description?.specJson ?? "")).toEqual({
|
||||
column: "vec",
|
||||
});
|
||||
expect(description?.failure?.message).toEqual("worker died");
|
||||
expect(await db.getJob("missing")).toBeNull();
|
||||
|
||||
expect(await db.cancelJob("job-1")).toBe(true);
|
||||
expect(await db.cancelJob("missing")).toBe(false);
|
||||
|
||||
// Opening a job hands back a populated handle; a missing one rejects.
|
||||
await expect(db.openJob("missing")).rejects.toThrow("not found");
|
||||
const finished = await db.openJob("job-2");
|
||||
expect(finished.state).toEqual("finished");
|
||||
expect(finished.result).toEqual({
|
||||
// biome-ignore lint/style/useNamingConvention: snake_case mandated by the server wire format
|
||||
rows_assigned: 1000000,
|
||||
});
|
||||
const history = await db.jobHistory("job-1");
|
||||
expect(history.numRows).toEqual(2);
|
||||
|
||||
const job = await db.openJob("job-1");
|
||||
const job = db.job("job-1");
|
||||
expect(job.id).toEqual("job-1");
|
||||
|
||||
// openJob already populated the handle; refresh() re-reads it.
|
||||
expect(job.state).toEqual("failed");
|
||||
await job.refresh();
|
||||
expect(job.state).toEqual("failed");
|
||||
expect(job.jobType).toEqual("create_index");
|
||||
expect(job.creationMs).toEqual(1000);
|
||||
expect(job.spec).toEqual({ column: "vec" });
|
||||
expect(job.result).toBeNull();
|
||||
expect(job.failure?.message).toEqual("worker died");
|
||||
|
||||
// The handle reaches its own events, supplying its job id.
|
||||
const jobEvents = await job.events({
|
||||
limit: 500,
|
||||
filter: "state = 'claim_complete'",
|
||||
});
|
||||
expect(jobEvents.numRows).toEqual(2);
|
||||
expect(queryEventsPayloads.pop()).toEqual({
|
||||
// biome-ignore lint/style/useNamingConvention: snake_case mandated by the server wire format
|
||||
job_id: "job-1",
|
||||
limit: 500,
|
||||
filter: "state = 'claim_complete'",
|
||||
});
|
||||
|
||||
// Printing lays every known field out on its own line, with the JSON
|
||||
// payloads indented rather than crammed onto one line.
|
||||
expect(`${job}`).toEqual(
|
||||
[
|
||||
"Job(",
|
||||
' id="job-1",',
|
||||
' state="failed",',
|
||||
' jobType="create_index",',
|
||||
" creationMs=1000,",
|
||||
" spec={",
|
||||
' "column": "vec"',
|
||||
" },",
|
||||
" failure={",
|
||||
' "phase": "execute",',
|
||||
' "message": "worker died",',
|
||||
' "retryable": true',
|
||||
" },",
|
||||
")",
|
||||
].join("\n"),
|
||||
);
|
||||
|
||||
expect(await job.status()).toEqual("failed");
|
||||
await expect(job.wait()).rejects.toThrow("worker died");
|
||||
},
|
||||
|
||||
@@ -281,7 +281,7 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
|
||||
numIndices: 0,
|
||||
numRows: 3,
|
||||
// Full on-disk size of the two data files, footers and metadata included.
|
||||
totalBytes: 550,
|
||||
totalBytes: 684,
|
||||
});
|
||||
|
||||
// Index files count toward totalBytes too (only deletion files and
|
||||
@@ -289,7 +289,7 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
|
||||
await table.createIndex("id", { config: Index.btree() });
|
||||
const statsWithIndex = await table.stats();
|
||||
expect(statsWithIndex.numIndices).toBe(1);
|
||||
expect(statsWithIndex.totalBytes).toBeGreaterThan(550);
|
||||
expect(statsWithIndex.totalBytes).toBeGreaterThan(684);
|
||||
});
|
||||
|
||||
it("should overwrite data if asked", async () => {
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
import { tableFromIPC } from "apache-arrow";
|
||||
import {
|
||||
Data,
|
||||
SchemaLike,
|
||||
@@ -15,7 +16,6 @@ import {
|
||||
makeEmptyTable,
|
||||
} from "./arrow";
|
||||
import { EmbeddingFunctionConfig, getRegistry } from "./embedding/registry";
|
||||
import { Job } from "./job";
|
||||
import {
|
||||
MaterializedView,
|
||||
MaterializedViewSelect,
|
||||
@@ -27,6 +27,8 @@ import type {
|
||||
CreateNamespaceResponse,
|
||||
DescribeNamespaceResponse,
|
||||
DropNamespaceResponse,
|
||||
Job,
|
||||
JobDescription,
|
||||
JobInfo,
|
||||
ListNamespacesResponse,
|
||||
ListTablesResponse,
|
||||
@@ -555,19 +557,24 @@ export abstract class Connection {
|
||||
): Promise<void>;
|
||||
|
||||
/**
|
||||
* Open a server-side job by id, returning a handle with its record already
|
||||
* populated. Rejects when the server has no such job, the way
|
||||
* {@link Connection.openTable} does for a missing table.
|
||||
* A {@link Job} handle for a server-side job by id.
|
||||
*
|
||||
* The returned {@link Job} answers for its own state, specification,
|
||||
* result, failure and event history, so there is no separate
|
||||
* connection-level call for any of them.
|
||||
* The handle is constructed without a server round trip; an unknown id
|
||||
* surfaces when the handle is used. Dropping the handle has no effect on
|
||||
* the job itself.
|
||||
*/
|
||||
abstract openJob(jobId: string): Promise<Job>;
|
||||
abstract job(jobId: string): Job;
|
||||
|
||||
/** List server-side jobs across the database's tables. */
|
||||
abstract listJobs(): Promise<JobInfo[]>;
|
||||
|
||||
/**
|
||||
* Describe a single server-side job by id.
|
||||
*
|
||||
* Resolves to `null` when the server has no such job.
|
||||
*/
|
||||
abstract getJob(jobId: string): Promise<JobDescription | null>;
|
||||
|
||||
/**
|
||||
* Request cancellation of a server-side job by id.
|
||||
*
|
||||
@@ -575,6 +582,13 @@ export abstract class Connection {
|
||||
* such job exists. Cancelling an already-terminal job is a no-op success.
|
||||
*/
|
||||
abstract cancelJob(jobId: string): Promise<boolean>;
|
||||
|
||||
/**
|
||||
* The lifecycle event history of a server-side job, as an Arrow table.
|
||||
*
|
||||
* Lists history across all jobs when `jobId` is omitted.
|
||||
*/
|
||||
abstract jobHistory(jobId?: string): Promise<ArrowTable>;
|
||||
}
|
||||
|
||||
/** @hideconstructor */
|
||||
@@ -855,7 +869,7 @@ export class LocalConnection extends Connection {
|
||||
}
|
||||
|
||||
async dropTableAsync(name: string, namespacePath?: string[]): Promise<Job> {
|
||||
return new Job(await this.inner.dropTableAsync(name, namespacePath ?? []));
|
||||
return this.inner.dropTableAsync(name, namespacePath ?? []);
|
||||
}
|
||||
|
||||
async dropAllTables(namespacePath?: string[]): Promise<void> {
|
||||
@@ -914,17 +928,29 @@ export class LocalConnection extends Connection {
|
||||
);
|
||||
}
|
||||
|
||||
async openJob(jobId: string): Promise<Job> {
|
||||
return new Job(await this.inner.openJob(jobId));
|
||||
job(jobId: string): Job {
|
||||
return this.inner.job(jobId);
|
||||
}
|
||||
|
||||
async listJobs(): Promise<JobInfo[]> {
|
||||
return this.inner.listJobs();
|
||||
}
|
||||
|
||||
async getJob(jobId: string): Promise<JobDescription | null> {
|
||||
return this.inner.getJob(jobId);
|
||||
}
|
||||
|
||||
async cancelJob(jobId: string): Promise<boolean> {
|
||||
return this.inner.cancelJob(jobId);
|
||||
}
|
||||
|
||||
async jobHistory(jobId?: string): Promise<ArrowTable> {
|
||||
const buf = await this.inner.jobHistory(jobId);
|
||||
if (buf.length === 0) {
|
||||
return new ArrowTable();
|
||||
}
|
||||
return tableFromIPC(buf);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -94,9 +94,13 @@ export {
|
||||
RenameTableOptions,
|
||||
} from "./connection";
|
||||
|
||||
export { JobFailureInfo, JobInfo, Session } from "./native.js";
|
||||
|
||||
export { Job, JobEventsOptions } from "./job";
|
||||
export {
|
||||
Job,
|
||||
JobDescription,
|
||||
JobFailureInfo,
|
||||
JobInfo,
|
||||
Session,
|
||||
} from "./native.js";
|
||||
|
||||
export {
|
||||
AutoQuery,
|
||||
|
||||
@@ -1,188 +0,0 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
import { Table as ArrowTable, tableFromIPC } from "apache-arrow";
|
||||
import { JobFailureInfo, Job as NativeJob } from "./native";
|
||||
|
||||
/** Which of a job's events {@link Job.events} returns. */
|
||||
export interface JobEventsOptions {
|
||||
/** Maximum event rows to return, up to the server maximum of 10,000. */
|
||||
limit?: number;
|
||||
/** SQL-like filter over the event columns. */
|
||||
filter?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* A handle to an operation that may still be running.
|
||||
*
|
||||
* The operation may already be complete when the handle is created.
|
||||
*
|
||||
* The detail getters read what the handle last observed. Submitting an
|
||||
* operation returns only a job id, so populating them eagerly would cost an
|
||||
* extra round trip on every call:
|
||||
*
|
||||
* - {@link Job.refresh} and {@link Job.status} fetch the whole record.
|
||||
* - {@link Job.wait} records the terminal state it establishes, but not the
|
||||
* rest of the record.
|
||||
* - Everything is null until one of those runs.
|
||||
*
|
||||
* @hideconstructor
|
||||
*/
|
||||
export class Job {
|
||||
private readonly inner: NativeJob;
|
||||
|
||||
constructor(inner: NativeJob) {
|
||||
this.inner = inner;
|
||||
}
|
||||
|
||||
/**
|
||||
* Identifies the operation on the server that is running it.
|
||||
*
|
||||
* Operations that run in this process have no server id. The value is
|
||||
* opaque: parsing it or storing it to resume the job later is not supported.
|
||||
*/
|
||||
get id(): string | null {
|
||||
return this.inner.id ?? null;
|
||||
}
|
||||
|
||||
/** The last observed lifecycle state, without contacting the backend. */
|
||||
get state(): string | null {
|
||||
return this.inner.state ?? null;
|
||||
}
|
||||
|
||||
/**
|
||||
* The job's type, as the server names it. Null for an in-process job, which
|
||||
* has no server-side record.
|
||||
*/
|
||||
get jobType(): string | null {
|
||||
return this.inner.jobType ?? null;
|
||||
}
|
||||
|
||||
/** When the job was created, in milliseconds since the epoch. */
|
||||
get creationMs(): number | null {
|
||||
return this.inner.creationMs ?? null;
|
||||
}
|
||||
|
||||
/** The job-type-specific specification it was submitted with. */
|
||||
// biome-ignore lint/suspicious/noExplicitAny: shape varies by job type
|
||||
get spec(): any | null {
|
||||
return parseJson(this.inner.specJson);
|
||||
}
|
||||
|
||||
/**
|
||||
* The job-type-specific terminal result. Null until the job succeeds, so a
|
||||
* job that never terminates reports its progress through {@link Job.events}
|
||||
* instead.
|
||||
*/
|
||||
// biome-ignore lint/suspicious/noExplicitAny: shape varies by job type
|
||||
get result(): any | null {
|
||||
return parseJson(this.inner.resultJson);
|
||||
}
|
||||
|
||||
/** Why the job failed, when it failed and the server reports a reason. */
|
||||
get failure(): JobFailureInfo | null {
|
||||
return this.inner.failure ?? null;
|
||||
}
|
||||
|
||||
/**
|
||||
* The operation's current lifecycle state: "running", "finished", "failed",
|
||||
* or "cancelled".
|
||||
*
|
||||
* A point snapshot; unlike {@link Job.wait} it does not block or reject on a
|
||||
* terminal failure state. Also refreshes the getters above.
|
||||
*/
|
||||
async status(): Promise<string> {
|
||||
return this.inner.status();
|
||||
}
|
||||
|
||||
/** Wait until the operation reaches a terminal state. */
|
||||
async wait(): Promise<void> {
|
||||
return this.inner.wait();
|
||||
}
|
||||
|
||||
/** Request cancellation. Cancelling a finished operation is a no-op. */
|
||||
async cancel(): Promise<void> {
|
||||
return this.inner.cancel();
|
||||
}
|
||||
|
||||
/**
|
||||
* Ask the backend for this job's current state, and for a server-side job
|
||||
* its full record, then cache it for the getters above.
|
||||
*/
|
||||
async refresh(): Promise<void> {
|
||||
return this.inner.refresh();
|
||||
}
|
||||
|
||||
/**
|
||||
* This job's recorded lifecycle events.
|
||||
*
|
||||
* Where the getters above report a terminal result only once the job reaches
|
||||
* one, events are written as the job runs and outlive the workers that
|
||||
* produced them. A distributed job records a `claim`/`claim_complete` pair
|
||||
* per unit of work, each carrying `rows_processed`, so a job that never
|
||||
* finishes still accounts for what it did.
|
||||
*
|
||||
* The server caps results at 1000 rows by default and 10,000 at most, and
|
||||
* truncates without saying so, so pass `limit` for a job that emits an event
|
||||
* per fragment. `filter` is a SQL-like expression over the `state`,
|
||||
* `updated_by`, `emitted_from`, `emitted_by`, and `claim_entity` columns.
|
||||
*/
|
||||
async events(options?: JobEventsOptions): Promise<ArrowTable> {
|
||||
const buf = await this.inner.events(options?.limit, options?.filter);
|
||||
if (buf.length === 0) {
|
||||
return new ArrowTable();
|
||||
}
|
||||
return tableFromIPC(buf);
|
||||
}
|
||||
|
||||
/**
|
||||
* Every field the handle currently knows, one per line, with the JSON
|
||||
* payloads indented -- a refresh job's spec and result are the point of
|
||||
* printing it.
|
||||
*/
|
||||
toString(): string {
|
||||
if (this.state === null) {
|
||||
const known = this.id === null ? "" : `id=${JSON.stringify(this.id)}, `;
|
||||
return `Job(${known}not refreshed)`;
|
||||
}
|
||||
const fields: string[] = [];
|
||||
if (this.id !== null) {
|
||||
fields.push(`id=${JSON.stringify(this.id)}`);
|
||||
}
|
||||
fields.push(`state=${JSON.stringify(this.state)}`);
|
||||
if (this.jobType !== null) {
|
||||
fields.push(`jobType=${JSON.stringify(this.jobType)}`);
|
||||
}
|
||||
if (this.creationMs !== null) {
|
||||
fields.push(`creationMs=${this.creationMs}`);
|
||||
}
|
||||
for (const [name, value] of [
|
||||
["spec", this.spec],
|
||||
["result", this.result],
|
||||
] as const) {
|
||||
if (value !== null) {
|
||||
fields.push(`${name}=${indentJson(value)}`);
|
||||
}
|
||||
}
|
||||
if (this.failure !== null) {
|
||||
fields.push(`failure=${indentJson(this.failure)}`);
|
||||
}
|
||||
return `Job(${fields.map((field) => `\n${REPR_INDENT}${field},`).join("")}\n)`;
|
||||
}
|
||||
|
||||
[Symbol.for("nodejs.util.inspect.custom")](): string {
|
||||
return this.toString();
|
||||
}
|
||||
}
|
||||
|
||||
const REPR_INDENT = " ";
|
||||
|
||||
// biome-ignore lint/suspicious/noExplicitAny: shape varies by job type
|
||||
function indentJson(value: any): string {
|
||||
return JSON.stringify(value, null, 4).replace(/\n/g, `\n${REPR_INDENT}`);
|
||||
}
|
||||
|
||||
// biome-ignore lint/suspicious/noExplicitAny: shape varies by job type
|
||||
function parseJson(raw: string | null | undefined): any | null {
|
||||
return raw === null || raw === undefined ? null : JSON.parse(raw);
|
||||
}
|
||||
+9
-11
@@ -19,7 +19,6 @@ import {
|
||||
|
||||
import { EmbeddingFunctionConfig, getRegistry } from "./embedding/registry";
|
||||
import { IndexOptions } from "./indices";
|
||||
import { Job } from "./job";
|
||||
import { MergeInsertBuilder } from "./merge";
|
||||
import {
|
||||
AddColumnsResult,
|
||||
@@ -31,6 +30,7 @@ import {
|
||||
DropColumnsResult,
|
||||
IndexConfig,
|
||||
IndexStatistics,
|
||||
Job,
|
||||
LsmStats,
|
||||
Branches as NativeBranches,
|
||||
OptimizeStats,
|
||||
@@ -1124,15 +1124,13 @@ export class LocalTable extends Table {
|
||||
): Promise<Job> {
|
||||
// biome-ignore lint/suspicious/noExplicitAny: skip
|
||||
const nativeIndex = (options?.config as any)?.inner;
|
||||
return new Job(
|
||||
await this.inner.createIndexAsync(
|
||||
nativeIndex,
|
||||
column,
|
||||
options?.replace,
|
||||
options?.waitTimeoutSeconds,
|
||||
options?.name,
|
||||
options?.train,
|
||||
),
|
||||
return await this.inner.createIndexAsync(
|
||||
nativeIndex,
|
||||
column,
|
||||
options?.replace,
|
||||
options?.waitTimeoutSeconds,
|
||||
options?.name,
|
||||
options?.train,
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1315,7 +1313,7 @@ export class LocalTable extends Table {
|
||||
}
|
||||
|
||||
async refreshColumnAsync(column: string): Promise<Job> {
|
||||
return new Job(await this.inner.refreshColumnAsync(column));
|
||||
return await this.inner.refreshColumnAsync(column);
|
||||
}
|
||||
|
||||
async refreshMaterializedView(
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-darwin-arm64",
|
||||
"version": "0.39.0-beta.6",
|
||||
"version": "0.39.0-beta.1",
|
||||
"os": ["darwin"],
|
||||
"cpu": ["arm64"],
|
||||
"main": "lancedb.darwin-arm64.node",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-linux-arm64-gnu",
|
||||
"version": "0.39.0-beta.6",
|
||||
"version": "0.39.0-beta.1",
|
||||
"os": ["linux"],
|
||||
"cpu": ["arm64"],
|
||||
"main": "lancedb.linux-arm64-gnu.node",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-linux-arm64-musl",
|
||||
"version": "0.39.0-beta.6",
|
||||
"version": "0.39.0-beta.1",
|
||||
"os": ["linux"],
|
||||
"cpu": ["arm64"],
|
||||
"main": "lancedb.linux-arm64-musl.node",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-linux-x64-gnu",
|
||||
"version": "0.39.0-beta.6",
|
||||
"version": "0.39.0-beta.1",
|
||||
"os": ["linux"],
|
||||
"cpu": ["x64"],
|
||||
"main": "lancedb.linux-x64-gnu.node",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-linux-x64-musl",
|
||||
"version": "0.39.0-beta.6",
|
||||
"version": "0.39.0-beta.1",
|
||||
"os": ["linux"],
|
||||
"cpu": ["x64"],
|
||||
"main": "lancedb.linux-x64-musl.node",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-win32-arm64-msvc",
|
||||
"version": "0.39.0-beta.6",
|
||||
"version": "0.39.0-beta.1",
|
||||
"os": [
|
||||
"win32"
|
||||
],
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-win32-x64-msvc",
|
||||
"version": "0.39.0-beta.6",
|
||||
"version": "0.39.0-beta.1",
|
||||
"os": ["win32"],
|
||||
"cpu": ["x64"],
|
||||
"main": "lancedb.win32-x64-msvc.node",
|
||||
|
||||
+3
-3
@@ -11,7 +11,7 @@
|
||||
"ann"
|
||||
],
|
||||
"private": false,
|
||||
"version": "0.39.0-beta.6",
|
||||
"version": "0.39.0-beta.1",
|
||||
"main": "dist/index.js",
|
||||
"exports": {
|
||||
".": "./dist/index.js",
|
||||
@@ -44,7 +44,7 @@
|
||||
"@biomejs/biome": "^1.7.3",
|
||||
"@jest/globals": "^29.7.0",
|
||||
"@napi-rs/cli": "3.7.0",
|
||||
"@opentelemetry/sdk-metrics": "^2.10.0",
|
||||
"@opentelemetry/sdk-metrics": "^1.30.0",
|
||||
"@types/axios": "^0.14.0",
|
||||
"@types/jest": "^29.1.2",
|
||||
"@types/node": "22.7.4",
|
||||
@@ -56,7 +56,7 @@
|
||||
"eslint": "^8.57.0",
|
||||
"jest": "^29.7.0",
|
||||
"shx": "^0.3.4",
|
||||
"tmp": "^0.2.7",
|
||||
"tmp": "^0.2.3",
|
||||
"ts-jest": "^29.1.2",
|
||||
"typedoc": "0.26.4",
|
||||
"typedoc-plugin-markdown": "4.2.1",
|
||||
|
||||
Generated
+443
-599
File diff suppressed because it is too large
Load Diff
@@ -16,41 +16,3 @@ allowBuilds:
|
||||
onnxruntime-node: true
|
||||
protobufjs: true
|
||||
sharp: true
|
||||
|
||||
minimumReleaseAgeExclude:
|
||||
- protobufjs@7.5.8
|
||||
- tmp@0.2.6
|
||||
- form-data@4.0.6
|
||||
- tar@7.5.16
|
||||
- markdown-it@14.1.2
|
||||
- linkify-it@5.0.1
|
||||
- js-yaml@3.15.0
|
||||
- js-yaml@4.1.2
|
||||
- protobufjs@7.6.1
|
||||
- protobufjs@7.6.3
|
||||
- '@babel/core@7.29.1'
|
||||
- axios@1.18.0
|
||||
- brace-expansion@2.1.2
|
||||
- brace-expansion@1.1.16
|
||||
- js-yaml@4.3.0
|
||||
- tar@7.5.18
|
||||
- tar@7.5.19
|
||||
- tar@7.5.17
|
||||
- protobufjs@7.6.5
|
||||
- linkify-it@5.0.2
|
||||
- sharp@0.35.0
|
||||
- brace-expansion@1.1.17
|
||||
- brace-expansion@2.1.3
|
||||
- brace-expansion@2.1.4
|
||||
- brace-expansion@1.1.18
|
||||
- js-yaml@3.15.1
|
||||
- js-yaml@4.3.1
|
||||
- tar@7.5.21
|
||||
- '@opentelemetry/core@2.8.0'
|
||||
|
||||
# @huggingface/transformers pins sharp ^0.33.5 and no released version has moved
|
||||
# past ^0.34.5, all of which inherit the libvips CVEs in GHSA-f88m-g3jw-g9cj.
|
||||
# Force the patched line. sharp is only reached by transformers' image pipeline,
|
||||
# which LanceDB's text embedding function never uses.
|
||||
overrides:
|
||||
sharp: ^0.35.4
|
||||
|
||||
@@ -442,15 +442,13 @@ impl Connection {
|
||||
self.get_inner()?.drop_all_tables(&ns).await.default_error()
|
||||
}
|
||||
|
||||
/// Open a server-side job by id, returning a handle with its record
|
||||
/// already populated. Rejects when the server has no such job.
|
||||
/// A `Job` handle for a server-side job by id.
|
||||
///
|
||||
/// The returned handle answers for its own state, specification, result,
|
||||
/// failure and event history, so there is no separate connection-level
|
||||
/// call for any of them.
|
||||
#[napi(catch_unwind)]
|
||||
pub async fn open_job(&self, job_id: String) -> napi::Result<crate::job::Job> {
|
||||
let job = self.get_inner()?.open_job(&job_id).await.default_error()?;
|
||||
/// The handle is constructed without a server round trip; an unknown id
|
||||
/// surfaces when the handle is used.
|
||||
#[napi]
|
||||
pub fn job(&self, job_id: String) -> napi::Result<crate::job::Job> {
|
||||
let job = self.get_inner()?.job(job_id).default_error()?;
|
||||
Ok(crate::job::Job::new(job))
|
||||
}
|
||||
|
||||
@@ -461,6 +459,17 @@ impl Connection {
|
||||
Ok(jobs.into_iter().map(Into::into).collect())
|
||||
}
|
||||
|
||||
/// Describe a single server-side job by id. `null` when the server has
|
||||
/// no such job.
|
||||
#[napi(catch_unwind)]
|
||||
pub async fn get_job(
|
||||
&self,
|
||||
job_id: String,
|
||||
) -> napi::Result<Option<crate::job::JobDescription>> {
|
||||
let description = self.get_inner()?.get_job(&job_id).await.default_error()?;
|
||||
Ok(description.map(Into::into))
|
||||
}
|
||||
|
||||
/// Request cancellation of a server-side job by id. Returns true if the
|
||||
/// server accepted the cancellation, false if no such job exists.
|
||||
#[napi(catch_unwind)]
|
||||
@@ -468,6 +477,34 @@ impl Connection {
|
||||
self.get_inner()?.cancel_job(&job_id).await.default_error()
|
||||
}
|
||||
|
||||
/// The lifecycle event history of a server-side job (all jobs when
|
||||
/// `job_id` is null), as an Arrow IPC stream buffer. Empty when there is
|
||||
/// no history.
|
||||
#[napi(catch_unwind)]
|
||||
pub async fn job_history(&self, job_id: Option<String>) -> napi::Result<Buffer> {
|
||||
let batches = self
|
||||
.get_inner()?
|
||||
.job_history(job_id.as_deref())
|
||||
.await
|
||||
.default_error()?;
|
||||
let Some(first) = batches.first() else {
|
||||
return Ok(Buffer::from(Vec::<u8>::new()));
|
||||
};
|
||||
let mut out = Vec::new();
|
||||
let mut writer = arrow_ipc::writer::StreamWriter::try_new(&mut out, &first.schema())
|
||||
.map_err(|e| napi::Error::from_reason(e.to_string()))?;
|
||||
for batch in &batches {
|
||||
writer
|
||||
.write(batch)
|
||||
.map_err(|e| napi::Error::from_reason(e.to_string()))?;
|
||||
}
|
||||
writer
|
||||
.finish()
|
||||
.map_err(|e| napi::Error::from_reason(e.to_string()))?;
|
||||
drop(writer);
|
||||
Ok(Buffer::from(out))
|
||||
}
|
||||
|
||||
#[napi(catch_unwind)]
|
||||
/// Describe a namespace and return its properties.
|
||||
pub async fn describe_namespace(
|
||||
|
||||
+34
-90
@@ -3,9 +3,6 @@
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use arrow_array::RecordBatch;
|
||||
use lancedb::job::JobEventsRequest;
|
||||
use napi::bindgen_prelude::Buffer;
|
||||
use napi_derive::napi;
|
||||
|
||||
use crate::error::NapiErrorExt;
|
||||
@@ -58,98 +55,12 @@ impl Job {
|
||||
pub async fn cancel(&self) -> napi::Result<()> {
|
||||
self.inner.cancel().await.default_error()
|
||||
}
|
||||
|
||||
/// Ask the backend for this job's current state, and for a server-side job
|
||||
/// its full record, then cache it for the getters below.
|
||||
///
|
||||
/// They are all null until this runs, because submitting an operation
|
||||
/// returns only a job id. {@link Job.status} fetches the whole record too;
|
||||
/// {@link Job.wait} records only the terminal state it establishes.
|
||||
#[napi(catch_unwind)]
|
||||
pub async fn refresh(&self) -> napi::Result<()> {
|
||||
self.inner.refresh().await.default_error()
|
||||
}
|
||||
|
||||
/// The last observed lifecycle state, without contacting the backend.
|
||||
#[napi(getter)]
|
||||
pub fn state(&self) -> Option<String> {
|
||||
self.inner.state()
|
||||
}
|
||||
|
||||
/// The job's type, as the server names it. Null for an in-process job,
|
||||
/// which has no server-side record.
|
||||
#[napi(getter)]
|
||||
pub fn job_type(&self) -> Option<String> {
|
||||
self.inner.job_type()
|
||||
}
|
||||
|
||||
/// When the job was created, in milliseconds since the epoch.
|
||||
#[napi(getter)]
|
||||
pub fn creation_ms(&self) -> Option<i64> {
|
||||
self.inner.creation_ms()
|
||||
}
|
||||
|
||||
/// The job-type-specific specification as a JSON string, when present.
|
||||
#[napi(getter)]
|
||||
pub fn spec_json(&self) -> Option<String> {
|
||||
self.inner.spec().map(|spec| spec.to_string())
|
||||
}
|
||||
|
||||
/// The job-type-specific terminal result as a JSON string. Null until the
|
||||
/// job succeeds, so a job that never terminates reports its progress
|
||||
/// through {@link Job.events} instead.
|
||||
#[napi(getter)]
|
||||
pub fn result_json(&self) -> Option<String> {
|
||||
self.inner.result().map(|result| result.to_string())
|
||||
}
|
||||
|
||||
/// Why the job failed, when it failed and the server reports a reason.
|
||||
#[napi(getter)]
|
||||
pub fn failure(&self) -> Option<JobFailureInfo> {
|
||||
self.inner.failure().map(|failure| JobFailureInfo {
|
||||
phase: failure.phase,
|
||||
message: failure.message,
|
||||
retryable: failure.retryable,
|
||||
})
|
||||
}
|
||||
|
||||
/// This job's recorded lifecycle events, as an Arrow IPC stream buffer.
|
||||
/// The TypeScript wrapper turns it into an Arrow table.
|
||||
#[napi(catch_unwind)]
|
||||
pub async fn events(&self, limit: Option<u32>, filter: Option<String>) -> napi::Result<Buffer> {
|
||||
let batches = self
|
||||
.inner
|
||||
.events(JobEventsRequest { limit, filter })
|
||||
.await
|
||||
.default_error()?;
|
||||
batches_to_ipc_buffer(&batches)
|
||||
}
|
||||
}
|
||||
|
||||
/// Serialise Arrow batches as a single IPC stream for the TypeScript layer.
|
||||
fn batches_to_ipc_buffer(batches: &[RecordBatch]) -> napi::Result<Buffer> {
|
||||
let Some(first) = batches.first() else {
|
||||
return Ok(Buffer::from(Vec::<u8>::new()));
|
||||
};
|
||||
let mut out = Vec::new();
|
||||
let mut writer = arrow_ipc::writer::StreamWriter::try_new(&mut out, &first.schema())
|
||||
.map_err(|e| napi::Error::from_reason(e.to_string()))?;
|
||||
for batch in batches {
|
||||
writer
|
||||
.write(batch)
|
||||
.map_err(|e| napi::Error::from_reason(e.to_string()))?;
|
||||
}
|
||||
writer
|
||||
.finish()
|
||||
.map_err(|e| napi::Error::from_reason(e.to_string()))?;
|
||||
drop(writer);
|
||||
Ok(Buffer::from(out))
|
||||
}
|
||||
|
||||
/// A row from `Connection.listJobs`: one server-side job.
|
||||
#[napi(object)]
|
||||
pub struct JobInfo {
|
||||
/// The job id -- what `Connection.openJob` and `Connection.cancelJob`
|
||||
/// The job id -- what `Connection.getJob` and `Connection.cancelJob`
|
||||
/// accept.
|
||||
pub job_id: String,
|
||||
/// The table the job runs against, without URI or namespace.
|
||||
@@ -180,3 +91,36 @@ pub struct JobFailureInfo {
|
||||
pub message: Option<String>,
|
||||
pub retryable: Option<bool>,
|
||||
}
|
||||
|
||||
/// A described job from `Connection.getJob`.
|
||||
#[napi(object)]
|
||||
pub struct JobDescription {
|
||||
pub job_id: String,
|
||||
pub job_type: String,
|
||||
/// Lifecycle state: "running", "finished", "failed", or "cancelled".
|
||||
pub state: String,
|
||||
/// When the job was created, in milliseconds since the epoch.
|
||||
pub creation_ms: i64,
|
||||
/// The job-type-specific specification as a JSON string, when present.
|
||||
pub spec_json: Option<String>,
|
||||
/// Why the job failed, when the job is failed and the server reports a
|
||||
/// reason.
|
||||
pub failure: Option<JobFailureInfo>,
|
||||
}
|
||||
|
||||
impl From<lancedb::database::JobDescription> for JobDescription {
|
||||
fn from(description: lancedb::database::JobDescription) -> Self {
|
||||
Self {
|
||||
job_id: description.job_id,
|
||||
job_type: description.job_type,
|
||||
state: description.state,
|
||||
creation_ms: description.creation_ms,
|
||||
spec_json: (!description.spec.is_null()).then(|| description.spec.to_string()),
|
||||
failure: description.failure.map(|failure| JobFailureInfo {
|
||||
phase: failure.phase,
|
||||
message: failure.message,
|
||||
retryable: failure.retryable,
|
||||
}),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+2
-3
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "lancedb-python"
|
||||
version = "0.39.0-beta.6"
|
||||
version = "0.39.0-beta.1"
|
||||
publish = false
|
||||
edition.workspace = true
|
||||
description = "Python bindings for LanceDB"
|
||||
@@ -28,7 +28,7 @@ env_logger.workspace = true
|
||||
log.workspace = true
|
||||
# Maturin enables extension-module mode for Python builds. Keeping it out of
|
||||
# Cargo features lets Rust unit tests link against libpython.
|
||||
pyo3 = { version = "0.28", features = ["abi3-py310", "chrono", "uuid"] }
|
||||
pyo3 = { version = "0.28", features = ["abi3-py310", "chrono"] }
|
||||
chrono.workspace = true
|
||||
pyo3-async-runtimes = { version = "0.28", features = [
|
||||
"attributes",
|
||||
@@ -40,7 +40,6 @@ serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
snafu.workspace = true
|
||||
tokio.workspace = true
|
||||
uuid.workspace = true
|
||||
libc = "0.2"
|
||||
|
||||
[build-dependencies]
|
||||
|
||||
@@ -139,7 +139,6 @@ include = [
|
||||
"python/lancedb/exceptions.py",
|
||||
"python/lancedb/background_loop.py",
|
||||
"python/lancedb/schema.py",
|
||||
"python/lancedb/sql.py",
|
||||
"python/lancedb/remote/__init__.py",
|
||||
"python/lancedb/remote/errors.py",
|
||||
"python/lancedb/embeddings/__init__.py",
|
||||
|
||||
@@ -22,11 +22,7 @@ from .remote.db import RemoteDBConnection
|
||||
from .expr import Expr, col, lit, func
|
||||
from .schema import blob, vector
|
||||
from .job import AsyncJob, Job
|
||||
from .sql import AsyncQuery as AsyncSqlQuery
|
||||
from .sql import Query as SqlQuery
|
||||
from .sql import QueryDescription
|
||||
from .functions import (
|
||||
AssignmentMapping as AssignmentMapping,
|
||||
FunctionArtifactRequest as FunctionArtifactRequest,
|
||||
FunctionApplication as FunctionApplication,
|
||||
FunctionBinding as FunctionBinding,
|
||||
@@ -105,7 +101,6 @@ def connect(
|
||||
api_key: Optional[str] = None,
|
||||
region: str = "us-east-1",
|
||||
host_override: Optional[str] = None,
|
||||
sql_host_override: Optional[str] = None,
|
||||
read_consistency_interval: Optional[timedelta] = None,
|
||||
request_thread_pool: Optional[Union[int, ThreadPoolExecutor]] = None,
|
||||
client_config: Union[ClientConfig, Dict[str, Any], None] = None,
|
||||
@@ -134,9 +129,6 @@ def connect(
|
||||
The region to use for LanceDB Cloud.
|
||||
host_override: str, optional
|
||||
The override url for LanceDB Cloud.
|
||||
sql_host_override: str, optional
|
||||
The remote SQL service endpoint override. The client connects lazily when SQL
|
||||
is first executed and retains that connection.
|
||||
read_consistency_interval: timedelta, default None
|
||||
The interval at which to check for updates to the table from other
|
||||
processes. If None, then consistency is not checked. For performance
|
||||
@@ -278,7 +270,6 @@ def connect(
|
||||
api_key,
|
||||
region,
|
||||
host_override,
|
||||
sql_host_override=sql_host_override,
|
||||
# TODO: remove this (deprecation warning downstream)
|
||||
request_thread_pool=request_thread_pool,
|
||||
client_config=client_config,
|
||||
@@ -421,7 +412,6 @@ def deserialize_conn(
|
||||
parsed["api_key"],
|
||||
parsed.get("region", "us-east-1"),
|
||||
host_override=parsed.get("host_override"),
|
||||
sql_host_override=parsed.get("sql_host_override"),
|
||||
client_config=parsed.get("client_config"),
|
||||
storage_options=storage_options,
|
||||
)
|
||||
@@ -435,7 +425,6 @@ async def connect_async(
|
||||
api_key: Optional[str] = None,
|
||||
region: str = "us-east-1",
|
||||
host_override: Optional[str] = None,
|
||||
sql_host_override: Optional[str] = None,
|
||||
read_consistency_interval: Optional[timedelta] = None,
|
||||
client_config: Optional[Union[ClientConfig, Dict[str, Any]]] = None,
|
||||
storage_options: Optional[Dict[str, str]] = None,
|
||||
@@ -458,9 +447,6 @@ async def connect_async(
|
||||
The region to use for LanceDB Cloud.
|
||||
host_override: str, optional
|
||||
The override url for LanceDB Cloud.
|
||||
sql_host_override: str, optional
|
||||
The remote SQL service endpoint override. The client connects lazily when SQL
|
||||
is first executed and retains that connection.
|
||||
read_consistency_interval: timedelta, default None
|
||||
The interval at which to check for updates to the table from other
|
||||
processes. If None, then consistency is not checked. For performance
|
||||
@@ -548,7 +534,6 @@ async def connect_async(
|
||||
api_key,
|
||||
region,
|
||||
host_override,
|
||||
sql_host_override,
|
||||
read_consistency_interval_secs,
|
||||
client_config,
|
||||
storage_options,
|
||||
@@ -571,7 +556,6 @@ __all__ = [
|
||||
"connect_namespace_async",
|
||||
"AsyncConnection",
|
||||
"AsyncJob",
|
||||
"AsyncSqlQuery",
|
||||
"AsyncLanceNamespaceDBConnection",
|
||||
"AsyncTable",
|
||||
"FtsToken",
|
||||
@@ -586,8 +570,6 @@ __all__ = [
|
||||
"vector",
|
||||
"DBConnection",
|
||||
"Job",
|
||||
"QueryDescription",
|
||||
"SqlQuery",
|
||||
"LanceDBConnection",
|
||||
"LanceNamespaceDBConnection",
|
||||
"LsmWriteSpec",
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
from datetime import date, datetime, timedelta
|
||||
from decimal import Decimal
|
||||
from typing import Dict, List, Optional, Tuple, Any, TypedDict, Union, Literal
|
||||
from uuid import UUID
|
||||
|
||||
import pyarrow as pa
|
||||
|
||||
@@ -148,20 +147,17 @@ class Connection(object):
|
||||
start_after: Optional[str],
|
||||
limit: Optional[int],
|
||||
) -> list[str]: ... # Deprecated: Use list_tables instead
|
||||
async def open_job(self, job_id: str) -> Job: ...
|
||||
def job(self, job_id: str) -> Job: ...
|
||||
async def create_function_async(self, request_json: str) -> Job: ...
|
||||
async def get_function(self, name: str, version: str) -> str: ...
|
||||
async def list_functions(self) -> List[str]: ...
|
||||
async def drop_function(self, name: str, version: str) -> bool: ...
|
||||
async def list_jobs(self) -> List[JobInfo]: ...
|
||||
async def get_job(self, job_id: str) -> Optional[JobDescription]: ...
|
||||
async def cancel_job(self, job_id: str) -> bool: ...
|
||||
async def execute_query_async(
|
||||
self,
|
||||
query: str,
|
||||
*,
|
||||
default_namespace_path: Optional[List[str]] = None,
|
||||
) -> SqlQuery: ...
|
||||
async def describe_query(self, query_id: UUID) -> QueryDescription: ...
|
||||
async def job_history(
|
||||
self, job_id: Optional[str] = None
|
||||
) -> List[pa.RecordBatch]: ...
|
||||
async def create_table(
|
||||
self,
|
||||
name: str,
|
||||
@@ -240,20 +236,9 @@ class BlobFile:
|
||||
class Job:
|
||||
@property
|
||||
def id(self) -> Optional[str]: ...
|
||||
@property
|
||||
def _state(self) -> Optional[str]: ...
|
||||
@property
|
||||
def _description(self) -> Optional[JobDescription]: ...
|
||||
async def status(self) -> str: ...
|
||||
async def wait(self) -> Optional[str]: ...
|
||||
async def cancel(self) -> None: ...
|
||||
async def refresh(self) -> None: ...
|
||||
async def events(
|
||||
self,
|
||||
*,
|
||||
limit: Optional[int] = None,
|
||||
filter: Optional[str] = None,
|
||||
) -> pa.Table: ...
|
||||
|
||||
class JobInfo:
|
||||
@property
|
||||
@@ -285,33 +270,10 @@ class JobDescription:
|
||||
@property
|
||||
def creation_ms(self) -> int: ...
|
||||
@property
|
||||
def _spec_json(self) -> Optional[str]: ...
|
||||
@property
|
||||
def _result_json(self) -> Optional[str]: ...
|
||||
@property
|
||||
def spec(self) -> Optional[Any]: ...
|
||||
@property
|
||||
def result(self) -> Optional[Any]: ...
|
||||
def spec_json(self) -> Optional[str]: ...
|
||||
@property
|
||||
def failure(self) -> Optional[JobFailureInfo]: ...
|
||||
|
||||
class SqlQuery:
|
||||
@property
|
||||
def id(self) -> UUID: ...
|
||||
async def describe(self) -> QueryDescription: ...
|
||||
async def reader(self) -> RecordBatchStream: ...
|
||||
async def cancel(self) -> None: ...
|
||||
|
||||
class QueryDescription:
|
||||
@property
|
||||
def id(self) -> UUID: ...
|
||||
@property
|
||||
def status(self) -> str: ...
|
||||
@property
|
||||
def progress(self) -> Optional[float]: ...
|
||||
@property
|
||||
def expires_at(self) -> Optional[datetime]: ...
|
||||
|
||||
class Table:
|
||||
def name(self) -> str: ...
|
||||
def __repr__(self) -> str: ...
|
||||
@@ -490,7 +452,6 @@ async def connect(
|
||||
api_key: Optional[str],
|
||||
region: Optional[str],
|
||||
host_override: Optional[str],
|
||||
sql_host_override: Optional[str],
|
||||
read_consistency_interval: Optional[float],
|
||||
client_config: Optional[Union[ClientConfig, Dict[str, Any]]],
|
||||
storage_options: Optional[Dict[str, str]],
|
||||
|
||||
+62
-93
@@ -19,7 +19,6 @@ from typing import (
|
||||
Optional,
|
||||
Union,
|
||||
)
|
||||
from uuid import UUID
|
||||
|
||||
if sys.version_info >= (3, 12):
|
||||
from typing import override
|
||||
@@ -48,9 +47,6 @@ from . import __version__
|
||||
from ._lancedb import connect as lancedb_connect # type: ignore
|
||||
from .functions import FunctionVersion, UdfDefinition
|
||||
from .job import AsyncJob, Job, _typed_job
|
||||
from .sql import AsyncQuery as AsyncSqlQuery
|
||||
from .sql import Query as SqlQuery
|
||||
from .sql import QueryDescription
|
||||
from .materialized_view import (
|
||||
AsyncMaterializedView,
|
||||
MaterializedView,
|
||||
@@ -72,11 +68,10 @@ import deprecation
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import pyarrow as pa
|
||||
from .arrow import AsyncRecordBatchReader
|
||||
from .pydantic import LanceModel
|
||||
|
||||
from ._lancedb import Connection as LanceDbConnection
|
||||
from ._lancedb import JobInfo
|
||||
from ._lancedb import JobDescription, JobInfo
|
||||
from .common import DATA, URI
|
||||
from .embeddings import EmbeddingFunctionConfig
|
||||
from ._lancedb import Session
|
||||
@@ -745,23 +740,26 @@ class DBConnection(EnforceOverrides):
|
||||
"Function catalog operations are not supported for this connection type"
|
||||
)
|
||||
|
||||
def open_job(self, job_id: str) -> Job:
|
||||
"""Open a server-side job by id, returning a handle with its record
|
||||
already populated.
|
||||
def job(self, job_id: str) -> Job:
|
||||
"""A [Job][lancedb.job.Job] handle for a server-side job by id.
|
||||
|
||||
The returned [Job][lancedb.job.Job] answers for its own state,
|
||||
specification, result, failure and event history, so there is no
|
||||
separate connection-level call for any of them.
|
||||
|
||||
Raises `JobNotFoundError` when the server has no such job, the way
|
||||
`open_table` does for a missing table.
|
||||
The handle is constructed without a server round trip; an unknown id
|
||||
surfaces when the handle is used. Dropping the handle has no effect
|
||||
on the job itself.
|
||||
"""
|
||||
raise NotImplementedError("open_job is not supported for this connection type")
|
||||
raise NotImplementedError("job is not supported for this connection type")
|
||||
|
||||
def list_jobs(self) -> List[JobInfo]:
|
||||
"""List server-side jobs across the database's tables."""
|
||||
raise NotImplementedError("list_jobs is not supported for this connection type")
|
||||
|
||||
def get_job(self, job_id: str) -> Optional[JobDescription]:
|
||||
"""Describe a single server-side job by id.
|
||||
|
||||
Returns None when the server has no such job.
|
||||
"""
|
||||
raise NotImplementedError("get_job is not supported for this connection type")
|
||||
|
||||
def cancel_job(self, job_id: str) -> bool:
|
||||
"""Request cancellation of a server-side job by id.
|
||||
|
||||
@@ -773,38 +771,14 @@ class DBConnection(EnforceOverrides):
|
||||
"cancel_job is not supported for this connection type"
|
||||
)
|
||||
|
||||
def execute_query(
|
||||
self,
|
||||
query: str,
|
||||
*,
|
||||
default_namespace_path: Optional[List[str]] = None,
|
||||
) -> pa.RecordBatchReader:
|
||||
"""Execute SQL and return a blocking Arrow reader.
|
||||
def job_history(self, job_id: Optional[str] = None) -> List[pa.RecordBatch]:
|
||||
"""The lifecycle event history of a server-side job, as Arrow batches.
|
||||
|
||||
This submits through :meth:`execute_query_async` and waits until the
|
||||
initial result stream is readable. It does not wait for the full query
|
||||
to finish.
|
||||
Lists history across all jobs when `job_id` is None.
|
||||
"""
|
||||
return self.execute_query_async(
|
||||
query,
|
||||
default_namespace_path=default_namespace_path,
|
||||
).reader()
|
||||
|
||||
def execute_query_async(
|
||||
self,
|
||||
query: str,
|
||||
*,
|
||||
default_namespace_path: Optional[List[str]] = None,
|
||||
) -> SqlQuery:
|
||||
"""Start executing SQL and return its query handle.
|
||||
|
||||
Local connections do not support SQL.
|
||||
"""
|
||||
raise NotImplementedError("SQL is not supported for this connection type")
|
||||
|
||||
def describe_query(self, query_id: UUID) -> QueryDescription:
|
||||
"""Describe a submitted SQL query by its connection-scoped id."""
|
||||
raise NotImplementedError("SQL is not supported for this connection type")
|
||||
raise NotImplementedError(
|
||||
"job_history is not supported for this connection type"
|
||||
)
|
||||
|
||||
|
||||
class LanceDBConnection(DBConnection):
|
||||
@@ -901,7 +875,6 @@ class LanceDBConnection(DBConnection):
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
read_consistency_interval_secs,
|
||||
None,
|
||||
storage_options,
|
||||
@@ -1450,11 +1423,14 @@ class LanceDBConnection(DBConnection):
|
||||
)
|
||||
|
||||
@override
|
||||
def open_job(self, job_id: str) -> Job:
|
||||
"""Open a server-side job by id. See
|
||||
[DBConnection.open_job][lancedb.db.DBConnection.open_job].
|
||||
def job(self, job_id: str) -> Job:
|
||||
"""A [Job][lancedb.job.Job] handle for a server-side job by id.
|
||||
|
||||
The handle is constructed without a server round trip; an unknown id
|
||||
surfaces when the handle is used. Dropping the handle has no effect
|
||||
on the job itself.
|
||||
"""
|
||||
return Job(LOOP.run(self._conn.open_job(job_id)))
|
||||
return Job(self._conn.job(job_id))
|
||||
|
||||
@override
|
||||
def create_function_async(self, definition: UdfDefinition) -> Job[FunctionVersion]:
|
||||
@@ -1478,6 +1454,14 @@ class LanceDBConnection(DBConnection):
|
||||
"""List server-side jobs across the database's tables."""
|
||||
return LOOP.run(self._conn.list_jobs())
|
||||
|
||||
@override
|
||||
def get_job(self, job_id: str) -> Optional[JobDescription]:
|
||||
"""Describe a single server-side job by id.
|
||||
|
||||
Returns None when the server has no such job.
|
||||
"""
|
||||
return LOOP.run(self._conn.get_job(job_id))
|
||||
|
||||
@override
|
||||
def cancel_job(self, job_id: str) -> bool:
|
||||
"""Request cancellation of a server-side job by id.
|
||||
@@ -1488,6 +1472,14 @@ class LanceDBConnection(DBConnection):
|
||||
"""
|
||||
return LOOP.run(self._conn.cancel_job(job_id))
|
||||
|
||||
@override
|
||||
def job_history(self, job_id: Optional[str] = None) -> List[pa.RecordBatch]:
|
||||
"""The lifecycle event history of a server-side job, as Arrow batches.
|
||||
|
||||
Lists history across all jobs when `job_id` is None.
|
||||
"""
|
||||
return LOOP.run(self._conn.job_history(job_id))
|
||||
|
||||
@override
|
||||
def namespace_client(self) -> LanceNamespace:
|
||||
"""Get the equivalent namespace client for this connection.
|
||||
@@ -2258,11 +2250,15 @@ class AsyncConnection(object):
|
||||
namespace_path = []
|
||||
await self._inner.drop_all_tables(namespace_path=namespace_path)
|
||||
|
||||
async def open_job(self, job_id: str) -> AsyncJob:
|
||||
"""Open a server-side job by id. See
|
||||
[DBConnection.open_job][lancedb.db.DBConnection.open_job].
|
||||
def job(self, job_id: str) -> AsyncJob:
|
||||
"""An [AsyncJob][lancedb.job.AsyncJob] handle for a server-side job
|
||||
by id.
|
||||
|
||||
The handle is constructed without a server round trip; an unknown id
|
||||
surfaces when the handle is used. Dropping the handle has no effect
|
||||
on the job itself.
|
||||
"""
|
||||
return AsyncJob(await self._inner.open_job(job_id))
|
||||
return AsyncJob(self._inner.job(job_id))
|
||||
|
||||
async def create_function_async(
|
||||
self, definition: UdfDefinition
|
||||
@@ -2302,6 +2298,13 @@ class AsyncConnection(object):
|
||||
"""List server-side jobs across the database's tables."""
|
||||
return await self._inner.list_jobs()
|
||||
|
||||
async def get_job(self, job_id: str) -> Optional[JobDescription]:
|
||||
"""Describe a single server-side job by id.
|
||||
|
||||
Returns None when the server has no such job.
|
||||
"""
|
||||
return await self._inner.get_job(job_id)
|
||||
|
||||
async def cancel_job(self, job_id: str) -> bool:
|
||||
"""Request cancellation of a server-side job by id.
|
||||
|
||||
@@ -2311,46 +2314,12 @@ class AsyncConnection(object):
|
||||
"""
|
||||
return await self._inner.cancel_job(job_id)
|
||||
|
||||
async def execute_query(
|
||||
self,
|
||||
query: str,
|
||||
*,
|
||||
default_namespace_path: Optional[List[str]] = None,
|
||||
) -> AsyncRecordBatchReader:
|
||||
"""Execute SQL and return an asynchronous Arrow reader.
|
||||
async def job_history(self, job_id: Optional[str] = None) -> List[pa.RecordBatch]:
|
||||
"""The lifecycle event history of a server-side job, as Arrow batches.
|
||||
|
||||
This submits through :meth:`execute_query_async` and waits until the
|
||||
initial result stream is readable. It does not wait for the full query
|
||||
to finish.
|
||||
Lists history across all jobs when `job_id` is None.
|
||||
"""
|
||||
submitted = await self.execute_query_async(
|
||||
query,
|
||||
default_namespace_path=default_namespace_path,
|
||||
)
|
||||
return await submitted.reader()
|
||||
|
||||
async def execute_query_async(
|
||||
self,
|
||||
query: str,
|
||||
*,
|
||||
default_namespace_path: Optional[List[str]] = None,
|
||||
) -> AsyncSqlQuery:
|
||||
"""Start executing SQL and return its query handle.
|
||||
|
||||
The database from ``connect_async`` is used for unqualified database
|
||||
references. The namespace defaults to ``["public"]``. Local
|
||||
connections raise ``NotImplementedError``.
|
||||
"""
|
||||
return AsyncSqlQuery(
|
||||
await self._inner.execute_query_async(
|
||||
query,
|
||||
default_namespace_path=default_namespace_path,
|
||||
)
|
||||
)
|
||||
|
||||
async def describe_query(self, query_id: UUID) -> QueryDescription:
|
||||
"""Describe a submitted SQL query by its connection-scoped id."""
|
||||
return await self._inner.describe_query(query_id)
|
||||
return await self._inner.job_history(job_id)
|
||||
|
||||
async def namespace_client(self) -> LanceNamespace:
|
||||
"""Get the equivalent namespace client for this connection.
|
||||
|
||||
@@ -35,9 +35,3 @@ class JobCancelledError(RuntimeError):
|
||||
"""Exception raised when an asynchronous job was cancelled."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class JobNotFoundError(ValueError):
|
||||
"""Exception raised when opening a job the server does not have."""
|
||||
|
||||
pass
|
||||
|
||||
@@ -470,7 +470,11 @@ class InputBinding(_RemoteValue):
|
||||
|
||||
|
||||
class OutputMapping(_RemoteValue):
|
||||
"""One stable result-field mapping."""
|
||||
"""One stable result-field mapping.
|
||||
|
||||
Assignment state is outside the Slice 1 client contract. During the NULL
|
||||
transition Lance exposes no public cell-flag identifier to persist here.
|
||||
"""
|
||||
|
||||
result_field: str
|
||||
output_name: str
|
||||
@@ -480,13 +484,6 @@ class OutputMapping(_RemoteValue):
|
||||
nullable: bool
|
||||
|
||||
|
||||
class AssignmentMapping(_RemoteValue):
|
||||
"""Internal physical column preserving flattened struct validity."""
|
||||
|
||||
output_name: str
|
||||
output_field_id: _Int32
|
||||
|
||||
|
||||
class FunctionBinding(_RemoteValue):
|
||||
"""Immutable Function binding persisted by the Enterprise table service."""
|
||||
|
||||
@@ -494,7 +491,6 @@ class FunctionBinding(_RemoteValue):
|
||||
function: FunctionVersionRef
|
||||
inputs: tuple[InputBinding, ...]
|
||||
outputs: tuple[OutputMapping, ...]
|
||||
assignment: Optional[AssignmentMapping] = None
|
||||
input_schema: Optional[Mapping[str, Any]] = None
|
||||
output_schema: Optional[Mapping[str, Any]] = None
|
||||
|
||||
@@ -915,6 +911,8 @@ def _function_output(output: pa.DataType | pa.Field | pa.Schema) -> FunctionOutp
|
||||
|
||||
if not fields:
|
||||
raise ValueError("named-struct Function output must contain at least one field")
|
||||
if any(field.nullable for field in fields):
|
||||
raise ValueError("Function output fields must be non-nullable")
|
||||
for field in fields:
|
||||
_validate_exact_arrow_field(field)
|
||||
names = [field.name for field in fields]
|
||||
@@ -926,7 +924,7 @@ def _function_output(output: pa.DataType | pa.Field | pa.Schema) -> FunctionOutp
|
||||
FunctionResultField(
|
||||
name=field.name,
|
||||
arrow_type=_canonical_arrow_field(field),
|
||||
nullable=field.nullable,
|
||||
nullable=False,
|
||||
)
|
||||
for field in fields
|
||||
),
|
||||
@@ -1309,9 +1307,8 @@ def udf(
|
||||
|
||||
Input and output signatures are inferred from supported annotations. For
|
||||
Arrow types annotations cannot express precisely, pass ``input_schema``
|
||||
and ``output_schema`` together. Scalar outputs must be non-nullable. Every
|
||||
named-struct field may be nullable; Enterprise preserves the struct's
|
||||
validity when the result is expanded into sibling columns.
|
||||
and ``output_schema`` together. Nullable outputs are rejected because V1
|
||||
uses physical NULL to represent unassigned computed-column rows.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
@@ -1323,8 +1320,8 @@ def udf(
|
||||
Explicit input fields in the exact order of the callable parameters.
|
||||
Must be provided together with ``output_schema``.
|
||||
output_schema : pyarrow.DataType, pyarrow.Field, or pyarrow.Schema, optional
|
||||
Explicit scalar or named-struct output. Scalar outputs must be
|
||||
non-nullable. Must be provided together with ``input_schema``.
|
||||
Explicit scalar or named-struct output. Must be non-nullable and be
|
||||
provided together with ``input_schema``.
|
||||
pip : sequence of str, optional
|
||||
Pip requirements for the remote environment.
|
||||
conda : sequence of str, optional
|
||||
@@ -1390,7 +1387,6 @@ def udf(
|
||||
|
||||
|
||||
__all__ = [
|
||||
"AssignmentMapping",
|
||||
"ApplicationInput",
|
||||
"FunctionApplication",
|
||||
"FunctionArtifact",
|
||||
|
||||
@@ -4,27 +4,15 @@
|
||||
"""Handles to operations a server may run asynchronously."""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from datetime import timedelta
|
||||
from typing import Any, Callable, Generic, Optional, TypeVar, cast
|
||||
|
||||
import pyarrow as pa
|
||||
|
||||
from lancedb.background_loop import LOOP
|
||||
|
||||
from . import _lancedb
|
||||
from ._lancedb import JobDescription, JobFailureInfo, JobInfo
|
||||
|
||||
T = TypeVar("T")
|
||||
|
||||
__all__ = [
|
||||
"AsyncJob",
|
||||
"Job",
|
||||
"JobDescription",
|
||||
"JobFailureInfo",
|
||||
"JobInfo",
|
||||
]
|
||||
|
||||
|
||||
class AsyncJob(Generic[T]):
|
||||
"""A handle to an operation that may still be running.
|
||||
@@ -90,149 +78,6 @@ class AsyncJob(Generic[T]):
|
||||
return
|
||||
await self._inner.cancel()
|
||||
|
||||
async def refresh(self) -> None:
|
||||
"""Ask the backend for this job's current state, and for a server-side
|
||||
job its full record, then cache it for the properties below.
|
||||
|
||||
The properties are all `None` until this runs, because submitting an
|
||||
operation returns only a job id. `status` fetches the whole record too;
|
||||
`wait` records only the terminal state it establishes.
|
||||
"""
|
||||
if self._inner is None:
|
||||
return
|
||||
await self._inner.refresh()
|
||||
|
||||
@property
|
||||
def state(self) -> Optional[str]:
|
||||
"""The last observed lifecycle state, without contacting the backend.
|
||||
|
||||
`None` until the handle has talked to it. See :meth:`AsyncJob.refresh`.
|
||||
"""
|
||||
if self._inner is None:
|
||||
return "finished"
|
||||
return self._inner._state
|
||||
|
||||
@property
|
||||
def job_type(self) -> Optional[str]:
|
||||
"""The job's type, as the server names it.
|
||||
|
||||
`None` for an in-process job, which has no server-side record.
|
||||
"""
|
||||
return self._field("job_type")
|
||||
|
||||
@property
|
||||
def creation_ms(self) -> Optional[int]:
|
||||
"""When the job was created, in milliseconds since the epoch."""
|
||||
return self._field("creation_ms")
|
||||
|
||||
@property
|
||||
def spec(self) -> Optional[Any]:
|
||||
"""The job-type-specific specification it was submitted with."""
|
||||
return self._field("spec")
|
||||
|
||||
@property
|
||||
def result(self) -> Optional[Any]:
|
||||
"""The job-type-specific terminal result, as reported data rather than
|
||||
the typed model :meth:`AsyncJob.wait` returns.
|
||||
|
||||
`None` until the job succeeds, so a job that never terminates reports
|
||||
its progress through :meth:`AsyncJob.events` instead.
|
||||
"""
|
||||
return self._field("result")
|
||||
|
||||
@property
|
||||
def failure(self) -> Optional[JobFailureInfo]:
|
||||
"""Why the job failed, when it failed and the server reports a reason."""
|
||||
return self._field("failure")
|
||||
|
||||
@property
|
||||
def _spec_json(self) -> Optional[str]:
|
||||
return self._field("_spec_json")
|
||||
|
||||
@property
|
||||
def _result_json(self) -> Optional[str]:
|
||||
return self._field("_result_json")
|
||||
|
||||
def _field(self, name: str) -> Optional[Any]:
|
||||
description = self._inner._description if self._inner is not None else None
|
||||
return getattr(description, name) if description is not None else None
|
||||
|
||||
async def events(
|
||||
self,
|
||||
*,
|
||||
limit: Optional[int] = None,
|
||||
filter: Optional[str] = None,
|
||||
) -> "pa.Table":
|
||||
"""This job's recorded lifecycle events.
|
||||
|
||||
Where the properties above report a terminal result only once the job
|
||||
reaches one, events are written as the job runs and outlive the workers
|
||||
that produced them. A distributed job records a `claim`/`claim_complete`
|
||||
pair per unit of work, each carrying `rows_processed`, so a job that
|
||||
never finishes still accounts for what it did.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
limit: int, optional
|
||||
Maximum event rows to return. The server caps results at 1000 by
|
||||
default and 10,000 at most, and truncates without saying so, so
|
||||
pass this for a job that emits an event per fragment.
|
||||
filter: str, optional
|
||||
SQL-like expression over the `state`, `updated_by`, `emitted_from`,
|
||||
`emitted_by`, and `claim_entity` columns, such as
|
||||
``state = 'claim_complete'``.
|
||||
"""
|
||||
if self._inner is None:
|
||||
raise NotImplementedError(
|
||||
"job event history is only available for server-side jobs"
|
||||
)
|
||||
return await self._inner.events(limit=limit, filter=filter)
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return _job_repr("AsyncJob", self)
|
||||
|
||||
|
||||
_REPR_INDENT = " " * 4
|
||||
|
||||
|
||||
def _repr_payload(value: Any) -> str:
|
||||
"""Render a job payload as indented JSON, aligned under its field."""
|
||||
try:
|
||||
rendered = json.dumps(value, indent=4)
|
||||
except TypeError:
|
||||
return repr(value)
|
||||
return rendered.replace("\n", "\n" + _REPR_INDENT)
|
||||
|
||||
|
||||
def _job_repr(kind: str, job: Any) -> str:
|
||||
"""Render every field the handle currently knows, omitting the rest.
|
||||
|
||||
One field per line, with the JSON payloads indented, because a refresh
|
||||
job's spec and result are the point of printing it.
|
||||
"""
|
||||
state = job.state
|
||||
if state is None:
|
||||
# Nothing has been fetched yet, so there is nothing to lay out.
|
||||
known = f"id={job.id!r}, " if job.id is not None else ""
|
||||
return f"{kind}({known}not refreshed)"
|
||||
|
||||
fields = []
|
||||
if job.id is not None:
|
||||
fields.append(f"id={job.id!r}")
|
||||
fields.append(f"state={state!r}")
|
||||
for name in ("job_type", "creation_ms"):
|
||||
value = getattr(job, name)
|
||||
if value is not None:
|
||||
fields.append(f"{name}={value!r}")
|
||||
for name in ("spec", "result"):
|
||||
value = getattr(job, name)
|
||||
if value is not None:
|
||||
fields.append(f"{name}={_repr_payload(value)}")
|
||||
if job.failure is not None:
|
||||
fields.append(f"failure={job.failure!r}")
|
||||
body = "".join(f"\n{_REPR_INDENT}{field}," for field in fields)
|
||||
return f"{kind}({body}\n)"
|
||||
|
||||
|
||||
class Job(Generic[T]):
|
||||
"""Synchronous counterpart of `AsyncJob` with the same result type."""
|
||||
@@ -277,75 +122,6 @@ class Job(Generic[T]):
|
||||
return
|
||||
LOOP.run(self._inner.cancel())
|
||||
|
||||
def refresh(self) -> None:
|
||||
"""Ask the backend for this job's current state and record.
|
||||
|
||||
See :meth:`AsyncJob.refresh`.
|
||||
"""
|
||||
if self._inner is None:
|
||||
return
|
||||
LOOP.run(self._inner.refresh())
|
||||
|
||||
@property
|
||||
def state(self) -> Optional[str]:
|
||||
"""The last observed lifecycle state. See :attr:`AsyncJob.state`."""
|
||||
return self._inner.state if self._inner is not None else "finished"
|
||||
|
||||
@property
|
||||
def job_type(self) -> Optional[str]:
|
||||
"""The job's type. See :attr:`AsyncJob.job_type`."""
|
||||
return self._field("job_type")
|
||||
|
||||
@property
|
||||
def creation_ms(self) -> Optional[int]:
|
||||
"""When the job was created. See :attr:`AsyncJob.creation_ms`."""
|
||||
return self._field("creation_ms")
|
||||
|
||||
@property
|
||||
def spec(self) -> Optional[Any]:
|
||||
"""The job's specification. See :attr:`AsyncJob.spec`."""
|
||||
return self._field("spec")
|
||||
|
||||
@property
|
||||
def result(self) -> Optional[Any]:
|
||||
"""The job's terminal result. See :attr:`AsyncJob.result`."""
|
||||
return self._field("result")
|
||||
|
||||
@property
|
||||
def failure(self) -> Optional[JobFailureInfo]:
|
||||
"""Why the job failed. See :attr:`AsyncJob.failure`."""
|
||||
return self._field("failure")
|
||||
|
||||
@property
|
||||
def _spec_json(self) -> Optional[str]:
|
||||
return self._field("_spec_json")
|
||||
|
||||
@property
|
||||
def _result_json(self) -> Optional[str]:
|
||||
return self._field("_result_json")
|
||||
|
||||
def _field(self, name: str) -> Optional[Any]:
|
||||
return getattr(self._inner, name) if self._inner is not None else None
|
||||
|
||||
def events(
|
||||
self,
|
||||
*,
|
||||
limit: Optional[int] = None,
|
||||
filter: Optional[str] = None,
|
||||
) -> "pa.Table":
|
||||
"""This job's recorded lifecycle events.
|
||||
|
||||
See :meth:`AsyncJob.events`.
|
||||
"""
|
||||
if self._inner is None:
|
||||
raise NotImplementedError(
|
||||
"job event history is only available for server-side jobs"
|
||||
)
|
||||
return LOOP.run(self._inner.events(limit=limit, filter=filter))
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return _job_repr("Job", self)
|
||||
|
||||
|
||||
def _typed_job(
|
||||
inner: "_lancedb.Job", result_decoder: Callable[[str], T]
|
||||
|
||||
@@ -12,7 +12,6 @@ from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Union
|
||||
from uuid import UUID
|
||||
|
||||
if sys.version_info >= (3, 12):
|
||||
from typing import override
|
||||
@@ -49,11 +48,8 @@ from lancedb._lancedb import (
|
||||
connect_namespace_client as _connect_namespace_client,
|
||||
)
|
||||
from lancedb.background_loop import LOOP
|
||||
from lancedb.arrow import AsyncRecordBatchReader
|
||||
from lancedb.db import AsyncConnection, DBConnection
|
||||
from lancedb.job import AsyncJob, Job
|
||||
from lancedb.sql import AsyncQuery as AsyncSqlQuery
|
||||
from lancedb.sql import QueryDescription
|
||||
from lance_namespace import (
|
||||
LanceNamespace,
|
||||
connect as namespace_connect,
|
||||
@@ -1451,37 +1447,6 @@ class AsyncLanceNamespaceDBConnection:
|
||||
namespace_path=namespace_path, page_token=page_token, limit=limit
|
||||
)
|
||||
|
||||
async def execute_query(
|
||||
self,
|
||||
query: str,
|
||||
*,
|
||||
default_namespace_path: Optional[List[str]] = None,
|
||||
) -> AsyncRecordBatchReader:
|
||||
"""Execute SQL when supported by the underlying connection."""
|
||||
return await self._inner.execute_query(
|
||||
query,
|
||||
default_namespace_path=default_namespace_path,
|
||||
)
|
||||
|
||||
async def execute_query_async(
|
||||
self,
|
||||
query: str,
|
||||
*,
|
||||
default_namespace_path: Optional[List[str]] = None,
|
||||
) -> AsyncSqlQuery:
|
||||
"""Start executing SQL when supported by the underlying connection.
|
||||
|
||||
Namespace-backed local connections do not support SQL.
|
||||
"""
|
||||
return await self._inner.execute_query_async(
|
||||
query,
|
||||
default_namespace_path=default_namespace_path,
|
||||
)
|
||||
|
||||
async def describe_query(self, query_id: UUID) -> QueryDescription:
|
||||
"""Describe a submitted SQL query when supported."""
|
||||
return await self._inner.describe_query(query_id)
|
||||
|
||||
async def namespace_client(self) -> LanceNamespace:
|
||||
"""Get the namespace client for this connection.
|
||||
|
||||
|
||||
@@ -9,7 +9,6 @@ from concurrent.futures import ThreadPoolExecutor
|
||||
import sys
|
||||
from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Union
|
||||
from urllib.parse import urlparse
|
||||
from uuid import UUID
|
||||
import warnings
|
||||
|
||||
if sys.version_info >= (3, 12):
|
||||
@@ -26,12 +25,10 @@ from ..common import DATA
|
||||
from ..db import DBConnection, LOOP
|
||||
from ..functions import FunctionVersion, UdfDefinition
|
||||
from ..job import AsyncJob, Job
|
||||
from ..sql import Query as SqlQuery
|
||||
from ..sql import QueryDescription
|
||||
from ..materialized_view import MaterializedView, SelectArg
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .._lancedb import JobInfo
|
||||
from .._lancedb import JobDescription, JobInfo
|
||||
from ..embeddings import EmbeddingFunctionConfig
|
||||
from lance_namespace import (
|
||||
LanceNamespace,
|
||||
@@ -119,7 +116,6 @@ class RemoteDBConnection(DBConnection):
|
||||
read_timeout: Optional[float] = None,
|
||||
storage_options: Optional[Dict[str, str]] = None,
|
||||
read_consistency_interval: Optional[timedelta] = None,
|
||||
sql_host_override: Optional[str] = None,
|
||||
):
|
||||
"""Connect to a remote LanceDB database."""
|
||||
if isinstance(client_config, dict):
|
||||
@@ -165,7 +161,6 @@ class RemoteDBConnection(DBConnection):
|
||||
self.api_key = api_key
|
||||
self.region = region
|
||||
self.host_override = host_override
|
||||
self.sql_host_override = sql_host_override
|
||||
self.storage_options = storage_options
|
||||
self.db_name = parsed.netloc
|
||||
|
||||
@@ -180,7 +175,6 @@ class RemoteDBConnection(DBConnection):
|
||||
api_key=api_key,
|
||||
region=region,
|
||||
host_override=host_override,
|
||||
sql_host_override=sql_host_override,
|
||||
client_config=client_config,
|
||||
storage_options=storage_options,
|
||||
read_consistency_interval=read_consistency_interval,
|
||||
@@ -199,7 +193,6 @@ class RemoteDBConnection(DBConnection):
|
||||
"api_key": self.api_key,
|
||||
"region": self.region,
|
||||
"host_override": self.host_override,
|
||||
"sql_host_override": self.sql_host_override,
|
||||
"client_config": _client_config_to_dict(self.client_config),
|
||||
"storage_options": self.storage_options,
|
||||
}
|
||||
@@ -739,11 +732,14 @@ class RemoteDBConnection(DBConnection):
|
||||
)
|
||||
|
||||
@override
|
||||
def open_job(self, job_id: str) -> Job:
|
||||
"""Open a server-side job by id. See
|
||||
[DBConnection.open_job][lancedb.db.DBConnection.open_job].
|
||||
def job(self, job_id: str) -> Job:
|
||||
"""A [Job][lancedb.job.Job] handle for a server-side job by id.
|
||||
|
||||
The handle is constructed without a server round trip; an unknown id
|
||||
surfaces when the handle is used. Dropping the handle has no effect
|
||||
on the job itself.
|
||||
"""
|
||||
return Job(LOOP.run(self._conn.open_job(job_id)))
|
||||
return Job(self._conn.job(job_id))
|
||||
|
||||
@override
|
||||
def create_function_async(self, definition: UdfDefinition) -> Job[FunctionVersion]:
|
||||
@@ -766,6 +762,14 @@ class RemoteDBConnection(DBConnection):
|
||||
"""List server-side jobs across the database's tables."""
|
||||
return LOOP.run(self._conn.list_jobs())
|
||||
|
||||
@override
|
||||
def get_job(self, job_id: str) -> Optional["JobDescription"]:
|
||||
"""Describe a single server-side job by id.
|
||||
|
||||
Returns None when the server has no such job.
|
||||
"""
|
||||
return LOOP.run(self._conn.get_job(job_id))
|
||||
|
||||
@override
|
||||
def cancel_job(self, job_id: str) -> bool:
|
||||
"""Request cancellation of a server-side job by id.
|
||||
@@ -777,35 +781,12 @@ class RemoteDBConnection(DBConnection):
|
||||
return LOOP.run(self._conn.cancel_job(job_id))
|
||||
|
||||
@override
|
||||
def execute_query_async(
|
||||
self,
|
||||
query: str,
|
||||
*,
|
||||
default_namespace_path: Optional[List[str]] = None,
|
||||
) -> SqlQuery:
|
||||
"""Start executing SQL through this remote connection.
|
||||
def job_history(self, job_id: Optional[str] = None) -> List[pa.RecordBatch]:
|
||||
"""The lifecycle event history of a server-side job, as Arrow batches.
|
||||
|
||||
Unqualified tables use this connection's database and the
|
||||
``["public"]`` namespace by default. Fully qualified table names may
|
||||
reference other databases available to the same deployment.
|
||||
Lists history across all jobs when `job_id` is None.
|
||||
"""
|
||||
return SqlQuery(
|
||||
LOOP.run(
|
||||
self._conn.execute_query_async(
|
||||
query,
|
||||
default_namespace_path=default_namespace_path,
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
@override
|
||||
def describe_query(self, query_id: UUID) -> QueryDescription:
|
||||
"""Describe a submitted SQL query by its connection-scoped id."""
|
||||
return LOOP.run(
|
||||
self._conn.describe_query(
|
||||
query_id,
|
||||
)
|
||||
)
|
||||
return LOOP.run(self._conn.job_history(job_id))
|
||||
|
||||
@override
|
||||
def namespace_client(self) -> LanceNamespace:
|
||||
|
||||
@@ -177,7 +177,4 @@ class OAuthProvider(HeaderProvider):
|
||||
if not self._current_token:
|
||||
raise RuntimeError("Failed to obtain OAuth token")
|
||||
|
||||
return {
|
||||
"Authorization": f"Bearer {self._current_token}",
|
||||
"x-lancedb-credential-type": "oidc",
|
||||
}
|
||||
return {"Authorization": f"Bearer {self._current_token}"}
|
||||
|
||||
@@ -1,88 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
"""Handles to SQL queries running on a remote database."""
|
||||
|
||||
from uuid import UUID
|
||||
|
||||
import pyarrow as pa
|
||||
|
||||
from lancedb.background_loop import LOOP
|
||||
|
||||
from . import _lancedb
|
||||
from .arrow import AsyncRecordBatchReader
|
||||
|
||||
QueryDescription = _lancedb.QueryDescription
|
||||
|
||||
|
||||
class AsyncQuery:
|
||||
"""A handle to a submitted SQL query on an asynchronous connection."""
|
||||
|
||||
def __init__(self, inner: "_lancedb.SqlQuery"):
|
||||
self._inner = inner
|
||||
|
||||
@property
|
||||
def id(self) -> UUID:
|
||||
"""The stable identifier scoped to the connection that submitted it."""
|
||||
return self._inner.id
|
||||
|
||||
async def describe(self) -> QueryDescription:
|
||||
"""Get a point-in-time description of the query."""
|
||||
return await self._inner.describe()
|
||||
|
||||
async def reader(self) -> AsyncRecordBatchReader:
|
||||
"""Wait for the initial result stream and return its Arrow reader.
|
||||
|
||||
Results are single-consumer. Calling this method more than once on the
|
||||
same query raises an error. Later batches are streamed as they become
|
||||
available without waiting for the full query to finish.
|
||||
"""
|
||||
return AsyncRecordBatchReader(await self._inner.reader())
|
||||
|
||||
async def cancel(self) -> None:
|
||||
"""Request cancellation of the query."""
|
||||
await self._inner.cancel()
|
||||
|
||||
|
||||
class Query:
|
||||
"""Synchronous counterpart of :class:`AsyncQuery`."""
|
||||
|
||||
def __init__(self, inner: AsyncQuery):
|
||||
self._inner = inner
|
||||
|
||||
@property
|
||||
def id(self) -> UUID:
|
||||
"""The stable identifier scoped to the connection that submitted it."""
|
||||
return self._inner.id
|
||||
|
||||
def describe(self) -> QueryDescription:
|
||||
"""Get a point-in-time description of the query."""
|
||||
return LOOP.run(self._inner.describe())
|
||||
|
||||
def reader(self) -> pa.RecordBatchReader:
|
||||
"""Wait for the initial result stream and return a blocking reader.
|
||||
|
||||
Results are single-consumer. Calling this method more than once on the
|
||||
same query raises an error. Later batches block only until they become
|
||||
available, without waiting for the full query to finish.
|
||||
"""
|
||||
reader = LOOP.run(self._inner.reader())
|
||||
|
||||
def next_batch():
|
||||
try:
|
||||
return LOOP.run(reader.__anext__())
|
||||
except StopAsyncIteration:
|
||||
return None
|
||||
|
||||
def batches():
|
||||
while (batch := next_batch()) is not None:
|
||||
yield batch
|
||||
|
||||
return pa.RecordBatchReader.from_batches(reader.schema, batches())
|
||||
|
||||
def cancel(self) -> None:
|
||||
"""Request cancellation of the query."""
|
||||
LOOP.run(self._inner.cancel())
|
||||
|
||||
|
||||
__all__ = ["AsyncQuery", "Query", "QueryDescription"]
|
||||
@@ -1793,9 +1793,6 @@ class Table(ABC):
|
||||
The result has the same length and order as ``row_ids``. Null blobs
|
||||
produce null slots; valid empty blobs produce ``b""``.
|
||||
|
||||
``_rowid`` values stay valid after compaction when the table has stable
|
||||
row ids.
|
||||
|
||||
Convenience for small payloads. For large values use
|
||||
:meth:`fetch_blob_files`.
|
||||
"""
|
||||
@@ -1813,9 +1810,6 @@ class Table(ABC):
|
||||
The result has the same length and order as ``requests``; null blobs
|
||||
produce null slots and empty ranges on non-null blobs produce ``b""``.
|
||||
|
||||
``_rowid`` values stay valid after compaction when the table has stable
|
||||
row ids.
|
||||
|
||||
Row IDs can be obtained from a query with ``with_row_id(True)``. This
|
||||
API is currently supported only by local tables.
|
||||
"""
|
||||
@@ -1831,9 +1825,6 @@ class Table(ABC):
|
||||
``_rowid`` or a ``_lance_row_id`` field on the blob descriptor. Null
|
||||
rows are ``None``. Remote tables require LanceDB Cloud server 0.5.0 or
|
||||
newer.
|
||||
|
||||
``_rowid`` values stay valid after compaction when the table has stable
|
||||
row ids.
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
|
||||
@@ -66,25 +66,6 @@ def _row_ids_by_id(table):
|
||||
return dict(zip(hits["id"].to_pylist(), hits["_rowid"].to_pylist()))
|
||||
|
||||
|
||||
def _assert_missing_blob_row_ids(exc_info):
|
||||
message = str(exc_info.value)
|
||||
assert "row ids" in message
|
||||
assert "rowaddr" not in message
|
||||
assert "fragment" not in message
|
||||
|
||||
|
||||
def _assert_fetch_apis_reject_missing_row_ids(table, row_ids):
|
||||
with pytest.raises(ValueError) as exc_info:
|
||||
table.fetch_blobs("image", row_ids)
|
||||
_assert_missing_blob_row_ids(exc_info)
|
||||
with pytest.raises(ValueError) as exc_info:
|
||||
table.fetch_blob_files("image", row_ids)
|
||||
_assert_missing_blob_row_ids(exc_info)
|
||||
with pytest.raises(ValueError) as exc_info:
|
||||
table.fetch_blob_ranges("image", [(row_id, 0, 1) for row_id in row_ids])
|
||||
_assert_missing_blob_row_ids(exc_info)
|
||||
|
||||
|
||||
def test_blob_factory_declares_v2_field():
|
||||
field = lancedb.blob("image")
|
||||
assert isinstance(field.type, pa.ExtensionType)
|
||||
@@ -297,10 +278,7 @@ def test_blob_v2_projection_sources_use_typed_column_name():
|
||||
|
||||
|
||||
def _legacy_v1_table(name):
|
||||
# Legacy v1 blob columns are only writable at file version <= 2.1.
|
||||
db = lancedb.connect(
|
||||
"memory:///", storage_options={"new_table_data_storage_version": "2.1"}
|
||||
)
|
||||
db = lancedb.connect("memory:///")
|
||||
schema = pa.schema(
|
||||
[
|
||||
pa.field("id", pa.int64()),
|
||||
@@ -713,25 +691,6 @@ def test_fetch_blobs_accepts_query_result():
|
||||
assert {blobs[i].as_py() for i in range(len(blobs))} == {b"gamma"}
|
||||
|
||||
|
||||
def test_fetch_blobs_after_compact_with_stable_row_ids(tmp_path):
|
||||
db = lancedb.connect(
|
||||
tmp_path, storage_options={"new_table_enable_stable_row_ids": "true"}
|
||||
)
|
||||
schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")])
|
||||
table = db.create_table("t", schema=schema)
|
||||
table.add([{"id": 1, "image": b"frag-one"}])
|
||||
table.add([{"id": 2, "image": b"frag-two"}])
|
||||
by_id = _row_ids_by_id(table)
|
||||
ids = [by_id[1], by_id[2]]
|
||||
|
||||
table.optimize()
|
||||
|
||||
blobs = table.fetch_blobs("image", ids)
|
||||
assert blobs.to_pylist() == [b"frag-one", b"frag-two"]
|
||||
ranges = table.fetch_blob_ranges("image", [(ids[0], 5, 3), (ids[1], 5, 3)])
|
||||
assert ranges.to_pylist() == [b"one", b"two"]
|
||||
|
||||
|
||||
def test_fetch_blobs_preserves_null_and_empty_values():
|
||||
table = _blob_table(
|
||||
"nulls",
|
||||
@@ -780,25 +739,8 @@ def test_fetch_blob_ranges_validates_requests():
|
||||
with pytest.raises(ValueError, match="offset \\+ length overflowed"):
|
||||
table.fetch_blob_ranges("image", [(row_id, 2**64 - 1, 1)])
|
||||
|
||||
with pytest.raises(ValueError) as exc_info:
|
||||
with pytest.raises(ValueError, match="row IDs"):
|
||||
table.fetch_blob_ranges("image", [(2**64 - 1, 0, 1)])
|
||||
_assert_missing_blob_row_ids(exc_info)
|
||||
|
||||
|
||||
def test_fetch_blob_apis_reject_missing_fragment_row_addr():
|
||||
table = _blob_table("missing_frag", [{"id": 1, "image": b"x"}])
|
||||
live = _row_ids_by_id(table)[1]
|
||||
_assert_fetch_apis_reject_missing_row_ids(table, [1 << 32, live])
|
||||
|
||||
|
||||
def test_fetch_blob_apis_reject_deleted_row_ids():
|
||||
table = _blob_table(
|
||||
"deleted_rows",
|
||||
[{"id": 1, "image": b"one"}, {"id": 2, "image": b"two"}],
|
||||
)
|
||||
by_id = _row_ids_by_id(table)
|
||||
table.delete("id = 2")
|
||||
_assert_fetch_apis_reject_missing_row_ids(table, [by_id[2], by_id[1]])
|
||||
|
||||
|
||||
def test_fetch_blob_ranges_empty_requests_returns_empty_array():
|
||||
|
||||
@@ -850,43 +850,6 @@ def test_named_struct_function_can_include_a_blob_result_field():
|
||||
]
|
||||
|
||||
|
||||
def test_named_struct_function_preserves_nullable_result_fields():
|
||||
@udf(
|
||||
input_schema=pa.schema([pa.field("value", pa.int64(), nullable=False)]),
|
||||
output_schema=pa.schema(
|
||||
[
|
||||
pa.field("result", pa.int64(), nullable=True),
|
||||
pa.field("failure_code", pa.int32(), nullable=False),
|
||||
]
|
||||
),
|
||||
)
|
||||
def nullable_result(value):
|
||||
return {"result": value, "failure_code": 0}
|
||||
|
||||
output = nullable_result.registration_request.signature.output
|
||||
assert [(field.name, field.nullable) for field in output.fields] == [
|
||||
("result", True),
|
||||
("failure_code", False),
|
||||
]
|
||||
|
||||
@udf(
|
||||
input_schema=pa.schema([pa.field("value", pa.int64(), nullable=False)]),
|
||||
output_schema=pa.schema(
|
||||
[
|
||||
pa.field("result", pa.int64(), nullable=True),
|
||||
pa.field("failure_code", pa.int32(), nullable=True),
|
||||
]
|
||||
),
|
||||
)
|
||||
def all_nullable(value):
|
||||
return {"result": value, "failure_code": None}
|
||||
|
||||
assert all(
|
||||
field.nullable
|
||||
for field in all_nullable.registration_request.signature.output.fields
|
||||
)
|
||||
|
||||
|
||||
def test_metadata_marked_blob_field_uses_the_semantic_type():
|
||||
extension = lancedb.blob("image", nullable=False).type
|
||||
storage = (
|
||||
|
||||
@@ -54,10 +54,7 @@ class TestOAuthProvider:
|
||||
provider = OAuthProvider(fetcher)
|
||||
headers = provider.get_headers()
|
||||
|
||||
assert headers == {
|
||||
"Authorization": "Bearer token123",
|
||||
"x-lancedb-credential-type": "oidc",
|
||||
}
|
||||
assert headers == {"Authorization": "Bearer token123"}
|
||||
assert provider._current_token == "token123"
|
||||
assert provider._token_expires_at is not None
|
||||
|
||||
@@ -76,20 +73,14 @@ class TestOAuthProvider:
|
||||
|
||||
# First call
|
||||
headers1 = provider.get_headers()
|
||||
assert headers1 == {
|
||||
"Authorization": "Bearer token1",
|
||||
"x-lancedb-credential-type": "oidc",
|
||||
}
|
||||
assert headers1 == {"Authorization": "Bearer token1"}
|
||||
|
||||
# Wait for token to expire
|
||||
time.sleep(1.1)
|
||||
|
||||
# Second call should refresh
|
||||
headers2 = provider.get_headers()
|
||||
assert headers2 == {
|
||||
"Authorization": "Bearer token2",
|
||||
"x-lancedb-credential-type": "oidc",
|
||||
}
|
||||
assert headers2 == {"Authorization": "Bearer token2"}
|
||||
assert call_count == 2
|
||||
|
||||
def test_no_expiry_info(self):
|
||||
@@ -101,18 +92,12 @@ class TestOAuthProvider:
|
||||
provider = OAuthProvider(fetcher)
|
||||
headers = provider.get_headers()
|
||||
|
||||
assert headers == {
|
||||
"Authorization": "Bearer permanent_token",
|
||||
"x-lancedb-credential-type": "oidc",
|
||||
}
|
||||
assert headers == {"Authorization": "Bearer permanent_token"}
|
||||
assert provider._token_expires_at is None
|
||||
|
||||
# Should not refresh on second call
|
||||
headers2 = provider.get_headers()
|
||||
assert headers2 == {
|
||||
"Authorization": "Bearer permanent_token",
|
||||
"x-lancedb-credential-type": "oidc",
|
||||
}
|
||||
assert headers2 == {"Authorization": "Bearer permanent_token"}
|
||||
|
||||
def test_missing_access_token(self):
|
||||
"""Test error handling when access_token is missing."""
|
||||
@@ -136,10 +121,7 @@ class TestOAuthProvider:
|
||||
provider = OAuthProvider(fetcher)
|
||||
headers = provider.get_headers()
|
||||
|
||||
assert headers == {
|
||||
"Authorization": "Bearer sync_token",
|
||||
"x-lancedb-credential-type": "oidc",
|
||||
}
|
||||
assert headers == {"Authorization": "Bearer sync_token"}
|
||||
|
||||
|
||||
class TestClientConfigIntegration:
|
||||
|
||||
@@ -193,13 +193,7 @@ class TestNamespaceConnection:
|
||||
),
|
||||
)
|
||||
|
||||
# Legacy v1 blob columns are only writable at file version <= 2.1.
|
||||
table = db.create_table(
|
||||
"blob_table",
|
||||
data,
|
||||
namespace_path=["test_ns"],
|
||||
storage_options={"new_table_data_storage_version": "2.1"},
|
||||
)
|
||||
table = db.create_table("blob_table", data, namespace_path=["test_ns"])
|
||||
df = table.to_pandas(blob_mode="lazy").sort_values("id")
|
||||
|
||||
blob = df["blob"].iloc[0]
|
||||
|
||||
@@ -40,10 +40,6 @@ from utils import exception_output
|
||||
from importlib.util import find_spec
|
||||
|
||||
|
||||
# Legacy v1 blob columns are only writable at file version <= 2.1.
|
||||
LEGACY_BLOB_STORAGE_OPTIONS = {"new_table_data_storage_version": "2.1"}
|
||||
|
||||
|
||||
def _blob_query_data():
|
||||
return pa.table(
|
||||
{
|
||||
@@ -123,17 +119,13 @@ def _assert_blob_bytes_projection(df):
|
||||
|
||||
def _blob_query_table(db, name, blob_schema):
|
||||
if blob_schema == "v1":
|
||||
return db.create_table(
|
||||
name, _blob_query_data(), storage_options=LEGACY_BLOB_STORAGE_OPTIONS
|
||||
)
|
||||
return db.create_table(name, _blob_query_data())
|
||||
return _create_blob_v2_query_table(db, name)
|
||||
|
||||
|
||||
async def _blob_query_table_async(db, name, blob_schema):
|
||||
if blob_schema == "v1":
|
||||
return await db.create_table(
|
||||
name, _blob_query_data(), storage_options=LEGACY_BLOB_STORAGE_OPTIONS
|
||||
)
|
||||
return await db.create_table(name, _blob_query_data())
|
||||
return await _create_blob_v2_query_table_async(db, name)
|
||||
|
||||
|
||||
@@ -283,9 +275,7 @@ async def test_query_to_pandas_kwargs(table, table_async):
|
||||
def test_plain_scan_query_to_pandas_blob_modes(tmp_db, blob_mode):
|
||||
pytest.importorskip("lance")
|
||||
table = tmp_db.create_table(
|
||||
f"test_query_to_pandas_blob_{blob_mode}",
|
||||
_blob_query_data(),
|
||||
storage_options=LEGACY_BLOB_STORAGE_OPTIONS,
|
||||
f"test_query_to_pandas_blob_{blob_mode}", _blob_query_data()
|
||||
)
|
||||
|
||||
df = (
|
||||
@@ -332,9 +322,7 @@ def test_plain_scan_query_to_pandas_blob_mode_does_not_collect_arrow(
|
||||
):
|
||||
pytest.importorskip("lance")
|
||||
table = tmp_db.create_table(
|
||||
"test_query_to_pandas_blob_no_arrow_collect",
|
||||
_blob_query_data(),
|
||||
storage_options=LEGACY_BLOB_STORAGE_OPTIONS,
|
||||
"test_query_to_pandas_blob_no_arrow_collect", _blob_query_data()
|
||||
)
|
||||
query = table.search().where("id = 1").select(["id", "blob"])
|
||||
|
||||
@@ -359,9 +347,7 @@ def test_plain_scan_query_to_pandas_blob_descriptions_flatten_uses_scanner(
|
||||
):
|
||||
pytest.importorskip("lance")
|
||||
table = tmp_db.create_table(
|
||||
"test_query_to_pandas_blob_desc_flatten",
|
||||
_blob_query_data(),
|
||||
storage_options=LEGACY_BLOB_STORAGE_OPTIONS,
|
||||
"test_query_to_pandas_blob_desc_flatten", _blob_query_data()
|
||||
)
|
||||
query = table.search().where("id = 1").select(["id", "blob"])
|
||||
|
||||
@@ -379,11 +365,7 @@ def test_plain_scan_query_to_pandas_blob_descriptions_flatten_uses_scanner(
|
||||
def test_plain_scan_query_to_pandas_scanner_state(tmp_db):
|
||||
pytest.importorskip("lance")
|
||||
data = _blob_query_data()
|
||||
table = tmp_db.create_table(
|
||||
"test_query_to_pandas_scanner_state",
|
||||
data.slice(0, 2),
|
||||
storage_options=LEGACY_BLOB_STORAGE_OPTIONS,
|
||||
)
|
||||
table = tmp_db.create_table("test_query_to_pandas_scanner_state", data.slice(0, 2))
|
||||
table.add(data.slice(2, 2))
|
||||
|
||||
fragments = table.to_lance().get_fragments()
|
||||
@@ -418,9 +400,7 @@ def test_plain_scan_query_to_pandas_scanner_state(tmp_db):
|
||||
async def test_async_plain_scan_query_to_pandas_blob_projection(tmp_db_async):
|
||||
pytest.importorskip("lance")
|
||||
table = await tmp_db_async.create_table(
|
||||
"test_async_query_to_pandas_blob_projection",
|
||||
_blob_query_data(),
|
||||
storage_options=LEGACY_BLOB_STORAGE_OPTIONS,
|
||||
"test_async_query_to_pandas_blob_projection", _blob_query_data()
|
||||
)
|
||||
|
||||
lazy_df = await (
|
||||
@@ -472,9 +452,7 @@ async def test_async_plain_scan_query_to_pandas_blob_mode_does_not_collect_arrow
|
||||
):
|
||||
pytest.importorskip("lance")
|
||||
table = await tmp_db_async.create_table(
|
||||
"test_async_query_to_pandas_blob_no_arrow_collect",
|
||||
_blob_query_data(),
|
||||
storage_options=LEGACY_BLOB_STORAGE_OPTIONS,
|
||||
"test_async_query_to_pandas_blob_no_arrow_collect", _blob_query_data()
|
||||
)
|
||||
query = table.query().where("id = 1").select(["id", "blob"])
|
||||
|
||||
@@ -496,11 +474,7 @@ async def test_async_plain_scan_query_to_pandas_blob_mode_does_not_collect_arrow
|
||||
|
||||
def test_vector_query_to_pandas_blob_mode_requires_native_path(tmp_db):
|
||||
pytest.importorskip("lance")
|
||||
table = tmp_db.create_table(
|
||||
"test_vector_query_blob_mode",
|
||||
_blob_query_data(),
|
||||
storage_options=LEGACY_BLOB_STORAGE_OPTIONS,
|
||||
)
|
||||
table = tmp_db.create_table("test_vector_query_blob_mode", _blob_query_data())
|
||||
|
||||
with pytest.raises(RuntimeError, match="Lance native pandas conversion"):
|
||||
table.search([1.0, 0.0]).select(["blob", "vector"]).limit(1).to_pandas(
|
||||
@@ -511,9 +485,7 @@ def test_vector_query_to_pandas_blob_mode_requires_native_path(tmp_db):
|
||||
def test_vector_query_to_pandas_blob_descriptions_requires_plain_scan(tmp_db):
|
||||
pytest.importorskip("lance")
|
||||
table = tmp_db.create_table(
|
||||
"test_vector_query_blob_descriptions",
|
||||
_blob_query_data(),
|
||||
storage_options=LEGACY_BLOB_STORAGE_OPTIONS,
|
||||
"test_vector_query_blob_descriptions", _blob_query_data()
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match="plain scan query"):
|
||||
|
||||
@@ -2467,7 +2467,7 @@ def test_remote_blob_byte_apis_not_supported_on_old_server():
|
||||
|
||||
|
||||
def test_remote_connection_jobs_surface():
|
||||
from lancedb.exceptions import JobFailedError, JobNotFoundError
|
||||
from lancedb.exceptions import JobFailedError
|
||||
|
||||
schema = pa.schema([("state", pa.string())])
|
||||
batch = pa.record_batch([pa.array(["created", "done"])], schema=schema)
|
||||
@@ -2475,7 +2475,6 @@ def test_remote_connection_jobs_surface():
|
||||
with pa.ipc.new_stream(sink, schema) as writer:
|
||||
writer.write_batch(batch)
|
||||
events_body = sink.getvalue().to_pybytes()
|
||||
query_events_payloads = []
|
||||
|
||||
def handler(request):
|
||||
content_len = int(request.headers.get("Content-Length", 0))
|
||||
@@ -2513,22 +2512,6 @@ def test_remote_connection_jobs_surface():
|
||||
request.end_headers()
|
||||
request.wfile.write(json.dumps(rsp).encode())
|
||||
elif request.path == "/v1/jobs/describe":
|
||||
if payload["job_id"] == "job-2":
|
||||
request.send_response(200)
|
||||
request.send_header("Content-Type", "application/json")
|
||||
request.end_headers()
|
||||
request.wfile.write(
|
||||
json.dumps(
|
||||
dict(
|
||||
job_id="job-2",
|
||||
job_type="refresh_column",
|
||||
job_state="DONE",
|
||||
creation_ms=2000,
|
||||
result=dict(rows_assigned=1000000, rows_failed=0),
|
||||
)
|
||||
).encode()
|
||||
)
|
||||
return
|
||||
if payload["job_id"] != "job-1":
|
||||
request.send_response(404)
|
||||
request.end_headers()
|
||||
@@ -2560,7 +2543,7 @@ def test_remote_connection_jobs_surface():
|
||||
request.end_headers()
|
||||
request.wfile.write(b'{"job_id": "job-1"}')
|
||||
elif request.path == "/v1/jobs/query_events":
|
||||
query_events_payloads.append(payload)
|
||||
assert payload["job_id"] == "job-1"
|
||||
request.send_response(200)
|
||||
request.send_header("Content-Type", "application/vnd.apache.arrow.stream")
|
||||
request.end_headers()
|
||||
@@ -2576,109 +2559,24 @@ def test_remote_connection_jobs_surface():
|
||||
assert jobs[0].table == "t1"
|
||||
assert jobs[1].state == "finished"
|
||||
|
||||
description = db.get_job("job-1")
|
||||
assert description.job_type == "create_index"
|
||||
assert description.state == "failed"
|
||||
assert json.loads(description.spec_json) == {"column": "vec"}
|
||||
assert description.failure.message == "worker died"
|
||||
assert description.failure.retryable is True
|
||||
assert db.get_job("missing") is None
|
||||
|
||||
assert db.cancel_job("job-1") is True
|
||||
assert db.cancel_job("missing") is False
|
||||
|
||||
# Opening a job hands back a populated handle; a missing one fails.
|
||||
with pytest.raises(JobNotFoundError, match="missing"):
|
||||
db.open_job("missing")
|
||||
finished = db.open_job("job-2")
|
||||
assert finished.state == "finished"
|
||||
assert finished.result == {"rows_assigned": 1000000, "rows_failed": 0}
|
||||
batches = db.job_history("job-1")
|
||||
assert len(batches) == 1
|
||||
assert batches[0].num_rows == 2
|
||||
assert batches[0].column("state").to_pylist() == ["created", "done"]
|
||||
|
||||
job = db.open_job("job-1")
|
||||
job = db.job("job-1")
|
||||
assert job.id == "job-1"
|
||||
# Opening already populated the handle.
|
||||
assert job.state == "failed"
|
||||
assert job.spec == {"column": "vec"}
|
||||
assert job.failure.message == "worker died"
|
||||
assert job.status() == "failed"
|
||||
with pytest.raises(JobFailedError, match="worker died"):
|
||||
job.wait(timeout=timedelta(seconds=5))
|
||||
|
||||
|
||||
def test_remote_job_handle_reports_its_own_detail():
|
||||
schema = pa.schema([("state", pa.string())])
|
||||
batch = pa.record_batch([pa.array(["claim_complete"])], schema=schema)
|
||||
sink = pa.BufferOutputStream()
|
||||
with pa.ipc.new_stream(sink, schema) as writer:
|
||||
writer.write_batch(batch)
|
||||
events_body = sink.getvalue().to_pybytes()
|
||||
event_payloads = []
|
||||
|
||||
def handler(request):
|
||||
content_len = int(request.headers.get("Content-Length", 0))
|
||||
body = request.rfile.read(content_len) if content_len > 0 else b""
|
||||
payload = json.loads(body) if body else {}
|
||||
if request.path == "/v1/jobs/describe":
|
||||
request.send_response(200)
|
||||
request.send_header("Content-Type", "application/json")
|
||||
request.end_headers()
|
||||
request.wfile.write(
|
||||
json.dumps(
|
||||
dict(
|
||||
job_id="job-1",
|
||||
job_type="refresh_column",
|
||||
job_state="DONE",
|
||||
creation_ms=2000,
|
||||
spec=dict(column="vec"),
|
||||
result=dict(rows_assigned=1000000),
|
||||
)
|
||||
).encode()
|
||||
)
|
||||
elif request.path == "/v1/jobs/query_events":
|
||||
event_payloads.append(payload)
|
||||
request.send_response(200)
|
||||
request.send_header("Content-Type", "application/vnd.apache.arrow.stream")
|
||||
request.end_headers()
|
||||
request.wfile.write(events_body)
|
||||
else:
|
||||
request.send_response(404)
|
||||
request.end_headers()
|
||||
|
||||
with mock_lancedb_connection(handler) as db:
|
||||
job = db.open_job("job-1")
|
||||
|
||||
# Opening populates the handle in the same round trip.
|
||||
assert job.state == "finished"
|
||||
job.refresh()
|
||||
assert job.job_type == "refresh_column"
|
||||
assert job.creation_ms == 2000
|
||||
assert job.spec == {"column": "vec"}
|
||||
assert job.result == {"rows_assigned": 1000000}
|
||||
assert job.failure is None
|
||||
# The JSON payloads stay reachable, but as internal APIs.
|
||||
assert json.loads(job._spec_json) == {"column": "vec"}
|
||||
assert json.loads(job._result_json) == {"rows_assigned": 1000000}
|
||||
|
||||
# print() shows everything the handle knows and nothing it does not.
|
||||
# print() lays every known field out on its own line, with the JSON
|
||||
# payloads indented rather than crammed onto one line.
|
||||
assert repr(job) == "\n".join(
|
||||
[
|
||||
"Job(",
|
||||
" id='job-1',",
|
||||
" state='finished',",
|
||||
" job_type='refresh_column',",
|
||||
" creation_ms=2000,",
|
||||
" spec={",
|
||||
' "column": "vec"',
|
||||
" },",
|
||||
" result={",
|
||||
' "rows_assigned": 1000000',
|
||||
" },",
|
||||
")",
|
||||
]
|
||||
)
|
||||
# Nothing it does not know shows up.
|
||||
assert "failure" not in repr(job)
|
||||
|
||||
events = job.events(filter="state = 'claim_complete'", limit=500)
|
||||
assert isinstance(events, pa.Table)
|
||||
assert events.column("state").to_pylist() == ["claim_complete"]
|
||||
# The handle supplies job_id; the caller only narrows the query.
|
||||
assert event_payloads[-1] == {
|
||||
"job_id": "job-1",
|
||||
"limit": 500,
|
||||
"filter": "state = 'claim_complete'",
|
||||
}
|
||||
|
||||
@@ -1,162 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
from uuid import UUID
|
||||
|
||||
import pytest
|
||||
import pyarrow as pa
|
||||
|
||||
import lancedb
|
||||
from lancedb import _lancedb
|
||||
from lancedb.arrow import AsyncRecordBatchReader
|
||||
from lancedb.db import AsyncConnection
|
||||
from lancedb.remote.db import RemoteDBConnection
|
||||
from lancedb.sql import AsyncQuery, Query
|
||||
|
||||
NIL_QUERY_ID = UUID(int=0)
|
||||
|
||||
|
||||
class FakeNativeQuery:
|
||||
id = UUID("0198f1b2-c3d4-7e5f-8123-456789abcdef")
|
||||
|
||||
async def reader(self):
|
||||
return pa.table({"value": [1, 2]})
|
||||
|
||||
|
||||
class FakeNativeConnection:
|
||||
async def execute_query_async(self, query, *, default_namespace_path=None):
|
||||
return FakeNativeQuery()
|
||||
|
||||
|
||||
class FakeAsyncConnection:
|
||||
async def execute_query_async(self, query, *, default_namespace_path=None):
|
||||
return AsyncQuery(FakeNativeQuery())
|
||||
|
||||
|
||||
def remote_connection(sql_host_override=None):
|
||||
return lancedb.connect(
|
||||
"db://analytics",
|
||||
api_key="test-key",
|
||||
host_override="http://localhost:10024",
|
||||
sql_host_override=sql_host_override,
|
||||
)
|
||||
|
||||
|
||||
def test_sql_is_connection_scoped():
|
||||
assert hasattr(lancedb, "sql")
|
||||
assert not callable(lancedb.sql)
|
||||
assert not hasattr(_lancedb, "sql")
|
||||
assert not hasattr(remote_connection(), "sql")
|
||||
assert hasattr(remote_connection(), "execute_query")
|
||||
assert hasattr(remote_connection(), "execute_query_async")
|
||||
assert hasattr(remote_connection(), "describe_query")
|
||||
|
||||
|
||||
def test_query_id_is_uuid():
|
||||
query = AsyncQuery(FakeNativeQuery())
|
||||
assert isinstance(query.id, UUID)
|
||||
assert Query(query).id == query.id
|
||||
|
||||
|
||||
def test_connection_serializes_sql_host_override():
|
||||
endpoint = "grpc+tls://sql.example.com:10026"
|
||||
restored = lancedb.deserialize_conn(
|
||||
remote_connection(sql_host_override=endpoint).serialize()
|
||||
)
|
||||
assert restored.sql_host_override == endpoint
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_sql_reader_is_record_batch_stream():
|
||||
reader = await AsyncQuery(FakeNativeQuery()).reader()
|
||||
assert isinstance(reader, AsyncRecordBatchReader)
|
||||
assert (await reader.read_all())[0].column(0).to_pylist() == [1, 2]
|
||||
|
||||
|
||||
def test_sync_sql_reader_is_record_batch_reader():
|
||||
reader = Query(AsyncQuery(FakeNativeQuery())).reader()
|
||||
assert isinstance(reader, pa.RecordBatchReader)
|
||||
assert reader.read_all().column(0).to_pylist() == [1, 2]
|
||||
|
||||
|
||||
def test_execute_query_returns_blocking_reader():
|
||||
connection = RemoteDBConnection.__new__(RemoteDBConnection)
|
||||
connection._conn = FakeAsyncConnection()
|
||||
reader = connection.execute_query("SELECT 1")
|
||||
assert isinstance(reader, pa.RecordBatchReader)
|
||||
assert reader.read_all().column(0).to_pylist() == [1, 2]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_execute_query_returns_async_reader():
|
||||
connection = AsyncConnection(FakeNativeConnection())
|
||||
reader = await connection.execute_query("SELECT 1")
|
||||
assert isinstance(reader, AsyncRecordBatchReader)
|
||||
assert (await reader.read_all())[0].column(0).to_pylist() == [1, 2]
|
||||
|
||||
|
||||
def test_local_connection_rejects_sql(tmp_path):
|
||||
connection = lancedb.connect(tmp_path)
|
||||
with pytest.raises(NotImplementedError, match="SQL"):
|
||||
connection.execute_query("SELECT 1")
|
||||
with pytest.raises(NotImplementedError, match="SQL"):
|
||||
connection.execute_query_async("SELECT 1")
|
||||
with pytest.raises(NotImplementedError, match="SQL"):
|
||||
connection.describe_query(NIL_QUERY_ID)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_local_async_connection_rejects_sql(tmp_path):
|
||||
connection = await lancedb.connect_async(tmp_path)
|
||||
with pytest.raises(NotImplementedError, match="SQL"):
|
||||
await connection.execute_query("SELECT 1")
|
||||
with pytest.raises(NotImplementedError, match="SQL"):
|
||||
await connection.execute_query_async("SELECT 1")
|
||||
with pytest.raises(NotImplementedError, match="SQL"):
|
||||
await connection.describe_query(NIL_QUERY_ID)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_namespace_connection_rejects_sql(tmp_path):
|
||||
connection = lancedb.connect_namespace_async("dir", {"root": str(tmp_path)})
|
||||
with pytest.raises(NotImplementedError, match="SQL"):
|
||||
await connection.execute_query("SELECT 1")
|
||||
with pytest.raises(NotImplementedError, match="SQL"):
|
||||
await connection.execute_query_async("SELECT 1")
|
||||
with pytest.raises(NotImplementedError, match="SQL"):
|
||||
await connection.describe_query(NIL_QUERY_ID)
|
||||
|
||||
|
||||
def test_describe_query_requires_uuid():
|
||||
with pytest.raises(TypeError, match="UUID"):
|
||||
remote_connection().describe_query(str(NIL_QUERY_ID))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"default_namespace_path",
|
||||
["public", ("public",), [1]],
|
||||
)
|
||||
def test_execute_query_async_requires_namespace_path_list(default_namespace_path):
|
||||
with pytest.raises(ValueError, match="default_namespace_path"):
|
||||
remote_connection().execute_query_async(
|
||||
"SELECT 1", default_namespace_path=default_namespace_path
|
||||
)
|
||||
|
||||
|
||||
def test_execute_query_async_rejects_invalid_endpoint():
|
||||
connection = remote_connection(sql_host_override="invalid://localhost")
|
||||
with pytest.raises(ValueError, match="sql_host_override"):
|
||||
connection.execute_query_async("SELECT 1")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"default_namespace_path",
|
||||
[[""], ["café"], ["pub\tlic"], ["events$raw"]],
|
||||
)
|
||||
def test_execute_query_async_rejects_invalid_namespace_components(
|
||||
default_namespace_path,
|
||||
):
|
||||
with pytest.raises(ValueError, match="default_namespace_path"):
|
||||
remote_connection().execute_query_async(
|
||||
"SELECT 1", default_namespace_path=default_namespace_path
|
||||
)
|
||||
@@ -64,23 +64,15 @@ async def _blob_v2_table_async(db: AsyncConnection, name: str):
|
||||
return table
|
||||
|
||||
|
||||
# Legacy v1 blob columns are only writable at file version <= 2.1.
|
||||
LEGACY_BLOB_STORAGE_OPTIONS = {"new_table_data_storage_version": "2.1"}
|
||||
|
||||
|
||||
def _blob_table(db: DBConnection, name: str, blob_schema: str):
|
||||
if blob_schema == "v1":
|
||||
return db.create_table(
|
||||
name, data=_blob_test_data(), storage_options=LEGACY_BLOB_STORAGE_OPTIONS
|
||||
)
|
||||
return db.create_table(name, data=_blob_test_data())
|
||||
return _blob_v2_table(db, name)
|
||||
|
||||
|
||||
async def _blob_table_async(db: AsyncConnection, name: str, blob_schema: str):
|
||||
if blob_schema == "v1":
|
||||
return await db.create_table(
|
||||
name, data=_blob_test_data(), storage_options=LEGACY_BLOB_STORAGE_OPTIONS
|
||||
)
|
||||
return await db.create_table(name, data=_blob_test_data())
|
||||
return await _blob_v2_table_async(db, name)
|
||||
|
||||
|
||||
@@ -155,11 +147,7 @@ def test_table_to_pandas_invalid_blob_mode_non_blob_table(tmp_db: DBConnection):
|
||||
@pytest.mark.parametrize("blob_mode", ["lazy", "bytes", "descriptions"])
|
||||
def test_table_to_pandas_blob_modes(tmp_db: DBConnection, blob_mode):
|
||||
pytest.importorskip("lance")
|
||||
table = tmp_db.create_table(
|
||||
f"test_to_pandas_blob_{blob_mode}",
|
||||
_blob_test_data(),
|
||||
storage_options=LEGACY_BLOB_STORAGE_OPTIONS,
|
||||
)
|
||||
table = tmp_db.create_table(f"test_to_pandas_blob_{blob_mode}", _blob_test_data())
|
||||
|
||||
df = table.to_pandas(blob_mode=blob_mode)
|
||||
|
||||
@@ -3971,7 +3959,7 @@ def test_stats(mem_db: DBConnection):
|
||||
print(f"{stats=}")
|
||||
assert stats == {
|
||||
# Full on-disk size of the data file, footer and metadata included.
|
||||
"total_bytes": 637,
|
||||
"total_bytes": 633,
|
||||
"num_rows": 2,
|
||||
"num_indices": 0,
|
||||
"fragment_stats": {
|
||||
|
||||
+36
-68
@@ -13,7 +13,11 @@ use crate::{
|
||||
runtime::future_into_py,
|
||||
table::Table,
|
||||
};
|
||||
use arrow::{datatypes::Schema, ffi_stream::ArrowArrayStreamReader, pyarrow::FromPyArrow};
|
||||
use arrow::{
|
||||
datatypes::Schema,
|
||||
ffi_stream::ArrowArrayStreamReader,
|
||||
pyarrow::{FromPyArrow, ToPyArrow},
|
||||
};
|
||||
use lancedb::{
|
||||
connection::Connection as LanceConnection,
|
||||
connection::NamespaceClientPushdownOperation,
|
||||
@@ -24,7 +28,7 @@ use pyo3::{
|
||||
Bound, FromPyObject, Py, PyAny, PyRef, PyResult, Python,
|
||||
exceptions::{PyRuntimeError, PyValueError},
|
||||
pyclass, pyfunction, pymethods,
|
||||
types::{PyAnyMethods, PyDict, PyDictMethods, PyList},
|
||||
types::{PyDict, PyDictMethods, PyList, PyListMethods},
|
||||
};
|
||||
|
||||
#[pyclass]
|
||||
@@ -82,24 +86,6 @@ impl Connection {
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_default_namespace_path(path: Option<Bound<'_, PyAny>>) -> PyResult<Vec<String>> {
|
||||
match path {
|
||||
Some(path) => {
|
||||
if !path.is_instance_of::<PyList>() {
|
||||
return Err(PyValueError::new_err(
|
||||
"Connection.execute_query_async default_namespace_path must be a list",
|
||||
));
|
||||
}
|
||||
path.extract::<Vec<String>>().map_err(|_| {
|
||||
PyValueError::new_err(
|
||||
"Connection.execute_query_async default_namespace_path components must be strings",
|
||||
)
|
||||
})
|
||||
}
|
||||
None => Ok(vec!["public".to_string()]),
|
||||
}
|
||||
}
|
||||
|
||||
#[pymethods]
|
||||
impl Connection {
|
||||
fn __repr__(&self) -> String {
|
||||
@@ -122,40 +108,6 @@ impl Connection {
|
||||
self.get_inner().map(|inner| inner.uri().to_string())
|
||||
}
|
||||
|
||||
#[pyo3(signature = (query, *, default_namespace_path=None))]
|
||||
pub fn execute_query_async<'a>(
|
||||
self_: PyRef<'a, Self>,
|
||||
query: String,
|
||||
default_namespace_path: Option<Bound<'_, PyAny>>,
|
||||
) -> PyResult<Bound<'a, PyAny>> {
|
||||
let inner = self_.get_inner()?.clone();
|
||||
let default_namespace_path = parse_default_namespace_path(default_namespace_path)?;
|
||||
future_into_py(self_.py(), async move {
|
||||
let operation = inner
|
||||
.execute_query_async(query)
|
||||
.default_namespace_path(default_namespace_path);
|
||||
operation
|
||||
.execute()
|
||||
.await
|
||||
.map(crate::sql::Query::new)
|
||||
.infer_error()
|
||||
})
|
||||
}
|
||||
|
||||
pub fn describe_query<'a>(
|
||||
self_: PyRef<'a, Self>,
|
||||
query_id: uuid::Uuid,
|
||||
) -> PyResult<Bound<'a, PyAny>> {
|
||||
let inner = self_.get_inner()?.clone();
|
||||
future_into_py(self_.py(), async move {
|
||||
inner
|
||||
.describe_query(query_id)
|
||||
.await
|
||||
.map(crate::sql::QueryDescription::from)
|
||||
.infer_error()
|
||||
})
|
||||
}
|
||||
|
||||
#[pyo3(signature = ())]
|
||||
pub fn get_read_consistency_interval(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
|
||||
let inner = self_.get_inner()?.clone();
|
||||
@@ -640,12 +592,9 @@ impl Connection {
|
||||
})
|
||||
}
|
||||
|
||||
pub fn open_job(self_: PyRef<'_, Self>, job_id: String) -> PyResult<Bound<'_, PyAny>> {
|
||||
let inner = self_.get_inner()?.clone();
|
||||
future_into_py(self_.py(), async move {
|
||||
let job = inner.open_job(&job_id).await.infer_error()?;
|
||||
Ok(crate::job::Job::new(job))
|
||||
})
|
||||
pub fn job(&self, job_id: String) -> PyResult<crate::job::Job> {
|
||||
let inner = self.get_inner()?.clone();
|
||||
Ok(crate::job::Job::new(inner.job(job_id).infer_error()?))
|
||||
}
|
||||
|
||||
pub fn create_function_async(
|
||||
@@ -715,16 +664,42 @@ impl Connection {
|
||||
})
|
||||
}
|
||||
|
||||
pub fn get_job(self_: PyRef<'_, Self>, job_id: String) -> PyResult<Bound<'_, PyAny>> {
|
||||
let inner = self_.get_inner()?.clone();
|
||||
future_into_py(self_.py(), async move {
|
||||
let description = inner.get_job(&job_id).await.infer_error()?;
|
||||
Ok(description.map(crate::job::JobDescription::from))
|
||||
})
|
||||
}
|
||||
|
||||
pub fn cancel_job(self_: PyRef<'_, Self>, job_id: String) -> PyResult<Bound<'_, PyAny>> {
|
||||
let inner = self_.get_inner()?.clone();
|
||||
future_into_py(self_.py(), async move {
|
||||
inner.cancel_job(&job_id).await.infer_error()
|
||||
})
|
||||
}
|
||||
|
||||
#[pyo3(signature = (job_id=None))]
|
||||
pub fn job_history(
|
||||
self_: PyRef<'_, Self>,
|
||||
job_id: Option<String>,
|
||||
) -> PyResult<Bound<'_, PyAny>> {
|
||||
let inner = self_.get_inner()?.clone();
|
||||
future_into_py(self_.py(), async move {
|
||||
let batches = inner.job_history(job_id.as_deref()).await.infer_error()?;
|
||||
Python::attach(|py| {
|
||||
let list = PyList::empty(py);
|
||||
for batch in batches {
|
||||
list.append(batch.to_pyarrow(py)?)?;
|
||||
}
|
||||
Ok(list.unbind())
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[pyfunction]
|
||||
#[pyo3(signature = (uri, api_key=None, region=None, host_override=None, sql_host_override=None, read_consistency_interval=None, client_config=None, storage_options=None, session=None, manifest_enabled=false, namespace_client_properties=None, oauth_config=None))]
|
||||
#[pyo3(signature = (uri, api_key=None, region=None, host_override=None, read_consistency_interval=None, client_config=None, storage_options=None, session=None, manifest_enabled=false, namespace_client_properties=None, oauth_config=None))]
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn connect(
|
||||
py: Python<'_>,
|
||||
@@ -732,7 +707,6 @@ pub fn connect(
|
||||
api_key: Option<String>,
|
||||
region: Option<String>,
|
||||
host_override: Option<String>,
|
||||
sql_host_override: Option<String>,
|
||||
read_consistency_interval: Option<f64>,
|
||||
client_config: Option<PyClientConfig>,
|
||||
storage_options: Option<HashMap<String, String>>,
|
||||
@@ -752,12 +726,6 @@ pub fn connect(
|
||||
if let Some(host_override) = host_override {
|
||||
builder = builder.host_override(&host_override);
|
||||
}
|
||||
#[cfg(feature = "remote")]
|
||||
if let Some(sql_host_override) = sql_host_override {
|
||||
builder = builder.sql_host_override(&sql_host_override);
|
||||
}
|
||||
#[cfg(not(feature = "remote"))]
|
||||
let _ = sql_host_override;
|
||||
if let Some(read_consistency_interval) = read_consistency_interval {
|
||||
let read_consistency_interval = Duration::from_secs_f64(read_consistency_interval);
|
||||
builder = builder.read_consistency_interval(read_consistency_interval);
|
||||
|
||||
@@ -114,12 +114,6 @@ impl<T> PythonErrorExt<T> for std::result::Result<T, LanceError> {
|
||||
.getattr(intern!(py, "JobCancelledError"))?;
|
||||
Err(PyErr::from_value(cls.call1((err.to_string(),))?))
|
||||
}),
|
||||
LanceError::JobNotFound { .. } => Python::attach(|py| {
|
||||
let cls = py
|
||||
.import(intern!(py, "lancedb.exceptions"))?
|
||||
.getattr(intern!(py, "JobNotFoundError"))?;
|
||||
Err(PyErr::from_value(cls.call1((err.to_string(),))?))
|
||||
}),
|
||||
_ => self.runtime_error(),
|
||||
},
|
||||
}
|
||||
|
||||
+9
-126
@@ -4,50 +4,11 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::runtime::future_into_py;
|
||||
use arrow::{
|
||||
datatypes::Schema,
|
||||
pyarrow::{IntoPyArrow, Table as PyArrowTable},
|
||||
};
|
||||
use lancedb::job::JobEventsRequest;
|
||||
use pyo3::{
|
||||
Bound, PyAny, PyRef, PyResult, Python,
|
||||
exceptions::PyValueError,
|
||||
pyclass, pymethods,
|
||||
types::{PyAnyMethods, PyDict, PyDictMethods},
|
||||
};
|
||||
use pyo3::{Bound, PyAny, PyRef, PyResult, pyclass, pymethods};
|
||||
use serde::Serialize;
|
||||
|
||||
use crate::error::PythonErrorExt;
|
||||
|
||||
const REPR_INDENT: &str = " ";
|
||||
|
||||
/// Parse a stored JSON payload into Python data. The bindings carry these as
|
||||
/// strings because that is what crosses the boundary cheaply; the public
|
||||
/// Python surface is the parsed form.
|
||||
fn parse_json_payload<'py>(
|
||||
py: Python<'py>,
|
||||
raw: Option<&str>,
|
||||
) -> PyResult<Option<Bound<'py, PyAny>>> {
|
||||
match raw {
|
||||
None => Ok(None),
|
||||
Some(raw) => Ok(Some(py.import("json")?.call_method1("loads", (raw,))?)),
|
||||
}
|
||||
}
|
||||
|
||||
/// A payload rendered as indented JSON, aligned under the field that holds it.
|
||||
fn pretty_json_payload(py: Python<'_>, raw: Option<&str>) -> PyResult<Option<String>> {
|
||||
let Some(parsed) = parse_json_payload(py, raw)? else {
|
||||
return Ok(None);
|
||||
};
|
||||
let kwargs = PyDict::new(py);
|
||||
kwargs.set_item("indent", 4)?;
|
||||
let rendered: String = py
|
||||
.import("json")?
|
||||
.call_method("dumps", (parsed,), Some(&kwargs))?
|
||||
.extract()?;
|
||||
Ok(Some(rendered.replace('\n', &format!("\n{REPR_INDENT}"))))
|
||||
}
|
||||
|
||||
#[pyclass]
|
||||
pub struct Job {
|
||||
inner: Arc<lancedb::Job<std::result::Result<Option<String>, String>>>,
|
||||
@@ -106,48 +67,6 @@ impl Job {
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
|
||||
pub fn refresh(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
|
||||
let inner = self_.inner.clone();
|
||||
future_into_py(self_.py(), async move {
|
||||
inner.refresh().await.infer_error()?;
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
|
||||
/// The last observed lifecycle state, without contacting the backend.
|
||||
#[getter]
|
||||
pub fn _state(&self) -> Option<String> {
|
||||
self.inner.state()
|
||||
}
|
||||
|
||||
/// The last observed server-side record. `None` for an in-process job.
|
||||
#[getter]
|
||||
pub fn _description(&self) -> Option<JobDescription> {
|
||||
self.inner.description().map(JobDescription::from)
|
||||
}
|
||||
|
||||
#[pyo3(signature = (*, limit=None, filter=None))]
|
||||
pub fn events(
|
||||
self_: PyRef<'_, Self>,
|
||||
limit: Option<u32>,
|
||||
filter: Option<String>,
|
||||
) -> PyResult<Bound<'_, PyAny>> {
|
||||
let inner = self_.inner.clone();
|
||||
let request = JobEventsRequest { limit, filter };
|
||||
future_into_py(self_.py(), async move {
|
||||
let batches = inner.events(request).await.infer_error()?;
|
||||
Python::attach(|py| {
|
||||
let schema = batches
|
||||
.first()
|
||||
.map(|batch| batch.schema())
|
||||
.unwrap_or_else(|| Arc::new(Schema::empty()));
|
||||
let table = PyArrowTable::try_new(batches, schema)
|
||||
.map_err(|err| PyValueError::new_err(err.to_string()))?;
|
||||
table.into_pyarrow(py).map(|table| table.unbind())
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// A row from `Connection.list_jobs`: one server-side job.
|
||||
@@ -202,7 +121,7 @@ impl JobFailureInfo {
|
||||
}
|
||||
}
|
||||
|
||||
/// The server-side record behind a `Job` handle.
|
||||
/// A described job from `Connection.get_job`.
|
||||
#[pyclass(get_all, skip_from_py_object)]
|
||||
#[derive(Clone)]
|
||||
pub struct JobDescription {
|
||||
@@ -210,49 +129,17 @@ pub struct JobDescription {
|
||||
job_type: String,
|
||||
state: String,
|
||||
creation_ms: i64,
|
||||
/// Internal: the wire form behind the `spec` property.
|
||||
_spec_json: Option<String>,
|
||||
/// Internal: the wire form behind the `result` property.
|
||||
_result_json: Option<String>,
|
||||
spec_json: Option<String>,
|
||||
failure: Option<JobFailureInfo>,
|
||||
}
|
||||
|
||||
#[pymethods]
|
||||
impl JobDescription {
|
||||
/// The job-type-specific specification it was submitted with.
|
||||
#[getter]
|
||||
fn spec<'py>(&self, py: Python<'py>) -> PyResult<Option<Bound<'py, PyAny>>> {
|
||||
parse_json_payload(py, self._spec_json.as_deref())
|
||||
}
|
||||
|
||||
/// The job-type-specific terminal result. `None` until the job succeeds.
|
||||
#[getter]
|
||||
fn result<'py>(&self, py: Python<'py>) -> PyResult<Option<Bound<'py, PyAny>>> {
|
||||
parse_json_payload(py, self._result_json.as_deref())
|
||||
}
|
||||
|
||||
fn __repr__(&self, py: Python<'_>) -> PyResult<String> {
|
||||
let mut fields = vec![
|
||||
format!("job_id={:?}", self.job_id),
|
||||
format!("job_type={:?}", self.job_type),
|
||||
format!("state={:?}", self.state),
|
||||
format!("creation_ms={}", self.creation_ms),
|
||||
];
|
||||
// Lay the payloads out as indented JSON, the same way the `Job` repr
|
||||
// does, so the two agree on how the same data looks.
|
||||
for (name, payload) in [("spec", &self._spec_json), ("result", &self._result_json)] {
|
||||
if let Some(rendered) = pretty_json_payload(py, payload.as_deref())? {
|
||||
fields.push(format!("{name}={rendered}"));
|
||||
}
|
||||
}
|
||||
if let Some(failure) = &self.failure {
|
||||
fields.push(format!("failure={}", failure.__repr__()));
|
||||
}
|
||||
let body = fields
|
||||
.iter()
|
||||
.map(|field| format!("\n{REPR_INDENT}{field},"))
|
||||
.collect::<String>();
|
||||
Ok(format!("JobDescription({body}\n)"))
|
||||
fn __repr__(&self) -> String {
|
||||
format!(
|
||||
"JobDescription(job_id={:?}, job_type={:?}, state={:?}, creation_ms={})",
|
||||
self.job_id, self.job_type, self.state, self.creation_ms
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -263,11 +150,7 @@ impl From<lancedb::database::JobDescription> for JobDescription {
|
||||
job_type: description.job_type,
|
||||
state: description.state,
|
||||
creation_ms: description.creation_ms,
|
||||
_spec_json: (!description.spec.is_null()).then(|| description.spec.to_string()),
|
||||
_result_json: description
|
||||
.result
|
||||
.filter(|result| !result.is_null())
|
||||
.map(|result| result.to_string()),
|
||||
spec_json: (!description.spec.is_null()).then(|| description.spec.to_string()),
|
||||
failure: description.failure.map(|failure| JobFailureInfo {
|
||||
phase: failure.phase,
|
||||
message: failure.message,
|
||||
|
||||
@@ -34,7 +34,6 @@ pub mod permutation;
|
||||
pub mod query;
|
||||
pub mod runtime;
|
||||
pub mod session;
|
||||
pub mod sql;
|
||||
pub mod table;
|
||||
pub mod util;
|
||||
|
||||
@@ -51,8 +50,6 @@ pub fn _lancedb(_py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> {
|
||||
m.add_class::<crate::job::JobInfo>()?;
|
||||
m.add_class::<crate::job::JobDescription>()?;
|
||||
m.add_class::<crate::job::JobFailureInfo>()?;
|
||||
m.add_class::<crate::sql::Query>()?;
|
||||
m.add_class::<crate::sql::QueryDescription>()?;
|
||||
m.add_class::<PyBlobFile>()?;
|
||||
m.add_class::<IndexConfig>()?;
|
||||
m.add_class::<Query>()?;
|
||||
|
||||
@@ -1,90 +0,0 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use pyo3::{Bound, PyAny, PyRef, PyResult, pyclass, pymethods};
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::arrow::RecordBatchStream;
|
||||
use crate::error::PythonErrorExt;
|
||||
use crate::runtime::future_into_py;
|
||||
|
||||
#[pyclass(name = "SqlQuery")]
|
||||
pub struct Query {
|
||||
inner: Arc<lancedb::sql::Query>,
|
||||
}
|
||||
|
||||
impl Query {
|
||||
pub(crate) fn new(inner: lancedb::sql::Query) -> Self {
|
||||
Self {
|
||||
inner: Arc::new(inner),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[pymethods]
|
||||
impl Query {
|
||||
#[getter]
|
||||
pub fn id(&self) -> Uuid {
|
||||
self.inner.id()
|
||||
}
|
||||
|
||||
pub fn describe(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
|
||||
let inner = self_.inner.clone();
|
||||
future_into_py(self_.py(), async move {
|
||||
inner
|
||||
.describe()
|
||||
.await
|
||||
.map(QueryDescription::from)
|
||||
.infer_error()
|
||||
})
|
||||
}
|
||||
|
||||
pub fn reader(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
|
||||
let inner = self_.inner.clone();
|
||||
future_into_py(self_.py(), async move {
|
||||
let stream = inner.reader().await.infer_error()?;
|
||||
Ok(RecordBatchStream::new(stream))
|
||||
})
|
||||
}
|
||||
|
||||
pub fn cancel(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
|
||||
let inner = self_.inner.clone();
|
||||
future_into_py(self_.py(), async move {
|
||||
inner.cancel().await.infer_error()?;
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[pyclass(get_all, skip_from_py_object)]
|
||||
#[derive(Clone)]
|
||||
pub struct QueryDescription {
|
||||
id: Uuid,
|
||||
status: String,
|
||||
progress: Option<f64>,
|
||||
expires_at: Option<DateTime<Utc>>,
|
||||
}
|
||||
|
||||
#[pymethods]
|
||||
impl QueryDescription {
|
||||
fn __repr__(&self) -> String {
|
||||
format!(
|
||||
"QueryDescription(id={:?}, status={:?}, progress={:?}, expires_at={:?})",
|
||||
self.id, self.status, self.progress, self.expires_at
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<lancedb::sql::QueryDescription> for QueryDescription {
|
||||
fn from(description: lancedb::sql::QueryDescription) -> Self {
|
||||
Self {
|
||||
id: description.id,
|
||||
status: description.status.to_string(),
|
||||
progress: description.progress,
|
||||
expires_at: description.expires_at,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "lancedb"
|
||||
version = "0.39.0-beta.6"
|
||||
version = "0.39.0-beta.1"
|
||||
edition.workspace = true
|
||||
description = "LanceDB: A serverless, low-latency vector database for AI applications"
|
||||
license.workspace = true
|
||||
@@ -21,8 +21,6 @@ arrow-select = { workspace = true }
|
||||
arrow-ord = { workspace = true }
|
||||
arrow-cast = { workspace = true }
|
||||
arrow-ipc.workspace = true
|
||||
arrow-flight = { workspace = true, optional = true }
|
||||
prost = { version = "0.14", optional = true }
|
||||
chrono = { workspace = true }
|
||||
datafusion-catalog.workspace = true
|
||||
datafusion-common.workspace = true
|
||||
@@ -79,7 +77,6 @@ reqwest = { version = "0.12.0", default-features = false, features = [
|
||||
"rustls-tls-native-roots",
|
||||
"stream",
|
||||
], optional = true }
|
||||
tonic = { workspace = true, optional = true }
|
||||
http = { version = "1", optional = true } # Matching what is in reqwest
|
||||
urlencoding = { version = "2", optional = true }
|
||||
uuid = { workspace = true, features = ["v5"] }
|
||||
@@ -148,11 +145,8 @@ huggingface = [
|
||||
]
|
||||
dynamodb = ["lance/dynamodb", "aws"]
|
||||
remote = [
|
||||
"dep:arrow-flight",
|
||||
"dep:prost",
|
||||
"dep:reqwest",
|
||||
"dep:http",
|
||||
"dep:tonic",
|
||||
"dep:urlencoding",
|
||||
"lance-namespace-impls/rest",
|
||||
"lance-namespace-impls/rest-adapter",
|
||||
|
||||
@@ -7,9 +7,7 @@
|
||||
//! raw `Binary` / `LargeBinary` into the blob struct layout. Queries return
|
||||
//! small descriptors, not bytes.
|
||||
//!
|
||||
//! Blob tables require Lance file format >= 2.2. `_rowid` values stay valid
|
||||
//! after compaction when the table has stable row ids. Overwrite is a new
|
||||
//! create and does not keep the previous table's stable row id setting.
|
||||
//! Blob tables require Lance file format >= 2.2 and stable row ids at create.
|
||||
|
||||
use std::ops::Range;
|
||||
use std::sync::Arc;
|
||||
@@ -326,7 +324,6 @@ pub(crate) fn blob_column_names(schema: &Schema) -> Vec<String> {
|
||||
}
|
||||
|
||||
/// Bumps storage format to at least [`LanceFileVersion::V2_2`] for blob schemas.
|
||||
/// Leaves `enable_stable_row_ids` unchanged.
|
||||
pub(crate) fn ensure_blob_storage_version(schema: &Schema, params: &mut WriteParams) {
|
||||
if !has_blob_columns(schema) {
|
||||
return;
|
||||
@@ -388,30 +385,6 @@ fn ensure_all_row_ids_resolved(column: &str, requested: usize, resolved: usize)
|
||||
}
|
||||
}
|
||||
|
||||
/// Lance take reports a missing physical row address as NotSupported or InvalidInput.
|
||||
fn map_blob_take_error(column: &str, requested: usize, err: lance::Error) -> Error {
|
||||
let missing_row_addr = match &err {
|
||||
lance::Error::NotSupported { source, .. } => {
|
||||
source.to_string().contains("must not target deleted rows")
|
||||
}
|
||||
lance::Error::InvalidInput { source, .. } => source
|
||||
.to_string()
|
||||
.contains("belongs to non-existent fragment"),
|
||||
_ => false,
|
||||
};
|
||||
|
||||
if missing_row_addr {
|
||||
Error::InvalidInput {
|
||||
message: format!(
|
||||
"blob read for column '{column}' requested {requested} row ids but some \
|
||||
do not exist in the table; pass row ids collected from this table"
|
||||
),
|
||||
}
|
||||
} else {
|
||||
err.into()
|
||||
}
|
||||
}
|
||||
|
||||
/// Materialize blob-local ranges (same length and order as `requests`, nulls preserved).
|
||||
pub(crate) async fn take_blob_ranges_aligned(
|
||||
dataset: &Arc<Dataset>,
|
||||
@@ -432,8 +405,7 @@ pub(crate) async fn take_blob_ranges_aligned(
|
||||
.with_row_ids(lance_requests)
|
||||
.preserve_order(true)
|
||||
.execute()
|
||||
.await
|
||||
.map_err(|err| map_blob_take_error(column, requests.len(), err))?;
|
||||
.await?;
|
||||
ensure_all_row_ids_resolved(column, requests.len(), payloads.len())?;
|
||||
|
||||
let mut builder = LargeBinaryBuilder::new();
|
||||
@@ -462,8 +434,7 @@ pub(crate) async fn take_blobs_aligned(
|
||||
.with_row_ids(row_ids.to_vec())
|
||||
.preserve_order(true)
|
||||
.execute()
|
||||
.await
|
||||
.map_err(|err| map_blob_take_error(column, row_ids.len(), err))?;
|
||||
.await?;
|
||||
ensure_all_row_ids_resolved(column, row_ids.len(), payloads.len())?;
|
||||
|
||||
let mut builder = LargeBinaryBuilder::new();
|
||||
@@ -487,10 +458,7 @@ pub(crate) async fn take_blob_files_aligned(
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
|
||||
let handles = dataset
|
||||
.take_blobs(row_ids, column)
|
||||
.await
|
||||
.map_err(|err| map_blob_take_error(column, row_ids.len(), err))?;
|
||||
let handles = dataset.take_blobs(row_ids, column).await?;
|
||||
ensure_all_row_ids_resolved(column, row_ids.len(), handles.len())?;
|
||||
Ok(handles
|
||||
.into_iter()
|
||||
@@ -532,27 +500,10 @@ mod tests {
|
||||
fn storage_version_bumps_to_v2_2() {
|
||||
let mut params = WriteParams::default();
|
||||
ensure_blob_storage_version(&blob_schema(), &mut params);
|
||||
let resolved = params
|
||||
.data_storage_version
|
||||
.unwrap_or(LanceFileVersion::Stable)
|
||||
.resolve();
|
||||
assert_eq!(resolved, ConcreteFileVersion::V2_2);
|
||||
assert!(!params.enable_stable_row_ids);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn storage_version_leaves_stable_row_ids_enabled() {
|
||||
let mut params = WriteParams {
|
||||
enable_stable_row_ids: true,
|
||||
..Default::default()
|
||||
};
|
||||
ensure_blob_storage_version(&blob_schema(), &mut params);
|
||||
assert!(params.enable_stable_row_ids);
|
||||
let resolved = params
|
||||
.data_storage_version
|
||||
.unwrap_or(LanceFileVersion::Stable)
|
||||
.resolve();
|
||||
assert_eq!(resolved, ConcreteFileVersion::V2_2);
|
||||
assert_eq!(
|
||||
params.data_storage_version.unwrap().resolve(),
|
||||
ConcreteFileVersion::V2_2
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -625,6 +576,5 @@ mod tests {
|
||||
let mut params = WriteParams::default();
|
||||
ensure_blob_storage_version(&schema, &mut params);
|
||||
assert!(params.data_storage_version.is_none());
|
||||
assert!(!params.enable_stable_row_ids);
|
||||
}
|
||||
}
|
||||
|
||||
+23
-151
@@ -23,18 +23,15 @@ use crate::connection::create_table::CreateTableBuilder;
|
||||
use crate::data::scannable::Scannable;
|
||||
use crate::database::listing::ListingDatabase;
|
||||
use crate::database::{
|
||||
CloneTableRequest, Database, DatabaseOptions, JobInfo, OpenTableRequest, ReadConsistency,
|
||||
TableNamesRequest,
|
||||
CloneTableRequest, Database, DatabaseOptions, JobDescription, JobInfo, OpenTableRequest,
|
||||
ReadConsistency, TableNamesRequest,
|
||||
};
|
||||
use crate::embeddings::{EmbeddingRegistry, MemoryRegistry};
|
||||
use crate::error::{Error, Result};
|
||||
#[cfg(feature = "remote")]
|
||||
use crate::remote::{
|
||||
client::ClientConfig,
|
||||
db::{
|
||||
OPT_REMOTE_API_KEY, OPT_REMOTE_HOST_OVERRIDE, OPT_REMOTE_REGION,
|
||||
OPT_REMOTE_SQL_HOST_OVERRIDE,
|
||||
},
|
||||
db::{OPT_REMOTE_API_KEY, OPT_REMOTE_HOST_OVERRIDE, OPT_REMOTE_REGION},
|
||||
};
|
||||
use lance::io::ObjectStoreParams;
|
||||
pub use lance_file::version::LanceFileVersion;
|
||||
@@ -325,43 +322,6 @@ pub struct CloneTableBuilder {
|
||||
request: CloneTableRequest,
|
||||
}
|
||||
|
||||
/// Builder for asynchronously executing a SQL statement on a remote database.
|
||||
pub struct ExecuteQueryAsyncBuilder {
|
||||
parent: Arc<dyn Database>,
|
||||
query: String,
|
||||
default_namespace_path: Vec<String>,
|
||||
}
|
||||
|
||||
impl ExecuteQueryAsyncBuilder {
|
||||
fn new(parent: Arc<dyn Database>, query: String) -> Self {
|
||||
Self {
|
||||
parent,
|
||||
query,
|
||||
default_namespace_path: vec!["public".to_string()],
|
||||
}
|
||||
}
|
||||
|
||||
/// Set the namespace used for unqualified table names.
|
||||
///
|
||||
/// An empty path is treated as `public`, which is the SQL name for the
|
||||
/// root Lance namespace.
|
||||
pub fn default_namespace_path<I, S>(mut self, path: I) -> Self
|
||||
where
|
||||
I: IntoIterator<Item = S>,
|
||||
S: Into<String>,
|
||||
{
|
||||
self.default_namespace_path = path.into_iter().map(Into::into).collect();
|
||||
self
|
||||
}
|
||||
|
||||
/// Start the statement and return its asynchronous query handle.
|
||||
pub async fn execute(self) -> Result<crate::sql::Query> {
|
||||
self.parent
|
||||
.execute_query_async(&self.query, &self.default_namespace_path)
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
impl CloneTableBuilder {
|
||||
fn new(parent: Arc<dyn Database>, target_table_name: String, source_uri: String) -> Self {
|
||||
Self {
|
||||
@@ -445,51 +405,6 @@ impl Connection {
|
||||
&self.internal
|
||||
}
|
||||
|
||||
/// Start executing SQL on a remote LanceDB database.
|
||||
///
|
||||
/// The query can reference tables in other databases with SQL dot notation.
|
||||
/// Use [`ExecuteQueryAsyncBuilder::default_namespace_path`] to avoid qualifying
|
||||
/// tables in the default namespace. Local connections return
|
||||
/// [`Error::NotSupported`].
|
||||
///
|
||||
/// # Example
|
||||
///
|
||||
/// ```no_run
|
||||
/// # async fn query(db: &lancedb::Connection) -> lancedb::Result<()> {
|
||||
/// use futures::TryStreamExt;
|
||||
///
|
||||
/// let query = db
|
||||
/// .execute_query_async("SELECT * FROM events LIMIT 10")
|
||||
/// .default_namespace_path(["public"])
|
||||
/// .execute()
|
||||
/// .await?;
|
||||
/// println!("query id: {}", query.id());
|
||||
/// let mut batches = query.reader().await?;
|
||||
/// while let Some(batch) = batches.try_next().await? {
|
||||
/// println!("received {} rows", batch.num_rows());
|
||||
/// }
|
||||
/// # Ok(())
|
||||
/// # }
|
||||
/// ```
|
||||
pub fn execute_query_async(&self, query: impl Into<String>) -> ExecuteQueryAsyncBuilder {
|
||||
ExecuteQueryAsyncBuilder::new(self.internal.clone(), query.into())
|
||||
}
|
||||
|
||||
/// Describe a submitted SQL query by its connection-scoped id.
|
||||
///
|
||||
/// This performs one bounded status poll using state retained by this
|
||||
/// connection. Running state with a live query handle is not evicted;
|
||||
/// abandoned state has bounded retention, and server expiration is
|
||||
/// honored. Terminal state is retained briefly.
|
||||
/// Query ids are not portable to another connection. Local connections
|
||||
/// return [`Error::NotSupported`].
|
||||
pub async fn describe_query(
|
||||
&self,
|
||||
query_id: uuid::Uuid,
|
||||
) -> Result<crate::sql::QueryDescription> {
|
||||
self.internal.describe_query(query_id).await
|
||||
}
|
||||
|
||||
/// Get the names of all tables in the database
|
||||
///
|
||||
/// The names will be returned in lexicographical order (ascending)
|
||||
@@ -670,34 +585,14 @@ impl Connection {
|
||||
self.internal.read_consistency().await
|
||||
}
|
||||
|
||||
/// Open a server-side job by id, returning a handle with its record
|
||||
/// already populated. Fails with [`crate::Error::JobNotFound`] when the
|
||||
/// server has no such job, the way [`Connection::open_table`] does for a
|
||||
/// missing table.
|
||||
/// A [`crate::job::Job`] handle for a server-side job by id, suitable for
|
||||
/// waiting on or cancelling the job.
|
||||
///
|
||||
/// This is the one way in: the returned [`crate::job::Job`] answers for
|
||||
/// its own state, specification, result, failure and event history, so
|
||||
/// there is no separate connection-level call for any of them.
|
||||
///
|
||||
/// # Example
|
||||
///
|
||||
/// ```no_run
|
||||
/// # use lancedb::job::JobEventsRequest;
|
||||
/// # async fn open_job(
|
||||
/// # connection: &lancedb::Connection,
|
||||
/// # job_id: &str,
|
||||
/// # ) -> Result<(), Box<dyn std::error::Error>> {
|
||||
/// let job = connection.open_job(job_id).await?;
|
||||
/// println!("{:?} {:?}", job.state(), job.result());
|
||||
/// let done = job
|
||||
/// .events(JobEventsRequest::default().filter("state = 'claim_complete'"))
|
||||
/// .await?;
|
||||
/// println!("{} completions", done.iter().map(|b| b.num_rows()).sum::<usize>());
|
||||
/// # Ok(())
|
||||
/// # }
|
||||
/// ```
|
||||
pub async fn open_job(&self, job_id: impl AsRef<str>) -> Result<crate::job::Job> {
|
||||
self.internal.open_job(job_id.as_ref()).await
|
||||
/// The handle is constructed without a server round trip; an unknown id
|
||||
/// surfaces when the handle is used. Only server-backed databases support
|
||||
/// job handles by id.
|
||||
pub fn job(&self, job_id: impl AsRef<str>) -> Result<crate::job::Job> {
|
||||
self.internal.job(job_id.as_ref())
|
||||
}
|
||||
|
||||
/// List server-side jobs across the database's tables.
|
||||
@@ -705,12 +600,24 @@ impl Connection {
|
||||
self.internal.list_jobs().await
|
||||
}
|
||||
|
||||
/// Describe a single server-side job by id. `None` when the server has no
|
||||
/// such job.
|
||||
pub async fn get_job(&self, job_id: impl AsRef<str>) -> Result<Option<JobDescription>> {
|
||||
self.internal.get_job(job_id.as_ref()).await
|
||||
}
|
||||
|
||||
/// Request cancellation of a server-side job by id. Returns true if the
|
||||
/// server accepted the cancellation, false if no such job exists.
|
||||
pub async fn cancel_job(&self, job_id: impl AsRef<str>) -> Result<bool> {
|
||||
self.internal.cancel_job(job_id.as_ref()).await
|
||||
}
|
||||
|
||||
/// The lifecycle event history of a server-side job (all jobs when
|
||||
/// `job_id` is `None`), as recorded Arrow batches.
|
||||
pub async fn job_history(&self, job_id: Option<&str>) -> Result<Vec<RecordBatch>> {
|
||||
self.internal.job_history(job_id).await
|
||||
}
|
||||
|
||||
/// Drop a table in the database.
|
||||
///
|
||||
/// # Arguments
|
||||
@@ -957,19 +864,6 @@ impl ConnectBuilder {
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the SQL service host override for a remote connection.
|
||||
///
|
||||
/// The SQL client is initialized lazily when the connection first executes
|
||||
/// SQL and is retained for the connection's lifetime.
|
||||
#[cfg(feature = "remote")]
|
||||
pub fn sql_host_override(mut self, sql_host_override: &str) -> Self {
|
||||
self.request.options.insert(
|
||||
OPT_REMOTE_SQL_HOST_OVERRIDE.to_string(),
|
||||
sql_host_override.to_string(),
|
||||
);
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the database specific options
|
||||
///
|
||||
/// See [crate::database::listing::ListingDatabaseOptions] for the options available for
|
||||
@@ -1159,7 +1053,6 @@ impl ConnectBuilder {
|
||||
|
||||
let mut merged_options = self.request.options.clone();
|
||||
Self::apply_env_defaults(&ENV_VARS_TO_STORAGE_OPTS, &mut merged_options);
|
||||
let sql_host_override = merged_options.get(OPT_REMOTE_SQL_HOST_OVERRIDE).cloned();
|
||||
let options = RemoteDatabaseOptions::parse_from_map(&merged_options)?;
|
||||
|
||||
let region = options.region.ok_or_else(|| Error::InvalidInput {
|
||||
@@ -1201,15 +1094,11 @@ impl ConnectBuilder {
|
||||
}
|
||||
|
||||
let storage_options = StorageOptions(options.storage_options.clone());
|
||||
let host_overrides = crate::remote::db::RemoteHostOverrides {
|
||||
rest: options.host_override,
|
||||
sql: sql_host_override,
|
||||
};
|
||||
let internal = Arc::new(crate::remote::db::RemoteDatabase::try_new(
|
||||
&self.request.uri,
|
||||
&api_key,
|
||||
®ion,
|
||||
host_overrides,
|
||||
options.host_override,
|
||||
client_config,
|
||||
storage_options.into(),
|
||||
self.request.read_consistency_interval,
|
||||
@@ -1503,23 +1392,6 @@ mod tests {
|
||||
assert_eq!(tc.connection.uri(), tc.uri);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_local_connection_rejects_sql_queries() {
|
||||
let directory = tempdir().unwrap();
|
||||
let connection = connect(directory.path().to_str().unwrap())
|
||||
.execute()
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(matches!(
|
||||
connection.execute_query_async("SELECT 1").execute().await,
|
||||
Err(Error::NotSupported { .. })
|
||||
));
|
||||
assert!(matches!(
|
||||
connection.describe_query(uuid::Uuid::nil()).await,
|
||||
Err(Error::NotSupported { .. })
|
||||
));
|
||||
}
|
||||
|
||||
#[cfg(feature = "remote")]
|
||||
#[test]
|
||||
fn test_apply_env_defaults() {
|
||||
|
||||
@@ -18,6 +18,8 @@ use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use arrow_array::RecordBatch;
|
||||
|
||||
use lance::dataset::ReadParams;
|
||||
use lance_namespace::LanceNamespace;
|
||||
use lance_namespace::models::{
|
||||
@@ -204,8 +206,8 @@ pub enum ReadConsistency {
|
||||
/// compaction, column refresh, ...).
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct JobInfo {
|
||||
/// The job id -- what [`Database::open_job`] and
|
||||
/// [`Database::cancel_job`] accept.
|
||||
/// The job id -- what [`Database::get_job`] and [`Database::cancel_job`]
|
||||
/// accept.
|
||||
pub job_id: String,
|
||||
/// The table the job runs against, without URI or namespace.
|
||||
pub table: String,
|
||||
@@ -216,8 +218,8 @@ pub struct JobInfo {
|
||||
pub created_at_millis: i64,
|
||||
}
|
||||
|
||||
/// The server-side record behind a [`crate::job::Job`] handle: lifecycle
|
||||
/// state plus the job-type-specific specification and result.
|
||||
/// A described job from [`Database::get_job`]: lifecycle state plus the
|
||||
/// job-type-specific specification.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct JobDescription {
|
||||
pub job_id: String,
|
||||
@@ -228,10 +230,6 @@ pub struct JobDescription {
|
||||
pub creation_ms: i64,
|
||||
/// The job-type-specific specification. Null when the server omits it.
|
||||
pub spec: serde_json::Value,
|
||||
/// The job-type-specific terminal result, for job types that define one.
|
||||
/// `None` until the job succeeds, so a job that never terminates reports
|
||||
/// its progress through [`crate::job::Job::events`] instead.
|
||||
pub result: Option<serde_json::Value>,
|
||||
/// Why the job failed, when the job is failed and the server reports a
|
||||
/// reason.
|
||||
pub failure: Option<crate::error::JobFailure>,
|
||||
@@ -317,37 +315,30 @@ pub trait Database:
|
||||
async fn drop_function(&self, _name: &str, _version: &str) -> Result<bool> {
|
||||
function_catalog_not_supported()
|
||||
}
|
||||
/// Open a job by id, returning a handle with its record already
|
||||
/// populated. Fails with [`crate::Error::JobNotFound`] when the server has
|
||||
/// no such job.
|
||||
async fn open_job(&self, _job_id: &str) -> Result<crate::job::Job> {
|
||||
job_op_not_supported("open_job")
|
||||
/// A [`crate::job::Job`] handle for a server-side job by id, suitable for
|
||||
/// waiting on or cancelling the job. The handle is constructed without a
|
||||
/// server round trip; an unknown id surfaces when the handle is used.
|
||||
fn job(&self, _job_id: &str) -> Result<crate::job::Job> {
|
||||
job_op_not_supported("job")
|
||||
}
|
||||
/// List server-side jobs across the database's tables.
|
||||
async fn list_jobs(&self) -> Result<Vec<JobInfo>> {
|
||||
job_op_not_supported("list_jobs")
|
||||
}
|
||||
/// Describe a single job by id. `None` when the server has no such job.
|
||||
async fn get_job(&self, _job_id: &str) -> Result<Option<JobDescription>> {
|
||||
job_op_not_supported("get_job")
|
||||
}
|
||||
/// Request cancellation of a job by id. Returns true if the server
|
||||
/// accepted the cancellation, false if no such job exists. Cancelling an
|
||||
/// already-terminal job is a no-op success.
|
||||
async fn cancel_job(&self, _job_id: &str) -> Result<bool> {
|
||||
job_op_not_supported("cancel_job")
|
||||
}
|
||||
/// Start executing a SQL statement on a remote database.
|
||||
async fn execute_query_async(
|
||||
&self,
|
||||
_query: &str,
|
||||
_default_namespace_path: &[String],
|
||||
) -> Result<crate::sql::Query> {
|
||||
Err(crate::error::Error::NotSupported {
|
||||
message: "SQL is not supported by this database".to_string(),
|
||||
})
|
||||
}
|
||||
/// Describe a submitted SQL query by its connection-scoped id.
|
||||
async fn describe_query(&self, _query_id: uuid::Uuid) -> Result<crate::sql::QueryDescription> {
|
||||
Err(crate::error::Error::NotSupported {
|
||||
message: "SQL is not supported by this database".to_string(),
|
||||
})
|
||||
/// The lifecycle event history of a job (all jobs when `job_id` is
|
||||
/// `None`), as recorded Arrow batches.
|
||||
async fn job_history(&self, _job_id: Option<&str>) -> Result<Vec<RecordBatch>> {
|
||||
job_op_not_supported("job_history")
|
||||
}
|
||||
/// Open a table in the database
|
||||
async fn open_table(&self, request: OpenTableRequest) -> Result<Arc<dyn BaseTable>>;
|
||||
|
||||
@@ -18,7 +18,7 @@ use lance_table::io::commit::commit_handler_from_url;
|
||||
use object_store::local::LocalFileSystem;
|
||||
use snafu::ResultExt;
|
||||
|
||||
use crate::blob::ensure_blob_storage_version;
|
||||
use crate::blob::{ensure_blob_storage_version, has_blob_columns};
|
||||
use crate::connection::ConnectRequest;
|
||||
use crate::database::ReadConsistency;
|
||||
use crate::database::namespace::LanceNamespaceDatabase;
|
||||
@@ -827,6 +827,7 @@ impl ListingDatabase {
|
||||
if let Some(enable_stable_row_ids) = overrides
|
||||
.enable_stable_row_ids
|
||||
.or(self.new_table_config.enable_stable_row_ids)
|
||||
.or(has_blob_columns(&data_schema).then_some(true))
|
||||
{
|
||||
write_params.enable_stable_row_ids = enable_stable_row_ids;
|
||||
}
|
||||
|
||||
@@ -3,7 +3,6 @@
|
||||
|
||||
//! Namespace-based database implementation that delegates table management to lance-namespace
|
||||
|
||||
use lance_datafusion::utils::StreamingWriteSource;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
@@ -24,7 +23,7 @@ use lance_namespace_impls::ConnectBuilder;
|
||||
use lance_table::io::commit::CommitHandler;
|
||||
use lance_table::io::commit::external_manifest::ExternalManifestCommitHandler;
|
||||
|
||||
use crate::blob::ensure_blob_storage_version;
|
||||
use crate::blob::{ensure_blob_storage_version, has_blob_columns};
|
||||
use crate::connection::NamespaceClientPushdownOperation;
|
||||
use crate::database::ReadConsistency;
|
||||
use crate::database::listing::{NewTableConfig, take_request_creation_overrides};
|
||||
@@ -218,6 +217,7 @@ impl LanceNamespaceDatabase {
|
||||
if let Some(enable_stable_row_ids) = overrides
|
||||
.enable_stable_row_ids
|
||||
.or(self.new_table_config.enable_stable_row_ids)
|
||||
.or(has_blob_columns(data_schema.as_ref()).then_some(true))
|
||||
{
|
||||
params.enable_stable_row_ids = enable_stable_row_ids;
|
||||
}
|
||||
@@ -305,10 +305,6 @@ impl Database for LanceNamespaceDatabase {
|
||||
}
|
||||
|
||||
async fn create_table(&self, request: DbCreateTableRequest) -> Result<Arc<dyn BaseTable>> {
|
||||
// Refuse a bad declaration before the namespace records a table.
|
||||
crate::table::computed_columns::ensure_declarations_are_planned(
|
||||
&request.data.arrow_schema(),
|
||||
)?;
|
||||
let mut table_id = request.namespace_path.clone();
|
||||
table_id.push(request.name.clone());
|
||||
let mut existing_table = None;
|
||||
|
||||
@@ -102,8 +102,6 @@ pub enum Error {
|
||||
},
|
||||
#[snafu(display("Job{} was cancelled", job_id.as_ref().map(|id| format!(" {id}")).unwrap_or_default()))]
|
||||
JobCancelled { job_id: Option<String> },
|
||||
#[snafu(display("Job '{job_id}' was not found"))]
|
||||
JobNotFound { job_id: String },
|
||||
|
||||
// 3rd party / external errors
|
||||
#[snafu(display("object_store error: {source}"))]
|
||||
|
||||
@@ -582,8 +582,8 @@ pub struct InputBinding {
|
||||
|
||||
/// Ordered result-field to table-field mapping for a Function binding.
|
||||
///
|
||||
/// `nullable` describes the logical Function result. Physical computed-column
|
||||
/// fields remain nullable while unassigned.
|
||||
/// Assignment state is not part of the Slice 1 client contract. During the
|
||||
/// NULL transition there is no public Lance cell-flag identifier to persist.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct OutputMapping {
|
||||
pub result_field: String,
|
||||
@@ -594,14 +594,6 @@ pub struct OutputMapping {
|
||||
pub nullable: bool,
|
||||
}
|
||||
|
||||
/// Internal physical column preserving the parent validity of a flattened
|
||||
/// named-struct result.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct AssignmentMapping {
|
||||
pub output_name: String,
|
||||
pub output_field_id: i32,
|
||||
}
|
||||
|
||||
/// Immutable Function binding persisted by the Enterprise table service.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct FunctionBinding {
|
||||
@@ -609,8 +601,6 @@ pub struct FunctionBinding {
|
||||
function: FunctionVersionRef,
|
||||
inputs: Vec<InputBinding>,
|
||||
outputs: Vec<OutputMapping>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
assignment: Option<AssignmentMapping>,
|
||||
/// Exact Arrow schema presented to the Function, encoded with the Lance
|
||||
/// Namespace Arrow JSON representation.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
@@ -637,10 +627,6 @@ impl FunctionBinding {
|
||||
&self.outputs
|
||||
}
|
||||
|
||||
pub fn assignment(&self) -> Option<&AssignmentMapping> {
|
||||
self.assignment.as_ref()
|
||||
}
|
||||
|
||||
pub fn input_schema(&self) -> Option<&Value> {
|
||||
self.input_schema.as_ref()
|
||||
}
|
||||
|
||||
+12
-247
@@ -3,53 +3,16 @@
|
||||
|
||||
//! Handles to operations a server may run asynchronously.
|
||||
|
||||
use std::sync::{Arc, RwLock, RwLockReadGuard, RwLockWriteGuard};
|
||||
use std::sync::Arc;
|
||||
|
||||
use arrow_array::RecordBatch;
|
||||
use async_trait::async_trait;
|
||||
use serde::{Serialize, de::DeserializeOwned};
|
||||
use serde_json::Value;
|
||||
use tokio::sync::watch;
|
||||
use tokio::task::{AbortHandle, JoinHandle};
|
||||
|
||||
use crate::database::JobDescription;
|
||||
use crate::error::{Error, JobFailure, Result};
|
||||
|
||||
/// Which of a job's events [`Job::events`] returns.
|
||||
///
|
||||
/// The handle already knows which job to ask about, so this narrows the
|
||||
/// query rather than naming one.
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct JobEventsRequest {
|
||||
/// Maximum event rows to return. The server applies its own default
|
||||
/// (1000 rows) and maximum (10,000 rows) when this is `None`, and
|
||||
/// truncates without saying so, which matters for a job with one event
|
||||
/// per fragment.
|
||||
pub limit: Option<u32>,
|
||||
/// SQL-like filter over the event columns `state`, `updated_by`,
|
||||
/// `emitted_from`, `emitted_by`, and `claim_entity`. For example
|
||||
/// `state = 'claim_complete'` selects only per-claim completions.
|
||||
pub filter: Option<String>,
|
||||
}
|
||||
|
||||
impl JobEventsRequest {
|
||||
pub fn limit(mut self, limit: u32) -> Self {
|
||||
self.limit = Some(limit);
|
||||
self
|
||||
}
|
||||
|
||||
pub fn filter(mut self, filter: impl Into<String>) -> Self {
|
||||
self.filter = Some(filter.into());
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
fn job_detail_not_supported<T>(what: &str) -> Result<T> {
|
||||
Err(Error::NotSupported {
|
||||
message: format!("{what} is only available for server-side jobs"),
|
||||
})
|
||||
}
|
||||
|
||||
/// Backend-specific tracking for an asynchronous operation.
|
||||
#[async_trait]
|
||||
pub(crate) trait JobHandle: Send + Sync {
|
||||
@@ -60,15 +23,6 @@ pub(crate) trait JobHandle: Send + Sync {
|
||||
async fn status(&self) -> Result<String>;
|
||||
async fn wait(&self) -> Result<TerminalResult>;
|
||||
async fn cancel(&self) -> Result<()>;
|
||||
/// The job's full server-side record. Backends that run the operation in
|
||||
/// this process have none and keep the default.
|
||||
async fn describe(&self) -> Result<JobDescription> {
|
||||
job_detail_not_supported("describing a job")
|
||||
}
|
||||
/// The job's recorded lifecycle events.
|
||||
async fn events(&self, _request: JobEventsRequest) -> Result<Vec<RecordBatch>> {
|
||||
job_detail_not_supported("job event history")
|
||||
}
|
||||
}
|
||||
|
||||
/// A backend-neutral successful terminal result.
|
||||
@@ -131,34 +85,16 @@ enum JobInner<T> {
|
||||
Completed(T),
|
||||
}
|
||||
|
||||
/// What a handle last learned about its job. `state` is separate because an
|
||||
/// in-process job can report one but has no server-side record behind it.
|
||||
#[derive(Default)]
|
||||
struct JobCache {
|
||||
state: Option<String>,
|
||||
description: Option<JobDescription>,
|
||||
}
|
||||
|
||||
/// A handle to an operation that may still be running.
|
||||
///
|
||||
/// The operation may already be complete when the handle is created. `T` is
|
||||
/// the endpoint's successful terminal result; unit-result operations use the
|
||||
/// default `Job<()>`.
|
||||
///
|
||||
/// The detail accessors ([`Job::state`], [`Job::job_type`], ...) read what the
|
||||
/// handle last observed. Submitting an operation returns only a job id, so
|
||||
/// populating them eagerly would cost an extra round trip on every call:
|
||||
///
|
||||
/// - [`Job::refresh`] and [`Job::status`] fetch the whole record.
|
||||
/// - [`Job::wait`] records the terminal state it establishes, but not the rest
|
||||
/// of the record; call [`Job::refresh`] for that.
|
||||
/// - Everything is `None` until one of those runs.
|
||||
pub struct Job<T = ()>
|
||||
where
|
||||
T: Clone + Send + Sync + 'static,
|
||||
{
|
||||
inner: JobInner<T>,
|
||||
cache: RwLock<JobCache>,
|
||||
}
|
||||
|
||||
impl<T> std::fmt::Debug for Job<T>
|
||||
@@ -166,40 +102,18 @@ where
|
||||
T: Clone + Send + Sync + 'static,
|
||||
{
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
let cache = self.cache_read();
|
||||
let mut out = f.debug_struct("Job");
|
||||
out.field("id", &self.id())
|
||||
.field("done", &matches!(self.inner, JobInner::Completed(_)));
|
||||
if let Some(state) = &cache.state {
|
||||
out.field("state", state);
|
||||
}
|
||||
if let Some(description) = &cache.description {
|
||||
out.field("job_type", &description.job_type)
|
||||
.field("creation_ms", &description.creation_ms);
|
||||
if !description.spec.is_null() {
|
||||
out.field("spec", &description.spec);
|
||||
}
|
||||
if let Some(result) = &description.result {
|
||||
out.field("result", result);
|
||||
}
|
||||
if let Some(failure) = &description.failure {
|
||||
out.field("failure", failure);
|
||||
}
|
||||
}
|
||||
out.finish()
|
||||
f.debug_struct("Job")
|
||||
.field("id", &self.id())
|
||||
.field("done", &matches!(self.inner, JobInner::Completed(_)))
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
impl Job<()> {
|
||||
/// A job whose operation finished before the handle was created. Its
|
||||
/// state is known without asking anyone, so the cache starts populated.
|
||||
/// A job whose operation finished before the handle was created.
|
||||
pub(crate) fn new_done() -> Self {
|
||||
Self {
|
||||
inner: JobInner::Completed(()),
|
||||
cache: RwLock::new(JobCache {
|
||||
state: Some("finished".to_string()),
|
||||
description: None,
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -209,21 +123,8 @@ impl Job<()> {
|
||||
handle,
|
||||
decode: Arc::new(|_| Ok(())),
|
||||
},
|
||||
cache: RwLock::default(),
|
||||
}
|
||||
}
|
||||
|
||||
/// A handle whose record the caller has already fetched, so the detail
|
||||
/// accessors answer without a second round trip.
|
||||
pub(crate) fn opened(handle: Box<dyn JobHandle>, description: JobDescription) -> Self {
|
||||
let job = Self::new(handle);
|
||||
{
|
||||
let mut cache = job.cache_write();
|
||||
cache.state = Some(description.state.clone());
|
||||
cache.description = Some(description);
|
||||
}
|
||||
job
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> Job<T>
|
||||
@@ -237,7 +138,6 @@ where
|
||||
handle,
|
||||
decode: Arc::new(TerminalResult::decode::<T>),
|
||||
},
|
||||
cache: RwLock::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -269,124 +169,16 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
fn cache_read(&self) -> RwLockReadGuard<'_, JobCache> {
|
||||
self.cache.read().unwrap_or_else(|err| err.into_inner())
|
||||
}
|
||||
|
||||
fn cache_write(&self) -> RwLockWriteGuard<'_, JobCache> {
|
||||
self.cache.write().unwrap_or_else(|err| err.into_inner())
|
||||
}
|
||||
|
||||
/// Asks the backend for this job's current state, and for a server-side
|
||||
/// job its full record, then caches the answer for the detail accessors.
|
||||
///
|
||||
/// In-process operations have no server-side record, so only
|
||||
/// [`Job::state`] is populated for them.
|
||||
pub async fn refresh(&self) -> Result<()> {
|
||||
self.refresh_state().await.map(|_| ())
|
||||
}
|
||||
|
||||
/// Refreshes and reports the state, which every backend can answer.
|
||||
async fn refresh_state(&self) -> Result<String> {
|
||||
let JobInner::Handle { handle, .. } = &self.inner else {
|
||||
let state = "finished".to_string();
|
||||
self.cache_write().state = Some(state.clone());
|
||||
return Ok(state);
|
||||
};
|
||||
match handle.describe().await {
|
||||
Ok(description) => {
|
||||
let state = description.state.clone();
|
||||
let mut cache = self.cache_write();
|
||||
cache.state = Some(state.clone());
|
||||
cache.description = Some(description);
|
||||
Ok(state)
|
||||
}
|
||||
// An in-process job knows its own state and nothing more.
|
||||
Err(Error::NotSupported { .. }) => {
|
||||
let state = handle.status().await?;
|
||||
self.cache_write().state = Some(state.clone());
|
||||
Ok(state)
|
||||
}
|
||||
Err(err) => Err(err),
|
||||
}
|
||||
}
|
||||
|
||||
/// The operation's current lifecycle state: "running", "finished",
|
||||
/// "failed", or "cancelled".
|
||||
///
|
||||
/// A point snapshot; unlike [`Job::wait`] it does not block, raise on a
|
||||
/// terminal failure state, or retry. States a newer server reports that
|
||||
/// this client version does not know pass through as-is. Also refreshes
|
||||
/// the detail accessors.
|
||||
/// this client version does not know pass through as-is.
|
||||
pub async fn status(&self) -> Result<String> {
|
||||
self.refresh_state().await
|
||||
}
|
||||
|
||||
/// The last lifecycle state this handle observed, without contacting the
|
||||
/// backend. `None` until the handle has.
|
||||
pub fn state(&self) -> Option<String> {
|
||||
self.cache_read().state.clone()
|
||||
}
|
||||
|
||||
/// The whole server-side record this handle last observed. The accessors
|
||||
/// below read individual fields out of it. `None` for an in-process job,
|
||||
/// which has no such record.
|
||||
pub fn description(&self) -> Option<JobDescription> {
|
||||
self.cache_read().description.clone()
|
||||
}
|
||||
|
||||
/// The job's type, as the server names it. `None` for an in-process job.
|
||||
pub fn job_type(&self) -> Option<String> {
|
||||
self.with_description(|description| description.job_type.clone())
|
||||
}
|
||||
|
||||
/// When the job was created, in milliseconds since the epoch. `None` for
|
||||
/// an in-process job.
|
||||
pub fn creation_ms(&self) -> Option<i64> {
|
||||
self.with_description(|description| description.creation_ms)
|
||||
}
|
||||
|
||||
/// The job-type-specific specification it was submitted with.
|
||||
pub fn spec(&self) -> Option<Value> {
|
||||
self.with_description(|description| description.spec.clone())
|
||||
.filter(|spec| !spec.is_null())
|
||||
}
|
||||
|
||||
/// The job-type-specific terminal result, as reported data rather than the
|
||||
/// typed model [`Job::wait`] returns. `None` until the job succeeds.
|
||||
pub fn result(&self) -> Option<Value> {
|
||||
self.with_description(|description| description.result.clone())
|
||||
.flatten()
|
||||
}
|
||||
|
||||
/// Why the job failed, when it failed and the server reports a reason.
|
||||
pub fn failure(&self) -> Option<JobFailure> {
|
||||
self.with_description(|description| description.failure.clone())
|
||||
.flatten()
|
||||
}
|
||||
|
||||
fn with_description<R>(&self, read: impl FnOnce(&JobDescription) -> R) -> Option<R> {
|
||||
self.cache_read().description.as_ref().map(read)
|
||||
}
|
||||
|
||||
/// This job's recorded lifecycle events.
|
||||
///
|
||||
/// Unlike the detail accessors, which report a terminal result only once
|
||||
/// the job reaches one, events are written as the job runs and outlive the
|
||||
/// workers that produced them. A distributed job records a
|
||||
/// `claim`/`claim_complete` pair per unit of work, each carrying
|
||||
/// `rows_processed`, so a job that never finishes still accounts for what
|
||||
/// it did. In-process operations keep no event history.
|
||||
pub async fn events(&self, request: JobEventsRequest) -> Result<Vec<RecordBatch>> {
|
||||
match &self.inner {
|
||||
JobInner::Handle { handle, .. } => handle.events(request).await,
|
||||
// The operation finished before the handle existed, so there is no
|
||||
// id to query with even when a server ran it.
|
||||
JobInner::Completed(_) => Err(Error::NotSupported {
|
||||
message: "this operation completed before its handle was created, so it \
|
||||
carries no job id to query events with"
|
||||
.to_string(),
|
||||
}),
|
||||
JobInner::Handle { handle, .. } => handle.status().await,
|
||||
JobInner::Completed(_) => Ok("finished".to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -398,19 +190,8 @@ where
|
||||
/// [`crate::Error::JobCancelled`] if it was cancelled.
|
||||
pub async fn wait(&self) -> Result<T> {
|
||||
match &self.inner {
|
||||
JobInner::Handle { handle, decode } => {
|
||||
let settled = handle.wait().await;
|
||||
// Waiting already established a terminal state; record it so
|
||||
// the detail accessors do not need another round trip for it.
|
||||
if let Some(state) = terminal_state(&settled) {
|
||||
self.cache_write().state = Some(state.to_string());
|
||||
}
|
||||
(decode)(settled?)
|
||||
}
|
||||
JobInner::Completed(result) => {
|
||||
self.cache_write().state = Some("finished".to_string());
|
||||
Ok(result.clone())
|
||||
}
|
||||
JobInner::Handle { handle, decode } => (decode)(handle.wait().await?),
|
||||
JobInner::Completed(result) => Ok(result.clone()),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -443,36 +224,20 @@ where
|
||||
U: Clone + Send + Sync + 'static,
|
||||
F: Fn(T) -> U + Send + Sync + 'static,
|
||||
{
|
||||
// The mapped handle tracks the same job, so it inherits what this one
|
||||
// has already learned about it.
|
||||
let Self { inner, cache } = self;
|
||||
match inner {
|
||||
match self.inner {
|
||||
JobInner::Handle { handle, decode } => Job {
|
||||
inner: JobInner::Handle {
|
||||
handle,
|
||||
decode: Arc::new(move |result| Ok(map((decode)(result)?))),
|
||||
},
|
||||
cache,
|
||||
},
|
||||
JobInner::Completed(result) => Job {
|
||||
inner: JobInner::Completed(map(result)),
|
||||
cache,
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The lifecycle state a settled [`JobHandle::wait`] implies.
|
||||
fn terminal_state(settled: &Result<TerminalResult>) -> Option<&'static str> {
|
||||
match settled {
|
||||
Ok(_) => Some("finished"),
|
||||
Err(Error::JobFailed { .. }) => Some("failed"),
|
||||
Err(Error::JobCancelled { .. }) => Some("cancelled"),
|
||||
// Anything else is a transport failure, not a verdict on the job.
|
||||
Err(_) => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// How an in-process operation ended. Cloneable so every waiter can be given
|
||||
/// the outcome; [`Error`] is not, so failures share one behind an [`Arc`].
|
||||
#[derive(Clone)]
|
||||
|
||||
@@ -195,7 +195,6 @@ pub mod query;
|
||||
#[cfg(feature = "remote")]
|
||||
pub mod remote;
|
||||
pub mod rerankers;
|
||||
pub mod sql;
|
||||
pub mod table;
|
||||
#[cfg(test)]
|
||||
pub mod test_utils;
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -24,8 +24,8 @@ use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use arrow_array::cast::AsArray;
|
||||
use arrow_array::types::UInt64Type;
|
||||
use arrow_array::{RecordBatch, UInt64Array, new_null_array};
|
||||
use arrow_schema::{FieldRef, Schema as ArrowSchema, SchemaRef};
|
||||
use arrow_array::{RecordBatch, UInt64Array};
|
||||
use arrow_schema::{Schema as ArrowSchema, SchemaRef};
|
||||
use datafusion::common::ScalarValue;
|
||||
use datafusion::error::DataFusionError;
|
||||
use datafusion::physical_plan::SendableRecordBatchStream;
|
||||
@@ -34,7 +34,7 @@ use datafusion::prelude::{col, lit};
|
||||
use futures::{StreamExt, TryStreamExt};
|
||||
use lance::Dataset;
|
||||
use lance::dataset::mem_wal::DatasetMemWalExt;
|
||||
use lance::dataset::transaction::{Operation, Transaction, UpdateMode};
|
||||
use lance::dataset::transaction::{Operation, Transaction};
|
||||
use lance::dataset::write::delete::DeleteBuilder;
|
||||
use lance::dataset::write::merge_insert::inserted_rows::{
|
||||
KeyExistenceFilter, KeyExistenceFilterBuilder, KeyValue,
|
||||
@@ -51,9 +51,6 @@ use super::{
|
||||
definition_to_metadata,
|
||||
};
|
||||
use crate::database::OpenTableRequest;
|
||||
use crate::table::computed_columns::{
|
||||
computed_column_from_field, computed_columns, ensure_declarations_are_planned,
|
||||
};
|
||||
use crate::table::{NativeTable, NativeTableExt, Table};
|
||||
use crate::{Error, Result};
|
||||
|
||||
@@ -170,52 +167,30 @@ pub(crate) async fn execute_refresh(
|
||||
.map(|p| (p.output.clone(), p.expression.clone()))
|
||||
.collect();
|
||||
validate_inputs(&source_ds, definition)?;
|
||||
let (replanned, planned_fields, _renames) = super::plan(
|
||||
let (replanned, mut planned_fields, _renames) = super::plan(
|
||||
source_schema,
|
||||
&definition.source_table,
|
||||
&definition.source_namespace,
|
||||
Some(&projections),
|
||||
&projections,
|
||||
definition.filter.as_deref(),
|
||||
definition.limit,
|
||||
)?;
|
||||
let mut planned_fields = planned_fields;
|
||||
planned_fields.push(arrow_schema::Field::new(
|
||||
SOURCE_ROW_ID_COLUMN,
|
||||
arrow_schema::DataType::UInt64,
|
||||
false,
|
||||
));
|
||||
// A computed column is not planned from the source: refresh writes it
|
||||
// NULL and its declaration's owner fills it. Its declaration must still
|
||||
// be complete, and it must be able to hold NULL.
|
||||
let physical = ArrowSchema::from(view_ds.schema());
|
||||
let mut computed = computed_columns(&physical).into_iter().map(|c| c.name);
|
||||
if let Some(name) = computed.by_ref().find(|name| {
|
||||
physical
|
||||
.field_with_name(name)
|
||||
.is_ok_and(|f| !f.is_nullable())
|
||||
}) {
|
||||
return Err(Error::Schema {
|
||||
message: format!(
|
||||
"computed column '{name}' of view '{}' cannot hold NULL; recreate the view",
|
||||
view.name()
|
||||
),
|
||||
});
|
||||
}
|
||||
ensure_declarations_are_planned(&physical)?;
|
||||
let physical_planned: Vec<&FieldRef> = physical
|
||||
let planned_shape: Vec<_> = planned_fields
|
||||
.iter()
|
||||
.map(|f| (f.name().clone(), f.data_type().clone(), f.is_nullable()))
|
||||
.collect();
|
||||
let physical_shape: Vec<_> = physical
|
||||
.fields()
|
||||
.iter()
|
||||
.filter(|f| computed_column_from_field(f).is_none())
|
||||
.map(|f| (f.name().clone(), f.data_type().clone(), f.is_nullable()))
|
||||
.collect();
|
||||
// A projected column that became nullable at the source still fits the
|
||||
// view's nullable field; the reverse would not.
|
||||
let matches = planned_fields.len() == physical_planned.len()
|
||||
&& planned_fields.iter().zip(&physical_planned).all(|(e, p)| {
|
||||
e.name() == p.name()
|
||||
&& e.data_type() == p.data_type()
|
||||
&& (p.is_nullable() || !e.is_nullable())
|
||||
});
|
||||
if !matches {
|
||||
if planned_shape != physical_shape {
|
||||
return Err(Error::Schema {
|
||||
message: format!(
|
||||
"the stored definition of view '{}' does not produce this \
|
||||
@@ -254,18 +229,11 @@ pub(crate) async fn execute_refresh(
|
||||
.get(SOURCE_VERSION_TS_META_KEY)
|
||||
.and_then(|raw| raw.parse().ok());
|
||||
// The watermark speaks only for the view state its refresh left behind;
|
||||
// any other commit on the view since then is drift, except a fill of its
|
||||
// computed columns, which rewrites nothing refresh certifies.
|
||||
let recorded_view_version = metadata
|
||||
// any other commit on the view since then is drift.
|
||||
let view_intact = metadata
|
||||
.get(VIEW_VERSION_META_KEY)
|
||||
.and_then(|raw| raw.parse::<u64>().ok());
|
||||
let view_intact = match recorded_view_version {
|
||||
Some(recorded) if recorded == view_ds.version().version => true,
|
||||
Some(recorded) if recorded < view_ds.version().version => {
|
||||
only_computed_rewrites_since(&view_ds, recorded).await?
|
||||
}
|
||||
_ => false,
|
||||
};
|
||||
.and_then(|raw| raw.parse::<u64>().ok())
|
||||
== Some(view_ds.version().version);
|
||||
|
||||
if !full && watermark == Some(source_version) && view_intact && recorded_ts == Some(source_ts) {
|
||||
return Ok(RefreshMaterializedViewResult {
|
||||
@@ -1122,69 +1090,6 @@ struct RowScope {
|
||||
limit: Option<u64>,
|
||||
}
|
||||
|
||||
/// Whether every commit on the view after `recorded` is a fill of its
|
||||
/// computed columns: a column rewrite or data replacement touching only
|
||||
/// those fields and neither adding nor removing rows. A version whose
|
||||
/// transaction cannot be read is not proven, so it counts as drift.
|
||||
async fn only_computed_rewrites_since(view_ds: &Dataset, recorded: u64) -> Result<bool> {
|
||||
// A fill may write any field under a computed column, so the whole
|
||||
// subtree counts, not only the root.
|
||||
let physical = ArrowSchema::from(view_ds.schema());
|
||||
fn subtree(field: &lance_core::datatypes::Field, ids: &mut Vec<u32>) {
|
||||
ids.push(field.id as u32);
|
||||
for child in &field.children {
|
||||
subtree(child, ids);
|
||||
}
|
||||
}
|
||||
let mut computed_fields = Vec::new();
|
||||
for column in computed_columns(&physical) {
|
||||
if let Some(field) = view_ds.schema().field(&column.name) {
|
||||
subtree(field, &mut computed_fields);
|
||||
}
|
||||
}
|
||||
if computed_fields.is_empty() {
|
||||
return Ok(false);
|
||||
}
|
||||
for version in recorded + 1..=view_ds.version().version {
|
||||
let Some(transaction) = view_ds.read_transaction_by_version(version).await? else {
|
||||
return Ok(false);
|
||||
};
|
||||
let fill = match &transaction.operation {
|
||||
Operation::Update {
|
||||
removed_fragment_ids,
|
||||
new_fragments,
|
||||
fields_modified,
|
||||
update_mode: Some(UpdateMode::RewriteColumns),
|
||||
..
|
||||
} => {
|
||||
removed_fragment_ids.is_empty()
|
||||
&& new_fragments.is_empty()
|
||||
&& !fields_modified.is_empty()
|
||||
&& fields_modified
|
||||
.iter()
|
||||
.all(|field| computed_fields.contains(field))
|
||||
}
|
||||
// What `refresh_column` commits for a SQL declaration.
|
||||
Operation::DataReplacement { replacements } => {
|
||||
!replacements.is_empty()
|
||||
&& replacements.iter().all(|group| {
|
||||
!group.1.fields.is_empty()
|
||||
&& group
|
||||
.1
|
||||
.fields
|
||||
.iter()
|
||||
.all(|field| computed_fields.contains(&(*field as u32)))
|
||||
})
|
||||
}
|
||||
_ => false,
|
||||
};
|
||||
if !fill {
|
||||
return Ok(false);
|
||||
}
|
||||
}
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
async fn compute_stream(
|
||||
source: &Dataset,
|
||||
definition: &MaterializedViewDefinition,
|
||||
@@ -1253,10 +1158,6 @@ async fn compute_stream(
|
||||
let batch = batch.map_err(|e| DataFusionError::External(Box::new(e)))?;
|
||||
let mut columns = Vec::with_capacity(out_schema.fields().len());
|
||||
for field in out_schema.fields() {
|
||||
if computed_column_from_field(field).is_some() {
|
||||
columns.push(new_null_array(field.data_type(), batch.num_rows()));
|
||||
continue;
|
||||
}
|
||||
let name = if field.name() == SOURCE_ROW_ID_COLUMN {
|
||||
ROW_ID
|
||||
} else {
|
||||
@@ -2867,7 +2768,7 @@ mod tests {
|
||||
let (conn, source) = db_with_source(vec![1]).await;
|
||||
let prepared = crate::materialized_view::prepare_declaration(
|
||||
&source,
|
||||
Some(&[("x".into(), "x".into()), ("twice".into(), "x * 2".into())]),
|
||||
&[("x".into(), "x".into()), ("twice".into(), "x * 2".into())],
|
||||
None,
|
||||
None,
|
||||
)
|
||||
@@ -3231,424 +3132,4 @@ mod tests {
|
||||
let err = view.refresh().execute().await.unwrap_err();
|
||||
assert!(err.to_string().contains("source table 'src'"), "{err}");
|
||||
}
|
||||
|
||||
/// A view with a computed column, declared over `people` and refreshed.
|
||||
async fn refreshed_computed_view(conn: &Connection) -> MaterializedView {
|
||||
use crate::materialized_view::tests::{computed_field, people, test_binding};
|
||||
let source = people(conn).await;
|
||||
let view = crate::materialized_view::prepare_declaration(
|
||||
&source,
|
||||
Some(&[
|
||||
("id".to_string(), "id".to_string()),
|
||||
("name".to_string(), "name".to_string()),
|
||||
]),
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.with_computed_columns(
|
||||
vec![(2, computed_field("emb", "fb_1", "name"))],
|
||||
&[test_binding("fb_1", "name", "emb")],
|
||||
)
|
||||
.unwrap()
|
||||
.create("v")
|
||||
.await
|
||||
.unwrap();
|
||||
let result = view.refresh().execute().await.unwrap();
|
||||
assert_eq!(result.mode, RefreshMode::Rebuild);
|
||||
view
|
||||
}
|
||||
|
||||
async fn unfilled(view: &MaterializedView) -> usize {
|
||||
view.table()
|
||||
.count_rows(Some("emb IS NULL".to_string()))
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
async fn append_people(conn: &Connection, ids: Vec<i32>, names: Vec<&str>) {
|
||||
let batch = record_batch!(("id", Int32, ids), ("name", Utf8, names)).unwrap();
|
||||
conn.open_table("people")
|
||||
.execute()
|
||||
.await
|
||||
.unwrap()
|
||||
.add(batch)
|
||||
.execute()
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
/// Commit the fill job's shape on the view: a column rewrite of
|
||||
/// `fields`, touching no rows. The data is left as it is; what matters
|
||||
/// here is how the next refresh classifies the commit.
|
||||
async fn commit_column_rewrite(view: &MaterializedView, fields: &[&str]) {
|
||||
let native = view.table().as_native().unwrap();
|
||||
native.dataset.reload().await.unwrap();
|
||||
let dataset = native.dataset.get().await.unwrap().as_ref().clone();
|
||||
let fields_modified = fields
|
||||
.iter()
|
||||
.map(|name| dataset.schema().field(name).unwrap().id as u32)
|
||||
.collect();
|
||||
let updated_fragments = dataset
|
||||
.get_fragments()
|
||||
.iter()
|
||||
.map(|fragment| fragment.metadata().clone())
|
||||
.collect();
|
||||
let operation = Operation::Update {
|
||||
removed_fragment_ids: Vec::new(),
|
||||
updated_fragments,
|
||||
new_fragments: Vec::new(),
|
||||
fields_modified,
|
||||
compacted_sstables: Vec::new(),
|
||||
fields_for_preserving_frag_bitmap: Vec::new(),
|
||||
update_mode: Some(UpdateMode::RewriteColumns),
|
||||
inserted_rows_filter: None,
|
||||
updated_fragment_offsets: None,
|
||||
};
|
||||
let read_version = dataset.version().version;
|
||||
CommitBuilder::new(WriteDestination::Dataset(Arc::new(dataset)))
|
||||
.execute(Transaction::new(read_version, operation, None))
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
/// Refresh never computes a computed column: every row it writes, on a
|
||||
/// rebuild, an append and a rewrite, carries NULL there, and the
|
||||
/// declaration survives all three.
|
||||
#[tokio::test]
|
||||
async fn test_computed_columns_are_written_null_and_kept() {
|
||||
let conn = connect("memory://").execute().await.unwrap();
|
||||
let view = refreshed_computed_view(&conn).await;
|
||||
assert_eq!(unfilled(&view).await, 3);
|
||||
|
||||
append_people(&conn, vec![4, 5], vec!["d", "e"]).await;
|
||||
let result = view.refresh().execute().await.unwrap();
|
||||
assert_eq!(result.mode, RefreshMode::Incremental);
|
||||
assert_eq!(unfilled(&view).await, 5);
|
||||
|
||||
conn.open_table("people")
|
||||
.execute()
|
||||
.await
|
||||
.unwrap()
|
||||
.update()
|
||||
.column("name", "'z'")
|
||||
.only_if("id = 1")
|
||||
.execute()
|
||||
.await
|
||||
.unwrap();
|
||||
view.refresh().execute().await.unwrap();
|
||||
assert_eq!(unfilled(&view).await, 5);
|
||||
assert_eq!(read(view.table(), "id").await, vec![1, 2, 3, 4, 5]);
|
||||
|
||||
let schema = view.table().schema().await.unwrap();
|
||||
assert!(
|
||||
crate::table::computed_columns::function_bindings(&schema)
|
||||
.unwrap()
|
||||
.iter()
|
||||
.any(|b| b.binding_id() == "fb_1"),
|
||||
"the binding envelope was lost"
|
||||
);
|
||||
assert!(
|
||||
computed_column_from_field(schema.field_with_name("emb").unwrap()).is_some(),
|
||||
"the declaration was lost"
|
||||
);
|
||||
assert_eq!(
|
||||
view.refresh().execute().await.unwrap().mode,
|
||||
RefreshMode::NoOp
|
||||
);
|
||||
}
|
||||
|
||||
/// The fill job's commit rewrites only computed columns. It is the one
|
||||
/// commit on a view that is not drift: the next refresh carries on from
|
||||
/// its watermark instead of rebuilding, which would null what the fill
|
||||
/// just wrote.
|
||||
#[tokio::test]
|
||||
async fn test_a_computed_column_fill_is_not_drift() {
|
||||
let conn = connect("memory://").execute().await.unwrap();
|
||||
let view = refreshed_computed_view(&conn).await;
|
||||
|
||||
commit_column_rewrite(&view, &["emb"]).await;
|
||||
assert_eq!(
|
||||
view.refresh().execute().await.unwrap().mode,
|
||||
RefreshMode::NoOp
|
||||
);
|
||||
|
||||
commit_column_rewrite(&view, &["emb"]).await;
|
||||
append_people(&conn, vec![4], vec!["d"]).await;
|
||||
let result = view.refresh().execute().await.unwrap();
|
||||
assert_eq!(result.mode, RefreshMode::Incremental);
|
||||
assert_eq!(result.rows_written, 1);
|
||||
assert_eq!(read(view.table(), "id").await, vec![1, 2, 3, 4]);
|
||||
}
|
||||
|
||||
/// A column rewrite that reaches a projected column is drift like any
|
||||
/// other write: refresh certifies those columns and must recompute them.
|
||||
#[tokio::test]
|
||||
async fn test_a_rewrite_of_a_projected_column_is_drift() {
|
||||
let conn = connect("memory://").execute().await.unwrap();
|
||||
let view = refreshed_computed_view(&conn).await;
|
||||
|
||||
commit_column_rewrite(&view, &["emb", "name"]).await;
|
||||
assert_eq!(
|
||||
view.refresh().execute().await.unwrap().mode,
|
||||
RefreshMode::Rebuild
|
||||
);
|
||||
}
|
||||
|
||||
/// The declaration contract is checked before any refresh mutation: a
|
||||
/// missing binding envelope and a column that lost its declaration both
|
||||
/// fail closed.
|
||||
#[tokio::test]
|
||||
async fn test_a_broken_declaration_is_refused_before_refresh() {
|
||||
let conn = connect("memory://").execute().await.unwrap();
|
||||
let view = refreshed_computed_view(&conn).await;
|
||||
let native = view.table().as_native().unwrap();
|
||||
let mut dataset = native.dataset.get().await.unwrap().as_ref().clone();
|
||||
dataset
|
||||
.update_schema_metadata(vec![(
|
||||
crate::table::computed_columns::FUNCTION_BINDINGS_META_KEY.to_string(),
|
||||
None,
|
||||
)])
|
||||
.await
|
||||
.unwrap();
|
||||
let err = view.refresh().execute().await.unwrap_err().to_string();
|
||||
assert!(err.contains("references missing binding 'fb_1'"), "{err}");
|
||||
|
||||
let conn = connect("memory://").execute().await.unwrap();
|
||||
let view = refreshed_computed_view(&conn).await;
|
||||
let native = view.table().as_native().unwrap();
|
||||
let mut dataset = native.dataset.get().await.unwrap().as_ref().clone();
|
||||
dataset
|
||||
.replace_field_metadata(vec![(
|
||||
dataset.schema().field("emb").unwrap().id as u32,
|
||||
HashMap::new(),
|
||||
)])
|
||||
.await
|
||||
.unwrap();
|
||||
let err = view.refresh().execute().await.unwrap_err().to_string();
|
||||
assert!(err.contains("does not match binding 'fb_1'"), "{err}");
|
||||
}
|
||||
|
||||
/// An input the view does not project is materialized on every refresh
|
||||
/// path, before the provenance column, with the source's values.
|
||||
#[tokio::test]
|
||||
async fn test_internal_inputs_are_materialized_and_refreshed() {
|
||||
use crate::materialized_view::tests::{computed_field, strict_people, test_binding};
|
||||
let conn = connect("memory://").execute().await.unwrap();
|
||||
let source = strict_people(&conn).await;
|
||||
let mut prepared = crate::materialized_view::prepare_declaration(
|
||||
&source,
|
||||
Some(&[("id".to_string(), "id".to_string())]),
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let input = prepared.input_column("name").unwrap();
|
||||
let view = prepared
|
||||
.with_computed_columns(
|
||||
vec![(1, computed_field("emb", "fb_1", &input))],
|
||||
&[test_binding("fb_1", &input, "emb")],
|
||||
)
|
||||
.unwrap()
|
||||
.create("v")
|
||||
.await
|
||||
.unwrap();
|
||||
let names: Vec<String> = view
|
||||
.table()
|
||||
.schema()
|
||||
.await
|
||||
.unwrap()
|
||||
.fields()
|
||||
.iter()
|
||||
.map(|f| f.name().clone())
|
||||
.collect();
|
||||
assert_eq!(names, ["id", "emb", "__input_name", SOURCE_ROW_ID_COLUMN]);
|
||||
|
||||
let unfilled_inputs = || async {
|
||||
view.table()
|
||||
.count_rows(Some("__input_name IS NULL".to_string()))
|
||||
.await
|
||||
.unwrap()
|
||||
};
|
||||
assert_eq!(
|
||||
view.refresh().execute().await.unwrap().mode,
|
||||
RefreshMode::Rebuild
|
||||
);
|
||||
assert_eq!(view.table().count_rows(None).await.unwrap(), 3);
|
||||
assert_eq!(unfilled_inputs().await, 0);
|
||||
|
||||
let more = arrow_array::RecordBatch::try_new(
|
||||
source.schema().await.unwrap(),
|
||||
vec![
|
||||
Arc::new(Int32Array::from(vec![4])),
|
||||
Arc::new(arrow_array::StringArray::from(vec!["d"])),
|
||||
],
|
||||
)
|
||||
.unwrap();
|
||||
source.add(more).execute().await.unwrap();
|
||||
assert_eq!(
|
||||
view.refresh().execute().await.unwrap().mode,
|
||||
RefreshMode::Incremental
|
||||
);
|
||||
assert_eq!(unfilled_inputs().await, 0);
|
||||
assert_eq!(
|
||||
view.table()
|
||||
.count_rows(Some("__input_name = 'd'".to_string()))
|
||||
.await
|
||||
.unwrap(),
|
||||
1
|
||||
);
|
||||
|
||||
source
|
||||
.update()
|
||||
.column("name", "'z'")
|
||||
.only_if("id = 1")
|
||||
.execute()
|
||||
.await
|
||||
.unwrap();
|
||||
view.refresh().execute().await.unwrap();
|
||||
assert_eq!(
|
||||
view.table()
|
||||
.count_rows(Some("__input_name = 'z'".to_string()))
|
||||
.await
|
||||
.unwrap(),
|
||||
1
|
||||
);
|
||||
assert_eq!(
|
||||
unfilled(&view).await,
|
||||
4,
|
||||
"rewritten and new rows are unfilled"
|
||||
);
|
||||
}
|
||||
|
||||
/// A SQL declaration is filled by `refresh_column` on the view, which
|
||||
/// commits a data replacement; the next refresh continues from its
|
||||
/// watermark and keeps what the fill wrote, and only rows the view added
|
||||
/// since come back unfilled.
|
||||
#[tokio::test]
|
||||
async fn test_a_sql_fill_is_not_drift() {
|
||||
use crate::materialized_view::tests::{people, sql_field};
|
||||
let conn = connect("memory://").execute().await.unwrap();
|
||||
let source = people(&conn).await;
|
||||
let view = crate::materialized_view::prepare_declaration(
|
||||
&source,
|
||||
Some(&[("id".to_string(), "id".to_string())]),
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.with_computed_columns(
|
||||
vec![(
|
||||
1,
|
||||
sql_field("next", arrow_schema::DataType::Int32, "id + 1", r#"["id"]"#),
|
||||
)],
|
||||
&[],
|
||||
)
|
||||
.unwrap()
|
||||
.create("v")
|
||||
.await
|
||||
.unwrap();
|
||||
let filled = || async {
|
||||
view.table()
|
||||
.count_rows(Some("next = id + 1".to_string()))
|
||||
.await
|
||||
.unwrap()
|
||||
};
|
||||
assert_eq!(
|
||||
view.refresh().execute().await.unwrap().mode,
|
||||
RefreshMode::Rebuild
|
||||
);
|
||||
assert_eq!(
|
||||
view.table()
|
||||
.refresh_column("next")
|
||||
.await
|
||||
.unwrap()
|
||||
.rows_filled,
|
||||
3
|
||||
);
|
||||
assert_eq!(filled().await, 3);
|
||||
assert_eq!(
|
||||
view.refresh().execute().await.unwrap().mode,
|
||||
RefreshMode::NoOp
|
||||
);
|
||||
assert_eq!(filled().await, 3);
|
||||
|
||||
append_people(&conn, vec![4], vec!["d"]).await;
|
||||
assert_eq!(
|
||||
view.refresh().execute().await.unwrap().mode,
|
||||
RefreshMode::Incremental
|
||||
);
|
||||
assert_eq!(filled().await, 3);
|
||||
assert_eq!(
|
||||
view.table()
|
||||
.refresh_column("next")
|
||||
.await
|
||||
.unwrap()
|
||||
.rows_filled,
|
||||
1
|
||||
);
|
||||
assert_eq!(filled().await, 4);
|
||||
assert_eq!(
|
||||
view.refresh().execute().await.unwrap().mode,
|
||||
RefreshMode::NoOp
|
||||
);
|
||||
}
|
||||
|
||||
/// A fill of a nested computed column writes its child fields; that is
|
||||
/// still a fill, not drift.
|
||||
#[tokio::test]
|
||||
async fn test_a_nested_sql_fill_is_not_drift() {
|
||||
use crate::materialized_view::tests::{people, sql_field};
|
||||
let conn = connect("memory://").execute().await.unwrap();
|
||||
let source = people(&conn).await;
|
||||
let payload = sql_field(
|
||||
"payload",
|
||||
arrow_schema::DataType::Struct(
|
||||
vec![arrow_schema::Field::new(
|
||||
"value",
|
||||
arrow_schema::DataType::Utf8,
|
||||
true,
|
||||
)]
|
||||
.into(),
|
||||
),
|
||||
"named_struct('value', name)",
|
||||
r#"["name"]"#,
|
||||
);
|
||||
let view = crate::materialized_view::prepare_declaration(
|
||||
&source,
|
||||
Some(&[("name".to_string(), "name".to_string())]),
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.with_computed_columns(vec![(1, payload)], &[])
|
||||
.unwrap()
|
||||
.create("v")
|
||||
.await
|
||||
.unwrap();
|
||||
view.refresh().execute().await.unwrap();
|
||||
assert_eq!(
|
||||
view.table()
|
||||
.refresh_column("payload")
|
||||
.await
|
||||
.unwrap()
|
||||
.rows_filled,
|
||||
3
|
||||
);
|
||||
assert_eq!(
|
||||
view.refresh().execute().await.unwrap().mode,
|
||||
RefreshMode::NoOp
|
||||
);
|
||||
assert_eq!(
|
||||
view.table()
|
||||
.count_rows(Some("payload.value = name".to_string()))
|
||||
.await
|
||||
.unwrap(),
|
||||
3
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -11,7 +11,6 @@ pub(crate) mod db;
|
||||
pub(crate) mod job;
|
||||
pub mod oauth;
|
||||
mod retry;
|
||||
pub(crate) mod sql;
|
||||
pub(crate) mod table;
|
||||
pub(crate) mod util;
|
||||
|
||||
|
||||
+92
-173
@@ -20,20 +20,19 @@ use lance_namespace::models::{
|
||||
|
||||
use crate::Error;
|
||||
use crate::database::{
|
||||
CloneTableRequest, CreateTableMode, CreateTableRequest, Database, DatabaseOptions, JobInfo,
|
||||
OpenTableRequest, ReadConsistency, TableNamesRequest,
|
||||
CloneTableRequest, CreateTableMode, CreateTableRequest, Database, DatabaseOptions,
|
||||
JobDescription, JobInfo, OpenTableRequest, ReadConsistency, TableNamesRequest,
|
||||
};
|
||||
use crate::error::Result;
|
||||
use crate::function::{FunctionRegistrationRequest, FunctionVersion};
|
||||
use crate::job::Job;
|
||||
use crate::remote::job::{RemoteJob, job_state_to_client};
|
||||
use crate::remote::job::{DescribeJobResponse, RemoteJob, job_state_to_client};
|
||||
use crate::remote::util::stream_as_body;
|
||||
use crate::table::BaseTable;
|
||||
|
||||
use super::client::{
|
||||
ClientConfig, HeaderProvider, HttpSend, RequestResultExt, RestfulLanceDbClient, Sender,
|
||||
};
|
||||
use super::sql::SqlClient;
|
||||
use super::table::RemoteTable;
|
||||
use super::util::parse_server_version;
|
||||
use super::{ARROW_STREAM_CONTENT_TYPE, extract_job_id};
|
||||
@@ -98,7 +97,6 @@ pub const OPT_REMOTE_PREFIX: &str = "remote_database_";
|
||||
pub const OPT_REMOTE_API_KEY: &str = "remote_database_api_key";
|
||||
pub const OPT_REMOTE_REGION: &str = "remote_database_region";
|
||||
pub const OPT_REMOTE_HOST_OVERRIDE: &str = "remote_database_host_override";
|
||||
pub const OPT_REMOTE_SQL_HOST_OVERRIDE: &str = "remote_database_sql_host_override";
|
||||
// TODO: add support for configuring client config via key/value options
|
||||
|
||||
#[derive(Clone, Debug, Default)]
|
||||
@@ -214,7 +212,6 @@ pub struct RemoteDatabase<S: HttpSend = Sender> {
|
||||
namespace_context_provider: Option<Arc<dyn DynamicContextProvider>>,
|
||||
/// TLS configuration for mTLS support
|
||||
tls_config: Option<super::client::TlsConfig>,
|
||||
sql_client: Option<SqlClient>,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
@@ -272,35 +269,22 @@ impl DynamicContextProvider for NamespaceHeaderProviderContext {
|
||||
}
|
||||
}
|
||||
|
||||
pub struct RemoteHostOverrides {
|
||||
pub rest: Option<String>,
|
||||
pub sql: Option<String>,
|
||||
}
|
||||
|
||||
impl RemoteDatabase {
|
||||
pub(crate) fn try_new(
|
||||
pub fn try_new(
|
||||
uri: &str,
|
||||
api_key: &str,
|
||||
region: &str,
|
||||
host_overrides: RemoteHostOverrides,
|
||||
host_override: Option<String>,
|
||||
client_config: ClientConfig,
|
||||
options: RemoteOptions,
|
||||
read_consistency_interval: Option<std::time::Duration>,
|
||||
) -> Result<Self> {
|
||||
let parsed = super::client::parse_db_url(uri)?;
|
||||
let sql_client = SqlClient::new(
|
||||
parsed.db_name.clone(),
|
||||
parsed.db_prefix.clone(),
|
||||
api_key.to_string(),
|
||||
host_overrides.rest.clone(),
|
||||
host_overrides.sql,
|
||||
client_config.clone(),
|
||||
);
|
||||
let header_map = RestfulLanceDbClient::<Sender>::default_headers(
|
||||
api_key,
|
||||
region,
|
||||
&parsed.db_name,
|
||||
host_overrides.rest.is_some(),
|
||||
host_override.is_some(),
|
||||
&options,
|
||||
parsed.db_prefix.as_deref(),
|
||||
&client_config,
|
||||
@@ -328,7 +312,7 @@ impl RemoteDatabase {
|
||||
let client = RestfulLanceDbClient::try_new(
|
||||
&parsed,
|
||||
region,
|
||||
host_overrides.rest,
|
||||
host_override,
|
||||
header_map,
|
||||
client_config.clone(),
|
||||
read_consistency_interval,
|
||||
@@ -346,7 +330,6 @@ impl RemoteDatabase {
|
||||
namespace_headers,
|
||||
namespace_context_provider,
|
||||
tls_config: client_config.tls_config,
|
||||
sql_client: Some(sql_client),
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -444,7 +427,6 @@ mod test_utils {
|
||||
namespace_headers: HashMap::new(),
|
||||
namespace_context_provider: None,
|
||||
tls_config: None,
|
||||
sql_client: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -467,7 +449,6 @@ mod test_utils {
|
||||
namespace_headers: config.extra_headers.clone(),
|
||||
namespace_context_provider,
|
||||
tls_config: config.tls_config.clone(),
|
||||
sql_client: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -671,18 +652,11 @@ impl<S: HttpSend> Database for RemoteDatabase<S> {
|
||||
Ok(response.dropped)
|
||||
}
|
||||
|
||||
async fn open_job(&self, job_id: &str) -> Result<Job> {
|
||||
let handle = super::job::RemoteJob::new(self.client.clone(), job_id.to_string());
|
||||
match crate::job::JobHandle::describe(&handle).await {
|
||||
Ok(description) => Ok(Job::opened(Box::new(handle), description)),
|
||||
Err(Error::Http {
|
||||
status_code: Some(StatusCode::NOT_FOUND),
|
||||
..
|
||||
}) => Err(Error::JobNotFound {
|
||||
job_id: job_id.to_string(),
|
||||
}),
|
||||
Err(err) => Err(err),
|
||||
}
|
||||
fn job(&self, job_id: &str) -> Result<crate::job::Job> {
|
||||
Ok(crate::job::Job::new(Box::new(super::job::RemoteJob::new(
|
||||
self.client.clone(),
|
||||
job_id.to_string(),
|
||||
))))
|
||||
}
|
||||
|
||||
async fn list_jobs(&self) -> Result<Vec<JobInfo>> {
|
||||
@@ -719,6 +693,31 @@ impl<S: HttpSend> Database for RemoteDatabase<S> {
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
async fn get_job(&self, job_id: &str) -> Result<Option<JobDescription>> {
|
||||
let req = self
|
||||
.client
|
||||
.post("/v1/jobs/describe")
|
||||
.json(&serde_json::json!({ "job_id": job_id }));
|
||||
let (request_id, rsp) = self.client.send(req).await?;
|
||||
let rsp = match self.client.check_response(&request_id, rsp).await {
|
||||
Ok(rsp) => rsp,
|
||||
Err(Error::Http {
|
||||
status_code: Some(StatusCode::NOT_FOUND),
|
||||
..
|
||||
}) => return Ok(None),
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
let body: DescribeJobResponse = rsp.json().await.err_to_http(request_id)?;
|
||||
Ok(Some(JobDescription {
|
||||
job_id: body.job_id,
|
||||
job_type: body.job_type,
|
||||
state: job_state_to_client(&body.job_state),
|
||||
creation_ms: body.creation_ms,
|
||||
spec: body.spec,
|
||||
failure: body.failure.map(|reported| reported.into_job_failure()),
|
||||
}))
|
||||
}
|
||||
|
||||
async fn cancel_job(&self, job_id: &str) -> Result<bool> {
|
||||
let req = self
|
||||
.client
|
||||
@@ -735,28 +734,19 @@ impl<S: HttpSend> Database for RemoteDatabase<S> {
|
||||
}
|
||||
}
|
||||
|
||||
async fn execute_query_async(
|
||||
&self,
|
||||
query: &str,
|
||||
default_namespace_path: &[String],
|
||||
) -> Result<crate::sql::Query> {
|
||||
let client = self
|
||||
.sql_client
|
||||
.as_ref()
|
||||
.ok_or_else(|| Error::NotSupported {
|
||||
message: "SQL is unavailable for this remote database client".to_string(),
|
||||
})?;
|
||||
client.submit(query, default_namespace_path).await
|
||||
}
|
||||
|
||||
async fn describe_query(&self, query_id: uuid::Uuid) -> Result<crate::sql::QueryDescription> {
|
||||
let client = self
|
||||
.sql_client
|
||||
.as_ref()
|
||||
.ok_or_else(|| Error::NotSupported {
|
||||
message: "SQL is unavailable for this remote database client".to_string(),
|
||||
})?;
|
||||
client.describe(query_id).await
|
||||
async fn job_history(&self, job_id: Option<&str>) -> Result<Vec<arrow_array::RecordBatch>> {
|
||||
let mut body = serde_json::json!({});
|
||||
if let Some(job_id) = job_id {
|
||||
body["job_id"] = serde_json::Value::String(job_id.to_string());
|
||||
}
|
||||
let req = self.client.post("/v1/jobs/query_events").json(&body);
|
||||
let (request_id, rsp) = self.client.send(req).await?;
|
||||
let rsp = self.client.check_response(&request_id, rsp).await?;
|
||||
let bytes = rsp.bytes().await.err_to_http(request_id)?;
|
||||
let reader = arrow_ipc::reader::StreamReader::try_new(std::io::Cursor::new(bytes), None)?;
|
||||
reader
|
||||
.collect::<std::result::Result<Vec<_>, _>>()
|
||||
.map_err(Into::into)
|
||||
}
|
||||
|
||||
async fn table_names(&self, request: TableNamesRequest) -> Result<Vec<String>> {
|
||||
@@ -1274,7 +1264,6 @@ mod tests {
|
||||
use crate::{
|
||||
Connection, Error,
|
||||
database::CreateTableMode,
|
||||
job::JobEventsRequest,
|
||||
remote::{ARROW_STREAM_CONTENT_TYPE, ClientConfig, HeaderProvider, JSON_CONTENT_TYPE},
|
||||
};
|
||||
|
||||
@@ -2623,7 +2612,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_open_job() {
|
||||
async fn test_get_job() {
|
||||
let conn = Connection::new_with_handler(|request| {
|
||||
assert_eq!(request.method(), &reqwest::Method::POST);
|
||||
assert_eq!(request.url().path(), "/v1/jobs/describe");
|
||||
@@ -2637,55 +2626,51 @@ mod tests {
|
||||
)
|
||||
.unwrap()
|
||||
});
|
||||
// Opening populates the handle, so the accessors answer without a
|
||||
// second round trip.
|
||||
let job = conn.open_job("job-1").await.unwrap();
|
||||
assert_eq!(job.id(), Some("job-1"));
|
||||
assert_eq!(job.job_type().as_deref(), Some("create_index"));
|
||||
assert_eq!(job.state().as_deref(), Some("failed"));
|
||||
assert_eq!(job.creation_ms(), Some(1000));
|
||||
assert_eq!(job.spec().unwrap()["column"], "vec");
|
||||
assert!(job.result().is_none());
|
||||
let failure = job.failure().unwrap();
|
||||
let job = conn.get_job("job-1").await.unwrap().unwrap();
|
||||
assert_eq!(job.job_id, "job-1");
|
||||
assert_eq!(job.job_type, "create_index");
|
||||
assert_eq!(job.state, "failed");
|
||||
assert_eq!(job.creation_ms, 1000);
|
||||
assert_eq!(job.spec["column"], "vec");
|
||||
let failure = job.failure.unwrap();
|
||||
assert_eq!(failure.phase.as_deref(), Some("execute"));
|
||||
assert_eq!(failure.message.as_deref(), Some("worker died"));
|
||||
assert_eq!(failure.retryable, Some(true));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_open_job_reports_the_terminal_result() {
|
||||
let conn = Connection::new_with_handler(|_| {
|
||||
http::Response::builder()
|
||||
.status(200)
|
||||
.body(
|
||||
r#"{"job_id": "job-1", "job_type": "refresh_column", "job_state": "DONE", "creation_ms": 1000, "result": {"rows_assigned": 1000000, "rows_failed": 0}}"#,
|
||||
)
|
||||
.unwrap()
|
||||
});
|
||||
let job = conn.open_job("job-1").await.unwrap();
|
||||
assert_eq!(job.state().as_deref(), Some("finished"));
|
||||
let result = job.result().unwrap();
|
||||
assert_eq!(result["rows_assigned"], 1_000_000);
|
||||
assert_eq!(result["rows_failed"], 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_open_job_missing_fails() {
|
||||
async fn test_get_job_missing_is_none() {
|
||||
let conn = Connection::new_with_handler(|_| {
|
||||
http::Response::builder()
|
||||
.status(404)
|
||||
.body("no such job")
|
||||
.unwrap()
|
||||
});
|
||||
let err = conn.open_job("nope").await.unwrap_err();
|
||||
assert!(
|
||||
matches!(&err, Error::JobNotFound { job_id } if job_id == "nope"),
|
||||
"{err:?}"
|
||||
);
|
||||
assert!(conn.get_job("nope").await.unwrap().is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_job_events_scope_to_that_job() {
|
||||
async fn test_cancel_job() {
|
||||
let conn = Connection::new_with_handler(|request| {
|
||||
assert_eq!(request.url().path(), "/v1/jobs/cancel");
|
||||
http::Response::builder()
|
||||
.status(200)
|
||||
.body(r#"{"job_id": "job-1"}"#)
|
||||
.unwrap()
|
||||
});
|
||||
assert!(conn.cancel_job("job-1").await.unwrap());
|
||||
|
||||
let conn = Connection::new_with_handler(|_| {
|
||||
http::Response::builder()
|
||||
.status(404)
|
||||
.body("no such job")
|
||||
.unwrap()
|
||||
});
|
||||
assert!(!conn.cancel_job("nope").await.unwrap());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_job_history_parses_arrow_stream() {
|
||||
let schema = Arc::new(Schema::new(vec![Field::new(
|
||||
"state",
|
||||
DataType::Utf8,
|
||||
@@ -2694,91 +2679,29 @@ mod tests {
|
||||
let batch = RecordBatch::try_new(
|
||||
schema.clone(),
|
||||
vec![Arc::new(arrow_array::StringArray::from(vec![
|
||||
"claim_complete",
|
||||
"created", "done",
|
||||
]))],
|
||||
)
|
||||
.unwrap();
|
||||
let mut events = Vec::new();
|
||||
let mut body = Vec::new();
|
||||
{
|
||||
let mut writer =
|
||||
arrow_ipc::writer::StreamWriter::try_new(&mut events, &schema).unwrap();
|
||||
let mut writer = arrow_ipc::writer::StreamWriter::try_new(&mut body, &schema).unwrap();
|
||||
writer.write(&batch).unwrap();
|
||||
writer.finish().unwrap();
|
||||
}
|
||||
let conn = Connection::new_with_handler(move |request| {
|
||||
let body: serde_json::Value =
|
||||
serde_json::from_slice(request.body().unwrap().as_bytes().unwrap()).unwrap();
|
||||
if request.url().path() == "/v1/jobs/describe" {
|
||||
return http::Response::builder()
|
||||
.status(200)
|
||||
.body(
|
||||
r#"{"job_id": "job-1", "job_type": "refresh_column", "job_state": "IN_PROGRESS", "creation_ms": 1}"#
|
||||
.as_bytes()
|
||||
.to_vec(),
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
assert_eq!(request.url().path(), "/v1/jobs/query_events");
|
||||
// The handle supplies job_id; the caller only narrows the query.
|
||||
assert_eq!(body["job_id"], "job-1");
|
||||
assert_eq!(body["limit"], 500);
|
||||
assert_eq!(body["filter"], "state = 'claim_complete'");
|
||||
http::Response::builder()
|
||||
.status(200)
|
||||
.body(events.clone())
|
||||
.unwrap()
|
||||
});
|
||||
let job = conn.open_job("job-1").await.unwrap();
|
||||
let batches = job
|
||||
.events(
|
||||
JobEventsRequest::default()
|
||||
.limit(500)
|
||||
.filter("state = 'claim_complete'"),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(batches.len(), 1);
|
||||
assert_eq!(batches[0].num_rows(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_job_events_keep_the_schema_when_nothing_matches() {
|
||||
let schema = Arc::new(Schema::new(vec![Field::new(
|
||||
"state",
|
||||
DataType::Utf8,
|
||||
false,
|
||||
)]));
|
||||
let mut events = Vec::new();
|
||||
{
|
||||
let mut writer =
|
||||
arrow_ipc::writer::StreamWriter::try_new(&mut events, &schema).unwrap();
|
||||
writer.finish().unwrap();
|
||||
}
|
||||
let conn = Connection::new_with_handler(move |request| {
|
||||
if request.url().path() == "/v1/jobs/describe" {
|
||||
return http::Response::builder()
|
||||
.status(200)
|
||||
.body(
|
||||
r#"{"job_id": "job-1", "job_type": "refresh_column", "job_state": "IN_PROGRESS", "creation_ms": 1}"#
|
||||
.as_bytes()
|
||||
.to_vec(),
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
let body: serde_json::Value =
|
||||
let req_body: serde_json::Value =
|
||||
serde_json::from_slice(request.body().unwrap().as_bytes().unwrap()).unwrap();
|
||||
// Only the job id when the caller narrows nothing.
|
||||
assert_eq!(body, serde_json::json!({ "job_id": "job-1" }));
|
||||
assert_eq!(req_body["job_id"], "job-1");
|
||||
http::Response::builder()
|
||||
.status(200)
|
||||
.body(events.clone())
|
||||
.body(body.clone())
|
||||
.unwrap()
|
||||
});
|
||||
let job = conn.open_job("job-1").await.unwrap();
|
||||
let batches = job.events(JobEventsRequest::default()).await.unwrap();
|
||||
let batches = conn.job_history(Some("job-1")).await.unwrap();
|
||||
assert_eq!(batches.len(), 1);
|
||||
assert_eq!(batches[0].num_rows(), 0);
|
||||
assert_eq!(batches[0].schema(), schema);
|
||||
assert_eq!(batches[0].num_rows(), 2);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -2973,9 +2896,7 @@ mod tests {
|
||||
let polls_ref = polls.clone();
|
||||
let conn = Connection::new_with_handler(move |request| {
|
||||
assert_eq!(request.url().path(), "/v1/jobs/describe");
|
||||
// Two in-progress answers: one for the load, one for the first
|
||||
// status poll.
|
||||
let state = if polls_ref.fetch_add(1, Ordering::SeqCst) < 2 {
|
||||
let state = if polls_ref.fetch_add(1, Ordering::SeqCst) == 0 {
|
||||
"IN_PROGRESS"
|
||||
} else {
|
||||
"DONE"
|
||||
@@ -2988,13 +2909,11 @@ mod tests {
|
||||
))
|
||||
.unwrap()
|
||||
});
|
||||
let job = conn.open_job("job-1").await.unwrap();
|
||||
let job = conn.job("job-1").unwrap();
|
||||
assert_eq!(job.id(), Some("job-1"));
|
||||
// Opening already answered the state; no extra call needed for it.
|
||||
assert_eq!(job.state().as_deref(), Some("running"));
|
||||
assert_eq!(job.status().await.unwrap(), "running");
|
||||
job.wait().await.unwrap();
|
||||
assert_eq!(job.status().await.unwrap(), "finished");
|
||||
assert!(polls.load(Ordering::SeqCst) >= 4);
|
||||
assert!(polls.load(Ordering::SeqCst) >= 3);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,15 +5,13 @@
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
use arrow_array::RecordBatch;
|
||||
use async_trait::async_trait;
|
||||
use tokio::time::sleep;
|
||||
|
||||
use serde::Deserialize;
|
||||
|
||||
use crate::database::JobDescription;
|
||||
use crate::error::{Error, JobFailure, Result};
|
||||
use crate::job::{JobEventsRequest, JobHandle, TerminalResult};
|
||||
use crate::job::{JobHandle, TerminalResult};
|
||||
use crate::remote::client::{HttpSend, RequestResultExt, RestfulLanceDbClient};
|
||||
|
||||
/// Delay before the second job-state poll; doubles up to [`MAX_POLL_INTERVAL`].
|
||||
@@ -88,7 +86,7 @@ pub(super) struct DescribeJobResponse {
|
||||
#[serde(default)]
|
||||
pub(super) spec: serde_json::Value,
|
||||
#[serde(default)]
|
||||
pub(super) result: Option<serde_json::Value>,
|
||||
result: Option<serde_json::Value>,
|
||||
#[serde(default)]
|
||||
pub(super) failure: Option<ReportedFailure>,
|
||||
}
|
||||
@@ -112,39 +110,6 @@ impl DescribeJobResponse {
|
||||
fn into_terminal_result(self, request_id: String) -> TerminalResult {
|
||||
TerminalResult::remote(self.result, request_id)
|
||||
}
|
||||
|
||||
/// The public description this wire envelope stands for.
|
||||
pub(super) fn into_description(self) -> JobDescription {
|
||||
JobDescription {
|
||||
job_id: self.job_id,
|
||||
job_type: self.job_type,
|
||||
state: JobState::from(self.job_state.as_str()).client_label(),
|
||||
creation_ms: self.creation_ms,
|
||||
spec: self.spec,
|
||||
result: self.result,
|
||||
failure: self.failure.map(ReportedFailure::into_job_failure),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// One `/v1/jobs/query_events` round trip.
|
||||
pub(super) async fn fetch_job_events<S: HttpSend>(
|
||||
client: &RestfulLanceDbClient<S>,
|
||||
body: serde_json::Value,
|
||||
) -> Result<Vec<RecordBatch>> {
|
||||
let request = client.post("/v1/jobs/query_events").json(&body);
|
||||
let (request_id, response) = client.send(request).await?;
|
||||
let response = client.check_response(&request_id, response).await?;
|
||||
let bytes = response.bytes().await.err_to_http(request_id)?;
|
||||
let reader = arrow_ipc::reader::StreamReader::try_new(std::io::Cursor::new(bytes), None)?;
|
||||
let schema = reader.schema();
|
||||
let mut batches = reader.collect::<std::result::Result<Vec<_>, _>>()?;
|
||||
// A query that matched nothing still describes the event columns.
|
||||
// Keep that schema so callers can build a typed empty result.
|
||||
if batches.is_empty() {
|
||||
batches.push(RecordBatch::new_empty(schema));
|
||||
}
|
||||
Ok(batches)
|
||||
}
|
||||
|
||||
pub struct RemoteJob<S: HttpSend> {
|
||||
@@ -158,7 +123,7 @@ impl<S: HttpSend> RemoteJob<S> {
|
||||
}
|
||||
|
||||
/// One `/v1/jobs/describe` round trip.
|
||||
async fn fetch_description(&self) -> Result<(String, DescribeJobResponse)> {
|
||||
async fn describe(&self) -> Result<(String, DescribeJobResponse)> {
|
||||
let request = self
|
||||
.client
|
||||
.post("/v1/jobs/describe")
|
||||
@@ -183,28 +148,13 @@ impl<S: HttpSend> JobHandle for RemoteJob<S> {
|
||||
}
|
||||
|
||||
async fn status(&self) -> Result<String> {
|
||||
Ok(self.fetch_description().await?.1.state().client_label())
|
||||
}
|
||||
|
||||
async fn describe(&self) -> Result<JobDescription> {
|
||||
Ok(self.fetch_description().await?.1.into_description())
|
||||
}
|
||||
|
||||
async fn events(&self, request: JobEventsRequest) -> Result<Vec<RecordBatch>> {
|
||||
let mut body = serde_json::json!({ "job_id": self.job_id });
|
||||
if let Some(limit) = request.limit {
|
||||
body["limit"] = serde_json::Value::from(limit);
|
||||
}
|
||||
if let Some(filter) = request.filter {
|
||||
body["filter"] = serde_json::Value::String(filter);
|
||||
}
|
||||
fetch_job_events(&self.client, body).await
|
||||
Ok(self.describe().await?.1.state().client_label())
|
||||
}
|
||||
|
||||
async fn wait(&self) -> Result<TerminalResult> {
|
||||
let mut interval = INITIAL_POLL_INTERVAL;
|
||||
loop {
|
||||
let (request_id, description) = self.fetch_description().await?;
|
||||
let (request_id, description) = self.describe().await?;
|
||||
match description.state() {
|
||||
JobState::Done => return Ok(description.into_terminal_result(request_id)),
|
||||
JobState::Failed => {
|
||||
|
||||
@@ -466,9 +466,7 @@ impl TokenSource for AzureImdsSource {
|
||||
/// OAuth header provider that manages the full token lifecycle.
|
||||
///
|
||||
/// Implements [`HeaderProvider`] to inject `Authorization: Bearer <token>`
|
||||
/// headers into every LanceDB request, with automatic token refresh. It also
|
||||
/// identifies the bearer credential as OIDC so LanceDB's SQL service selects
|
||||
/// OIDC validation instead of API-key validation.
|
||||
/// headers into every LanceDB request, with automatic token refresh.
|
||||
pub struct OAuthHeaderProvider {
|
||||
token_source: Box<dyn TokenSource>,
|
||||
token_state: Arc<RwLock<TokenState>>,
|
||||
@@ -556,10 +554,10 @@ impl OAuthHeaderProvider {
|
||||
impl HeaderProvider for OAuthHeaderProvider {
|
||||
async fn get_headers(&self) -> Result<HashMap<String, String>> {
|
||||
let token = self.get_valid_token().await?;
|
||||
Ok(HashMap::from([
|
||||
("authorization".to_string(), format!("Bearer {token}")),
|
||||
("x-lancedb-credential-type".to_string(), "oidc".to_string()),
|
||||
]))
|
||||
Ok(HashMap::from([(
|
||||
"authorization".to_string(),
|
||||
format!("Bearer {token}"),
|
||||
)]))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -264,17 +264,6 @@ impl<S: HttpSend> crate::job::JobHandle for FreshnessJob<S> {
|
||||
crate::job::JobHandle::status(&self.inner).await
|
||||
}
|
||||
|
||||
async fn describe(&self) -> Result<crate::database::JobDescription> {
|
||||
crate::job::JobHandle::describe(&self.inner).await
|
||||
}
|
||||
|
||||
async fn events(
|
||||
&self,
|
||||
request: crate::job::JobEventsRequest,
|
||||
) -> Result<Vec<arrow_array::RecordBatch>> {
|
||||
crate::job::JobHandle::events(&self.inner, request).await
|
||||
}
|
||||
|
||||
async fn wait(&self) -> Result<crate::job::TerminalResult> {
|
||||
let result = crate::job::JobHandle::wait(&self.inner).await?;
|
||||
let version = self.version.read().await;
|
||||
@@ -7993,72 +7982,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
/// The refresh handle is wrapped for read-freshness tracking, so it has to
|
||||
/// forward the detail APIs too -- this is the job an operator is holding
|
||||
/// when a backfill goes quiet.
|
||||
#[tokio::test]
|
||||
async fn test_refresh_job_handle_reports_detail_and_events() {
|
||||
let schema = Arc::new(Schema::new(vec![Field::new(
|
||||
"state",
|
||||
DataType::Utf8,
|
||||
false,
|
||||
)]));
|
||||
let batch = RecordBatch::try_new(
|
||||
schema.clone(),
|
||||
vec![Arc::new(arrow_array::StringArray::from(vec![
|
||||
"claim_complete",
|
||||
]))],
|
||||
)
|
||||
.unwrap();
|
||||
let mut events = Vec::new();
|
||||
{
|
||||
let mut writer =
|
||||
arrow_ipc::writer::StreamWriter::try_new(&mut events, &schema).unwrap();
|
||||
writer.write(&batch).unwrap();
|
||||
writer.finish().unwrap();
|
||||
}
|
||||
let table = Table::new_with_handler("my_table", move |request| {
|
||||
match request.url().path() {
|
||||
"/v1/table/my_table/backfill_column" => http::Response::builder()
|
||||
.status(202)
|
||||
.body(br#"{"job_id": "j-42"}"#.to_vec())
|
||||
.unwrap(),
|
||||
"/v1/jobs/describe" => http::Response::builder()
|
||||
.status(200)
|
||||
.body(
|
||||
r#"{"job_id": "j-42", "job_type": "refresh_column", "job_state": "IN_PROGRESS", "creation_ms": 7, "spec": {"column": "doubled"}}"#
|
||||
.as_bytes()
|
||||
.to_vec(),
|
||||
)
|
||||
.unwrap(),
|
||||
"/v1/jobs/query_events" => {
|
||||
let body: serde_json::Value =
|
||||
serde_json::from_slice(request.body().unwrap().as_bytes().unwrap())
|
||||
.unwrap();
|
||||
assert_eq!(body["job_id"], "j-42");
|
||||
http::Response::builder()
|
||||
.status(200)
|
||||
.body(events.clone())
|
||||
.unwrap()
|
||||
}
|
||||
other => panic!("unexpected path {other}"),
|
||||
}
|
||||
});
|
||||
|
||||
let job = table.refresh_column_async("doubled").await.unwrap();
|
||||
job.refresh().await.unwrap();
|
||||
assert_eq!(job.state().as_deref(), Some("running"));
|
||||
assert_eq!(job.job_type().as_deref(), Some("refresh_column"));
|
||||
assert_eq!(job.creation_ms(), Some(7));
|
||||
assert_eq!(job.spec().unwrap()["column"], "doubled");
|
||||
|
||||
let batches = job
|
||||
.events(crate::job::JobEventsRequest::default())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(batches.iter().map(|b| b.num_rows()).sum::<usize>(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_refresh_submission_uses_add_columns_version_fence() {
|
||||
let table = Table::new_with_handler("my_table", |request| match request.url().path() {
|
||||
|
||||
@@ -1,124 +0,0 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
//! Handles to SQL queries running on a remote database.
|
||||
|
||||
use std::{fmt, sync::Arc};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use chrono::{DateTime, Utc};
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::{Result, arrow::SendableRecordBatchStream};
|
||||
|
||||
/// The externally visible lifecycle state of a submitted SQL query.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum QueryStatus {
|
||||
/// The server is still executing the query.
|
||||
Running,
|
||||
/// The server has made the complete result available.
|
||||
Finished,
|
||||
/// The server accepted cancellation but has not confirmed it yet.
|
||||
Cancelling,
|
||||
/// The server confirmed cancellation.
|
||||
Cancelled,
|
||||
}
|
||||
|
||||
impl fmt::Display for QueryStatus {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter.write_str(match self {
|
||||
Self::Running => "running",
|
||||
Self::Finished => "finished",
|
||||
Self::Cancelling => "cancelling",
|
||||
Self::Cancelled => "cancelled",
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// A point-in-time description of a submitted SQL query.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct QueryDescription {
|
||||
/// The stable, connection-scoped identifier assigned when the query was submitted.
|
||||
pub id: Uuid,
|
||||
/// The server-visible lifecycle state.
|
||||
pub status: QueryStatus,
|
||||
/// Server-reported completion progress, when known. Values are in `[0.0, 1.0]`,
|
||||
/// with `1.0` meaning complete.
|
||||
pub progress: Option<f64>,
|
||||
/// When the server may stop accepting this query's continuation token.
|
||||
pub expires_at: Option<DateTime<Utc>>,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
pub(crate) trait QueryHandle: Send + Sync {
|
||||
fn id(&self) -> Uuid;
|
||||
async fn describe(&self) -> Result<QueryDescription>;
|
||||
async fn reader(&self) -> Result<SendableRecordBatchStream>;
|
||||
async fn cancel(&self) -> Result<()>;
|
||||
}
|
||||
|
||||
/// A handle to a submitted SQL query.
|
||||
///
|
||||
/// The handle can be inspected, opened as an Arrow reader, or cancelled.
|
||||
/// Dropping it does not cancel the server-side query.
|
||||
/// Identifier lookup is scoped to the connection that submitted the query and
|
||||
/// is not a durable resume mechanism.
|
||||
pub struct Query {
|
||||
handle: Arc<dyn QueryHandle>,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for Query {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
formatter
|
||||
.debug_struct("Query")
|
||||
.field("id", &self.id())
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
impl Query {
|
||||
#[cfg(feature = "remote")]
|
||||
pub(crate) fn new(handle: Arc<dyn QueryHandle>) -> Self {
|
||||
Self { handle }
|
||||
}
|
||||
|
||||
/// Return the stable, connection-scoped identifier for this query.
|
||||
pub fn id(&self) -> Uuid {
|
||||
self.handle.id()
|
||||
}
|
||||
|
||||
/// Get a point-in-time description of the query.
|
||||
pub async fn describe(&self) -> Result<QueryDescription> {
|
||||
self.handle.describe().await
|
||||
}
|
||||
|
||||
/// Wait for the initial result stream and return its Arrow record batches.
|
||||
///
|
||||
/// The stream can begin yielding partial results before query execution is
|
||||
/// complete. It continues polling for newly available result endpoints
|
||||
/// until the query finishes and all endpoints have been consumed.
|
||||
///
|
||||
/// Results are single-consumer. Calling this method more than once on the
|
||||
/// same handle returns an error.
|
||||
pub async fn reader(&self) -> Result<SendableRecordBatchStream> {
|
||||
self.handle.reader().await
|
||||
}
|
||||
|
||||
/// Request cancellation of the query.
|
||||
pub async fn cancel(&self) -> Result<()> {
|
||||
self.handle.cancel().await
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::QueryStatus;
|
||||
|
||||
#[test]
|
||||
fn query_status_display_is_stable() {
|
||||
assert_eq!(QueryStatus::Running.to_string(), "running");
|
||||
assert_eq!(QueryStatus::Finished.to_string(), "finished");
|
||||
assert_eq!(QueryStatus::Cancelling.to_string(), "cancelling");
|
||||
assert_eq!(QueryStatus::Cancelled.to_string(), "cancelled");
|
||||
}
|
||||
}
|
||||
@@ -1192,9 +1192,6 @@ impl Table {
|
||||
/// valid empty blobs contain empty byte strings. Prefer
|
||||
/// [`Self::fetch_blob_files`] for large selections.
|
||||
///
|
||||
/// `_rowid` values stay valid after compaction when the table has stable
|
||||
/// row ids.
|
||||
///
|
||||
/// ```
|
||||
/// use arrow_array::UInt64Array;
|
||||
/// use futures::TryStreamExt;
|
||||
@@ -1236,9 +1233,6 @@ impl Table {
|
||||
/// the requests. Null blobs produce null output slots; empty ranges on
|
||||
/// non-null blobs produce empty byte strings.
|
||||
///
|
||||
/// `_rowid` values stay valid after compaction when the table has stable
|
||||
/// row ids.
|
||||
///
|
||||
/// ```
|
||||
/// use lancedb::blob::BlobRangeRequest;
|
||||
///
|
||||
@@ -1277,9 +1271,6 @@ impl Table {
|
||||
/// Same length and order as `row_ids`. Null rows are `None`. Bytes are not
|
||||
/// read from disk until a call to [`BlobFile::read`].
|
||||
///
|
||||
/// `_rowid` values stay valid after compaction when the table has stable
|
||||
/// row ids.
|
||||
///
|
||||
/// ```
|
||||
/// # use lancedb::Table;
|
||||
/// # async fn lazy_read(table: &Table, row_ids: &[u64]) -> Result<(), Box<dyn std::error::Error>> {
|
||||
@@ -2804,7 +2795,7 @@ impl NativeTable {
|
||||
namespace_client: Option<Arc<dyn LanceNamespace>>,
|
||||
pushdown_operations: HashSet<NamespaceClientPushdownOperation>,
|
||||
) -> Result<Self> {
|
||||
let batches = computed_columns::admit_create_source(batches)?;
|
||||
computed_columns::ensure_no_foreign_declarations(batches.arrow_schema().fields())?;
|
||||
// Default params uses format v1.
|
||||
let params = params.unwrap_or(WriteParams {
|
||||
..Default::default()
|
||||
@@ -2904,7 +2895,6 @@ impl NativeTable {
|
||||
pushdown_operations: HashSet<NamespaceClientPushdownOperation>,
|
||||
session: Option<Arc<lance::session::Session>>,
|
||||
) -> Result<Self> {
|
||||
let batches = computed_columns::admit_create_source(batches)?;
|
||||
// Build table_id from namespace + name for the storage options provider
|
||||
let mut table_id = namespace.clone();
|
||||
table_id.push(name.to_string());
|
||||
@@ -5678,7 +5668,7 @@ mod tests {
|
||||
TableStatistics {
|
||||
num_rows: 250,
|
||||
num_indices: 0,
|
||||
total_bytes: 8969,
|
||||
total_bytes: 8925,
|
||||
fragment_stats: FragmentStatistics {
|
||||
num_fragments: 11,
|
||||
num_small_fragments: 11,
|
||||
|
||||
@@ -21,7 +21,6 @@
|
||||
//! [`computed_columns`] and [`computed_column_from_field`] read declarations
|
||||
//! back off a schema.
|
||||
|
||||
use futures::StreamExt;
|
||||
use std::collections::{BTreeSet, HashMap, HashSet};
|
||||
use std::sync::Arc;
|
||||
|
||||
@@ -61,10 +60,6 @@ pub const FUNCTION_BINDING_ID_META_KEY: &str = "computed_column.function.binding
|
||||
/// Field metadata key holding this sibling's ordered Function output ordinal.
|
||||
pub const FUNCTION_OUTPUT_ORDINAL_META_KEY: &str = "computed_column.function.output_ordinal";
|
||||
|
||||
/// Reserved Function output ordinal for an internal flattened-result
|
||||
/// assignment column.
|
||||
pub const FUNCTION_ASSIGNMENT_OUTPUT_ORDINAL: u32 = u32::MAX;
|
||||
|
||||
/// Schema metadata key holding all immutable Function bindings.
|
||||
pub const FUNCTION_BINDINGS_META_KEY: &str = "lancedb::function_bindings";
|
||||
|
||||
@@ -317,29 +312,22 @@ pub(crate) fn ensure_supported_function_metadata(schema: &ArrowSchema) -> Result
|
||||
binding_id
|
||||
),
|
||||
})?;
|
||||
let destination = if output_ordinal == FUNCTION_ASSIGNMENT_OUTPUT_ORDINAL {
|
||||
binding
|
||||
.assignment()
|
||||
.map(|assignment| assignment.output_name.as_str())
|
||||
} else {
|
||||
binding
|
||||
.outputs()
|
||||
.get(output_ordinal as usize)
|
||||
.map(|output| output.output_name.as_str())
|
||||
}
|
||||
.ok_or_else(|| Error::InvalidInput {
|
||||
message: format!(
|
||||
"Function output '{}' has invalid ordinal {}",
|
||||
field.name(),
|
||||
output_ordinal
|
||||
),
|
||||
})?;
|
||||
if destination != field.name().as_str() {
|
||||
let output = binding
|
||||
.outputs()
|
||||
.get(output_ordinal as usize)
|
||||
.ok_or_else(|| Error::InvalidInput {
|
||||
message: format!(
|
||||
"Function output '{}' has invalid ordinal {}",
|
||||
field.name(),
|
||||
output_ordinal
|
||||
),
|
||||
})?;
|
||||
if output.output_name != field.name().as_str() {
|
||||
return Err(Error::InvalidInput {
|
||||
message: format!(
|
||||
"Function output '{}' does not match binding destination '{}'",
|
||||
field.name(),
|
||||
destination
|
||||
output.output_name
|
||||
),
|
||||
});
|
||||
}
|
||||
@@ -510,7 +498,6 @@ fn ensure_known_binding_shape(value: &Value) -> Result<()> {
|
||||
"function",
|
||||
"inputs",
|
||||
"outputs",
|
||||
"assignment",
|
||||
"input_schema",
|
||||
"output_schema",
|
||||
],
|
||||
@@ -559,13 +546,6 @@ fn ensure_known_binding_shape(value: &Value) -> Result<()> {
|
||||
"output mapping",
|
||||
)?;
|
||||
}
|
||||
if let Some(assignment) = object.get("assignment") {
|
||||
reject_unknown_object_fields(
|
||||
assignment,
|
||||
&["output_name", "output_field_id"],
|
||||
"assignment mapping",
|
||||
)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -760,33 +740,14 @@ fn function_output_field(name: &str, nullable: bool, raw: &str) -> Result<JsonAr
|
||||
Ok(field)
|
||||
}
|
||||
|
||||
/// Whether two fields describe the same Function output.
|
||||
///
|
||||
/// `compare_identity` covers the field's own name and nullability. Struct
|
||||
/// children carry both as part of the declaration and compare with it on. List
|
||||
/// children do not: Lance rewrites a list item's name and nullability when it
|
||||
/// writes, so a stored `fixed_size_list<item: float not null>` comes back as
|
||||
/// `fixed_size_list<item: float>` and never matches the declaration again.
|
||||
/// Comparing those by type alone keeps this agreeing with the server, which
|
||||
/// draws the same distinction and is what accepted the column when it was
|
||||
/// declared.
|
||||
fn function_output_field_matches(
|
||||
expected: &ArrowField,
|
||||
actual: &ArrowField,
|
||||
compare_identity: bool,
|
||||
) -> bool {
|
||||
if compare_identity
|
||||
&& (expected.name() != actual.name() || expected.is_nullable() != actual.is_nullable())
|
||||
{
|
||||
return false;
|
||||
}
|
||||
match (expected.is_blob_v2(), actual.is_blob_v2()) {
|
||||
(false, false) => function_output_type_matches(expected.data_type(), actual.data_type()),
|
||||
(true, true) => {
|
||||
fn function_output_field_matches(expected: &ArrowField, actual: &ArrowField) -> bool {
|
||||
expected.name() == actual.name()
|
||||
&& expected.is_nullable() == actual.is_nullable()
|
||||
&& if expected.is_blob_v2() {
|
||||
has_supported_blob_v2_layout(expected) && has_supported_blob_v2_layout(actual)
|
||||
} else {
|
||||
function_output_type_matches(expected.data_type(), actual.data_type())
|
||||
}
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
fn function_output_type_matches(expected: &DataType, actual: &DataType) -> bool {
|
||||
@@ -799,19 +760,33 @@ fn function_output_type_matches(expected: &DataType, actual: &DataType) -> bool
|
||||
&& expected
|
||||
.iter()
|
||||
.zip(actual)
|
||||
.all(|(expected, actual)| function_output_field_matches(expected, actual, true))
|
||||
.all(|(expected, actual)| function_output_field_matches(expected, actual))
|
||||
}
|
||||
(DataType::List(expected), DataType::List(actual))
|
||||
| (DataType::LargeList(expected), DataType::LargeList(actual)) => {
|
||||
function_output_field_matches(expected, actual, false)
|
||||
function_output_field_matches(expected, actual)
|
||||
}
|
||||
(
|
||||
DataType::FixedSizeList(expected, expected_size),
|
||||
DataType::FixedSizeList(actual, actual_size),
|
||||
) => expected_size == actual_size && function_output_field_matches(expected, actual, false),
|
||||
) => expected_size == actual_size && function_output_field_matches(expected, actual),
|
||||
(DataType::Map(expected, expected_sorted), DataType::Map(actual, actual_sorted)) => {
|
||||
expected_sorted == actual_sorted
|
||||
&& function_output_field_matches(expected, actual, true)
|
||||
expected_sorted == actual_sorted && function_output_field_matches(expected, actual)
|
||||
}
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
fn function_output_type_has_blob(data_type: &DataType) -> bool {
|
||||
match data_type {
|
||||
DataType::Struct(fields) => fields
|
||||
.iter()
|
||||
.any(|field| field.is_blob_v2() || function_output_type_has_blob(field.data_type())),
|
||||
DataType::List(field)
|
||||
| DataType::LargeList(field)
|
||||
| DataType::FixedSizeList(field, _)
|
||||
| DataType::Map(field, _) => {
|
||||
field.is_blob_v2() || function_output_type_has_blob(field.data_type())
|
||||
}
|
||||
_ => false,
|
||||
}
|
||||
@@ -890,20 +865,23 @@ fn ensure_binding_matches_schema(schema: &ArrowSchema, binding: &FunctionBinding
|
||||
output.output_name
|
||||
))
|
||||
})?;
|
||||
if field.name() != &output.output_name || !field.is_nullable() {
|
||||
if field.name() != &output.output_name || !field.is_nullable() || output.nullable {
|
||||
return Err(invalid_function(format!(
|
||||
"Function output '{}' no longer matches binding '{}'",
|
||||
output.output_name,
|
||||
binding.binding_id()
|
||||
)));
|
||||
}
|
||||
let type_matches = if output.arrow_type == FUNCTION_BLOB_V2_TYPE {
|
||||
has_supported_blob_v2_layout(field)
|
||||
let (type_matches, has_semantic_blob) = if output.arrow_type == FUNCTION_BLOB_V2_TYPE {
|
||||
(has_supported_blob_v2_layout(field), true)
|
||||
} else {
|
||||
let expected_type = parse_output_arrow_type(&output.arrow_type)?;
|
||||
let expected_type = lance_namespace::schema::convert_json_arrow_type(&expected_type)
|
||||
.map_err(|e| invalid_function(format!("invalid Function output type: {e}")))?;
|
||||
function_output_type_matches(&expected_type, field.data_type())
|
||||
(
|
||||
function_output_type_matches(&expected_type, field.data_type()),
|
||||
function_output_type_has_blob(&expected_type),
|
||||
)
|
||||
};
|
||||
if !type_matches {
|
||||
return Err(invalid_function(format!(
|
||||
@@ -934,71 +912,19 @@ fn ensure_binding_matches_schema(schema: &ArrowSchema, binding: &FunctionBinding
|
||||
binding.binding_id()
|
||||
)));
|
||||
}
|
||||
// Rebuild from the declaration rather than from the stored field. The
|
||||
// stored field carries Lance's write-time normalization, which would
|
||||
// never round-trip back to the schema the binding recorded -- the same
|
||||
// reason list children compare by type above. Whether the column on
|
||||
// disk still matches is settled by that comparison, not here.
|
||||
output_fields.push(function_output_field(
|
||||
field.name(),
|
||||
true,
|
||||
&output.arrow_type,
|
||||
)?);
|
||||
}
|
||||
if let Some(assignment) = binding.assignment() {
|
||||
if binding
|
||||
.outputs()
|
||||
.iter()
|
||||
.any(|output| output.result_field == WHOLE_RESULT_FIELD)
|
||||
{
|
||||
return Err(invalid_function(format!(
|
||||
"Function binding '{}' cannot attach an assignment column to a whole result",
|
||||
binding.binding_id()
|
||||
)));
|
||||
if has_semantic_blob {
|
||||
output_fields.push(function_output_field(
|
||||
field.name(),
|
||||
true,
|
||||
&output.arrow_type,
|
||||
)?);
|
||||
} else {
|
||||
let json = lance_namespace::schema::arrow_schema_to_json(&ArrowSchema::new(vec![
|
||||
ArrowField::new(field.name().clone(), field.data_type().clone(), true),
|
||||
]))
|
||||
.map_err(|e| invalid_function(format!("invalid Function output schema: {e}")))?;
|
||||
output_fields.push(json.fields.into_iter().next().unwrap());
|
||||
}
|
||||
let field = schema
|
||||
.field_with_name(&assignment.output_name)
|
||||
.map_err(|_| {
|
||||
invalid_function(format!(
|
||||
"Function binding '{}' assignment column '{}' is missing",
|
||||
binding.binding_id(),
|
||||
assignment.output_name
|
||||
))
|
||||
})?;
|
||||
let metadata = field.metadata();
|
||||
if field.data_type() != &DataType::Boolean
|
||||
|| !field.is_nullable()
|
||||
|| metadata.get(COMPUTED_COLUMN_META_KEY).map(String::as_str) != Some("true")
|
||||
|| metadata.get(KIND_META_KEY).map(String::as_str) != Some(FUNCTION_KIND)
|
||||
|| metadata
|
||||
.get(FUNCTION_BINDING_ID_META_KEY)
|
||||
.map(String::as_str)
|
||||
!= Some(binding.binding_id())
|
||||
|| metadata
|
||||
.get(FUNCTION_OUTPUT_ORDINAL_META_KEY)
|
||||
.and_then(|value| value.parse::<u32>().ok())
|
||||
!= Some(FUNCTION_ASSIGNMENT_OUTPUT_ORDINAL)
|
||||
{
|
||||
return Err(invalid_function(format!(
|
||||
"Function binding '{}' assignment column no longer matches its declaration",
|
||||
binding.binding_id()
|
||||
)));
|
||||
}
|
||||
let json = lance_namespace::schema::arrow_schema_to_json(&ArrowSchema::new(vec![
|
||||
ArrowField::new(assignment.output_name.clone(), DataType::Boolean, true),
|
||||
]))
|
||||
.map_err(|e| invalid_function(format!("invalid Function assignment schema: {e}")))?;
|
||||
output_fields.push(json.fields.into_iter().next().unwrap());
|
||||
} else if binding.outputs().iter().all(|output| output.nullable)
|
||||
&& binding
|
||||
.outputs()
|
||||
.iter()
|
||||
.all(|output| output.result_field != WHOLE_RESULT_FIELD)
|
||||
{
|
||||
return Err(invalid_function(format!(
|
||||
"Function binding '{}' has no flattened-result assignment column",
|
||||
binding.binding_id()
|
||||
)));
|
||||
}
|
||||
let output_schema = JsonArrowSchema::new(output_fields);
|
||||
let output_schema = serde_json::to_value(output_schema).map_err(|e| {
|
||||
@@ -1155,6 +1081,11 @@ pub(crate) fn plan_function_application(
|
||||
"named-struct Function result field names must be unique",
|
||||
));
|
||||
}
|
||||
if output.fields.iter().any(|field| field.nullable) {
|
||||
return Err(invalid_function(
|
||||
"Function logical outputs must be non-nullable during NULL assignment",
|
||||
));
|
||||
}
|
||||
let unknown = application
|
||||
.columns()
|
||||
.keys()
|
||||
@@ -1176,9 +1107,7 @@ pub(crate) fn plan_function_application(
|
||||
let fields = output
|
||||
.fields
|
||||
.iter()
|
||||
.map(|field| {
|
||||
function_output_field(&field.name, field.nullable, &field.arrow_type)
|
||||
})
|
||||
.map(|field| function_output_field(&field.name, false, &field.arrow_type))
|
||||
.collect::<Result<Vec<_>>>()?;
|
||||
let mut data_type = JsonArrowDataType::new("struct".to_string());
|
||||
data_type.fields = Some(fields);
|
||||
@@ -1339,106 +1268,6 @@ pub(crate) fn ensure_batch_writes_no_computed_values(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Validate every computed-column declaration `schema` carries against the
|
||||
/// schema itself: every field with declaration metadata is a complete
|
||||
/// declaration, a SQL declaration re-plans to the field it declares, a
|
||||
/// Function declaration satisfies the binding contract, and no declaration
|
||||
/// reads another computed column. What passes here is what `refresh_column`
|
||||
/// can execute.
|
||||
pub(crate) fn ensure_declarations_are_planned(schema: &ArrowSchema) -> Result<()> {
|
||||
let invalid = |message: String| Error::InvalidInput { message };
|
||||
// A field with any declaration key is a declaration; a partial one is
|
||||
// not "no declaration", it is a broken one.
|
||||
for field in schema.fields() {
|
||||
if field.metadata().keys().any(|k| is_declaration_key(k))
|
||||
&& computed_column_from_field(field).is_none()
|
||||
{
|
||||
return Err(invalid(format!(
|
||||
"field '{}' carries an incomplete computed-column declaration",
|
||||
field.name()
|
||||
)));
|
||||
}
|
||||
}
|
||||
let declared: HashSet<String> = computed_columns(schema)
|
||||
.into_iter()
|
||||
.map(|c| c.name)
|
||||
.collect();
|
||||
for column in computed_columns(schema) {
|
||||
let field = schema.field_with_name(&column.name)?;
|
||||
if !field.is_nullable() {
|
||||
return Err(invalid(format!(
|
||||
"computed column '{}' must be nullable until a refresh fills it",
|
||||
column.name
|
||||
)));
|
||||
}
|
||||
match &column.kind {
|
||||
ComputedColumnKind::Sql { expression } => {
|
||||
let others: Vec<ArrowField> = schema
|
||||
.fields()
|
||||
.iter()
|
||||
.filter(|f| f.name() != &column.name)
|
||||
.map(|f| f.as_ref().clone())
|
||||
.collect();
|
||||
let bound = bind(Arc::new(ArrowSchema::new(others)), &column.name, expression)?;
|
||||
if let Some(input) = bound.roots.iter().find(|r| declared.contains(*r)) {
|
||||
return Err(invalid(format!(
|
||||
"computed column '{}' reads computed column '{input}'",
|
||||
column.name
|
||||
)));
|
||||
}
|
||||
if &bound.data_type != field.data_type() {
|
||||
return Err(invalid(format!(
|
||||
"computed column '{}' is declared as {} but its expression yields {}",
|
||||
column.name,
|
||||
field.data_type(),
|
||||
bound.data_type
|
||||
)));
|
||||
}
|
||||
let mut declared_inputs = column.inputs.clone();
|
||||
declared_inputs.sort();
|
||||
if declared_inputs != bound.inputs {
|
||||
return Err(invalid(format!(
|
||||
"computed column '{}' declares inputs {:?} but its expression reads {:?}",
|
||||
column.name, declared_inputs, bound.inputs
|
||||
)));
|
||||
}
|
||||
}
|
||||
ComputedColumnKind::Function { binding_id, .. } => {
|
||||
// The binding validator resolves each input's leaf; the
|
||||
// no-computed-input rule is about the root it hangs from.
|
||||
let bindings = function_bindings(schema)?;
|
||||
let Some(binding) = bindings.iter().find(|b| b.binding_id() == binding_id) else {
|
||||
continue; // reported by the binding validator below
|
||||
};
|
||||
// Roots come from the canonical path parser: a quoted
|
||||
// top-level name may itself contain a dot.
|
||||
if let Some(input) = binding
|
||||
.inputs()
|
||||
.iter()
|
||||
.filter_map(|input| resolve_field_path(schema, &input.field_path).ok())
|
||||
.map(|resolved| resolved.root.name().as_str())
|
||||
.find(|r| declared.contains(*r))
|
||||
{
|
||||
return Err(invalid(format!(
|
||||
"computed column '{}' reads computed column '{input}'",
|
||||
column.name
|
||||
)));
|
||||
}
|
||||
}
|
||||
ComputedColumnKind::Unrecognized { kind } => {
|
||||
return Err(Error::NotSupported {
|
||||
message: format!(
|
||||
"computed column '{}' is defined by '{kind}', which this version \
|
||||
of lancedb cannot fill",
|
||||
column.name
|
||||
),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
ensure_supported_function_metadata(schema)
|
||||
}
|
||||
|
||||
/// Reject fields carrying declaration metadata that did not come through
|
||||
/// [`plan`]. One authority for creation, overwrite and raw transforms.
|
||||
pub(crate) fn ensure_no_foreign_declarations<'a>(
|
||||
@@ -1897,54 +1726,6 @@ pub(super) async fn add_foreign_kind(table: &crate::Table, name: &str, kind: &st
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
/// Admit a table's initial data: every declaration it carries is validated,
|
||||
/// and the stream refuses any batch with values in a computed column, whose
|
||||
/// values come from refresh alone. One boundary for every way a table is
|
||||
/// created.
|
||||
pub(crate) fn admit_create_source<S: lance_datafusion::utils::StreamingWriteSource>(
|
||||
batches: S,
|
||||
) -> Result<UnfilledDeclarations<S>> {
|
||||
let schema = batches.arrow_schema();
|
||||
ensure_declarations_are_planned(&schema)?;
|
||||
let declared = computed_columns(&schema)
|
||||
.into_iter()
|
||||
.map(|c| c.name)
|
||||
.collect();
|
||||
Ok(UnfilledDeclarations {
|
||||
inner: batches,
|
||||
declared,
|
||||
})
|
||||
}
|
||||
|
||||
/// A write source whose computed columns must arrive unfilled.
|
||||
pub(crate) struct UnfilledDeclarations<S> {
|
||||
inner: S,
|
||||
declared: Vec<String>,
|
||||
}
|
||||
|
||||
impl<S: lance_datafusion::utils::StreamingWriteSource> lance_datafusion::utils::StreamingWriteSource
|
||||
for UnfilledDeclarations<S>
|
||||
{
|
||||
fn arrow_schema(&self) -> SchemaRef {
|
||||
self.inner.arrow_schema()
|
||||
}
|
||||
|
||||
fn into_stream(self) -> datafusion_physical_plan::SendableRecordBatchStream {
|
||||
if self.declared.is_empty() {
|
||||
return self.inner.into_stream();
|
||||
}
|
||||
let schema = self.inner.arrow_schema();
|
||||
let declared = self.declared;
|
||||
let stream = self.inner.into_stream().map(move |batch| {
|
||||
let batch = batch?;
|
||||
ensure_batch_writes_no_computed_values(&declared, &batch)
|
||||
.map_err(|e| datafusion_common::DataFusionError::External(Box::new(e)))?;
|
||||
Ok(batch)
|
||||
});
|
||||
Box::pin(datafusion_physical_plan::stream::RecordBatchStreamAdapter::new(schema, stream))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
/// The gate's reproducer: the validator applies the same schema-level
|
||||
@@ -1963,69 +1744,6 @@ mod tests {
|
||||
assert!(super::validate_declarations(schema, &declarations).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn list_children_match_by_type_but_struct_children_by_identity() {
|
||||
use arrow_schema::Field as F;
|
||||
|
||||
// Lance rewrites a list item's name and nullability on write, so the
|
||||
// stored field is no longer identical to what was declared. Comparing
|
||||
// those by type keeps a table with a vector output usable.
|
||||
let declared =
|
||||
DataType::FixedSizeList(Arc::new(F::new("item", DataType::Float32, false)), 4);
|
||||
let stored = DataType::FixedSizeList(Arc::new(F::new("item", DataType::Float32, true)), 4);
|
||||
assert!(super::function_output_type_matches(&declared, &stored));
|
||||
|
||||
let renamed =
|
||||
DataType::FixedSizeList(Arc::new(F::new("element", DataType::Float32, true)), 4);
|
||||
assert!(super::function_output_type_matches(&declared, &renamed));
|
||||
|
||||
// The dimension is still part of the declaration.
|
||||
let resized = DataType::FixedSizeList(Arc::new(F::new("item", DataType::Float32, true)), 8);
|
||||
assert!(!super::function_output_type_matches(&declared, &resized));
|
||||
|
||||
// Struct children keep comparing by name and nullability.
|
||||
let struct_declared =
|
||||
DataType::Struct(vec![F::new("changed", DataType::Boolean, false)].into());
|
||||
let struct_nullable =
|
||||
DataType::Struct(vec![F::new("changed", DataType::Boolean, true)].into());
|
||||
let struct_renamed =
|
||||
DataType::Struct(vec![F::new("altered", DataType::Boolean, false)].into());
|
||||
assert!(super::function_output_type_matches(
|
||||
&struct_declared,
|
||||
&struct_declared
|
||||
));
|
||||
assert!(!super::function_output_type_matches(
|
||||
&struct_declared,
|
||||
&struct_nullable
|
||||
));
|
||||
assert!(!super::function_output_type_matches(
|
||||
&struct_declared,
|
||||
&struct_renamed
|
||||
));
|
||||
|
||||
// A list nested inside a struct gets the list rule.
|
||||
let nested_declared = DataType::Struct(
|
||||
vec![F::new(
|
||||
"tokens",
|
||||
DataType::List(Arc::new(F::new("item", DataType::Utf8, false))),
|
||||
true,
|
||||
)]
|
||||
.into(),
|
||||
);
|
||||
let nested_stored = DataType::Struct(
|
||||
vec![F::new(
|
||||
"tokens",
|
||||
DataType::List(Arc::new(F::new("item", DataType::Utf8, true))),
|
||||
true,
|
||||
)]
|
||||
.into(),
|
||||
);
|
||||
assert!(super::function_output_type_matches(
|
||||
&nested_declared,
|
||||
&nested_stored
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn output_arrow_type_grammar_matches_the_shared_golden() {
|
||||
let golden: serde_json::Value = serde_json::from_str(include_str!(
|
||||
@@ -2795,8 +2513,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
/// A create carries a declaration only if it re-plans completely; this
|
||||
/// one lacks its inputs and is refused before its forged value matters.
|
||||
#[tokio::test]
|
||||
async fn test_create_table_cannot_inject_a_declaration() {
|
||||
let conn = connect("memory://").execute().await.unwrap();
|
||||
@@ -2824,7 +2540,7 @@ mod tests {
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(
|
||||
matches!(&err, Error::InvalidInput { message } if message.contains("computed column 'doubled'")),
|
||||
matches!(&err, Error::InvalidInput { message } if message.contains("computed()")),
|
||||
"{err:?}"
|
||||
);
|
||||
}
|
||||
@@ -3142,17 +2858,6 @@ mod tests {
|
||||
&inputs,
|
||||
));
|
||||
}
|
||||
if let Some(assignment) = binding.assignment() {
|
||||
fields.push(
|
||||
ArrowField::new(&assignment.output_name, DataType::Boolean, true).with_metadata(
|
||||
function_computed_column_metadata(
|
||||
binding.binding_id(),
|
||||
FUNCTION_ASSIGNMENT_OUTPUT_ORDINAL,
|
||||
&inputs,
|
||||
),
|
||||
),
|
||||
);
|
||||
}
|
||||
ArrowSchema::new(fields)
|
||||
}
|
||||
|
||||
@@ -3170,74 +2875,6 @@ mod tests {
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_binding_preserves_all_nullable_outputs_with_an_assignment_column() {
|
||||
let mut raw_binding: Value = serde_json::from_str(include_str!(
|
||||
"../../tests/fixtures/first_class_functions/v1/remote_function_binding.json"
|
||||
))
|
||||
.unwrap();
|
||||
raw_binding["outputs"][0]["nullable"] = Value::Bool(true);
|
||||
raw_binding["outputs"][1]["nullable"] = Value::Bool(true);
|
||||
let without_assignment: FunctionBinding =
|
||||
serde_json::from_value(raw_binding.clone()).unwrap();
|
||||
let error = ensure_binding_matches_schema(
|
||||
&valid_function_binding_schema(true, true, &without_assignment),
|
||||
&without_assignment,
|
||||
)
|
||||
.unwrap_err();
|
||||
assert!(
|
||||
error
|
||||
.to_string()
|
||||
.contains("flattened-result assignment column")
|
||||
);
|
||||
|
||||
raw_binding["assignment"] = serde_json::json!({
|
||||
"output_name": "__function_assignment_fb_01K3TEXT",
|
||||
"output_field_id": -1,
|
||||
});
|
||||
raw_binding["output_schema"]["fields"]
|
||||
.as_array_mut()
|
||||
.unwrap()
|
||||
.push(serde_json::json!({
|
||||
"name": "__function_assignment_fb_01K3TEXT",
|
||||
"nullable": true,
|
||||
"type": {"type": "bool"},
|
||||
}));
|
||||
let binding: FunctionBinding = serde_json::from_value(raw_binding).unwrap();
|
||||
ensure_binding_matches_schema(
|
||||
&valid_function_binding_schema(true, true, &binding),
|
||||
&binding,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let schema = ArrowSchema::new_with_metadata(
|
||||
valid_function_binding_schema(true, true, &binding)
|
||||
.fields()
|
||||
.to_vec(),
|
||||
HashMap::from([(
|
||||
FUNCTION_BINDINGS_META_KEY.to_string(),
|
||||
function_bindings_metadata(std::slice::from_ref(&binding)).unwrap(),
|
||||
)]),
|
||||
);
|
||||
ensure_supported_function_metadata(&schema).unwrap();
|
||||
|
||||
let mut metadata: Value =
|
||||
serde_json::from_str(schema.metadata().get(FUNCTION_BINDINGS_META_KEY).unwrap())
|
||||
.unwrap();
|
||||
metadata["bindings"][0]["assignment"]["future"] = Value::Bool(true);
|
||||
let future_schema = ArrowSchema::new_with_metadata(
|
||||
schema.fields().to_vec(),
|
||||
HashMap::from([(
|
||||
FUNCTION_BINDINGS_META_KEY.to_string(),
|
||||
serde_json::to_string(&metadata).unwrap(),
|
||||
)]),
|
||||
);
|
||||
assert!(matches!(
|
||||
ensure_supported_function_metadata(&future_schema),
|
||||
Err(Error::NotSupported { .. })
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_nullable_function_input_cannot_bind_to_non_nullable_parameter() {
|
||||
let mut raw_binding: Value = serde_json::from_str(include_str!(
|
||||
@@ -3457,35 +3094,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_named_struct_plan_preserves_nullable_result_fields() {
|
||||
let mut value = serde_json::to_value(named_struct_application("{}")).unwrap();
|
||||
value["output"]["fields"][0]["nullable"] = Value::Bool(true);
|
||||
value["output"]["fields"][1]["nullable"] = Value::Bool(true);
|
||||
let application = FunctionApplication::from_json(&value.to_string()).unwrap();
|
||||
|
||||
let expanded =
|
||||
plan_function_application(&function_input_schema(), &application, None).unwrap();
|
||||
assert!(
|
||||
expanded
|
||||
.output_schema
|
||||
.fields
|
||||
.iter()
|
||||
.all(|field| field.nullable)
|
||||
);
|
||||
|
||||
let whole =
|
||||
plan_function_application(&function_input_schema(), &application, Some("features"))
|
||||
.unwrap();
|
||||
let fields = whole.output_schema.fields[0]
|
||||
.r#type
|
||||
.fields
|
||||
.as_ref()
|
||||
.unwrap();
|
||||
assert!(fields[0].nullable);
|
||||
assert!(fields[1].nullable);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_blob_function_plans_semantic_input_and_scalar_output() {
|
||||
let schema = ArrowSchema::new(vec![crate::blob("image", false)]);
|
||||
@@ -3502,70 +3110,6 @@ mod tests {
|
||||
assert!(output_schema.field(0).is_blob_v2());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn binding_accepts_a_lance_normalized_list_child() {
|
||||
// The whole guard, not just the type helper: this also reaches the
|
||||
// output-schema comparison at the end of ensure_binding_matches_schema,
|
||||
// which used to rebuild the schema from the stored field and so failed
|
||||
// on exactly the same normalization.
|
||||
let input = ArrowField::new("value", DataType::Int64, false);
|
||||
let application = FunctionApplication::from_json(
|
||||
&serde_json::json!({
|
||||
"function": {"name": "embed", "version": "fv_embed"},
|
||||
"inputs": [{
|
||||
"parameter": "value",
|
||||
"kind": "column",
|
||||
"value": {"path": "value"}
|
||||
}],
|
||||
"output": {
|
||||
"kind": "scalar",
|
||||
"arrow_type": "fixed_size_list<float32, 4>",
|
||||
"nullable": false
|
||||
}
|
||||
})
|
||||
.to_string(),
|
||||
)
|
||||
.unwrap();
|
||||
let plan = plan_function_application(
|
||||
&ArrowSchema::new(vec![input.clone()]),
|
||||
&application,
|
||||
Some("embedding"),
|
||||
)
|
||||
.unwrap();
|
||||
let binding = binding_from_plan(&plan);
|
||||
|
||||
// The declaration says the item is non-nullable; Lance rewrites it to
|
||||
// nullable on write, so this is what the column looks like on disk.
|
||||
let stored = DataType::FixedSizeList(
|
||||
Arc::new(ArrowField::new("item", DataType::Float32, true)),
|
||||
4,
|
||||
);
|
||||
let output = ArrowField::new("embedding", stored, true).with_metadata(
|
||||
function_computed_column_metadata(binding.binding_id(), 0, &["value".into()]),
|
||||
);
|
||||
|
||||
ensure_binding_matches_schema(&ArrowSchema::new(vec![input.clone(), output]), &binding)
|
||||
.unwrap();
|
||||
|
||||
// A different element type is still a mismatch.
|
||||
let wrong = ArrowField::new(
|
||||
"embedding",
|
||||
DataType::FixedSizeList(
|
||||
Arc::new(ArrowField::new("item", DataType::Float64, true)),
|
||||
4,
|
||||
),
|
||||
true,
|
||||
)
|
||||
.with_metadata(function_computed_column_metadata(
|
||||
binding.binding_id(),
|
||||
0,
|
||||
&["value".into()],
|
||||
));
|
||||
assert!(
|
||||
ensure_binding_matches_schema(&ArrowSchema::new(vec![input, wrong]), &binding).is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_blob_scalar_binding_accepts_full_logical_layout() {
|
||||
let input = crate::blob("image", false);
|
||||
|
||||
@@ -133,7 +133,7 @@ impl NativeTable {
|
||||
),
|
||||
});
|
||||
}
|
||||
(resolved.canonical_path, resolved.terminal_field)
|
||||
(resolved.canonical_path, resolved.field)
|
||||
} else {
|
||||
Self::resolve_index_field(dataset.schema(), &opts.columns[0])?
|
||||
};
|
||||
@@ -439,8 +439,7 @@ mod tests {
|
||||
use arrow_array::record_batch;
|
||||
use arrow_array::{
|
||||
Array, ArrayRef, BinaryArray, BooleanArray, FixedSizeListArray, Float32Array, Int32Array,
|
||||
LargeBinaryArray, LargeStringArray, ListArray, RecordBatch, StringArray, StructArray,
|
||||
UInt32Array,
|
||||
LargeBinaryArray, LargeStringArray, RecordBatch, StringArray, StructArray,
|
||||
};
|
||||
use arrow_data::ArrayDataBuilder;
|
||||
use arrow_schema::{DataType, Field, Schema};
|
||||
@@ -459,7 +458,6 @@ mod tests {
|
||||
use crate::query::{ExecutableQuery, QueryBase};
|
||||
use crate::table::optimize::{CompactionOptions, OptimizeAction};
|
||||
use lance_index::scalar::FullTextSearchQuery;
|
||||
use lance_index::scalar::inverted::query::{FtsQuery, MatchQuery};
|
||||
|
||||
fn create_fixed_size_list<T: Array>(
|
||||
values: T,
|
||||
@@ -601,80 +599,6 @@ mod tests {
|
||||
assert!(invalid_granularity.is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_nested_list_fts_uses_deepest_document_coordinates() {
|
||||
let conn = connect("memory://").execute().await.unwrap();
|
||||
let mut docs = ListBuilder::new(ListBuilder::new(StringBuilder::new()));
|
||||
|
||||
docs.values().values().append_value("alpha");
|
||||
docs.values().values().append_value("beta");
|
||||
docs.values().append(true);
|
||||
docs.values().values().append_value("gamma");
|
||||
docs.values().values().append_value("alpha delta");
|
||||
docs.values().append(true);
|
||||
docs.append(true);
|
||||
|
||||
docs.values().append(true);
|
||||
docs.values().values().append_value("alpha");
|
||||
docs.values().append(true);
|
||||
docs.append(true);
|
||||
|
||||
let batch = RecordBatch::try_from_iter(vec![
|
||||
("id", Arc::new(Int32Array::from(vec![0, 1])) as ArrayRef),
|
||||
("docs", Arc::new(docs.finish()) as ArrayRef),
|
||||
])
|
||||
.unwrap();
|
||||
let table = conn.create_table("nested", batch).execute().await.unwrap();
|
||||
|
||||
let job = table
|
||||
.create_index(
|
||||
&["docs"],
|
||||
Index::FTS(
|
||||
FtsIndexBuilder::default()
|
||||
.document_granularity(DocumentGranularity::ListElement),
|
||||
),
|
||||
)
|
||||
.execute_async()
|
||||
.await
|
||||
.unwrap();
|
||||
job.wait().await.unwrap();
|
||||
|
||||
let query = FullTextSearchQuery::new_query(FtsQuery::Match(
|
||||
MatchQuery::new("alpha".to_string())
|
||||
.with_column(Some("docs".to_string()))
|
||||
.with_document_granularity(DocumentGranularity::ListElement),
|
||||
));
|
||||
let batches = table
|
||||
.query()
|
||||
.full_text_search(query)
|
||||
.limit(10)
|
||||
.execute()
|
||||
.await
|
||||
.unwrap()
|
||||
.try_collect::<Vec<_>>()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let mut hits = Vec::new();
|
||||
for batch in batches {
|
||||
let ids = batch["id"].as_any().downcast_ref::<Int32Array>().unwrap();
|
||||
let coordinates = batch["_doc_index"]
|
||||
.as_any()
|
||||
.downcast_ref::<ListArray>()
|
||||
.unwrap();
|
||||
for row in 0..batch.num_rows() {
|
||||
let coordinate = coordinates.value(row);
|
||||
let coordinate = coordinate.as_any().downcast_ref::<UInt32Array>().unwrap();
|
||||
hits.push((ids.value(row), coordinate.values().to_vec()));
|
||||
}
|
||||
}
|
||||
hits.sort_unstable();
|
||||
assert_eq!(
|
||||
hits,
|
||||
vec![(0, vec![0, 0]), (0, vec![1, 1]), (1, vec![1, 0])]
|
||||
);
|
||||
}
|
||||
|
||||
/// Concurrent waiters, and a wait issued after the job settled, all
|
||||
/// succeed once the build does.
|
||||
#[tokio::test]
|
||||
|
||||
@@ -5,17 +5,12 @@
|
||||
//!
|
||||
//! [`super::cast::cast_to_table_schema`] calls [`coerce_blob_expr`].
|
||||
|
||||
use std::fmt;
|
||||
use std::hash::{Hash, Hasher};
|
||||
use std::sync::Arc;
|
||||
|
||||
use arrow_array::{Array, BooleanArray, RecordBatch};
|
||||
use arrow_schema::{DataType, Field, FieldRef, Fields, Schema};
|
||||
use arrow_select::nullif::nullif;
|
||||
use arrow_schema::{DataType, Field, FieldRef, Fields};
|
||||
use datafusion::functions::core::{get_field, named_struct};
|
||||
use datafusion_common::ScalarValue;
|
||||
use datafusion_common::config::ConfigOptions;
|
||||
use datafusion_expr::ColumnarValue;
|
||||
use datafusion_physical_expr::ScalarFunctionExpr;
|
||||
use datafusion_physical_expr::expressions::{CastExpr, Literal};
|
||||
use datafusion_physical_plan::PhysicalExpr;
|
||||
@@ -138,102 +133,16 @@ pub(super) fn coerce_blob_expr(
|
||||
ns_args.push(value);
|
||||
}
|
||||
|
||||
let built: Arc<dyn PhysicalExpr> = Arc::new(ScalarFunctionExpr::new(
|
||||
let expr: Arc<dyn PhysicalExpr> = Arc::new(ScalarFunctionExpr::new(
|
||||
&format!("named_struct({})", table_field.name()),
|
||||
named_struct(),
|
||||
ns_args,
|
||||
table_field.clone(),
|
||||
config.clone(),
|
||||
));
|
||||
|
||||
// `named_struct` always yields a valid struct, so a null input would land
|
||||
// as a row that set neither `data` nor `uri` -- not an absent blob but a
|
||||
// malformed one, which Lance rejects on write.
|
||||
let expr: Arc<dyn PhysicalExpr> = Arc::new(AbsentBlobIsNull {
|
||||
source: input_expr,
|
||||
built,
|
||||
field: table_field.clone(),
|
||||
});
|
||||
Ok((expr, table_field.clone()))
|
||||
}
|
||||
|
||||
/// Carries the source column's nullity onto the struct built for it.
|
||||
///
|
||||
/// This is its own expression rather than a `CASE` because the projection
|
||||
/// takes its output field from `return_field`, and the generic implementation
|
||||
/// rebuilds a bare field -- which would drop the `lance.blob.v2` extension
|
||||
/// metadata and stop the column being recognised as a blob at all.
|
||||
#[derive(Debug, Clone)]
|
||||
struct AbsentBlobIsNull {
|
||||
source: Arc<dyn PhysicalExpr>,
|
||||
built: Arc<dyn PhysicalExpr>,
|
||||
field: FieldRef,
|
||||
}
|
||||
|
||||
impl fmt::Display for AbsentBlobIsNull {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
write!(f, "absent_blob_is_null({}, {})", self.source, self.built)
|
||||
}
|
||||
}
|
||||
|
||||
impl PartialEq for AbsentBlobIsNull {
|
||||
fn eq(&self, other: &Self) -> bool {
|
||||
self.source.eq(&other.source) && self.built.eq(&other.built) && self.field == other.field
|
||||
}
|
||||
}
|
||||
|
||||
impl Eq for AbsentBlobIsNull {}
|
||||
|
||||
impl Hash for AbsentBlobIsNull {
|
||||
fn hash<H: Hasher>(&self, state: &mut H) {
|
||||
self.source.hash(state);
|
||||
self.built.hash(state);
|
||||
self.field.hash(state);
|
||||
}
|
||||
}
|
||||
|
||||
impl PhysicalExpr for AbsentBlobIsNull {
|
||||
fn return_field(&self, _input_schema: &Schema) -> datafusion_common::Result<FieldRef> {
|
||||
Ok(self.field.clone())
|
||||
}
|
||||
|
||||
fn nullable(&self, _input_schema: &Schema) -> datafusion_common::Result<bool> {
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
fn evaluate(&self, batch: &RecordBatch) -> datafusion_common::Result<ColumnarValue> {
|
||||
let rows = batch.num_rows();
|
||||
let built = self.built.evaluate(batch)?.into_array(rows)?;
|
||||
let source = self.source.evaluate(batch)?.into_array(rows)?;
|
||||
let Some(nulls) = source.logical_nulls() else {
|
||||
return Ok(ColumnarValue::Array(built));
|
||||
};
|
||||
// `nullif` nulls the rows the mask marks true, which is where the
|
||||
// source had no value.
|
||||
let absent = BooleanArray::new(!nulls.inner(), None);
|
||||
Ok(ColumnarValue::Array(nullif(built.as_ref(), &absent)?))
|
||||
}
|
||||
|
||||
fn children(&self) -> Vec<&Arc<dyn PhysicalExpr>> {
|
||||
vec![&self.source, &self.built]
|
||||
}
|
||||
|
||||
fn with_new_children(
|
||||
self: Arc<Self>,
|
||||
children: Vec<Arc<dyn PhysicalExpr>>,
|
||||
) -> datafusion_common::Result<Arc<dyn PhysicalExpr>> {
|
||||
Ok(Arc::new(Self {
|
||||
source: children[0].clone(),
|
||||
built: children[1].clone(),
|
||||
field: self.field.clone(),
|
||||
}))
|
||||
}
|
||||
|
||||
fn fmt_sql(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
write!(f, "{self}")
|
||||
}
|
||||
}
|
||||
|
||||
enum BlobInputShape<'a> {
|
||||
Bytes,
|
||||
String,
|
||||
@@ -404,11 +313,6 @@ mod tests {
|
||||
let data = image.column_by_name("data").unwrap();
|
||||
assert!(!data.is_null(0));
|
||||
assert!(data.is_null(1));
|
||||
// The row itself has to be null, not merely a struct whose children
|
||||
// are. A present-but-empty struct set neither `data` nor `uri`, which
|
||||
// Lance rejects as malformed rather than reading as an absent blob.
|
||||
assert!(!image.is_null(0));
|
||||
assert!(image.is_null(1));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -227,7 +227,7 @@ pub(crate) fn resolve_arrow_field_path(schema: &Schema, column: &str) -> Result<
|
||||
|
||||
pub(crate) struct ResolvedFtsField {
|
||||
pub canonical_path: String,
|
||||
pub terminal_field: Field,
|
||||
pub field: Field,
|
||||
pub list_depth: usize,
|
||||
}
|
||||
|
||||
@@ -309,7 +309,7 @@ pub(crate) fn resolve_lance_fts_field_path(
|
||||
);
|
||||
Ok(ResolvedFtsField {
|
||||
canonical_path,
|
||||
terminal_field: Field::from(terminal),
|
||||
field: Field::from(field),
|
||||
list_depth,
|
||||
})
|
||||
}
|
||||
@@ -375,7 +375,7 @@ pub(crate) fn resolve_arrow_fts_field_path(
|
||||
message: format!("Invalid schema: {}", e),
|
||||
})?;
|
||||
let resolved = resolve_lance_fts_field_path(&lance_schema, column)?;
|
||||
Ok((resolved.canonical_path, resolved.terminal_field))
|
||||
Ok((resolved.canonical_path, resolved.field))
|
||||
}
|
||||
|
||||
pub fn supported_btree_data_type(dtype: &DataType) -> bool {
|
||||
@@ -647,9 +647,8 @@ mod tests {
|
||||
Field::new("docs", text_list(), true),
|
||||
]);
|
||||
|
||||
let (path, field) = resolve_arrow_fts_field_path(&schema, "docs.content").unwrap();
|
||||
let (path, _) = resolve_arrow_fts_field_path(&schema, "docs.content").unwrap();
|
||||
assert_eq!(path, "docs.content");
|
||||
assert_eq!(field.data_type(), &DataType::Utf8);
|
||||
|
||||
let lance_schema = lance_core::datatypes::Schema::try_from(&schema).unwrap();
|
||||
let field_id = lance_schema
|
||||
|
||||
@@ -19,7 +19,6 @@ use lancedb::{
|
||||
connect, connect_namespace,
|
||||
database::listing::{
|
||||
ListingDatabaseOptions, NewTableConfig, OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS,
|
||||
OPT_NEW_TABLE_STORAGE_VERSION,
|
||||
},
|
||||
query::{ExecutableQuery, QueryBase},
|
||||
table::{AddDataMode, CompactionOptions, OptimizeAction, OptimizeStats, WriteOptions},
|
||||
@@ -112,7 +111,7 @@ async fn query_image_struct(table: &Table) -> StructArray {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn declaring_blob_column_uses_v2_2_and_default_row_ids() -> Result<()> {
|
||||
async fn declaring_blob_column_bumps_format_and_enables_stable_row_ids() -> Result<()> {
|
||||
let tmp = tempdir().unwrap();
|
||||
let db = connect(tmp.path().to_str().unwrap()).execute().await?;
|
||||
let table = db
|
||||
@@ -121,12 +120,12 @@ async fn declaring_blob_column_uses_v2_2_and_default_row_ids() -> Result<()> {
|
||||
.await?;
|
||||
|
||||
assert!(supports_blob_v2(storage_format_version(&table).await));
|
||||
assert!(!uses_stable_row_ids(&table).await);
|
||||
assert!(uses_stable_row_ids(&table).await);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn blob_create_honors_disabled_stable_row_ids() -> Result<()> {
|
||||
async fn explicit_stable_row_id_setting_wins_over_blob_default() -> Result<()> {
|
||||
let tmp = tempdir().unwrap();
|
||||
let db = connect(tmp.path().to_str().unwrap()).execute().await?;
|
||||
let table = db
|
||||
@@ -147,10 +146,7 @@ async fn non_blob_table_keeps_default_format_and_row_id_setting() -> Result<()>
|
||||
let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)]));
|
||||
let table = db.create_empty_table("t", schema).execute().await?;
|
||||
|
||||
assert_eq!(
|
||||
storage_format_version(&table).await,
|
||||
LanceFileVersion::Stable.resolve()
|
||||
);
|
||||
assert!(!supports_blob_v2(storage_format_version(&table).await));
|
||||
assert!(!uses_stable_row_ids(&table).await);
|
||||
Ok(())
|
||||
}
|
||||
@@ -183,7 +179,7 @@ async fn creating_with_blob_data_bumps_format() -> Result<()> {
|
||||
let table = db.create_table("t", batch).execute().await?;
|
||||
|
||||
assert!(supports_blob_v2(storage_format_version(&table).await));
|
||||
assert!(!uses_stable_row_ids(&table).await);
|
||||
assert!(uses_stable_row_ids(&table).await);
|
||||
assert_eq!(table.count_rows(None).await?, 1);
|
||||
Ok(())
|
||||
}
|
||||
@@ -281,7 +277,7 @@ async fn add_rejects_uncoercible_blob_input() -> Result<()> {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn connection_disables_stable_row_ids_on_blob_create() -> Result<()> {
|
||||
async fn connection_level_stable_row_id_setting_wins_over_blob_default() -> Result<()> {
|
||||
let tmp = tempdir().unwrap();
|
||||
let db = connect(tmp.path().to_str().unwrap())
|
||||
.storage_option(OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS, "false")
|
||||
@@ -298,7 +294,7 @@ async fn connection_disables_stable_row_ids_on_blob_create() -> Result<()> {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn namespace_blob_create_uses_v2_2_and_default_row_ids() -> Result<()> {
|
||||
async fn namespace_create_applies_blob_defaults() -> Result<()> {
|
||||
let tmp = tempdir().unwrap();
|
||||
let mut properties = std::collections::HashMap::new();
|
||||
properties.insert("root".to_string(), tmp.path().to_str().unwrap().to_string());
|
||||
@@ -308,23 +304,6 @@ async fn namespace_blob_create_uses_v2_2_and_default_row_ids() -> Result<()> {
|
||||
.execute()
|
||||
.await?;
|
||||
|
||||
assert!(supports_blob_v2(storage_format_version(&table).await));
|
||||
assert!(!uses_stable_row_ids(&table).await);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn namespace_create_honors_enabled_stable_row_ids() -> Result<()> {
|
||||
let tmp = tempdir().unwrap();
|
||||
let mut properties = std::collections::HashMap::new();
|
||||
properties.insert("root".to_string(), tmp.path().to_str().unwrap().to_string());
|
||||
let db = connect_namespace("dir", properties).execute().await?;
|
||||
let table = db
|
||||
.create_empty_table("t", blob_table_schema())
|
||||
.storage_option(OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS, "true")
|
||||
.execute()
|
||||
.await?;
|
||||
|
||||
assert!(supports_blob_v2(storage_format_version(&table).await));
|
||||
assert!(uses_stable_row_ids(&table).await);
|
||||
Ok(())
|
||||
@@ -451,35 +430,6 @@ async fn collect_id_rowid(table: &Table) -> Result<Vec<(i64, u64)>> {
|
||||
.collect())
|
||||
}
|
||||
|
||||
fn assert_missing_blob_row_ids(err: &Error) {
|
||||
assert!(matches!(err, Error::InvalidInput { .. }), "got {err:?}");
|
||||
let message = err.to_string();
|
||||
assert!(message.contains("row ids"), "{message}");
|
||||
assert!(!message.contains("rowaddr"), "{message}");
|
||||
assert!(!message.contains("fragment"), "{message}");
|
||||
}
|
||||
|
||||
async fn assert_fetch_apis_reject_missing_row_ids(table: &Table, row_ids: &[u64]) -> Result<()> {
|
||||
let err = table.fetch_blobs("image", row_ids).await.unwrap_err();
|
||||
assert_missing_blob_row_ids(&err);
|
||||
|
||||
let err = table.fetch_blob_files("image", row_ids).await.unwrap_err();
|
||||
assert_missing_blob_row_ids(&err);
|
||||
|
||||
let err = table
|
||||
.fetch_blob_ranges(
|
||||
"image",
|
||||
row_ids
|
||||
.iter()
|
||||
.copied()
|
||||
.map(|row_id| BlobRangeRequest::new(row_id, 0, 1)),
|
||||
)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_missing_blob_row_ids(&err);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn fetch_blobs_round_trips_bytes() -> Result<()> {
|
||||
let tmp = tempdir().unwrap();
|
||||
@@ -532,7 +482,7 @@ async fn fetch_blobs_round_trips_nested_blob_column() -> Result<()> {
|
||||
let table = db.create_table("t", batch).execute().await?;
|
||||
|
||||
assert!(supports_blob_v2(storage_format_version(&table).await));
|
||||
assert!(!uses_stable_row_ids(&table).await);
|
||||
assert!(uses_stable_row_ids(&table).await);
|
||||
|
||||
let ids = collect_row_ids(&table).await?;
|
||||
let bytes = table.fetch_blobs("info.blob", &ids).await?;
|
||||
@@ -706,7 +656,8 @@ async fn fetch_blob_ranges_validates_requests() -> Result<()> {
|
||||
.fetch_blob_ranges("image", [BlobRangeRequest::new(u64::MAX, 0, 1)])
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_missing_blob_row_ids(&err);
|
||||
assert!(matches!(&err, Error::InvalidInput { .. }), "got {err:?}");
|
||||
assert!(err.to_string().contains("row IDs"));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -739,21 +690,7 @@ async fn fetch_blobs_out_of_range_id_errors_without_panic() -> Result<()> {
|
||||
let table = create_inline_blob_table(&db, "t", &[1], &[Some(b"x".as_slice())]).await?;
|
||||
|
||||
let err = table.fetch_blobs("image", &[u64::MAX]).await.unwrap_err();
|
||||
assert_missing_blob_row_ids(&err);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn fetch_blob_files_rejects_missing_fragment_row_addr() -> Result<()> {
|
||||
let tmp = tempdir().unwrap();
|
||||
let db = connect(tmp.path().to_str().unwrap()).execute().await?;
|
||||
let table = create_inline_blob_table(&db, "t", &[1], &[Some(b"x".as_slice())]).await?;
|
||||
|
||||
let err = table
|
||||
.fetch_blob_files("image", &[1u64 << 32])
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_missing_blob_row_ids(&err);
|
||||
assert!(err.to_string().contains("row IDs"));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -763,25 +700,24 @@ async fn fetch_blob_apis_reject_mixed_valid_and_missing_row_ids() -> Result<()>
|
||||
let db = connect(tmp.path().to_str().unwrap()).execute().await?;
|
||||
let table = create_inline_blob_table(&db, "t", &[1], &[Some(b"x".as_slice())]).await?;
|
||||
let row_id = collect_row_ids(&table).await?[0];
|
||||
let missing_row_addr = 1u64 << 32;
|
||||
let row_ids = [missing_row_addr, row_id];
|
||||
assert_fetch_apis_reject_missing_row_ids(&table, &row_ids).await
|
||||
}
|
||||
let row_ids = [u64::MAX, row_id];
|
||||
|
||||
#[tokio::test]
|
||||
async fn fetch_blob_apis_reject_deleted_row_ids() -> Result<()> {
|
||||
let tmp = tempdir().unwrap();
|
||||
let db = connect(tmp.path().to_str().unwrap()).execute().await?;
|
||||
let table =
|
||||
create_inline_blob_table(&db, "t", &[1, 2], &[Some(b"one".as_slice()), Some(b"two")])
|
||||
.await?;
|
||||
let pairs = collect_id_rowid(&table).await?;
|
||||
let deleted_row_addr = pairs.iter().find(|(id, _)| *id == 2).unwrap().1;
|
||||
let live_row_addr = pairs.iter().find(|(id, _)| *id == 1).unwrap().1;
|
||||
let err = table.fetch_blobs("image", &row_ids).await.unwrap_err();
|
||||
assert!(matches!(&err, Error::InvalidInput { .. }), "got {err:?}");
|
||||
assert!(err.to_string().contains("row IDs"));
|
||||
|
||||
table.delete("id = 2").await?;
|
||||
let err = table.fetch_blob_files("image", &row_ids).await.unwrap_err();
|
||||
assert!(matches!(&err, Error::InvalidInput { .. }), "got {err:?}");
|
||||
assert!(err.to_string().contains("row IDs"));
|
||||
|
||||
assert_fetch_apis_reject_missing_row_ids(&table, &[deleted_row_addr, live_row_addr]).await
|
||||
let requests = row_ids.map(|row_id| BlobRangeRequest::new(row_id, 0, 1));
|
||||
let err = table
|
||||
.fetch_blob_ranges("image", requests)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(matches!(&err, Error::InvalidInput { .. }), "got {err:?}");
|
||||
assert!(err.to_string().contains("row IDs"));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -813,11 +749,7 @@ async fn fetch_blobs_rejects_unknown_column() -> Result<()> {
|
||||
#[tokio::test]
|
||||
async fn fetch_blobs_rejects_legacy_v1_blob_column() -> Result<()> {
|
||||
let tmp = tempdir().unwrap();
|
||||
// Legacy v1 blob columns are only writable at file version <= 2.1.
|
||||
let db = connect(tmp.path().to_str().unwrap())
|
||||
.storage_options([(OPT_NEW_TABLE_STORAGE_VERSION, "2.1")])
|
||||
.execute()
|
||||
.await?;
|
||||
let db = connect(tmp.path().to_str().unwrap()).execute().await?;
|
||||
let legacy = Field::new("image", DataType::LargeBinary, true).with_metadata(
|
||||
std::collections::HashMap::from([("lance-encoding:blob".to_string(), "true".to_string())]),
|
||||
);
|
||||
@@ -988,10 +920,7 @@ async fn fetch_blobs_after_delete() -> Result<()> {
|
||||
#[tokio::test]
|
||||
async fn fetch_blobs_with_precompaction_row_ids_survives_compaction() -> Result<()> {
|
||||
let tmp = tempdir().unwrap();
|
||||
let db = connect(tmp.path().to_str().unwrap())
|
||||
.storage_option(OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS, "true")
|
||||
.execute()
|
||||
.await?;
|
||||
let db = connect(tmp.path().to_str().unwrap()).execute().await?;
|
||||
let table = create_inline_blob_table(&db, "t", &[1], &[Some(b"frag-one".as_slice())]).await?;
|
||||
table
|
||||
.add(binary_input_batch(&[2], &[Some(b"frag-two".as_slice())]))
|
||||
|
||||
Reference in New Issue
Block a user