mirror of
https://github.com/lancedb/lancedb.git
synced 2026-09-09 06:42:30 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
cacf972b97 |
+1
-1
@@ -1,5 +1,5 @@
|
||||
[tool.bumpversion]
|
||||
current_version = "0.39.0-beta.6"
|
||||
current_version = "0.39.0-beta.1"
|
||||
parse = """(?x)
|
||||
(?P<major>0|[1-9]\\d*)\\.
|
||||
(?P<minor>0|[1-9]\\d*)\\.
|
||||
|
||||
Generated
+51
-53
@@ -3526,8 +3526,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
|
||||
|
||||
[[package]]
|
||||
name = "fsst"
|
||||
version = "12.0.0-beta.15"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
|
||||
version = "12.0.0-beta.11"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"rand 0.9.5",
|
||||
@@ -4886,8 +4886,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"
|
||||
|
||||
[[package]]
|
||||
name = "lance"
|
||||
version = "12.0.0-beta.15"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
|
||||
version = "12.0.0-beta.11"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"arrow",
|
||||
@@ -4959,8 +4959,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-arrow"
|
||||
version = "12.0.0-beta.15"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
|
||||
version = "12.0.0-beta.11"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -4982,7 +4982,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "lance-arrow-scalar"
|
||||
version = "58.0.0"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -4996,7 +4996,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "lance-arrow-stats"
|
||||
version = "58.0.0"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-schema",
|
||||
@@ -5005,8 +5005,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-bitpacking"
|
||||
version = "12.0.0-beta.15"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
|
||||
version = "12.0.0-beta.11"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
|
||||
dependencies = [
|
||||
"arrayref",
|
||||
"crunchy",
|
||||
@@ -5016,8 +5016,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-core"
|
||||
version = "12.0.0-beta.15"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
|
||||
version = "12.0.0-beta.11"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -5054,8 +5054,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-datafusion"
|
||||
version = "12.0.0-beta.15"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
|
||||
version = "12.0.0-beta.11"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -5085,8 +5085,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-datagen"
|
||||
version = "12.0.0-beta.15"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
|
||||
version = "12.0.0-beta.11"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -5103,8 +5103,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-derive"
|
||||
version = "12.0.0-beta.15"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
|
||||
version = "12.0.0-beta.11"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -5113,8 +5113,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-encoding"
|
||||
version = "12.0.0-beta.15"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
|
||||
version = "12.0.0-beta.11"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
|
||||
dependencies = [
|
||||
"arrow-arith",
|
||||
"arrow-array",
|
||||
@@ -5147,8 +5147,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-file"
|
||||
version = "12.0.0-beta.15"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
|
||||
version = "12.0.0-beta.11"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
|
||||
dependencies = [
|
||||
"arrow-arith",
|
||||
"arrow-array",
|
||||
@@ -5179,8 +5179,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-index"
|
||||
version = "12.0.0-beta.15"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
|
||||
version = "12.0.0-beta.11"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"arrow",
|
||||
@@ -5244,8 +5244,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-index-core"
|
||||
version = "12.0.0-beta.15"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
|
||||
version = "12.0.0-beta.11"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-schema",
|
||||
@@ -5267,8 +5267,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-io"
|
||||
version = "12.0.0-beta.15"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
|
||||
version = "12.0.0-beta.11"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -5308,8 +5308,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-linalg"
|
||||
version = "12.0.0-beta.15"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
|
||||
version = "12.0.0-beta.11"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-schema",
|
||||
@@ -5323,23 +5323,21 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-namespace"
|
||||
version = "12.0.0-beta.15"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
|
||||
version = "12.0.0-beta.11"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"async-trait",
|
||||
"bytes",
|
||||
"lance-core",
|
||||
"lance-namespace-reqwest-client",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"snafu 0.9.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lance-namespace-impls"
|
||||
version = "12.0.0-beta.15"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
|
||||
version = "12.0.0-beta.11"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-ipc",
|
||||
@@ -5378,9 +5376,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-namespace-reqwest-client"
|
||||
version = "0.12.0"
|
||||
version = "0.11.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d8d23e54b1634d5bbb434f8dd33dc3c05f6e58d876a9a27b3b4aef58ddbe11af"
|
||||
checksum = "1d06b1fbb5d41f93bc652b61e2872af92e8a6c5f6b4ce8839a8ecfa05365d359"
|
||||
dependencies = [
|
||||
"reqwest 0.12.28",
|
||||
"serde",
|
||||
@@ -5392,8 +5390,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-select"
|
||||
version = "12.0.0-beta.15"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
|
||||
version = "12.0.0-beta.11"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -5407,8 +5405,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-table"
|
||||
version = "12.0.0-beta.15"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
|
||||
version = "12.0.0-beta.11"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -5448,8 +5446,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-testing"
|
||||
version = "12.0.0-beta.15"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
|
||||
version = "12.0.0-beta.11"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-schema",
|
||||
@@ -5462,8 +5460,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-tokenizer"
|
||||
version = "12.0.0-beta.15"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
|
||||
version = "12.0.0-beta.11"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
|
||||
dependencies = [
|
||||
"frostem",
|
||||
"icu_segmenter",
|
||||
@@ -5476,7 +5474,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lancedb"
|
||||
version = "0.39.0-beta.5"
|
||||
version = "0.39.0-beta.1"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"anyhow",
|
||||
@@ -5567,7 +5565,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lancedb-nodejs"
|
||||
version = "0.39.0-beta.5"
|
||||
version = "0.39.0-beta.1"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -5592,7 +5590,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lancedb-python"
|
||||
version = "0.39.0-beta.5"
|
||||
version = "0.39.0-beta.1"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"async-trait",
|
||||
@@ -7703,9 +7701,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "prost"
|
||||
version = "0.14.3"
|
||||
version = "0.14.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d2ea70524a2f82d518bce41317d0fae74151505651af45faf1ffbd6fd33f0568"
|
||||
checksum = "528ac67416ff8646872a3c02cad9cc4ee5dc9f9540c9b10771855c95cb2e5ae1"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"prost-derive",
|
||||
@@ -7732,9 +7730,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "prost-derive"
|
||||
version = "0.14.3"
|
||||
version = "0.14.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b"
|
||||
checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"itertools 0.14.0",
|
||||
|
||||
+14
-14
@@ -13,20 +13,20 @@ categories = ["database-implementations"]
|
||||
rust-version = "1.91.0"
|
||||
|
||||
[workspace.dependencies]
|
||||
lance = { "version" = "=12.0.0-beta.15", default-features = false, "tag" = "v12.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-core = { "version" = "=12.0.0-beta.15", "tag" = "v12.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datagen = { "version" = "=12.0.0-beta.15", "tag" = "v12.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-file = { "version" = "=12.0.0-beta.15", "tag" = "v12.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-io = { "version" = "=12.0.0-beta.15", default-features = false, "tag" = "v12.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-index = { "version" = "=12.0.0-beta.15", "tag" = "v12.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-linalg = { "version" = "=12.0.0-beta.15", "tag" = "v12.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace = { "version" = "=12.0.0-beta.15", "tag" = "v12.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace-impls = { "version" = "=12.0.0-beta.15", default-features = false, "tag" = "v12.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-table = { "version" = "=12.0.0-beta.15", "tag" = "v12.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-testing = { "version" = "=12.0.0-beta.15", "tag" = "v12.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datafusion = { "version" = "=12.0.0-beta.15", "tag" = "v12.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-encoding = { "version" = "=12.0.0-beta.15", "tag" = "v12.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-arrow = { "version" = "=12.0.0-beta.15", "tag" = "v12.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance = { "version" = "=12.0.0-beta.11", default-features = false, "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-core = { "version" = "=12.0.0-beta.11", "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datagen = { "version" = "=12.0.0-beta.11", "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-file = { "version" = "=12.0.0-beta.11", "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-io = { "version" = "=12.0.0-beta.11", default-features = false, "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-index = { "version" = "=12.0.0-beta.11", "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-linalg = { "version" = "=12.0.0-beta.11", "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace = { "version" = "=12.0.0-beta.11", "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace-impls = { "version" = "=12.0.0-beta.11", default-features = false, "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-table = { "version" = "=12.0.0-beta.11", "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-testing = { "version" = "=12.0.0-beta.11", "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datafusion = { "version" = "=12.0.0-beta.11", "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-encoding = { "version" = "=12.0.0-beta.11", "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-arrow = { "version" = "=12.0.0-beta.11", "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lancedb = { path = "rust/lancedb", default-features = false }
|
||||
ahash = "0.8"
|
||||
# Note that this one does not include pyarrow
|
||||
|
||||
@@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`:
|
||||
<dependency>
|
||||
<groupId>com.lancedb</groupId>
|
||||
<artifactId>lancedb-core</artifactId>
|
||||
<version>0.39.0-beta.6</version>
|
||||
<version>0.39.0-beta.1</version>
|
||||
</dependency>
|
||||
```
|
||||
|
||||
|
||||
@@ -448,6 +448,26 @@ on the returned job to know when cleanup has finished.
|
||||
|
||||
***
|
||||
|
||||
### getJob()
|
||||
|
||||
```ts
|
||||
abstract getJob(jobId): Promise<null | JobDescription>
|
||||
```
|
||||
|
||||
Describe a single server-side job by id.
|
||||
|
||||
Resolves to `null` when the server has no such job.
|
||||
|
||||
#### Parameters
|
||||
|
||||
* **jobId**: `string`
|
||||
|
||||
#### Returns
|
||||
|
||||
`Promise`<`null` \| [`JobDescription`](../interfaces/JobDescription.md)>
|
||||
|
||||
***
|
||||
|
||||
### isOpen()
|
||||
|
||||
```ts
|
||||
@@ -462,6 +482,48 @@ Return true if the connection has not been closed
|
||||
|
||||
***
|
||||
|
||||
### job()
|
||||
|
||||
```ts
|
||||
abstract job(jobId): Job
|
||||
```
|
||||
|
||||
A [Job](Job.md) handle for a server-side job by id.
|
||||
|
||||
The handle is constructed without a server round trip; an unknown id
|
||||
surfaces when the handle is used. Dropping the handle has no effect on
|
||||
the job itself.
|
||||
|
||||
#### Parameters
|
||||
|
||||
* **jobId**: `string`
|
||||
|
||||
#### Returns
|
||||
|
||||
[`Job`](Job.md)
|
||||
|
||||
***
|
||||
|
||||
### jobHistory()
|
||||
|
||||
```ts
|
||||
abstract jobHistory(jobId?): Promise<Table<any>>
|
||||
```
|
||||
|
||||
The lifecycle event history of a server-side job, as an Arrow table.
|
||||
|
||||
Lists history across all jobs when `jobId` is omitted.
|
||||
|
||||
#### Parameters
|
||||
|
||||
* **jobId?**: `string`
|
||||
|
||||
#### Returns
|
||||
|
||||
`Promise`<`Table`<`any`>>
|
||||
|
||||
***
|
||||
|
||||
### listJobs()
|
||||
|
||||
```ts
|
||||
@@ -586,30 +648,6 @@ A page of table names and an
|
||||
|
||||
***
|
||||
|
||||
### openJob()
|
||||
|
||||
```ts
|
||||
abstract openJob(jobId): Promise<Job>
|
||||
```
|
||||
|
||||
Open a server-side job by id, returning a handle with its record already
|
||||
populated. Rejects when the server has no such job, the way
|
||||
[Connection.openTable](Connection.md#opentable) does for a missing table.
|
||||
|
||||
The returned [Job](Job.md) answers for its own state, specification,
|
||||
result, failure and event history, so there is no separate
|
||||
connection-level call for any of them.
|
||||
|
||||
#### Parameters
|
||||
|
||||
* **jobId**: `string`
|
||||
|
||||
#### Returns
|
||||
|
||||
`Promise`<[`Job`](Job.md)>
|
||||
|
||||
***
|
||||
|
||||
### openMaterializedView()
|
||||
|
||||
```ts
|
||||
|
||||
+16
-163
@@ -8,116 +8,28 @@
|
||||
|
||||
A handle to an operation that may still be running.
|
||||
|
||||
The operation may already be complete when the handle is created.
|
||||
## Constructors
|
||||
|
||||
The detail getters read what the handle last observed. Submitting an
|
||||
operation returns only a job id, so populating them eagerly would cost an
|
||||
extra round trip on every call:
|
||||
### new Job()
|
||||
|
||||
- [Job.refresh](Job.md#refresh) and [Job.status](Job.md#status) fetch the whole record.
|
||||
- [Job.wait](Job.md#wait) records the terminal state it establishes, but not the
|
||||
rest of the record.
|
||||
- Everything is null until one of those runs.
|
||||
```ts
|
||||
new Job(): Job
|
||||
```
|
||||
|
||||
#### Returns
|
||||
|
||||
[`Job`](Job.md)
|
||||
|
||||
## Accessors
|
||||
|
||||
### creationMs
|
||||
|
||||
```ts
|
||||
get creationMs(): null | number
|
||||
```
|
||||
|
||||
When the job was created, in milliseconds since the epoch.
|
||||
|
||||
#### Returns
|
||||
|
||||
`null` \| `number`
|
||||
|
||||
***
|
||||
|
||||
### failure
|
||||
|
||||
```ts
|
||||
get failure(): null | JobFailureInfo
|
||||
```
|
||||
|
||||
Why the job failed, when it failed and the server reports a reason.
|
||||
|
||||
#### Returns
|
||||
|
||||
`null` \| [`JobFailureInfo`](../interfaces/JobFailureInfo.md)
|
||||
|
||||
***
|
||||
|
||||
### id
|
||||
|
||||
```ts
|
||||
get id(): null | string
|
||||
```
|
||||
|
||||
Identifies the operation on the server that is running it.
|
||||
|
||||
Operations that run in this process have no server id. The value is
|
||||
opaque: parsing it or storing it to resume the job later is not supported.
|
||||
|
||||
#### Returns
|
||||
|
||||
`null` \| `string`
|
||||
|
||||
***
|
||||
|
||||
### jobType
|
||||
|
||||
```ts
|
||||
get jobType(): null | string
|
||||
```
|
||||
|
||||
The job's type, as the server names it. Null for an in-process job, which
|
||||
has no server-side record.
|
||||
|
||||
#### Returns
|
||||
|
||||
`null` \| `string`
|
||||
|
||||
***
|
||||
|
||||
### result
|
||||
|
||||
```ts
|
||||
get result(): any
|
||||
```
|
||||
|
||||
The job-type-specific terminal result. Null until the job succeeds, so a
|
||||
job that never terminates reports its progress through [Job.events](Job.md#events)
|
||||
instead.
|
||||
|
||||
#### Returns
|
||||
|
||||
`any`
|
||||
|
||||
***
|
||||
|
||||
### spec
|
||||
|
||||
```ts
|
||||
get spec(): any
|
||||
```
|
||||
|
||||
The job-type-specific specification it was submitted with.
|
||||
|
||||
#### Returns
|
||||
|
||||
`any`
|
||||
|
||||
***
|
||||
|
||||
### state
|
||||
|
||||
```ts
|
||||
get state(): null | string
|
||||
```
|
||||
|
||||
The last observed lifecycle state, without contacting the backend.
|
||||
Identifies the operation on the server that is running it. Operations
|
||||
that run in this process have no server id. The value is opaque.
|
||||
|
||||
#### Returns
|
||||
|
||||
@@ -139,61 +51,18 @@ Request cancellation. Cancelling a finished operation is a no-op.
|
||||
|
||||
***
|
||||
|
||||
### events()
|
||||
|
||||
```ts
|
||||
events(options?): Promise<Table<any>>
|
||||
```
|
||||
|
||||
This job's recorded lifecycle events.
|
||||
|
||||
Where the getters above report a terminal result only once the job reaches
|
||||
one, events are written as the job runs and outlive the workers that
|
||||
produced them. A distributed job records a `claim`/`claim_complete` pair
|
||||
per unit of work, each carrying `rows_processed`, so a job that never
|
||||
finishes still accounts for what it did.
|
||||
|
||||
The server caps results at 1000 rows by default and 10,000 at most, and
|
||||
truncates without saying so, so pass `limit` for a job that emits an event
|
||||
per fragment. `filter` is a SQL-like expression over the `state`,
|
||||
`updated_by`, `emitted_from`, `emitted_by`, and `claim_entity` columns.
|
||||
|
||||
#### Parameters
|
||||
|
||||
* **options?**: [`JobEventsOptions`](../interfaces/JobEventsOptions.md)
|
||||
|
||||
#### Returns
|
||||
|
||||
`Promise`<`Table`<`any`>>
|
||||
|
||||
***
|
||||
|
||||
### refresh()
|
||||
|
||||
```ts
|
||||
refresh(): Promise<void>
|
||||
```
|
||||
|
||||
Ask the backend for this job's current state, and for a server-side job
|
||||
its full record, then cache it for the getters above.
|
||||
|
||||
#### Returns
|
||||
|
||||
`Promise`<`void`>
|
||||
|
||||
***
|
||||
|
||||
### status()
|
||||
|
||||
```ts
|
||||
status(): Promise<string>
|
||||
```
|
||||
|
||||
The operation's current lifecycle state: "running", "finished", "failed",
|
||||
or "cancelled".
|
||||
The operation's current lifecycle state: "running", "finished",
|
||||
"failed", or "cancelled".
|
||||
|
||||
A point snapshot; unlike [Job.wait](Job.md#wait) it does not block or reject on a
|
||||
terminal failure state. Also refreshes the getters above.
|
||||
A point snapshot; unlike [Job.wait](Job.md#wait) it does not block or reject
|
||||
on a terminal failure state. States a newer server reports that this
|
||||
client version does not know pass through as-is.
|
||||
|
||||
#### Returns
|
||||
|
||||
@@ -201,22 +70,6 @@ terminal failure state. Also refreshes the getters above.
|
||||
|
||||
***
|
||||
|
||||
### toString()
|
||||
|
||||
```ts
|
||||
toString(): string
|
||||
```
|
||||
|
||||
Every field the handle currently knows, one per line, with the JSON
|
||||
payloads indented -- a refresh job's spec and result are the point of
|
||||
printing it.
|
||||
|
||||
#### Returns
|
||||
|
||||
`string`
|
||||
|
||||
***
|
||||
|
||||
### wait()
|
||||
|
||||
```ts
|
||||
|
||||
@@ -96,7 +96,7 @@
|
||||
- [IvfFlatOptions](interfaces/IvfFlatOptions.md)
|
||||
- [IvfPqOptions](interfaces/IvfPqOptions.md)
|
||||
- [IvfRqOptions](interfaces/IvfRqOptions.md)
|
||||
- [JobEventsOptions](interfaces/JobEventsOptions.md)
|
||||
- [JobDescription](interfaces/JobDescription.md)
|
||||
- [JobFailureInfo](interfaces/JobFailureInfo.md)
|
||||
- [JobInfo](interfaces/JobInfo.md)
|
||||
- [ListNamespacesOptions](interfaces/ListNamespacesOptions.md)
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
[**@lancedb/lancedb**](../README.md) • **Docs**
|
||||
|
||||
***
|
||||
|
||||
[@lancedb/lancedb](../globals.md) / JobDescription
|
||||
|
||||
# Interface: JobDescription
|
||||
|
||||
A described job from `Connection.getJob`.
|
||||
|
||||
## Properties
|
||||
|
||||
### creationMs
|
||||
|
||||
```ts
|
||||
creationMs: number;
|
||||
```
|
||||
|
||||
When the job was created, in milliseconds since the epoch.
|
||||
|
||||
***
|
||||
|
||||
### failure?
|
||||
|
||||
```ts
|
||||
optional failure: JobFailureInfo;
|
||||
```
|
||||
|
||||
Why the job failed, when the job is failed and the server reports a
|
||||
reason.
|
||||
|
||||
***
|
||||
|
||||
### jobId
|
||||
|
||||
```ts
|
||||
jobId: string;
|
||||
```
|
||||
|
||||
***
|
||||
|
||||
### jobType
|
||||
|
||||
```ts
|
||||
jobType: string;
|
||||
```
|
||||
|
||||
***
|
||||
|
||||
### specJson?
|
||||
|
||||
```ts
|
||||
optional specJson: string;
|
||||
```
|
||||
|
||||
The job-type-specific specification as a JSON string, when present.
|
||||
|
||||
***
|
||||
|
||||
### state
|
||||
|
||||
```ts
|
||||
state: string;
|
||||
```
|
||||
|
||||
Lifecycle state: "running", "finished", "failed", or "cancelled".
|
||||
@@ -1,29 +0,0 @@
|
||||
[**@lancedb/lancedb**](../README.md) • **Docs**
|
||||
|
||||
***
|
||||
|
||||
[@lancedb/lancedb](../globals.md) / JobEventsOptions
|
||||
|
||||
# Interface: JobEventsOptions
|
||||
|
||||
Which of a job's events [Job.events](../classes/Job.md#events) returns.
|
||||
|
||||
## Properties
|
||||
|
||||
### filter?
|
||||
|
||||
```ts
|
||||
optional filter: string;
|
||||
```
|
||||
|
||||
SQL-like filter over the event columns.
|
||||
|
||||
***
|
||||
|
||||
### limit?
|
||||
|
||||
```ts
|
||||
optional limit: number;
|
||||
```
|
||||
|
||||
Maximum event rows to return, up to the server maximum of 10,000.
|
||||
@@ -26,7 +26,7 @@ When the job was created, in milliseconds since the epoch.
|
||||
jobId: string;
|
||||
```
|
||||
|
||||
The job id -- what `Connection.openJob` and `Connection.cancelJob`
|
||||
The job id -- what `Connection.getJob` and `Connection.cancelJob`
|
||||
accept.
|
||||
|
||||
***
|
||||
|
||||
@@ -157,12 +157,6 @@ listing a storage directory.
|
||||
|
||||
::: lancedb.job.AsyncJob
|
||||
|
||||
::: lancedb.job.JobInfo
|
||||
|
||||
::: lancedb.job.JobDescription
|
||||
|
||||
::: lancedb.job.JobFailureInfo
|
||||
|
||||
::: lancedb.sql.Query
|
||||
|
||||
::: lancedb.sql.AsyncQuery
|
||||
@@ -316,12 +310,6 @@ still work. Queries return descriptors. Call
|
||||
|
||||
::: lancedb.exceptions.MissingColumnError
|
||||
|
||||
::: lancedb.exceptions.JobNotFoundError
|
||||
|
||||
::: lancedb.exceptions.JobFailedError
|
||||
|
||||
::: lancedb.exceptions.JobCancelledError
|
||||
|
||||
## Integrations
|
||||
|
||||
## Pydantic
|
||||
|
||||
@@ -8,7 +8,7 @@
|
||||
<parent>
|
||||
<groupId>com.lancedb</groupId>
|
||||
<artifactId>lancedb-parent</artifactId>
|
||||
<version>0.39.0-beta.6</version>
|
||||
<version>0.39.0-beta.1</version>
|
||||
<relativePath>../pom.xml</relativePath>
|
||||
</parent>
|
||||
|
||||
|
||||
+2
-2
@@ -6,7 +6,7 @@
|
||||
|
||||
<groupId>com.lancedb</groupId>
|
||||
<artifactId>lancedb-parent</artifactId>
|
||||
<version>0.39.0-beta.6</version>
|
||||
<version>0.39.0-beta.1</version>
|
||||
<packaging>pom</packaging>
|
||||
<name>${project.artifactId}</name>
|
||||
<description>LanceDB Java SDK Parent POM</description>
|
||||
@@ -28,7 +28,7 @@
|
||||
<properties>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<arrow.version>15.0.0</arrow.version>
|
||||
<lance-core.version>12.0.0-beta.15</lance-core.version>
|
||||
<lance-core.version>12.0.0-beta.11</lance-core.version>
|
||||
<spotless.skip>false</spotless.skip>
|
||||
<spotless.version>2.30.0</spotless.version>
|
||||
<spotless.java.googlejavaformat.version>1.7</spotless.java.googlejavaformat.version>
|
||||
|
||||
+1
-1
@@ -1,7 +1,7 @@
|
||||
[package]
|
||||
name = "lancedb-nodejs"
|
||||
edition.workspace = true
|
||||
version = "0.39.0-beta.6"
|
||||
version = "0.39.0-beta.1"
|
||||
publish = false
|
||||
license.workspace = true
|
||||
description.workspace = true
|
||||
|
||||
@@ -939,7 +939,6 @@ describe("remote connection jobs surface", () => {
|
||||
const { tableFromArrays, tableToIPC } = await import("apache-arrow");
|
||||
const eventsTable = tableFromArrays({ state: ["created", "succeeded"] });
|
||||
const eventsBody = Buffer.from(tableToIPC(eventsTable, "stream"));
|
||||
const queryEventsPayloads: Record<string, unknown>[] = [];
|
||||
|
||||
await withMockDatabase(
|
||||
(req, res) => {
|
||||
@@ -968,16 +967,6 @@ describe("remote connection jobs surface", () => {
|
||||
);
|
||||
}
|
||||
} else if (req.url === "/v1/jobs/describe") {
|
||||
if (payload["job_id"] === "job-2") {
|
||||
res
|
||||
.writeHead(200, { "Content-Type": "application/json" })
|
||||
.end(
|
||||
'{"job_id": "job-2", "job_type": "refresh_column", ' +
|
||||
'"job_state": "DONE", "creation_ms": 2000, ' +
|
||||
'"result": {"rows_assigned": 1000000}}',
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (payload["job_id"] !== "job-1") {
|
||||
res.writeHead(404).end("no such job");
|
||||
return;
|
||||
@@ -999,7 +988,6 @@ describe("remote connection jobs surface", () => {
|
||||
.writeHead(200, { "Content-Type": "application/json" })
|
||||
.end('{"job_id": "job-1"}');
|
||||
} else if (req.url === "/v1/jobs/query_events") {
|
||||
queryEventsPayloads.push(payload);
|
||||
res
|
||||
.writeHead(200, {
|
||||
"Content-Type": "application/vnd.apache.arrow.stream",
|
||||
@@ -1016,65 +1004,22 @@ describe("remote connection jobs surface", () => {
|
||||
expect(jobs[0].state).toEqual("running");
|
||||
expect(jobs[1].state).toEqual("finished");
|
||||
|
||||
const description = await db.getJob("job-1");
|
||||
expect(description?.state).toEqual("failed");
|
||||
expect(JSON.parse(description?.specJson ?? "")).toEqual({
|
||||
column: "vec",
|
||||
});
|
||||
expect(description?.failure?.message).toEqual("worker died");
|
||||
expect(await db.getJob("missing")).toBeNull();
|
||||
|
||||
expect(await db.cancelJob("job-1")).toBe(true);
|
||||
expect(await db.cancelJob("missing")).toBe(false);
|
||||
|
||||
// Opening a job hands back a populated handle; a missing one rejects.
|
||||
await expect(db.openJob("missing")).rejects.toThrow("not found");
|
||||
const finished = await db.openJob("job-2");
|
||||
expect(finished.state).toEqual("finished");
|
||||
expect(finished.result).toEqual({
|
||||
// biome-ignore lint/style/useNamingConvention: snake_case mandated by the server wire format
|
||||
rows_assigned: 1000000,
|
||||
});
|
||||
const history = await db.jobHistory("job-1");
|
||||
expect(history.numRows).toEqual(2);
|
||||
|
||||
const job = await db.openJob("job-1");
|
||||
const job = db.job("job-1");
|
||||
expect(job.id).toEqual("job-1");
|
||||
|
||||
// openJob already populated the handle; refresh() re-reads it.
|
||||
expect(job.state).toEqual("failed");
|
||||
await job.refresh();
|
||||
expect(job.state).toEqual("failed");
|
||||
expect(job.jobType).toEqual("create_index");
|
||||
expect(job.creationMs).toEqual(1000);
|
||||
expect(job.spec).toEqual({ column: "vec" });
|
||||
expect(job.result).toBeNull();
|
||||
expect(job.failure?.message).toEqual("worker died");
|
||||
|
||||
// The handle reaches its own events, supplying its job id.
|
||||
const jobEvents = await job.events({
|
||||
limit: 500,
|
||||
filter: "state = 'claim_complete'",
|
||||
});
|
||||
expect(jobEvents.numRows).toEqual(2);
|
||||
expect(queryEventsPayloads.pop()).toEqual({
|
||||
// biome-ignore lint/style/useNamingConvention: snake_case mandated by the server wire format
|
||||
job_id: "job-1",
|
||||
limit: 500,
|
||||
filter: "state = 'claim_complete'",
|
||||
});
|
||||
|
||||
// Printing lays every known field out on its own line, with the JSON
|
||||
// payloads indented rather than crammed onto one line.
|
||||
expect(`${job}`).toEqual(
|
||||
[
|
||||
"Job(",
|
||||
' id="job-1",',
|
||||
' state="failed",',
|
||||
' jobType="create_index",',
|
||||
" creationMs=1000,",
|
||||
" spec={",
|
||||
' "column": "vec"',
|
||||
" },",
|
||||
" failure={",
|
||||
' "phase": "execute",',
|
||||
' "message": "worker died",',
|
||||
' "retryable": true',
|
||||
" },",
|
||||
")",
|
||||
].join("\n"),
|
||||
);
|
||||
|
||||
expect(await job.status()).toEqual("failed");
|
||||
await expect(job.wait()).rejects.toThrow("worker died");
|
||||
},
|
||||
|
||||
@@ -281,7 +281,7 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
|
||||
numIndices: 0,
|
||||
numRows: 3,
|
||||
// Full on-disk size of the two data files, footers and metadata included.
|
||||
totalBytes: 550,
|
||||
totalBytes: 684,
|
||||
});
|
||||
|
||||
// Index files count toward totalBytes too (only deletion files and
|
||||
@@ -289,7 +289,7 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
|
||||
await table.createIndex("id", { config: Index.btree() });
|
||||
const statsWithIndex = await table.stats();
|
||||
expect(statsWithIndex.numIndices).toBe(1);
|
||||
expect(statsWithIndex.totalBytes).toBeGreaterThan(550);
|
||||
expect(statsWithIndex.totalBytes).toBeGreaterThan(684);
|
||||
});
|
||||
|
||||
it("should overwrite data if asked", async () => {
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
import { tableFromIPC } from "apache-arrow";
|
||||
import {
|
||||
Data,
|
||||
SchemaLike,
|
||||
@@ -15,7 +16,6 @@ import {
|
||||
makeEmptyTable,
|
||||
} from "./arrow";
|
||||
import { EmbeddingFunctionConfig, getRegistry } from "./embedding/registry";
|
||||
import { Job } from "./job";
|
||||
import {
|
||||
MaterializedView,
|
||||
MaterializedViewSelect,
|
||||
@@ -27,6 +27,8 @@ import type {
|
||||
CreateNamespaceResponse,
|
||||
DescribeNamespaceResponse,
|
||||
DropNamespaceResponse,
|
||||
Job,
|
||||
JobDescription,
|
||||
JobInfo,
|
||||
ListNamespacesResponse,
|
||||
ListTablesResponse,
|
||||
@@ -555,19 +557,24 @@ export abstract class Connection {
|
||||
): Promise<void>;
|
||||
|
||||
/**
|
||||
* Open a server-side job by id, returning a handle with its record already
|
||||
* populated. Rejects when the server has no such job, the way
|
||||
* {@link Connection.openTable} does for a missing table.
|
||||
* A {@link Job} handle for a server-side job by id.
|
||||
*
|
||||
* The returned {@link Job} answers for its own state, specification,
|
||||
* result, failure and event history, so there is no separate
|
||||
* connection-level call for any of them.
|
||||
* The handle is constructed without a server round trip; an unknown id
|
||||
* surfaces when the handle is used. Dropping the handle has no effect on
|
||||
* the job itself.
|
||||
*/
|
||||
abstract openJob(jobId: string): Promise<Job>;
|
||||
abstract job(jobId: string): Job;
|
||||
|
||||
/** List server-side jobs across the database's tables. */
|
||||
abstract listJobs(): Promise<JobInfo[]>;
|
||||
|
||||
/**
|
||||
* Describe a single server-side job by id.
|
||||
*
|
||||
* Resolves to `null` when the server has no such job.
|
||||
*/
|
||||
abstract getJob(jobId: string): Promise<JobDescription | null>;
|
||||
|
||||
/**
|
||||
* Request cancellation of a server-side job by id.
|
||||
*
|
||||
@@ -575,6 +582,13 @@ export abstract class Connection {
|
||||
* such job exists. Cancelling an already-terminal job is a no-op success.
|
||||
*/
|
||||
abstract cancelJob(jobId: string): Promise<boolean>;
|
||||
|
||||
/**
|
||||
* The lifecycle event history of a server-side job, as an Arrow table.
|
||||
*
|
||||
* Lists history across all jobs when `jobId` is omitted.
|
||||
*/
|
||||
abstract jobHistory(jobId?: string): Promise<ArrowTable>;
|
||||
}
|
||||
|
||||
/** @hideconstructor */
|
||||
@@ -855,7 +869,7 @@ export class LocalConnection extends Connection {
|
||||
}
|
||||
|
||||
async dropTableAsync(name: string, namespacePath?: string[]): Promise<Job> {
|
||||
return new Job(await this.inner.dropTableAsync(name, namespacePath ?? []));
|
||||
return this.inner.dropTableAsync(name, namespacePath ?? []);
|
||||
}
|
||||
|
||||
async dropAllTables(namespacePath?: string[]): Promise<void> {
|
||||
@@ -914,17 +928,29 @@ export class LocalConnection extends Connection {
|
||||
);
|
||||
}
|
||||
|
||||
async openJob(jobId: string): Promise<Job> {
|
||||
return new Job(await this.inner.openJob(jobId));
|
||||
job(jobId: string): Job {
|
||||
return this.inner.job(jobId);
|
||||
}
|
||||
|
||||
async listJobs(): Promise<JobInfo[]> {
|
||||
return this.inner.listJobs();
|
||||
}
|
||||
|
||||
async getJob(jobId: string): Promise<JobDescription | null> {
|
||||
return this.inner.getJob(jobId);
|
||||
}
|
||||
|
||||
async cancelJob(jobId: string): Promise<boolean> {
|
||||
return this.inner.cancelJob(jobId);
|
||||
}
|
||||
|
||||
async jobHistory(jobId?: string): Promise<ArrowTable> {
|
||||
const buf = await this.inner.jobHistory(jobId);
|
||||
if (buf.length === 0) {
|
||||
return new ArrowTable();
|
||||
}
|
||||
return tableFromIPC(buf);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -94,9 +94,13 @@ export {
|
||||
RenameTableOptions,
|
||||
} from "./connection";
|
||||
|
||||
export { JobFailureInfo, JobInfo, Session } from "./native.js";
|
||||
|
||||
export { Job, JobEventsOptions } from "./job";
|
||||
export {
|
||||
Job,
|
||||
JobDescription,
|
||||
JobFailureInfo,
|
||||
JobInfo,
|
||||
Session,
|
||||
} from "./native.js";
|
||||
|
||||
export {
|
||||
AutoQuery,
|
||||
|
||||
@@ -1,188 +0,0 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
import { Table as ArrowTable, tableFromIPC } from "apache-arrow";
|
||||
import { JobFailureInfo, Job as NativeJob } from "./native";
|
||||
|
||||
/** Which of a job's events {@link Job.events} returns. */
|
||||
export interface JobEventsOptions {
|
||||
/** Maximum event rows to return, up to the server maximum of 10,000. */
|
||||
limit?: number;
|
||||
/** SQL-like filter over the event columns. */
|
||||
filter?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* A handle to an operation that may still be running.
|
||||
*
|
||||
* The operation may already be complete when the handle is created.
|
||||
*
|
||||
* The detail getters read what the handle last observed. Submitting an
|
||||
* operation returns only a job id, so populating them eagerly would cost an
|
||||
* extra round trip on every call:
|
||||
*
|
||||
* - {@link Job.refresh} and {@link Job.status} fetch the whole record.
|
||||
* - {@link Job.wait} records the terminal state it establishes, but not the
|
||||
* rest of the record.
|
||||
* - Everything is null until one of those runs.
|
||||
*
|
||||
* @hideconstructor
|
||||
*/
|
||||
export class Job {
|
||||
private readonly inner: NativeJob;
|
||||
|
||||
constructor(inner: NativeJob) {
|
||||
this.inner = inner;
|
||||
}
|
||||
|
||||
/**
|
||||
* Identifies the operation on the server that is running it.
|
||||
*
|
||||
* Operations that run in this process have no server id. The value is
|
||||
* opaque: parsing it or storing it to resume the job later is not supported.
|
||||
*/
|
||||
get id(): string | null {
|
||||
return this.inner.id ?? null;
|
||||
}
|
||||
|
||||
/** The last observed lifecycle state, without contacting the backend. */
|
||||
get state(): string | null {
|
||||
return this.inner.state ?? null;
|
||||
}
|
||||
|
||||
/**
|
||||
* The job's type, as the server names it. Null for an in-process job, which
|
||||
* has no server-side record.
|
||||
*/
|
||||
get jobType(): string | null {
|
||||
return this.inner.jobType ?? null;
|
||||
}
|
||||
|
||||
/** When the job was created, in milliseconds since the epoch. */
|
||||
get creationMs(): number | null {
|
||||
return this.inner.creationMs ?? null;
|
||||
}
|
||||
|
||||
/** The job-type-specific specification it was submitted with. */
|
||||
// biome-ignore lint/suspicious/noExplicitAny: shape varies by job type
|
||||
get spec(): any | null {
|
||||
return parseJson(this.inner.specJson);
|
||||
}
|
||||
|
||||
/**
|
||||
* The job-type-specific terminal result. Null until the job succeeds, so a
|
||||
* job that never terminates reports its progress through {@link Job.events}
|
||||
* instead.
|
||||
*/
|
||||
// biome-ignore lint/suspicious/noExplicitAny: shape varies by job type
|
||||
get result(): any | null {
|
||||
return parseJson(this.inner.resultJson);
|
||||
}
|
||||
|
||||
/** Why the job failed, when it failed and the server reports a reason. */
|
||||
get failure(): JobFailureInfo | null {
|
||||
return this.inner.failure ?? null;
|
||||
}
|
||||
|
||||
/**
|
||||
* The operation's current lifecycle state: "running", "finished", "failed",
|
||||
* or "cancelled".
|
||||
*
|
||||
* A point snapshot; unlike {@link Job.wait} it does not block or reject on a
|
||||
* terminal failure state. Also refreshes the getters above.
|
||||
*/
|
||||
async status(): Promise<string> {
|
||||
return this.inner.status();
|
||||
}
|
||||
|
||||
/** Wait until the operation reaches a terminal state. */
|
||||
async wait(): Promise<void> {
|
||||
return this.inner.wait();
|
||||
}
|
||||
|
||||
/** Request cancellation. Cancelling a finished operation is a no-op. */
|
||||
async cancel(): Promise<void> {
|
||||
return this.inner.cancel();
|
||||
}
|
||||
|
||||
/**
|
||||
* Ask the backend for this job's current state, and for a server-side job
|
||||
* its full record, then cache it for the getters above.
|
||||
*/
|
||||
async refresh(): Promise<void> {
|
||||
return this.inner.refresh();
|
||||
}
|
||||
|
||||
/**
|
||||
* This job's recorded lifecycle events.
|
||||
*
|
||||
* Where the getters above report a terminal result only once the job reaches
|
||||
* one, events are written as the job runs and outlive the workers that
|
||||
* produced them. A distributed job records a `claim`/`claim_complete` pair
|
||||
* per unit of work, each carrying `rows_processed`, so a job that never
|
||||
* finishes still accounts for what it did.
|
||||
*
|
||||
* The server caps results at 1000 rows by default and 10,000 at most, and
|
||||
* truncates without saying so, so pass `limit` for a job that emits an event
|
||||
* per fragment. `filter` is a SQL-like expression over the `state`,
|
||||
* `updated_by`, `emitted_from`, `emitted_by`, and `claim_entity` columns.
|
||||
*/
|
||||
async events(options?: JobEventsOptions): Promise<ArrowTable> {
|
||||
const buf = await this.inner.events(options?.limit, options?.filter);
|
||||
if (buf.length === 0) {
|
||||
return new ArrowTable();
|
||||
}
|
||||
return tableFromIPC(buf);
|
||||
}
|
||||
|
||||
/**
|
||||
* Every field the handle currently knows, one per line, with the JSON
|
||||
* payloads indented -- a refresh job's spec and result are the point of
|
||||
* printing it.
|
||||
*/
|
||||
toString(): string {
|
||||
if (this.state === null) {
|
||||
const known = this.id === null ? "" : `id=${JSON.stringify(this.id)}, `;
|
||||
return `Job(${known}not refreshed)`;
|
||||
}
|
||||
const fields: string[] = [];
|
||||
if (this.id !== null) {
|
||||
fields.push(`id=${JSON.stringify(this.id)}`);
|
||||
}
|
||||
fields.push(`state=${JSON.stringify(this.state)}`);
|
||||
if (this.jobType !== null) {
|
||||
fields.push(`jobType=${JSON.stringify(this.jobType)}`);
|
||||
}
|
||||
if (this.creationMs !== null) {
|
||||
fields.push(`creationMs=${this.creationMs}`);
|
||||
}
|
||||
for (const [name, value] of [
|
||||
["spec", this.spec],
|
||||
["result", this.result],
|
||||
] as const) {
|
||||
if (value !== null) {
|
||||
fields.push(`${name}=${indentJson(value)}`);
|
||||
}
|
||||
}
|
||||
if (this.failure !== null) {
|
||||
fields.push(`failure=${indentJson(this.failure)}`);
|
||||
}
|
||||
return `Job(${fields.map((field) => `\n${REPR_INDENT}${field},`).join("")}\n)`;
|
||||
}
|
||||
|
||||
[Symbol.for("nodejs.util.inspect.custom")](): string {
|
||||
return this.toString();
|
||||
}
|
||||
}
|
||||
|
||||
const REPR_INDENT = " ";
|
||||
|
||||
// biome-ignore lint/suspicious/noExplicitAny: shape varies by job type
|
||||
function indentJson(value: any): string {
|
||||
return JSON.stringify(value, null, 4).replace(/\n/g, `\n${REPR_INDENT}`);
|
||||
}
|
||||
|
||||
// biome-ignore lint/suspicious/noExplicitAny: shape varies by job type
|
||||
function parseJson(raw: string | null | undefined): any | null {
|
||||
return raw === null || raw === undefined ? null : JSON.parse(raw);
|
||||
}
|
||||
+9
-11
@@ -19,7 +19,6 @@ import {
|
||||
|
||||
import { EmbeddingFunctionConfig, getRegistry } from "./embedding/registry";
|
||||
import { IndexOptions } from "./indices";
|
||||
import { Job } from "./job";
|
||||
import { MergeInsertBuilder } from "./merge";
|
||||
import {
|
||||
AddColumnsResult,
|
||||
@@ -31,6 +30,7 @@ import {
|
||||
DropColumnsResult,
|
||||
IndexConfig,
|
||||
IndexStatistics,
|
||||
Job,
|
||||
LsmStats,
|
||||
Branches as NativeBranches,
|
||||
OptimizeStats,
|
||||
@@ -1124,15 +1124,13 @@ export class LocalTable extends Table {
|
||||
): Promise<Job> {
|
||||
// biome-ignore lint/suspicious/noExplicitAny: skip
|
||||
const nativeIndex = (options?.config as any)?.inner;
|
||||
return new Job(
|
||||
await this.inner.createIndexAsync(
|
||||
nativeIndex,
|
||||
column,
|
||||
options?.replace,
|
||||
options?.waitTimeoutSeconds,
|
||||
options?.name,
|
||||
options?.train,
|
||||
),
|
||||
return await this.inner.createIndexAsync(
|
||||
nativeIndex,
|
||||
column,
|
||||
options?.replace,
|
||||
options?.waitTimeoutSeconds,
|
||||
options?.name,
|
||||
options?.train,
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1315,7 +1313,7 @@ export class LocalTable extends Table {
|
||||
}
|
||||
|
||||
async refreshColumnAsync(column: string): Promise<Job> {
|
||||
return new Job(await this.inner.refreshColumnAsync(column));
|
||||
return await this.inner.refreshColumnAsync(column);
|
||||
}
|
||||
|
||||
async refreshMaterializedView(
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-darwin-arm64",
|
||||
"version": "0.39.0-beta.6",
|
||||
"version": "0.39.0-beta.1",
|
||||
"os": ["darwin"],
|
||||
"cpu": ["arm64"],
|
||||
"main": "lancedb.darwin-arm64.node",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-linux-arm64-gnu",
|
||||
"version": "0.39.0-beta.6",
|
||||
"version": "0.39.0-beta.1",
|
||||
"os": ["linux"],
|
||||
"cpu": ["arm64"],
|
||||
"main": "lancedb.linux-arm64-gnu.node",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-linux-arm64-musl",
|
||||
"version": "0.39.0-beta.6",
|
||||
"version": "0.39.0-beta.1",
|
||||
"os": ["linux"],
|
||||
"cpu": ["arm64"],
|
||||
"main": "lancedb.linux-arm64-musl.node",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-linux-x64-gnu",
|
||||
"version": "0.39.0-beta.6",
|
||||
"version": "0.39.0-beta.1",
|
||||
"os": ["linux"],
|
||||
"cpu": ["x64"],
|
||||
"main": "lancedb.linux-x64-gnu.node",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-linux-x64-musl",
|
||||
"version": "0.39.0-beta.6",
|
||||
"version": "0.39.0-beta.1",
|
||||
"os": ["linux"],
|
||||
"cpu": ["x64"],
|
||||
"main": "lancedb.linux-x64-musl.node",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-win32-arm64-msvc",
|
||||
"version": "0.39.0-beta.6",
|
||||
"version": "0.39.0-beta.1",
|
||||
"os": [
|
||||
"win32"
|
||||
],
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-win32-x64-msvc",
|
||||
"version": "0.39.0-beta.6",
|
||||
"version": "0.39.0-beta.1",
|
||||
"os": ["win32"],
|
||||
"cpu": ["x64"],
|
||||
"main": "lancedb.win32-x64-msvc.node",
|
||||
|
||||
+1
-1
@@ -11,7 +11,7 @@
|
||||
"ann"
|
||||
],
|
||||
"private": false,
|
||||
"version": "0.39.0-beta.6",
|
||||
"version": "0.39.0-beta.1",
|
||||
"main": "dist/index.js",
|
||||
"exports": {
|
||||
".": "./dist/index.js",
|
||||
|
||||
Generated
+24
-24
@@ -41,7 +41,7 @@ importers:
|
||||
version: 3.7.0(@emnapi/core@1.10.0)(@emnapi/runtime@1.11.3)(@types/node@22.7.4)
|
||||
'@opentelemetry/sdk-metrics':
|
||||
specifier: ^2.10.0
|
||||
version: 2.11.0(@opentelemetry/api@1.9.1)
|
||||
version: 2.10.0(@opentelemetry/api@1.9.1)
|
||||
'@types/axios':
|
||||
specifier: ^0.14.0
|
||||
version: 0.14.4
|
||||
@@ -80,7 +80,7 @@ importers:
|
||||
version: 0.2.7
|
||||
ts-jest:
|
||||
specifier: ^29.1.2
|
||||
version: 29.4.12(@babel/core@7.29.7)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.29.7))(jest-util@29.7.0)(jest@29.7.0(@types/node@22.7.4))(typescript@5.5.4)
|
||||
version: 29.4.9(@babel/core@7.29.7)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.29.7))(jest-util@29.7.0)(jest@29.7.0(@types/node@22.7.4))(typescript@5.5.4)
|
||||
typedoc:
|
||||
specifier: 0.26.4
|
||||
version: 0.26.4(typescript@5.5.4)
|
||||
@@ -1394,20 +1394,20 @@ packages:
|
||||
resolution: {integrity: sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q==}
|
||||
engines: {node: '>=8.0.0'}
|
||||
|
||||
'@opentelemetry/core@2.11.0':
|
||||
resolution: {integrity: sha512-7YP44XH0tV6+Mb54x2YGf84i7yi+31MBZlE8JwvozkxyTvXbSp10X7cI7YE49ChJ3shMJoBmCJF3+1QFBJctGA==}
|
||||
'@opentelemetry/core@2.10.0':
|
||||
resolution: {integrity: sha512-/wNZ8twnEQQA4HoHu22+vcsdru6pWPWxW+7w+FlxT6Id7PE/WIbZmVKkte+PF72e0F2dnImFeHD2syyE1Mw6MQ==}
|
||||
engines: {node: ^18.19.0 || >=20.6.0}
|
||||
peerDependencies:
|
||||
'@opentelemetry/api': '>=1.0.0 <1.10.0'
|
||||
|
||||
'@opentelemetry/resources@2.11.0':
|
||||
resolution: {integrity: sha512-Ie7+8q8MDF4FAEQCKVMTx3ReUvxiIAgIiiW3c9JdmP8+HMcDy20puT+AHjexnExgnbvBxjQ9fjkFDWrikJ2jQA==}
|
||||
'@opentelemetry/resources@2.10.0':
|
||||
resolution: {integrity: sha512-q6MMm2zhggzsHVNbabYwut+a6nbuQQe3URUoxaojM/8K1IBfwwPzvxIjNi2/lI1TFe+fMHMW9MWhrtDLEXEnkA==}
|
||||
engines: {node: ^18.19.0 || >=20.6.0}
|
||||
peerDependencies:
|
||||
'@opentelemetry/api': '>=1.3.0 <1.10.0'
|
||||
|
||||
'@opentelemetry/sdk-metrics@2.11.0':
|
||||
resolution: {integrity: sha512-7GXXcObyHyDUUSG+L+kJoquty01bzm7ivE7+SSgXXJcHuPzGviptxwARmI2c+bnnxjexGQbJnyNlN8HxBP/Y7A==}
|
||||
'@opentelemetry/sdk-metrics@2.10.0':
|
||||
resolution: {integrity: sha512-t6r1VSvXNtSDnPXU1FbZeetJb7yyovHmgu0wRSoftxtE0g2rSNhQZQUy69sRUCL+iioJpX8SN/S6wq6ZtvLySQ==}
|
||||
engines: {node: ^18.19.0 || >=20.6.0}
|
||||
peerDependencies:
|
||||
'@opentelemetry/api': '>=1.9.0 <1.10.0'
|
||||
@@ -1480,7 +1480,6 @@ packages:
|
||||
'@smithy/core@3.24.1':
|
||||
resolution: {integrity: sha512-3mT7o4qQyUWttYnVK3A0Z/u3Xha3E81tXn32Tz6vjZiUXhBrkEivpw1hBYfh84iFF9CSzkBU9Y1DJ3Q6RQ231g==}
|
||||
engines: {node: '>=18.0.0'}
|
||||
deprecated: Deprecated due to bug in browser bundling instructions https://github.com/smithy-lang/smithy-typescript/issues/2025
|
||||
|
||||
'@smithy/credential-provider-imds@4.3.1':
|
||||
resolution: {integrity: sha512-0S/acwHnqX4WrjXzhdiDRxsG2s9SC0cpPIK9nZ1R6UOHd+j7uL28+4bHu22urbLk2TVw3fkp6na/+fkUt/pLNQ==}
|
||||
@@ -3239,8 +3238,8 @@ packages:
|
||||
peerDependencies:
|
||||
typescript: '>=4.2.0'
|
||||
|
||||
ts-jest@29.4.12:
|
||||
resolution: {integrity: sha512-Ov6ClY53Fflh6BGAnY2DlTq1hYDrTycz2PVTXBWFW2CU+9zrEqAp9fWdGXl42EXO5RLSFAcAZ2JFKbP+zBTFfw==}
|
||||
ts-jest@29.4.9:
|
||||
resolution: {integrity: sha512-LTb9496gYPMCqjeDLdPrKuXtncudeV1yRZnF4Wo5l3SFi0RYEnYRNgMrFIdg+FHvfzjCyQk1cLncWVqiSX+EvQ==}
|
||||
engines: {node: ^14.15.0 || ^16.10.0 || ^18.0.0 || >=20.0.0}
|
||||
hasBin: true
|
||||
peerDependencies:
|
||||
@@ -5111,22 +5110,22 @@ snapshots:
|
||||
|
||||
'@opentelemetry/api@1.9.1': {}
|
||||
|
||||
'@opentelemetry/core@2.11.0(@opentelemetry/api@1.9.1)':
|
||||
'@opentelemetry/core@2.10.0(@opentelemetry/api@1.9.1)':
|
||||
dependencies:
|
||||
'@opentelemetry/api': 1.9.1
|
||||
'@opentelemetry/semantic-conventions': 1.43.0
|
||||
|
||||
'@opentelemetry/resources@2.11.0(@opentelemetry/api@1.9.1)':
|
||||
'@opentelemetry/resources@2.10.0(@opentelemetry/api@1.9.1)':
|
||||
dependencies:
|
||||
'@opentelemetry/api': 1.9.1
|
||||
'@opentelemetry/core': 2.11.0(@opentelemetry/api@1.9.1)
|
||||
'@opentelemetry/core': 2.10.0(@opentelemetry/api@1.9.1)
|
||||
'@opentelemetry/semantic-conventions': 1.43.0
|
||||
|
||||
'@opentelemetry/sdk-metrics@2.11.0(@opentelemetry/api@1.9.1)':
|
||||
'@opentelemetry/sdk-metrics@2.10.0(@opentelemetry/api@1.9.1)':
|
||||
dependencies:
|
||||
'@opentelemetry/api': 1.9.1
|
||||
'@opentelemetry/core': 2.11.0(@opentelemetry/api@1.9.1)
|
||||
'@opentelemetry/resources': 2.11.0(@opentelemetry/api@1.9.1)
|
||||
'@opentelemetry/core': 2.10.0(@opentelemetry/api@1.9.1)
|
||||
'@opentelemetry/resources': 2.10.0(@opentelemetry/api@1.9.1)
|
||||
|
||||
'@opentelemetry/semantic-conventions@1.43.0': {}
|
||||
|
||||
@@ -5575,7 +5574,7 @@ snapshots:
|
||||
globby: 11.1.0
|
||||
is-glob: 4.0.3
|
||||
minimatch: 9.0.9
|
||||
semver: 7.8.5
|
||||
semver: 7.8.0
|
||||
ts-api-utils: 1.4.3(typescript@5.5.4)
|
||||
optionalDependencies:
|
||||
typescript: 5.5.4
|
||||
@@ -6427,7 +6426,7 @@ snapshots:
|
||||
'@babel/parser': 7.29.3
|
||||
'@istanbuljs/schema': 0.1.6
|
||||
istanbul-lib-coverage: 3.2.2
|
||||
semver: 7.8.5
|
||||
semver: 7.8.0
|
||||
transitivePeerDependencies:
|
||||
- supports-color
|
||||
|
||||
@@ -6706,7 +6705,7 @@ snapshots:
|
||||
jest-util: 29.7.0
|
||||
natural-compare: 1.4.0
|
||||
pretty-format: 29.7.0
|
||||
semver: 7.8.5
|
||||
semver: 7.8.0
|
||||
transitivePeerDependencies:
|
||||
- supports-color
|
||||
|
||||
@@ -6829,7 +6828,7 @@ snapshots:
|
||||
|
||||
make-dir@4.0.0:
|
||||
dependencies:
|
||||
semver: 7.8.5
|
||||
semver: 7.8.0
|
||||
|
||||
make-error@1.3.6: {}
|
||||
|
||||
@@ -7163,7 +7162,8 @@ snapshots:
|
||||
|
||||
semver@7.8.0: {}
|
||||
|
||||
semver@7.8.5: {}
|
||||
semver@7.8.5:
|
||||
optional: true
|
||||
|
||||
sharp@0.35.4(@types/node@22.7.4):
|
||||
dependencies:
|
||||
@@ -7327,7 +7327,7 @@ snapshots:
|
||||
dependencies:
|
||||
typescript: 5.5.4
|
||||
|
||||
ts-jest@29.4.12(@babel/core@7.29.7)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.29.7))(jest-util@29.7.0)(jest@29.7.0(@types/node@22.7.4))(typescript@5.5.4):
|
||||
ts-jest@29.4.9(@babel/core@7.29.7)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.29.7))(jest-util@29.7.0)(jest@29.7.0(@types/node@22.7.4))(typescript@5.5.4):
|
||||
dependencies:
|
||||
bs-logger: 0.2.6
|
||||
fast-json-stable-stringify: 2.1.0
|
||||
@@ -7336,7 +7336,7 @@ snapshots:
|
||||
json5: 2.2.3
|
||||
lodash.memoize: 4.1.2
|
||||
make-error: 1.3.6
|
||||
semver: 7.8.5
|
||||
semver: 7.8.0
|
||||
type-fest: 4.41.0
|
||||
typescript: 5.5.4
|
||||
yargs-parser: 21.1.1
|
||||
|
||||
@@ -442,15 +442,13 @@ impl Connection {
|
||||
self.get_inner()?.drop_all_tables(&ns).await.default_error()
|
||||
}
|
||||
|
||||
/// Open a server-side job by id, returning a handle with its record
|
||||
/// already populated. Rejects when the server has no such job.
|
||||
/// A `Job` handle for a server-side job by id.
|
||||
///
|
||||
/// The returned handle answers for its own state, specification, result,
|
||||
/// failure and event history, so there is no separate connection-level
|
||||
/// call for any of them.
|
||||
#[napi(catch_unwind)]
|
||||
pub async fn open_job(&self, job_id: String) -> napi::Result<crate::job::Job> {
|
||||
let job = self.get_inner()?.open_job(&job_id).await.default_error()?;
|
||||
/// The handle is constructed without a server round trip; an unknown id
|
||||
/// surfaces when the handle is used.
|
||||
#[napi]
|
||||
pub fn job(&self, job_id: String) -> napi::Result<crate::job::Job> {
|
||||
let job = self.get_inner()?.job(job_id).default_error()?;
|
||||
Ok(crate::job::Job::new(job))
|
||||
}
|
||||
|
||||
@@ -461,6 +459,17 @@ impl Connection {
|
||||
Ok(jobs.into_iter().map(Into::into).collect())
|
||||
}
|
||||
|
||||
/// Describe a single server-side job by id. `null` when the server has
|
||||
/// no such job.
|
||||
#[napi(catch_unwind)]
|
||||
pub async fn get_job(
|
||||
&self,
|
||||
job_id: String,
|
||||
) -> napi::Result<Option<crate::job::JobDescription>> {
|
||||
let description = self.get_inner()?.get_job(&job_id).await.default_error()?;
|
||||
Ok(description.map(Into::into))
|
||||
}
|
||||
|
||||
/// Request cancellation of a server-side job by id. Returns true if the
|
||||
/// server accepted the cancellation, false if no such job exists.
|
||||
#[napi(catch_unwind)]
|
||||
@@ -468,6 +477,34 @@ impl Connection {
|
||||
self.get_inner()?.cancel_job(&job_id).await.default_error()
|
||||
}
|
||||
|
||||
/// The lifecycle event history of a server-side job (all jobs when
|
||||
/// `job_id` is null), as an Arrow IPC stream buffer. Empty when there is
|
||||
/// no history.
|
||||
#[napi(catch_unwind)]
|
||||
pub async fn job_history(&self, job_id: Option<String>) -> napi::Result<Buffer> {
|
||||
let batches = self
|
||||
.get_inner()?
|
||||
.job_history(job_id.as_deref())
|
||||
.await
|
||||
.default_error()?;
|
||||
let Some(first) = batches.first() else {
|
||||
return Ok(Buffer::from(Vec::<u8>::new()));
|
||||
};
|
||||
let mut out = Vec::new();
|
||||
let mut writer = arrow_ipc::writer::StreamWriter::try_new(&mut out, &first.schema())
|
||||
.map_err(|e| napi::Error::from_reason(e.to_string()))?;
|
||||
for batch in &batches {
|
||||
writer
|
||||
.write(batch)
|
||||
.map_err(|e| napi::Error::from_reason(e.to_string()))?;
|
||||
}
|
||||
writer
|
||||
.finish()
|
||||
.map_err(|e| napi::Error::from_reason(e.to_string()))?;
|
||||
drop(writer);
|
||||
Ok(Buffer::from(out))
|
||||
}
|
||||
|
||||
#[napi(catch_unwind)]
|
||||
/// Describe a namespace and return its properties.
|
||||
pub async fn describe_namespace(
|
||||
|
||||
+34
-90
@@ -3,9 +3,6 @@
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use arrow_array::RecordBatch;
|
||||
use lancedb::job::JobEventsRequest;
|
||||
use napi::bindgen_prelude::Buffer;
|
||||
use napi_derive::napi;
|
||||
|
||||
use crate::error::NapiErrorExt;
|
||||
@@ -58,98 +55,12 @@ impl Job {
|
||||
pub async fn cancel(&self) -> napi::Result<()> {
|
||||
self.inner.cancel().await.default_error()
|
||||
}
|
||||
|
||||
/// Ask the backend for this job's current state, and for a server-side job
|
||||
/// its full record, then cache it for the getters below.
|
||||
///
|
||||
/// They are all null until this runs, because submitting an operation
|
||||
/// returns only a job id. {@link Job.status} fetches the whole record too;
|
||||
/// {@link Job.wait} records only the terminal state it establishes.
|
||||
#[napi(catch_unwind)]
|
||||
pub async fn refresh(&self) -> napi::Result<()> {
|
||||
self.inner.refresh().await.default_error()
|
||||
}
|
||||
|
||||
/// The last observed lifecycle state, without contacting the backend.
|
||||
#[napi(getter)]
|
||||
pub fn state(&self) -> Option<String> {
|
||||
self.inner.state()
|
||||
}
|
||||
|
||||
/// The job's type, as the server names it. Null for an in-process job,
|
||||
/// which has no server-side record.
|
||||
#[napi(getter)]
|
||||
pub fn job_type(&self) -> Option<String> {
|
||||
self.inner.job_type()
|
||||
}
|
||||
|
||||
/// When the job was created, in milliseconds since the epoch.
|
||||
#[napi(getter)]
|
||||
pub fn creation_ms(&self) -> Option<i64> {
|
||||
self.inner.creation_ms()
|
||||
}
|
||||
|
||||
/// The job-type-specific specification as a JSON string, when present.
|
||||
#[napi(getter)]
|
||||
pub fn spec_json(&self) -> Option<String> {
|
||||
self.inner.spec().map(|spec| spec.to_string())
|
||||
}
|
||||
|
||||
/// The job-type-specific terminal result as a JSON string. Null until the
|
||||
/// job succeeds, so a job that never terminates reports its progress
|
||||
/// through {@link Job.events} instead.
|
||||
#[napi(getter)]
|
||||
pub fn result_json(&self) -> Option<String> {
|
||||
self.inner.result().map(|result| result.to_string())
|
||||
}
|
||||
|
||||
/// Why the job failed, when it failed and the server reports a reason.
|
||||
#[napi(getter)]
|
||||
pub fn failure(&self) -> Option<JobFailureInfo> {
|
||||
self.inner.failure().map(|failure| JobFailureInfo {
|
||||
phase: failure.phase,
|
||||
message: failure.message,
|
||||
retryable: failure.retryable,
|
||||
})
|
||||
}
|
||||
|
||||
/// This job's recorded lifecycle events, as an Arrow IPC stream buffer.
|
||||
/// The TypeScript wrapper turns it into an Arrow table.
|
||||
#[napi(catch_unwind)]
|
||||
pub async fn events(&self, limit: Option<u32>, filter: Option<String>) -> napi::Result<Buffer> {
|
||||
let batches = self
|
||||
.inner
|
||||
.events(JobEventsRequest { limit, filter })
|
||||
.await
|
||||
.default_error()?;
|
||||
batches_to_ipc_buffer(&batches)
|
||||
}
|
||||
}
|
||||
|
||||
/// Serialise Arrow batches as a single IPC stream for the TypeScript layer.
|
||||
fn batches_to_ipc_buffer(batches: &[RecordBatch]) -> napi::Result<Buffer> {
|
||||
let Some(first) = batches.first() else {
|
||||
return Ok(Buffer::from(Vec::<u8>::new()));
|
||||
};
|
||||
let mut out = Vec::new();
|
||||
let mut writer = arrow_ipc::writer::StreamWriter::try_new(&mut out, &first.schema())
|
||||
.map_err(|e| napi::Error::from_reason(e.to_string()))?;
|
||||
for batch in batches {
|
||||
writer
|
||||
.write(batch)
|
||||
.map_err(|e| napi::Error::from_reason(e.to_string()))?;
|
||||
}
|
||||
writer
|
||||
.finish()
|
||||
.map_err(|e| napi::Error::from_reason(e.to_string()))?;
|
||||
drop(writer);
|
||||
Ok(Buffer::from(out))
|
||||
}
|
||||
|
||||
/// A row from `Connection.listJobs`: one server-side job.
|
||||
#[napi(object)]
|
||||
pub struct JobInfo {
|
||||
/// The job id -- what `Connection.openJob` and `Connection.cancelJob`
|
||||
/// The job id -- what `Connection.getJob` and `Connection.cancelJob`
|
||||
/// accept.
|
||||
pub job_id: String,
|
||||
/// The table the job runs against, without URI or namespace.
|
||||
@@ -180,3 +91,36 @@ pub struct JobFailureInfo {
|
||||
pub message: Option<String>,
|
||||
pub retryable: Option<bool>,
|
||||
}
|
||||
|
||||
/// A described job from `Connection.getJob`.
|
||||
#[napi(object)]
|
||||
pub struct JobDescription {
|
||||
pub job_id: String,
|
||||
pub job_type: String,
|
||||
/// Lifecycle state: "running", "finished", "failed", or "cancelled".
|
||||
pub state: String,
|
||||
/// When the job was created, in milliseconds since the epoch.
|
||||
pub creation_ms: i64,
|
||||
/// The job-type-specific specification as a JSON string, when present.
|
||||
pub spec_json: Option<String>,
|
||||
/// Why the job failed, when the job is failed and the server reports a
|
||||
/// reason.
|
||||
pub failure: Option<JobFailureInfo>,
|
||||
}
|
||||
|
||||
impl From<lancedb::database::JobDescription> for JobDescription {
|
||||
fn from(description: lancedb::database::JobDescription) -> Self {
|
||||
Self {
|
||||
job_id: description.job_id,
|
||||
job_type: description.job_type,
|
||||
state: description.state,
|
||||
creation_ms: description.creation_ms,
|
||||
spec_json: (!description.spec.is_null()).then(|| description.spec.to_string()),
|
||||
failure: description.failure.map(|failure| JobFailureInfo {
|
||||
phase: failure.phase,
|
||||
message: failure.message,
|
||||
retryable: failure.retryable,
|
||||
}),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "lancedb-python"
|
||||
version = "0.39.0-beta.6"
|
||||
version = "0.39.0-beta.1"
|
||||
publish = false
|
||||
edition.workspace = true
|
||||
description = "Python bindings for LanceDB"
|
||||
|
||||
@@ -148,13 +148,17 @@ class Connection(object):
|
||||
start_after: Optional[str],
|
||||
limit: Optional[int],
|
||||
) -> list[str]: ... # Deprecated: Use list_tables instead
|
||||
async def open_job(self, job_id: str) -> Job: ...
|
||||
def job(self, job_id: str) -> Job: ...
|
||||
async def create_function_async(self, request_json: str) -> Job: ...
|
||||
async def get_function(self, name: str, version: str) -> str: ...
|
||||
async def list_functions(self) -> List[str]: ...
|
||||
async def drop_function(self, name: str, version: str) -> bool: ...
|
||||
async def list_jobs(self) -> List[JobInfo]: ...
|
||||
async def get_job(self, job_id: str) -> Optional[JobDescription]: ...
|
||||
async def cancel_job(self, job_id: str) -> bool: ...
|
||||
async def job_history(
|
||||
self, job_id: Optional[str] = None
|
||||
) -> List[pa.RecordBatch]: ...
|
||||
async def execute_query_async(
|
||||
self,
|
||||
query: str,
|
||||
@@ -240,20 +244,9 @@ class BlobFile:
|
||||
class Job:
|
||||
@property
|
||||
def id(self) -> Optional[str]: ...
|
||||
@property
|
||||
def _state(self) -> Optional[str]: ...
|
||||
@property
|
||||
def _description(self) -> Optional[JobDescription]: ...
|
||||
async def status(self) -> str: ...
|
||||
async def wait(self) -> Optional[str]: ...
|
||||
async def cancel(self) -> None: ...
|
||||
async def refresh(self) -> None: ...
|
||||
async def events(
|
||||
self,
|
||||
*,
|
||||
limit: Optional[int] = None,
|
||||
filter: Optional[str] = None,
|
||||
) -> pa.Table: ...
|
||||
|
||||
class JobInfo:
|
||||
@property
|
||||
@@ -285,13 +278,7 @@ class JobDescription:
|
||||
@property
|
||||
def creation_ms(self) -> int: ...
|
||||
@property
|
||||
def _spec_json(self) -> Optional[str]: ...
|
||||
@property
|
||||
def _result_json(self) -> Optional[str]: ...
|
||||
@property
|
||||
def spec(self) -> Optional[Any]: ...
|
||||
@property
|
||||
def result(self) -> Optional[Any]: ...
|
||||
def spec_json(self) -> Optional[str]: ...
|
||||
@property
|
||||
def failure(self) -> Optional[JobFailureInfo]: ...
|
||||
|
||||
|
||||
+68
-19
@@ -76,7 +76,7 @@ if TYPE_CHECKING:
|
||||
from .pydantic import LanceModel
|
||||
|
||||
from ._lancedb import Connection as LanceDbConnection
|
||||
from ._lancedb import JobInfo
|
||||
from ._lancedb import JobDescription, JobInfo
|
||||
from .common import DATA, URI
|
||||
from .embeddings import EmbeddingFunctionConfig
|
||||
from ._lancedb import Session
|
||||
@@ -745,23 +745,26 @@ class DBConnection(EnforceOverrides):
|
||||
"Function catalog operations are not supported for this connection type"
|
||||
)
|
||||
|
||||
def open_job(self, job_id: str) -> Job:
|
||||
"""Open a server-side job by id, returning a handle with its record
|
||||
already populated.
|
||||
def job(self, job_id: str) -> Job:
|
||||
"""A [Job][lancedb.job.Job] handle for a server-side job by id.
|
||||
|
||||
The returned [Job][lancedb.job.Job] answers for its own state,
|
||||
specification, result, failure and event history, so there is no
|
||||
separate connection-level call for any of them.
|
||||
|
||||
Raises `JobNotFoundError` when the server has no such job, the way
|
||||
`open_table` does for a missing table.
|
||||
The handle is constructed without a server round trip; an unknown id
|
||||
surfaces when the handle is used. Dropping the handle has no effect
|
||||
on the job itself.
|
||||
"""
|
||||
raise NotImplementedError("open_job is not supported for this connection type")
|
||||
raise NotImplementedError("job is not supported for this connection type")
|
||||
|
||||
def list_jobs(self) -> List[JobInfo]:
|
||||
"""List server-side jobs across the database's tables."""
|
||||
raise NotImplementedError("list_jobs is not supported for this connection type")
|
||||
|
||||
def get_job(self, job_id: str) -> Optional[JobDescription]:
|
||||
"""Describe a single server-side job by id.
|
||||
|
||||
Returns None when the server has no such job.
|
||||
"""
|
||||
raise NotImplementedError("get_job is not supported for this connection type")
|
||||
|
||||
def cancel_job(self, job_id: str) -> bool:
|
||||
"""Request cancellation of a server-side job by id.
|
||||
|
||||
@@ -773,6 +776,15 @@ class DBConnection(EnforceOverrides):
|
||||
"cancel_job is not supported for this connection type"
|
||||
)
|
||||
|
||||
def job_history(self, job_id: Optional[str] = None) -> List[pa.RecordBatch]:
|
||||
"""The lifecycle event history of a server-side job, as Arrow batches.
|
||||
|
||||
Lists history across all jobs when `job_id` is None.
|
||||
"""
|
||||
raise NotImplementedError(
|
||||
"job_history is not supported for this connection type"
|
||||
)
|
||||
|
||||
def execute_query(
|
||||
self,
|
||||
query: str,
|
||||
@@ -1450,11 +1462,14 @@ class LanceDBConnection(DBConnection):
|
||||
)
|
||||
|
||||
@override
|
||||
def open_job(self, job_id: str) -> Job:
|
||||
"""Open a server-side job by id. See
|
||||
[DBConnection.open_job][lancedb.db.DBConnection.open_job].
|
||||
def job(self, job_id: str) -> Job:
|
||||
"""A [Job][lancedb.job.Job] handle for a server-side job by id.
|
||||
|
||||
The handle is constructed without a server round trip; an unknown id
|
||||
surfaces when the handle is used. Dropping the handle has no effect
|
||||
on the job itself.
|
||||
"""
|
||||
return Job(LOOP.run(self._conn.open_job(job_id)))
|
||||
return Job(self._conn.job(job_id))
|
||||
|
||||
@override
|
||||
def create_function_async(self, definition: UdfDefinition) -> Job[FunctionVersion]:
|
||||
@@ -1478,6 +1493,14 @@ class LanceDBConnection(DBConnection):
|
||||
"""List server-side jobs across the database's tables."""
|
||||
return LOOP.run(self._conn.list_jobs())
|
||||
|
||||
@override
|
||||
def get_job(self, job_id: str) -> Optional[JobDescription]:
|
||||
"""Describe a single server-side job by id.
|
||||
|
||||
Returns None when the server has no such job.
|
||||
"""
|
||||
return LOOP.run(self._conn.get_job(job_id))
|
||||
|
||||
@override
|
||||
def cancel_job(self, job_id: str) -> bool:
|
||||
"""Request cancellation of a server-side job by id.
|
||||
@@ -1488,6 +1511,14 @@ class LanceDBConnection(DBConnection):
|
||||
"""
|
||||
return LOOP.run(self._conn.cancel_job(job_id))
|
||||
|
||||
@override
|
||||
def job_history(self, job_id: Optional[str] = None) -> List[pa.RecordBatch]:
|
||||
"""The lifecycle event history of a server-side job, as Arrow batches.
|
||||
|
||||
Lists history across all jobs when `job_id` is None.
|
||||
"""
|
||||
return LOOP.run(self._conn.job_history(job_id))
|
||||
|
||||
@override
|
||||
def namespace_client(self) -> LanceNamespace:
|
||||
"""Get the equivalent namespace client for this connection.
|
||||
@@ -2258,11 +2289,15 @@ class AsyncConnection(object):
|
||||
namespace_path = []
|
||||
await self._inner.drop_all_tables(namespace_path=namespace_path)
|
||||
|
||||
async def open_job(self, job_id: str) -> AsyncJob:
|
||||
"""Open a server-side job by id. See
|
||||
[DBConnection.open_job][lancedb.db.DBConnection.open_job].
|
||||
def job(self, job_id: str) -> AsyncJob:
|
||||
"""An [AsyncJob][lancedb.job.AsyncJob] handle for a server-side job
|
||||
by id.
|
||||
|
||||
The handle is constructed without a server round trip; an unknown id
|
||||
surfaces when the handle is used. Dropping the handle has no effect
|
||||
on the job itself.
|
||||
"""
|
||||
return AsyncJob(await self._inner.open_job(job_id))
|
||||
return AsyncJob(self._inner.job(job_id))
|
||||
|
||||
async def create_function_async(
|
||||
self, definition: UdfDefinition
|
||||
@@ -2302,6 +2337,13 @@ class AsyncConnection(object):
|
||||
"""List server-side jobs across the database's tables."""
|
||||
return await self._inner.list_jobs()
|
||||
|
||||
async def get_job(self, job_id: str) -> Optional[JobDescription]:
|
||||
"""Describe a single server-side job by id.
|
||||
|
||||
Returns None when the server has no such job.
|
||||
"""
|
||||
return await self._inner.get_job(job_id)
|
||||
|
||||
async def cancel_job(self, job_id: str) -> bool:
|
||||
"""Request cancellation of a server-side job by id.
|
||||
|
||||
@@ -2311,6 +2353,13 @@ class AsyncConnection(object):
|
||||
"""
|
||||
return await self._inner.cancel_job(job_id)
|
||||
|
||||
async def job_history(self, job_id: Optional[str] = None) -> List[pa.RecordBatch]:
|
||||
"""The lifecycle event history of a server-side job, as Arrow batches.
|
||||
|
||||
Lists history across all jobs when `job_id` is None.
|
||||
"""
|
||||
return await self._inner.job_history(job_id)
|
||||
|
||||
async def execute_query(
|
||||
self,
|
||||
query: str,
|
||||
|
||||
@@ -35,9 +35,3 @@ class JobCancelledError(RuntimeError):
|
||||
"""Exception raised when an asynchronous job was cancelled."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class JobNotFoundError(ValueError):
|
||||
"""Exception raised when opening a job the server does not have."""
|
||||
|
||||
pass
|
||||
|
||||
@@ -4,27 +4,15 @@
|
||||
"""Handles to operations a server may run asynchronously."""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from datetime import timedelta
|
||||
from typing import Any, Callable, Generic, Optional, TypeVar, cast
|
||||
|
||||
import pyarrow as pa
|
||||
|
||||
from lancedb.background_loop import LOOP
|
||||
|
||||
from . import _lancedb
|
||||
from ._lancedb import JobDescription, JobFailureInfo, JobInfo
|
||||
|
||||
T = TypeVar("T")
|
||||
|
||||
__all__ = [
|
||||
"AsyncJob",
|
||||
"Job",
|
||||
"JobDescription",
|
||||
"JobFailureInfo",
|
||||
"JobInfo",
|
||||
]
|
||||
|
||||
|
||||
class AsyncJob(Generic[T]):
|
||||
"""A handle to an operation that may still be running.
|
||||
@@ -90,149 +78,6 @@ class AsyncJob(Generic[T]):
|
||||
return
|
||||
await self._inner.cancel()
|
||||
|
||||
async def refresh(self) -> None:
|
||||
"""Ask the backend for this job's current state, and for a server-side
|
||||
job its full record, then cache it for the properties below.
|
||||
|
||||
The properties are all `None` until this runs, because submitting an
|
||||
operation returns only a job id. `status` fetches the whole record too;
|
||||
`wait` records only the terminal state it establishes.
|
||||
"""
|
||||
if self._inner is None:
|
||||
return
|
||||
await self._inner.refresh()
|
||||
|
||||
@property
|
||||
def state(self) -> Optional[str]:
|
||||
"""The last observed lifecycle state, without contacting the backend.
|
||||
|
||||
`None` until the handle has talked to it. See :meth:`AsyncJob.refresh`.
|
||||
"""
|
||||
if self._inner is None:
|
||||
return "finished"
|
||||
return self._inner._state
|
||||
|
||||
@property
|
||||
def job_type(self) -> Optional[str]:
|
||||
"""The job's type, as the server names it.
|
||||
|
||||
`None` for an in-process job, which has no server-side record.
|
||||
"""
|
||||
return self._field("job_type")
|
||||
|
||||
@property
|
||||
def creation_ms(self) -> Optional[int]:
|
||||
"""When the job was created, in milliseconds since the epoch."""
|
||||
return self._field("creation_ms")
|
||||
|
||||
@property
|
||||
def spec(self) -> Optional[Any]:
|
||||
"""The job-type-specific specification it was submitted with."""
|
||||
return self._field("spec")
|
||||
|
||||
@property
|
||||
def result(self) -> Optional[Any]:
|
||||
"""The job-type-specific terminal result, as reported data rather than
|
||||
the typed model :meth:`AsyncJob.wait` returns.
|
||||
|
||||
`None` until the job succeeds, so a job that never terminates reports
|
||||
its progress through :meth:`AsyncJob.events` instead.
|
||||
"""
|
||||
return self._field("result")
|
||||
|
||||
@property
|
||||
def failure(self) -> Optional[JobFailureInfo]:
|
||||
"""Why the job failed, when it failed and the server reports a reason."""
|
||||
return self._field("failure")
|
||||
|
||||
@property
|
||||
def _spec_json(self) -> Optional[str]:
|
||||
return self._field("_spec_json")
|
||||
|
||||
@property
|
||||
def _result_json(self) -> Optional[str]:
|
||||
return self._field("_result_json")
|
||||
|
||||
def _field(self, name: str) -> Optional[Any]:
|
||||
description = self._inner._description if self._inner is not None else None
|
||||
return getattr(description, name) if description is not None else None
|
||||
|
||||
async def events(
|
||||
self,
|
||||
*,
|
||||
limit: Optional[int] = None,
|
||||
filter: Optional[str] = None,
|
||||
) -> "pa.Table":
|
||||
"""This job's recorded lifecycle events.
|
||||
|
||||
Where the properties above report a terminal result only once the job
|
||||
reaches one, events are written as the job runs and outlive the workers
|
||||
that produced them. A distributed job records a `claim`/`claim_complete`
|
||||
pair per unit of work, each carrying `rows_processed`, so a job that
|
||||
never finishes still accounts for what it did.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
limit: int, optional
|
||||
Maximum event rows to return. The server caps results at 1000 by
|
||||
default and 10,000 at most, and truncates without saying so, so
|
||||
pass this for a job that emits an event per fragment.
|
||||
filter: str, optional
|
||||
SQL-like expression over the `state`, `updated_by`, `emitted_from`,
|
||||
`emitted_by`, and `claim_entity` columns, such as
|
||||
``state = 'claim_complete'``.
|
||||
"""
|
||||
if self._inner is None:
|
||||
raise NotImplementedError(
|
||||
"job event history is only available for server-side jobs"
|
||||
)
|
||||
return await self._inner.events(limit=limit, filter=filter)
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return _job_repr("AsyncJob", self)
|
||||
|
||||
|
||||
_REPR_INDENT = " " * 4
|
||||
|
||||
|
||||
def _repr_payload(value: Any) -> str:
|
||||
"""Render a job payload as indented JSON, aligned under its field."""
|
||||
try:
|
||||
rendered = json.dumps(value, indent=4)
|
||||
except TypeError:
|
||||
return repr(value)
|
||||
return rendered.replace("\n", "\n" + _REPR_INDENT)
|
||||
|
||||
|
||||
def _job_repr(kind: str, job: Any) -> str:
|
||||
"""Render every field the handle currently knows, omitting the rest.
|
||||
|
||||
One field per line, with the JSON payloads indented, because a refresh
|
||||
job's spec and result are the point of printing it.
|
||||
"""
|
||||
state = job.state
|
||||
if state is None:
|
||||
# Nothing has been fetched yet, so there is nothing to lay out.
|
||||
known = f"id={job.id!r}, " if job.id is not None else ""
|
||||
return f"{kind}({known}not refreshed)"
|
||||
|
||||
fields = []
|
||||
if job.id is not None:
|
||||
fields.append(f"id={job.id!r}")
|
||||
fields.append(f"state={state!r}")
|
||||
for name in ("job_type", "creation_ms"):
|
||||
value = getattr(job, name)
|
||||
if value is not None:
|
||||
fields.append(f"{name}={value!r}")
|
||||
for name in ("spec", "result"):
|
||||
value = getattr(job, name)
|
||||
if value is not None:
|
||||
fields.append(f"{name}={_repr_payload(value)}")
|
||||
if job.failure is not None:
|
||||
fields.append(f"failure={job.failure!r}")
|
||||
body = "".join(f"\n{_REPR_INDENT}{field}," for field in fields)
|
||||
return f"{kind}({body}\n)"
|
||||
|
||||
|
||||
class Job(Generic[T]):
|
||||
"""Synchronous counterpart of `AsyncJob` with the same result type."""
|
||||
@@ -277,75 +122,6 @@ class Job(Generic[T]):
|
||||
return
|
||||
LOOP.run(self._inner.cancel())
|
||||
|
||||
def refresh(self) -> None:
|
||||
"""Ask the backend for this job's current state and record.
|
||||
|
||||
See :meth:`AsyncJob.refresh`.
|
||||
"""
|
||||
if self._inner is None:
|
||||
return
|
||||
LOOP.run(self._inner.refresh())
|
||||
|
||||
@property
|
||||
def state(self) -> Optional[str]:
|
||||
"""The last observed lifecycle state. See :attr:`AsyncJob.state`."""
|
||||
return self._inner.state if self._inner is not None else "finished"
|
||||
|
||||
@property
|
||||
def job_type(self) -> Optional[str]:
|
||||
"""The job's type. See :attr:`AsyncJob.job_type`."""
|
||||
return self._field("job_type")
|
||||
|
||||
@property
|
||||
def creation_ms(self) -> Optional[int]:
|
||||
"""When the job was created. See :attr:`AsyncJob.creation_ms`."""
|
||||
return self._field("creation_ms")
|
||||
|
||||
@property
|
||||
def spec(self) -> Optional[Any]:
|
||||
"""The job's specification. See :attr:`AsyncJob.spec`."""
|
||||
return self._field("spec")
|
||||
|
||||
@property
|
||||
def result(self) -> Optional[Any]:
|
||||
"""The job's terminal result. See :attr:`AsyncJob.result`."""
|
||||
return self._field("result")
|
||||
|
||||
@property
|
||||
def failure(self) -> Optional[JobFailureInfo]:
|
||||
"""Why the job failed. See :attr:`AsyncJob.failure`."""
|
||||
return self._field("failure")
|
||||
|
||||
@property
|
||||
def _spec_json(self) -> Optional[str]:
|
||||
return self._field("_spec_json")
|
||||
|
||||
@property
|
||||
def _result_json(self) -> Optional[str]:
|
||||
return self._field("_result_json")
|
||||
|
||||
def _field(self, name: str) -> Optional[Any]:
|
||||
return getattr(self._inner, name) if self._inner is not None else None
|
||||
|
||||
def events(
|
||||
self,
|
||||
*,
|
||||
limit: Optional[int] = None,
|
||||
filter: Optional[str] = None,
|
||||
) -> "pa.Table":
|
||||
"""This job's recorded lifecycle events.
|
||||
|
||||
See :meth:`AsyncJob.events`.
|
||||
"""
|
||||
if self._inner is None:
|
||||
raise NotImplementedError(
|
||||
"job event history is only available for server-side jobs"
|
||||
)
|
||||
return LOOP.run(self._inner.events(limit=limit, filter=filter))
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return _job_repr("Job", self)
|
||||
|
||||
|
||||
def _typed_job(
|
||||
inner: "_lancedb.Job", result_decoder: Callable[[str], T]
|
||||
|
||||
@@ -31,7 +31,7 @@ from ..sql import QueryDescription
|
||||
from ..materialized_view import MaterializedView, SelectArg
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .._lancedb import JobInfo
|
||||
from .._lancedb import JobDescription, JobInfo
|
||||
from ..embeddings import EmbeddingFunctionConfig
|
||||
from lance_namespace import (
|
||||
LanceNamespace,
|
||||
@@ -739,11 +739,14 @@ class RemoteDBConnection(DBConnection):
|
||||
)
|
||||
|
||||
@override
|
||||
def open_job(self, job_id: str) -> Job:
|
||||
"""Open a server-side job by id. See
|
||||
[DBConnection.open_job][lancedb.db.DBConnection.open_job].
|
||||
def job(self, job_id: str) -> Job:
|
||||
"""A [Job][lancedb.job.Job] handle for a server-side job by id.
|
||||
|
||||
The handle is constructed without a server round trip; an unknown id
|
||||
surfaces when the handle is used. Dropping the handle has no effect
|
||||
on the job itself.
|
||||
"""
|
||||
return Job(LOOP.run(self._conn.open_job(job_id)))
|
||||
return Job(self._conn.job(job_id))
|
||||
|
||||
@override
|
||||
def create_function_async(self, definition: UdfDefinition) -> Job[FunctionVersion]:
|
||||
@@ -766,6 +769,14 @@ class RemoteDBConnection(DBConnection):
|
||||
"""List server-side jobs across the database's tables."""
|
||||
return LOOP.run(self._conn.list_jobs())
|
||||
|
||||
@override
|
||||
def get_job(self, job_id: str) -> Optional["JobDescription"]:
|
||||
"""Describe a single server-side job by id.
|
||||
|
||||
Returns None when the server has no such job.
|
||||
"""
|
||||
return LOOP.run(self._conn.get_job(job_id))
|
||||
|
||||
@override
|
||||
def cancel_job(self, job_id: str) -> bool:
|
||||
"""Request cancellation of a server-side job by id.
|
||||
@@ -776,6 +787,14 @@ class RemoteDBConnection(DBConnection):
|
||||
"""
|
||||
return LOOP.run(self._conn.cancel_job(job_id))
|
||||
|
||||
@override
|
||||
def job_history(self, job_id: Optional[str] = None) -> List[pa.RecordBatch]:
|
||||
"""The lifecycle event history of a server-side job, as Arrow batches.
|
||||
|
||||
Lists history across all jobs when `job_id` is None.
|
||||
"""
|
||||
return LOOP.run(self._conn.job_history(job_id))
|
||||
|
||||
@override
|
||||
def execute_query_async(
|
||||
self,
|
||||
|
||||
@@ -297,10 +297,7 @@ def test_blob_v2_projection_sources_use_typed_column_name():
|
||||
|
||||
|
||||
def _legacy_v1_table(name):
|
||||
# Legacy v1 blob columns are only writable at file version <= 2.1.
|
||||
db = lancedb.connect(
|
||||
"memory:///", storage_options={"new_table_data_storage_version": "2.1"}
|
||||
)
|
||||
db = lancedb.connect("memory:///")
|
||||
schema = pa.schema(
|
||||
[
|
||||
pa.field("id", pa.int64()),
|
||||
|
||||
@@ -193,13 +193,7 @@ class TestNamespaceConnection:
|
||||
),
|
||||
)
|
||||
|
||||
# Legacy v1 blob columns are only writable at file version <= 2.1.
|
||||
table = db.create_table(
|
||||
"blob_table",
|
||||
data,
|
||||
namespace_path=["test_ns"],
|
||||
storage_options={"new_table_data_storage_version": "2.1"},
|
||||
)
|
||||
table = db.create_table("blob_table", data, namespace_path=["test_ns"])
|
||||
df = table.to_pandas(blob_mode="lazy").sort_values("id")
|
||||
|
||||
blob = df["blob"].iloc[0]
|
||||
|
||||
@@ -40,10 +40,6 @@ from utils import exception_output
|
||||
from importlib.util import find_spec
|
||||
|
||||
|
||||
# Legacy v1 blob columns are only writable at file version <= 2.1.
|
||||
LEGACY_BLOB_STORAGE_OPTIONS = {"new_table_data_storage_version": "2.1"}
|
||||
|
||||
|
||||
def _blob_query_data():
|
||||
return pa.table(
|
||||
{
|
||||
@@ -123,17 +119,13 @@ def _assert_blob_bytes_projection(df):
|
||||
|
||||
def _blob_query_table(db, name, blob_schema):
|
||||
if blob_schema == "v1":
|
||||
return db.create_table(
|
||||
name, _blob_query_data(), storage_options=LEGACY_BLOB_STORAGE_OPTIONS
|
||||
)
|
||||
return db.create_table(name, _blob_query_data())
|
||||
return _create_blob_v2_query_table(db, name)
|
||||
|
||||
|
||||
async def _blob_query_table_async(db, name, blob_schema):
|
||||
if blob_schema == "v1":
|
||||
return await db.create_table(
|
||||
name, _blob_query_data(), storage_options=LEGACY_BLOB_STORAGE_OPTIONS
|
||||
)
|
||||
return await db.create_table(name, _blob_query_data())
|
||||
return await _create_blob_v2_query_table_async(db, name)
|
||||
|
||||
|
||||
@@ -283,9 +275,7 @@ async def test_query_to_pandas_kwargs(table, table_async):
|
||||
def test_plain_scan_query_to_pandas_blob_modes(tmp_db, blob_mode):
|
||||
pytest.importorskip("lance")
|
||||
table = tmp_db.create_table(
|
||||
f"test_query_to_pandas_blob_{blob_mode}",
|
||||
_blob_query_data(),
|
||||
storage_options=LEGACY_BLOB_STORAGE_OPTIONS,
|
||||
f"test_query_to_pandas_blob_{blob_mode}", _blob_query_data()
|
||||
)
|
||||
|
||||
df = (
|
||||
@@ -332,9 +322,7 @@ def test_plain_scan_query_to_pandas_blob_mode_does_not_collect_arrow(
|
||||
):
|
||||
pytest.importorskip("lance")
|
||||
table = tmp_db.create_table(
|
||||
"test_query_to_pandas_blob_no_arrow_collect",
|
||||
_blob_query_data(),
|
||||
storage_options=LEGACY_BLOB_STORAGE_OPTIONS,
|
||||
"test_query_to_pandas_blob_no_arrow_collect", _blob_query_data()
|
||||
)
|
||||
query = table.search().where("id = 1").select(["id", "blob"])
|
||||
|
||||
@@ -359,9 +347,7 @@ def test_plain_scan_query_to_pandas_blob_descriptions_flatten_uses_scanner(
|
||||
):
|
||||
pytest.importorskip("lance")
|
||||
table = tmp_db.create_table(
|
||||
"test_query_to_pandas_blob_desc_flatten",
|
||||
_blob_query_data(),
|
||||
storage_options=LEGACY_BLOB_STORAGE_OPTIONS,
|
||||
"test_query_to_pandas_blob_desc_flatten", _blob_query_data()
|
||||
)
|
||||
query = table.search().where("id = 1").select(["id", "blob"])
|
||||
|
||||
@@ -379,11 +365,7 @@ def test_plain_scan_query_to_pandas_blob_descriptions_flatten_uses_scanner(
|
||||
def test_plain_scan_query_to_pandas_scanner_state(tmp_db):
|
||||
pytest.importorskip("lance")
|
||||
data = _blob_query_data()
|
||||
table = tmp_db.create_table(
|
||||
"test_query_to_pandas_scanner_state",
|
||||
data.slice(0, 2),
|
||||
storage_options=LEGACY_BLOB_STORAGE_OPTIONS,
|
||||
)
|
||||
table = tmp_db.create_table("test_query_to_pandas_scanner_state", data.slice(0, 2))
|
||||
table.add(data.slice(2, 2))
|
||||
|
||||
fragments = table.to_lance().get_fragments()
|
||||
@@ -418,9 +400,7 @@ def test_plain_scan_query_to_pandas_scanner_state(tmp_db):
|
||||
async def test_async_plain_scan_query_to_pandas_blob_projection(tmp_db_async):
|
||||
pytest.importorskip("lance")
|
||||
table = await tmp_db_async.create_table(
|
||||
"test_async_query_to_pandas_blob_projection",
|
||||
_blob_query_data(),
|
||||
storage_options=LEGACY_BLOB_STORAGE_OPTIONS,
|
||||
"test_async_query_to_pandas_blob_projection", _blob_query_data()
|
||||
)
|
||||
|
||||
lazy_df = await (
|
||||
@@ -472,9 +452,7 @@ async def test_async_plain_scan_query_to_pandas_blob_mode_does_not_collect_arrow
|
||||
):
|
||||
pytest.importorskip("lance")
|
||||
table = await tmp_db_async.create_table(
|
||||
"test_async_query_to_pandas_blob_no_arrow_collect",
|
||||
_blob_query_data(),
|
||||
storage_options=LEGACY_BLOB_STORAGE_OPTIONS,
|
||||
"test_async_query_to_pandas_blob_no_arrow_collect", _blob_query_data()
|
||||
)
|
||||
query = table.query().where("id = 1").select(["id", "blob"])
|
||||
|
||||
@@ -496,11 +474,7 @@ async def test_async_plain_scan_query_to_pandas_blob_mode_does_not_collect_arrow
|
||||
|
||||
def test_vector_query_to_pandas_blob_mode_requires_native_path(tmp_db):
|
||||
pytest.importorskip("lance")
|
||||
table = tmp_db.create_table(
|
||||
"test_vector_query_blob_mode",
|
||||
_blob_query_data(),
|
||||
storage_options=LEGACY_BLOB_STORAGE_OPTIONS,
|
||||
)
|
||||
table = tmp_db.create_table("test_vector_query_blob_mode", _blob_query_data())
|
||||
|
||||
with pytest.raises(RuntimeError, match="Lance native pandas conversion"):
|
||||
table.search([1.0, 0.0]).select(["blob", "vector"]).limit(1).to_pandas(
|
||||
@@ -511,9 +485,7 @@ def test_vector_query_to_pandas_blob_mode_requires_native_path(tmp_db):
|
||||
def test_vector_query_to_pandas_blob_descriptions_requires_plain_scan(tmp_db):
|
||||
pytest.importorskip("lance")
|
||||
table = tmp_db.create_table(
|
||||
"test_vector_query_blob_descriptions",
|
||||
_blob_query_data(),
|
||||
storage_options=LEGACY_BLOB_STORAGE_OPTIONS,
|
||||
"test_vector_query_blob_descriptions", _blob_query_data()
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match="plain scan query"):
|
||||
|
||||
@@ -2467,7 +2467,7 @@ def test_remote_blob_byte_apis_not_supported_on_old_server():
|
||||
|
||||
|
||||
def test_remote_connection_jobs_surface():
|
||||
from lancedb.exceptions import JobFailedError, JobNotFoundError
|
||||
from lancedb.exceptions import JobFailedError
|
||||
|
||||
schema = pa.schema([("state", pa.string())])
|
||||
batch = pa.record_batch([pa.array(["created", "done"])], schema=schema)
|
||||
@@ -2475,7 +2475,6 @@ def test_remote_connection_jobs_surface():
|
||||
with pa.ipc.new_stream(sink, schema) as writer:
|
||||
writer.write_batch(batch)
|
||||
events_body = sink.getvalue().to_pybytes()
|
||||
query_events_payloads = []
|
||||
|
||||
def handler(request):
|
||||
content_len = int(request.headers.get("Content-Length", 0))
|
||||
@@ -2513,22 +2512,6 @@ def test_remote_connection_jobs_surface():
|
||||
request.end_headers()
|
||||
request.wfile.write(json.dumps(rsp).encode())
|
||||
elif request.path == "/v1/jobs/describe":
|
||||
if payload["job_id"] == "job-2":
|
||||
request.send_response(200)
|
||||
request.send_header("Content-Type", "application/json")
|
||||
request.end_headers()
|
||||
request.wfile.write(
|
||||
json.dumps(
|
||||
dict(
|
||||
job_id="job-2",
|
||||
job_type="refresh_column",
|
||||
job_state="DONE",
|
||||
creation_ms=2000,
|
||||
result=dict(rows_assigned=1000000, rows_failed=0),
|
||||
)
|
||||
).encode()
|
||||
)
|
||||
return
|
||||
if payload["job_id"] != "job-1":
|
||||
request.send_response(404)
|
||||
request.end_headers()
|
||||
@@ -2560,7 +2543,7 @@ def test_remote_connection_jobs_surface():
|
||||
request.end_headers()
|
||||
request.wfile.write(b'{"job_id": "job-1"}')
|
||||
elif request.path == "/v1/jobs/query_events":
|
||||
query_events_payloads.append(payload)
|
||||
assert payload["job_id"] == "job-1"
|
||||
request.send_response(200)
|
||||
request.send_header("Content-Type", "application/vnd.apache.arrow.stream")
|
||||
request.end_headers()
|
||||
@@ -2576,109 +2559,24 @@ def test_remote_connection_jobs_surface():
|
||||
assert jobs[0].table == "t1"
|
||||
assert jobs[1].state == "finished"
|
||||
|
||||
description = db.get_job("job-1")
|
||||
assert description.job_type == "create_index"
|
||||
assert description.state == "failed"
|
||||
assert json.loads(description.spec_json) == {"column": "vec"}
|
||||
assert description.failure.message == "worker died"
|
||||
assert description.failure.retryable is True
|
||||
assert db.get_job("missing") is None
|
||||
|
||||
assert db.cancel_job("job-1") is True
|
||||
assert db.cancel_job("missing") is False
|
||||
|
||||
# Opening a job hands back a populated handle; a missing one fails.
|
||||
with pytest.raises(JobNotFoundError, match="missing"):
|
||||
db.open_job("missing")
|
||||
finished = db.open_job("job-2")
|
||||
assert finished.state == "finished"
|
||||
assert finished.result == {"rows_assigned": 1000000, "rows_failed": 0}
|
||||
batches = db.job_history("job-1")
|
||||
assert len(batches) == 1
|
||||
assert batches[0].num_rows == 2
|
||||
assert batches[0].column("state").to_pylist() == ["created", "done"]
|
||||
|
||||
job = db.open_job("job-1")
|
||||
job = db.job("job-1")
|
||||
assert job.id == "job-1"
|
||||
# Opening already populated the handle.
|
||||
assert job.state == "failed"
|
||||
assert job.spec == {"column": "vec"}
|
||||
assert job.failure.message == "worker died"
|
||||
assert job.status() == "failed"
|
||||
with pytest.raises(JobFailedError, match="worker died"):
|
||||
job.wait(timeout=timedelta(seconds=5))
|
||||
|
||||
|
||||
def test_remote_job_handle_reports_its_own_detail():
|
||||
schema = pa.schema([("state", pa.string())])
|
||||
batch = pa.record_batch([pa.array(["claim_complete"])], schema=schema)
|
||||
sink = pa.BufferOutputStream()
|
||||
with pa.ipc.new_stream(sink, schema) as writer:
|
||||
writer.write_batch(batch)
|
||||
events_body = sink.getvalue().to_pybytes()
|
||||
event_payloads = []
|
||||
|
||||
def handler(request):
|
||||
content_len = int(request.headers.get("Content-Length", 0))
|
||||
body = request.rfile.read(content_len) if content_len > 0 else b""
|
||||
payload = json.loads(body) if body else {}
|
||||
if request.path == "/v1/jobs/describe":
|
||||
request.send_response(200)
|
||||
request.send_header("Content-Type", "application/json")
|
||||
request.end_headers()
|
||||
request.wfile.write(
|
||||
json.dumps(
|
||||
dict(
|
||||
job_id="job-1",
|
||||
job_type="refresh_column",
|
||||
job_state="DONE",
|
||||
creation_ms=2000,
|
||||
spec=dict(column="vec"),
|
||||
result=dict(rows_assigned=1000000),
|
||||
)
|
||||
).encode()
|
||||
)
|
||||
elif request.path == "/v1/jobs/query_events":
|
||||
event_payloads.append(payload)
|
||||
request.send_response(200)
|
||||
request.send_header("Content-Type", "application/vnd.apache.arrow.stream")
|
||||
request.end_headers()
|
||||
request.wfile.write(events_body)
|
||||
else:
|
||||
request.send_response(404)
|
||||
request.end_headers()
|
||||
|
||||
with mock_lancedb_connection(handler) as db:
|
||||
job = db.open_job("job-1")
|
||||
|
||||
# Opening populates the handle in the same round trip.
|
||||
assert job.state == "finished"
|
||||
job.refresh()
|
||||
assert job.job_type == "refresh_column"
|
||||
assert job.creation_ms == 2000
|
||||
assert job.spec == {"column": "vec"}
|
||||
assert job.result == {"rows_assigned": 1000000}
|
||||
assert job.failure is None
|
||||
# The JSON payloads stay reachable, but as internal APIs.
|
||||
assert json.loads(job._spec_json) == {"column": "vec"}
|
||||
assert json.loads(job._result_json) == {"rows_assigned": 1000000}
|
||||
|
||||
# print() shows everything the handle knows and nothing it does not.
|
||||
# print() lays every known field out on its own line, with the JSON
|
||||
# payloads indented rather than crammed onto one line.
|
||||
assert repr(job) == "\n".join(
|
||||
[
|
||||
"Job(",
|
||||
" id='job-1',",
|
||||
" state='finished',",
|
||||
" job_type='refresh_column',",
|
||||
" creation_ms=2000,",
|
||||
" spec={",
|
||||
' "column": "vec"',
|
||||
" },",
|
||||
" result={",
|
||||
' "rows_assigned": 1000000',
|
||||
" },",
|
||||
")",
|
||||
]
|
||||
)
|
||||
# Nothing it does not know shows up.
|
||||
assert "failure" not in repr(job)
|
||||
|
||||
events = job.events(filter="state = 'claim_complete'", limit=500)
|
||||
assert isinstance(events, pa.Table)
|
||||
assert events.column("state").to_pylist() == ["claim_complete"]
|
||||
# The handle supplies job_id; the caller only narrows the query.
|
||||
assert event_payloads[-1] == {
|
||||
"job_id": "job-1",
|
||||
"limit": 500,
|
||||
"filter": "state = 'claim_complete'",
|
||||
}
|
||||
|
||||
@@ -64,23 +64,15 @@ async def _blob_v2_table_async(db: AsyncConnection, name: str):
|
||||
return table
|
||||
|
||||
|
||||
# Legacy v1 blob columns are only writable at file version <= 2.1.
|
||||
LEGACY_BLOB_STORAGE_OPTIONS = {"new_table_data_storage_version": "2.1"}
|
||||
|
||||
|
||||
def _blob_table(db: DBConnection, name: str, blob_schema: str):
|
||||
if blob_schema == "v1":
|
||||
return db.create_table(
|
||||
name, data=_blob_test_data(), storage_options=LEGACY_BLOB_STORAGE_OPTIONS
|
||||
)
|
||||
return db.create_table(name, data=_blob_test_data())
|
||||
return _blob_v2_table(db, name)
|
||||
|
||||
|
||||
async def _blob_table_async(db: AsyncConnection, name: str, blob_schema: str):
|
||||
if blob_schema == "v1":
|
||||
return await db.create_table(
|
||||
name, data=_blob_test_data(), storage_options=LEGACY_BLOB_STORAGE_OPTIONS
|
||||
)
|
||||
return await db.create_table(name, data=_blob_test_data())
|
||||
return await _blob_v2_table_async(db, name)
|
||||
|
||||
|
||||
@@ -155,11 +147,7 @@ def test_table_to_pandas_invalid_blob_mode_non_blob_table(tmp_db: DBConnection):
|
||||
@pytest.mark.parametrize("blob_mode", ["lazy", "bytes", "descriptions"])
|
||||
def test_table_to_pandas_blob_modes(tmp_db: DBConnection, blob_mode):
|
||||
pytest.importorskip("lance")
|
||||
table = tmp_db.create_table(
|
||||
f"test_to_pandas_blob_{blob_mode}",
|
||||
_blob_test_data(),
|
||||
storage_options=LEGACY_BLOB_STORAGE_OPTIONS,
|
||||
)
|
||||
table = tmp_db.create_table(f"test_to_pandas_blob_{blob_mode}", _blob_test_data())
|
||||
|
||||
df = table.to_pandas(blob_mode=blob_mode)
|
||||
|
||||
@@ -3971,7 +3959,7 @@ def test_stats(mem_db: DBConnection):
|
||||
print(f"{stats=}")
|
||||
assert stats == {
|
||||
# Full on-disk size of the data file, footer and metadata included.
|
||||
"total_bytes": 637,
|
||||
"total_bytes": 633,
|
||||
"num_rows": 2,
|
||||
"num_indices": 0,
|
||||
"fragment_stats": {
|
||||
|
||||
@@ -13,7 +13,11 @@ use crate::{
|
||||
runtime::future_into_py,
|
||||
table::Table,
|
||||
};
|
||||
use arrow::{datatypes::Schema, ffi_stream::ArrowArrayStreamReader, pyarrow::FromPyArrow};
|
||||
use arrow::{
|
||||
datatypes::Schema,
|
||||
ffi_stream::ArrowArrayStreamReader,
|
||||
pyarrow::{FromPyArrow, ToPyArrow},
|
||||
};
|
||||
use lancedb::{
|
||||
connection::Connection as LanceConnection,
|
||||
connection::NamespaceClientPushdownOperation,
|
||||
@@ -24,7 +28,7 @@ use pyo3::{
|
||||
Bound, FromPyObject, Py, PyAny, PyRef, PyResult, Python,
|
||||
exceptions::{PyRuntimeError, PyValueError},
|
||||
pyclass, pyfunction, pymethods,
|
||||
types::{PyAnyMethods, PyDict, PyDictMethods, PyList},
|
||||
types::{PyAnyMethods, PyDict, PyDictMethods, PyList, PyListMethods},
|
||||
};
|
||||
|
||||
#[pyclass]
|
||||
@@ -640,12 +644,9 @@ impl Connection {
|
||||
})
|
||||
}
|
||||
|
||||
pub fn open_job(self_: PyRef<'_, Self>, job_id: String) -> PyResult<Bound<'_, PyAny>> {
|
||||
let inner = self_.get_inner()?.clone();
|
||||
future_into_py(self_.py(), async move {
|
||||
let job = inner.open_job(&job_id).await.infer_error()?;
|
||||
Ok(crate::job::Job::new(job))
|
||||
})
|
||||
pub fn job(&self, job_id: String) -> PyResult<crate::job::Job> {
|
||||
let inner = self.get_inner()?.clone();
|
||||
Ok(crate::job::Job::new(inner.job(job_id).infer_error()?))
|
||||
}
|
||||
|
||||
pub fn create_function_async(
|
||||
@@ -715,12 +716,38 @@ impl Connection {
|
||||
})
|
||||
}
|
||||
|
||||
pub fn get_job(self_: PyRef<'_, Self>, job_id: String) -> PyResult<Bound<'_, PyAny>> {
|
||||
let inner = self_.get_inner()?.clone();
|
||||
future_into_py(self_.py(), async move {
|
||||
let description = inner.get_job(&job_id).await.infer_error()?;
|
||||
Ok(description.map(crate::job::JobDescription::from))
|
||||
})
|
||||
}
|
||||
|
||||
pub fn cancel_job(self_: PyRef<'_, Self>, job_id: String) -> PyResult<Bound<'_, PyAny>> {
|
||||
let inner = self_.get_inner()?.clone();
|
||||
future_into_py(self_.py(), async move {
|
||||
inner.cancel_job(&job_id).await.infer_error()
|
||||
})
|
||||
}
|
||||
|
||||
#[pyo3(signature = (job_id=None))]
|
||||
pub fn job_history(
|
||||
self_: PyRef<'_, Self>,
|
||||
job_id: Option<String>,
|
||||
) -> PyResult<Bound<'_, PyAny>> {
|
||||
let inner = self_.get_inner()?.clone();
|
||||
future_into_py(self_.py(), async move {
|
||||
let batches = inner.job_history(job_id.as_deref()).await.infer_error()?;
|
||||
Python::attach(|py| {
|
||||
let list = PyList::empty(py);
|
||||
for batch in batches {
|
||||
list.append(batch.to_pyarrow(py)?)?;
|
||||
}
|
||||
Ok(list.unbind())
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[pyfunction]
|
||||
|
||||
@@ -114,12 +114,6 @@ impl<T> PythonErrorExt<T> for std::result::Result<T, LanceError> {
|
||||
.getattr(intern!(py, "JobCancelledError"))?;
|
||||
Err(PyErr::from_value(cls.call1((err.to_string(),))?))
|
||||
}),
|
||||
LanceError::JobNotFound { .. } => Python::attach(|py| {
|
||||
let cls = py
|
||||
.import(intern!(py, "lancedb.exceptions"))?
|
||||
.getattr(intern!(py, "JobNotFoundError"))?;
|
||||
Err(PyErr::from_value(cls.call1((err.to_string(),))?))
|
||||
}),
|
||||
_ => self.runtime_error(),
|
||||
},
|
||||
}
|
||||
|
||||
+9
-126
@@ -4,50 +4,11 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::runtime::future_into_py;
|
||||
use arrow::{
|
||||
datatypes::Schema,
|
||||
pyarrow::{IntoPyArrow, Table as PyArrowTable},
|
||||
};
|
||||
use lancedb::job::JobEventsRequest;
|
||||
use pyo3::{
|
||||
Bound, PyAny, PyRef, PyResult, Python,
|
||||
exceptions::PyValueError,
|
||||
pyclass, pymethods,
|
||||
types::{PyAnyMethods, PyDict, PyDictMethods},
|
||||
};
|
||||
use pyo3::{Bound, PyAny, PyRef, PyResult, pyclass, pymethods};
|
||||
use serde::Serialize;
|
||||
|
||||
use crate::error::PythonErrorExt;
|
||||
|
||||
const REPR_INDENT: &str = " ";
|
||||
|
||||
/// Parse a stored JSON payload into Python data. The bindings carry these as
|
||||
/// strings because that is what crosses the boundary cheaply; the public
|
||||
/// Python surface is the parsed form.
|
||||
fn parse_json_payload<'py>(
|
||||
py: Python<'py>,
|
||||
raw: Option<&str>,
|
||||
) -> PyResult<Option<Bound<'py, PyAny>>> {
|
||||
match raw {
|
||||
None => Ok(None),
|
||||
Some(raw) => Ok(Some(py.import("json")?.call_method1("loads", (raw,))?)),
|
||||
}
|
||||
}
|
||||
|
||||
/// A payload rendered as indented JSON, aligned under the field that holds it.
|
||||
fn pretty_json_payload(py: Python<'_>, raw: Option<&str>) -> PyResult<Option<String>> {
|
||||
let Some(parsed) = parse_json_payload(py, raw)? else {
|
||||
return Ok(None);
|
||||
};
|
||||
let kwargs = PyDict::new(py);
|
||||
kwargs.set_item("indent", 4)?;
|
||||
let rendered: String = py
|
||||
.import("json")?
|
||||
.call_method("dumps", (parsed,), Some(&kwargs))?
|
||||
.extract()?;
|
||||
Ok(Some(rendered.replace('\n', &format!("\n{REPR_INDENT}"))))
|
||||
}
|
||||
|
||||
#[pyclass]
|
||||
pub struct Job {
|
||||
inner: Arc<lancedb::Job<std::result::Result<Option<String>, String>>>,
|
||||
@@ -106,48 +67,6 @@ impl Job {
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
|
||||
pub fn refresh(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
|
||||
let inner = self_.inner.clone();
|
||||
future_into_py(self_.py(), async move {
|
||||
inner.refresh().await.infer_error()?;
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
|
||||
/// The last observed lifecycle state, without contacting the backend.
|
||||
#[getter]
|
||||
pub fn _state(&self) -> Option<String> {
|
||||
self.inner.state()
|
||||
}
|
||||
|
||||
/// The last observed server-side record. `None` for an in-process job.
|
||||
#[getter]
|
||||
pub fn _description(&self) -> Option<JobDescription> {
|
||||
self.inner.description().map(JobDescription::from)
|
||||
}
|
||||
|
||||
#[pyo3(signature = (*, limit=None, filter=None))]
|
||||
pub fn events(
|
||||
self_: PyRef<'_, Self>,
|
||||
limit: Option<u32>,
|
||||
filter: Option<String>,
|
||||
) -> PyResult<Bound<'_, PyAny>> {
|
||||
let inner = self_.inner.clone();
|
||||
let request = JobEventsRequest { limit, filter };
|
||||
future_into_py(self_.py(), async move {
|
||||
let batches = inner.events(request).await.infer_error()?;
|
||||
Python::attach(|py| {
|
||||
let schema = batches
|
||||
.first()
|
||||
.map(|batch| batch.schema())
|
||||
.unwrap_or_else(|| Arc::new(Schema::empty()));
|
||||
let table = PyArrowTable::try_new(batches, schema)
|
||||
.map_err(|err| PyValueError::new_err(err.to_string()))?;
|
||||
table.into_pyarrow(py).map(|table| table.unbind())
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// A row from `Connection.list_jobs`: one server-side job.
|
||||
@@ -202,7 +121,7 @@ impl JobFailureInfo {
|
||||
}
|
||||
}
|
||||
|
||||
/// The server-side record behind a `Job` handle.
|
||||
/// A described job from `Connection.get_job`.
|
||||
#[pyclass(get_all, skip_from_py_object)]
|
||||
#[derive(Clone)]
|
||||
pub struct JobDescription {
|
||||
@@ -210,49 +129,17 @@ pub struct JobDescription {
|
||||
job_type: String,
|
||||
state: String,
|
||||
creation_ms: i64,
|
||||
/// Internal: the wire form behind the `spec` property.
|
||||
_spec_json: Option<String>,
|
||||
/// Internal: the wire form behind the `result` property.
|
||||
_result_json: Option<String>,
|
||||
spec_json: Option<String>,
|
||||
failure: Option<JobFailureInfo>,
|
||||
}
|
||||
|
||||
#[pymethods]
|
||||
impl JobDescription {
|
||||
/// The job-type-specific specification it was submitted with.
|
||||
#[getter]
|
||||
fn spec<'py>(&self, py: Python<'py>) -> PyResult<Option<Bound<'py, PyAny>>> {
|
||||
parse_json_payload(py, self._spec_json.as_deref())
|
||||
}
|
||||
|
||||
/// The job-type-specific terminal result. `None` until the job succeeds.
|
||||
#[getter]
|
||||
fn result<'py>(&self, py: Python<'py>) -> PyResult<Option<Bound<'py, PyAny>>> {
|
||||
parse_json_payload(py, self._result_json.as_deref())
|
||||
}
|
||||
|
||||
fn __repr__(&self, py: Python<'_>) -> PyResult<String> {
|
||||
let mut fields = vec![
|
||||
format!("job_id={:?}", self.job_id),
|
||||
format!("job_type={:?}", self.job_type),
|
||||
format!("state={:?}", self.state),
|
||||
format!("creation_ms={}", self.creation_ms),
|
||||
];
|
||||
// Lay the payloads out as indented JSON, the same way the `Job` repr
|
||||
// does, so the two agree on how the same data looks.
|
||||
for (name, payload) in [("spec", &self._spec_json), ("result", &self._result_json)] {
|
||||
if let Some(rendered) = pretty_json_payload(py, payload.as_deref())? {
|
||||
fields.push(format!("{name}={rendered}"));
|
||||
}
|
||||
}
|
||||
if let Some(failure) = &self.failure {
|
||||
fields.push(format!("failure={}", failure.__repr__()));
|
||||
}
|
||||
let body = fields
|
||||
.iter()
|
||||
.map(|field| format!("\n{REPR_INDENT}{field},"))
|
||||
.collect::<String>();
|
||||
Ok(format!("JobDescription({body}\n)"))
|
||||
fn __repr__(&self) -> String {
|
||||
format!(
|
||||
"JobDescription(job_id={:?}, job_type={:?}, state={:?}, creation_ms={})",
|
||||
self.job_id, self.job_type, self.state, self.creation_ms
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -263,11 +150,7 @@ impl From<lancedb::database::JobDescription> for JobDescription {
|
||||
job_type: description.job_type,
|
||||
state: description.state,
|
||||
creation_ms: description.creation_ms,
|
||||
_spec_json: (!description.spec.is_null()).then(|| description.spec.to_string()),
|
||||
_result_json: description
|
||||
.result
|
||||
.filter(|result| !result.is_null())
|
||||
.map(|result| result.to_string()),
|
||||
spec_json: (!description.spec.is_null()).then(|| description.spec.to_string()),
|
||||
failure: description.failure.map(|failure| JobFailureInfo {
|
||||
phase: failure.phase,
|
||||
message: failure.message,
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "lancedb"
|
||||
version = "0.39.0-beta.6"
|
||||
version = "0.39.0-beta.1"
|
||||
edition.workspace = true
|
||||
description = "LanceDB: A serverless, low-latency vector database for AI applications"
|
||||
license.workspace = true
|
||||
|
||||
@@ -532,11 +532,10 @@ mod tests {
|
||||
fn storage_version_bumps_to_v2_2() {
|
||||
let mut params = WriteParams::default();
|
||||
ensure_blob_storage_version(&blob_schema(), &mut params);
|
||||
let resolved = params
|
||||
.data_storage_version
|
||||
.unwrap_or(LanceFileVersion::Stable)
|
||||
.resolve();
|
||||
assert_eq!(resolved, ConcreteFileVersion::V2_2);
|
||||
assert_eq!(
|
||||
params.data_storage_version.unwrap().resolve(),
|
||||
ConcreteFileVersion::V2_2
|
||||
);
|
||||
assert!(!params.enable_stable_row_ids);
|
||||
}
|
||||
|
||||
@@ -548,11 +547,10 @@ mod tests {
|
||||
};
|
||||
ensure_blob_storage_version(&blob_schema(), &mut params);
|
||||
assert!(params.enable_stable_row_ids);
|
||||
let resolved = params
|
||||
.data_storage_version
|
||||
.unwrap_or(LanceFileVersion::Stable)
|
||||
.resolve();
|
||||
assert_eq!(resolved, ConcreteFileVersion::V2_2);
|
||||
assert_eq!(
|
||||
params.data_storage_version.unwrap().resolve(),
|
||||
ConcreteFileVersion::V2_2
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -23,8 +23,8 @@ use crate::connection::create_table::CreateTableBuilder;
|
||||
use crate::data::scannable::Scannable;
|
||||
use crate::database::listing::ListingDatabase;
|
||||
use crate::database::{
|
||||
CloneTableRequest, Database, DatabaseOptions, JobInfo, OpenTableRequest, ReadConsistency,
|
||||
TableNamesRequest,
|
||||
CloneTableRequest, Database, DatabaseOptions, JobDescription, JobInfo, OpenTableRequest,
|
||||
ReadConsistency, TableNamesRequest,
|
||||
};
|
||||
use crate::embeddings::{EmbeddingRegistry, MemoryRegistry};
|
||||
use crate::error::{Error, Result};
|
||||
@@ -670,34 +670,14 @@ impl Connection {
|
||||
self.internal.read_consistency().await
|
||||
}
|
||||
|
||||
/// Open a server-side job by id, returning a handle with its record
|
||||
/// already populated. Fails with [`crate::Error::JobNotFound`] when the
|
||||
/// server has no such job, the way [`Connection::open_table`] does for a
|
||||
/// missing table.
|
||||
/// A [`crate::job::Job`] handle for a server-side job by id, suitable for
|
||||
/// waiting on or cancelling the job.
|
||||
///
|
||||
/// This is the one way in: the returned [`crate::job::Job`] answers for
|
||||
/// its own state, specification, result, failure and event history, so
|
||||
/// there is no separate connection-level call for any of them.
|
||||
///
|
||||
/// # Example
|
||||
///
|
||||
/// ```no_run
|
||||
/// # use lancedb::job::JobEventsRequest;
|
||||
/// # async fn open_job(
|
||||
/// # connection: &lancedb::Connection,
|
||||
/// # job_id: &str,
|
||||
/// # ) -> Result<(), Box<dyn std::error::Error>> {
|
||||
/// let job = connection.open_job(job_id).await?;
|
||||
/// println!("{:?} {:?}", job.state(), job.result());
|
||||
/// let done = job
|
||||
/// .events(JobEventsRequest::default().filter("state = 'claim_complete'"))
|
||||
/// .await?;
|
||||
/// println!("{} completions", done.iter().map(|b| b.num_rows()).sum::<usize>());
|
||||
/// # Ok(())
|
||||
/// # }
|
||||
/// ```
|
||||
pub async fn open_job(&self, job_id: impl AsRef<str>) -> Result<crate::job::Job> {
|
||||
self.internal.open_job(job_id.as_ref()).await
|
||||
/// The handle is constructed without a server round trip; an unknown id
|
||||
/// surfaces when the handle is used. Only server-backed databases support
|
||||
/// job handles by id.
|
||||
pub fn job(&self, job_id: impl AsRef<str>) -> Result<crate::job::Job> {
|
||||
self.internal.job(job_id.as_ref())
|
||||
}
|
||||
|
||||
/// List server-side jobs across the database's tables.
|
||||
@@ -705,12 +685,24 @@ impl Connection {
|
||||
self.internal.list_jobs().await
|
||||
}
|
||||
|
||||
/// Describe a single server-side job by id. `None` when the server has no
|
||||
/// such job.
|
||||
pub async fn get_job(&self, job_id: impl AsRef<str>) -> Result<Option<JobDescription>> {
|
||||
self.internal.get_job(job_id.as_ref()).await
|
||||
}
|
||||
|
||||
/// Request cancellation of a server-side job by id. Returns true if the
|
||||
/// server accepted the cancellation, false if no such job exists.
|
||||
pub async fn cancel_job(&self, job_id: impl AsRef<str>) -> Result<bool> {
|
||||
self.internal.cancel_job(job_id.as_ref()).await
|
||||
}
|
||||
|
||||
/// The lifecycle event history of a server-side job (all jobs when
|
||||
/// `job_id` is `None`), as recorded Arrow batches.
|
||||
pub async fn job_history(&self, job_id: Option<&str>) -> Result<Vec<RecordBatch>> {
|
||||
self.internal.job_history(job_id).await
|
||||
}
|
||||
|
||||
/// Drop a table in the database.
|
||||
///
|
||||
/// # Arguments
|
||||
|
||||
@@ -18,6 +18,8 @@ use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use arrow_array::RecordBatch;
|
||||
|
||||
use lance::dataset::ReadParams;
|
||||
use lance_namespace::LanceNamespace;
|
||||
use lance_namespace::models::{
|
||||
@@ -204,8 +206,8 @@ pub enum ReadConsistency {
|
||||
/// compaction, column refresh, ...).
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct JobInfo {
|
||||
/// The job id -- what [`Database::open_job`] and
|
||||
/// [`Database::cancel_job`] accept.
|
||||
/// The job id -- what [`Database::get_job`] and [`Database::cancel_job`]
|
||||
/// accept.
|
||||
pub job_id: String,
|
||||
/// The table the job runs against, without URI or namespace.
|
||||
pub table: String,
|
||||
@@ -216,8 +218,8 @@ pub struct JobInfo {
|
||||
pub created_at_millis: i64,
|
||||
}
|
||||
|
||||
/// The server-side record behind a [`crate::job::Job`] handle: lifecycle
|
||||
/// state plus the job-type-specific specification and result.
|
||||
/// A described job from [`Database::get_job`]: lifecycle state plus the
|
||||
/// job-type-specific specification.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct JobDescription {
|
||||
pub job_id: String,
|
||||
@@ -228,10 +230,6 @@ pub struct JobDescription {
|
||||
pub creation_ms: i64,
|
||||
/// The job-type-specific specification. Null when the server omits it.
|
||||
pub spec: serde_json::Value,
|
||||
/// The job-type-specific terminal result, for job types that define one.
|
||||
/// `None` until the job succeeds, so a job that never terminates reports
|
||||
/// its progress through [`crate::job::Job::events`] instead.
|
||||
pub result: Option<serde_json::Value>,
|
||||
/// Why the job failed, when the job is failed and the server reports a
|
||||
/// reason.
|
||||
pub failure: Option<crate::error::JobFailure>,
|
||||
@@ -317,22 +315,31 @@ pub trait Database:
|
||||
async fn drop_function(&self, _name: &str, _version: &str) -> Result<bool> {
|
||||
function_catalog_not_supported()
|
||||
}
|
||||
/// Open a job by id, returning a handle with its record already
|
||||
/// populated. Fails with [`crate::Error::JobNotFound`] when the server has
|
||||
/// no such job.
|
||||
async fn open_job(&self, _job_id: &str) -> Result<crate::job::Job> {
|
||||
job_op_not_supported("open_job")
|
||||
/// A [`crate::job::Job`] handle for a server-side job by id, suitable for
|
||||
/// waiting on or cancelling the job. The handle is constructed without a
|
||||
/// server round trip; an unknown id surfaces when the handle is used.
|
||||
fn job(&self, _job_id: &str) -> Result<crate::job::Job> {
|
||||
job_op_not_supported("job")
|
||||
}
|
||||
/// List server-side jobs across the database's tables.
|
||||
async fn list_jobs(&self) -> Result<Vec<JobInfo>> {
|
||||
job_op_not_supported("list_jobs")
|
||||
}
|
||||
/// Describe a single job by id. `None` when the server has no such job.
|
||||
async fn get_job(&self, _job_id: &str) -> Result<Option<JobDescription>> {
|
||||
job_op_not_supported("get_job")
|
||||
}
|
||||
/// Request cancellation of a job by id. Returns true if the server
|
||||
/// accepted the cancellation, false if no such job exists. Cancelling an
|
||||
/// already-terminal job is a no-op success.
|
||||
async fn cancel_job(&self, _job_id: &str) -> Result<bool> {
|
||||
job_op_not_supported("cancel_job")
|
||||
}
|
||||
/// The lifecycle event history of a job (all jobs when `job_id` is
|
||||
/// `None`), as recorded Arrow batches.
|
||||
async fn job_history(&self, _job_id: Option<&str>) -> Result<Vec<RecordBatch>> {
|
||||
job_op_not_supported("job_history")
|
||||
}
|
||||
/// Start executing a SQL statement on a remote database.
|
||||
async fn execute_query_async(
|
||||
&self,
|
||||
|
||||
@@ -3,7 +3,6 @@
|
||||
|
||||
//! Namespace-based database implementation that delegates table management to lance-namespace
|
||||
|
||||
use lance_datafusion::utils::StreamingWriteSource;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
@@ -305,10 +304,6 @@ impl Database for LanceNamespaceDatabase {
|
||||
}
|
||||
|
||||
async fn create_table(&self, request: DbCreateTableRequest) -> Result<Arc<dyn BaseTable>> {
|
||||
// Refuse a bad declaration before the namespace records a table.
|
||||
crate::table::computed_columns::ensure_declarations_are_planned(
|
||||
&request.data.arrow_schema(),
|
||||
)?;
|
||||
let mut table_id = request.namespace_path.clone();
|
||||
table_id.push(request.name.clone());
|
||||
let mut existing_table = None;
|
||||
|
||||
@@ -102,8 +102,6 @@ pub enum Error {
|
||||
},
|
||||
#[snafu(display("Job{} was cancelled", job_id.as_ref().map(|id| format!(" {id}")).unwrap_or_default()))]
|
||||
JobCancelled { job_id: Option<String> },
|
||||
#[snafu(display("Job '{job_id}' was not found"))]
|
||||
JobNotFound { job_id: String },
|
||||
|
||||
// 3rd party / external errors
|
||||
#[snafu(display("object_store error: {source}"))]
|
||||
|
||||
+12
-247
@@ -3,53 +3,16 @@
|
||||
|
||||
//! Handles to operations a server may run asynchronously.
|
||||
|
||||
use std::sync::{Arc, RwLock, RwLockReadGuard, RwLockWriteGuard};
|
||||
use std::sync::Arc;
|
||||
|
||||
use arrow_array::RecordBatch;
|
||||
use async_trait::async_trait;
|
||||
use serde::{Serialize, de::DeserializeOwned};
|
||||
use serde_json::Value;
|
||||
use tokio::sync::watch;
|
||||
use tokio::task::{AbortHandle, JoinHandle};
|
||||
|
||||
use crate::database::JobDescription;
|
||||
use crate::error::{Error, JobFailure, Result};
|
||||
|
||||
/// Which of a job's events [`Job::events`] returns.
|
||||
///
|
||||
/// The handle already knows which job to ask about, so this narrows the
|
||||
/// query rather than naming one.
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct JobEventsRequest {
|
||||
/// Maximum event rows to return. The server applies its own default
|
||||
/// (1000 rows) and maximum (10,000 rows) when this is `None`, and
|
||||
/// truncates without saying so, which matters for a job with one event
|
||||
/// per fragment.
|
||||
pub limit: Option<u32>,
|
||||
/// SQL-like filter over the event columns `state`, `updated_by`,
|
||||
/// `emitted_from`, `emitted_by`, and `claim_entity`. For example
|
||||
/// `state = 'claim_complete'` selects only per-claim completions.
|
||||
pub filter: Option<String>,
|
||||
}
|
||||
|
||||
impl JobEventsRequest {
|
||||
pub fn limit(mut self, limit: u32) -> Self {
|
||||
self.limit = Some(limit);
|
||||
self
|
||||
}
|
||||
|
||||
pub fn filter(mut self, filter: impl Into<String>) -> Self {
|
||||
self.filter = Some(filter.into());
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
fn job_detail_not_supported<T>(what: &str) -> Result<T> {
|
||||
Err(Error::NotSupported {
|
||||
message: format!("{what} is only available for server-side jobs"),
|
||||
})
|
||||
}
|
||||
|
||||
/// Backend-specific tracking for an asynchronous operation.
|
||||
#[async_trait]
|
||||
pub(crate) trait JobHandle: Send + Sync {
|
||||
@@ -60,15 +23,6 @@ pub(crate) trait JobHandle: Send + Sync {
|
||||
async fn status(&self) -> Result<String>;
|
||||
async fn wait(&self) -> Result<TerminalResult>;
|
||||
async fn cancel(&self) -> Result<()>;
|
||||
/// The job's full server-side record. Backends that run the operation in
|
||||
/// this process have none and keep the default.
|
||||
async fn describe(&self) -> Result<JobDescription> {
|
||||
job_detail_not_supported("describing a job")
|
||||
}
|
||||
/// The job's recorded lifecycle events.
|
||||
async fn events(&self, _request: JobEventsRequest) -> Result<Vec<RecordBatch>> {
|
||||
job_detail_not_supported("job event history")
|
||||
}
|
||||
}
|
||||
|
||||
/// A backend-neutral successful terminal result.
|
||||
@@ -131,34 +85,16 @@ enum JobInner<T> {
|
||||
Completed(T),
|
||||
}
|
||||
|
||||
/// What a handle last learned about its job. `state` is separate because an
|
||||
/// in-process job can report one but has no server-side record behind it.
|
||||
#[derive(Default)]
|
||||
struct JobCache {
|
||||
state: Option<String>,
|
||||
description: Option<JobDescription>,
|
||||
}
|
||||
|
||||
/// A handle to an operation that may still be running.
|
||||
///
|
||||
/// The operation may already be complete when the handle is created. `T` is
|
||||
/// the endpoint's successful terminal result; unit-result operations use the
|
||||
/// default `Job<()>`.
|
||||
///
|
||||
/// The detail accessors ([`Job::state`], [`Job::job_type`], ...) read what the
|
||||
/// handle last observed. Submitting an operation returns only a job id, so
|
||||
/// populating them eagerly would cost an extra round trip on every call:
|
||||
///
|
||||
/// - [`Job::refresh`] and [`Job::status`] fetch the whole record.
|
||||
/// - [`Job::wait`] records the terminal state it establishes, but not the rest
|
||||
/// of the record; call [`Job::refresh`] for that.
|
||||
/// - Everything is `None` until one of those runs.
|
||||
pub struct Job<T = ()>
|
||||
where
|
||||
T: Clone + Send + Sync + 'static,
|
||||
{
|
||||
inner: JobInner<T>,
|
||||
cache: RwLock<JobCache>,
|
||||
}
|
||||
|
||||
impl<T> std::fmt::Debug for Job<T>
|
||||
@@ -166,40 +102,18 @@ where
|
||||
T: Clone + Send + Sync + 'static,
|
||||
{
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
let cache = self.cache_read();
|
||||
let mut out = f.debug_struct("Job");
|
||||
out.field("id", &self.id())
|
||||
.field("done", &matches!(self.inner, JobInner::Completed(_)));
|
||||
if let Some(state) = &cache.state {
|
||||
out.field("state", state);
|
||||
}
|
||||
if let Some(description) = &cache.description {
|
||||
out.field("job_type", &description.job_type)
|
||||
.field("creation_ms", &description.creation_ms);
|
||||
if !description.spec.is_null() {
|
||||
out.field("spec", &description.spec);
|
||||
}
|
||||
if let Some(result) = &description.result {
|
||||
out.field("result", result);
|
||||
}
|
||||
if let Some(failure) = &description.failure {
|
||||
out.field("failure", failure);
|
||||
}
|
||||
}
|
||||
out.finish()
|
||||
f.debug_struct("Job")
|
||||
.field("id", &self.id())
|
||||
.field("done", &matches!(self.inner, JobInner::Completed(_)))
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
impl Job<()> {
|
||||
/// A job whose operation finished before the handle was created. Its
|
||||
/// state is known without asking anyone, so the cache starts populated.
|
||||
/// A job whose operation finished before the handle was created.
|
||||
pub(crate) fn new_done() -> Self {
|
||||
Self {
|
||||
inner: JobInner::Completed(()),
|
||||
cache: RwLock::new(JobCache {
|
||||
state: Some("finished".to_string()),
|
||||
description: None,
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -209,21 +123,8 @@ impl Job<()> {
|
||||
handle,
|
||||
decode: Arc::new(|_| Ok(())),
|
||||
},
|
||||
cache: RwLock::default(),
|
||||
}
|
||||
}
|
||||
|
||||
/// A handle whose record the caller has already fetched, so the detail
|
||||
/// accessors answer without a second round trip.
|
||||
pub(crate) fn opened(handle: Box<dyn JobHandle>, description: JobDescription) -> Self {
|
||||
let job = Self::new(handle);
|
||||
{
|
||||
let mut cache = job.cache_write();
|
||||
cache.state = Some(description.state.clone());
|
||||
cache.description = Some(description);
|
||||
}
|
||||
job
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> Job<T>
|
||||
@@ -237,7 +138,6 @@ where
|
||||
handle,
|
||||
decode: Arc::new(TerminalResult::decode::<T>),
|
||||
},
|
||||
cache: RwLock::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -269,124 +169,16 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
fn cache_read(&self) -> RwLockReadGuard<'_, JobCache> {
|
||||
self.cache.read().unwrap_or_else(|err| err.into_inner())
|
||||
}
|
||||
|
||||
fn cache_write(&self) -> RwLockWriteGuard<'_, JobCache> {
|
||||
self.cache.write().unwrap_or_else(|err| err.into_inner())
|
||||
}
|
||||
|
||||
/// Asks the backend for this job's current state, and for a server-side
|
||||
/// job its full record, then caches the answer for the detail accessors.
|
||||
///
|
||||
/// In-process operations have no server-side record, so only
|
||||
/// [`Job::state`] is populated for them.
|
||||
pub async fn refresh(&self) -> Result<()> {
|
||||
self.refresh_state().await.map(|_| ())
|
||||
}
|
||||
|
||||
/// Refreshes and reports the state, which every backend can answer.
|
||||
async fn refresh_state(&self) -> Result<String> {
|
||||
let JobInner::Handle { handle, .. } = &self.inner else {
|
||||
let state = "finished".to_string();
|
||||
self.cache_write().state = Some(state.clone());
|
||||
return Ok(state);
|
||||
};
|
||||
match handle.describe().await {
|
||||
Ok(description) => {
|
||||
let state = description.state.clone();
|
||||
let mut cache = self.cache_write();
|
||||
cache.state = Some(state.clone());
|
||||
cache.description = Some(description);
|
||||
Ok(state)
|
||||
}
|
||||
// An in-process job knows its own state and nothing more.
|
||||
Err(Error::NotSupported { .. }) => {
|
||||
let state = handle.status().await?;
|
||||
self.cache_write().state = Some(state.clone());
|
||||
Ok(state)
|
||||
}
|
||||
Err(err) => Err(err),
|
||||
}
|
||||
}
|
||||
|
||||
/// The operation's current lifecycle state: "running", "finished",
|
||||
/// "failed", or "cancelled".
|
||||
///
|
||||
/// A point snapshot; unlike [`Job::wait`] it does not block, raise on a
|
||||
/// terminal failure state, or retry. States a newer server reports that
|
||||
/// this client version does not know pass through as-is. Also refreshes
|
||||
/// the detail accessors.
|
||||
/// this client version does not know pass through as-is.
|
||||
pub async fn status(&self) -> Result<String> {
|
||||
self.refresh_state().await
|
||||
}
|
||||
|
||||
/// The last lifecycle state this handle observed, without contacting the
|
||||
/// backend. `None` until the handle has.
|
||||
pub fn state(&self) -> Option<String> {
|
||||
self.cache_read().state.clone()
|
||||
}
|
||||
|
||||
/// The whole server-side record this handle last observed. The accessors
|
||||
/// below read individual fields out of it. `None` for an in-process job,
|
||||
/// which has no such record.
|
||||
pub fn description(&self) -> Option<JobDescription> {
|
||||
self.cache_read().description.clone()
|
||||
}
|
||||
|
||||
/// The job's type, as the server names it. `None` for an in-process job.
|
||||
pub fn job_type(&self) -> Option<String> {
|
||||
self.with_description(|description| description.job_type.clone())
|
||||
}
|
||||
|
||||
/// When the job was created, in milliseconds since the epoch. `None` for
|
||||
/// an in-process job.
|
||||
pub fn creation_ms(&self) -> Option<i64> {
|
||||
self.with_description(|description| description.creation_ms)
|
||||
}
|
||||
|
||||
/// The job-type-specific specification it was submitted with.
|
||||
pub fn spec(&self) -> Option<Value> {
|
||||
self.with_description(|description| description.spec.clone())
|
||||
.filter(|spec| !spec.is_null())
|
||||
}
|
||||
|
||||
/// The job-type-specific terminal result, as reported data rather than the
|
||||
/// typed model [`Job::wait`] returns. `None` until the job succeeds.
|
||||
pub fn result(&self) -> Option<Value> {
|
||||
self.with_description(|description| description.result.clone())
|
||||
.flatten()
|
||||
}
|
||||
|
||||
/// Why the job failed, when it failed and the server reports a reason.
|
||||
pub fn failure(&self) -> Option<JobFailure> {
|
||||
self.with_description(|description| description.failure.clone())
|
||||
.flatten()
|
||||
}
|
||||
|
||||
fn with_description<R>(&self, read: impl FnOnce(&JobDescription) -> R) -> Option<R> {
|
||||
self.cache_read().description.as_ref().map(read)
|
||||
}
|
||||
|
||||
/// This job's recorded lifecycle events.
|
||||
///
|
||||
/// Unlike the detail accessors, which report a terminal result only once
|
||||
/// the job reaches one, events are written as the job runs and outlive the
|
||||
/// workers that produced them. A distributed job records a
|
||||
/// `claim`/`claim_complete` pair per unit of work, each carrying
|
||||
/// `rows_processed`, so a job that never finishes still accounts for what
|
||||
/// it did. In-process operations keep no event history.
|
||||
pub async fn events(&self, request: JobEventsRequest) -> Result<Vec<RecordBatch>> {
|
||||
match &self.inner {
|
||||
JobInner::Handle { handle, .. } => handle.events(request).await,
|
||||
// The operation finished before the handle existed, so there is no
|
||||
// id to query with even when a server ran it.
|
||||
JobInner::Completed(_) => Err(Error::NotSupported {
|
||||
message: "this operation completed before its handle was created, so it \
|
||||
carries no job id to query events with"
|
||||
.to_string(),
|
||||
}),
|
||||
JobInner::Handle { handle, .. } => handle.status().await,
|
||||
JobInner::Completed(_) => Ok("finished".to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -398,19 +190,8 @@ where
|
||||
/// [`crate::Error::JobCancelled`] if it was cancelled.
|
||||
pub async fn wait(&self) -> Result<T> {
|
||||
match &self.inner {
|
||||
JobInner::Handle { handle, decode } => {
|
||||
let settled = handle.wait().await;
|
||||
// Waiting already established a terminal state; record it so
|
||||
// the detail accessors do not need another round trip for it.
|
||||
if let Some(state) = terminal_state(&settled) {
|
||||
self.cache_write().state = Some(state.to_string());
|
||||
}
|
||||
(decode)(settled?)
|
||||
}
|
||||
JobInner::Completed(result) => {
|
||||
self.cache_write().state = Some("finished".to_string());
|
||||
Ok(result.clone())
|
||||
}
|
||||
JobInner::Handle { handle, decode } => (decode)(handle.wait().await?),
|
||||
JobInner::Completed(result) => Ok(result.clone()),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -443,36 +224,20 @@ where
|
||||
U: Clone + Send + Sync + 'static,
|
||||
F: Fn(T) -> U + Send + Sync + 'static,
|
||||
{
|
||||
// The mapped handle tracks the same job, so it inherits what this one
|
||||
// has already learned about it.
|
||||
let Self { inner, cache } = self;
|
||||
match inner {
|
||||
match self.inner {
|
||||
JobInner::Handle { handle, decode } => Job {
|
||||
inner: JobInner::Handle {
|
||||
handle,
|
||||
decode: Arc::new(move |result| Ok(map((decode)(result)?))),
|
||||
},
|
||||
cache,
|
||||
},
|
||||
JobInner::Completed(result) => Job {
|
||||
inner: JobInner::Completed(map(result)),
|
||||
cache,
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The lifecycle state a settled [`JobHandle::wait`] implies.
|
||||
fn terminal_state(settled: &Result<TerminalResult>) -> Option<&'static str> {
|
||||
match settled {
|
||||
Ok(_) => Some("finished"),
|
||||
Err(Error::JobFailed { .. }) => Some("failed"),
|
||||
Err(Error::JobCancelled { .. }) => Some("cancelled"),
|
||||
// Anything else is a transport failure, not a verdict on the job.
|
||||
Err(_) => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// How an in-process operation ended. Cloneable so every waiter can be given
|
||||
/// the outcome; [`Error`] is not, so failures share one behind an [`Arc`].
|
||||
#[derive(Clone)]
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -24,8 +24,8 @@ use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use arrow_array::cast::AsArray;
|
||||
use arrow_array::types::UInt64Type;
|
||||
use arrow_array::{RecordBatch, UInt64Array, new_null_array};
|
||||
use arrow_schema::{FieldRef, Schema as ArrowSchema, SchemaRef};
|
||||
use arrow_array::{RecordBatch, UInt64Array};
|
||||
use arrow_schema::{Schema as ArrowSchema, SchemaRef};
|
||||
use datafusion::common::ScalarValue;
|
||||
use datafusion::error::DataFusionError;
|
||||
use datafusion::physical_plan::SendableRecordBatchStream;
|
||||
@@ -34,7 +34,7 @@ use datafusion::prelude::{col, lit};
|
||||
use futures::{StreamExt, TryStreamExt};
|
||||
use lance::Dataset;
|
||||
use lance::dataset::mem_wal::DatasetMemWalExt;
|
||||
use lance::dataset::transaction::{Operation, Transaction, UpdateMode};
|
||||
use lance::dataset::transaction::{Operation, Transaction};
|
||||
use lance::dataset::write::delete::DeleteBuilder;
|
||||
use lance::dataset::write::merge_insert::inserted_rows::{
|
||||
KeyExistenceFilter, KeyExistenceFilterBuilder, KeyValue,
|
||||
@@ -51,9 +51,6 @@ use super::{
|
||||
definition_to_metadata,
|
||||
};
|
||||
use crate::database::OpenTableRequest;
|
||||
use crate::table::computed_columns::{
|
||||
computed_column_from_field, computed_columns, ensure_declarations_are_planned,
|
||||
};
|
||||
use crate::table::{NativeTable, NativeTableExt, Table};
|
||||
use crate::{Error, Result};
|
||||
|
||||
@@ -170,52 +167,30 @@ pub(crate) async fn execute_refresh(
|
||||
.map(|p| (p.output.clone(), p.expression.clone()))
|
||||
.collect();
|
||||
validate_inputs(&source_ds, definition)?;
|
||||
let (replanned, planned_fields, _renames) = super::plan(
|
||||
let (replanned, mut planned_fields, _renames) = super::plan(
|
||||
source_schema,
|
||||
&definition.source_table,
|
||||
&definition.source_namespace,
|
||||
Some(&projections),
|
||||
&projections,
|
||||
definition.filter.as_deref(),
|
||||
definition.limit,
|
||||
)?;
|
||||
let mut planned_fields = planned_fields;
|
||||
planned_fields.push(arrow_schema::Field::new(
|
||||
SOURCE_ROW_ID_COLUMN,
|
||||
arrow_schema::DataType::UInt64,
|
||||
false,
|
||||
));
|
||||
// A computed column is not planned from the source: refresh writes it
|
||||
// NULL and its declaration's owner fills it. Its declaration must still
|
||||
// be complete, and it must be able to hold NULL.
|
||||
let physical = ArrowSchema::from(view_ds.schema());
|
||||
let mut computed = computed_columns(&physical).into_iter().map(|c| c.name);
|
||||
if let Some(name) = computed.by_ref().find(|name| {
|
||||
physical
|
||||
.field_with_name(name)
|
||||
.is_ok_and(|f| !f.is_nullable())
|
||||
}) {
|
||||
return Err(Error::Schema {
|
||||
message: format!(
|
||||
"computed column '{name}' of view '{}' cannot hold NULL; recreate the view",
|
||||
view.name()
|
||||
),
|
||||
});
|
||||
}
|
||||
ensure_declarations_are_planned(&physical)?;
|
||||
let physical_planned: Vec<&FieldRef> = physical
|
||||
let planned_shape: Vec<_> = planned_fields
|
||||
.iter()
|
||||
.map(|f| (f.name().clone(), f.data_type().clone(), f.is_nullable()))
|
||||
.collect();
|
||||
let physical_shape: Vec<_> = physical
|
||||
.fields()
|
||||
.iter()
|
||||
.filter(|f| computed_column_from_field(f).is_none())
|
||||
.map(|f| (f.name().clone(), f.data_type().clone(), f.is_nullable()))
|
||||
.collect();
|
||||
// A projected column that became nullable at the source still fits the
|
||||
// view's nullable field; the reverse would not.
|
||||
let matches = planned_fields.len() == physical_planned.len()
|
||||
&& planned_fields.iter().zip(&physical_planned).all(|(e, p)| {
|
||||
e.name() == p.name()
|
||||
&& e.data_type() == p.data_type()
|
||||
&& (p.is_nullable() || !e.is_nullable())
|
||||
});
|
||||
if !matches {
|
||||
if planned_shape != physical_shape {
|
||||
return Err(Error::Schema {
|
||||
message: format!(
|
||||
"the stored definition of view '{}' does not produce this \
|
||||
@@ -254,18 +229,11 @@ pub(crate) async fn execute_refresh(
|
||||
.get(SOURCE_VERSION_TS_META_KEY)
|
||||
.and_then(|raw| raw.parse().ok());
|
||||
// The watermark speaks only for the view state its refresh left behind;
|
||||
// any other commit on the view since then is drift, except a fill of its
|
||||
// computed columns, which rewrites nothing refresh certifies.
|
||||
let recorded_view_version = metadata
|
||||
// any other commit on the view since then is drift.
|
||||
let view_intact = metadata
|
||||
.get(VIEW_VERSION_META_KEY)
|
||||
.and_then(|raw| raw.parse::<u64>().ok());
|
||||
let view_intact = match recorded_view_version {
|
||||
Some(recorded) if recorded == view_ds.version().version => true,
|
||||
Some(recorded) if recorded < view_ds.version().version => {
|
||||
only_computed_rewrites_since(&view_ds, recorded).await?
|
||||
}
|
||||
_ => false,
|
||||
};
|
||||
.and_then(|raw| raw.parse::<u64>().ok())
|
||||
== Some(view_ds.version().version);
|
||||
|
||||
if !full && watermark == Some(source_version) && view_intact && recorded_ts == Some(source_ts) {
|
||||
return Ok(RefreshMaterializedViewResult {
|
||||
@@ -1122,69 +1090,6 @@ struct RowScope {
|
||||
limit: Option<u64>,
|
||||
}
|
||||
|
||||
/// Whether every commit on the view after `recorded` is a fill of its
|
||||
/// computed columns: a column rewrite or data replacement touching only
|
||||
/// those fields and neither adding nor removing rows. A version whose
|
||||
/// transaction cannot be read is not proven, so it counts as drift.
|
||||
async fn only_computed_rewrites_since(view_ds: &Dataset, recorded: u64) -> Result<bool> {
|
||||
// A fill may write any field under a computed column, so the whole
|
||||
// subtree counts, not only the root.
|
||||
let physical = ArrowSchema::from(view_ds.schema());
|
||||
fn subtree(field: &lance_core::datatypes::Field, ids: &mut Vec<u32>) {
|
||||
ids.push(field.id as u32);
|
||||
for child in &field.children {
|
||||
subtree(child, ids);
|
||||
}
|
||||
}
|
||||
let mut computed_fields = Vec::new();
|
||||
for column in computed_columns(&physical) {
|
||||
if let Some(field) = view_ds.schema().field(&column.name) {
|
||||
subtree(field, &mut computed_fields);
|
||||
}
|
||||
}
|
||||
if computed_fields.is_empty() {
|
||||
return Ok(false);
|
||||
}
|
||||
for version in recorded + 1..=view_ds.version().version {
|
||||
let Some(transaction) = view_ds.read_transaction_by_version(version).await? else {
|
||||
return Ok(false);
|
||||
};
|
||||
let fill = match &transaction.operation {
|
||||
Operation::Update {
|
||||
removed_fragment_ids,
|
||||
new_fragments,
|
||||
fields_modified,
|
||||
update_mode: Some(UpdateMode::RewriteColumns),
|
||||
..
|
||||
} => {
|
||||
removed_fragment_ids.is_empty()
|
||||
&& new_fragments.is_empty()
|
||||
&& !fields_modified.is_empty()
|
||||
&& fields_modified
|
||||
.iter()
|
||||
.all(|field| computed_fields.contains(field))
|
||||
}
|
||||
// What `refresh_column` commits for a SQL declaration.
|
||||
Operation::DataReplacement { replacements } => {
|
||||
!replacements.is_empty()
|
||||
&& replacements.iter().all(|group| {
|
||||
!group.1.fields.is_empty()
|
||||
&& group
|
||||
.1
|
||||
.fields
|
||||
.iter()
|
||||
.all(|field| computed_fields.contains(&(*field as u32)))
|
||||
})
|
||||
}
|
||||
_ => false,
|
||||
};
|
||||
if !fill {
|
||||
return Ok(false);
|
||||
}
|
||||
}
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
async fn compute_stream(
|
||||
source: &Dataset,
|
||||
definition: &MaterializedViewDefinition,
|
||||
@@ -1253,10 +1158,6 @@ async fn compute_stream(
|
||||
let batch = batch.map_err(|e| DataFusionError::External(Box::new(e)))?;
|
||||
let mut columns = Vec::with_capacity(out_schema.fields().len());
|
||||
for field in out_schema.fields() {
|
||||
if computed_column_from_field(field).is_some() {
|
||||
columns.push(new_null_array(field.data_type(), batch.num_rows()));
|
||||
continue;
|
||||
}
|
||||
let name = if field.name() == SOURCE_ROW_ID_COLUMN {
|
||||
ROW_ID
|
||||
} else {
|
||||
@@ -2867,7 +2768,7 @@ mod tests {
|
||||
let (conn, source) = db_with_source(vec![1]).await;
|
||||
let prepared = crate::materialized_view::prepare_declaration(
|
||||
&source,
|
||||
Some(&[("x".into(), "x".into()), ("twice".into(), "x * 2".into())]),
|
||||
&[("x".into(), "x".into()), ("twice".into(), "x * 2".into())],
|
||||
None,
|
||||
None,
|
||||
)
|
||||
@@ -3231,424 +3132,4 @@ mod tests {
|
||||
let err = view.refresh().execute().await.unwrap_err();
|
||||
assert!(err.to_string().contains("source table 'src'"), "{err}");
|
||||
}
|
||||
|
||||
/// A view with a computed column, declared over `people` and refreshed.
|
||||
async fn refreshed_computed_view(conn: &Connection) -> MaterializedView {
|
||||
use crate::materialized_view::tests::{computed_field, people, test_binding};
|
||||
let source = people(conn).await;
|
||||
let view = crate::materialized_view::prepare_declaration(
|
||||
&source,
|
||||
Some(&[
|
||||
("id".to_string(), "id".to_string()),
|
||||
("name".to_string(), "name".to_string()),
|
||||
]),
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.with_computed_columns(
|
||||
vec![(2, computed_field("emb", "fb_1", "name"))],
|
||||
&[test_binding("fb_1", "name", "emb")],
|
||||
)
|
||||
.unwrap()
|
||||
.create("v")
|
||||
.await
|
||||
.unwrap();
|
||||
let result = view.refresh().execute().await.unwrap();
|
||||
assert_eq!(result.mode, RefreshMode::Rebuild);
|
||||
view
|
||||
}
|
||||
|
||||
async fn unfilled(view: &MaterializedView) -> usize {
|
||||
view.table()
|
||||
.count_rows(Some("emb IS NULL".to_string()))
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
async fn append_people(conn: &Connection, ids: Vec<i32>, names: Vec<&str>) {
|
||||
let batch = record_batch!(("id", Int32, ids), ("name", Utf8, names)).unwrap();
|
||||
conn.open_table("people")
|
||||
.execute()
|
||||
.await
|
||||
.unwrap()
|
||||
.add(batch)
|
||||
.execute()
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
/// Commit the fill job's shape on the view: a column rewrite of
|
||||
/// `fields`, touching no rows. The data is left as it is; what matters
|
||||
/// here is how the next refresh classifies the commit.
|
||||
async fn commit_column_rewrite(view: &MaterializedView, fields: &[&str]) {
|
||||
let native = view.table().as_native().unwrap();
|
||||
native.dataset.reload().await.unwrap();
|
||||
let dataset = native.dataset.get().await.unwrap().as_ref().clone();
|
||||
let fields_modified = fields
|
||||
.iter()
|
||||
.map(|name| dataset.schema().field(name).unwrap().id as u32)
|
||||
.collect();
|
||||
let updated_fragments = dataset
|
||||
.get_fragments()
|
||||
.iter()
|
||||
.map(|fragment| fragment.metadata().clone())
|
||||
.collect();
|
||||
let operation = Operation::Update {
|
||||
removed_fragment_ids: Vec::new(),
|
||||
updated_fragments,
|
||||
new_fragments: Vec::new(),
|
||||
fields_modified,
|
||||
compacted_sstables: Vec::new(),
|
||||
fields_for_preserving_frag_bitmap: Vec::new(),
|
||||
update_mode: Some(UpdateMode::RewriteColumns),
|
||||
inserted_rows_filter: None,
|
||||
updated_fragment_offsets: None,
|
||||
};
|
||||
let read_version = dataset.version().version;
|
||||
CommitBuilder::new(WriteDestination::Dataset(Arc::new(dataset)))
|
||||
.execute(Transaction::new(read_version, operation, None))
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
/// Refresh never computes a computed column: every row it writes, on a
|
||||
/// rebuild, an append and a rewrite, carries NULL there, and the
|
||||
/// declaration survives all three.
|
||||
#[tokio::test]
|
||||
async fn test_computed_columns_are_written_null_and_kept() {
|
||||
let conn = connect("memory://").execute().await.unwrap();
|
||||
let view = refreshed_computed_view(&conn).await;
|
||||
assert_eq!(unfilled(&view).await, 3);
|
||||
|
||||
append_people(&conn, vec![4, 5], vec!["d", "e"]).await;
|
||||
let result = view.refresh().execute().await.unwrap();
|
||||
assert_eq!(result.mode, RefreshMode::Incremental);
|
||||
assert_eq!(unfilled(&view).await, 5);
|
||||
|
||||
conn.open_table("people")
|
||||
.execute()
|
||||
.await
|
||||
.unwrap()
|
||||
.update()
|
||||
.column("name", "'z'")
|
||||
.only_if("id = 1")
|
||||
.execute()
|
||||
.await
|
||||
.unwrap();
|
||||
view.refresh().execute().await.unwrap();
|
||||
assert_eq!(unfilled(&view).await, 5);
|
||||
assert_eq!(read(view.table(), "id").await, vec![1, 2, 3, 4, 5]);
|
||||
|
||||
let schema = view.table().schema().await.unwrap();
|
||||
assert!(
|
||||
crate::table::computed_columns::function_bindings(&schema)
|
||||
.unwrap()
|
||||
.iter()
|
||||
.any(|b| b.binding_id() == "fb_1"),
|
||||
"the binding envelope was lost"
|
||||
);
|
||||
assert!(
|
||||
computed_column_from_field(schema.field_with_name("emb").unwrap()).is_some(),
|
||||
"the declaration was lost"
|
||||
);
|
||||
assert_eq!(
|
||||
view.refresh().execute().await.unwrap().mode,
|
||||
RefreshMode::NoOp
|
||||
);
|
||||
}
|
||||
|
||||
/// The fill job's commit rewrites only computed columns. It is the one
|
||||
/// commit on a view that is not drift: the next refresh carries on from
|
||||
/// its watermark instead of rebuilding, which would null what the fill
|
||||
/// just wrote.
|
||||
#[tokio::test]
|
||||
async fn test_a_computed_column_fill_is_not_drift() {
|
||||
let conn = connect("memory://").execute().await.unwrap();
|
||||
let view = refreshed_computed_view(&conn).await;
|
||||
|
||||
commit_column_rewrite(&view, &["emb"]).await;
|
||||
assert_eq!(
|
||||
view.refresh().execute().await.unwrap().mode,
|
||||
RefreshMode::NoOp
|
||||
);
|
||||
|
||||
commit_column_rewrite(&view, &["emb"]).await;
|
||||
append_people(&conn, vec![4], vec!["d"]).await;
|
||||
let result = view.refresh().execute().await.unwrap();
|
||||
assert_eq!(result.mode, RefreshMode::Incremental);
|
||||
assert_eq!(result.rows_written, 1);
|
||||
assert_eq!(read(view.table(), "id").await, vec![1, 2, 3, 4]);
|
||||
}
|
||||
|
||||
/// A column rewrite that reaches a projected column is drift like any
|
||||
/// other write: refresh certifies those columns and must recompute them.
|
||||
#[tokio::test]
|
||||
async fn test_a_rewrite_of_a_projected_column_is_drift() {
|
||||
let conn = connect("memory://").execute().await.unwrap();
|
||||
let view = refreshed_computed_view(&conn).await;
|
||||
|
||||
commit_column_rewrite(&view, &["emb", "name"]).await;
|
||||
assert_eq!(
|
||||
view.refresh().execute().await.unwrap().mode,
|
||||
RefreshMode::Rebuild
|
||||
);
|
||||
}
|
||||
|
||||
/// The declaration contract is checked before any refresh mutation: a
|
||||
/// missing binding envelope and a column that lost its declaration both
|
||||
/// fail closed.
|
||||
#[tokio::test]
|
||||
async fn test_a_broken_declaration_is_refused_before_refresh() {
|
||||
let conn = connect("memory://").execute().await.unwrap();
|
||||
let view = refreshed_computed_view(&conn).await;
|
||||
let native = view.table().as_native().unwrap();
|
||||
let mut dataset = native.dataset.get().await.unwrap().as_ref().clone();
|
||||
dataset
|
||||
.update_schema_metadata(vec![(
|
||||
crate::table::computed_columns::FUNCTION_BINDINGS_META_KEY.to_string(),
|
||||
None,
|
||||
)])
|
||||
.await
|
||||
.unwrap();
|
||||
let err = view.refresh().execute().await.unwrap_err().to_string();
|
||||
assert!(err.contains("references missing binding 'fb_1'"), "{err}");
|
||||
|
||||
let conn = connect("memory://").execute().await.unwrap();
|
||||
let view = refreshed_computed_view(&conn).await;
|
||||
let native = view.table().as_native().unwrap();
|
||||
let mut dataset = native.dataset.get().await.unwrap().as_ref().clone();
|
||||
dataset
|
||||
.replace_field_metadata(vec![(
|
||||
dataset.schema().field("emb").unwrap().id as u32,
|
||||
HashMap::new(),
|
||||
)])
|
||||
.await
|
||||
.unwrap();
|
||||
let err = view.refresh().execute().await.unwrap_err().to_string();
|
||||
assert!(err.contains("does not match binding 'fb_1'"), "{err}");
|
||||
}
|
||||
|
||||
/// An input the view does not project is materialized on every refresh
|
||||
/// path, before the provenance column, with the source's values.
|
||||
#[tokio::test]
|
||||
async fn test_internal_inputs_are_materialized_and_refreshed() {
|
||||
use crate::materialized_view::tests::{computed_field, strict_people, test_binding};
|
||||
let conn = connect("memory://").execute().await.unwrap();
|
||||
let source = strict_people(&conn).await;
|
||||
let mut prepared = crate::materialized_view::prepare_declaration(
|
||||
&source,
|
||||
Some(&[("id".to_string(), "id".to_string())]),
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let input = prepared.input_column("name").unwrap();
|
||||
let view = prepared
|
||||
.with_computed_columns(
|
||||
vec![(1, computed_field("emb", "fb_1", &input))],
|
||||
&[test_binding("fb_1", &input, "emb")],
|
||||
)
|
||||
.unwrap()
|
||||
.create("v")
|
||||
.await
|
||||
.unwrap();
|
||||
let names: Vec<String> = view
|
||||
.table()
|
||||
.schema()
|
||||
.await
|
||||
.unwrap()
|
||||
.fields()
|
||||
.iter()
|
||||
.map(|f| f.name().clone())
|
||||
.collect();
|
||||
assert_eq!(names, ["id", "emb", "__input_name", SOURCE_ROW_ID_COLUMN]);
|
||||
|
||||
let unfilled_inputs = || async {
|
||||
view.table()
|
||||
.count_rows(Some("__input_name IS NULL".to_string()))
|
||||
.await
|
||||
.unwrap()
|
||||
};
|
||||
assert_eq!(
|
||||
view.refresh().execute().await.unwrap().mode,
|
||||
RefreshMode::Rebuild
|
||||
);
|
||||
assert_eq!(view.table().count_rows(None).await.unwrap(), 3);
|
||||
assert_eq!(unfilled_inputs().await, 0);
|
||||
|
||||
let more = arrow_array::RecordBatch::try_new(
|
||||
source.schema().await.unwrap(),
|
||||
vec![
|
||||
Arc::new(Int32Array::from(vec![4])),
|
||||
Arc::new(arrow_array::StringArray::from(vec!["d"])),
|
||||
],
|
||||
)
|
||||
.unwrap();
|
||||
source.add(more).execute().await.unwrap();
|
||||
assert_eq!(
|
||||
view.refresh().execute().await.unwrap().mode,
|
||||
RefreshMode::Incremental
|
||||
);
|
||||
assert_eq!(unfilled_inputs().await, 0);
|
||||
assert_eq!(
|
||||
view.table()
|
||||
.count_rows(Some("__input_name = 'd'".to_string()))
|
||||
.await
|
||||
.unwrap(),
|
||||
1
|
||||
);
|
||||
|
||||
source
|
||||
.update()
|
||||
.column("name", "'z'")
|
||||
.only_if("id = 1")
|
||||
.execute()
|
||||
.await
|
||||
.unwrap();
|
||||
view.refresh().execute().await.unwrap();
|
||||
assert_eq!(
|
||||
view.table()
|
||||
.count_rows(Some("__input_name = 'z'".to_string()))
|
||||
.await
|
||||
.unwrap(),
|
||||
1
|
||||
);
|
||||
assert_eq!(
|
||||
unfilled(&view).await,
|
||||
4,
|
||||
"rewritten and new rows are unfilled"
|
||||
);
|
||||
}
|
||||
|
||||
/// A SQL declaration is filled by `refresh_column` on the view, which
|
||||
/// commits a data replacement; the next refresh continues from its
|
||||
/// watermark and keeps what the fill wrote, and only rows the view added
|
||||
/// since come back unfilled.
|
||||
#[tokio::test]
|
||||
async fn test_a_sql_fill_is_not_drift() {
|
||||
use crate::materialized_view::tests::{people, sql_field};
|
||||
let conn = connect("memory://").execute().await.unwrap();
|
||||
let source = people(&conn).await;
|
||||
let view = crate::materialized_view::prepare_declaration(
|
||||
&source,
|
||||
Some(&[("id".to_string(), "id".to_string())]),
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.with_computed_columns(
|
||||
vec![(
|
||||
1,
|
||||
sql_field("next", arrow_schema::DataType::Int32, "id + 1", r#"["id"]"#),
|
||||
)],
|
||||
&[],
|
||||
)
|
||||
.unwrap()
|
||||
.create("v")
|
||||
.await
|
||||
.unwrap();
|
||||
let filled = || async {
|
||||
view.table()
|
||||
.count_rows(Some("next = id + 1".to_string()))
|
||||
.await
|
||||
.unwrap()
|
||||
};
|
||||
assert_eq!(
|
||||
view.refresh().execute().await.unwrap().mode,
|
||||
RefreshMode::Rebuild
|
||||
);
|
||||
assert_eq!(
|
||||
view.table()
|
||||
.refresh_column("next")
|
||||
.await
|
||||
.unwrap()
|
||||
.rows_filled,
|
||||
3
|
||||
);
|
||||
assert_eq!(filled().await, 3);
|
||||
assert_eq!(
|
||||
view.refresh().execute().await.unwrap().mode,
|
||||
RefreshMode::NoOp
|
||||
);
|
||||
assert_eq!(filled().await, 3);
|
||||
|
||||
append_people(&conn, vec![4], vec!["d"]).await;
|
||||
assert_eq!(
|
||||
view.refresh().execute().await.unwrap().mode,
|
||||
RefreshMode::Incremental
|
||||
);
|
||||
assert_eq!(filled().await, 3);
|
||||
assert_eq!(
|
||||
view.table()
|
||||
.refresh_column("next")
|
||||
.await
|
||||
.unwrap()
|
||||
.rows_filled,
|
||||
1
|
||||
);
|
||||
assert_eq!(filled().await, 4);
|
||||
assert_eq!(
|
||||
view.refresh().execute().await.unwrap().mode,
|
||||
RefreshMode::NoOp
|
||||
);
|
||||
}
|
||||
|
||||
/// A fill of a nested computed column writes its child fields; that is
|
||||
/// still a fill, not drift.
|
||||
#[tokio::test]
|
||||
async fn test_a_nested_sql_fill_is_not_drift() {
|
||||
use crate::materialized_view::tests::{people, sql_field};
|
||||
let conn = connect("memory://").execute().await.unwrap();
|
||||
let source = people(&conn).await;
|
||||
let payload = sql_field(
|
||||
"payload",
|
||||
arrow_schema::DataType::Struct(
|
||||
vec![arrow_schema::Field::new(
|
||||
"value",
|
||||
arrow_schema::DataType::Utf8,
|
||||
true,
|
||||
)]
|
||||
.into(),
|
||||
),
|
||||
"named_struct('value', name)",
|
||||
r#"["name"]"#,
|
||||
);
|
||||
let view = crate::materialized_view::prepare_declaration(
|
||||
&source,
|
||||
Some(&[("name".to_string(), "name".to_string())]),
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.with_computed_columns(vec![(1, payload)], &[])
|
||||
.unwrap()
|
||||
.create("v")
|
||||
.await
|
||||
.unwrap();
|
||||
view.refresh().execute().await.unwrap();
|
||||
assert_eq!(
|
||||
view.table()
|
||||
.refresh_column("payload")
|
||||
.await
|
||||
.unwrap()
|
||||
.rows_filled,
|
||||
3
|
||||
);
|
||||
assert_eq!(
|
||||
view.refresh().execute().await.unwrap().mode,
|
||||
RefreshMode::NoOp
|
||||
);
|
||||
assert_eq!(
|
||||
view.table()
|
||||
.count_rows(Some("payload.value = name".to_string()))
|
||||
.await
|
||||
.unwrap(),
|
||||
3
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
+90
-128
@@ -20,13 +20,13 @@ use lance_namespace::models::{
|
||||
|
||||
use crate::Error;
|
||||
use crate::database::{
|
||||
CloneTableRequest, CreateTableMode, CreateTableRequest, Database, DatabaseOptions, JobInfo,
|
||||
OpenTableRequest, ReadConsistency, TableNamesRequest,
|
||||
CloneTableRequest, CreateTableMode, CreateTableRequest, Database, DatabaseOptions,
|
||||
JobDescription, JobInfo, OpenTableRequest, ReadConsistency, TableNamesRequest,
|
||||
};
|
||||
use crate::error::Result;
|
||||
use crate::function::{FunctionRegistrationRequest, FunctionVersion};
|
||||
use crate::job::Job;
|
||||
use crate::remote::job::{RemoteJob, job_state_to_client};
|
||||
use crate::remote::job::{DescribeJobResponse, RemoteJob, job_state_to_client};
|
||||
use crate::remote::util::stream_as_body;
|
||||
use crate::table::BaseTable;
|
||||
|
||||
@@ -671,18 +671,11 @@ impl<S: HttpSend> Database for RemoteDatabase<S> {
|
||||
Ok(response.dropped)
|
||||
}
|
||||
|
||||
async fn open_job(&self, job_id: &str) -> Result<Job> {
|
||||
let handle = super::job::RemoteJob::new(self.client.clone(), job_id.to_string());
|
||||
match crate::job::JobHandle::describe(&handle).await {
|
||||
Ok(description) => Ok(Job::opened(Box::new(handle), description)),
|
||||
Err(Error::Http {
|
||||
status_code: Some(StatusCode::NOT_FOUND),
|
||||
..
|
||||
}) => Err(Error::JobNotFound {
|
||||
job_id: job_id.to_string(),
|
||||
}),
|
||||
Err(err) => Err(err),
|
||||
}
|
||||
fn job(&self, job_id: &str) -> Result<crate::job::Job> {
|
||||
Ok(crate::job::Job::new(Box::new(super::job::RemoteJob::new(
|
||||
self.client.clone(),
|
||||
job_id.to_string(),
|
||||
))))
|
||||
}
|
||||
|
||||
async fn list_jobs(&self) -> Result<Vec<JobInfo>> {
|
||||
@@ -719,6 +712,31 @@ impl<S: HttpSend> Database for RemoteDatabase<S> {
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
async fn get_job(&self, job_id: &str) -> Result<Option<JobDescription>> {
|
||||
let req = self
|
||||
.client
|
||||
.post("/v1/jobs/describe")
|
||||
.json(&serde_json::json!({ "job_id": job_id }));
|
||||
let (request_id, rsp) = self.client.send(req).await?;
|
||||
let rsp = match self.client.check_response(&request_id, rsp).await {
|
||||
Ok(rsp) => rsp,
|
||||
Err(Error::Http {
|
||||
status_code: Some(StatusCode::NOT_FOUND),
|
||||
..
|
||||
}) => return Ok(None),
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
let body: DescribeJobResponse = rsp.json().await.err_to_http(request_id)?;
|
||||
Ok(Some(JobDescription {
|
||||
job_id: body.job_id,
|
||||
job_type: body.job_type,
|
||||
state: job_state_to_client(&body.job_state),
|
||||
creation_ms: body.creation_ms,
|
||||
spec: body.spec,
|
||||
failure: body.failure.map(|reported| reported.into_job_failure()),
|
||||
}))
|
||||
}
|
||||
|
||||
async fn cancel_job(&self, job_id: &str) -> Result<bool> {
|
||||
let req = self
|
||||
.client
|
||||
@@ -735,6 +753,21 @@ impl<S: HttpSend> Database for RemoteDatabase<S> {
|
||||
}
|
||||
}
|
||||
|
||||
async fn job_history(&self, job_id: Option<&str>) -> Result<Vec<arrow_array::RecordBatch>> {
|
||||
let mut body = serde_json::json!({});
|
||||
if let Some(job_id) = job_id {
|
||||
body["job_id"] = serde_json::Value::String(job_id.to_string());
|
||||
}
|
||||
let req = self.client.post("/v1/jobs/query_events").json(&body);
|
||||
let (request_id, rsp) = self.client.send(req).await?;
|
||||
let rsp = self.client.check_response(&request_id, rsp).await?;
|
||||
let bytes = rsp.bytes().await.err_to_http(request_id)?;
|
||||
let reader = arrow_ipc::reader::StreamReader::try_new(std::io::Cursor::new(bytes), None)?;
|
||||
reader
|
||||
.collect::<std::result::Result<Vec<_>, _>>()
|
||||
.map_err(Into::into)
|
||||
}
|
||||
|
||||
async fn execute_query_async(
|
||||
&self,
|
||||
query: &str,
|
||||
@@ -1274,7 +1307,6 @@ mod tests {
|
||||
use crate::{
|
||||
Connection, Error,
|
||||
database::CreateTableMode,
|
||||
job::JobEventsRequest,
|
||||
remote::{ARROW_STREAM_CONTENT_TYPE, ClientConfig, HeaderProvider, JSON_CONTENT_TYPE},
|
||||
};
|
||||
|
||||
@@ -2623,7 +2655,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_open_job() {
|
||||
async fn test_get_job() {
|
||||
let conn = Connection::new_with_handler(|request| {
|
||||
assert_eq!(request.method(), &reqwest::Method::POST);
|
||||
assert_eq!(request.url().path(), "/v1/jobs/describe");
|
||||
@@ -2637,55 +2669,51 @@ mod tests {
|
||||
)
|
||||
.unwrap()
|
||||
});
|
||||
// Opening populates the handle, so the accessors answer without a
|
||||
// second round trip.
|
||||
let job = conn.open_job("job-1").await.unwrap();
|
||||
assert_eq!(job.id(), Some("job-1"));
|
||||
assert_eq!(job.job_type().as_deref(), Some("create_index"));
|
||||
assert_eq!(job.state().as_deref(), Some("failed"));
|
||||
assert_eq!(job.creation_ms(), Some(1000));
|
||||
assert_eq!(job.spec().unwrap()["column"], "vec");
|
||||
assert!(job.result().is_none());
|
||||
let failure = job.failure().unwrap();
|
||||
let job = conn.get_job("job-1").await.unwrap().unwrap();
|
||||
assert_eq!(job.job_id, "job-1");
|
||||
assert_eq!(job.job_type, "create_index");
|
||||
assert_eq!(job.state, "failed");
|
||||
assert_eq!(job.creation_ms, 1000);
|
||||
assert_eq!(job.spec["column"], "vec");
|
||||
let failure = job.failure.unwrap();
|
||||
assert_eq!(failure.phase.as_deref(), Some("execute"));
|
||||
assert_eq!(failure.message.as_deref(), Some("worker died"));
|
||||
assert_eq!(failure.retryable, Some(true));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_open_job_reports_the_terminal_result() {
|
||||
let conn = Connection::new_with_handler(|_| {
|
||||
http::Response::builder()
|
||||
.status(200)
|
||||
.body(
|
||||
r#"{"job_id": "job-1", "job_type": "refresh_column", "job_state": "DONE", "creation_ms": 1000, "result": {"rows_assigned": 1000000, "rows_failed": 0}}"#,
|
||||
)
|
||||
.unwrap()
|
||||
});
|
||||
let job = conn.open_job("job-1").await.unwrap();
|
||||
assert_eq!(job.state().as_deref(), Some("finished"));
|
||||
let result = job.result().unwrap();
|
||||
assert_eq!(result["rows_assigned"], 1_000_000);
|
||||
assert_eq!(result["rows_failed"], 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_open_job_missing_fails() {
|
||||
async fn test_get_job_missing_is_none() {
|
||||
let conn = Connection::new_with_handler(|_| {
|
||||
http::Response::builder()
|
||||
.status(404)
|
||||
.body("no such job")
|
||||
.unwrap()
|
||||
});
|
||||
let err = conn.open_job("nope").await.unwrap_err();
|
||||
assert!(
|
||||
matches!(&err, Error::JobNotFound { job_id } if job_id == "nope"),
|
||||
"{err:?}"
|
||||
);
|
||||
assert!(conn.get_job("nope").await.unwrap().is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_job_events_scope_to_that_job() {
|
||||
async fn test_cancel_job() {
|
||||
let conn = Connection::new_with_handler(|request| {
|
||||
assert_eq!(request.url().path(), "/v1/jobs/cancel");
|
||||
http::Response::builder()
|
||||
.status(200)
|
||||
.body(r#"{"job_id": "job-1"}"#)
|
||||
.unwrap()
|
||||
});
|
||||
assert!(conn.cancel_job("job-1").await.unwrap());
|
||||
|
||||
let conn = Connection::new_with_handler(|_| {
|
||||
http::Response::builder()
|
||||
.status(404)
|
||||
.body("no such job")
|
||||
.unwrap()
|
||||
});
|
||||
assert!(!conn.cancel_job("nope").await.unwrap());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_job_history_parses_arrow_stream() {
|
||||
let schema = Arc::new(Schema::new(vec![Field::new(
|
||||
"state",
|
||||
DataType::Utf8,
|
||||
@@ -2694,91 +2722,29 @@ mod tests {
|
||||
let batch = RecordBatch::try_new(
|
||||
schema.clone(),
|
||||
vec![Arc::new(arrow_array::StringArray::from(vec![
|
||||
"claim_complete",
|
||||
"created", "done",
|
||||
]))],
|
||||
)
|
||||
.unwrap();
|
||||
let mut events = Vec::new();
|
||||
let mut body = Vec::new();
|
||||
{
|
||||
let mut writer =
|
||||
arrow_ipc::writer::StreamWriter::try_new(&mut events, &schema).unwrap();
|
||||
let mut writer = arrow_ipc::writer::StreamWriter::try_new(&mut body, &schema).unwrap();
|
||||
writer.write(&batch).unwrap();
|
||||
writer.finish().unwrap();
|
||||
}
|
||||
let conn = Connection::new_with_handler(move |request| {
|
||||
let body: serde_json::Value =
|
||||
serde_json::from_slice(request.body().unwrap().as_bytes().unwrap()).unwrap();
|
||||
if request.url().path() == "/v1/jobs/describe" {
|
||||
return http::Response::builder()
|
||||
.status(200)
|
||||
.body(
|
||||
r#"{"job_id": "job-1", "job_type": "refresh_column", "job_state": "IN_PROGRESS", "creation_ms": 1}"#
|
||||
.as_bytes()
|
||||
.to_vec(),
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
assert_eq!(request.url().path(), "/v1/jobs/query_events");
|
||||
// The handle supplies job_id; the caller only narrows the query.
|
||||
assert_eq!(body["job_id"], "job-1");
|
||||
assert_eq!(body["limit"], 500);
|
||||
assert_eq!(body["filter"], "state = 'claim_complete'");
|
||||
http::Response::builder()
|
||||
.status(200)
|
||||
.body(events.clone())
|
||||
.unwrap()
|
||||
});
|
||||
let job = conn.open_job("job-1").await.unwrap();
|
||||
let batches = job
|
||||
.events(
|
||||
JobEventsRequest::default()
|
||||
.limit(500)
|
||||
.filter("state = 'claim_complete'"),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(batches.len(), 1);
|
||||
assert_eq!(batches[0].num_rows(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_job_events_keep_the_schema_when_nothing_matches() {
|
||||
let schema = Arc::new(Schema::new(vec![Field::new(
|
||||
"state",
|
||||
DataType::Utf8,
|
||||
false,
|
||||
)]));
|
||||
let mut events = Vec::new();
|
||||
{
|
||||
let mut writer =
|
||||
arrow_ipc::writer::StreamWriter::try_new(&mut events, &schema).unwrap();
|
||||
writer.finish().unwrap();
|
||||
}
|
||||
let conn = Connection::new_with_handler(move |request| {
|
||||
if request.url().path() == "/v1/jobs/describe" {
|
||||
return http::Response::builder()
|
||||
.status(200)
|
||||
.body(
|
||||
r#"{"job_id": "job-1", "job_type": "refresh_column", "job_state": "IN_PROGRESS", "creation_ms": 1}"#
|
||||
.as_bytes()
|
||||
.to_vec(),
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
let body: serde_json::Value =
|
||||
let req_body: serde_json::Value =
|
||||
serde_json::from_slice(request.body().unwrap().as_bytes().unwrap()).unwrap();
|
||||
// Only the job id when the caller narrows nothing.
|
||||
assert_eq!(body, serde_json::json!({ "job_id": "job-1" }));
|
||||
assert_eq!(req_body["job_id"], "job-1");
|
||||
http::Response::builder()
|
||||
.status(200)
|
||||
.body(events.clone())
|
||||
.body(body.clone())
|
||||
.unwrap()
|
||||
});
|
||||
let job = conn.open_job("job-1").await.unwrap();
|
||||
let batches = job.events(JobEventsRequest::default()).await.unwrap();
|
||||
let batches = conn.job_history(Some("job-1")).await.unwrap();
|
||||
assert_eq!(batches.len(), 1);
|
||||
assert_eq!(batches[0].num_rows(), 0);
|
||||
assert_eq!(batches[0].schema(), schema);
|
||||
assert_eq!(batches[0].num_rows(), 2);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -2973,9 +2939,7 @@ mod tests {
|
||||
let polls_ref = polls.clone();
|
||||
let conn = Connection::new_with_handler(move |request| {
|
||||
assert_eq!(request.url().path(), "/v1/jobs/describe");
|
||||
// Two in-progress answers: one for the load, one for the first
|
||||
// status poll.
|
||||
let state = if polls_ref.fetch_add(1, Ordering::SeqCst) < 2 {
|
||||
let state = if polls_ref.fetch_add(1, Ordering::SeqCst) == 0 {
|
||||
"IN_PROGRESS"
|
||||
} else {
|
||||
"DONE"
|
||||
@@ -2988,13 +2952,11 @@ mod tests {
|
||||
))
|
||||
.unwrap()
|
||||
});
|
||||
let job = conn.open_job("job-1").await.unwrap();
|
||||
let job = conn.job("job-1").unwrap();
|
||||
assert_eq!(job.id(), Some("job-1"));
|
||||
// Opening already answered the state; no extra call needed for it.
|
||||
assert_eq!(job.state().as_deref(), Some("running"));
|
||||
assert_eq!(job.status().await.unwrap(), "running");
|
||||
job.wait().await.unwrap();
|
||||
assert_eq!(job.status().await.unwrap(), "finished");
|
||||
assert!(polls.load(Ordering::SeqCst) >= 4);
|
||||
assert!(polls.load(Ordering::SeqCst) >= 3);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,15 +5,13 @@
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
use arrow_array::RecordBatch;
|
||||
use async_trait::async_trait;
|
||||
use tokio::time::sleep;
|
||||
|
||||
use serde::Deserialize;
|
||||
|
||||
use crate::database::JobDescription;
|
||||
use crate::error::{Error, JobFailure, Result};
|
||||
use crate::job::{JobEventsRequest, JobHandle, TerminalResult};
|
||||
use crate::job::{JobHandle, TerminalResult};
|
||||
use crate::remote::client::{HttpSend, RequestResultExt, RestfulLanceDbClient};
|
||||
|
||||
/// Delay before the second job-state poll; doubles up to [`MAX_POLL_INTERVAL`].
|
||||
@@ -88,7 +86,7 @@ pub(super) struct DescribeJobResponse {
|
||||
#[serde(default)]
|
||||
pub(super) spec: serde_json::Value,
|
||||
#[serde(default)]
|
||||
pub(super) result: Option<serde_json::Value>,
|
||||
result: Option<serde_json::Value>,
|
||||
#[serde(default)]
|
||||
pub(super) failure: Option<ReportedFailure>,
|
||||
}
|
||||
@@ -112,39 +110,6 @@ impl DescribeJobResponse {
|
||||
fn into_terminal_result(self, request_id: String) -> TerminalResult {
|
||||
TerminalResult::remote(self.result, request_id)
|
||||
}
|
||||
|
||||
/// The public description this wire envelope stands for.
|
||||
pub(super) fn into_description(self) -> JobDescription {
|
||||
JobDescription {
|
||||
job_id: self.job_id,
|
||||
job_type: self.job_type,
|
||||
state: JobState::from(self.job_state.as_str()).client_label(),
|
||||
creation_ms: self.creation_ms,
|
||||
spec: self.spec,
|
||||
result: self.result,
|
||||
failure: self.failure.map(ReportedFailure::into_job_failure),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// One `/v1/jobs/query_events` round trip.
|
||||
pub(super) async fn fetch_job_events<S: HttpSend>(
|
||||
client: &RestfulLanceDbClient<S>,
|
||||
body: serde_json::Value,
|
||||
) -> Result<Vec<RecordBatch>> {
|
||||
let request = client.post("/v1/jobs/query_events").json(&body);
|
||||
let (request_id, response) = client.send(request).await?;
|
||||
let response = client.check_response(&request_id, response).await?;
|
||||
let bytes = response.bytes().await.err_to_http(request_id)?;
|
||||
let reader = arrow_ipc::reader::StreamReader::try_new(std::io::Cursor::new(bytes), None)?;
|
||||
let schema = reader.schema();
|
||||
let mut batches = reader.collect::<std::result::Result<Vec<_>, _>>()?;
|
||||
// A query that matched nothing still describes the event columns.
|
||||
// Keep that schema so callers can build a typed empty result.
|
||||
if batches.is_empty() {
|
||||
batches.push(RecordBatch::new_empty(schema));
|
||||
}
|
||||
Ok(batches)
|
||||
}
|
||||
|
||||
pub struct RemoteJob<S: HttpSend> {
|
||||
@@ -158,7 +123,7 @@ impl<S: HttpSend> RemoteJob<S> {
|
||||
}
|
||||
|
||||
/// One `/v1/jobs/describe` round trip.
|
||||
async fn fetch_description(&self) -> Result<(String, DescribeJobResponse)> {
|
||||
async fn describe(&self) -> Result<(String, DescribeJobResponse)> {
|
||||
let request = self
|
||||
.client
|
||||
.post("/v1/jobs/describe")
|
||||
@@ -183,28 +148,13 @@ impl<S: HttpSend> JobHandle for RemoteJob<S> {
|
||||
}
|
||||
|
||||
async fn status(&self) -> Result<String> {
|
||||
Ok(self.fetch_description().await?.1.state().client_label())
|
||||
}
|
||||
|
||||
async fn describe(&self) -> Result<JobDescription> {
|
||||
Ok(self.fetch_description().await?.1.into_description())
|
||||
}
|
||||
|
||||
async fn events(&self, request: JobEventsRequest) -> Result<Vec<RecordBatch>> {
|
||||
let mut body = serde_json::json!({ "job_id": self.job_id });
|
||||
if let Some(limit) = request.limit {
|
||||
body["limit"] = serde_json::Value::from(limit);
|
||||
}
|
||||
if let Some(filter) = request.filter {
|
||||
body["filter"] = serde_json::Value::String(filter);
|
||||
}
|
||||
fetch_job_events(&self.client, body).await
|
||||
Ok(self.describe().await?.1.state().client_label())
|
||||
}
|
||||
|
||||
async fn wait(&self) -> Result<TerminalResult> {
|
||||
let mut interval = INITIAL_POLL_INTERVAL;
|
||||
loop {
|
||||
let (request_id, description) = self.fetch_description().await?;
|
||||
let (request_id, description) = self.describe().await?;
|
||||
match description.state() {
|
||||
JobState::Done => return Ok(description.into_terminal_result(request_id)),
|
||||
JobState::Failed => {
|
||||
|
||||
@@ -264,17 +264,6 @@ impl<S: HttpSend> crate::job::JobHandle for FreshnessJob<S> {
|
||||
crate::job::JobHandle::status(&self.inner).await
|
||||
}
|
||||
|
||||
async fn describe(&self) -> Result<crate::database::JobDescription> {
|
||||
crate::job::JobHandle::describe(&self.inner).await
|
||||
}
|
||||
|
||||
async fn events(
|
||||
&self,
|
||||
request: crate::job::JobEventsRequest,
|
||||
) -> Result<Vec<arrow_array::RecordBatch>> {
|
||||
crate::job::JobHandle::events(&self.inner, request).await
|
||||
}
|
||||
|
||||
async fn wait(&self) -> Result<crate::job::TerminalResult> {
|
||||
let result = crate::job::JobHandle::wait(&self.inner).await?;
|
||||
let version = self.version.read().await;
|
||||
@@ -7993,72 +7982,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
/// The refresh handle is wrapped for read-freshness tracking, so it has to
|
||||
/// forward the detail APIs too -- this is the job an operator is holding
|
||||
/// when a backfill goes quiet.
|
||||
#[tokio::test]
|
||||
async fn test_refresh_job_handle_reports_detail_and_events() {
|
||||
let schema = Arc::new(Schema::new(vec![Field::new(
|
||||
"state",
|
||||
DataType::Utf8,
|
||||
false,
|
||||
)]));
|
||||
let batch = RecordBatch::try_new(
|
||||
schema.clone(),
|
||||
vec![Arc::new(arrow_array::StringArray::from(vec![
|
||||
"claim_complete",
|
||||
]))],
|
||||
)
|
||||
.unwrap();
|
||||
let mut events = Vec::new();
|
||||
{
|
||||
let mut writer =
|
||||
arrow_ipc::writer::StreamWriter::try_new(&mut events, &schema).unwrap();
|
||||
writer.write(&batch).unwrap();
|
||||
writer.finish().unwrap();
|
||||
}
|
||||
let table = Table::new_with_handler("my_table", move |request| {
|
||||
match request.url().path() {
|
||||
"/v1/table/my_table/backfill_column" => http::Response::builder()
|
||||
.status(202)
|
||||
.body(br#"{"job_id": "j-42"}"#.to_vec())
|
||||
.unwrap(),
|
||||
"/v1/jobs/describe" => http::Response::builder()
|
||||
.status(200)
|
||||
.body(
|
||||
r#"{"job_id": "j-42", "job_type": "refresh_column", "job_state": "IN_PROGRESS", "creation_ms": 7, "spec": {"column": "doubled"}}"#
|
||||
.as_bytes()
|
||||
.to_vec(),
|
||||
)
|
||||
.unwrap(),
|
||||
"/v1/jobs/query_events" => {
|
||||
let body: serde_json::Value =
|
||||
serde_json::from_slice(request.body().unwrap().as_bytes().unwrap())
|
||||
.unwrap();
|
||||
assert_eq!(body["job_id"], "j-42");
|
||||
http::Response::builder()
|
||||
.status(200)
|
||||
.body(events.clone())
|
||||
.unwrap()
|
||||
}
|
||||
other => panic!("unexpected path {other}"),
|
||||
}
|
||||
});
|
||||
|
||||
let job = table.refresh_column_async("doubled").await.unwrap();
|
||||
job.refresh().await.unwrap();
|
||||
assert_eq!(job.state().as_deref(), Some("running"));
|
||||
assert_eq!(job.job_type().as_deref(), Some("refresh_column"));
|
||||
assert_eq!(job.creation_ms(), Some(7));
|
||||
assert_eq!(job.spec().unwrap()["column"], "doubled");
|
||||
|
||||
let batches = job
|
||||
.events(crate::job::JobEventsRequest::default())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(batches.iter().map(|b| b.num_rows()).sum::<usize>(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_refresh_submission_uses_add_columns_version_fence() {
|
||||
let table = Table::new_with_handler("my_table", |request| match request.url().path() {
|
||||
|
||||
@@ -2804,7 +2804,7 @@ impl NativeTable {
|
||||
namespace_client: Option<Arc<dyn LanceNamespace>>,
|
||||
pushdown_operations: HashSet<NamespaceClientPushdownOperation>,
|
||||
) -> Result<Self> {
|
||||
let batches = computed_columns::admit_create_source(batches)?;
|
||||
computed_columns::ensure_no_foreign_declarations(batches.arrow_schema().fields())?;
|
||||
// Default params uses format v1.
|
||||
let params = params.unwrap_or(WriteParams {
|
||||
..Default::default()
|
||||
@@ -2904,7 +2904,6 @@ impl NativeTable {
|
||||
pushdown_operations: HashSet<NamespaceClientPushdownOperation>,
|
||||
session: Option<Arc<lance::session::Session>>,
|
||||
) -> Result<Self> {
|
||||
let batches = computed_columns::admit_create_source(batches)?;
|
||||
// Build table_id from namespace + name for the storage options provider
|
||||
let mut table_id = namespace.clone();
|
||||
table_id.push(name.to_string());
|
||||
@@ -5678,7 +5677,7 @@ mod tests {
|
||||
TableStatistics {
|
||||
num_rows: 250,
|
||||
num_indices: 0,
|
||||
total_bytes: 8969,
|
||||
total_bytes: 8925,
|
||||
fragment_stats: FragmentStatistics {
|
||||
num_fragments: 11,
|
||||
num_small_fragments: 11,
|
||||
|
||||
@@ -21,7 +21,6 @@
|
||||
//! [`computed_columns`] and [`computed_column_from_field`] read declarations
|
||||
//! back off a schema.
|
||||
|
||||
use futures::StreamExt;
|
||||
use std::collections::{BTreeSet, HashMap, HashSet};
|
||||
use std::sync::Arc;
|
||||
|
||||
@@ -760,33 +759,14 @@ fn function_output_field(name: &str, nullable: bool, raw: &str) -> Result<JsonAr
|
||||
Ok(field)
|
||||
}
|
||||
|
||||
/// Whether two fields describe the same Function output.
|
||||
///
|
||||
/// `compare_identity` covers the field's own name and nullability. Struct
|
||||
/// children carry both as part of the declaration and compare with it on. List
|
||||
/// children do not: Lance rewrites a list item's name and nullability when it
|
||||
/// writes, so a stored `fixed_size_list<item: float not null>` comes back as
|
||||
/// `fixed_size_list<item: float>` and never matches the declaration again.
|
||||
/// Comparing those by type alone keeps this agreeing with the server, which
|
||||
/// draws the same distinction and is what accepted the column when it was
|
||||
/// declared.
|
||||
fn function_output_field_matches(
|
||||
expected: &ArrowField,
|
||||
actual: &ArrowField,
|
||||
compare_identity: bool,
|
||||
) -> bool {
|
||||
if compare_identity
|
||||
&& (expected.name() != actual.name() || expected.is_nullable() != actual.is_nullable())
|
||||
{
|
||||
return false;
|
||||
}
|
||||
match (expected.is_blob_v2(), actual.is_blob_v2()) {
|
||||
(false, false) => function_output_type_matches(expected.data_type(), actual.data_type()),
|
||||
(true, true) => {
|
||||
fn function_output_field_matches(expected: &ArrowField, actual: &ArrowField) -> bool {
|
||||
expected.name() == actual.name()
|
||||
&& expected.is_nullable() == actual.is_nullable()
|
||||
&& if expected.is_blob_v2() {
|
||||
has_supported_blob_v2_layout(expected) && has_supported_blob_v2_layout(actual)
|
||||
} else {
|
||||
function_output_type_matches(expected.data_type(), actual.data_type())
|
||||
}
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
fn function_output_type_matches(expected: &DataType, actual: &DataType) -> bool {
|
||||
@@ -799,19 +779,33 @@ fn function_output_type_matches(expected: &DataType, actual: &DataType) -> bool
|
||||
&& expected
|
||||
.iter()
|
||||
.zip(actual)
|
||||
.all(|(expected, actual)| function_output_field_matches(expected, actual, true))
|
||||
.all(|(expected, actual)| function_output_field_matches(expected, actual))
|
||||
}
|
||||
(DataType::List(expected), DataType::List(actual))
|
||||
| (DataType::LargeList(expected), DataType::LargeList(actual)) => {
|
||||
function_output_field_matches(expected, actual, false)
|
||||
function_output_field_matches(expected, actual)
|
||||
}
|
||||
(
|
||||
DataType::FixedSizeList(expected, expected_size),
|
||||
DataType::FixedSizeList(actual, actual_size),
|
||||
) => expected_size == actual_size && function_output_field_matches(expected, actual, false),
|
||||
) => expected_size == actual_size && function_output_field_matches(expected, actual),
|
||||
(DataType::Map(expected, expected_sorted), DataType::Map(actual, actual_sorted)) => {
|
||||
expected_sorted == actual_sorted
|
||||
&& function_output_field_matches(expected, actual, true)
|
||||
expected_sorted == actual_sorted && function_output_field_matches(expected, actual)
|
||||
}
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
fn function_output_type_has_blob(data_type: &DataType) -> bool {
|
||||
match data_type {
|
||||
DataType::Struct(fields) => fields
|
||||
.iter()
|
||||
.any(|field| field.is_blob_v2() || function_output_type_has_blob(field.data_type())),
|
||||
DataType::List(field)
|
||||
| DataType::LargeList(field)
|
||||
| DataType::FixedSizeList(field, _)
|
||||
| DataType::Map(field, _) => {
|
||||
field.is_blob_v2() || function_output_type_has_blob(field.data_type())
|
||||
}
|
||||
_ => false,
|
||||
}
|
||||
@@ -897,13 +891,16 @@ fn ensure_binding_matches_schema(schema: &ArrowSchema, binding: &FunctionBinding
|
||||
binding.binding_id()
|
||||
)));
|
||||
}
|
||||
let type_matches = if output.arrow_type == FUNCTION_BLOB_V2_TYPE {
|
||||
has_supported_blob_v2_layout(field)
|
||||
let (type_matches, has_semantic_blob) = if output.arrow_type == FUNCTION_BLOB_V2_TYPE {
|
||||
(has_supported_blob_v2_layout(field), true)
|
||||
} else {
|
||||
let expected_type = parse_output_arrow_type(&output.arrow_type)?;
|
||||
let expected_type = lance_namespace::schema::convert_json_arrow_type(&expected_type)
|
||||
.map_err(|e| invalid_function(format!("invalid Function output type: {e}")))?;
|
||||
function_output_type_matches(&expected_type, field.data_type())
|
||||
(
|
||||
function_output_type_matches(&expected_type, field.data_type()),
|
||||
function_output_type_has_blob(&expected_type),
|
||||
)
|
||||
};
|
||||
if !type_matches {
|
||||
return Err(invalid_function(format!(
|
||||
@@ -934,16 +931,19 @@ fn ensure_binding_matches_schema(schema: &ArrowSchema, binding: &FunctionBinding
|
||||
binding.binding_id()
|
||||
)));
|
||||
}
|
||||
// Rebuild from the declaration rather than from the stored field. The
|
||||
// stored field carries Lance's write-time normalization, which would
|
||||
// never round-trip back to the schema the binding recorded -- the same
|
||||
// reason list children compare by type above. Whether the column on
|
||||
// disk still matches is settled by that comparison, not here.
|
||||
output_fields.push(function_output_field(
|
||||
field.name(),
|
||||
true,
|
||||
&output.arrow_type,
|
||||
)?);
|
||||
if has_semantic_blob {
|
||||
output_fields.push(function_output_field(
|
||||
field.name(),
|
||||
true,
|
||||
&output.arrow_type,
|
||||
)?);
|
||||
} else {
|
||||
let json = lance_namespace::schema::arrow_schema_to_json(&ArrowSchema::new(vec![
|
||||
ArrowField::new(field.name().clone(), field.data_type().clone(), true),
|
||||
]))
|
||||
.map_err(|e| invalid_function(format!("invalid Function output schema: {e}")))?;
|
||||
output_fields.push(json.fields.into_iter().next().unwrap());
|
||||
}
|
||||
}
|
||||
if let Some(assignment) = binding.assignment() {
|
||||
if binding
|
||||
@@ -1339,106 +1339,6 @@ pub(crate) fn ensure_batch_writes_no_computed_values(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Validate every computed-column declaration `schema` carries against the
|
||||
/// schema itself: every field with declaration metadata is a complete
|
||||
/// declaration, a SQL declaration re-plans to the field it declares, a
|
||||
/// Function declaration satisfies the binding contract, and no declaration
|
||||
/// reads another computed column. What passes here is what `refresh_column`
|
||||
/// can execute.
|
||||
pub(crate) fn ensure_declarations_are_planned(schema: &ArrowSchema) -> Result<()> {
|
||||
let invalid = |message: String| Error::InvalidInput { message };
|
||||
// A field with any declaration key is a declaration; a partial one is
|
||||
// not "no declaration", it is a broken one.
|
||||
for field in schema.fields() {
|
||||
if field.metadata().keys().any(|k| is_declaration_key(k))
|
||||
&& computed_column_from_field(field).is_none()
|
||||
{
|
||||
return Err(invalid(format!(
|
||||
"field '{}' carries an incomplete computed-column declaration",
|
||||
field.name()
|
||||
)));
|
||||
}
|
||||
}
|
||||
let declared: HashSet<String> = computed_columns(schema)
|
||||
.into_iter()
|
||||
.map(|c| c.name)
|
||||
.collect();
|
||||
for column in computed_columns(schema) {
|
||||
let field = schema.field_with_name(&column.name)?;
|
||||
if !field.is_nullable() {
|
||||
return Err(invalid(format!(
|
||||
"computed column '{}' must be nullable until a refresh fills it",
|
||||
column.name
|
||||
)));
|
||||
}
|
||||
match &column.kind {
|
||||
ComputedColumnKind::Sql { expression } => {
|
||||
let others: Vec<ArrowField> = schema
|
||||
.fields()
|
||||
.iter()
|
||||
.filter(|f| f.name() != &column.name)
|
||||
.map(|f| f.as_ref().clone())
|
||||
.collect();
|
||||
let bound = bind(Arc::new(ArrowSchema::new(others)), &column.name, expression)?;
|
||||
if let Some(input) = bound.roots.iter().find(|r| declared.contains(*r)) {
|
||||
return Err(invalid(format!(
|
||||
"computed column '{}' reads computed column '{input}'",
|
||||
column.name
|
||||
)));
|
||||
}
|
||||
if &bound.data_type != field.data_type() {
|
||||
return Err(invalid(format!(
|
||||
"computed column '{}' is declared as {} but its expression yields {}",
|
||||
column.name,
|
||||
field.data_type(),
|
||||
bound.data_type
|
||||
)));
|
||||
}
|
||||
let mut declared_inputs = column.inputs.clone();
|
||||
declared_inputs.sort();
|
||||
if declared_inputs != bound.inputs {
|
||||
return Err(invalid(format!(
|
||||
"computed column '{}' declares inputs {:?} but its expression reads {:?}",
|
||||
column.name, declared_inputs, bound.inputs
|
||||
)));
|
||||
}
|
||||
}
|
||||
ComputedColumnKind::Function { binding_id, .. } => {
|
||||
// The binding validator resolves each input's leaf; the
|
||||
// no-computed-input rule is about the root it hangs from.
|
||||
let bindings = function_bindings(schema)?;
|
||||
let Some(binding) = bindings.iter().find(|b| b.binding_id() == binding_id) else {
|
||||
continue; // reported by the binding validator below
|
||||
};
|
||||
// Roots come from the canonical path parser: a quoted
|
||||
// top-level name may itself contain a dot.
|
||||
if let Some(input) = binding
|
||||
.inputs()
|
||||
.iter()
|
||||
.filter_map(|input| resolve_field_path(schema, &input.field_path).ok())
|
||||
.map(|resolved| resolved.root.name().as_str())
|
||||
.find(|r| declared.contains(*r))
|
||||
{
|
||||
return Err(invalid(format!(
|
||||
"computed column '{}' reads computed column '{input}'",
|
||||
column.name
|
||||
)));
|
||||
}
|
||||
}
|
||||
ComputedColumnKind::Unrecognized { kind } => {
|
||||
return Err(Error::NotSupported {
|
||||
message: format!(
|
||||
"computed column '{}' is defined by '{kind}', which this version \
|
||||
of lancedb cannot fill",
|
||||
column.name
|
||||
),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
ensure_supported_function_metadata(schema)
|
||||
}
|
||||
|
||||
/// Reject fields carrying declaration metadata that did not come through
|
||||
/// [`plan`]. One authority for creation, overwrite and raw transforms.
|
||||
pub(crate) fn ensure_no_foreign_declarations<'a>(
|
||||
@@ -1897,54 +1797,6 @@ pub(super) async fn add_foreign_kind(table: &crate::Table, name: &str, kind: &st
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
/// Admit a table's initial data: every declaration it carries is validated,
|
||||
/// and the stream refuses any batch with values in a computed column, whose
|
||||
/// values come from refresh alone. One boundary for every way a table is
|
||||
/// created.
|
||||
pub(crate) fn admit_create_source<S: lance_datafusion::utils::StreamingWriteSource>(
|
||||
batches: S,
|
||||
) -> Result<UnfilledDeclarations<S>> {
|
||||
let schema = batches.arrow_schema();
|
||||
ensure_declarations_are_planned(&schema)?;
|
||||
let declared = computed_columns(&schema)
|
||||
.into_iter()
|
||||
.map(|c| c.name)
|
||||
.collect();
|
||||
Ok(UnfilledDeclarations {
|
||||
inner: batches,
|
||||
declared,
|
||||
})
|
||||
}
|
||||
|
||||
/// A write source whose computed columns must arrive unfilled.
|
||||
pub(crate) struct UnfilledDeclarations<S> {
|
||||
inner: S,
|
||||
declared: Vec<String>,
|
||||
}
|
||||
|
||||
impl<S: lance_datafusion::utils::StreamingWriteSource> lance_datafusion::utils::StreamingWriteSource
|
||||
for UnfilledDeclarations<S>
|
||||
{
|
||||
fn arrow_schema(&self) -> SchemaRef {
|
||||
self.inner.arrow_schema()
|
||||
}
|
||||
|
||||
fn into_stream(self) -> datafusion_physical_plan::SendableRecordBatchStream {
|
||||
if self.declared.is_empty() {
|
||||
return self.inner.into_stream();
|
||||
}
|
||||
let schema = self.inner.arrow_schema();
|
||||
let declared = self.declared;
|
||||
let stream = self.inner.into_stream().map(move |batch| {
|
||||
let batch = batch?;
|
||||
ensure_batch_writes_no_computed_values(&declared, &batch)
|
||||
.map_err(|e| datafusion_common::DataFusionError::External(Box::new(e)))?;
|
||||
Ok(batch)
|
||||
});
|
||||
Box::pin(datafusion_physical_plan::stream::RecordBatchStreamAdapter::new(schema, stream))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
/// The gate's reproducer: the validator applies the same schema-level
|
||||
@@ -1963,69 +1815,6 @@ mod tests {
|
||||
assert!(super::validate_declarations(schema, &declarations).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn list_children_match_by_type_but_struct_children_by_identity() {
|
||||
use arrow_schema::Field as F;
|
||||
|
||||
// Lance rewrites a list item's name and nullability on write, so the
|
||||
// stored field is no longer identical to what was declared. Comparing
|
||||
// those by type keeps a table with a vector output usable.
|
||||
let declared =
|
||||
DataType::FixedSizeList(Arc::new(F::new("item", DataType::Float32, false)), 4);
|
||||
let stored = DataType::FixedSizeList(Arc::new(F::new("item", DataType::Float32, true)), 4);
|
||||
assert!(super::function_output_type_matches(&declared, &stored));
|
||||
|
||||
let renamed =
|
||||
DataType::FixedSizeList(Arc::new(F::new("element", DataType::Float32, true)), 4);
|
||||
assert!(super::function_output_type_matches(&declared, &renamed));
|
||||
|
||||
// The dimension is still part of the declaration.
|
||||
let resized = DataType::FixedSizeList(Arc::new(F::new("item", DataType::Float32, true)), 8);
|
||||
assert!(!super::function_output_type_matches(&declared, &resized));
|
||||
|
||||
// Struct children keep comparing by name and nullability.
|
||||
let struct_declared =
|
||||
DataType::Struct(vec![F::new("changed", DataType::Boolean, false)].into());
|
||||
let struct_nullable =
|
||||
DataType::Struct(vec![F::new("changed", DataType::Boolean, true)].into());
|
||||
let struct_renamed =
|
||||
DataType::Struct(vec![F::new("altered", DataType::Boolean, false)].into());
|
||||
assert!(super::function_output_type_matches(
|
||||
&struct_declared,
|
||||
&struct_declared
|
||||
));
|
||||
assert!(!super::function_output_type_matches(
|
||||
&struct_declared,
|
||||
&struct_nullable
|
||||
));
|
||||
assert!(!super::function_output_type_matches(
|
||||
&struct_declared,
|
||||
&struct_renamed
|
||||
));
|
||||
|
||||
// A list nested inside a struct gets the list rule.
|
||||
let nested_declared = DataType::Struct(
|
||||
vec![F::new(
|
||||
"tokens",
|
||||
DataType::List(Arc::new(F::new("item", DataType::Utf8, false))),
|
||||
true,
|
||||
)]
|
||||
.into(),
|
||||
);
|
||||
let nested_stored = DataType::Struct(
|
||||
vec![F::new(
|
||||
"tokens",
|
||||
DataType::List(Arc::new(F::new("item", DataType::Utf8, true))),
|
||||
true,
|
||||
)]
|
||||
.into(),
|
||||
);
|
||||
assert!(super::function_output_type_matches(
|
||||
&nested_declared,
|
||||
&nested_stored
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn output_arrow_type_grammar_matches_the_shared_golden() {
|
||||
let golden: serde_json::Value = serde_json::from_str(include_str!(
|
||||
@@ -2795,8 +2584,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
/// A create carries a declaration only if it re-plans completely; this
|
||||
/// one lacks its inputs and is refused before its forged value matters.
|
||||
#[tokio::test]
|
||||
async fn test_create_table_cannot_inject_a_declaration() {
|
||||
let conn = connect("memory://").execute().await.unwrap();
|
||||
@@ -2824,7 +2611,7 @@ mod tests {
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(
|
||||
matches!(&err, Error::InvalidInput { message } if message.contains("computed column 'doubled'")),
|
||||
matches!(&err, Error::InvalidInput { message } if message.contains("computed()")),
|
||||
"{err:?}"
|
||||
);
|
||||
}
|
||||
@@ -3502,70 +3289,6 @@ mod tests {
|
||||
assert!(output_schema.field(0).is_blob_v2());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn binding_accepts_a_lance_normalized_list_child() {
|
||||
// The whole guard, not just the type helper: this also reaches the
|
||||
// output-schema comparison at the end of ensure_binding_matches_schema,
|
||||
// which used to rebuild the schema from the stored field and so failed
|
||||
// on exactly the same normalization.
|
||||
let input = ArrowField::new("value", DataType::Int64, false);
|
||||
let application = FunctionApplication::from_json(
|
||||
&serde_json::json!({
|
||||
"function": {"name": "embed", "version": "fv_embed"},
|
||||
"inputs": [{
|
||||
"parameter": "value",
|
||||
"kind": "column",
|
||||
"value": {"path": "value"}
|
||||
}],
|
||||
"output": {
|
||||
"kind": "scalar",
|
||||
"arrow_type": "fixed_size_list<float32, 4>",
|
||||
"nullable": false
|
||||
}
|
||||
})
|
||||
.to_string(),
|
||||
)
|
||||
.unwrap();
|
||||
let plan = plan_function_application(
|
||||
&ArrowSchema::new(vec![input.clone()]),
|
||||
&application,
|
||||
Some("embedding"),
|
||||
)
|
||||
.unwrap();
|
||||
let binding = binding_from_plan(&plan);
|
||||
|
||||
// The declaration says the item is non-nullable; Lance rewrites it to
|
||||
// nullable on write, so this is what the column looks like on disk.
|
||||
let stored = DataType::FixedSizeList(
|
||||
Arc::new(ArrowField::new("item", DataType::Float32, true)),
|
||||
4,
|
||||
);
|
||||
let output = ArrowField::new("embedding", stored, true).with_metadata(
|
||||
function_computed_column_metadata(binding.binding_id(), 0, &["value".into()]),
|
||||
);
|
||||
|
||||
ensure_binding_matches_schema(&ArrowSchema::new(vec![input.clone(), output]), &binding)
|
||||
.unwrap();
|
||||
|
||||
// A different element type is still a mismatch.
|
||||
let wrong = ArrowField::new(
|
||||
"embedding",
|
||||
DataType::FixedSizeList(
|
||||
Arc::new(ArrowField::new("item", DataType::Float64, true)),
|
||||
4,
|
||||
),
|
||||
true,
|
||||
)
|
||||
.with_metadata(function_computed_column_metadata(
|
||||
binding.binding_id(),
|
||||
0,
|
||||
&["value".into()],
|
||||
));
|
||||
assert!(
|
||||
ensure_binding_matches_schema(&ArrowSchema::new(vec![input, wrong]), &binding).is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_blob_scalar_binding_accepts_full_logical_layout() {
|
||||
let input = crate::blob("image", false);
|
||||
|
||||
@@ -5,17 +5,12 @@
|
||||
//!
|
||||
//! [`super::cast::cast_to_table_schema`] calls [`coerce_blob_expr`].
|
||||
|
||||
use std::fmt;
|
||||
use std::hash::{Hash, Hasher};
|
||||
use std::sync::Arc;
|
||||
|
||||
use arrow_array::{Array, BooleanArray, RecordBatch};
|
||||
use arrow_schema::{DataType, Field, FieldRef, Fields, Schema};
|
||||
use arrow_select::nullif::nullif;
|
||||
use arrow_schema::{DataType, Field, FieldRef, Fields};
|
||||
use datafusion::functions::core::{get_field, named_struct};
|
||||
use datafusion_common::ScalarValue;
|
||||
use datafusion_common::config::ConfigOptions;
|
||||
use datafusion_expr::ColumnarValue;
|
||||
use datafusion_physical_expr::ScalarFunctionExpr;
|
||||
use datafusion_physical_expr::expressions::{CastExpr, Literal};
|
||||
use datafusion_physical_plan::PhysicalExpr;
|
||||
@@ -138,102 +133,16 @@ pub(super) fn coerce_blob_expr(
|
||||
ns_args.push(value);
|
||||
}
|
||||
|
||||
let built: Arc<dyn PhysicalExpr> = Arc::new(ScalarFunctionExpr::new(
|
||||
let expr: Arc<dyn PhysicalExpr> = Arc::new(ScalarFunctionExpr::new(
|
||||
&format!("named_struct({})", table_field.name()),
|
||||
named_struct(),
|
||||
ns_args,
|
||||
table_field.clone(),
|
||||
config.clone(),
|
||||
));
|
||||
|
||||
// `named_struct` always yields a valid struct, so a null input would land
|
||||
// as a row that set neither `data` nor `uri` -- not an absent blob but a
|
||||
// malformed one, which Lance rejects on write.
|
||||
let expr: Arc<dyn PhysicalExpr> = Arc::new(AbsentBlobIsNull {
|
||||
source: input_expr,
|
||||
built,
|
||||
field: table_field.clone(),
|
||||
});
|
||||
Ok((expr, table_field.clone()))
|
||||
}
|
||||
|
||||
/// Carries the source column's nullity onto the struct built for it.
|
||||
///
|
||||
/// This is its own expression rather than a `CASE` because the projection
|
||||
/// takes its output field from `return_field`, and the generic implementation
|
||||
/// rebuilds a bare field -- which would drop the `lance.blob.v2` extension
|
||||
/// metadata and stop the column being recognised as a blob at all.
|
||||
#[derive(Debug, Clone)]
|
||||
struct AbsentBlobIsNull {
|
||||
source: Arc<dyn PhysicalExpr>,
|
||||
built: Arc<dyn PhysicalExpr>,
|
||||
field: FieldRef,
|
||||
}
|
||||
|
||||
impl fmt::Display for AbsentBlobIsNull {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
write!(f, "absent_blob_is_null({}, {})", self.source, self.built)
|
||||
}
|
||||
}
|
||||
|
||||
impl PartialEq for AbsentBlobIsNull {
|
||||
fn eq(&self, other: &Self) -> bool {
|
||||
self.source.eq(&other.source) && self.built.eq(&other.built) && self.field == other.field
|
||||
}
|
||||
}
|
||||
|
||||
impl Eq for AbsentBlobIsNull {}
|
||||
|
||||
impl Hash for AbsentBlobIsNull {
|
||||
fn hash<H: Hasher>(&self, state: &mut H) {
|
||||
self.source.hash(state);
|
||||
self.built.hash(state);
|
||||
self.field.hash(state);
|
||||
}
|
||||
}
|
||||
|
||||
impl PhysicalExpr for AbsentBlobIsNull {
|
||||
fn return_field(&self, _input_schema: &Schema) -> datafusion_common::Result<FieldRef> {
|
||||
Ok(self.field.clone())
|
||||
}
|
||||
|
||||
fn nullable(&self, _input_schema: &Schema) -> datafusion_common::Result<bool> {
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
fn evaluate(&self, batch: &RecordBatch) -> datafusion_common::Result<ColumnarValue> {
|
||||
let rows = batch.num_rows();
|
||||
let built = self.built.evaluate(batch)?.into_array(rows)?;
|
||||
let source = self.source.evaluate(batch)?.into_array(rows)?;
|
||||
let Some(nulls) = source.logical_nulls() else {
|
||||
return Ok(ColumnarValue::Array(built));
|
||||
};
|
||||
// `nullif` nulls the rows the mask marks true, which is where the
|
||||
// source had no value.
|
||||
let absent = BooleanArray::new(!nulls.inner(), None);
|
||||
Ok(ColumnarValue::Array(nullif(built.as_ref(), &absent)?))
|
||||
}
|
||||
|
||||
fn children(&self) -> Vec<&Arc<dyn PhysicalExpr>> {
|
||||
vec![&self.source, &self.built]
|
||||
}
|
||||
|
||||
fn with_new_children(
|
||||
self: Arc<Self>,
|
||||
children: Vec<Arc<dyn PhysicalExpr>>,
|
||||
) -> datafusion_common::Result<Arc<dyn PhysicalExpr>> {
|
||||
Ok(Arc::new(Self {
|
||||
source: children[0].clone(),
|
||||
built: children[1].clone(),
|
||||
field: self.field.clone(),
|
||||
}))
|
||||
}
|
||||
|
||||
fn fmt_sql(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
write!(f, "{self}")
|
||||
}
|
||||
}
|
||||
|
||||
enum BlobInputShape<'a> {
|
||||
Bytes,
|
||||
String,
|
||||
@@ -404,11 +313,6 @@ mod tests {
|
||||
let data = image.column_by_name("data").unwrap();
|
||||
assert!(!data.is_null(0));
|
||||
assert!(data.is_null(1));
|
||||
// The row itself has to be null, not merely a struct whose children
|
||||
// are. A present-but-empty struct set neither `data` nor `uri`, which
|
||||
// Lance rejects as malformed rather than reading as an absent blob.
|
||||
assert!(!image.is_null(0));
|
||||
assert!(image.is_null(1));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -19,7 +19,6 @@ use lancedb::{
|
||||
connect, connect_namespace,
|
||||
database::listing::{
|
||||
ListingDatabaseOptions, NewTableConfig, OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS,
|
||||
OPT_NEW_TABLE_STORAGE_VERSION,
|
||||
},
|
||||
query::{ExecutableQuery, QueryBase},
|
||||
table::{AddDataMode, CompactionOptions, OptimizeAction, OptimizeStats, WriteOptions},
|
||||
@@ -147,10 +146,7 @@ async fn non_blob_table_keeps_default_format_and_row_id_setting() -> Result<()>
|
||||
let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)]));
|
||||
let table = db.create_empty_table("t", schema).execute().await?;
|
||||
|
||||
assert_eq!(
|
||||
storage_format_version(&table).await,
|
||||
LanceFileVersion::Stable.resolve()
|
||||
);
|
||||
assert!(!supports_blob_v2(storage_format_version(&table).await));
|
||||
assert!(!uses_stable_row_ids(&table).await);
|
||||
Ok(())
|
||||
}
|
||||
@@ -813,11 +809,7 @@ async fn fetch_blobs_rejects_unknown_column() -> Result<()> {
|
||||
#[tokio::test]
|
||||
async fn fetch_blobs_rejects_legacy_v1_blob_column() -> Result<()> {
|
||||
let tmp = tempdir().unwrap();
|
||||
// Legacy v1 blob columns are only writable at file version <= 2.1.
|
||||
let db = connect(tmp.path().to_str().unwrap())
|
||||
.storage_options([(OPT_NEW_TABLE_STORAGE_VERSION, "2.1")])
|
||||
.execute()
|
||||
.await?;
|
||||
let db = connect(tmp.path().to_str().unwrap()).execute().await?;
|
||||
let legacy = Field::new("image", DataType::LargeBinary, true).with_metadata(
|
||||
std::collections::HashMap::from([("lance-encoding:blob".to_string(), "true".to_string())]),
|
||||
);
|
||||
|
||||
Reference in New Issue
Block a user