From e0bd4b5fa1afdb10d2d551e867ae7d5304179c38 Mon Sep 17 00:00:00 2001 From: LanceDB Robot Date: Fri, 11 Sep 2026 09:59:19 -0700 Subject: [PATCH 1/8] chore: update lance dependency to v12.0.0-beta.17 (#4162) Update the Rust workspace Lance dependencies and Java lance-core to [v12.0.0-beta.17](https://github.com/lance-format/lance/releases/tag/v12.0.0-beta.17). Align object_store to 0.14.1 for compatibility with Lance and refresh the Cargo lockfile, including the required reqsign updates. Validation passed: `cargo clippy --quiet --workspace --tests --all-features -- -D warnings` and `cargo fmt --all --quiet`. --- Cargo.lock | 489 ++++++++++++++++++++++++++------------------------- Cargo.toml | 30 ++-- java/pom.xml | 2 +- 3 files changed, 266 insertions(+), 255 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3afa04044..209ee4942 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -141,7 +141,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -152,7 +152,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -581,6 +581,16 @@ dependencies = [ "loom", ] +[[package]] +name = "asyncband" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94a214ba60d6231afd0e805e3c27c45a1626d9debaa5a5061c45a1ea1b2f1ed0" +dependencies = [ + "hashbrown 0.17.1", + "slab", +] + [[package]] name = "atoi" version = "2.0.0" @@ -937,7 +947,7 @@ dependencies = [ "http 1.5.0", "http-body 1.1.0", "http-body-util", - "md-5 0.11.0", + "md-5", "pin-project-lite", "sha1 0.11.0", "sha2 0.11.0", @@ -1767,7 +1777,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" dependencies = [ "clap_builder", - "clap_derive", ] [[package]] @@ -1776,22 +1785,8 @@ version = "4.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" dependencies = [ - "anstream", "anstyle", "clap_lex", - "strsim", -] - -[[package]] -name = "clap_derive" -version = "4.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" -dependencies = [ - "heck 0.5.0", - "proc-macro2", - "quote", - "syn 2.0.117", ] [[package]] @@ -1827,7 +1822,7 @@ version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -2230,16 +2225,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "ctor" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "424e0138278faeb2b401f174ad17e715c829512d74f3d1e81eb43365c2e0590e" -dependencies = [ - "ctor-proc-macro", - "dtor", -] - [[package]] name = "ctor" version = "1.0.12" @@ -2250,12 +2235,6 @@ dependencies = [ "linktime-proc-macro", ] -[[package]] -name = "ctor-proc-macro" -version = "0.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52560adf09603e58c9a7ee1fe1dcb95a16927b17c127f0ac02d6e768a0e25bc1" - [[package]] name = "ctutils" version = "0.4.2" @@ -2392,7 +2371,7 @@ dependencies = [ "indexmap 2.14.0", "itertools 0.14.0", "log", - "object_store", + "object_store 0.13.2", "parking_lot", "sqlparser 0.62.0", "tempfile", @@ -2421,7 +2400,7 @@ dependencies = [ "futures", "itertools 0.14.0", "log", - "object_store", + "object_store 0.13.2", "parking_lot", "tokio", ] @@ -2446,7 +2425,7 @@ dependencies = [ "futures", "itertools 0.14.0", "log", - "object_store", + "object_store 0.13.2", ] [[package]] @@ -2466,7 +2445,7 @@ dependencies = [ "itertools 0.14.0", "libc", "log", - "object_store", + "object_store 0.13.2", "sqlparser 0.62.0", "tokio", "uuid", @@ -2507,7 +2486,7 @@ dependencies = [ "glob", "itertools 0.14.0", "log", - "object_store", + "object_store 0.13.2", "parking_lot", "rand 0.9.5", "tokio", @@ -2534,7 +2513,7 @@ dependencies = [ "datafusion-session", "futures", "itertools 0.14.0", - "object_store", + "object_store 0.13.2", "tokio", ] @@ -2556,7 +2535,7 @@ dependencies = [ "datafusion-physical-plan", "datafusion-session", "futures", - "object_store", + "object_store 0.13.2", "regex", "tokio", ] @@ -2579,7 +2558,7 @@ dependencies = [ "datafusion-physical-plan", "datafusion-session", "futures", - "object_store", + "object_store 0.13.2", "tokio", "tokio-stream", ] @@ -2605,7 +2584,7 @@ dependencies = [ "datafusion-physical-expr-common", "futures", "log", - "object_store", + "object_store 0.13.2", "parking_lot", "rand 0.9.5", "tempfile", @@ -2669,7 +2648,7 @@ dependencies = [ "hex", "itertools 0.14.0", "log", - "md-5 0.11.0", + "md-5", "memchr", "num-traits", "rand 0.9.5", @@ -3105,7 +3084,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -3137,21 +3116,6 @@ dependencies = [ "litrs", ] -[[package]] -name = "dtor" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "404d02eeb088a82cfd873006cb713fe411306c7d182c344905e101fb1167d301" -dependencies = [ - "dtor-proc-macro", -] - -[[package]] -name = "dtor-proc-macro" -version = "0.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f678cf4a922c215c63e0de95eb1ff08a958a81d47e485cf9da1e27bf6305cfa5" - [[package]] name = "dunce" version = "1.0.5" @@ -3328,7 +3292,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -3526,8 +3490,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "fsst" -version = "12.0.0-beta.16" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.16#f7df098f5860cf9b3ac5339e7a641430cfbc6351" +version = "12.0.0-beta.17" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.17#d7b031f9ca86e7cbb8c3732ab87b2b18548996ba" dependencies = [ "arrow-array", "rand 0.9.5", @@ -4085,18 +4049,21 @@ dependencies = [ [[package]] name = "hf-xet" -version = "1.5.2" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "430b33fa84f92796d4d263070b6c0d3ca219df7b9a0e1853ee431029b1612bcd" +checksum = "c237ef4fb0ce1962a5117f8bd8c74454b41629826a9df17d14a1840ca18f0754" dependencies = [ + "anyhow", "async-trait", "bytes", "http 1.5.0", "more-asserts", "serde", + "serde_json", "thiserror 2.0.18", "tokio", "tokio-util", + "tokio_with_wasm", "tracing", "uuid", "xet-client", @@ -4333,7 +4300,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.6.3", + "socket2 0.5.10", "system-configuration", "tokio", "tower-service", @@ -4632,7 +4599,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -4677,6 +4644,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.18" @@ -4886,8 +4862,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a" [[package]] name = "lance" -version = "12.0.0-beta.16" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.16#f7df098f5860cf9b3ac5339e7a641430cfbc6351" +version = "12.0.0-beta.17" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.17#d7b031f9ca86e7cbb8c3732ab87b2b18548996ba" dependencies = [ "arc-swap", "arrow", @@ -4935,7 +4911,7 @@ dependencies = [ "lance-tokenizer", "log", "moka", - "object_store", + "object_store 0.14.1", "permutation", "pin-project", "prost", @@ -4959,8 +4935,8 @@ dependencies = [ [[package]] name = "lance-arrow" -version = "12.0.0-beta.16" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.16#f7df098f5860cf9b3ac5339e7a641430cfbc6351" +version = "12.0.0-beta.17" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.17#d7b031f9ca86e7cbb8c3732ab87b2b18548996ba" dependencies = [ "arrow-array", "arrow-buffer", @@ -4982,7 +4958,7 @@ dependencies = [ [[package]] name = "lance-arrow-scalar" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.16#f7df098f5860cf9b3ac5339e7a641430cfbc6351" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.17#d7b031f9ca86e7cbb8c3732ab87b2b18548996ba" dependencies = [ "arrow-array", "arrow-buffer", @@ -4996,7 +4972,7 @@ dependencies = [ [[package]] name = "lance-arrow-stats" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.16#f7df098f5860cf9b3ac5339e7a641430cfbc6351" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.17#d7b031f9ca86e7cbb8c3732ab87b2b18548996ba" dependencies = [ "arrow-array", "arrow-schema", @@ -5005,8 +4981,8 @@ dependencies = [ [[package]] name = "lance-bitpacking" -version = "12.0.0-beta.16" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.16#f7df098f5860cf9b3ac5339e7a641430cfbc6351" +version = "12.0.0-beta.17" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.17#d7b031f9ca86e7cbb8c3732ab87b2b18548996ba" dependencies = [ "arrayref", "crunchy", @@ -5016,8 +4992,8 @@ dependencies = [ [[package]] name = "lance-core" -version = "12.0.0-beta.16" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.16#f7df098f5860cf9b3ac5339e7a641430cfbc6351" +version = "12.0.0-beta.17" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.17#d7b031f9ca86e7cbb8c3732ab87b2b18548996ba" dependencies = [ "arrow-array", "arrow-buffer", @@ -5036,7 +5012,7 @@ dependencies = [ "log", "moka", "num_cpus", - "object_store", + "object_store 0.14.1", "pin-project", "prost", "quick_cache", @@ -5054,8 +5030,8 @@ dependencies = [ [[package]] name = "lance-datafusion" -version = "12.0.0-beta.16" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.16#f7df098f5860cf9b3ac5339e7a641430cfbc6351" +version = "12.0.0-beta.17" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.17#d7b031f9ca86e7cbb8c3732ab87b2b18548996ba" dependencies = [ "arrow", "arrow-array", @@ -5085,8 +5061,8 @@ dependencies = [ [[package]] name = "lance-datagen" -version = "12.0.0-beta.16" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.16#f7df098f5860cf9b3ac5339e7a641430cfbc6351" +version = "12.0.0-beta.17" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.17#d7b031f9ca86e7cbb8c3732ab87b2b18548996ba" dependencies = [ "arrow", "arrow-array", @@ -5103,8 +5079,8 @@ dependencies = [ [[package]] name = "lance-derive" -version = "12.0.0-beta.16" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.16#f7df098f5860cf9b3ac5339e7a641430cfbc6351" +version = "12.0.0-beta.17" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.17#d7b031f9ca86e7cbb8c3732ab87b2b18548996ba" dependencies = [ "proc-macro2", "quote", @@ -5113,8 +5089,8 @@ dependencies = [ [[package]] name = "lance-encoding" -version = "12.0.0-beta.16" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.16#f7df098f5860cf9b3ac5339e7a641430cfbc6351" +version = "12.0.0-beta.17" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.17#d7b031f9ca86e7cbb8c3732ab87b2b18548996ba" dependencies = [ "arrow-arith", "arrow-array", @@ -5147,8 +5123,8 @@ dependencies = [ [[package]] name = "lance-file" -version = "12.0.0-beta.16" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.16#f7df098f5860cf9b3ac5339e7a641430cfbc6351" +version = "12.0.0-beta.17" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.17#d7b031f9ca86e7cbb8c3732ab87b2b18548996ba" dependencies = [ "arrow-arith", "arrow-array", @@ -5169,7 +5145,7 @@ dependencies = [ "lance-io", "log", "num-traits", - "object_store", + "object_store 0.14.1", "prost", "prost-build", "prost-types", @@ -5179,8 +5155,8 @@ dependencies = [ [[package]] name = "lance-index" -version = "12.0.0-beta.16" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.16#f7df098f5860cf9b3ac5339e7a641430cfbc6351" +version = "12.0.0-beta.17" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.17#d7b031f9ca86e7cbb8c3732ab87b2b18548996ba" dependencies = [ "arc-swap", "arrow", @@ -5224,7 +5200,7 @@ dependencies = [ "log", "ndarray", "num-traits", - "object_store", + "object_store 0.14.1", "prost", "prost-build", "prost-types", @@ -5244,8 +5220,8 @@ dependencies = [ [[package]] name = "lance-index-core" -version = "12.0.0-beta.16" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.16#f7df098f5860cf9b3ac5339e7a641430cfbc6351" +version = "12.0.0-beta.17" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.17#d7b031f9ca86e7cbb8c3732ab87b2b18548996ba" dependencies = [ "arrow-array", "arrow-schema", @@ -5267,8 +5243,8 @@ dependencies = [ [[package]] name = "lance-io" -version = "12.0.0-beta.16" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.16#f7df098f5860cf9b3ac5339e7a641430cfbc6351" +version = "12.0.0-beta.17" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.17#d7b031f9ca86e7cbb8c3732ab87b2b18548996ba" dependencies = [ "arrow", "arrow-array", @@ -5287,7 +5263,7 @@ dependencies = [ "log", "metrics", "moka", - "object_store", + "object_store 0.14.1", "object_store_opendal", "opendal", "path_abs", @@ -5308,8 +5284,8 @@ dependencies = [ [[package]] name = "lance-linalg" -version = "12.0.0-beta.16" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.16#f7df098f5860cf9b3ac5339e7a641430cfbc6351" +version = "12.0.0-beta.17" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.17#d7b031f9ca86e7cbb8c3732ab87b2b18548996ba" dependencies = [ "arrow-array", "arrow-schema", @@ -5323,8 +5299,8 @@ dependencies = [ [[package]] name = "lance-namespace" -version = "12.0.0-beta.16" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.16#f7df098f5860cf9b3ac5339e7a641430cfbc6351" +version = "12.0.0-beta.17" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.17#d7b031f9ca86e7cbb8c3732ab87b2b18548996ba" dependencies = [ "arrow", "async-trait", @@ -5338,8 +5314,8 @@ dependencies = [ [[package]] name = "lance-namespace-impls" -version = "12.0.0-beta.16" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.16#f7df098f5860cf9b3ac5339e7a641430cfbc6351" +version = "12.0.0-beta.17" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.17#d7b031f9ca86e7cbb8c3732ab87b2b18548996ba" dependencies = [ "arrow", "arrow-ipc", @@ -5361,7 +5337,7 @@ dependencies = [ "lance-namespace", "lance-table", "log", - "object_store", + "object_store 0.14.1", "quick-xml 0.40.1", "rand 0.9.5", "reqwest 0.12.28", @@ -5392,8 +5368,8 @@ dependencies = [ [[package]] name = "lance-select" -version = "12.0.0-beta.16" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.16#f7df098f5860cf9b3ac5339e7a641430cfbc6351" +version = "12.0.0-beta.17" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.17#d7b031f9ca86e7cbb8c3732ab87b2b18548996ba" dependencies = [ "arrow-array", "arrow-buffer", @@ -5407,8 +5383,8 @@ dependencies = [ [[package]] name = "lance-table" -version = "12.0.0-beta.16" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.16#f7df098f5860cf9b3ac5339e7a641430cfbc6351" +version = "12.0.0-beta.17" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.17#d7b031f9ca86e7cbb8c3732ab87b2b18548996ba" dependencies = [ "arrow", "arrow-array", @@ -5429,7 +5405,7 @@ dependencies = [ "lance-io", "lance-select", "log", - "object_store", + "object_store 0.14.1", "prost", "prost-build", "prost-types", @@ -5448,8 +5424,8 @@ dependencies = [ [[package]] name = "lance-testing" -version = "12.0.0-beta.16" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.16#f7df098f5860cf9b3ac5339e7a641430cfbc6351" +version = "12.0.0-beta.17" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.17#d7b031f9ca86e7cbb8c3732ab87b2b18548996ba" dependencies = [ "arrow-array", "arrow-schema", @@ -5462,8 +5438,8 @@ dependencies = [ [[package]] name = "lance-tokenizer" -version = "12.0.0-beta.16" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.16#f7df098f5860cf9b3ac5339e7a641430cfbc6351" +version = "12.0.0-beta.17" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.17#d7b031f9ca86e7cbb8c3732ab87b2b18548996ba" dependencies = [ "frostem", "icu_segmenter", @@ -5536,7 +5512,7 @@ dependencies = [ "metrics-util", "moka", "num-traits", - "object_store", + "object_store 0.14.1", "pin-project", "polars", "polars-arrow", @@ -5956,16 +5932,6 @@ dependencies = [ "thread-tree", ] -[[package]] -name = "md-5" -version = "0.10.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" -dependencies = [ - "cfg-if 1.0.4", - "digest 0.10.7", -] - [[package]] name = "md-5" version = "0.11.0" @@ -5978,10 +5944,11 @@ dependencies = [ [[package]] name = "mea" -version = "0.6.3" +version = "0.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6747f54621d156e1b47eb6b25f39a941b9fc347f98f67d25d8881ff99e8ed832" +checksum = "c709842c4ce65cb91e2666ad5319dfc1efc3af0d34f02075eddca9000d9f8afb" dependencies = [ + "hashbrown 0.17.1", "slab", ] @@ -6187,7 +6154,7 @@ checksum = "58c5f4d5375213fdb7be2655e152386e82f026f9a5ba36a75556e11359aafe09" dependencies = [ "bitflags 2.11.1", "chrono", - "ctor 1.0.12", + "ctor", "futures", "libc", "napi-build", @@ -6212,7 +6179,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fa55ea69990c90b888e9e77044410e304ce7f35de599dc6d0b5c1923d2e59af" dependencies = [ "convert_case", - "ctor 1.0.12", + "ctor", "napi-derive-backend", "proc-macro2", "quote", @@ -6276,6 +6243,18 @@ dependencies = [ "libc", ] +[[package]] +name = "nix" +version = "0.31.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" +dependencies = [ + "bitflags 2.11.1", + "cfg-if 1.0.4", + "cfg_aliases", + "libc", +] + [[package]] name = "nohash-hasher" version = "0.2.0" @@ -6325,7 +6304,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -6470,9 +6449,37 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "622acbc9100d3c10e2ee15804b0caa40e55c933d5aa53814cd520805b7958a49" dependencies = [ "async-trait", + "bytes", + "chrono", + "futures-channel", + "futures-core", + "futures-util", + "http 1.5.0", + "humantime", + "itertools 0.14.0", + "parking_lot", + "percent-encoding", + "thiserror 2.0.18", + "tokio", + "tracing", + "url", + "walkdir", + "wasm-bindgen-futures", + "web-time", +] + +[[package]] +name = "object_store" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d354792e39fa5f0009e47623cf8b15b099bf9a652fa55c6f817fe28ac84fea50" +dependencies = [ + "async-trait", + "aws-lc-rs", "base64 0.22.1", "bytes", "chrono", + "crc-fast", "form_urlencoded", "futures-channel", "futures-core", @@ -6482,14 +6489,14 @@ dependencies = [ "httparse", "humantime", "hyper 1.9.0", - "itertools 0.14.0", - "md-5 0.10.6", + "itertools 0.15.0", + "md-5", + "nix 0.31.3", "parking_lot", "percent-encoding", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "rand 0.10.1", - "reqwest 0.12.28", - "ring", + "reqwest 0.13.4", "rustls-pki-types", "serde", "serde_json", @@ -6501,20 +6508,21 @@ dependencies = [ "walkdir", "wasm-bindgen-futures", "web-time", + "windows-sys 0.61.2", ] [[package]] name = "object_store_opendal" -version = "0.58.0" +version = "0.60.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88f165780495c17aa3ce86846600504198c3fffd99073521552751c2430fa6ac" +checksum = "f0206382328a82a28b549e5b2d18b6b9384ac1d82fa1f3c63da06f5ee6f7f054" dependencies = [ "async-trait", + "asyncband", "bytes", "chrono", "futures", - "mea", - "object_store", + "object_store 0.14.1", "opendal", "pin-project", "tokio", @@ -6568,11 +6576,11 @@ checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" [[package]] name = "opendal" -version = "0.58.1" +version = "0.59.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f20562cc7447fcc915fc5c23df305a412ea80a733c9f2fd9e2d267e2815be6d" +checksum = "f950151f9587a51a7bed70a15fa0cff464eae96e41ae7499f97067bdafdf43eb" dependencies = [ - "ctor 1.0.12", + "ctor", "opendal-core", "opendal-http-transport-reqwest", "opendal-layer-concurrent-limit", @@ -6591,19 +6599,19 @@ dependencies = [ [[package]] name = "opendal-core" -version = "0.58.1" +version = "0.59.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec75551ff4cf3e57da98979f6a937aaa9ddb3915bf68cc17d03df733be6646ed" +checksum = "a43405d217dfdfb543f58847336d3af672897dd1939bb7dcf314b63cf364f1c9" dependencies = [ "anyhow", + "asyncband", "base64 0.23.1", "bytes", "futures", "http 1.5.0", "jiff", "log", - "md-5 0.11.0", - "mea", + "md-5", "percent-encoding", "quick-xml 0.41.0", "reqsign-core", @@ -6617,9 +6625,9 @@ dependencies = [ [[package]] name = "opendal-http-transport-reqwest" -version = "0.58.1" +version = "0.59.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad4d4f19c3ce01126a30611f8e544eaa217104a278c889ac17c9374fe4f9e4ef" +checksum = "401999057db611e592f883fcf2cbd6754ff37af587deaadd07b8c1398b2b6b06" dependencies = [ "bytes", "futures", @@ -6631,21 +6639,21 @@ dependencies = [ [[package]] name = "opendal-layer-concurrent-limit" -version = "0.58.1" +version = "0.59.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "249ac5b0aa5a7a6c3737342d10456067937f9c9a6f3f02544271f7908ab91081" +checksum = "fba1dd0742261925fc0eb910773ec39cbc1336c55d41e13b19f3af970ad5a126" dependencies = [ + "asyncband", "futures", "http 1.5.0", - "mea", "opendal-core", ] [[package]] name = "opendal-layer-logging" -version = "0.58.1" +version = "0.59.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c75411ab00f77851ff086b686c1e9ca8175ac18c15afa2cb75b9036436cb06c" +checksum = "d0fd963f9d32dd276521479d7f1f3a265d669b03a75f2062a4570a26b1b17421" dependencies = [ "log", "opendal-core", @@ -6653,9 +6661,9 @@ dependencies = [ [[package]] name = "opendal-layer-retry" -version = "0.58.1" +version = "0.59.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80b7738bd5f233ad8da39af9b9316b9b7a4eaddd91e8e32a1e19b7030688121d" +checksum = "06306202c97c54fb41bdbbdcb854c8798823f0022ae7aac7a40c8a1023aa83a6" dependencies = [ "backon", "log", @@ -6664,9 +6672,9 @@ dependencies = [ [[package]] name = "opendal-layer-timeout" -version = "0.58.1" +version = "0.59.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a704141924500f3803c05ed871b53305d2a2f11cb5ef20160c3ee688a1857f66" +checksum = "6bd334cbd0a0bc934146733e74a80a5faf8db8e014781a25f6d9d80d7b87c981" dependencies = [ "opendal-core", "tokio", @@ -6674,9 +6682,9 @@ dependencies = [ [[package]] name = "opendal-service-azblob" -version = "0.58.1" +version = "0.59.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3310fbbb48f111c6f590473c2cd15e1b7f8e384444b0d4e328f0464c864d767" +checksum = "ba0d2662ddf0de1f838db5dc48fafb8212ed1a5c7980bc164aa67809696c309c" dependencies = [ "base64 0.23.1", "bytes", @@ -6695,15 +6703,15 @@ dependencies = [ [[package]] name = "opendal-service-azdls" -version = "0.58.1" +version = "0.59.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e3c406729935fe214ce574d68681a1ff7e0b322548f14094912bdbfe50e5c53" +checksum = "9064ed464286bffbea5955d470082a1e72b9b89f1e8f7a61b9e303c55ec08e4f" dependencies = [ + "asyncband", "base64 0.23.1", "bytes", "http 1.5.0", "log", - "mea", "opendal-core", "opendal-service-azure-common", "quick-xml 0.41.0", @@ -6716,9 +6724,9 @@ dependencies = [ [[package]] name = "opendal-service-azure-common" -version = "0.58.1" +version = "0.59.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7348c88edf15af435b7be930077746b569fac5e738c1bf6a363b675e7317c9df" +checksum = "6348e3c0d7ff77a9b05c5b7f3d744ed395c2b2511812d3b37a934218002248f8" dependencies = [ "http 1.5.0", "opendal-core", @@ -6726,9 +6734,9 @@ dependencies = [ [[package]] name = "opendal-service-cos" -version = "0.58.1" +version = "0.59.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d533d4582105d269c8aebeee5f0e8bcf960f41b8aab6197df7012254d9f39bf0" +checksum = "49a652eadc76b94f9cffa497b3de5250dff53cce394d2974c00dde62c4e4cd81" dependencies = [ "bytes", "http 1.5.0", @@ -6743,9 +6751,9 @@ dependencies = [ [[package]] name = "opendal-service-gcs" -version = "0.58.1" +version = "0.59.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "007f3fba63c21e516c956b891e96ff9892d8175662bfb781cdada9d3766a11e6" +checksum = "1ccbf8450652bfe7b3ae69b7decce090c95c120ad565048f9a5a77dac2917a19" dependencies = [ "async-trait", "bytes", @@ -6760,13 +6768,14 @@ dependencies = [ "serde", "serde_json", "tokio", + "uuid", ] [[package]] name = "opendal-service-goosefs" -version = "0.58.1" +version = "0.59.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60871e6386f04d831e6a5bdbc032af4a91aeba49963252d0ef456a2cf36a9b78" +checksum = "4b53d8c3e1db3176add7aff9ad2637c907409bac19e025ee8e9c072c0061d9c5" dependencies = [ "bytes", "goosefs-sdk", @@ -6778,10 +6787,11 @@ dependencies = [ [[package]] name = "opendal-service-hf" -version = "0.58.1" +version = "0.59.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b41fd41eb7ed03c5e66cefda61e8e117808ffd2908f2916737cb020a6beb02c7" +checksum = "5c17b59cf22bd2da9f751b8e66db595d5fe546b4c6b6ff0fb84c7bfd27455d7a" dependencies = [ + "asyncband", "bytes", "hf-xet", "http 1.5.0", @@ -6794,9 +6804,9 @@ dependencies = [ [[package]] name = "opendal-service-oss" -version = "0.58.1" +version = "0.59.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd528ec2d49c5ca69e674ffed7b3e0686fb9cfcfea0596870de381467fda4f1b" +checksum = "284373c4a1143d8efaa7d010c1db05856d33a7cad475aaee8405dbcb0660cd96" dependencies = [ "bytes", "http 1.5.0", @@ -6811,16 +6821,16 @@ dependencies = [ [[package]] name = "opendal-service-s3" -version = "0.58.1" +version = "0.59.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58e80cdf192d7eff05feed747894d64f81905ac4eaf132edf7ea270abdd2d663" +checksum = "388b1d39b62535c62803754ebef89808859558697366dbedd0299345887ba461" dependencies = [ "base64 0.23.1", "bytes", "crc-fast", "http 1.5.0", "log", - "md-5 0.11.0", + "md-5", "opendal-core", "quick-xml 0.41.0", "reqsign-aws-v4", @@ -7633,7 +7643,7 @@ dependencies = [ "inferno", "libc", "log", - "nix", + "nix 0.26.4", "once_cell", "smallvec", "spin 0.10.1", @@ -7655,7 +7665,7 @@ dependencies = [ "inferno", "libc", "log", - "nix", + "nix 0.26.4", "once_cell", "smallvec", "spin 0.10.1", @@ -7915,16 +7925,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "quick-xml" -version = "0.39.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdcc8dd4e2f670d309a5f0e83fe36dfdc05af317008fea29144da1a2ac858e5e" -dependencies = [ - "memchr", - "serde", -] - [[package]] name = "quick-xml" version = "0.40.1" @@ -7969,7 +7969,7 @@ dependencies = [ "quinn-udp", "rustc-hash", "rustls 0.23.40", - "socket2 0.6.3", + "socket2 0.5.10", "thiserror 2.0.18", "tokio", "tracing", @@ -8007,7 +8007,7 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.6.3", + "socket2 0.5.10", "tracing", "windows-sys 0.60.2", ] @@ -8382,9 +8382,9 @@ dependencies = [ [[package]] name = "reqsign-aws-core" -version = "3.0.3" +version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4af084e1f3cbf3e67e0c972765399bce54ecec804cceba46b39a8331f3c1bff" +checksum = "bac4749b7dfa7bfaccd01eb03e9dc795ed37e3f20d6f0f38e2c67ee85ad6bc86" dependencies = [ "bytes", "form_urlencoded", @@ -8403,9 +8403,9 @@ dependencies = [ [[package]] name = "reqsign-aws-v4" -version = "3.1.0" +version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ac5b3b7cefa28933792b439186459f77f19f9b6edbeab41b8b187150361a206" +checksum = "ff250f0fd0b913fbd565e405acc553da0f13bde30bfb5403178c9d0313cdc15f" dependencies = [ "bytes", "http 1.5.0", @@ -8439,9 +8439,9 @@ dependencies = [ [[package]] name = "reqsign-core" -version = "3.2.1" +version = "3.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c07dd510b1e1b9b241883e483358147fb2ed2d497a7b39b065ba61eb93deceb0" +checksum = "ff052daffb0599681c50f85c59e7236438976efe991ab864edd9f3b235501a0f" dependencies = [ "anyhow", "base64 0.23.1", @@ -8452,6 +8452,7 @@ dependencies = [ "http 1.5.0", "jiff", "log", + "mea", "percent-encoding", "rsa", "serde", @@ -8463,9 +8464,9 @@ dependencies = [ [[package]] name = "reqsign-file-read-tokio" -version = "3.0.4" +version = "3.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "663d9d55abd0df0830ef0ae43708297cc1371cf4e8ca91f3ac813c309cca8c98" +checksum = "b3235df90a6bca681aa47dd86f2393d122a6d77042aa8a7c81e218cd45c5bfc0" dependencies = [ "anyhow", "reqsign-core", @@ -8474,10 +8475,11 @@ dependencies = [ [[package]] name = "reqsign-google" -version = "3.0.4" +version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4080a227f82a09f68540ecd028622065d7ac4c0bcb8727a25bdcfc0526235792" +checksum = "272a5813571885c1455ffe106f0c768577e6ce69313446d4476e8d41dcf6ac5a" dependencies = [ + "bytes", "form_urlencoded", "http 1.5.0", "log", @@ -8561,6 +8563,7 @@ dependencies = [ "bytes", "futures-core", "futures-util", + "h2 0.4.16", "http 1.5.0", "http-body 1.1.0", "http-body-util", @@ -8787,7 +8790,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -8858,7 +8861,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -9255,7 +9258,6 @@ dependencies = [ "cfg-if 1.0.4", "cpufeatures 0.2.17", "digest 0.10.7", - "sha2-asm", ] [[package]] @@ -9269,15 +9271,6 @@ dependencies = [ "digest 0.11.3", ] -[[package]] -name = "sha2-asm" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b845214d6175804686b2bd482bcffe96651bb2d1200742b712003504a2dac1ab" -dependencies = [ - "cc", -] - [[package]] name = "sharded-slab" version = "0.1.7" @@ -9450,7 +9443,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -9557,7 +9550,7 @@ dependencies = [ "cfg-if 1.0.4", "libc", "psm", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -9849,7 +9842,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -10134,6 +10127,30 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio_with_wasm" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34e40fbbbd95441133fe9483f522db15dbfd26dc636164ebd8f2dd28759a6aa6" +dependencies = [ + "js-sys", + "tokio", + "tokio_with_wasm_proc", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "tokio_with_wasm_proc" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d01145a2c788d6aae4cd653afec1e8332534d7d783d01897cefcafe4428de992" +dependencies = [ + "quote", + "syn 2.0.117", +] + [[package]] name = "toml_datetime" version = "1.1.1+spec-1.1.0" @@ -10829,7 +10846,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -11263,20 +11280,18 @@ dependencies = [ [[package]] name = "xet-client" -version = "1.5.2" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e1e496dcbe6a09017acdfaf48e1a646735e7ff5b2a49e2c7e081cca77a59bc8" +checksum = "c3b8da8cc70aa2e3c500c0400e012df82c656ab9fca47f9f939fffc5afd89aca" dependencies = [ "anyhow", "async-trait", "base64 0.22.1", "bytes", - "clap", "crc32fast", "futures", "http 1.5.0", "hyper 1.9.0", - "lazy_static", "more-asserts", "rand 0.10.1", "redb", @@ -11290,8 +11305,8 @@ dependencies = [ "thiserror 2.0.18", "tokio", "tokio-retry", + "tokio_with_wasm", "tracing", - "tracing-subscriber", "url", "urlencoding", "web-time", @@ -11301,24 +11316,21 @@ dependencies = [ [[package]] name = "xet-core-structures" -version = "1.5.2" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb838aa8eb67d730af301584cf003caad407487606058292a6750711b603fbee" +checksum = "73503c223783dccc864abde22115e09d12f190448a0baf58ab2c54bc709e2f99" dependencies = [ "async-trait", "base64 0.22.1", "blake3", "bytemuck", "bytes", - "clap", "countio", - "csv", "futures", "futures-util", "getrandom 0.4.2", "heapify", "itertools 0.14.0", - "lazy_static", "lz4_flex", "more-asserts", "rand 0.10.1", @@ -11326,7 +11338,6 @@ dependencies = [ "safe-transmute", "serde", "static_assertions", - "tempfile", "thiserror 2.0.18", "tokio", "tokio-util", @@ -11338,32 +11349,31 @@ dependencies = [ [[package]] name = "xet-data" -version = "1.5.2" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67fd409bef621411a9d9013798540bb8036cb2678f03ab39af89a5e88034ed8c" +checksum = "c89052ec5dec2187cad30b86af92cc24fd61c4a57a795f1ff7ff5f38d49184eb" dependencies = [ "anyhow", "async-trait", "bytes", "chrono", - "clap", "gearhash", "http 1.5.0", "itertools 0.14.0", - "lazy_static", "more-asserts", "rand 0.10.1", "serde", "serde_json", - "sha2 0.10.9", + "sha2 0.11.0", "tempfile", "thiserror 2.0.18", "tokio", "tokio-util", + "tokio_with_wasm", "tracing", "url", "uuid", - "walkdir", + "web-time", "xet-client", "xet-core-structures", "xet-runtime", @@ -11371,9 +11381,9 @@ dependencies = [ [[package]] name = "xet-runtime" -version = "1.5.2" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15d8f121c33866f7648b737abe70d0e2dd9c0af4ffdd7219207531d0283aa63d" +checksum = "af5c60d5eed38ab4c576f4421bae835e7bd07631fb381705605529d2015c106b" dependencies = [ "anyhow", "async-trait", @@ -11381,13 +11391,12 @@ dependencies = [ "chrono", "colored", "const-str", - "ctor 0.6.3", + "ctor", "dirs", "futures", "git-version", "humantime", "konst", - "lazy_static", "libc", "more-asserts", "oneshot", @@ -11401,9 +11410,11 @@ dependencies = [ "thiserror 2.0.18", "tokio", "tokio-util", + "tokio_with_wasm", "tracing", "tracing-appender", "tracing-subscriber", + "web-time", "whoami", "winapi", ] diff --git a/Cargo.toml b/Cargo.toml index a0a908489..2b16a6bf0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,20 +13,20 @@ categories = ["database-implementations"] rust-version = "1.91.0" [workspace.dependencies] -lance = { "version" = "=12.0.0-beta.16", default-features = false, "tag" = "v12.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" } -lance-core = { "version" = "=12.0.0-beta.16", "tag" = "v12.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" } -lance-datagen = { "version" = "=12.0.0-beta.16", "tag" = "v12.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" } -lance-file = { "version" = "=12.0.0-beta.16", "tag" = "v12.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" } -lance-io = { "version" = "=12.0.0-beta.16", default-features = false, "tag" = "v12.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" } -lance-index = { "version" = "=12.0.0-beta.16", "tag" = "v12.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" } -lance-linalg = { "version" = "=12.0.0-beta.16", "tag" = "v12.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace = { "version" = "=12.0.0-beta.16", "tag" = "v12.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace-impls = { "version" = "=12.0.0-beta.16", default-features = false, "tag" = "v12.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" } -lance-table = { "version" = "=12.0.0-beta.16", "tag" = "v12.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" } -lance-testing = { "version" = "=12.0.0-beta.16", "tag" = "v12.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" } -lance-datafusion = { "version" = "=12.0.0-beta.16", "tag" = "v12.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" } -lance-encoding = { "version" = "=12.0.0-beta.16", "tag" = "v12.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" } -lance-arrow = { "version" = "=12.0.0-beta.16", "tag" = "v12.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" } +lance = { "version" = "=12.0.0-beta.17", default-features = false, "tag" = "v12.0.0-beta.17", "git" = "https://github.com/lance-format/lance.git" } +lance-core = { "version" = "=12.0.0-beta.17", "tag" = "v12.0.0-beta.17", "git" = "https://github.com/lance-format/lance.git" } +lance-datagen = { "version" = "=12.0.0-beta.17", "tag" = "v12.0.0-beta.17", "git" = "https://github.com/lance-format/lance.git" } +lance-file = { "version" = "=12.0.0-beta.17", "tag" = "v12.0.0-beta.17", "git" = "https://github.com/lance-format/lance.git" } +lance-io = { "version" = "=12.0.0-beta.17", default-features = false, "tag" = "v12.0.0-beta.17", "git" = "https://github.com/lance-format/lance.git" } +lance-index = { "version" = "=12.0.0-beta.17", "tag" = "v12.0.0-beta.17", "git" = "https://github.com/lance-format/lance.git" } +lance-linalg = { "version" = "=12.0.0-beta.17", "tag" = "v12.0.0-beta.17", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace = { "version" = "=12.0.0-beta.17", "tag" = "v12.0.0-beta.17", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace-impls = { "version" = "=12.0.0-beta.17", default-features = false, "tag" = "v12.0.0-beta.17", "git" = "https://github.com/lance-format/lance.git" } +lance-table = { "version" = "=12.0.0-beta.17", "tag" = "v12.0.0-beta.17", "git" = "https://github.com/lance-format/lance.git" } +lance-testing = { "version" = "=12.0.0-beta.17", "tag" = "v12.0.0-beta.17", "git" = "https://github.com/lance-format/lance.git" } +lance-datafusion = { "version" = "=12.0.0-beta.17", "tag" = "v12.0.0-beta.17", "git" = "https://github.com/lance-format/lance.git" } +lance-encoding = { "version" = "=12.0.0-beta.17", "tag" = "v12.0.0-beta.17", "git" = "https://github.com/lance-format/lance.git" } +lance-arrow = { "version" = "=12.0.0-beta.17", "tag" = "v12.0.0-beta.17", "git" = "https://github.com/lance-format/lance.git" } lancedb = { path = "rust/lancedb", default-features = false } ahash = "0.8" # Note that this one does not include pyarrow @@ -60,7 +60,7 @@ log = "0.4" metrics = "0.24" metrics-util = "0.19" moka = { version = "0.12", features = ["future"] } -object_store = "0.13.2" +object_store = "0.14.1" pin-project = "1.0.7" rand = "0.9" snafu = "0.8" diff --git a/java/pom.xml b/java/pom.xml index b3dbe413c..7cbb819d9 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -28,7 +28,7 @@ UTF-8 15.0.0 - 12.0.0-beta.16 + 12.0.0-beta.17 false 2.30.0 1.7 From 6702e3fec1a2de38aa9e1c04fb69bcd93bfd210a Mon Sep 17 00:00:00 2001 From: Drew Date: Fri, 11 Sep 2026 14:46:27 -0700 Subject: [PATCH 2/8] feat(node): add blob v2 fetch and field helpers (#4155) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit this PR blob v2 field helpers and reads to the Node SDK. `blob()` marks a field as blob v2 and lets you set the storage thresholds. Inputs can be bytes, a URI, or a data/uri struct. Queries return descriptors. `fetchBlobs()` reads the bytes by row ID, and `fetchBlobFiles()` gives you lazy handles for full or range reads. `blobColumns()` lists the blob fields, including nested ones. Fetch uses the table’s current checkout. It preserves order, duplicates, and nulls. Holding row IDs across compaction still requires stable row IDs. ```javascript const db = await connect("./data"); const video = await readFile("clip.mp4"); const table = await db.createTable( "videos", [{ id: 1n, video }], { schema: new Schema([ new Field("id", new Int64()), blob("video"), ]), }, ); const rows = await table.query().select(["id"]).withRowId().toArray(); const rowIds = rows.map((row) => row._rowid as bigint); const bytes = await table.fetchBlobs("video", rowIds); const [handle] = await table.fetchBlobFiles("video", rowIds); const header = await handle!.readRange(0n, 65536n); ``` ### Testing - cover input validation, thresholds, nested fields, fetch ordering, nulls, and range reads. --- docs/src/js/classes/BlobFile.md | 62 ++++++ docs/src/js/classes/Table.md | 62 ++++++ docs/src/js/functions/blob.md | 55 +++++ docs/src/js/functions/isBlobField.md | 22 ++ docs/src/js/globals.md | 4 + docs/src/js/type-aliases/BlobOptions.md | 48 +++++ nodejs/__test__/blob.test.ts | 185 ++++++++++++++++ nodejs/__test__/table.test.ts | 271 ++++++++++++++++++++++++ nodejs/lancedb/arrow.ts | 103 ++++++++- nodejs/lancedb/blob.ts | 236 +++++++++++++++++++++ nodejs/lancedb/index.ts | 3 + nodejs/lancedb/table.ts | 91 ++++++-- nodejs/src/blob.rs | 95 +++++++++ nodejs/src/lib.rs | 1 + nodejs/src/table.rs | 39 ++++ 15 files changed, 1260 insertions(+), 17 deletions(-) create mode 100644 docs/src/js/classes/BlobFile.md create mode 100644 docs/src/js/functions/blob.md create mode 100644 docs/src/js/functions/isBlobField.md create mode 100644 docs/src/js/type-aliases/BlobOptions.md create mode 100644 nodejs/__test__/blob.test.ts create mode 100644 nodejs/lancedb/blob.ts create mode 100644 nodejs/src/blob.rs diff --git a/docs/src/js/classes/BlobFile.md b/docs/src/js/classes/BlobFile.md new file mode 100644 index 000000000..84a596d1a --- /dev/null +++ b/docs/src/js/classes/BlobFile.md @@ -0,0 +1,62 @@ +[**@lancedb/lancedb**](../README.md) • **Docs** + +*** + +[@lancedb/lancedb](../globals.md) / BlobFile + +# Class: BlobFile + +A lazy handle to blob bytes. Create one with [Table.fetchBlobFiles](Table.md#fetchblobfiles). + +## Methods + +### read() + +```ts +read(): Promise +``` + +Reads from the cursor to the end and advances the cursor. + +A second call returns an empty buffer. [BlobFile.readRange](BlobFile.md#readrange) does +not move the cursor. + +#### Returns + +`Promise`<`Buffer`> + +*** + +### readRange() + +```ts +readRange(start, end): Promise +``` + +Reads the half-open byte range `[start, end)`. + +Fails when `end` is past the blob size. Does not move the cursor. + +#### Parameters + +* **start**: `bigint` + +* **end**: `bigint` + +#### Returns + +`Promise`<`Buffer`> + +*** + +### size() + +```ts +size(): bigint +``` + +Returns the blob size in bytes. + +#### Returns + +`bigint` diff --git a/docs/src/js/classes/Table.md b/docs/src/js/classes/Table.md index 3847f3a39..ef6e9535a 100644 --- a/docs/src/js/classes/Table.md +++ b/docs/src/js/classes/Table.md @@ -137,6 +137,20 @@ containing the new version number of the table after altering the columns. *** +### blobColumns() + +```ts +abstract blobColumns(): Promise +``` + +Blob v2 columns, including nested dotted paths. + +#### Returns + +`Promise`<`string`[]> + +*** + ### branches() ```ts @@ -499,6 +513,54 @@ Drop an index from the table. *** +### fetchBlobFiles() + +```ts +abstract fetchBlobFiles(column, rowIds): Promise<(null | BlobFile)[]> +``` + +Opens lazy blob handles for `column` at the given row IDs using the +table's current checkout. + +Preserves input order, duplicates, and nulls. Use this for large payloads. +See [Table.fetchBlobs](Table.md#fetchblobs) for row-ID validity across versions. + +#### Parameters + +* **column**: `string` + +* **rowIds**: readonly (`number` \| `bigint`)[] + +#### Returns + +`Promise`<(`null` \| [`BlobFile`](BlobFile.md))[]> + +*** + +### fetchBlobs() + +```ts +abstract fetchBlobs(column, rowIds): Promise<(null | Buffer)[]> +``` + +Bytes for `column` at row IDs from [Query.withRowId](Query.md#withrowid). + +Reads the table's current checkout. IDs from another version can fail after +compaction unless stable row ids are enabled. Results keep input order and +duplicates. Null blobs are `null`. Empty blobs are empty buffers. + +#### Parameters + +* **column**: `string` + +* **rowIds**: readonly (`number` \| `bigint`)[] + +#### Returns + +`Promise`<(`null` \| `Buffer`)[]> + +*** + ### flushLsm() ```ts diff --git a/docs/src/js/functions/blob.md b/docs/src/js/functions/blob.md new file mode 100644 index 000000000..20a734cd4 --- /dev/null +++ b/docs/src/js/functions/blob.md @@ -0,0 +1,55 @@ +[**@lancedb/lancedb**](../README.md) • **Docs** + +*** + +[@lancedb/lancedb](../globals.md) / blob + +# Function: blob() + +```ts +function blob(name, options): Field +``` + +Declares a `lance.blob.v2` column. + +Query results are descriptors, not payload bytes. Use [Table.fetchBlobs](../classes/Table.md#fetchblobs) +or [Table.fetchBlobFiles](../classes/Table.md#fetchblobfiles) to read bytes. + +## Parameters + +* **name**: `string` + +* **options**: [`BlobOptions`](../type-aliases/BlobOptions.md) = `{}` + +## Returns + +`Field` + +## Example + +```ts +import { readFile } from "node:fs/promises"; +import { Field, Int64, Schema } from "apache-arrow"; +import { blob, connect } from "@lancedb/lancedb"; + +const db = await connect("./data"); +const video = await readFile("clip.mp4"); +const table = await db.createTable( + "videos", + [{ id: 1n, video }], + { + schema: new Schema([ + new Field("id", new Int64()), + blob("video"), + ]), + }, +); + +const rows = await table.query().select(["id"]).withRowId().toArray(); +const rowIds = rows.map((row) => row._rowid as bigint); +const bytes = await table.fetchBlobs("video", rowIds); + +const [handle] = await table.fetchBlobFiles("video", rowIds); +const size = handle!.size(); +const header = await handle!.readRange(0n, size < 65536n ? size : 65536n); +``` diff --git a/docs/src/js/functions/isBlobField.md b/docs/src/js/functions/isBlobField.md new file mode 100644 index 000000000..944309f90 --- /dev/null +++ b/docs/src/js/functions/isBlobField.md @@ -0,0 +1,22 @@ +[**@lancedb/lancedb**](../README.md) • **Docs** + +*** + +[@lancedb/lancedb](../globals.md) / isBlobField + +# Function: isBlobField() + +```ts +function isBlobField(field): boolean +``` + +Checks for the `lance.blob.v2` extension marker. Does not validate the +field's storage type. + +## Parameters + +* **field**: `Field`<`any`> + +## Returns + +`boolean` diff --git a/docs/src/js/globals.md b/docs/src/js/globals.md index eb0fc7d5a..4a5effaae 100644 --- a/docs/src/js/globals.md +++ b/docs/src/js/globals.md @@ -19,6 +19,7 @@ ## Classes - [AutoQuery](classes/AutoQuery.md) +- [BlobFile](classes/BlobFile.md) - [BooleanQuery](classes/BooleanQuery.md) - [BoostQuery](classes/BoostQuery.md) - [BranchContents](classes/BranchContents.md) @@ -143,6 +144,7 @@ - [AnalyzePlanDistributedMetrics](type-aliases/AnalyzePlanDistributedMetrics.md) - [BaseTokenizer](type-aliases/BaseTokenizer.md) +- [BlobOptions](type-aliases/BlobOptions.md) - [Data](type-aliases/Data.md) - [DataLike](type-aliases/DataLike.md) - [FieldLike](type-aliases/FieldLike.md) @@ -158,9 +160,11 @@ ## Functions - [RecordBatchIterator](functions/RecordBatchIterator.md) +- [blob](functions/blob.md) - [connect](functions/connect.md) - [connectNamespace](functions/connectNamespace.md) - [instrumentLanceDbMetrics](functions/instrumentLanceDbMetrics.md) +- [isBlobField](functions/isBlobField.md) - [makeArrowTable](functions/makeArrowTable.md) - [packBits](functions/packBits.md) - [permutationBuilder](functions/permutationBuilder.md) diff --git a/docs/src/js/type-aliases/BlobOptions.md b/docs/src/js/type-aliases/BlobOptions.md new file mode 100644 index 000000000..41dbe92c2 --- /dev/null +++ b/docs/src/js/type-aliases/BlobOptions.md @@ -0,0 +1,48 @@ +[**@lancedb/lancedb**](../README.md) • **Docs** + +*** + +[@lancedb/lancedb](../globals.md) / BlobOptions + +# Type Alias: BlobOptions + +```ts +type BlobOptions: object; +``` + +## Type declaration + +### dedicatedSizeThreshold? + +```ts +optional dedicatedSizeThreshold: number; +``` + +Max payload bytes stored in a packed sidecar before a dedicated file. Must +be a positive safe integer. + +### inlineSizeThreshold? + +```ts +optional inlineSizeThreshold: number; +``` + +Max payload bytes kept inline in the data file. Zero is allowed. Must be a +safe integer. + +### nullable? + +```ts +optional nullable: boolean; +``` + +Defaults to true. + +### packFileSizeThreshold? + +```ts +optional packFileSizeThreshold: number; +``` + +Max bytes in one packed sidecar before starting another. Must be a positive +safe integer. diff --git a/nodejs/__test__/blob.test.ts b/nodejs/__test__/blob.test.ts new file mode 100644 index 000000000..e8e9357e3 --- /dev/null +++ b/nodejs/__test__/blob.test.ts @@ -0,0 +1,185 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +import { Field, Int64, List, Schema, Struct, Utf8 } from "apache-arrow"; +import { makeArrowTable } from "../lancedb/arrow"; +import { BlobFile, blob, coerceBlobValue, isBlobField } from "../lancedb/blob"; + +describe("blob()", () => { + it("marks the field as lance.blob.v2", () => { + const field = blob("image", { nullable: false }); + expect(field.nullable).toBe(false); + expect(isBlobField(field)).toBe(true); + expect(field.metadata.get("ARROW:extension:name")).toBe("lance.blob.v2"); + }); + + it("writes encoding thresholds as field metadata", () => { + const field = blob("video", { + inlineSizeThreshold: 1024, + dedicatedSizeThreshold: 2 * 1024 * 1024, + packFileSizeThreshold: 64 * 1024 * 1024, + }); + expect( + field.metadata.get("lance-encoding:blob-inline-size-threshold"), + ).toBe("1024"); + expect( + field.metadata.get("lance-encoding:blob-dedicated-size-threshold"), + ).toBe(String(2 * 1024 * 1024)); + expect( + field.metadata.get("lance-encoding:blob-pack-file-size-threshold"), + ).toBe(String(64 * 1024 * 1024)); + }); + + it("rejects invalid thresholds", () => { + expect(() => blob("image", { inlineSizeThreshold: -1 })).toThrow( + /inlineSizeThreshold must be non-negative/, + ); + expect(() => blob("image", { dedicatedSizeThreshold: 0 })).toThrow( + /dedicatedSizeThreshold must be positive/, + ); + expect(() => blob("image", { packFileSizeThreshold: 1.5 })).toThrow( + /packFileSizeThreshold must be a safe integer/, + ); + expect(() => + blob("image", { dedicatedSizeThreshold: Number.MAX_SAFE_INTEGER + 1 }), + ).toThrow(/dedicatedSizeThreshold must be a safe integer/); + }); +}); + +describe("coerceBlobValue", () => { + it.each([ + ["Buffer", Buffer.from("x"), { data: Buffer.from("x"), uri: null }], + [ + "Uint8Array", + new Uint8Array([120]), + { data: new Uint8Array([120]), uri: null }, + ], + ["URI string", "s3://bucket/key", { data: null, uri: "s3://bucket/key" }], + [ + "data struct", + { data: Buffer.from("y") }, + { data: Buffer.from("y"), uri: null }, + ], + [ + "uri struct", + { uri: "s3://bucket/key" }, + { data: null, uri: "s3://bucket/key" }, + ], + ["null", null, null], + ])("accepts %s", (_name, input, expected) => { + expect(coerceBlobValue(input)).toEqual(expected); + }); + + it.each([ + ["empty URI", "", /uri cannot be empty/], + ["object without data or uri", { position: 0 }, /data' or 'uri/], + [ + "Int16Array", + new Int16Array([1]), + /Blob data must be Buffer or Uint8Array/, + ], + [ + "both data and uri", + { data: Buffer.from("y"), uri: "s3://bucket/key" }, + /exactly one of 'data' or 'uri'/, + ], + [ + "neither data nor uri", + { data: null, uri: null }, + /exactly one of 'data' or 'uri'/, + ], + ])("rejects %s", (_name, input, message) => { + expect(() => coerceBlobValue(input)).toThrow(message); + }); +}); + +describe("BlobFile", () => { + it("rejects constructing BlobFile without a native handle", () => { + expect(() => new (BlobFile as unknown as { new (): BlobFile })()).toThrow( + /fetchBlobFiles/, + ); + }); +}); + +describe("makeArrowTable blob columns", () => { + it("coerces Buffer input onto a blob field", () => { + const schema = new Schema([ + new Field("id", new Int64(), true), + blob("image"), + ]); + const table = makeArrowTable([{ id: 1n, image: Buffer.from("hello") }], { + schema, + }); + expect(isBlobField(table.schema.fields[1])).toBe(true); + const image = table.getChild("image")!; + expect(image.nullCount).toBe(0); + expect(image.getChild("uri")!.get(0)).toBeNull(); + expect(image.getChild("data")!.nullCount).toBe(0); + expect(Buffer.from(image.getChild("data")!.get(0)!).toString()).toBe( + "hello", + ); + }); + + it("coerces Buffer elements inside a list and keeps null slots", () => { + const schema = new Schema([ + new Field("id", new Int64(), true), + new Field("images", new List(blob("image")), true), + ]); + const table = makeArrowTable( + [ + { id: 1n, images: [Buffer.from("a"), Buffer.from("bb")] }, + { id: 2n, images: null }, + { id: 3n, images: [Buffer.from("c"), null] }, + { id: 4n, images: [] }, + ], + { schema }, + ); + const images = table.getChild("images")!; + expect(images.nullCount).toBe(1); + const rows = images.toArray(); + expect(rows[1]).toBeNull(); + expect(Array.from(rows[3] as Iterable)).toHaveLength(0); + const first = Array.from(rows[0] as Iterable<{ data: Uint8Array | null }>); + expect(Buffer.from(first[0].data!).toString()).toBe("a"); + expect(Buffer.from(first[1].data!).toString()).toBe("bb"); + const third = Array.from( + rows[2] as Iterable<{ data: Uint8Array | null } | null>, + ); + expect(Buffer.from(third[0]!.data!).toString()).toBe("c"); + expect(third[1]).toBeNull(); + }); + + it("coerces Buffer fields inside list structs", () => { + const schema = new Schema([ + new Field("id", new Int64(), true), + new Field( + "items", + new List( + new Field( + "item", + new Struct([new Field("name", new Utf8(), true), blob("image")]), + true, + ), + ), + true, + ), + ]); + const table = makeArrowTable( + [ + { + id: 1n, + items: [{ name: "one", image: Buffer.from("alpha") }], + }, + ], + { schema }, + ); + const items = Array.from( + table.getChild("items")!.toArray()[0] as Iterable<{ + name: string; + image: { data: Uint8Array | null }; + }>, + ); + expect(items[0].name).toBe("one"); + expect(Buffer.from(items[0].image.data!).toString()).toBe("alpha"); + }); +}); diff --git a/nodejs/__test__/table.test.ts b/nodejs/__test__/table.test.ts index e8b97bf77..cae01d9d5 100644 --- a/nodejs/__test__/table.test.ts +++ b/nodejs/__test__/table.test.ts @@ -18,6 +18,7 @@ import { Query, Table, VectorQuery, + blob, connect, tokenize, } from "../lancedb"; @@ -2401,6 +2402,276 @@ describe("when dealing with versioning", () => { }); }); +describe("when dealing with blob columns", () => { + let tmpDir: tmp.DirResult; + beforeEach(() => { + tmpDir = tmp.dirSync({ unsafeCleanup: true }); + }); + afterEach(() => { + tmpDir.removeCallback(); + }); + + it("discovers blob columns", async () => { + const { table } = await openBlobTable(); + expect(await table.blobColumns()).toEqual(["image"]); + }); + + it("preserves order, duplicates, and nulls", async () => { + const { table, rowIds } = await openBlobTable(); + const [alphaId, betaId, nullId] = rowIds; + const bytes = await table.fetchBlobs("image", [ + betaId, + alphaId, + betaId, + nullId, + ]); + expect(bytes.map((b) => (b == null ? null : b.toString()))).toEqual([ + "beta", + "alpha", + "beta", + null, + ]); + const files = await table.fetchBlobFiles("image", [ + betaId, + nullId, + alphaId, + ]); + expect(files.map((f) => f == null)).toEqual([false, true, false]); + }); + + it("reads full blob contents", async () => { + const { table, rowIds, alpha, beta } = await openBlobTable(); + const bytes = await table.fetchBlobs("image", rowIds); + expect(bytes[0]!.equals(alpha)).toBe(true); + expect(bytes[1]!.equals(beta)).toBe(true); + const files = await table.fetchBlobFiles("image", rowIds); + expect(files[0]!.size()).toBe(BigInt(alpha.length)); + expect(Buffer.from(await files[0]!.read()).toString()).toBe("alpha"); + expect(Buffer.from(await files[1]!.read()).toString()).toBe("beta"); + }); + + it("reads a half-open range", async () => { + const { table, rowIds } = await openBlobTable(); + const files = await table.fetchBlobFiles("image", rowIds); + expect(Buffer.from(await files[0]!.readRange(0n, 2n)).toString()).toBe( + "al", + ); + }); + + it("readRange does not move the cursor", async () => { + const { table, rowIds, alpha } = await openBlobTable(); + const [handle] = await table.fetchBlobFiles("image", rowIds); + expect((await handle!.readRange(1n, 3n)).toString()).toBe("lp"); + expect(await handle!.read()).toEqual(alpha); + expect(await handle!.read()).toEqual(Buffer.alloc(0)); + }); + + it("fails when readRange end is past the blob size", async () => { + const { table, rowIds, alpha } = await openBlobTable(); + const files = await table.fetchBlobFiles("image", rowIds); + await expect( + files[0]!.readRange(0n, BigInt(alpha.length + 1)), + ).rejects.toThrow(/exceeds blob size/); + }); + + it("rejects fetchBlobs on a non-blob column", async () => { + const { table, rowIds } = await openBlobTable(); + await expect(table.fetchBlobs("id", rowIds)).rejects.toThrow(/blob/i); + }); + + it("discovers and fetches nested blob columns", async () => { + const db = await connect(tmpDir.name); + const schema = new Schema([ + new Field("id", new Int64(), true), + new Field("info", new Struct([blob("image")]), true), + ]); + const payload = Buffer.from("nested"); + const table = await db.createTable( + "nested_blobs", + [{ id: 1n, info: { image: payload } }], + { schema }, + ); + expect(await table.blobColumns()).toEqual(["info.image"]); + const rows = await table.query().withRowId().toArray(); + const bytes = await table.fetchBlobs("info.image", [ + rows[0]._rowid as bigint, + ]); + expect(bytes[0]!.equals(payload)).toBe(true); + }); + + it("creates and adds list blob columns", async () => { + const db = await connect(tmpDir.name); + const schema = new Schema([ + new Field("id", new Int64(), true), + new Field("images", new List(blob("image")), true), + ]); + const alpha = Buffer.from("alpha"); + const beta = Buffer.from("beta"); + const gamma = Buffer.from("gamma"); + const table = await db.createTable( + "list_blobs", + [{ id: 1n, images: [alpha, beta] }], + { schema }, + ); + await table.add([ + { id: 2n, images: null }, + { id: 3n, images: [gamma, null] }, + { id: 4n, images: [] }, + ]); + expect(await table.blobColumns()).toEqual(["images.image"]); + const rows = await table.query().toArray(); + const byId = new Map(rows.map((row) => [Number(row.id), row])); + expect(descriptorSizes(byId.get(1)!.images)).toEqual([ + alpha.length, + beta.length, + ]); + expect(byId.get(2)!.images).toBeNull(); + expect(descriptorSizes(byId.get(3)!.images)).toEqual([gamma.length, null]); + expect(Array.from(byId.get(4)!.images as Iterable)).toHaveLength( + 0, + ); + }); + + it("creates and adds list struct blob columns", async () => { + const db = await connect(tmpDir.name); + const schema = new Schema([ + new Field("id", new Int64(), true), + new Field( + "items", + new List( + new Field( + "item", + new Struct([new Field("name", new Utf8(), true), blob("image")]), + true, + ), + ), + true, + ), + ]); + const alpha = Buffer.from("nested-alpha"); + const beta = Buffer.from("nested-beta"); + const table = await db.createTable( + "list_struct_blobs", + [{ id: 1n, items: [{ name: "one", image: alpha }] }], + { schema }, + ); + await table.add([ + { + id: 2n, + items: [ + { name: "two", image: beta }, + { name: "three", image: null }, + ], + }, + ]); + const rows = await table.query().toArray(); + const byId = new Map(rows.map((row) => [Number(row.id), row])); + expect( + descriptorSizes( + Array.from(byId.get(1)!.items as Iterable<{ image: unknown }>).map( + (item) => item.image, + ), + ), + ).toEqual([alpha.length]); + expect( + descriptorSizes( + Array.from(byId.get(2)!.items as Iterable<{ image: unknown }>).map( + (item) => item.image, + ), + ), + ).toEqual([beta.length, null]); + }); + + it("rejects blob fields inside a fixed-size list", async () => { + const db = await connect(tmpDir.name); + const schema = new Schema([ + new Field("id", new Int64(), true), + new Field("frames", new FixedSizeList(2, blob("frame")), true), + ]); + await expect( + db.createTable( + "fsl_blobs", + [{ id: 1n, frames: [Buffer.from("a"), Buffer.from("b")] }], + { schema }, + ), + ).rejects.toThrow( + "Blob fields inside FixedSizeList are not supported. Use List instead.", + ); + }); + + it("rejects blob fields inside a nested fixed-size list", async () => { + const db = await connect(tmpDir.name); + const schema = new Schema([ + new Field("id", new Int64(), true), + new Field( + "clip", + new Struct([ + new Field("frames", new FixedSizeList(2, blob("frame")), true), + ]), + true, + ), + ]); + await expect( + db.createTable( + "nested_fsl_blobs", + [ + { + id: 1n, + clip: { frames: [Buffer.from("a"), Buffer.from("b")] }, + }, + ], + { schema }, + ), + ).rejects.toThrow( + "Blob fields inside FixedSizeList are not supported. Use List instead.", + ); + }); + + it("rejects an Arrow table with blob fields inside a fixed-size list", async () => { + const db = await connect(tmpDir.name); + const schema = new Schema([ + new Field("id", new Int64(), true), + new Field("frames", new FixedSizeList(2, blob("frame")), true), + ]); + await expect( + db.createTable("fsl_blobs_ipc", new ArrowTable(schema)), + ).rejects.toThrow( + "Blob fields inside FixedSizeList are not supported. Use List instead.", + ); + }); + + function descriptorSizes(values: unknown): (number | null)[] { + return Array.from( + values as Iterable<{ size?: bigint | number } | null>, + ).map((value) => (value == null ? null : Number(value.size))); + } + + async function openBlobTable() { + const db = await connect(tmpDir.name); + const schema = new Schema([ + new Field("id", new Int64(), true), + blob("image"), + ]); + const alpha = Buffer.from("alpha"); + const beta = Buffer.from("beta"); + const table = await db.createTable( + "blobs", + [ + { id: 1n, image: alpha }, + { id: 2n, image: beta }, + { id: 3n, image: null }, + ], + { schema }, + ); + const rows = await table.query().withRowId().toArray(); + const rowIdById = new Map( + rows.map((r) => [Number(r.id), r._rowid as bigint]), + ); + const rowIds = [1, 2, 3].map((id) => rowIdById.get(id)!); + return { table, rowIds, alpha, beta }; + } +}); + describe("when dealing with tags", () => { let tmpDir: tmp.DirResult; beforeEach(() => { diff --git a/nodejs/lancedb/arrow.ts b/nodejs/lancedb/arrow.ts index 119887704..81b140da7 100644 --- a/nodejs/lancedb/arrow.ts +++ b/nodejs/lancedb/arrow.ts @@ -40,6 +40,7 @@ import { } from "apache-arrow"; import { Buffers } from "apache-arrow/data"; import { typedArrayToArrowType } from "./arrow_type"; +import { coerceBlobValue, isBlobField } from "./blob"; import { type EmbeddingFunction } from "./embedding/embedding_function"; import { EmbeddingFunctionConfig, @@ -430,12 +431,14 @@ export function makeArrowTable( throw new Error("A schema must be provided if data is empty"); } else { schema = new Schema(schema.fields, schemaMetadata); + validateBlobSchema(schema); return new ArrowTable(schema); } } let inferredSchema = inferSchema(data, schema, opt); inferredSchema = new Schema(inferredSchema.fields, schemaMetadata); + validateBlobSchema(inferredSchema); const finalColumns: Record = {}; for (const field of inferredSchema.fields) { @@ -445,6 +448,35 @@ export function makeArrowTable( return new ArrowTable(inferredSchema, finalColumns); } +function validateBlobSchema(schema: Schema): void { + for (const field of schema.fields) { + validateBlobField(field); + } +} + +function validateBlobField(field: Field): void { + if ( + isFixedSizeList(field.type) && + containsBlobField(field.type.children[0]) + ) { + throw new Error( + "Blob fields inside FixedSizeList are not supported. Use List instead.", + ); + } + for (const child of field.type.children ?? []) { + validateBlobField(child); + } +} + +function containsBlobField(field: Field): boolean { + if (isBlobField(field)) { + return true; + } + return (field.type.children ?? []).some((child: Field) => + containsBlobField(child), + ); +} + function isObject(value: unknown): value is Record { return ( typeof value === "object" && @@ -480,6 +512,32 @@ function transposeData( path: string[] = [], ): Vector { const valuesPath = [...path, field.name]; + if (isBlobField(field) && field.type instanceof Struct) { + const blobRows = data.map((datum) => + coerceBlobValue(valueAtPath(datum, valuesPath)), + ); + const childVectors = field.type.children.map((child) => { + const values = blobRows.map((row) => + row == null ? null : (row[child.name as "data" | "uri"] ?? null), + ); + return makeVector(values, child.type, undefined, child.nullable); + }); + const nullCount = blobRows.filter((row) => row === null).length; + const structData = makeData({ + type: field.type, + length: blobRows.length, + nullCount, + nullBitmap: + nullCount > 0 + ? arrowUtil.packBools(blobRows.map((row) => row !== null)) + : undefined, + children: childVectors.map((v) => v.data[0]), + }); + return arrowMakeVector(structData); + } + if (isList(field.type) && containsBlobField(field.type.children[0])) { + return transposeListData(data, field, valuesPath); + } const values = data.map((datum) => valueAtPath(datum, valuesPath)); if (field.type instanceof Struct) { const childFields = field.type.children; @@ -495,7 +553,7 @@ function transposeData( nullCount > 0 ? arrowUtil.packBools(values.map((value) => value !== null)) : undefined, - children: childVectors as unknown as ArrowData[], + children: childVectors.map((v) => v.data[0]), }); return arrowMakeVector(structData); } else { @@ -503,6 +561,48 @@ function transposeData( } } +function transposeListData( + data: Record[], + field: Field, + valuesPath: string[], +): Vector { + const listType = field.type as List; + const childField = listType.children[0]; + const lists = data.map((datum) => valueAtPath(datum, valuesPath)); + const flattened: Record[] = []; + const validity: boolean[] = []; + const offsets: number[] = [0]; + + for (const list of lists) { + if (list == null) { + validity.push(false); + offsets.push(flattened.length); + continue; + } + if (!Array.isArray(list)) { + throw new Error(`expected an array for list field '${field.name}'`); + } + validity.push(true); + for (const element of list) { + flattened.push({ [childField.name]: element }); + } + offsets.push(flattened.length); + } + + const childVector = transposeData(flattened, childField, []); + const nullCount = validity.filter((valid) => !valid).length; + return arrowMakeVector( + makeData({ + type: listType, + length: lists.length, + nullCount, + nullBitmap: nullCount > 0 ? arrowUtil.packBools(validity) : undefined, + valueOffsets: Int32Array.from(offsets), + child: childVector.data[0], + }), + ); +} + /** * Create an empty Arrow table with the provided schema */ @@ -952,6 +1052,7 @@ export async function fromTableToBuffer( schema = sanitizeSchema(schema); } const tableWithEmbeddings = await applyEmbeddings(table, embeddings, schema); + validateBlobSchema(tableWithEmbeddings.schema); const writer = RecordBatchFileWriter.writeAll(tableWithEmbeddings); return Buffer.from(await writer.toUint8Array()); } diff --git a/nodejs/lancedb/blob.ts b/nodejs/lancedb/blob.ts new file mode 100644 index 000000000..244faac09 --- /dev/null +++ b/nodejs/lancedb/blob.ts @@ -0,0 +1,236 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +import { Field, LargeBinary, Struct, Utf8 } from "apache-arrow"; +import { BlobFile as NativeBlobFile } from "./native"; + +const BLOB_V2_EXTENSION_NAME = "lance.blob.v2"; + +const INLINE_SIZE_THRESHOLD_KEY = "lance-encoding:blob-inline-size-threshold"; +const DEDICATED_SIZE_THRESHOLD_KEY = + "lance-encoding:blob-dedicated-size-threshold"; +const PACK_FILE_SIZE_THRESHOLD_KEY = + "lance-encoding:blob-pack-file-size-threshold"; + +export type BlobInput = { + data: Buffer | Uint8Array | null; + uri: string | null; +}; + +export type BlobOptions = { + /** Defaults to true. */ + nullable?: boolean; + /** + * Max payload bytes kept inline in the data file. Zero is allowed. Must be a + * safe integer. + */ + inlineSizeThreshold?: number; + /** + * Max payload bytes stored in a packed sidecar before a dedicated file. Must + * be a positive safe integer. + */ + dedicatedSizeThreshold?: number; + /** + * Max bytes in one packed sidecar before starting another. Must be a positive + * safe integer. + */ + packFileSizeThreshold?: number; +}; + +/** + * Declares a `lance.blob.v2` column. + * + * Query results are descriptors, not payload bytes. Use {@link Table.fetchBlobs} + * or {@link Table.fetchBlobFiles} to read bytes. + * + * @example + * ```ts + * import { readFile } from "node:fs/promises"; + * import { Field, Int64, Schema } from "apache-arrow"; + * import { blob, connect } from "@lancedb/lancedb"; + * + * const db = await connect("./data"); + * const video = await readFile("clip.mp4"); + * const table = await db.createTable( + * "videos", + * [{ id: 1n, video }], + * { + * schema: new Schema([ + * new Field("id", new Int64()), + * blob("video"), + * ]), + * }, + * ); + * + * const rows = await table.query().select(["id"]).withRowId().toArray(); + * const rowIds = rows.map((row) => row._rowid as bigint); + * const bytes = await table.fetchBlobs("video", rowIds); + * + * const [handle] = await table.fetchBlobFiles("video", rowIds); + * const size = handle!.size(); + * const header = await handle!.readRange(0n, size < 65536n ? size : 65536n); + * ``` + */ +export function blob(name: string, options: BlobOptions = {}): Field { + const metadata = new Map([ + ["ARROW:extension:name", BLOB_V2_EXTENSION_NAME], + ]); + setThreshold( + metadata, + INLINE_SIZE_THRESHOLD_KEY, + "inlineSizeThreshold", + options.inlineSizeThreshold, + 0, + ); + setThreshold( + metadata, + DEDICATED_SIZE_THRESHOLD_KEY, + "dedicatedSizeThreshold", + options.dedicatedSizeThreshold, + 1, + ); + setThreshold( + metadata, + PACK_FILE_SIZE_THRESHOLD_KEY, + "packFileSizeThreshold", + options.packFileSizeThreshold, + 1, + ); + return new Field( + name, + new Struct([ + new Field("data", new LargeBinary(), true), + new Field("uri", new Utf8(), true), + ]), + options.nullable ?? true, + metadata, + ); +} + +/** + * Checks for the `lance.blob.v2` extension marker. Does not validate the + * field's storage type. + */ +export function isBlobField(field: Field): boolean { + return field.metadata?.get("ARROW:extension:name") === BLOB_V2_EXTENSION_NAME; +} + +/** + * A lazy handle to blob bytes. Create one with {@link Table.fetchBlobFiles}. + * + * @hideconstructor + */ +export class BlobFile { + private readonly inner: NativeBlobFile; + + private constructor(inner: NativeBlobFile) { + if (!(inner instanceof NativeBlobFile)) { + throw new Error("BlobFile handles come from Table.fetchBlobFiles"); + } + this.inner = inner; + } + + /** @ignore */ + static fromNative(inner: NativeBlobFile): BlobFile { + return new BlobFile(inner); + } + + /** Returns the blob size in bytes. */ + size(): bigint { + return this.inner.size(); + } + + /** + * Reads from the cursor to the end and advances the cursor. + * + * A second call returns an empty buffer. {@link BlobFile.readRange} does + * not move the cursor. + */ + read(): Promise { + return this.inner.read(); + } + + /** + * Reads the half-open byte range `[start, end)`. + * + * Fails when `end` is past the blob size. Does not move the cursor. + */ + readRange(start: bigint, end: bigint): Promise { + return this.inner.readRange(start, end); + } +} + +export function coerceBlobValue(value: unknown): BlobInput | null { + if (value == null) { + return null; + } + if (isBlobBytes(value)) { + return { data: value, uri: null }; + } + if (ArrayBuffer.isView(value)) { + throw new Error("Blob data must be Buffer or Uint8Array"); + } + if (typeof value === "string") { + if (value === "") { + throw new Error("Blob uri cannot be empty"); + } + return { data: null, uri: value }; + } + if (typeof value === "object") { + const record = value as Record; + if (!("data" in record) && !("uri" in record)) { + throw new Error( + "Blob struct values must include a 'data' or 'uri' field", + ); + } + const uri = record.uri; + if (uri === "") { + throw new Error("Blob uri cannot be empty"); + } + if (uri != null && typeof uri !== "string") { + throw new Error(`Blob uri must be a string or null, got ${typeof uri}`); + } + const data = record.data; + if (data != null && !isBlobBytes(data)) { + throw new Error("Blob data must be Buffer, Uint8Array, or null"); + } + const bytes = (data as Buffer | Uint8Array | null | undefined) ?? null; + const uriValue = uri ?? null; + if ((bytes == null) === (uriValue == null)) { + throw new Error( + "Blob struct values must set exactly one of 'data' or 'uri'", + ); + } + return { data: bytes, uri: uriValue }; + } + throw new Error( + "Blob column values must be Buffer, Uint8Array, a URI string, null, or { data?, uri? }", + ); +} + +function isBlobBytes(value: unknown): value is Buffer | Uint8Array { + return Buffer.isBuffer(value) || value instanceof Uint8Array; +} + +function setThreshold( + metadata: Map, + key: string, + optionName: string, + value: number | undefined, + minimum: number, +): void { + if (value === undefined) { + return; + } + if (!Number.isSafeInteger(value)) { + throw new Error(`${optionName} must be a safe integer`); + } + if (value < minimum) { + throw new Error( + minimum <= 0 + ? `${optionName} must be non-negative` + : `${optionName} must be positive`, + ); + } + metadata.set(key, String(value)); +} diff --git a/nodejs/lancedb/index.ts b/nodejs/lancedb/index.ts index 4f8ff77e5..d94007a11 100644 --- a/nodejs/lancedb/index.ts +++ b/nodejs/lancedb/index.ts @@ -77,6 +77,9 @@ export { VectorColumnOptions, } from "./arrow"; +export { blob, isBlobField, BlobFile } from "./blob"; +export type { BlobOptions } from "./blob"; + export { Connection, CreateTableOptions, diff --git a/nodejs/lancedb/table.ts b/nodejs/lancedb/table.ts index d1fb8acd8..eac9f490d 100644 --- a/nodejs/lancedb/table.ts +++ b/nodejs/lancedb/table.ts @@ -17,6 +17,7 @@ import { tableFromIPC, } from "./arrow"; +import { BlobFile } from "./blob"; import { EmbeddingFunctionConfig, getRegistry } from "./embedding/registry"; import { IndexOptions } from "./indices"; import { Job } from "./job"; @@ -510,6 +511,35 @@ export abstract class Table { */ abstract takeRowIds(rowIds: readonly (bigint | number)[]): TakeQuery; + /** + * Blob v2 columns, including nested dotted paths. + */ + abstract blobColumns(): Promise; + + /** + * Bytes for `column` at row IDs from {@link Query.withRowId}. + * + * Reads the table's current checkout. IDs from another version can fail after + * compaction unless stable row ids are enabled. Results keep input order and + * duplicates. Null blobs are `null`. Empty blobs are empty buffers. + */ + abstract fetchBlobs( + column: string, + rowIds: readonly (bigint | number)[], + ): Promise<(Buffer | null)[]>; + + /** + * Opens lazy blob handles for `column` at the given row IDs using the + * table's current checkout. + * + * Preserves input order, duplicates, and nulls. Use this for large payloads. + * See {@link Table.fetchBlobs} for row-ID validity across versions. + */ + abstract fetchBlobFiles( + column: string, + rowIds: readonly (bigint | number)[], + ): Promise<(BlobFile | null)[]>; + /** * Create a search query to find the nearest neighbors * of the given query @@ -1160,23 +1190,34 @@ export class LocalTable extends Table { } takeRowIds(rowIds: readonly (bigint | number)[]): TakeQuery { - const ids = rowIds.map((id) => { - if (typeof id === "bigint") { - return id; - } - if (!Number.isInteger(id)) { - throw new Error("Row id must be an integer (or bigint)"); - } - if (id < 0) { - throw new Error("Row id cannot be negative"); - } - if (!Number.isSafeInteger(id)) { - throw new Error("Row id is too large for number; use bigint instead"); - } - return BigInt(id); - }); + return new TakeQuery(this.inner.takeRowIds(rowIdsToBigInts(rowIds))); + } - return new TakeQuery(this.inner.takeRowIds(ids)); + blobColumns(): Promise { + return this.inner.blobColumns(); + } + + async fetchBlobs( + column: string, + rowIds: readonly (bigint | number)[], + ): Promise<(Buffer | null)[]> { + const values = await this.inner.fetchBlobs(column, rowIdsToBigInts(rowIds)); + // N-API Option maps missing values to undefined. Collapse those to null. + return values.map((value) => value ?? null); + } + + async fetchBlobFiles( + column: string, + rowIds: readonly (bigint | number)[], + ): Promise<(BlobFile | null)[]> { + const files = await this.inner.fetchBlobFiles( + column, + rowIdsToBigInts(rowIds), + ); + // N-API Option maps missing values to undefined. Collapse those to null. + return files.map((file) => + file == null ? null : BlobFile.fromNative(file), + ); } query(): Query { @@ -1733,3 +1774,21 @@ export class Branches { )) as unknown as CherryPickResult; } } + +function rowIdsToBigInts(rowIds: readonly (bigint | number)[]): bigint[] { + return rowIds.map((id) => { + if (typeof id === "bigint") { + return id; + } + if (!Number.isInteger(id)) { + throw new Error("Row id must be an integer (or bigint)"); + } + if (id < 0) { + throw new Error("Row id cannot be negative"); + } + if (!Number.isSafeInteger(id)) { + throw new Error("Row id is too large for number; use bigint instead"); + } + return BigInt(id); + }); +} diff --git a/nodejs/src/blob.rs b/nodejs/src/blob.rs new file mode 100644 index 000000000..0e19a9a8a --- /dev/null +++ b/nodejs/src/blob.rs @@ -0,0 +1,95 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +use std::ops::Range; +use std::sync::Arc; + +use arrow_array::{Array, LargeBinaryArray}; +use lancedb::blob::BlobFile as LanceBlobFile; +use napi::bindgen_prelude::*; +use napi_derive::napi; + +use crate::error::convert_error; + +#[napi] +pub struct BlobFile { + inner: Arc, +} + +impl BlobFile { + pub(crate) fn new(inner: LanceBlobFile) -> Self { + Self { + inner: Arc::new(inner), + } + } +} + +#[napi] +impl BlobFile { + #[napi] + pub fn size(&self) -> BigInt { + BigInt::from(self.inner.size()) + } + + #[napi] + pub async fn read(&self) -> napi::Result { + let bytes = self.inner.read().await.map_err(|err| convert_error(&err))?; + Ok(Buffer::from(bytes.as_ref())) + } + + #[napi] + pub async fn read_range(&self, start: BigInt, end: BigInt) -> napi::Result { + let range = bigint_range(start, end)?; + let bytes = self + .inner + .read_range(range) + .await + .map_err(|err| convert_error(&err))?; + Ok(Buffer::from(bytes.as_ref())) + } +} + +fn bigint_range(start: BigInt, end: BigInt) -> napi::Result> { + let start = parse_u64(start, "start")?; + let end = parse_u64(end, "end")?; + if start > end { + return Err(napi::Error::from_reason(format!( + "invalid blob range: start ({start}) > end ({end})" + ))); + } + Ok(start..end) +} + +pub(crate) fn parse_u64(value: BigInt, name: &str) -> napi::Result { + let (negative, value, lossless) = value.get_u64(); + if negative { + return Err(napi::Error::from_reason(format!( + "{name} cannot be negative" + ))); + } + if !lossless { + return Err(napi::Error::from_reason(format!( + "{name} is too large to fit in u64" + ))); + } + Ok(value) +} + +pub(crate) fn parse_row_ids(row_ids: Vec) -> napi::Result> { + row_ids + .into_iter() + .map(|id| parse_u64(id, "row id")) + .collect() +} + +pub(crate) fn copy_blob_buffers(array: LargeBinaryArray) -> Vec> { + (0..array.len()) + .map(|i| { + if array.is_null(i) { + None + } else { + Some(Buffer::from(array.value(i).to_vec())) + } + }) + .collect() +} diff --git a/nodejs/src/lib.rs b/nodejs/src/lib.rs index 1110f6203..288a2b925 100644 --- a/nodejs/src/lib.rs +++ b/nodejs/src/lib.rs @@ -10,6 +10,7 @@ use std::collections::HashMap; use env_logger::Env; use napi_derive::*; +mod blob; mod connection; mod error; mod header; diff --git a/nodejs/src/table.rs b/nodejs/src/table.rs index db74d38fa..7ae4402ab 100644 --- a/nodejs/src/table.rs +++ b/nodejs/src/table.rs @@ -15,6 +15,7 @@ use napi::bindgen_prelude::*; use napi::threadsafe_function::{ThreadsafeFunction, ThreadsafeFunctionCallMode}; use napi_derive::napi; +use crate::blob::{BlobFile, copy_blob_buffers, parse_row_ids}; use crate::error::NapiErrorExt; use crate::index::Index; use crate::merge::NativeMergeInsertBuilder; @@ -329,6 +330,44 @@ impl Table { )) } + #[napi(catch_unwind)] + pub async fn blob_columns(&self) -> napi::Result> { + self.inner_ref()?.blob_columns().await.default_error() + } + + #[napi(catch_unwind)] + pub async fn fetch_blobs( + &self, + column: String, + row_ids: Vec, + ) -> napi::Result>> { + let row_ids = parse_row_ids(row_ids)?; + let array = self + .inner_ref()? + .fetch_blobs(column.as_str(), &row_ids) + .await + .default_error()?; + Ok(copy_blob_buffers(array)) + } + + #[napi(catch_unwind)] + pub async fn fetch_blob_files( + &self, + column: String, + row_ids: Vec, + ) -> napi::Result>> { + let row_ids = parse_row_ids(row_ids)?; + let files = self + .inner_ref()? + .fetch_blob_files(column.as_str(), &row_ids) + .await + .default_error()?; + Ok(files + .into_iter() + .map(|file| file.map(BlobFile::new)) + .collect()) + } + #[napi(catch_unwind)] pub fn vector_search(&self, vector: Float32Array) -> napi::Result { self.query()?.nearest_to(vector) From b8f0048b5a0f4bd9f9d735cb60fc444db3f5a42d Mon Sep 17 00:00:00 2001 From: LanceDB Robot Date: Sat, 12 Sep 2026 22:25:12 -0700 Subject: [PATCH 3/8] chore: update lance dependency to v12.0.0-beta.18 (#4164) Update the Rust workspace Lance dependencies, Cargo lockfile, and Java lance-core to [v12.0.0-beta.18](https://github.com/lance-format/lance/releases/tag/v12.0.0-beta.18). Fix redundant visibility declarations in the Node.js Rust blob helpers required by the workspace Clippy check. Validated with workspace Clippy (all features and tests, warnings denied), cargo fmt, pnpm build, and 13 targeted Node.js blob tests. --- Cargo.lock | 84 +++++++++++++++++++++++----------------------- Cargo.toml | 28 ++++++++-------- java/pom.xml | 2 +- nodejs/src/blob.rs | 6 ++-- 4 files changed, 60 insertions(+), 60 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 209ee4942..7176d8c76 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3490,8 +3490,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "fsst" -version = "12.0.0-beta.17" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.17#d7b031f9ca86e7cbb8c3732ab87b2b18548996ba" +version = "12.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93" dependencies = [ "arrow-array", "rand 0.9.5", @@ -4862,8 +4862,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a" [[package]] name = "lance" -version = "12.0.0-beta.17" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.17#d7b031f9ca86e7cbb8c3732ab87b2b18548996ba" +version = "12.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93" dependencies = [ "arc-swap", "arrow", @@ -4935,8 +4935,8 @@ dependencies = [ [[package]] name = "lance-arrow" -version = "12.0.0-beta.17" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.17#d7b031f9ca86e7cbb8c3732ab87b2b18548996ba" +version = "12.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93" dependencies = [ "arrow-array", "arrow-buffer", @@ -4958,7 +4958,7 @@ dependencies = [ [[package]] name = "lance-arrow-scalar" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.17#d7b031f9ca86e7cbb8c3732ab87b2b18548996ba" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93" dependencies = [ "arrow-array", "arrow-buffer", @@ -4972,7 +4972,7 @@ dependencies = [ [[package]] name = "lance-arrow-stats" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.17#d7b031f9ca86e7cbb8c3732ab87b2b18548996ba" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93" dependencies = [ "arrow-array", "arrow-schema", @@ -4981,8 +4981,8 @@ dependencies = [ [[package]] name = "lance-bitpacking" -version = "12.0.0-beta.17" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.17#d7b031f9ca86e7cbb8c3732ab87b2b18548996ba" +version = "12.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93" dependencies = [ "arrayref", "crunchy", @@ -4992,8 +4992,8 @@ dependencies = [ [[package]] name = "lance-core" -version = "12.0.0-beta.17" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.17#d7b031f9ca86e7cbb8c3732ab87b2b18548996ba" +version = "12.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93" dependencies = [ "arrow-array", "arrow-buffer", @@ -5030,8 +5030,8 @@ dependencies = [ [[package]] name = "lance-datafusion" -version = "12.0.0-beta.17" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.17#d7b031f9ca86e7cbb8c3732ab87b2b18548996ba" +version = "12.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93" dependencies = [ "arrow", "arrow-array", @@ -5061,8 +5061,8 @@ dependencies = [ [[package]] name = "lance-datagen" -version = "12.0.0-beta.17" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.17#d7b031f9ca86e7cbb8c3732ab87b2b18548996ba" +version = "12.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93" dependencies = [ "arrow", "arrow-array", @@ -5079,8 +5079,8 @@ dependencies = [ [[package]] name = "lance-derive" -version = "12.0.0-beta.17" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.17#d7b031f9ca86e7cbb8c3732ab87b2b18548996ba" +version = "12.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93" dependencies = [ "proc-macro2", "quote", @@ -5089,8 +5089,8 @@ dependencies = [ [[package]] name = "lance-encoding" -version = "12.0.0-beta.17" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.17#d7b031f9ca86e7cbb8c3732ab87b2b18548996ba" +version = "12.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93" dependencies = [ "arrow-arith", "arrow-array", @@ -5123,8 +5123,8 @@ dependencies = [ [[package]] name = "lance-file" -version = "12.0.0-beta.17" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.17#d7b031f9ca86e7cbb8c3732ab87b2b18548996ba" +version = "12.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93" dependencies = [ "arrow-arith", "arrow-array", @@ -5155,8 +5155,8 @@ dependencies = [ [[package]] name = "lance-index" -version = "12.0.0-beta.17" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.17#d7b031f9ca86e7cbb8c3732ab87b2b18548996ba" +version = "12.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93" dependencies = [ "arc-swap", "arrow", @@ -5220,8 +5220,8 @@ dependencies = [ [[package]] name = "lance-index-core" -version = "12.0.0-beta.17" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.17#d7b031f9ca86e7cbb8c3732ab87b2b18548996ba" +version = "12.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93" dependencies = [ "arrow-array", "arrow-schema", @@ -5243,8 +5243,8 @@ dependencies = [ [[package]] name = "lance-io" -version = "12.0.0-beta.17" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.17#d7b031f9ca86e7cbb8c3732ab87b2b18548996ba" +version = "12.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93" dependencies = [ "arrow", "arrow-array", @@ -5284,8 +5284,8 @@ dependencies = [ [[package]] name = "lance-linalg" -version = "12.0.0-beta.17" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.17#d7b031f9ca86e7cbb8c3732ab87b2b18548996ba" +version = "12.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93" dependencies = [ "arrow-array", "arrow-schema", @@ -5299,8 +5299,8 @@ dependencies = [ [[package]] name = "lance-namespace" -version = "12.0.0-beta.17" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.17#d7b031f9ca86e7cbb8c3732ab87b2b18548996ba" +version = "12.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93" dependencies = [ "arrow", "async-trait", @@ -5314,8 +5314,8 @@ dependencies = [ [[package]] name = "lance-namespace-impls" -version = "12.0.0-beta.17" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.17#d7b031f9ca86e7cbb8c3732ab87b2b18548996ba" +version = "12.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93" dependencies = [ "arrow", "arrow-ipc", @@ -5368,8 +5368,8 @@ dependencies = [ [[package]] name = "lance-select" -version = "12.0.0-beta.17" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.17#d7b031f9ca86e7cbb8c3732ab87b2b18548996ba" +version = "12.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93" dependencies = [ "arrow-array", "arrow-buffer", @@ -5383,8 +5383,8 @@ dependencies = [ [[package]] name = "lance-table" -version = "12.0.0-beta.17" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.17#d7b031f9ca86e7cbb8c3732ab87b2b18548996ba" +version = "12.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93" dependencies = [ "arrow", "arrow-array", @@ -5424,8 +5424,8 @@ dependencies = [ [[package]] name = "lance-testing" -version = "12.0.0-beta.17" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.17#d7b031f9ca86e7cbb8c3732ab87b2b18548996ba" +version = "12.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93" dependencies = [ "arrow-array", "arrow-schema", @@ -5438,8 +5438,8 @@ dependencies = [ [[package]] name = "lance-tokenizer" -version = "12.0.0-beta.17" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.17#d7b031f9ca86e7cbb8c3732ab87b2b18548996ba" +version = "12.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93" dependencies = [ "frostem", "icu_segmenter", diff --git a/Cargo.toml b/Cargo.toml index 2b16a6bf0..1068d48ee 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,20 +13,20 @@ categories = ["database-implementations"] rust-version = "1.91.0" [workspace.dependencies] -lance = { "version" = "=12.0.0-beta.17", default-features = false, "tag" = "v12.0.0-beta.17", "git" = "https://github.com/lance-format/lance.git" } -lance-core = { "version" = "=12.0.0-beta.17", "tag" = "v12.0.0-beta.17", "git" = "https://github.com/lance-format/lance.git" } -lance-datagen = { "version" = "=12.0.0-beta.17", "tag" = "v12.0.0-beta.17", "git" = "https://github.com/lance-format/lance.git" } -lance-file = { "version" = "=12.0.0-beta.17", "tag" = "v12.0.0-beta.17", "git" = "https://github.com/lance-format/lance.git" } -lance-io = { "version" = "=12.0.0-beta.17", default-features = false, "tag" = "v12.0.0-beta.17", "git" = "https://github.com/lance-format/lance.git" } -lance-index = { "version" = "=12.0.0-beta.17", "tag" = "v12.0.0-beta.17", "git" = "https://github.com/lance-format/lance.git" } -lance-linalg = { "version" = "=12.0.0-beta.17", "tag" = "v12.0.0-beta.17", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace = { "version" = "=12.0.0-beta.17", "tag" = "v12.0.0-beta.17", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace-impls = { "version" = "=12.0.0-beta.17", default-features = false, "tag" = "v12.0.0-beta.17", "git" = "https://github.com/lance-format/lance.git" } -lance-table = { "version" = "=12.0.0-beta.17", "tag" = "v12.0.0-beta.17", "git" = "https://github.com/lance-format/lance.git" } -lance-testing = { "version" = "=12.0.0-beta.17", "tag" = "v12.0.0-beta.17", "git" = "https://github.com/lance-format/lance.git" } -lance-datafusion = { "version" = "=12.0.0-beta.17", "tag" = "v12.0.0-beta.17", "git" = "https://github.com/lance-format/lance.git" } -lance-encoding = { "version" = "=12.0.0-beta.17", "tag" = "v12.0.0-beta.17", "git" = "https://github.com/lance-format/lance.git" } -lance-arrow = { "version" = "=12.0.0-beta.17", "tag" = "v12.0.0-beta.17", "git" = "https://github.com/lance-format/lance.git" } +lance = { "version" = "=12.0.0-beta.18", default-features = false, "tag" = "v12.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } +lance-core = { "version" = "=12.0.0-beta.18", "tag" = "v12.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } +lance-datagen = { "version" = "=12.0.0-beta.18", "tag" = "v12.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } +lance-file = { "version" = "=12.0.0-beta.18", "tag" = "v12.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } +lance-io = { "version" = "=12.0.0-beta.18", default-features = false, "tag" = "v12.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } +lance-index = { "version" = "=12.0.0-beta.18", "tag" = "v12.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } +lance-linalg = { "version" = "=12.0.0-beta.18", "tag" = "v12.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace = { "version" = "=12.0.0-beta.18", "tag" = "v12.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace-impls = { "version" = "=12.0.0-beta.18", default-features = false, "tag" = "v12.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } +lance-table = { "version" = "=12.0.0-beta.18", "tag" = "v12.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } +lance-testing = { "version" = "=12.0.0-beta.18", "tag" = "v12.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } +lance-datafusion = { "version" = "=12.0.0-beta.18", "tag" = "v12.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } +lance-encoding = { "version" = "=12.0.0-beta.18", "tag" = "v12.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } +lance-arrow = { "version" = "=12.0.0-beta.18", "tag" = "v12.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } lancedb = { path = "rust/lancedb", default-features = false } ahash = "0.8" # Note that this one does not include pyarrow diff --git a/java/pom.xml b/java/pom.xml index 7cbb819d9..a2456103d 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -28,7 +28,7 @@ UTF-8 15.0.0 - 12.0.0-beta.17 + 12.0.0-beta.18 false 2.30.0 1.7 diff --git a/nodejs/src/blob.rs b/nodejs/src/blob.rs index 0e19a9a8a..230a11bae 100644 --- a/nodejs/src/blob.rs +++ b/nodejs/src/blob.rs @@ -60,7 +60,7 @@ fn bigint_range(start: BigInt, end: BigInt) -> napi::Result> { Ok(start..end) } -pub(crate) fn parse_u64(value: BigInt, name: &str) -> napi::Result { +fn parse_u64(value: BigInt, name: &str) -> napi::Result { let (negative, value, lossless) = value.get_u64(); if negative { return Err(napi::Error::from_reason(format!( @@ -75,14 +75,14 @@ pub(crate) fn parse_u64(value: BigInt, name: &str) -> napi::Result { Ok(value) } -pub(crate) fn parse_row_ids(row_ids: Vec) -> napi::Result> { +pub fn parse_row_ids(row_ids: Vec) -> napi::Result> { row_ids .into_iter() .map(|id| parse_u64(id, "row id")) .collect() } -pub(crate) fn copy_blob_buffers(array: LargeBinaryArray) -> Vec> { +pub fn copy_blob_buffers(array: LargeBinaryArray) -> Vec> { (0..array.len()) .map(|i| { if array.is_null(i) { From 9fe10c7362b1fdfb9bfa1378412ce4d178d3e7a4 Mon Sep 17 00:00:00 2001 From: Jack Ye Date: Sun, 13 Sep 2026 21:44:13 -0700 Subject: [PATCH 4/8] fix(remote): align Function CRUD routes (#4166) Align the experimental Function HTTP transport with the equivalent Table CRUD API shape. This is an intentional breaking change to the experimental Function routes; public Rust and Python APIs remain unchanged. ## Route comparison | Operation | Function before | Function after | Equivalent Table API | | --- | --- | --- | --- | | Create | `POST /v1/functions/create` | `POST /v1/function/{id}/create` | `POST /v1/table/{id}/create` | | Describe | `POST /v1/functions/describe` | `POST /v1/function/{id}/describe` | `POST /v1/table/{id}/describe` | | List | `POST /v1/functions/list` | `GET /v1/namespace/{id}/function/list` | `GET /v1/namespace/{id}/table/list` | | Drop | `POST /v1/functions/drop` | `POST /v1/function/{id}/drop` | `POST /v1/table/{id}/drop` | ## Contract details - Create, describe, and drop use a singular resource path. Their `{id}` path parameter is the URL-encoded Function name, and the duplicate Function identifier is removed from each request body. - Create continues to accept `202 Accepted`. - List changes from a POST with a JSON body to a namespace-scoped GET. Its `{id}` path parameter is the namespace identifier rather than a Function name. - Functions do not support nested namespaces yet, so the client lists against the root namespace identifier (`$` with the default delimiter). A non-root namespace is rejected. - The optional list filter is named `name`. `limit`, `page_token`, and `include_definition` remain available as query parameters. - The paginated list response shape is unchanged. --- .../tests/test_first_class_function_slice2.py | 101 ++++++++++-------- rust/lancedb/src/remote/db.rs | 80 ++++++++------ 2 files changed, 101 insertions(+), 80 deletions(-) diff --git a/python/python/tests/test_first_class_function_slice2.py b/python/python/tests/test_first_class_function_slice2.py index 4816019d6..900f69f8d 100644 --- a/python/python/tests/test_first_class_function_slice2.py +++ b/python/python/tests/test_first_class_function_slice2.py @@ -15,6 +15,7 @@ from pathlib import Path import subprocess import sys import threading +import urllib.parse from typing import Optional import pyarrow as pa @@ -1213,14 +1214,22 @@ def _mock_remote_function_catalog(): def log_message(self, *args): pass + def _write_response(self, status, response): + encoded = json.dumps(response).encode() + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(encoded))) + self.end_headers() + self.wfile.write(encoded) + def do_POST(self): length = int(self.headers.get("Content-Length", "0")) body = json.loads(self.rfile.read(length) or b"{}") state["requests"].append((self.path, body)) status = 200 - if self.path == "/v1/functions/create": + if self.path == "/v1/function/normalize_score/create": state["version"] = { - "name": body["name"], + "name": "normalize_score", "version": "fv_exact", "artifact": { key: body["artifact"][key] @@ -1242,43 +1251,43 @@ def _mock_remote_function_catalog(): "job_state": "DONE", "result": state["version"], } - elif self.path == "/v1/functions/describe": - assert body == { - "name": "normalize_score", - "version": "fv_exact", - } + elif self.path == "/v1/function/normalize_score/describe": + assert body == {"version": "fv_exact"} response = state["version"] - elif self.path == "/v1/functions/list": - assert body["include_definition"] is True - if "page_token" not in body: - response = { - "functions": [ - { - "name": "normalize_score", - "version": "fv_exact", - "definition": state["version"], - } - ], - "page_token": "next", - } - else: - assert body["page_token"] == "next" - response = {"functions": []} - elif self.path == "/v1/functions/drop": - assert body == { - "name": "normalize_score", - "version": "fv_exact", - } + elif self.path == "/v1/function/normalize_score/drop": + assert body == {"version": "fv_exact"} response = {"dropped": True} else: status = 404 response = {"error": "not found"} - encoded = json.dumps(response).encode() - self.send_response(status) - self.send_header("Content-Type", "application/json") - self.send_header("Content-Length", str(len(encoded))) - self.end_headers() - self.wfile.write(encoded) + self._write_response(status, response) + + def do_GET(self): + url = urllib.parse.urlsplit(self.path) + query = { + key: values[-1] + for key, values in urllib.parse.parse_qs(url.query).items() + } + state["requests"].append((url.path, query)) + if url.path != "/v1/namespace/$/function/list": + self._write_response(404, {"error": "not found"}) + return + assert query["include_definition"] == "true" + if "page_token" not in query: + response = { + "functions": [ + { + "name": "normalize_score", + "version": "fv_exact", + "definition": state["version"], + } + ], + "page_token": "next", + } + else: + assert query["page_token"] == "next" + response = {"functions": []} + self._write_response(200, response) with http.server.HTTPServer(("localhost", 0), Handler) as server: thread = threading.Thread(target=server.serve_forever) @@ -1307,9 +1316,11 @@ def test_remote_registration_job_and_exact_version_reopen_round_trip(): assert reopened.name == "normalize_score" assert reopened.version == "fv_exact" create_request = state["requests"][0][1] - assert create_request == json.loads( + expected_request = json.loads( normalize_score.registration_request.to_canonical_json() ) + expected_request.pop("name") + assert create_request == expected_request def test_blocking_remote_registration_returns_function_version(): @@ -1325,7 +1336,7 @@ def test_blocking_remote_registration_returns_function_version(): assert created.name == "normalize_score" assert created.version == "fv_exact" assert [path for path, _ in state["requests"]] == [ - "/v1/functions/create", + "/v1/function/normalize_score/create", "/v1/jobs/describe", ] @@ -1344,10 +1355,10 @@ def test_remote_list_functions_paginates_and_returns_typed_versions(): assert functions == [created] assert state["requests"] == [ - ("/v1/functions/list", {"include_definition": True}), + ("/v1/namespace/$/function/list", {"include_definition": "true"}), ( - "/v1/functions/list", - {"include_definition": True, "page_token": "next"}, + "/v1/namespace/$/function/list", + {"include_definition": "true", "page_token": "next"}, ), ] @@ -1368,8 +1379,8 @@ async def test_async_remote_list_functions_returns_typed_versions(): assert functions == [created] assert [path for path, _ in state["requests"]] == [ - "/v1/functions/list", - "/v1/functions/list", + "/v1/namespace/$/function/list", + "/v1/namespace/$/function/list", ] @@ -1385,8 +1396,8 @@ def test_remote_drop_function_sends_exact_version(): assert state["requests"] == [ ( - "/v1/functions/drop", - {"name": "normalize_score", "version": "fv_exact"}, + "/v1/function/normalize_score/drop", + {"version": "fv_exact"}, ) ] @@ -1404,7 +1415,7 @@ async def test_async_remote_drop_function_sends_exact_version(): assert state["requests"] == [ ( - "/v1/functions/drop", - {"name": "normalize_score", "version": "fv_exact"}, + "/v1/function/normalize_score/drop", + {"version": "fv_exact"}, ) ] diff --git a/rust/lancedb/src/remote/db.rs b/rust/lancedb/src/remote/db.rs index 32ace368e..917bf909b 100644 --- a/rust/lancedb/src/remote/db.rs +++ b/rust/lancedb/src/remote/db.rs @@ -591,7 +591,15 @@ impl Database for RemoteDatabase { &self, request: FunctionRegistrationRequest, ) -> Result> { - let req = self.client.post("/v1/functions/create").json(&request); + let function_id = urlencoding::encode(&request.name); + let req = self + .client + .post(&format!("/v1/function/{function_id}/create")) + .json(&serde_json::json!({ + "artifact": request.artifact, + "signature": request.signature, + "runtime": request.runtime, + })); let (request_id, response) = self.client.send(req).await?; let response = self.client.check_response(&request_id, response).await?; let status = response.status(); @@ -608,11 +616,11 @@ impl Database for RemoteDatabase { } async fn get_function(&self, name: &str, version: &str) -> Result { + let function_id = urlencoding::encode(name); let req = self .client - .post("/v1/functions/describe") + .post(&format!("/v1/function/{function_id}/describe")) .json(&serde_json::json!({ - "name": name, "version": version, })); let (request_id, response) = self.client.send(req).await?; @@ -621,15 +629,19 @@ impl Database for RemoteDatabase { } async fn list_functions(&self) -> Result> { + let namespace_id = build_namespace_identifier(&[], &self.client.id_delimiter); + let path = format!("/v1/namespace/{namespace_id}/function/list"); let mut functions = Vec::new(); let mut page_token: Option = None; let mut seen_page_tokens = HashSet::new(); loop { - let mut body = serde_json::json!({ "include_definition": true }); + let mut req = self + .client + .get(&path) + .query(&[("include_definition", true)]); if let Some(token) = &page_token { - body["page_token"] = serde_json::Value::String(token.clone()); + req = req.query(&[("page_token", token)]); } - let req = self.client.post("/v1/functions/list").json(&body); let (request_id, response) = self.client.send(req).await?; let response = self.client.check_response(&request_id, response).await?; let status = response.status(); @@ -658,11 +670,11 @@ impl Database for RemoteDatabase { } async fn drop_function(&self, name: &str, version: &str) -> Result { + let function_id = urlencoding::encode(name); let req = self .client - .post("/v1/functions/drop") + .post(&format!("/v1/function/{function_id}/drop")) .json(&serde_json::json!({ - "name": name, "version": version, })); let (request_id, response) = self.client.send(req).await?; @@ -2788,9 +2800,10 @@ mod tests { ); const FUNCTION_JOB: &str = include_str!("../../tests/fixtures/first_class_functions/v1/remote_function_job.json"); - let expected: serde_json::Value = serde_json::from_str(REQUEST).unwrap(); + let mut expected: serde_json::Value = serde_json::from_str(REQUEST).unwrap(); + expected.as_object_mut().unwrap().remove("name"); let conn = Connection::new_with_handler(move |request| match request.url().path() { - "/v1/functions/create" => { + "/v1/function/normalize_score/create" => { assert_eq!(request.method(), &reqwest::Method::POST); let body: serde_json::Value = serde_json::from_slice(request.body().unwrap().as_bytes().unwrap()).unwrap(); @@ -2821,13 +2834,10 @@ mod tests { ); let conn = Connection::new_with_handler(|request| { assert_eq!(request.method(), &reqwest::Method::POST); - assert_eq!(request.url().path(), "/v1/functions/describe"); + assert_eq!(request.url().path(), "/v1/function/embed/describe"); let body: serde_json::Value = serde_json::from_slice(request.body().unwrap().as_bytes().unwrap()).unwrap(); - assert_eq!( - body, - serde_json::json!({"name": "embed", "version": "fv_01K3EXACT"}) - ); + assert_eq!(body, serde_json::json!({"version": "fv_01K3EXACT"})); http::Response::builder().status(200).body(VERSION).unwrap() }); let version = conn.get_function("embed", "fv_01K3EXACT").await.unwrap(); @@ -2843,21 +2853,20 @@ mod tests { let version: serde_json::Value = serde_json::from_str(VERSION).unwrap(); let page = Arc::new(AtomicUsize::new(0)); let conn = Connection::new_with_handler(move |request| { - assert_eq!(request.method(), &reqwest::Method::POST); - assert_eq!(request.url().path(), "/v1/functions/list"); - let body: serde_json::Value = - serde_json::from_slice(request.body().unwrap().as_bytes().unwrap()).unwrap(); - assert_eq!(body["include_definition"], true); + assert_eq!(request.method(), &reqwest::Method::GET); + assert_eq!(request.url().path(), "/v1/namespace/$/function/list"); + let query = request.url().query_pairs().collect::>(); + assert_eq!(query.get("include_definition").unwrap(), "true"); match page.fetch_add(1, Ordering::SeqCst) { 0 => { - assert!(body.get("page_token").is_none()); + assert!(!query.contains_key("page_token")); http::Response::builder() .status(200) .body(r#"{"functions": [], "page_token": "next"}"#.to_string()) .unwrap() } _ => { - assert_eq!(body["page_token"], "next"); + assert_eq!(query.get("page_token").unwrap(), "next"); http::Response::builder() .status(200) .body( @@ -2886,9 +2895,11 @@ mod tests { let seen = requests.clone(); let conn = Connection::new_with_handler(move |request| { seen.fetch_add(1, Ordering::SeqCst); - let body: serde_json::Value = - serde_json::from_slice(request.body().unwrap().as_bytes().unwrap()).unwrap(); - assert!(body.get("page_token").is_none()); + assert_eq!(request.method(), &reqwest::Method::GET); + assert_eq!(request.url().path(), "/v1/namespace/$/function/list"); + let query = request.url().query_pairs().collect::>(); + assert_eq!(query.get("include_definition").unwrap(), "true"); + assert!(!query.contains_key("page_token")); http::Response::builder() .status(200) .body(r#"{"functions": [], "page_token": ""}"#) @@ -2905,19 +2916,21 @@ mod tests { let page = Arc::new(AtomicUsize::new(0)); let requests = page.clone(); let conn = Connection::new_with_handler(move |request| { - let body: serde_json::Value = - serde_json::from_slice(request.body().unwrap().as_bytes().unwrap()).unwrap(); + assert_eq!(request.method(), &reqwest::Method::GET); + assert_eq!(request.url().path(), "/v1/namespace/$/function/list"); + let query = request.url().query_pairs().collect::>(); + assert_eq!(query.get("include_definition").unwrap(), "true"); let next_page_token = match page.fetch_add(1, Ordering::SeqCst) { 0 => { - assert!(body.get("page_token").is_none()); + assert!(!query.contains_key("page_token")); "one" } 1 => { - assert_eq!(body["page_token"], "one"); + assert_eq!(query.get("page_token").unwrap(), "one"); "two" } 2 => { - assert_eq!(body["page_token"], "two"); + assert_eq!(query.get("page_token").unwrap(), "two"); "one" } page => panic!("unexpected page: {page}"), @@ -2952,13 +2965,10 @@ mod tests { async fn test_drop_function_sends_exact_version_and_decodes_replay() { let conn = Connection::new_with_handler(|request| { assert_eq!(request.method(), &reqwest::Method::POST); - assert_eq!(request.url().path(), "/v1/functions/drop"); + assert_eq!(request.url().path(), "/v1/function/embed/drop"); let body: serde_json::Value = serde_json::from_slice(request.body().unwrap().as_bytes().unwrap()).unwrap(); - assert_eq!( - body, - serde_json::json!({"name": "embed", "version": "fv_01K3EXACT"}) - ); + assert_eq!(body, serde_json::json!({"version": "fv_01K3EXACT"})); http::Response::builder() .status(200) .body(r#"{"dropped":false}"#) From ec410e015aef4787e3bb6282e298ce70874bc474 Mon Sep 17 00:00:00 2001 From: Lance Release Date: Mon, 14 Sep 2026 04:45:01 +0000 Subject: [PATCH 5/8] =?UTF-8?q?Bump=20version:=200.39.0-beta.6=20=E2=86=92?= =?UTF-8?q?=200.39.0-beta.7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.toml | 2 +- Cargo.lock | 6 +++--- docs/src/java/java.md | 2 +- java/lancedb-core/pom.xml | 2 +- java/pom.xml | 2 +- nodejs/Cargo.toml | 2 +- nodejs/npm/darwin-arm64/package.json | 2 +- nodejs/npm/linux-arm64-gnu/package.json | 2 +- nodejs/npm/linux-arm64-musl/package.json | 2 +- nodejs/npm/linux-x64-gnu/package.json | 2 +- nodejs/npm/linux-x64-musl/package.json | 2 +- nodejs/npm/win32-arm64-msvc/package.json | 2 +- nodejs/npm/win32-x64-msvc/package.json | 2 +- nodejs/package.json | 2 +- python/Cargo.toml | 2 +- rust/lancedb/Cargo.toml | 2 +- 16 files changed, 18 insertions(+), 18 deletions(-) diff --git a/.bumpversion.toml b/.bumpversion.toml index de5f91973..a2c5fdd61 100644 --- a/.bumpversion.toml +++ b/.bumpversion.toml @@ -1,5 +1,5 @@ [tool.bumpversion] -current_version = "0.39.0-beta.6" +current_version = "0.39.0-beta.7" parse = """(?x) (?P0|[1-9]\\d*)\\. (?P0|[1-9]\\d*)\\. diff --git a/Cargo.lock b/Cargo.lock index 7176d8c76..be5040d37 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5452,7 +5452,7 @@ dependencies = [ [[package]] name = "lancedb" -version = "0.39.0-beta.6" +version = "0.39.0-beta.7" dependencies = [ "ahash", "anyhow", @@ -5543,7 +5543,7 @@ dependencies = [ [[package]] name = "lancedb-nodejs" -version = "0.39.0-beta.6" +version = "0.39.0-beta.7" dependencies = [ "arrow-array", "arrow-buffer", @@ -5568,7 +5568,7 @@ dependencies = [ [[package]] name = "lancedb-python" -version = "0.39.0-beta.6" +version = "0.39.0-beta.7" dependencies = [ "arrow", "async-trait", diff --git a/docs/src/java/java.md b/docs/src/java/java.md index 17d14df15..59c995344 100644 --- a/docs/src/java/java.md +++ b/docs/src/java/java.md @@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`: com.lancedb lancedb-core - 0.39.0-beta.6 + 0.39.0-beta.7 ``` diff --git a/java/lancedb-core/pom.xml b/java/lancedb-core/pom.xml index c5c928219..afe9b8c5c 100644 --- a/java/lancedb-core/pom.xml +++ b/java/lancedb-core/pom.xml @@ -8,7 +8,7 @@ com.lancedb lancedb-parent - 0.39.0-beta.6 + 0.39.0-beta.7 ../pom.xml diff --git a/java/pom.xml b/java/pom.xml index a2456103d..e66e1bbd4 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -6,7 +6,7 @@ com.lancedb lancedb-parent - 0.39.0-beta.6 + 0.39.0-beta.7 pom ${project.artifactId} LanceDB Java SDK Parent POM diff --git a/nodejs/Cargo.toml b/nodejs/Cargo.toml index c4ef09cda..340d8e500 100644 --- a/nodejs/Cargo.toml +++ b/nodejs/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "lancedb-nodejs" edition.workspace = true -version = "0.39.0-beta.6" +version = "0.39.0-beta.7" publish = false license.workspace = true description.workspace = true diff --git a/nodejs/npm/darwin-arm64/package.json b/nodejs/npm/darwin-arm64/package.json index 7ebfcaacb..e1ea88e62 100644 --- a/nodejs/npm/darwin-arm64/package.json +++ b/nodejs/npm/darwin-arm64/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-darwin-arm64", - "version": "0.39.0-beta.6", + "version": "0.39.0-beta.7", "os": ["darwin"], "cpu": ["arm64"], "main": "lancedb.darwin-arm64.node", diff --git a/nodejs/npm/linux-arm64-gnu/package.json b/nodejs/npm/linux-arm64-gnu/package.json index 1e2ad10b3..7bbbd3221 100644 --- a/nodejs/npm/linux-arm64-gnu/package.json +++ b/nodejs/npm/linux-arm64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-arm64-gnu", - "version": "0.39.0-beta.6", + "version": "0.39.0-beta.7", "os": ["linux"], "cpu": ["arm64"], "main": "lancedb.linux-arm64-gnu.node", diff --git a/nodejs/npm/linux-arm64-musl/package.json b/nodejs/npm/linux-arm64-musl/package.json index d4843f743..58c488e47 100644 --- a/nodejs/npm/linux-arm64-musl/package.json +++ b/nodejs/npm/linux-arm64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-arm64-musl", - "version": "0.39.0-beta.6", + "version": "0.39.0-beta.7", "os": ["linux"], "cpu": ["arm64"], "main": "lancedb.linux-arm64-musl.node", diff --git a/nodejs/npm/linux-x64-gnu/package.json b/nodejs/npm/linux-x64-gnu/package.json index d4107659a..ca364168e 100644 --- a/nodejs/npm/linux-x64-gnu/package.json +++ b/nodejs/npm/linux-x64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-x64-gnu", - "version": "0.39.0-beta.6", + "version": "0.39.0-beta.7", "os": ["linux"], "cpu": ["x64"], "main": "lancedb.linux-x64-gnu.node", diff --git a/nodejs/npm/linux-x64-musl/package.json b/nodejs/npm/linux-x64-musl/package.json index d14af6c09..c088a4d7d 100644 --- a/nodejs/npm/linux-x64-musl/package.json +++ b/nodejs/npm/linux-x64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-x64-musl", - "version": "0.39.0-beta.6", + "version": "0.39.0-beta.7", "os": ["linux"], "cpu": ["x64"], "main": "lancedb.linux-x64-musl.node", diff --git a/nodejs/npm/win32-arm64-msvc/package.json b/nodejs/npm/win32-arm64-msvc/package.json index 823db7c7c..af28f68d3 100644 --- a/nodejs/npm/win32-arm64-msvc/package.json +++ b/nodejs/npm/win32-arm64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-win32-arm64-msvc", - "version": "0.39.0-beta.6", + "version": "0.39.0-beta.7", "os": [ "win32" ], diff --git a/nodejs/npm/win32-x64-msvc/package.json b/nodejs/npm/win32-x64-msvc/package.json index c2f0b0a80..ff612c4df 100644 --- a/nodejs/npm/win32-x64-msvc/package.json +++ b/nodejs/npm/win32-x64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-win32-x64-msvc", - "version": "0.39.0-beta.6", + "version": "0.39.0-beta.7", "os": ["win32"], "cpu": ["x64"], "main": "lancedb.win32-x64-msvc.node", diff --git a/nodejs/package.json b/nodejs/package.json index dae336765..e9621ba60 100644 --- a/nodejs/package.json +++ b/nodejs/package.json @@ -11,7 +11,7 @@ "ann" ], "private": false, - "version": "0.39.0-beta.6", + "version": "0.39.0-beta.7", "main": "dist/index.js", "exports": { ".": "./dist/index.js", diff --git a/python/Cargo.toml b/python/Cargo.toml index e0c72df48..ff7a98234 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lancedb-python" -version = "0.39.0-beta.6" +version = "0.39.0-beta.7" publish = false edition.workspace = true description = "Python bindings for LanceDB" diff --git a/rust/lancedb/Cargo.toml b/rust/lancedb/Cargo.toml index c8f17d50c..404753d7a 100644 --- a/rust/lancedb/Cargo.toml +++ b/rust/lancedb/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lancedb" -version = "0.39.0-beta.6" +version = "0.39.0-beta.7" edition.workspace = true description = "LanceDB: A serverless, low-latency vector database for AI applications" license.workspace = true From 255da8a8546cac3fe30e15939ff2ef516e4f640c Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Mon, 14 Sep 2026 15:09:29 +0800 Subject: [PATCH 6/8] fix(python): resolve native job types in API docs (#4170) Native job metadata types report `builtins` as their module, so Griffe cannot resolve the public `lancedb.job` re-exports and the Python API reference build fails. Set the PyO3 module metadata for `JobInfo`, `JobDescription`, and `JobFailureInfo`, and cover import resolution in the existing package metadata tests. Reproduced the failure and validated the fix with the docs CI toolchain (`griffe==0.49.0`, `mkdocstrings==0.25.2`, and `mkdocstrings-python==1.10.9`). After rebuilding the native extension, the full `PYTHONPATH=. mkdocs build` succeeds and all three classes and their public members appear in the generated reference. --- python/python/tests/test_package_metadata.py | 9 +++++++++ python/src/job.rs | 6 +++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/python/python/tests/test_package_metadata.py b/python/python/tests/test_package_metadata.py index 5792f457b..27def814f 100644 --- a/python/python/tests/test_package_metadata.py +++ b/python/python/tests/test_package_metadata.py @@ -9,6 +9,15 @@ from pathlib import Path import pytest +@pytest.mark.parametrize("name", ["JobInfo", "JobDescription", "JobFailureInfo"]) +def test_job_metadata_types_have_resolvable_modules(name): + """Documentation tools resolve re-exports through each type's module.""" + public_type = getattr(importlib.import_module("lancedb.job"), name) + defining_module = importlib.import_module(public_type.__module__) + + assert getattr(defining_module, public_type.__name__, None) is public_type + + def test_pyo3_abi_matches_minimum_supported_python(): project_dir = Path(__file__).parents[2] pyproject = (project_dir / "pyproject.toml").read_text() diff --git a/python/src/job.rs b/python/src/job.rs index 4922c701a..e22b2f897 100644 --- a/python/src/job.rs +++ b/python/src/job.rs @@ -151,7 +151,7 @@ impl Job { } /// A row from `Connection.list_jobs`: one server-side job. -#[pyclass(get_all, skip_from_py_object)] +#[pyclass(module = "lancedb._lancedb", get_all, skip_from_py_object)] #[derive(Clone)] pub struct JobInfo { job_id: String, @@ -184,7 +184,7 @@ impl From for JobInfo { } /// The server's account of why a job failed. -#[pyclass(get_all, skip_from_py_object)] +#[pyclass(module = "lancedb._lancedb", get_all, skip_from_py_object)] #[derive(Clone)] pub struct JobFailureInfo { phase: Option, @@ -203,7 +203,7 @@ impl JobFailureInfo { } /// The server-side record behind a `Job` handle. -#[pyclass(get_all, skip_from_py_object)] +#[pyclass(module = "lancedb._lancedb", get_all, skip_from_py_object)] #[derive(Clone)] pub struct JobDescription { job_id: String, From 6bb64c3edb50f372088d8ed34af9509daf807d0d Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Mon, 14 Sep 2026 20:50:54 +0800 Subject: [PATCH 7/8] fix: reject repeated job pagination tokens (#4139) Fixes #4138 `RemoteDatabase::list_jobs` followed every returned pagination token without remembering previously seen values. A server-side token cycle therefore caused repeated requests and duplicate accumulation until the 100-page safeguard returned partial results as a success. This change tracks non-empty job-list page tokens and returns an HTTP-context error as soon as a token repeats, matching the existing `list_functions` behavior. A mock-handler regression test verifies a repeated `loop` token is rejected after two requests. Validation: - `cargo test --quiet --features remote -p lancedb test_list_jobs` - `cargo fmt --all` - `cargo check --quiet --features remote --tests --examples` Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Co-authored-by: Xuanwo --- rust/lancedb/src/remote/db.rs | 46 ++++++++++++++++++++++++++++++++--- 1 file changed, 43 insertions(+), 3 deletions(-) diff --git a/rust/lancedb/src/remote/db.rs b/rust/lancedb/src/remote/db.rs index 917bf909b..1d6f9abe8 100644 --- a/rust/lancedb/src/remote/db.rs +++ b/rust/lancedb/src/remote/db.rs @@ -700,6 +700,7 @@ impl Database for RemoteDatabase { async fn list_jobs(&self) -> Result> { let mut out = Vec::new(); let mut page_token: Option = None; + let mut seen_page_tokens = HashSet::new(); for page in 0..MAX_LIST_JOBS_PAGES { let mut body = serde_json::json!({}); if let Some(token) = &page_token { @@ -708,7 +709,8 @@ impl Database for RemoteDatabase { let req = self.client.post("/v1/jobs/list").json(&body); let (request_id, rsp) = self.client.send(req).await?; let rsp = self.client.check_response(&request_id, rsp).await?; - let body: RemoteListJobsResponse = rsp.json().await.err_to_http(request_id)?; + let status = rsp.status(); + let body: RemoteListJobsResponse = rsp.json().await.err_to_http(request_id.clone())?; out.extend(body.jobs.into_iter().map(|row| JobInfo { job_id: row.job_id, table: row.table, @@ -716,10 +718,17 @@ impl Database for RemoteDatabase { state: job_state_to_client(&row.state), created_at_millis: row.created_at_millis, })); - page_token = body.page_token; - if page_token.is_none() { + let Some(next_page_token) = body.page_token.filter(|token| !token.is_empty()) else { break; + }; + if !seen_page_tokens.insert(next_page_token.clone()) { + return Err(Error::Http { + source: "Job listing response repeated a page_token".into(), + request_id, + status_code: Some(status), + }); } + page_token = Some(next_page_token); if page + 1 == MAX_LIST_JOBS_PAGES { log::warn!( "list_jobs truncated after {} pages ({} jobs)", @@ -2634,6 +2643,37 @@ mod tests { assert_eq!(jobs[2].state, "failed"); } + #[tokio::test] + async fn test_list_jobs_rejects_a_page_token_cycle() { + let requests = Arc::new(AtomicUsize::new(0)); + let seen = requests.clone(); + let conn = Connection::new_with_handler(move |request| { + let body: serde_json::Value = + serde_json::from_slice(request.body().unwrap().as_bytes().unwrap()).unwrap(); + match seen.fetch_add(1, Ordering::SeqCst) { + 0 => assert!(body.get("page_token").is_none()), + _ => assert_eq!(body["page_token"], "loop"), + } + http::Response::builder() + .status(200) + .body(r#"{"jobs": [], "page_token": "loop"}"#) + .unwrap() + }); + + let error = conn.list_jobs().await.unwrap_err(); + assert!( + matches!( + &error, + Error::Http { + status_code: Some(http::StatusCode::OK), + .. + } + ), + "got {error:?}" + ); + assert_eq!(requests.load(Ordering::SeqCst), 2); + } + #[tokio::test] async fn test_open_job() { let conn = Connection::new_with_handler(|request| { From 0113cee48976c5c4632ff251ff42a46f913ba4e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E5=BF=97=E8=B0=A6?= <89645338+simpleqt@users.noreply.github.com> Date: Mon, 14 Sep 2026 22:40:50 +0800 Subject: [PATCH 8/8] docs(embeddings): correct the documented max_retries default (#4145) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `lancedb/embeddings/utils.py` documents `max_retries` with "(default is 10)" — the signature default is `7`. Docs-only; conventional title per the contribution guide. Co-authored-by: Xuanwo --- python/python/lancedb/embeddings/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/python/lancedb/embeddings/utils.py b/python/python/lancedb/embeddings/utils.py index 189bbe53c..98806fa52 100644 --- a/python/python/lancedb/embeddings/utils.py +++ b/python/python/lancedb/embeddings/utils.py @@ -249,7 +249,7 @@ def retry_with_exponential_backoff( initial_delay (float): Initial delay in seconds (default is 1). exponential_base (float): The base for exponential backoff (default is 2). jitter (bool): Whether to add jitter to the delay (default is True). - max_retries (int): Maximum number of retries (default is 10). + max_retries (int): Maximum number of retries (default is 7). Returns: function: The decorated function.