From 7e171f3abb94b85077d6c468fd755cd6a5850a84 Mon Sep 17 00:00:00 2001 From: LanceDB Robot Date: Mon, 21 Sep 2026 11:08:11 -0700 Subject: [PATCH 1/3] chore: update lance dependency to v13.0.0-beta.8 (#4242) Updates the Rust workspace Lance dependencies and Java lance-core to [v13.0.0-beta.8](https://github.com/lance-format/lance/releases/tag/v13.0.0-beta.8), refreshing Cargo.lock including the required goosefs-sdk 0.2.2 update. No source compatibility changes were required. Validation passed: `cargo clippy --quiet --workspace --tests --all-features -- -D warnings`, `cargo fmt --all --quiet`, and `git diff --check`. --- Cargo.lock | 227 ++++++++++++++++++++++++++++++++++++--------------- Cargo.toml | 28 +++---- java/pom.xml | 2 +- 3 files changed, 174 insertions(+), 83 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f4491ba13..bb5f388cd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -141,7 +141,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -152,7 +152,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -1819,7 +1819,7 @@ version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -2937,6 +2937,12 @@ dependencies = [ "sqlparser 0.62.0", ] +[[package]] +name = "datasketches" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46c4cf71a36b46dcfc00e5014c0c20ccad2b1b6a008304d7d57d2749b2d41b3d" + [[package]] name = "debugid" version = "0.8.0" @@ -3081,7 +3087,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -3309,7 +3315,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -3499,6 +3505,65 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "foyer-common" +version = "0.22.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "643b47d510032a01e5af70dca288b0c5ec531646c1a8392e793fb5b0c13bef30" +dependencies = [ + "anyhow", + "bytes", + "cfg-if 1.0.4", + "foyer-tokio", + "mixtrics", + "parking_lot", + "pin-project", + "twox-hash", +] + +[[package]] +name = "foyer-intrusive-collections" +version = "0.10.0-dev" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e4fee46bea69e0596130e3210e65d3424e0ac1e6df3bde6636304bdf1ca4a3b" +dependencies = [ + "memoffset", +] + +[[package]] +name = "foyer-memory" +version = "0.22.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae51f5089f3d9025ae77f17ea66d2489ac0f82fbb1d91462807bea174d486d82" +dependencies = [ + "anyhow", + "asyncband", + "bitflags 2.11.1", + "datasketches", + "equivalent", + "foyer-common", + "foyer-intrusive-collections", + "foyer-tokio", + "futures-util", + "hashbrown 0.17.1", + "itertools 0.15.0", + "mixtrics", + "parking_lot", + "paste", + "pin-project", + "serde", + "tracing", +] + +[[package]] +name = "foyer-tokio" +version = "0.22.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6741d1133dfaab64d3f8a9290bbfed3685084add28f8b6ee7a6264aa4dc26918" +dependencies = [ + "tokio", +] + [[package]] name = "frostem" version = "1.20260804.1" @@ -3523,8 +3588,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "fsst" -version = "13.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.7#ba4192424e7a871a448431f247366df2f3609ed7" +version = "13.0.0-beta.8" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d" dependencies = [ "arrow-array", "rand 0.9.5", @@ -4004,18 +4069,23 @@ dependencies = [ [[package]] name = "goosefs-sdk" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f521e3beacc809ced36a8aff4a468b1bf61302c8f4dca41c3f21ac0557c22ed" +checksum = "1e471f7bce40936cf2023855c7350e31e11b3f703eb7cb2ce73c9ffc451f87d2" dependencies = [ "arc-swap", "async-trait", "bytes", "dashmap", "fastrand", + "foyer-common", + "foyer-memory", "futures", "hostname", + "io-uring", "itoa", + "libc", + "lru", "prost", "rand 0.9.5", "serde", @@ -4468,7 +4538,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.6.3", + "socket2 0.5.10", "system-configuration", "tokio", "tower-service", @@ -4810,7 +4880,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -5073,8 +5143,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a" [[package]] name = "lance" -version = "13.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.7#ba4192424e7a871a448431f247366df2f3609ed7" +version = "13.0.0-beta.8" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d" dependencies = [ "arc-swap", "arrow", @@ -5146,8 +5216,8 @@ dependencies = [ [[package]] name = "lance-arrow" -version = "13.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.7#ba4192424e7a871a448431f247366df2f3609ed7" +version = "13.0.0-beta.8" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d" dependencies = [ "arrow-array", "arrow-buffer", @@ -5169,7 +5239,7 @@ dependencies = [ [[package]] name = "lance-arrow-scalar" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.7#ba4192424e7a871a448431f247366df2f3609ed7" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d" dependencies = [ "arrow-array", "arrow-buffer", @@ -5183,7 +5253,7 @@ dependencies = [ [[package]] name = "lance-arrow-stats" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.7#ba4192424e7a871a448431f247366df2f3609ed7" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d" dependencies = [ "arrow-array", "arrow-schema", @@ -5192,8 +5262,8 @@ dependencies = [ [[package]] name = "lance-bitpacking" -version = "13.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.7#ba4192424e7a871a448431f247366df2f3609ed7" +version = "13.0.0-beta.8" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d" dependencies = [ "arrayref", "crunchy", @@ -5203,8 +5273,8 @@ dependencies = [ [[package]] name = "lance-core" -version = "13.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.7#ba4192424e7a871a448431f247366df2f3609ed7" +version = "13.0.0-beta.8" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d" dependencies = [ "arrow-array", "arrow-buffer", @@ -5241,8 +5311,8 @@ dependencies = [ [[package]] name = "lance-datafusion" -version = "13.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.7#ba4192424e7a871a448431f247366df2f3609ed7" +version = "13.0.0-beta.8" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d" dependencies = [ "arrow", "arrow-array", @@ -5273,8 +5343,8 @@ dependencies = [ [[package]] name = "lance-datagen" -version = "13.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.7#ba4192424e7a871a448431f247366df2f3609ed7" +version = "13.0.0-beta.8" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d" dependencies = [ "arrow", "arrow-array", @@ -5291,8 +5361,8 @@ dependencies = [ [[package]] name = "lance-derive" -version = "13.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.7#ba4192424e7a871a448431f247366df2f3609ed7" +version = "13.0.0-beta.8" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d" dependencies = [ "proc-macro2", "quote", @@ -5301,8 +5371,8 @@ dependencies = [ [[package]] name = "lance-encoding" -version = "13.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.7#ba4192424e7a871a448431f247366df2f3609ed7" +version = "13.0.0-beta.8" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d" dependencies = [ "arrow-arith", "arrow-array", @@ -5335,8 +5405,8 @@ dependencies = [ [[package]] name = "lance-file" -version = "13.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.7#ba4192424e7a871a448431f247366df2f3609ed7" +version = "13.0.0-beta.8" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d" dependencies = [ "arrow-arith", "arrow-array", @@ -5368,8 +5438,8 @@ dependencies = [ [[package]] name = "lance-geo" -version = "13.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.7#ba4192424e7a871a448431f247366df2f3609ed7" +version = "13.0.0-beta.8" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d" dependencies = [ "datafusion", "geo-traits", @@ -5383,8 +5453,8 @@ dependencies = [ [[package]] name = "lance-index" -version = "13.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.7#ba4192424e7a871a448431f247366df2f3609ed7" +version = "13.0.0-beta.8" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d" dependencies = [ "arc-swap", "arrow", @@ -5452,8 +5522,8 @@ dependencies = [ [[package]] name = "lance-index-core" -version = "13.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.7#ba4192424e7a871a448431f247366df2f3609ed7" +version = "13.0.0-beta.8" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d" dependencies = [ "arrow-array", "arrow-schema", @@ -5475,8 +5545,8 @@ dependencies = [ [[package]] name = "lance-io" -version = "13.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.7#ba4192424e7a871a448431f247366df2f3609ed7" +version = "13.0.0-beta.8" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d" dependencies = [ "arrow", "arrow-array", @@ -5488,6 +5558,7 @@ dependencies = [ "bytes", "chrono", "futures", + "goosefs-sdk", "http 1.5.0", "io-uring", "lance-core", @@ -5498,6 +5569,7 @@ dependencies = [ "object_store 0.14.1", "object_store_opendal", "opendal", + "opendal-service-goosefs", "path_abs", "pin-project", "prost", @@ -5516,8 +5588,8 @@ dependencies = [ [[package]] name = "lance-linalg" -version = "13.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.7#ba4192424e7a871a448431f247366df2f3609ed7" +version = "13.0.0-beta.8" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d" dependencies = [ "arrow-array", "arrow-schema", @@ -5531,8 +5603,8 @@ dependencies = [ [[package]] name = "lance-namespace" -version = "13.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.7#ba4192424e7a871a448431f247366df2f3609ed7" +version = "13.0.0-beta.8" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d" dependencies = [ "arrow", "async-trait", @@ -5546,8 +5618,8 @@ dependencies = [ [[package]] name = "lance-namespace-impls" -version = "13.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.7#ba4192424e7a871a448431f247366df2f3609ed7" +version = "13.0.0-beta.8" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d" dependencies = [ "arrow", "arrow-ipc", @@ -5600,8 +5672,8 @@ dependencies = [ [[package]] name = "lance-select" -version = "13.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.7#ba4192424e7a871a448431f247366df2f3609ed7" +version = "13.0.0-beta.8" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d" dependencies = [ "arrow-array", "arrow-buffer", @@ -5615,8 +5687,8 @@ dependencies = [ [[package]] name = "lance-table" -version = "13.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.7#ba4192424e7a871a448431f247366df2f3609ed7" +version = "13.0.0-beta.8" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d" dependencies = [ "arrow", "arrow-array", @@ -5656,8 +5728,8 @@ dependencies = [ [[package]] name = "lance-testing" -version = "13.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.7#ba4192424e7a871a448431f247366df2f3609ed7" +version = "13.0.0-beta.8" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d" dependencies = [ "arrow-array", "arrow-schema", @@ -5670,8 +5742,8 @@ dependencies = [ [[package]] name = "lance-tokenizer" -version = "13.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.7#ba4192424e7a871a448431f247366df2f3609ed7" +version = "13.0.0-beta.8" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d" dependencies = [ "frostem", "icu_segmenter", @@ -6207,6 +6279,15 @@ dependencies = [ "stable_deref_trait", ] +[[package]] +name = "memoffset" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" +dependencies = [ + "autocfg", +] + [[package]] name = "metrics" version = "0.24.6" @@ -6280,6 +6361,16 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "mixtrics" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c46b5adfb7a3ae4996d327a5bdc90e78fec025806dd312bdbe6f07a755e0ec9" +dependencies = [ + "itertools 0.15.0", + "parking_lot", +] + [[package]] name = "moka" version = "0.12.16" @@ -6540,7 +6631,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -6669,7 +6760,7 @@ version = "5.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51e219e79014df21a225b1860a479e2dcd7cbd9130f4defd4bd0e191ea31d67d" dependencies = [ - "base64 0.22.1", + "base64 0.21.7", "chrono", "getrandom 0.2.17", "http 1.5.0", @@ -8043,8 +8134,8 @@ version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7" dependencies = [ - "heck 0.5.0", - "itertools 0.14.0", + "heck 0.4.1", + "itertools 0.11.0", "log", "multimap", "petgraph", @@ -8063,7 +8154,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", - "itertools 0.14.0", + "itertools 0.11.0", "proc-macro2", "quote", "syn 2.0.117", @@ -8285,7 +8376,7 @@ dependencies = [ "quinn-udp", "rustc-hash", "rustls 0.23.45", - "socket2 0.6.3", + "socket2 0.5.10", "thiserror 2.0.18", "tokio", "tracing", @@ -8323,7 +8414,7 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.6.3", + "socket2 0.5.10", "tracing", "windows-sys 0.60.2", ] @@ -9123,7 +9214,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -9194,7 +9285,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -9741,7 +9832,7 @@ version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451" dependencies = [ - "heck 0.5.0", + "heck 0.4.1", "proc-macro2", "quote", "syn 2.0.117", @@ -9753,7 +9844,7 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "54254b8531cafa275c5e096f62d48c81435d1015405a91198ddb11e967301d40" dependencies = [ - "heck 0.5.0", + "heck 0.4.1", "proc-macro2", "quote", "syn 2.0.117", @@ -9776,7 +9867,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -9895,7 +9986,7 @@ dependencies = [ "cfg-if 1.0.4", "libc", "psm", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -10187,7 +10278,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -11208,7 +11299,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index cb67b9f8c..eb691ac48 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,20 +13,20 @@ categories = ["database-implementations"] rust-version = "1.91.0" [workspace.dependencies] -lance = { "version" = "=13.0.0-beta.7", default-features = false, "tag" = "v13.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } -lance-core = { "version" = "=13.0.0-beta.7", "tag" = "v13.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } -lance-datagen = { "version" = "=13.0.0-beta.7", "tag" = "v13.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } -lance-file = { "version" = "=13.0.0-beta.7", "tag" = "v13.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } -lance-io = { "version" = "=13.0.0-beta.7", default-features = false, "tag" = "v13.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } -lance-index = { "version" = "=13.0.0-beta.7", "tag" = "v13.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } -lance-linalg = { "version" = "=13.0.0-beta.7", "tag" = "v13.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace = { "version" = "=13.0.0-beta.7", "tag" = "v13.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace-impls = { "version" = "=13.0.0-beta.7", default-features = false, "tag" = "v13.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } -lance-table = { "version" = "=13.0.0-beta.7", "tag" = "v13.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } -lance-testing = { "version" = "=13.0.0-beta.7", "tag" = "v13.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } -lance-datafusion = { "version" = "=13.0.0-beta.7", "tag" = "v13.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } -lance-encoding = { "version" = "=13.0.0-beta.7", "tag" = "v13.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } -lance-arrow = { "version" = "=13.0.0-beta.7", "tag" = "v13.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } +lance = { "version" = "=13.0.0-beta.8", default-features = false, "tag" = "v13.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" } +lance-core = { "version" = "=13.0.0-beta.8", "tag" = "v13.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" } +lance-datagen = { "version" = "=13.0.0-beta.8", "tag" = "v13.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" } +lance-file = { "version" = "=13.0.0-beta.8", "tag" = "v13.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" } +lance-io = { "version" = "=13.0.0-beta.8", default-features = false, "tag" = "v13.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" } +lance-index = { "version" = "=13.0.0-beta.8", "tag" = "v13.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" } +lance-linalg = { "version" = "=13.0.0-beta.8", "tag" = "v13.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace = { "version" = "=13.0.0-beta.8", "tag" = "v13.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace-impls = { "version" = "=13.0.0-beta.8", default-features = false, "tag" = "v13.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" } +lance-table = { "version" = "=13.0.0-beta.8", "tag" = "v13.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" } +lance-testing = { "version" = "=13.0.0-beta.8", "tag" = "v13.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" } +lance-datafusion = { "version" = "=13.0.0-beta.8", "tag" = "v13.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" } +lance-encoding = { "version" = "=13.0.0-beta.8", "tag" = "v13.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" } +lance-arrow = { "version" = "=13.0.0-beta.8", "tag" = "v13.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" } lancedb = { path = "rust/lancedb", default-features = false } ahash = "0.8" # Note that this one does not include pyarrow diff --git a/java/pom.xml b/java/pom.xml index 5e5068552..a42d31540 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -28,7 +28,7 @@ UTF-8 15.0.0 - 13.0.0-beta.7 + 13.0.0-beta.8 false 2.30.0 1.7 From df2e785df73f75cc17c87485d5345969056837dd Mon Sep 17 00:00:00 2001 From: Wyatt Alt Date: Mon, 21 Sep 2026 11:51:44 -0700 Subject: [PATCH 2/3] feat: retire a Function binding when its output columns are dropped (#4231) Dropping a column a Function binding writes was refused outright, which left a bad declaration unrecoverable: the binding is immutable, there is no rebind, and so the column could never be filled again. A drop that names every output of a binding now retires it instead. A binding lives in three places -- the envelope in schema metadata, a `computed_column.*` marker on each output field, and the columns themselves -- and readers cross-check the first two, so removing one and leaving the others is a table that refuses every write. The retirement therefore commits in two steps, each landing a state that stands on its own: one `UpdateConfig` rewrites the envelope and clears the markers together, leaving the outputs as ordinary columns holding their last values, and the drop follows. Interrupted between them, the columns are still there to be dropped again. Folding the metadata into the drop's own `Project` was the obvious alternative and does not work: the transaction proto records `Project` as fields alone, so the edit would survive only in the writer's memory. Naming one output of a multi-output binding is refused and names the missing siblings, since one refresh writes them in one commit. A multi-output binding's hidden `__function_assignment_*` column goes with it. Inputs stay protected while a surviving binding reads them, and drop in the request that retires the last one. --- rust/lancedb/src/table/computed_columns.rs | 190 ++++++++- rust/lancedb/src/table/schema_evolution.rs | 447 +++++++++++++++++++-- 2 files changed, 596 insertions(+), 41 deletions(-) diff --git a/rust/lancedb/src/table/computed_columns.rs b/rust/lancedb/src/table/computed_columns.rs index 23a870b42..679928cd3 100644 --- a/rust/lancedb/src/table/computed_columns.rs +++ b/rust/lancedb/src/table/computed_columns.rs @@ -418,23 +418,18 @@ pub(crate) fn ensure_not_function_bound>( touched: impl IntoIterator, ) -> Result<()> { ensure_supported_function_metadata(schema)?; - let mut protected = BTreeSet::new(); - for binding in function_bindings(schema)? { - for input in binding.inputs() { - protected.insert(field_root(&input.field_path)?); - } - protected.extend( - binding - .outputs() - .iter() - .map(|output| output.output_name.clone()), - ); - protected.extend( - binding - .assignment() - .map(|assignment| assignment.output_name.clone()), - ); - } + ensure_not_bound_by(&function_bindings(schema)?, operation, touched) +} + +/// [`ensure_not_function_bound`] against an explicit set of bindings, for a +/// caller that is retiring some of the schema's own in the same operation and +/// must be checked against what survives it. +pub(crate) fn ensure_not_bound_by>( + bindings: &[FunctionBinding], + operation: &str, + touched: impl IntoIterator, +) -> Result<()> { + let protected = protected_roots(bindings)?; if protected.is_empty() { return Ok(()); } @@ -452,6 +447,129 @@ pub(crate) fn ensure_not_function_bound>( Ok(()) } +/// Every column `bindings` depend on, inputs taken at their root. +fn protected_roots(bindings: &[FunctionBinding]) -> Result> { + let mut protected = BTreeSet::new(); + for binding in bindings { + for input in binding.inputs() { + protected.insert(field_root(&input.field_path)?); + } + protected.extend( + binding + .outputs() + .iter() + .map(|output| output.output_name.clone()), + ); + protected.extend( + binding + .assignment() + .map(|assignment| assignment.output_name.clone()), + ); + } + Ok(protected) +} + +/// What a drop must do besides removing columns, so that the Function +/// bindings it covers are retired rather than stranded. +#[derive(Debug, Default)] +pub struct FunctionUnbinding { + /// The bindings the drop leaves in place. Later columns in the same + /// operation are checked against these, not against the schema's. + pub retained: Vec, + /// Assignment columns of the retired bindings that the caller did not + /// name. Internal bookkeeping columns, so the drop adds them itself. + pub assignment_columns: Vec, + /// Columns whose `computed_column.*` declaration metadata the unbind + /// commit clears. + pub cleared_columns: Vec, + /// The new value of [`FUNCTION_BINDINGS_META_KEY`]; `None` deletes it. + pub bindings_metadata: Option, +} + +impl FunctionUnbinding { + /// True when the drop retires nothing and is an ordinary column drop. + pub fn is_noop(&self) -> bool { + self.cleared_columns.is_empty() + } + + /// Refuse `operation` on a path that a binding surviving this drop still + /// reads or writes. The retired ones no longer protect anything. + pub fn ensure_retained_unaffected>( + &self, + operation: &str, + touched: impl IntoIterator, + ) -> Result<()> { + ensure_not_bound_by(&self.retained, operation, touched) + } +} + +/// Plan the retirement of every Function binding whose outputs `columns` +/// covers. +/// +/// Naming one output of a binding means naming all of them: the outputs of +/// one Function are written by one refresh in one commit, so a binding that +/// kept some of them would have no coherent shape to write. A partial drop is +/// refused and names the siblings that are missing. +pub fn plan_function_unbinding( + schema: &ArrowSchema, + columns: &[&str], +) -> Result { + ensure_supported_function_metadata(schema)?; + let named = columns + .iter() + .map(|column| whole_column(column)) + .collect::>>>()? + .into_iter() + .flatten() + .collect::>(); + + let mut plan = FunctionUnbinding::default(); + for binding in function_bindings(schema)? { + let outputs = binding + .outputs() + .iter() + .map(|output| output.output_name.clone()) + .collect::>(); + let Some(dropped) = outputs.iter().find(|name| named.contains(name.as_str())) else { + plan.retained.push(binding); + continue; + }; + let missing = outputs + .iter() + .filter(|name| !named.contains(name.as_str())) + .cloned() + .collect::>(); + if !missing.is_empty() { + return Err(Error::InvalidInput { + message: format!( + "dropping Function output '{dropped}' must drop every output of its \ + binding; the same request must also name {}", + missing.join(", ") + ), + }); + } + plan.cleared_columns.extend(outputs); + if let Some(assignment) = binding.assignment() { + plan.cleared_columns.push(assignment.output_name.clone()); + if !named.contains(assignment.output_name.as_str()) { + plan.assignment_columns.push(assignment.output_name.clone()); + } + } + } + + // Re-encoding an untouched set would rewrite bytes a stricter reader + // compares against its own encoding, so leave the key alone when the drop + // retires nothing. + if !plan.is_noop() { + plan.bindings_metadata = if plan.retained.is_empty() { + None + } else { + Some(function_bindings_metadata(&plan.retained)?) + }; + } + Ok(plan) +} + /// Read a field's computed-column declaration, if it carries one. /// /// A field flagged computed but carrying no kind, or a SQL one missing its @@ -1320,7 +1438,30 @@ pub(crate) fn plan_function_application( /// Paths are compared at their root: a declaration reading `metadata` is /// invalidated by a change to `metadata.age` just as surely. pub(crate) fn ensure_not_an_input(schema: &SchemaRef, paths: &[&str]) -> Result<()> { + ensure_not_an_input_of(schema, paths, &[]) +} + +/// [`ensure_not_an_input`] where `removed` names columns whose own +/// declarations the same operation deletes. A reader that is going away +/// cannot be stranded by dropping what it reads, so a group drops together. +pub(crate) fn ensure_not_an_input_of( + schema: &SchemaRef, + paths: &[&str], + removed: &[&str], +) -> Result<()> { + // By identity, not spelling: a quoted path names the same column, while + // a path *into* one removes no declaration at all. + let removed = removed + .iter() + .map(|path| whole_column(path)) + .collect::>>()? + .into_iter() + .flatten() + .collect::>(); for declaration in computed_columns(schema) { + if removed.contains(&declaration.name) { + continue; + } // The expression, not stored inputs, is the source of truth; an // expression that no longer parses proves nothing, so refuse. let inputs = match &declaration.kind { @@ -1583,6 +1724,21 @@ pub(crate) fn field_root(path: &str) -> Result { }) } +/// The column a path names when it names the whole column, and `None` when it +/// addresses a field inside one. Retiring a binding takes the whole output; +/// a path into one is left to the ordinary guard to refuse. +fn whole_column(path: &str) -> Result> { + let mut segments = parse_field_path(path) + .map_err(|e| Error::InvalidInput { + message: format!("invalid column path '{path}': {e}"), + })? + .into_iter(); + let column = segments.next().ok_or_else(|| Error::InvalidInput { + message: format!("column path '{path}' is empty"), + })?; + Ok(segments.next().is_none().then_some(column)) +} + /// [`field_root`], falling back to the text before the first dot for a /// spelling lance would not resolve. pub(crate) fn root(path: &str) -> String { diff --git a/rust/lancedb/src/table/schema_evolution.rs b/rust/lancedb/src/table/schema_evolution.rs index 796db09bd..4d0e2815d 100644 --- a/rust/lancedb/src/table/schema_evolution.rs +++ b/rust/lancedb/src/table/schema_evolution.rs @@ -9,7 +9,8 @@ //! - [`drop_columns`](execute_drop_columns): Remove columns from the table use arrow_schema::Schema as ArrowSchema; -use lance::dataset::{ColumnAlteration, NewColumnTransform}; +use lance::dataset::transaction::{Operation, Transaction, UpdateMap, UpdateMapEntry}; +use lance::dataset::{ColumnAlteration, CommitBuilder, Dataset, NewColumnTransform}; use serde::{Deserialize, Serialize}; use std::collections::HashMap; @@ -240,14 +241,96 @@ pub(crate) async fn execute_drop_columns( table.dataset.ensure_mutable()?; let mut dataset = (*table.dataset.get().await?).clone(); let schema = std::sync::Arc::new(ArrowSchema::from(dataset.schema())); - computed_columns::ensure_not_function_bound(schema.as_ref(), "schema evolution", columns)?; - computed_columns::ensure_not_an_input(&schema, columns)?; - dataset.drop_columns(columns).await?; + let unbinding = computed_columns::plan_function_unbinding(schema.as_ref(), columns)?; + + let mut names = columns.iter().map(|c| c.to_string()).collect::>(); + names.extend(unbinding.assignment_columns.iter().cloned()); + let dropped = names.iter().map(String::as_str).collect::>(); + computed_columns::ensure_not_bound_by(&unbinding.retained, "schema evolution", &dropped)?; + computed_columns::ensure_not_an_input_of(&schema, &dropped, &dropped)?; + + if !unbinding.is_noop() { + // The retirement commits before the projection, so the whole + // projection has to be known valid against this revision first -- + // otherwise the binding is gone and the columns are not. Lance plans + // it rather than this repeating the rules: dropping a struct's last + // child removes the struct, and data files that disagree on metadata + // semantics are refused, neither of which an approximation here would + // get right. + dataset.plan_drop_columns(&dropped)?; + dataset = commit_function_unbinding(dataset, &unbinding).await?; + } + dataset.drop_columns(&dropped).await?; let version = dataset.version().version; table.dataset.update(dataset); Ok(DropColumnsResult { version }) } +/// Retire the bindings a drop covers, in the commit before it. +/// +/// A binding lives in three places -- the schema-metadata envelope, each +/// output's `computed_column.*` field metadata, and the columns themselves -- +/// and readers cross-check the first two, so a state with one of them removed +/// is a table that refuses every write. Clearing both metadata halves in one +/// `UpdateConfig` leaves the outputs as ordinary columns holding their last +/// values: valid on its own, and the drop that follows is then an ordinary +/// drop. Interrupted in between, the columns are still there to be dropped +/// again. +/// +/// Not folded into the drop's own `Operation::Project`, though it carries a +/// schema: the transaction proto records `Project` as fields alone, so the +/// metadata edit would survive only in this writer's memory. +async fn commit_function_unbinding( + dataset: Dataset, + unbinding: &computed_columns::FunctionUnbinding, +) -> Result { + let bindings = UpdateMap { + update_entries: vec![UpdateMapEntry { + key: computed_columns::FUNCTION_BINDINGS_META_KEY.to_string(), + value: unbinding.bindings_metadata.clone(), + }], + replace: false, + }; + let mut field_metadata_updates = HashMap::new(); + for column in &unbinding.cleared_columns { + let field = dataset + .schema() + .field(column) + .ok_or_else(|| Error::InvalidInput { + message: format!("Function output '{column}' does not exist in the table"), + })?; + let cleared = field + .metadata + .keys() + .filter(|key| computed_columns::is_declaration_key(key)) + .map(|key| UpdateMapEntry { + key: key.clone(), + value: None, + }) + .collect::>(); + field_metadata_updates.insert( + field.id, + UpdateMap { + update_entries: cleared, + replace: false, + }, + ); + } + let transaction = Transaction::new( + dataset.manifest.version, + Operation::UpdateConfig { + config_updates: None, + table_metadata_updates: None, + schema_metadata_updates: Some(bindings), + field_metadata_updates, + }, + None, + ); + Ok(CommitBuilder::new(std::sync::Arc::new(dataset)) + .execute(transaction) + .await?) +} + /// Internal implementation of the update field metadata logic. /// /// Merges or replaces per-field metadata, addressing fields by dot-path. @@ -330,8 +413,9 @@ mod tests { use crate::query::{ExecutableQuery, QueryBase, Select}; use crate::table::NewColumnTransform; use crate::table::computed_columns::{ - FUNCTION_BINDINGS_META_KEY, ensure_supported_function_metadata, function_bindings, - function_bindings_metadata, function_computed_column_metadata, + FUNCTION_ASSIGNMENT_OUTPUT_ORDINAL, FUNCTION_BINDINGS_META_KEY, + ensure_supported_function_metadata, function_bindings, function_bindings_metadata, + function_computed_column_metadata, is_declaration_key, }; use crate::{Error, Table}; use std::collections::HashMap; @@ -351,38 +435,71 @@ mod tests { ) .unwrap(); let table = conn.create_table("bound", batch).execute().await.unwrap(); - let binding = FunctionBinding::from_json(include_str!( + stamp_bindings(&table, &[text_features_binding("")]).await; + table + } + + /// The fixture binding, its id and output names suffixed so a table can + /// carry more than one. + fn text_features_binding(suffix: &str) -> FunctionBinding { + let raw = include_str!( "../../tests/fixtures/first_class_functions/v1/remote_function_binding.json" - )) - .unwrap(); + ); + if suffix.is_empty() { + return FunctionBinding::from_json(raw).unwrap(); + } + FunctionBinding::from_json( + &raw.replace("fb_01K3TEXT", &format!("fb_01K3TEXT{suffix}")) + .replace("search_text", &format!("search_text{suffix}")) + .replace("search_token_count", &format!("search_token_count{suffix}")), + ) + .unwrap() + } + + /// Stamp bindings the way the server does it, since no local path + /// declares one: the envelope on the schema, a declaration marker on + /// every output field. + async fn stamp_bindings(table: &Table, bindings: &[FunctionBinding]) { let native = table.as_native().unwrap(); let mut dataset = native.dataset.get().await.unwrap().as_ref().clone(); dataset .update_schema_metadata(vec![( FUNCTION_BINDINGS_META_KEY.to_string(), - Some(function_bindings_metadata(std::slice::from_ref(&binding)).unwrap()), + Some(function_bindings_metadata(bindings).unwrap()), )]) .await .unwrap(); let inputs = ["title".to_string(), "body".to_string()]; - let outputs = binding - .outputs() + let outputs = bindings .iter() - .map(|output| { - ( - dataset.schema().field(&output.output_name).unwrap().id as u32, - function_computed_column_metadata( - binding.binding_id(), - output.output_ordinal, - &inputs, - ), - ) + .flat_map(|binding| { + let assignment = binding.assignment().map(|assignment| { + ( + assignment.output_name.clone(), + FUNCTION_ASSIGNMENT_OUTPUT_ORDINAL, + ) + }); + binding + .outputs() + .iter() + .map(|output| (output.output_name.clone(), output.output_ordinal)) + .chain(assignment) + .map(|(name, ordinal)| { + ( + dataset.schema().field(&name).unwrap().id as u32, + function_computed_column_metadata( + binding.binding_id(), + ordinal, + &inputs, + ), + ) + }) + .collect::>() }) .collect::>(); dataset.replace_field_metadata(outputs).await.unwrap(); native.dataset.update(dataset); ensure_supported_function_metadata(&table.schema().await.unwrap()).unwrap(); - table } fn metadata_update(path: &str) -> FieldMetadataUpdate { @@ -438,6 +555,18 @@ mod tests { ); } + fn incomplete_group(err: Error) { + assert!( + matches!(&err, Error::InvalidInput { message } + if message.contains("must drop every output of its binding")), + "{err:?}" + ); + } + + async fn bindings_of(table: &Table) -> Vec { + function_bindings(table.schema().await.unwrap().as_ref()).unwrap() + } + /// Every schema-evolution door refuses a column a binding reads or /// writes, including a rename onto one. #[tokio::test] @@ -445,7 +574,14 @@ mod tests { let table = bound_table().await; let version = table.version().await.unwrap(); for column in ["title", "body", "search_text", "search_token_count"] { - bound(table.drop_columns(&[column]).await.unwrap_err()); + // A lone drop of an output is refused too, but for its siblings + // rather than for the binding; see the retirement tests below. + let err = table.drop_columns(&[column]).await.unwrap_err(); + if column.starts_with("search_") { + incomplete_group(err); + } else { + bound(err); + } bound( table .alter_columns(&[ColumnAlteration::new(column.into()).rename("moved".into())]) @@ -500,7 +636,12 @@ mod tests { let table = bound_table().await; let version = table.version().await.unwrap(); for path in ["`title`", "`search_text`", "`title`.nested"] { - bound(table.drop_columns(&[path]).await.unwrap_err()); + let err = table.drop_columns(&[path]).await.unwrap_err(); + if path == "`search_text`" { + incomplete_group(err); + } else { + bound(err); + } bound( table .alter_columns(&[ColumnAlteration::new(path.into()).set_nullable(false)]) @@ -1038,6 +1179,264 @@ mod tests { assert_eq!(drop_result.version, v4); } + // Retiring a Function binding by dropping its outputs (ENT-2669). + + /// The whole output set goes, and the binding with it: no envelope, no + /// declaration markers, and the inputs it held are ordinary again. + #[tokio::test] + async fn dropping_every_output_retires_the_binding() { + let table = bound_table().await; + table + .drop_columns(&["search_text", "search_token_count"]) + .await + .unwrap(); + + let schema = table.schema().await.unwrap(); + assert!(schema.field_with_name("search_text").is_err()); + assert!(schema.field_with_name("search_token_count").is_err()); + assert!(!schema.metadata().contains_key(FUNCTION_BINDINGS_META_KEY)); + assert!(bindings_of(&table).await.is_empty()); + + // What the binding used to protect is now ordinary schema. + table.drop_columns(&["title"]).await.unwrap(); + table + .update_field_metadata(&[metadata_update("body")]) + .await + .unwrap(); + ensure_supported_function_metadata(&table.schema().await.unwrap()).unwrap(); + } + + /// One output of a multi-output binding cannot go alone: its siblings are + /// written by the same refresh in the same commit. + #[tokio::test] + async fn dropping_part_of_an_output_group_names_the_missing_siblings() { + let table = bound_table().await; + let version = table.version().await.unwrap(); + for (named, missing) in [ + ("search_text", "search_token_count"), + ("search_token_count", "search_text"), + ] { + let err = table.drop_columns(&[named]).await.unwrap_err(); + let Error::InvalidInput { message } = &err else { + panic!("{err:?}"); + }; + assert!( + message.contains("must drop every output of its binding") + && message.contains(missing), + "{message}" + ); + } + assert_eq!(table.version().await.unwrap(), version); + assert_eq!(bindings_of(&table).await.len(), 1); + } + + /// An input is protected by the binding, not by its own name, so it drops + /// in the request that retires the binding reading it. + #[tokio::test] + async fn an_input_drops_with_the_binding_that_reads_it() { + let table = bound_table().await; + bound(table.drop_columns(&["title"]).await.unwrap_err()); + + table + .drop_columns(&["title", "body", "search_text", "search_token_count"]) + .await + .unwrap(); + let schema = table.schema().await.unwrap(); + assert_eq!(schema.fields().len(), 1); + assert!(schema.field_with_name("spare").is_ok()); + assert!(bindings_of(&table).await.is_empty()); + } + + /// Lance resolves a quoted spelling to the same field, so it retires the + /// same binding rather than slipping past the group rule. + #[tokio::test] + async fn a_quoted_output_spelling_retires_the_same_binding() { + let table = bound_table().await; + incomplete_group(table.drop_columns(&["`search_text`"]).await.unwrap_err()); + table + .drop_columns(&["`search_text`", "`search_token_count`"]) + .await + .unwrap(); + assert!(bindings_of(&table).await.is_empty()); + } + + /// The unbind lands before the drop, and on its own: at that version the + /// outputs are still there as plain columns, with no binding and no + /// declaration metadata left pointing at one. A crash in between leaves + /// that state, which every later write must accept. + #[tokio::test] + async fn the_unbind_commit_leaves_a_valid_table_before_the_drop() { + let table = bound_table().await; + let before = table.version().await.unwrap(); + table + .drop_columns(&["search_text", "search_token_count"]) + .await + .unwrap(); + assert_eq!(table.version().await.unwrap(), before + 2); + + table.checkout(before + 1).await.unwrap(); + let schema = table.schema().await.unwrap(); + assert!(!schema.metadata().contains_key(FUNCTION_BINDINGS_META_KEY)); + for column in ["search_text", "search_token_count"] { + let field = schema.field_with_name(column).unwrap(); + assert!( + !field.metadata().keys().any(|key| is_declaration_key(key)), + "{column}: {:?}", + field.metadata() + ); + } + ensure_supported_function_metadata(&schema).unwrap(); + assert!(function_bindings(&schema).unwrap().is_empty()); + } + + /// A table carrying two bindings retires only the one whose outputs the + /// drop names; the other keeps its envelope entry and its protection. + #[tokio::test] + async fn retiring_one_binding_leaves_the_others_standing() { + let conn = connect("memory://").execute().await.unwrap(); + let batch = record_batch!( + ("title", Utf8, ["a"]), + ("body", Utf8, ["b"]), + ("search_text", Utf8, ["a b"]), + ("search_token_count", Int64, [2]), + ("search_text_2", Utf8, ["a b"]), + ("search_token_count_2", Int64, [2]) + ) + .unwrap(); + let table = conn.create_table("two", batch).execute().await.unwrap(); + stamp_bindings( + &table, + &[text_features_binding(""), text_features_binding("_2")], + ) + .await; + + table + .drop_columns(&["search_text", "search_token_count"]) + .await + .unwrap(); + let remaining = bindings_of(&table).await; + assert_eq!(remaining.len(), 1); + assert_eq!(remaining[0].binding_id(), "fb_01K3TEXT_2"); + // The survivor still protects the inputs it shared with the retired one. + bound(table.drop_columns(&["title"]).await.unwrap_err()); + // And its own outputs are still a group. + incomplete_group(table.drop_columns(&["search_text_2"]).await.unwrap_err()); + } + + /// A multi-output binding also owns a hidden `__function_assignment_*` + /// column, which the caller never names. Retiring the binding has to take + /// it: leaving it behind orphans a column nothing can fill or explain. + #[tokio::test] + async fn retiring_a_binding_drops_its_assignment_column() { + const ASSIGNMENT: &str = "__function_assignment_fb_01K3TEXT"; + + let conn = connect("memory://").execute().await.unwrap(); + let batch = record_batch!( + ("title", Utf8, ["a"]), + ("body", Utf8, ["b"]), + ("search_text", Utf8, ["a b"]), + ("search_token_count", Int64, [2]), + (ASSIGNMENT, Boolean, [Some(true)]), + ("spare", Int32, [1]) + ) + .unwrap(); + let table = conn + .create_table("assigned", batch) + .execute() + .await + .unwrap(); + + let raw = include_str!( + "../../tests/fixtures/first_class_functions/v1/remote_function_binding.json" + ) + .replace( + r#""future_binding""#, + &format!( + r#""assignment": {{"output_name": "{ASSIGNMENT}", "output_field_id": -1}}, "future_binding""# + ), + ) + // The assignment column is a physical sibling, so it belongs to the + // binding's output schema too -- the server writes it that way. + .replace( + r#"{"name": "search_token_count", "nullable": true, "type": {"type": "int64"}}"#, + &format!( + r#"{{"name": "search_token_count", "nullable": true, "type": {{"type": "int64"}}}}, {{"name": "{ASSIGNMENT}", "nullable": true, "type": {{"type": "bool"}}}}"# + ), + ); + let binding = FunctionBinding::from_json(&raw).unwrap(); + assert!(binding.assignment().is_some(), "fixture must carry one"); + stamp_bindings(&table, std::slice::from_ref(&binding)).await; + + // The assignment column is the binding's, so it is protected too... + bound(table.drop_columns(&[ASSIGNMENT]).await.unwrap_err()); + + // ...and goes with the outputs without the caller naming it. + table + .drop_columns(&["search_text", "search_token_count"]) + .await + .unwrap(); + let schema = table.schema().await.unwrap(); + assert!( + schema.field_with_name(ASSIGNMENT).is_err(), + "the assignment column outlived its binding: {:?}", + schema.fields().iter().map(|f| f.name()).collect::>() + ); + assert!(bindings_of(&table).await.is_empty()); + ensure_supported_function_metadata(&schema).unwrap(); + } + + /// An input error must not cost the binding. The retirement commits + /// first, so a projection that could never succeed is refused before it, + /// leaving the table exactly as it was. + #[rstest::rstest] + #[case::unknown_column( + &["search_text", "search_token_count", "nope"], + "does not exist" + )] + #[case::every_column( + &["title", "body", "search_text", "search_token_count", "spare"], + "Cannot drop all columns" + )] + #[case::every_column_quoted( + &["`title`", "`body`", "`search_text`", "`search_token_count`", "`spare`"], + "Cannot drop all columns" + )] + #[tokio::test] + async fn an_invalid_drop_leaves_the_binding_in_place( + #[case] columns: &[&str], + #[case] expected: &str, + ) { + let table = bound_table().await; + let version = table.version().await.unwrap(); + + let err = table.drop_columns(columns).await.unwrap_err(); + let Error::InvalidInput { message } = &err else { + panic!("{err:?}"); + }; + assert!(message.contains(expected), "{message}"); + + // Read durable state, not this handle: the error path never calls + // `dataset.update`, so the cached handle would report the old version + // even if the unbind had committed. + table.checkout_latest().await.unwrap(); + assert_eq!(table.version().await.unwrap(), version); + let schema = table.schema().await.unwrap(); + assert!(schema.metadata().contains_key(FUNCTION_BINDINGS_META_KEY)); + assert_eq!(bindings_of(&table).await.len(), 1); + for column in ["search_text", "search_token_count"] { + assert!( + schema + .field_with_name(column) + .unwrap() + .metadata() + .keys() + .any(|key| is_declaration_key(key)), + "{column} lost its declaration metadata" + ); + } + ensure_supported_function_metadata(&schema).unwrap(); + } + #[tokio::test] async fn test_update_field_metadata() { let conn = connect("memory://").execute().await.unwrap(); From a8aa19f513b05600c065176e5effd04d459ff14d Mon Sep 17 00:00:00 2001 From: Lance Release Date: Mon, 21 Sep 2026 18:52:46 +0000 Subject: [PATCH 3/3] =?UTF-8?q?Bump=20version:=200.40.0-beta.4=20=E2=86=92?= =?UTF-8?q?=200.40.0-beta.5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.toml | 2 +- Cargo.lock | 6 +++--- docs/src/java/java.md | 2 +- java/lancedb-core/pom.xml | 2 +- java/pom.xml | 2 +- nodejs/Cargo.toml | 2 +- nodejs/npm/darwin-arm64/package.json | 2 +- nodejs/npm/linux-arm64-gnu/package.json | 2 +- nodejs/npm/linux-arm64-musl/package.json | 2 +- nodejs/npm/linux-x64-gnu/package.json | 2 +- nodejs/npm/linux-x64-musl/package.json | 2 +- nodejs/npm/win32-arm64-msvc/package.json | 2 +- nodejs/npm/win32-x64-msvc/package.json | 2 +- nodejs/package.json | 2 +- python/Cargo.toml | 2 +- rust/lancedb/Cargo.toml | 2 +- 16 files changed, 18 insertions(+), 18 deletions(-) diff --git a/.bumpversion.toml b/.bumpversion.toml index 00c38dc79..2dfbe973d 100644 --- a/.bumpversion.toml +++ b/.bumpversion.toml @@ -1,5 +1,5 @@ [tool.bumpversion] -current_version = "0.40.0-beta.4" +current_version = "0.40.0-beta.5" parse = """(?x) (?P0|[1-9]\\d*)\\. (?P0|[1-9]\\d*)\\. diff --git a/Cargo.lock b/Cargo.lock index bb5f388cd..9905e8b86 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5756,7 +5756,7 @@ dependencies = [ [[package]] name = "lancedb" -version = "0.40.0-beta.4" +version = "0.40.0-beta.5" dependencies = [ "ahash", "anyhow", @@ -5853,7 +5853,7 @@ dependencies = [ [[package]] name = "lancedb-nodejs" -version = "0.40.0-beta.4" +version = "0.40.0-beta.5" dependencies = [ "arrow-array", "arrow-buffer", @@ -5878,7 +5878,7 @@ dependencies = [ [[package]] name = "lancedb-python" -version = "0.40.0-beta.4" +version = "0.40.0-beta.5" dependencies = [ "arc-swap", "arrow", diff --git a/docs/src/java/java.md b/docs/src/java/java.md index a877b25c4..f6e21c69e 100644 --- a/docs/src/java/java.md +++ b/docs/src/java/java.md @@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`: com.lancedb lancedb-core - 0.40.0-beta.4 + 0.40.0-beta.5 ``` diff --git a/java/lancedb-core/pom.xml b/java/lancedb-core/pom.xml index 85d149ed0..e38de7a79 100644 --- a/java/lancedb-core/pom.xml +++ b/java/lancedb-core/pom.xml @@ -8,7 +8,7 @@ com.lancedb lancedb-parent - 0.40.0-beta.4 + 0.40.0-beta.5 ../pom.xml diff --git a/java/pom.xml b/java/pom.xml index a42d31540..845d1ea07 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -6,7 +6,7 @@ com.lancedb lancedb-parent - 0.40.0-beta.4 + 0.40.0-beta.5 pom ${project.artifactId} LanceDB Java SDK Parent POM diff --git a/nodejs/Cargo.toml b/nodejs/Cargo.toml index 05b336f04..e2357dfc3 100644 --- a/nodejs/Cargo.toml +++ b/nodejs/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "lancedb-nodejs" edition.workspace = true -version = "0.40.0-beta.4" +version = "0.40.0-beta.5" publish = false license.workspace = true description.workspace = true diff --git a/nodejs/npm/darwin-arm64/package.json b/nodejs/npm/darwin-arm64/package.json index d41265130..b517a7b8a 100644 --- a/nodejs/npm/darwin-arm64/package.json +++ b/nodejs/npm/darwin-arm64/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-darwin-arm64", - "version": "0.40.0-beta.4", + "version": "0.40.0-beta.5", "os": ["darwin"], "cpu": ["arm64"], "main": "lancedb.darwin-arm64.node", diff --git a/nodejs/npm/linux-arm64-gnu/package.json b/nodejs/npm/linux-arm64-gnu/package.json index a93727003..25c6698c3 100644 --- a/nodejs/npm/linux-arm64-gnu/package.json +++ b/nodejs/npm/linux-arm64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-arm64-gnu", - "version": "0.40.0-beta.4", + "version": "0.40.0-beta.5", "os": ["linux"], "cpu": ["arm64"], "main": "lancedb.linux-arm64-gnu.node", diff --git a/nodejs/npm/linux-arm64-musl/package.json b/nodejs/npm/linux-arm64-musl/package.json index 8d5843ec8..c46f40f96 100644 --- a/nodejs/npm/linux-arm64-musl/package.json +++ b/nodejs/npm/linux-arm64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-arm64-musl", - "version": "0.40.0-beta.4", + "version": "0.40.0-beta.5", "os": ["linux"], "cpu": ["arm64"], "main": "lancedb.linux-arm64-musl.node", diff --git a/nodejs/npm/linux-x64-gnu/package.json b/nodejs/npm/linux-x64-gnu/package.json index a3bb2e449..fc030ef53 100644 --- a/nodejs/npm/linux-x64-gnu/package.json +++ b/nodejs/npm/linux-x64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-x64-gnu", - "version": "0.40.0-beta.4", + "version": "0.40.0-beta.5", "os": ["linux"], "cpu": ["x64"], "main": "lancedb.linux-x64-gnu.node", diff --git a/nodejs/npm/linux-x64-musl/package.json b/nodejs/npm/linux-x64-musl/package.json index 0c5e31824..4f23a3261 100644 --- a/nodejs/npm/linux-x64-musl/package.json +++ b/nodejs/npm/linux-x64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-x64-musl", - "version": "0.40.0-beta.4", + "version": "0.40.0-beta.5", "os": ["linux"], "cpu": ["x64"], "main": "lancedb.linux-x64-musl.node", diff --git a/nodejs/npm/win32-arm64-msvc/package.json b/nodejs/npm/win32-arm64-msvc/package.json index fdcc00e3b..0594f5610 100644 --- a/nodejs/npm/win32-arm64-msvc/package.json +++ b/nodejs/npm/win32-arm64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-win32-arm64-msvc", - "version": "0.40.0-beta.4", + "version": "0.40.0-beta.5", "os": [ "win32" ], diff --git a/nodejs/npm/win32-x64-msvc/package.json b/nodejs/npm/win32-x64-msvc/package.json index 02be1aba2..2c819c574 100644 --- a/nodejs/npm/win32-x64-msvc/package.json +++ b/nodejs/npm/win32-x64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-win32-x64-msvc", - "version": "0.40.0-beta.4", + "version": "0.40.0-beta.5", "os": ["win32"], "cpu": ["x64"], "main": "lancedb.win32-x64-msvc.node", diff --git a/nodejs/package.json b/nodejs/package.json index b7549d143..b35b5eb28 100644 --- a/nodejs/package.json +++ b/nodejs/package.json @@ -11,7 +11,7 @@ "ann" ], "private": false, - "version": "0.40.0-beta.4", + "version": "0.40.0-beta.5", "main": "dist/index.js", "exports": { ".": "./dist/index.js", diff --git a/python/Cargo.toml b/python/Cargo.toml index b2d7450e3..8fcdf8914 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lancedb-python" -version = "0.40.0-beta.4" +version = "0.40.0-beta.5" publish = false edition.workspace = true description = "Python bindings for LanceDB" diff --git a/rust/lancedb/Cargo.toml b/rust/lancedb/Cargo.toml index 57a21eee2..ebec41318 100644 --- a/rust/lancedb/Cargo.toml +++ b/rust/lancedb/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lancedb" -version = "0.40.0-beta.4" +version = "0.40.0-beta.5" edition.workspace = true description = "LanceDB: A serverless, low-latency vector database for AI applications" license.workspace = true