From d1d71518245e5c3fa212452c7c67714a2b999ba4 Mon Sep 17 00:00:00 2001 From: lancedb automation Date: Wed, 21 Jan 2026 08:51:12 +0000 Subject: [PATCH] chore: update lance dependency to v1.0.3-rc.1 --- Cargo.lock | 658 ++++++++++++++++----------- Cargo.toml | 56 +-- python/Cargo.toml | 8 +- python/src/arrow.rs | 8 +- python/src/permutation.rs | 6 +- python/src/query.rs | 24 +- python/src/table.rs | 6 +- rust/lancedb/src/connection.rs | 114 +++-- rust/lancedb/src/database/listing.rs | 68 ++- rust/lancedb/src/table.rs | 33 +- rust/lancedb/src/table/dataset.rs | 3 +- 11 files changed, 568 insertions(+), 416 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 206cc208e..226eb9643 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -181,9 +181,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "arrow" -version = "57.2.0" +version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a2b10dcb159faf30d3f81f6d56c1211a5bea2ca424eabe477648a44b993320e" +checksum = "6e833808ff2d94ed40d9379848a950d995043c7fb3e81a30b383f4c6033821cc" dependencies = [ "arrow-arith", "arrow-array", @@ -203,23 +203,23 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "57.2.0" +version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "288015089e7931843c80ed4032c5274f02b37bcb720c4a42096d50b390e70372" +checksum = "ad08897b81588f60ba983e3ca39bda2b179bdd84dced378e7df81a5313802ef8" dependencies = [ "arrow-array", "arrow-buffer", "arrow-data", "arrow-schema", "chrono", - "num-traits", + "num", ] [[package]] name = "arrow-array" -version = "57.2.0" +version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65ca404ea6191e06bf30956394173337fa9c35f445bd447fe6c21ab944e1a23c" +checksum = "8548ca7c070d8db9ce7aa43f37393e4bfcf3f2d3681df278490772fd1673d08d" dependencies = [ "ahash", "arrow-buffer", @@ -229,33 +229,29 @@ dependencies = [ "chrono-tz 0.10.4", "half", "hashbrown 0.16.0", - "num-complex", - "num-integer", - "num-traits", + "num", ] [[package]] name = "arrow-buffer" -version = "57.2.0" +version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36356383099be0151dacc4245309895f16ba7917d79bdb71a7148659c9206c56" +checksum = "e003216336f70446457e280807a73899dd822feaf02087d31febca1363e2fccc" dependencies = [ "bytes", "half", - "num-bigint", - "num-traits", + "num", ] [[package]] name = "arrow-cast" -version = "57.2.0" +version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c8e372ed52bd4ee88cc1e6c3859aa7ecea204158ac640b10e187936e7e87074" +checksum = "919418a0681298d3a77d1a315f625916cb5678ad0d74b9c60108eb15fd083023" dependencies = [ "arrow-array", "arrow-buffer", "arrow-data", - "arrow-ord", "arrow-schema", "arrow-select", "atoi", @@ -264,15 +260,15 @@ dependencies = [ "comfy-table", "half", "lexical-core", - "num-traits", + "num", "ryu", ] [[package]] name = "arrow-csv" -version = "57.2.0" +version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e4100b729fe656f2e4fb32bc5884f14acf9118d4ad532b7b33c1132e4dce896" +checksum = "bfa9bf02705b5cf762b6f764c65f04ae9082c7cfc4e96e0c33548ee3f67012eb" dependencies = [ "arrow-array", "arrow-cast", @@ -285,22 +281,21 @@ dependencies = [ [[package]] name = "arrow-data" -version = "57.2.0" +version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf87f4ff5fc13290aa47e499a8b669a82c5977c6a1fedce22c7f542c1fd5a597" +checksum = "a5c64fff1d142f833d78897a772f2e5b55b36cb3e6320376f0961ab0db7bd6d0" dependencies = [ "arrow-buffer", "arrow-schema", "half", - "num-integer", - "num-traits", + "num", ] [[package]] name = "arrow-ipc" -version = "57.2.0" +version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb3ca63edd2073fcb42ba112f8ae165df1de935627ead6e203d07c99445f2081" +checksum = "1d3594dcddccc7f20fd069bc8e9828ce37220372680ff638c5e00dea427d88f5" dependencies = [ "arrow-array", "arrow-buffer", @@ -308,15 +303,15 @@ dependencies = [ "arrow-schema", "arrow-select", "flatbuffers", - "lz4_flex 0.12.0", + "lz4_flex", "zstd", ] [[package]] name = "arrow-json" -version = "57.2.0" +version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a36b2332559d3310ebe3e173f75b29989b4412df4029a26a30cc3f7da0869297" +checksum = "88cf36502b64a127dc659e3b305f1d993a544eab0d48cce704424e62074dc04b" dependencies = [ "arrow-array", "arrow-buffer", @@ -326,21 +321,19 @@ dependencies = [ "chrono", "half", "indexmap 2.12.0", - "itoa", "lexical-core", "memchr", - "num-traits", - "ryu", - "serde_core", + "num", + "serde", "serde_json", "simdutf8", ] [[package]] name = "arrow-ord" -version = "57.2.0" +version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13c4e0530272ca755d6814218dffd04425c5b7854b87fa741d5ff848bf50aa39" +checksum = "3c8f82583eb4f8d84d4ee55fd1cb306720cddead7596edce95b50ee418edf66f" dependencies = [ "arrow-array", "arrow-buffer", @@ -351,9 +344,9 @@ dependencies = [ [[package]] name = "arrow-pyarrow" -version = "57.2.0" +version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f45c7989cb70214b2f362eaa10266d15e1a433692f2ea1514018be3aace679f4" +checksum = "7d924b32e96f8bb74d94cd82bd97b313c432fcb0ea331689ef9e7c6b8be4b258" dependencies = [ "arrow-array", "arrow-data", @@ -363,9 +356,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "57.2.0" +version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b07f52788744cc71c4628567ad834cadbaeb9f09026ff1d7a4120f69edf7abd3" +checksum = "9d07ba24522229d9085031df6b94605e0f4b26e099fb7cdeec37abd941a73753" dependencies = [ "arrow-array", "arrow-buffer", @@ -376,34 +369,34 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "57.2.0" +version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bb63203e8e0e54b288d0d8043ca8fa1013820822a27692ef1b78a977d879f2c" +checksum = "b3aa9e59c611ebc291c28582077ef25c97f1975383f1479b12f3b9ffee2ffabe" dependencies = [ "bitflags 2.9.4", - "serde_core", + "serde", "serde_json", ] [[package]] name = "arrow-select" -version = "57.2.0" +version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c96d8a1c180b44ecf2e66c9a2f2bbcb8b1b6f14e165ce46ac8bde211a363411b" +checksum = "8c41dbbd1e97bfcaee4fcb30e29105fb2c75e4d82ae4de70b792a5d3f66b2e7a" dependencies = [ "ahash", "arrow-array", "arrow-buffer", "arrow-data", "arrow-schema", - "num-traits", + "num", ] [[package]] name = "arrow-string" -version = "57.2.0" +version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8ad6a81add9d3ea30bf8374ee8329992c7fd246ffd8b7e2f48a3cea5aa0cc9a" +checksum = "53f5183c150fbc619eede22b861ea7c0eebed8eaac0333eaa7f6da5205fd504d" dependencies = [ "arrow-array", "arrow-buffer", @@ -411,7 +404,7 @@ dependencies = [ "arrow-schema", "arrow-select", "memchr", - "num-traits", + "num", "regex", "regex-syntax", ] @@ -434,11 +427,15 @@ version = "0.4.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06575e6a9673580f52661c92107baabffbf41e2141373441cbcdc47cb733003c" dependencies = [ + "bzip2 0.5.2", "flate2", "futures-core", "memchr", "pin-project-lite", "tokio", + "xz2", + "zstd", + "zstd-safe", ] [[package]] @@ -1403,6 +1400,34 @@ dependencies = [ "either", ] +[[package]] +name = "bzip2" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49ecfb22d906f800d4fe833b6282cf4dc1c298f5057ca0b5445e5c209735ca47" +dependencies = [ + "bzip2-sys", +] + +[[package]] +name = "bzip2" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3a53fac24f34a81bc9954b5d6cfce0c21e18ec6959f44f56e8e90e4bb7c346c" +dependencies = [ + "libbz2-rs-sys", +] + +[[package]] +name = "bzip2-sys" +version = "0.1.13+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" +dependencies = [ + "cc", + "pkg-config", +] + [[package]] name = "candle-core" version = "0.9.1" @@ -1977,23 +2002,25 @@ dependencies = [ [[package]] name = "datafusion" -version = "51.0.0" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ba7cb113e9c0bedf9e9765926031e132fa05a1b09ba6e93a6d1a4d7044457b8" +checksum = "2af15bb3c6ffa33011ef579f6b0bcbe7c26584688bd6c994f548e44df67f011a" dependencies = [ "arrow", + "arrow-ipc", "arrow-schema", "async-trait", "bytes", + "bzip2 0.6.1", "chrono", "datafusion-catalog", "datafusion-catalog-listing", "datafusion-common", "datafusion-common-runtime", "datafusion-datasource", - "datafusion-datasource-arrow", "datafusion-datasource-csv", "datafusion-datasource-json", + "datafusion-datasource-parquet", "datafusion-execution", "datafusion-expr", "datafusion-expr-common", @@ -2010,26 +2037,29 @@ dependencies = [ "datafusion-physical-plan", "datafusion-session", "datafusion-sql", + "flate2", "futures", "itertools 0.14.0", "log", "object_store", "parking_lot", + "parquet", "rand 0.9.2", "regex", - "rstest 0.26.1", - "sqlparser 0.59.0", + "sqlparser 0.58.0", "tempfile", "tokio", "url", "uuid", + "xz2", + "zstd", ] [[package]] name = "datafusion-catalog" -version = "51.0.0" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66a3a799f914a59b1ea343906a0486f17061f39509af74e874a866428951130d" +checksum = "187622262ad8f7d16d3be9202b4c1e0116f1c9aa387e5074245538b755261621" dependencies = [ "arrow", "async-trait", @@ -2042,6 +2072,7 @@ dependencies = [ "datafusion-physical-expr", "datafusion-physical-plan", "datafusion-session", + "datafusion-sql", "futures", "itertools 0.14.0", "log", @@ -2052,9 +2083,9 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" -version = "51.0.0" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db1b113c80d7a0febcd901476a57aef378e717c54517a163ed51417d87621b0" +checksum = "9657314f0a32efd0382b9a46fdeb2d233273ece64baa68a7c45f5a192daf0f83" dependencies = [ "arrow", "async-trait", @@ -2064,11 +2095,10 @@ dependencies = [ "datafusion-execution", "datafusion-expr", "datafusion-physical-expr", - "datafusion-physical-expr-adapter", "datafusion-physical-expr-common", "datafusion-physical-plan", + "datafusion-session", "futures", - "itertools 0.14.0", "log", "object_store", "tokio", @@ -2076,13 +2106,14 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "51.0.0" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c10f7659e96127d25e8366be7c8be4109595d6a2c3eac70421f380a7006a1b0" +checksum = "5a83760d9a13122d025fbdb1d5d5aaf93dd9ada5e90ea229add92aa30898b2d1" dependencies = [ "ahash", "arrow", "arrow-ipc", + "base64 0.22.1", "chrono", "half", "hashbrown 0.14.5", @@ -2090,17 +2121,19 @@ dependencies = [ "libc", "log", "object_store", + "parquet", "paste", - "sqlparser 0.59.0", + "recursive", + "sqlparser 0.58.0", "tokio", "web-time", ] [[package]] name = "datafusion-common-runtime" -version = "51.0.0" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b92065bbc6532c6651e2f7dd30b55cba0c7a14f860c7e1d15f165c41a1868d95" +checksum = "5b6234a6c7173fe5db1c6c35c01a12b2aa0f803a3007feee53483218817f8b1e" dependencies = [ "futures", "log", @@ -2109,13 +2142,15 @@ dependencies = [ [[package]] name = "datafusion-datasource" -version = "51.0.0" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fde13794244bc7581cd82f6fff217068ed79cdc344cafe4ab2c3a1c3510b38d6" +checksum = "7256c9cb27a78709dd42d0c80f0178494637209cac6e29d5c93edd09b6721b86" dependencies = [ "arrow", + "async-compression", "async-trait", "bytes", + "bzip2 0.6.1", "chrono", "datafusion-common", "datafusion-common-runtime", @@ -2126,54 +2161,38 @@ dependencies = [ "datafusion-physical-expr-common", "datafusion-physical-plan", "datafusion-session", + "flate2", "futures", "glob", "itertools 0.14.0", "log", "object_store", + "parquet", "rand 0.9.2", + "tempfile", "tokio", + "tokio-util", "url", -] - -[[package]] -name = "datafusion-datasource-arrow" -version = "51.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "804fa9b4ecf3157982021770617200ef7c1b2979d57bec9044748314775a9aea" -dependencies = [ - "arrow", - "arrow-ipc", - "async-trait", - "bytes", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-session", - "futures", - "itertools 0.14.0", - "object_store", - "tokio", + "xz2", + "zstd", ] [[package]] name = "datafusion-datasource-csv" -version = "51.0.0" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61a1641a40b259bab38131c5e6f48fac0717bedb7dc93690e604142a849e0568" +checksum = "64533a90f78e1684bfb113d200b540f18f268134622d7c96bbebc91354d04825" dependencies = [ "arrow", "async-trait", "bytes", + "datafusion-catalog", "datafusion-common", "datafusion-common-runtime", "datafusion-datasource", "datafusion-execution", "datafusion-expr", + "datafusion-physical-expr", "datafusion-physical-expr-common", "datafusion-physical-plan", "datafusion-session", @@ -2185,37 +2204,73 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" -version = "51.0.0" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adeacdb00c1d37271176f8fb6a1d8ce096baba16ea7a4b2671840c5c9c64fe85" +checksum = "8d7ebeb12c77df0aacad26f21b0d033aeede423a64b2b352f53048a75bf1d6e6" dependencies = [ "arrow", "async-trait", "bytes", + "datafusion-catalog", "datafusion-common", "datafusion-common-runtime", "datafusion-datasource", "datafusion-execution", "datafusion-expr", + "datafusion-physical-expr", "datafusion-physical-expr-common", "datafusion-physical-plan", "datafusion-session", "futures", "object_store", + "serde_json", + "tokio", +] + +[[package]] +name = "datafusion-datasource-parquet" +version = "50.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09e783c4c7d7faa1199af2df4761c68530634521b176a8d1331ddbc5a5c75133" +dependencies = [ + "arrow", + "async-trait", + "bytes", + "datafusion-catalog", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions-aggregate", + "datafusion-physical-expr", + "datafusion-physical-expr-adapter", + "datafusion-physical-expr-common", + "datafusion-physical-optimizer", + "datafusion-physical-plan", + "datafusion-pruning", + "datafusion-session", + "futures", + "itertools 0.14.0", + "log", + "object_store", + "parking_lot", + "parquet", + "rand 0.9.2", "tokio", ] [[package]] name = "datafusion-doc" -version = "51.0.0" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b99e13947667b36ad713549237362afb054b2d8f8cc447751e23ec61202db07" +checksum = "99ee6b1d9a80d13f9deb2291f45c07044b8e62fb540dbde2453a18be17a36429" [[package]] name = "datafusion-execution" -version = "51.0.0" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63695643190679037bc946ad46a263b62016931547bf119859c511f7ff2f5178" +checksum = "a4cec0a57653bec7b933fb248d3ffa3fa3ab3bd33bd140dc917f714ac036f531" dependencies = [ "arrow", "async-trait", @@ -2233,9 +2288,9 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "51.0.0" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9a4787cbf5feb1ab351f789063398f67654a6df75c4d37d7f637dc96f951a91" +checksum = "ef76910bdca909722586389156d0aa4da4020e1631994d50fadd8ad4b1aa05fe" dependencies = [ "arrow", "async-trait", @@ -2247,17 +2302,17 @@ dependencies = [ "datafusion-functions-window-common", "datafusion-physical-expr-common", "indexmap 2.12.0", - "itertools 0.14.0", "paste", + "recursive", "serde_json", - "sqlparser 0.59.0", + "sqlparser 0.58.0", ] [[package]] name = "datafusion-expr-common" -version = "51.0.0" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ce2fb1b8c15c9ac45b0863c30b268c69dc9ee7a1ee13ecf5d067738338173dc" +checksum = "6d155ccbda29591ca71a1344dd6bed26c65a4438072b400df9db59447f590bb6" dependencies = [ "arrow", "datafusion-common", @@ -2268,9 +2323,9 @@ dependencies = [ [[package]] name = "datafusion-functions" -version = "51.0.0" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "794a9db7f7b96b3346fc007ff25e994f09b8f0511b4cf7dff651fadfe3ebb28f" +checksum = "7de2782136bd6014670fd84fe3b0ca3b3e4106c96403c3ae05c0598577139977" dependencies = [ "arrow", "arrow-buffer", @@ -2288,7 +2343,6 @@ dependencies = [ "itertools 0.14.0", "log", "md-5", - "num-traits", "rand 0.9.2", "regex", "sha2", @@ -2298,9 +2352,9 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" -version = "51.0.0" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c25210520a9dcf9c2b2cbbce31ebd4131ef5af7fc60ee92b266dc7d159cb305" +checksum = "07331fc13603a9da97b74fd8a273f4238222943dffdbbed1c4c6f862a30105bf" dependencies = [ "ahash", "arrow", @@ -2319,9 +2373,9 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" -version = "51.0.0" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62f4a66f3b87300bb70f4124b55434d2ae3fe80455f3574701d0348da040b55d" +checksum = "b5951e572a8610b89968a09b5420515a121fbc305c0258651f318dc07c97ab17" dependencies = [ "ahash", "arrow", @@ -2332,9 +2386,9 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" -version = "51.0.0" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae5c06eed03918dc7fe7a9f082a284050f0e9ecf95d72f57712d1496da03b8c4" +checksum = "fdacca9302c3d8fc03f3e94f338767e786a88a33f5ebad6ffc0e7b50364b9ea3" dependencies = [ "arrow", "arrow-ord", @@ -2342,7 +2396,6 @@ dependencies = [ "datafusion-doc", "datafusion-execution", "datafusion-expr", - "datafusion-expr-common", "datafusion-functions", "datafusion-functions-aggregate", "datafusion-functions-aggregate-common", @@ -2355,9 +2408,9 @@ dependencies = [ [[package]] name = "datafusion-functions-table" -version = "51.0.0" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db4fed1d71738fbe22e2712d71396db04c25de4111f1ec252b8f4c6d3b25d7f5" +checksum = "8c37ff8a99434fbbad604a7e0669717c58c7c4f14c472d45067c4b016621d981" dependencies = [ "arrow", "async-trait", @@ -2371,9 +2424,9 @@ dependencies = [ [[package]] name = "datafusion-functions-window" -version = "51.0.0" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d92206aa5ae21892f1552b4d61758a862a70956e6fd7a95cb85db1de74bc6d1" +checksum = "48e2aea7c79c926cffabb13dc27309d4eaeb130f4a21c8ba91cdd241c813652b" dependencies = [ "arrow", "datafusion-common", @@ -2389,9 +2442,9 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" -version = "51.0.0" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53ae9bcc39800820d53a22d758b3b8726ff84a5a3e24cecef04ef4e5fdf1c7cc" +checksum = "0fead257ab5fd2ffc3b40fda64da307e20de0040fe43d49197241d9de82a487f" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -2399,20 +2452,20 @@ dependencies = [ [[package]] name = "datafusion-macros" -version = "51.0.0" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1063ad4c9e094b3f798acee16d9a47bd7372d9699be2de21b05c3bd3f34ab848" +checksum = "ec6f637bce95efac05cdfb9b6c19579ed4aa5f6b94d951cfa5bb054b7bb4f730" dependencies = [ - "datafusion-doc", + "datafusion-expr", "quote", "syn 2.0.114", ] [[package]] name = "datafusion-optimizer" -version = "51.0.0" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f35f9ec5d08b87fd1893a30c2929f2559c2f9806ca072d8fefca5009dc0f06a" +checksum = "c6583ef666ae000a613a837e69e456681a9faa96347bf3877661e9e89e141d8a" dependencies = [ "arrow", "chrono", @@ -2423,15 +2476,16 @@ dependencies = [ "indexmap 2.12.0", "itertools 0.14.0", "log", + "recursive", "regex", "regex-syntax", ] [[package]] name = "datafusion-physical-expr" -version = "51.0.0" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c30cc8012e9eedcb48bbe112c6eff4ae5ed19cf3003cb0f505662e88b7014c5d" +checksum = "c8668103361a272cbbe3a61f72eca60c9b7c706e87cc3565bcf21e2b277b84f6" dependencies = [ "ahash", "arrow", @@ -2444,16 +2498,17 @@ dependencies = [ "hashbrown 0.14.5", "indexmap 2.12.0", "itertools 0.14.0", + "log", "parking_lot", "paste", - "petgraph", + "petgraph 0.8.3", ] [[package]] name = "datafusion-physical-expr-adapter" -version = "51.0.0" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f9ff2dbd476221b1f67337699eff432781c4e6e1713d2aefdaa517dfbf79768" +checksum = "815acced725d30601b397e39958e0e55630e0a10d66ef7769c14ae6597298bb0" dependencies = [ "arrow", "datafusion-common", @@ -2466,9 +2521,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" -version = "51.0.0" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90da43e1ec550b172f34c87ec68161986ced70fd05c8d2a2add66eef9c276f03" +checksum = "6652fe7b5bf87e85ed175f571745305565da2c0b599d98e697bcbedc7baa47c3" dependencies = [ "ahash", "arrow", @@ -2480,9 +2535,9 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" -version = "51.0.0" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce9804f799acd7daef3be7aaffe77c0033768ed8fdbf5fb82fc4c5f2e6bc14e6" +checksum = "49b7d623eb6162a3332b564a0907ba00895c505d101b99af78345f1acf929b5c" dependencies = [ "arrow", "datafusion-common", @@ -2494,13 +2549,15 @@ dependencies = [ "datafusion-physical-plan", "datafusion-pruning", "itertools 0.14.0", + "log", + "recursive", ] [[package]] name = "datafusion-physical-plan" -version = "51.0.0" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0acf0ad6b6924c6b1aa7d213b181e012e2d3ec0a64ff5b10ee6282ab0f8532ac" +checksum = "e2f7f778a1a838dec124efb96eae6144237d546945587557c9e6936b3414558c" dependencies = [ "ahash", "arrow", @@ -2529,11 +2586,12 @@ dependencies = [ [[package]] name = "datafusion-pruning" -version = "51.0.0" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac2c2498a1f134a9e11a9f5ed202a2a7d7e9774bd9249295593053ea3be999db" +checksum = "cd1e59e2ca14fe3c30f141600b10ad8815e2856caa59ebbd0e3e07cd3d127a65" dependencies = [ "arrow", + "arrow-schema", "datafusion-common", "datafusion-datasource", "datafusion-expr-common", @@ -2546,33 +2604,43 @@ dependencies = [ [[package]] name = "datafusion-session" -version = "51.0.0" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f96eebd17555386f459037c65ab73aae8df09f464524c709d6a3134ad4f4776" +checksum = "21ef8e2745583619bd7a49474e8f45fbe98ebb31a133f27802217125a7b3d58d" dependencies = [ + "arrow", "async-trait", + "dashmap", "datafusion-common", + "datafusion-common-runtime", "datafusion-execution", "datafusion-expr", + "datafusion-physical-expr", "datafusion-physical-plan", + "datafusion-sql", + "futures", + "itertools 0.14.0", + "log", + "object_store", "parking_lot", + "tokio", ] [[package]] name = "datafusion-sql" -version = "51.0.0" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fc195fe60634b2c6ccfd131b487de46dc30eccae8a3c35a13f136e7f440414f" +checksum = "89abd9868770386fede29e5a4b14f49c0bf48d652c3b9d7a8a0332329b87d50b" dependencies = [ "arrow", "bigdecimal", - "chrono", "datafusion-common", "datafusion-expr", "indexmap 2.12.0", "log", + "recursive", "regex", - "sqlparser 0.59.0", + "sqlparser 0.58.0", ] [[package]] @@ -3018,6 +3086,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc5a4e564e38c699f2880d3fda590bedc2e69f3f84cd48b457bd892ce61d0aa9" dependencies = [ "crc32fast", + "libz-rs-sys", "miniz_oxide", ] @@ -3072,8 +3141,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "fsst" -version = "2.0.0-beta.8" -source = "git+https://github.com/lance-format/lance.git?tag=v2.0.0-beta.8#5bffcfc3f9716fd6eb069579eb5027ed9c69feb7" +version = "1.0.3-rc.1" +source = "git+https://github.com/lance-format/lance.git?tag=v1.0.3-rc.1#695a77bbe5abe1695d98d4d397acc23c456c4217" dependencies = [ "arrow-array", "rand 0.9.2", @@ -3492,9 +3561,9 @@ dependencies = [ [[package]] name = "geoarrow-array" -version = "0.7.0" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc1cc4106ac0a0a512c398961ce95d8150475c84a84e17c4511c3643fa120a17" +checksum = "7d1884b17253d8572e88833c282fcbb442365e4ae5f9052ced2831608253436c" dependencies = [ "arrow-array", "arrow-buffer", @@ -3508,9 +3577,9 @@ dependencies = [ [[package]] name = "geoarrow-expr-geo" -version = "0.7.0" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa84300361ce57fb875bcaa6e32b95b0aff5c6b1af692b936bdd58ff343f4394" +checksum = "a67d3b543bc3ebeffdc204b67d69b8f9fcd33d76269ddd4a4618df99f053a934" dependencies = [ "arrow-array", "arrow-buffer", @@ -3522,9 +3591,9 @@ dependencies = [ [[package]] name = "geoarrow-schema" -version = "0.7.0" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e97be4e9f523f92bd6a0e0458323f4b783d073d011664decd8dbf05651704f34" +checksum = "02f1b18b1c9a44ecd72be02e53d6e63bbccfdc8d1765206226af227327e2be6e" dependencies = [ "arrow-schema", "geo-traits", @@ -3535,9 +3604,9 @@ dependencies = [ [[package]] name = "geodatafusion" -version = "0.2.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773cfa1fb0d7f7661b76b3fde00f3ffd8e0ff7b3635096f0ff6294fe5ca62a2b" +checksum = "83d676b8d8b5f391ab4270ba31e9b599ee2c3d780405a38e272a0a7565ea189c" dependencies = [ "arrow-arith", "arrow-array", @@ -4236,6 +4305,12 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "integer-encoding" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" + [[package]] name = "ipnet" version = "2.11.0" @@ -4380,7 +4455,7 @@ dependencies = [ "jiff", "nom 8.0.0", "num-traits", - "ordered-float", + "ordered-float 5.1.0", "rand 0.9.2", "ryu", "serde", @@ -4404,8 +4479,8 @@ dependencies = [ [[package]] name = "lance" -version = "2.0.0-beta.8" -source = "git+https://github.com/lance-format/lance.git?tag=v2.0.0-beta.8#5bffcfc3f9716fd6eb069579eb5027ed9c69feb7" +version = "1.0.3-rc.1" +source = "git+https://github.com/lance-format/lance.git?tag=v1.0.3-rc.1#695a77bbe5abe1695d98d4d397acc23c456c4217" dependencies = [ "arrow", "arrow-arith", @@ -4470,14 +4545,13 @@ dependencies = [ [[package]] name = "lance-arrow" -version = "2.0.0-beta.8" -source = "git+https://github.com/lance-format/lance.git?tag=v2.0.0-beta.8#5bffcfc3f9716fd6eb069579eb5027ed9c69feb7" +version = "1.0.3-rc.1" +source = "git+https://github.com/lance-format/lance.git?tag=v1.0.3-rc.1#695a77bbe5abe1695d98d4d397acc23c456c4217" dependencies = [ "arrow-array", "arrow-buffer", "arrow-cast", "arrow-data", - "arrow-ord", "arrow-schema", "arrow-select", "bytes", @@ -4490,8 +4564,8 @@ dependencies = [ [[package]] name = "lance-bitpacking" -version = "2.0.0-beta.8" -source = "git+https://github.com/lance-format/lance.git?tag=v2.0.0-beta.8#5bffcfc3f9716fd6eb069579eb5027ed9c69feb7" +version = "1.0.3-rc.1" +source = "git+https://github.com/lance-format/lance.git?tag=v1.0.3-rc.1#695a77bbe5abe1695d98d4d397acc23c456c4217" dependencies = [ "arrayref", "paste", @@ -4500,8 +4574,8 @@ dependencies = [ [[package]] name = "lance-core" -version = "2.0.0-beta.8" -source = "git+https://github.com/lance-format/lance.git?tag=v2.0.0-beta.8#5bffcfc3f9716fd6eb069579eb5027ed9c69feb7" +version = "1.0.3-rc.1" +source = "git+https://github.com/lance-format/lance.git?tag=v1.0.3-rc.1#695a77bbe5abe1695d98d4d397acc23c456c4217" dependencies = [ "arrow-array", "arrow-buffer", @@ -4514,7 +4588,6 @@ dependencies = [ "datafusion-sql", "deepsize", "futures", - "itertools 0.13.0", "lance-arrow", "libc", "log", @@ -4538,8 +4611,8 @@ dependencies = [ [[package]] name = "lance-datafusion" -version = "2.0.0-beta.8" -source = "git+https://github.com/lance-format/lance.git?tag=v2.0.0-beta.8#5bffcfc3f9716fd6eb069579eb5027ed9c69feb7" +version = "1.0.3-rc.1" +source = "git+https://github.com/lance-format/lance.git?tag=v1.0.3-rc.1#695a77bbe5abe1695d98d4d397acc23c456c4217" dependencies = [ "arrow", "arrow-array", @@ -4569,8 +4642,8 @@ dependencies = [ [[package]] name = "lance-datagen" -version = "2.0.0-beta.8" -source = "git+https://github.com/lance-format/lance.git?tag=v2.0.0-beta.8#5bffcfc3f9716fd6eb069579eb5027ed9c69feb7" +version = "1.0.3-rc.1" +source = "git+https://github.com/lance-format/lance.git?tag=v1.0.3-rc.1#695a77bbe5abe1695d98d4d397acc23c456c4217" dependencies = [ "arrow", "arrow-array", @@ -4581,15 +4654,14 @@ dependencies = [ "half", "hex", "rand 0.9.2", - "rand_distr 0.5.1", "rand_xoshiro", "random_word 0.5.2", ] [[package]] name = "lance-encoding" -version = "2.0.0-beta.8" -source = "git+https://github.com/lance-format/lance.git?tag=v2.0.0-beta.8#5bffcfc3f9716fd6eb069579eb5027ed9c69feb7" +version = "1.0.3-rc.1" +source = "git+https://github.com/lance-format/lance.git?tag=v1.0.3-rc.1#695a77bbe5abe1695d98d4d397acc23c456c4217" dependencies = [ "arrow-arith", "arrow-array", @@ -4626,8 +4698,8 @@ dependencies = [ [[package]] name = "lance-file" -version = "2.0.0-beta.8" -source = "git+https://github.com/lance-format/lance.git?tag=v2.0.0-beta.8#5bffcfc3f9716fd6eb069579eb5027ed9c69feb7" +version = "1.0.3-rc.1" +source = "git+https://github.com/lance-format/lance.git?tag=v1.0.3-rc.1#695a77bbe5abe1695d98d4d397acc23c456c4217" dependencies = [ "arrow-arith", "arrow-array", @@ -4659,23 +4731,20 @@ dependencies = [ [[package]] name = "lance-geo" -version = "2.0.0-beta.8" -source = "git+https://github.com/lance-format/lance.git?tag=v2.0.0-beta.8#5bffcfc3f9716fd6eb069579eb5027ed9c69feb7" +version = "1.0.3-rc.1" +source = "git+https://github.com/lance-format/lance.git?tag=v1.0.3-rc.1#695a77bbe5abe1695d98d4d397acc23c456c4217" dependencies = [ "datafusion", - "geo-traits", "geo-types", "geoarrow-array", "geoarrow-schema", "geodatafusion", - "lance-core", - "serde", ] [[package]] name = "lance-index" -version = "2.0.0-beta.8" -source = "git+https://github.com/lance-format/lance.git?tag=v2.0.0-beta.8#5bffcfc3f9716fd6eb069579eb5027ed9c69feb7" +version = "1.0.3-rc.1" +source = "git+https://github.com/lance-format/lance.git?tag=v1.0.3-rc.1#695a77bbe5abe1695d98d4d397acc23c456c4217" dependencies = [ "arrow", "arrow-arith", @@ -4699,9 +4768,6 @@ dependencies = [ "dirs", "fst", "futures", - "geo-types", - "geoarrow-array", - "geoarrow-schema", "half", "itertools 0.13.0", "jsonb", @@ -4711,7 +4777,6 @@ dependencies = [ "lance-datagen", "lance-encoding", "lance-file", - "lance-geo", "lance-io", "lance-linalg", "lance-table", @@ -4725,12 +4790,10 @@ dependencies = [ "prost-types", "rand 0.9.2", "rand_distr 0.5.1", - "rangemap", "rayon", "roaring", "serde", "serde_json", - "smallvec", "snafu", "tantivy", "tempfile", @@ -4742,8 +4805,8 @@ dependencies = [ [[package]] name = "lance-io" -version = "2.0.0-beta.8" -source = "git+https://github.com/lance-format/lance.git?tag=v2.0.0-beta.8#5bffcfc3f9716fd6eb069579eb5027ed9c69feb7" +version = "1.0.3-rc.1" +source = "git+https://github.com/lance-format/lance.git?tag=v1.0.3-rc.1#695a77bbe5abe1695d98d4d397acc23c456c4217" dependencies = [ "arrow", "arrow-arith", @@ -4783,8 +4846,8 @@ dependencies = [ [[package]] name = "lance-linalg" -version = "2.0.0-beta.8" -source = "git+https://github.com/lance-format/lance.git?tag=v2.0.0-beta.8#5bffcfc3f9716fd6eb069579eb5027ed9c69feb7" +version = "1.0.3-rc.1" +source = "git+https://github.com/lance-format/lance.git?tag=v1.0.3-rc.1#695a77bbe5abe1695d98d4d397acc23c456c4217" dependencies = [ "arrow-array", "arrow-buffer", @@ -4800,8 +4863,8 @@ dependencies = [ [[package]] name = "lance-namespace" -version = "2.0.0-beta.8" -source = "git+https://github.com/lance-format/lance.git?tag=v2.0.0-beta.8#5bffcfc3f9716fd6eb069579eb5027ed9c69feb7" +version = "1.0.3-rc.1" +source = "git+https://github.com/lance-format/lance.git?tag=v1.0.3-rc.1#695a77bbe5abe1695d98d4d397acc23c456c4217" dependencies = [ "arrow", "async-trait", @@ -4813,8 +4876,8 @@ dependencies = [ [[package]] name = "lance-namespace-impls" -version = "2.0.0-beta.8" -source = "git+https://github.com/lance-format/lance.git?tag=v2.0.0-beta.8#5bffcfc3f9716fd6eb069579eb5027ed9c69feb7" +version = "1.0.3-rc.1" +source = "git+https://github.com/lance-format/lance.git?tag=v1.0.3-rc.1#695a77bbe5abe1695d98d4d397acc23c456c4217" dependencies = [ "arrow", "arrow-ipc", @@ -4857,8 +4920,8 @@ dependencies = [ [[package]] name = "lance-table" -version = "2.0.0-beta.8" -source = "git+https://github.com/lance-format/lance.git?tag=v2.0.0-beta.8#5bffcfc3f9716fd6eb069579eb5027ed9c69feb7" +version = "1.0.3-rc.1" +source = "git+https://github.com/lance-format/lance.git?tag=v1.0.3-rc.1#695a77bbe5abe1695d98d4d397acc23c456c4217" dependencies = [ "arrow", "arrow-array", @@ -4897,8 +4960,8 @@ dependencies = [ [[package]] name = "lance-testing" -version = "2.0.0-beta.8" -source = "git+https://github.com/lance-format/lance.git?tag=v2.0.0-beta.8#5bffcfc3f9716fd6eb069579eb5027ed9c69feb7" +version = "1.0.3-rc.1" +source = "git+https://github.com/lance-format/lance.git?tag=v1.0.3-rc.1#695a77bbe5abe1695d98d4d397acc23c456c4217" dependencies = [ "arrow-array", "arrow-schema", @@ -4971,7 +5034,7 @@ dependencies = [ "random_word 0.4.3", "regex", "reqwest", - "rstest 0.23.0", + "rstest", "semver", "serde", "serde_json", @@ -5104,6 +5167,12 @@ dependencies = [ "lexical-util", ] +[[package]] +name = "libbz2-rs-sys" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c4a545a15244c7d945065b5d392b2d2d7f21526fba56ce51467b06ed445e8f7" + [[package]] name = "libc" version = "0.2.180" @@ -5136,6 +5205,15 @@ dependencies = [ "libc", ] +[[package]] +name = "libz-rs-sys" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c10501e7805cee23da17c7790e59df2870c0d4043ec6d03f67d31e2b53e77415" +dependencies = [ + "zlib-rs", +] + [[package]] name = "linux-raw-sys" version = "0.4.15" @@ -5221,12 +5299,6 @@ name = "lz4_flex" version = "0.11.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08ab2867e3eeeca90e844d1940eab391c9dc5228783db2ed999acbc0a9ed375a" - -[[package]] -name = "lz4_flex" -version = "0.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab6473172471198271ff72e9379150e9dfd70d8e533e0752a27e515b48dd375e" dependencies = [ "twox-hash", ] @@ -5856,6 +5928,15 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" +[[package]] +name = "ordered-float" +version = "2.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c" +dependencies = [ + "num-traits", +] + [[package]] name = "ordered-float" version = "5.1.0" @@ -5930,6 +6011,43 @@ dependencies = [ "windows-link 0.2.1", ] +[[package]] +name = "parquet" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0dbd48ad52d7dccf8ea1b90a3ddbfaea4f69878dd7683e51c507d4bc52b5b27" +dependencies = [ + "ahash", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-ipc", + "arrow-schema", + "arrow-select", + "base64 0.22.1", + "brotli 8.0.2", + "bytes", + "chrono", + "flate2", + "futures", + "half", + "hashbrown 0.16.0", + "lz4_flex", + "num", + "num-bigint", + "object_store", + "paste", + "ring", + "seq-macro", + "simdutf8", + "snap", + "thrift", + "tokio", + "twox-hash", + "zstd", +] + [[package]] name = "parquet-format-safe" version = "0.2.4" @@ -6004,6 +6122,16 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df202b0b0f5b8e389955afd5f27b007b00fb948162953f1db9c70d2c7e3157d7" +[[package]] +name = "petgraph" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" +dependencies = [ + "fixedbitset", + "indexmap 2.12.0", +] + [[package]] name = "petgraph" version = "0.8.3" @@ -6585,9 +6713,9 @@ dependencies = [ [[package]] name = "prost" -version = "0.14.3" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2ea70524a2f82d518bce41317d0fae74151505651af45faf1ffbd6fd33f0568" +checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" dependencies = [ "bytes", "prost-derive", @@ -6595,15 +6723,16 @@ dependencies = [ [[package]] name = "prost-build" -version = "0.14.3" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7" +checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" dependencies = [ "heck 0.5.0", "itertools 0.14.0", "log", "multimap", - "petgraph", + "once_cell", + "petgraph 0.7.1", "prettyplease", "prost", "prost-types", @@ -6614,9 +6743,9 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.14.3" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" +checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6627,9 +6756,9 @@ dependencies = [ [[package]] name = "prost-types" -version = "0.14.3" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8991c4cbdb8bc5b11f0b074ffe286c30e523de90fee5ba8132f1399f23cb3dd7" +checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16" dependencies = [ "prost", ] @@ -6671,9 +6800,9 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.26.0" +version = "0.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ba0117f4212101ee6544044dae45abe1083d30ce7b29c4b5cbdfa2354e07383" +checksum = "8970a78afe0628a3e3430376fc5fd76b6b45c4d43360ffd6cdd40bdde72b682a" dependencies = [ "indoc", "libc", @@ -6688,9 +6817,9 @@ dependencies = [ [[package]] name = "pyo3-async-runtimes" -version = "0.26.0" +version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6ee6d4cb3e8d5b925f5cdb38da183e0ff18122eb2048d4041c9e7034d026e23" +checksum = "d73cc6b1b7d8b3cef02101d37390dbdfe7e450dfea14921cae80a9534ba59ef2" dependencies = [ "futures", "once_cell", @@ -6702,9 +6831,9 @@ dependencies = [ [[package]] name = "pyo3-async-runtimes-macros" -version = "0.26.0" +version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c29bc5c673e36a8102d0b9179149c1bb59990d8db4f3ae58bd7dceccab90b951" +checksum = "ca31e43a0f205f2960208938135e37e579e61e10b36b4e7f49b0e8f60fab5b83" dependencies = [ "proc-macro2", "quote", @@ -6713,18 +6842,19 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.26.0" +version = "0.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fc6ddaf24947d12a9aa31ac65431fb1b851b8f4365426e182901eabfb87df5f" +checksum = "458eb0c55e7ece017adeba38f2248ff3ac615e53660d7c71a238d7d2a01c7598" dependencies = [ + "once_cell", "target-lexicon", ] [[package]] name = "pyo3-ffi" -version = "0.26.0" +version = "0.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "025474d3928738efb38ac36d4744a74a400c901c7596199e20e45d98eb194105" +checksum = "7114fe5457c61b276ab77c5055f206295b812608083644a5c5b2640c3102565c" dependencies = [ "libc", "pyo3-build-config", @@ -6732,9 +6862,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.26.0" +version = "0.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e64eb489f22fe1c95911b77c44cc41e7c19f3082fc81cce90f657cdc42ffded" +checksum = "a8725c0a622b374d6cb051d11a0983786448f7785336139c3c94f5aa6bef7e50" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -6744,9 +6874,9 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.26.0" +version = "0.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "100246c0ecf400b475341b8455a9213344569af29a3c841d29270e53102e0fcf" +checksum = "4109984c22491085343c05b0dbc54ddc405c3cf7b4374fc533f5c3313a572ccc" dependencies = [ "heck 0.5.0", "proc-macro2", @@ -7310,21 +7440,10 @@ checksum = "0a2c585be59b6b5dd66a9d2084aa1d8bd52fbdb806eafdeffb52791147862035" dependencies = [ "futures", "futures-timer", - "rstest_macros 0.23.0", + "rstest_macros", "rustc_version", ] -[[package]] -name = "rstest" -version = "0.26.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5a3193c063baaa2a95a33f03035c8a72b83d97a54916055ba22d35ed3839d49" -dependencies = [ - "futures-timer", - "futures-util", - "rstest_macros 0.26.1", -] - [[package]] name = "rstest_macros" version = "0.23.0" @@ -7343,24 +7462,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "rstest_macros" -version = "0.26.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c845311f0ff7951c5506121a9ad75aec44d083c31583b2ea5a30bcb0b0abba0" -dependencies = [ - "cfg-if", - "glob", - "proc-macro-crate", - "proc-macro2", - "quote", - "regex", - "relative-path", - "rustc_version", - "syn 2.0.114", - "unicode-ident", -] - [[package]] name = "rust-ini" version = "0.21.3" @@ -7983,6 +8084,12 @@ dependencies = [ "syn 2.0.114", ] +[[package]] +name = "snap" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b" + [[package]] name = "socket2" version = "0.5.10" @@ -8075,11 +8182,12 @@ dependencies = [ [[package]] name = "sqlparser" -version = "0.59.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4591acadbcf52f0af60eafbb2c003232b2b4cd8de5f0e9437cb8b1b59046cc0f" +checksum = "ec4b661c54b1e4b603b37873a18c59920e4c51ea8ea2cf527d925424dbd4437c" dependencies = [ "log", + "recursive", "sqlparser_derive", ] @@ -8335,7 +8443,7 @@ dependencies = [ "levenshtein_automata", "log", "lru", - "lz4_flex 0.11.5", + "lz4_flex", "measure_time", "memmap2 0.9.8", "once_cell", @@ -8567,6 +8675,17 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "thrift" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e54bc85fc7faa8bc175c4bab5b92ba8d9a3ce893d0e9f42cc455c8ab16a9e09" +dependencies = [ + "byteorder", + "integer-encoding", + "ordered-float 2.10.1", +] + [[package]] name = "time" version = "0.3.44" @@ -9672,6 +9791,15 @@ version = "0.8.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3" +[[package]] +name = "xz2" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388c44dc09d76f1536602ead6d325eb532f5c122f17782bd57fb47baeeb767e2" +dependencies = [ + "lzma-sys", +] + [[package]] name = "yoke" version = "0.7.5" @@ -9815,6 +9943,12 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "zlib-rs" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40990edd51aae2c2b6907af74ffb635029d5788228222c4bb811e9351c0caad3" + [[package]] name = "zstd" version = "0.13.3" diff --git a/Cargo.toml b/Cargo.toml index 54be799f5..47c75fc29 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,37 +15,37 @@ categories = ["database-implementations"] rust-version = "1.78.0" [workspace.dependencies] -lance = { "version" = "=2.0.0-beta.8", default-features = false, "tag" = "v2.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" } -lance-core = { "version" = "=2.0.0-beta.8", "tag" = "v2.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" } -lance-datagen = { "version" = "=2.0.0-beta.8", "tag" = "v2.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" } -lance-file = { "version" = "=2.0.0-beta.8", "tag" = "v2.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" } -lance-io = { "version" = "=2.0.0-beta.8", default-features = false, "tag" = "v2.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" } -lance-index = { "version" = "=2.0.0-beta.8", "tag" = "v2.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" } -lance-linalg = { "version" = "=2.0.0-beta.8", "tag" = "v2.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace = { "version" = "=2.0.0-beta.8", "tag" = "v2.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace-impls = { "version" = "=2.0.0-beta.8", default-features = false, "tag" = "v2.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" } -lance-table = { "version" = "=2.0.0-beta.8", "tag" = "v2.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" } -lance-testing = { "version" = "=2.0.0-beta.8", "tag" = "v2.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" } -lance-datafusion = { "version" = "=2.0.0-beta.8", "tag" = "v2.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" } -lance-encoding = { "version" = "=2.0.0-beta.8", "tag" = "v2.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" } -lance-arrow = { "version" = "=2.0.0-beta.8", "tag" = "v2.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" } +lance = { "version" = "=1.0.3-rc.1", default-features = false, "tag" = "v1.0.3-rc.1", "git" = "https://github.com/lance-format/lance.git" } +lance-core = { "version" = "=1.0.3-rc.1", "tag" = "v1.0.3-rc.1", "git" = "https://github.com/lance-format/lance.git" } +lance-datagen = { "version" = "=1.0.3-rc.1", "tag" = "v1.0.3-rc.1", "git" = "https://github.com/lance-format/lance.git" } +lance-file = { "version" = "=1.0.3-rc.1", "tag" = "v1.0.3-rc.1", "git" = "https://github.com/lance-format/lance.git" } +lance-io = { "version" = "=1.0.3-rc.1", default-features = false, "tag" = "v1.0.3-rc.1", "git" = "https://github.com/lance-format/lance.git" } +lance-index = { "version" = "=1.0.3-rc.1", "tag" = "v1.0.3-rc.1", "git" = "https://github.com/lance-format/lance.git" } +lance-linalg = { "version" = "=1.0.3-rc.1", "tag" = "v1.0.3-rc.1", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace = { "version" = "=1.0.3-rc.1", "tag" = "v1.0.3-rc.1", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace-impls = { "version" = "=1.0.3-rc.1", default-features = false, "tag" = "v1.0.3-rc.1", "git" = "https://github.com/lance-format/lance.git" } +lance-table = { "version" = "=1.0.3-rc.1", "tag" = "v1.0.3-rc.1", "git" = "https://github.com/lance-format/lance.git" } +lance-testing = { "version" = "=1.0.3-rc.1", "tag" = "v1.0.3-rc.1", "git" = "https://github.com/lance-format/lance.git" } +lance-datafusion = { "version" = "=1.0.3-rc.1", "tag" = "v1.0.3-rc.1", "git" = "https://github.com/lance-format/lance.git" } +lance-encoding = { "version" = "=1.0.3-rc.1", "tag" = "v1.0.3-rc.1", "git" = "https://github.com/lance-format/lance.git" } +lance-arrow = { "version" = "=1.0.3-rc.1", "tag" = "v1.0.3-rc.1", "git" = "https://github.com/lance-format/lance.git" } ahash = "0.8" # Note that this one does not include pyarrow -arrow = { version = "57.2", optional = false } -arrow-array = "57.2" -arrow-data = "57.2" -arrow-ipc = "57.2" -arrow-ord = "57.2" -arrow-schema = "57.2" -arrow-select = "57.2" -arrow-cast = "57.2" +arrow = { version = "56.1", optional = false } +arrow-array = "56.1" +arrow-data = "56.1" +arrow-ipc = "56.1" +arrow-ord = "56.1" +arrow-schema = "56.1" +arrow-select = "56.1" +arrow-cast = "56.1" async-trait = "0" -datafusion = { version = "51.0", default-features = false } -datafusion-catalog = "51.0" -datafusion-common = { version = "51.0", default-features = false } -datafusion-execution = "51.0" -datafusion-expr = "51.0" -datafusion-physical-plan = "51.0" +datafusion = { version = "50.0.0", default-features = false } +datafusion-catalog = "50.0.0" +datafusion-common = { version = "50.0.0", default-features = false } +datafusion-execution = "50.0.0" +datafusion-expr = "50.0.0" +datafusion-physical-plan = "50.0.0" env_logger = "0.11" half = { "version" = "2.7.1", default-features = false, features = [ "num-traits", diff --git a/python/Cargo.toml b/python/Cargo.toml index a9a8c3904..63504ede8 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -14,15 +14,15 @@ name = "_lancedb" crate-type = ["cdylib"] [dependencies] -arrow = { version = "57.2", features = ["pyarrow"] } +arrow = { version = "56.1", features = ["pyarrow"] } async-trait = "0.1" lancedb = { path = "../rust/lancedb", default-features = false } lance-core.workspace = true lance-namespace.workspace = true lance-io.workspace = true env_logger.workspace = true -pyo3 = { version = "0.26", features = ["extension-module", "abi3-py39"] } -pyo3-async-runtimes = { version = "0.26", features = [ +pyo3 = { version = "0.25.1", features = ["extension-module", "abi3-py39"] } +pyo3-async-runtimes = { version = "0.25.0", features = [ "attributes", "tokio-runtime", ] } @@ -32,7 +32,7 @@ snafu.workspace = true tokio = { version = "1.40", features = ["sync"] } [build-dependencies] -pyo3-build-config = { version = "0.26", features = [ +pyo3-build-config = { version = "0.25.0", features = [ "extension-module", "abi3-py39", ] } diff --git a/python/src/arrow.rs b/python/src/arrow.rs index 700f29cef..7eec8be24 100644 --- a/python/src/arrow.rs +++ b/python/src/arrow.rs @@ -36,10 +36,7 @@ impl RecordBatchStream { impl RecordBatchStream { #[getter] pub fn schema(&self, py: Python) -> PyResult> { - (*self.schema) - .clone() - .into_pyarrow(py) - .map(|obj| obj.unbind()) + (*self.schema).clone().into_pyarrow(py) } pub fn __aiter__(self_: PyRef<'_, Self>) -> PyRef<'_, Self> { @@ -57,8 +54,7 @@ impl RecordBatchStream { .ok_or_else(|| PyStopAsyncIteration::new_err(""))?; #[allow(deprecated)] let py_obj: Py = Python::with_gil(|py| -> PyResult> { - let bound = inner_next.infer_error()?.to_pyarrow(py)?; - Ok(bound.unbind()) + inner_next.infer_error()?.to_pyarrow(py) })?; Ok(py_obj) }) diff --git a/python/src/permutation.rs b/python/src/permutation.rs index e9487a8f2..0ac681f02 100644 --- a/python/src/permutation.rs +++ b/python/src/permutation.rs @@ -282,10 +282,8 @@ impl PyPermutationReader { future_into_py(slf.py(), async move { let schema = reader.output_schema(selection).await.infer_error()?; #[allow(deprecated)] - let py_obj: Py = Python::with_gil(|py| -> PyResult> { - let bound = schema.to_pyarrow(py)?; - Ok(bound.unbind()) - })?; + let py_obj: Py = + Python::with_gil(|py| -> PyResult> { schema.to_pyarrow(py) })?; Ok(py_obj) }) } diff --git a/python/src/query.rs b/python/src/query.rs index 6fbe27595..53a43a91b 100644 --- a/python/src/query.rs +++ b/python/src/query.rs @@ -455,10 +455,8 @@ impl Query { future_into_py(self_.py(), async move { let schema = inner.output_schema().await.infer_error()?; #[allow(deprecated)] - let py_obj: Py = Python::with_gil(|py| -> PyResult> { - let bound = schema.to_pyarrow(py)?; - Ok(bound.unbind()) - })?; + let py_obj: Py = + Python::with_gil(|py| -> PyResult> { schema.to_pyarrow(py) })?; Ok(py_obj) }) } @@ -539,10 +537,8 @@ impl TakeQuery { future_into_py(self_.py(), async move { let schema = inner.output_schema().await.infer_error()?; #[allow(deprecated)] - let py_obj: Py = Python::with_gil(|py| -> PyResult> { - let bound = schema.to_pyarrow(py)?; - Ok(bound.unbind()) - })?; + let py_obj: Py = + Python::with_gil(|py| -> PyResult> { schema.to_pyarrow(py) })?; Ok(py_obj) }) } @@ -639,10 +635,8 @@ impl FTSQuery { future_into_py(self_.py(), async move { let schema = inner.output_schema().await.infer_error()?; #[allow(deprecated)] - let py_obj: Py = Python::with_gil(|py| -> PyResult> { - let bound = schema.to_pyarrow(py)?; - Ok(bound.unbind()) - })?; + let py_obj: Py = + Python::with_gil(|py| -> PyResult> { schema.to_pyarrow(py) })?; Ok(py_obj) }) } @@ -823,10 +817,8 @@ impl VectorQuery { future_into_py(self_.py(), async move { let schema = inner.output_schema().await.infer_error()?; #[allow(deprecated)] - let py_obj: Py = Python::with_gil(|py| -> PyResult> { - let bound = schema.to_pyarrow(py)?; - Ok(bound.unbind()) - })?; + let py_obj: Py = + Python::with_gil(|py| -> PyResult> { schema.to_pyarrow(py) })?; Ok(py_obj) }) } diff --git a/python/src/table.rs b/python/src/table.rs index 088712c4b..4782bf4af 100644 --- a/python/src/table.rs +++ b/python/src/table.rs @@ -288,10 +288,8 @@ impl Table { future_into_py(self_.py(), async move { let schema = inner.schema().await.infer_error()?; #[allow(deprecated)] - let py_obj: Py = Python::with_gil(|py| -> PyResult> { - let bound = schema.to_pyarrow(py)?; - Ok(bound.unbind()) - })?; + let py_obj: Py = + Python::with_gil(|py| -> PyResult> { schema.to_pyarrow(py) })?; Ok(py_obj) }) } diff --git a/rust/lancedb/src/connection.rs b/rust/lancedb/src/connection.rs index 7743e3396..f18757fb8 100644 --- a/rust/lancedb/src/connection.rs +++ b/rust/lancedb/src/connection.rs @@ -9,6 +9,7 @@ use std::sync::Arc; use arrow_array::RecordBatchReader; use arrow_schema::{Field, SchemaRef}; use lance::dataset::ReadParams; +use lance::io::ObjectStoreParams; use lance_namespace::models::{ CreateNamespaceRequest, CreateNamespaceResponse, DescribeNamespaceRequest, DescribeNamespaceResponse, DropNamespaceRequest, DropNamespaceResponse, ListNamespacesRequest, @@ -39,7 +40,18 @@ use crate::Table; pub use lance_encoding::version::LanceFileVersion; #[cfg(feature = "remote")] use lance_io::object_store::StorageOptions; -use lance_io::object_store::StorageOptionsProvider; +use lance_io::object_store::{StorageOptionsAccessor, StorageOptionsProvider}; + +fn update_storage_options(store_params: &mut ObjectStoreParams, update: F) +where + F: FnOnce(&mut HashMap), +{ + let mut options = store_params.storage_options().cloned().unwrap_or_default(); + update(&mut options); + store_params.storage_options_accessor = Some(Arc::new( + StorageOptionsAccessor::with_static_options(options), + )); +} /// A builder for configuring a [`Connection::table_names`] operation pub struct TableNamesBuilder { @@ -246,16 +258,16 @@ impl CreateTableBuilder { /// /// See available options at pub fn storage_option(mut self, key: impl Into, value: impl Into) -> Self { - let store_options = self + let store_params = self .request .write_options .lance_write_params .get_or_insert(Default::default()) .store_params - .get_or_insert(Default::default()) - .storage_options .get_or_insert(Default::default()); - store_options.insert(key.into(), value.into()); + update_storage_options(store_params, |options| { + options.insert(key.into(), value.into()); + }); self } @@ -269,19 +281,18 @@ impl CreateTableBuilder { mut self, pairs: impl IntoIterator, impl Into)>, ) -> Self { - let store_options = self + let store_params = self .request .write_options .lance_write_params .get_or_insert(Default::default()) .store_params - .get_or_insert(Default::default()) - .storage_options .get_or_insert(Default::default()); - - for (key, value) in pairs { - store_options.insert(key.into(), value.into()); - } + update_storage_options(store_params, |options| { + for (key, value) in pairs { + options.insert(key.into(), value.into()); + } + }); self } @@ -318,24 +329,23 @@ impl CreateTableBuilder { /// This has no effect in LanceDB Cloud. #[deprecated(since = "0.15.1", note = "Use `database_options` instead")] pub fn enable_v2_manifest_paths(mut self, use_v2_manifest_paths: bool) -> Self { - let storage_options = self + let store_params = self .request .write_options .lance_write_params .get_or_insert_with(Default::default) .store_params - .get_or_insert_with(Default::default) - .storage_options .get_or_insert_with(Default::default); - - storage_options.insert( - OPT_NEW_TABLE_V2_MANIFEST_PATHS.to_string(), - if use_v2_manifest_paths { - "true".to_string() - } else { - "false".to_string() - }, - ); + update_storage_options(store_params, |options| { + options.insert( + OPT_NEW_TABLE_V2_MANIFEST_PATHS.to_string(), + if use_v2_manifest_paths { + "true".to_string() + } else { + "false".to_string() + }, + ); + }); self } @@ -344,20 +354,19 @@ impl CreateTableBuilder { /// The default is `LanceFileVersion::Stable`. #[deprecated(since = "0.15.1", note = "Use `database_options` instead")] pub fn data_storage_version(mut self, data_storage_version: LanceFileVersion) -> Self { - let storage_options = self + let store_params = self .request .write_options .lance_write_params .get_or_insert_with(Default::default) .store_params - .get_or_insert_with(Default::default) - .storage_options .get_or_insert_with(Default::default); - - storage_options.insert( - OPT_NEW_TABLE_STORAGE_VERSION.to_string(), - data_storage_version.to_string(), - ); + update_storage_options(store_params, |options| { + options.insert( + OPT_NEW_TABLE_STORAGE_VERSION.to_string(), + data_storage_version.to_string(), + ); + }); self } @@ -381,13 +390,17 @@ impl CreateTableBuilder { /// This allows tables to automatically refresh cloud storage credentials /// when they expire, enabling long-running operations on remote storage. pub fn storage_options_provider(mut self, provider: Arc) -> Self { - self.request + let store_params = self + .request .write_options .lance_write_params .get_or_insert(Default::default()) .store_params - .get_or_insert(Default::default()) - .storage_options_provider = Some(provider); + .get_or_insert(Default::default()); + let initial = store_params.storage_options().cloned().unwrap_or_default(); + store_params.storage_options_accessor = Some(Arc::new( + StorageOptionsAccessor::with_initial_and_provider(initial, provider), + )); self } } @@ -450,15 +463,15 @@ impl OpenTableBuilder { /// /// See available options at pub fn storage_option(mut self, key: impl Into, value: impl Into) -> Self { - let storage_options = self + let store_params = self .request .lance_read_params .get_or_insert(Default::default()) .store_options - .get_or_insert(Default::default()) - .storage_options .get_or_insert(Default::default()); - storage_options.insert(key.into(), value.into()); + update_storage_options(store_params, |options| { + options.insert(key.into(), value.into()); + }); self } @@ -472,18 +485,17 @@ impl OpenTableBuilder { mut self, pairs: impl IntoIterator, impl Into)>, ) -> Self { - let storage_options = self + let store_params = self .request .lance_read_params .get_or_insert(Default::default()) .store_options - .get_or_insert(Default::default()) - .storage_options .get_or_insert(Default::default()); - - for (key, value) in pairs { - storage_options.insert(key.into(), value.into()); - } + update_storage_options(store_params, |options| { + for (key, value) in pairs { + options.insert(key.into(), value.into()); + } + }); self } @@ -507,12 +519,16 @@ impl OpenTableBuilder { /// This allows tables to automatically refresh cloud storage credentials /// when they expire, enabling long-running operations on remote storage. pub fn storage_options_provider(mut self, provider: Arc) -> Self { - self.request + let store_params = self + .request .lance_read_params .get_or_insert(Default::default()) .store_options - .get_or_insert(Default::default()) - .storage_options_provider = Some(provider); + .get_or_insert(Default::default()); + let initial = store_params.storage_options().cloned().unwrap_or_default(); + store_params.storage_options_accessor = Some(Arc::new( + StorageOptionsAccessor::with_initial_and_provider(initial, provider), + )); self } diff --git a/rust/lancedb/src/database/listing.rs b/rust/lancedb/src/database/listing.rs index 8edb505ef..5096017bd 100644 --- a/rust/lancedb/src/database/listing.rs +++ b/rust/lancedb/src/database/listing.rs @@ -12,7 +12,7 @@ use lance::dataset::{builder::DatasetBuilder, ReadParams, WriteMode}; use lance::io::{ObjectStore, ObjectStoreParams, WrappingObjectStore}; use lance_datafusion::utils::StreamingWriteSource; use lance_encoding::version::LanceFileVersion; -use lance_io::object_store::StorageOptionsProvider; +use lance_io::object_store::{StorageOptionsAccessor, StorageOptionsProvider}; use lance_table::io::commit::commit_handler_from_url; use object_store::local::LocalFileSystem; use snafu::ResultExt; @@ -356,7 +356,11 @@ impl ListingDatabase { .clone() .unwrap_or_else(|| Arc::new(lance::session::Session::default())); let os_params = ObjectStoreParams { - storage_options: Some(options.storage_options.clone()), + storage_options_accessor: Some(Arc::new( + StorageOptionsAccessor::with_static_options( + options.storage_options.clone(), + ), + )), ..Default::default() }; let (object_store, base_path) = ObjectStore::from_uri_and_params( @@ -492,7 +496,9 @@ impl ListingDatabase { async fn drop_tables(&self, names: Vec) -> Result<()> { let object_store_params = ObjectStoreParams { - storage_options: Some(self.storage_options.clone()), + storage_options_accessor: Some(Arc::new(StorageOptionsAccessor::with_static_options( + self.storage_options.clone(), + ))), ..Default::default() }; let mut uri = self.uri.clone(); @@ -541,7 +547,7 @@ impl ListingDatabase { .lance_write_params .as_ref() .and_then(|p| p.store_params.as_ref()) - .and_then(|sp| sp.storage_options.as_ref()); + .and_then(|sp| sp.storage_options()); let storage_version_override = storage_options .and_then(|opts| opts.get(OPT_NEW_TABLE_STORAGE_VERSION)) @@ -593,20 +599,25 @@ impl ListingDatabase { // be dropped from the cache when python GCs the table object, which // confounds reuse across tables. if !self.storage_options.is_empty() { - let storage_options = write_params + let store_params = write_params .store_params - .get_or_insert_with(Default::default) - .storage_options .get_or_insert_with(Default::default); - self.inherit_storage_options(storage_options); + let mut storage_options = store_params.storage_options().cloned().unwrap_or_default(); + self.inherit_storage_options(&mut storage_options); + store_params.storage_options_accessor = Some(Arc::new( + StorageOptionsAccessor::with_static_options(storage_options), + )); } // Set storage options provider if available - if self.storage_options_provider.is_some() { - write_params + if let Some(provider) = self.storage_options_provider.clone() { + let store_params = write_params .store_params - .get_or_insert_with(Default::default) - .storage_options_provider = self.storage_options_provider.clone(); + .get_or_insert_with(Default::default); + let initial = store_params.storage_options().cloned().unwrap_or_default(); + store_params.storage_options_accessor = Some(Arc::new( + StorageOptionsAccessor::with_initial_and_provider(initial, provider), + )); } write_params.data_storage_version = self @@ -892,7 +903,9 @@ impl Database for ListingDatabase { validate_table_name(&request.target_table_name)?; let storage_params = ObjectStoreParams { - storage_options: Some(self.storage_options.clone()), + storage_options_accessor: Some(Arc::new(StorageOptionsAccessor::with_static_options( + self.storage_options.clone(), + ))), ..Default::default() }; let read_params = ReadParams { @@ -957,24 +970,29 @@ impl Database for ListingDatabase { // be dropped from the cache when python GCs the table object, which // confounds reuse across tables. if !self.storage_options.is_empty() { - let storage_options = request + let store_params = request .lance_read_params .get_or_insert_with(Default::default) .store_options - .get_or_insert_with(Default::default) - .storage_options .get_or_insert_with(Default::default); - self.inherit_storage_options(storage_options); + let mut storage_options = store_params.storage_options().cloned().unwrap_or_default(); + self.inherit_storage_options(&mut storage_options); + store_params.storage_options_accessor = Some(Arc::new( + StorageOptionsAccessor::with_static_options(storage_options), + )); } // Set storage options provider if available - if self.storage_options_provider.is_some() { - request + if let Some(provider) = self.storage_options_provider.clone() { + let store_params = request .lance_read_params .get_or_insert_with(Default::default) .store_options - .get_or_insert_with(Default::default) - .storage_options_provider = self.storage_options_provider.clone(); + .get_or_insert_with(Default::default); + let initial = store_params.storage_options().cloned().unwrap_or_default(); + store_params.storage_options_accessor = Some(Arc::new( + StorageOptionsAccessor::with_initial_and_provider(initial, provider), + )); } // Some ReadParams are exposed in the OpenTableBuilder, but we also @@ -1881,7 +1899,9 @@ mod tests { let write_options = WriteOptions { lance_write_params: Some(lance::dataset::WriteParams { store_params: Some(lance::io::ObjectStoreParams { - storage_options: Some(storage_options), + storage_options_accessor: Some(Arc::new( + StorageOptionsAccessor::with_static_options(storage_options), + )), ..Default::default() }), ..Default::default() @@ -1955,7 +1975,9 @@ mod tests { let write_options = WriteOptions { lance_write_params: Some(lance::dataset::WriteParams { store_params: Some(lance::io::ObjectStoreParams { - storage_options: Some(storage_options), + storage_options_accessor: Some(Arc::new( + StorageOptionsAccessor::with_static_options(storage_options), + )), ..Default::default() }), ..Default::default() diff --git a/rust/lancedb/src/table.rs b/rust/lancedb/src/table.rs index 9d7148b14..64e022623 100644 --- a/rust/lancedb/src/table.rs +++ b/rust/lancedb/src/table.rs @@ -40,7 +40,7 @@ use lance_index::vector::pq::PQBuildParams; use lance_index::vector::sq::builder::SQBuildParams; use lance_index::DatasetIndexExt; use lance_index::IndexType; -use lance_io::object_store::LanceNamespaceStorageOptionsProvider; +use lance_io::object_store::{LanceNamespaceStorageOptionsProvider, StorageOptionsAccessor}; use lance_namespace::models::{ QueryTableRequest as NsQueryTableRequest, QueryTableRequestColumns, QueryTableRequestFullTextQuery, QueryTableRequestVector, StringFtsQuery, @@ -1425,9 +1425,7 @@ impl Table { }) .collect::>(); - let unioned = UnionExec::try_new(projected_plans).map_err(|e| Error::Runtime { - message: format!("Failed to build union plan: {e}"), - })?; + let unioned = Arc::new(UnionExec::new(projected_plans)); // We require 1 partition in the final output let repartitioned = RepartitionExec::try_new( unioned, @@ -1668,18 +1666,14 @@ impl NativeTable { // Use DatasetBuilder::from_namespace which automatically fetches location // and storage options from the namespace - let builder = DatasetBuilder::from_namespace( - namespace_client.clone(), - table_id, - false, // Don't ignore namespace storage options - ) - .await - .map_err(|e| match e { - lance::Error::Namespace { source, .. } => Error::Runtime { - message: format!("Failed to get table info from namespace: {:?}", source), - }, - source => Error::Lance { source }, - })?; + let builder = DatasetBuilder::from_namespace(namespace_client.clone(), table_id) + .await + .map_err(|e| match e { + lance::Error::Namespace { source, .. } => Error::Runtime { + message: format!("Failed to get table info from namespace: {:?}", source), + }, + source => Error::Lance { source }, + })?; let dataset = builder .with_read_params(params) @@ -1883,7 +1877,10 @@ impl NativeTable { let store_params = params .store_params .get_or_insert_with(ObjectStoreParams::default); - store_params.storage_options_provider = Some(storage_options_provider); + let initial = store_params.storage_options().cloned().unwrap_or_default(); + store_params.storage_options_accessor = Some(Arc::new( + StorageOptionsAccessor::with_initial_and_provider(initial, storage_options_provider), + )); // Patch the params if we have a write store wrapper let params = match write_store_wrapper.clone() { @@ -3244,7 +3241,7 @@ impl BaseTable for NativeTable { .get() .await .ok() - .and_then(|dataset| dataset.storage_options().cloned()) + .and_then(|dataset| dataset.initial_storage_options().cloned()) } async fn index_stats(&self, index_name: &str) -> Result> { diff --git a/rust/lancedb/src/table/dataset.rs b/rust/lancedb/src/table/dataset.rs index 50f6b6472..ea1734299 100644 --- a/rust/lancedb/src/table/dataset.rs +++ b/rust/lancedb/src/table/dataset.rs @@ -100,8 +100,7 @@ impl DatasetRef { let should_checkout = match &target_ref { refs::Ref::Version(_, Some(target_ver)) => version != target_ver, refs::Ref::Version(_, None) => true, // No specific version, always checkout - refs::Ref::VersionNumber(target_ver) => version != target_ver, - refs::Ref::Tag(_) => true, // Always checkout for tags + refs::Ref::Tag(_) => true, // Always checkout for tags }; if should_checkout {