diff --git a/Cargo.lock b/Cargo.lock index 8839b990..b88a5c67 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1039,6 +1039,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "backon" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "302eaff5357a264a2c42f127ecb8bac761cf99749fc3dc95677e2743991f99e7" +dependencies = [ + "fastrand", + "gloo-timers", + "tokio", +] + [[package]] name = "backtrace" version = "0.3.75" @@ -2477,6 +2488,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", + "const-oid", "crypto-common", "subtle", ] @@ -2840,9 +2852,10 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "fsst" -version = "0.31.2" -source = "git+https://github.com/lancedb/lance.git?tag=v0.31.2-beta.3#6e987921d0efbe42c018047ea45d1b6f624d8280" +version = "0.32.0" +source = "git+https://github.com/lancedb/lance.git?tag=v0.32.0-beta.2#f9ebd7c3b9ecf07acf285a5ee9195399d9455696" dependencies = [ + "arrow-array", "rand 0.8.5", ] @@ -3256,6 +3269,18 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" +[[package]] +name = "gloo-timers" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb143cf96099802033e0d4f4963b19fd2e0b728bcf076cd9cf7f6634f092994" +dependencies = [ + "futures-channel", + "futures-core", + "js-sys", + "wasm-bindgen", +] + [[package]] name = "group" version = "0.12.1" @@ -3792,6 +3817,17 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "io-uring" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b86e202f00093dcba4275d4636b93ef9dd75d025ae560d2521b45ea28ab49013" +dependencies = [ + "bitflags 2.9.1", + "cfg-if", + "libc", +] + [[package]] name = "ipnet" version = "2.11.0" @@ -3930,8 +3966,8 @@ dependencies = [ [[package]] name = "lance" -version = "0.31.2" -source = "git+https://github.com/lancedb/lance.git?tag=v0.31.2-beta.3#6e987921d0efbe42c018047ea45d1b6f624d8280" +version = "0.32.0" +source = "git+https://github.com/lancedb/lance.git?tag=v0.32.0-beta.2#f9ebd7c3b9ecf07acf285a5ee9195399d9455696" dependencies = [ "arrow", "arrow-arith", @@ -3993,8 +4029,8 @@ dependencies = [ [[package]] name = "lance-arrow" -version = "0.31.2" -source = "git+https://github.com/lancedb/lance.git?tag=v0.31.2-beta.3#6e987921d0efbe42c018047ea45d1b6f624d8280" +version = "0.32.0" +source = "git+https://github.com/lancedb/lance.git?tag=v0.32.0-beta.2#f9ebd7c3b9ecf07acf285a5ee9195399d9455696" dependencies = [ "arrow-array", "arrow-buffer", @@ -4011,8 +4047,8 @@ dependencies = [ [[package]] name = "lance-core" -version = "0.31.2" -source = "git+https://github.com/lancedb/lance.git?tag=v0.31.2-beta.3#6e987921d0efbe42c018047ea45d1b6f624d8280" +version = "0.32.0" +source = "git+https://github.com/lancedb/lance.git?tag=v0.32.0-beta.2#f9ebd7c3b9ecf07acf285a5ee9195399d9455696" dependencies = [ "arrow-array", "arrow-buffer", @@ -4047,8 +4083,8 @@ dependencies = [ [[package]] name = "lance-datafusion" -version = "0.31.2" -source = "git+https://github.com/lancedb/lance.git?tag=v0.31.2-beta.3#6e987921d0efbe42c018047ea45d1b6f624d8280" +version = "0.32.0" +source = "git+https://github.com/lancedb/lance.git?tag=v0.32.0-beta.2#f9ebd7c3b9ecf07acf285a5ee9195399d9455696" dependencies = [ "arrow", "arrow-array", @@ -4076,8 +4112,8 @@ dependencies = [ [[package]] name = "lance-datagen" -version = "0.31.2" -source = "git+https://github.com/lancedb/lance.git?tag=v0.31.2-beta.3#6e987921d0efbe42c018047ea45d1b6f624d8280" +version = "0.32.0" +source = "git+https://github.com/lancedb/lance.git?tag=v0.32.0-beta.2#f9ebd7c3b9ecf07acf285a5ee9195399d9455696" dependencies = [ "arrow", "arrow-array", @@ -4093,8 +4129,8 @@ dependencies = [ [[package]] name = "lance-encoding" -version = "0.31.2" -source = "git+https://github.com/lancedb/lance.git?tag=v0.31.2-beta.3#6e987921d0efbe42c018047ea45d1b6f624d8280" +version = "0.32.0" +source = "git+https://github.com/lancedb/lance.git?tag=v0.32.0-beta.2#f9ebd7c3b9ecf07acf285a5ee9195399d9455696" dependencies = [ "arrayref", "arrow", @@ -4133,8 +4169,8 @@ dependencies = [ [[package]] name = "lance-file" -version = "0.31.2" -source = "git+https://github.com/lancedb/lance.git?tag=v0.31.2-beta.3#6e987921d0efbe42c018047ea45d1b6f624d8280" +version = "0.32.0" +source = "git+https://github.com/lancedb/lance.git?tag=v0.32.0-beta.2#f9ebd7c3b9ecf07acf285a5ee9195399d9455696" dependencies = [ "arrow-arith", "arrow-array", @@ -4168,8 +4204,8 @@ dependencies = [ [[package]] name = "lance-index" -version = "0.31.2" -source = "git+https://github.com/lancedb/lance.git?tag=v0.31.2-beta.3#6e987921d0efbe42c018047ea45d1b6f624d8280" +version = "0.32.0" +source = "git+https://github.com/lancedb/lance.git?tag=v0.32.0-beta.2#f9ebd7c3b9ecf07acf285a5ee9195399d9455696" dependencies = [ "arrow", "arrow-array", @@ -4203,7 +4239,6 @@ dependencies = [ "lance-linalg", "lance-table", "log", - "moka", "num-traits", "object_store", "prost", @@ -4223,8 +4258,8 @@ dependencies = [ [[package]] name = "lance-io" -version = "0.31.2" -source = "git+https://github.com/lancedb/lance.git?tag=v0.31.2-beta.3#6e987921d0efbe42c018047ea45d1b6f624d8280" +version = "0.32.0" +source = "git+https://github.com/lancedb/lance.git?tag=v0.32.0-beta.2#f9ebd7c3b9ecf07acf285a5ee9195399d9455696" dependencies = [ "arrow", "arrow-arith", @@ -4248,6 +4283,8 @@ dependencies = [ "lance-core", "log", "object_store", + "object_store_opendal", + "opendal", "path_abs", "pin-project", "prost", @@ -4262,8 +4299,8 @@ dependencies = [ [[package]] name = "lance-linalg" -version = "0.31.2" -source = "git+https://github.com/lancedb/lance.git?tag=v0.31.2-beta.3#6e987921d0efbe42c018047ea45d1b6f624d8280" +version = "0.32.0" +source = "git+https://github.com/lancedb/lance.git?tag=v0.32.0-beta.2#f9ebd7c3b9ecf07acf285a5ee9195399d9455696" dependencies = [ "arrow-array", "arrow-buffer", @@ -4286,8 +4323,8 @@ dependencies = [ [[package]] name = "lance-table" -version = "0.31.2" -source = "git+https://github.com/lancedb/lance.git?tag=v0.31.2-beta.3#6e987921d0efbe42c018047ea45d1b6f624d8280" +version = "0.32.0" +source = "git+https://github.com/lancedb/lance.git?tag=v0.32.0-beta.2#f9ebd7c3b9ecf07acf285a5ee9195399d9455696" dependencies = [ "arrow", "arrow-array", @@ -4325,8 +4362,8 @@ dependencies = [ [[package]] name = "lance-testing" -version = "0.31.2" -source = "git+https://github.com/lancedb/lance.git?tag=v0.31.2-beta.3#6e987921d0efbe42c018047ea45d1b6f624d8280" +version = "0.32.0" +source = "git+https://github.com/lancedb/lance.git?tag=v0.32.0-beta.2#f9ebd7c3b9ecf07acf285a5ee9195399d9455696" dependencies = [ "arrow-array", "arrow-schema", @@ -5215,6 +5252,21 @@ dependencies = [ "web-time", ] +[[package]] +name = "object_store_opendal" +version = "0.54.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ce697ee723fdc3eaf6c457abf4059034be15167022b18b619993802cd1443d5" +dependencies = [ + "async-trait", + "bytes", + "futures", + "object_store", + "opendal", + "pin-project", + "tokio", +] + [[package]] name = "once_cell" version = "1.21.3" @@ -5255,6 +5307,33 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "opendal" +version = "0.54.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffb9838d0575c6dbaf3fcec7255af8d5771996d4af900bbb6fa9a314dec00a1a" +dependencies = [ + "anyhow", + "backon", + "base64 0.22.1", + "bytes", + "chrono", + "futures", + "getrandom 0.2.16", + "http 1.3.1", + "http-body 1.0.1", + "log", + "md-5", + "percent-encoding", + "quick-xml", + "reqsign", + "reqwest", + "serde", + "serde_json", + "tokio", + "uuid", +] + [[package]] name = "openssl-probe" version = "0.1.6" @@ -6460,6 +6539,33 @@ version = "1.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" +[[package]] +name = "reqsign" +version = "0.16.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43451dbf3590a7590684c25fb8d12ecdcc90ed3ac123433e500447c7d77ed701" +dependencies = [ + "anyhow", + "async-trait", + "base64 0.22.1", + "chrono", + "form_urlencoded", + "getrandom 0.2.16", + "hex", + "hmac", + "home", + "http 1.3.1", + "log", + "once_cell", + "percent-encoding", + "rand 0.8.5", + "reqwest", + "serde", + "serde_json", + "sha1", + "sha2", +] + [[package]] name = "reqwest" version = "0.12.20" @@ -7732,16 +7838,18 @@ dependencies = [ [[package]] name = "tokio" -version = "1.45.1" +version = "1.46.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75ef51a33ef1da925cea3e4eb122833cb377c61439ca401b770f54902b806779" +checksum = "0cc3a2344dafbe23a245241fe8b09735b521110d30fcefbbd5feb1797ca35d17" dependencies = [ "backtrace", "bytes", + "io-uring", "libc", "mio", "pin-project-lite", "signal-hook-registry", + "slab", "socket2", "tokio-macros", "windows-sys 0.52.0", diff --git a/Cargo.toml b/Cargo.toml index 6ab050ed..4da64a6d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,16 +21,16 @@ categories = ["database-implementations"] rust-version = "1.78.0" [workspace.dependencies] -lance = { "version" = "=0.31.2", "features" = [ +lance = { "version" = "=0.32.0", "features" = [ "dynamodb", -], "tag" = "v0.31.2-beta.3", "git" = "https://github.com/lancedb/lance.git" } -lance-io = { "version" = "=0.31.2", "tag" = "v0.31.2-beta.3", "git" = "https://github.com/lancedb/lance.git" } -lance-index = { "version" = "=0.31.2", "tag" = "v0.31.2-beta.3", "git" = "https://github.com/lancedb/lance.git" } -lance-linalg = { "version" = "=0.31.2", "tag" = "v0.31.2-beta.3", "git" = "https://github.com/lancedb/lance.git" } -lance-table = { "version" = "=0.31.2", "tag" = "v0.31.2-beta.3", "git" = "https://github.com/lancedb/lance.git" } -lance-testing = { "version" = "=0.31.2", "tag" = "v0.31.2-beta.3", "git" = "https://github.com/lancedb/lance.git" } -lance-datafusion = { "version" = "=0.31.2", "tag" = "v0.31.2-beta.3", "git" = "https://github.com/lancedb/lance.git" } -lance-encoding = { "version" = "=0.31.2", "tag" = "v0.31.2-beta.3", "git" = "https://github.com/lancedb/lance.git" } +], "tag" = "v0.32.0-beta.2", "git" = "https://github.com/lancedb/lance.git" } +lance-io = { "version" = "=0.32.0", "tag" = "v0.32.0-beta.2", "git" = "https://github.com/lancedb/lance.git" } +lance-index = { "version" = "=0.32.0", "tag" = "v0.32.0-beta.2", "git" = "https://github.com/lancedb/lance.git" } +lance-linalg = { "version" = "=0.32.0", "tag" = "v0.32.0-beta.2", "git" = "https://github.com/lancedb/lance.git" } +lance-table = { "version" = "=0.32.0", "tag" = "v0.32.0-beta.2", "git" = "https://github.com/lancedb/lance.git" } +lance-testing = { "version" = "=0.32.0", "tag" = "v0.32.0-beta.2", "git" = "https://github.com/lancedb/lance.git" } +lance-datafusion = { "version" = "=0.32.0", "tag" = "v0.32.0-beta.2", "git" = "https://github.com/lancedb/lance.git" } +lance-encoding = { "version" = "=0.32.0", "tag" = "v0.32.0-beta.2", "git" = "https://github.com/lancedb/lance.git" } # Note that this one does not include pyarrow arrow = { version = "55.1", optional = false } arrow-array = "55.1" diff --git a/java/core/lancedb-jni/Cargo.toml b/java/core/lancedb-jni/Cargo.toml index 6fa63fba..be8ef6e8 100644 --- a/java/core/lancedb-jni/Cargo.toml +++ b/java/core/lancedb-jni/Cargo.toml @@ -19,7 +19,7 @@ lancedb = { path = "../../../rust/lancedb" } lance = { workspace = true } arrow = { workspace = true, features = ["ffi"] } arrow-schema.workspace = true -tokio = "1.23" +tokio = "1.46" jni = "0.21.1" snafu.workspace = true lazy_static.workspace = true diff --git a/node/package-lock.json b/node/package-lock.json index bf27f926..02126b31 100644 --- a/node/package-lock.json +++ b/node/package-lock.json @@ -326,6 +326,66 @@ "@jridgewell/sourcemap-codec": "^1.4.10" } }, + "node_modules/@lancedb/vectordb-darwin-arm64": { + "version": "0.21.2-beta.1", + "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-arm64/-/vectordb-darwin-arm64-0.21.2-beta.1.tgz", + "integrity": "sha512-7QXVJNTei7PMuXRyyc+F3WGiudRNq9HfeOaMmMOJJpuCAO0zLq1pM9DCl5aPF5MddrodPHJxi+IWV+iAFH7zcg==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@lancedb/vectordb-darwin-x64": { + "version": "0.21.2-beta.1", + "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-x64/-/vectordb-darwin-x64-0.21.2-beta.1.tgz", + "integrity": "sha512-M/TWcJ3WVc6DNFgG/lWI7L5tQ05IF3WoWuZfRfbbimGhRvY7xf1O3uOt+jMcNJCa5mHFGCg2SZDA8mebd/mL7g==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@lancedb/vectordb-linux-arm64-gnu": { + "version": "0.21.2-beta.1", + "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-arm64-gnu/-/vectordb-linux-arm64-gnu-0.21.2-beta.1.tgz", + "integrity": "sha512-OEsM9znf9DDmdwGuTg2EVu+ebwuWQ1lCx0cYy4+hNy3ntolwMC39ePg2H9WD9SsEnQ2vcGJgBJTQLPKgXww+iQ==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@lancedb/vectordb-linux-x64-gnu": { + "version": "0.21.2-beta.1", + "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-x64-gnu/-/vectordb-linux-x64-gnu-0.21.2-beta.1.tgz", + "integrity": "sha512-7FTq/O1zNzD71rgX2PEVmkct4jk2wc+ADU3rss+0VqoBSO9XeMqZEVD2WgZWuSTg6bYai//FHGDHSaknHBNsdw==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@lancedb/vectordb-win32-x64-msvc": { + "version": "0.21.2-beta.1", + "resolved": "https://registry.npmjs.org/@lancedb/vectordb-win32-x64-msvc/-/vectordb-win32-x64-msvc-0.21.2-beta.1.tgz", + "integrity": "sha512-mN1p/J0kdqy6MrlKtmA8set/PibqFPyytQJFAuxSLXC/rwD7vgqUCt0SI0zVWPGG7J5Y65kvdc99l7Yl7lJtwQ==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "win32" + ] + }, "node_modules/@neon-rs/cli": { "version": "0.0.160", "resolved": "https://registry.npmjs.org/@neon-rs/cli/-/cli-0.0.160.tgz", diff --git a/rust/lancedb/src/database/listing.rs b/rust/lancedb/src/database/listing.rs index d63e46f5..8aa9e437 100644 --- a/rust/lancedb/src/database/listing.rs +++ b/rust/lancedb/src/database/listing.rs @@ -678,7 +678,8 @@ impl Database for ListingDatabase { let mut read_params = request.lance_read_params.unwrap_or_else(|| { let mut default_params = ReadParams::default(); if let Some(index_cache_size) = request.index_cache_size { - default_params.index_cache_size = index_cache_size as usize; + #[allow(deprecated)] + default_params.index_cache_size(index_cache_size as usize); } default_params }); diff --git a/rust/lancedb/src/table/datafusion.rs b/rust/lancedb/src/table/datafusion.rs index f136075e..940b8831 100644 --- a/rust/lancedb/src/table/datafusion.rs +++ b/rust/lancedb/src/table/datafusion.rs @@ -486,11 +486,9 @@ pub mod tests { TestFixture::check_plan( plan, "MetadataEraserExec - CoalesceBatchesExec:... - FilterExec: i@0 >= 5 RepartitionExec:... ProjectionExec:... - LanceScan:...", + LanceRead:...", ) .await;