rebase

Cargo.lock

@@ -1852,7 +1852,7 @@ dependencies = [
  "futures",
  "itertools 0.14.0",
  "log",
- "object_store 0.12.2",
+ "object_store",
  "parking_lot",
  "rand 0.8.5",
  "regex",
@@ -1884,7 +1884,7 @@ dependencies = [
  "futures",
  "itertools 0.14.0",
  "log",
- "object_store 0.12.2",
+ "object_store",
  "parking_lot",
  "tokio",
 ]
@@ -1908,7 +1908,7 @@ dependencies = [
  "datafusion-session",
  "futures",
  "log",
- "object_store 0.12.2",
+ "object_store",
  "tokio",
 ]
 
@@ -1927,7 +1927,7 @@ dependencies = [
  "indexmap 2.9.0",
  "libc",
  "log",
- "object_store 0.12.2",
+ "object_store",
  "paste",
  "sqlparser 0.55.0",
  "tokio",
@@ -1967,7 +1967,7 @@ dependencies = [
  "glob",
  "itertools 0.14.0",
  "log",
- "object_store 0.12.2",
+ "object_store",
  "rand 0.8.5",
  "tokio",
  "url",
@@ -1993,7 +1993,7 @@ dependencies = [
  "datafusion-physical-plan",
  "datafusion-session",
  "futures",
- "object_store 0.12.2",
+ "object_store",
  "regex",
  "tokio",
 ]
@@ -2018,7 +2018,7 @@ dependencies = [
  "datafusion-physical-plan",
  "datafusion-session",
  "futures",
- "object_store 0.12.2",
+ "object_store",
  "serde_json",
  "tokio",
 ]
@@ -2041,7 +2041,7 @@ dependencies = [
  "datafusion-expr",
  "futures",
  "log",
- "object_store 0.12.2",
+ "object_store",
  "parking_lot",
  "rand 0.8.5",
  "tempfile",
@@ -2340,7 +2340,7 @@ dependencies = [
  "futures",
  "itertools 0.14.0",
  "log",
- "object_store 0.12.2",
+ "object_store",
  "parking_lot",
  "tokio",
 ]
@@ -2813,9 +2813,7 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
 
 [[package]]
 name = "fsst"
-version = "0.30.0"
+version = "0.31.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3b6a55335126d20524dc83cf0638b7ca1b5d9736f9064a89c47e4d028cbaccdb"
 dependencies = [
  "rand 0.8.5",
 ]
@@ -3907,9 +3905,7 @@ dependencies = [
 
 [[package]]
 name = "lance"
-version = "0.30.0"
+version = "0.31.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "84a9bf2cf9ff1d8b8a8c822cf4aaec7023fbe056d3348dce347957695470bd19"
 dependencies = [
  "arrow",
  "arrow-arith",
@@ -3952,7 +3948,7 @@ dependencies = [
  "lazy_static",
  "log",
  "moka",
- "object_store 0.11.2",
+ "object_store",
  "permutation",
  "pin-project",
  "prost",
@@ -3972,9 +3968,7 @@ dependencies = [
 
 [[package]]
 name = "lance-arrow"
-version = "0.30.0"
+version = "0.31.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "82fc2b0dd2598f4b390445d63a3906f84d928c250b208d382d4cfc22681b23c0"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -3991,9 +3985,7 @@ dependencies = [
 
 [[package]]
 name = "lance-core"
-version = "0.30.0"
+version = "0.31.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4118c6e2ac2d26ff80e55708f337c4593381a32751f2a79a03d92452885bd648"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -4013,7 +4005,7 @@ dependencies = [
  "mock_instant",
  "moka",
  "num_cpus",
- "object_store 0.11.2",
+ "object_store",
  "pin-project",
  "prost",
  "rand 0.8.5",
@@ -4029,9 +4021,7 @@ dependencies = [
 
 [[package]]
 name = "lance-datafusion"
-version = "0.30.0"
+version = "0.31.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ccf8b01e9a5f15d4975423ea1495df85cf36f9036c3ed999190d4631ffbd28b6"
 dependencies = [
  "arrow",
  "arrow-array",
@@ -4060,9 +4050,7 @@ dependencies = [
 
 [[package]]
 name = "lance-datagen"
-version = "0.30.0"
+version = "0.31.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3fbedb84243fb2fe255b4e9ac298019d2e93e83fcc9ce2eb67a4ac7cab427dda"
 dependencies = [
  "arrow",
  "arrow-array",
@@ -4077,9 +4065,7 @@ dependencies = [
 
 [[package]]
 name = "lance-encoding"
-version = "0.30.0"
+version = "0.31.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9a0e078414cce96da2e2b37290d0b38a81ba6b0ebcad6806b231c2cd8d04427a"
 dependencies = [
  "arrayref",
  "arrow",
@@ -4113,14 +4099,13 @@ dependencies = [
  "snafu",
  "tokio",
  "tracing",
+ "xxhash-rust",
  "zstd",
 ]
 
 [[package]]
 name = "lance-file"
-version = "0.30.0"
+version = "0.31.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ce7deba5b59118f7ef726859ace192b7cc7da4e6639147d2a3908a2de621ce98"
 dependencies = [
  "arrow-arith",
  "arrow-array",
@@ -4141,7 +4126,7 @@ dependencies = [
  "lance-io",
  "log",
  "num-traits",
- "object_store 0.11.2",
+ "object_store",
  "prost",
  "prost-build",
  "prost-types",
@@ -4154,9 +4139,7 @@ dependencies = [
 
 [[package]]
 name = "lance-index"
-version = "0.30.0"
+version = "0.31.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5bee1aecc60c759436d8f952e2d9c4e93d1940bfbdc1869068b4ac6b01e86b2f"
 dependencies = [
  "arrow",
  "arrow-array",
@@ -4193,7 +4176,7 @@ dependencies = [
  "log",
  "moka",
  "num-traits",
- "object_store 0.11.2",
+ "object_store",
  "prost",
  "prost-build",
  "rand 0.8.5",
@@ -4211,9 +4194,7 @@ dependencies = [
 
 [[package]]
 name = "lance-io"
-version = "0.30.0"
+version = "0.31.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "61a48f6a3f5433ca5095993fcd8bb47efbf473af852b9aca1e175a3d7bbf67fd"
 dependencies = [
  "arrow",
  "arrow-arith",
@@ -4237,7 +4218,7 @@ dependencies = [
  "lance-core",
  "lazy_static",
  "log",
- "object_store 0.11.2",
+ "object_store",
  "path_abs",
  "pin-project",
  "prost",
@@ -4252,9 +4233,7 @@ dependencies = [
 
 [[package]]
 name = "lance-linalg"
-version = "0.30.0"
+version = "0.31.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "620dedc792311862fc336b2651e825d2b450bbade7bfc819b7b182c3bb585c1e"
 dependencies = [
  "arrow-array",
  "arrow-ord",
@@ -4277,9 +4256,7 @@ dependencies = [
 
 [[package]]
 name = "lance-table"
-version = "0.30.0"
+version = "0.31.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9b010312330943c5e81628722a50e3679688d96065348659b7913964f13765cf"
 dependencies = [
  "arrow",
  "arrow-array",
@@ -4300,7 +4277,7 @@ dependencies = [
  "lance-io",
  "lazy_static",
  "log",
- "object_store 0.11.2",
+ "object_store",
  "prost",
  "prost-build",
  "prost-types",
@@ -4318,9 +4295,7 @@ dependencies = [
 
 [[package]]
 name = "lance-testing"
-version = "0.30.0"
+version = "0.31.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "efa10957cdadef40e853896a67282cd29898775b29715eec42dd49bc3b3c8554"
 dependencies = [
  "arrow-array",
  "arrow-schema",
@@ -4378,7 +4353,7 @@ dependencies = [
  "log",
  "moka",
  "num-traits",
- "object_store 0.11.2",
+ "object_store",
  "pin-project",
  "polars",
  "polars-arrow",
@@ -4435,7 +4410,7 @@ dependencies = [
  "lancedb",
  "lzma-sys",
  "neon",
- "object_store 0.11.2",
+ "object_store",
  "once_cell",
  "snafu",
  "tokio",
@@ -5172,38 +5147,6 @@ dependencies = [
  "memchr",
 ]
 
-[[package]]
-name = "object_store"
-version = "0.11.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3cfccb68961a56facde1163f9319e0d15743352344e7808a11795fb99698dcaf"
-dependencies = [
- "async-trait",
- "base64 0.22.1",
- "bytes",
- "chrono",
- "futures",
- "httparse",
- "humantime",
- "hyper 1.6.0",
- "itertools 0.13.0",
- "md-5",
- "parking_lot",
- "percent-encoding",
- "quick-xml",
- "rand 0.8.5",
- "reqwest",
- "ring",
- "rustls-pemfile 2.2.0",
- "serde",
- "serde_json",
- "snafu",
- "tokio",
- "tracing",
- "url",
- "walkdir",
-]
-
 [[package]]
 name = "object_store"
 version = "0.12.2"
@@ -5211,14 +5154,28 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7781f96d79ed0f961a7021424ab01840efbda64ae7a505aaea195efc91eaaec4"
 dependencies = [
  "async-trait",
+ "base64 0.22.1",
  "bytes",
  "chrono",
+ "form_urlencoded",
  "futures",
  "http 1.3.1",
+ "http-body-util",
+ "httparse",
  "humantime",
+ "hyper 1.6.0",
  "itertools 0.14.0",
+ "md-5",
  "parking_lot",
  "percent-encoding",
+ "quick-xml",
+ "rand 0.9.1",
+ "reqwest",
+ "ring",
+ "rustls-pemfile 2.2.0",
+ "serde",
+ "serde_json",
+ "serde_urlencoded",
  "thiserror 2.0.12",
  "tokio",
  "tracing",

Cargo.toml

@@ -21,14 +21,14 @@ categories = ["database-implementations"]
 rust-version = "1.78.0"
 
 [workspace.dependencies]
-lance = { "version" = "=0.30.0", "features" = ["dynamodb"] }
+lance = { path = "../lance/rust/lance", "features" = ["dynamodb"] }
-lance-io = "=0.30.0"
+lance-io = { path = "../lance/rust/lance-io" }
-lance-index = "=0.30.0"
+lance-index = { path = "../lance/rust/lance-index" }
-lance-linalg = "=0.30.0"
+lance-linalg = { path = "../lance/rust/lance-linalg" }
-lance-table = "=0.30.0"
+lance-table = { path = "../lance/rust/lance-table" }
-lance-testing = "=0.30.0"
+lance-testing = { path = "../lance/rust/lance-testing" }
-lance-datafusion = "=0.30.0"
+lance-datafusion = { path = "../lance/rust/lance-datafusion" }
-lance-encoding = "=0.30.0"
+lance-encoding = { path = "../lance/rust/lance-encoding" }
 # Note that this one does not include pyarrow
 arrow = { version = "55.1", optional = false }
 arrow-array = "55.1"
@@ -52,7 +52,7 @@ half = { "version" = "=2.5.0", default-features = false, features = [
 futures = "0"
 log = "0.4"
 moka = { version = "0.12", features = ["future"] }
-object_store = "0.11.0"
+object_store = "0.12.0"
 pin-project = "1.0.7"
 snafu = "0.8"
 url = "2"

rust/lancedb/src/io/object_store.rs

@@ -107,7 +107,7 @@ impl ObjectStore for MirroringObjectStore {
         self.primary.delete(location).await
     }
 
-    fn list(&self, prefix: Option<&Path>) -> BoxStream<'_, Result<ObjectMeta>> {
+    fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, Result<ObjectMeta>> {
         self.primary.list(prefix)
     }
 

rust/lancedb/src/io/object_store/io_tracking.rs

@@ -133,7 +133,7 @@ impl ObjectStore for IoTrackingStore {
         result
     }
 
-    async fn get_range(&self, location: &Path, range: std::ops::Range<usize>) -> OSResult<Bytes> {
+    async fn get_range(&self, location: &Path, range: std::ops::Range<u64>) -> OSResult<Bytes> {
         let result = self.target.get_range(location, range).await;
         if let Ok(result) = &result {
             self.record_read(result.len() as u64);
@@ -144,7 +144,7 @@ impl ObjectStore for IoTrackingStore {
     async fn get_ranges(
         &self,
         location: &Path,
-        ranges: &[std::ops::Range<usize>],
+        ranges: &[std::ops::Range<u64>],
     ) -> OSResult<Vec<Bytes>> {
         let result = self.target.get_ranges(location, ranges).await;
         if let Ok(result) = &result {
@@ -170,7 +170,7 @@ impl ObjectStore for IoTrackingStore {
         self.target.delete_stream(locations)
     }
 
-    fn list(&self, prefix: Option<&Path>) -> BoxStream<'_, OSResult<ObjectMeta>> {
+    fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, OSResult<ObjectMeta>> {
         self.record_read(0);
         self.target.list(prefix)
     }
@@ -179,7 +179,7 @@ impl ObjectStore for IoTrackingStore {
         &self,
         prefix: Option<&Path>,
         offset: &Path,
-    ) -> BoxStream<'_, OSResult<ObjectMeta>> {
+    ) -> BoxStream<'static, OSResult<ObjectMeta>> {
         self.record_read(0);
         self.target.list_with_offset(prefix, offset)
     }