rust/lancedb
1
0
Fork 0
mirror of https://github.com/lancedb/lancedb.git synced 2025-12-22 21:09:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

rebase

Browse Source
This commit is contained in:
Will Jones
2025-05-29 15:32:10 -07:00
committed by Weston Pace
parent 4beb2d2877
commit 5a732bbf39
4 changed files with 59 additions and 102 deletions
Download Patch File Download Diff File Expand all files Collapse all files

133
Cargo.lock generated
View File

@@ -1852,7 +1852,7 @@ dependencies = [
"futures",
"itertools 0.14.0",
"log",
"object_store 0.12.2",
"object_store",
"parking_lot",
"rand 0.8.5",
"regex",
@@ -1884,7 +1884,7 @@ dependencies = [
"futures",
"itertools 0.14.0",
"log",
"object_store 0.12.2",
"object_store",
"parking_lot",
"tokio",
]
@@ -1908,7 +1908,7 @@ dependencies = [
"datafusion-session",
"futures",
"log",
"object_store 0.12.2",
"object_store",
"tokio",
]
@@ -1927,7 +1927,7 @@ dependencies = [
"indexmap 2.9.0",
"libc",
"log",
"object_store 0.12.2",
"object_store",
"paste",
"sqlparser 0.55.0",
"tokio",
@@ -1967,7 +1967,7 @@ dependencies = [
"glob",
"itertools 0.14.0",
"log",
"object_store 0.12.2",
"object_store",
"rand 0.8.5",
"tokio",
"url",
@@ -1993,7 +1993,7 @@ dependencies = [
"datafusion-physical-plan",
"datafusion-session",
"futures",
"object_store 0.12.2",
"object_store",
"regex",
"tokio",
]
@@ -2018,7 +2018,7 @@ dependencies = [
"datafusion-physical-plan",
"datafusion-session",
"futures",
"object_store 0.12.2",
"object_store",
"serde_json",
"tokio",
]
@@ -2041,7 +2041,7 @@ dependencies = [
"datafusion-expr",
"futures",
"log",
"object_store 0.12.2",
"object_store",
"parking_lot",
"rand 0.8.5",
"tempfile",
@@ -2340,7 +2340,7 @@ dependencies = [
"futures",
"itertools 0.14.0",
"log",
"object_store 0.12.2",
"object_store",
"parking_lot",
"tokio",
]
@@ -2813,9 +2813,7 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
[[package]]
name = "fsst"
version = "0.30.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3b6a55335126d20524dc83cf0638b7ca1b5d9736f9064a89c47e4d028cbaccdb"
version = "0.31.0"
dependencies = [
"rand 0.8.5",
]
@@ -3907,9 +3905,7 @@ dependencies = [
[[package]]
name = "lance"
version = "0.30.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "84a9bf2cf9ff1d8b8a8c822cf4aaec7023fbe056d3348dce347957695470bd19"
version = "0.31.0"
dependencies = [
"arrow",
"arrow-arith",
@@ -3952,7 +3948,7 @@ dependencies = [
"lazy_static",
"log",
"moka",
"object_store 0.11.2",
"object_store",
"permutation",
"pin-project",
"prost",
@@ -3972,9 +3968,7 @@ dependencies = [
[[package]]
name = "lance-arrow"
version = "0.30.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "82fc2b0dd2598f4b390445d63a3906f84d928c250b208d382d4cfc22681b23c0"
version = "0.31.0"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -3991,9 +3985,7 @@ dependencies = [
[[package]]
name = "lance-core"
version = "0.30.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4118c6e2ac2d26ff80e55708f337c4593381a32751f2a79a03d92452885bd648"
version = "0.31.0"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -4013,7 +4005,7 @@ dependencies = [
"mock_instant",
"moka",
"num_cpus",
"object_store 0.11.2",
"object_store",
"pin-project",
"prost",
"rand 0.8.5",
@@ -4029,9 +4021,7 @@ dependencies = [
[[package]]
name = "lance-datafusion"
version = "0.30.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ccf8b01e9a5f15d4975423ea1495df85cf36f9036c3ed999190d4631ffbd28b6"
version = "0.31.0"
dependencies = [
"arrow",
"arrow-array",
@@ -4060,9 +4050,7 @@ dependencies = [
[[package]]
name = "lance-datagen"
version = "0.30.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3fbedb84243fb2fe255b4e9ac298019d2e93e83fcc9ce2eb67a4ac7cab427dda"
version = "0.31.0"
dependencies = [
"arrow",
"arrow-array",
@@ -4077,9 +4065,7 @@ dependencies = [
[[package]]
name = "lance-encoding"
version = "0.30.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9a0e078414cce96da2e2b37290d0b38a81ba6b0ebcad6806b231c2cd8d04427a"
version = "0.31.0"
dependencies = [
"arrayref",
"arrow",
@@ -4113,14 +4099,13 @@ dependencies = [
"snafu",
"tokio",
"tracing",
"xxhash-rust",
"zstd",
]
[[package]]
name = "lance-file"
version = "0.30.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ce7deba5b59118f7ef726859ace192b7cc7da4e6639147d2a3908a2de621ce98"
version = "0.31.0"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -4141,7 +4126,7 @@ dependencies = [
"lance-io",
"log",
"num-traits",
"object_store 0.11.2",
"object_store",
"prost",
"prost-build",
"prost-types",
@@ -4154,9 +4139,7 @@ dependencies = [
[[package]]
name = "lance-index"
version = "0.30.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5bee1aecc60c759436d8f952e2d9c4e93d1940bfbdc1869068b4ac6b01e86b2f"
version = "0.31.0"
dependencies = [
"arrow",
"arrow-array",
@@ -4193,7 +4176,7 @@ dependencies = [
"log",
"moka",
"num-traits",
"object_store 0.11.2",
"object_store",
"prost",
"prost-build",
"rand 0.8.5",
@@ -4211,9 +4194,7 @@ dependencies = [
[[package]]
name = "lance-io"
version = "0.30.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "61a48f6a3f5433ca5095993fcd8bb47efbf473af852b9aca1e175a3d7bbf67fd"
version = "0.31.0"
dependencies = [
"arrow",
"arrow-arith",
@@ -4237,7 +4218,7 @@ dependencies = [
"lance-core",
"lazy_static",
"log",
"object_store 0.11.2",
"object_store",
"path_abs",
"pin-project",
"prost",
@@ -4252,9 +4233,7 @@ dependencies = [
[[package]]
name = "lance-linalg"
version = "0.30.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "620dedc792311862fc336b2651e825d2b450bbade7bfc819b7b182c3bb585c1e"
version = "0.31.0"
dependencies = [
"arrow-array",
"arrow-ord",
@@ -4277,9 +4256,7 @@ dependencies = [
[[package]]
name = "lance-table"
version = "0.30.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9b010312330943c5e81628722a50e3679688d96065348659b7913964f13765cf"
version = "0.31.0"
dependencies = [
"arrow",
"arrow-array",
@@ -4300,7 +4277,7 @@ dependencies = [
"lance-io",
"lazy_static",
"log",
"object_store 0.11.2",
"object_store",
"prost",
"prost-build",
"prost-types",
@@ -4318,9 +4295,7 @@ dependencies = [
[[package]]
name = "lance-testing"
version = "0.30.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "efa10957cdadef40e853896a67282cd29898775b29715eec42dd49bc3b3c8554"
version = "0.31.0"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -4378,7 +4353,7 @@ dependencies = [
"log",
"moka",
"num-traits",
"object_store 0.11.2",
"object_store",
"pin-project",
"polars",
"polars-arrow",
@@ -4435,7 +4410,7 @@ dependencies = [
"lancedb",
"lzma-sys",
"neon",
"object_store 0.11.2",
"object_store",
"once_cell",
"snafu",
"tokio",
@@ -5172,38 +5147,6 @@ dependencies = [
"memchr",
]
[[package]]
name = "object_store"
version = "0.11.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3cfccb68961a56facde1163f9319e0d15743352344e7808a11795fb99698dcaf"
dependencies = [
"async-trait",
"base64 0.22.1",
"bytes",
"chrono",
"futures",
"httparse",
"humantime",
"hyper 1.6.0",
"itertools 0.13.0",
"md-5",
"parking_lot",
"percent-encoding",
"quick-xml",
"rand 0.8.5",
"reqwest",
"ring",
"rustls-pemfile 2.2.0",
"serde",
"serde_json",
"snafu",
"tokio",
"tracing",
"url",
"walkdir",
]
[[package]]
name = "object_store"
version = "0.12.2"
@@ -5211,14 +5154,28 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7781f96d79ed0f961a7021424ab01840efbda64ae7a505aaea195efc91eaaec4"
dependencies = [
"async-trait",
"base64 0.22.1",
"bytes",
"chrono",
"form_urlencoded",
"futures",
"http 1.3.1",
"http-body-util",
"httparse",
"humantime",
"hyper 1.6.0",
"itertools 0.14.0",
"md-5",
"parking_lot",
"percent-encoding",
"quick-xml",
"rand 0.9.1",
"reqwest",
"ring",
"rustls-pemfile 2.2.0",
"serde",
"serde_json",
"serde_urlencoded",
"thiserror 2.0.12",
"tokio",
"tracing",

18
Cargo.toml
View File

@@ -21,14 +21,14 @@ categories = ["database-implementations"]
rust-version = "1.78.0"
[workspace.dependencies]
lance = { "version" = "=0.30.0", "features" = ["dynamodb"] }
lance-io = "=0.30.0"
lance-index = "=0.30.0"
lance-linalg = "=0.30.0"
lance-table = "=0.30.0"
lance-testing = "=0.30.0"
lance-datafusion = "=0.30.0"
lance-encoding = "=0.30.0"
lance = { path = "../lance/rust/lance", "features" = ["dynamodb"] }
lance-io = { path = "../lance/rust/lance-io" }
lance-index = { path = "../lance/rust/lance-index" }
lance-linalg = { path = "../lance/rust/lance-linalg" }
lance-table = { path = "../lance/rust/lance-table" }
lance-testing = { path = "../lance/rust/lance-testing" }
lance-datafusion = { path = "../lance/rust/lance-datafusion" }
lance-encoding = { path = "../lance/rust/lance-encoding" }
# Note that this one does not include pyarrow
arrow = { version = "55.1", optional = false }
arrow-array = "55.1"
@@ -52,7 +52,7 @@ half = { "version" = "=2.5.0", default-features = false, features = [
futures = "0"
log = "0.4"
moka = { version = "0.12", features = ["future"] }
object_store = "0.11.0"
object_store = "0.12.0"
pin-project = "1.0.7"
snafu = "0.8"
url = "2"

2
rust/lancedb/src/io/object_store.rs
View File

@@ -107,7 +107,7 @@ impl ObjectStore for MirroringObjectStore {
self.primary.delete(location).await
}
fn list(&self, prefix: Option<&Path>) -> BoxStream<'_, Result<ObjectMeta>> {
fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, Result<ObjectMeta>> {
self.primary.list(prefix)
}

8
rust/lancedb/src/io/object_store/io_tracking.rs
View File

@@ -133,7 +133,7 @@ impl ObjectStore for IoTrackingStore {
result
}
async fn get_range(&self, location: &Path, range: std::ops::Range<usize>) -> OSResult<Bytes> {
async fn get_range(&self, location: &Path, range: std::ops::Range<u64>) -> OSResult<Bytes> {
let result = self.target.get_range(location, range).await;
if let Ok(result) = &result {
self.record_read(result.len() as u64);
@@ -144,7 +144,7 @@ impl ObjectStore for IoTrackingStore {
async fn get_ranges(
&self,
location: &Path,
ranges: &[std::ops::Range<usize>],
ranges: &[std::ops::Range<u64>],
) -> OSResult<Vec<Bytes>> {
let result = self.target.get_ranges(location, ranges).await;
if let Ok(result) = &result {
@@ -170,7 +170,7 @@ impl ObjectStore for IoTrackingStore {
self.target.delete_stream(locations)
}
fn list(&self, prefix: Option<&Path>) -> BoxStream<'_, OSResult<ObjectMeta>> {
fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, OSResult<ObjectMeta>> {
self.record_read(0);
self.target.list(prefix)
}
@@ -179,7 +179,7 @@ impl ObjectStore for IoTrackingStore {
&self,
prefix: Option<&Path>,
offset: &Path,
) -> BoxStream<'_, OSResult<ObjectMeta>> {
) -> BoxStream<'static, OSResult<ObjectMeta>> {
self.record_read(0);
self.target.list_with_offset(prefix, offset)
}