diff --git a/Cargo.lock b/Cargo.lock index 3a2a4a8a3..8172c1b75 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3455,9 +3455,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "fsst" -version = "11.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f727719438dfdb74f358a347c91ff81b6e7084a6421f34de3e473ce271f10caa" +version = "12.0.0-beta.5" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259" dependencies = [ "arrow-array", "rand 0.9.5", @@ -4816,9 +4815,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a" [[package]] name = "lance" -version = "11.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be781f40c7a75f9eae2188a2f71174acb7a360dca97163db40b041d0828dea48" +version = "12.0.0-beta.5" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259" dependencies = [ "arc-swap", "arrow", @@ -4890,9 +4888,8 @@ dependencies = [ [[package]] name = "lance-arrow" -version = "11.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb97fd9875f3036d7c2561aa5b16eb87b80ccabaa4eeb5e6099b19cc662f1cd8" +version = "12.0.0-beta.5" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259" dependencies = [ "arrow-array", "arrow-buffer", @@ -4914,8 +4911,7 @@ dependencies = [ [[package]] name = "lance-arrow-scalar" version = "58.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "771f68b04b47f3addf781116f65061808de94b05e1e9411c23c18f32d14ebe79" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259" dependencies = [ "arrow-array", "arrow-buffer", @@ -4929,20 +4925,17 @@ dependencies = [ [[package]] name = "lance-arrow-stats" version = "58.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd47ec33c90bf29f688fd02118e37d3a5ad5c339caa3163f89e417dc0867001f" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259" dependencies = [ "arrow-array", "arrow-schema", - "half", "lance-arrow-scalar", ] [[package]] name = "lance-bitpacking" -version = "11.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f45658c5b2dc9aada41b66ee44b83af3fa888b7385ae414bae951b12a9f1cd3" +version = "12.0.0-beta.5" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259" dependencies = [ "arrayref", "crunchy", @@ -4952,9 +4945,8 @@ dependencies = [ [[package]] name = "lance-core" -version = "11.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27af3df3a7d08897efccd04461df31cedf0880c4b86a055ddce48e423d27f967" +version = "12.0.0-beta.5" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259" dependencies = [ "arrow-array", "arrow-buffer", @@ -4991,9 +4983,8 @@ dependencies = [ [[package]] name = "lance-datafusion" -version = "11.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c377f837df5296e92f9fad724c83c1bef4e74d5af6e5a9312e9307e1dead8614" +version = "12.0.0-beta.5" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259" dependencies = [ "arrow", "arrow-array", @@ -5022,9 +5013,8 @@ dependencies = [ [[package]] name = "lance-datagen" -version = "11.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "778e1a5065fa4bc184e36e32681f10f8f4680ad8cedc9377b4c088dce8c5b8da" +version = "12.0.0-beta.5" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259" dependencies = [ "arrow", "arrow-array", @@ -5041,9 +5031,8 @@ dependencies = [ [[package]] name = "lance-derive" -version = "11.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13e5e95e0fd3d74f7938f4bee623041421b323b5c61f242c8622a1f48a202527" +version = "12.0.0-beta.5" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259" dependencies = [ "proc-macro2", "quote", @@ -5052,9 +5041,8 @@ dependencies = [ [[package]] name = "lance-encoding" -version = "11.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1625653c55c65f3426e281f6e29b54c603f38a40bd4cebd707bd4f3ea48be6c5" +version = "12.0.0-beta.5" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259" dependencies = [ "arrow-arith", "arrow-array", @@ -5087,9 +5075,8 @@ dependencies = [ [[package]] name = "lance-file" -version = "11.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7e13c9266b478fc98f36ee19347c4658f7a6613fed77778b1a455fe1b88552e" +version = "12.0.0-beta.5" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259" dependencies = [ "arrow-arith", "arrow-array", @@ -5120,9 +5107,8 @@ dependencies = [ [[package]] name = "lance-index" -version = "11.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0e0cb95f2c4f341c4dd04ac60f6a89ea26a6f75e09570225cbda4854c8b088e" +version = "12.0.0-beta.5" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259" dependencies = [ "arc-swap", "arrow", @@ -5186,9 +5172,8 @@ dependencies = [ [[package]] name = "lance-index-core" -version = "11.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79ccd371977c1f7168da259d66ad37154f23146f093d46136bc7f79559f00f2c" +version = "12.0.0-beta.5" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259" dependencies = [ "arrow-array", "arrow-schema", @@ -5210,9 +5195,8 @@ dependencies = [ [[package]] name = "lance-io" -version = "11.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "414d50997391b1ac83dc183c1612fdff88f58b959806078dc4c5e465154566de" +version = "12.0.0-beta.5" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259" dependencies = [ "arrow", "arrow-array", @@ -5252,9 +5236,8 @@ dependencies = [ [[package]] name = "lance-linalg" -version = "11.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ff55b152ef23a56d7ba7e4d1b2c9cf0cc79aef6ee607c115597557ea4059f41" +version = "12.0.0-beta.5" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259" dependencies = [ "arrow-array", "arrow-schema", @@ -5268,9 +5251,8 @@ dependencies = [ [[package]] name = "lance-namespace" -version = "11.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09991c13ab282b731e323619613914e08da9cc82b312f904e58c128b23f2f0e3" +version = "12.0.0-beta.5" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259" dependencies = [ "arrow", "async-trait", @@ -5282,9 +5264,8 @@ dependencies = [ [[package]] name = "lance-namespace-impls" -version = "11.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ec0bc005f6bb8f120774eb4a9ba02e10463d8338167a46cbb1391d46680a174" +version = "12.0.0-beta.5" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259" dependencies = [ "arrow", "arrow-ipc", @@ -5337,9 +5318,8 @@ dependencies = [ [[package]] name = "lance-select" -version = "11.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8f676a2a1837cc85b77feb5326d3296827da964e40d67144f646563302a6ce9" +version = "12.0.0-beta.5" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259" dependencies = [ "arrow-array", "arrow-buffer", @@ -5353,9 +5333,8 @@ dependencies = [ [[package]] name = "lance-table" -version = "11.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd33054347395048b1d842dfb85a13f7801392c2da5f39425db62f00a481744b" +version = "12.0.0-beta.5" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259" dependencies = [ "arrow", "arrow-array", @@ -5395,9 +5374,8 @@ dependencies = [ [[package]] name = "lance-testing" -version = "11.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecc9ad9ae24f045dfddd538a39e55a28fa7e1ca6ad9f23e20d4087eaf2bb66f7" +version = "12.0.0-beta.5" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259" dependencies = [ "arrow-array", "arrow-schema", @@ -5410,9 +5388,8 @@ dependencies = [ [[package]] name = "lance-tokenizer" -version = "11.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3bfa6f0164c8b7056150f5682ce4d415a335b59b04c479873fda04b200117d27" +version = "12.0.0-beta.5" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259" dependencies = [ "frostem", "icu_segmenter", diff --git a/Cargo.toml b/Cargo.toml index 276658157..033da5907 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,20 +13,20 @@ categories = ["database-implementations"] rust-version = "1.91.0" [workspace.dependencies] -lance = { "version" = "=11.0.0", default-features = false } -lance-core = "=11.0.0" -lance-datagen = "=11.0.0" -lance-file = "=11.0.0" -lance-io = { "version" = "=11.0.0", default-features = false } -lance-index = "=11.0.0" -lance-linalg = "=11.0.0" -lance-namespace = "=11.0.0" -lance-namespace-impls = { "version" = "=11.0.0", default-features = false } -lance-table = "=11.0.0" -lance-testing = "=11.0.0" -lance-datafusion = "=11.0.0" -lance-encoding = "=11.0.0" -lance-arrow = "=11.0.0" +lance = { "version" = "=12.0.0-beta.5", default-features = false, "tag" = "v12.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" } +lance-core = { "version" = "=12.0.0-beta.5", "tag" = "v12.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" } +lance-datagen = { "version" = "=12.0.0-beta.5", "tag" = "v12.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" } +lance-file = { "version" = "=12.0.0-beta.5", "tag" = "v12.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" } +lance-io = { "version" = "=12.0.0-beta.5", default-features = false, "tag" = "v12.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" } +lance-index = { "version" = "=12.0.0-beta.5", "tag" = "v12.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" } +lance-linalg = { "version" = "=12.0.0-beta.5", "tag" = "v12.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace = { "version" = "=12.0.0-beta.5", "tag" = "v12.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace-impls = { "version" = "=12.0.0-beta.5", default-features = false, "tag" = "v12.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" } +lance-table = { "version" = "=12.0.0-beta.5", "tag" = "v12.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" } +lance-testing = { "version" = "=12.0.0-beta.5", "tag" = "v12.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" } +lance-datafusion = { "version" = "=12.0.0-beta.5", "tag" = "v12.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" } +lance-encoding = { "version" = "=12.0.0-beta.5", "tag" = "v12.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" } +lance-arrow = { "version" = "=12.0.0-beta.5", "tag" = "v12.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" } lancedb = { path = "rust/lancedb", default-features = false } ahash = "0.8" # Note that this one does not include pyarrow diff --git a/java/pom.xml b/java/pom.xml index 3b0b84667..c5481e022 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -28,7 +28,7 @@ UTF-8 15.0.0 - 11.0.0 + 12.0.0-beta.5 false 2.30.0 1.7 diff --git a/rust/lancedb/src/database/listing.rs b/rust/lancedb/src/database/listing.rs index 71e4016dd..c22b73dd7 100644 --- a/rust/lancedb/src/database/listing.rs +++ b/rust/lancedb/src/database/listing.rs @@ -13,7 +13,7 @@ use lance::dataset::{ReadParams, WriteMode, builder::DatasetBuilder}; use lance::io::{ObjectStore, ObjectStoreParams, WrappingObjectStore}; use lance_datafusion::utils::StreamingWriteSource; use lance_file::version::LanceFileVersion; -use lance_io::object_store::{StorageOptionsAccessor, StorageOptionsProvider}; +use lance_io::object_store::{ReadDirOptions, StorageOptionsAccessor, StorageOptionsProvider}; use lance_table::io::commit::commit_handler_from_url; use object_store::local::LocalFileSystem; use snafu::ResultExt; @@ -282,11 +282,14 @@ impl std::fmt::Display for ListingDatabase { const LANCE_EXTENSION: &str = "lance"; -/// The table a listed child directory holds, or `None` if it is not a table at all. +/// The table a listed child of the database names, or `None` if the child is not a table. /// /// A table is the directory `.lance`; a loose file or any other directory under the /// database prefix belongs to something else. `dir_suffix` is `.lance`, built once by the /// caller rather than per child. +/// The table a listed child directory holds, or `None` if it is not a table at all. +/// +/// Only directories are considered, so a loose object named like a table is not one. fn table_name(location: &object_store::path::Path, dir_suffix: &str) -> Option { location .filename()? @@ -294,75 +297,6 @@ fn table_name(location: &object_store::path::Path, dir_suffix: &str) -> Option, - /// Resumes after this page, or `None` when the page reached the end of the level. - page_token: Option, -} - -/// Where a listed location sits inside the database directory — the space page tokens live -/// in — or `None` if it is not a child of that directory at all. Matching both halves of the -/// prefix drops a location that merely starts with the directory's name (`dbx/y` against -/// `db/`) as well as the marker object some stores keep for the directory itself. -fn relative_key<'a>(prefix: Option<&str>, location: &'a str) -> Option<&'a str> { - let relative = match prefix { - Some(prefix) => location.strip_prefix(prefix)?, - None => location, - }; - (!relative.is_empty()).then_some(relative) -} - -/// One page of the table directories under `base_path`, one directory level deep. -/// -/// Lance 11 exposes no paginated directory listing, so the level is listed in full and paged -/// locally: table directories go into key order (a directory's key keeps its trailing `/`, -/// so a token is never a table name), the page is the smallest `limit` of them past -/// `page_token`, and the token handed back is the key of the last directory the page took — -/// so a page that took nothing ends the listing rather than resuming from a position no page -/// ever reached. Only `.lance/` directories enter the page: loose objects, other -/// directories, and a bare `.lance/` never take a page slot or name a token, which keeps a -/// page to exactly one listing of the level. Correct on every store, at the cost of that one -/// full-level listing per page. -async fn read_dir_page( - object_store: &ObjectStore, - base_path: &object_store::path::Path, - page_token: Option, - limit: Option, -) -> Result { - let listed = object_store.list_with_delimiter(Some(base_path)).await?; - let prefix = { - let base = base_path.as_ref(); - (!base.is_empty()).then(|| format!("{base}/")) - }; - let table_dir_suffix = format!(".{LANCE_EXTENSION}/"); - let mut children: Vec<(String, object_store::path::Path)> = listed - .common_prefixes - .into_iter() - .filter_map(|location| { - let key = format!("{}/", relative_key(prefix.as_deref(), location.as_ref())?); - (key.len() > table_dir_suffix.len() && key.ends_with(&table_dir_suffix)) - .then_some((key, location)) - }) - .collect(); - children.sort_unstable_by(|(left, _), (right, _)| left.cmp(right)); - if let Some(resume) = &page_token { - children.retain(|(key, _)| key > resume); - } - let total = children.len(); - children.truncate(limit.unwrap_or(total).min(total)); - let page_token = match children.last() { - Some((last, _)) if children.len() < total => Some(last.clone()), - _ => None, - }; - Ok(DirPage { - common_prefixes: children.into_iter().map(|(_, location)| location).collect(), - page_token, - }) -} - const ENGINE: &str = "engine"; const MIRRORED_STORE: &str = "mirroredStore"; @@ -1048,7 +982,8 @@ impl Database for ListingDatabase { let mut tables = Vec::new(); let mut page_token = request.page_token.filter(|token| !token.is_empty()); - // A page of nothing: no table was handed over for a token to resume after. + // A page of nothing: the store rejects a limit of zero, and no table was handed over + // for a token to resume after. if limit == Some(0) { return Ok(ListTablesResponse { context: None, @@ -1057,21 +992,35 @@ impl Database for ListingDatabase { }); } - // The page holds only table directories, so one call — and the one full-level - // listing behind it — fills it. - let page = read_dir_page( - &self.object_store, - &self.base_path, - page_token.take(), - limit, - ) - .await?; - page_token = page.page_token; - tables.extend( - page.common_prefixes - .iter() - .filter_map(|location| table_name(location, &dir_suffix)), - ); + loop { + // Ask only for what the page still has room for, so a database holding more + // than one page costs one request per page rather than one per table. + let listing = self + .object_store + .read_dir_page( + self.base_path.clone(), + ReadDirOptions { + page_token: page_token.take(), + limit: limit.map(|limit| limit - tables.len()), + }, + ) + .await?; + page_token = listing.page_token; + // Only child directories can be tables, and the store already separates them + // out, so the objects in the page are not looked at. + tables.extend( + listing + .result + .common_prefixes + .iter() + .filter_map(|location| table_name(location, &dir_suffix)), + ); + // Children that are not tables leave the page short of the limit, so keep + // going until the page is full or the database runs out. + if page_token.is_none() || limit.is_none_or(|limit| tables.len() >= limit) { + break; + } + } Ok(ListTablesResponse { context: None, @@ -1717,8 +1666,8 @@ mod tests { } /// Only directories named `.lance` are tables; loose files and other directories - /// under the database prefix are not. They never take a page slot, so even a `limit` - /// smaller than the clutter ahead of the first table returns that table. + /// under the database prefix are not. A page spent on them is filled from the next one, + /// so a page holding only non-tables does not read as an empty database. #[tokio::test] async fn test_listing_ignores_non_table_children() { let (tempdir, db) = setup_database().await; @@ -1737,37 +1686,6 @@ mod tests { assert_eq!(page.tables, vec!["real"]); } - /// The Lance 11 fallback pages locally over one full-level listing, so a bounded page - /// costs exactly one listing call — clutter ahead of the first table must not buy extra - /// round trips. - #[tokio::test] - async fn test_one_full_listing_per_public_page() { - use crate::io::object_store::io_tracking::IoStatsHolder; - use lance_io::object_store::WrappingObjectStore; - - let (tempdir, mut db) = setup_database().await; - create_tables(&db, &["real"]).await; - std::fs::write(tempdir.path().join("aaa-loose.lance"), b"not a table").unwrap(); - create_dir_all(tempdir.path().join("aaa-scratch")).unwrap(); - - let io_stats = IoStatsHolder::default(); - let mut tracked_store = (*db.object_store).clone(); - tracked_store.inner = - io_stats.wrap(&tracked_store.store_prefix, tracked_store.inner.clone()); - db.object_store = Arc::new(tracked_store); - - let page = db - .list_tables(ListTablesRequest { - limit: Some(1), - ..Default::default() - }) - .await - .unwrap(); - - assert_eq!(page.tables, vec!["real"]); - assert_eq!(io_stats.incremental_stats().read_iops, 1); - } - #[tokio::test] async fn listing_ignores_empty_table_name() { let (tempdir, db) = setup_database().await; diff --git a/rust/lancedb/src/io/object_store.rs b/rust/lancedb/src/io/object_store.rs index d594bd857..c4a9a4f7e 100644 --- a/rust/lancedb/src/io/object_store.rs +++ b/rust/lancedb/src/io/object_store.rs @@ -10,7 +10,7 @@ use lance::io::WrappingObjectStore; use object_store::{ CopyOptions, Error, GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta, ObjectStore, ObjectStoreExt, PutMultipartOptions, PutOptions, PutPayload, PutResult, Result, - UploadPart, path::Path, + UploadPart, list::PaginatedListStore, path::Path, }; use async_trait::async_trait; @@ -187,6 +187,14 @@ impl WrappingObjectStore for MirroringObjectStoreWrapper { secondary: self.secondary.clone(), }) } + + fn wrap_paginated( + &self, + _store_prefix: &str, + original: Arc, + ) -> Option> { + Some(original) + } } // windows pathing can't be simply concatenated diff --git a/rust/lancedb/src/io/object_store/io_tracking.rs b/rust/lancedb/src/io/object_store/io_tracking.rs index bd4f8f54a..7f9750216 100644 --- a/rust/lancedb/src/io/object_store/io_tracking.rs +++ b/rust/lancedb/src/io/object_store/io_tracking.rs @@ -12,7 +12,7 @@ use lance::io::WrappingObjectStore; use object_store::{ CopyOptions, GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta, ObjectStore, PutMultipartOptions, PutOptions, PutPayload, PutResult, RenameOptions, Result as OSResult, - UploadPart, path::Path, + UploadPart, list::PaginatedListStore, path::Path, }; #[derive(Debug, Default)] @@ -57,6 +57,14 @@ impl WrappingObjectStore for IoStatsHolder { stats: self.0.clone(), }) } + + fn wrap_paginated( + &self, + _store_prefix: &str, + original: Arc, + ) -> Option> { + Some(original) + } } impl IoTrackingStore { diff --git a/rust/lancedb/src/table.rs b/rust/lancedb/src/table.rs index d152f3616..efc705e3c 100644 --- a/rust/lancedb/src/table.rs +++ b/rust/lancedb/src/table.rs @@ -4183,6 +4183,14 @@ mod tests { parent_list_calls: self.parent_list_calls.clone(), }) } + + fn wrap_paginated( + &self, + _store_prefix: &str, + _original: Arc, + ) -> Option> { + None + } } #[tokio::test] @@ -4286,6 +4294,14 @@ mod tests { self.called.store(true, Ordering::Relaxed); original } + + fn wrap_paginated( + &self, + _store_prefix: &str, + original: Arc, + ) -> Option> { + Some(original) + } } #[tokio::test] diff --git a/rust/lancedb/src/table/query/lsm.rs b/rust/lancedb/src/table/query/lsm.rs index 07ea7fb81..86c1fe5f2 100644 --- a/rust/lancedb/src/table/query/lsm.rs +++ b/rust/lancedb/src/table/query/lsm.rs @@ -300,7 +300,7 @@ async fn build_read_context( for shard_id in shard_ids { let manifest_store = ShardManifestStore::new(store.clone(), &base_path, shard_id, scan_batch_size); - if let Some(manifest) = manifest_store.read_latest().await? { + if let Some(manifest) = manifest_store.latest().await? { snapshots.push(snapshot_from_manifest(shard_id, &manifest, &exclude)); } }