From dc771eb94fc933b965f281b5469b1b136429abe9 Mon Sep 17 00:00:00 2001 From: Bilal Atique <74999164+BilalAtique@users.noreply.github.com> Date: Tue, 3 Nov 2026 16:52:01 +0500 Subject: [PATCH] test(rust): run the mirroring e2e test again through file-object-store Closes #3059. `io::object_store::test::test_e2e` has been `#[ignore]`d since Lance started writing local tables through `LocalWriter`. For a `file://` store, `ObjectStore::create()` returns a `LocalWriter` that writes straight to disk, so `MirroringObjectStoreWrapper` never sees the writes and the secondary stays empty. Lance's `file-object-store://` scheme uses the same local provider, but `create()` only takes the `LocalWriter` path for `file`, so every other scheme writes through the wrapped store. This connects the test to `file-object-store://` and drops the `#[ignore]`. The table files still land in `dir1`, so the existing file-list comparison covers the mirroring wrapper on real files again. The two relative file lists are sorted before comparison so the assertion detects missing and extra files independently of `WalkDir` traversal order. The row and vector-query assertions remain unchanged. Validation: - Rust object-store module at `b11f7a9684d94100f40818d1249479c726b2b006`: 3 passed, with the focused `test_e2e` included. - Hosted Linux test merge: `test_e2e` passed and the library reported 2046 passed. - Complete applicable CI at the final head: 8 workflows and 39 successful checks passed; 2 conditional jobs were skipped. Co-authored-by: Bilal Atique --- rust/lancedb/src/io/object_store.rs | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/rust/lancedb/src/io/object_store.rs b/rust/lancedb/src/io/object_store.rs index a40694463..3c0c0fe58 100644 --- a/rust/lancedb/src/io/object_store.rs +++ b/rust/lancedb/src/io/object_store.rs @@ -345,11 +345,6 @@ mod test { )); } - // This test is ignored because lance 3.0 introduced LocalWriter optimization - // that bypasses the object store wrapper for local writes. The mirroring feature - // still works for remote/cloud storage, but can't be tested with local storage. - // See lance commit c878af433 "perf: create local writer for efficient local writes" - #[ignore] #[tokio::test] async fn test_e2e() { let dir1 = tempfile::tempdir().unwrap().keep().canonicalize().unwrap(); @@ -360,7 +355,13 @@ mod test { secondary: Arc::new(secondary_store), }); - let db = connect(dir1.to_str().unwrap()).execute().await.unwrap(); + // Lance writes `file://` tables through LocalWriter, which skips object store + // wrappers. `file-object-store://` writes the same local files through the + // wrapped store. + let db = connect(&format!("file-object-store://{}", dir1.display())) + .execute() + .await + .unwrap(); let mut param = WriteParams::default(); let store_params = ObjectStoreParams { @@ -413,7 +414,7 @@ mod test { || path_str.contains("_transactions") }; - let primary_files: Vec<_> = WalkDir::new(&primary_location) + let mut primary_files: Vec<_> = WalkDir::new(&primary_location) .into_iter() .filter_entry(|e| !should_skip(e.path())) .filter_map(|e| e.ok()) @@ -424,8 +425,9 @@ mod test { .to_path_buf() }) .collect(); + primary_files.sort(); - let secondary_files: Vec<_> = WalkDir::new(&secondary_location) + let mut secondary_files: Vec<_> = WalkDir::new(&secondary_location) .into_iter() .filter_entry(|e| !should_skip(e.path())) .filter_map(|e| e.ok()) @@ -436,6 +438,7 @@ mod test { .to_path_buf() }) .collect(); + secondary_files.sort(); assert_eq!(primary_files, secondary_files, "File lists should match"); }