test(rust): run the mirroring e2e test again through file-object-store

Closes #3059.

`io::object_store::test::test_e2e` has been `#[ignore]`d since Lance started writing local tables through `LocalWriter`. For a `file://` store, `ObjectStore::create()` returns a `LocalWriter` that writes straight to disk, so `MirroringObjectStoreWrapper` never sees the writes and the secondary stays empty.

Lance's `file-object-store://` scheme uses the same local provider, but `create()` only takes the `LocalWriter` path for `file`, so every other scheme writes through the wrapped store. This connects the test to `file-object-store://<dir1>` and drops the `#[ignore]`. The table files still land in `dir1`, so the existing file-list comparison covers the mirroring wrapper on real files again.

The two relative file lists are sorted before comparison so the assertion detects missing and extra files independently of `WalkDir` traversal order. The row and vector-query assertions remain unchanged.

Validation:
- Rust object-store module at `b11f7a9684d94100f40818d1249479c726b2b006`: 3 passed, with the focused `test_e2e` included.
- Hosted Linux test merge: `test_e2e` passed and the library reported 2046 passed.
- Complete applicable CI at the final head: 8 workflows and 39 successful checks passed; 2 conditional jobs were skipped.

Co-authored-by: Bilal Atique <bilalbinatique@gmail.com>
This commit is contained in:
Bilal Atique
2026-11-03 19:52:01 +08:00
committed by GitHub
co-authored by Bilal Atique
parent 8172298044
commit dc771eb94f
+11 -8
View File
@@ -345,11 +345,6 @@ mod test {
));
}
// This test is ignored because lance 3.0 introduced LocalWriter optimization
// that bypasses the object store wrapper for local writes. The mirroring feature
// still works for remote/cloud storage, but can't be tested with local storage.
// See lance commit c878af433 "perf: create local writer for efficient local writes"
#[ignore]
#[tokio::test]
async fn test_e2e() {
let dir1 = tempfile::tempdir().unwrap().keep().canonicalize().unwrap();
@@ -360,7 +355,13 @@ mod test {
secondary: Arc::new(secondary_store),
});
let db = connect(dir1.to_str().unwrap()).execute().await.unwrap();
// Lance writes `file://` tables through LocalWriter, which skips object store
// wrappers. `file-object-store://` writes the same local files through the
// wrapped store.
let db = connect(&format!("file-object-store://{}", dir1.display()))
.execute()
.await
.unwrap();
let mut param = WriteParams::default();
let store_params = ObjectStoreParams {
@@ -413,7 +414,7 @@ mod test {
|| path_str.contains("_transactions")
};
let primary_files: Vec<_> = WalkDir::new(&primary_location)
let mut primary_files: Vec<_> = WalkDir::new(&primary_location)
.into_iter()
.filter_entry(|e| !should_skip(e.path()))
.filter_map(|e| e.ok())
@@ -424,8 +425,9 @@ mod test {
.to_path_buf()
})
.collect();
primary_files.sort();
let secondary_files: Vec<_> = WalkDir::new(&secondary_location)
let mut secondary_files: Vec<_> = WalkDir::new(&secondary_location)
.into_iter()
.filter_entry(|e| !should_skip(e.path()))
.filter_map(|e| e.ok())
@@ -436,6 +438,7 @@ mod test {
.to_path_buf()
})
.collect();
secondary_files.sort();
assert_eq!(primary_files, secondary_files, "File lists should match");
}