mirror of
https://github.com/lancedb/lancedb.git
synced 2026-11-18 16:43:59 +00:00
test(rust): run the mirroring e2e test again through file-object-store
Closes #3059. `io::object_store::test::test_e2e` has been `#[ignore]`d since Lance started writing local tables through `LocalWriter`. For a `file://` store, `ObjectStore::create()` returns a `LocalWriter` that writes straight to disk, so `MirroringObjectStoreWrapper` never sees the writes and the secondary stays empty. Lance's `file-object-store://` scheme uses the same local provider, but `create()` only takes the `LocalWriter` path for `file`, so every other scheme writes through the wrapped store. This connects the test to `file-object-store://<dir1>` and drops the `#[ignore]`. The table files still land in `dir1`, so the existing file-list comparison covers the mirroring wrapper on real files again. The two relative file lists are sorted before comparison so the assertion detects missing and extra files independently of `WalkDir` traversal order. The row and vector-query assertions remain unchanged. Validation: - Rust object-store module at `b11f7a9684d94100f40818d1249479c726b2b006`: 3 passed, with the focused `test_e2e` included. - Hosted Linux test merge: `test_e2e` passed and the library reported 2046 passed. - Complete applicable CI at the final head: 8 workflows and 39 successful checks passed; 2 conditional jobs were skipped. Co-authored-by: Bilal Atique <bilalbinatique@gmail.com>
This commit is contained in:
co-authored by
Bilal Atique
parent
8172298044
commit
dc771eb94f
@@ -345,11 +345,6 @@ mod test {
|
||||
));
|
||||
}
|
||||
|
||||
// This test is ignored because lance 3.0 introduced LocalWriter optimization
|
||||
// that bypasses the object store wrapper for local writes. The mirroring feature
|
||||
// still works for remote/cloud storage, but can't be tested with local storage.
|
||||
// See lance commit c878af433 "perf: create local writer for efficient local writes"
|
||||
#[ignore]
|
||||
#[tokio::test]
|
||||
async fn test_e2e() {
|
||||
let dir1 = tempfile::tempdir().unwrap().keep().canonicalize().unwrap();
|
||||
@@ -360,7 +355,13 @@ mod test {
|
||||
secondary: Arc::new(secondary_store),
|
||||
});
|
||||
|
||||
let db = connect(dir1.to_str().unwrap()).execute().await.unwrap();
|
||||
// Lance writes `file://` tables through LocalWriter, which skips object store
|
||||
// wrappers. `file-object-store://` writes the same local files through the
|
||||
// wrapped store.
|
||||
let db = connect(&format!("file-object-store://{}", dir1.display()))
|
||||
.execute()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let mut param = WriteParams::default();
|
||||
let store_params = ObjectStoreParams {
|
||||
@@ -413,7 +414,7 @@ mod test {
|
||||
|| path_str.contains("_transactions")
|
||||
};
|
||||
|
||||
let primary_files: Vec<_> = WalkDir::new(&primary_location)
|
||||
let mut primary_files: Vec<_> = WalkDir::new(&primary_location)
|
||||
.into_iter()
|
||||
.filter_entry(|e| !should_skip(e.path()))
|
||||
.filter_map(|e| e.ok())
|
||||
@@ -424,8 +425,9 @@ mod test {
|
||||
.to_path_buf()
|
||||
})
|
||||
.collect();
|
||||
primary_files.sort();
|
||||
|
||||
let secondary_files: Vec<_> = WalkDir::new(&secondary_location)
|
||||
let mut secondary_files: Vec<_> = WalkDir::new(&secondary_location)
|
||||
.into_iter()
|
||||
.filter_entry(|e| !should_skip(e.path()))
|
||||
.filter_map(|e| e.ok())
|
||||
@@ -436,6 +438,7 @@ mod test {
|
||||
.to_path_buf()
|
||||
})
|
||||
.collect();
|
||||
secondary_files.sort();
|
||||
|
||||
assert_eq!(primary_files, secondary_files, "File lists should match");
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user