From 5d3f06c44aa6c82291f81ebcd79503470180fe19 Mon Sep 17 00:00:00 2001 From: Will Jones Date: Tue, 14 Jul 2026 21:23:10 -0700 Subject: [PATCH] test: update hash-split expectations for DataFusion 54 foldhash DataFusion 54's `create_hashes` uses foldhash instead of ahash, so the concrete hash values (and therefore split assignments) differ from the DF53 baseline: - `test_hash_split` (Rust): recompute the expected per-split counts. - `test_split_hash_with_discard` (Python): hash a high-cardinality column instead of the 2-value `category`, so the discard ratio no longer hinges on where two specific hashes land. Co-Authored-By: Claude Opus 4.8 (1M context) --- python/python/tests/test_permutation.py | 5 ++++- rust/lancedb/src/dataloader/permutation/split.rs | 8 ++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/python/python/tests/test_permutation.py b/python/python/tests/test_permutation.py index 0da12c7dc..7ff35e5e2 100644 --- a/python/python/tests/test_permutation.py +++ b/python/python/tests/test_permutation.py @@ -134,8 +134,11 @@ def test_split_hash_with_discard(mem_db): ) permutation_tbl = ( + # Hash a high-cardinality column: "category" has only two distinct + # values, so whether anything is discarded would hinge on where those + # two hashes land rather than on the discard weight. permutation_builder(tbl) - .split_hash(["category"], [1, 1], discard_weight=2) # Should discard ~50% + .split_hash(["id"], [1, 1], discard_weight=2) # Should discard ~50% .execute() ) diff --git a/rust/lancedb/src/dataloader/permutation/split.rs b/rust/lancedb/src/dataloader/permutation/split.rs index 202691bf1..b33544100 100644 --- a/rust/lancedb/src/dataloader/permutation/split.rs +++ b/rust/lancedb/src/dataloader/permutation/split.rs @@ -796,15 +796,15 @@ mod tests { // These assertions are all based on fixed seed in data generation but they match // up roughly to what we expect (25% discarded, 25% in split 0, 50% in split 1) - // 14 rows (28%) are discarded because discard_weight is 1 - assert_eq!(split_batch.num_rows(), 36); + // 8 rows (16%) are discarded because discard_weight is 1 + assert_eq!(split_batch.num_rows(), 42); assert_eq!(split_batch.num_columns(), 2); let split_ids = split_batch.column(1).as_primitive::().values(); let num_in_split_0 = split_ids.iter().filter(|v| **v == 0).count(); let num_in_split_1 = split_ids.iter().filter(|v| **v == 1).count(); - assert_eq!(num_in_split_0, 11); // 22% - assert_eq!(num_in_split_1, 25); // 50% + assert_eq!(num_in_split_0, 12); // 24% + assert_eq!(num_in_split_1, 30); // 60% } }