test: update hash-split expectations for DataFusion 54 foldhash

DataFusion 54's `create_hashes` uses foldhash instead of ahash, so the
concrete hash values (and therefore split assignments) differ from the
DF53 baseline:

- `test_hash_split` (Rust): recompute the expected per-split counts.
- `test_split_hash_with_discard` (Python): hash a high-cardinality
  column instead of the 2-value `category`, so the discard ratio no
  longer hinges on where two specific hashes land.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Will Jones
2026-07-14 21:23:10 -07:00
parent 4d6d9082e7
commit 5d3f06c44a
2 changed files with 8 additions and 5 deletions
+4 -1
View File
@@ -134,8 +134,11 @@ def test_split_hash_with_discard(mem_db):
)
permutation_tbl = (
# Hash a high-cardinality column: "category" has only two distinct
# values, so whether anything is discarded would hinge on where those
# two hashes land rather than on the discard weight.
permutation_builder(tbl)
.split_hash(["category"], [1, 1], discard_weight=2) # Should discard ~50%
.split_hash(["id"], [1, 1], discard_weight=2) # Should discard ~50%
.execute()
)
@@ -796,15 +796,15 @@ mod tests {
// These assertions are all based on fixed seed in data generation but they match
// up roughly to what we expect (25% discarded, 25% in split 0, 50% in split 1)
// 14 rows (28%) are discarded because discard_weight is 1
assert_eq!(split_batch.num_rows(), 36);
// 8 rows (16%) are discarded because discard_weight is 1
assert_eq!(split_batch.num_rows(), 42);
assert_eq!(split_batch.num_columns(), 2);
let split_ids = split_batch.column(1).as_primitive::<UInt64Type>().values();
let num_in_split_0 = split_ids.iter().filter(|v| **v == 0).count();
let num_in_split_1 = split_ids.iter().filter(|v| **v == 1).count();
assert_eq!(num_in_split_0, 11); // 22%
assert_eq!(num_in_split_1, 25); // 50%
assert_eq!(num_in_split_0, 12); // 24%
assert_eq!(num_in_split_1, 30); // 60%
}
}