mirror of
https://github.com/lancedb/lancedb.git
synced 2026-08-18 12:08:35 +00:00
test: update hash-split expectations for DataFusion 54 foldhash
DataFusion 54's `create_hashes` uses foldhash instead of ahash, so the concrete hash values (and therefore split assignments) differ from the DF53 baseline: - `test_hash_split` (Rust): recompute the expected per-split counts. - `test_split_hash_with_discard` (Python): hash a high-cardinality column instead of the 2-value `category`, so the discard ratio no longer hinges on where two specific hashes land. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -134,8 +134,11 @@ def test_split_hash_with_discard(mem_db):
|
||||
)
|
||||
|
||||
permutation_tbl = (
|
||||
# Hash a high-cardinality column: "category" has only two distinct
|
||||
# values, so whether anything is discarded would hinge on where those
|
||||
# two hashes land rather than on the discard weight.
|
||||
permutation_builder(tbl)
|
||||
.split_hash(["category"], [1, 1], discard_weight=2) # Should discard ~50%
|
||||
.split_hash(["id"], [1, 1], discard_weight=2) # Should discard ~50%
|
||||
.execute()
|
||||
)
|
||||
|
||||
|
||||
@@ -796,15 +796,15 @@ mod tests {
|
||||
// These assertions are all based on fixed seed in data generation but they match
|
||||
// up roughly to what we expect (25% discarded, 25% in split 0, 50% in split 1)
|
||||
|
||||
// 14 rows (28%) are discarded because discard_weight is 1
|
||||
assert_eq!(split_batch.num_rows(), 36);
|
||||
// 8 rows (16%) are discarded because discard_weight is 1
|
||||
assert_eq!(split_batch.num_rows(), 42);
|
||||
assert_eq!(split_batch.num_columns(), 2);
|
||||
|
||||
let split_ids = split_batch.column(1).as_primitive::<UInt64Type>().values();
|
||||
let num_in_split_0 = split_ids.iter().filter(|v| **v == 0).count();
|
||||
let num_in_split_1 = split_ids.iter().filter(|v| **v == 1).count();
|
||||
|
||||
assert_eq!(num_in_split_0, 11); // 22%
|
||||
assert_eq!(num_in_split_1, 25); // 50%
|
||||
assert_eq!(num_in_split_0, 12); // 24%
|
||||
assert_eq!(num_in_split_1, 30); // 60%
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user