From 7b65ad922d0a573d7a1c92066dfd76953deb57e7 Mon Sep 17 00:00:00 2001 From: PSeitz Date: Wed, 16 Oct 2024 11:41:33 +0800 Subject: [PATCH] use binggan for stacker bench (#2492) * use binggan for stacker bench ``` alice (num terms: 174693) hashmap Memory: 1.3 MB Avg: 367.19 MiB/s (-1.34%) Median: 368.10 MiB/s (-1.34%) [378.75 MiB/s .. 352.81 MiB/s] hasmap with postings Memory: 2.4 MB Avg: 237.29 MiB/s (-2.19%) Median: 240.22 MiB/s (-1.61%) [248.26 MiB/s .. 210.66 MiB/s] fxhashmap ref postings Memory: 2.9 MB Avg: 171.94 MiB/s (-3.22%) Median: 174.13 MiB/s (-2.69%) [185.94 MiB/s .. 152.43 MiB/s] fxhasmap owned postings Memory: 3.5 MB Avg: 96.993 MiB/s (-4.20%) Median: 97.410 MiB/s (-4.48%) [102.78 MiB/s .. 82.745 MiB/s] numbers unique 100k hashmap Memory: 5.2 MB Avg: 334.17 MiB/s (-3.06%) Median: 352.61 MiB/s (+0.77%) [362.60 MiB/s .. 213.03 MiB/s] hasmap with postings Memory: 6.3 MB Avg: 316.96 MiB/s (-0.02%) Median: 325.16 MiB/s (-0.04%) [338.36 MiB/s .. 218.60 MiB/s] zipfs numbers 100k hashmap Memory: 1.3 MB Avg: 1.2342 GiB/s (+2.87%) Median: 1.2677 GiB/s (+4.66%) [1.3130 GiB/s .. 915.93 MiB/s] hasmap with postings Memory: 2.4 MB Avg: 485.16 MiB/s (+2.68%) Median: 494.70 MiB/s (+4.42%) [505.31 MiB/s .. 413.14 MiB/s] numbers unique 1mio hashmap Memory: 35.7 MB Avg: 169.68 MiB/s (-1.08%) Median: 166.80 MiB/s (-3.87%) [201.33 MiB/s .. 154.26 MiB/s] hasmap with postings Memory: 39.8 MB Avg: 149.49 MiB/s (-3.07%) Median: 150.85 MiB/s (-1.45%) [160.76 MiB/s .. 130.94 MiB/s] zipfs numbers 1mio hashmap Memory: 1.3 MB Avg: 1.2185 GiB/s (-2.33%) Median: 1.2291 GiB/s (-2.33%) [1.2905 GiB/s .. 1.0742 GiB/s] hasmap with postings Memory: 5.5 MB Avg: 358.43 MiB/s (-11.63%) Median: 356.95 MiB/s (-12.85%) [444.94 MiB/s .. 302.46 MiB/s] numbers unique 2mio hashmap Memory: 70.3 MB Avg: 163.65 MiB/s (+8.37%) Median: 162.83 MiB/s (+8.80%) [190.20 MiB/s .. 144.70 MiB/s] hasmap with postings Memory: 78.6 MB Avg: 148.00 MiB/s (+7.75%) Median: 151.53 MiB/s (+9.11%) [166.92 MiB/s .. 120.09 MiB/s] zipfs numbers 2mio hashmap Memory: 1.3 MB Avg: 1.2535 GiB/s (+2.59%) Median: 1.2654 GiB/s (+0.36%) [1.2938 GiB/s .. 1.0592 GiB/s] hasmap with postings Memory: 9.7 MB Avg: 377.96 MiB/s (-4.94%) Median: 381.82 MiB/s (-3.67%) [426.14 MiB/s .. 335.66 MiB/s] numbers unique 5mio hashmap Memory: 277.9 MB Avg: 121.30 MiB/s (+2.00%) Median: 121.99 MiB/s (+2.99%) [132.51 MiB/s .. 110.32 MiB/s] hasmap with postings Memory: 295.7 MB Avg: 114.23 MiB/s (+2.13%) Median: 115.26 MiB/s (+2.94%) [124.08 MiB/s .. 103.38 MiB/s] zipfs numbers 5mio hashmap Memory: 1.3 MB Avg: 1.2326 GiB/s (+0.63%) Median: 1.2400 GiB/s (+0.71%) [1.2755 GiB/s .. 1.0923 GiB/s] hasmap with postings Memory: 25.4 MB Avg: 360.49 MiB/s (+1.07%) Median: 363.44 MiB/s (+1.27%) [404.88 MiB/s .. 300.38 MiB/s] ``` * rename bench * update binggan * rename to HASHMAP_CAPACITY --- stacker/Cargo.toml | 7 +- stacker/benches/bench.rs | 206 ++++++++++++++++++++++++++++++++++ stacker/benches/crit_bench.rs | 175 ----------------------------- 3 files changed, 210 insertions(+), 178 deletions(-) create mode 100644 stacker/benches/bench.rs delete mode 100644 stacker/benches/crit_bench.rs diff --git a/stacker/Cargo.toml b/stacker/Cargo.toml index 1702940cd..c9eec6dc3 100644 --- a/stacker/Cargo.toml +++ b/stacker/Cargo.toml @@ -13,10 +13,11 @@ common = { version = "0.7", path = "../common/", package = "tantivy-common" } ahash = { version = "0.8.11", default-features = false, optional = true } rand_distr = "0.4.3" + [[bench]] harness = false -name = "crit_bench" -path = "benches/crit_bench.rs" +name = "bench" +path = "benches/bench.rs" [[example]] name = "hashmap" @@ -25,9 +26,9 @@ path = "example/hashmap.rs" [dev-dependencies] rand = "0.8.5" zipf = "7.0.0" -criterion = { git = "https://github.com/PSeitz/criterion.rs/", rev = "e6f98ee"} # This fork includes stack randomization to reduce caching effects rustc-hash = "1.1.0" proptest = "1.2.0" +binggan = { version = "0.12.0" } [features] compare_hash_only = ["ahash"] # Compare hash only, not the key in the Hashmap diff --git a/stacker/benches/bench.rs b/stacker/benches/bench.rs new file mode 100644 index 000000000..346586969 --- /dev/null +++ b/stacker/benches/bench.rs @@ -0,0 +1,206 @@ +use binggan::plugins::PeakMemAllocPlugin; +use binggan::{black_box, BenchRunner, PeakMemAlloc, INSTRUMENTED_SYSTEM}; +use rand::SeedableRng; +use rustc_hash::FxHashMap; +use tantivy_stacker::{ArenaHashMap, ExpUnrolledLinkedList, MemoryArena}; + +const ALICE: &str = include_str!("../../benches/alice.txt"); + +#[global_allocator] +pub static GLOBAL: &PeakMemAlloc = &INSTRUMENTED_SYSTEM; + +fn bench_vint() { + let mut runner = BenchRunner::new(); + // Set the peak mem allocator. This will enable peak memory reporting. + runner.add_plugin(PeakMemAllocPlugin::new(GLOBAL)); + + { + let input_bytes = ALICE.len(); + + let alice_terms_as_bytes: Vec<&[u8]> = ALICE + .split_ascii_whitespace() + .map(|el| el.as_bytes()) + .collect(); + + let alice_terms_as_bytes_with_docid: Vec<(u32, &[u8])> = ALICE + .split_ascii_whitespace() + .map(|el| el.as_bytes()) + .enumerate() + .map(|(docid, el)| (docid as u32, el)) + .collect(); + + // Alice benchmark + let mut group = runner.new_group(); + group.set_name(format!("alice (num terms: {})", ALICE.len())); + group.set_input_size(input_bytes); + group.register_with_input("hashmap", &alice_terms_as_bytes, move |data| { + black_box(create_hash_map(data.iter())); + Some(()) + }); + group.register_with_input( + "hasmap with postings", + &alice_terms_as_bytes_with_docid, + move |data| { + black_box(create_hash_map_with_expull(data.iter().cloned())); + Some(()) + }, + ); + group.register_with_input( + "fxhashmap ref postings", + &alice_terms_as_bytes, + move |data| { + black_box(create_fx_hash_ref_map_with_expull(data.iter().cloned())); + Some(()) + }, + ); + group.register_with_input( + "fxhasmap owned postings", + &alice_terms_as_bytes, + move |data| { + black_box(create_fx_hash_owned_map_with_expull(data.iter().cloned())); + Some(()) + }, + ); + group.run(); + } + + { + for (num_numbers, num_numbers_label) in [ + (100_000u64, "100k"), + (1_000_000, "1mio"), + (2_000_000, "2mio"), + (5_000_000, "5mio"), + ] { + // benchmark unique numbers + { + let numbers: Vec<[u8; 8]> = (0..num_numbers).map(|el| el.to_le_bytes()).collect(); + let numbers_with_doc: Vec<_> = numbers + .iter() + .enumerate() + .map(|(docid, el)| (docid as u32, el)) + .collect(); + + let input_bytes = numbers.len() * 8; + let mut group = runner.new_group(); + group.set_name(format!("numbers unique {}", num_numbers_label)); + group.set_input_size(input_bytes); + group.register_with_input("only hashmap", &numbers, move |data| { + black_box(create_hash_map(data.iter())); + Some(()) + }); + group.register_with_input("hasmap with postings", &numbers_with_doc, move |data| { + black_box(create_hash_map_with_expull(data.iter().cloned())); + Some(()) + }); + group.run(); + } + // benchmark zipfs distribution numbers + { + use rand::distributions::Distribution; + use rand::rngs::StdRng; + let mut rng = StdRng::from_seed([3u8; 32]); + let zipf = zipf::ZipfDistribution::new(10_000, 1.03).unwrap(); + let numbers: Vec<[u8; 8]> = (0..num_numbers) + .map(|_| zipf.sample(&mut rng).to_le_bytes()) + .collect(); + let numbers_with_doc: Vec<_> = numbers + .iter() + .enumerate() + .map(|(docid, el)| (docid as u32, el)) + .collect(); + + let input_bytes = numbers.len() * 8; + let mut group = runner.new_group(); + group.set_name(format!("zipfs numbers {}", num_numbers_label)); + group.set_input_size(input_bytes); + group.register_with_input("hashmap", &numbers, move |data| { + black_box(create_hash_map(data.iter())); + Some(()) + }); + group.register_with_input("hasmap with postings", &numbers_with_doc, move |data| { + black_box(create_hash_map_with_expull(data.iter().cloned())); + Some(()) + }); + group.run(); + } + } + } +} + +fn main() { + bench_vint(); +} + +const HASHMAP_CAPACITY: usize = 1 << 15; + +/// Only records the doc ids +#[derive(Clone, Default, Copy)] +pub struct DocIdRecorder { + stack: ExpUnrolledLinkedList, +} +impl DocIdRecorder { + fn new_doc(&mut self, doc: u32, arena: &mut MemoryArena) { + self.stack.writer(arena).write_u32_vint(doc); + } +} + +fn create_hash_map>(terms: impl Iterator) -> ArenaHashMap { + let mut map = ArenaHashMap::with_capacity(HASHMAP_CAPACITY); + for term in terms { + map.mutate_or_create(term.as_ref(), |val| { + if let Some(mut val) = val { + val += 1; + val + } else { + 1u64 + } + }); + } + + map +} + +fn create_hash_map_with_expull>( + terms: impl Iterator, +) -> ArenaHashMap { + let mut memory_arena = MemoryArena::default(); + let mut map = ArenaHashMap::with_capacity(HASHMAP_CAPACITY); + for (i, term) in terms { + map.mutate_or_create(term.as_ref(), |val: Option| { + if let Some(mut rec) = val { + rec.new_doc(i, &mut memory_arena); + rec + } else { + DocIdRecorder::default() + } + }); + } + + map +} + +fn create_fx_hash_ref_map_with_expull( + terms: impl Iterator, +) -> FxHashMap<&'static [u8], Vec> { + let terms = terms.enumerate(); + let mut map = FxHashMap::with_capacity_and_hasher(HASHMAP_CAPACITY, Default::default()); + for (i, term) in terms { + map.entry(term.as_ref()) + .or_insert_with(Vec::new) + .push(i as u32); + } + map +} + +fn create_fx_hash_owned_map_with_expull( + terms: impl Iterator, +) -> FxHashMap, Vec> { + let terms = terms.enumerate(); + let mut map = FxHashMap::with_capacity_and_hasher(HASHMAP_CAPACITY, Default::default()); + for (i, term) in terms { + map.entry(term.as_ref().to_vec()) + .or_insert_with(Vec::new) + .push(i as u32); + } + map +} diff --git a/stacker/benches/crit_bench.rs b/stacker/benches/crit_bench.rs deleted file mode 100644 index a7618883b..000000000 --- a/stacker/benches/crit_bench.rs +++ /dev/null @@ -1,175 +0,0 @@ -#![allow(dead_code)] -extern crate criterion; - -use criterion::*; -use rand::SeedableRng; -use rustc_hash::FxHashMap; -use tantivy_stacker::{ArenaHashMap, ExpUnrolledLinkedList, MemoryArena}; - -const ALICE: &str = include_str!("../../benches/alice.txt"); - -fn bench_hashmap_throughput(c: &mut Criterion) { - let plot_config = PlotConfiguration::default().summary_scale(AxisScale::Linear); - - let mut group = c.benchmark_group("CreateHashMap"); - group.plot_config(plot_config); - - let input_bytes = ALICE.len() as u64; - - let alice_terms_as_bytes: Vec<&[u8]> = ALICE - .split_ascii_whitespace() - .map(|el| el.as_bytes()) - .collect(); - - let alice_terms_as_bytes_with_docid: Vec<(u32, &[u8])> = ALICE - .split_ascii_whitespace() - .map(|el| el.as_bytes()) - .enumerate() - .map(|(docid, el)| (docid as u32, el)) - .collect(); - - group.throughput(Throughput::Bytes(input_bytes)); - - group.bench_with_input( - BenchmarkId::new("alice".to_string(), input_bytes), - &alice_terms_as_bytes, - |b, i| b.iter(|| create_hash_map(i.iter())), - ); - group.bench_with_input( - BenchmarkId::new("alice_expull".to_string(), input_bytes), - &alice_terms_as_bytes_with_docid, - |b, i| b.iter(|| create_hash_map_with_expull(i.iter().cloned())), - ); - - group.bench_with_input( - BenchmarkId::new("alice_fx_hashmap_ref_expull".to_string(), input_bytes), - &alice_terms_as_bytes, - |b, i| b.iter(|| create_fx_hash_ref_map_with_expull(i.iter().cloned())), - ); - - group.bench_with_input( - BenchmarkId::new("alice_fx_hashmap_owned_expull".to_string(), input_bytes), - &alice_terms_as_bytes, - |b, i| b.iter(|| create_fx_hash_owned_map_with_expull(i.iter().cloned())), - ); - - // numbers - let input_bytes = 1_000_000 * 8; - group.throughput(Throughput::Bytes(input_bytes)); - let numbers: Vec<[u8; 8]> = (0..1_000_000u64).map(|el| el.to_le_bytes()).collect(); - - group.bench_with_input( - BenchmarkId::new("numbers".to_string(), input_bytes), - &numbers, - |b, i| b.iter(|| create_hash_map(i.iter().cloned())), - ); - - let numbers_with_doc: Vec<_> = numbers - .iter() - .enumerate() - .map(|(docid, el)| (docid as u32, el)) - .collect(); - - group.bench_with_input( - BenchmarkId::new("ids_expull".to_string(), input_bytes), - &numbers_with_doc, - |b, i| b.iter(|| create_hash_map_with_expull(i.iter().cloned())), - ); - - // numbers zipf - use rand::distributions::Distribution; - use rand::rngs::StdRng; - let mut rng = StdRng::from_seed([3u8; 32]); - let zipf = zipf::ZipfDistribution::new(10_000, 1.03).unwrap(); - - let input_bytes = 1_000_000 * 8; - group.throughput(Throughput::Bytes(input_bytes)); - let zipf_numbers: Vec<[u8; 8]> = (0..1_000_000u64) - .map(|_| zipf.sample(&mut rng).to_le_bytes()) - .collect(); - - group.bench_with_input( - BenchmarkId::new("numbers_zipf".to_string(), input_bytes), - &zipf_numbers, - |b, i| b.iter(|| create_hash_map(i.iter().cloned())), - ); - - group.finish(); -} - -const HASHMAP_SIZE: usize = 1 << 15; - -/// Only records the doc ids -#[derive(Clone, Default, Copy)] -pub struct DocIdRecorder { - stack: ExpUnrolledLinkedList, -} -impl DocIdRecorder { - fn new_doc(&mut self, doc: u32, arena: &mut MemoryArena) { - self.stack.writer(arena).write_u32_vint(doc); - } -} - -fn create_hash_map>(terms: impl Iterator) -> ArenaHashMap { - let mut map = ArenaHashMap::with_capacity(HASHMAP_SIZE); - for term in terms { - map.mutate_or_create(term.as_ref(), |val| { - if let Some(mut val) = val { - val += 1; - val - } else { - 1u64 - } - }); - } - - map -} - -fn create_hash_map_with_expull>( - terms: impl Iterator, -) -> ArenaHashMap { - let mut memory_arena = MemoryArena::default(); - let mut map = ArenaHashMap::with_capacity(HASHMAP_SIZE); - for (i, term) in terms { - map.mutate_or_create(term.as_ref(), |val: Option| { - if let Some(mut rec) = val { - rec.new_doc(i, &mut memory_arena); - rec - } else { - DocIdRecorder::default() - } - }); - } - - map -} - -fn create_fx_hash_ref_map_with_expull( - terms: impl Iterator, -) -> FxHashMap<&'static [u8], Vec> { - let terms = terms.enumerate(); - let mut map = FxHashMap::with_capacity_and_hasher(HASHMAP_SIZE, Default::default()); - for (i, term) in terms { - map.entry(term.as_ref()) - .or_insert_with(Vec::new) - .push(i as u32); - } - map -} - -fn create_fx_hash_owned_map_with_expull( - terms: impl Iterator, -) -> FxHashMap, Vec> { - let terms = terms.enumerate(); - let mut map = FxHashMap::with_capacity_and_hasher(HASHMAP_SIZE, Default::default()); - for (i, term) in terms { - map.entry(term.as_ref().to_vec()) - .or_insert_with(Vec::new) - .push(i as u32); - } - map -} - -criterion_group!(block_benches, bench_hashmap_throughput,); -criterion_main!(block_benches);