diff --git a/Cargo.toml b/Cargo.toml index 16e6ee408..436971b56 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,9 +6,8 @@ build = "build.rs" [dependencies] -byteorder = "0.4.2" -itertools = "0.4.5" -memmap = "0.2.3" +byteorder = "0.4" +memmap = "0.2" lazy_static = "0.1.*" regex = "0.1" fst = "0.1.27" diff --git a/src/core/simdcompression.rs b/src/core/simdcompression.rs index 734a79ef1..1d43d4fbf 100644 --- a/src/core/simdcompression.rs +++ b/src/core/simdcompression.rs @@ -2,6 +2,8 @@ use libc::size_t; use std::ptr; + + extern { fn encode_native(data: *mut u32, num_els: size_t, output: *mut u32, output_capacity: size_t) -> size_t; fn decode_native(compressed_data: *const u32, compressed_size: size_t, uncompressed: *mut u32, output_capacity: size_t) -> size_t; @@ -82,3 +84,37 @@ fn test_encode_big() { assert_eq!(num_ints, decoder.decode(&encoded_data[..], &mut decoded_data)); assert_eq!(decoded_data, input); } + + +#[cfg(test)] +mod tests { + + + use super::*; + use test::Bencher; + use rand::Rng; + use rand::SeedableRng; + use rand::StdRng; + + fn generate_array(n: usize, ratio: f32) -> Vec { + let seed: &[_] = &[1, 2, 3, 4]; + let mut rng: StdRng = SeedableRng::from_seed(seed); + (0..u32::max_value()) + .filter(|_| rng.next_f32()< ratio) + .take(n) + .collect() + } + + #[bench] + fn bench_decode(b: &mut Bencher) { + const TEST_SIZE: usize = 100_000; + let arr = generate_array(TEST_SIZE, 0.1); + let mut encoder = Encoder::new(); + let encoded = encoder.encode(&arr); + let mut uncompressed: Vec = (0..TEST_SIZE as u32).collect(); + let decoder = Decoder; + b.iter(|| { + decoder.decode(&encoded, &mut uncompressed); + }); + } +} diff --git a/src/core/store.rs b/src/core/store.rs index 844d99ada..4fbd6a699 100644 --- a/src/core/store.rs +++ b/src/core/store.rs @@ -174,24 +174,27 @@ impl StoreReader { #[cfg(test)] mod tests { + use super::*; + use test::Bencher; + use rand::Rng; + use rand::SeedableRng; + use rand::StdRng; + use std::io::Write; use tempfile; use core::schema::Schema; use core::schema::FieldOptions; use core::schema::FieldValue; use fst::raw::MmapReadOnly; - use core::store::StoreWriter; - use core::store::StoreReader; + use std::fs::File; - #[test] - fn test_store() { - let offsets; - let store_file = tempfile::NamedTempFile::new().unwrap(); + + fn write_lorem_ipsum_store(store_file: File) -> Schema { let mut schema = Schema::new(); let field_body = schema.add_field("body", &FieldOptions::new().set_stored()); let field_title = schema.add_field("title", &FieldOptions::new().set_stored()); let lorem = String::from("Doc Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum."); { - let mut store_writer = StoreWriter::new(store_file.reopen().unwrap()); + let mut store_writer = StoreWriter::new(store_file); for i in 0..1000 { let mut fields: Vec = Vec::new(); { @@ -213,14 +216,41 @@ mod tests { store_writer.store(&fields_refs); } store_writer.close(); - offsets = store_writer.offsets.clone(); } + schema + } + #[test] + + fn test_store() { + let store_file = tempfile::NamedTempFile::new().unwrap(); + let mut schema = write_lorem_ipsum_store(store_file.reopen().unwrap()); + let field_title = schema.field("title").unwrap(); let store_mmap = MmapReadOnly::open(&store_file).unwrap(); let store = StoreReader::new(store_mmap); - assert_eq!(offsets, store.offsets); for i in (0..10).map(|i| i * 3 / 2) { assert_eq!(*store.get(&i).get_one(&field_title).unwrap(), format!("Doc {}", i)); } } + #[bench] + fn bench_store_encode(b: &mut Bencher) { + let mut store_file = tempfile::NamedTempFile::new().unwrap(); + b.iter(|| { + write_lorem_ipsum_store(store_file.reopen().unwrap()); + }); + } + + + #[bench] + fn bench_store_decode(b: &mut Bencher) { + + let store_file = tempfile::NamedTempFile::new().unwrap(); + write_lorem_ipsum_store(store_file.reopen().unwrap()); + let store_mmap = MmapReadOnly::open(&store_file).unwrap(); + let store = StoreReader::new(store_mmap); + b.iter(|| { + store.get(&12); + }); + + } } diff --git a/src/lib.rs b/src/lib.rs index 00e6eb154..658727f6e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,11 +1,10 @@ +#![feature(test)] #[allow(unused_imports)] #[macro_use] extern crate lazy_static; #[macro_use] -extern crate log; - extern crate fst; extern crate byteorder; extern crate memmap; @@ -22,4 +21,6 @@ extern crate serde; extern crate libc; extern crate lz4; +#[cfg(test)] extern crate test; + pub mod core; diff --git a/tests/core.rs b/tests/core.rs index 9a31011ba..c5e3852b2 100644 --- a/tests/core.rs +++ b/tests/core.rs @@ -1,3 +1,5 @@ +#![feature(test)] + extern crate tantivy; extern crate regex; extern crate tempdir; diff --git a/tests/query.rs b/tests/query.rs deleted file mode 100644 index e69de29bb..000000000