diff --git a/src/collector/count_collector.rs b/src/collector/count_collector.rs index 67f563f1a..6707e687e 100644 --- a/src/collector/count_collector.rs +++ b/src/collector/count_collector.rs @@ -20,7 +20,6 @@ impl CountCollector { } } - impl Collector for CountCollector { fn set_segment(&mut self, _: SegmentLocalId, _: &SegmentReader) -> Result<()> { Ok(()) diff --git a/src/collector/facet_collector.rs b/src/collector/facet_collector.rs index 26e14d14f..41dd0bcf5 100644 --- a/src/collector/facet_collector.rs +++ b/src/collector/facet_collector.rs @@ -13,7 +13,7 @@ use termdict::TermStreamerBuilder; use std::collections::BTreeSet; use termdict::TermMerger; use postings::SkipResult; -use std::{u64, usize}; +use std::{usize, u64}; use std::iter::Peekable; use DocId; @@ -48,7 +48,6 @@ impl<'a> Ord for Hit<'a> { } } - struct SegmentFacetCounter { pub facet_reader: FacetReader, pub facet_ords: Vec, @@ -59,16 +58,10 @@ fn facet_depth(facet_bytes: &[u8]) -> usize { if facet_bytes.is_empty() { 0 } else { - facet_bytes - .iter() - .cloned() - .filter(|b| *b == 0u8) - .count() + 1 + facet_bytes.iter().cloned().filter(|b| *b == 0u8).count() + 1 } } - - /// Collector for faceting /// /// The collector collects all facets. You need to configure it @@ -215,23 +208,24 @@ pub struct FacetCollector { // collapse facet_id -> facet_ord current_collapse_facet_ords: Vec, - facets: BTreeSet + facets: BTreeSet, } -fn skip<'a, I: Iterator>(target: &[u8], collapse_it: &mut Peekable) -> SkipResult { +fn skip<'a, I: Iterator>( + target: &[u8], + collapse_it: &mut Peekable, +) -> SkipResult { loop { match collapse_it.peek() { - Some(facet_bytes) => { - match facet_bytes.encoded_bytes().cmp(&target) { - Ordering::Less => {} - Ordering::Greater => { - return SkipResult::OverStep; - } - Ordering::Equal => { - return SkipResult::Reached; - } + Some(facet_bytes) => match facet_bytes.encoded_bytes().cmp(&target) { + Ordering::Less => {} + Ordering::Greater => { + return SkipResult::OverStep; } - } + Ordering::Equal => { + return SkipResult::Reached; + } + }, None => { return SkipResult::End; } @@ -241,7 +235,6 @@ fn skip<'a, I: Iterator>(target: &[u8], collapse_it: &mut Peekab } impl FacetCollector { - /// Create a facet collector to collect the facets /// from a specific facet `Field`. /// @@ -261,7 +254,6 @@ impl FacetCollector { } } - /// Adds a facet that we want to record counts /// /// Adding facet `Facet::from("/country")` for instance, @@ -272,12 +264,15 @@ impl FacetCollector { /// If you need the correct number of unique documents for two such facets, /// just add them in separate `FacetCollector`. pub fn add_facet(&mut self, facet_from: T) - where Facet: From { + where + Facet: From, + { let facet = Facet::from(facet_from); for old_facet in &self.facets { assert!( !old_facet.is_prefix_of(&facet), - "Tried to add a facet which is a descendant of an already added facet."); + "Tried to add a facet which is a descendant of an already added facet." + ); assert!( !facet.is_prefix_of(&old_facet), "Tried to add a facet which is an ancestor of an already added facet." @@ -292,10 +287,7 @@ impl FacetCollector { self.current_segment_counts.clear(); let mut collapse_facet_it = self.facets.iter().peekable(); self.current_collapse_facet_ords.push(0); - let mut facet_streamer = facet_reader - .facet_dict() - .range() - .into_stream(); + let mut facet_streamer = facet_reader.facet_dict().range().into_stream(); if !facet_streamer.advance() { return; } @@ -315,7 +307,8 @@ impl FacetCollector { continue 'outer; } else if depth == collapse_depth + 1 { collapsed_id = self.current_collapse_facet_ords.len(); - self.current_collapse_facet_ords.push(facet_streamer.term_ord()); + self.current_collapse_facet_ords + .push(facet_streamer.term_ord()); self.current_segment_collapse_mapping.push(collapsed_id); } else { self.current_segment_collapse_mapping.push(collapsed_id); @@ -335,13 +328,11 @@ impl FacetCollector { fn finalize_segment(&mut self) { if self.ff_reader.is_some() { - self.segment_counters.push( - SegmentFacetCounter { - facet_reader: unsafe { self.ff_reader.take().unwrap().into_inner() }, - facet_ords: mem::replace(&mut self.current_collapse_facet_ords, Vec::new()), - facet_counts: mem::replace(&mut self.current_segment_counts, Vec::new()), - } - ); + self.segment_counters.push(SegmentFacetCounter { + facet_reader: unsafe { self.ff_reader.take().unwrap().into_inner() }, + facet_ords: mem::replace(&mut self.current_collapse_facet_ords, Vec::new()), + facet_counts: mem::replace(&mut self.current_segment_counts, Vec::new()), + }); } } @@ -361,14 +352,9 @@ impl FacetCollector { .map(|segment_counter| &segment_counter.facet_counts[..]) .collect(); - let facet_streams = self.segment_counters .iter() - .map(|seg_counts| seg_counts - .facet_reader - .facet_dict() - .range() - .into_stream()) + .map(|seg_counts| seg_counts.facet_reader.facet_dict().range().into_stream()) .collect::>(); let mut facet_merger = TermMerger::new(facet_streams); @@ -398,51 +384,43 @@ impl FacetCollector { facet_counts.insert(Facet::from_encoded(bytes), count); } } - FacetCounts { - facet_counts - } + FacetCounts { facet_counts } } } - - - impl Collector for FacetCollector { fn set_segment(&mut self, _: SegmentLocalId, reader: &SegmentReader) -> Result<()> { self.finalize_segment(); let facet_reader = reader.facet_reader(self.field)?; self.set_collapse_mapping(&facet_reader); - self.current_segment_counts.resize(self.current_collapse_facet_ords.len(), 0); + self.current_segment_counts + .resize(self.current_collapse_facet_ords.len(), 0); self.ff_reader = Some(UnsafeCell::new(facet_reader)); Ok(()) } fn collect(&mut self, doc: DocId, _: Score) { - let facet_reader: &mut FacetReader = - unsafe { - &mut *self.ff_reader - .as_ref() - .expect("collect() was called before set_segment. This should never happen.") - .get() - }; + let facet_reader: &mut FacetReader = unsafe { + &mut *self.ff_reader + .as_ref() + .expect("collect() was called before set_segment. This should never happen.") + .get() + }; facet_reader.facet_ords(doc, &mut self.facet_ords); let mut previous_collapsed_ord: usize = usize::MAX; for &facet_ord in &self.facet_ords { let collapsed_ord = self.current_segment_collapse_mapping[facet_ord as usize]; - self.current_segment_counts[collapsed_ord] += - if collapsed_ord == previous_collapsed_ord { - 0 - } else { - 1 - }; + self.current_segment_counts[collapsed_ord] += if collapsed_ord == previous_collapsed_ord + { + 0 + } else { + 1 + }; previous_collapsed_ord = collapsed_ord; } } } - - - /// Intermediary result of the `FacetCollector` that stores /// the facet counts for all the segments. pub struct FacetCounts { @@ -450,20 +428,20 @@ pub struct FacetCounts { } impl FacetCounts { - - pub fn get<'a, T>(&'a self, facet_from: T) -> impl Iterator - where Facet: From { + pub fn get<'a, T>(&'a self, facet_from: T) -> impl Iterator + where + Facet: From, + { let facet = Facet::from(facet_from); let left_bound = Bound::Excluded(facet.clone()); - let right_bound = - if facet.is_root() { - Bound::Unbounded - } else { - let mut facet_after_bytes = facet.encoded_bytes().to_owned(); - facet_after_bytes.push(1u8); - let facet_after = Facet::from_encoded(facet_after_bytes); - Bound::Excluded(facet_after) - }; + let right_bound = if facet.is_root() { + Bound::Unbounded + } else { + let mut facet_after_bytes = facet.encoded_bytes().to_owned(); + facet_after_bytes.push(1u8); + let facet_after = Facet::from_encoded(facet_after_bytes); + Bound::Excluded(facet_after) + }; self.facet_counts .range((left_bound, right_bound)) @@ -471,8 +449,9 @@ impl FacetCounts { } pub fn top_k(&self, facet: T, k: usize) -> Vec<(&Facet, u64)> - where Facet: From { - + where + Facet: From, + { let mut heap = BinaryHeap::with_capacity(k); let mut it = self.get(facet); @@ -480,9 +459,7 @@ impl FacetCounts { heap.push(Hit { count, facet }); } - let mut lowest_count: u64 = heap.peek() - .map(|hit| hit.count) - .unwrap_or(u64::MIN); + let mut lowest_count: u64 = heap.peek().map(|hit| hit.count).unwrap_or(u64::MIN); for (facet, count) in it { if count > lowest_count { lowest_count = count; @@ -496,16 +473,13 @@ impl FacetCounts { .map(|hit| (hit.facet, hit.count)) .collect::>() } - } - - #[cfg(test)] mod tests { use test::Bencher; use core::Index; - use schema::{SchemaBuilder, Document, Facet}; + use schema::{Document, Facet, SchemaBuilder}; use query::AllQuery; use super::{FacetCollector, FacetCounts}; use std::iter; @@ -550,21 +524,23 @@ mod tests { .get("/top1") .map(|(facet, count)| (facet.to_string(), count)) .collect(); - assert_eq!(facets, [ - ("/top1/mid0", 50), - ("/top1/mid1", 50), - ("/top1/mid2", 50), - ("/top1/mid3", 50), - ].iter() - .map(|&(facet_str, count)| { - (String::from(facet_str), count) - }) - .collect::>()); + assert_eq!( + facets, + [ + ("/top1/mid0", 50), + ("/top1/mid1", 50), + ("/top1/mid2", 50), + ("/top1/mid3", 50), + ].iter() + .map(|&(facet_str, count)| (String::from(facet_str), count)) + .collect::>() + ); } } #[test] - #[should_panic(expected="Tried to add a facet which is a descendant of an already added facet.")] + #[should_panic(expected = "Tried to add a facet which is a descendant of \ + an already added facet.")] fn test_misused_facet_collector() { let mut facet_collector = FacetCollector::for_field(Field(0)); facet_collector.add_facet(Facet::from("/country")); @@ -585,18 +561,14 @@ mod tests { let schema = schema_builder.build(); let index = Index::create_in_ram(schema); - let mut docs: Vec = vec![ - ("a", 10), - ("b", 100), - ("c", 7), - ("d", 12), - ("e", 21) - ].into_iter() - .flat_map(|(c, count)| { - let facet = Facet::from(&format!("/facet_{}", c)); - let doc = doc!(facet_field => facet); - iter::repeat(doc).take(count) - }).collect(); + let mut docs: Vec = vec![("a", 10), ("b", 100), ("c", 7), ("d", 12), ("e", 21)] + .into_iter() + .flat_map(|(c, count)| { + let facet = Facet::from(&format!("/facet_{}", c)); + let doc = doc!(facet_field => facet); + iter::repeat(doc).take(count) + }) + .collect(); thread_rng().shuffle(&mut docs[..]); let mut index_writer = index.writer(3_000_000).unwrap(); @@ -620,8 +592,9 @@ mod tests { vec![ (&Facet::from("/facet_b"), 100), (&Facet::from("/facet_e"), 21), - (&Facet::from("/facet_d"), 12) - ]); + (&Facet::from("/facet_d"), 12), + ] + ); } } @@ -632,12 +605,12 @@ mod tests { let schema = schema_builder.build(); let index = Index::create_in_ram(schema); - let mut docs = vec!(); + let mut docs = vec![]; for val in 0..50 { let facet = Facet::from(&format!("/facet_{}", val)); - for _ in 0..val*val { - docs.push(doc!(facet_field=>facet.clone())); - } + for _ in 0..val * val { + docs.push(doc!(facet_field=>facet.clone())); + } } // 40425 docs thread_rng().shuffle(&mut docs[..]); @@ -656,4 +629,3 @@ mod tests { }); } } - diff --git a/src/common/bitpacker.rs b/src/common/bitpacker.rs index cb3b81e72..b78a32746 100644 --- a/src/common/bitpacker.rs +++ b/src/common/bitpacker.rs @@ -134,7 +134,8 @@ where addr + 8 <= data.len(), "The fast field field should have been padded with 7 bytes." ); - let val_unshifted_unmasked: u64 = unsafe { *(data[addr..].as_ptr() as *const u64) }; + let val_unshifted_unmasked: u64 = + unsafe { *(data[addr..].as_ptr() as *const u64) }; let val_shifted = (val_unshifted_unmasked >> bit_shift) as u64; (val_shifted & mask) } else { @@ -165,7 +166,8 @@ where for output_val in output.iter_mut() { let addr = addr_in_bits >> 3; let bit_shift = addr_in_bits & 7; - let val_unshifted_unmasked: u64 = unsafe { *(data[addr..].as_ptr() as *const u64) }; + let val_unshifted_unmasked: u64 = + unsafe { *(data[addr..].as_ptr() as *const u64) }; let val_shifted = (val_unshifted_unmasked >> bit_shift) as u64; *output_val = val_shifted & mask; addr_in_bits += num_bits; diff --git a/src/common/composite_file.rs b/src/common/composite_file.rs index 341dc051c..cceafce49 100644 --- a/src/common/composite_file.rs +++ b/src/common/composite_file.rs @@ -8,7 +8,6 @@ use std::io::{self, Read}; use directory::ReadOnlySource; use common::BinarySerializable; - #[derive(Eq, PartialEq, Hash, Copy, Ord, PartialOrd, Clone, Debug)] pub struct FileAddr { field: Field, @@ -19,7 +18,7 @@ impl FileAddr { fn new(field: Field, idx: usize) -> FileAddr { FileAddr { field: field, - idx: idx + idx: idx, } } } @@ -36,7 +35,7 @@ impl BinarySerializable for FileAddr { let idx = VInt::deserialize(reader)?.0 as usize; Ok(FileAddr { field: field, - idx: idx + idx: idx, }) } } @@ -59,7 +58,7 @@ impl CompositeWrite { /// Start writing a new field. pub fn for_field(&mut self, field: Field) -> &mut CountingWriter { - self.for_field_with_idx(field, 0) + self.for_field_with_idx(field, 0) } /// Start writing a new field. @@ -71,7 +70,6 @@ impl CompositeWrite { &mut self.write } - /// Close the composite file. /// /// An index of the different field offsets @@ -89,9 +87,7 @@ impl CompositeWrite { let mut prev_offset = 0; for (offset, file_addr) in offset_fields { - VInt((offset - prev_offset) as u64).serialize( - &mut self.write, - )?; + VInt((offset - prev_offset) as u64).serialize(&mut self.write)?; file_addr.serialize(&mut self.write)?; prev_offset = offset; } @@ -103,7 +99,6 @@ impl CompositeWrite { } } - /// A composite file is an abstraction to store a /// file partitioned by field. /// @@ -174,20 +169,20 @@ impl CompositeFile { /// to a given `Field` and stored in a `CompositeFile`. pub fn open_read_with_idx(&self, field: Field, idx: usize) -> Option { self.offsets_index - .get(&FileAddr {field: field, idx: idx}) - .map(|&(from, to)| { - self.data.slice(from, to) + .get(&FileAddr { + field: field, + idx: idx, }) + .map(|&(from, to)| self.data.slice(from, to)) } } - #[cfg(test)] mod test { use std::io::Write; - use super::{CompositeWrite, CompositeFile}; - use directory::{RAMDirectory, Directory}; + use super::{CompositeFile, CompositeWrite}; + use directory::{Directory, RAMDirectory}; use schema::Field; use common::VInt; use common::BinarySerializable; @@ -231,7 +226,6 @@ mod test { assert_eq!(payload_4, 2u64); } } - } } diff --git a/src/compression/pack/compression_pack_simd.rs b/src/compression/pack/compression_pack_simd.rs index 2a900e9ed..2db372630 100644 --- a/src/compression/pack/compression_pack_simd.rs +++ b/src/compression/pack/compression_pack_simd.rs @@ -25,7 +25,9 @@ fn compress_sorted(vals: &[u32], output: &mut [u8], offset: u32) -> usize { } fn uncompress_sorted(compressed_data: &[u8], output: &mut [u32], offset: u32) -> usize { - unsafe { simdcomp::uncompress_sorted(compressed_data.as_ptr(), output.as_mut_ptr(), offset) } + unsafe { + simdcomp::uncompress_sorted(compressed_data.as_ptr(), output.as_mut_ptr(), offset) + } } fn compress_unsorted(vals: &[u32], output: &mut [u8]) -> usize { diff --git a/src/core/segment_reader.rs b/src/core/segment_reader.rs index 939873d08..a62b0befc 100644 --- a/src/core/segment_reader.rs +++ b/src/core/segment_reader.rs @@ -108,17 +108,21 @@ impl SegmentReader { pub fn facet_reader(&self, field: Field) -> Result { let field_entry = self.schema.get_field_entry(field); if field_entry.field_type() != &FieldType::HierarchicalFacet { - return Err(ErrorKind::InvalidArgument(format!("The field {:?} is not a \ - hierarchical facet.", field_entry)).into()) + return Err(ErrorKind::InvalidArgument(format!( + "The field {:?} is not a \ + hierarchical facet.", + field_entry + )).into()); } let term_ords_reader = self.multi_value_reader(field)?; - let termdict_source = self.termdict_composite - .open_read(field) - .ok_or_else(|| { - ErrorKind::InvalidArgument(format!("The field \"{}\" is a hierarchical \ - but this segment does not seem to have the field term \ - dictionary.", field_entry.name())) - })?; + let termdict_source = self.termdict_composite.open_read(field).ok_or_else(|| { + ErrorKind::InvalidArgument(format!( + "The field \"{}\" is a hierarchical \ + but this segment does not seem to have the field term \ + dictionary.", + field_entry.name() + )) + })?; let termdict = TermDictionaryImpl::from_source(termdict_source); let facet_reader = FacetReader::new(term_ords_reader, termdict); Ok(facet_reader) diff --git a/src/datastruct/stacker/hashmap.rs b/src/datastruct/stacker/hashmap.rs index ea57258df..18025b657 100644 --- a/src/datastruct/stacker/hashmap.rs +++ b/src/datastruct/stacker/hashmap.rs @@ -1,7 +1,7 @@ use std::iter; use std::mem; use postings::UnorderedTermId; -use super::heap::{Heap, HeapAllocable, BytesRef}; +use super::heap::{BytesRef, Heap, HeapAllocable}; mod murmurhash2 { @@ -53,9 +53,6 @@ mod murmurhash2 { } } - - - /// Split the thread memory budget into /// - the heap size /// - the hash table "table" itself. @@ -63,14 +60,10 @@ mod murmurhash2 { /// Returns (the heap size in bytes, the hash table size in number of bits) pub(crate) fn split_memory(per_thread_memory_budget: usize) -> (usize, usize) { let table_size_limit: usize = per_thread_memory_budget / 3; - let compute_table_size = |num_bits: usize| { - (1 << num_bits) * mem::size_of::() - }; + let compute_table_size = |num_bits: usize| (1 << num_bits) * mem::size_of::(); let table_num_bits: usize = (1..) .into_iter() - .take_while(|num_bits: &usize| { - compute_table_size(*num_bits) < table_size_limit - }) + .take_while(|num_bits: &usize| compute_table_size(*num_bits) < table_size_limit) .last() .expect(&format!( "Per thread memory is too small: {}", @@ -81,7 +74,6 @@ pub(crate) fn split_memory(per_thread_memory_budget: usize) -> (usize, usize) { (heap_size, table_num_bits) } - /// `KeyValue` is the item stored in the hash table. /// The key is actually a `BytesRef` object stored in an external heap. /// The `value_addr` also points to an address in the heap. @@ -101,7 +93,6 @@ impl KeyValue { } } - /// Customized `HashMap` with string keys /// /// This `HashMap` takes String as keys. Keys are @@ -118,7 +109,6 @@ pub struct TermHashMap<'a> { occupied: Vec, } - struct QuadraticProbing { hash: usize, i: usize, @@ -141,7 +131,6 @@ impl QuadraticProbing { } } - impl<'a> TermHashMap<'a> { pub fn new(num_bucket_power_of_2: usize, heap: &'a Heap) -> TermHashMap<'a> { let table_size = 1 << num_bucket_power_of_2; @@ -178,18 +167,17 @@ impl<'a> TermHashMap<'a> { } pub fn iter<'b: 'a>(&'b self) -> impl Iterator + 'b { - self.occupied - .iter() - .cloned() - .map(move |bucket: usize| { - let kv = self.table[bucket]; - let (key, offset) = self.get_key_value(kv.key_value_addr); - (key, offset, bucket as UnorderedTermId) - }) + self.occupied.iter().cloned().map(move |bucket: usize| { + let kv = self.table[bucket]; + let (key, offset) = self.get_key_value(kv.key_value_addr); + (key, offset, bucket as UnorderedTermId) + }) } - - pub fn get_or_create, V: HeapAllocable>(&mut self, key: S) -> (UnorderedTermId, &mut V) { + pub fn get_or_create, V: HeapAllocable>( + &mut self, + key: S, + ) -> (UnorderedTermId, &mut V) { let key_bytes: &[u8] = key.as_ref(); let hash = murmurhash2::murmurhash2(key.as_ref()); let mut probe = self.probe(hash); @@ -212,7 +200,6 @@ impl<'a> TermHashMap<'a> { } } - #[cfg(test)] mod tests { @@ -223,7 +210,6 @@ mod tests { use std::collections::HashSet; use super::split_memory; - struct TestValue { val: u32, _addr: u32, @@ -245,7 +231,6 @@ mod tests { assert_eq!(split_memory(10_000_000), (7902848, 18)); } - #[test] fn test_hash_map() { let heap = Heap::with_capacity(2_000_000); @@ -319,5 +304,4 @@ mod tests { }); } - } diff --git a/src/datastruct/stacker/mod.rs b/src/datastruct/stacker/mod.rs index 1d6cac450..811bfeee7 100644 --- a/src/datastruct/stacker/mod.rs +++ b/src/datastruct/stacker/mod.rs @@ -39,6 +39,5 @@ fn test_unrolled_linked_list() { assert!(!it.next().is_some()); } } - } } diff --git a/src/directory/mmap_directory.rs b/src/directory/mmap_directory.rs index 5dc1166af..bb7a99bf1 100644 --- a/src/directory/mmap_directory.rs +++ b/src/directory/mmap_directory.rs @@ -20,19 +20,17 @@ use std::sync::Arc; use std::sync::RwLock; use tempdir::TempDir; - /// Returns None iff the file exists, can be read, but is empty (and hence /// cannot be mmapped). /// fn open_mmap(full_path: &PathBuf) -> result::Result, OpenReadError> { - let file = File::open(&full_path) - .map_err(|e| { - if e.kind() == io::ErrorKind::NotFound { - OpenReadError::FileDoesNotExist(full_path.clone()) - } else { - OpenReadError::IOError(IOError::with_path(full_path.to_owned(), e)) - } - })?; + let file = File::open(&full_path).map_err(|e| { + if e.kind() == io::ErrorKind::NotFound { + OpenReadError::FileDoesNotExist(full_path.clone()) + } else { + OpenReadError::IOError(IOError::with_path(full_path.to_owned(), e)) + } + })?; let meta_data = file.metadata() .map_err(|e| IOError::with_path(full_path.to_owned(), e))?; @@ -44,9 +42,7 @@ fn open_mmap(full_path: &PathBuf) -> result::Result, OpenRe } MmapReadOnly::open(&file) .map(Some) - .map_err(|e| { - From::from(IOError::with_path(full_path.to_owned(), e)) - }) + .map_err(|e| From::from(IOError::with_path(full_path.to_owned(), e))) } #[derive(Default, Clone, Debug, Serialize, Deserialize)] @@ -79,7 +75,6 @@ impl Default for MmapCache { } impl MmapCache { - /// Removes a `MmapReadOnly` entry from the mmap cache. fn discard_from_cache(&mut self, full_path: &Path) -> bool { self.cache.remove(full_path).is_some() @@ -95,21 +90,21 @@ impl MmapCache { fn get_mmap(&mut self, full_path: PathBuf) -> Result, OpenReadError> { Ok(match self.cache.entry(full_path.clone()) { - HashMapEntry::Occupied(occupied_entry) => { - let mmap = occupied_entry.get(); - self.counters.hit += 1; - Some(mmap.clone()) - } - HashMapEntry::Vacant(vacant_entry) => { - self.counters.miss += 1; - if let Some(mmap) = open_mmap(&full_path)? { - vacant_entry.insert(mmap.clone()); - Some(mmap) - } else { - None - } - } - }) + HashMapEntry::Occupied(occupied_entry) => { + let mmap = occupied_entry.get(); + self.counters.hit += 1; + Some(mmap.clone()) + } + HashMapEntry::Vacant(vacant_entry) => { + self.counters.miss += 1; + if let Some(mmap) = open_mmap(&full_path)? { + vacant_entry.insert(mmap.clone()); + Some(mmap) + } else { + None + } + } + }) } } @@ -257,9 +252,9 @@ impl Directory for MmapDirectory { })?; Ok(mmap_cache - .get_mmap(full_path)? - .map(ReadOnlySource::Mmap) - .unwrap_or_else(|| ReadOnlySource::Anonymous(SharedVecSlice::empty()))) + .get_mmap(full_path)? + .map(ReadOnlySource::Mmap) + .unwrap_or_else(|| ReadOnlySource::Anonymous(SharedVecSlice::empty()))) } fn open_write(&mut self, path: &Path) -> Result { @@ -292,20 +287,19 @@ impl Directory for MmapDirectory { Ok(BufWriter::new(Box::new(writer))) } - /// Any entry associated to the path in the mmap will be /// removed before the file is deleted. fn delete(&self, path: &Path) -> result::Result<(), DeleteError> { debug!("Deleting file {:?}", path); let full_path = self.resolve_path(path); - let mut mmap_cache = self.mmap_cache - .write() - .map_err(|_| { - let msg = format!("Failed to acquired write lock \ - on mmap cache while deleting {:?}", - path); - IOError::with_path(path.to_owned(), make_io_err(msg)) - })?; + let mut mmap_cache = self.mmap_cache.write().map_err(|_| { + let msg = format!( + "Failed to acquired write lock \ + on mmap cache while deleting {:?}", + path + ); + IOError::with_path(path.to_owned(), make_io_err(msg)) + })?; mmap_cache.discard_from_cache(path); // Removing the entry in the MMap cache. @@ -415,7 +409,10 @@ mod tests { } for (i, path) in paths.iter().enumerate() { mmap_directory.delete(path).unwrap(); - assert_eq!(mmap_directory.get_cache_info().mmapped.len(), num_paths - i - 1); + assert_eq!( + mmap_directory.get_cache_info().mmapped.len(), + num_paths - i - 1 + ); } } assert_eq!(mmap_directory.get_cache_info().counters.hit, 10); diff --git a/src/directory/read_only_source.rs b/src/directory/read_only_source.rs index 10d9a85d9..161fec11a 100644 --- a/src/directory/read_only_source.rs +++ b/src/directory/read_only_source.rs @@ -4,7 +4,7 @@ use super::shared_vec_slice::SharedVecSlice; use common::HasLen; use std::slice; use std::io::{self, Read}; -use stable_deref_trait::{StableDeref, CloneStableDeref}; +use stable_deref_trait::{CloneStableDeref, StableDeref}; /// Read object that represents files in tantivy. /// diff --git a/src/fastfield/facet_reader.rs b/src/fastfield/facet_reader.rs index d7a9d8ce9..ea5a9e25a 100644 --- a/src/fastfield/facet_reader.rs +++ b/src/fastfield/facet_reader.rs @@ -4,7 +4,6 @@ use termdict::TermOrdinal; use schema::Facet; use termdict::{TermDictionary, TermDictionaryImpl}; - /// The facet reader makes it possible to access the list of /// facets associated to a given document in a specific /// segment. @@ -24,7 +23,6 @@ pub struct FacetReader { } impl FacetReader { - /// Creates a new `FacetReader`. /// /// A facet reader just wraps : @@ -32,8 +30,10 @@ impl FacetReader { /// access the list of facet ords for a given document. /// - a `TermDictionaryImpl` that helps associating a facet to /// an ordinal and vice versa. - pub fn new(term_ords: MultiValueIntFastFieldReader, - term_dict: TermDictionaryImpl) -> FacetReader { + pub fn new( + term_ords: MultiValueIntFastFieldReader, + term_dict: TermDictionaryImpl, + ) -> FacetReader { FacetReader { term_ords: term_ords, term_dict: term_dict, @@ -56,7 +56,8 @@ impl FacetReader { /// Given a term ordinal returns the term associated to it. pub fn facet_from_ord(&self, facet_ord: TermOrdinal, output: &mut Facet) { - let found_term = self.term_dict.ord_to_term(facet_ord as u64, output.inner_buffer_mut()); + let found_term = self.term_dict + .ord_to_term(facet_ord as u64, output.inner_buffer_mut()); assert!(found_term, "Term ordinal {} no found.", facet_ord); } @@ -64,4 +65,4 @@ impl FacetReader { pub fn facet_ords(&mut self, doc: DocId, output: &mut Vec) { self.term_ords.get_vals(doc, output); } -} \ No newline at end of file +} diff --git a/src/fastfield/mod.rs b/src/fastfield/mod.rs index 207012a3f..0afc50c7e 100644 --- a/src/fastfield/mod.rs +++ b/src/fastfield/mod.rs @@ -95,7 +95,9 @@ mod tests { add_single_field_doc(&mut fast_field_writers, *FIELD, 13u64); add_single_field_doc(&mut fast_field_writers, *FIELD, 14u64); add_single_field_doc(&mut fast_field_writers, *FIELD, 2u64); - fast_field_writers.serialize(&mut serializer, HashMap::new()).unwrap(); + fast_field_writers + .serialize(&mut serializer, HashMap::new()) + .unwrap(); serializer.close().unwrap(); } let source = directory.open_read(&path).unwrap(); @@ -129,7 +131,9 @@ mod tests { add_single_field_doc(&mut fast_field_writers, *FIELD, 1_002u64); add_single_field_doc(&mut fast_field_writers, *FIELD, 1_501u64); add_single_field_doc(&mut fast_field_writers, *FIELD, 215u64); - fast_field_writers.serialize(&mut serializer, HashMap::new()).unwrap(); + fast_field_writers + .serialize(&mut serializer, HashMap::new()) + .unwrap(); serializer.close().unwrap(); } let source = directory.open_read(&path).unwrap(); @@ -164,7 +168,9 @@ mod tests { for _ in 0..10_000 { add_single_field_doc(&mut fast_field_writers, *FIELD, 100_000u64); } - fast_field_writers.serialize(&mut serializer, HashMap::new()).unwrap(); + fast_field_writers + .serialize(&mut serializer, HashMap::new()) + .unwrap(); serializer.close().unwrap(); } let source = directory.open_read(&path).unwrap(); @@ -199,7 +205,9 @@ mod tests { 5_000_000_000_000_000_000u64 + i, ); } - fast_field_writers.serialize(&mut serializer, HashMap::new()).unwrap(); + fast_field_writers + .serialize(&mut serializer, HashMap::new()) + .unwrap(); serializer.close().unwrap(); } let source = directory.open_read(&path).unwrap(); @@ -238,7 +246,9 @@ mod tests { doc.add_i64(i64_field, i); fast_field_writers.add_document(&doc); } - fast_field_writers.serialize(&mut serializer, HashMap::new()).unwrap(); + fast_field_writers + .serialize(&mut serializer, HashMap::new()) + .unwrap(); serializer.close().unwrap(); } let source = directory.open_read(&path).unwrap(); @@ -277,7 +287,9 @@ mod tests { let mut fast_field_writers = FastFieldsWriter::from_schema(&schema); let doc = Document::default(); fast_field_writers.add_document(&doc); - fast_field_writers.serialize(&mut serializer, HashMap::new()).unwrap(); + fast_field_writers + .serialize(&mut serializer, HashMap::new()) + .unwrap(); serializer.close().unwrap(); } @@ -311,7 +323,9 @@ mod tests { for x in &permutation { add_single_field_doc(&mut fast_field_writers, *FIELD, *x); } - fast_field_writers.serialize(&mut serializer, HashMap::new()).unwrap(); + fast_field_writers + .serialize(&mut serializer, HashMap::new()) + .unwrap(); serializer.close().unwrap(); } let source = directory.open_read(&path).unwrap(); @@ -366,7 +380,9 @@ mod tests { for x in &permutation { add_single_field_doc(&mut fast_field_writers, *FIELD, *x); } - fast_field_writers.serialize(&mut serializer, HashMap::new()).unwrap(); + fast_field_writers + .serialize(&mut serializer, HashMap::new()) + .unwrap(); serializer.close().unwrap(); } let source = directory.open_read(&path).unwrap(); @@ -398,7 +414,9 @@ mod tests { for x in &permutation { add_single_field_doc(&mut fast_field_writers, *FIELD, *x); } - fast_field_writers.serialize(&mut serializer, HashMap::new()).unwrap(); + fast_field_writers + .serialize(&mut serializer, HashMap::new()) + .unwrap(); serializer.close().unwrap(); } let source = directory.open_read(&path).unwrap(); diff --git a/src/fastfield/multivalued/mod.rs b/src/fastfield/multivalued/mod.rs index 726682826..0043e7783 100644 --- a/src/fastfield/multivalued/mod.rs +++ b/src/fastfield/multivalued/mod.rs @@ -2,4 +2,4 @@ mod writer; mod reader; pub use self::writer::MultiValueIntFastFieldWriter; -pub use self::reader::MultiValueIntFastFieldReader; \ No newline at end of file +pub use self::reader::MultiValueIntFastFieldReader; diff --git a/src/fastfield/multivalued/reader.rs b/src/fastfield/multivalued/reader.rs index bbf5102ac..147cee89a 100644 --- a/src/fastfield/multivalued/reader.rs +++ b/src/fastfield/multivalued/reader.rs @@ -18,8 +18,10 @@ pub struct MultiValueIntFastFieldReader { } impl MultiValueIntFastFieldReader { - - pub(crate) fn open(idx_reader: U64FastFieldReader, vals_reader: U64FastFieldReader) -> MultiValueIntFastFieldReader { + pub(crate) fn open( + idx_reader: U64FastFieldReader, + vals_reader: U64FastFieldReader, + ) -> MultiValueIntFastFieldReader { MultiValueIntFastFieldReader { idx_reader: idx_reader, vals_reader: vals_reader, @@ -38,12 +40,11 @@ impl MultiValueIntFastFieldReader { } } - #[cfg(test)] mod tests { use core::Index; - use schema::{Facet, Document, SchemaBuilder}; + use schema::{Document, Facet, SchemaBuilder}; #[test] fn test_multifastfield_reader() { @@ -51,7 +52,9 @@ mod tests { let facet_field = schema_builder.add_facet_field("facets"); let schema = schema_builder.build(); let index = Index::create_in_ram(schema); - let mut index_writer = index.writer_with_num_threads(1, 30_000_000).expect("Failed to create index writer."); + let mut index_writer = index + .writer_with_num_threads(1, 30_000_000) + .expect("Failed to create index writer."); { let mut doc = Document::new(); doc.add_facet(facet_field, "/category/cat2"); @@ -72,9 +75,7 @@ mod tests { index.load_searchers().expect("Reloading searchers"); let searcher = index.searcher(); let segment_reader = searcher.segment_reader(0); - let mut facet_reader = segment_reader - .facet_reader(facet_field) - .unwrap(); + let mut facet_reader = segment_reader.facet_reader(facet_field).unwrap(); let mut facet = Facet::root(); { @@ -108,7 +109,5 @@ mod tests { facet_reader.facet_ords(2, &mut vals); assert_eq!(&vals[..], &[4]); } - - } -} \ No newline at end of file +} diff --git a/src/fastfield/multivalued/writer.rs b/src/fastfield/multivalued/writer.rs index 499b8bad4..d988656ce 100644 --- a/src/fastfield/multivalued/writer.rs +++ b/src/fastfield/multivalued/writer.rs @@ -38,10 +38,15 @@ impl MultiValueIntFastFieldWriter { } /// Push the fast fields value to the `FastFieldWriter`. - pub fn serialize(&self, serializer: &mut FastFieldSerializer, mapping: &HashMap) -> io::Result<()> { + pub fn serialize( + &self, + serializer: &mut FastFieldSerializer, + mapping: &HashMap, + ) -> io::Result<()> { { // writing the offset index - let mut doc_index_serializer = serializer.new_u64_fast_field_with_idx(self.field, 0, self.vals.len() as u64, 0)?; + let mut doc_index_serializer = + serializer.new_u64_fast_field_with_idx(self.field, 0, self.vals.len() as u64, 0)?; for &offset in &self.doc_index { doc_index_serializer.add_val(offset)?; } @@ -50,13 +55,13 @@ impl MultiValueIntFastFieldWriter { } { // writing the values themselves. - let mut value_serializer = serializer.new_u64_fast_field_with_idx(self.field, 0u64, mapping.len() as u64, 1)?; + let mut value_serializer = + serializer.new_u64_fast_field_with_idx(self.field, 0u64, mapping.len() as u64, 1)?; for val in &self.vals { value_serializer.add_val(*mapping.get(val).expect("Missing term ordinal") as u64)?; } value_serializer.close_field()?; } Ok(()) - } } diff --git a/src/fastfield/reader.rs b/src/fastfield/reader.rs index 3e0486a1c..b556e39f3 100644 --- a/src/fastfield/reader.rs +++ b/src/fastfield/reader.rs @@ -5,7 +5,7 @@ use DocId; use schema::SchemaBuilder; use std::path::Path; use schema::FAST; -use directory::{WritePtr, RAMDirectory, Directory}; +use directory::{Directory, RAMDirectory, WritePtr}; use fastfield::{FastFieldSerializer, FastFieldsWriter}; use schema::FieldType; use std::mem; @@ -88,7 +88,7 @@ impl FastFieldReader for U64FastFieldReader { fn is_enabled(field_type: &FieldType) -> bool { match *field_type { FieldType::U64(ref integer_options) => integer_options.is_fast(), - FieldType::HierarchicalFacet => { true }, + FieldType::HierarchicalFacet => true, _ => false, } } @@ -113,7 +113,6 @@ impl FastFieldReader for U64FastFieldReader { u64::deserialize(&mut cursor).expect("Failed to read the min_value of fast field."); amplitude = u64::deserialize(&mut cursor).expect("Failed to read the amplitude of fast field."); - } let max_value = min_value + amplitude; let num_bits = compute_num_bits(amplitude); @@ -127,7 +126,6 @@ impl FastFieldReader for U64FastFieldReader { } } - impl From> for U64FastFieldReader { fn from(vals: Vec) -> U64FastFieldReader { let mut schema_builder = SchemaBuilder::default(); @@ -136,22 +134,23 @@ impl From> for U64FastFieldReader { let path = Path::new("__dummy__"); let mut directory: RAMDirectory = RAMDirectory::create(); { - let write: WritePtr = directory.open_write(path).expect( - "With a RAMDirectory, this should never fail.", - ); - let mut serializer = FastFieldSerializer::from_write(write).expect( - "With a RAMDirectory, this should never fail.", - ); + let write: WritePtr = directory + .open_write(path) + .expect("With a RAMDirectory, this should never fail."); + let mut serializer = FastFieldSerializer::from_write(write) + .expect("With a RAMDirectory, this should never fail."); let mut fast_field_writers = FastFieldsWriter::from_schema(&schema); { - let fast_field_writer = fast_field_writers.get_field_writer(field).expect( - "With a RAMDirectory, this should never fail.", - ); + let fast_field_writer = fast_field_writers + .get_field_writer(field) + .expect("With a RAMDirectory, this should never fail."); for val in vals { fast_field_writer.add_val(val); } } - fast_field_writers.serialize(&mut serializer, HashMap::new()).unwrap(); + fast_field_writers + .serialize(&mut serializer, HashMap::new()) + .unwrap(); serializer.close().unwrap(); } @@ -159,9 +158,9 @@ impl From> for U64FastFieldReader { let composite_file = CompositeFile::open(&source).expect("Failed to read the composite file"); - let field_source = composite_file.open_read(field).expect( - "File component not found", - ); + let field_source = composite_file + .open_read(field) + .expect("File component not found"); U64FastFieldReader::open(field_source) } } @@ -222,7 +221,9 @@ impl FastFieldReader for I64FastFieldReader { /// # Panics /// Panics if the data is corrupted. fn open(data: ReadOnlySource) -> I64FastFieldReader { - I64FastFieldReader { underlying: U64FastFieldReader::open(data) } + I64FastFieldReader { + underlying: U64FastFieldReader::open(data), + } } fn is_enabled(field_type: &FieldType) -> bool { diff --git a/src/fastfield/serializer.rs b/src/fastfield/serializer.rs index afce4f053..bde080e0e 100644 --- a/src/fastfield/serializer.rs +++ b/src/fastfield/serializer.rs @@ -35,7 +35,9 @@ impl FastFieldSerializer { pub fn from_write(write: WritePtr) -> io::Result { // just making room for the pointer to header. let composite_write = CompositeWrite::wrap(write); - Ok(FastFieldSerializer { composite_write: composite_write }) + Ok(FastFieldSerializer { + composite_write: composite_write, + }) } /// Start serializing a new u64 fast field @@ -54,12 +56,12 @@ impl FastFieldSerializer { field: Field, min_value: u64, max_value: u64, - idx: usize) -> io::Result>> { + idx: usize, + ) -> io::Result>> { let field_write = self.composite_write.for_field_with_idx(field, idx); FastSingleFieldSerializer::open(field_write, min_value, max_value) } - /// Closes the serializer /// /// After this call the data must be persistently save on disk. diff --git a/src/fastfield/writer.rs b/src/fastfield/writer.rs index 780ea9bf2..4ea73e702 100644 --- a/src/fastfield/writer.rs +++ b/src/fastfield/writer.rs @@ -1,4 +1,4 @@ -use schema::{Schema, Field, Document, Cardinality}; +use schema::{Cardinality, Document, Field, Schema}; use fastfield::FastFieldSerializer; use std::io; use schema::Value; @@ -25,12 +25,11 @@ impl FastFieldsWriter { for (field_id, field_entry) in schema.fields().iter().enumerate() { let field = Field(field_id as u32); - let default_value = - if let FieldType::I64(_) = *field_entry.field_type() { - common::i64_to_u64(0i64) - } else { - 0u64 - }; + let default_value = if let FieldType::I64(_) = *field_entry.field_type() { + common::i64_to_u64(0i64) + } else { + 0u64 + }; match *field_entry.field_type() { FieldType::I64(ref int_options) | FieldType::U64(ref int_options) => { match int_options.get_fastfield_cardinality() { @@ -50,7 +49,7 @@ impl FastFieldsWriter { let fast_field_writer = MultiValueIntFastFieldWriter::new(field); multi_values_writers.push(fast_field_writer); } - _ => {}, + _ => {} } } FastFieldsWriter { @@ -64,7 +63,7 @@ impl FastFieldsWriter { pub(crate) fn new(fields: Vec) -> FastFieldsWriter { FastFieldsWriter { single_value_writers: fields.into_iter().map(IntFastFieldWriter::new).collect(), - multi_values_writers: vec!(), + multi_values_writers: vec![], } } @@ -73,23 +72,22 @@ impl FastFieldsWriter { // TODO optimize self.single_value_writers .iter_mut() - .find(|field_writer| { - field_writer.field() == field - }) + .find(|field_writer| field_writer.field() == field) } /// Returns the fast field multi-value writer for the given field. /// /// Returns None if the field does not exist, or is not /// configured as a multivalued fastfield in the schema. - pub(crate) fn get_multivalue_writer(&mut self, field: Field) -> Option<&mut MultiValueIntFastFieldWriter> { + pub(crate) fn get_multivalue_writer( + &mut self, + field: Field, + ) -> Option<&mut MultiValueIntFastFieldWriter> { // TODO optimize // TODO expose for users self.multi_values_writers .iter_mut() - .find(|multivalue_writer| { - multivalue_writer.field() == field - }) + .find(|multivalue_writer| multivalue_writer.field() == field) } /// Indexes all of the fastfields of a new document. @@ -104,9 +102,11 @@ impl FastFieldsWriter { /// Serializes all of the `FastFieldWriter`s by pushing them in /// order to the fast field serializer. - pub fn serialize(&self, - serializer: &mut FastFieldSerializer, - mapping: HashMap>) -> io::Result<()> { + pub fn serialize( + &self, + serializer: &mut FastFieldSerializer, + mapping: HashMap>, + ) -> io::Result<()> { for field_writer in &self.single_value_writers { field_writer.serialize(serializer)?; } @@ -201,9 +201,9 @@ impl IntFastFieldWriter { /// associated to the document with the `DocId` n. /// (Well, `n-1` actually because of 0-indexing) pub fn add_val(&mut self, val: u64) { - VInt(val).serialize(&mut self.vals).expect( - "unable to serialize VInt to Vec", - ); + VInt(val) + .serialize(&mut self.vals) + .expect("unable to serialize VInt to Vec"); if val > self.val_max { self.val_max = val; @@ -215,7 +215,6 @@ impl IntFastFieldWriter { self.val_count += 1; } - /// Extract the value associated to the fast field for /// this document. /// @@ -228,13 +227,11 @@ impl IntFastFieldWriter { /// only the first one is taken in account. fn extract_val(&self, doc: &Document) -> u64 { match doc.get_first(self.field) { - Some(v) => { - match *v { - Value::U64(ref val) => *val, - Value::I64(ref val) => common::i64_to_u64(*val), - _ => panic!("Expected a u64field, got {:?} ", v), - } - } + Some(v) => match *v { + Value::U64(ref val) => *val, + Value::I64(ref val) => common::i64_to_u64(*val), + _ => panic!("Expected a u64field, got {:?} ", v), + }, None => self.val_if_missing, } } diff --git a/src/indexer/segment_writer.rs b/src/indexer/segment_writer.rs index cb4fdc16e..f3b318356 100644 --- a/src/indexer/segment_writer.rs +++ b/src/indexer/segment_writer.rs @@ -17,7 +17,7 @@ use super::operation::AddOperation; use postings::MultiFieldPostingsWriter; use tokenizer::BoxedTokenizer; use tokenizer::FacetTokenizer; -use tokenizer::{Tokenizer, TokenStream}; +use tokenizer::{TokenStream, Tokenizer}; use schema::Value; /// A `SegmentWriter` is in charge of creating segment index from a @@ -126,11 +126,7 @@ impl<'a> SegmentWriter<'a> { /// Indexes a new document /// /// As a user, you should rather use `IndexWriter`'s add_document. - pub fn add_document( - &mut self, - add_operation: AddOperation, - schema: &Schema, - ) -> io::Result<()> { + pub fn add_document(&mut self, add_operation: AddOperation, schema: &Schema) -> io::Result<()> { let doc_id = self.max_doc; let mut doc = add_operation.document; self.doc_opstamps.push(add_operation.opstamp); @@ -144,17 +140,16 @@ impl<'a> SegmentWriter<'a> { } match *field_options.field_type() { FieldType::HierarchicalFacet => { - let facets: Vec<&[u8]> = field_values.iter() - .flat_map(|field_value| { - match field_value.value() { - &Value::Facet(ref facet) => Some(facet.encoded_bytes()), - _ => { - panic!("Expected hierarchical facet"); - } + let facets: Vec<&[u8]> = field_values + .iter() + .flat_map(|field_value| match field_value.value() { + &Value::Facet(ref facet) => Some(facet.encoded_bytes()), + _ => { + panic!("Expected hierarchical facet"); } }) .collect(); - let mut term = unsafe {Term::with_capacity(100)}; + let mut term = unsafe { Term::with_capacity(100) }; term.set_field(field); for facet_bytes in facets { let mut unordered_term_id_opt = None; @@ -163,7 +158,8 @@ impl<'a> SegmentWriter<'a> { .token_stream(&fake_str) .process(&mut |ref token| { term.set_text(&token.text); - let unordered_term_id = self.multifield_postings.subscribe(doc_id, &term); + let unordered_term_id = + self.multifield_postings.subscribe(doc_id, &term); unordered_term_id_opt = Some(unordered_term_id); }); @@ -176,25 +172,26 @@ impl<'a> SegmentWriter<'a> { } } FieldType::Str(_) => { - let num_tokens = - if let Some(ref mut tokenizer) = self.tokenizers[field.0 as usize] { - let texts: Vec<&str> = field_values - .iter() - .flat_map(|field_value| match *field_value.value() { - Value::Str(ref text) => Some(text.as_str()), - _ => None, - }) - .collect(); - if texts.is_empty() { - 0 - } else { - let mut token_stream = tokenizer.token_stream_texts(&texts[..]); - self.multifield_postings - .index_text(doc_id, field, &mut token_stream) - } - } else { + let num_tokens = if let Some(ref mut tokenizer) = + self.tokenizers[field.0 as usize] + { + let texts: Vec<&str> = field_values + .iter() + .flat_map(|field_value| match *field_value.value() { + Value::Str(ref text) => Some(text.as_str()), + _ => None, + }) + .collect(); + if texts.is_empty() { 0 - }; + } else { + let mut token_stream = tokenizer.token_stream_texts(&texts[..]); + self.multifield_postings + .index_text(doc_id, field, &mut token_stream) + } + } else { + 0 + }; self.fieldnorms_writer .get_field_writer(field) .map(|field_norms_writer| { @@ -226,9 +223,7 @@ impl<'a> SegmentWriter<'a> { } } self.fieldnorms_writer.fill_val_up_to(doc_id); - doc.filter_fields(|field| { - schema.get_field_entry(field).is_stored() - }); + doc.filter_fields(|field| schema.get_field_entry(field).is_stored()); let doc_writer = self.segment_serializer.get_store_writer(); doc_writer.store(&doc)?; self.max_doc += 1; diff --git a/src/lib.rs b/src/lib.rs index 46f537067..ca6c0dfc1 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -34,8 +34,6 @@ extern crate log; #[macro_use] extern crate error_chain; -extern crate regex; -extern crate tempfile; extern crate atomicwrites; extern crate bit_set; extern crate byteorder; @@ -49,11 +47,13 @@ extern crate itertools; extern crate lz4; extern crate num_cpus; extern crate owning_ref; +extern crate regex; extern crate rust_stemmers; extern crate serde; extern crate serde_json; extern crate stable_deref_trait; extern crate tempdir; +extern crate tempfile; extern crate time; extern crate uuid; diff --git a/src/postings/mod.rs b/src/postings/mod.rs index 972d38aae..fd86266f4 100644 --- a/src/postings/mod.rs +++ b/src/postings/mod.rs @@ -94,39 +94,52 @@ mod tests { index_writer.commit().unwrap(); index.load_searchers().unwrap(); let searcher = index.searcher(); - let query = TermQuery::new(Term::from_field_text(title, "abc"), IndexRecordOption::WithFreqsAndPositions); + let query = TermQuery::new( + Term::from_field_text(title, "abc"), + IndexRecordOption::WithFreqsAndPositions, + ); let weight = query.specialized_weight(&*searcher); { - let mut scorer = weight.specialized_scorer(searcher.segment_reader(0u32)).unwrap(); + let mut scorer = weight + .specialized_scorer(searcher.segment_reader(0u32)) + .unwrap(); scorer.advance(); - assert_eq!(&[0,1,2], scorer.postings().positions()); + assert_eq!(&[0, 1, 2], scorer.postings().positions()); scorer.advance(); - assert_eq!(&[0,5], scorer.postings().positions()); + assert_eq!(&[0, 5], scorer.postings().positions()); } { - let mut scorer = weight.specialized_scorer(searcher.segment_reader(0u32)).unwrap(); + let mut scorer = weight + .specialized_scorer(searcher.segment_reader(0u32)) + .unwrap(); scorer.advance(); scorer.advance(); - assert_eq!(&[0,5], scorer.postings().positions()); + assert_eq!(&[0, 5], scorer.postings().positions()); } { - let mut scorer = weight.specialized_scorer(searcher.segment_reader(0u32)).unwrap(); + let mut scorer = weight + .specialized_scorer(searcher.segment_reader(0u32)) + .unwrap(); assert_eq!(scorer.skip_next(1), SkipResult::Reached); assert_eq!(scorer.doc(), 1); - assert_eq!(&[0,5], scorer.postings().positions()); + assert_eq!(&[0, 5], scorer.postings().positions()); } { - let mut scorer = weight.specialized_scorer(searcher.segment_reader(0u32)).unwrap(); + let mut scorer = weight + .specialized_scorer(searcher.segment_reader(0u32)) + .unwrap(); assert_eq!(scorer.skip_next(1002), SkipResult::Reached); assert_eq!(scorer.doc(), 1002); - assert_eq!(&[0,5], scorer.postings().positions()); + assert_eq!(&[0, 5], scorer.postings().positions()); } { - let mut scorer = weight.specialized_scorer(searcher.segment_reader(0u32)).unwrap(); + let mut scorer = weight + .specialized_scorer(searcher.segment_reader(0u32)) + .unwrap(); assert_eq!(scorer.skip_next(100), SkipResult::Reached); assert_eq!(scorer.skip_next(1002), SkipResult::Reached); assert_eq!(scorer.doc(), 1002); - assert_eq!(&[0,5], scorer.postings().positions()); + assert_eq!(&[0, 5], scorer.postings().positions()); } } diff --git a/src/postings/postings_writer.rs b/src/postings/postings_writer.rs index b63e0d527..d5061ebd2 100644 --- a/src/postings/postings_writer.rs +++ b/src/postings/postings_writer.rs @@ -1,15 +1,15 @@ use DocId; use schema::Term; -use postings::{InvertedIndexSerializer, FieldSerializer}; +use postings::{FieldSerializer, InvertedIndexSerializer}; use std::io; use std::collections::HashMap; use postings::Recorder; use Result; -use schema::{Schema, Field}; +use schema::{Field, Schema}; use std::marker::PhantomData; use std::ops::DerefMut; -use datastruct::stacker::{TermHashMap, Heap}; -use postings::{NothingRecorder, TermFrequencyRecorder, TFAndPositionRecorder}; +use datastruct::stacker::{Heap, TermHashMap}; +use postings::{NothingRecorder, TFAndPositionRecorder, TermFrequencyRecorder}; use schema::FieldEntry; use schema::FieldType; use tokenizer::Token; @@ -17,39 +17,31 @@ use tokenizer::TokenStream; use schema::IndexRecordOption; use postings::UnorderedTermId; - fn posting_from_field_entry<'a>( field_entry: &FieldEntry, heap: &'a Heap, ) -> Box { match *field_entry.field_type() { - FieldType::Str(ref text_options) => { - text_options + FieldType::Str(ref text_options) => text_options .get_indexing_options() - .map(|indexing_options| { - match indexing_options.index_option() { - IndexRecordOption::Basic => { - SpecializedPostingsWriter::::new_boxed(heap) - } - IndexRecordOption::WithFreqs => { - SpecializedPostingsWriter::::new_boxed(heap) - } - IndexRecordOption::WithFreqsAndPositions => { - SpecializedPostingsWriter::::new_boxed(heap) - } + .map(|indexing_options| match indexing_options.index_option() { + IndexRecordOption::Basic => { + SpecializedPostingsWriter::::new_boxed(heap) + } + IndexRecordOption::WithFreqs => { + SpecializedPostingsWriter::::new_boxed(heap) + } + IndexRecordOption::WithFreqsAndPositions => { + SpecializedPostingsWriter::::new_boxed(heap) } }) - .unwrap_or_else(|| { - SpecializedPostingsWriter::::new_boxed(heap) - }) + .unwrap_or_else(|| SpecializedPostingsWriter::::new_boxed(heap)), + FieldType::U64(_) | FieldType::I64(_) | FieldType::HierarchicalFacet => { + SpecializedPostingsWriter::::new_boxed(heap) } - FieldType::U64(_) | - FieldType::I64(_) | - FieldType::HierarchicalFacet => SpecializedPostingsWriter::::new_boxed(heap), } } - pub struct MultiFieldPostingsWriter<'a> { heap: &'a Heap, term_index: TermHashMap<'a>, @@ -88,7 +80,10 @@ impl<'a> MultiFieldPostingsWriter<'a> { /// It pushes all term, one field at a time, towards the /// postings serializer. #[allow(needless_range_loop)] - pub fn serialize(&self, serializer: &mut InvertedIndexSerializer) -> Result>> { + pub fn serialize( + &self, + serializer: &mut InvertedIndexSerializer, + ) -> Result>> { let mut term_offsets: Vec<(&[u8], u32, UnorderedTermId)> = self.term_index.iter().collect(); term_offsets.sort_by_key(|&(k, _, _)| k); @@ -99,8 +94,8 @@ impl<'a> MultiFieldPostingsWriter<'a> { .map(|(key, _, _)| Term::wrap(key).field()) .enumerate(); - - let mut unordered_term_mappings: HashMap> = HashMap::new(); + let mut unordered_term_mappings: HashMap> = + HashMap::new(); let mut prev_field = Field(u32::max_value()); for (offset, field) in term_offsets_it { @@ -120,8 +115,9 @@ impl<'a> MultiFieldPostingsWriter<'a> { let mut mapping = HashMap::new(); for (term_ord, term_unord_id) in term_offsets[start..stop] .iter() - .map(|&(_,_,bucket)| bucket) - .enumerate() { + .map(|&(_, _, bucket)| bucket) + .enumerate() + { mapping.insert(term_unord_id, term_ord); } unordered_term_mappings.insert(field, mapping); @@ -144,7 +140,6 @@ impl<'a> MultiFieldPostingsWriter<'a> { } } - /// The `PostingsWriter` is in charge of receiving documenting /// and building a `Segment` in anonymous memory. /// @@ -168,20 +163,22 @@ pub trait PostingsWriter { /// Serializes the postings on disk. /// The actual serialization format is handled by the `PostingsSerializer`. - fn serialize(&self, - term_addrs: &[(&[u8], u32, UnorderedTermId)], - serializer: &mut FieldSerializer, - heap: &Heap) - -> io::Result<()>; + fn serialize( + &self, + term_addrs: &[(&[u8], u32, UnorderedTermId)], + serializer: &mut FieldSerializer, + heap: &Heap, + ) -> io::Result<()>; /// Tokenize a text and subscribe all of its token. - fn index_text<'a>(&mut self, - term_index: &mut TermHashMap, - doc_id: DocId, - field: Field, - token_stream: &mut TokenStream, - heap: &Heap) - -> u32 { + fn index_text<'a>( + &mut self, + term_index: &mut TermHashMap, + doc_id: DocId, + field: Field, + token_stream: &mut TokenStream, + heap: &Heap, + ) -> u32 { let mut term = unsafe { Term::with_capacity(100) }; term.set_field(field); let mut sink = |token: &Token| { @@ -215,7 +212,6 @@ impl<'a, Rec: Recorder + 'static> SpecializedPostingsWriter<'a, Rec> { } impl<'a, Rec: Recorder + 'static> PostingsWriter for SpecializedPostingsWriter<'a, Rec> { - fn subscribe( &mut self, term_index: &mut TermHashMap, @@ -237,8 +233,6 @@ impl<'a, Rec: Recorder + 'static> PostingsWriter for SpecializedPostingsWriter<' term_ord } - - fn serialize( &self, term_addrs: &[(&[u8], u32, UnorderedTermId)], diff --git a/src/query/all_query.rs b/src/query/all_query.rs index 632693cef..da36bf38b 100644 --- a/src/query/all_query.rs +++ b/src/query/all_query.rs @@ -9,7 +9,6 @@ use DocId; use std::any::Any; use core::Searcher; - /// Query that matches all of the documents. /// /// All of the document get the score 1f32. @@ -34,12 +33,11 @@ impl Weight for AllWeight { Ok(box AllScorer { started: false, doc: 0u32, - max_doc: reader.max_doc() + max_doc: reader.max_doc(), }) } } - /// Scorer associated to the `AllQuery` query. pub struct AllScorer { started: bool, @@ -51,8 +49,7 @@ impl DocSet for AllScorer { fn advance(&mut self) -> bool { if self.started { self.doc += 1u32; - } - else { + } else { self.started = true; } self.doc < self.max_doc @@ -71,4 +68,4 @@ impl Scorer for AllScorer { fn score(&self) -> Score { 1f32 } -} \ No newline at end of file +} diff --git a/src/query/mod.rs b/src/query/mod.rs index 7177871b5..7303baebb 100644 --- a/src/query/mod.rs +++ b/src/query/mod.rs @@ -24,4 +24,4 @@ pub use self::scorer::EmptyScorer; pub use self::scorer::Scorer; pub use self::term_query::TermQuery; pub use self::weight::Weight; -pub use self::all_query::{AllQuery, AllWeight, AllScorer}; \ No newline at end of file +pub use self::all_query::{AllQuery, AllScorer, AllWeight}; diff --git a/src/query/phrase_query/mod.rs b/src/query/phrase_query/mod.rs index 8e78394cb..b1aaaa6ca 100644 --- a/src/query/phrase_query/mod.rs +++ b/src/query/phrase_query/mod.rs @@ -6,7 +6,6 @@ pub use self::phrase_query::PhraseQuery; pub use self::phrase_weight::PhraseWeight; pub use self::phrase_scorer::PhraseScorer; - #[cfg(test)] mod tests { @@ -75,8 +74,6 @@ mod tests { assert_eq!(test_query(vec!["g", "a"]), empty_vec); } - - #[test] // motivated by #234 pub fn test_phrase_query_docfreq_order() { let mut schema_builder = SchemaBuilder::default(); @@ -90,11 +87,13 @@ mod tests { let doc = doc!(text_field=>"b"); index_writer.add_document(doc); } - { // 1 + { + // 1 let doc = doc!(text_field=>"a b"); index_writer.add_document(doc); } - { // 2 + { + // 2 let doc = doc!(text_field=>"b a"); index_writer.add_document(doc); } diff --git a/src/query/phrase_query/phrase_scorer.rs b/src/query/phrase_query/phrase_scorer.rs index 4b7a26095..f6775cb60 100644 --- a/src/query/phrase_query/phrase_scorer.rs +++ b/src/query/phrase_query/phrase_scorer.rs @@ -1,6 +1,6 @@ use query::Scorer; use DocId; -use postings::{SkipResult, IntersectionDocSet, DocSet, Postings, SegmentPostings}; +use postings::{DocSet, IntersectionDocSet, Postings, SegmentPostings, SkipResult}; struct PostingsWithOffset { offset: u32, @@ -11,7 +11,7 @@ impl PostingsWithOffset { pub fn new(segment_postings: SegmentPostings, offset: u32) -> PostingsWithOffset { PostingsWithOffset { offset, - segment_postings + segment_postings, } } } @@ -49,7 +49,6 @@ pub struct PhraseScorer { } impl PhraseScorer { - pub fn new(term_postings: Vec) -> PhraseScorer { let postings_with_offsets: Vec<_> = term_postings .into_iter() @@ -57,12 +56,11 @@ impl PhraseScorer { .map(|(offset, postings)| PostingsWithOffset::new(postings, offset as u32)) .collect(); PhraseScorer { - intersection_docset: IntersectionDocSet::from(postings_with_offsets) + intersection_docset: IntersectionDocSet::from(postings_with_offsets), } } fn phrase_match(&self) -> bool { - // TODO maybe we could avoid decoding positions lazily for all terms // when there is > 2 terms. // @@ -74,7 +72,6 @@ impl PhraseScorer { positions_arr[docset.offset as usize] = docset.positions(); } - let num_postings = positions_arr.len() as u32; let mut ord = 1u32; diff --git a/src/query/phrase_query/phrase_weight.rs b/src/query/phrase_query/phrase_weight.rs index 5c73bd2a0..b70ba3747 100644 --- a/src/query/phrase_query/phrase_weight.rs +++ b/src/query/phrase_query/phrase_weight.rs @@ -23,7 +23,8 @@ impl Weight for PhraseWeight { for term in &self.phrase_terms { if let Some(postings) = reader .inverted_index(term.field()) - .read_postings(term, IndexRecordOption::WithFreqsAndPositions) { + .read_postings(term, IndexRecordOption::WithFreqsAndPositions) + { term_postings_list.push(postings); } else { return Ok(box EmptyScorer); diff --git a/src/schema/document.rs b/src/schema/document.rs index 7c15d9c16..82a07f184 100644 --- a/src/schema/document.rs +++ b/src/schema/document.rs @@ -21,9 +21,7 @@ pub struct Document { impl From> for Document { fn from(field_values: Vec) -> Self { - Document { - field_values - } + Document { field_values } } } @@ -38,7 +36,6 @@ impl PartialEq for Document { } } - impl Eq for Document {} impl Document { @@ -59,14 +56,16 @@ impl Document { /// Retain only the field that are matching the /// predicate given in argument. - pub fn filter_fieldsbool>(&mut self, predicate: P) { + pub fn filter_fields bool>(&mut self, predicate: P) { self.field_values .retain(|field_value| predicate(field_value.field())); } /// Adding a facet to the document. pub fn add_facet(&mut self, field: Field, path: F) - where Facet: From { + where + Facet: From, + { let facet = Facet::from(path); let value = Value::Facet(facet); self.add(FieldValue::new(field, value)); @@ -144,9 +143,7 @@ impl BinarySerializable for Document { fn deserialize(reader: &mut R) -> io::Result { let num_field_values = VInt::deserialize(reader)?.val() as usize; let field_values = (0..num_field_values) - .map(|_| { - FieldValue::deserialize(reader) - }) + .map(|_| FieldValue::deserialize(reader)) .collect::>>()?; Ok(Document::from(field_values)) } diff --git a/src/schema/facet.rs b/src/schema/facet.rs index 9c67fb8bb..15e2dca65 100644 --- a/src/schema/facet.rs +++ b/src/schema/facet.rs @@ -1,4 +1,4 @@ -use std::fmt::{self, Display, Debug, Formatter}; +use std::fmt::{self, Debug, Display, Formatter}; use std::str; use std::io::{self, Read, Write}; use regex::Regex; @@ -7,7 +7,6 @@ use serde::{Deserialize, Deserializer, Serialize, Serializer}; use std::borrow::Cow; use common::BinarySerializable; - const SLASH_BYTE: u8 = '/' as u8; const ESCAPE_BYTE: u8 = '\\' as u8; @@ -30,7 +29,6 @@ pub const FACET_SEP_BYTE: u8 = 0u8; pub struct Facet(Vec); impl Facet { - /// Returns a new instance of the "root facet" /// Equivalent to `/`. pub fn root() -> Facet { @@ -66,7 +64,9 @@ impl Facet { /// contains a `/` or a `\`, it should be escaped /// using an anti-slash `/`. pub fn from_text<'a, T>(path: &'a T) -> Facet - where T: ?Sized + AsRef { + where + T: ?Sized + AsRef, + { From::from(path) } @@ -75,9 +75,10 @@ impl Facet { /// /// The steps are expected to be unescaped. pub fn from_path(path: Path) -> Facet - where - Path: IntoIterator, - Path::Item: ToString { + where + Path: IntoIterator, + Path::Item: ToString, + { let mut facet_bytes: Vec = Vec::with_capacity(100); let mut step_it = path.into_iter(); if let Some(step) = step_it.next() { @@ -95,7 +96,6 @@ impl Facet { &mut self.0 } - /// Returns `true` iff other is a subfacet of `self`. pub fn is_prefix_of(&self, other: &Facet) -> bool { let self_bytes: &[u8] = self.encoded_bytes(); @@ -116,7 +116,6 @@ impl Borrow<[u8]> for Facet { } impl<'a, T: ?Sized + AsRef> From<&'a T> for Facet { - fn from(path_asref: &'a T) -> Facet { #[derive(Copy, Clone)] enum State { @@ -129,9 +128,7 @@ impl<'a, T: ?Sized + AsRef> From<&'a T> for Facet { let path_bytes = path.as_bytes(); for &c in &path_bytes[1..] { match (state, c) { - (State::Idle, ESCAPE_BYTE) => { - state = State::Escaped - } + (State::Idle, ESCAPE_BYTE) => state = State::Escaped, (State::Idle, SLASH_BYTE) => { facet_encoded.push(FACET_SEP_BYTE); } @@ -179,16 +176,19 @@ fn escape_slashes(s: &str) -> Cow { impl Serialize for Facet { fn serialize(&self, serializer: S) -> Result - where S: Serializer { + where + S: Serializer, + { serializer.serialize_str(&self.to_string()) } } impl<'de> Deserialize<'de> for Facet { - fn deserialize(deserializer: D) -> Result where - D: Deserializer<'de> { - <&'de str as Deserialize<'de>>::deserialize(deserializer) - .map(Facet::from) + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + <&'de str as Deserialize<'de>>::deserialize(deserializer).map(Facet::from) } } @@ -199,7 +199,6 @@ impl Debug for Facet { } } - #[cfg(test)] mod tests { @@ -226,7 +225,6 @@ mod tests { } } - #[test] fn test_facet_debug() { let v = ["first", "second", "third"]; @@ -234,4 +232,4 @@ mod tests { assert_eq!(format!("{:?}", facet), "Facet(/first/second/third)"); } -} \ No newline at end of file +} diff --git a/src/schema/field_entry.rs b/src/schema/field_entry.rs index 5d5d49273..f120375bd 100644 --- a/src/schema/field_entry.rs +++ b/src/schema/field_entry.rs @@ -71,7 +71,7 @@ impl FieldEntry { match self.field_type { FieldType::Str(ref options) => options.get_indexing_options().is_some(), FieldType::U64(ref options) | FieldType::I64(ref options) => options.is_indexed(), - FieldType::HierarchicalFacet => true + FieldType::HierarchicalFacet => true, } } diff --git a/src/schema/field_type.rs b/src/schema/field_type.rs index 3a39490b3..3069a3b37 100644 --- a/src/schema/field_type.rs +++ b/src/schema/field_type.rs @@ -39,7 +39,7 @@ impl FieldType { FieldType::U64(ref int_options) | FieldType::I64(ref int_options) => { int_options.is_indexed() } - FieldType::HierarchicalFacet => true + FieldType::HierarchicalFacet => true, } } @@ -59,7 +59,7 @@ impl FieldType { None } } - FieldType::HierarchicalFacet => Some(IndexRecordOption::Basic) + FieldType::HierarchicalFacet => Some(IndexRecordOption::Basic), } } @@ -75,9 +75,7 @@ impl FieldType { FieldType::U64(_) | FieldType::I64(_) => Err(ValueParsingError::TypeError( format!("Expected an integer, got {:?}", json), )), - FieldType::HierarchicalFacet => { - Ok(Value::Facet(Facet::from(field_text))) - } + FieldType::HierarchicalFacet => Ok(Value::Facet(Facet::from(field_text))), }, JsonValue::Number(ref field_val_num) => match *self { FieldType::I64(_) => { diff --git a/src/schema/int_options.rs b/src/schema/int_options.rs index f47d79319..cd1fd8a22 100644 --- a/src/schema/int_options.rs +++ b/src/schema/int_options.rs @@ -1,6 +1,5 @@ use std::ops::BitOr; - /// Express whether a field is single-value or multi-valued. #[derive(Clone, Copy, PartialEq, Eq, Debug, Serialize, Deserialize)] pub enum Cardinality { @@ -10,15 +9,14 @@ pub enum Cardinality { /// The document can have any number of values associated to the document. /// This is more memory and CPU expensive than the SingleValue solution. #[serde(rename = "multi")] - MultiValues + MultiValues, } /// Define how an int field should be handled by tantivy. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct IntOptions { indexed: bool, - #[serde(skip_serializing_if="Option::is_none")] - fast: Option, + #[serde(skip_serializing_if = "Option::is_none")] fast: Option, stored: bool, } @@ -86,7 +84,6 @@ impl Default for IntOptions { } } - /// Shortcut for a u64 fast field. /// /// Such a shortcut can be composed as follows `STORED | FAST | INT_INDEXED` @@ -114,7 +111,6 @@ pub const INT_STORED: IntOptions = IntOptions { fast: None, }; - impl BitOr for IntOptions { type Output = IntOptions; diff --git a/src/schema/schema.rs b/src/schema/schema.rs index 864992790..b93944b8e 100644 --- a/src/schema/schema.rs +++ b/src/schema/schema.rs @@ -334,8 +334,12 @@ mod tests { #[test] pub fn test_schema_serialization() { let mut schema_builder = SchemaBuilder::default(); - let count_options = IntOptions::default().set_stored().set_fast(Cardinality::SingleValue); - let popularity_options = IntOptions::default().set_stored().set_fast(Cardinality::SingleValue); + let count_options = IntOptions::default() + .set_stored() + .set_fast(Cardinality::SingleValue); + let popularity_options = IntOptions::default() + .set_stored() + .set_fast(Cardinality::SingleValue); schema_builder.add_text_field("title", TEXT); schema_builder.add_text_field("author", STRING); schema_builder.add_u64_field("count", count_options); @@ -399,7 +403,9 @@ mod tests { #[test] pub fn test_document_to_json() { let mut schema_builder = SchemaBuilder::default(); - let count_options = IntOptions::default().set_stored().set_fast(Cardinality::SingleValue); + let count_options = IntOptions::default() + .set_stored() + .set_fast(Cardinality::SingleValue); schema_builder.add_text_field("title", TEXT); schema_builder.add_text_field("author", STRING); schema_builder.add_u64_field("count", count_options); @@ -418,8 +424,12 @@ mod tests { #[test] pub fn test_parse_document() { let mut schema_builder = SchemaBuilder::default(); - let count_options = IntOptions::default().set_stored().set_fast(Cardinality::SingleValue); - let popularity_options = IntOptions::default().set_stored().set_fast(Cardinality::SingleValue); + let count_options = IntOptions::default() + .set_stored() + .set_fast(Cardinality::SingleValue); + let popularity_options = IntOptions::default() + .set_stored() + .set_fast(Cardinality::SingleValue); let title_field = schema_builder.add_text_field("title", TEXT); let author_field = schema_builder.add_text_field("author", STRING); let count_field = schema_builder.add_u64_field("count", count_options); diff --git a/src/schema/text_options.rs b/src/schema/text_options.rs index 18202847e..401ed65a9 100644 --- a/src/schema/text_options.rs +++ b/src/schema/text_options.rs @@ -44,7 +44,7 @@ impl Default for TextOptions { /// Configuration defining indexing for a text field. /// It wraps: -/// +/// /// * record (See [`IndexRecordOption`](./enum.IndexRecordOption.html)) /// * tokenizer #[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)] diff --git a/src/schema/value.rs b/src/schema/value.rs index eaf66f101..90e573ade 100644 --- a/src/schema/value.rs +++ b/src/schema/value.rs @@ -26,7 +26,7 @@ impl Serialize for Value { Value::Str(ref v) => serializer.serialize_str(v), Value::U64(u) => serializer.serialize_u64(u), Value::I64(u) => serializer.serialize_i64(u), - Value::Facet(ref facet) => facet.serialize(serializer) + Value::Facet(ref facet) => facet.serialize(serializer), } } } @@ -178,9 +178,7 @@ mod binary_serialize { let value = i64::deserialize(reader)?; Ok(Value::I64(value)) } - HIERARCHICAL_FACET_CODE => { - Ok(Value::Facet(Facet::deserialize(reader)?)) - } + HIERARCHICAL_FACET_CODE => Ok(Value::Facet(Facet::deserialize(reader)?)), _ => Err(io::Error::new( io::ErrorKind::InvalidData, format!("No field type is associated with code {:?}", type_code), diff --git a/src/store/reader.rs b/src/store/reader.rs index 4ed8cdab2..7f4343f8f 100644 --- a/src/store/reader.rs +++ b/src/store/reader.rs @@ -39,9 +39,7 @@ impl StoreReader { } fn block_offset(&self, doc_id: DocId) -> (DocId, u64) { - self.block_index() - .seek(doc_id + 1) - .unwrap_or((0u32, 0u64)) + self.block_index().seek(doc_id + 1).unwrap_or((0u32, 0u64)) } pub(crate) fn block_data(&self) -> &[u8] { diff --git a/src/store/writer.rs b/src/store/writer.rs index 1742a3d00..402158278 100644 --- a/src/store/writer.rs +++ b/src/store/writer.rs @@ -1,6 +1,6 @@ use directory::WritePtr; use DocId; -use common::{VInt, BinarySerializable}; +use common::{BinarySerializable, VInt}; use std::io::{self, Write}; use super::StoreReader; use lz4; @@ -66,10 +66,8 @@ impl StoreWriter { pub fn stack(&mut self, store_reader: &StoreReader) -> io::Result<()> { if !self.current_block.is_empty() { self.write_and_compress_block()?; - self.offset_index_writer.insert( - self.doc, - &(self.writer.written_bytes() as u64), - )?; + self.offset_index_writer + .insert(self.doc, &(self.writer.written_bytes() as u64))?; } let doc_offset = self.doc; let start_offset = self.writer.written_bytes() as u64; @@ -81,9 +79,8 @@ impl StoreWriter { // its start doc id and its start file offset. for (next_doc_id, block_addr) in store_reader.block_index() { self.doc = doc_offset + next_doc_id; - self.offset_index_writer.insert( - self.doc, - &(start_offset + block_addr))?; + self.offset_index_writer + .insert(self.doc, &(start_offset + block_addr))?; } Ok(()) } diff --git a/src/termdict/fstdict/streamer.rs b/src/termdict/fstdict/streamer.rs index 9af6d42e2..4ebc3630c 100644 --- a/src/termdict/fstdict/streamer.rs +++ b/src/termdict/fstdict/streamer.rs @@ -1,8 +1,8 @@ use fst::{IntoStreamer, Streamer}; -use fst::map::{StreamBuilder, Stream}; +use fst::map::{Stream, StreamBuilder}; use postings::TermInfo; use super::TermDictionaryImpl; -use termdict::{TermOrdinal, TermDictionary, TermStreamerBuilder, TermStreamer}; +use termdict::{TermDictionary, TermOrdinal, TermStreamer, TermStreamerBuilder}; /// See [`TermStreamerBuilder`](./trait.TermStreamerBuilder.html) pub struct TermStreamerBuilderImpl<'a> { @@ -53,7 +53,6 @@ impl<'a> TermStreamerBuilder for TermStreamerBuilderImpl<'a> { } } - /// See [`TermStreamer`](./trait.TermStreamer.html) pub struct TermStreamerImpl<'a> { fst_map: &'a TermDictionaryImpl, @@ -88,4 +87,3 @@ impl<'a> TermStreamer for TermStreamerImpl<'a> { &self.current_value } } - diff --git a/src/termdict/fstdict/termdict.rs b/src/termdict/fstdict/termdict.rs index 9ae69d914..96c2be3f9 100644 --- a/src/termdict/fstdict/termdict.rs +++ b/src/termdict/fstdict/termdict.rs @@ -6,7 +6,7 @@ use common::BinarySerializable; use schema::FieldType; use postings::TermInfo; use termdict::{TermDictionary, TermDictionaryBuilder, TermOrdinal}; -use super::{TermStreamerImpl, TermStreamerBuilderImpl}; +use super::{TermStreamerBuilderImpl, TermStreamerImpl}; fn convert_fst_error(e: fst::Error) -> io::Error { io::Error::new(io::ErrorKind::Other, e) @@ -95,7 +95,6 @@ pub struct TermDictionaryImpl { values_mmap: ReadOnlySource, } - impl<'a> TermDictionary<'a> for TermDictionaryImpl { type Streamer = TermStreamerImpl<'a>; @@ -105,9 +104,8 @@ impl<'a> TermDictionary<'a> for TermDictionaryImpl { let total_len = source.len(); let length_offset = total_len - 4; let mut split_len_buffer: &[u8] = &source.as_slice()[length_offset..]; - let footer_size = u32::deserialize(&mut split_len_buffer).expect( - "Deserializing 4 bytes should always work", - ) as usize; + let footer_size = u32::deserialize(&mut split_len_buffer) + .expect("Deserializing 4 bytes should always work") as usize; let split_len = length_offset - footer_size; let fst_source = source.slice(0, split_len); let values_source = source.slice(split_len, length_offset); @@ -128,16 +126,14 @@ impl<'a> TermDictionary<'a> for TermDictionaryImpl { let mut node = fst.root(); while ord != 0 || !node.is_final() { if let Some(transition) = node.transitions() - .take_while(|transition| { - transition.out.value() <= ord - }) - .last() { + .take_while(|transition| transition.out.value() <= ord) + .last() + { ord -= transition.out.value(); bytes.push(transition.inp); let new_node_addr = transition.addr; node = fst.node(new_node_addr); - } - else { + } else { return false; } } diff --git a/src/termdict/mod.rs b/src/termdict/mod.rs index ef07aae86..5f84d933a 100644 --- a/src/termdict/mod.rs +++ b/src/termdict/mod.rs @@ -48,34 +48,30 @@ followed by a streaming through at most `1024` elements in the term `stream`. */ -use schema::{Term, Field, FieldType}; +use schema::{Field, FieldType, Term}; use directory::ReadOnlySource; use postings::TermInfo; - /// Position of the term in the sorted list of terms. pub type TermOrdinal = u64; - pub use self::merger::TermMerger; #[cfg(not(feature = "streamdict"))] mod fstdict; #[cfg(not(feature = "streamdict"))] -pub use self::fstdict::{TermDictionaryImpl, TermDictionaryBuilderImpl, TermStreamerImpl, - TermStreamerBuilderImpl}; - +pub use self::fstdict::{TermDictionaryBuilderImpl, TermDictionaryImpl, TermStreamerBuilderImpl, + TermStreamerImpl}; #[cfg(feature = "streamdict")] mod streamdict; #[cfg(feature = "streamdict")] -pub use self::streamdict::{TermDictionaryImpl, TermDictionaryBuilderImpl, TermStreamerImpl, - TermStreamerBuilderImpl}; +pub use self::streamdict::{TermDictionaryBuilderImpl, TermDictionaryImpl, TermStreamerBuilderImpl, + TermStreamerImpl}; mod merger; use std::io; - /// Dictionary associating sorted `&[u8]` to values pub trait TermDictionary<'a> where @@ -154,7 +150,6 @@ where fn finish(self) -> io::Result; } - /// `TermStreamer` acts as a cursor over a range of terms of a segment. /// Terms are guaranteed to be sorted. pub trait TermStreamer: Sized { @@ -202,7 +197,6 @@ pub trait TermStreamer: Sized { } } - /// `TermStreamerBuilder` is an helper object used to define /// a range of terms that should be streamed. pub trait TermStreamerBuilder { @@ -226,13 +220,12 @@ pub trait TermStreamerBuilder { fn into_stream(self) -> Self::Streamer; } - #[cfg(test)] mod tests { - use super::{TermDictionaryImpl, TermDictionaryBuilderImpl, TermStreamerImpl}; - use directory::{RAMDirectory, Directory, ReadOnlySource}; + use super::{TermDictionaryBuilderImpl, TermDictionaryImpl, TermStreamerImpl}; + use directory::{Directory, RAMDirectory, ReadOnlySource}; use std::path::PathBuf; - use schema::{FieldType, SchemaBuilder, Document, TEXT}; + use schema::{Document, FieldType, SchemaBuilder, TEXT}; use core::Index; use std::str; use termdict::TermStreamer; @@ -243,17 +236,15 @@ mod tests { const BLOCK_SIZE: usize = 1_500; - fn make_term_info(val: u64) -> TermInfo { TermInfo { doc_freq: val as u32, - positions_offset: val * 2u64, + positions_offset: val * 2u64, postings_offset: val * 3u64, positions_inner_offset: 5u8, } } - #[test] fn test_term_ordinals() { const COUNTRIES: [&'static str; 7] = [ @@ -263,15 +254,15 @@ mod tests { "Slovenia", "Spain", "Sweden", - "Switzerland" + "Switzerland", ]; let mut directory = RAMDirectory::create(); let path = PathBuf::from("TermDictionary"); { let write = directory.open_write(&path).unwrap(); let field_type = FieldType::Str(TEXT); - let mut term_dictionary_builder = TermDictionaryBuilderImpl::new(write, field_type) - .unwrap(); + let mut term_dictionary_builder = + TermDictionaryBuilderImpl::new(write, field_type).unwrap(); for term in COUNTRIES.iter() { term_dictionary_builder .insert(term.as_bytes(), &make_term_info(0u64)) @@ -283,7 +274,7 @@ mod tests { let term_dict: TermDictionaryImpl = TermDictionaryImpl::from_source(source); for (term_ord, term) in COUNTRIES.iter().enumerate() { assert_eq!(term_dict.term_ord(term).unwrap(), term_ord as u64); - let mut bytes = vec!(); + let mut bytes = vec![]; assert!(term_dict.ord_to_term(term_ord as u64, &mut bytes)); assert_eq!(bytes, term.as_bytes()); } @@ -296,8 +287,8 @@ mod tests { { let write = directory.open_write(&path).unwrap(); let field_type = FieldType::Str(TEXT); - let mut term_dictionary_builder = TermDictionaryBuilderImpl::new(write, field_type) - .unwrap(); + let mut term_dictionary_builder = + TermDictionaryBuilderImpl::new(write, field_type).unwrap(); term_dictionary_builder .insert("abc".as_bytes(), &make_term_info(34u64)) .unwrap(); @@ -377,7 +368,6 @@ mod tests { assert_eq!(&*term_string, "abcdef"); } - #[test] fn test_term_dictionary_stream() { let ids: Vec<_> = (0u32..10_000u32) @@ -385,8 +375,8 @@ mod tests { .collect(); let field_type = FieldType::Str(TEXT); let buffer: Vec = { - let mut term_dictionary_builder = TermDictionaryBuilderImpl::new(vec![], field_type) - .unwrap(); + let mut term_dictionary_builder = + TermDictionaryBuilderImpl::new(vec![], field_type).unwrap(); for &(ref id, ref i) in &ids { term_dictionary_builder .insert(id.as_bytes(), &make_term_info(*i as u64)) @@ -411,13 +401,12 @@ mod tests { term_dictionary.get(key.as_bytes()); } - #[test] fn test_stream_high_range_prefix_suffix() { let field_type = FieldType::Str(TEXT); let buffer: Vec = { - let mut term_dictionary_builder = TermDictionaryBuilderImpl::new(vec![], field_type) - .unwrap(); + let mut term_dictionary_builder = + TermDictionaryBuilderImpl::new(vec![], field_type).unwrap(); // term requires more than 16bits term_dictionary_builder .insert("abcdefghijklmnopqrstuvwxy", &make_term_info(1)) @@ -451,8 +440,8 @@ mod tests { .collect(); let field_type = FieldType::Str(TEXT); let buffer: Vec = { - let mut term_dictionary_builder = TermDictionaryBuilderImpl::new(vec![], field_type) - .unwrap(); + let mut term_dictionary_builder = + TermDictionaryBuilderImpl::new(vec![], field_type).unwrap(); for &(ref id, ref i) in &ids { term_dictionary_builder .insert(id.as_bytes(), &make_term_info(*i as u64)) @@ -520,14 +509,15 @@ mod tests { fn test_empty_string() { let field_type = FieldType::Str(TEXT); let buffer: Vec = { - let mut term_dictionary_builder = TermDictionaryBuilderImpl::new(vec![], field_type) + let mut term_dictionary_builder = + TermDictionaryBuilderImpl::new(vec![], field_type).unwrap(); + term_dictionary_builder + .insert(&[], &make_term_info(1 as u64)) .unwrap(); term_dictionary_builder - .insert(&[], &make_term_info(1 as u64)).unwrap(); - term_dictionary_builder - .insert(&[1u8], &make_term_info(2 as u64)).unwrap(); - term_dictionary_builder - .finish().unwrap() + .insert(&[1u8], &make_term_info(2 as u64)) + .unwrap(); + term_dictionary_builder.finish().unwrap() }; let source = ReadOnlySource::from(buffer); let term_dictionary: TermDictionaryImpl = TermDictionaryImpl::from_source(source); @@ -543,8 +533,8 @@ mod tests { fn test_stream_range_boundaries() { let field_type = FieldType::Str(TEXT); let buffer: Vec = { - let mut term_dictionary_builder = TermDictionaryBuilderImpl::new(vec![], field_type) - .unwrap(); + let mut term_dictionary_builder = + TermDictionaryBuilderImpl::new(vec![], field_type).unwrap(); for i in 0u8..10u8 { let number_arr = [i; 1]; term_dictionary_builder diff --git a/src/termdict/streamdict/delta_encoder.rs b/src/termdict/streamdict/delta_encoder.rs index cf47105f5..b7e64f3cf 100644 --- a/src/termdict/streamdict/delta_encoder.rs +++ b/src/termdict/streamdict/delta_encoder.rs @@ -49,7 +49,6 @@ impl TermDeltaDecoder { } } - // code // first bit represents whether the prefix / suffix len can be encoded // on the same byte. (the next one) @@ -57,18 +56,17 @@ impl TermDeltaDecoder { #[inline(always)] pub fn decode<'a>(&mut self, code: u8, mut cursor: &'a [u8]) -> &'a [u8] { - let (prefix_len, suffix_len): (usize, usize) = - if (code & 1u8) == 1u8 { - let b = cursor[0]; - cursor = &cursor[1..]; - let prefix_len = (b & 15u8) as usize; - let suffix_len = (b >> 4u8) as usize; - (prefix_len, suffix_len) - } else { - let prefix_len = u32::deserialize(&mut cursor).unwrap(); - let suffix_len = u32::deserialize(&mut cursor).unwrap(); - (prefix_len as usize, suffix_len as usize) - }; + let (prefix_len, suffix_len): (usize, usize) = if (code & 1u8) == 1u8 { + let b = cursor[0]; + cursor = &cursor[1..]; + let prefix_len = (b & 15u8) as usize; + let suffix_len = (b >> 4u8) as usize; + (prefix_len, suffix_len) + } else { + let prefix_len = u32::deserialize(&mut cursor).unwrap(); + let suffix_len = u32::deserialize(&mut cursor).unwrap(); + (prefix_len as usize, suffix_len as usize) + }; unsafe { self.term.set_len(prefix_len) }; self.term.extend_from_slice(&(*cursor)[..suffix_len]); &cursor[suffix_len..] diff --git a/src/tokenizer/alphanum_only.rs b/src/tokenizer/alphanum_only.rs index c3bcab050..b5c70178c 100644 --- a/src/tokenizer/alphanum_only.rs +++ b/src/tokenizer/alphanum_only.rs @@ -6,32 +6,28 @@ use super::{Token, TokenFilter, TokenStream}; pub struct AlphaNumOnlyFilter; pub struct AlphaNumOnlyFilterStream - where TailTokenStream: TokenStream +where + TailTokenStream: TokenStream, { tail: TailTokenStream, } - impl AlphaNumOnlyFilterStream - where TailTokenStream: TokenStream +where + TailTokenStream: TokenStream, { fn predicate(&self, token: &Token) -> bool { token.text.chars().all(|c| c.is_ascii_alphanumeric()) } - fn wrap( - tail: TailTokenStream, - ) -> AlphaNumOnlyFilterStream { - AlphaNumOnlyFilterStream { - tail - } + fn wrap(tail: TailTokenStream) -> AlphaNumOnlyFilterStream { + AlphaNumOnlyFilterStream { tail } } } - impl TokenFilter for AlphaNumOnlyFilter - where - TailTokenStream: TokenStream, +where + TailTokenStream: TokenStream, { type ResultTokenStream = AlphaNumOnlyFilterStream; @@ -41,8 +37,8 @@ impl TokenFilter for AlphaNumOnlyFilter } impl TokenStream for AlphaNumOnlyFilterStream - where - TailTokenStream: TokenStream +where + TailTokenStream: TokenStream, { fn token(&self) -> &Token { self.tail.token() diff --git a/src/tokenizer/facet_tokenizer.rs b/src/tokenizer/facet_tokenizer.rs index b6138ec7a..982c35f7b 100644 --- a/src/tokenizer/facet_tokenizer.rs +++ b/src/tokenizer/facet_tokenizer.rs @@ -1,8 +1,7 @@ -use super::{Token, Tokenizer, TokenStream}; +use super::{Token, TokenStream, Tokenizer}; use std::str; use schema::FACET_SEP_BYTE; - /// The `FacetTokenizer` process a `Facet` binary representation /// and emits a token for all of its parent. /// @@ -39,27 +38,27 @@ impl<'a> Tokenizer<'a> for FacetTokenizer { } } - impl<'a> TokenStream for FacetTokenStream<'a> { fn advance(&mut self) -> bool { match self.state { State::RootFacetNotEmitted => { - self.state = - if self.text.is_empty() { - State::Terminated - } else { - State::UpToPosition(0) - }; + self.state = if self.text.is_empty() { + State::Terminated + } else { + State::UpToPosition(0) + }; true } State::UpToPosition(cursor) => { let bytes: &[u8] = self.text.as_bytes(); - if let Some(next_sep_pos) = bytes[cursor+1..] + if let Some(next_sep_pos) = bytes[cursor + 1..] .iter() .cloned() .position(|b| b == FACET_SEP_BYTE) - .map(|pos| cursor + 1 + pos) { - let facet_part = unsafe { str::from_utf8_unchecked(&bytes[cursor..next_sep_pos]) }; + .map(|pos| cursor + 1 + pos) + { + let facet_part = + unsafe { str::from_utf8_unchecked(&bytes[cursor..next_sep_pos]) }; self.token.text.push_str(facet_part); self.state = State::UpToPosition(next_sep_pos); } else { @@ -69,9 +68,7 @@ impl<'a> TokenStream for FacetTokenStream<'a> { } true } - State::Terminated => { - false - } + State::Terminated => false, } } @@ -87,7 +84,7 @@ impl<'a> TokenStream for FacetTokenStream<'a> { #[cfg(test)] mod tests { - use tokenizer::{TokenStream, Token, Tokenizer}; + use tokenizer::{Token, TokenStream, Tokenizer}; use super::FacetTokenizer; use schema::Facet; @@ -101,7 +98,9 @@ mod tests { tokens.push(format!("{}", facet)); }; FacetTokenizer - .token_stream(unsafe { ::std::str::from_utf8_unchecked(facet.encoded_bytes()) }) + .token_stream(unsafe { + ::std::str::from_utf8_unchecked(facet.encoded_bytes()) + }) .process(&mut add_token); } assert_eq!(tokens.len(), 4); @@ -121,10 +120,12 @@ mod tests { tokens.push(format!("{}", facet)); }; FacetTokenizer - .token_stream(unsafe { ::std::str::from_utf8_unchecked(facet.encoded_bytes()) }) + .token_stream(unsafe { + ::std::str::from_utf8_unchecked(facet.encoded_bytes()) + }) .process(&mut add_token); } assert_eq!(tokens.len(), 1); assert_eq!(tokens[0], "/"); } -} \ No newline at end of file +} diff --git a/src/tokenizer/tokenizer.rs b/src/tokenizer/tokenizer.rs index 2909bcdd1..bbd2d0d41 100644 --- a/src/tokenizer/tokenizer.rs +++ b/src/tokenizer/tokenizer.rs @@ -253,7 +253,6 @@ where } } - /// Trait for the pluggable components of `Tokenizer`s. pub trait TokenFilter: Clone { /// The resulting `TokenStream` type.