diff --git a/src/common/bitpacker.rs b/src/common/bitpacker.rs index 14a96312f..1a8b8208d 100644 --- a/src/common/bitpacker.rs +++ b/src/common/bitpacker.rs @@ -82,6 +82,9 @@ impl BitPacker { pub fn close(&mut self, output: &mut TWrite) -> io::Result { self.flush(output)?; + // Padding the write file to simplify reads. + output.write_all(&[0u8; 7])?; + self.written_size += 7; Ok(self.written_size) } } @@ -117,17 +120,8 @@ impl BitUnpacker { let addr = (idx * self.num_bits) / 8; let bit_shift = idx * self.num_bits - addr * 8; let val_unshifted_unmasked: u64; - val_unshifted_unmasked = if addr + 8 <= self.data_len { - unsafe { *(self.data_ptr.offset(addr as isize) as *const u64) } - } else { - let mut arr = [0u8; 8]; - if addr < self.data_len { - for i in 0..self.data_len - addr { - arr[i] = unsafe { *self.data_ptr.offset((addr + i) as isize) }; - } - } - unsafe { mem::transmute::<[u8; 8], u64>(arr) } - }; + debug_assert!(addr + 8 <= self.data_len, "The fast field field should have been padded with 7 bytes."); + val_unshifted_unmasked = unsafe { *(self.data_ptr.offset(addr as isize) as *const u64) }; let val_shifted = (val_unshifted_unmasked >> bit_shift) as u64; (val_shifted & self.mask) } @@ -135,7 +129,7 @@ impl BitUnpacker { - + #[cfg(test)] mod test { use super::{BitPacker, BitUnpacker, compute_num_bits}; @@ -163,7 +157,7 @@ mod test { bitpacker.write(val, &mut data).unwrap(); } let num_bytes = bitpacker.close(&mut data).unwrap(); - assert_eq!(num_bytes, (num_bits * len + 7) / 8); + assert_eq!(num_bytes, (num_bits * len + 7) / 8 + 7); assert_eq!(data.len(), num_bytes); let bitunpacker = BitUnpacker::new(&data, num_bits); for (i, val) in vals.iter().enumerate() { diff --git a/src/fastfield/mod.rs b/src/fastfield/mod.rs index d470f041f..061ed3910 100644 --- a/src/fastfield/mod.rs +++ b/src/fastfield/mod.rs @@ -92,7 +92,7 @@ mod tests { } let source = directory.open_read(&path).unwrap(); { - assert_eq!(source.len(), 31 as usize); + assert_eq!(source.len(), 38 as usize); } { let fast_field_readers = FastFieldsReader::open(source).unwrap(); @@ -126,7 +126,7 @@ mod tests { } let source = directory.open_read(&path).unwrap(); { - assert_eq!(source.len(), 56 as usize); + assert_eq!(source.len(), 63 as usize); } { let fast_field_readers = FastFieldsReader::open(source).unwrap(); @@ -162,7 +162,7 @@ mod tests { } let source = directory.open_read(&path).unwrap(); { - assert_eq!(source.len(), 29 as usize); + assert_eq!(source.len(), 36 as usize); } { let fast_field_readers = FastFieldsReader::open(source).unwrap(); @@ -195,7 +195,7 @@ mod tests { } let source = directory.open_read(&path).unwrap(); { - assert_eq!(source.len(), 80037 as usize); + assert_eq!(source.len(), 80044 as usize); } { let fast_field_readers = FastFieldsReader::open(source).unwrap(); @@ -232,7 +232,7 @@ mod tests { } let source = directory.open_read(&path).unwrap(); { - assert_eq!(source.len(), 17704 as usize); + assert_eq!(source.len(), 17711 as usize); } { let fast_field_readers = FastFieldsReader::open(source).unwrap(); diff --git a/src/indexer/index_writer.rs b/src/indexer/index_writer.rs index 3db8aa339..94d9d9567 100644 --- a/src/indexer/index_writer.rs +++ b/src/indexer/index_writer.rs @@ -159,7 +159,7 @@ pub fn open_index_writer(index: &Index, pub fn compute_deleted_bitset(delete_bitset: &mut BitSet, segment_reader: &SegmentReader, delete_cursor: &mut DeleteCursor, - doc_opstamps: DocToOpstampMapping, + doc_opstamps: &DocToOpstampMapping, target_opstamp: u64) -> Result { @@ -225,7 +225,7 @@ pub fn advance_deletes(mut segment: Segment, compute_deleted_bitset(&mut delete_bitset, &segment_reader, delete_cursor, - DocToOpstampMapping::None, + &DocToOpstampMapping::None, target_opstamp)?; for doc in 0u32..max_doc { @@ -285,7 +285,7 @@ fn index_documents(heap: &mut Heap, let may_have_deletes = compute_deleted_bitset(&mut deleted_bitset, &segment_reader, &mut delete_cursor, - doc_to_opstamps, + &doc_to_opstamps, last_docstamp)?; let segment_entry = SegmentEntry::new(segment_meta, delete_cursor, { diff --git a/src/query/boolean_query/mod.rs b/src/query/boolean_query/mod.rs index 21ea30575..c9dfdfb53 100644 --- a/src/query/boolean_query/mod.rs +++ b/src/query/boolean_query/mod.rs @@ -113,7 +113,7 @@ mod tests { let occurs = vec![Occur::Should, Occur::Should]; let occur_filter = OccurFilter::new(&occurs); - let left_fieldnorms = U64FastFieldReader::from(vec![100, 200, 300]); + let left_fieldnorms = U64FastFieldReader::from((0u64..9u64).map(|doc| doc*3).collect::>()); let left = VecPostings::from(vec![1, 2, 3]); let left_scorer = TermScorer { @@ -122,7 +122,7 @@ mod tests { postings: left, }; - let right_fieldnorms = U64FastFieldReader::from(vec![15, 25, 35]); + let right_fieldnorms = U64FastFieldReader::from((0u64..9u64).map(|doc| doc*5).collect::>()); let right = VecPostings::from(vec![1, 3, 8]); let right_scorer = TermScorer { @@ -133,12 +133,12 @@ mod tests { let mut boolean_scorer = BooleanScorer::new(vec![left_scorer, right_scorer], occur_filter); assert_eq!(boolean_scorer.next(), Some(1u32)); - assert!(abs_diff(boolean_scorer.score(), 0.8707107) < 0.001); + assert!(abs_diff(boolean_scorer.score(), 2.3662047) < 0.001); assert_eq!(boolean_scorer.next(), Some(2u32)); - assert!(abs_diff(boolean_scorer.score(), 0.028867513) < 0.001f32); + assert!(abs_diff(boolean_scorer.score(), 0.20412415) < 0.001f32); assert_eq!(boolean_scorer.next(), Some(3u32)); assert_eq!(boolean_scorer.next(), Some(8u32)); - assert!(abs_diff(boolean_scorer.score(), 0.5163978) < 0.001f32); + assert!(abs_diff(boolean_scorer.score(), 0.31622776) < 0.001f32); assert!(!boolean_scorer.advance()); } diff --git a/src/query/phrase_query/phrase_scorer.rs b/src/query/phrase_query/phrase_scorer.rs index f80416712..23721037b 100644 --- a/src/query/phrase_query/phrase_scorer.rs +++ b/src/query/phrase_query/phrase_scorer.rs @@ -29,8 +29,7 @@ impl<'a> PhraseScorer<'a> { 'outer: loop { let target = pos_candidate + ord; let positions = positions_arr[ord as usize]; - for i in 0..positions.len() { - let pos_i = positions[i]; + for (i, pos_i) in positions.iter().cloned().enumerate() { if pos_i < target { continue; } diff --git a/src/store/reader.rs b/src/store/reader.rs index 79eb4d4dd..060be4525 100644 --- a/src/store/reader.rs +++ b/src/store/reader.rs @@ -64,7 +64,7 @@ impl StoreReader { } } - +#[allow(needless_pass_by_value)] fn split_source(data: ReadOnlySource) -> (ReadOnlySource, ReadOnlySource, DocId) { let data_len = data.len(); let footer_offset = data_len - size_of::() - size_of::();