diff --git a/Cargo.toml b/Cargo.toml index 11711a088..845f1d31d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -61,7 +61,7 @@ debug-assertions = false [features] -default = ["simdcompression", "streamdict"] +default = ["simdcompression"] simdcompression = ["libc", "gcc"] streamdict = [] diff --git a/src/common/composite_file.rs b/src/common/composite_file.rs index 60f9286df..db9b8ba31 100644 --- a/src/common/composite_file.rs +++ b/src/common/composite_file.rs @@ -64,7 +64,7 @@ pub struct CompositeFile { impl CompositeFile { pub fn open(data: ReadOnlySource) -> io::Result { let end = data.len(); - let footer_len_data = data.slice(end - 4, end); + let footer_len_data = data.slice_from(end - 4); let footer_len = u32::deserialize(&mut footer_len_data.as_slice())? as usize; let footer_start = end - 4 - footer_len; @@ -93,7 +93,7 @@ impl CompositeFile { } Ok(CompositeFile { - data: data.slice(0, footer_start), + data: data.slice_to(footer_start), offsets_index: field_index, }) } diff --git a/src/compression/mod.rs b/src/compression/mod.rs index 43096622c..a6bb0eb17 100644 --- a/src/compression/mod.rs +++ b/src/compression/mod.rs @@ -83,7 +83,7 @@ impl VIntDecoder for BlockDecoder { } -pub const NUM_DOCS_PER_BLOCK: usize = 128; //< should be a power of 2 to let the compiler optimize. +pub const COMPRESSION_BLOCK_SIZE: usize = 128; #[cfg(test)] pub mod tests { @@ -186,14 +186,14 @@ pub mod tests { #[bench] fn bench_compress(b: &mut Bencher) { let mut encoder = BlockEncoder::new(); - let data = tests::generate_array(NUM_DOCS_PER_BLOCK, 0.1); + let data = tests::generate_array(COMPRESSION_BLOCK_SIZE, 0.1); b.iter(|| { encoder.compress_block_sorted(&data, 0u32); }); } #[bench] fn bench_uncompress(b: &mut Bencher) { let mut encoder = BlockEncoder::new(); - let data = tests::generate_array(NUM_DOCS_PER_BLOCK, 0.1); + let data = tests::generate_array(COMPRESSION_BLOCK_SIZE, 0.1); let compressed = encoder.compress_block_sorted(&data, 0u32); let mut decoder = BlockDecoder::new(); b.iter(|| { decoder.uncompress_block_sorted(compressed, 0u32); }); diff --git a/src/compression/pack/compression_pack_nosimd.rs b/src/compression/pack/compression_pack_nosimd.rs index 1c15567e4..24379b9a4 100644 --- a/src/compression/pack/compression_pack_nosimd.rs +++ b/src/compression/pack/compression_pack_nosimd.rs @@ -2,15 +2,15 @@ use common::bitpacker::compute_num_bits; use common::bitpacker::{BitPacker, BitUnpacker}; use std::cmp; use std::io::Write; -use super::super::NUM_DOCS_PER_BLOCK; +use super::super::COMPRESSION_BLOCK_SIZE; -const COMPRESSED_BLOCK_MAX_SIZE: usize = NUM_DOCS_PER_BLOCK * 4 + 1; +const COMPRESSED_BLOCK_MAX_SIZE: usize = COMPRESSION_BLOCK_SIZE * 4 + 1; pub fn compress_sorted(vals: &mut [u32], mut output: &mut [u8], offset: u32) -> usize { let mut max_delta = 0; { let mut local_offset = offset; - for i in 0..NUM_DOCS_PER_BLOCK { + for i in 0..COMPRESSION_BLOCK_SIZE { let val = vals[i]; let delta = val - local_offset; max_delta = cmp::max(max_delta, delta); @@ -35,7 +35,7 @@ pub fn compress_sorted(vals: &mut [u32], mut output: &mut [u8], offset: u32) -> pub struct BlockEncoder { pub output: [u8; COMPRESSED_BLOCK_MAX_SIZE], pub output_len: usize, - input_buffer: [u32; NUM_DOCS_PER_BLOCK], + input_buffer: [u32; COMPRESSION_BLOCK_SIZE], } impl BlockEncoder { @@ -43,7 +43,7 @@ impl BlockEncoder { BlockEncoder { output: [0u8; COMPRESSED_BLOCK_MAX_SIZE], output_len: 0, - input_buffer: [0u32; NUM_DOCS_PER_BLOCK], + input_buffer: [0u32; COMPRESSION_BLOCK_SIZE], } } @@ -100,26 +100,26 @@ impl BlockDecoder { let consumed_size = { let num_bits = compressed_data[0]; let bit_unpacker = BitUnpacker::new(&compressed_data[1..], num_bits as usize); - for i in 0..NUM_DOCS_PER_BLOCK { + for i in 0..COMPRESSION_BLOCK_SIZE { let delta = bit_unpacker.get(i); let val = offset + delta; self.output[i] = val; offset = val; } - 1 + (num_bits as usize * NUM_DOCS_PER_BLOCK + 7) / 8 + 1 + (num_bits as usize * COMPRESSION_BLOCK_SIZE + 7) / 8 }; - self.output_len = NUM_DOCS_PER_BLOCK; + self.output_len = COMPRESSION_BLOCK_SIZE; &compressed_data[consumed_size..] } pub fn uncompress_block_unsorted<'a>(&mut self, compressed_data: &'a [u8]) -> &'a [u8] { let num_bits = compressed_data[0]; let bit_unpacker = BitUnpacker::new(&compressed_data[1..], num_bits as usize); - for i in 0..NUM_DOCS_PER_BLOCK { + for i in 0..COMPRESSION_BLOCK_SIZE { self.output[i] = bit_unpacker.get(i); } - let consumed_size = 1 + (num_bits as usize * NUM_DOCS_PER_BLOCK + 7) / 8; - self.output_len = NUM_DOCS_PER_BLOCK; + let consumed_size = 1 + (num_bits as usize * COMPRESSION_BLOCK_SIZE + 7) / 8; + self.output_len = COMPRESSION_BLOCK_SIZE; &compressed_data[consumed_size..] } diff --git a/src/compression/pack/compression_pack_simd.rs b/src/compression/pack/compression_pack_simd.rs index c430a728f..d24d0f65b 100644 --- a/src/compression/pack/compression_pack_simd.rs +++ b/src/compression/pack/compression_pack_simd.rs @@ -1,6 +1,6 @@ -use super::super::NUM_DOCS_PER_BLOCK; +use super::super::COMPRESSION_BLOCK_SIZE; -const COMPRESSED_BLOCK_MAX_SIZE: usize = NUM_DOCS_PER_BLOCK * 4 + 1; +const COMPRESSED_BLOCK_MAX_SIZE: usize = COMPRESSION_BLOCK_SIZE * 4 + 1; mod simdcomp { use libc::size_t; @@ -83,13 +83,13 @@ impl BlockDecoder { offset: u32) -> usize { let consumed_size = uncompress_sorted(compressed_data, &mut self.output, offset); - self.output_len = NUM_DOCS_PER_BLOCK; + self.output_len = COMPRESSION_BLOCK_SIZE; consumed_size } pub fn uncompress_block_unsorted<'a>(&mut self, compressed_data: &'a [u8]) -> usize { let consumed_size = uncompress_unsorted(compressed_data, &mut self.output); - self.output_len = NUM_DOCS_PER_BLOCK; + self.output_len = COMPRESSION_BLOCK_SIZE; consumed_size } diff --git a/src/compression/stream.rs b/src/compression/stream.rs index de902da85..9829fe20a 100644 --- a/src/compression/stream.rs +++ b/src/compression/stream.rs @@ -1,5 +1,5 @@ use compression::BlockDecoder; -use compression::NUM_DOCS_PER_BLOCK; +use compression::COMPRESSION_BLOCK_SIZE; use compression::compressed_block_size; use directory::{ReadOnlySource, SourceRead}; @@ -14,7 +14,7 @@ impl CompressedIntStream { CompressedIntStream { buffer: SourceRead::from(source), block_decoder: BlockDecoder::new(), - inner_offset: NUM_DOCS_PER_BLOCK, + inner_offset: COMPRESSION_BLOCK_SIZE, } } @@ -22,7 +22,7 @@ impl CompressedIntStream { let mut num_els: usize = output.len(); let mut start: usize = 0; loop { - let available = NUM_DOCS_PER_BLOCK - self.inner_offset; + let available = COMPRESSION_BLOCK_SIZE - self.inner_offset; if num_els >= available { if available > 0 { let uncompressed_block = &self.block_decoder.output_array()[self.inner_offset..]; @@ -44,15 +44,15 @@ impl CompressedIntStream { } pub fn skip(&mut self, mut skip_len: usize) { - let available = NUM_DOCS_PER_BLOCK - self.inner_offset; + let available = COMPRESSION_BLOCK_SIZE - self.inner_offset; if available >= skip_len { self.inner_offset += skip_len; } else { skip_len -= available; // entirely skip decompressing some blocks. - while skip_len >= NUM_DOCS_PER_BLOCK { - skip_len -= NUM_DOCS_PER_BLOCK; + while skip_len >= COMPRESSION_BLOCK_SIZE { + skip_len -= COMPRESSION_BLOCK_SIZE; let num_bits: u8 = self.buffer.as_ref()[0]; let block_len = compressed_block_size(num_bits); self.buffer.advance(block_len); @@ -70,7 +70,7 @@ pub mod tests { use super::CompressedIntStream; use compression::compressed_block_size; - use compression::NUM_DOCS_PER_BLOCK; + use compression::COMPRESSION_BLOCK_SIZE; use compression::BlockEncoder; use directory::ReadOnlySource; @@ -78,7 +78,7 @@ pub mod tests { let mut buffer: Vec = vec!(); let mut encoder = BlockEncoder::new(); let vals: Vec = (0u32..1_025u32).collect(); - for chunk in vals.chunks(NUM_DOCS_PER_BLOCK) { + for chunk in vals.chunks(COMPRESSION_BLOCK_SIZE) { let compressed_block = encoder.compress_block_unsorted(chunk); let num_bits = compressed_block[0]; assert_eq!(compressed_block_size(num_bits), compressed_block.len()); @@ -91,7 +91,7 @@ pub mod tests { fn test_compressed_int_stream() { let buffer = create_stream_buffer(); let mut stream = CompressedIntStream::wrap(buffer); - let mut block: [u32; NUM_DOCS_PER_BLOCK] = [0u32; NUM_DOCS_PER_BLOCK]; + let mut block: [u32; COMPRESSION_BLOCK_SIZE] = [0u32; COMPRESSION_BLOCK_SIZE]; stream.read(&mut block[0..2]); assert_eq!(block[0], 0); diff --git a/src/directory/read_only_source.rs b/src/directory/read_only_source.rs index 8ddec278b..32423ff96 100644 --- a/src/directory/read_only_source.rs +++ b/src/directory/read_only_source.rs @@ -43,6 +43,8 @@ impl ReadOnlySource { } } + /// Splits into 2 `ReadOnlySource`, at the offset given + /// as an argument. pub fn split(self, addr: usize) -> (ReadOnlySource, ReadOnlySource) { let left = self.slice(0, addr); let right = self.slice_from(addr); @@ -73,10 +75,20 @@ impl ReadOnlySource { /// Like `.slice(...)` but enforcing only the `from` /// boundary. + /// + /// Equivalent to `.slice(from_offset, self.len())` pub fn slice_from(&self, from_offset: usize) -> ReadOnlySource { let len = self.len(); self.slice(from_offset, len) } + + /// Like `.slice(...)` but enforcing only the `to` + /// boundary. + /// + /// Equivalent to `.slice(0, to_offset)` + pub fn slice_to(&self, to_offset: usize) -> ReadOnlySource { + self.slice(0, to_offset) + } } impl HasLen for ReadOnlySource { diff --git a/src/postings/segment_postings.rs b/src/postings/segment_postings.rs index 0866a5fe5..7d2bab7f8 100644 --- a/src/postings/segment_postings.rs +++ b/src/postings/segment_postings.rs @@ -1,4 +1,4 @@ -use compression::{NUM_DOCS_PER_BLOCK, BlockDecoder, VIntDecoder, CompressedIntStream}; +use compression::{COMPRESSION_BLOCK_SIZE, BlockDecoder, VIntDecoder, CompressedIntStream}; use DocId; use postings::{Postings, DocSet, HasLen, SkipResult}; use std::cmp; @@ -92,7 +92,7 @@ impl SegmentPostings { }); SegmentPostings { block_cursor: segment_block_postings, - cur: NUM_DOCS_PER_BLOCK, // cursor within the block + cur: COMPRESSION_BLOCK_SIZE, // cursor within the block delete_bitset: delete_bitset, position_computer: position_computer, } @@ -104,7 +104,7 @@ impl SegmentPostings { SegmentPostings { block_cursor: empty_block_cursor, delete_bitset: DeleteBitSet::empty(), - cur: NUM_DOCS_PER_BLOCK, + cur: COMPRESSION_BLOCK_SIZE, position_computer: None, } } @@ -131,7 +131,7 @@ impl DocSet for SegmentPostings { if self.cur >= self.block_cursor.block_len() { self.cur = 0; if !self.block_cursor.advance() { - self.cur = NUM_DOCS_PER_BLOCK; + self.cur = COMPRESSION_BLOCK_SIZE; return false; } } @@ -315,8 +315,8 @@ impl BlockSegmentPostings { data: SourceRead, has_freq: bool) -> BlockSegmentPostings { - let num_binpacked_blocks: usize = (doc_freq as usize) / NUM_DOCS_PER_BLOCK; - let num_vint_docs = (doc_freq as usize) - NUM_DOCS_PER_BLOCK * num_binpacked_blocks; + let num_binpacked_blocks: usize = (doc_freq as usize) / COMPRESSION_BLOCK_SIZE; + let num_vint_docs = (doc_freq as usize) - COMPRESSION_BLOCK_SIZE * num_binpacked_blocks; BlockSegmentPostings { num_binpacked_blocks: num_binpacked_blocks, num_vint_docs: num_vint_docs, @@ -343,8 +343,8 @@ impl BlockSegmentPostings { // // This does not reset the positions list. pub(crate) fn reset(&mut self, doc_freq: usize, postings_data: SourceRead) { - let num_binpacked_blocks: usize = doc_freq / NUM_DOCS_PER_BLOCK; - let num_vint_docs = doc_freq & (NUM_DOCS_PER_BLOCK - 1); + let num_binpacked_blocks: usize = doc_freq / COMPRESSION_BLOCK_SIZE; + let num_vint_docs = doc_freq & (COMPRESSION_BLOCK_SIZE - 1); self.num_binpacked_blocks = num_binpacked_blocks; self.num_vint_docs = num_vint_docs; self.remaining_data = postings_data; @@ -414,7 +414,7 @@ impl BlockSegmentPostings { self.remaining_data.advance(num_consumed_bytes); } // it will be used as the next offset. - self.doc_offset = self.doc_decoder.output(NUM_DOCS_PER_BLOCK - 1); + self.doc_offset = self.doc_decoder.output(COMPRESSION_BLOCK_SIZE - 1); self.num_binpacked_blocks -= 1; true } else if self.num_vint_docs > 0 { diff --git a/src/postings/serializer.rs b/src/postings/serializer.rs index 5c24256cc..14a22ccea 100644 --- a/src/postings/serializer.rs +++ b/src/postings/serializer.rs @@ -6,7 +6,7 @@ use schema::FieldEntry; use schema::FieldType; use schema::Schema; use directory::WritePtr; -use compression::{NUM_DOCS_PER_BLOCK, BlockEncoder}; +use compression::{COMPRESSION_BLOCK_SIZE, BlockEncoder}; use DocId; use core::Segment; use std::io::{self, Write}; @@ -264,7 +264,7 @@ impl PostingsSerializer { if self.termfreq_enabled { self.term_freqs.push(term_freq as u32); } - if self.doc_ids.len() == NUM_DOCS_PER_BLOCK { + if self.doc_ids.len() == COMPRESSION_BLOCK_SIZE { { // encode the doc ids let block_encoded: &[u8] = @@ -336,7 +336,7 @@ struct PositionSerializer { impl PositionSerializer { fn new(write: W) -> PositionSerializer { PositionSerializer { - buffer: Vec::with_capacity(NUM_DOCS_PER_BLOCK), + buffer: Vec::with_capacity(COMPRESSION_BLOCK_SIZE), write: CountingWriter::wrap(write), block_encoder: BlockEncoder::new(), } @@ -347,7 +347,7 @@ impl PositionSerializer { } fn write_block(&mut self) -> io::Result<()> { - assert_eq!(self.buffer.len(), NUM_DOCS_PER_BLOCK); + assert_eq!(self.buffer.len(), COMPRESSION_BLOCK_SIZE); let block_compressed: &[u8] = self.block_encoder.compress_block_unsorted(&self.buffer); self.write.write_all(block_compressed)?; self.buffer.clear(); @@ -356,8 +356,8 @@ impl PositionSerializer { fn write(&mut self, mut vals: &[u32]) -> io::Result<()> { let mut buffer_len = self.buffer.len(); - while vals.len() + buffer_len >= NUM_DOCS_PER_BLOCK { - let len_to_completion = NUM_DOCS_PER_BLOCK - buffer_len; + while vals.len() + buffer_len >= COMPRESSION_BLOCK_SIZE { + let len_to_completion = COMPRESSION_BLOCK_SIZE - buffer_len; self.buffer.extend_from_slice(&vals[..len_to_completion]); self.write_block()?; vals = &vals[len_to_completion..]; @@ -368,7 +368,7 @@ impl PositionSerializer { } fn close(mut self) -> io::Result<()> { - self.buffer.resize(NUM_DOCS_PER_BLOCK, 0u32); + self.buffer.resize(COMPRESSION_BLOCK_SIZE, 0u32); self.write_block()?; self.write.flush() } diff --git a/src/termdict/mod.rs b/src/termdict/mod.rs index 5930eeaaf..171a70c2f 100644 --- a/src/termdict/mod.rs +++ b/src/termdict/mod.rs @@ -338,9 +338,35 @@ mod tests { term_dictionary.get(key.as_bytes()); } + + #[test] + fn test_stream_high_range_prefix_suffix() { + let field_type = FieldType::Str(TEXT); + let buffer: Vec = { + let mut term_dictionary_builder = TermDictionaryBuilderImpl::new(vec![], field_type).unwrap(); + // term requires more than 16bits + term_dictionary_builder.insert("abcdefghijklmnopqrstuvwxyz", &make_term_info(1)).unwrap(); + term_dictionary_builder.insert("abcdefghijklmnopqrstuvwxyz", &make_term_info(2)).unwrap(); + term_dictionary_builder.insert("abr", &make_term_info(2)).unwrap(); + term_dictionary_builder.finish().unwrap() + }; + let source = ReadOnlySource::from(buffer); + let term_dictionary: TermDictionaryImpl = TermDictionaryImpl::from_source(source) + .unwrap(); + let mut kv_stream = term_dictionary.stream(); + assert!(kv_stream.advance()); + assert_eq!(kv_stream.key(), "abcdefghijklmnopqrstuvwxyz".as_bytes()); + assert_eq!(kv_stream.value(), &make_term_info(1)); + assert!(kv_stream.advance()); + assert_eq!(kv_stream.key(), "abcdefghijklmnopqrstuvwxyz".as_bytes()); + assert_eq!(kv_stream.value(), &make_term_info(2)); + assert!(kv_stream.advance()); + assert_eq!(kv_stream.key(), "abr".as_bytes()); + assert!(!kv_stream.advance()); + } + #[test] fn test_stream_range() { -// let ids: Vec<_> = (0u32..10_000u32) let ids: Vec<_> = (0u32..10_000u32) .map(|i| (format!("doc{:0>6}", i), i)) .collect(); diff --git a/src/termdict/streamdict/delta_encoder.rs b/src/termdict/streamdict/delta_encoder.rs index 1666e0161..9a2a4173e 100644 --- a/src/termdict/streamdict/delta_encoder.rs +++ b/src/termdict/streamdict/delta_encoder.rs @@ -1,6 +1,7 @@ use postings::TermInfo; use super::CheckPoint; use std::mem; +use common::BinarySerializable; /// Returns the len of the longest /// common prefix of `s1` and `s2`. @@ -49,9 +50,24 @@ impl TermDeltaDecoder { } } - pub fn decode(&mut self, prefix_len: usize, suffix: &[u8]) { - self.term.truncate(prefix_len); - self.term.extend_from_slice(suffix); + #[inline(always)] + pub fn decode<'a>(&mut self, code: u8, mut cursor: &'a [u8]) -> &'a [u8] { + let (prefix_len, suffix_len): (usize, usize) = + if (code & 1u8) == 1u8 { + let b = cursor[0]; + cursor = &cursor[1..]; + let prefix_len = (b & 15u8) as usize; + let suffix_len = (b >> 4u8) as usize; + (prefix_len, suffix_len) + } + else { + let prefix_len = u32::deserialize(&mut cursor).unwrap(); + let suffix_len = u32::deserialize(&mut cursor).unwrap(); + (prefix_len as usize, suffix_len as usize) + }; + unsafe { self.term.set_len(prefix_len) }; + self.term.extend_from_slice(&(*cursor)[..suffix_len]); + &cursor[suffix_len..] } pub fn term(&self) -> &[u8] { @@ -108,6 +124,12 @@ pub struct TermInfoDeltaDecoder { } +#[inline(always)] +pub fn make_mask(num_bytes: usize) -> u32 { + const MASK: [u32; 4] = [0xffu32, 0xffffu32, 0xffffffu32, 0xffffffffu32]; + *unsafe { MASK.get_unchecked(num_bytes.wrapping_sub(1) as usize) } +} + impl TermInfoDeltaDecoder { pub fn from_term_info(term_info: TermInfo, has_positions: bool) -> TermInfoDeltaDecoder { @@ -129,27 +151,26 @@ impl TermInfoDeltaDecoder { } } - pub fn decode(&mut self, code: u8, cursor: &mut &[u8]) { - let num_bytes_docfreq: usize = ((code >> 1) & 3) as usize; - let num_bytes_postings_offset: usize = ((code >> 3) & 3) as usize; - const MASK: [u32; 4] = [ - 0xffu32, - 0xffffu32, - 0xffffffu32, - 0xffffffffu32, - ]; - let doc_freq: u32 = unsafe { *(cursor.as_ptr() as *const u32) } & MASK[num_bytes_docfreq]; - *cursor = &cursor[num_bytes_docfreq + 1 ..]; - let delta_postings_offset: u32 = unsafe { *(cursor.as_ptr() as *const u32) } & MASK[num_bytes_postings_offset]; - *cursor = &cursor[num_bytes_postings_offset + 1..]; + #[inline(always)] + pub fn decode<'a>(&mut self, code: u8, mut cursor: &'a [u8]) -> &'a [u8] { + let num_bytes_docfreq: usize = ((code >> 1) & 3) as usize + 1; + let num_bytes_postings_offset: usize = ((code >> 3) & 3) as usize + 1; + let mut v: u64 = unsafe { *(cursor.as_ptr() as *const u64) }; + let doc_freq: u32 = (v as u32) & make_mask(num_bytes_docfreq); + v >>= (num_bytes_docfreq as u64) * 8u64; + let delta_postings_offset: u32 = (v as u32) & make_mask(num_bytes_postings_offset); + cursor = &cursor[num_bytes_docfreq + num_bytes_postings_offset..]; self.term_info.doc_freq = doc_freq; self.term_info.postings_offset += delta_postings_offset; if self.has_positions { - let num_bytes_positions_offset = ((code >> 5) & 3) as usize; - let delta_positions_offset: u32 = unsafe { *(cursor.as_ptr() as *const u32) } & MASK[num_bytes_positions_offset]; + let num_bytes_positions_offset = ((code >> 5) & 3) as usize + 1; + let delta_positions_offset: u32 = unsafe { *(cursor.as_ptr() as *const u32) } & make_mask(num_bytes_positions_offset); self.term_info.positions_offset += delta_positions_offset; - self.term_info.positions_inner_offset = cursor[num_bytes_positions_offset + 1]; - *cursor = &cursor[num_bytes_positions_offset + 2..]; + self.term_info.positions_inner_offset = cursor[num_bytes_positions_offset]; + &cursor[num_bytes_positions_offset + 1..] + } + else { + cursor } } @@ -157,4 +178,3 @@ impl TermInfoDeltaDecoder { &self.term_info } } - diff --git a/src/termdict/streamdict/streamer.rs b/src/termdict/streamdict/streamer.rs index eacffe5dc..dcb4b8bdb 100644 --- a/src/termdict/streamdict/streamer.rs +++ b/src/termdict/streamdict/streamer.rs @@ -4,7 +4,6 @@ use std::cmp::max; use super::TermDictionaryImpl; use termdict::{TermStreamerBuilder, TermStreamer}; use postings::TermInfo; -use common::BinarySerializable; use super::delta_encoder::{TermInfoDeltaDecoder, TermDeltaDecoder}; @@ -163,28 +162,10 @@ impl<'a> TermStreamer for TermStreamerImpl<'a> if self.cursor.is_empty() { return false; } - let code: u8 = self.cursor[0]; - let mut cursor: &[u8] = &self.cursor[1..]; - - let prefix_suffix_packed = (code & 1u8) == 1u8; - let (prefix_len, suffix_len): (usize, usize) = - if prefix_suffix_packed { - let b = cursor[0]; - cursor = &cursor[1..]; - let prefix_len = (b & 15u8) as usize; - let suffix_len = (b >> 4u8) as usize; - (prefix_len, suffix_len) - } - else { - let prefix_len = u32::deserialize(&mut cursor).unwrap(); - let suffix_len = u32::deserialize(&mut cursor).unwrap(); - (prefix_len as usize, suffix_len as usize) - }; - - let suffix = &cursor[..suffix_len]; - self.term_delta_decoder.decode(prefix_len, suffix); - cursor = &cursor[suffix_len..]; - self.term_info_decoder.decode(code, &mut cursor); + let mut cursor: &[u8] = &self.cursor; + let code: u8 = cursor[0]; + cursor = self.term_delta_decoder.decode(code, &cursor[1..]); + cursor = self.term_info_decoder.decode(code, cursor); self.cursor = cursor; true } diff --git a/src/termdict/streamdict/termdict.rs b/src/termdict/streamdict/termdict.rs index c2bdbbe68..bf4d899fd 100644 --- a/src/termdict/streamdict/termdict.rs +++ b/src/termdict/streamdict/termdict.rs @@ -17,7 +17,7 @@ use super::{TermStreamerImpl, TermStreamerBuilderImpl}; use termdict::TermStreamerBuilder; use std::mem::transmute; -const PADDING_SIZE: usize = 16; +const PADDING_SIZE: usize = 4; const INDEX_INTERVAL: usize = 1024; fn convert_fst_error(e: fst::Error) -> io::Error {