Added unit test.

Cargo.toml

@@ -61,7 +61,7 @@ debug-assertions = false
 
 
 [features]
-default = ["simdcompression", "streamdict"]
+default = ["simdcompression"]
 simdcompression = ["libc", "gcc"]
 streamdict = []
 

src/common/composite_file.rs

@@ -64,7 +64,7 @@ pub struct CompositeFile {
 impl CompositeFile {
     pub fn open(data: ReadOnlySource) -> io::Result<CompositeFile> {
         let end = data.len();
-        let footer_len_data = data.slice(end - 4, end);
+        let footer_len_data = data.slice_from(end - 4);
         let footer_len = u32::deserialize(&mut footer_len_data.as_slice())? as usize;
 
         let footer_start = end - 4 - footer_len;
@@ -93,7 +93,7 @@ impl CompositeFile {
         }
 
         Ok(CompositeFile {
-            data: data.slice(0, footer_start),
+            data: data.slice_to(footer_start),
             offsets_index: field_index,
         })
     }

src/compression/mod.rs

@@ -83,7 +83,7 @@ impl VIntDecoder for BlockDecoder {
 }
 
 
-pub const NUM_DOCS_PER_BLOCK: usize = 128; //< should be a power of 2 to let the compiler optimize.
+pub const COMPRESSION_BLOCK_SIZE: usize = 128;
 
 #[cfg(test)]
 pub mod tests {
@@ -186,14 +186,14 @@ pub mod tests {
     #[bench]
     fn bench_compress(b: &mut Bencher) {
         let mut encoder = BlockEncoder::new();
-        let data = tests::generate_array(NUM_DOCS_PER_BLOCK, 0.1);
+        let data = tests::generate_array(COMPRESSION_BLOCK_SIZE, 0.1);
         b.iter(|| { encoder.compress_block_sorted(&data, 0u32); });
     }
 
     #[bench]
     fn bench_uncompress(b: &mut Bencher) {
         let mut encoder = BlockEncoder::new();
-        let data = tests::generate_array(NUM_DOCS_PER_BLOCK, 0.1);
+        let data = tests::generate_array(COMPRESSION_BLOCK_SIZE, 0.1);
         let compressed = encoder.compress_block_sorted(&data, 0u32);
         let mut decoder = BlockDecoder::new();
         b.iter(|| { decoder.uncompress_block_sorted(compressed, 0u32); });

src/compression/pack/compression_pack_nosimd.rs

@@ -2,15 +2,15 @@ use common::bitpacker::compute_num_bits;
 use common::bitpacker::{BitPacker, BitUnpacker};
 use std::cmp;
 use std::io::Write;
-use super::super::NUM_DOCS_PER_BLOCK;
+use super::super::COMPRESSION_BLOCK_SIZE;
 
-const COMPRESSED_BLOCK_MAX_SIZE: usize = NUM_DOCS_PER_BLOCK * 4 + 1;
+const COMPRESSED_BLOCK_MAX_SIZE: usize = COMPRESSION_BLOCK_SIZE * 4 + 1;
 
 pub fn compress_sorted(vals: &mut [u32], mut output: &mut [u8], offset: u32) -> usize {
     let mut max_delta = 0;
     {
         let mut local_offset = offset;
-        for i in 0..NUM_DOCS_PER_BLOCK {
+        for i in 0..COMPRESSION_BLOCK_SIZE {
             let val = vals[i];
             let delta = val - local_offset;
             max_delta = cmp::max(max_delta, delta);
@@ -35,7 +35,7 @@ pub fn compress_sorted(vals: &mut [u32], mut output: &mut [u8], offset: u32) ->
 pub struct BlockEncoder {
     pub output: [u8; COMPRESSED_BLOCK_MAX_SIZE],
     pub output_len: usize,
-    input_buffer: [u32; NUM_DOCS_PER_BLOCK],
+    input_buffer: [u32; COMPRESSION_BLOCK_SIZE],
 }
 
 impl BlockEncoder {
@@ -43,7 +43,7 @@ impl BlockEncoder {
         BlockEncoder {
             output: [0u8; COMPRESSED_BLOCK_MAX_SIZE],
             output_len: 0,
-            input_buffer: [0u32; NUM_DOCS_PER_BLOCK],
+            input_buffer: [0u32; COMPRESSION_BLOCK_SIZE],
         }
     }
 
@@ -100,26 +100,26 @@ impl BlockDecoder {
         let consumed_size = {
             let num_bits = compressed_data[0];
             let bit_unpacker = BitUnpacker::new(&compressed_data[1..], num_bits as usize);
-            for i in 0..NUM_DOCS_PER_BLOCK {
+            for i in 0..COMPRESSION_BLOCK_SIZE {
                 let delta = bit_unpacker.get(i);
                 let val = offset + delta;
                 self.output[i] = val;
                 offset = val;
             }
-            1 + (num_bits as usize * NUM_DOCS_PER_BLOCK + 7) / 8
+            1 + (num_bits as usize * COMPRESSION_BLOCK_SIZE + 7) / 8
         };
-        self.output_len = NUM_DOCS_PER_BLOCK;
+        self.output_len = COMPRESSION_BLOCK_SIZE;
         &compressed_data[consumed_size..]
     }
 
     pub fn uncompress_block_unsorted<'a>(&mut self, compressed_data: &'a [u8]) -> &'a [u8] {
         let num_bits = compressed_data[0];
         let bit_unpacker = BitUnpacker::new(&compressed_data[1..], num_bits as usize);
-        for i in 0..NUM_DOCS_PER_BLOCK {
+        for i in 0..COMPRESSION_BLOCK_SIZE {
             self.output[i] = bit_unpacker.get(i);
         }
-        let consumed_size = 1 + (num_bits as usize * NUM_DOCS_PER_BLOCK + 7) / 8;
-        self.output_len = NUM_DOCS_PER_BLOCK;
+        let consumed_size = 1 + (num_bits as usize * COMPRESSION_BLOCK_SIZE + 7) / 8;
+        self.output_len = COMPRESSION_BLOCK_SIZE;
         &compressed_data[consumed_size..]
     }
 

src/compression/pack/compression_pack_simd.rs

@@ -1,6 +1,6 @@
-use super::super::NUM_DOCS_PER_BLOCK;
+use super::super::COMPRESSION_BLOCK_SIZE;
 
-const COMPRESSED_BLOCK_MAX_SIZE: usize = NUM_DOCS_PER_BLOCK * 4 + 1;
+const COMPRESSED_BLOCK_MAX_SIZE: usize = COMPRESSION_BLOCK_SIZE * 4 + 1;
 
 mod simdcomp {
     use libc::size_t;
@@ -83,13 +83,13 @@ impl BlockDecoder {
                                    offset: u32)
                                        -> usize {
         let consumed_size = uncompress_sorted(compressed_data, &mut self.output, offset);
-        self.output_len = NUM_DOCS_PER_BLOCK;
+        self.output_len = COMPRESSION_BLOCK_SIZE;
         consumed_size
     }
 
     pub fn uncompress_block_unsorted<'a>(&mut self, compressed_data: &'a [u8]) -> usize {
         let consumed_size = uncompress_unsorted(compressed_data, &mut self.output);
-        self.output_len = NUM_DOCS_PER_BLOCK;
+        self.output_len = COMPRESSION_BLOCK_SIZE;
         consumed_size
     }
 

src/compression/stream.rs

@@ -1,5 +1,5 @@
 use compression::BlockDecoder;
-use compression::NUM_DOCS_PER_BLOCK;
+use compression::COMPRESSION_BLOCK_SIZE;
 use compression::compressed_block_size;
 use directory::{ReadOnlySource, SourceRead};
 
@@ -14,7 +14,7 @@ impl CompressedIntStream {
         CompressedIntStream {
             buffer: SourceRead::from(source),
             block_decoder: BlockDecoder::new(),
-            inner_offset: NUM_DOCS_PER_BLOCK,
+            inner_offset: COMPRESSION_BLOCK_SIZE,
         }
     }
 
@@ -22,7 +22,7 @@ impl CompressedIntStream {
         let mut num_els: usize = output.len();
         let mut start: usize = 0;
         loop {
-            let available = NUM_DOCS_PER_BLOCK - self.inner_offset;
+            let available = COMPRESSION_BLOCK_SIZE - self.inner_offset;
             if num_els >= available {
                 if available > 0 {
                     let uncompressed_block = &self.block_decoder.output_array()[self.inner_offset..];
@@ -44,15 +44,15 @@ impl CompressedIntStream {
     }
 
     pub fn skip(&mut self, mut skip_len: usize) {
-        let available = NUM_DOCS_PER_BLOCK - self.inner_offset;
+        let available = COMPRESSION_BLOCK_SIZE - self.inner_offset;
         if available >= skip_len {
             self.inner_offset += skip_len;
         }
         else {
             skip_len -= available;
             // entirely skip decompressing some blocks.
-            while skip_len >= NUM_DOCS_PER_BLOCK {
-                skip_len -= NUM_DOCS_PER_BLOCK;
+            while skip_len >= COMPRESSION_BLOCK_SIZE {
+                skip_len -= COMPRESSION_BLOCK_SIZE;
                 let num_bits: u8 = self.buffer.as_ref()[0];
                 let block_len = compressed_block_size(num_bits);
                 self.buffer.advance(block_len);
@@ -70,7 +70,7 @@ pub mod tests {
 
     use super::CompressedIntStream;
     use compression::compressed_block_size;
-    use compression::NUM_DOCS_PER_BLOCK;
+    use compression::COMPRESSION_BLOCK_SIZE;
     use compression::BlockEncoder;
     use directory::ReadOnlySource;
 
@@ -78,7 +78,7 @@ pub mod tests {
         let mut buffer: Vec<u8> = vec!();
         let mut encoder = BlockEncoder::new();
         let vals: Vec<u32> = (0u32..1_025u32).collect();
-        for chunk in vals.chunks(NUM_DOCS_PER_BLOCK) {
+        for chunk in vals.chunks(COMPRESSION_BLOCK_SIZE) {
             let compressed_block = encoder.compress_block_unsorted(chunk);
             let num_bits = compressed_block[0];
             assert_eq!(compressed_block_size(num_bits), compressed_block.len());
@@ -91,7 +91,7 @@ pub mod tests {
     fn test_compressed_int_stream() {
         let buffer = create_stream_buffer();
         let mut stream = CompressedIntStream::wrap(buffer);
-        let mut block: [u32; NUM_DOCS_PER_BLOCK] = [0u32; NUM_DOCS_PER_BLOCK];
+        let mut block: [u32; COMPRESSION_BLOCK_SIZE] = [0u32; COMPRESSION_BLOCK_SIZE];
 
         stream.read(&mut block[0..2]);
         assert_eq!(block[0], 0);

src/directory/read_only_source.rs

@@ -43,6 +43,8 @@ impl ReadOnlySource {
         }
     }
 
+    /// Splits into 2 `ReadOnlySource`, at the offset given
+    /// as an argument.
     pub fn split(self, addr: usize) -> (ReadOnlySource, ReadOnlySource) {
         let left = self.slice(0, addr);
         let right = self.slice_from(addr);
@@ -73,10 +75,20 @@ impl ReadOnlySource {
 
     /// Like `.slice(...)` but enforcing only the `from`
     /// boundary.
+    ///
+    /// Equivalent to `.slice(from_offset, self.len())`
     pub fn slice_from(&self, from_offset: usize) -> ReadOnlySource {
         let len = self.len();
         self.slice(from_offset, len)
     }
+
+    /// Like `.slice(...)` but enforcing only the `to`
+    /// boundary.
+    ///
+    /// Equivalent to `.slice(0, to_offset)`
+    pub fn slice_to(&self, to_offset: usize) -> ReadOnlySource {
+        self.slice(0, to_offset)
+    }
 }
 
 impl HasLen for ReadOnlySource {

src/postings/segment_postings.rs

@@ -1,4 +1,4 @@
-use compression::{NUM_DOCS_PER_BLOCK, BlockDecoder, VIntDecoder, CompressedIntStream};
+use compression::{COMPRESSION_BLOCK_SIZE, BlockDecoder, VIntDecoder, CompressedIntStream};
 use DocId;
 use postings::{Postings, DocSet, HasLen, SkipResult};
 use std::cmp;
@@ -92,7 +92,7 @@ impl SegmentPostings {
         });
         SegmentPostings {
             block_cursor: segment_block_postings,
-            cur: NUM_DOCS_PER_BLOCK, // cursor within the block
+            cur: COMPRESSION_BLOCK_SIZE, // cursor within the block
             delete_bitset: delete_bitset,
             position_computer: position_computer,
         }
@@ -104,7 +104,7 @@ impl SegmentPostings {
         SegmentPostings {
             block_cursor: empty_block_cursor,
             delete_bitset: DeleteBitSet::empty(),
-            cur: NUM_DOCS_PER_BLOCK,
+            cur: COMPRESSION_BLOCK_SIZE,
             position_computer: None,
         }
     }
@@ -131,7 +131,7 @@ impl DocSet for SegmentPostings {
             if self.cur >= self.block_cursor.block_len() {
                 self.cur = 0;
                 if !self.block_cursor.advance() {
-                    self.cur = NUM_DOCS_PER_BLOCK;
+                    self.cur = COMPRESSION_BLOCK_SIZE;
                     return false;
                 }
             }
@@ -315,8 +315,8 @@ impl BlockSegmentPostings {
                             data: SourceRead,
                             has_freq: bool)
                             -> BlockSegmentPostings {
-        let num_binpacked_blocks: usize = (doc_freq as usize) / NUM_DOCS_PER_BLOCK;
-        let num_vint_docs = (doc_freq as usize) - NUM_DOCS_PER_BLOCK * num_binpacked_blocks;
+        let num_binpacked_blocks: usize = (doc_freq as usize) / COMPRESSION_BLOCK_SIZE;
+        let num_vint_docs = (doc_freq as usize) - COMPRESSION_BLOCK_SIZE * num_binpacked_blocks;
         BlockSegmentPostings {
             num_binpacked_blocks: num_binpacked_blocks,
             num_vint_docs: num_vint_docs,
@@ -343,8 +343,8 @@ impl BlockSegmentPostings {
     //
     // This does not reset the positions list.
     pub(crate) fn reset(&mut self, doc_freq: usize, postings_data: SourceRead) {
-        let num_binpacked_blocks: usize = doc_freq / NUM_DOCS_PER_BLOCK;
-        let num_vint_docs = doc_freq & (NUM_DOCS_PER_BLOCK - 1);
+        let num_binpacked_blocks: usize = doc_freq / COMPRESSION_BLOCK_SIZE;
+        let num_vint_docs = doc_freq & (COMPRESSION_BLOCK_SIZE - 1);
         self.num_binpacked_blocks = num_binpacked_blocks;
         self.num_vint_docs = num_vint_docs;
         self.remaining_data = postings_data;
@@ -414,7 +414,7 @@ impl BlockSegmentPostings {
                 self.remaining_data.advance(num_consumed_bytes);
             }
             // it will be used as the next offset.
-            self.doc_offset = self.doc_decoder.output(NUM_DOCS_PER_BLOCK - 1);
+            self.doc_offset = self.doc_decoder.output(COMPRESSION_BLOCK_SIZE - 1);
             self.num_binpacked_blocks -= 1;
             true
         } else if self.num_vint_docs > 0 {

src/postings/serializer.rs

@@ -6,7 +6,7 @@ use schema::FieldEntry;
 use schema::FieldType;
 use schema::Schema;
 use directory::WritePtr;
-use compression::{NUM_DOCS_PER_BLOCK, BlockEncoder};
+use compression::{COMPRESSION_BLOCK_SIZE, BlockEncoder};
 use DocId;
 use core::Segment;
 use std::io::{self, Write};
@@ -264,7 +264,7 @@ impl<W: Write> PostingsSerializer<W> {
         if self.termfreq_enabled {
             self.term_freqs.push(term_freq as u32);
         }
-        if self.doc_ids.len() == NUM_DOCS_PER_BLOCK {
+        if self.doc_ids.len() == COMPRESSION_BLOCK_SIZE {
             {
                 // encode the doc ids
                 let block_encoded: &[u8] =
@@ -336,7 +336,7 @@ struct PositionSerializer<W: Write> {
 impl<W: Write> PositionSerializer<W> {
     fn new(write: W) -> PositionSerializer<W> {
         PositionSerializer {
-            buffer: Vec::with_capacity(NUM_DOCS_PER_BLOCK),
+            buffer: Vec::with_capacity(COMPRESSION_BLOCK_SIZE),
             write: CountingWriter::wrap(write),
             block_encoder: BlockEncoder::new(),
         }
@@ -347,7 +347,7 @@ impl<W: Write> PositionSerializer<W> {
     }
 
     fn write_block(&mut self) -> io::Result<()> {
-        assert_eq!(self.buffer.len(), NUM_DOCS_PER_BLOCK);
+        assert_eq!(self.buffer.len(), COMPRESSION_BLOCK_SIZE);
         let block_compressed: &[u8] = self.block_encoder.compress_block_unsorted(&self.buffer);
         self.write.write_all(block_compressed)?;
         self.buffer.clear();
@@ -356,8 +356,8 @@ impl<W: Write> PositionSerializer<W> {
 
     fn write(&mut self, mut vals: &[u32]) -> io::Result<()> {
         let mut buffer_len = self.buffer.len();
-        while vals.len() + buffer_len >= NUM_DOCS_PER_BLOCK {
-            let len_to_completion = NUM_DOCS_PER_BLOCK - buffer_len;
+        while vals.len() + buffer_len >= COMPRESSION_BLOCK_SIZE {
+            let len_to_completion = COMPRESSION_BLOCK_SIZE - buffer_len;
             self.buffer.extend_from_slice(&vals[..len_to_completion]);
             self.write_block()?;
             vals = &vals[len_to_completion..];
@@ -368,7 +368,7 @@ impl<W: Write> PositionSerializer<W> {
     }
 
     fn close(mut self) -> io::Result<()> {
-        self.buffer.resize(NUM_DOCS_PER_BLOCK, 0u32);
+        self.buffer.resize(COMPRESSION_BLOCK_SIZE, 0u32);
         self.write_block()?;
         self.write.flush()
     }

src/termdict/mod.rs

@@ -338,9 +338,35 @@ mod tests {
         term_dictionary.get(key.as_bytes());
     }
 
+
+    #[test]
+    fn test_stream_high_range_prefix_suffix() {
+        let field_type = FieldType::Str(TEXT);
+        let buffer: Vec<u8> = {
+            let mut term_dictionary_builder = TermDictionaryBuilderImpl::new(vec![], field_type).unwrap();
+            // term requires more than 16bits
+            term_dictionary_builder.insert("abcdefghijklmnopqrstuvwxyz", &make_term_info(1)).unwrap();
+            term_dictionary_builder.insert("abcdefghijklmnopqrstuvwxyz", &make_term_info(2)).unwrap();
+            term_dictionary_builder.insert("abr", &make_term_info(2)).unwrap();
+            term_dictionary_builder.finish().unwrap()
+        };
+        let source = ReadOnlySource::from(buffer);
+        let term_dictionary: TermDictionaryImpl = TermDictionaryImpl::from_source(source)
+            .unwrap();
+        let mut kv_stream = term_dictionary.stream();
+        assert!(kv_stream.advance());
+        assert_eq!(kv_stream.key(), "abcdefghijklmnopqrstuvwxyz".as_bytes());
+        assert_eq!(kv_stream.value(), &make_term_info(1));
+        assert!(kv_stream.advance());
+        assert_eq!(kv_stream.key(), "abcdefghijklmnopqrstuvwxyz".as_bytes());
+        assert_eq!(kv_stream.value(), &make_term_info(2));
+        assert!(kv_stream.advance());
+        assert_eq!(kv_stream.key(), "abr".as_bytes());
+        assert!(!kv_stream.advance());
+    }
+
     #[test]
     fn test_stream_range() {
-//        let ids: Vec<_> = (0u32..10_000u32)
         let ids: Vec<_> = (0u32..10_000u32)
             .map(|i| (format!("doc{:0>6}", i), i))
             .collect();

src/termdict/streamdict/delta_encoder.rs

@@ -1,6 +1,7 @@
 use postings::TermInfo;
 use super::CheckPoint;
 use std::mem;
+use common::BinarySerializable;
 
 /// Returns the len of the longest
 /// common prefix of `s1` and `s2`.
@@ -49,9 +50,24 @@ impl TermDeltaDecoder {
         }
     }
 
-    pub fn decode(&mut self, prefix_len: usize, suffix: &[u8]) {
-        self.term.truncate(prefix_len);
-        self.term.extend_from_slice(suffix);
+    #[inline(always)]
+    pub fn decode<'a>(&mut self, code: u8, mut cursor: &'a [u8]) -> &'a [u8] {
+        let (prefix_len, suffix_len): (usize, usize) =
+            if (code & 1u8) == 1u8 {
+                let b = cursor[0];
+                cursor = &cursor[1..];
+                let prefix_len = (b & 15u8) as usize;
+                let suffix_len = (b >> 4u8) as usize;
+                (prefix_len, suffix_len)
+            }
+            else {
+                let prefix_len = u32::deserialize(&mut cursor).unwrap();
+                let suffix_len = u32::deserialize(&mut cursor).unwrap();
+                (prefix_len as usize, suffix_len as usize)
+            };
+        unsafe { self.term.set_len(prefix_len) };
+        self.term.extend_from_slice(&(*cursor)[..suffix_len]);
+        &cursor[suffix_len..]
     }
 
     pub fn term(&self) -> &[u8]  {
@@ -108,6 +124,12 @@ pub struct TermInfoDeltaDecoder {
 }
 
 
+#[inline(always)]
+pub fn make_mask(num_bytes: usize) -> u32 {
+    const MASK: [u32; 4] = [0xffu32, 0xffffu32, 0xffffffu32, 0xffffffffu32];
+    *unsafe { MASK.get_unchecked(num_bytes.wrapping_sub(1) as usize) }
+}
+
 impl TermInfoDeltaDecoder {
 
     pub fn from_term_info(term_info: TermInfo, has_positions: bool) -> TermInfoDeltaDecoder {
@@ -129,27 +151,26 @@ impl TermInfoDeltaDecoder {
         }
     }
 
-    pub fn decode(&mut self, code: u8, cursor: &mut &[u8]) {
-        let num_bytes_docfreq: usize = ((code >> 1) & 3) as usize;
-        let num_bytes_postings_offset: usize = ((code >> 3) & 3) as usize;
-        const MASK: [u32; 4] = [
-            0xffu32,
-            0xffffu32,
-            0xffffffu32,
-            0xffffffffu32,
-        ];
-        let doc_freq: u32 = unsafe { *(cursor.as_ptr() as *const u32) } & MASK[num_bytes_docfreq];
-        *cursor = &cursor[num_bytes_docfreq + 1 ..];
-        let delta_postings_offset: u32 = unsafe { *(cursor.as_ptr() as *const u32) } & MASK[num_bytes_postings_offset];
-        *cursor = &cursor[num_bytes_postings_offset + 1..];
+    #[inline(always)]
+    pub fn decode<'a>(&mut self, code: u8, mut cursor: &'a [u8]) -> &'a [u8] {
+        let num_bytes_docfreq: usize = ((code >> 1) & 3) as usize + 1;
+        let num_bytes_postings_offset: usize = ((code >> 3) & 3) as usize + 1;
+        let mut v: u64 = unsafe { *(cursor.as_ptr() as *const u64) };
+        let doc_freq: u32 = (v as u32) & make_mask(num_bytes_docfreq);
+        v >>= (num_bytes_docfreq as u64) * 8u64;
+        let delta_postings_offset: u32 = (v as u32) & make_mask(num_bytes_postings_offset);
+        cursor = &cursor[num_bytes_docfreq + num_bytes_postings_offset..];
         self.term_info.doc_freq = doc_freq;
         self.term_info.postings_offset += delta_postings_offset;
         if self.has_positions {
-            let num_bytes_positions_offset = ((code >> 5) & 3) as usize;
-            let delta_positions_offset: u32 = unsafe { *(cursor.as_ptr() as *const u32) } & MASK[num_bytes_positions_offset];
+            let num_bytes_positions_offset = ((code >> 5) & 3) as usize + 1;
+            let delta_positions_offset: u32 = unsafe { *(cursor.as_ptr() as *const u32) } & make_mask(num_bytes_positions_offset);
             self.term_info.positions_offset += delta_positions_offset;
-            self.term_info.positions_inner_offset = cursor[num_bytes_positions_offset + 1];
-            *cursor = &cursor[num_bytes_positions_offset + 2..];
+            self.term_info.positions_inner_offset = cursor[num_bytes_positions_offset];
+            &cursor[num_bytes_positions_offset + 1..]
+        }
+        else {
+            cursor
         }
     }
 
@@ -157,4 +178,3 @@ impl TermInfoDeltaDecoder {
         &self.term_info
     }
 }
-

src/termdict/streamdict/streamer.rs

@@ -4,7 +4,6 @@ use std::cmp::max;
 use super::TermDictionaryImpl;
 use termdict::{TermStreamerBuilder, TermStreamer};
 use postings::TermInfo;
-use common::BinarySerializable;
 use super::delta_encoder::{TermInfoDeltaDecoder, TermDeltaDecoder};
 
 
@@ -163,28 +162,10 @@ impl<'a> TermStreamer for TermStreamerImpl<'a>
         if self.cursor.is_empty() {
             return false;
         }
-        let code: u8 = self.cursor[0];
-        let mut cursor: &[u8] = &self.cursor[1..];
-
-        let prefix_suffix_packed = (code & 1u8) == 1u8;
-        let (prefix_len, suffix_len): (usize, usize) =
-            if prefix_suffix_packed {
-                let b = cursor[0];
-                cursor = &cursor[1..];
-                let prefix_len = (b & 15u8) as usize;
-                let suffix_len = (b >> 4u8) as usize;
-                (prefix_len, suffix_len)
-            }
-            else {
-                let prefix_len = u32::deserialize(&mut cursor).unwrap();
-                let suffix_len = u32::deserialize(&mut cursor).unwrap();
-                (prefix_len as usize, suffix_len as usize)
-            };
-
-        let suffix = &cursor[..suffix_len];
-        self.term_delta_decoder.decode(prefix_len, suffix);
-        cursor = &cursor[suffix_len..];
-        self.term_info_decoder.decode(code, &mut cursor);
+        let mut cursor: &[u8] = &self.cursor;
+        let code: u8 = cursor[0];
+        cursor = self.term_delta_decoder.decode(code, &cursor[1..]);
+        cursor = self.term_info_decoder.decode(code, cursor);
         self.cursor = cursor;
         true
     }

src/termdict/streamdict/termdict.rs

@@ -17,7 +17,7 @@ use super::{TermStreamerImpl, TermStreamerBuilderImpl};
 use termdict::TermStreamerBuilder;
 use std::mem::transmute;
 
-const PADDING_SIZE: usize = 16;
+const PADDING_SIZE: usize = 4;
 const INDEX_INTERVAL: usize = 1024;
 
 fn convert_fst_error(e: fst::Error) -> io::Error {