first stab

Updates the requirements on [lru](https://github.com/jeromefroe/lru-rs) to permit the latest version.
- [Release notes](https://github.com/jeromefroe/lru-rs/releases)
- [Changelog](https://github.com/jeromefroe/lru-rs/blob/master/CHANGELOG.md)
- [Commits](https://github.com/jeromefroe/lru-rs/compare/0.6.5...0.7.0)

---
updated-dependencies:
- dependency-name: lru
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <support@github.com>

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

.github/workflows/coverage.yml

@@ -0,0 +1,25 @@
+name: Coverage
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  coverage:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Install Rust
+        run: rustup toolchain install nightly --component llvm-tools-preview
+      - name: Install cargo-llvm-cov
+        run: curl -LsSf https://github.com/taiki-e/cargo-llvm-cov/releases/latest/download/cargo-llvm-cov-x86_64-unknown-linux-gnu.tar.gz | tar xzf - -C ~/.cargo/bin
+      - name: Generate code coverage
+        run: cargo llvm-cov --all-features --workspace --lcov --output-path lcov.info
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v1
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos
+          files: lcov.info
+          fail_ci_if_error: true

.github/workflows/long_running.yml

@@ -0,0 +1,24 @@
+name: Rust
+
+on:
+  push:
+    branches: [ main ]
+
+env:
+  CARGO_TERM_COLOR: always
+  NUM_FUNCTIONAL_TEST_ITERATIONS: 20000
+
+jobs:
+  functional_test_unsorted:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Run indexing_unsorted
+      run: cargo test indexing_unsorted -- --ignored
+  functional_test_sorted:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Run indexing_sorted
+      run: cargo test indexing_sorted -- --ignored
+

.github/workflows/test.yml

@@ -10,7 +10,7 @@ env:
   CARGO_TERM_COLOR: always
 
 jobs:
-  build:
+  test:
 
     runs-on: ubuntu-latest
 
@@ -18,7 +18,13 @@ jobs:
     - uses: actions/checkout@v2
     - name: Build
       run: cargo build --verbose --workspace
+    - name: Install latest nightly to test also against unstable feature flag
+      uses: actions-rs/toolchain@v1
+      with:
+            toolchain: nightly
+            override: true
+            components: rustfmt
     - name: Run tests
-      run: cargo test --verbose --workspace
+      run: cargo test --all-features --verbose --workspace
     - name: Check Formatting
       run: cargo fmt --all -- --check

CHANGELOG.md

@@ -1,3 +1,21 @@
+Tantivy 0.16.1
+========================
+- Major Bugfix on multivalued fastfield.  #1151
+
+Tantivy 0.16.0
+=========================
+- Bugfix in the filesum check. (@evanxg852000) #1127
+- Bugfix in positions when the index is sorted by a field. (@appaquet) #1125
+
+Tantivy 0.15.3
+=========================
+- Major bugfix. Deleting documents was broken when the index was sorted by a field. (@appaquet, @fulmicoton) #1101
+
+
+Tantivy 0.15.2
+========================
+- Major bugfix. DocStore still panics when a deleted doc is at the beginning of a block. (@appaquet) #1088
+
 Tantivy 0.15.1
 =========================
 - Major bugfix. DocStore panics when first block is deleted. (@appaquet) #1077

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "tantivy"
-version = "0.15.1"
+version = "0.16.1"
 authors = ["Paul Masurel <paul.masurel@gmail.com>"]
 license = "MIT"
 categories = ["database-implementations", "data-structures"]
@@ -19,22 +19,26 @@ crc32fast = "1.2.1"
 once_cell = "1.7.2"
 regex ={ version = "1.5.4", default-features = false, features = ["std"] }
 tantivy-fst = "0.3"
-memmap = {version = "0.7", optional=true}
-lz4_flex = { version = "0.8.0", default-features = false, features = ["checked-decode"], optional = true }
+memmap2 = {version = "0.5", optional=true}
+lz4_flex = { version = "0.9.0", default-features = false, features = ["checked-decode"], optional = true }
 brotli = { version = "3.3", optional = true }
 snap = { version = "1.0.5", optional = true }
 tempfile = { version = "3.2", optional = true }
 log = "0.4.14"
 serde = { version = "1.0.126", features = ["derive"] }
+serde_closure = "0.3"
 serde_json = "1.0.64"
 num_cpus = "1.13"
 fs2={ version = "0.4.3", optional = true }
 levenshtein_automata = "0.2"
 uuid = { version = "0.8.2", features = ["v4", "serde"] }
-crossbeam = "0.8"
+crossbeam = "0.8.1"
 futures = { version = "0.3.15", features = ["thread-pool"] }
 tantivy-query-grammar = { version="0.15.0", path="./query-grammar" }
 tantivy-bitpacker = { version="0.1", path="./bitpacker" }
+common = { version = "0.1", path = "./common/", package = "tantivy-common" }
+fastfield_codecs = { version="0.1", path="./fastfield_codecs", default-features = false }
+ownedbytes = { version="0.1", path="./ownedbytes" }
 stable_deref_trait = "1.2"
 rust-stemmers = "1.2"
 downcast-rs = "1.2"
@@ -47,10 +51,12 @@ fail = "0.4"
 murmurhash32 = "0.2"
 chrono = "0.4.19"
 smallvec = "1.6.1"
-rayon = "1.5"
-lru = "0.6.5"
+lru = "0.7.0"
 fastdivide = "0.3"
 itertools = "0.10.0"
+measure_time = "0.7.0"
+wasm-mt = "0.1"
+wasm-mt-pool = "0.1"
 
 [target.'cfg(windows)'.dependencies]
 winapi = "0.3.9"
@@ -60,7 +66,9 @@ rand = "0.8.3"
 maplit = "1.0.2"
 matches = "0.1.8"
 proptest = "1.0"
-criterion = "0.3.4"
+criterion = "0.3.5"
+test-env-log = "0.2.7"
+env_logger = "0.9.0"
 
 [dev-dependencies.fail]
 version = "0.4"
@@ -77,7 +85,7 @@ overflow-checks = true
 
 [features]
 default = ["mmap", "lz4-compression" ]
-mmap = ["fs2", "tempfile", "memmap"]
+mmap = ["fs2", "tempfile", "memmap2"]
 
 brotli-compression = ["brotli"]
 lz4-compression = ["lz4_flex"]
@@ -88,7 +96,7 @@ unstable = [] # useful for benches.
 wasm-bindgen = ["uuid/wasm-bindgen"]
 
 [workspace]
-members = ["query-grammar", "bitpacker"]
+members = ["query-grammar", "bitpacker", "common", "fastfield_codecs", "ownedbytes"]
 
 [badges]
 travis-ci = { repository = "tantivy-search/tantivy" }

README.md

@@ -1,9 +1,9 @@
 
-[![Build Status](https://travis-ci.org/tantivy-search/tantivy.svg?branch=main)](https://travis-ci.org/tantivy-search/tantivy)
+[![Docs](https://docs.rs/tantivy/badge.svg)](https://docs.rs/crate/tantivy/)
+[![Build Status](https://github.com/tantivy-search/tantivy/actions/workflows/test.yml/badge.svg)](https://github.com/tantivy-search/tantivy/actions/workflows/test.yml)
 [![codecov](https://codecov.io/gh/tantivy-search/tantivy/branch/main/graph/badge.svg)](https://codecov.io/gh/tantivy-search/tantivy)
 [![Join the chat at https://gitter.im/tantivy-search/tantivy](https://badges.gitter.im/tantivy-search/tantivy.svg)](https://gitter.im/tantivy-search/tantivy?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
-[![Build status](https://ci.appveyor.com/api/projects/status/r7nb13kj23u8m9pj/branch/main?svg=true)](https://ci.appveyor.com/project/fulmicoton/tantivy/branch/main)
 [![Crates.io](https://img.shields.io/crates/v/tantivy.svg)](https://crates.io/crates/tantivy)
 
 ![Tantivy](https://tantivy-search.github.io/logo/tantivy-logo.png)

benches/analyzer.rs

@@ -1,7 +1,7 @@
 use criterion::{criterion_group, criterion_main, Criterion};
 use tantivy::tokenizer::TokenizerManager;
 
-const ALICE_TXT: &'static str = include_str!("alice.txt");
+const ALICE_TXT: &str = include_str!("alice.txt");
 
 pub fn criterion_benchmark(c: &mut Criterion) {
     let tokenizer_manager = TokenizerManager::default();

bitpacker/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "tantivy-bitpacker"
-version = "0.1.0"
+version = "0.1.1"
 edition = "2018"
 authors = ["Paul Masurel <paul.masurel@gmail.com>"]
 license = "MIT"

bitpacker/src/bitpacker.rs

@@ -49,6 +49,7 @@ impl BitPacker {
             let bytes = self.mini_buffer.to_le_bytes();
             output.write_all(&bytes[..num_bytes])?;
             self.mini_buffer_written = 0;
+            self.mini_buffer = 0;
         }
         Ok(())
     }
@@ -61,7 +62,7 @@ impl BitPacker {
     }
 }
 
-#[derive(Clone)]
+#[derive(Clone, Debug, Default)]
 pub struct BitUnpacker {
     num_bits: u64,
     mask: u64,

bitpacker/src/lib.rs

@@ -50,3 +50,32 @@ where
     }
     None
 }
+
+#[test]
+fn test_compute_num_bits() {
+    assert_eq!(compute_num_bits(1), 1u8);
+    assert_eq!(compute_num_bits(0), 0u8);
+    assert_eq!(compute_num_bits(2), 2u8);
+    assert_eq!(compute_num_bits(3), 2u8);
+    assert_eq!(compute_num_bits(4), 3u8);
+    assert_eq!(compute_num_bits(255), 8u8);
+    assert_eq!(compute_num_bits(256), 9u8);
+    assert_eq!(compute_num_bits(5_000_000_000), 33u8);
+}
+
+#[test]
+fn test_minmax_empty() {
+    let vals: Vec<u32> = vec![];
+    assert_eq!(minmax(vals.into_iter()), None);
+}
+
+#[test]
+fn test_minmax_one() {
+    assert_eq!(minmax(vec![1].into_iter()), Some((1, 1)));
+}
+
+#[test]
+fn test_minmax_two() {
+    assert_eq!(minmax(vec![1, 2].into_iter()), Some((1, 2)));
+    assert_eq!(minmax(vec![2, 1].into_iter()), Some((1, 2)));
+}

common/Cargo.toml

@@ -0,0 +1,17 @@
+[package]
+name = "tantivy-common"
+version = "0.1.0"
+authors = ["Paul Masurel <paul@quickwit.io>", "Pascal Seitz <pascal@quickwit.io>"]
+license = "MIT"
+edition = "2018"
+description = "common traits and utility functions used by multiple tantivy subcrates"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+byteorder = "1.4.3"
+ownedbytes = { version="0.1", path="../ownedbytes" }
+
+[dev-dependencies]
+proptest = "1.0.0"
+rand = "0.8.4"

common/src/bitset.rs

@@ -1,8 +1,11 @@
-use std::fmt;
+use ownedbytes::OwnedBytes;
+use std::convert::TryInto;
+use std::io::Write;
 use std::u64;
+use std::{fmt, io};
 
 #[derive(Clone, Copy, Eq, PartialEq)]
-pub(crate) struct TinySet(u64);
+pub struct TinySet(u64);
 
 impl fmt::Debug for TinySet {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
@@ -14,6 +17,7 @@ pub struct TinySetIterator(TinySet);
 impl Iterator for TinySetIterator {
     type Item = u32;
 
+    #[inline]
     fn next(&mut self) -> Option<Self::Item> {
         self.0.pop_lowest()
     }
@@ -28,30 +32,54 @@ impl IntoIterator for TinySet {
 }
 
 impl TinySet {
+    pub fn serialize<T: Write>(&self, writer: &mut T) -> io::Result<()> {
+        writer.write_all(self.0.to_le_bytes().as_ref())
+    }
+
+    #[inline]
+    pub fn deserialize(data: [u8; 8]) -> io::Result<Self> {
+        let val: u64 = u64::from_le_bytes(data);
+        Ok(TinySet(val))
+    }
+
     /// Returns an empty `TinySet`.
+    #[inline]
     pub fn empty() -> TinySet {
         TinySet(0u64)
     }
 
+    /// Returns a full `TinySet`.
+    #[inline]
+    pub fn full() -> TinySet {
+        TinySet::empty().complement()
+    }
+
     pub fn clear(&mut self) {
         self.0 = 0u64;
     }
 
+    #[inline]
     /// Returns the complement of the set in `[0, 64[`.
+    ///
+    /// Careful on making this function public, as it will break the padding handling in the last
+    /// bucket.
     fn complement(self) -> TinySet {
         TinySet(!self.0)
     }
 
+    #[inline]
     /// Returns true iff the `TinySet` contains the element `el`.
     pub fn contains(self, el: u32) -> bool {
         !self.intersect(TinySet::singleton(el)).is_empty()
     }
 
+    #[inline]
     /// Returns the number of elements in the TinySet.
     pub fn len(self) -> u32 {
         self.0.count_ones()
     }
 
+    #[inline]
     /// Returns the intersection of `self` and `other`
     pub fn intersect(self, other: TinySet) -> TinySet {
         TinySet(self.0 & other.0)
@@ -64,13 +92,21 @@ impl TinySet {
         TinySet(1u64 << u64::from(el))
     }
 
-    /// Insert a new element within [0..64[
+    /// Insert a new element within [0..64)
     #[inline]
     pub fn insert(self, el: u32) -> TinySet {
         self.union(TinySet::singleton(el))
     }
 
-    /// Insert a new element within [0..64[
+    /// Removes an element within [0..64)
+    #[inline]
+    pub fn remove(self, el: u32) -> TinySet {
+        self.intersect(TinySet::singleton(el).complement())
+    }
+
+    /// Insert a new element within [0..64)
+    ///
+    /// returns true if the set changed
     #[inline]
     pub fn insert_mut(&mut self, el: u32) -> bool {
         let old = *self;
@@ -78,6 +114,16 @@ impl TinySet {
         old != *self
     }
 
+    /// Remove a element within [0..64)
+    ///
+    /// returns true if the set changed
+    #[inline]
+    pub fn remove_mut(&mut self, el: u32) -> bool {
+        let old = *self;
+        *self = old.remove(el);
+        old != *self
+    }
+
     /// Returns the union of two tinysets
     #[inline]
     pub fn union(self, other: TinySet) -> TinySet {
@@ -123,7 +169,7 @@ impl TinySet {
 #[derive(Clone)]
 pub struct BitSet {
     tinysets: Box<[TinySet]>,
-    len: usize,
+    len: u64,
     max_value: u32,
 }
 
@@ -132,8 +178,41 @@ fn num_buckets(max_val: u32) -> u32 {
 }
 
 impl BitSet {
+    /// serialize a `BitSet`.
+    ///
+    pub fn serialize<T: Write>(&self, writer: &mut T) -> io::Result<()> {
+        writer.write_all(self.max_value.to_le_bytes().as_ref())?;
+
+        for tinyset in self.tinysets.iter() {
+            tinyset.serialize(writer)?;
+        }
+        writer.flush()?;
+        Ok(())
+    }
+
+    /// Deserialize a `BitSet`.
+    ///
+    #[cfg(test)]
+    pub fn deserialize(mut data: &[u8]) -> io::Result<Self> {
+        let max_value: u32 = u32::from_le_bytes(data[..4].try_into().unwrap());
+        data = &data[4..];
+
+        let mut len: u64 = 0;
+        let mut tinysets = vec![];
+        for chunk in data.chunks_exact(8) {
+            let tinyset = TinySet::deserialize(chunk.try_into().unwrap())?;
+            len += tinyset.len() as u64;
+            tinysets.push(tinyset);
+        }
+        Ok(BitSet {
+            tinysets: tinysets.into_boxed_slice(),
+            len,
+            max_value,
+        })
+    }
+
     /// Create a new `BitSet` that may contain elements
-    /// within `[0, max_val[`.
+    /// within `[0, max_val)`.
     pub fn with_max_value(max_value: u32) -> BitSet {
         let num_buckets = num_buckets(max_value);
         let tinybisets = vec![TinySet::empty(); num_buckets as usize].into_boxed_slice();
@@ -144,6 +223,23 @@ impl BitSet {
         }
     }
 
+    /// Create a new `BitSet` that may contain elements. Initially all values will be set.
+    /// within `[0, max_val)`.
+    pub fn with_max_value_and_full(max_value: u32) -> BitSet {
+        let num_buckets = num_buckets(max_value);
+        let mut tinybisets = vec![TinySet::full(); num_buckets as usize].into_boxed_slice();
+
+        // Fix padding
+        let lower = max_value % 64u32;
+        tinybisets[tinybisets.len() - 1] = TinySet::range_lower(lower);
+
+        BitSet {
+            tinysets: tinybisets,
+            len: max_value as u64,
+            max_value,
+        }
+    }
+
     /// Removes all elements from the `BitSet`.
     pub fn clear(&mut self) {
         for tinyset in self.tinysets.iter_mut() {
@@ -153,10 +249,11 @@ impl BitSet {
 
     /// Returns the number of elements in the `BitSet`.
     pub fn len(&self) -> usize {
-        self.len
+        self.len as usize
     }
 
     /// Inserts an element in the `BitSet`
+    #[inline]
     pub fn insert(&mut self, el: u32) {
         // we do not check saturated els.
         let higher = el / 64u32;
@@ -168,7 +265,21 @@ impl BitSet {
         };
     }
 
+    /// Inserts an element in the `BitSet`
+    #[inline]
+    pub fn remove(&mut self, el: u32) {
+        // we do not check saturated els.
+        let higher = el / 64u32;
+        let lower = el % 64u32;
+        self.len -= if self.tinysets[higher as usize].remove_mut(lower) {
+            1
+        } else {
+            0
+        };
+    }
+
     /// Returns true iff the elements is in the `BitSet`.
+    #[inline]
     pub fn contains(&self, el: u32) -> bool {
         self.tinyset(el / 64u32).contains(el % 64)
     }
@@ -178,7 +289,7 @@ impl BitSet {
     ///
     /// Reminder: the tiny set with the bucket `bucket`, represents the
     /// elements from `bucket * 64` to `(bucket+1) * 64`.
-    pub(crate) fn first_non_empty_bucket(&self, bucket: u32) -> Option<u32> {
+    pub fn first_non_empty_bucket(&self, bucket: u32) -> Option<u32> {
         self.tinysets[bucket as usize..]
             .iter()
             .cloned()
@@ -193,23 +304,149 @@ impl BitSet {
     /// Returns the tiny bitset representing the
     /// the set restricted to the number range from
     /// `bucket * 64` to `(bucket + 1) * 64`.
-    pub(crate) fn tinyset(&self, bucket: u32) -> TinySet {
+    pub fn tinyset(&self, bucket: u32) -> TinySet {
         self.tinysets[bucket as usize]
     }
 }
 
+/// Serialized BitSet.
+#[derive(Clone)]
+pub struct ReadSerializedBitSet {
+    data: OwnedBytes,
+    max_value: u32,
+}
+
+impl ReadSerializedBitSet {
+    pub fn open(data: OwnedBytes) -> Self {
+        let (max_value_data, data) = data.split(4);
+        let max_value: u32 = u32::from_le_bytes(max_value_data.as_ref().try_into().unwrap());
+        ReadSerializedBitSet { data, max_value }
+    }
+
+    /// Number of elements in the bitset.
+    #[inline]
+    pub fn len(&self) -> usize {
+        self.iter_tinysets()
+            .map(|tinyset| tinyset.len() as usize)
+            .sum()
+    }
+
+    /// Iterate the tinyset on the fly from serialized data.
+    ///
+    #[inline]
+    fn iter_tinysets<'a>(&'a self) -> impl Iterator<Item = TinySet> + 'a {
+        assert!((self.data.len()) % 8 == 0);
+        self.data.chunks_exact(8).map(move |chunk| {
+            let tinyset: TinySet = TinySet::deserialize(chunk.try_into().unwrap()).unwrap();
+            tinyset
+        })
+    }
+
+    /// Iterate over the positions of the elements.
+    ///
+    #[inline]
+    pub fn iter<'a>(&'a self) -> impl Iterator<Item = u32> + 'a {
+        self.iter_tinysets()
+            .enumerate()
+            .flat_map(move |(chunk_num, tinyset)| {
+                let chunk_base_val = chunk_num as u32 * 64;
+                tinyset
+                    .into_iter()
+                    .map(move |val| val + chunk_base_val)
+                    .take_while(move |doc| *doc < self.max_value)
+            })
+    }
+
+    /// Returns true iff the elements is in the `BitSet`.
+    #[inline]
+    pub fn contains(&self, el: u32) -> bool {
+        let byte_offset = el / 8u32;
+        let b: u8 = self.data[byte_offset as usize];
+        let shift = (el % 8) as u8;
+        b & (1u8 << shift) != 0
+    }
+
+    /// Maximum value the bitset may contain.
+    /// (Note this is not the maximum value contained in the set.)
+    ///
+    /// A bitset has an intrinsic capacity.
+    /// It only stores elements within [0..max_value).
+    #[inline]
+    pub fn max_value(&self) -> u32 {
+        self.max_value
+    }
+}
+
 #[cfg(test)]
 mod tests {
 
     use super::BitSet;
+    use super::ReadSerializedBitSet;
     use super::TinySet;
-    use crate::docset::{DocSet, TERMINATED};
-    use crate::query::BitSetDocSet;
-    use crate::tests;
-    use crate::tests::generate_nonunique_unsorted;
-    use std::collections::BTreeSet;
+    use ownedbytes::OwnedBytes;
+    use rand::distributions::Bernoulli;
+    use rand::rngs::StdRng;
+    use rand::{Rng, SeedableRng};
     use std::collections::HashSet;
+    use std::convert::TryInto;
 
+    #[test]
+    fn test_read_serialized_bitset_full() {
+        let mut bitset = BitSet::with_max_value_and_full(5);
+        bitset.remove(3);
+        let mut out = vec![];
+        bitset.serialize(&mut out).unwrap();
+
+        let bitset = ReadSerializedBitSet::open(OwnedBytes::new(out));
+        assert_eq!(bitset.len(), 4);
+    }
+
+    #[test]
+    fn test_read_serialized_bitset_empty() {
+        let mut bitset = BitSet::with_max_value(5);
+        bitset.insert(3);
+        let mut out = vec![];
+        bitset.serialize(&mut out).unwrap();
+
+        let bitset = ReadSerializedBitSet::open(OwnedBytes::new(out));
+        assert_eq!(bitset.len(), 1);
+
+        {
+            let bitset = BitSet::with_max_value(5);
+            let mut out = vec![];
+            bitset.serialize(&mut out).unwrap();
+            let bitset = ReadSerializedBitSet::open(OwnedBytes::new(out));
+            assert_eq!(bitset.len(), 0);
+        }
+    }
+
+    #[test]
+    fn test_tiny_set_remove() {
+        {
+            let mut u = TinySet::empty().insert(63u32).insert(5).remove(63u32);
+            assert_eq!(u.pop_lowest(), Some(5u32));
+            assert!(u.pop_lowest().is_none());
+        }
+        {
+            let mut u = TinySet::empty()
+                .insert(63u32)
+                .insert(1)
+                .insert(5)
+                .remove(63u32);
+            assert_eq!(u.pop_lowest(), Some(1u32));
+            assert_eq!(u.pop_lowest(), Some(5u32));
+            assert!(u.pop_lowest().is_none());
+        }
+        {
+            let mut u = TinySet::empty().insert(1).remove(63u32);
+            assert_eq!(u.pop_lowest(), Some(1u32));
+            assert!(u.pop_lowest().is_none());
+        }
+        {
+            let mut u = TinySet::empty().insert(1).remove(1u32);
+            assert!(u.pop_lowest().is_none());
+        }
+    }
     #[test]
     fn test_tiny_set() {
         assert!(TinySet::empty().is_empty());
@@ -235,6 +472,21 @@ mod tests {
             assert_eq!(u.pop_lowest(), Some(63u32));
             assert!(u.pop_lowest().is_none());
         }
+        {
+            let mut u = TinySet::empty().insert(63u32).insert(5);
+            assert_eq!(u.pop_lowest(), Some(5u32));
+            assert_eq!(u.pop_lowest(), Some(63u32));
+            assert!(u.pop_lowest().is_none());
+        }
+        {
+            let u = TinySet::empty().insert(63u32).insert(5);
+            let mut data = vec![];
+            u.serialize(&mut data).unwrap();
+            let mut u = TinySet::deserialize(data[..8].try_into().unwrap()).unwrap();
+            assert_eq!(u.pop_lowest(), Some(5u32));
+            assert_eq!(u.pop_lowest(), Some(63u32));
+            assert!(u.pop_lowest().is_none());
+        }
     }
 
     #[test]
@@ -251,6 +503,16 @@ mod tests {
                 assert_eq!(hashset.contains(&el), bitset.contains(el));
             }
             assert_eq!(bitset.max_value(), max_value);
+
+            // test deser
+            let mut data = vec![];
+            bitset.serialize(&mut data).unwrap();
+            let bitset = BitSet::deserialize(&data).unwrap();
+            for el in 0..max_value {
+                assert_eq!(hashset.contains(&el), bitset.contains(el));
+            }
+            assert_eq!(bitset.max_value(), max_value);
+            assert_eq!(bitset.len(), els.len());
         };
 
         test_against_hashset(&[], 0);
@@ -263,29 +525,6 @@ mod tests {
         test_against_hashset(&[62u32, 63u32], 64);
     }
 
-    #[test]
-    fn test_bitset_large() {
-        let arr = generate_nonunique_unsorted(100_000, 5_000);
-        let mut btreeset: BTreeSet<u32> = BTreeSet::new();
-        let mut bitset = BitSet::with_max_value(100_000);
-        for el in arr {
-            btreeset.insert(el);
-            bitset.insert(el);
-        }
-        for i in 0..100_000 {
-            assert_eq!(btreeset.contains(&i), bitset.contains(i));
-        }
-        assert_eq!(btreeset.len(), bitset.len());
-        let mut bitset_docset = BitSetDocSet::from(bitset);
-        let mut remaining = true;
-        for el in btreeset.into_iter() {
-            assert!(remaining);
-            assert_eq!(bitset_docset.doc(), el);
-            remaining = bitset_docset.advance() != TERMINATED;
-        }
-        assert!(!remaining);
-    }
-
     #[test]
     fn test_bitset_num_buckets() {
         use super::num_buckets;
@@ -338,12 +577,33 @@ mod tests {
         assert_eq!(bitset.len(), 2);
         bitset.insert(104u32);
         assert_eq!(bitset.len(), 3);
+        bitset.remove(105u32);
+        assert_eq!(bitset.len(), 3);
+        bitset.remove(104u32);
+        assert_eq!(bitset.len(), 2);
+        bitset.remove(3u32);
+        assert_eq!(bitset.len(), 1);
+        bitset.remove(103u32);
+        assert_eq!(bitset.len(), 0);
+    }
+
+    pub fn sample_with_seed(n: u32, ratio: f64, seed_val: u8) -> Vec<u32> {
+        StdRng::from_seed([seed_val; 32])
+            .sample_iter(&Bernoulli::new(ratio).unwrap())
+            .take(n as usize)
+            .enumerate()
+            .filter_map(|(val, keep)| if keep { Some(val as u32) } else { None })
+            .collect()
+    }
+
+    pub fn sample(n: u32, ratio: f64) -> Vec<u32> {
+        sample_with_seed(n, ratio, 4)
     }
 
     #[test]
     fn test_bitset_clear() {
         let mut bitset = BitSet::with_max_value(1_000);
-        let els = tests::sample(1_000, 0.01f64);
+        let els = sample(1_000, 0.01f64);
         for &el in &els {
             bitset.insert(el);
         }

common/src/lib.rs

@@ -1,23 +1,16 @@
-mod bitset;
-mod composite_file;
-mod counting_writer;
-mod serialize;
-mod vint;
+use std::ops::Deref;
 
-pub use self::bitset::BitSet;
-pub(crate) use self::bitset::TinySet;
-pub(crate) use self::composite_file::{CompositeFile, CompositeWrite};
-pub use self::counting_writer::CountingWriter;
-pub use self::serialize::{BinarySerializable, DeserializeFrom, FixedSize};
-pub use self::vint::{
-    read_u32_vint, read_u32_vint_no_advance, serialize_vint_u32, write_u32_vint, VInt,
-};
 pub use byteorder::LittleEndian as Endianness;
 
-/// Segment's max doc must be `< MAX_DOC_LIMIT`.
-///
-/// We do not allow segments with more than
-pub const MAX_DOC_LIMIT: u32 = 1 << 31;
+mod bitset;
+mod serialize;
+mod vint;
+mod writer;
+
+pub use bitset::*;
+pub use serialize::{BinarySerializable, DeserializeFrom, FixedSize};
+pub use vint::{read_u32_vint, read_u32_vint_no_advance, serialize_vint_u32, write_u32_vint, VInt};
+pub use writer::{AntiCallToken, CountingWriter, TerminatingWrite};
 
 /// Has length trait
 pub trait HasLen {
@@ -30,6 +23,12 @@ pub trait HasLen {
     }
 }
 
+impl<T: Deref<Target = [u8]>> HasLen for T {
+    fn len(&self) -> usize {
+        self.deref().len()
+    }
+}
+
 const HIGHEST_BIT: u64 = 1 << 63;
 
 /// Maps a `i64` to `u64`
@@ -101,14 +100,12 @@ pub fn u64_to_f64(val: u64) -> f64 {
 }
 
 #[cfg(test)]
-pub(crate) mod test {
+pub mod test {
 
-    pub use super::serialize::test::fixed_size_test;
     use super::{f64_to_u64, i64_to_u64, u64_to_f64, u64_to_i64};
+    use super::{BinarySerializable, FixedSize};
     use proptest::prelude::*;
     use std::f64;
-    use tantivy_bitpacker::compute_num_bits;
-    pub use tantivy_bitpacker::minmax;
 
     fn test_i64_converter_helper(val: i64) {
         assert_eq!(u64_to_i64(i64_to_u64(val)), val);
@@ -118,6 +115,12 @@ pub(crate) mod test {
         assert_eq!(u64_to_f64(f64_to_u64(val)), val);
     }
 
+    pub fn fixed_size_test<O: BinarySerializable + FixedSize + Default>() {
+        let mut buffer = Vec::new();
+        O::default().serialize(&mut buffer).unwrap();
+        assert_eq!(buffer.len(), O::SIZE_IN_BYTES);
+    }
+
     proptest! {
         #[test]
         fn test_f64_converter_monotonicity_proptest((left, right) in (proptest::num::f64::NORMAL, proptest::num::f64::NORMAL)) {
@@ -161,40 +164,4 @@ pub(crate) mod test {
         assert!(f64_to_u64(-2.0) < f64_to_u64(1.0));
         assert!(f64_to_u64(-2.0) < f64_to_u64(-1.5));
     }
-
-    #[test]
-    fn test_compute_num_bits() {
-        assert_eq!(compute_num_bits(1), 1u8);
-        assert_eq!(compute_num_bits(0), 0u8);
-        assert_eq!(compute_num_bits(2), 2u8);
-        assert_eq!(compute_num_bits(3), 2u8);
-        assert_eq!(compute_num_bits(4), 3u8);
-        assert_eq!(compute_num_bits(255), 8u8);
-        assert_eq!(compute_num_bits(256), 9u8);
-        assert_eq!(compute_num_bits(5_000_000_000), 33u8);
-    }
-
-    #[test]
-    fn test_max_doc() {
-        // this is the first time I write a unit test for a constant.
-        assert!(((super::MAX_DOC_LIMIT - 1) as i32) >= 0);
-        assert!((super::MAX_DOC_LIMIT as i32) < 0);
-    }
-
-    #[test]
-    fn test_minmax_empty() {
-        let vals: Vec<u32> = vec![];
-        assert_eq!(minmax(vals.into_iter()), None);
-    }
-
-    #[test]
-    fn test_minmax_one() {
-        assert_eq!(minmax(vec![1].into_iter()), Some((1, 1)));
-    }
-
-    #[test]
-    fn test_minmax_two() {
-        assert_eq!(minmax(vec![1, 2].into_iter()), Some((1, 2)));
-        assert_eq!(minmax(vec![2, 1].into_iter()), Some((1, 2)));
-    }
 }

common/src/serialize.rs

@@ -1,5 +1,5 @@
-use crate::common::Endianness;
-use crate::common::VInt;
+use crate::Endianness;
+use crate::VInt;
 use byteorder::{ReadBytesExt, WriteBytesExt};
 use std::fmt;
 use std::io;
@@ -160,6 +160,28 @@ impl FixedSize for u8 {
     const SIZE_IN_BYTES: usize = 1;
 }
 
+impl BinarySerializable for bool {
+    fn serialize<W: Write>(&self, writer: &mut W) -> io::Result<()> {
+        let val = if *self { 1 } else { 0 };
+        writer.write_u8(val)
+    }
+    fn deserialize<R: Read>(reader: &mut R) -> io::Result<bool> {
+        let val = reader.read_u8()?;
+        match val {
+            0 => Ok(false),
+            1 => Ok(true),
+            _ => Err(io::Error::new(
+                io::ErrorKind::InvalidData,
+                "invalid bool value on deserialization, data corrupted",
+            )),
+        }
+    }
+}
+
+impl FixedSize for bool {
+    const SIZE_IN_BYTES: usize = 1;
+}
+
 impl BinarySerializable for String {
     fn serialize<W: Write>(&self, writer: &mut W) -> io::Result<()> {
         let data: &[u8] = self.as_bytes();
@@ -180,9 +202,9 @@ impl BinarySerializable for String {
 #[cfg(test)]
 pub mod test {
 
+    use super::VInt;
     use super::*;
-    use crate::common::VInt;
-
+    use crate::serialize::BinarySerializable;
     pub fn fixed_size_test<O: BinarySerializable + FixedSize + Default>() {
         let mut buffer = Vec::new();
         O::default().serialize(&mut buffer).unwrap();

common/src/vint.rs

@@ -106,7 +106,7 @@ pub fn read_u32_vint_no_advance(data: &[u8]) -> (u32, usize) {
 pub fn write_u32_vint<W: io::Write>(val: u32, writer: &mut W) -> io::Result<()> {
     let mut buf = [0u8; 8];
     let data = serialize_vint_u32(val, &mut buf);
-    writer.write_all(&data)
+    writer.write_all(data)
 }
 
 impl VInt {
@@ -175,14 +175,14 @@ impl BinarySerializable for VInt {
 mod tests {
 
     use super::serialize_vint_u32;
+    use super::BinarySerializable;
     use super::VInt;
-    use crate::common::BinarySerializable;
 
     fn aux_test_vint(val: u64) {
         let mut v = [14u8; 10];
         let num_bytes = VInt(val).serialize_into(&mut v);
-        for i in num_bytes..10 {
-            assert_eq!(v[i], 14u8);
+        for el in &v[num_bytes..10] {
+            assert_eq!(el, &14u8);
         }
         assert!(num_bytes > 0);
         if num_bytes < 10 {

common/src/writer.rs

@@ -1,7 +1,4 @@
-use crate::directory::AntiCallToken;
-use crate::directory::TerminatingWrite;
-use std::io;
-use std::io::Write;
+use std::io::{self, BufWriter, Write};
 
 pub struct CountingWriter<W> {
     underlying: W,
@@ -16,41 +13,87 @@ impl<W: Write> CountingWriter<W> {
         }
     }
 
+    #[inline]
     pub fn written_bytes(&self) -> u64 {
         self.written_bytes
     }
 
     /// Returns the underlying write object.
     /// Note that this method does not trigger any flushing.
+    #[inline]
     pub fn finish(self) -> W {
         self.underlying
     }
 }
 
 impl<W: Write> Write for CountingWriter<W> {
+    #[inline]
     fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
         let written_size = self.underlying.write(buf)?;
         self.written_bytes += written_size as u64;
         Ok(written_size)
     }
 
+    #[inline]
     fn write_all(&mut self, buf: &[u8]) -> io::Result<()> {
         self.underlying.write_all(buf)?;
         self.written_bytes += buf.len() as u64;
         Ok(())
     }
 
+    #[inline]
     fn flush(&mut self) -> io::Result<()> {
         self.underlying.flush()
     }
 }
 
 impl<W: TerminatingWrite> TerminatingWrite for CountingWriter<W> {
+    #[inline]
     fn terminate_ref(&mut self, token: AntiCallToken) -> io::Result<()> {
         self.underlying.terminate_ref(token)
     }
 }
 
+/// Struct used to prevent from calling [`terminate_ref`](trait.TerminatingWrite#method.terminate_ref) directly
+///
+/// The point is that while the type is public, it cannot be built by anyone
+/// outside of this module.
+pub struct AntiCallToken(());
+
+/// Trait used to indicate when no more write need to be done on a writer
+pub trait TerminatingWrite: Write {
+    /// Indicate that the writer will no longer be used. Internally call terminate_ref.
+    fn terminate(mut self) -> io::Result<()>
+    where
+        Self: Sized,
+    {
+        self.terminate_ref(AntiCallToken(()))
+    }
+
+    /// You should implement this function to define custom behavior.
+    /// This function should flush any buffer it may hold.
+    fn terminate_ref(&mut self, _: AntiCallToken) -> io::Result<()>;
+}
+
+impl<W: TerminatingWrite + ?Sized> TerminatingWrite for Box<W> {
+    fn terminate_ref(&mut self, token: AntiCallToken) -> io::Result<()> {
+        self.as_mut().terminate_ref(token)
+    }
+}
+
+impl<W: TerminatingWrite> TerminatingWrite for BufWriter<W> {
+    fn terminate_ref(&mut self, a: AntiCallToken) -> io::Result<()> {
+        self.flush()?;
+        self.get_mut().terminate_ref(a)
+    }
+}
+
+impl<'a> TerminatingWrite for &'a mut Vec<u8> {
+    fn terminate_ref(&mut self, _a: AntiCallToken) -> io::Result<()> {
+        self.flush()
+    }
+}
+
 #[cfg(test)]
 mod test {
 

doc/src/SUMMARY.md

@@ -7,6 +7,7 @@
 - [Segments](./basis.md)
 - [Defining your schema](./schema.md)
 - [Facetting](./facetting.md)
+- [Index Sorting](./index_sorting.md)
 - [Innerworkings](./innerworkings.md)
   - [Inverted index](./inverted_index.md)
 - [Best practise](./inverted_index.md)

doc/src/index_sorting.md

@@ -0,0 +1,61 @@
+
+- [Index Sorting](#index-sorting)
+    + [Why Sorting](#why-sorting)
+        * [Compression](#compression)
+        * [Top-N Optimization](#top-n-optimization)
+        * [Pruning](#pruning)
+        * [Other](#other)
+    + [Usage](#usage)
+
+# Index Sorting
+
+Tantivy allows you to sort the index according to a property.
+
+## Why Sorting
+
+Presorting an index has several advantages:
+
+###### Compression
+
+When data is sorted it is easier to compress the data. E.g. the numbers sequence [5, 2, 3, 1, 4] would be sorted to [1, 2, 3, 4, 5]. 
+If we apply delta encoding this list would be unsorted [5, -3, 1, -2, 3] vs. [1, 1, 1, 1, 1].
+Compression ratio is mainly affected on the fast field of the sorted property, every thing else is likely unaffected. 
+###### Top-N Optimization
+
+When data is presorted by a field and search queries request sorting by the same field, we can leverage the natural order of the documents. 
+E.g. if the data is sorted by timestamp and want the top n newest docs containing a term, we can simply leveraging the order of the docids.
+
+Note: Tantivy 0.16 does not do this optimization yet.
+
+###### Pruning
+
+Let's say we want all documents and want to apply the filter `>= 2010-08-11`. When the data is sorted, we could make a lookup in the fast field to find the docid range and use this as the filter.
+
+Note: Tantivy 0.16 does not do this optimization yet.
+
+###### Other?
+
+In principle there are many algorithms possible that exploit the monotonically increasing nature. (aggregations maybe?)
+
+## Usage
+The index sorting can be configured setting [`sort_by_field`](https://github.com/tantivy-search/tantivy/blob/000d76b11a139a84b16b9b95060a1c93e8b9851c/src/core/index_meta.rs#L238) on `IndexSettings` and passing it to a `IndexBuilder`. As of tantvy 0.16 only fast fields are allowed to be used.
+
+```
+let settings = IndexSettings {
+    sort_by_field: Some(IndexSortByField {
+        field: "intval".to_string(),
+        order: Order::Desc,
+    }),
+    ..Default::default()
+};
+let mut index_builder = Index::builder().schema(schema);
+index_builder = index_builder.settings(settings);
+let index = index_builder.create_in_ram().unwrap();
+```
+
+## Implementation details
+
+Sorting an index is applied in the serialization step. In general there are two serialization steps: [Finishing a single segment](https://github.com/tantivy-search/tantivy/blob/000d76b11a139a84b16b9b95060a1c93e8b9851c/src/indexer/segment_writer.rs#L338) and [merging multiple segments](https://github.com/tantivy-search/tantivy/blob/000d76b11a139a84b16b9b95060a1c93e8b9851c/src/indexer/merger.rs#L1073).
+
+In both cases we generate a docid mapping reflecting the sort. This mapping is used when serializing the different components (doc store, fastfields, posting list, normfield, facets).
+

examples/custom_collector.rs

@@ -86,12 +86,10 @@ impl Collector for StatsCollector {
 
     fn merge_fruits(&self, segment_stats: Vec<Option<Stats>>) -> tantivy::Result<Option<Stats>> {
         let mut stats = Stats::default();
-        for segment_stats_opt in segment_stats {
-            if let Some(segment_stats) = segment_stats_opt {
-                stats.count += segment_stats.count;
-                stats.sum += segment_stats.sum;
-                stats.squared_sum += segment_stats.squared_sum;
-            }
+        for segment_stats in segment_stats.into_iter().flatten() {
+            stats.count += segment_stats.count;
+            stats.sum += segment_stats.sum;
+            stats.squared_sum += segment_stats.squared_sum;
         }
         Ok(stats.non_zero_count())
     }
@@ -139,7 +137,7 @@ fn main() -> tantivy::Result<()> {
     //
     // Lets index a bunch of fake documents for the sake of
     // this example.
-    let index = Index::create_in_ram(schema.clone());
+    let index = Index::create_in_ram(schema);
 
     let mut index_writer = index.writer(50_000_000)?;
     index_writer.add_document(doc!(

examples/faceted_search_with_tweaked_score.rs

@@ -12,7 +12,7 @@ fn main() -> tantivy::Result<()> {
     let ingredient = schema_builder.add_facet_field("ingredient", INDEXED);
 
     let schema = schema_builder.build();
-    let index = Index::create_in_ram(schema.clone());
+    let index = Index::create_in_ram(schema);
 
     let mut index_writer = index.writer(30_000_000)?;
 
@@ -51,7 +51,7 @@ fn main() -> tantivy::Result<()> {
         let query = BooleanQuery::new_multiterms_query(
             facets
                 .iter()
-                .map(|key| Term::from_facet(ingredient, &key))
+                .map(|key| Term::from_facet(ingredient, key))
                 .collect(),
         );
         let top_docs_by_custom_score =

examples/iterating_docs_and_positions.rs

@@ -22,7 +22,7 @@ fn main() -> tantivy::Result<()> {
     let title = schema_builder.add_text_field("title", TEXT | STORED);
     let schema = schema_builder.build();
 
-    let index = Index::create_in_ram(schema.clone());
+    let index = Index::create_in_ram(schema);
 
     let mut index_writer = index.writer_with_num_threads(1, 50_000_000)?;
     index_writer.add_document(doc!(title => "The Old Man and the Sea"));

examples/pre_tokenized_text.rs

@@ -82,7 +82,7 @@ fn main() -> tantivy::Result<()> {
         }]
     }"#;
 
-    let short_man_doc = schema.parse_document(&short_man_json)?;
+    let short_man_doc = schema.parse_document(short_man_json)?;
 
     index_writer.add_document(short_man_doc);
 

examples/snippet.rs

@@ -25,7 +25,7 @@ fn main() -> tantivy::Result<()> {
     let schema = schema_builder.build();
 
     // # Indexing documents
-    let index = Index::create_in_dir(&index_path, schema.clone())?;
+    let index = Index::create_in_dir(&index_path, schema)?;
 
     let mut index_writer = index.writer(50_000_000)?;
 

examples/working_with_json.rs

@@ -1,4 +1,3 @@
-use tantivy;
 use tantivy::schema::*;
 
 // # Document from json
@@ -22,7 +21,7 @@ fn main() -> tantivy::Result<()> {
     }"#;
 
     // We can parse our document
-    let _mice_and_men_doc = schema.parse_document(&mice_and_men_doc_json)?;
+    let _mice_and_men_doc = schema.parse_document(mice_and_men_doc_json)?;
 
     // Multi-valued field are allowed, they are
     // expressed in JSON by an array.
@@ -31,7 +30,7 @@ fn main() -> tantivy::Result<()> {
        "title": ["Frankenstein", "The Modern Prometheus"],
        "year": 1818
     }"#;
-    let _frankenstein_doc = schema.parse_document(&frankenstein_json)?;
+    let _frankenstein_doc = schema.parse_document(frankenstein_json)?;
 
     // Note that the schema is saved in your index directory.
     //

fastfield_codecs/Cargo.toml

@@ -0,0 +1,24 @@
+[package]
+name = "fastfield_codecs"
+version = "0.1.0"
+authors = ["Pascal Seitz <pascal@quickwit.io>"]
+license = "MIT"
+edition = "2018"
+description = "Fast field codecs used by tantivy"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+common = { version = "0.1", path = "../common/", package = "tantivy-common" }
+tantivy-bitpacker = { version="0.1.1", path = "../bitpacker/" }
+prettytable-rs = {version="0.8.0", optional= true}
+rand = {version="0.8.3", optional= true}
+
+[dev-dependencies]
+more-asserts = "0.2.1"
+rand = "0.8.3"
+
+[features]
+bin = ["prettytable-rs", "rand"]
+default = ["bin"]
+

fastfield_codecs/README.md

@@ -0,0 +1,68 @@
+
+
+# Fast Field Codecs
+
+This crate contains various fast field codecs, used to compress/decompress fast field data in tantivy.
+
+## Contributing
+
+Contributing is pretty straightforward. Since the bitpacking is the simplest compressor, you can check it for reference.
+
+A codec needs to implement 2 traits:
+
+- A reader implementing `FastFieldCodecReader` to read the codec.
+- A serializer implementing `FastFieldCodecSerializer` for compression estimation and codec name + id.
+
+### Tests
+
+Once the traits are implemented test and benchmark integration is pretty easy (see `test_with_codec_data_sets` and `bench.rs`).
+
+Make sure to add the codec to the main.rs, which tests the compression ratio and estimation against different data sets. You can run it with:
+```
+cargo run --features bin
+```
+
+### TODO
+- Add real world data sets in comparison
+- Add codec to cover sparse data sets
+
+
+### Codec Comparison
+```
++----------------------------------+-------------------+------------------------+
+|                                  | Compression Ratio | Compression Estimation |
++----------------------------------+-------------------+------------------------+
+| Autoincrement                    |                   |                        |
++----------------------------------+-------------------+------------------------+
+| LinearInterpol                   | 0.000039572664    | 0.000004396963         |
++----------------------------------+-------------------+------------------------+
+| MultiLinearInterpol              | 0.1477348         | 0.17275847             |
++----------------------------------+-------------------+------------------------+
+| Bitpacked                        | 0.28126493        | 0.28125                |
++----------------------------------+-------------------+------------------------+
+| Monotonically increasing concave |                   |                        |
++----------------------------------+-------------------+------------------------+
+| LinearInterpol                   | 0.25003937        | 0.26562938             |
++----------------------------------+-------------------+------------------------+
+| MultiLinearInterpol              | 0.190665          | 0.1883836              |
++----------------------------------+-------------------+------------------------+
+| Bitpacked                        | 0.31251436        | 0.3125                 |
++----------------------------------+-------------------+------------------------+
+| Monotonically increasing convex  |                   |                        |
++----------------------------------+-------------------+------------------------+
+| LinearInterpol                   | 0.25003937        | 0.28125438             |
++----------------------------------+-------------------+------------------------+
+| MultiLinearInterpol              | 0.18676           | 0.2040086              |
++----------------------------------+-------------------+------------------------+
+| Bitpacked                        | 0.31251436        | 0.3125                 |
++----------------------------------+-------------------+------------------------+
+| Almost monotonically increasing  |                   |                        |
++----------------------------------+-------------------+------------------------+
+| LinearInterpol                   | 0.14066513        | 0.1562544              |
++----------------------------------+-------------------+------------------------+
+| MultiLinearInterpol              | 0.16335973        | 0.17275847             |
++----------------------------------+-------------------+------------------------+
+| Bitpacked                        | 0.28126493        | 0.28125                |
++----------------------------------+-------------------+------------------------+
+
+```

fastfield_codecs/benches/bench.rs

@@ -0,0 +1,108 @@
+#![feature(test)]
+
+extern crate test;
+
+#[cfg(test)]
+mod tests {
+    use fastfield_codecs::{
+        bitpacked::{BitpackedFastFieldReader, BitpackedFastFieldSerializer},
+        linearinterpol::{LinearInterpolFastFieldReader, LinearInterpolFastFieldSerializer},
+        multilinearinterpol::{
+            MultiLinearInterpolFastFieldReader, MultiLinearInterpolFastFieldSerializer,
+        },
+        *,
+    };
+
+    fn get_data() -> Vec<u64> {
+        let mut data: Vec<_> = (100..55000_u64)
+            .map(|num| num + rand::random::<u8>() as u64)
+            .collect();
+        data.push(99_000);
+        data.insert(1000, 2000);
+        data.insert(2000, 100);
+        data.insert(3000, 4100);
+        data.insert(4000, 100);
+        data.insert(5000, 800);
+        data
+    }
+
+    fn value_iter() -> impl Iterator<Item = u64> {
+        0..20_000
+    }
+    fn bench_get<S: FastFieldCodecSerializer, R: FastFieldCodecReader>(
+        b: &mut Bencher,
+        data: &[u64],
+    ) {
+        let mut bytes = vec![];
+        S::serialize(
+            &mut bytes,
+            &data,
+            stats_from_vec(data),
+            data.iter().cloned(),
+            data.iter().cloned(),
+        )
+        .unwrap();
+        let reader = R::open_from_bytes(&bytes).unwrap();
+        b.iter(|| {
+            for pos in value_iter() {
+                reader.get_u64(pos as u64, &bytes);
+            }
+        });
+    }
+    fn bench_create<S: FastFieldCodecSerializer>(b: &mut Bencher, data: &[u64]) {
+        let mut bytes = vec![];
+        b.iter(|| {
+            S::serialize(
+                &mut bytes,
+                &data,
+                stats_from_vec(data),
+                data.iter().cloned(),
+                data.iter().cloned(),
+            )
+            .unwrap();
+        });
+    }
+
+    use test::Bencher;
+    #[bench]
+    fn bench_fastfield_bitpack_create(b: &mut Bencher) {
+        let data: Vec<_> = get_data();
+        bench_create::<BitpackedFastFieldSerializer>(b, &data);
+    }
+    #[bench]
+    fn bench_fastfield_linearinterpol_create(b: &mut Bencher) {
+        let data: Vec<_> = get_data();
+        bench_create::<LinearInterpolFastFieldSerializer>(b, &data);
+    }
+    #[bench]
+    fn bench_fastfield_multilinearinterpol_create(b: &mut Bencher) {
+        let data: Vec<_> = get_data();
+        bench_create::<MultiLinearInterpolFastFieldSerializer>(b, &data);
+    }
+    #[bench]
+    fn bench_fastfield_bitpack_get(b: &mut Bencher) {
+        let data: Vec<_> = get_data();
+        bench_get::<BitpackedFastFieldSerializer, BitpackedFastFieldReader>(b, &data);
+    }
+    #[bench]
+    fn bench_fastfield_linearinterpol_get(b: &mut Bencher) {
+        let data: Vec<_> = get_data();
+        bench_get::<LinearInterpolFastFieldSerializer, LinearInterpolFastFieldReader>(b, &data);
+    }
+    #[bench]
+    fn bench_fastfield_multilinearinterpol_get(b: &mut Bencher) {
+        let data: Vec<_> = get_data();
+        bench_get::<MultiLinearInterpolFastFieldSerializer, MultiLinearInterpolFastFieldReader>(
+            b, &data,
+        );
+    }
+    pub fn stats_from_vec(data: &[u64]) -> FastFieldStats {
+        let min_value = data.iter().cloned().min().unwrap_or(0);
+        let max_value = data.iter().cloned().max().unwrap_or(0);
+        FastFieldStats {
+            min_value,
+            max_value,
+            num_vals: data.len() as u64,
+        }
+    }
+}

fastfield_codecs/src/bitpacked.rs

@@ -0,0 +1,176 @@
+use crate::FastFieldCodecReader;
+use crate::FastFieldCodecSerializer;
+use crate::FastFieldDataAccess;
+use crate::FastFieldStats;
+use common::BinarySerializable;
+use std::io::{self, Write};
+use tantivy_bitpacker::compute_num_bits;
+use tantivy_bitpacker::BitPacker;
+
+use tantivy_bitpacker::BitUnpacker;
+
+/// Depending on the field type, a different
+/// fast field is required.
+#[derive(Clone)]
+pub struct BitpackedFastFieldReader {
+    bit_unpacker: BitUnpacker,
+    pub min_value_u64: u64,
+    pub max_value_u64: u64,
+}
+
+impl<'data> FastFieldCodecReader for BitpackedFastFieldReader {
+    /// Opens a fast field given a file.
+    fn open_from_bytes(bytes: &[u8]) -> io::Result<Self> {
+        let (_data, mut footer) = bytes.split_at(bytes.len() - 16);
+        let min_value = u64::deserialize(&mut footer)?;
+        let amplitude = u64::deserialize(&mut footer)?;
+        let max_value = min_value + amplitude;
+        let num_bits = compute_num_bits(amplitude);
+        let bit_unpacker = BitUnpacker::new(num_bits);
+        Ok(BitpackedFastFieldReader {
+            min_value_u64: min_value,
+            max_value_u64: max_value,
+            bit_unpacker,
+        })
+    }
+    #[inline]
+    fn get_u64(&self, doc: u64, data: &[u8]) -> u64 {
+        self.min_value_u64 + self.bit_unpacker.get(doc, data)
+    }
+    #[inline]
+    fn min_value(&self) -> u64 {
+        self.min_value_u64
+    }
+    #[inline]
+    fn max_value(&self) -> u64 {
+        self.max_value_u64
+    }
+}
+pub struct BitpackedFastFieldSerializerLegacy<'a, W: 'a + Write> {
+    bit_packer: BitPacker,
+    write: &'a mut W,
+    min_value: u64,
+    amplitude: u64,
+    num_bits: u8,
+}
+
+impl<'a, W: Write> BitpackedFastFieldSerializerLegacy<'a, W> {
+    /// Creates a new fast field serializer.
+    ///
+    /// The serializer in fact encode the values by bitpacking
+    /// `(val - min_value)`.
+    ///
+    /// It requires a `min_value` and a `max_value` to compute
+    /// compute the minimum number of bits required to encode
+    /// values.
+    pub fn open(
+        write: &'a mut W,
+        min_value: u64,
+        max_value: u64,
+    ) -> io::Result<BitpackedFastFieldSerializerLegacy<'a, W>> {
+        assert!(min_value <= max_value);
+        let amplitude = max_value - min_value;
+        let num_bits = compute_num_bits(amplitude);
+        let bit_packer = BitPacker::new();
+        Ok(BitpackedFastFieldSerializerLegacy {
+            bit_packer,
+            write,
+            min_value,
+            amplitude,
+            num_bits,
+        })
+    }
+    /// Pushes a new value to the currently open u64 fast field.
+    #[inline]
+    pub fn add_val(&mut self, val: u64) -> io::Result<()> {
+        let val_to_write: u64 = val - self.min_value;
+        self.bit_packer
+            .write(val_to_write, self.num_bits, &mut self.write)?;
+        Ok(())
+    }
+    pub fn close_field(mut self) -> io::Result<()> {
+        self.bit_packer.close(&mut self.write)?;
+        self.min_value.serialize(&mut self.write)?;
+        self.amplitude.serialize(&mut self.write)?;
+        Ok(())
+    }
+}
+
+pub struct BitpackedFastFieldSerializer {}
+
+impl FastFieldCodecSerializer for BitpackedFastFieldSerializer {
+    const NAME: &'static str = "Bitpacked";
+    const ID: u8 = 1;
+    /// Serializes data with the BitpackedFastFieldSerializer.
+    ///
+    /// The serializer in fact encode the values by bitpacking
+    /// `(val - min_value)`.
+    ///
+    /// It requires a `min_value` and a `max_value` to compute
+    /// compute the minimum number of bits required to encode
+    /// values.
+    fn serialize(
+        write: &mut impl Write,
+        _fastfield_accessor: &impl FastFieldDataAccess,
+        stats: FastFieldStats,
+        data_iter: impl Iterator<Item = u64>,
+        _data_iter1: impl Iterator<Item = u64>,
+    ) -> io::Result<()> {
+        let mut serializer =
+            BitpackedFastFieldSerializerLegacy::open(write, stats.min_value, stats.max_value)?;
+
+        for val in data_iter {
+            serializer.add_val(val)?;
+        }
+        serializer.close_field()?;
+
+        Ok(())
+    }
+    fn is_applicable(
+        _fastfield_accessor: &impl FastFieldDataAccess,
+        _stats: FastFieldStats,
+    ) -> bool {
+        true
+    }
+    fn estimate(_fastfield_accessor: &impl FastFieldDataAccess, stats: FastFieldStats) -> f32 {
+        let amplitude = stats.max_value - stats.min_value;
+        let num_bits = compute_num_bits(amplitude);
+        let num_bits_uncompressed = 64;
+        num_bits as f32 / num_bits_uncompressed as f32
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::tests::get_codec_test_data_sets;
+
+    fn create_and_validate(data: &[u64], name: &str) {
+        crate::tests::create_and_validate::<BitpackedFastFieldSerializer, BitpackedFastFieldReader>(
+            data, name,
+        );
+    }
+
+    #[test]
+    fn test_with_codec_data_sets() {
+        let data_sets = get_codec_test_data_sets();
+        for (mut data, name) in data_sets {
+            create_and_validate(&data, name);
+            data.reverse();
+            create_and_validate(&data, name);
+        }
+    }
+
+    #[test]
+    fn bitpacked_fast_field_rand() {
+        for _ in 0..500 {
+            let mut data = (0..1 + rand::random::<u8>() as usize)
+                .map(|_| rand::random::<i64>() as u64 / 2)
+                .collect::<Vec<_>>();
+            create_and_validate(&data, "rand");
+
+            data.reverse();
+            create_and_validate(&data, "rand");
+        }
+    }
+}

fastfield_codecs/src/lib.rs

@@ -0,0 +1,225 @@
+#[cfg(test)]
+#[macro_use]
+extern crate more_asserts;
+
+use std::io;
+use std::io::Write;
+
+pub mod bitpacked;
+pub mod linearinterpol;
+pub mod multilinearinterpol;
+
+pub trait FastFieldCodecReader: Sized {
+    /// reads the metadata and returns the CodecReader
+    fn open_from_bytes(bytes: &[u8]) -> std::io::Result<Self>;
+
+    fn get_u64(&self, doc: u64, data: &[u8]) -> u64;
+
+    fn min_value(&self) -> u64;
+    fn max_value(&self) -> u64;
+}
+
+/// The FastFieldSerializerEstimate trait is required on all variants
+/// of fast field compressions, to decide which one to choose.
+pub trait FastFieldCodecSerializer {
+    /// A codex needs to provide a unique name and id, which is
+    /// used for debugging and de/serialization.
+    const NAME: &'static str;
+    const ID: u8;
+
+    /// Check if the Codec is able to compress the data
+    fn is_applicable(fastfield_accessor: &impl FastFieldDataAccess, stats: FastFieldStats) -> bool;
+
+    /// Returns an estimate of the compression ratio.
+    /// The baseline is uncompressed 64bit data.
+    ///
+    /// It could make sense to also return a value representing
+    /// computational complexity.
+    fn estimate(fastfield_accessor: &impl FastFieldDataAccess, stats: FastFieldStats) -> f32;
+
+    /// Serializes the data using the serializer into write.
+    /// There are multiple iterators, in case the codec needs to read the data multiple times.
+    /// The iterators should be preferred over using fastfield_accessor for performance reasons.
+    fn serialize(
+        write: &mut impl Write,
+        fastfield_accessor: &impl FastFieldDataAccess,
+        stats: FastFieldStats,
+        data_iter: impl Iterator<Item = u64>,
+        data_iter1: impl Iterator<Item = u64>,
+    ) -> io::Result<()>;
+}
+
+/// FastFieldDataAccess is the trait to access fast field data during serialization and estimation.
+pub trait FastFieldDataAccess {
+    /// Return the value associated to the given position.
+    ///
+    /// Whenever possible use the Iterator passed to the fastfield creation instead, for performance reasons.
+    ///
+    /// # Panics
+    ///
+    /// May panic if `position` is greater than the index.
+    fn get_val(&self, position: u64) -> u64;
+}
+
+#[derive(Debug, Clone)]
+pub struct FastFieldStats {
+    pub min_value: u64,
+    pub max_value: u64,
+    pub num_vals: u64,
+}
+
+impl<'a> FastFieldDataAccess for &'a [u64] {
+    fn get_val(&self, position: u64) -> u64 {
+        self[position as usize]
+    }
+}
+
+impl FastFieldDataAccess for Vec<u64> {
+    fn get_val(&self, position: u64) -> u64 {
+        self[position as usize]
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::{
+        bitpacked::{BitpackedFastFieldReader, BitpackedFastFieldSerializer},
+        linearinterpol::{LinearInterpolFastFieldReader, LinearInterpolFastFieldSerializer},
+        multilinearinterpol::{
+            MultiLinearInterpolFastFieldReader, MultiLinearInterpolFastFieldSerializer,
+        },
+    };
+
+    pub fn create_and_validate<S: FastFieldCodecSerializer, R: FastFieldCodecReader>(
+        data: &[u64],
+        name: &str,
+    ) -> (f32, f32) {
+        if !S::is_applicable(&data, crate::tests::stats_from_vec(data)) {
+            return (f32::MAX, 0.0);
+        }
+        let estimation = S::estimate(&data, crate::tests::stats_from_vec(data));
+        let mut out = vec![];
+        S::serialize(
+            &mut out,
+            &data,
+            crate::tests::stats_from_vec(data),
+            data.iter().cloned(),
+            data.iter().cloned(),
+        )
+        .unwrap();
+
+        let reader = R::open_from_bytes(&out).unwrap();
+        for (doc, orig_val) in data.iter().enumerate() {
+            let val = reader.get_u64(doc as u64, &out);
+            if val != *orig_val {
+                panic!(
+                    "val {:?} does not match orig_val {:?}, in data set {}, data {:?}",
+                    val, orig_val, name, data
+                );
+            }
+        }
+        let actual_compression = out.len() as f32 / (data.len() as f32 * 8.0);
+        (estimation, actual_compression)
+    }
+    pub fn get_codec_test_data_sets() -> Vec<(Vec<u64>, &'static str)> {
+        let mut data_and_names = vec![];
+
+        let data = (10..=20_u64).collect::<Vec<_>>();
+        data_and_names.push((data, "simple monotonically increasing"));
+
+        data_and_names.push((
+            vec![5, 6, 7, 8, 9, 10, 99, 100],
+            "offset in linear interpol",
+        ));
+        data_and_names.push((vec![5, 50, 3, 13, 1, 1000, 35], "rand small"));
+        data_and_names.push((vec![10], "single value"));
+
+        data_and_names
+    }
+
+    fn test_codec<S: FastFieldCodecSerializer, R: FastFieldCodecReader>() {
+        let codec_name = S::NAME;
+        for (data, data_set_name) in get_codec_test_data_sets() {
+            let (estimate, actual) =
+                crate::tests::create_and_validate::<S, R>(&data, data_set_name);
+            let result = if estimate == f32::MAX {
+                "Disabled".to_string()
+            } else {
+                format!("Estimate {:?} Actual {:?} ", estimate, actual)
+            };
+            println!(
+                "Codec {}, DataSet {}, {}",
+                codec_name, data_set_name, result
+            );
+        }
+    }
+    #[test]
+    fn test_codec_bitpacking() {
+        test_codec::<BitpackedFastFieldSerializer, BitpackedFastFieldReader>();
+    }
+    #[test]
+    fn test_codec_interpolation() {
+        test_codec::<LinearInterpolFastFieldSerializer, LinearInterpolFastFieldReader>();
+    }
+    #[test]
+    fn test_codec_multi_interpolation() {
+        test_codec::<MultiLinearInterpolFastFieldSerializer, MultiLinearInterpolFastFieldReader>();
+    }
+
+    use super::*;
+    pub fn stats_from_vec(data: &[u64]) -> FastFieldStats {
+        let min_value = data.iter().cloned().min().unwrap_or(0);
+        let max_value = data.iter().cloned().max().unwrap_or(0);
+        FastFieldStats {
+            min_value,
+            max_value,
+            num_vals: data.len() as u64,
+        }
+    }
+
+    #[test]
+    fn estimation_good_interpolation_case() {
+        let data = (10..=20000_u64).collect::<Vec<_>>();
+
+        let linear_interpol_estimation =
+            LinearInterpolFastFieldSerializer::estimate(&data, stats_from_vec(&data));
+        assert_le!(linear_interpol_estimation, 0.01);
+
+        let multi_linear_interpol_estimation =
+            MultiLinearInterpolFastFieldSerializer::estimate(&data, stats_from_vec(&data));
+        assert_le!(multi_linear_interpol_estimation, 0.2);
+        assert_le!(linear_interpol_estimation, multi_linear_interpol_estimation);
+
+        let bitpacked_estimation =
+            BitpackedFastFieldSerializer::estimate(&data, stats_from_vec(&data));
+        assert_le!(linear_interpol_estimation, bitpacked_estimation);
+    }
+    #[test]
+    fn estimation_test_bad_interpolation_case() {
+        let data = vec![200, 10, 10, 10, 10, 1000, 20];
+
+        let linear_interpol_estimation =
+            LinearInterpolFastFieldSerializer::estimate(&data, stats_from_vec(&data));
+        assert_le!(linear_interpol_estimation, 0.32);
+
+        let bitpacked_estimation =
+            BitpackedFastFieldSerializer::estimate(&data, stats_from_vec(&data));
+        assert_le!(bitpacked_estimation, linear_interpol_estimation);
+    }
+    #[test]
+    fn estimation_test_bad_interpolation_case_monotonically_increasing() {
+        let mut data = (200..=20000_u64).collect::<Vec<_>>();
+        data.push(1_000_000);
+
+        // in this case the linear interpolation can't in fact not be worse than bitpacking,
+        // but the estimator adds some threshold, which leads to estimated worse behavior
+        let linear_interpol_estimation =
+            LinearInterpolFastFieldSerializer::estimate(&data, stats_from_vec(&data));
+        assert_le!(linear_interpol_estimation, 0.35);
+
+        let bitpacked_estimation =
+            BitpackedFastFieldSerializer::estimate(&data, stats_from_vec(&data));
+        assert_le!(bitpacked_estimation, 0.32);
+        assert_le!(bitpacked_estimation, linear_interpol_estimation);
+    }
+}

fastfield_codecs/src/linearinterpol.rs

@@ -0,0 +1,305 @@
+use crate::FastFieldCodecReader;
+use crate::FastFieldCodecSerializer;
+use crate::FastFieldDataAccess;
+use crate::FastFieldStats;
+use std::io::{self, Read, Write};
+use std::ops::Sub;
+use tantivy_bitpacker::compute_num_bits;
+use tantivy_bitpacker::BitPacker;
+
+use common::BinarySerializable;
+use common::FixedSize;
+use tantivy_bitpacker::BitUnpacker;
+
+/// Depending on the field type, a different
+/// fast field is required.
+#[derive(Clone)]
+pub struct LinearInterpolFastFieldReader {
+    bit_unpacker: BitUnpacker,
+    pub footer: LinearInterpolFooter,
+    pub slope: f32,
+}
+
+#[derive(Clone, Debug)]
+pub struct LinearInterpolFooter {
+    pub relative_max_value: u64,
+    pub offset: u64,
+    pub first_val: u64,
+    pub last_val: u64,
+    pub num_vals: u64,
+    pub min_value: u64,
+    pub max_value: u64,
+}
+
+impl BinarySerializable for LinearInterpolFooter {
+    fn serialize<W: Write>(&self, write: &mut W) -> io::Result<()> {
+        self.relative_max_value.serialize(write)?;
+        self.offset.serialize(write)?;
+        self.first_val.serialize(write)?;
+        self.last_val.serialize(write)?;
+        self.num_vals.serialize(write)?;
+        self.min_value.serialize(write)?;
+        self.max_value.serialize(write)?;
+        Ok(())
+    }
+
+    fn deserialize<R: Read>(reader: &mut R) -> io::Result<LinearInterpolFooter> {
+        Ok(LinearInterpolFooter {
+            relative_max_value: u64::deserialize(reader)?,
+            offset: u64::deserialize(reader)?,
+            first_val: u64::deserialize(reader)?,
+            last_val: u64::deserialize(reader)?,
+            num_vals: u64::deserialize(reader)?,
+            min_value: u64::deserialize(reader)?,
+            max_value: u64::deserialize(reader)?,
+        })
+    }
+}
+
+impl FixedSize for LinearInterpolFooter {
+    const SIZE_IN_BYTES: usize = 56;
+}
+
+impl FastFieldCodecReader for LinearInterpolFastFieldReader {
+    /// Opens a fast field given a file.
+    fn open_from_bytes(bytes: &[u8]) -> io::Result<Self> {
+        let (_data, mut footer) = bytes.split_at(bytes.len() - LinearInterpolFooter::SIZE_IN_BYTES);
+        let footer = LinearInterpolFooter::deserialize(&mut footer)?;
+        let slope = get_slope(footer.first_val, footer.last_val, footer.num_vals);
+
+        let num_bits = compute_num_bits(footer.relative_max_value);
+        let bit_unpacker = BitUnpacker::new(num_bits);
+        Ok(LinearInterpolFastFieldReader {
+            bit_unpacker,
+            footer,
+            slope,
+        })
+    }
+    #[inline]
+    fn get_u64(&self, doc: u64, data: &[u8]) -> u64 {
+        let calculated_value = get_calculated_value(self.footer.first_val, doc, self.slope);
+        (calculated_value + self.bit_unpacker.get(doc, data)) - self.footer.offset
+    }
+
+    #[inline]
+    fn min_value(&self) -> u64 {
+        self.footer.min_value
+    }
+    #[inline]
+    fn max_value(&self) -> u64 {
+        self.footer.max_value
+    }
+}
+
+/// Fastfield serializer, which tries to guess values by linear interpolation
+/// and stores the difference bitpacked.
+pub struct LinearInterpolFastFieldSerializer {}
+
+#[inline]
+fn get_slope(first_val: u64, last_val: u64, num_vals: u64) -> f32 {
+    if num_vals <= 1 {
+        return 0.0;
+    }
+    //  We calculate the slope with f64 high precision and use the result in lower precision f32
+    //  This is done in order to handle estimations for very large values like i64::MAX
+    ((last_val as f64 - first_val as f64) / (num_vals as u64 - 1) as f64) as f32
+}
+
+#[inline]
+fn get_calculated_value(first_val: u64, pos: u64, slope: f32) -> u64 {
+    first_val + (pos as f32 * slope) as u64
+}
+
+impl FastFieldCodecSerializer for LinearInterpolFastFieldSerializer {
+    const NAME: &'static str = "LinearInterpol";
+    const ID: u8 = 2;
+    /// Creates a new fast field serializer.
+    fn serialize(
+        write: &mut impl Write,
+        fastfield_accessor: &impl FastFieldDataAccess,
+        stats: FastFieldStats,
+        data_iter: impl Iterator<Item = u64>,
+        data_iter1: impl Iterator<Item = u64>,
+    ) -> io::Result<()> {
+        assert!(stats.min_value <= stats.max_value);
+
+        let first_val = fastfield_accessor.get_val(0);
+        let last_val = fastfield_accessor.get_val(stats.num_vals as u64 - 1);
+        let slope = get_slope(first_val, last_val, stats.num_vals);
+        // calculate offset to ensure all values are positive
+        let mut offset = 0;
+        let mut rel_positive_max = 0;
+        for (pos, actual_value) in data_iter1.enumerate() {
+            let calculated_value = get_calculated_value(first_val, pos as u64, slope);
+            if calculated_value > actual_value {
+                // negative value we need to apply an offset
+                // we ignore negative values in the max value calculation, because negative values
+                // will be offset to 0
+                offset = offset.max(calculated_value - actual_value);
+            } else {
+                //positive value no offset reuqired
+                rel_positive_max = rel_positive_max.max(actual_value - calculated_value);
+            }
+        }
+
+        // rel_positive_max will be adjusted by offset
+        let relative_max_value = rel_positive_max + offset;
+
+        let num_bits = compute_num_bits(relative_max_value);
+        let mut bit_packer = BitPacker::new();
+        for (pos, val) in data_iter.enumerate() {
+            let calculated_value = get_calculated_value(first_val, pos as u64, slope);
+            let diff = (val + offset) - calculated_value;
+            bit_packer.write(diff, num_bits, write)?;
+        }
+        bit_packer.close(write)?;
+
+        let footer = LinearInterpolFooter {
+            relative_max_value,
+            offset,
+            first_val,
+            last_val,
+            num_vals: stats.num_vals,
+            min_value: stats.min_value,
+            max_value: stats.max_value,
+        };
+        footer.serialize(write)?;
+        Ok(())
+    }
+    fn is_applicable(
+        _fastfield_accessor: &impl FastFieldDataAccess,
+        stats: FastFieldStats,
+    ) -> bool {
+        if stats.num_vals < 3 {
+            return false; //disable compressor for this case
+        }
+        // On serialisation the offset is added to the actual value.
+        // We need to make sure this won't run into overflow calculation issues.
+        // For this we take the maximum theroretical offset and add this to the max value.
+        // If this doesn't overflow the algortihm should be fine
+        let theorethical_maximum_offset = stats.max_value - stats.min_value;
+        if stats
+            .max_value
+            .checked_add(theorethical_maximum_offset)
+            .is_none()
+        {
+            return false;
+        }
+        true
+    }
+    /// estimation for linear interpolation is hard because, you don't know
+    /// where the local maxima for the deviation of the calculated value are and
+    /// the offset to shift all values to >=0 is also unknown.
+    fn estimate(fastfield_accessor: &impl FastFieldDataAccess, stats: FastFieldStats) -> f32 {
+        let first_val = fastfield_accessor.get_val(0);
+        let last_val = fastfield_accessor.get_val(stats.num_vals as u64 - 1);
+        let slope = get_slope(first_val, last_val, stats.num_vals);
+
+        // let's sample at 0%, 5%, 10% .. 95%, 100%
+        let num_vals = stats.num_vals as f32 / 100.0;
+        let sample_positions = (0..20)
+            .map(|pos| (num_vals * pos as f32 * 5.0) as usize)
+            .collect::<Vec<_>>();
+
+        let max_distance = sample_positions
+            .iter()
+            .map(|pos| {
+                let calculated_value = get_calculated_value(first_val, *pos as u64, slope);
+                let actual_value = fastfield_accessor.get_val(*pos as u64);
+                distance(calculated_value, actual_value)
+            })
+            .max()
+            .unwrap_or(0);
+
+        // the theory would be that we don't have the actual max_distance, but we are close within 50%
+        // threshold.
+        // It is multiplied by 2 because in a log case scenario the line would be as much above as
+        // below. So the offset would = max_distance
+        //
+        let relative_max_value = (max_distance as f32 * 1.5) * 2.0;
+
+        let num_bits = compute_num_bits(relative_max_value as u64) as u64 * stats.num_vals as u64
+            + LinearInterpolFooter::SIZE_IN_BYTES as u64;
+        let num_bits_uncompressed = 64 * stats.num_vals;
+        num_bits as f32 / num_bits_uncompressed as f32
+    }
+}
+
+#[inline]
+fn distance<T: Sub<Output = T> + Ord>(x: T, y: T) -> T {
+    if x < y {
+        y - x
+    } else {
+        x - y
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::tests::get_codec_test_data_sets;
+
+    fn create_and_validate(data: &[u64], name: &str) -> (f32, f32) {
+        crate::tests::create_and_validate::<
+            LinearInterpolFastFieldSerializer,
+            LinearInterpolFastFieldReader,
+        >(data, name)
+    }
+
+    #[test]
+    fn test_compression() {
+        let data = (10..=6_000_u64).collect::<Vec<_>>();
+        let (estimate, actual_compression) =
+            create_and_validate(&data, "simple monotonically large");
+
+        assert!(actual_compression < 0.01);
+        assert!(estimate < 0.01);
+    }
+
+    #[test]
+    fn test_with_codec_data_sets() {
+        let data_sets = get_codec_test_data_sets();
+        for (mut data, name) in data_sets {
+            create_and_validate(&data, name);
+            data.reverse();
+            create_and_validate(&data, name);
+        }
+    }
+    #[test]
+    fn linear_interpol_fast_field_test_large_amplitude() {
+        let data = vec![
+            i64::MAX as u64 / 2,
+            i64::MAX as u64 / 3,
+            i64::MAX as u64 / 2,
+        ];
+
+        create_and_validate(&data, "large amplitude");
+    }
+    #[test]
+    fn linear_interpol_fast_concave_data() {
+        let data = vec![0, 1, 2, 5, 8, 10, 20, 50];
+        create_and_validate(&data, "concave data");
+    }
+    #[test]
+    fn linear_interpol_fast_convex_data() {
+        let data = vec![0, 40, 60, 70, 75, 77];
+        create_and_validate(&data, "convex data");
+    }
+    #[test]
+    fn linear_interpol_fast_field_test_simple() {
+        let data = (10..=20_u64).collect::<Vec<_>>();
+
+        create_and_validate(&data, "simple monotonically");
+    }
+
+    #[test]
+    fn linear_interpol_fast_field_rand() {
+        for _ in 0..5000 {
+            let mut data = (0..50).map(|_| rand::random::<u64>()).collect::<Vec<_>>();
+            create_and_validate(&data, "random");
+
+            data.reverse();
+            create_and_validate(&data, "random");
+        }
+    }
+}

fastfield_codecs/src/main.rs

@@ -0,0 +1,127 @@
+#[macro_use]
+extern crate prettytable;
+use fastfield_codecs::{
+    linearinterpol::LinearInterpolFastFieldSerializer,
+    multilinearinterpol::MultiLinearInterpolFastFieldSerializer, FastFieldCodecSerializer,
+    FastFieldStats,
+};
+use prettytable::{Cell, Row, Table};
+
+fn main() {
+    let mut table = Table::new();
+
+    // Add a row per time
+    table.add_row(row!["", "Compression Ratio", "Compression Estimation"]);
+
+    for (data, data_set_name) in get_codec_test_data_sets() {
+        let mut results = vec![];
+        let res = serialize_with_codec::<LinearInterpolFastFieldSerializer>(&data);
+        results.push(res);
+        let res = serialize_with_codec::<MultiLinearInterpolFastFieldSerializer>(&data);
+        results.push(res);
+        let res = serialize_with_codec::<fastfield_codecs::bitpacked::BitpackedFastFieldSerializer>(
+            &data,
+        );
+        results.push(res);
+
+        //let best_estimation_codec = results
+        //.iter()
+        //.min_by(|res1, res2| res1.partial_cmp(&res2).unwrap())
+        //.unwrap();
+        let best_compression_ratio_codec = results
+            .iter()
+            .min_by(|res1, res2| res1.partial_cmp(res2).unwrap())
+            .cloned()
+            .unwrap();
+
+        table.add_row(Row::new(vec![Cell::new(data_set_name).style_spec("Bbb")]));
+        for (is_applicable, est, comp, name) in results {
+            let (est_cell, ratio_cell) = if !is_applicable {
+                ("Codec Disabled".to_string(), "".to_string())
+            } else {
+                (est.to_string(), comp.to_string())
+            };
+            #[allow(clippy::all)]
+            let style = if comp == best_compression_ratio_codec.1 {
+                "Fb"
+            } else {
+                ""
+            };
+
+            table.add_row(Row::new(vec![
+                Cell::new(&name.to_string()).style_spec("bFg"),
+                Cell::new(&ratio_cell).style_spec(style),
+                Cell::new(&est_cell).style_spec(""),
+            ]));
+        }
+    }
+
+    table.printstd();
+}
+
+pub fn get_codec_test_data_sets() -> Vec<(Vec<u64>, &'static str)> {
+    let mut data_and_names = vec![];
+
+    let data = (1000..=200_000_u64).collect::<Vec<_>>();
+    data_and_names.push((data, "Autoincrement"));
+
+    let mut current_cumulative = 0;
+    let data = (1..=200_000_u64)
+        .map(|num| {
+            let num = (num as f32 + num as f32).log10() as u64;
+            current_cumulative += num;
+            current_cumulative
+        })
+        .collect::<Vec<_>>();
+    //let data = (1..=200000_u64).map(|num| num + num).collect::<Vec<_>>();
+    data_and_names.push((data, "Monotonically increasing concave"));
+
+    let mut current_cumulative = 0;
+    let data = (1..=200_000_u64)
+        .map(|num| {
+            let num = (200_000.0 - num as f32).log10() as u64;
+            current_cumulative += num;
+            current_cumulative
+        })
+        .collect::<Vec<_>>();
+    data_and_names.push((data, "Monotonically increasing convex"));
+
+    let data = (1000..=200_000_u64)
+        .map(|num| num + rand::random::<u8>() as u64)
+        .collect::<Vec<_>>();
+    data_and_names.push((data, "Almost monotonically increasing"));
+
+    data_and_names
+}
+
+pub fn serialize_with_codec<S: FastFieldCodecSerializer>(
+    data: &[u64],
+) -> (bool, f32, f32, &'static str) {
+    let is_applicable = S::is_applicable(&data, stats_from_vec(data));
+    if !is_applicable {
+        return (false, 0.0, 0.0, S::NAME);
+    }
+    let estimation = S::estimate(&data, stats_from_vec(data));
+    let mut out = vec![];
+    S::serialize(
+        &mut out,
+        &data,
+        stats_from_vec(data),
+        data.iter().cloned(),
+        data.iter().cloned(),
+    )
+    .unwrap();
+
+    let actual_compression = out.len() as f32 / (data.len() * 8) as f32;
+    (true, estimation, actual_compression, S::NAME)
+}
+
+pub fn stats_from_vec(data: &[u64]) -> FastFieldStats {
+    let min_value = data.iter().cloned().min().unwrap_or(0);
+    let max_value = data.iter().cloned().max().unwrap_or(0);
+    FastFieldStats {
+        min_value,
+        max_value,
+        num_vals: data.len() as u64,
+    }
+}

fastfield_codecs/src/multilinearinterpol.rs

@@ -0,0 +1,438 @@
+/*!
+
+MultiLinearInterpol compressor uses linear interpolation to guess a values and stores the offset, but in blocks of 512.
+
+With a CHUNK_SIZE of 512 and 29 byte metadata per block, we get a overhead for metadata of 232 / 512 = 0,45 bits per element.
+The additional space required per element in a block is the the maximum deviation of the linear interpolation estimation function.
+
+E.g. if the maximum deviation of an element is 12, all elements cost 4bits.
+
+Size per block:
+Num Elements * Maximum Deviation from Interpolation + 29 Byte Metadata
+
+*/
+
+use crate::FastFieldCodecReader;
+use crate::FastFieldCodecSerializer;
+use crate::FastFieldDataAccess;
+use crate::FastFieldStats;
+use common::CountingWriter;
+use std::io::{self, Read, Write};
+use std::ops::Sub;
+use tantivy_bitpacker::compute_num_bits;
+use tantivy_bitpacker::BitPacker;
+
+use common::BinarySerializable;
+use common::DeserializeFrom;
+use tantivy_bitpacker::BitUnpacker;
+
+const CHUNK_SIZE: u64 = 512;
+
+/// Depending on the field type, a different
+/// fast field is required.
+#[derive(Clone)]
+pub struct MultiLinearInterpolFastFieldReader {
+    pub footer: MultiLinearInterpolFooter,
+}
+
+#[derive(Clone, Debug, Default)]
+struct Function {
+    // The offset in the data is required, because we have diffrent bit_widths per block
+    data_start_offset: u64,
+    // start_pos in the block will be CHUNK_SIZE * BLOCK_NUM
+    start_pos: u64,
+    // only used during serialization, 0 after deserialization
+    end_pos: u64,
+    // only used during serialization, 0 after deserialization
+    value_start_pos: u64,
+    // only used during serialization, 0 after deserialization
+    value_end_pos: u64,
+    slope: f32,
+    // The offset so that all values are positive when writing them
+    positive_val_offset: u64,
+    num_bits: u8,
+    bit_unpacker: BitUnpacker,
+}
+
+impl Function {
+    fn calc_slope(&mut self) {
+        let num_vals = self.end_pos - self.start_pos;
+        self.slope = get_slope(self.value_start_pos, self.value_end_pos, num_vals);
+    }
+    // split the interpolation into two function, change self and return the second split
+    fn split(&mut self, split_pos: u64, split_pos_value: u64) -> Function {
+        let mut new_function = Function {
+            start_pos: split_pos,
+            end_pos: self.end_pos,
+            value_start_pos: split_pos_value,
+            value_end_pos: self.value_end_pos,
+            ..Default::default()
+        };
+        new_function.calc_slope();
+        self.end_pos = split_pos;
+        self.value_end_pos = split_pos_value;
+        self.calc_slope();
+        new_function
+    }
+}
+
+impl BinarySerializable for Function {
+    fn serialize<W: Write>(&self, write: &mut W) -> io::Result<()> {
+        self.data_start_offset.serialize(write)?;
+        self.value_start_pos.serialize(write)?;
+        self.positive_val_offset.serialize(write)?;
+        self.slope.serialize(write)?;
+        self.num_bits.serialize(write)?;
+        Ok(())
+    }
+
+    fn deserialize<R: Read>(reader: &mut R) -> io::Result<Function> {
+        let data_start_offset = u64::deserialize(reader)?;
+        let value_start_pos = u64::deserialize(reader)?;
+        let offset = u64::deserialize(reader)?;
+        let slope = f32::deserialize(reader)?;
+        let num_bits = u8::deserialize(reader)?;
+        let interpolation = Function {
+            data_start_offset,
+            value_start_pos,
+            positive_val_offset: offset,
+            num_bits,
+            bit_unpacker: BitUnpacker::new(num_bits),
+            slope,
+            ..Default::default()
+        };
+
+        Ok(interpolation)
+    }
+}
+
+#[derive(Clone, Debug)]
+pub struct MultiLinearInterpolFooter {
+    pub num_vals: u64,
+    pub min_value: u64,
+    pub max_value: u64,
+    interpolations: Vec<Function>,
+}
+
+impl BinarySerializable for MultiLinearInterpolFooter {
+    fn serialize<W: Write>(&self, write: &mut W) -> io::Result<()> {
+        let mut out = vec![];
+        self.num_vals.serialize(&mut out)?;
+        self.min_value.serialize(&mut out)?;
+        self.max_value.serialize(&mut out)?;
+        self.interpolations.serialize(&mut out)?;
+        write.write_all(&out)?;
+        (out.len() as u32).serialize(write)?;
+        Ok(())
+    }
+
+    fn deserialize<R: Read>(reader: &mut R) -> io::Result<MultiLinearInterpolFooter> {
+        let mut footer = MultiLinearInterpolFooter {
+            num_vals: u64::deserialize(reader)?,
+            min_value: u64::deserialize(reader)?,
+            max_value: u64::deserialize(reader)?,
+            interpolations: Vec::<Function>::deserialize(reader)?,
+        };
+        for (num, interpol) in footer.interpolations.iter_mut().enumerate() {
+            interpol.start_pos = CHUNK_SIZE * num as u64;
+        }
+        Ok(footer)
+    }
+}
+
+#[inline]
+fn get_interpolation_position(doc: u64) -> usize {
+    let index = doc / CHUNK_SIZE;
+    index as usize
+}
+
+#[inline]
+fn get_interpolation_function(doc: u64, interpolations: &[Function]) -> &Function {
+    &interpolations[get_interpolation_position(doc)]
+}
+
+impl FastFieldCodecReader for MultiLinearInterpolFastFieldReader {
+    /// Opens a fast field given a file.
+    fn open_from_bytes(bytes: &[u8]) -> io::Result<Self> {
+        let footer_len: u32 = (&bytes[bytes.len() - 4..]).deserialize()?;
+
+        let (_data, mut footer) = bytes.split_at(bytes.len() - (4 + footer_len) as usize);
+        let footer = MultiLinearInterpolFooter::deserialize(&mut footer)?;
+
+        Ok(MultiLinearInterpolFastFieldReader { footer })
+    }
+
+    #[inline]
+    fn get_u64(&self, doc: u64, data: &[u8]) -> u64 {
+        let interpolation = get_interpolation_function(doc, &self.footer.interpolations);
+        let doc = doc - interpolation.start_pos;
+        let calculated_value =
+            get_calculated_value(interpolation.value_start_pos, doc, interpolation.slope);
+        let diff = interpolation
+            .bit_unpacker
+            .get(doc, &data[interpolation.data_start_offset as usize..]);
+        (calculated_value + diff) - interpolation.positive_val_offset
+    }
+
+    #[inline]
+    fn min_value(&self) -> u64 {
+        self.footer.min_value
+    }
+    #[inline]
+    fn max_value(&self) -> u64 {
+        self.footer.max_value
+    }
+}
+
+#[inline]
+fn get_slope(first_val: u64, last_val: u64, num_vals: u64) -> f32 {
+    ((last_val as f64 - first_val as f64) / (num_vals as u64 - 1) as f64) as f32
+}
+
+#[inline]
+fn get_calculated_value(first_val: u64, pos: u64, slope: f32) -> u64 {
+    (first_val as i64 + (pos as f32 * slope) as i64) as u64
+}
+
+/// Same as LinearInterpolFastFieldSerializer, but working on chunks of CHUNK_SIZE elements.
+pub struct MultiLinearInterpolFastFieldSerializer {}
+
+impl FastFieldCodecSerializer for MultiLinearInterpolFastFieldSerializer {
+    const NAME: &'static str = "MultiLinearInterpol";
+    const ID: u8 = 3;
+    /// Creates a new fast field serializer.
+    fn serialize(
+        write: &mut impl Write,
+        fastfield_accessor: &impl FastFieldDataAccess,
+        stats: FastFieldStats,
+        data_iter: impl Iterator<Item = u64>,
+        _data_iter1: impl Iterator<Item = u64>,
+    ) -> io::Result<()> {
+        assert!(stats.min_value <= stats.max_value);
+
+        let first_val = fastfield_accessor.get_val(0);
+        let last_val = fastfield_accessor.get_val(stats.num_vals as u64 - 1);
+
+        let mut first_function = Function {
+            end_pos: stats.num_vals,
+            value_start_pos: first_val,
+            value_end_pos: last_val,
+            ..Default::default()
+        };
+        first_function.calc_slope();
+        let mut interpolations = vec![first_function];
+
+        // Since we potentially apply multiple passes over the data, the data is cached.
+        // Multiple iteration can be expensive (merge with index sorting can add lot of overhead per
+        // iteration)
+        let data = data_iter.collect::<Vec<_>>();
+
+        //// let's split this into chunks of CHUNK_SIZE
+        for data_pos in (0..data.len() as u64).step_by(CHUNK_SIZE as usize).skip(1) {
+            let new_fun = {
+                let current_interpolation = interpolations.last_mut().unwrap();
+                current_interpolation.split(data_pos, data[data_pos as usize])
+            };
+            interpolations.push(new_fun);
+        }
+        // calculate offset and max (-> numbits) for each function
+        for interpolation in &mut interpolations {
+            let mut offset = 0;
+            let mut rel_positive_max = 0;
+            for (pos, actual_value) in data
+                [interpolation.start_pos as usize..interpolation.end_pos as usize]
+                .iter()
+                .cloned()
+                .enumerate()
+            {
+                let calculated_value = get_calculated_value(
+                    interpolation.value_start_pos,
+                    pos as u64,
+                    interpolation.slope,
+                );
+                if calculated_value > actual_value {
+                    // negative value we need to apply an offset
+                    // we ignore negative values in the max value calculation, because negative values
+                    // will be offset to 0
+                    offset = offset.max(calculated_value - actual_value);
+                } else {
+                    //positive value no offset reuqired
+                    rel_positive_max = rel_positive_max.max(actual_value - calculated_value);
+                }
+            }
+
+            interpolation.positive_val_offset = offset;
+            interpolation.num_bits = compute_num_bits(rel_positive_max + offset);
+        }
+        let mut bit_packer = BitPacker::new();
+
+        let write = &mut CountingWriter::wrap(write);
+        for interpolation in &mut interpolations {
+            interpolation.data_start_offset = write.written_bytes();
+            let num_bits = interpolation.num_bits;
+            for (pos, actual_value) in data
+                [interpolation.start_pos as usize..interpolation.end_pos as usize]
+                .iter()
+                .cloned()
+                .enumerate()
+            {
+                let calculated_value = get_calculated_value(
+                    interpolation.value_start_pos,
+                    pos as u64,
+                    interpolation.slope,
+                );
+                let diff = (actual_value + interpolation.positive_val_offset) - calculated_value;
+                bit_packer.write(diff, num_bits, write)?;
+            }
+            bit_packer.flush(write)?;
+        }
+        bit_packer.close(write)?;
+
+        let footer = MultiLinearInterpolFooter {
+            num_vals: stats.num_vals,
+            min_value: stats.min_value,
+            max_value: stats.max_value,
+            interpolations,
+        };
+        footer.serialize(write)?;
+        Ok(())
+    }
+
+    fn is_applicable(
+        _fastfield_accessor: &impl FastFieldDataAccess,
+        stats: FastFieldStats,
+    ) -> bool {
+        if stats.num_vals < 5_000 {
+            return false;
+        }
+        // On serialization the offset is added to the actual value.
+        // We need to make sure this won't run into overflow calculation issues.
+        // For this we take the maximum theroretical offset and add this to the max value.
+        // If this doesn't overflow the algortihm should be fine
+        let theorethical_maximum_offset = stats.max_value - stats.min_value;
+        if stats
+            .max_value
+            .checked_add(theorethical_maximum_offset)
+            .is_none()
+        {
+            return false;
+        }
+        true
+    }
+    /// estimation for linear interpolation is hard because, you don't know
+    /// where the local maxima are for the deviation of the calculated value and
+    /// the offset is also unknown.
+    fn estimate(fastfield_accessor: &impl FastFieldDataAccess, stats: FastFieldStats) -> f32 {
+        let first_val_in_first_block = fastfield_accessor.get_val(0);
+        let last_elem_in_first_chunk = CHUNK_SIZE.min(stats.num_vals);
+        let last_val_in_first_block =
+            fastfield_accessor.get_val(last_elem_in_first_chunk as u64 - 1);
+        let slope = get_slope(
+            first_val_in_first_block,
+            last_val_in_first_block,
+            stats.num_vals,
+        );
+
+        // let's sample at 0%, 5%, 10% .. 95%, 100%, but for the first block only
+        let sample_positions = (0..20)
+            .map(|pos| (last_elem_in_first_chunk as f32 / 100.0 * pos as f32 * 5.0) as usize)
+            .collect::<Vec<_>>();
+
+        let max_distance = sample_positions
+            .iter()
+            .map(|pos| {
+                let calculated_value =
+                    get_calculated_value(first_val_in_first_block, *pos as u64, slope);
+                let actual_value = fastfield_accessor.get_val(*pos as u64);
+                distance(calculated_value, actual_value)
+            })
+            .max()
+            .unwrap();
+
+        // Estimate one block and extrapolate the cost to all blocks.
+        // the theory would be that we don't have the actual max_distance, but we are close within 50%
+        // threshold.
+        // It is multiplied by 2 because in a log case scenario the line would be as much above as
+        // below. So the offset would = max_distance
+        //
+        let relative_max_value = (max_distance as f32 * 1.5) * 2.0;
+
+        let num_bits = compute_num_bits(relative_max_value as u64) as u64 * stats.num_vals as u64
+            // function metadata per block
+            + 29 * (stats.num_vals / CHUNK_SIZE);
+        let num_bits_uncompressed = 64 * stats.num_vals;
+        num_bits as f32 / num_bits_uncompressed as f32
+    }
+}
+
+fn distance<T: Sub<Output = T> + Ord>(x: T, y: T) -> T {
+    if x < y {
+        y - x
+    } else {
+        x - y
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::tests::get_codec_test_data_sets;
+
+    fn create_and_validate(data: &[u64], name: &str) -> (f32, f32) {
+        crate::tests::create_and_validate::<
+            MultiLinearInterpolFastFieldSerializer,
+            MultiLinearInterpolFastFieldReader,
+        >(data, name)
+    }
+
+    #[test]
+    fn test_compression() {
+        let data = (10..=6_000_u64).collect::<Vec<_>>();
+        let (estimate, actual_compression) =
+            create_and_validate(&data, "simple monotonically large");
+        assert!(actual_compression < 0.2);
+        assert!(estimate < 0.20);
+        assert!(estimate > 0.15);
+        assert!(actual_compression > 0.01);
+    }
+
+    #[test]
+    fn test_with_codec_data_sets() {
+        let data_sets = get_codec_test_data_sets();
+        for (mut data, name) in data_sets {
+            create_and_validate(&data, name);
+            data.reverse();
+            create_and_validate(&data, name);
+        }
+    }
+    #[test]
+    fn test_simple() {
+        let data = (10..=20_u64).collect::<Vec<_>>();
+        create_and_validate(&data, "simple monotonically");
+    }
+
+    #[test]
+    fn border_cases_1() {
+        let data = (0..1024).collect::<Vec<_>>();
+        create_and_validate(&data, "border case");
+    }
+    #[test]
+    fn border_case_2() {
+        let data = (0..1025).collect::<Vec<_>>();
+        create_and_validate(&data, "border case");
+    }
+    #[test]
+    fn rand() {
+        for _ in 0..10 {
+            let mut data = (5_000..20_000)
+                .map(|_| rand::random::<u32>() as u64)
+                .collect::<Vec<_>>();
+            let (estimate, actual_compression) = create_and_validate(&data, "random");
+            dbg!(estimate);
+            dbg!(actual_compression);
+
+            data.reverse();
+            create_and_validate(&data, "random");
+        }
+    }
+}

ownedbytes/Cargo.toml

@@ -0,0 +1,11 @@
+[package]
+authors = ["Paul Masurel <paul@quickwit.io>", "Pascal Seitz <pascal@quickwit.io>"]
+name = "ownedbytes"
+version = "0.1.0"
+edition = "2018"
+description = "Expose data as static slice"
+license = "MIT"
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+stable_deref_trait = "1.2.0"

ownedbytes/src/lib.rs

@@ -0,0 +1,290 @@
+use stable_deref_trait::StableDeref;
+use std::convert::TryInto;
+use std::mem;
+use std::ops::{Deref, Range};
+use std::sync::Arc;
+use std::{fmt, io};
+
+/// An OwnedBytes simply wraps an object that owns a slice of data and exposes
+/// this data as a static slice.
+///
+/// The backing object is required to be `StableDeref`.
+#[derive(Clone)]
+pub struct OwnedBytes {
+    data: &'static [u8],
+    box_stable_deref: Arc<dyn Deref<Target = [u8]> + Sync + Send>,
+}
+
+impl OwnedBytes {
+    /// Creates an empty `OwnedBytes`.
+    pub fn empty() -> OwnedBytes {
+        OwnedBytes::new(&[][..])
+    }
+
+    /// Creates an `OwnedBytes` intance given a `StableDeref` object.
+    pub fn new<T: StableDeref + Deref<Target = [u8]> + 'static + Send + Sync>(
+        data_holder: T,
+    ) -> OwnedBytes {
+        let box_stable_deref = Arc::new(data_holder);
+        let bytes: &[u8] = box_stable_deref.as_ref();
+        let data = unsafe { mem::transmute::<_, &'static [u8]>(bytes.deref()) };
+        OwnedBytes {
+            data,
+            box_stable_deref,
+        }
+    }
+
+    /// creates a fileslice that is just a view over a slice of the data.
+    pub fn slice(&self, range: Range<usize>) -> Self {
+        OwnedBytes {
+            data: &self.data[range],
+            box_stable_deref: self.box_stable_deref.clone(),
+        }
+    }
+
+    /// Returns the underlying slice of data.
+    /// `Deref` and `AsRef` are also available.
+    #[inline]
+    pub fn as_slice(&self) -> &[u8] {
+        self.data
+    }
+
+    /// Returns the len of the slice.
+    #[inline]
+    pub fn len(&self) -> usize {
+        self.data.len()
+    }
+
+    /// Splits the OwnedBytes into two OwnedBytes `(left, right)`.
+    ///
+    /// Left will hold `split_len` bytes.
+    ///
+    /// This operation is cheap and does not require to copy any memory.
+    /// On the other hand, both `left` and `right` retain a handle over
+    /// the entire slice of memory. In other words, the memory will only
+    /// be released when both left and right are dropped.
+    pub fn split(self, split_len: usize) -> (OwnedBytes, OwnedBytes) {
+        let right_box_stable_deref = self.box_stable_deref.clone();
+        let left = OwnedBytes {
+            data: &self.data[..split_len],
+            box_stable_deref: self.box_stable_deref,
+        };
+        let right = OwnedBytes {
+            data: &self.data[split_len..],
+            box_stable_deref: right_box_stable_deref,
+        };
+        (left, right)
+    }
+
+    /// Returns true iff this `OwnedBytes` is empty.
+    #[inline]
+    pub fn is_empty(&self) -> bool {
+        self.as_slice().is_empty()
+    }
+
+    /// Drops the left most `advance_len` bytes.
+    ///
+    /// See also [.clip(clip_len: usize))](#method.clip).
+    #[inline]
+    pub fn advance(&mut self, advance_len: usize) {
+        self.data = &self.data[advance_len..]
+    }
+
+    /// Reads an `u8` from the `OwnedBytes` and advance by one byte.
+    #[inline]
+    pub fn read_u8(&mut self) -> u8 {
+        assert!(!self.is_empty());
+
+        let byte = self.as_slice()[0];
+        self.advance(1);
+        byte
+    }
+
+    /// Reads an `u64` encoded as little-endian from the `OwnedBytes` and advance by 8 bytes.
+    #[inline]
+    pub fn read_u64(&mut self) -> u64 {
+        assert!(self.len() > 7);
+
+        let octlet: [u8; 8] = self.as_slice()[..8].try_into().unwrap();
+        self.advance(8);
+        u64::from_le_bytes(octlet)
+    }
+}
+
+impl fmt::Debug for OwnedBytes {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        // We truncate the bytes in order to make sure the debug string
+        // is not too long.
+        let bytes_truncated: &[u8] = if self.len() > 8 {
+            &self.as_slice()[..10]
+        } else {
+            self.as_slice()
+        };
+        write!(f, "OwnedBytes({:?}, len={})", bytes_truncated, self.len())
+    }
+}
+
+impl Deref for OwnedBytes {
+    type Target = [u8];
+
+    #[inline]
+    fn deref(&self) -> &Self::Target {
+        self.as_slice()
+    }
+}
+
+impl io::Read for OwnedBytes {
+    #[inline]
+    fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
+        let read_len = {
+            let data = self.as_slice();
+            if data.len() >= buf.len() {
+                let buf_len = buf.len();
+                buf.copy_from_slice(&data[..buf_len]);
+                buf.len()
+            } else {
+                let data_len = data.len();
+                buf[..data_len].copy_from_slice(data);
+                data_len
+            }
+        };
+        self.advance(read_len);
+        Ok(read_len)
+    }
+    #[inline]
+    fn read_to_end(&mut self, buf: &mut Vec<u8>) -> io::Result<usize> {
+        let read_len = {
+            let data = self.as_slice();
+            buf.extend(data);
+            data.len()
+        };
+        self.advance(read_len);
+        Ok(read_len)
+    }
+    #[inline]
+    fn read_exact(&mut self, buf: &mut [u8]) -> io::Result<()> {
+        let read_len = self.read(buf)?;
+        if read_len != buf.len() {
+            return Err(io::Error::new(
+                io::ErrorKind::UnexpectedEof,
+                "failed to fill whole buffer",
+            ));
+        }
+        Ok(())
+    }
+}
+
+impl AsRef<[u8]> for OwnedBytes {
+    #[inline]
+    fn as_ref(&self) -> &[u8] {
+        self.as_slice()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::io::{self, Read};
+
+    use super::OwnedBytes;
+
+    #[test]
+    fn test_owned_bytes_debug() {
+        let short_bytes = OwnedBytes::new(b"abcd".as_ref());
+        assert_eq!(
+            format!("{:?}", short_bytes),
+            "OwnedBytes([97, 98, 99, 100], len=4)"
+        );
+        let long_bytes = OwnedBytes::new(b"abcdefghijklmnopq".as_ref());
+        assert_eq!(
+            format!("{:?}", long_bytes),
+            "OwnedBytes([97, 98, 99, 100, 101, 102, 103, 104, 105, 106], len=17)"
+        );
+    }
+
+    #[test]
+    fn test_owned_bytes_read() -> io::Result<()> {
+        let mut bytes = OwnedBytes::new(b"abcdefghiklmnopqrstuvwxyz".as_ref());
+        {
+            let mut buf = [0u8; 5];
+            bytes.read_exact(&mut buf[..]).unwrap();
+            assert_eq!(&buf, b"abcde");
+            assert_eq!(bytes.as_slice(), b"fghiklmnopqrstuvwxyz")
+        }
+        {
+            let mut buf = [0u8; 2];
+            bytes.read_exact(&mut buf[..]).unwrap();
+            assert_eq!(&buf, b"fg");
+            assert_eq!(bytes.as_slice(), b"hiklmnopqrstuvwxyz")
+        }
+        Ok(())
+    }
+
+    #[test]
+    fn test_owned_bytes_read_right_at_the_end() -> io::Result<()> {
+        let mut bytes = OwnedBytes::new(b"abcde".as_ref());
+        let mut buf = [0u8; 5];
+        assert_eq!(bytes.read(&mut buf[..]).unwrap(), 5);
+        assert_eq!(&buf, b"abcde");
+        assert_eq!(bytes.as_slice(), b"");
+        assert_eq!(bytes.read(&mut buf[..]).unwrap(), 0);
+        assert_eq!(&buf, b"abcde");
+        Ok(())
+    }
+    #[test]
+    fn test_owned_bytes_read_incomplete() -> io::Result<()> {
+        let mut bytes = OwnedBytes::new(b"abcde".as_ref());
+        let mut buf = [0u8; 7];
+        assert_eq!(bytes.read(&mut buf[..]).unwrap(), 5);
+        assert_eq!(&buf[..5], b"abcde");
+        assert_eq!(bytes.read(&mut buf[..]).unwrap(), 0);
+        Ok(())
+    }
+
+    #[test]
+    fn test_owned_bytes_read_to_end() -> io::Result<()> {
+        let mut bytes = OwnedBytes::new(b"abcde".as_ref());
+        let mut buf = Vec::new();
+        bytes.read_to_end(&mut buf)?;
+        assert_eq!(buf.as_slice(), b"abcde".as_ref());
+        Ok(())
+    }
+
+    #[test]
+    fn test_owned_bytes_read_u8() -> io::Result<()> {
+        let mut bytes = OwnedBytes::new(b"\xFF".as_ref());
+        assert_eq!(bytes.read_u8(), 255);
+        assert_eq!(bytes.len(), 0);
+        Ok(())
+    }
+
+    #[test]
+    fn test_owned_bytes_read_u64() -> io::Result<()> {
+        let mut bytes = OwnedBytes::new(b"\0\xFF\xFF\xFF\xFF\xFF\xFF\xFF".as_ref());
+        assert_eq!(bytes.read_u64(), u64::MAX - 255);
+        assert_eq!(bytes.len(), 0);
+        Ok(())
+    }
+
+    #[test]
+    fn test_owned_bytes_split() {
+        let bytes = OwnedBytes::new(b"abcdefghi".as_ref());
+        let (left, right) = bytes.split(3);
+        assert_eq!(left.as_slice(), b"abc");
+        assert_eq!(right.as_slice(), b"defghi");
+    }
+
+    #[test]
+    fn test_owned_bytes_split_boundary() {
+        let bytes = OwnedBytes::new(b"abcdefghi".as_ref());
+        {
+            let (left, right) = bytes.clone().split(0);
+            assert_eq!(left.as_slice(), b"");
+            assert_eq!(right.as_slice(), b"abcdefghi");
+        }
+        {
+            let (left, right) = bytes.split(9);
+            assert_eq!(left.as_slice(), b"abcdefghi");
+            assert_eq!(right.as_slice(), b"");
+        }
+    }
+}

query-grammar/Cargo.toml

@@ -14,3 +14,5 @@ edition = "2018"
 
 [dependencies]
 combine = {version="4", default-features=false, features=[] }
+once_cell = "1.7.2"
+regex ={ version = "1.5.4", default-features = false, features = ["std"] }

query-grammar/src/query_grammar.rs

@@ -1,21 +1,44 @@
 use super::user_input_ast::{UserInputAst, UserInputBound, UserInputLeaf, UserInputLiteral};
 use crate::Occur;
-use combine::parser::char::{char, digit, letter, space, spaces, string};
+use combine::parser::char::{char, digit, space, spaces, string};
+use combine::parser::range::{take_while, take_while1};
+use combine::parser::repeat::escaped;
 use combine::parser::Parser;
 use combine::{
     attempt, choice, eof, many, many1, one_of, optional, parser, satisfy, skip_many1, value,
 };
 use combine::{error::StringStreamError, parser::combinator::recognize};
+use once_cell::sync::Lazy;
+use regex::Regex;
 
-fn field<'a>() -> impl Parser<&'a str, Output = String> {
-    (
-        (letter().or(char('_'))),
-        many(satisfy(|c: char| {
-            c.is_alphanumeric() || c == '_' || c == '-'
-        })),
-    )
-        .skip(char(':'))
-        .map(|(s1, s2): (char, String)| format!("{}{}", s1, s2))
+// Note: '-' char is only forbidden at the beginning of a field name, would be clearer to add it to special characters.
+const SPECIAL_CHARS: &[char] = &[
+    '+', '^', '`', ':', '{', '}', '"', '[', ']', '(', ')', '~', '!', '\\', '*', ' ',
+];
+const ESCAPED_SPECIAL_CHARS_PATTERN: &str = r#"\\(\+|\^|`|:|\{|\}|"|\[|\]|\(|\)|\~|!|\\|\*| )"#;
+
+/// Parses a field_name
+/// A field name must have at least one character and be followed by a colon.
+/// All characters are allowed including special characters `SPECIAL_CHARS`, but these
+/// need to be escaped with a backslack character '\'.
+fn field_name<'a>() -> impl Parser<&'a str, Output = String> {
+    static ESCAPED_SPECIAL_CHARS_RE: Lazy<Regex> =
+        Lazy::new(|| Regex::new(ESCAPED_SPECIAL_CHARS_PATTERN).unwrap());
+
+    recognize::<String, _, _>(escaped(
+        (
+            take_while1(|c| !SPECIAL_CHARS.contains(&c) && c != '-'),
+            take_while(|c| !SPECIAL_CHARS.contains(&c)),
+        ),
+        '\\',
+        satisfy(|c| SPECIAL_CHARS.contains(&c)),
+    ))
+    .skip(char(':'))
+    .map(|s| ESCAPED_SPECIAL_CHARS_RE.replace_all(&s, "$1").to_string())
+    .and_then(|s: String| match s.is_empty() {
+        true => Err(StringStreamError::UnexpectedParse),
+        _ => Ok(s),
+    })
 }
 
 fn word<'a>() -> impl Parser<&'a str, Output = String> {
@@ -98,7 +121,7 @@ fn term_val<'a>() -> impl Parser<&'a str, Output = String> {
 
 fn term_query<'a>() -> impl Parser<&'a str, Output = UserInputLiteral> {
     let term_val_with_field = negative_number().or(term_val());
-    (field(), term_val_with_field).map(|(field_name, phrase)| UserInputLiteral {
+    (field_name(), term_val_with_field).map(|(field_name, phrase)| UserInputLiteral {
         field_name: Some(field_name),
         phrase,
     })
@@ -195,7 +218,7 @@ fn range<'a>() -> impl Parser<&'a str, Output = UserInputLeaf> {
     );
 
     (
-        optional(field()).skip(spaces()),
+        optional(field_name()).skip(spaces()),
         // try elastic first, if it matches, the range is unbounded
         attempt(elastic_unbounded_range).or(lower_to_upper),
     )
@@ -464,21 +487,21 @@ mod test {
 
     #[test]
     fn test_parse_elastic_query_ranges() {
-        test_parse_query_to_ast_helper("title: >a", "title:{\"a\" TO \"*\"}");
-        test_parse_query_to_ast_helper("title:>=a", "title:[\"a\" TO \"*\"}");
-        test_parse_query_to_ast_helper("title: <a", "title:{\"*\" TO \"a\"}");
-        test_parse_query_to_ast_helper("title:<=a", "title:{\"*\" TO \"a\"]");
-        test_parse_query_to_ast_helper("title:<=bsd", "title:{\"*\" TO \"bsd\"]");
+        test_parse_query_to_ast_helper("title: >a", "\"title\":{\"a\" TO \"*\"}");
+        test_parse_query_to_ast_helper("title:>=a", "\"title\":[\"a\" TO \"*\"}");
+        test_parse_query_to_ast_helper("title: <a", "\"title\":{\"*\" TO \"a\"}");
+        test_parse_query_to_ast_helper("title:<=a", "\"title\":{\"*\" TO \"a\"]");
+        test_parse_query_to_ast_helper("title:<=bsd", "\"title\":{\"*\" TO \"bsd\"]");
 
-        test_parse_query_to_ast_helper("weight: >70", "weight:{\"70\" TO \"*\"}");
-        test_parse_query_to_ast_helper("weight:>=70", "weight:[\"70\" TO \"*\"}");
-        test_parse_query_to_ast_helper("weight: <70", "weight:{\"*\" TO \"70\"}");
-        test_parse_query_to_ast_helper("weight:<=70", "weight:{\"*\" TO \"70\"]");
-        test_parse_query_to_ast_helper("weight: >60.7", "weight:{\"60.7\" TO \"*\"}");
+        test_parse_query_to_ast_helper("weight: >70", "\"weight\":{\"70\" TO \"*\"}");
+        test_parse_query_to_ast_helper("weight:>=70", "\"weight\":[\"70\" TO \"*\"}");
+        test_parse_query_to_ast_helper("weight: <70", "\"weight\":{\"*\" TO \"70\"}");
+        test_parse_query_to_ast_helper("weight:<=70", "\"weight\":{\"*\" TO \"70\"]");
+        test_parse_query_to_ast_helper("weight: >60.7", "\"weight\":{\"60.7\" TO \"*\"}");
 
-        test_parse_query_to_ast_helper("weight: <= 70", "weight:{\"*\" TO \"70\"]");
+        test_parse_query_to_ast_helper("weight: <= 70", "\"weight\":{\"*\" TO \"70\"]");
 
-        test_parse_query_to_ast_helper("weight: <= 70.5", "weight:{\"*\" TO \"70.5\"]");
+        test_parse_query_to_ast_helper("weight: <= 70.5", "\"weight\":{\"*\" TO \"70.5\"]");
     }
 
     #[test]
@@ -491,22 +514,43 @@ mod test {
     #[test]
     fn test_field_name() -> TestParseResult {
         assert_eq!(
-            super::field().parse("my-field-name:a")?,
-            ("my-field-name".to_string(), "a")
+            super::field_name().parse(".my.field.name:a"),
+            Ok((".my.field.name".to_string(), "a"))
         );
         assert_eq!(
-            super::field().parse("my_field_name:a")?,
-            ("my_field_name".to_string(), "a")
+            super::field_name().parse("my\\ field\\ name:a"),
+            Ok(("my field name".to_string(), "a"))
         );
-        assert!(super::field().parse(":a").is_err());
-        assert!(super::field().parse("-my_field:a").is_err());
+        assert!(super::field_name().parse("my field:a").is_err());
         assert_eq!(
-            super::field().parse("_my_field:a")?,
+            super::field_name().parse("\\(1\\+1\\):2"),
+            Ok(("(1+1)".to_string(), "2"))
+        );
+        assert_eq!(
+            super::field_name().parse("my_field_name:a"),
+            Ok(("my_field_name".to_string(), "a"))
+        );
+        assert!(super::field_name().parse("my_field_name").is_err());
+        assert!(super::field_name().parse(":a").is_err());
+        assert!(super::field_name().parse("-my_field:a").is_err());
+        assert_eq!(
+            super::field_name().parse("_my_field:a")?,
             ("_my_field".to_string(), "a")
         );
         Ok(())
     }
 
+    #[test]
+    fn test_field_name_re() {
+        let escaped_special_chars_re = Regex::new(ESCAPED_SPECIAL_CHARS_PATTERN).unwrap();
+        for special_char in SPECIAL_CHARS.iter() {
+            assert_eq!(
+                escaped_special_chars_re.replace_all(&format!("\\{}", special_char), "$1"),
+                special_char.to_string()
+            );
+        }
+    }
+
     #[test]
     fn test_range_parser() {
         // testing the range() parser separately
@@ -600,12 +644,14 @@ mod test {
 
     #[test]
     fn test_single_term_with_field() {
-        test_parse_query_to_ast_helper("abc:toto", "abc:\"toto\"");
+        test_parse_query_to_ast_helper("abc:toto", "\"abc\":\"toto\"");
     }
 
     #[test]
     fn test_single_term_with_float() {
-        test_parse_query_to_ast_helper("abc:1.1", "abc:\"1.1\"");
+        test_parse_query_to_ast_helper("abc:1.1", "\"abc\":\"1.1\"");
+        test_parse_query_to_ast_helper("a.b.c:1.1", "\"a.b.c\":\"1.1\"");
+        test_parse_query_to_ast_helper("a\\ b\\ c:1.1", "\"a b c\":\"1.1\"");
     }
 
     #[test]
@@ -621,22 +667,27 @@ mod test {
     #[test]
     fn test_parse_test_query_other() {
         test_parse_query_to_ast_helper("(+a +b) d", "(*(+\"a\" +\"b\") *\"d\")");
-        test_parse_query_to_ast_helper("+abc:toto", "abc:\"toto\"");
-        test_parse_query_to_ast_helper("(+abc:toto -titi)", "(+abc:\"toto\" -\"titi\")");
-        test_parse_query_to_ast_helper("-abc:toto", "(-abc:\"toto\")");
-        test_parse_query_to_ast_helper("abc:a b", "(*abc:\"a\" *\"b\")");
-        test_parse_query_to_ast_helper("abc:\"a b\"", "abc:\"a b\"");
-        test_parse_query_to_ast_helper("foo:[1 TO 5]", "foo:[\"1\" TO \"5\"]");
+        test_parse_query_to_ast_helper("+abc:toto", "\"abc\":\"toto\"");
+        test_parse_query_to_ast_helper("+a\\+b\\+c:toto", "\"a+b+c\":\"toto\"");
+        test_parse_query_to_ast_helper("(+abc:toto -titi)", "(+\"abc\":\"toto\" -\"titi\")");
+        test_parse_query_to_ast_helper("-abc:toto", "(-\"abc\":\"toto\")");
+        test_is_parse_err("--abc:toto");
+        test_parse_query_to_ast_helper("abc:a b", "(*\"abc\":\"a\" *\"b\")");
+        test_parse_query_to_ast_helper("abc:\"a b\"", "\"abc\":\"a b\"");
+        test_parse_query_to_ast_helper("foo:[1 TO 5]", "\"foo\":[\"1\" TO \"5\"]");
     }
 
     #[test]
     fn test_parse_query_with_range() {
         test_parse_query_to_ast_helper("[1 TO 5]", "[\"1\" TO \"5\"]");
-        test_parse_query_to_ast_helper("foo:{a TO z}", "foo:{\"a\" TO \"z\"}");
-        test_parse_query_to_ast_helper("foo:[1 TO toto}", "foo:[\"1\" TO \"toto\"}");
-        test_parse_query_to_ast_helper("foo:[* TO toto}", "foo:{\"*\" TO \"toto\"}");
-        test_parse_query_to_ast_helper("foo:[1 TO *}", "foo:[\"1\" TO \"*\"}");
-        test_parse_query_to_ast_helper("foo:[1.1 TO *}", "foo:[\"1.1\" TO \"*\"}");
+        test_parse_query_to_ast_helper("foo:{a TO z}", "\"foo\":{\"a\" TO \"z\"}");
+        test_parse_query_to_ast_helper("foo:[1 TO toto}", "\"foo\":[\"1\" TO \"toto\"}");
+        test_parse_query_to_ast_helper("foo:[* TO toto}", "\"foo\":{\"*\" TO \"toto\"}");
+        test_parse_query_to_ast_helper("foo:[1 TO *}", "\"foo\":[\"1\" TO \"*\"}");
+        test_parse_query_to_ast_helper(
+            "1.2.foo.bar:[1.1 TO *}",
+            "\"1.2.foo.bar\":[\"1.1\" TO \"*\"}",
+        );
         test_is_parse_err("abc +    ");
     }
 }

query-grammar/src/user_input_ast.rs

@@ -24,7 +24,7 @@ impl Debug for UserInputLeaf {
                 ref upper,
             } => {
                 if let Some(ref field) = field {
-                    write!(formatter, "{}:", field)?;
+                    write!(formatter, "\"{}\":", field)?;
                 }
                 lower.display_lower(formatter)?;
                 write!(formatter, " TO ")?;
@@ -45,7 +45,7 @@ pub struct UserInputLiteral {
 impl fmt::Debug for UserInputLiteral {
     fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> {
         match self.field_name {
-            Some(ref field_name) => write!(formatter, "{}:\"{}\"", field_name, self.phrase),
+            Some(ref field_name) => write!(formatter, "\"{}\":\"{}\"", field_name, self.phrase),
             None => write!(formatter, "\"{}\"", self.phrase),
         }
     }
@@ -79,7 +79,7 @@ impl UserInputBound {
         match *self {
             UserInputBound::Inclusive(ref contents) => contents,
             UserInputBound::Exclusive(ref contents) => contents,
-            UserInputBound::Unbounded => &"*",
+            UserInputBound::Unbounded => "*",
         }
     }
 }

src/collector/facet_collector.rs

@@ -297,10 +297,8 @@ impl Collector for FacetCollector {
                                 if depth == collapse_depth + 1 {
                                     collapsed_id = collapse_facet_ords.len();
                                     collapse_facet_ords.push(facet_streamer.term_ord());
-                                    collapse_mapping.push(collapsed_id);
-                                } else {
-                                    collapse_mapping.push(collapsed_id);
                                 }
+                                collapse_mapping.push(collapsed_id);
                             }
                             break;
                         }

src/collector/filter_collector_wrapper.rs

@@ -16,7 +16,7 @@ use crate::fastfield::{DynamicFastFieldReader, FastFieldReader, FastValue};
 use crate::schema::Field;
 use crate::{Score, SegmentReader, TantivyError};
 
-/// The `FilterCollector` collector filters docs using a u64 fast field value and a predicate.
+/// The `FilterCollector` collector filters docs using a fast field value and a predicate.
 /// Only the documents for which the predicate returned "true" will be passed on to the next collector.
 ///
 /// ```rust

src/collector/mod.rs

@@ -178,9 +178,9 @@ pub trait Collector: Sync + Send {
     ) -> crate::Result<<Self::Child as SegmentCollector>::Fruit> {
         let mut segment_collector = self.for_segment(segment_ord as u32, reader)?;
 
-        if let Some(delete_bitset) = reader.delete_bitset() {
+        if let Some(alive_bitset) = reader.alive_bitset() {
             weight.for_each(reader, &mut |doc, score| {
-                if delete_bitset.is_alive(doc) {
+                if alive_bitset.is_alive(doc) {
                     segment_collector.collect(doc, score);
                 }
             })?;

src/collector/multi_collector.rs

@@ -276,7 +276,7 @@ mod tests {
         let mut collectors = MultiCollector::new();
         let topdocs_handler = collectors.add_collector(TopDocs::with_limit(2));
         let count_handler = collectors.add_collector(Count);
-        let mut multifruits = searcher.search(&query, &mut collectors).unwrap();
+        let mut multifruits = searcher.search(&query, &collectors).unwrap();
 
         assert_eq!(count_handler.extract(&mut multifruits), 5);
         assert_eq!(topdocs_handler.extract(&mut multifruits).len(), 2);

src/collector/top_score_collector.rs

@@ -629,10 +629,10 @@ impl Collector for TopDocs {
         let heap_len = self.0.limit + self.0.offset;
         let mut heap: BinaryHeap<ComparableDoc<Score, DocId>> = BinaryHeap::with_capacity(heap_len);
 
-        if let Some(delete_bitset) = reader.delete_bitset() {
+        if let Some(alive_bitset) = reader.alive_bitset() {
             let mut threshold = Score::MIN;
             weight.for_each_pruning(threshold, reader, &mut |doc, score| {
-                if delete_bitset.is_deleted(doc) {
+                if alive_bitset.is_deleted(doc) {
                     return threshold;
                 }
                 let heap_item = ComparableDoc {
@@ -1080,7 +1080,7 @@ mod tests {
         query: &str,
         query_field: Field,
         schema: Schema,
-        mut doc_adder: impl FnMut(&mut IndexWriter) -> (),
+        mut doc_adder: impl FnMut(&mut IndexWriter),
     ) -> (Index, Box<dyn Query>) {
         let index = Index::create_in_ram(schema);
         let mut index_writer = index.writer_with_num_threads(1, 10_000_000).unwrap();

src/core/executor.rs

@@ -1,5 +1,4 @@
 use crossbeam::channel;
-use rayon::{ThreadPool, ThreadPoolBuilder};
 
 /// Search executor whether search request are single thread or multithread.
 ///
@@ -11,8 +10,6 @@ use rayon::{ThreadPool, ThreadPoolBuilder};
 pub enum Executor {
     /// Single thread variant of an Executor
     SingleThread,
-    /// Thread pool variant of an Executor
-    ThreadPool(ThreadPool),
 }
 
 impl Executor {
@@ -21,15 +18,6 @@ impl Executor {
         Executor::SingleThread
     }
 
-    /// Creates an Executor that dispatches the tasks in a thread pool.
-    pub fn multi_thread(num_threads: usize, prefix: &'static str) -> crate::Result<Executor> {
-        let pool = ThreadPoolBuilder::new()
-            .num_threads(num_threads)
-            .thread_name(move |num| format!("{}{}", prefix, num))
-            .build()?;
-        Ok(Executor::ThreadPool(pool))
-    }
-
     /// Perform a map in the thread pool.
     ///
     /// Regardless of the executor (`SingleThread` or `ThreadPool`), panics in the task
@@ -46,40 +34,6 @@ impl Executor {
     ) -> crate::Result<Vec<R>> {
         match self {
             Executor::SingleThread => args.map(f).collect::<crate::Result<_>>(),
-            Executor::ThreadPool(pool) => {
-                let args_with_indices: Vec<(usize, A)> = args.enumerate().collect();
-                let num_fruits = args_with_indices.len();
-                let fruit_receiver = {
-                    let (fruit_sender, fruit_receiver) = channel::unbounded();
-                    pool.scope(|scope| {
-                        for arg_with_idx in args_with_indices {
-                            scope.spawn(|_| {
-                                let (idx, arg) = arg_with_idx;
-                                let fruit = f(arg);
-                                if let Err(err) = fruit_sender.send((idx, fruit)) {
-                                    error!("Failed to send search task. It probably means all search threads have panicked. {:?}", err);
-                                }
-                            });
-                        }
-                    });
-                    fruit_receiver
-                    // This ends the scope of fruit_sender.
-                    // This is important as it makes it possible for the fruit_receiver iteration to
-                    // terminate.
-                };
-                // This is lame, but safe.
-                let mut results_with_position = Vec::with_capacity(num_fruits);
-                for (pos, fruit_res) in fruit_receiver {
-                    let fruit = fruit_res?;
-                    results_with_position.push((pos, fruit));
-                }
-                results_with_position.sort_by_key(|(pos, _)| *pos);
-                assert_eq!(results_with_position.len(), num_fruits);
-                Ok(results_with_position
-                    .into_iter()
-                    .map(|(_, fruit)| fruit)
-                    .collect::<Vec<_>>())
-            }
         }
     }
 }

src/core/index.rs

@@ -42,7 +42,7 @@ fn load_metas(
             "Meta file does not contain valid utf8 file.".to_string(),
         )
     })?;
-    IndexMeta::deserialize(&meta_string, &inventory)
+    IndexMeta::deserialize(&meta_string, inventory)
         .map_err(|e| {
             DataCorruption::new(
                 META_FILEPATH.to_path_buf(),
@@ -120,7 +120,7 @@ impl IndexBuilder {
     /// Creates a new index in a given filepath.
     /// The index will use the `MMapDirectory`.
     ///
-    /// If a previous index was in this directory, then its meta file will be destroyed.
+    /// If a previous index was in this directory, it returns an `IndexAlreadyExists` error.
     #[cfg(feature = "mmap")]
     pub fn create_in_dir<P: AsRef<Path>>(self, directory_path: P) -> crate::Result<Index> {
         let mmap_directory = MmapDirectory::open(directory_path)?;
@@ -229,7 +229,8 @@ impl Index {
     /// Creates a new index using the `RamDirectory`.
     ///
     /// The index will be allocated in anonymous memory.
-    /// This should only be used for unit tests.
+    /// This is useful for indexing small set of documents
+    /// for instances like unit test or temporary in memory index.
     pub fn create_in_ram(schema: Schema) -> Index {
         IndexBuilder::new().schema(schema).create_in_ram().unwrap()
     }
@@ -237,7 +238,7 @@ impl Index {
     /// Creates a new index in a given filepath.
     /// The index will use the `MMapDirectory`.
     ///
-    /// If a previous index was in this directory, then its meta file will be destroyed.
+    /// If a previous index was in this directory, then it returns  an `IndexAlreadyExists` error.
     #[cfg(feature = "mmap")]
     pub fn create_in_dir<P: AsRef<Path>>(
         directory_path: P,
@@ -523,7 +524,22 @@ impl Index {
 
     /// Returns the set of corrupted files
     pub fn validate_checksum(&self) -> crate::Result<HashSet<PathBuf>> {
-        self.directory.list_damaged().map_err(Into::into)
+        let managed_files = self.directory.list_managed_files();
+        let active_segments_files: HashSet<PathBuf> = self
+            .searchable_segment_metas()?
+            .iter()
+            .flat_map(|segment_meta| segment_meta.list_files())
+            .collect();
+        let active_existing_files: HashSet<&PathBuf> =
+            active_segments_files.intersection(&managed_files).collect();
+
+        let mut damaged_files = HashSet::new();
+        for path in active_existing_files {
+            if !self.directory.validate_checksum(path)? {
+                damaged_files.insert((*path).clone());
+            }
+        }
+        Ok(damaged_files)
     }
 }
 
@@ -604,7 +620,7 @@ mod tests {
         .is_ok());
         assert!(Index::exists(&directory).unwrap());
         assert!(Index::create(
-            directory.clone(),
+            directory,
             Schema::builder().build(),
             IndexSettings::default()
         )

src/core/index_meta.rs

@@ -101,6 +101,7 @@ impl SegmentMeta {
 
     /// Returns the list of files that
     /// are required for the segment meta.
+    /// Note: Some of the returned files may not exist depending on the state of the segment.
     ///
     /// This is useful as the way tantivy removes files
     /// is by removing all files that have been created by tantivy
@@ -369,7 +370,6 @@ mod tests {
         schema::{Schema, TEXT},
         IndexSettings, IndexSortByField, Order,
     };
-    use serde_json;
 
     #[test]
     fn test_serialize_metas() {

src/core/inverted_index_reader.rs

@@ -1,6 +1,5 @@
 use std::io;
 
-use crate::common::BinarySerializable;
 use crate::directory::FileSlice;
 use crate::positions::PositionReader;
 use crate::postings::TermInfo;
@@ -8,6 +7,7 @@ use crate::postings::{BlockSegmentPostings, SegmentPostings};
 use crate::schema::IndexRecordOption;
 use crate::schema::Term;
 use crate::termdict::TermDictionary;
+use common::BinarySerializable;
 
 /// The inverted index reader is in charge of accessing
 /// the inverted index associated to a specific field.

src/core/mod.rs

@@ -16,7 +16,6 @@ pub use self::index_meta::{
 pub use self::inverted_index_reader::InvertedIndexReader;
 pub use self::searcher::Searcher;
 pub use self::segment::Segment;
-pub use self::segment::SerializableSegment;
 pub use self::segment_component::SegmentComponent;
 pub use self::segment_id::SegmentId;
 pub use self::segment_reader::SegmentReader;

src/core/segment.rs

@@ -1,13 +1,12 @@
 use super::SegmentComponent;
+use crate::core::Index;
 use crate::core::SegmentId;
 use crate::core::SegmentMeta;
 use crate::directory::error::{OpenReadError, OpenWriteError};
 use crate::directory::Directory;
 use crate::directory::{FileSlice, WritePtr};
-use crate::indexer::segment_serializer::SegmentSerializer;
 use crate::schema::Schema;
 use crate::Opstamp;
-use crate::{core::Index, indexer::doc_id_mapping::DocIdMapping};
 use std::fmt;
 use std::path::PathBuf;
 
@@ -90,20 +89,3 @@ impl Segment {
         Ok(write)
     }
 }
-
-pub trait SerializableSegment {
-    /// Writes a view of a segment by pushing information
-    /// to the `SegmentSerializer`.
-    ///
-    /// # Returns
-    /// The number of documents in the segment.
-    ///
-    /// doc_id_map is used when index is created and sorted, to map to the new doc_id order.
-    /// It is not used by the `IndexMerger`, since the doc_id_mapping on cross-segments works
-    /// differently
-    fn write(
-        &self,
-        serializer: SegmentSerializer,
-        doc_id_map: Option<&DocIdMapping>,
-    ) -> crate::Result<u32>;
-}

src/core/segment_id.rs

@@ -22,7 +22,7 @@ use std::sync::atomic;
 pub struct SegmentId(Uuid);
 
 #[cfg(test)]
-static AUTO_INC_COUNTER: Lazy<atomic::AtomicUsize> = Lazy::new(|| atomic::AtomicUsize::default());
+static AUTO_INC_COUNTER: Lazy<atomic::AtomicUsize> = Lazy::new(atomic::AtomicUsize::default);
 
 #[cfg(test)]
 const ZERO_ARRAY: [u8; 8] = [0u8; 8];
@@ -108,6 +108,12 @@ impl fmt::Debug for SegmentId {
     }
 }
 
+impl fmt::Display for SegmentId {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "Seg({:?})", self.short_uuid_string())
+    }
+}
+
 impl PartialOrd for SegmentId {
     fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
         Some(self.cmp(other))

src/core/segment_reader.rs

@@ -1,10 +1,11 @@
-use crate::common::HasLen;
 use crate::core::InvertedIndexReader;
 use crate::core::Segment;
 use crate::core::SegmentComponent;
 use crate::core::SegmentId;
+use crate::directory::CompositeFile;
 use crate::directory::FileSlice;
-use crate::fastfield::DeleteBitSet;
+use crate::error::DataCorruption;
+use crate::fastfield::AliveBitSet;
 use crate::fastfield::FacetReader;
 use crate::fastfield::FastFieldReaders;
 use crate::fieldnorm::{FieldNormReader, FieldNormReaders};
@@ -15,7 +16,6 @@ use crate::space_usage::SegmentSpaceUsage;
 use crate::store::StoreReader;
 use crate::termdict::TermDictionary;
 use crate::DocId;
-use crate::{common::CompositeFile, error::DataCorruption};
 use fail::fail_point;
 use std::fmt;
 use std::sync::Arc;
@@ -32,9 +32,6 @@ use std::{collections::HashMap, io};
 ///
 /// The segment reader has a very low memory footprint,
 /// as close to all of the memory data is mmapped.
-///
-///
-/// TODO fix not decoding docfreq
 #[derive(Clone)]
 pub struct SegmentReader {
     inv_idx_reader_cache: Arc<RwLock<HashMap<Field, Arc<InvertedIndexReader>>>>,
@@ -50,24 +47,19 @@ pub struct SegmentReader {
     fieldnorm_readers: FieldNormReaders,
 
     store_file: FileSlice,
-    delete_bitset_opt: Option<DeleteBitSet>,
+    alive_bitset_opt: Option<AliveBitSet>,
     schema: Schema,
 }
 
 impl SegmentReader {
     /// Returns the highest document id ever attributed in
     /// this segment + 1.
-    /// Today, `tantivy` does not handle deletes, so it happens
-    /// to also be the number of documents in the index.
     pub fn max_doc(&self) -> DocId {
         self.max_doc
     }
 
-    /// Returns the number of documents.
+    /// Returns the number of alive documents.
     /// Deleted documents are not counted.
-    ///
-    /// Today, `tantivy` does not handle deletes so max doc and
-    /// num_docs are the same.
     pub fn num_docs(&self) -> DocId {
         self.num_docs
     }
@@ -80,14 +72,12 @@ impl SegmentReader {
     /// Return the number of documents that have been
     /// deleted in the segment.
     pub fn num_deleted_docs(&self) -> DocId {
-        self.delete_bitset()
-            .map(|delete_set| delete_set.len() as DocId)
-            .unwrap_or(0u32)
+        self.max_doc - self.num_docs
     }
 
     /// Returns true iff some of the documents of the segment have been deleted.
     pub fn has_deletes(&self) -> bool {
-        self.delete_bitset().is_some()
+        self.num_deleted_docs() > 0
     }
 
     /// Accessor to a segment's fast field reader given a field.
@@ -178,10 +168,10 @@ impl SegmentReader {
         let fieldnorm_data = segment.open_read(SegmentComponent::FieldNorms)?;
         let fieldnorm_readers = FieldNormReaders::open(fieldnorm_data)?;
 
-        let delete_bitset_opt = if segment.meta().has_deletes() {
-            let delete_data = segment.open_read(SegmentComponent::Delete)?;
-            let delete_bitset = DeleteBitSet::open(delete_data)?;
-            Some(delete_bitset)
+        let alive_bitset_opt = if segment.meta().has_deletes() {
+            let alive_bitset_bytes = segment.open_read(SegmentComponent::Delete)?.read_bytes()?;
+            let alive_bitset = AliveBitSet::open(alive_bitset_bytes);
+            Some(alive_bitset)
         } else {
             None
         };
@@ -196,7 +186,7 @@ impl SegmentReader {
             fieldnorm_readers,
             segment_id: segment.id(),
             store_file,
-            delete_bitset_opt,
+            alive_bitset_opt,
             positions_composite,
             schema,
         })
@@ -282,21 +272,25 @@ impl SegmentReader {
 
     /// Returns the bitset representing
     /// the documents that have been deleted.
-    pub fn delete_bitset(&self) -> Option<&DeleteBitSet> {
-        self.delete_bitset_opt.as_ref()
+    pub fn alive_bitset(&self) -> Option<&AliveBitSet> {
+        self.alive_bitset_opt.as_ref()
     }
 
     /// Returns true iff the `doc` is marked
     /// as deleted.
     pub fn is_deleted(&self, doc: DocId) -> bool {
-        self.delete_bitset()
+        self.alive_bitset()
             .map(|delete_set| delete_set.is_deleted(doc))
             .unwrap_or(false)
     }
 
     /// Returns an iterator that will iterate over the alive document ids
-    pub fn doc_ids_alive(&self) -> impl Iterator<Item = DocId> + '_ {
-        (0u32..self.max_doc).filter(move |doc| !self.is_deleted(*doc))
+    pub fn doc_ids_alive(&self) -> Box<dyn Iterator<Item = DocId> + '_> {
+        if let Some(alive_bitset) = &self.alive_bitset_opt {
+            Box::new(alive_bitset.iter_alive())
+        } else {
+            Box::new(0u32..self.max_doc)
+        }
     }
 
     /// Summarize total space usage of this segment.
@@ -309,9 +303,9 @@ impl SegmentReader {
             self.fast_fields_readers.space_usage(),
             self.fieldnorm_readers.space_usage(),
             self.get_store_reader()?.space_usage(),
-            self.delete_bitset_opt
+            self.alive_bitset_opt
                 .as_ref()
-                .map(DeleteBitSet::space_usage)
+                .map(AliveBitSet::space_usage)
                 .unwrap_or(0),
         ))
     }
@@ -329,6 +323,32 @@ mod test {
     use crate::schema::{Schema, Term, STORED, TEXT};
     use crate::DocId;
 
+    #[test]
+    fn test_num_alive() -> crate::Result<()> {
+        let mut schema_builder = Schema::builder();
+        schema_builder.add_text_field("name", TEXT | STORED);
+        let schema = schema_builder.build();
+        let index = Index::create_in_ram(schema.clone());
+        let name = schema.get_field("name").unwrap();
+
+        {
+            let mut index_writer = index.writer_for_tests()?;
+            index_writer.add_document(doc!(name => "tantivy"));
+            index_writer.add_document(doc!(name => "horse"));
+            index_writer.add_document(doc!(name => "jockey"));
+            index_writer.add_document(doc!(name => "cap"));
+            // we should now have one segment with two docs
+            index_writer.delete_term(Term::from_field_text(name, "horse"));
+            index_writer.delete_term(Term::from_field_text(name, "cap"));
+
+            // ok, now we should have a deleted doc
+            index_writer.commit()?;
+        }
+        let searcher = index.reader()?.searcher();
+        assert_eq!(2, searcher.segment_reader(0).num_docs());
+        assert_eq!(4, searcher.segment_reader(0).max_doc());
+        Ok(())
+    }
     #[test]
     fn test_alive_docs_iterator() -> crate::Result<()> {
         let mut schema_builder = Schema::builder();

src/directory/composite_file.rs

@@ -1,18 +1,17 @@
-use crate::common::BinarySerializable;
-use crate::common::CountingWriter;
-use crate::common::VInt;
 use crate::directory::FileSlice;
 use crate::directory::{TerminatingWrite, WritePtr};
 use crate::schema::Field;
 use crate::space_usage::FieldUsage;
 use crate::space_usage::PerFieldSpaceUsage;
+use common::BinarySerializable;
+use common::CountingWriter;
+use common::HasLen;
+use common::VInt;
 use std::collections::HashMap;
 use std::io::{self, Read, Write};
 use std::iter::ExactSizeIterator;
 use std::ops::Range;
 
-use super::HasLen;
-
 #[derive(Eq, PartialEq, Hash, Copy, Ord, PartialOrd, Clone, Debug)]
 pub struct FileAddr {
     field: Field,
@@ -188,10 +187,10 @@ impl CompositeFile {
 mod test {
 
     use super::{CompositeFile, CompositeWrite};
-    use crate::common::BinarySerializable;
-    use crate::common::VInt;
     use crate::directory::{Directory, RamDirectory};
     use crate::schema::Field;
+    use common::BinarySerializable;
+    use common::VInt;
     use std::io::Write;
     use std::path::Path;
 

src/directory/file_slice.rs

@@ -1,7 +1,7 @@
 use stable_deref_trait::StableDeref;
 
-use crate::common::HasLen;
 use crate::directory::OwnedBytes;
+use common::HasLen;
 use std::fmt;
 use std::ops::Range;
 use std::sync::{Arc, Weak};
@@ -32,12 +32,6 @@ impl FileHandle for &'static [u8] {
     }
 }
 
-impl<T: Deref<Target = [u8]>> HasLen for T {
-    fn len(&self) -> usize {
-        self.deref().len()
-    }
-}
-
 impl<B> From<B> for FileSlice
 where
     B: StableDeref + Deref<Target = [u8]> + 'static + Send + Sync,
@@ -178,7 +172,7 @@ impl HasLen for FileSlice {
 #[cfg(test)]
 mod tests {
     use super::{FileHandle, FileSlice};
-    use crate::common::HasLen;
+    use common::HasLen;
     use std::io;
 
     #[test]
@@ -211,7 +205,7 @@ mod tests {
             assert_eq!(right.read_bytes()?.as_slice(), b"");
         }
         {
-            let (left, right) = file_slice.clone().split_from_end(2);
+            let (left, right) = file_slice.split_from_end(2);
             assert_eq!(left.read_bytes()?.as_slice(), b"abcd");
             assert_eq!(right.read_bytes()?.as_slice(), b"ef");
         }

src/directory/footer.rs

@@ -1,10 +1,10 @@
 use crate::directory::error::Incompatibility;
 use crate::directory::FileSlice;
 use crate::{
-    common::{BinarySerializable, CountingWriter, DeserializeFrom, FixedSize, HasLen},
     directory::{AntiCallToken, TerminatingWrite},
     Version, INDEX_FORMAT_VERSION,
 };
+use common::{BinarySerializable, CountingWriter, DeserializeFrom, FixedSize, HasLen};
 use crc32fast::Hasher;
 use serde::{Deserialize, Serialize};
 use std::io;
@@ -156,10 +156,8 @@ mod tests {
 
     use crate::directory::footer::Footer;
     use crate::directory::OwnedBytes;
-    use crate::{
-        common::BinarySerializable,
-        directory::{footer::FOOTER_MAGIC_NUMBER, FileSlice},
-    };
+    use crate::directory::{footer::FOOTER_MAGIC_NUMBER, FileSlice};
+    use common::BinarySerializable;
     use std::io;
 
     #[test]

src/directory/managed_directory.rs

@@ -1,4 +1,4 @@
-use crate::core::{MANAGED_FILEPATH, META_FILEPATH};
+use crate::core::MANAGED_FILEPATH;
 use crate::directory::error::{DeleteError, LockError, OpenReadError, OpenWriteError};
 use crate::directory::footer::{Footer, FooterProxy};
 use crate::directory::GarbageCollectionResult;
@@ -248,24 +248,15 @@ impl ManagedDirectory {
         Ok(footer.crc() == crc)
     }
 
-    /// List files for which checksum does not match content
-    pub fn list_damaged(&self) -> result::Result<HashSet<PathBuf>, OpenReadError> {
-        let mut managed_paths = self
+    /// List all managed files
+    pub fn list_managed_files(&self) -> HashSet<PathBuf> {
+        let managed_paths = self
             .meta_informations
             .read()
             .expect("Managed directory rlock poisoned in list damaged.")
             .managed_paths
             .clone();
-
-        managed_paths.remove(*META_FILEPATH);
-
-        let mut damaged_files = HashSet::new();
-        for path in managed_paths {
-            if !self.validate_checksum(&path)? {
-                damaged_files.insert(path);
-            }
-        }
-        Ok(damaged_files)
+        managed_paths
     }
 }
 
@@ -336,7 +327,6 @@ mod tests_mmap_specific {
 
     use crate::directory::{Directory, ManagedDirectory, MmapDirectory, TerminatingWrite};
     use std::collections::HashSet;
-    use std::fs::OpenOptions;
     use std::io::Write;
     use std::path::{Path, PathBuf};
     use tempfile::TempDir;
@@ -402,44 +392,7 @@ mod tests_mmap_specific {
             // The file should still be in the list of managed file and
             // eventually be deleted once mmap is released.
             assert!(managed_directory.garbage_collect(|| living_files).is_ok());
-            assert!(!managed_directory.exists(test_path1).unwrap());
-        } else {
-            assert!(!managed_directory.exists(test_path1).unwrap());
         }
-    }
-
-    #[test]
-    fn test_checksum() -> crate::Result<()> {
-        let test_path1: &'static Path = Path::new("some_path_for_test");
-        let test_path2: &'static Path = Path::new("other_test_path");
-
-        let tempdir = TempDir::new().unwrap();
-        let tempdir_path = PathBuf::from(tempdir.path());
-
-        let mmap_directory = MmapDirectory::open(&tempdir_path)?;
-        let managed_directory = ManagedDirectory::wrap(mmap_directory)?;
-        let mut write = managed_directory.open_write(test_path1)?;
-        write.write_all(&[0u8, 1u8])?;
-        write.terminate()?;
-
-        let mut write = managed_directory.open_write(test_path2)?;
-        write.write_all(&[3u8, 4u8, 5u8])?;
-        write.terminate()?;
-
-        let read_file = managed_directory.open_read(test_path2)?.read_bytes()?;
-        assert_eq!(read_file.as_slice(), &[3u8, 4u8, 5u8]);
-        assert!(managed_directory.list_damaged().unwrap().is_empty());
-
-        let mut corrupted_path = tempdir_path.clone();
-        corrupted_path.push(test_path2);
-        let mut file = OpenOptions::new().write(true).open(&corrupted_path)?;
-        file.write_all(&[255u8])?;
-        file.flush()?;
-        drop(file);
-
-        let damaged = managed_directory.list_damaged()?;
-        assert_eq!(damaged.len(), 1);
-        assert!(damaged.contains(test_path2));
-        Ok(())
+        assert!(!managed_directory.exists(test_path1).unwrap());
     }
 }

src/directory/mmap_directory.rs

@@ -11,7 +11,7 @@ use crate::directory::{AntiCallToken, FileHandle, OwnedBytes};
 use crate::directory::{ArcBytes, WeakArcBytes};
 use crate::directory::{TerminatingWrite, WritePtr};
 use fs2::FileExt;
-use memmap::Mmap;
+use memmap2::Mmap;
 use serde::{Deserialize, Serialize};
 use stable_deref_trait::StableDeref;
 use std::convert::From;
@@ -53,7 +53,7 @@ fn open_mmap(full_path: &Path) -> result::Result<Option<Mmap>, OpenReadError> {
         return Ok(None);
     }
     unsafe {
-        memmap::Mmap::map(&file)
+        memmap2::Mmap::map(&file)
             .map(Some)
             .map_err(|io_err| OpenReadError::wrap_io_error(io_err, full_path.to_path_buf()))
     }
@@ -485,13 +485,14 @@ mod tests {
     // The following tests are specific to the MmapDirectory
 
     use super::*;
+    use crate::indexer::LogMergePolicy;
     use crate::Index;
     use crate::ReloadPolicy;
-    use crate::{common::HasLen, indexer::LogMergePolicy};
     use crate::{
         schema::{Schema, SchemaBuilder, TEXT},
         IndexSettings,
     };
+    use common::HasLen;
 
     #[test]
     fn test_open_non_existent_path() {

src/directory/mod.rs

@@ -20,6 +20,9 @@ mod watch_event_router;
 /// Errors specific to the directory module.
 pub mod error;
 
+mod composite_file;
+
+pub(crate) use self::composite_file::{CompositeFile, CompositeWrite};
 pub use self::directory::DirectoryLock;
 pub use self::directory::{Directory, DirectoryClone};
 pub use self::directory_lock::{Lock, INDEX_WRITER_LOCK, META_LOCK};
@@ -28,7 +31,9 @@ pub use self::file_slice::{FileHandle, FileSlice};
 pub use self::owned_bytes::OwnedBytes;
 pub use self::ram_directory::RamDirectory;
 pub use self::watch_event_router::{WatchCallback, WatchCallbackList, WatchHandle};
-use std::io::{self, BufWriter, Write};
+pub use common::AntiCallToken;
+pub use common::TerminatingWrite;
+use std::io::BufWriter;
 use std::path::PathBuf;
 
 /// Outcome of the Garbage collection
@@ -50,47 +55,6 @@ pub use self::mmap_directory::MmapDirectory;
 
 pub use self::managed_directory::ManagedDirectory;
 
-/// Struct used to prevent from calling [`terminate_ref`](trait.TerminatingWrite#method.terminate_ref) directly
-///
-/// The point is that while the type is public, it cannot be built by anyone
-/// outside of this module.
-pub struct AntiCallToken(());
-
-/// Trait used to indicate when no more write need to be done on a writer
-pub trait TerminatingWrite: Write {
-    /// Indicate that the writer will no longer be used. Internally call terminate_ref.
-    fn terminate(mut self) -> io::Result<()>
-    where
-        Self: Sized,
-    {
-        self.terminate_ref(AntiCallToken(()))
-    }
-
-    /// You should implement this function to define custom behavior.
-    /// This function should flush any buffer it may hold.
-    fn terminate_ref(&mut self, _: AntiCallToken) -> io::Result<()>;
-}
-
-impl<W: TerminatingWrite + ?Sized> TerminatingWrite for Box<W> {
-    fn terminate_ref(&mut self, token: AntiCallToken) -> io::Result<()> {
-        self.as_mut().terminate_ref(token)
-    }
-}
-
-impl<W: TerminatingWrite> TerminatingWrite for BufWriter<W> {
-    fn terminate_ref(&mut self, a: AntiCallToken) -> io::Result<()> {
-        self.flush()?;
-        self.get_mut().terminate_ref(a)
-    }
-}
-
-#[cfg(test)]
-impl<'a> TerminatingWrite for &'a mut Vec<u8> {
-    fn terminate_ref(&mut self, _a: AntiCallToken) -> io::Result<()> {
-        self.flush()
-    }
-}
-
 /// Write object for Directory.
 ///
 /// `WritePtr` are required to implement both Write

src/directory/owned_bytes.rs

@@ -1,290 +1,11 @@
 use crate::directory::FileHandle;
-use stable_deref_trait::StableDeref;
-use std::convert::TryInto;
-use std::mem;
-use std::ops::{Deref, Range};
-use std::sync::Arc;
-use std::{fmt, io};
+use std::io;
+use std::ops::Range;
 
-/// An OwnedBytes simply wraps an object that owns a slice of data and exposes
-/// this data as a static slice.
-///
-/// The backing object is required to be `StableDeref`.
-#[derive(Clone)]
-pub struct OwnedBytes {
-    data: &'static [u8],
-    box_stable_deref: Arc<dyn Deref<Target = [u8]> + Sync + Send>,
-}
+pub use ownedbytes::OwnedBytes;
 
 impl FileHandle for OwnedBytes {
     fn read_bytes(&self, range: Range<usize>) -> io::Result<OwnedBytes> {
         Ok(self.slice(range))
     }
 }
-
-impl OwnedBytes {
-    /// Creates an empty `OwnedBytes`.
-    pub fn empty() -> OwnedBytes {
-        OwnedBytes::new(&[][..])
-    }
-
-    /// Creates an `OwnedBytes` intance given a `StableDeref` object.
-    pub fn new<T: StableDeref + Deref<Target = [u8]> + 'static + Send + Sync>(
-        data_holder: T,
-    ) -> OwnedBytes {
-        let box_stable_deref = Arc::new(data_holder);
-        let bytes: &[u8] = box_stable_deref.as_ref();
-        let data = unsafe { mem::transmute::<_, &'static [u8]>(bytes.deref()) };
-        OwnedBytes {
-            data,
-            box_stable_deref,
-        }
-    }
-
-    /// creates a fileslice that is just a view over a slice of the data.
-    pub fn slice(&self, range: Range<usize>) -> Self {
-        OwnedBytes {
-            data: &self.data[range],
-            box_stable_deref: self.box_stable_deref.clone(),
-        }
-    }
-
-    /// Returns the underlying slice of data.
-    /// `Deref` and `AsRef` are also available.
-    #[inline]
-    pub fn as_slice(&self) -> &[u8] {
-        self.data
-    }
-
-    /// Returns the len of the slice.
-    #[inline]
-    pub fn len(&self) -> usize {
-        self.data.len()
-    }
-
-    /// Splits the OwnedBytes into two OwnedBytes `(left, right)`.
-    ///
-    /// Left will hold `split_len` bytes.
-    ///
-    /// This operation is cheap and does not require to copy any memory.
-    /// On the other hand, both `left` and `right` retain a handle over
-    /// the entire slice of memory. In other words, the memory will only
-    /// be released when both left and right are dropped.
-    pub fn split(self, split_len: usize) -> (OwnedBytes, OwnedBytes) {
-        let right_box_stable_deref = self.box_stable_deref.clone();
-        let left = OwnedBytes {
-            data: &self.data[..split_len],
-            box_stable_deref: self.box_stable_deref,
-        };
-        let right = OwnedBytes {
-            data: &self.data[split_len..],
-            box_stable_deref: right_box_stable_deref,
-        };
-        (left, right)
-    }
-
-    /// Returns true iff this `OwnedBytes` is empty.
-    #[inline]
-    pub fn is_empty(&self) -> bool {
-        self.as_slice().is_empty()
-    }
-
-    /// Drops the left most `advance_len` bytes.
-    ///
-    /// See also [.clip(clip_len: usize))](#method.clip).
-    #[inline]
-    pub fn advance(&mut self, advance_len: usize) {
-        self.data = &self.data[advance_len..]
-    }
-
-    /// Reads an `u8` from the `OwnedBytes` and advance by one byte.
-    pub fn read_u8(&mut self) -> u8 {
-        assert!(!self.is_empty());
-
-        let byte = self.as_slice()[0];
-        self.advance(1);
-        byte
-    }
-
-    /// Reads an `u64` encoded as little-endian from the `OwnedBytes` and advance by 8 bytes.
-    pub fn read_u64(&mut self) -> u64 {
-        assert!(self.len() > 7);
-
-        let octlet: [u8; 8] = self.as_slice()[..8].try_into().unwrap();
-        self.advance(8);
-        u64::from_le_bytes(octlet)
-    }
-}
-
-impl fmt::Debug for OwnedBytes {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        // We truncate the bytes in order to make sure the debug string
-        // is not too long.
-        let bytes_truncated: &[u8] = if self.len() > 8 {
-            &self.as_slice()[..10]
-        } else {
-            self.as_slice()
-        };
-        write!(f, "OwnedBytes({:?}, len={})", bytes_truncated, self.len())
-    }
-}
-
-impl Deref for OwnedBytes {
-    type Target = [u8];
-
-    fn deref(&self) -> &Self::Target {
-        self.as_slice()
-    }
-}
-
-impl io::Read for OwnedBytes {
-    fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
-        let read_len = {
-            let data = self.as_slice();
-            if data.len() >= buf.len() {
-                let buf_len = buf.len();
-                buf.copy_from_slice(&data[..buf_len]);
-                buf.len()
-            } else {
-                let data_len = data.len();
-                buf[..data_len].copy_from_slice(data);
-                data_len
-            }
-        };
-        self.advance(read_len);
-        Ok(read_len)
-    }
-    fn read_to_end(&mut self, buf: &mut Vec<u8>) -> io::Result<usize> {
-        let read_len = {
-            let data = self.as_slice();
-            buf.extend(data);
-            data.len()
-        };
-        self.advance(read_len);
-        Ok(read_len)
-    }
-    fn read_exact(&mut self, buf: &mut [u8]) -> io::Result<()> {
-        let read_len = self.read(buf)?;
-        if read_len != buf.len() {
-            return Err(io::Error::new(
-                io::ErrorKind::UnexpectedEof,
-                "failed to fill whole buffer",
-            ));
-        }
-        Ok(())
-    }
-}
-
-impl AsRef<[u8]> for OwnedBytes {
-    fn as_ref(&self) -> &[u8] {
-        self.as_slice()
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use std::io::{self, Read};
-
-    use super::OwnedBytes;
-
-    #[test]
-    fn test_owned_bytes_debug() {
-        let short_bytes = OwnedBytes::new(b"abcd".as_ref());
-        assert_eq!(
-            format!("{:?}", short_bytes),
-            "OwnedBytes([97, 98, 99, 100], len=4)"
-        );
-        let long_bytes = OwnedBytes::new(b"abcdefghijklmnopq".as_ref());
-        assert_eq!(
-            format!("{:?}", long_bytes),
-            "OwnedBytes([97, 98, 99, 100, 101, 102, 103, 104, 105, 106], len=17)"
-        );
-    }
-
-    #[test]
-    fn test_owned_bytes_read() -> io::Result<()> {
-        let mut bytes = OwnedBytes::new(b"abcdefghiklmnopqrstuvwxyz".as_ref());
-        {
-            let mut buf = [0u8; 5];
-            bytes.read_exact(&mut buf[..]).unwrap();
-            assert_eq!(&buf, b"abcde");
-            assert_eq!(bytes.as_slice(), b"fghiklmnopqrstuvwxyz")
-        }
-        {
-            let mut buf = [0u8; 2];
-            bytes.read_exact(&mut buf[..]).unwrap();
-            assert_eq!(&buf, b"fg");
-            assert_eq!(bytes.as_slice(), b"hiklmnopqrstuvwxyz")
-        }
-        Ok(())
-    }
-
-    #[test]
-    fn test_owned_bytes_read_right_at_the_end() -> io::Result<()> {
-        let mut bytes = OwnedBytes::new(b"abcde".as_ref());
-        let mut buf = [0u8; 5];
-        assert_eq!(bytes.read(&mut buf[..]).unwrap(), 5);
-        assert_eq!(&buf, b"abcde");
-        assert_eq!(bytes.as_slice(), b"");
-        assert_eq!(bytes.read(&mut buf[..]).unwrap(), 0);
-        assert_eq!(&buf, b"abcde");
-        Ok(())
-    }
-    #[test]
-    fn test_owned_bytes_read_incomplete() -> io::Result<()> {
-        let mut bytes = OwnedBytes::new(b"abcde".as_ref());
-        let mut buf = [0u8; 7];
-        assert_eq!(bytes.read(&mut buf[..]).unwrap(), 5);
-        assert_eq!(&buf[..5], b"abcde");
-        assert_eq!(bytes.read(&mut buf[..]).unwrap(), 0);
-        Ok(())
-    }
-
-    #[test]
-    fn test_owned_bytes_read_to_end() -> io::Result<()> {
-        let mut bytes = OwnedBytes::new(b"abcde".as_ref());
-        let mut buf = Vec::new();
-        bytes.read_to_end(&mut buf)?;
-        assert_eq!(buf.as_slice(), b"abcde".as_ref());
-        Ok(())
-    }
-
-    #[test]
-    fn test_owned_bytes_read_u8() -> io::Result<()> {
-        let mut bytes = OwnedBytes::new(b"\xFF".as_ref());
-        assert_eq!(bytes.read_u8(), 255);
-        assert_eq!(bytes.len(), 0);
-        Ok(())
-    }
-
-    #[test]
-    fn test_owned_bytes_read_u64() -> io::Result<()> {
-        let mut bytes = OwnedBytes::new(b"\0\xFF\xFF\xFF\xFF\xFF\xFF\xFF".as_ref());
-        assert_eq!(bytes.read_u64(), u64::MAX - 255);
-        assert_eq!(bytes.len(), 0);
-        Ok(())
-    }
-
-    #[test]
-    fn test_owned_bytes_split() {
-        let bytes = OwnedBytes::new(b"abcdefghi".as_ref());
-        let (left, right) = bytes.split(3);
-        assert_eq!(left.as_slice(), b"abc");
-        assert_eq!(right.as_slice(), b"defghi");
-    }
-
-    #[test]
-    fn test_owned_bytes_split_boundary() {
-        let bytes = OwnedBytes::new(b"abcdefghi".as_ref());
-        {
-            let (left, right) = bytes.clone().split(0);
-            assert_eq!(left.as_slice(), b"");
-            assert_eq!(right.as_slice(), b"abcdefghi");
-        }
-        {
-            let (left, right) = bytes.split(9);
-            assert_eq!(left.as_slice(), b"abcdefghi");
-            assert_eq!(right.as_slice(), b"");
-        }
-    }
-}

src/directory/ram_directory.rs

@@ -1,9 +1,10 @@
+use crate::core::META_FILEPATH;
 use crate::directory::error::{DeleteError, OpenReadError, OpenWriteError};
 use crate::directory::AntiCallToken;
 use crate::directory::WatchCallbackList;
 use crate::directory::{Directory, FileSlice, WatchCallback, WatchHandle};
 use crate::directory::{TerminatingWrite, WritePtr};
-use crate::{common::HasLen, core::META_FILEPATH};
+use common::HasLen;
 use fail::fail_point;
 use std::collections::HashMap;
 use std::fmt;
@@ -46,7 +47,7 @@ impl Drop for VecWriter {
     fn drop(&mut self) {
         if !self.is_flushed {
             panic!(
-                "You forgot to flush {:?} before its writter got Drop. Do not rely on drop.",
+                "You forgot to flush {:?} before its writter got Drop. Do not rely on drop. This also occurs when the indexer crashed, so you may want to check the logs for the root cause.",
                 self.path
             )
         }

src/directory/tests.rs

@@ -166,26 +166,26 @@ fn test_write_create_the_file(directory: &dyn Directory) {
 fn test_directory_delete(directory: &dyn Directory) -> crate::Result<()> {
     let test_path: &'static Path = Path::new("some_path_for_test");
     assert!(directory.open_read(test_path).is_err());
-    let mut write_file = directory.open_write(&test_path)?;
+    let mut write_file = directory.open_write(test_path)?;
     write_file.write_all(&[1, 2, 3, 4])?;
     write_file.flush()?;
     {
-        let read_handle = directory.open_read(&test_path)?.read_bytes()?;
+        let read_handle = directory.open_read(test_path)?.read_bytes()?;
         assert_eq!(read_handle.as_slice(), &[1u8, 2u8, 3u8, 4u8]);
         // Mapped files can't be deleted on Windows
         if !cfg!(windows) {
-            assert!(directory.delete(&test_path).is_ok());
+            assert!(directory.delete(test_path).is_ok());
             assert_eq!(read_handle.as_slice(), &[1u8, 2u8, 3u8, 4u8]);
         }
         assert!(directory.delete(Path::new("SomeOtherPath")).is_err());
     }
 
     if cfg!(windows) {
-        assert!(directory.delete(&test_path).is_ok());
+        assert!(directory.delete(test_path).is_ok());
     }
 
-    assert!(directory.open_read(&test_path).is_err());
-    assert!(directory.delete(&test_path).is_err());
+    assert!(directory.open_read(test_path).is_err());
+    assert!(directory.delete(test_path).is_err());
     Ok(())
 }
 

src/docset.rs

@@ -1,4 +1,4 @@
-use crate::fastfield::DeleteBitSet;
+use crate::fastfield::AliveBitSet;
 use crate::DocId;
 use std::borrow::Borrow;
 use std::borrow::BorrowMut;
@@ -85,11 +85,11 @@ pub trait DocSet: Send {
 
     /// Returns the number documents matching.
     /// Calling this method consumes the `DocSet`.
-    fn count(&mut self, delete_bitset: &DeleteBitSet) -> u32 {
+    fn count(&mut self, alive_bitset: &AliveBitSet) -> u32 {
         let mut count = 0u32;
         let mut doc = self.doc();
         while doc != TERMINATED {
-            if !delete_bitset.is_deleted(doc) {
+            if alive_bitset.is_alive(doc) {
                 count += 1u32;
             }
             doc = self.advance();
@@ -130,8 +130,8 @@ impl<'a> DocSet for &'a mut dyn DocSet {
         (**self).size_hint()
     }
 
-    fn count(&mut self, delete_bitset: &DeleteBitSet) -> u32 {
-        (**self).count(delete_bitset)
+    fn count(&mut self, alive_bitset: &AliveBitSet) -> u32 {
+        (**self).count(alive_bitset)
     }
 
     fn count_including_deleted(&mut self) -> u32 {
@@ -160,9 +160,9 @@ impl<TDocSet: DocSet + ?Sized> DocSet for Box<TDocSet> {
         unboxed.size_hint()
     }
 
-    fn count(&mut self, delete_bitset: &DeleteBitSet) -> u32 {
+    fn count(&mut self, alive_bitset: &AliveBitSet) -> u32 {
         let unboxed: &mut TDocSet = self.borrow_mut();
-        unboxed.count(delete_bitset)
+        unboxed.count(alive_bitset)
     }
 
     fn count_including_deleted(&mut self) -> u32 {

src/fastfield/alive_bitset.rs

@@ -0,0 +1,202 @@
+use crate::space_usage::ByteCount;
+use crate::DocId;
+use common::BitSet;
+use common::ReadSerializedBitSet;
+use ownedbytes::OwnedBytes;
+use std::io;
+use std::io::Write;
+
+/// Write a alive `BitSet`
+///
+/// where `alive_bitset` is the set of alive `DocId`.
+/// Warning: this function does not call terminate. The caller is in charge of
+/// closing the writer properly.
+pub fn write_alive_bitset<T: Write>(alive_bitset: &BitSet, writer: &mut T) -> io::Result<()> {
+    alive_bitset.serialize(writer)?;
+    Ok(())
+}
+
+/// Set of alive `DocId`s.
+#[derive(Clone)]
+pub struct AliveBitSet {
+    num_alive_docs: usize,
+    bitset: ReadSerializedBitSet,
+    num_bytes: ByteCount,
+}
+
+impl AliveBitSet {
+    #[cfg(test)]
+    pub(crate) fn for_test_from_deleted_docs(deleted_docs: &[DocId], max_doc: u32) -> AliveBitSet {
+        assert!(deleted_docs.iter().all(|&doc| doc < max_doc));
+        let mut bitset = BitSet::with_max_value_and_full(max_doc);
+        for &doc in deleted_docs {
+            bitset.remove(doc);
+        }
+        let mut alive_bitset_buffer = Vec::new();
+        write_alive_bitset(&bitset, &mut alive_bitset_buffer).unwrap();
+        let alive_bitset_bytes = OwnedBytes::new(alive_bitset_buffer);
+        Self::open(alive_bitset_bytes)
+    }
+
+    /// Opens a delete bitset given its file.
+    pub fn open(bytes: OwnedBytes) -> AliveBitSet {
+        let num_bytes = bytes.len();
+        let bitset = ReadSerializedBitSet::open(bytes);
+        AliveBitSet {
+            num_alive_docs: bitset.len(),
+            bitset,
+            num_bytes,
+        }
+    }
+
+    /// Returns true iff the document is still "alive". In other words, if it has not been deleted.
+    #[inline]
+    pub fn is_alive(&self, doc: DocId) -> bool {
+        self.bitset.contains(doc)
+    }
+
+    /// Returns true iff the document has been marked as deleted.
+    #[inline]
+    pub fn is_deleted(&self, doc: DocId) -> bool {
+        !self.is_alive(doc)
+    }
+
+    /// Iterate over the alive docids.
+    #[inline]
+    pub fn iter_alive(&self) -> impl Iterator<Item = DocId> + '_ {
+        self.bitset.iter()
+    }
+
+    /// Get underlying bitset
+    #[inline]
+    pub fn bitset(&self) -> &ReadSerializedBitSet {
+        &self.bitset
+    }
+
+    /// The number of deleted docs
+    pub fn num_alive_docs(&self) -> usize {
+        self.num_alive_docs
+    }
+
+    /// Summarize total space usage of this bitset.
+    pub fn space_usage(&self) -> ByteCount {
+        self.num_bytes
+    }
+}
+
+#[cfg(test)]
+mod tests {
+
+    use super::AliveBitSet;
+
+    #[test]
+    fn test_alive_bitset_empty() {
+        let alive_bitset = AliveBitSet::for_test_from_deleted_docs(&[], 10);
+        for doc in 0..10 {
+            assert_eq!(alive_bitset.is_deleted(doc), !alive_bitset.is_alive(doc));
+            assert!(!alive_bitset.is_deleted(doc));
+        }
+        assert_eq!(alive_bitset.num_alive_docs(), 10);
+    }
+
+    #[test]
+    fn test_alive_bitset() {
+        let alive_bitset = AliveBitSet::for_test_from_deleted_docs(&[1, 9], 10);
+        assert!(alive_bitset.is_alive(0));
+        assert!(alive_bitset.is_deleted(1));
+        assert!(alive_bitset.is_alive(2));
+        assert!(alive_bitset.is_alive(3));
+        assert!(alive_bitset.is_alive(4));
+        assert!(alive_bitset.is_alive(5));
+        assert!(alive_bitset.is_alive(6));
+        assert!(alive_bitset.is_alive(6));
+        assert!(alive_bitset.is_alive(7));
+        assert!(alive_bitset.is_alive(8));
+        assert!(alive_bitset.is_deleted(9));
+        for doc in 0..10 {
+            assert_eq!(alive_bitset.is_deleted(doc), !alive_bitset.is_alive(doc));
+        }
+        assert_eq!(alive_bitset.num_alive_docs(), 8);
+    }
+
+    #[test]
+    fn test_alive_bitset_iter_minimal() {
+        let alive_bitset = AliveBitSet::for_test_from_deleted_docs(&[7], 8);
+
+        let data: Vec<_> = alive_bitset.iter_alive().collect();
+        assert_eq!(data, vec![0, 1, 2, 3, 4, 5, 6]);
+    }
+
+    #[test]
+    fn test_alive_bitset_iter_small() {
+        let alive_bitset = AliveBitSet::for_test_from_deleted_docs(&[0, 2, 3, 6], 7);
+
+        let data: Vec<_> = alive_bitset.iter_alive().collect();
+        assert_eq!(data, vec![1, 4, 5]);
+    }
+    #[test]
+    fn test_alive_bitset_iter() {
+        let alive_bitset = AliveBitSet::for_test_from_deleted_docs(&[0, 1, 1000], 1001);
+
+        let data: Vec<_> = alive_bitset.iter_alive().collect();
+        assert_eq!(data, (2..=999).collect::<Vec<_>>());
+    }
+}
+
+#[cfg(all(test, feature = "unstable"))]
+mod bench {
+
+    use super::AliveBitSet;
+    use rand::prelude::IteratorRandom;
+    use rand::thread_rng;
+    use test::Bencher;
+
+    fn get_alive() -> Vec<u32> {
+        let mut data = (0..1_000_000_u32).collect::<Vec<u32>>();
+        for _ in 0..(1_000_000) * 1 / 8 {
+            remove_rand(&mut data);
+        }
+        data
+    }
+
+    fn remove_rand(raw: &mut Vec<u32>) {
+        let i = (0..raw.len()).choose(&mut thread_rng()).unwrap();
+        raw.remove(i);
+    }
+
+    #[bench]
+    fn bench_deletebitset_iter_deser_on_fly(bench: &mut Bencher) {
+        let alive_bitset = AliveBitSet::for_test_from_deleted_docs(&[0, 1, 1000, 10000], 1_000_000);
+
+        bench.iter(|| alive_bitset.iter_alive().collect::<Vec<_>>());
+    }
+
+    #[bench]
+    fn bench_deletebitset_access(bench: &mut Bencher) {
+        let alive_bitset = AliveBitSet::for_test_from_deleted_docs(&[0, 1, 1000, 10000], 1_000_000);
+
+        bench.iter(|| {
+            (0..1_000_000_u32)
+                .filter(|doc| alive_bitset.is_alive(*doc))
+                .collect::<Vec<_>>()
+        });
+    }
+
+    #[bench]
+    fn bench_deletebitset_iter_deser_on_fly_1_8_alive(bench: &mut Bencher) {
+        let alive_bitset = AliveBitSet::for_test_from_deleted_docs(&get_alive(), 1_000_000);
+
+        bench.iter(|| alive_bitset.iter_alive().collect::<Vec<_>>());
+    }
+
+    #[bench]
+    fn bench_deletebitset_access_1_8_alive(bench: &mut Bencher) {
+        let alive_bitset = AliveBitSet::for_test_from_deleted_docs(&get_alive(), 1_000_000);
+
+        bench.iter(|| {
+            (0..1_000_000_u32)
+                .filter(|doc| alive_bitset.is_alive(*doc))
+                .collect::<Vec<_>>()
+        });
+    }
+}

src/fastfield/bytes/writer.rs

@@ -1,6 +1,5 @@
 use std::io;
 
-use crate::fastfield::serializer::FastFieldSerializer;
 use crate::schema::{Document, Field, Value};
 use crate::DocId;
 use crate::{
@@ -84,11 +83,11 @@ impl BytesFastFieldWriter {
         &'a self,
         doc_id_map: Option<&'b DocIdMapping>,
     ) -> impl Iterator<Item = &'b [u8]> {
-        let doc_id_iter = if let Some(doc_id_map) = doc_id_map {
-            Box::new(doc_id_map.iter_old_doc_ids().cloned()) as Box<dyn Iterator<Item = u32>>
+        let doc_id_iter: Box<dyn Iterator<Item = u32>> = if let Some(doc_id_map) = doc_id_map {
+            Box::new(doc_id_map.iter_old_doc_ids())
         } else {
-            Box::new(self.doc_index.iter().enumerate().map(|el| el.0 as u32))
-                as Box<dyn Iterator<Item = u32>>
+            let max_doc = self.doc_index.len() as u32;
+            Box::new(0..max_doc)
         };
         doc_id_iter.map(move |doc_id| self.get_values_for_doc_id(doc_id))
     }

src/fastfield/delete.rs

@@ -1,139 +0,0 @@
-use crate::common::{BitSet, HasLen};
-use crate::directory::FileSlice;
-use crate::directory::OwnedBytes;
-use crate::directory::WritePtr;
-use crate::space_usage::ByteCount;
-use crate::DocId;
-use std::io;
-use std::io::Write;
-
-/// Write a delete `BitSet`
-///
-/// where `delete_bitset` is the set of deleted `DocId`.
-/// Warning: this function does not call terminate. The caller is in charge of
-/// closing the writer properly.
-pub fn write_delete_bitset(
-    delete_bitset: &BitSet,
-    max_doc: u32,
-    writer: &mut WritePtr,
-) -> io::Result<()> {
-    let mut byte = 0u8;
-    let mut shift = 0u8;
-    for doc in 0..max_doc {
-        if delete_bitset.contains(doc) {
-            byte |= 1 << shift;
-        }
-        if shift == 7 {
-            writer.write_all(&[byte])?;
-            shift = 0;
-            byte = 0;
-        } else {
-            shift += 1;
-        }
-    }
-    if max_doc % 8 > 0 {
-        writer.write_all(&[byte])?;
-    }
-    Ok(())
-}
-
-/// Set of deleted `DocId`s.
-#[derive(Clone)]
-pub struct DeleteBitSet {
-    data: OwnedBytes,
-    num_deleted: usize,
-}
-
-impl DeleteBitSet {
-    #[cfg(test)]
-    pub(crate) fn for_test(docs: &[DocId], max_doc: u32) -> DeleteBitSet {
-        use crate::directory::{Directory, RamDirectory, TerminatingWrite};
-        use std::path::Path;
-        assert!(docs.iter().all(|&doc| doc < max_doc));
-        let mut bitset = BitSet::with_max_value(max_doc);
-        for &doc in docs {
-            bitset.insert(doc);
-        }
-        let directory = RamDirectory::create();
-        let path = Path::new("dummydeletebitset");
-        let mut wrt = directory.open_write(path).unwrap();
-        write_delete_bitset(&bitset, max_doc, &mut wrt).unwrap();
-        wrt.terminate().unwrap();
-        let file = directory.open_read(path).unwrap();
-        Self::open(file).unwrap()
-    }
-
-    /// Opens a delete bitset given its file.
-    pub fn open(file: FileSlice) -> crate::Result<DeleteBitSet> {
-        let bytes = file.read_bytes()?;
-        let num_deleted: usize = bytes
-            .as_slice()
-            .iter()
-            .map(|b| b.count_ones() as usize)
-            .sum();
-        Ok(DeleteBitSet {
-            data: bytes,
-            num_deleted,
-        })
-    }
-
-    /// Returns true iff the document is still "alive". In other words, if it has not been deleted.
-    pub fn is_alive(&self, doc: DocId) -> bool {
-        !self.is_deleted(doc)
-    }
-
-    /// Returns true iff the document has been marked as deleted.
-    #[inline]
-    pub fn is_deleted(&self, doc: DocId) -> bool {
-        let byte_offset = doc / 8u32;
-        let b: u8 = self.data.as_slice()[byte_offset as usize];
-        let shift = (doc & 7u32) as u8;
-        b & (1u8 << shift) != 0
-    }
-
-    /// Summarize total space usage of this bitset.
-    pub fn space_usage(&self) -> ByteCount {
-        self.data.len()
-    }
-}
-
-impl HasLen for DeleteBitSet {
-    fn len(&self) -> usize {
-        self.num_deleted
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::DeleteBitSet;
-    use crate::common::HasLen;
-
-    #[test]
-    fn test_delete_bitset_empty() {
-        let delete_bitset = DeleteBitSet::for_test(&[], 10);
-        for doc in 0..10 {
-            assert_eq!(delete_bitset.is_deleted(doc), !delete_bitset.is_alive(doc));
-        }
-        assert_eq!(delete_bitset.len(), 0);
-    }
-
-    #[test]
-    fn test_delete_bitset() {
-        let delete_bitset = DeleteBitSet::for_test(&[1, 9], 10);
-        assert!(delete_bitset.is_alive(0));
-        assert!(delete_bitset.is_deleted(1));
-        assert!(delete_bitset.is_alive(2));
-        assert!(delete_bitset.is_alive(3));
-        assert!(delete_bitset.is_alive(4));
-        assert!(delete_bitset.is_alive(5));
-        assert!(delete_bitset.is_alive(6));
-        assert!(delete_bitset.is_alive(6));
-        assert!(delete_bitset.is_alive(7));
-        assert!(delete_bitset.is_alive(8));
-        assert!(delete_bitset.is_deleted(9));
-        for doc in 0..10 {
-            assert_eq!(delete_bitset.is_deleted(doc), !delete_bitset.is_alive(doc));
-        }
-        assert_eq!(delete_bitset.len(), 2);
-    }
-}

src/fastfield/mod.rs

@@ -23,30 +23,31 @@ values stored.
 Read access performance is comparable to that of an array lookup.
 */
 
+pub use self::alive_bitset::write_alive_bitset;
+pub use self::alive_bitset::AliveBitSet;
 pub use self::bytes::{BytesFastFieldReader, BytesFastFieldWriter};
-pub use self::delete::write_delete_bitset;
-pub use self::delete::DeleteBitSet;
 pub use self::error::{FastFieldNotAvailableError, Result};
 pub use self::facet_reader::FacetReader;
 pub use self::multivalued::{MultiValuedFastFieldReader, MultiValuedFastFieldWriter};
-pub use self::reader::BitpackedFastFieldReader;
+pub(crate) use self::reader::BitpackedFastFieldReader;
 pub use self::reader::DynamicFastFieldReader;
 pub use self::reader::FastFieldReader;
 pub use self::readers::FastFieldReaders;
 pub use self::serializer::CompositeFastFieldSerializer;
-pub use self::serializer::FastFieldSerializer;
+pub use self::serializer::FastFieldDataAccess;
+pub use self::serializer::FastFieldStats;
 pub use self::writer::{FastFieldsWriter, IntFastFieldWriter};
 use crate::schema::Cardinality;
 use crate::schema::FieldType;
 use crate::schema::Value;
+use crate::DocId;
 use crate::{
     chrono::{NaiveDateTime, Utc},
     schema::Type,
 };
-use crate::{common, DocId};
 
+mod alive_bitset;
 mod bytes;
-mod delete;
 mod error;
 mod facet_reader;
 mod multivalued;
@@ -212,9 +213,8 @@ fn value_to_u64(value: &Value) -> u64 {
 mod tests {
 
     use super::*;
-    use crate::common::CompositeFile;
+    use crate::directory::CompositeFile;
     use crate::directory::{Directory, RamDirectory, WritePtr};
-    use crate::fastfield::BitpackedFastFieldReader;
     use crate::merge_policy::NoMergePolicy;
     use crate::schema::Field;
     use crate::schema::Schema;
@@ -239,7 +239,7 @@ mod tests {
 
     #[test]
     pub fn test_fastfield() {
-        let test_fastfield = BitpackedFastFieldReader::<u64>::from(vec![100, 200, 300]);
+        let test_fastfield = DynamicFastFieldReader::<u64>::from(vec![100, 200, 300]);
         assert_eq!(test_fastfield.get(0), 100);
         assert_eq!(test_fastfield.get(1), 200);
         assert_eq!(test_fastfield.get(2), 300);
@@ -267,11 +267,11 @@ mod tests {
                 .unwrap();
             serializer.close().unwrap();
         }
-        let file = directory.open_read(&path).unwrap();
-        assert_eq!(file.len(), 36 as usize);
+        let file = directory.open_read(path).unwrap();
+        assert_eq!(file.len(), 37);
         let composite_file = CompositeFile::open(&file)?;
         let file = composite_file.open_read(*FIELD).unwrap();
-        let fast_field_reader = BitpackedFastFieldReader::<u64>::open(file)?;
+        let fast_field_reader = DynamicFastFieldReader::<u64>::open(file)?;
         assert_eq!(fast_field_reader.get(0), 13u64);
         assert_eq!(fast_field_reader.get(1), 14u64);
         assert_eq!(fast_field_reader.get(2), 2u64);
@@ -298,12 +298,12 @@ mod tests {
             fast_field_writers.serialize(&mut serializer, &HashMap::new(), None)?;
             serializer.close()?;
         }
-        let file = directory.open_read(&path)?;
-        assert_eq!(file.len(), 61 as usize);
+        let file = directory.open_read(path)?;
+        assert_eq!(file.len(), 62);
         {
             let fast_fields_composite = CompositeFile::open(&file)?;
             let data = fast_fields_composite.open_read(*FIELD).unwrap();
-            let fast_field_reader = BitpackedFastFieldReader::<u64>::open(data)?;
+            let fast_field_reader = DynamicFastFieldReader::<u64>::open(data)?;
             assert_eq!(fast_field_reader.get(0), 4u64);
             assert_eq!(fast_field_reader.get(1), 14_082_001u64);
             assert_eq!(fast_field_reader.get(2), 3_052u64);
@@ -334,12 +334,12 @@ mod tests {
                 .unwrap();
             serializer.close().unwrap();
         }
-        let file = directory.open_read(&path).unwrap();
-        assert_eq!(file.len(), 34 as usize);
+        let file = directory.open_read(path).unwrap();
+        assert_eq!(file.len(), 35);
         {
             let fast_fields_composite = CompositeFile::open(&file).unwrap();
             let data = fast_fields_composite.open_read(*FIELD).unwrap();
-            let fast_field_reader = BitpackedFastFieldReader::<u64>::open(data)?;
+            let fast_field_reader = DynamicFastFieldReader::<u64>::open(data)?;
             for doc in 0..10_000 {
                 assert_eq!(fast_field_reader.get(doc), 100_000u64);
             }
@@ -366,12 +366,12 @@ mod tests {
                 .unwrap();
             serializer.close().unwrap();
         }
-        let file = directory.open_read(&path).unwrap();
-        assert_eq!(file.len(), 80042 as usize);
+        let file = directory.open_read(path).unwrap();
+        assert_eq!(file.len(), 80043);
         {
             let fast_fields_composite = CompositeFile::open(&file)?;
             let data = fast_fields_composite.open_read(*FIELD).unwrap();
-            let fast_field_reader = BitpackedFastFieldReader::<u64>::open(data)?;
+            let fast_field_reader = DynamicFastFieldReader::<u64>::open(data)?;
             assert_eq!(fast_field_reader.get(0), 0u64);
             for doc in 1..10_001 {
                 assert_eq!(
@@ -405,12 +405,13 @@ mod tests {
                 .unwrap();
             serializer.close().unwrap();
         }
-        let file = directory.open_read(&path).unwrap();
-        assert_eq!(file.len(), 17709 as usize);
+        let file = directory.open_read(path).unwrap();
+        //assert_eq!(file.len(), 17710 as usize); //bitpacked size
+        assert_eq!(file.len(), 10175_usize); // linear interpol size
         {
             let fast_fields_composite = CompositeFile::open(&file)?;
             let data = fast_fields_composite.open_read(i64_field).unwrap();
-            let fast_field_reader = BitpackedFastFieldReader::<i64>::open(data)?;
+            let fast_field_reader = DynamicFastFieldReader::<i64>::open(data)?;
 
             assert_eq!(fast_field_reader.min_value(), -100i64);
             assert_eq!(fast_field_reader.max_value(), 9_999i64);
@@ -446,11 +447,11 @@ mod tests {
             serializer.close().unwrap();
         }
 
-        let file = directory.open_read(&path).unwrap();
+        let file = directory.open_read(path).unwrap();
         {
             let fast_fields_composite = CompositeFile::open(&file).unwrap();
             let data = fast_fields_composite.open_read(i64_field).unwrap();
-            let fast_field_reader = BitpackedFastFieldReader::<i64>::open(data)?;
+            let fast_field_reader = DynamicFastFieldReader::<i64>::open(data)?;
             assert_eq!(fast_field_reader.get(0u32), 0i64);
         }
         Ok(())
@@ -479,11 +480,11 @@ mod tests {
             fast_field_writers.serialize(&mut serializer, &HashMap::new(), None)?;
             serializer.close()?;
         }
-        let file = directory.open_read(&path)?;
+        let file = directory.open_read(path)?;
         {
             let fast_fields_composite = CompositeFile::open(&file)?;
             let data = fast_fields_composite.open_read(*FIELD).unwrap();
-            let fast_field_reader = BitpackedFastFieldReader::<u64>::open(data)?;
+            let fast_field_reader = DynamicFastFieldReader::<u64>::open(data)?;
 
             let mut a = 0u64;
             for _ in 0..n {
@@ -587,7 +588,7 @@ mod bench {
     use super::tests::FIELD;
     use super::tests::{generate_permutation, SCHEMA};
     use super::*;
-    use crate::common::CompositeFile;
+    use crate::directory::CompositeFile;
     use crate::directory::{Directory, RamDirectory, WritePtr};
     use crate::fastfield::FastFieldReader;
     use std::collections::HashMap;
@@ -627,7 +628,7 @@ mod bench {
         let directory: RamDirectory = RamDirectory::create();
         {
             let write: WritePtr = directory.open_write(Path::new("test")).unwrap();
-            let mut serializer = FastFieldSerializer::from_write(write).unwrap();
+            let mut serializer = CompositeFastFieldSerializer::from_write(write).unwrap();
             let mut fast_field_writers = FastFieldsWriter::from_schema(&SCHEMA);
             for &x in &permutation {
                 fast_field_writers.add_document(&doc!(*FIELD=>x));
@@ -641,7 +642,7 @@ mod bench {
         {
             let fast_fields_composite = CompositeFile::open(&file).unwrap();
             let data = fast_fields_composite.open_read(*FIELD).unwrap();
-            let fast_field_reader = FastFieldReader::<u64>::open(data).unwrap();
+            let fast_field_reader = DynamicFastFieldReader::<u64>::open(data).unwrap();
 
             b.iter(|| {
                 let n = test::black_box(7000u32);
@@ -661,7 +662,7 @@ mod bench {
         let directory: RamDirectory = RamDirectory::create();
         {
             let write: WritePtr = directory.open_write(Path::new("test")).unwrap();
-            let mut serializer = FastFieldSerializer::from_write(write).unwrap();
+            let mut serializer = CompositeFastFieldSerializer::from_write(write).unwrap();
             let mut fast_field_writers = FastFieldsWriter::from_schema(&SCHEMA);
             for &x in &permutation {
                 fast_field_writers.add_document(&doc!(*FIELD=>x));
@@ -675,7 +676,7 @@ mod bench {
         {
             let fast_fields_composite = CompositeFile::open(&file).unwrap();
             let data = fast_fields_composite.open_read(*FIELD).unwrap();
-            let fast_field_reader = FastFieldReader::<u64>::open(data).unwrap();
+            let fast_field_reader = DynamicFastFieldReader::<u64>::open(data).unwrap();
 
             b.iter(|| {
                 let n = test::black_box(1000u32);

src/fastfield/multivalued/mod.rs

@@ -8,14 +8,22 @@ pub use self::writer::MultiValuedFastFieldWriter;
 mod tests {
 
     use crate::collector::TopDocs;
+    use crate::indexer::NoMergePolicy;
     use crate::query::QueryParser;
     use crate::schema::Cardinality;
     use crate::schema::Facet;
     use crate::schema::IntOptions;
     use crate::schema::Schema;
     use crate::schema::INDEXED;
+    use crate::Document;
     use crate::Index;
+    use crate::Term;
     use chrono::Duration;
+    use futures::executor::block_on;
+    use proptest::prop_oneof;
+    use proptest::proptest;
+    use proptest::strategy::Strategy;
+    use test_env_log::test;
 
     #[test]
     fn test_multivalued_u64() {
@@ -90,7 +98,7 @@ mod tests {
         {
             let parser = QueryParser::for_index(&index, vec![date_field]);
             let query = parser
-                .parse_query(&format!("\"{}\"", first_time_stamp.to_rfc3339()).to_string())
+                .parse_query(&format!("\"{}\"", first_time_stamp.to_rfc3339()))
                 .expect("could not parse query");
             let results = searcher
                 .search(&query, &TopDocs::with_limit(5))
@@ -121,7 +129,7 @@ mod tests {
         {
             let parser = QueryParser::for_index(&index, vec![date_field]);
             let query = parser
-                .parse_query(&format!("\"{}\"", two_secs_ahead.to_rfc3339()).to_string())
+                .parse_query(&format!("\"{}\"", two_secs_ahead.to_rfc3339()))
                 .expect("could not parse query");
             let results = searcher
                 .search(&query, &TopDocs::with_limit(5))
@@ -225,6 +233,111 @@ mod tests {
         multi_value_reader.get_vals(3, &mut vals);
         assert_eq!(&vals, &[-5i64, -20i64, 1i64]);
     }
+
+    fn test_multivalued_no_panic(ops: &[IndexingOp]) {
+        let mut schema_builder = Schema::builder();
+        let field = schema_builder.add_u64_field(
+            "multifield",
+            IntOptions::default()
+                .set_fast(Cardinality::MultiValues)
+                .set_indexed(),
+        );
+        let schema = schema_builder.build();
+        let index = Index::create_in_ram(schema);
+        let mut index_writer = index.writer_for_tests().unwrap();
+        index_writer.set_merge_policy(Box::new(NoMergePolicy));
+
+        for &op in ops {
+            match op {
+                IndexingOp::AddDoc { id } => {
+                    match id % 3 {
+                        0 => {
+                            index_writer.add_document(doc!());
+                        }
+                        1 => {
+                            let mut doc = Document::new();
+                            for _ in 0..5001 {
+                                doc.add_u64(field, id as u64);
+                            }
+                            index_writer.add_document(doc);
+                        }
+                        _ => {
+                            let mut doc = Document::new();
+                            doc.add_u64(field, id as u64);
+                            index_writer.add_document(doc);
+                        }
+                    };
+                }
+                IndexingOp::DeleteDoc { id } => {
+                    index_writer.delete_term(Term::from_field_u64(field, id as u64));
+                }
+                IndexingOp::Commit => {
+                    index_writer.commit().unwrap();
+                }
+                IndexingOp::Merge => {
+                    let segment_ids = index
+                        .searchable_segment_ids()
+                        .expect("Searchable segments failed.");
+                    if segment_ids.len() >= 2 {
+                        block_on(index_writer.merge(&segment_ids)).unwrap();
+                        assert!(index_writer.segment_updater().wait_merging_thread().is_ok());
+                    }
+                }
+            }
+        }
+
+        assert!(index_writer.commit().is_ok());
+
+        // Merging the segments
+        {
+            let segment_ids = index
+                .searchable_segment_ids()
+                .expect("Searchable segments failed.");
+            if !segment_ids.is_empty() {
+                block_on(index_writer.merge(&segment_ids)).unwrap();
+                assert!(index_writer.wait_merging_threads().is_ok());
+            }
+        }
+    }
+
+    #[derive(Debug, Clone, Copy)]
+    enum IndexingOp {
+        AddDoc { id: u32 },
+        DeleteDoc { id: u32 },
+        Commit,
+        Merge,
+    }
+
+    fn operation_strategy() -> impl Strategy<Value = IndexingOp> {
+        prop_oneof![
+            (0u32..10u32).prop_map(|id| IndexingOp::DeleteDoc { id }),
+            (0u32..10u32).prop_map(|id| IndexingOp::AddDoc { id }),
+            (0u32..2u32).prop_map(|_| IndexingOp::Commit),
+            (0u32..1u32).prop_map(|_| IndexingOp::Merge),
+        ]
+    }
+
+    proptest! {
+        #[test]
+        fn test_multivalued_proptest(ops in proptest::collection::vec(operation_strategy(), 1..10)) {
+            test_multivalued_no_panic(&ops[..]);
+        }
+    }
+
+    #[test]
+    fn test_multivalued_proptest_off_by_one_bug_1151() {
+        use IndexingOp::*;
+        let ops = [
+            AddDoc { id: 3 },
+            AddDoc { id: 1 },
+            AddDoc { id: 3 },
+            Commit,
+            Merge,
+        ];
+
+        test_multivalued_no_panic(&ops[..]);
+    }
+
     #[test]
     #[ignore]
     fn test_many_facets() {

src/fastfield/multivalued/reader.rs

@@ -1,6 +1,6 @@
 use std::ops::Range;
 
-use crate::fastfield::{BitpackedFastFieldReader, FastFieldReader, FastValue, MultiValueLength};
+use crate::fastfield::{DynamicFastFieldReader, FastFieldReader, FastValue, MultiValueLength};
 use crate::DocId;
 
 /// Reader for a multivalued `u64` fast field.
@@ -13,14 +13,14 @@ use crate::DocId;
 ///
 #[derive(Clone)]
 pub struct MultiValuedFastFieldReader<Item: FastValue> {
-    idx_reader: BitpackedFastFieldReader<u64>,
-    vals_reader: BitpackedFastFieldReader<Item>,
+    idx_reader: DynamicFastFieldReader<u64>,
+    vals_reader: DynamicFastFieldReader<Item>,
 }
 
 impl<Item: FastValue> MultiValuedFastFieldReader<Item> {
     pub(crate) fn open(
-        idx_reader: BitpackedFastFieldReader<u64>,
-        vals_reader: BitpackedFastFieldReader<Item>,
+        idx_reader: DynamicFastFieldReader<u64>,
+        vals_reader: DynamicFastFieldReader<Item>,
     ) -> MultiValuedFastFieldReader<Item> {
         MultiValuedFastFieldReader {
             idx_reader,
@@ -30,6 +30,7 @@ impl<Item: FastValue> MultiValuedFastFieldReader<Item> {
 
     /// Returns `(start, stop)`, such that the values associated
     /// to the given document are `start..stop`.
+    #[inline]
     fn range(&self, doc: DocId) -> Range<u64> {
         let start = self.idx_reader.get(doc);
         let stop = self.idx_reader.get(doc + 1);
@@ -37,20 +38,41 @@ impl<Item: FastValue> MultiValuedFastFieldReader<Item> {
     }
 
     /// Returns the array of values associated to the given `doc`.
+    #[inline]
     pub fn get_vals(&self, doc: DocId, vals: &mut Vec<Item>) {
         let range = self.range(doc);
         let len = (range.end - range.start) as usize;
         vals.resize(len, Item::make_zero());
-        self.vals_reader.get_range_u64(range.start, &mut vals[..]);
+        self.vals_reader.get_range(range.start, &mut vals[..]);
+    }
+
+    /// Returns the minimum value for this fast field.
+    ///
+    /// The min value does not take in account of possible
+    /// deleted document, and should be considered as a lower bound
+    /// of the actual mimimum value.
+    pub fn min_value(&self) -> Item {
+        self.vals_reader.min_value()
+    }
+
+    /// Returns the maximum value for this fast field.
+    ///
+    /// The max value does not take in account of possible
+    /// deleted document, and should be considered as an upper bound
+    /// of the actual maximum value.
+    pub fn max_value(&self) -> Item {
+        self.vals_reader.max_value()
     }
 
     /// Returns the number of values associated with the document `DocId`.
+    #[inline]
     pub fn num_vals(&self, doc: DocId) -> usize {
         let range = self.range(doc);
         (range.end - range.start) as usize
     }
 
     /// Returns the overall number of values in this field  .
+    #[inline]
     pub fn total_num_vals(&self) -> u64 {
         self.idx_reader.max_value()
     }
@@ -69,7 +91,7 @@ impl<Item: FastValue> MultiValueLength for MultiValuedFastFieldReader<Item> {
 mod tests {
 
     use crate::core::Index;
-    use crate::schema::{Facet, Schema, INDEXED};
+    use crate::schema::{Cardinality, Facet, IntOptions, Schema, INDEXED};
 
     #[test]
     fn test_multifastfield_reader() {
@@ -124,4 +146,32 @@ mod tests {
             assert_eq!(&vals[..], &[4]);
         }
     }
+
+    #[test]
+    fn test_multifastfield_reader_min_max() {
+        let mut schema_builder = Schema::builder();
+        let field_options = IntOptions::default()
+            .set_indexed()
+            .set_fast(Cardinality::MultiValues);
+        let item_field = schema_builder.add_i64_field("items", field_options);
+        let schema = schema_builder.build();
+        let index = Index::create_in_ram(schema);
+        let mut index_writer = index
+            .writer_for_tests()
+            .expect("Failed to create index writer.");
+        index_writer.add_document(doc!(
+            item_field => 2i64,
+            item_field => 3i64,
+            item_field => -2i64,
+        ));
+        index_writer.add_document(doc!(item_field => 6i64, item_field => 3i64));
+        index_writer.add_document(doc!(item_field => 4i64));
+        index_writer.commit().expect("Commit failed");
+        let searcher = index.reader().unwrap().searcher();
+        let segment_reader = searcher.segment_reader(0);
+        let field_reader = segment_reader.fast_fields().i64s(item_field).unwrap();
+
+        assert_eq!(field_reader.min_value(), -2);
+        assert_eq!(field_reader.max_value(), 6);
+    }
 }

src/fastfield/multivalued/writer.rs

@@ -1,5 +1,4 @@
-use crate::fastfield::serializer::DynamicFastFieldSerializer;
-use crate::fastfield::serializer::FastFieldSerializer;
+use crate::fastfield::serializer::BitpackedFastFieldSerializerLegacy;
 use crate::fastfield::CompositeFastFieldSerializer;
 use crate::postings::UnorderedTermId;
 use crate::schema::{Document, Field};
@@ -103,11 +102,11 @@ impl MultiValuedFastFieldWriter {
         &'a self,
         doc_id_map: Option<&'b DocIdMapping>,
     ) -> impl Iterator<Item = &'b [u64]> {
-        let doc_id_iter = if let Some(doc_id_map) = doc_id_map {
-            Box::new(doc_id_map.iter_old_doc_ids().cloned()) as Box<dyn Iterator<Item = u32>>
+        let doc_id_iter: Box<dyn Iterator<Item = u32>> = if let Some(doc_id_map) = doc_id_map {
+            Box::new(doc_id_map.iter_old_doc_ids())
         } else {
-            Box::new(self.doc_index.iter().enumerate().map(|el| el.0 as u32))
-                as Box<dyn Iterator<Item = u32>>
+            let max_doc = self.doc_index.len() as DocId;
+            Box::new(0..max_doc)
         };
         doc_id_iter.map(move |doc_id| self.get_values_for_doc_id(doc_id))
     }
@@ -155,7 +154,7 @@ impl MultiValuedFastFieldWriter {
         }
         {
             // writing the values themselves.
-            let mut value_serializer: DynamicFastFieldSerializer<'_, _>;
+            let mut value_serializer: BitpackedFastFieldSerializerLegacy<'_, _>;
             match mapping_opt {
                 Some(mapping) => {
                     value_serializer = serializer.new_u64_fast_field_with_idx(

src/fastfield/reader.rs

@@ -1,6 +1,5 @@
 use super::FastValue;
-use crate::common::BinarySerializable;
-use crate::common::CompositeFile;
+use crate::directory::CompositeFile;
 use crate::directory::FileSlice;
 use crate::directory::OwnedBytes;
 use crate::directory::{Directory, RamDirectory, WritePtr};
@@ -8,11 +7,18 @@ use crate::fastfield::{CompositeFastFieldSerializer, FastFieldsWriter};
 use crate::schema::Schema;
 use crate::schema::FAST;
 use crate::DocId;
+use common::BinarySerializable;
+use fastfield_codecs::bitpacked::BitpackedFastFieldReader as BitpackedReader;
+use fastfield_codecs::bitpacked::BitpackedFastFieldSerializer;
+use fastfield_codecs::linearinterpol::LinearInterpolFastFieldReader;
+use fastfield_codecs::linearinterpol::LinearInterpolFastFieldSerializer;
+use fastfield_codecs::multilinearinterpol::MultiLinearInterpolFastFieldReader;
+use fastfield_codecs::multilinearinterpol::MultiLinearInterpolFastFieldSerializer;
+use fastfield_codecs::FastFieldCodecReader;
+use fastfield_codecs::FastFieldCodecSerializer;
 use std::collections::HashMap;
 use std::marker::PhantomData;
 use std::path::Path;
-use tantivy_bitpacker::compute_num_bits;
-use tantivy_bitpacker::BitUnpacker;
 
 /// FastFieldReader is the trait to access fast field data.
 pub trait FastFieldReader<Item: FastValue>: Clone {
@@ -38,13 +44,13 @@ pub trait FastFieldReader<Item: FastValue>: Clone {
     ///
     /// May panic if `start + output.len()` is greater than
     /// the segment's `maxdoc`.
-    fn get_range(&self, start: DocId, output: &mut [Item]);
+    fn get_range(&self, start: u64, output: &mut [Item]);
 
     /// Returns the minimum value for this fast field.
     ///
-    /// The max value does not take in account of possible
-    /// deleted document, and should be considered as an upper bound
-    /// of the actual maximum value.
+    /// The min value does not take in account of possible
+    /// deleted document, and should be considered as a lower bound
+    /// of the actual mimimum value.
     fn min_value(&self) -> Item;
 
     /// Returns the maximum value for this fast field.
@@ -61,15 +67,48 @@ pub trait FastFieldReader<Item: FastValue>: Clone {
 ///
 pub enum DynamicFastFieldReader<Item: FastValue> {
     /// Bitpacked compressed fastfield data.
-    Bitpacked(BitpackedFastFieldReader<Item>),
+    Bitpacked(FastFieldReaderCodecWrapper<Item, BitpackedReader>),
+    /// Linear interpolated values + bitpacked
+    LinearInterpol(FastFieldReaderCodecWrapper<Item, LinearInterpolFastFieldReader>),
+    /// Blockwise linear interpolated values + bitpacked
+    MultiLinearInterpol(FastFieldReaderCodecWrapper<Item, MultiLinearInterpolFastFieldReader>),
 }
 
 impl<Item: FastValue> DynamicFastFieldReader<Item> {
     /// Returns correct the reader wrapped in the `DynamicFastFieldReader` enum for the data.
     pub fn open(file: FileSlice) -> crate::Result<DynamicFastFieldReader<Item>> {
-        Ok(DynamicFastFieldReader::Bitpacked(
-            BitpackedFastFieldReader::open(file)?,
-        ))
+        let mut bytes = file.read_bytes()?;
+        let id = bytes.read_u8();
+
+        let reader = match id {
+            BitpackedFastFieldSerializer::ID => {
+                DynamicFastFieldReader::Bitpacked(FastFieldReaderCodecWrapper::<
+                    Item,
+                    BitpackedReader,
+                >::open_from_bytes(bytes)?)
+            }
+            LinearInterpolFastFieldSerializer::ID => {
+                DynamicFastFieldReader::LinearInterpol(FastFieldReaderCodecWrapper::<
+                    Item,
+                    LinearInterpolFastFieldReader,
+                >::open_from_bytes(bytes)?)
+            }
+            MultiLinearInterpolFastFieldSerializer::ID => {
+                DynamicFastFieldReader::MultiLinearInterpol(FastFieldReaderCodecWrapper::<
+                    Item,
+                    MultiLinearInterpolFastFieldReader,
+                >::open_from_bytes(
+                    bytes
+                )?)
+            }
+            _ => {
+                panic!(
+                    "unknown fastfield id {:?}. Data corrupted or using old tantivy version.",
+                    id
+                )
+            }
+        };
+        Ok(reader)
     }
 }
 
@@ -77,57 +116,67 @@ impl<Item: FastValue> FastFieldReader<Item> for DynamicFastFieldReader<Item> {
     fn get(&self, doc: DocId) -> Item {
         match self {
             Self::Bitpacked(reader) => reader.get(doc),
+            Self::LinearInterpol(reader) => reader.get(doc),
+            Self::MultiLinearInterpol(reader) => reader.get(doc),
         }
     }
-    fn get_range(&self, start: DocId, output: &mut [Item]) {
+    fn get_range(&self, start: u64, output: &mut [Item]) {
         match self {
             Self::Bitpacked(reader) => reader.get_range(start, output),
+            Self::LinearInterpol(reader) => reader.get_range(start, output),
+            Self::MultiLinearInterpol(reader) => reader.get_range(start, output),
         }
     }
     fn min_value(&self) -> Item {
         match self {
             Self::Bitpacked(reader) => reader.min_value(),
+            Self::LinearInterpol(reader) => reader.min_value(),
+            Self::MultiLinearInterpol(reader) => reader.min_value(),
         }
     }
     fn max_value(&self) -> Item {
         match self {
             Self::Bitpacked(reader) => reader.max_value(),
+            Self::LinearInterpol(reader) => reader.max_value(),
+            Self::MultiLinearInterpol(reader) => reader.max_value(),
         }
     }
 }
 
-/// Trait for accessing a fastfield.
+/// Wrapper for accessing a fastfield.
+///
+/// Holds the data and the codec to the read the data.
 ///
-/// Depending on the field type, a different
-/// fast field is required.
 #[derive(Clone)]
-pub struct BitpackedFastFieldReader<Item: FastValue> {
+pub struct FastFieldReaderCodecWrapper<Item: FastValue, CodecReader> {
+    reader: CodecReader,
     bytes: OwnedBytes,
-    bit_unpacker: BitUnpacker,
-    min_value_u64: u64,
-    max_value_u64: u64,
     _phantom: PhantomData<Item>,
 }
 
-impl<Item: FastValue> BitpackedFastFieldReader<Item> {
+impl<Item: FastValue, C: FastFieldCodecReader> FastFieldReaderCodecWrapper<Item, C> {
     /// Opens a fast field given a file.
     pub fn open(file: FileSlice) -> crate::Result<Self> {
         let mut bytes = file.read_bytes()?;
-        let min_value = u64::deserialize(&mut bytes)?;
-        let amplitude = u64::deserialize(&mut bytes)?;
-        let max_value = min_value + amplitude;
-        let num_bits = compute_num_bits(amplitude);
-        let bit_unpacker = BitUnpacker::new(num_bits);
-        Ok(BitpackedFastFieldReader {
+        let id = u8::deserialize(&mut bytes)?;
+        assert_eq!(
+            BitpackedFastFieldSerializer::ID,
+            id,
+            "Tried to open fast field as bitpacked encoded (id=1), but got serializer with different id"
+        );
+        Self::open_from_bytes(bytes)
+    }
+    /// Opens a fast field given the bytes.
+    pub fn open_from_bytes(bytes: OwnedBytes) -> crate::Result<Self> {
+        let reader = C::open_from_bytes(bytes.as_slice())?;
+        Ok(FastFieldReaderCodecWrapper {
+            reader,
             bytes,
-            min_value_u64: min_value,
-            max_value_u64: max_value,
-            bit_unpacker,
             _phantom: PhantomData,
         })
     }
     pub(crate) fn get_u64(&self, doc: u64) -> Item {
-        Item::from_u64(self.min_value_u64 + self.bit_unpacker.get(doc, &self.bytes))
+        Item::from_u64(self.reader.get_u64(doc, self.bytes.as_slice()))
     }
 
     /// Internally `multivalued` also use SingleValue Fast fields.
@@ -149,7 +198,9 @@ impl<Item: FastValue> BitpackedFastFieldReader<Item> {
     }
 }
 
-impl<Item: FastValue> FastFieldReader<Item> for BitpackedFastFieldReader<Item> {
+impl<Item: FastValue, C: FastFieldCodecReader + Clone> FastFieldReader<Item>
+    for FastFieldReaderCodecWrapper<Item, C>
+{
     /// Return the value associated to the given document.
     ///
     /// This accessor should return as fast as possible.
@@ -175,8 +226,8 @@ impl<Item: FastValue> FastFieldReader<Item> for BitpackedFastFieldReader<Item> {
     ///
     /// May panic if `start + output.len()` is greater than
     /// the segment's `maxdoc`.
-    fn get_range(&self, start: DocId, output: &mut [Item]) {
-        self.get_range_u64(u64::from(start), output);
+    fn get_range(&self, start: u64, output: &mut [Item]) {
+        self.get_range_u64(start, output);
     }
 
     /// Returns the minimum value for this fast field.
@@ -185,7 +236,7 @@ impl<Item: FastValue> FastFieldReader<Item> for BitpackedFastFieldReader<Item> {
     /// deleted document, and should be considered as an upper bound
     /// of the actual maximum value.
     fn min_value(&self) -> Item {
-        Item::from_u64(self.min_value_u64)
+        Item::from_u64(self.reader.min_value())
     }
 
     /// Returns the maximum value for this fast field.
@@ -194,12 +245,14 @@ impl<Item: FastValue> FastFieldReader<Item> for BitpackedFastFieldReader<Item> {
     /// deleted document, and should be considered as an upper bound
     /// of the actual maximum value.
     fn max_value(&self) -> Item {
-        Item::from_u64(self.max_value_u64)
+        Item::from_u64(self.reader.max_value())
     }
 }
 
-impl<Item: FastValue> From<Vec<Item>> for BitpackedFastFieldReader<Item> {
-    fn from(vals: Vec<Item>) -> BitpackedFastFieldReader<Item> {
+pub(crate) type BitpackedFastFieldReader<Item> = FastFieldReaderCodecWrapper<Item, BitpackedReader>;
+
+impl<Item: FastValue> From<Vec<Item>> for DynamicFastFieldReader<Item> {
+    fn from(vals: Vec<Item>) -> DynamicFastFieldReader<Item> {
         let mut schema_builder = Schema::builder();
         let field = schema_builder.add_u64_field("field", FAST);
         let schema = schema_builder.build();
@@ -231,6 +284,6 @@ impl<Item: FastValue> From<Vec<Item>> for BitpackedFastFieldReader<Item> {
         let field_file = composite_file
             .open_read(field)
             .expect("File component not found");
-        BitpackedFastFieldReader::open(field_file).unwrap()
+        DynamicFastFieldReader::open(field_file).unwrap()
     }
 }

src/fastfield/readers.rs

@@ -1,4 +1,4 @@
-use crate::common::CompositeFile;
+use crate::directory::CompositeFile;
 use crate::directory::FileSlice;
 use crate::fastfield::MultiValuedFastFieldReader;
 use crate::fastfield::{BitpackedFastFieldReader, FastFieldNotAvailableError};
@@ -8,7 +8,6 @@ use crate::space_usage::PerFieldSpaceUsage;
 use crate::TantivyError;
 
 use super::reader::DynamicFastFieldReader;
-use super::FastFieldReader;
 
 /// Provides access to all of the BitpackedFastFieldReader.
 ///
@@ -100,23 +99,27 @@ impl FastFieldReaders {
         Ok(())
     }
 
+    pub(crate) fn typed_fast_field_reader_with_idx<TFastValue: FastValue>(
+        &self,
+        field: Field,
+        index: usize,
+    ) -> crate::Result<DynamicFastFieldReader<TFastValue>> {
+        let fast_field_slice = self.fast_field_data(field, index)?;
+        DynamicFastFieldReader::open(fast_field_slice)
+    }
     pub(crate) fn typed_fast_field_reader<TFastValue: FastValue>(
         &self,
         field: Field,
     ) -> crate::Result<DynamicFastFieldReader<TFastValue>> {
-        let fast_field_slice = self.fast_field_data(field, 0)?;
-        DynamicFastFieldReader::open(fast_field_slice)
+        self.typed_fast_field_reader_with_idx(field, 0)
     }
 
     pub(crate) fn typed_fast_field_multi_reader<TFastValue: FastValue>(
         &self,
         field: Field,
     ) -> crate::Result<MultiValuedFastFieldReader<TFastValue>> {
-        let fast_field_slice_idx = self.fast_field_data(field, 0)?;
-        let fast_field_slice_vals = self.fast_field_data(field, 1)?;
-        let idx_reader = BitpackedFastFieldReader::open(fast_field_slice_idx)?;
-        let vals_reader: BitpackedFastFieldReader<TFastValue> =
-            BitpackedFastFieldReader::open(fast_field_slice_vals)?;
+        let idx_reader = self.typed_fast_field_reader(field)?;
+        let vals_reader = self.typed_fast_field_reader_with_idx(field, 1)?;
         Ok(MultiValuedFastFieldReader::open(idx_reader, vals_reader))
     }
 
@@ -139,7 +142,7 @@ impl FastFieldReaders {
     /// Returns the `i64` fast field reader reader associated to `field`.
     ///
     /// If `field` is not a i64 fast field, this method returns an Error.
-    pub fn i64(&self, field: Field) -> crate::Result<impl FastFieldReader<i64>> {
+    pub fn i64(&self, field: Field) -> crate::Result<DynamicFastFieldReader<i64>> {
         self.check_type(field, FastType::I64, Cardinality::SingleValue)?;
         self.typed_fast_field_reader(field)
     }
@@ -147,7 +150,7 @@ impl FastFieldReaders {
     /// Returns the `i64` fast field reader reader associated to `field`.
     ///
     /// If `field` is not a i64 fast field, this method returns an Error.
-    pub fn date(&self, field: Field) -> crate::Result<impl FastFieldReader<crate::DateTime>> {
+    pub fn date(&self, field: Field) -> crate::Result<DynamicFastFieldReader<crate::DateTime>> {
         self.check_type(field, FastType::Date, Cardinality::SingleValue)?;
         self.typed_fast_field_reader(field)
     }
@@ -155,7 +158,7 @@ impl FastFieldReaders {
     /// Returns the `f64` fast field reader reader associated to `field`.
     ///
     /// If `field` is not a f64 fast field, this method returns an Error.
-    pub fn f64(&self, field: Field) -> crate::Result<impl FastFieldReader<f64>> {
+    pub fn f64(&self, field: Field) -> crate::Result<DynamicFastFieldReader<f64>> {
         self.check_type(field, FastType::F64, Cardinality::SingleValue)?;
         self.typed_fast_field_reader(field)
     }

src/fastfield/serializer.rs

@@ -1,256 +0,0 @@
-use crate::common::BinarySerializable;
-use crate::common::CompositeWrite;
-use crate::common::CountingWriter;
-use crate::directory::WritePtr;
-use crate::schema::Field;
-use std::io::{self, Write};
-use tantivy_bitpacker::compute_num_bits;
-use tantivy_bitpacker::BitPacker;
-
-/// `CompositeFastFieldSerializer` is in charge of serializing
-/// fastfields on disk.
-///
-/// Fast fields have differnt encodings like bit-packing.
-///
-/// `FastFieldWriter`s are in charge of pushing the data to
-/// the serializer.
-/// The serializer expects to receive the following calls.
-///
-/// * `new_u64_fast_field(...)`
-/// * `add_val(...)`
-/// * `add_val(...)`
-/// * `add_val(...)`
-/// * ...
-/// * `close_field()`
-/// * `new_u64_fast_field(...)`
-/// * `add_val(...)`
-/// * ...
-/// * `close_field()`
-/// * `close()`
-pub struct CompositeFastFieldSerializer {
-    composite_write: CompositeWrite<WritePtr>,
-}
-
-impl CompositeFastFieldSerializer {
-    /// Constructor
-    pub fn from_write(write: WritePtr) -> io::Result<CompositeFastFieldSerializer> {
-        // just making room for the pointer to header.
-        let composite_write = CompositeWrite::wrap(write);
-        Ok(CompositeFastFieldSerializer { composite_write })
-    }
-
-    /// Start serializing a new u64 fast field
-    pub fn new_u64_fast_field(
-        &mut self,
-        field: Field,
-        min_value: u64,
-        max_value: u64,
-    ) -> io::Result<DynamicFastFieldSerializer<'_, CountingWriter<WritePtr>>> {
-        self.new_u64_fast_field_with_idx(field, min_value, max_value, 0)
-    }
-
-    /// Start serializing a new u64 fast field
-    pub fn new_u64_fast_field_with_idx(
-        &mut self,
-        field: Field,
-        min_value: u64,
-        max_value: u64,
-        idx: usize,
-    ) -> io::Result<DynamicFastFieldSerializer<'_, CountingWriter<WritePtr>>> {
-        let field_write = self.composite_write.for_field_with_idx(field, idx);
-        DynamicFastFieldSerializer::open(field_write, min_value, max_value)
-    }
-
-    /// Start serializing a new [u8] fast field
-    pub fn new_bytes_fast_field_with_idx(
-        &mut self,
-        field: Field,
-        idx: usize,
-    ) -> FastBytesFieldSerializer<'_, CountingWriter<WritePtr>> {
-        let field_write = self.composite_write.for_field_with_idx(field, idx);
-        FastBytesFieldSerializer { write: field_write }
-    }
-
-    /// Closes the serializer
-    ///
-    /// After this call the data must be persistently save on disk.
-    pub fn close(self) -> io::Result<()> {
-        self.composite_write.close()
-    }
-}
-
-#[derive(Debug, Clone)]
-pub struct EstimationStats {
-    min_value: u64,
-    max_value: u64,
-}
-/// The FastFieldSerializer trait is the common interface
-/// implemented by every fastfield serializer variant.
-///
-/// `DynamicFastFieldSerializer` is the enum wrapping all variants.
-/// It is used to create an serializer instance.
-pub trait FastFieldSerializer {
-    /// add value to serializer
-    fn add_val(&mut self, val: u64) -> io::Result<()>;
-    /// finish serializing a field.
-    fn close_field(self) -> io::Result<()>;
-}
-
-/// The FastFieldSerializerEstimate trait is required on all variants
-/// of fast field compressions, to decide which one to choose.
-pub trait FastFieldSerializerEstimate {
-    /// returns an estimate of the compression ratio.
-    fn estimate(
-        /*fastfield_accessor: impl FastFieldReader<u64>,*/ stats: EstimationStats,
-    ) -> (f32, &'static str);
-    /// the unique name of the compressor
-    fn name() -> &'static str;
-}
-
-pub enum DynamicFastFieldSerializer<'a, W: Write> {
-    Bitpacked(BitpackedFastFieldSerializer<'a, W>),
-}
-
-impl<'a, W: Write> DynamicFastFieldSerializer<'a, W> {
-    /// Creates a new fast field serializer.
-    ///
-    /// The serializer in fact encode the values by bitpacking
-    /// `(val - min_value)`.
-    ///
-    /// It requires a `min_value` and a `max_value` to compute
-    /// compute the minimum number of bits required to encode
-    /// values.
-    pub fn open(
-        write: &'a mut W,
-        min_value: u64,
-        max_value: u64,
-    ) -> io::Result<DynamicFastFieldSerializer<'a, W>> {
-        let stats = EstimationStats {
-            min_value,
-            max_value,
-        };
-        let (_ratio, name) = (
-            BitpackedFastFieldSerializer::<Vec<u8>>::estimate(stats),
-            BitpackedFastFieldSerializer::<Vec<u8>>::name(),
-        );
-        Self::open_from_name(write, min_value, max_value, name)
-    }
-
-    /// Creates a new fast field serializer.
-    ///
-    /// The serializer in fact encode the values by bitpacking
-    /// `(val - min_value)`.
-    ///
-    /// It requires a `min_value` and a `max_value` to compute
-    /// compute the minimum number of bits required to encode
-    /// values.
-    pub fn open_from_name(
-        write: &'a mut W,
-        min_value: u64,
-        max_value: u64,
-        name: &str,
-    ) -> io::Result<DynamicFastFieldSerializer<'a, W>> {
-        // Weirdly the W generic on BitpackedFastFieldSerializer needs to be set,
-        // although name() doesn't use it
-        let variant = if name == BitpackedFastFieldSerializer::<Vec<u8>>::name() {
-            DynamicFastFieldSerializer::Bitpacked(BitpackedFastFieldSerializer::open(
-                write, min_value, max_value,
-            )?)
-        } else {
-            panic!("unknown fastfield serializer {}", name);
-        };
-
-        Ok(variant)
-    }
-}
-impl<'a, W: Write> FastFieldSerializer for DynamicFastFieldSerializer<'a, W> {
-    fn add_val(&mut self, val: u64) -> io::Result<()> {
-        match self {
-            Self::Bitpacked(serializer) => serializer.add_val(val),
-        }
-    }
-    fn close_field(self) -> io::Result<()> {
-        match self {
-            Self::Bitpacked(serializer) => serializer.close_field(),
-        }
-    }
-}
-
-pub struct BitpackedFastFieldSerializer<'a, W: Write> {
-    bit_packer: BitPacker,
-    write: &'a mut W,
-    min_value: u64,
-    num_bits: u8,
-}
-
-impl<'a, W: Write> BitpackedFastFieldSerializer<'a, W> {
-    /// Creates a new fast field serializer.
-    ///
-    /// The serializer in fact encode the values by bitpacking
-    /// `(val - min_value)`.
-    ///
-    /// It requires a `min_value` and a `max_value` to compute
-    /// compute the minimum number of bits required to encode
-    /// values.
-    fn open(
-        write: &'a mut W,
-        min_value: u64,
-        max_value: u64,
-    ) -> io::Result<BitpackedFastFieldSerializer<'a, W>> {
-        assert!(min_value <= max_value);
-        min_value.serialize(write)?;
-        let amplitude = max_value - min_value;
-        amplitude.serialize(write)?;
-        let num_bits = compute_num_bits(amplitude);
-        let bit_packer = BitPacker::new();
-        Ok(BitpackedFastFieldSerializer {
-            bit_packer,
-            write,
-            min_value,
-            num_bits,
-        })
-    }
-}
-
-impl<'a, W: 'a + Write> FastFieldSerializer for BitpackedFastFieldSerializer<'a, W> {
-    /// Pushes a new value to the currently open u64 fast field.
-    fn add_val(&mut self, val: u64) -> io::Result<()> {
-        let val_to_write: u64 = val - self.min_value;
-        self.bit_packer
-            .write(val_to_write, self.num_bits, &mut self.write)?;
-        Ok(())
-    }
-    fn close_field(mut self) -> io::Result<()> {
-        self.bit_packer.close(&mut self.write)
-    }
-}
-
-impl<'a, W: 'a + Write> FastFieldSerializerEstimate for BitpackedFastFieldSerializer<'a, W> {
-    fn estimate(
-        /*_fastfield_accessor: impl FastFieldReader<u64>, */ stats: EstimationStats,
-    ) -> (f32, &'static str) {
-        let amplitude = stats.max_value - stats.min_value;
-        let num_bits = compute_num_bits(amplitude);
-        let num_bits_uncompressed = 64;
-        let ratio = num_bits as f32 / num_bits_uncompressed as f32;
-        let name = Self::name();
-        (ratio, name)
-    }
-    fn name() -> &'static str {
-        "Bitpacked"
-    }
-}
-
-pub struct FastBytesFieldSerializer<'a, W: Write> {
-    write: &'a mut W,
-}
-
-impl<'a, W: Write> FastBytesFieldSerializer<'a, W> {
-    pub fn write_all(&mut self, vals: &[u8]) -> io::Result<()> {
-        self.write.write_all(vals)
-    }
-
-    pub fn flush(&mut self) -> io::Result<()> {
-        self.write.flush()
-    }
-}

src/fastfield/serializer/mod.rs

@@ -0,0 +1,216 @@
+use crate::directory::CompositeWrite;
+use crate::directory::WritePtr;
+use crate::schema::Field;
+use common::BinarySerializable;
+use common::CountingWriter;
+pub use fastfield_codecs::bitpacked::BitpackedFastFieldSerializer;
+pub use fastfield_codecs::bitpacked::BitpackedFastFieldSerializerLegacy;
+use fastfield_codecs::linearinterpol::LinearInterpolFastFieldSerializer;
+use fastfield_codecs::multilinearinterpol::MultiLinearInterpolFastFieldSerializer;
+pub use fastfield_codecs::FastFieldCodecSerializer;
+pub use fastfield_codecs::FastFieldDataAccess;
+pub use fastfield_codecs::FastFieldStats;
+use std::io::{self, Write};
+
+/// `CompositeFastFieldSerializer` is in charge of serializing
+/// fastfields on disk.
+///
+/// Fast fields have different encodings like bit-packing.
+///
+/// `FastFieldWriter`s are in charge of pushing the data to
+/// the serializer.
+/// The serializer expects to receive the following calls.
+///
+/// * `new_u64_fast_field(...)`
+/// * `add_val(...)`
+/// * `add_val(...)`
+/// * `add_val(...)`
+/// * ...
+/// * `close_field()`
+/// * `new_u64_fast_field(...)`
+/// * `add_val(...)`
+/// * ...
+/// * `close_field()`
+/// * `close()`
+pub struct CompositeFastFieldSerializer {
+    composite_write: CompositeWrite<WritePtr>,
+}
+
+// use this, when this is merged and stabilized explicit_generic_args_with_impl_trait
+// https://github.com/rust-lang/rust/pull/86176
+fn codec_estimation<T: FastFieldCodecSerializer, A: FastFieldDataAccess>(
+    stats: FastFieldStats,
+    fastfield_accessor: &A,
+    estimations: &mut Vec<(f32, &str, u8)>,
+) {
+    if !T::is_applicable(fastfield_accessor, stats.clone()) {
+        return;
+    }
+    let (ratio, name, id) = (T::estimate(fastfield_accessor, stats), T::NAME, T::ID);
+    estimations.push((ratio, name, id));
+}
+
+impl CompositeFastFieldSerializer {
+    /// Constructor
+    pub fn from_write(write: WritePtr) -> io::Result<CompositeFastFieldSerializer> {
+        // just making room for the pointer to header.
+        let composite_write = CompositeWrite::wrap(write);
+        Ok(CompositeFastFieldSerializer { composite_write })
+    }
+
+    /// Serialize data into a new u64 fast field. The best compression codec will be chosen automatically.
+    pub fn create_auto_detect_u64_fast_field(
+        &mut self,
+        field: Field,
+        stats: FastFieldStats,
+        fastfield_accessor: impl FastFieldDataAccess,
+        data_iter_1: impl Iterator<Item = u64>,
+        data_iter_2: impl Iterator<Item = u64>,
+    ) -> io::Result<()> {
+        self.create_auto_detect_u64_fast_field_with_idx(
+            field,
+            stats,
+            fastfield_accessor,
+            data_iter_1,
+            data_iter_2,
+            0,
+        )
+    }
+    /// Serialize data into a new u64 fast field. The best compression codec will be chosen automatically.
+    pub fn create_auto_detect_u64_fast_field_with_idx(
+        &mut self,
+        field: Field,
+        stats: FastFieldStats,
+        fastfield_accessor: impl FastFieldDataAccess,
+        data_iter_1: impl Iterator<Item = u64>,
+        data_iter_2: impl Iterator<Item = u64>,
+        idx: usize,
+    ) -> io::Result<()> {
+        let field_write = self.composite_write.for_field_with_idx(field, idx);
+
+        let mut estimations = vec![];
+
+        codec_estimation::<BitpackedFastFieldSerializer, _>(
+            stats.clone(),
+            &fastfield_accessor,
+            &mut estimations,
+        );
+        codec_estimation::<LinearInterpolFastFieldSerializer, _>(
+            stats.clone(),
+            &fastfield_accessor,
+            &mut estimations,
+        );
+        codec_estimation::<MultiLinearInterpolFastFieldSerializer, _>(
+            stats.clone(),
+            &fastfield_accessor,
+            &mut estimations,
+        );
+        if let Some(broken_estimation) = estimations.iter().find(|estimation| estimation.0.is_nan())
+        {
+            warn!(
+                "broken estimation for fast field codec {}",
+                broken_estimation.1
+            );
+        }
+        // removing nan values for codecs with broken calculations, and max values which disables codecs
+        estimations.retain(|estimation| !estimation.0.is_nan() && estimation.0 != f32::MAX);
+        estimations.sort_by(|a, b| a.0.partial_cmp(&b.0).unwrap());
+        let (_ratio, name, id) = estimations[0];
+        debug!(
+            "choosing fast field codec {} for field_id {:?}",
+            name, field
+        ); // todo print actual field name
+        id.serialize(field_write)?;
+        match name {
+            BitpackedFastFieldSerializer::NAME => {
+                BitpackedFastFieldSerializer::serialize(
+                    field_write,
+                    &fastfield_accessor,
+                    stats,
+                    data_iter_1,
+                    data_iter_2,
+                )?;
+            }
+            LinearInterpolFastFieldSerializer::NAME => {
+                LinearInterpolFastFieldSerializer::serialize(
+                    field_write,
+                    &fastfield_accessor,
+                    stats,
+                    data_iter_1,
+                    data_iter_2,
+                )?;
+            }
+            MultiLinearInterpolFastFieldSerializer::NAME => {
+                MultiLinearInterpolFastFieldSerializer::serialize(
+                    field_write,
+                    &fastfield_accessor,
+                    stats,
+                    data_iter_1,
+                    data_iter_2,
+                )?;
+            }
+            _ => {
+                panic!("unknown fastfield serializer {}", name)
+            }
+        };
+        field_write.flush()?;
+
+        Ok(())
+    }
+
+    /// Start serializing a new u64 fast field
+    pub fn new_u64_fast_field(
+        &mut self,
+        field: Field,
+        min_value: u64,
+        max_value: u64,
+    ) -> io::Result<BitpackedFastFieldSerializerLegacy<'_, CountingWriter<WritePtr>>> {
+        self.new_u64_fast_field_with_idx(field, min_value, max_value, 0)
+    }
+
+    /// Start serializing a new u64 fast field
+    pub fn new_u64_fast_field_with_idx(
+        &mut self,
+        field: Field,
+        min_value: u64,
+        max_value: u64,
+        idx: usize,
+    ) -> io::Result<BitpackedFastFieldSerializerLegacy<'_, CountingWriter<WritePtr>>> {
+        let field_write = self.composite_write.for_field_with_idx(field, idx);
+        // Prepend codec id to field data for compatibility with DynamicFastFieldReader.
+        let id = BitpackedFastFieldSerializer::ID;
+        id.serialize(field_write)?;
+        BitpackedFastFieldSerializerLegacy::open(field_write, min_value, max_value)
+    }
+
+    /// Start serializing a new [u8] fast field
+    pub fn new_bytes_fast_field_with_idx(
+        &mut self,
+        field: Field,
+        idx: usize,
+    ) -> FastBytesFieldSerializer<'_, CountingWriter<WritePtr>> {
+        let field_write = self.composite_write.for_field_with_idx(field, idx);
+        FastBytesFieldSerializer { write: field_write }
+    }
+
+    /// Closes the serializer
+    ///
+    /// After this call the data must be persistently save on disk.
+    pub fn close(self) -> io::Result<()> {
+        self.composite_write.close()
+    }
+}
+
+pub struct FastBytesFieldSerializer<'a, W: Write> {
+    write: &'a mut W,
+}
+
+impl<'a, W: Write> FastBytesFieldSerializer<'a, W> {
+    pub fn write_all(&mut self, vals: &[u8]) -> io::Result<()> {
+        self.write.write_all(vals)
+    }
+
+    pub fn flush(&mut self) -> io::Result<()> {
+        self.write.flush()
+    }
+}

src/fastfield/writer.rs

@@ -1,11 +1,12 @@
 use super::multivalued::MultiValuedFastFieldWriter;
-use crate::common;
-use crate::fastfield::serializer::FastFieldSerializer;
+use super::serializer::FastFieldStats;
+use super::FastFieldDataAccess;
 use crate::fastfield::{BytesFastFieldWriter, CompositeFastFieldSerializer};
 use crate::indexer::doc_id_mapping::DocIdMapping;
 use crate::postings::UnorderedTermId;
 use crate::schema::{Cardinality, Document, Field, FieldEntry, FieldType, Schema};
 use crate::termdict::TermOrdinal;
+use common;
 use fnv::FnvHashMap;
 use std::collections::HashMap;
 use std::io;
@@ -265,9 +266,9 @@ impl IntFastFieldWriter {
         self.add_val(val);
     }
 
-    /// Extract the stored data
-    pub(crate) fn get_data(&self) -> Vec<u64> {
-        self.vals.iter().collect::<Vec<u64>>()
+    /// get iterator over the data
+    pub(crate) fn iter(&self) -> impl Iterator<Item = u64> + '_ {
+        self.vals.iter()
     }
 
     /// Push the fast fields value to the `FastFieldWriter`.
@@ -281,17 +282,59 @@ impl IntFastFieldWriter {
         } else {
             (self.val_min, self.val_max)
         };
-        let mut single_field_serializer = serializer.new_u64_fast_field(self.field, min, max)?;
-        if let Some(doc_id_map) = doc_id_map {
-            for doc_id in doc_id_map.iter_old_doc_ids() {
-                single_field_serializer.add_val(self.vals.get(*doc_id as usize))?;
-            }
-        } else {
-            for val in self.vals.iter() {
-                single_field_serializer.add_val(val)?;
-            }
+        let fastfield_accessor = WriterFastFieldAccessProvider {
+            doc_id_map,
+            vals: &self.vals,
+        };
+        let stats = FastFieldStats {
+            min_value: min,
+            max_value: max,
+            num_vals: self.val_count as u64,
         };
 
-        single_field_serializer.close_field()
+        if let Some(doc_id_map) = doc_id_map {
+            let iter = doc_id_map
+                .iter_old_doc_ids()
+                .map(|doc_id| self.vals.get(doc_id as usize));
+            serializer.create_auto_detect_u64_fast_field(
+                self.field,
+                stats,
+                fastfield_accessor,
+                iter.clone(),
+                iter,
+            )?;
+        } else {
+            serializer.create_auto_detect_u64_fast_field(
+                self.field,
+                stats,
+                fastfield_accessor,
+                self.vals.iter(),
+                self.vals.iter(),
+            )?;
+        };
+        Ok(())
+    }
+}
+
+#[derive(Clone)]
+struct WriterFastFieldAccessProvider<'map, 'bitp> {
+    doc_id_map: Option<&'map DocIdMapping>,
+    vals: &'bitp BlockedBitpacker,
+}
+impl<'map, 'bitp> FastFieldDataAccess for WriterFastFieldAccessProvider<'map, 'bitp> {
+    /// Return the value associated to the given doc.
+    ///
+    /// Whenever possible use the Iterator passed to the fastfield creation instead, for performance reasons.
+    ///
+    /// # Panics
+    ///
+    /// May panic if `doc` is greater than the index.
+    fn get_val(&self, doc: u64) -> u64 {
+        if let Some(doc_id_map) = self.doc_id_map {
+            self.vals
+                .get(doc_id_map.get_old_doc_id(doc as u32) as usize) // consider extra FastFieldReader wrapper for non doc_id_map
+        } else {
+            self.vals.get(doc as usize)
+        }
     }
 }

src/fieldnorm/reader.rs

@@ -1,5 +1,5 @@
 use super::{fieldnorm_to_id, id_to_fieldnorm};
-use crate::common::CompositeFile;
+use crate::directory::CompositeFile;
 use crate::directory::FileSlice;
 use crate::directory::OwnedBytes;
 use crate::schema::Field;

src/fieldnorm/serializer.rs

@@ -1,4 +1,4 @@
-use crate::common::CompositeWrite;
+use crate::directory::CompositeWrite;
 use crate::directory::WritePtr;
 use crate::schema::Field;
 use std::io;

src/fieldnorm/writer.rs

@@ -98,7 +98,7 @@ impl FieldNormsWriter {
                 let mut mapped_fieldnorm_values = vec![];
                 mapped_fieldnorm_values.resize(fieldnorm_values.len(), 0u8);
                 for (new_doc_id, old_doc_id) in doc_id_map.iter_old_doc_ids().enumerate() {
-                    mapped_fieldnorm_values[new_doc_id] = fieldnorm_values[*old_doc_id as usize];
+                    mapped_fieldnorm_values[new_doc_id] = fieldnorm_values[old_doc_id as usize];
                 }
                 fieldnorms_serializer.serialize_field(field, &mapped_fieldnorm_values)?;
             } else {

src/functional_test.rs

@@ -1,4 +1,8 @@
+use crate::schema;
 use crate::Index;
+use crate::IndexSettings;
+use crate::IndexSortByField;
+use crate::Order;
 use crate::Searcher;
 use crate::{doc, schema::*};
 use rand::thread_rng;
@@ -35,10 +39,10 @@ fn test_functional_store() -> crate::Result<()> {
     let mut doc_set: Vec<u64> = Vec::new();
 
     let mut doc_id = 0u64;
-    for iteration in 0..500 {
+    for iteration in 0..get_num_iterations() {
         dbg!(iteration);
         let num_docs: usize = rng.gen_range(0..4);
-        if doc_set.len() >= 1 {
+        if !doc_set.is_empty() {
             let doc_to_remove_id = rng.gen_range(0..doc_set.len());
             let removed_doc_id = doc_set.swap_remove(doc_to_remove_id);
             index_writer.delete_term(Term::from_field_u64(id_field, removed_doc_id));
@@ -56,16 +60,37 @@ fn test_functional_store() -> crate::Result<()> {
     Ok(())
 }
 
+fn get_num_iterations() -> usize {
+    std::env::var("NUM_FUNCTIONAL_TEST_ITERATIONS")
+        .map(|str| str.parse().unwrap())
+        .unwrap_or(2000)
+}
 #[test]
 #[ignore]
-fn test_functional_indexing() -> crate::Result<()> {
+fn test_functional_indexing_sorted() -> crate::Result<()> {
     let mut schema_builder = Schema::builder();
 
-    let id_field = schema_builder.add_u64_field("id", INDEXED);
+    let id_field = schema_builder.add_u64_field("id", INDEXED | FAST);
     let multiples_field = schema_builder.add_u64_field("multiples", INDEXED);
+    let text_field_options = TextOptions::default()
+        .set_indexing_options(
+            TextFieldIndexing::default()
+                .set_index_option(schema::IndexRecordOption::WithFreqsAndPositions),
+        )
+        .set_stored();
+    let text_field = schema_builder.add_text_field("text_field", text_field_options);
     let schema = schema_builder.build();
 
-    let index = Index::create_from_tempdir(schema)?;
+    let mut index_builder = Index::builder().schema(schema);
+    index_builder = index_builder.settings(IndexSettings {
+        sort_by_field: Some(IndexSortByField {
+            field: "id".to_string(),
+            order: Order::Desc,
+        }),
+        ..Default::default()
+    });
+    let index = index_builder.create_from_tempdir().unwrap();
+
     let reader = index.reader()?;
 
     let mut rng = thread_rng();
@@ -75,7 +100,7 @@ fn test_functional_indexing() -> crate::Result<()> {
     let mut committed_docs: HashSet<u64> = HashSet::new();
     let mut uncommitted_docs: HashSet<u64> = HashSet::new();
 
-    for _ in 0..200 {
+    for _ in 0..get_num_iterations() {
         let random_val = rng.gen_range(0..20);
         if random_val == 0 {
             index_writer.commit()?;
@@ -88,19 +113,95 @@ fn test_functional_indexing() -> crate::Result<()> {
                 &searcher,
                 &committed_docs.iter().cloned().collect::<Vec<u64>>(),
             )?;
+        } else if committed_docs.remove(&random_val) || uncommitted_docs.remove(&random_val) {
+            let doc_id_term = Term::from_field_u64(id_field, random_val);
+            index_writer.delete_term(doc_id_term);
         } else {
-            if committed_docs.remove(&random_val) || uncommitted_docs.remove(&random_val) {
-                let doc_id_term = Term::from_field_u64(id_field, random_val);
-                index_writer.delete_term(doc_id_term);
-            } else {
-                uncommitted_docs.insert(random_val);
-                let mut doc = Document::new();
-                doc.add_u64(id_field, random_val);
-                for i in 1u64..10u64 {
-                    doc.add_u64(multiples_field, random_val * i);
-                }
-                index_writer.add_document(doc);
+            uncommitted_docs.insert(random_val);
+            let mut doc = Document::new();
+            doc.add_u64(id_field, random_val);
+            for i in 1u64..10u64 {
+                doc.add_u64(multiples_field, random_val * i);
             }
+            doc.add_text(text_field, get_text());
+            index_writer.add_document(doc);
+        }
+    }
+    Ok(())
+}
+
+const LOREM: &str = "Doc Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed \
+             do eiusmod tempor incididunt ut labore et dolore magna aliqua. \
+             Ut enim ad minim veniam, quis nostrud exercitation ullamco \
+             laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure \
+             dolor in reprehenderit in voluptate velit esse cillum dolore eu \
+             fugiat nulla pariatur. Excepteur sint occaecat cupidatat non \
+             proident, sunt in culpa qui officia deserunt mollit anim id est \
+             laborum.";
+fn get_text() -> String {
+    use rand::seq::SliceRandom;
+    let mut rng = thread_rng();
+    let tokens: Vec<_> = LOREM.split(' ').collect();
+    let random_val = rng.gen_range(0..20);
+
+    (0..random_val)
+        .map(|_| tokens.choose(&mut rng).unwrap())
+        .cloned()
+        .collect::<Vec<_>>()
+        .join(" ")
+}
+
+#[test]
+#[ignore]
+fn test_functional_indexing_unsorted() -> crate::Result<()> {
+    let mut schema_builder = Schema::builder();
+
+    let id_field = schema_builder.add_u64_field("id", INDEXED);
+    let multiples_field = schema_builder.add_u64_field("multiples", INDEXED);
+    let text_field_options = TextOptions::default()
+        .set_indexing_options(
+            TextFieldIndexing::default()
+                .set_index_option(schema::IndexRecordOption::WithFreqsAndPositions),
+        )
+        .set_stored();
+    let text_field = schema_builder.add_text_field("text_field", text_field_options);
+    let schema = schema_builder.build();
+
+    let index = Index::create_from_tempdir(schema)?;
+    let reader = index.reader()?;
+
+    let mut rng = thread_rng();
+
+    let mut index_writer = index.writer_with_num_threads(3, 120_000_000)?;
+
+    let mut committed_docs: HashSet<u64> = HashSet::new();
+    let mut uncommitted_docs: HashSet<u64> = HashSet::new();
+
+    for _ in 0..get_num_iterations() {
+        let random_val = rng.gen_range(0..20);
+        if random_val == 0 {
+            index_writer.commit()?;
+            committed_docs.extend(&uncommitted_docs);
+            uncommitted_docs.clear();
+            reader.reload()?;
+            let searcher = reader.searcher();
+            // check that everything is correct.
+            check_index_content(
+                &searcher,
+                &committed_docs.iter().cloned().collect::<Vec<u64>>(),
+            )?;
+        } else if committed_docs.remove(&random_val) || uncommitted_docs.remove(&random_val) {
+            let doc_id_term = Term::from_field_u64(id_field, random_val);
+            index_writer.delete_term(doc_id_term);
+        } else {
+            uncommitted_docs.insert(random_val);
+            let mut doc = Document::new();
+            doc.add_u64(id_field, random_val);
+            for i in 1u64..10u64 {
+                doc.add_u64(multiples_field, random_val * i);
+            }
+            doc.add_text(text_field, get_text());
+            index_writer.add_document(doc);
         }
     }
     Ok(())

src/indexer/delete_queue.rs

@@ -1,6 +1,6 @@
 use super::operation::DeleteOperation;
 use crate::Opstamp;
-use std::mem;
+
 use std::ops::DerefMut;
 use std::sync::{Arc, RwLock, Weak};
 
@@ -105,7 +105,7 @@ impl DeleteQueue {
             return None;
         }
 
-        let delete_operations = mem::replace(&mut self_wlock.writer, vec![]);
+        let delete_operations = std::mem::take(&mut self_wlock.writer);
 
         let new_block = Arc::new(Block {
             operations: Arc::from(delete_operations.into_boxed_slice()),
@@ -286,7 +286,7 @@ mod tests {
             operations_it.advance();
         }
         {
-            let mut operations_it = snapshot.clone();
+            let mut operations_it = snapshot;
             assert_eq!(operations_it.get().unwrap().opstamp, 1);
             operations_it.advance();
             assert_eq!(operations_it.get().unwrap().opstamp, 2);

src/indexer/doc_id_mapping.rs

@@ -5,10 +5,58 @@
 use super::SegmentWriter;
 use crate::{
     schema::{Field, Schema},
-    DocId, IndexSortByField, Order, TantivyError,
+    DocId, IndexSortByField, Order, SegmentOrdinal, TantivyError,
 };
-use std::cmp::Reverse;
-/// Struct to provide mapping from old doc_id to new doc_id and vice versa
+use std::{cmp::Reverse, ops::Index};
+
+/// Struct to provide mapping from new doc_id to old doc_id and segment.
+#[derive(Clone)]
+pub(crate) struct SegmentDocidMapping {
+    new_doc_id_to_old_and_segment: Vec<(DocId, SegmentOrdinal)>,
+    is_trivial: bool,
+}
+
+impl SegmentDocidMapping {
+    pub(crate) fn new(
+        new_doc_id_to_old_and_segment: Vec<(DocId, SegmentOrdinal)>,
+        is_trivial: bool,
+    ) -> Self {
+        Self {
+            new_doc_id_to_old_and_segment,
+            is_trivial,
+        }
+    }
+    pub(crate) fn iter(&self) -> impl Iterator<Item = &(DocId, SegmentOrdinal)> {
+        self.new_doc_id_to_old_and_segment.iter()
+    }
+    pub(crate) fn len(&self) -> usize {
+        self.new_doc_id_to_old_and_segment.len()
+    }
+    /// This flags means the segments are simply stacked in the order of their ordinal.
+    /// e.g. [(0, 1), .. (n, 1), (0, 2)..., (m, 2)]
+    ///
+    /// This allows for some optimization.
+    pub(crate) fn is_trivial(&self) -> bool {
+        self.is_trivial
+    }
+}
+impl Index<usize> for SegmentDocidMapping {
+    type Output = (DocId, SegmentOrdinal);
+
+    fn index(&self, idx: usize) -> &Self::Output {
+        &self.new_doc_id_to_old_and_segment[idx]
+    }
+}
+impl IntoIterator for SegmentDocidMapping {
+    type Item = (DocId, SegmentOrdinal);
+    type IntoIter = std::vec::IntoIter<Self::Item>;
+
+    fn into_iter(self) -> Self::IntoIter {
+        self.new_doc_id_to_old_and_segment.into_iter()
+    }
+}
+
+/// Struct to provide mapping from old doc_id to new doc_id and vice versa within a segment.
 pub struct DocIdMapping {
     new_doc_id_to_old: Vec<DocId>,
     old_doc_id_to_new: Vec<DocId>,
@@ -24,8 +72,8 @@ impl DocIdMapping {
         self.new_doc_id_to_old[doc_id as usize]
     }
     /// iterate over old doc_ids in order of the new doc_ids
-    pub fn iter_old_doc_ids(&self) -> std::slice::Iter<'_, DocId> {
-        self.new_doc_id_to_old.iter()
+    pub fn iter_old_doc_ids(&self) -> impl Iterator<Item = DocId> + Clone + '_ {
+        self.new_doc_id_to_old.iter().cloned()
     }
 }
 
@@ -60,9 +108,8 @@ pub(crate) fn get_doc_id_mapping_from_field(
         })?;
 
     // create new doc_id to old doc_id index (used in fast_field_writers)
-    let data = fast_field.get_data();
-    let mut doc_id_and_data = data
-        .into_iter()
+    let mut doc_id_and_data = fast_field
+        .iter()
         .enumerate()
         .map(|el| (el.0 as DocId, el.1))
         .collect::<Vec<_>>();

src/indexer/doc_opstamp_mapping.rs

@@ -14,35 +14,27 @@ use crate::Opstamp;
 // The doc to opstamp mapping stores precisely an array
 // indexed by doc id and storing the opstamp of the document.
 //
-// This mapping is (for the moment) stricly increasing
-// because of the way document id are allocated.
+// This mapping is NOT necessarily increasing, because
+// we might be sorting documents according to a fast field.
 #[derive(Clone)]
 pub enum DocToOpstampMapping<'a> {
     WithMap(&'a [Opstamp]),
     None,
 }
 
-impl<'a> From<&'a [u64]> for DocToOpstampMapping<'a> {
-    fn from(opstamps: &[Opstamp]) -> DocToOpstampMapping {
-        DocToOpstampMapping::WithMap(opstamps)
-    }
-}
-
 impl<'a> DocToOpstampMapping<'a> {
-    /// Given an opstamp return the limit doc id L
-    /// such that all doc id D such that
-    // D >= L iff opstamp(D) >= than `target_opstamp`.
-    //
-    // The edge case opstamp = some doc opstamp is in practise
-    // never called.
-    pub fn compute_doc_limit(&self, target_opstamp: Opstamp) -> DocId {
-        match *self {
-            DocToOpstampMapping::WithMap(ref doc_opstamps) => {
-                match doc_opstamps.binary_search(&target_opstamp) {
-                    Ok(doc_id) | Err(doc_id) => doc_id as DocId,
-                }
+    /// Assess whether a document should be considered deleted given that it contains
+    /// a deleted term that was deleted at the opstamp: `delete_opstamp`.
+    ///
+    /// This function returns true if the `DocToOpstamp` mapping is none or if
+    /// the `doc_opstamp` is anterior to the delete opstamp.
+    pub fn is_deleted(&self, doc_id: DocId, delete_opstamp: Opstamp) -> bool {
+        match self {
+            Self::WithMap(doc_opstamps) => {
+                let doc_opstamp = doc_opstamps[doc_id as usize];
+                doc_opstamp < delete_opstamp
             }
-            DocToOpstampMapping::None => DocId::max_value(),
+            Self::None => true,
         }
     }
 }
@@ -55,40 +47,17 @@ mod tests {
     #[test]
     fn test_doc_to_opstamp_mapping_none() {
         let doc_to_opstamp_mapping = DocToOpstampMapping::None;
-        assert_eq!(
-            doc_to_opstamp_mapping.compute_doc_limit(1),
-            u32::max_value()
-        );
+        assert!(doc_to_opstamp_mapping.is_deleted(1u32, 0u64));
+        assert!(doc_to_opstamp_mapping.is_deleted(1u32, 2u64));
     }
 
     #[test]
-    fn test_doc_to_opstamp_mapping_complex() {
-        {
-            let doc_to_opstamp_mapping = DocToOpstampMapping::from(&[][..]);
-            assert_eq!(doc_to_opstamp_mapping.compute_doc_limit(0u64), 0);
-            assert_eq!(doc_to_opstamp_mapping.compute_doc_limit(2u64), 0);
-        }
-        {
-            let doc_to_opstamp_mapping = DocToOpstampMapping::from(&[1u64][..]);
-            assert_eq!(doc_to_opstamp_mapping.compute_doc_limit(0u64), 0);
-            assert_eq!(doc_to_opstamp_mapping.compute_doc_limit(2u64), 1);
-        }
-        {
-            let doc_to_opstamp_mapping =
-                DocToOpstampMapping::from(&[1u64, 12u64, 17u64, 23u64][..]);
-            assert_eq!(doc_to_opstamp_mapping.compute_doc_limit(0u64), 0);
-            for i in 2u64..13u64 {
-                assert_eq!(doc_to_opstamp_mapping.compute_doc_limit(i), 1);
-            }
-            for i in 13u64..18u64 {
-                assert_eq!(doc_to_opstamp_mapping.compute_doc_limit(i), 2);
-            }
-            for i in 18u64..24u64 {
-                assert_eq!(doc_to_opstamp_mapping.compute_doc_limit(i), 3);
-            }
-            for i in 24u64..30u64 {
-                assert_eq!(doc_to_opstamp_mapping.compute_doc_limit(i), 4);
-            }
-        }
+    fn test_doc_to_opstamp_mapping_with_map() {
+        let doc_to_opstamp_mapping = DocToOpstampMapping::WithMap(&[5u64, 1u64, 0u64, 4u64, 3u64]);
+        assert_eq!(doc_to_opstamp_mapping.is_deleted(0u32, 2u64), false);
+        assert_eq!(doc_to_opstamp_mapping.is_deleted(1u32, 2u64), true);
+        assert_eq!(doc_to_opstamp_mapping.is_deleted(2u32, 2u64), true);
+        assert_eq!(doc_to_opstamp_mapping.is_deleted(3u32, 2u64), false);
+        assert_eq!(doc_to_opstamp_mapping.is_deleted(4u32, 2u64), false);
     }
 }

src/indexer/index_writer.rs

@@ -1,7 +1,6 @@
 use super::operation::{AddOperation, UserOperation};
 use super::segment_updater::SegmentUpdater;
 use super::PreparedCommit;
-use crate::common::BitSet;
 use crate::core::Index;
 use crate::core::Segment;
 use crate::core::SegmentComponent;
@@ -12,7 +11,7 @@ use crate::directory::TerminatingWrite;
 use crate::directory::{DirectoryLock, GarbageCollectionResult};
 use crate::docset::{DocSet, TERMINATED};
 use crate::error::TantivyError;
-use crate::fastfield::write_delete_bitset;
+use crate::fastfield::write_alive_bitset;
 use crate::indexer::delete_queue::{DeleteCursor, DeleteQueue};
 use crate::indexer::doc_opstamp_mapping::DocToOpstampMapping;
 use crate::indexer::operation::DeleteOperation;
@@ -24,14 +23,18 @@ use crate::schema::Document;
 use crate::schema::IndexRecordOption;
 use crate::schema::Term;
 use crate::Opstamp;
+use common::BitSet;
 use crossbeam::channel;
 use futures::executor::block_on;
 use futures::future::Future;
 use smallvec::smallvec;
 use smallvec::SmallVec;
+use wasm_mt_pool::pool_exec;
+use wasm_mt::prelude::*;
 use std::mem;
 use std::ops::Range;
 use std::sync::Arc;
+use wasm_mt_pool::prelude::*;
 use std::thread;
 use std::thread::JoinHandle;
 
@@ -75,7 +78,7 @@ pub struct IndexWriter {
 
     heap_size_in_bytes_per_thread: usize,
 
-    workers_join_handle: Vec<JoinHandle<crate::Result<()>>>,
+    workers_join_handle: Vec<JoinHandle<Result<JsValue, JsValue>>>,
 
     operation_receiver: OperationReceiver,
     operation_sender: OperationSender,
@@ -90,10 +93,12 @@ pub struct IndexWriter {
 
     stamper: Stamper,
     committed_opstamp: Opstamp,
+
+    worker_pool: wasm_mt_pool::ThreadPool,
 }
 
 fn compute_deleted_bitset(
-    delete_bitset: &mut BitSet,
+    alive_bitset: &mut BitSet,
     segment_reader: &SegmentReader,
     delete_cursor: &mut DeleteCursor,
     doc_opstamps: &DocToOpstampMapping,
@@ -106,22 +111,18 @@ fn compute_deleted_bitset(
         }
 
         // A delete operation should only affect
-        // document that were inserted after it.
-        //
-        // Limit doc helps identify the first document
-        // that may be affected by the delete operation.
-        let limit_doc = doc_opstamps.compute_doc_limit(delete_op.opstamp);
+        // document that were inserted before it.
         let inverted_index = segment_reader.inverted_index(delete_op.term.field())?;
         if let Some(mut docset) =
             inverted_index.read_postings(&delete_op.term, IndexRecordOption::Basic)?
         {
-            let mut deleted_doc = docset.doc();
-            while deleted_doc != TERMINATED {
-                if deleted_doc < limit_doc {
-                    delete_bitset.insert(deleted_doc);
+            let mut doc_matching_deleted_term = docset.doc();
+            while doc_matching_deleted_term != TERMINATED {
+                if doc_opstamps.is_deleted(doc_matching_deleted_term, delete_op.opstamp) {
+                    alive_bitset.remove(doc_matching_deleted_term);
                     might_have_changed = true;
                 }
-                deleted_doc = docset.advance();
+                doc_matching_deleted_term = docset.advance();
             }
         }
         delete_cursor.advance();
@@ -145,7 +146,7 @@ pub(crate) fn advance_deletes(
         return Ok(());
     }
 
-    if segment_entry.delete_bitset().is_none() && segment_entry.delete_cursor().get().is_none() {
+    if segment_entry.alive_bitset().is_none() && segment_entry.delete_cursor().get().is_none() {
         // There has been no `DeleteOperation` between the segment status and `target_opstamp`.
         return Ok(());
     }
@@ -153,15 +154,15 @@ pub(crate) fn advance_deletes(
     let segment_reader = SegmentReader::open(&segment)?;
 
     let max_doc = segment_reader.max_doc();
-    let mut delete_bitset: BitSet = match segment_entry.delete_bitset() {
-        Some(previous_delete_bitset) => (*previous_delete_bitset).clone(),
-        None => BitSet::with_max_value(max_doc),
+    let mut alive_bitset: BitSet = match segment_entry.alive_bitset() {
+        Some(previous_alive_bitset) => (*previous_alive_bitset).clone(),
+        None => BitSet::with_max_value_and_full(max_doc),
     };
 
     let num_deleted_docs_before = segment.meta().num_deleted_docs();
 
     compute_deleted_bitset(
-        &mut delete_bitset,
+        &mut alive_bitset,
         &segment_reader,
         segment_entry.delete_cursor(),
         &DocToOpstampMapping::None,
@@ -171,20 +172,21 @@ pub(crate) fn advance_deletes(
     // TODO optimize
     // It should be possible to do something smarter by manipulation bitsets directly
     // to compute this union.
-    if let Some(seg_delete_bitset) = segment_reader.delete_bitset() {
+    if let Some(seg_alive_bitset) = segment_reader.alive_bitset() {
         for doc in 0u32..max_doc {
-            if seg_delete_bitset.is_deleted(doc) {
-                delete_bitset.insert(doc);
+            if seg_alive_bitset.is_deleted(doc) {
+                alive_bitset.remove(doc);
             }
         }
     }
 
-    let num_deleted_docs: u32 = delete_bitset.len() as u32;
+    let num_alive_docs: u32 = alive_bitset.len() as u32;
+    let num_deleted_docs = max_doc - num_alive_docs;
     if num_deleted_docs > num_deleted_docs_before {
         // There are new deletes. We need to write a new delete file.
         segment = segment.with_delete_meta(num_deleted_docs as u32, target_opstamp);
         let mut delete_file = segment.open_write(SegmentComponent::Delete)?;
-        write_delete_bitset(&delete_bitset, max_doc, &mut delete_file)?;
+        write_alive_bitset(&alive_bitset, &mut delete_file)?;
         delete_file.terminate()?;
     }
 
@@ -230,45 +232,45 @@ fn index_documents(
 
     let segment_with_max_doc = segment.with_max_doc(max_doc);
 
-    let last_docstamp: Opstamp = *(doc_opstamps.last().unwrap());
-
-    let delete_bitset_opt = apply_deletes(
-        &segment_with_max_doc,
-        &mut delete_cursor,
-        &doc_opstamps,
-        last_docstamp,
-    )?;
+    let alive_bitset_opt = apply_deletes(&segment_with_max_doc, &mut delete_cursor, &doc_opstamps)?;
 
     let meta = segment_with_max_doc.meta().clone();
     meta.untrack_temp_docstore();
     // update segment_updater inventory to remove tempstore
-    let segment_entry = SegmentEntry::new(meta, delete_cursor, delete_bitset_opt);
+    let segment_entry = SegmentEntry::new(meta, delete_cursor, alive_bitset_opt);
     block_on(segment_updater.schedule_add_segment(segment_entry))?;
     Ok(true)
 }
 
+/// `doc_opstamps` is required to be non-empty.
 fn apply_deletes(
     segment: &Segment,
     mut delete_cursor: &mut DeleteCursor,
     doc_opstamps: &[Opstamp],
-    last_docstamp: Opstamp,
 ) -> crate::Result<Option<BitSet>> {
     if delete_cursor.get().is_none() {
         // if there are no delete operation in the queue, no need
         // to even open the segment.
         return Ok(None);
     }
+
+    let max_doc_opstamp: Opstamp = doc_opstamps
+        .iter()
+        .cloned()
+        .max()
+        .expect("Empty DocOpstamp is forbidden");
+
     let segment_reader = SegmentReader::open(segment)?;
-    let doc_to_opstamps = DocToOpstampMapping::from(doc_opstamps);
+    let doc_to_opstamps = DocToOpstampMapping::WithMap(doc_opstamps);
 
     let max_doc = segment.meta().max_doc();
-    let mut deleted_bitset = BitSet::with_max_value(max_doc);
+    let mut deleted_bitset = BitSet::with_max_value_and_full(max_doc);
     let may_have_deletes = compute_deleted_bitset(
         &mut deleted_bitset,
         &segment_reader,
         &mut delete_cursor,
         &doc_to_opstamps,
-        last_docstamp,
+        max_doc_opstamp,
     )?;
     Ok(if may_have_deletes {
         Some(deleted_bitset)
@@ -321,6 +323,7 @@ impl IndexWriter {
         let segment_updater =
             SegmentUpdater::create(index.clone(), stamper.clone(), &delete_queue.cursor())?;
 
+        let worker_pool = block_on(wasm_mt_pool::ThreadPool::new(num_threads, crate::PKG_JS).and_init()).unwrap();
         let mut index_writer = IndexWriter {
             _directory_lock: Some(directory_lock),
 
@@ -341,6 +344,7 @@ impl IndexWriter {
             stamper,
 
             worker_id: 0,
+            worker_pool,
         };
         index_writer.start_workers()?;
         Ok(index_writer)
@@ -351,6 +355,11 @@ impl IndexWriter {
         self.operation_sender = sender;
     }
 
+    /// Accessor to the index.
+    pub fn index(&self) -> &Index {
+        &self.index
+    }
+
     /// If there are some merging threads, blocks until they all finish their work and
     /// then drop the `IndexWriter`.
     pub fn wait_merging_threads(mut self) -> crate::Result<()> {
@@ -358,7 +367,7 @@ impl IndexWriter {
         // dropping the last reference to the segment_updater.
         self.drop_sender();
 
-        let former_workers_handles = mem::replace(&mut self.workers_join_handle, vec![]);
+        let former_workers_handles = std::mem::take(&mut self.workers_join_handle);
         for join_handle in former_workers_handles {
             join_handle
                 .join()
@@ -409,9 +418,8 @@ impl IndexWriter {
 
         let mem_budget = self.heap_size_in_bytes_per_thread;
         let index = self.index.clone();
-        let join_handle: JoinHandle<crate::Result<()>> = thread::Builder::new()
-            .name(format!("thrd-tantivy-index{}", self.worker_id))
-            .spawn(move || {
+        let join_handle: JoinHandle<crate::Result<_>> = pool_exec!(self.worker_pool,
+            move || {
                 loop {
                     let mut document_iterator =
                         document_receiver_clone.clone().into_iter().peekable();
@@ -628,7 +636,7 @@ impl IndexWriter {
         // and recreate a new one.
         self.recreate_document_channel();
 
-        let former_workers_join_handle = mem::replace(&mut self.workers_join_handle, Vec::new());
+        let former_workers_join_handle = std::mem::take(&mut self.workers_join_handle);
 
         for worker_handle in former_workers_join_handle {
             let indexing_worker_result = worker_handle
@@ -784,17 +792,44 @@ impl Drop for IndexWriter {
 
 #[cfg(test)]
 mod tests {
+    use std::collections::HashMap;
+    use std::collections::HashSet;
+
+    use futures::executor::block_on;
+    use proptest::prelude::*;
+    use proptest::prop_oneof;
+    use proptest::strategy::Strategy;
 
     use super::super::operation::UserOperation;
     use crate::collector::TopDocs;
     use crate::directory::error::LockError;
     use crate::error::*;
+    use crate::fastfield::FastFieldReader;
     use crate::indexer::NoMergePolicy;
+    use crate::query::QueryParser;
     use crate::query::TermQuery;
-    use crate::schema::{self, IndexRecordOption, STRING};
+    use crate::schema::Cardinality;
+    use crate::schema::Facet;
+    use crate::schema::IntOptions;
+    use crate::schema::TextFieldIndexing;
+    use crate::schema::TextOptions;
+    use crate::schema::STORED;
+    use crate::schema::TEXT;
+    use crate::schema::{self, IndexRecordOption, FAST, INDEXED, STRING};
+    use crate::DocAddress;
     use crate::Index;
     use crate::ReloadPolicy;
     use crate::Term;
+    use crate::{IndexSettings, IndexSortByField, Order};
+
+    const LOREM: &str = "Doc Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed \
+             do eiusmod tempor incididunt ut labore et dolore magna aliqua. \
+             Ut enim ad minim veniam, quis nostrud exercitation ullamco \
+             laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure \
+             dolor in reprehenderit in voluptate velit esse cillum dolore eu \
+             fugiat nulla pariatur. Excepteur sint occaecat cupidatat non \
+             proident, sunt in culpa qui officia deserunt mollit anim id est \
+             laborum.";
 
     #[test]
     fn test_operations_group() {
@@ -825,7 +860,7 @@ mod tests {
         let reader = index.reader().unwrap();
         let searcher = reader.searcher();
         assert_eq!(searcher.segment_readers().len(), 1);
-        assert_eq!(searcher.segment_reader(0u32).num_deleted_docs(), 0);
+        assert_eq!(searcher.segment_reader(0u32).num_docs(), 2);
 
         index_writer.delete_term(Term::from_field_text(text_field, "hello1"));
         assert!(index_writer.commit().is_ok());
@@ -833,7 +868,7 @@ mod tests {
         assert!(reader.reload().is_ok());
         let searcher = reader.searcher();
         assert_eq!(searcher.segment_readers().len(), 1);
-        assert_eq!(searcher.segment_reader(0u32).num_deleted_docs(), 1);
+        assert_eq!(searcher.segment_reader(0u32).num_docs(), 1);
 
         let previous_delete_opstamp = index.load_metas().unwrap().segments[0].delete_opstamp();
 
@@ -845,7 +880,7 @@ mod tests {
         assert!(reader.reload().is_ok());
         let searcher = reader.searcher();
         assert_eq!(searcher.segment_readers().len(), 1);
-        assert_eq!(searcher.segment_reader(0u32).num_deleted_docs(), 1);
+        assert_eq!(searcher.segment_reader(0u32).num_docs(), 1);
 
         let after_delete_opstamp = index.load_metas().unwrap().segments[0].delete_opstamp();
         assert_eq!(after_delete_opstamp, previous_delete_opstamp);
@@ -1282,6 +1317,343 @@ mod tests {
         assert!(commit_again.is_ok());
     }
 
+    #[test]
+    fn test_delete_with_sort_by_field() -> crate::Result<()> {
+        let mut schema_builder = schema::Schema::builder();
+        let id_field =
+            schema_builder.add_u64_field("id", schema::INDEXED | schema::STORED | schema::FAST);
+        let schema = schema_builder.build();
+
+        let settings = IndexSettings {
+            sort_by_field: Some(IndexSortByField {
+                field: "id".to_string(),
+                order: Order::Desc,
+            }),
+            ..Default::default()
+        };
+
+        let index = Index::builder()
+            .schema(schema)
+            .settings(settings)
+            .create_in_ram()?;
+        let index_reader = index.reader()?;
+        let mut index_writer = index.writer_for_tests()?;
+
+        // create and delete docs in same commit
+        for id in 0u64..5u64 {
+            index_writer.add_document(doc!(id_field => id));
+        }
+        for id in 2u64..4u64 {
+            index_writer.delete_term(Term::from_field_u64(id_field, id));
+        }
+        for id in 5u64..10u64 {
+            index_writer.add_document(doc!(id_field => id));
+        }
+        index_writer.commit()?;
+        index_reader.reload()?;
+
+        let searcher = index_reader.searcher();
+        assert_eq!(searcher.segment_readers().len(), 1);
+
+        let segment_reader = searcher.segment_reader(0);
+        assert_eq!(segment_reader.num_docs(), 8);
+        assert_eq!(segment_reader.max_doc(), 10);
+        let fast_field_reader = segment_reader.fast_fields().u64(id_field)?;
+        let in_order_alive_ids: Vec<u64> = segment_reader
+            .doc_ids_alive()
+            .map(|doc| fast_field_reader.get(doc))
+            .collect();
+        assert_eq!(&in_order_alive_ids[..], &[9, 8, 7, 6, 5, 4, 1, 0]);
+        Ok(())
+    }
+
+    #[derive(Debug, Clone, Copy)]
+    enum IndexingOp {
+        AddDoc { id: u64 },
+        DeleteDoc { id: u64 },
+        Commit,
+        Merge,
+    }
+
+    fn operation_strategy() -> impl Strategy<Value = IndexingOp> {
+        prop_oneof![
+            (0u64..10u64).prop_map(|id| IndexingOp::DeleteDoc { id }),
+            (0u64..10u64).prop_map(|id| IndexingOp::AddDoc { id }),
+            (0u64..2u64).prop_map(|_| IndexingOp::Commit),
+            (0u64..1u64).prop_map(|_| IndexingOp::Merge),
+        ]
+    }
+
+    fn expected_ids(ops: &[IndexingOp]) -> (HashMap<u64, u64>, HashSet<u64>) {
+        let mut existing_ids = HashMap::new();
+        let mut deleted_ids = HashSet::new();
+        for &op in ops {
+            match op {
+                IndexingOp::AddDoc { id } => {
+                    *existing_ids.entry(id).or_insert(0) += 1;
+                    deleted_ids.remove(&id);
+                }
+                IndexingOp::DeleteDoc { id } => {
+                    existing_ids.remove(&id);
+                    deleted_ids.insert(id);
+                }
+                _ => {}
+            }
+        }
+        (existing_ids, deleted_ids)
+    }
+
+    fn test_operation_strategy(
+        ops: &[IndexingOp],
+        sort_index: bool,
+        force_end_merge: bool,
+    ) -> crate::Result<()> {
+        let mut schema_builder = schema::Schema::builder();
+        let id_field = schema_builder.add_u64_field("id", FAST | INDEXED | STORED);
+        let text_field = schema_builder.add_text_field(
+            "text_field",
+            TextOptions::default()
+                .set_indexing_options(
+                    TextFieldIndexing::default()
+                        .set_index_option(schema::IndexRecordOption::WithFreqsAndPositions),
+                )
+                .set_stored(),
+        );
+
+        let large_text_field = schema_builder.add_text_field("large_text_field", TEXT | STORED);
+
+        let multi_numbers = schema_builder.add_u64_field(
+            "multi_numbers",
+            IntOptions::default()
+                .set_fast(Cardinality::MultiValues)
+                .set_stored(),
+        );
+        let facet_field = schema_builder.add_facet_field("facet", INDEXED);
+        let schema = schema_builder.build();
+        let settings = if sort_index {
+            IndexSettings {
+                sort_by_field: Some(IndexSortByField {
+                    field: "id".to_string(),
+                    order: Order::Asc,
+                }),
+                ..Default::default()
+            }
+        } else {
+            IndexSettings {
+                ..Default::default()
+            }
+        };
+        let index = Index::builder()
+            .schema(schema)
+            .settings(settings)
+            .create_in_ram()?;
+        let mut index_writer = index.writer_for_tests()?;
+        index_writer.set_merge_policy(Box::new(NoMergePolicy));
+
+        for &op in ops {
+            match op {
+                IndexingOp::AddDoc { id } => {
+                    let facet = Facet::from(&("/cola/".to_string() + &id.to_string()));
+                    index_writer
+                        .add_document(doc!(id_field=>id, multi_numbers=> id, multi_numbers => id, text_field => id.to_string(), facet_field => facet, large_text_field=> LOREM));
+                }
+                IndexingOp::DeleteDoc { id } => {
+                    index_writer.delete_term(Term::from_field_u64(id_field, id));
+                }
+                IndexingOp::Commit => {
+                    index_writer.commit()?;
+                }
+                IndexingOp::Merge => {
+                    let segment_ids = index
+                        .searchable_segment_ids()
+                        .expect("Searchable segments failed.");
+                    if segment_ids.len() >= 2 {
+                        block_on(index_writer.merge(&segment_ids)).unwrap();
+                        assert!(index_writer.segment_updater().wait_merging_thread().is_ok());
+                    }
+                }
+            }
+        }
+        index_writer.commit()?;
+
+        let searcher = index.reader()?.searcher();
+        if force_end_merge {
+            index_writer.wait_merging_threads()?;
+            let mut index_writer = index.writer_for_tests()?;
+            let segment_ids = index
+                .searchable_segment_ids()
+                .expect("Searchable segments failed.");
+            if segment_ids.len() >= 2 {
+                block_on(index_writer.merge(&segment_ids)).unwrap();
+                assert!(index_writer.wait_merging_threads().is_ok());
+            }
+        }
+        let ids: HashSet<u64> = searcher
+            .segment_readers()
+            .iter()
+            .flat_map(|segment_reader| {
+                let ff_reader = segment_reader.fast_fields().u64(id_field).unwrap();
+                segment_reader
+                    .doc_ids_alive()
+                    .map(move |doc| ff_reader.get(doc))
+            })
+            .collect();
+
+        let (expected_ids_and_num_occurences, deleted_ids) = expected_ids(ops);
+        assert_eq!(
+            ids,
+            expected_ids_and_num_occurences
+                .keys()
+                .cloned()
+                .collect::<HashSet<_>>()
+        );
+
+        // multivalue fast field tests
+        for segment_reader in searcher.segment_readers().iter() {
+            let ff_reader = segment_reader.fast_fields().u64s(multi_numbers).unwrap();
+            for doc in segment_reader.doc_ids_alive() {
+                let mut vals = vec![];
+                ff_reader.get_vals(doc, &mut vals);
+                assert_eq!(vals.len(), 2);
+                assert_eq!(vals[0], vals[1]);
+                assert!(expected_ids_and_num_occurences.contains_key(&vals[0]));
+            }
+        }
+
+        // doc store tests
+        for segment_reader in searcher.segment_readers().iter() {
+            let store_reader = segment_reader.get_store_reader().unwrap();
+            // test store iterator
+            for doc in store_reader.iter(segment_reader.alive_bitset()) {
+                let id = doc
+                    .unwrap()
+                    .get_first(id_field)
+                    .unwrap()
+                    .u64_value()
+                    .unwrap();
+                assert!(expected_ids_and_num_occurences.contains_key(&id));
+            }
+            // test store random access
+            for doc_id in segment_reader.doc_ids_alive() {
+                let id = store_reader
+                    .get(doc_id)
+                    .unwrap()
+                    .get_first(id_field)
+                    .unwrap()
+                    .u64_value()
+                    .unwrap();
+                assert!(expected_ids_and_num_occurences.contains_key(&id));
+                let id2 = store_reader
+                    .get(doc_id)
+                    .unwrap()
+                    .get_first(multi_numbers)
+                    .unwrap()
+                    .u64_value()
+                    .unwrap();
+                assert_eq!(id, id2);
+            }
+        }
+        // test search
+        let my_text_field = index.schema().get_field("text_field").unwrap();
+
+        let do_search = |term: &str| {
+            let query = QueryParser::for_index(&index, vec![my_text_field])
+                .parse_query(term)
+                .unwrap();
+            let top_docs: Vec<(f32, DocAddress)> =
+                searcher.search(&query, &TopDocs::with_limit(1000)).unwrap();
+
+            top_docs.iter().map(|el| el.1).collect::<Vec<_>>()
+        };
+
+        for (existing_id, count) in expected_ids_and_num_occurences {
+            assert_eq!(do_search(&existing_id.to_string()).len() as u64, count);
+        }
+        for existing_id in deleted_ids {
+            assert_eq!(do_search(&existing_id.to_string()).len(), 0);
+        }
+        // test facets
+        for segment_reader in searcher.segment_readers().iter() {
+            let mut facet_reader = segment_reader.facet_reader(facet_field).unwrap();
+            let ff_reader = segment_reader.fast_fields().u64(id_field).unwrap();
+            for doc_id in segment_reader.doc_ids_alive() {
+                let mut facet_ords = Vec::new();
+                facet_reader.facet_ords(doc_id, &mut facet_ords);
+                assert_eq!(facet_ords.len(), 1);
+                let mut facet = Facet::default();
+                facet_reader
+                    .facet_from_ord(facet_ords[0], &mut facet)
+                    .unwrap();
+                let id = ff_reader.get(doc_id);
+                let facet_expected = Facet::from(&("/cola/".to_string() + &id.to_string()));
+
+                assert_eq!(facet, facet_expected);
+            }
+        }
+        Ok(())
+    }
+
+    proptest! {
+        #[test]
+        fn test_delete_with_sort_proptest(ops in proptest::collection::vec(operation_strategy(), 1..10)) {
+            assert!(test_operation_strategy(&ops[..], true, false).is_ok());
+        }
+        #[test]
+        fn test_delete_without_sort_proptest(ops in proptest::collection::vec(operation_strategy(), 1..10)) {
+            assert!(test_operation_strategy(&ops[..], false, false).is_ok());
+        }
+        #[test]
+        fn test_delete_with_sort_proptest_with_merge(ops in proptest::collection::vec(operation_strategy(), 1..10)) {
+            assert!(test_operation_strategy(&ops[..], true, true).is_ok());
+        }
+        #[test]
+        fn test_delete_without_sort_proptest_with_merge(ops in proptest::collection::vec(operation_strategy(), 1..10)) {
+            assert!(test_operation_strategy(&ops[..], false, true).is_ok());
+        }
+    }
+
+    #[test]
+    fn test_delete_with_sort_by_field_last_opstamp_is_not_max() -> crate::Result<()> {
+        let mut schema_builder = schema::Schema::builder();
+        let sort_by_field = schema_builder.add_u64_field("sort_by", FAST);
+        let id_field = schema_builder.add_u64_field("id", INDEXED);
+        let schema = schema_builder.build();
+
+        let settings = IndexSettings {
+            sort_by_field: Some(IndexSortByField {
+                field: "sort_by".to_string(),
+                order: Order::Asc,
+            }),
+            ..Default::default()
+        };
+
+        let index = Index::builder()
+            .schema(schema)
+            .settings(settings)
+            .create_in_ram()?;
+        let mut index_writer = index.writer_for_tests()?;
+
+        // We add a doc...
+        index_writer.add_document(doc!(sort_by_field => 2u64, id_field => 0u64));
+        // And remove it.
+        index_writer.delete_term(Term::from_field_u64(id_field, 0u64));
+        // We add another doc.
+        index_writer.add_document(doc!(sort_by_field=>1u64, id_field => 0u64));
+
+        // The expected result is a segment with
+        // maxdoc = 2
+        // numdoc = 1.
+        index_writer.commit()?;
+
+        let searcher = index.reader()?.searcher();
+        assert_eq!(searcher.segment_readers().len(), 1);
+
+        let segment_reader = searcher.segment_reader(0);
+        assert_eq!(segment_reader.max_doc(), 2);
+        assert_eq!(segment_reader.num_docs(), 1);
+        Ok(())
+    }
+
     #[test]
     fn test_index_doc_missing_field() {
         let mut schema_builder = schema::Schema::builder();

src/indexer/merger_sorted_index_test.rs

@@ -1,6 +1,7 @@
 #[cfg(test)]
 mod tests {
-    use crate::fastfield::FastFieldReader;
+    use crate::fastfield::{AliveBitSet, FastFieldReader};
+    use crate::schema::IndexRecordOption;
     use crate::{
         collector::TopDocs,
         schema::{Cardinality, TextFieldIndexing},
@@ -16,7 +17,7 @@ mod tests {
         schema::{self, BytesOptions},
         DocAddress,
     };
-    use crate::{IndexSettings, Term};
+    use crate::{DocSet, IndexSettings, Postings, Term};
     use futures::executor::block_on;
 
     fn create_test_index_posting_list_issue(index_settings: Option<IndexSettings>) -> Index {
@@ -104,9 +105,11 @@ mod tests {
             index_writer.add_document(
                 doc!(int_field=>3_u64, multi_numbers => 3_u64, multi_numbers => 4_u64, bytes_field => vec![1, 2, 3], text_field => "some text", facet_field=> Facet::from("/book/crime")),
             );
-            index_writer.add_document(doc!(int_field=>1_u64, text_field=> "deleteme"));
             index_writer.add_document(
-                doc!(int_field=>2_u64, multi_numbers => 2_u64, multi_numbers => 3_u64),
+                doc!(int_field=>1_u64, text_field=> "deleteme",  text_field => "ok text more text"),
+            );
+            index_writer.add_document(
+                doc!(int_field=>2_u64, multi_numbers => 2_u64, multi_numbers => 3_u64, text_field => "ok text more text"),
             );
 
             assert!(index_writer.commit().is_ok());
@@ -118,7 +121,7 @@ mod tests {
             } else {
                 1
             };
-            index_writer.add_document(doc!(int_field=>in_val, text_field=> "deleteme", facet_field=> Facet::from("/book/crime")));
+            index_writer.add_document(doc!(int_field=>in_val, text_field=> "deleteme" , text_field => "ok text more text", facet_field=> Facet::from("/book/crime")));
             assert!(index_writer.commit().is_ok());
             // segment 3 - range 5-1000, with force_disjunct_segment_sort_values 50-1000
             let int_vals = if force_disjunct_segment_sort_values {
@@ -243,6 +246,36 @@ mod tests {
             assert_eq!(do_search("biggest"), vec![0]);
         }
 
+        // postings file
+        {
+            let my_text_field = index.schema().get_field("text_field").unwrap();
+            let term_a = Term::from_field_text(my_text_field, "text");
+            let inverted_index = segment_reader.inverted_index(my_text_field).unwrap();
+            let mut postings = inverted_index
+                .read_postings(&term_a, IndexRecordOption::WithFreqsAndPositions)
+                .unwrap()
+                .unwrap();
+
+            assert_eq!(postings.doc_freq(), 2);
+            let fallback_bitset = AliveBitSet::for_test_from_deleted_docs(&[0], 100);
+            assert_eq!(
+                postings.doc_freq_given_deletes(
+                    segment_reader.alive_bitset().unwrap_or(&fallback_bitset)
+                ),
+                2
+            );
+
+            assert_eq!(postings.term_freq(), 1);
+            let mut output = vec![];
+            postings.positions(&mut output);
+            assert_eq!(output, vec![1]);
+            postings.advance();
+
+            assert_eq!(postings.term_freq(), 2);
+            postings.positions(&mut output);
+            assert_eq!(output, vec![1, 3]);
+        }
+
         // access doc store
         {
             let blubber_pos = if force_disjunct_segment_sort_values {
@@ -260,6 +293,69 @@ mod tests {
         }
     }
 
+    #[test]
+    fn test_merge_unsorted_index() {
+        let index = create_test_index(
+            Some(IndexSettings {
+                ..Default::default()
+            }),
+            false,
+        );
+
+        let reader = index.reader().unwrap();
+        let searcher = reader.searcher();
+        assert_eq!(searcher.segment_readers().len(), 1);
+        let segment_reader = searcher.segment_readers().last().unwrap();
+
+        let searcher = index.reader().unwrap().searcher();
+        {
+            let my_text_field = index.schema().get_field("text_field").unwrap();
+
+            let do_search = |term: &str| {
+                let query = QueryParser::for_index(&index, vec![my_text_field])
+                    .parse_query(term)
+                    .unwrap();
+                let top_docs: Vec<(f32, DocAddress)> =
+                    searcher.search(&query, &TopDocs::with_limit(3)).unwrap();
+
+                top_docs.iter().map(|el| el.1.doc_id).collect::<Vec<_>>()
+            };
+
+            assert_eq!(do_search("some"), vec![1]);
+            assert_eq!(do_search("blubber"), vec![3]);
+            assert_eq!(do_search("biggest"), vec![4]);
+        }
+
+        // postings file
+        {
+            let my_text_field = index.schema().get_field("text_field").unwrap();
+            let term_a = Term::from_field_text(my_text_field, "text");
+            let inverted_index = segment_reader.inverted_index(my_text_field).unwrap();
+            let mut postings = inverted_index
+                .read_postings(&term_a, IndexRecordOption::WithFreqsAndPositions)
+                .unwrap()
+                .unwrap();
+            assert_eq!(postings.doc_freq(), 2);
+            let fallback_bitset = AliveBitSet::for_test_from_deleted_docs(&[0], 100);
+            assert_eq!(
+                postings.doc_freq_given_deletes(
+                    segment_reader.alive_bitset().unwrap_or(&fallback_bitset)
+                ),
+                2
+            );
+
+            assert_eq!(postings.term_freq(), 1);
+            let mut output = vec![];
+            postings.positions(&mut output);
+            assert_eq!(output, vec![1]);
+            postings.advance();
+
+            assert_eq!(postings.term_freq(), 2);
+            postings.positions(&mut output);
+            assert_eq!(output, vec![1, 3]);
+        }
+    }
+
     #[test]
     fn test_merge_sorted_index_asc() {
         let index = create_test_index(
@@ -314,7 +410,7 @@ mod tests {
             let my_text_field = index.schema().get_field("text_field").unwrap();
             let fieldnorm_reader = segment_reader.get_fieldnorms_reader(my_text_field).unwrap();
             assert_eq!(fieldnorm_reader.fieldnorm(0), 0);
-            assert_eq!(fieldnorm_reader.fieldnorm(1), 0);
+            assert_eq!(fieldnorm_reader.fieldnorm(1), 4);
             assert_eq!(fieldnorm_reader.fieldnorm(2), 2); // some text
             assert_eq!(fieldnorm_reader.fieldnorm(3), 1);
             assert_eq!(fieldnorm_reader.fieldnorm(5), 3); // the biggest num
@@ -339,6 +435,34 @@ mod tests {
             assert_eq!(do_search("biggest"), vec![5]);
         }
 
+        // postings file
+        {
+            let my_text_field = index.schema().get_field("text_field").unwrap();
+            let term_a = Term::from_field_text(my_text_field, "text");
+            let inverted_index = segment_reader.inverted_index(my_text_field).unwrap();
+            let mut postings = inverted_index
+                .read_postings(&term_a, IndexRecordOption::WithFreqsAndPositions)
+                .unwrap()
+                .unwrap();
+
+            assert_eq!(postings.doc_freq(), 2);
+            let fallback_bitset = AliveBitSet::for_test_from_deleted_docs(&[0], 100);
+            assert_eq!(
+                postings.doc_freq_given_deletes(
+                    segment_reader.alive_bitset().unwrap_or(&fallback_bitset)
+                ),
+                2
+            );
+
+            let mut output = vec![];
+            postings.positions(&mut output);
+            assert_eq!(output, vec![1, 3]);
+            postings.advance();
+
+            postings.positions(&mut output);
+            assert_eq!(output, vec![1]);
+        }
+
         // access doc store
         {
             let doc = searcher.doc(DocAddress::new(0, 0)).unwrap();
@@ -362,6 +486,7 @@ mod bench_sorted_index_merge {
 
     use crate::core::Index;
     //use cratedoc_id, readerdoc_id_mappinglet vals = reader.fate::schema;
+    use crate::fastfield::DynamicFastFieldReader;
     use crate::fastfield::FastFieldReader;
     use crate::indexer::merger::IndexMerger;
     use crate::schema::Cardinality;
@@ -421,8 +546,9 @@ mod bench_sorted_index_merge {
         let doc_id_mapping = merger.generate_doc_id_mapping(&sort_by_field).unwrap();
         b.iter(|| {
 
-            let sorted_doc_ids = doc_id_mapping.iter().map(|(doc_id, reader)|{
-            let u64_reader: FastFieldReader<u64> = reader.reader
+            let sorted_doc_ids = doc_id_mapping.iter().map(|(doc_id, ordinal)|{
+            let reader = &merger.readers[*ordinal as usize];
+            let u64_reader: DynamicFastFieldReader<u64> = reader
                 .fast_fields()
                 .typed_fast_field_reader(field)
                 .expect("Failed to find a reader for single fast field. This is a tantivy bug and it should never happen.");

src/indexer/segment_entry.rs

@@ -1,7 +1,7 @@
-use crate::common::BitSet;
 use crate::core::SegmentId;
 use crate::core::SegmentMeta;
 use crate::indexer::delete_queue::DeleteCursor;
+use common::BitSet;
 use std::fmt;
 
 /// A segment entry describes the state of
@@ -9,18 +9,16 @@ use std::fmt;
 ///
 /// In addition to segment `meta`,
 /// it contains a few transient states
-/// - `state` expresses whether the segment is already in the
-/// middle of a merge
-/// - `delete_bitset` is a bitset describing
-/// documents that were deleted during the commit
+/// - `alive_bitset` is a bitset describing
+/// documents that were alive during the commit
 /// itself.
 /// - `delete_cursor` is the position in the delete queue.
 /// Deletes happening before the cursor are reflected either
-/// in the .del file or in the `delete_bitset`.
+/// in the .del file or in the `alive_bitset`.
 #[derive(Clone)]
 pub struct SegmentEntry {
     meta: SegmentMeta,
-    delete_bitset: Option<BitSet>,
+    alive_bitset: Option<BitSet>,
     delete_cursor: DeleteCursor,
 }
 
@@ -29,11 +27,11 @@ impl SegmentEntry {
     pub fn new(
         segment_meta: SegmentMeta,
         delete_cursor: DeleteCursor,
-        delete_bitset: Option<BitSet>,
+        alive_bitset: Option<BitSet>,
     ) -> SegmentEntry {
         SegmentEntry {
             meta: segment_meta,
-            delete_bitset,
+            alive_bitset,
             delete_cursor,
         }
     }
@@ -41,8 +39,8 @@ impl SegmentEntry {
     /// Return a reference to the segment entry deleted bitset.
     ///
     /// `DocId` in this bitset are flagged as deleted.
-    pub fn delete_bitset(&self) -> Option<&BitSet> {
-        self.delete_bitset.as_ref()
+    pub fn alive_bitset(&self) -> Option<&BitSet> {
+        self.alive_bitset.as_ref()
     }
 
     /// Set the `SegmentMeta` for this segment.

src/indexer/segment_manager.rs

@@ -32,6 +32,12 @@ impl SegmentRegisters {
         } else if self.committed.contains_all(segment_ids) {
             Some(SegmentsStatus::Committed)
         } else {
+            warn!(
+                "segment_ids: {:?}, committed_ids: {:?}, uncommitted_ids {:?}",
+                segment_ids,
+                self.committed.segment_ids(),
+                self.uncommitted.segment_ids()
+            );
             None
         }
     }
@@ -58,21 +64,6 @@ impl Debug for SegmentManager {
     }
 }
 
-pub fn get_mergeable_segments(
-    in_merge_segment_ids: &HashSet<SegmentId>,
-    segment_manager: &SegmentManager,
-) -> (Vec<SegmentMeta>, Vec<SegmentMeta>) {
-    let registers_lock = segment_manager.read();
-    (
-        registers_lock
-            .committed
-            .get_mergeable_segments(in_merge_segment_ids),
-        registers_lock
-            .uncommitted
-            .get_mergeable_segments(in_merge_segment_ids),
-    )
-}
-
 impl SegmentManager {
     pub fn from_segments(
         segment_metas: Vec<SegmentMeta>,
@@ -86,6 +77,20 @@ impl SegmentManager {
         }
     }
 
+    pub fn get_mergeable_segments(
+        &self,
+        in_merge_segment_ids: &HashSet<SegmentId>,
+    ) -> (Vec<SegmentMeta>, Vec<SegmentMeta>) {
+        let registers_lock = self.read();
+        (
+            registers_lock
+                .committed
+                .get_mergeable_segments(in_merge_segment_ids),
+            registers_lock
+                .uncommitted
+                .get_mergeable_segments(in_merge_segment_ids),
+        )
+    }
     /// Returns all of the segment entries (committed or uncommitted)
     pub fn segment_entries(&self) -> Vec<SegmentEntry> {
         let registers_lock = self.read();

src/indexer/segment_register.rs

@@ -4,6 +4,7 @@ use crate::indexer::delete_queue::DeleteCursor;
 use crate::indexer::segment_entry::SegmentEntry;
 use std::collections::HashMap;
 use std::collections::HashSet;
+use std::fmt::Display;
 use std::fmt::{self, Debug, Formatter};
 
 /// The segment register keeps track
@@ -29,6 +30,16 @@ impl Debug for SegmentRegister {
         Ok(())
     }
 }
+impl Display for SegmentRegister {
+    fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), fmt::Error> {
+        write!(f, "SegmentRegister(")?;
+        for k in self.segment_states.keys() {
+            write!(f, "{}, ", k.short_uuid_string())?;
+        }
+        write!(f, ")")?;
+        Ok(())
+    }
+}
 
 impl SegmentRegister {
     pub fn clear(&mut self) {
@@ -46,6 +57,10 @@ impl SegmentRegister {
             .collect()
     }
 
+    pub fn segment_ids(&self) -> Vec<SegmentId> {
+        self.segment_states.keys().cloned().collect()
+    }
+
     pub fn segment_entries(&self) -> Vec<SegmentEntry> {
         self.segment_states.values().cloned().collect()
     }

src/indexer/segment_updater.rs

@@ -1,11 +1,10 @@
-use super::segment_manager::{get_mergeable_segments, SegmentManager};
+use super::segment_manager::SegmentManager;
 use crate::core::Index;
 use crate::core::IndexMeta;
 use crate::core::IndexSettings;
 use crate::core::Segment;
 use crate::core::SegmentId;
 use crate::core::SegmentMeta;
-use crate::core::SerializableSegment;
 use crate::core::META_FILEPATH;
 use crate::directory::{Directory, DirectoryClone, GarbageCollectionResult};
 use crate::indexer::delete_queue::DeleteCursor;
@@ -140,7 +139,7 @@ fn merge(
     // ... we just serialize this index merger in our new segment to merge the segments.
     let segment_serializer = SegmentSerializer::for_segment(merged_segment.clone(), true)?;
 
-    let num_docs = merger.write(segment_serializer, None)?;
+    let num_docs = merger.write(segment_serializer)?;
 
     let merged_segment_id = merged_segment.id();
 
@@ -209,7 +208,7 @@ pub fn merge_segments<Dir: Directory>(
         &segments[..],
     )?;
     let segment_serializer = SegmentSerializer::for_segment(merged_segment, true)?;
-    let num_docs = merger.write(segment_serializer, None)?;
+    let num_docs = merger.write(segment_serializer)?;
 
     let segment_meta = merged_index.new_segment_meta(merged_segment_id, num_docs);
 
@@ -528,10 +527,14 @@ impl SegmentUpdater {
         }))
     }
 
-    async fn consider_merge_options(&self) {
+    pub(crate) fn get_mergeable_segments(&self) -> (Vec<SegmentMeta>, Vec<SegmentMeta>) {
         let merge_segment_ids: HashSet<SegmentId> = self.merge_operations.segment_in_merge();
-        let (committed_segments, uncommitted_segments) =
-            get_mergeable_segments(&merge_segment_ids, &self.segment_manager);
+        self.segment_manager
+            .get_mergeable_segments(&merge_segment_ids)
+    }
+
+    async fn consider_merge_options(&self) {
+        let (committed_segments, uncommitted_segments) = self.get_mergeable_segments();
 
         // Committed segments cannot be merged with uncommitted_segments.
         // We therefore consider merges using these two sets of segments independently.
@@ -717,7 +720,7 @@ mod tests {
 
         let seg_ids = index.searchable_segment_ids()?;
         // docs exist, should have at least 1 segment
-        assert!(seg_ids.len() > 0);
+        assert!(!seg_ids.is_empty());
 
         let term_vals = vec!["a", "b", "c", "d", "e", "f"];
         for term_val in term_vals {

src/indexer/segment_writer.rs

@@ -12,12 +12,12 @@ use crate::schema::Schema;
 use crate::schema::Term;
 use crate::schema::Value;
 use crate::schema::{Field, FieldEntry};
+use crate::store::StoreReader;
 use crate::tokenizer::{BoxTokenStream, PreTokenizedStream};
 use crate::tokenizer::{FacetTokenizer, TextAnalyzer};
 use crate::tokenizer::{TokenStreamChain, Tokenizer};
 use crate::Opstamp;
 use crate::{core::Segment, store::StoreWriter};
-use crate::{core::SerializableSegment, store::StoreReader};
 use crate::{DocId, SegmentComponent};
 
 /// Computes the initial size of the hash table.
@@ -36,6 +36,20 @@ fn initial_table_size(per_thread_memory_budget: usize) -> crate::Result<usize> {
     }
 }
 
+fn remap_doc_opstamps(
+    opstamps: Vec<Opstamp>,
+    doc_id_mapping_opt: Option<&DocIdMapping>,
+) -> Vec<Opstamp> {
+    if let Some(doc_id_mapping_opt) = doc_id_mapping_opt {
+        doc_id_mapping_opt
+            .iter_old_doc_ids()
+            .map(|doc| opstamps[doc as usize])
+            .collect()
+    } else {
+        opstamps
+    }
+}
+
 /// A `SegmentWriter` is in charge of creating segment index from a
 /// set of documents.
 ///
@@ -112,14 +126,15 @@ impl SegmentWriter {
             .clone()
             .map(|sort_by_field| get_doc_id_mapping_from_field(sort_by_field, &self))
             .transpose()?;
-        write(
+        remap_and_write(
             &self.multifield_postings,
             &self.fast_field_writers,
             &self.fieldnorms_writer,
             self.segment_serializer,
             mapping.as_ref(),
         )?;
-        Ok(self.doc_opstamps)
+        let doc_opstamps = remap_doc_opstamps(self.doc_opstamps, mapping.as_ref());
+        Ok(doc_opstamps)
     }
 
     pub fn mem_usage(&self) -> usize {
@@ -176,7 +191,7 @@ impl SegmentWriter {
                             .process(&mut |token| {
                                 term_buffer.set_text(&token.text);
                                 let unordered_term_id =
-                                    multifield_postings.subscribe(doc_id, &term_buffer);
+                                    multifield_postings.subscribe(doc_id, term_buffer);
                                 unordered_term_id_opt = Some(unordered_term_id);
                             });
                         if let Some(unordered_term_id) = unordered_term_id_opt {
@@ -237,7 +252,7 @@ impl SegmentWriter {
                             .u64_value()
                             .ok_or_else(make_schema_error)?;
                         term_buffer.set_u64(u64_val);
-                        multifield_postings.subscribe(doc_id, &term_buffer);
+                        multifield_postings.subscribe(doc_id, term_buffer);
                     }
                 }
                 FieldType::Date(_) => {
@@ -248,7 +263,7 @@ impl SegmentWriter {
                             .date_value()
                             .ok_or_else(make_schema_error)?;
                         term_buffer.set_i64(date_val.timestamp());
-                        multifield_postings.subscribe(doc_id, &term_buffer);
+                        multifield_postings.subscribe(doc_id, term_buffer);
                     }
                 }
                 FieldType::I64(_) => {
@@ -259,7 +274,7 @@ impl SegmentWriter {
                             .i64_value()
                             .ok_or_else(make_schema_error)?;
                         term_buffer.set_i64(i64_val);
-                        multifield_postings.subscribe(doc_id, &term_buffer);
+                        multifield_postings.subscribe(doc_id, term_buffer);
                     }
                 }
                 FieldType::F64(_) => {
@@ -270,7 +285,7 @@ impl SegmentWriter {
                             .f64_value()
                             .ok_or_else(make_schema_error)?;
                         term_buffer.set_f64(f64_val);
-                        multifield_postings.subscribe(doc_id, &term_buffer);
+                        multifield_postings.subscribe(doc_id, term_buffer);
                     }
                 }
                 FieldType::Bytes(_) => {
@@ -281,7 +296,7 @@ impl SegmentWriter {
                             .bytes_value()
                             .ok_or_else(make_schema_error)?;
                         term_buffer.set_bytes(bytes);
-                        self.multifield_postings.subscribe(doc_id, &term_buffer);
+                        self.multifield_postings.subscribe(doc_id, term_buffer);
                     }
                 }
             }
@@ -315,8 +330,12 @@ impl SegmentWriter {
     }
 }
 
-// This method is used as a trick to workaround the borrow checker
-fn write(
+/// This method is used as a trick to workaround the borrow checker
+/// Writes a view of a segment by pushing information
+/// to the `SegmentSerializer`.
+///
+/// `doc_id_map` is used to map to the new doc_id order.
+fn remap_and_write(
     multifield_postings: &MultiFieldPostingsWriter,
     fast_field_writers: &FastFieldsWriter,
     fieldnorms_writer: &FieldNormsWriter,
@@ -340,6 +359,7 @@ fn write(
         &term_ord_map,
         doc_id_map,
     )?;
+
     // finalize temp docstore and create version, which reflects the doc_id_map
     if let Some(doc_id_map) = doc_id_map {
         let store_write = serializer
@@ -356,31 +376,16 @@ fn write(
                 .segment()
                 .open_read(SegmentComponent::TempStore)?,
         )?;
+
         for old_doc_id in doc_id_map.iter_old_doc_ids() {
-            let doc_bytes = store_read.get_document_bytes(*old_doc_id)?;
+            let doc_bytes = store_read.get_document_bytes(old_doc_id)?;
             serializer.get_store_writer().store_bytes(&doc_bytes)?;
         }
     }
-    serializer.close()?;
-    Ok(())
-}
 
-impl SerializableSegment for SegmentWriter {
-    fn write(
-        &self,
-        serializer: SegmentSerializer,
-        doc_id_map: Option<&DocIdMapping>,
-    ) -> crate::Result<u32> {
-        let max_doc = self.max_doc;
-        write(
-            &self.multifield_postings,
-            &self.fast_field_writers,
-            &self.fieldnorms_writer,
-            serializer,
-            doc_id_map,
-        )?;
-        Ok(max_doc)
-    }
+    serializer.close()?;
+
+    Ok(())
 }
 
 #[cfg(test)]

src/lib.rs

@@ -1,9 +1,18 @@
 #![doc(html_logo_url = "http://fulmicoton.com/tantivy-logo/tantivy-logo.png")]
 #![cfg_attr(all(feature = "unstable", test), feature(test))]
-#![cfg_attr(feature = "cargo-clippy", allow(clippy::module_inception))]
+#![cfg_attr(
+    feature = "cargo-clippy",
+    allow(
+        clippy::module_inception,
+        clippy::needless_range_loop,
+        clippy::bool_assert_comparison
+    )
+)]
 #![doc(test(attr(allow(unused_variables), deny(warnings))))]
 #![warn(missing_docs)]
 
+#![feature(async_closure)]
+
 //! # `tantivy`
 //!
 //! Tantivy is a search engine library.
@@ -100,7 +109,6 @@
 
 #[cfg_attr(test, macro_use)]
 extern crate serde_json;
-
 #[macro_use]
 extern crate log;
 
@@ -120,6 +128,8 @@ mod macros;
 pub use crate::error::TantivyError;
 pub use chrono;
 
+pub const PKG_JS: &'static str = "./pkg/pool_exec.js"; // path to `wasm-bindgen`'s JS binding
+
 /// Tantivy result.
 ///
 /// Within tantivy, please avoid importing `Result` using `use crate::Result`
@@ -129,7 +139,6 @@ pub type Result<T> = std::result::Result<T, TantivyError>;
 /// Tantivy DateTime
 pub type DateTime = chrono::DateTime<chrono::Utc>;
 
-mod common;
 mod core;
 mod indexer;
 
@@ -157,8 +166,6 @@ pub use self::snippet::{Snippet, SnippetGenerator};
 
 mod docset;
 pub use self::docset::{DocSet, TERMINATED};
-pub use crate::common::HasLen;
-pub use crate::common::{f64_to_u64, i64_to_u64, u64_to_f64, u64_to_i64};
 pub use crate::core::{Executor, SegmentComponent};
 pub use crate::core::{
     Index, IndexBuilder, IndexMeta, IndexSettings, IndexSortByField, Order, Searcher, Segment,
@@ -172,6 +179,8 @@ pub use crate::indexer::IndexWriter;
 pub use crate::postings::Postings;
 pub use crate::reader::LeasedItem;
 pub use crate::schema::{Document, Term};
+pub use common::HasLen;
+pub use common::{f64_to_u64, i64_to_u64, u64_to_f64, u64_to_i64};
 use std::fmt;
 
 use once_cell::sync::Lazy;
@@ -287,7 +296,7 @@ pub struct DocAddress {
 }
 
 #[cfg(test)]
-mod tests {
+pub mod tests {
     use crate::collector::tests::TEST_COLLECTOR_WITH_SCORE;
     use crate::core::SegmentReader;
     use crate::docset::{DocSet, TERMINATED};
@@ -298,11 +307,18 @@ mod tests {
     use crate::Index;
     use crate::Postings;
     use crate::ReloadPolicy;
+    use common::{BinarySerializable, FixedSize};
     use rand::distributions::Bernoulli;
     use rand::distributions::Uniform;
     use rand::rngs::StdRng;
     use rand::{Rng, SeedableRng};
 
+    pub fn fixed_size_test<O: BinarySerializable + FixedSize + Default>() {
+        let mut buffer = Vec::new();
+        O::default().serialize(&mut buffer).unwrap();
+        assert_eq!(buffer.len(), O::SIZE_IN_BYTES);
+    }
+
     /// Checks if left and right are close one to each other.
     /// Panics if the two values are more than 0.5% apart.
     #[macro_export]
@@ -934,7 +950,7 @@ mod tests {
         let id = schema_builder.add_u64_field("id", INDEXED);
         let schema = schema_builder.build();
 
-        let index = Index::create_in_ram(schema.clone());
+        let index = Index::create_in_ram(schema);
         let index_reader = index.reader()?;
 
         let mut index_writer = index.writer_for_tests()?;
@@ -973,7 +989,7 @@ mod tests {
         let searcher = index_reader.searcher();
         let segment_ids: Vec<SegmentId> = searcher
             .segment_readers()
-            .into_iter()
+            .iter()
             .map(|reader| reader.segment_id())
             .collect();
         block_on(index_writer.merge(&segment_ids)).unwrap();
@@ -987,8 +1003,24 @@ mod tests {
     #[test]
     fn test_validate_checksum() -> crate::Result<()> {
         let index_path = tempfile::tempdir().expect("dir");
-        let schema = Schema::builder().build();
+        let mut builder = Schema::builder();
+        let body = builder.add_text_field("body", TEXT | STORED);
+        let schema = builder.build();
         let index = Index::create_in_dir(&index_path, schema)?;
+        let mut writer = index.writer(50_000_000)?;
+        for _ in 0..5000 {
+            writer.add_document(doc!(body => "foo"));
+            writer.add_document(doc!(body => "boo"));
+        }
+        writer.commit()?;
+        assert!(index.validate_checksum()?.is_empty());
+
+        // delete few docs
+        writer.delete_term(Term::from_field_text(body, "foo"));
+        writer.commit()?;
+        let segment_ids = index.searchable_segment_ids()?;
+        let _ = futures::executor::block_on(writer.merge(&segment_ids));
+
         assert!(index.validate_checksum()?.is_empty());
         Ok(())
     }

src/positions/mod.rs

@@ -46,7 +46,7 @@ pub mod tests {
     fn create_positions_data(vals: &[u32]) -> crate::Result<OwnedBytes> {
         let mut positions_buffer = vec![];
         let mut serializer = PositionSerializer::new(&mut positions_buffer);
-        serializer.write_positions_delta(&vals);
+        serializer.write_positions_delta(vals);
         serializer.close_term()?;
         serializer.close()?;
         Ok(OwnedBytes::new(positions_buffer))
@@ -169,7 +169,7 @@ pub mod tests {
         let positions_delta: Vec<u32> = (0..2_000_000).collect();
         let positions_data = create_positions_data(&positions_delta[..])?;
         assert_eq!(positions_data.len(), 5003499);
-        let mut position_reader = PositionReader::open(positions_data.clone())?;
+        let mut position_reader = PositionReader::open(positions_data)?;
         let mut buf = [0u32; 256];
         position_reader.read(128, &mut buf);
         for i in 0..256 {

src/positions/reader.rs

@@ -1,9 +1,9 @@
 use std::io;
 
-use crate::common::{BinarySerializable, VInt};
 use crate::directory::OwnedBytes;
 use crate::positions::COMPRESSION_BLOCK_SIZE;
 use crate::postings::compression::{BlockDecoder, VIntDecoder};
+use common::{BinarySerializable, VInt};
 
 /// When accessing the position of a term, we get a positions_idx from the `Terminfo`.
 /// This means we need to skip to the `nth` positions efficiently.

src/positions/serializer.rs

@@ -1,7 +1,7 @@
-use crate::common::{BinarySerializable, CountingWriter, VInt};
 use crate::positions::COMPRESSION_BLOCK_SIZE;
 use crate::postings::compression::BlockEncoder;
 use crate::postings::compression::VIntEncoder;
+use common::{BinarySerializable, CountingWriter, VInt};
 use std::io::{self, Write};
 
 /// The PositionSerializer is in charge of serializing all of the positions

src/postings/block_search.rs

@@ -1,241 +1,109 @@
-use std::ops::Range;
+use crate::postings::compression::COMPRESSION_BLOCK_SIZE;
 
-use crate::postings::compression::AlignedBuffer;
+unsafe fn binary_search_step(ptr: *const u32, target: u32, half_size: isize) -> *const u32 {
+    let mid = ptr.offset(half_size);
+    if *mid < target {
+        mid.offset(1)
+    } else {
+        ptr
+    }
+}
 
-/// This modules define the logic used to search for a doc in a given
-/// block. (at most 128 docs)
+/// Search the first index containing an element greater or equal to
+/// the target.
 ///
-/// Searching within a block is a hotspot when running intersection.
-/// so it was worth defining it in its own module.
-
-#[cfg(target_arch = "x86_64")]
-mod sse2 {
-    use crate::postings::compression::{AlignedBuffer, COMPRESSION_BLOCK_SIZE};
-    use std::arch::x86_64::__m128i as DataType;
-    use std::arch::x86_64::_mm_add_epi32 as op_add;
-    use std::arch::x86_64::_mm_cmplt_epi32 as op_lt;
-    use std::arch::x86_64::_mm_load_si128 as op_load; // requires 128-bits alignment
-    use std::arch::x86_64::_mm_set1_epi32 as set1;
-    use std::arch::x86_64::_mm_setzero_si128 as set0;
-    use std::arch::x86_64::_mm_sub_epi32 as op_sub;
-    use std::arch::x86_64::{_mm_cvtsi128_si32, _mm_shuffle_epi32};
-
-    const MASK1: i32 = 78;
-    const MASK2: i32 = 177;
-
-    /// Performs an exhaustive linear search over the
-    ///
-    /// There is no early exit here. We simply count the
-    /// number of elements that are `< target`.
-    pub(crate) fn linear_search_sse2_128(arr: &AlignedBuffer, target: u32) -> usize {
-        unsafe {
-            let ptr = arr as *const AlignedBuffer as *const DataType;
-            let vkey = set1(target as i32);
-            let mut cnt = set0();
-            // We work over 4 `__m128i` at a time.
-            // A single `__m128i` actual contains 4 `u32`.
-            for i in 0..(COMPRESSION_BLOCK_SIZE as isize) / (4 * 4) {
-                let cmp1 = op_lt(op_load(ptr.offset(i * 4)), vkey);
-                let cmp2 = op_lt(op_load(ptr.offset(i * 4 + 1)), vkey);
-                let cmp3 = op_lt(op_load(ptr.offset(i * 4 + 2)), vkey);
-                let cmp4 = op_lt(op_load(ptr.offset(i * 4 + 3)), vkey);
-                let sum = op_add(op_add(cmp1, cmp2), op_add(cmp3, cmp4));
-                cnt = op_sub(cnt, sum);
-            }
-            cnt = op_add(cnt, _mm_shuffle_epi32(cnt, MASK1));
-            cnt = op_add(cnt, _mm_shuffle_epi32(cnt, MASK2));
-            _mm_cvtsi128_si32(cnt) as usize
-        }
-    }
-
-    #[cfg(test)]
-    mod test {
-        use super::linear_search_sse2_128;
-        use crate::postings::compression::{AlignedBuffer, COMPRESSION_BLOCK_SIZE};
-
-        #[test]
-        fn test_linear_search_sse2_128_u32() {
-            let mut block = [0u32; COMPRESSION_BLOCK_SIZE];
-            for el in 0u32..128u32 {
-                block[el as usize] = el * 2 + 1 << 18;
-            }
-            let target = block[64] + 1;
-            assert_eq!(linear_search_sse2_128(&AlignedBuffer(block), target), 65);
-        }
-    }
-}
-
-/// This `linear search` browser exhaustively through the array.
-/// but the early exit is very difficult to predict.
+/// The results should be equivalent to
+/// ```compile_fail
+/// block[..]
+//       .iter()
+//       .take_while(|&&val| val < target)
+//       .count()
+/// ```
 ///
-/// Coupled with `exponential search` this function is likely
-/// to be called with the same `len`
-fn linear_search(arr: &[u32], target: u32) -> usize {
-    arr.iter().map(|&el| if el < target { 1 } else { 0 }).sum()
-}
-
-fn exponential_search(arr: &[u32], target: u32) -> Range<usize> {
-    let end = arr.len();
-    let mut begin = 0;
-    for &pivot in &[1, 3, 7, 15, 31, 63] {
-        if pivot >= end {
-            break;
-        }
-        if arr[pivot] > target {
-            return begin..pivot;
-        }
-        begin = pivot;
-    }
-    begin..end
-}
-
-#[inline(never)]
-fn galloping(block_docs: &[u32], target: u32) -> usize {
-    let range = exponential_search(&block_docs, target);
-    range.start + linear_search(&block_docs[range], target)
-}
-
-/// Tantivy may rely on SIMD instructions to search for a specific document within
-/// a given block.
-#[derive(Clone, Copy, PartialEq)]
-pub enum BlockSearcher {
-    #[cfg(target_arch = "x86_64")]
-    Sse2,
-    Scalar,
-}
-
-impl BlockSearcher {
-    /// Search the first index containing an element greater or equal to
-    /// the target.
-    ///
-    /// The results should be equivalent to
-    /// ```compile_fail
-    /// block[..]
-    //       .iter()
-    //       .take_while(|&&val| val < target)
-    //       .count()
-    /// ```
-    ///
-    /// The `start` argument is just used to hint that the response is
-    /// greater than beyond `start`. The implementation may or may not use
-    /// it for optimization.
-    ///
-    /// # Assumption
-    ///
-    /// The array len is > start.
-    /// The block is sorted
-    /// The target is assumed greater or equal to the `arr[start]`.
-    /// The target is assumed smaller or equal to the last element of the block.
-    ///
-    /// Currently the scalar implementation starts by an exponential search, and
-    /// then operates a linear search in the result subarray.
-    ///
-    /// If SSE2 instructions are available in the `(platform, running CPU)`,
-    /// then we use a different implementation that does an exhaustive linear search over
-    /// the block regardless of whether the block is full or not.
-    ///
-    /// Indeed, if the block is not full, the remaining items are TERMINATED.
-    /// It is surprisingly faster, most likely because of the lack of branch misprediction.
-    pub(crate) fn search_in_block(self, block_docs: &AlignedBuffer, target: u32) -> usize {
-        #[cfg(target_arch = "x86_64")]
-        {
-            if self == BlockSearcher::Sse2 {
-                return sse2::linear_search_sse2_128(block_docs, target);
-            }
-        }
-        galloping(&block_docs.0[..], target)
-    }
-}
-
-impl Default for BlockSearcher {
-    fn default() -> BlockSearcher {
-        #[cfg(target_arch = "x86_64")]
-        {
-            if is_x86_feature_detected!("sse2") {
-                return BlockSearcher::Sse2;
-            }
-        }
-        BlockSearcher::Scalar
+/// the `start` argument is just used to hint that the response is
+/// greater than beyond `start`. the implementation may or may not use
+/// it for optimization.
+///
+/// # Assumption
+///
+/// - The block is sorted. Some elements may appear several times. This is the case at the
+/// end of the last block for instance.
+/// - The target is assumed smaller or equal to the last element of the block.
+pub fn branchless_binary_search(arr: &[u32; COMPRESSION_BLOCK_SIZE], target: u32) -> usize {
+    let start_ptr: *const u32 = &arr[0] as *const u32;
+    unsafe {
+        let mut ptr = start_ptr;
+        ptr = binary_search_step(ptr, target, 63);
+        ptr = binary_search_step(ptr, target, 31);
+        ptr = binary_search_step(ptr, target, 15);
+        ptr = binary_search_step(ptr, target, 7);
+        ptr = binary_search_step(ptr, target, 3);
+        ptr = binary_search_step(ptr, target, 1);
+        let extra = if *ptr < target { 1 } else { 0 };
+        (ptr.offset_from(start_ptr) as usize) + extra
     }
 }
 
 #[cfg(test)]
 mod tests {
-    use super::exponential_search;
-    use super::linear_search;
-    use super::BlockSearcher;
+    use super::branchless_binary_search;
     use crate::docset::TERMINATED;
-    use crate::postings::compression::{AlignedBuffer, COMPRESSION_BLOCK_SIZE};
-
-    #[test]
-    fn test_linear_search() {
-        let len: usize = 50;
-        let arr: Vec<u32> = (0..len).map(|el| 1u32 + (el as u32) * 2).collect();
-        for target in 1..*arr.last().unwrap() {
-            let res = linear_search(&arr[..], target);
-            if res > 0 {
-                assert!(arr[res - 1] < target);
-            }
-            if res < len {
-                assert!(arr[res] >= target);
-            }
-        }
-    }
-
-    #[test]
-    fn test_exponentiel_search() {
-        assert_eq!(exponential_search(&[1, 2], 0), 0..1);
-        assert_eq!(exponential_search(&[1, 2], 1), 0..1);
-        assert_eq!(
-            exponential_search(&[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], 7),
-            3..7
-        );
-    }
-
-    fn util_test_search_in_block(block_searcher: BlockSearcher, block: &[u32], target: u32) {
-        let cursor = search_in_block_trivial_but_slow(block, target);
-        assert!(block.len() < COMPRESSION_BLOCK_SIZE);
-        let mut output_buffer = [TERMINATED; COMPRESSION_BLOCK_SIZE];
-        output_buffer[..block.len()].copy_from_slice(block);
-        assert_eq!(
-            block_searcher.search_in_block(&AlignedBuffer(output_buffer), target),
-            cursor
-        );
-    }
-
-    fn util_test_search_in_block_all(block_searcher: BlockSearcher, block: &[u32]) {
-        use std::collections::HashSet;
-        let mut targets = HashSet::new();
-        for (i, val) in block.iter().cloned().enumerate() {
-            if i > 0 {
-                targets.insert(val - 1);
-            }
-            targets.insert(val);
-        }
-        for target in targets {
-            util_test_search_in_block(block_searcher, block, target);
-        }
-    }
+    use crate::postings::compression::COMPRESSION_BLOCK_SIZE;
+    use proptest::prelude::*;
+    use std::collections::HashSet;
 
     fn search_in_block_trivial_but_slow(block: &[u32], target: u32) -> usize {
         block.iter().take_while(|&&val| val < target).count()
     }
 
-    fn test_search_in_block_util(block_searcher: BlockSearcher) {
-        for len in 1u32..128u32 {
-            let v: Vec<u32> = (0..len).map(|i| i * 2).collect();
-            util_test_search_in_block_all(block_searcher, &v[..]);
+    fn util_test_search_in_block(block: &[u32], target: u32) {
+        let cursor = search_in_block_trivial_but_slow(block, target);
+        assert!(cursor < COMPRESSION_BLOCK_SIZE);
+        assert!(block[cursor] >= target);
+        if cursor > 0 {
+            assert!(block[cursor - 1] < target);
+        }
+        assert_eq!(block.len(), COMPRESSION_BLOCK_SIZE);
+        let mut output_buffer = [TERMINATED; COMPRESSION_BLOCK_SIZE];
+        output_buffer[..block.len()].copy_from_slice(block);
+        assert_eq!(branchless_binary_search(&output_buffer, target), cursor);
+    }
+
+    fn util_test_search_in_block_all(block: &[u32]) {
+        let mut targets = HashSet::new();
+        targets.insert(0);
+        for &val in block {
+            if val > 0 {
+                targets.insert(val - 1);
+            }
+            targets.insert(val);
+        }
+        for target in targets {
+            util_test_search_in_block(block, target);
         }
     }
 
     #[test]
-    fn test_search_in_block_scalar() {
-        test_search_in_block_util(BlockSearcher::Scalar);
+    fn test_search_in_branchless_binary_search() {
+        let v: Vec<u32> = (0..COMPRESSION_BLOCK_SIZE).map(|i| i as u32 * 2).collect();
+        util_test_search_in_block_all(&v[..]);
     }
 
-    #[cfg(target_arch = "x86_64")]
-    #[test]
-    fn test_search_in_block_sse2() {
-        test_search_in_block_util(BlockSearcher::Sse2);
+    fn monotonous_block() -> impl Strategy<Value = Vec<u32>> {
+        prop::collection::vec(0u32..5u32, COMPRESSION_BLOCK_SIZE).prop_map(|mut deltas| {
+            let mut el = 0;
+            for i in 0..COMPRESSION_BLOCK_SIZE {
+                el += deltas[i];
+                deltas[i] = el;
+            }
+            deltas
+        })
+    }
+
+    proptest! {
+        #[test]
+        fn test_proptest_branchless_binary_search(block in monotonous_block()) {
+            util_test_search_in_block_all(&block[..]);
+        }
     }
 }

src/postings/block_segment_postings.rs

@@ -1,23 +1,17 @@
 use std::io;
 
-use crate::common::{BinarySerializable, VInt};
 use crate::directory::FileSlice;
 use crate::directory::OwnedBytes;
 use crate::fieldnorm::FieldNormReader;
-use crate::postings::compression::{
-    AlignedBuffer, BlockDecoder, VIntDecoder, COMPRESSION_BLOCK_SIZE,
-};
+use crate::postings::compression::{BlockDecoder, VIntDecoder, COMPRESSION_BLOCK_SIZE};
 use crate::postings::{BlockInfo, FreqReadingOption, SkipReader};
 use crate::query::Bm25Weight;
 use crate::schema::IndexRecordOption;
 use crate::{DocId, Score, TERMINATED};
+use common::{BinarySerializable, VInt};
 
 fn max_score<I: Iterator<Item = Score>>(mut it: I) -> Option<Score> {
-    if let Some(first) = it.next() {
-        Some(it.fold(first, Score::max))
-    } else {
-        None
-    }
+    it.next().map(|first| it.fold(first, Score::max))
 }
 
 /// `BlockSegmentPostings` is a cursor iterating over blocks
@@ -213,9 +207,9 @@ impl BlockSegmentPostings {
     ///
     /// This method is useful to run SSE2 linear search.
     #[inline]
-    pub(crate) fn docs_aligned(&self) -> &AlignedBuffer {
+    pub(crate) fn full_block(&self) -> &[DocId; COMPRESSION_BLOCK_SIZE] {
         debug_assert!(self.block_is_loaded());
-        self.doc_decoder.output_aligned()
+        self.doc_decoder.full_output()
     }
 
     /// Return the document at index `idx` of the block.
@@ -353,7 +347,6 @@ impl BlockSegmentPostings {
 #[cfg(test)]
 mod tests {
     use super::BlockSegmentPostings;
-    use crate::common::HasLen;
     use crate::core::Index;
     use crate::docset::{DocSet, TERMINATED};
     use crate::postings::compression::COMPRESSION_BLOCK_SIZE;
@@ -364,6 +357,7 @@ mod tests {
     use crate::schema::Term;
     use crate::schema::INDEXED;
     use crate::DocId;
+    use common::HasLen;
 
     #[test]
     fn test_empty_segment_postings() {
@@ -482,11 +476,11 @@ mod tests {
             docs.push((i * i / 100) + i);
         }
         let mut block_postings = build_block_postings(&docs[..]);
-        for i in vec![0, 424, 10000] {
-            block_postings.seek(i);
+        for i in &[0, 424, 10000] {
+            block_postings.seek(*i);
             let docs = block_postings.docs();
-            assert!(docs[0] <= i);
-            assert!(docs.last().cloned().unwrap_or(0u32) >= i);
+            assert!(docs[0] <= *i);
+            assert!(docs.last().cloned().unwrap_or(0u32) >= *i);
         }
         block_postings.seek(100_000);
         assert_eq!(block_postings.doc(COMPRESSION_BLOCK_SIZE - 1), TERMINATED);

src/postings/compression/mod.rs

@@ -1,5 +1,5 @@
-use crate::common::FixedSize;
 use bitpacking::{BitPacker, BitPacker4x};
+use common::FixedSize;
 
 pub const COMPRESSION_BLOCK_SIZE: usize = BitPacker4x::BLOCK_LEN;
 const COMPRESSED_BLOCK_MAX_SIZE: usize = COMPRESSION_BLOCK_SIZE * u32::SIZE_IN_BYTES;
@@ -49,16 +49,10 @@ impl BlockEncoder {
     }
 }
 
-/// We ensure that the OutputBuffer is align on 128 bits
-/// in order to run SSE2 linear search on it.
-#[repr(align(128))]
-#[derive(Clone)]
-pub(crate) struct AlignedBuffer(pub [u32; COMPRESSION_BLOCK_SIZE]);
-
 #[derive(Clone)]
 pub struct BlockDecoder {
     bitpacker: BitPacker4x,
-    output: AlignedBuffer,
+    output: [u32; COMPRESSION_BLOCK_SIZE],
     pub output_len: usize,
 }
 
@@ -72,7 +66,7 @@ impl BlockDecoder {
     pub fn with_val(val: u32) -> BlockDecoder {
         BlockDecoder {
             bitpacker: BitPacker4x::new(),
-            output: AlignedBuffer([val; COMPRESSION_BLOCK_SIZE]),
+            output: [val; COMPRESSION_BLOCK_SIZE],
             output_len: 0,
         }
     }
@@ -85,28 +79,28 @@ impl BlockDecoder {
     ) -> usize {
         self.output_len = COMPRESSION_BLOCK_SIZE;
         self.bitpacker
-            .decompress_sorted(offset, &compressed_data, &mut self.output.0, num_bits)
+            .decompress_sorted(offset, compressed_data, &mut self.output, num_bits)
     }
 
     pub fn uncompress_block_unsorted(&mut self, compressed_data: &[u8], num_bits: u8) -> usize {
         self.output_len = COMPRESSION_BLOCK_SIZE;
         self.bitpacker
-            .decompress(&compressed_data, &mut self.output.0, num_bits)
+            .decompress(compressed_data, &mut self.output, num_bits)
     }
 
     #[inline]
     pub fn output_array(&self) -> &[u32] {
-        &self.output.0[..self.output_len]
+        &self.output[..self.output_len]
     }
 
     #[inline]
-    pub(crate) fn output_aligned(&self) -> &AlignedBuffer {
+    pub(crate) fn full_output(&self) -> &[u32; COMPRESSION_BLOCK_SIZE] {
         &self.output
     }
 
     #[inline]
     pub fn output(&self, idx: usize) -> u32 {
-        self.output.0[idx]
+        self.output[idx]
     }
 }
 
@@ -190,8 +184,8 @@ impl VIntDecoder for BlockDecoder {
         padding: u32,
     ) -> usize {
         self.output_len = num_els;
-        self.output.0.iter_mut().for_each(|el| *el = padding);
-        vint::uncompress_sorted(compressed_data, &mut self.output.0[..num_els], offset)
+        self.output.iter_mut().for_each(|el| *el = padding);
+        vint::uncompress_sorted(compressed_data, &mut self.output[..num_els], offset)
     }
 
     fn uncompress_vint_unsorted(
@@ -201,12 +195,12 @@ impl VIntDecoder for BlockDecoder {
         padding: u32,
     ) -> usize {
         self.output_len = num_els;
-        self.output.0.iter_mut().for_each(|el| *el = padding);
-        vint::uncompress_unsorted(compressed_data, &mut self.output.0[..num_els])
+        self.output.iter_mut().for_each(|el| *el = padding);
+        vint::uncompress_unsorted(compressed_data, &mut self.output[..num_els])
     }
 
     fn uncompress_vint_unsorted_until_end(&mut self, compressed_data: &[u8]) {
-        let num_els = vint::uncompress_unsorted_until_end(compressed_data, &mut self.output.0);
+        let num_els = vint::uncompress_unsorted_until_end(compressed_data, &mut self.output);
         self.output_len = num_els;
     }
 }
@@ -303,7 +297,7 @@ pub mod tests {
             assert!(encoded_data.len() <= expected_length);
             let mut decoder = BlockDecoder::default();
             let consumed_num_bytes =
-                decoder.uncompress_vint_sorted(&encoded_data, *offset, input.len(), PADDING_VALUE);
+                decoder.uncompress_vint_sorted(encoded_data, *offset, input.len(), PADDING_VALUE);
             assert_eq!(consumed_num_bytes, encoded_data.len());
             assert_eq!(input, decoder.output_array());
             for i in input.len()..COMPRESSION_BLOCK_SIZE {
@@ -354,18 +348,18 @@ mod bench {
         });
     }
 
-    #[test]
-    fn test_all_docs_compression_numbits() {
-        for expected_num_bits in 0u8.. {
-            let mut data = [0u32; 128];
-            if expected_num_bits > 0 {
-                data[0] = (1u64 << (expected_num_bits as usize) - 1) as u32;
-            }
-            let mut encoder = BlockEncoder::new();
-            let (num_bits, compressed) = encoder.compress_block_unsorted(&data);
-            assert_eq!(compressed.len(), compressed_block_size(num_bits));
-        }
-    }
+    //#[test]
+    //fn test_all_docs_compression_numbits() {
+    //for expected_num_bits in 0u8.. {
+    //let mut data = [0u32; 128];
+    //if expected_num_bits > 0 {
+    //data[0] = (1u64 << (expected_num_bits as usize) - 1) as u32;
+    //}
+    //let mut encoder = BlockEncoder::new();
+    //let (num_bits, compressed) = encoder.compress_block_unsorted(&data);
+    //assert_eq!(compressed.len(), compressed_block_size(num_bits));
+    //}
+    //}
 
     const NUM_INTS_BENCH_VINT: usize = 10;
 

src/postings/mod.rs

@@ -3,6 +3,9 @@ Postings module (also called inverted index)
 */
 
 mod block_search;
+
+pub(crate) use self::block_search::branchless_binary_search;
+
 mod block_segment_postings;
 pub(crate) mod compression;
 mod postings;
@@ -14,7 +17,6 @@ mod skip;
 mod stacker;
 mod term_info;
 
-pub(crate) use self::block_search::BlockSearcher;
 pub use self::block_segment_postings::BlockSegmentPostings;
 pub use self::postings::Postings;
 pub(crate) use self::postings_writer::MultiFieldPostingsWriter;
@@ -54,7 +56,7 @@ pub mod tests {
     use crate::DocId;
     use crate::HasLen;
     use crate::Score;
-    use std::{iter, mem};
+    use std::mem;
 
     #[test]
     pub fn test_position_write() -> crate::Result<()> {
@@ -153,8 +155,8 @@ pub mod tests {
 
     #[test]
     pub fn test_drop_token_that_are_too_long() -> crate::Result<()> {
-        let ok_token_text: String = iter::repeat('A').take(MAX_TOKEN_LEN).collect();
-        let mut exceeding_token_text: String = iter::repeat('A').take(MAX_TOKEN_LEN + 1).collect();
+        let ok_token_text: String = "A".repeat(MAX_TOKEN_LEN);
+        let mut exceeding_token_text: String = "A".repeat(MAX_TOKEN_LEN + 1);
         exceeding_token_text.push_str(" hello");
         let mut schema_builder = Schema::builder();
         let text_options = TextOptions::default().set_indexing_options(
@@ -164,7 +166,7 @@ pub mod tests {
         );
         let text_field = schema_builder.add_text_field("text", text_options);
         let schema = schema_builder.build();
-        let index = Index::create_in_ram(schema.clone());
+        let index = Index::create_in_ram(schema);
         index
             .tokenizers()
             .register("simple_no_truncation", SimpleTokenizer);
@@ -229,7 +231,7 @@ pub mod tests {
                 segment_writer.add_document(op, &schema).unwrap();
             }
             for i in 2..1000 {
-                let mut text: String = iter::repeat("e ").take(i).collect();
+                let mut text: String = "e ".repeat(i);
                 text.push_str(" a");
                 let op = AddOperation {
                     opstamp: 2u64,
@@ -409,7 +411,7 @@ pub mod tests {
                 .unwrap();
 
             for i in 0..num_docs / 2 - 1 {
-                assert!(segment_postings.seek(i * 2 + 1) > (i * 1) * 2);
+                assert!(segment_postings.seek(i * 2 + 1) > i * 2);
                 assert_eq!(segment_postings.doc(), (i + 1) * 2);
             }
         }
@@ -431,13 +433,10 @@ pub mod tests {
                 .read_postings(&term_2, IndexRecordOption::Basic)?
                 .unwrap();
 
+            assert_eq!(segment_postings.seek(i), i);
+            assert_eq!(segment_postings.doc(), i);
             if i % 2 == 0 {
-                assert_eq!(segment_postings.seek(i), i);
-                assert_eq!(segment_postings.doc(), i);
                 assert!(segment_reader.is_deleted(i));
-            } else {
-                assert_eq!(segment_postings.seek(i), i);
-                assert_eq!(segment_postings.doc(), i);
             }
         }
 

src/postings/postings.rs

@@ -11,7 +11,7 @@ use crate::docset::DocSet;
 /// but other implementations mocking `SegmentPostings` exist,
 /// for merging segments or for testing.
 pub trait Postings: DocSet + 'static {
-    /// Returns the term frequency
+    /// The number of times the term appears in the document.
     fn term_freq(&self) -> u32;
 
     /// Returns the positions offseted with a given value.

src/postings/postings_writer.rs

@@ -133,7 +133,8 @@ impl MultiFieldPostingsWriter {
         doc_id_map: Option<&DocIdMapping>,
     ) -> crate::Result<HashMap<Field, FnvHashMap<UnorderedTermId, TermOrdinal>>> {
         let mut term_offsets: Vec<(&[u8], Addr, UnorderedTermId)> =
-            self.term_index.iter().collect();
+            Vec::with_capacity(self.term_index.len());
+        term_offsets.extend(self.term_index.iter());
         term_offsets.sort_unstable_by_key(|&(k, _, _)| k);
 
         let mut unordered_term_mappings: HashMap<Field, FnvHashMap<UnorderedTermId, TermOrdinal>> =
@@ -231,13 +232,7 @@ pub trait PostingsWriter {
             // We skip all tokens with a len greater than u16.
             if token.text.len() <= MAX_TOKEN_LEN {
                 term_buffer.set_text(token.text.as_str());
-                self.subscribe(
-                    term_index,
-                    doc_id,
-                    token.position as u32,
-                    &term_buffer,
-                    heap,
-                );
+                self.subscribe(term_index, doc_id, token.position as u32, term_buffer, heap);
             } else {
                 warn!(
                     "A token exceeding MAX_TOKEN_LEN ({}>{}) was dropped. Search for \

src/postings/recorder.rs

@@ -1,10 +1,8 @@
 use super::stacker::{ExpUnrolledLinkedList, MemoryArena};
+use crate::indexer::doc_id_mapping::DocIdMapping;
 use crate::postings::FieldSerializer;
 use crate::DocId;
-use crate::{
-    common::{read_u32_vint, write_u32_vint},
-    indexer::doc_id_mapping::DocIdMapping,
-};
+use common::{read_u32_vint, write_u32_vint};
 
 const POSITION_END: u32 = 0;
 
@@ -282,7 +280,7 @@ impl Recorder for TfAndPositionRecorder {
                 doc_id_and_positions
                     .push((doc_id_map.get_new_doc_id(doc), buffer_positions.to_vec()));
             } else {
-                serializer.write_doc(doc, buffer_positions.len() as u32, &buffer_positions);
+                serializer.write_doc(doc, buffer_positions.len() as u32, buffer_positions);
             }
         }
         if doc_id_map.is_some() {

src/postings/segment_postings.rs

@@ -1,12 +1,12 @@
-use crate::common::HasLen;
 use crate::docset::DocSet;
-use crate::fastfield::DeleteBitSet;
+use crate::fastfield::AliveBitSet;
 use crate::positions::PositionReader;
+use crate::postings::branchless_binary_search;
 use crate::postings::compression::COMPRESSION_BLOCK_SIZE;
-use crate::postings::BlockSearcher;
 use crate::postings::BlockSegmentPostings;
 use crate::postings::Postings;
 use crate::{DocId, TERMINATED};
+use common::HasLen;
 
 /// `SegmentPostings` represents the inverted list or postings associated to
 /// a term in a `Segment`.
@@ -18,7 +18,6 @@ pub struct SegmentPostings {
     pub(crate) block_cursor: BlockSegmentPostings,
     cur: usize,
     position_reader: Option<PositionReader>,
-    block_searcher: BlockSearcher,
 }
 
 impl SegmentPostings {
@@ -28,7 +27,6 @@ impl SegmentPostings {
             block_cursor: BlockSegmentPostings::empty(),
             cur: 0,
             position_reader: None,
-            block_searcher: BlockSearcher::default(),
         }
     }
 
@@ -36,7 +34,7 @@ impl SegmentPostings {
     ///
     /// This method will clone and scan through the posting lists.
     /// (this is a rather expensive operation).
-    pub fn doc_freq_given_deletes(&self, delete_bitset: &DeleteBitSet) -> u32 {
+    pub fn doc_freq_given_deletes(&self, alive_bitset: &AliveBitSet) -> u32 {
         let mut docset = self.clone();
         let mut doc_freq = 0;
         loop {
@@ -44,7 +42,7 @@ impl SegmentPostings {
             if doc == TERMINATED {
                 return doc_freq;
             }
-            if delete_bitset.is_alive(doc) {
+            if alive_bitset.is_alive(doc) {
                 doc_freq += 1u32;
             }
             docset.advance();
@@ -107,7 +105,7 @@ impl SegmentPostings {
         let fieldnorm_reader = fieldnorms.map(FieldNormReader::for_test);
         let average_field_norm = fieldnorms
             .map(|fieldnorms| {
-                if fieldnorms.len() == 0 {
+                if fieldnorms.is_empty() {
                     return 0.0;
                 }
                 let total_num_tokens: u64 = fieldnorms
@@ -154,7 +152,6 @@ impl SegmentPostings {
             block_cursor: segment_block_postings,
             cur: 0, // cursor within the block
             position_reader,
-            block_searcher: BlockSearcher::default(),
         }
     }
 }
@@ -183,8 +180,8 @@ impl DocSet for SegmentPostings {
         self.block_cursor.seek(target);
 
         // At this point we are on the block, that might contain our document.
-        let output = self.block_cursor.docs_aligned();
-        self.cur = self.block_searcher.search_in_block(&output, target);
+        let output = self.block_cursor.full_block();
+        self.cur = branchless_binary_search(output, target);
 
         // The last block is not full and padded with the value TERMINATED,
         // so that we are guaranteed to have at least doc in the block (a real one or the padding)
@@ -197,7 +194,7 @@ impl DocSet for SegmentPostings {
         // with the value `TERMINATED`.
         //
         // After the search, the cursor should point to the first value of TERMINATED.
-        let doc = output.0[self.cur];
+        let doc = output[self.cur];
         debug_assert!(doc >= target);
         debug_assert_eq!(doc, self.doc());
         doc
@@ -268,10 +265,10 @@ impl Postings for SegmentPostings {
 mod tests {
 
     use super::SegmentPostings;
-    use crate::common::HasLen;
+    use common::HasLen;
 
     use crate::docset::{DocSet, TERMINATED};
-    use crate::fastfield::DeleteBitSet;
+    use crate::fastfield::AliveBitSet;
     use crate::postings::postings::Postings;
 
     #[test]
@@ -299,9 +296,10 @@ mod tests {
     fn test_doc_freq() {
         let docs = SegmentPostings::create_from_docs(&[0, 2, 10]);
         assert_eq!(docs.doc_freq(), 3);
-        let delete_bitset = DeleteBitSet::for_test(&[2], 12);
-        assert_eq!(docs.doc_freq_given_deletes(&delete_bitset), 2);
-        let all_deleted = DeleteBitSet::for_test(&[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], 12);
+        let alive_bitset = AliveBitSet::for_test_from_deleted_docs(&[2], 12);
+        assert_eq!(docs.doc_freq_given_deletes(&alive_bitset), 2);
+        let all_deleted =
+            AliveBitSet::for_test_from_deleted_docs(&[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], 12);
         assert_eq!(docs.doc_freq_given_deletes(&all_deleted), 0);
     }
 }

src/postings/serializer.rs

@@ -1,7 +1,6 @@
 use super::TermInfo;
-use crate::common::{BinarySerializable, VInt};
-use crate::common::{CompositeWrite, CountingWriter};
 use crate::core::Segment;
+use crate::directory::CompositeWrite;
 use crate::directory::WritePtr;
 use crate::fieldnorm::FieldNormReader;
 use crate::positions::PositionSerializer;
@@ -12,6 +11,8 @@ use crate::schema::{Field, FieldEntry, FieldType};
 use crate::schema::{IndexRecordOption, Schema};
 use crate::termdict::{TermDictionaryBuilder, TermOrdinal};
 use crate::{DocId, Score};
+use common::CountingWriter;
+use common::{BinarySerializable, VInt};
 use std::cmp::Ordering;
 use std::io::{self, Write};
 
@@ -356,7 +357,7 @@ impl<W: Write> PostingsSerializer<W> {
             // encode the doc ids
             let (num_bits, block_encoded): (u8, &[u8]) = self
                 .block_encoder
-                .compress_block_sorted(&self.block.doc_ids(), self.last_doc_id_encoded);
+                .compress_block_sorted(self.block.doc_ids(), self.last_doc_id_encoded);
             self.last_doc_id_encoded = self.block.last_doc();
             self.skip_write
                 .write_doc(self.last_doc_id_encoded, num_bits);
@@ -366,7 +367,7 @@ impl<W: Write> PostingsSerializer<W> {
         if self.mode.has_freq() {
             let (num_bits, block_encoded): (u8, &[u8]) = self
                 .block_encoder
-                .compress_block_unsorted(&self.block.term_freqs());
+                .compress_block_unsorted(self.block.term_freqs());
             self.postings_write.extend(block_encoded);
             self.skip_write.write_term_freq(num_bits);
             if self.mode.has_positions() {
@@ -426,7 +427,7 @@ impl<W: Write> PostingsSerializer<W> {
             {
                 let block_encoded = self
                     .block_encoder
-                    .compress_vint_sorted(&self.block.doc_ids(), self.last_doc_id_encoded);
+                    .compress_vint_sorted(self.block.doc_ids(), self.last_doc_id_encoded);
                 self.postings_write.write_all(block_encoded)?;
             }
             // ... Idem for term frequencies
@@ -442,10 +443,8 @@ impl<W: Write> PostingsSerializer<W> {
             let skip_data = self.skip_write.data();
             VInt(skip_data.len() as u64).serialize(&mut self.output_write)?;
             self.output_write.write_all(skip_data)?;
-            self.output_write.write_all(&self.postings_write[..])?;
-        } else {
-            self.output_write.write_all(&self.postings_write[..])?;
         }
+        self.output_write.write_all(&self.postings_write[..])?;
         self.skip_write.clear();
         self.postings_write.clear();
         self.bm25_weight = None;