fix typing

Cargo.toml

@@ -11,7 +11,6 @@ repository = "https://github.com/quickwit-oss/tantivy"
 readme = "README.md"
 keywords = ["search", "information", "retrieval"]
 edition = "2021"
-rust-version = "1.62"
 
 [dependencies]
 oneshot = "0.1.3"
@@ -20,11 +19,11 @@ byteorder = "1.4.3"
 crc32fast = "1.3.2"
 once_cell = "1.10.0"
 regex = { version = "1.5.5", default-features = false, features = ["std", "unicode"] }
-tantivy-fst = "0.4.0"
+tantivy-fst = "0.3.0"
 memmap2 = { version = "0.5.3", optional = true }
 lz4_flex = { version = "0.9.2", default-features = false, features = ["checked-decode"], optional = true }
 brotli = { version = "3.3.4", optional = true }
-zstd = { version = "0.11", optional = true, default-features = false }
+zstd = { version = "0.11", optional = true }
 snap = { version = "1.0.5", optional = true }
 tempfile = { version = "3.3.0", optional = true }
 log = "0.4.16"

README.md

@@ -58,7 +58,7 @@ Distributed search is out of the scope of Tantivy, but if you are looking for th
 
 # Getting started
 
-Tantivy works on stable Rust and supports Linux, macOS, and Windows.
+Tantivy works on stable Rust (>= 1.27) and supports Linux, macOS, and Windows.
 
 - [Tantivy's simple search example](https://tantivy-search.github.io/examples/basic_search.html)
 - [tantivy-cli and its tutorial](https://github.com/quickwit-oss/tantivy-cli) - `tantivy-cli` is an actual command-line interface that makes it easy for you to create a search engine,
@@ -81,13 +81,9 @@ There are many ways to support this project.
 
 We use the GitHub Pull Request workflow: reference a GitHub ticket and/or include a comprehensive commit message when opening a PR.
 
-## Minimum supported Rust version
-
-Tantivy currently requires at least Rust 1.62 or later to compile.
-
 ## Clone and build locally
 
-Tantivy compiles on stable Rust.
+Tantivy compiles on stable Rust but requires `Rust >= 1.27`.
 To check out and run tests, you can simply run:
 
 ```bash

common/src/lib.rs

@@ -34,8 +34,7 @@ impl<T: Deref<Target = [u8]>> HasLen for T {
     }
 }
 
-const HIGHEST_BIT_64: u64 = 1 << 63;
-const HIGHEST_BIT_32: u32 = 1 << 31;
+const HIGHEST_BIT: u64 = 1 << 63;
 
 /// Maps a `i64` to `u64`
 ///
@@ -59,13 +58,13 @@ const HIGHEST_BIT_32: u32 = 1 << 31;
 /// The reverse mapping is [`u64_to_i64()`].
 #[inline]
 pub fn i64_to_u64(val: i64) -> u64 {
-    (val as u64) ^ HIGHEST_BIT_64
+    (val as u64) ^ HIGHEST_BIT
 }
 
 /// Reverse the mapping given by [`i64_to_u64()`].
 #[inline]
 pub fn u64_to_i64(val: u64) -> i64 {
-    (val ^ HIGHEST_BIT_64) as i64
+    (val ^ HIGHEST_BIT) as i64
 }
 
 /// Maps a `f64` to `u64`
@@ -89,7 +88,7 @@ pub fn u64_to_i64(val: u64) -> i64 {
 pub fn f64_to_u64(val: f64) -> u64 {
     let bits = val.to_bits();
     if val.is_sign_positive() {
-        bits ^ HIGHEST_BIT_64
+        bits ^ HIGHEST_BIT
     } else {
         !bits
     }
@@ -98,148 +97,26 @@ pub fn f64_to_u64(val: f64) -> u64 {
 /// Reverse the mapping given by [`f64_to_u64()`].
 #[inline]
 pub fn u64_to_f64(val: u64) -> f64 {
-    f64::from_bits(if val & HIGHEST_BIT_64 != 0 {
-        val ^ HIGHEST_BIT_64
+    f64::from_bits(if val & HIGHEST_BIT != 0 {
+        val ^ HIGHEST_BIT
     } else {
         !val
     })
 }
 
-/// Maps a `f32` to `u64`
-///
-/// # See also
-/// Similar mapping for f64 [`u64_to_f64()`].
-#[inline]
-pub fn f32_to_u64(val: f32) -> u64 {
-    let bits = val.to_bits();
-    let res32 = if val.is_sign_positive() {
-        bits ^ HIGHEST_BIT_32
-    } else {
-        !bits
-    };
-    res32 as u64
-}
-
-/// Reverse the mapping given by [`f32_to_u64()`].
-#[inline]
-pub fn u64_to_f32(val: u64) -> f32 {
-    debug_assert!(val <= 1 << 32);
-    let val = val as u32;
-    f32::from_bits(if val & HIGHEST_BIT_32 != 0 {
-        val ^ HIGHEST_BIT_32
-    } else {
-        !val
-    })
-}
-
-/// Maps a `f64` to a fixed point representation.
-/// Lower bound is inclusive, upper bound is exclusive.
-/// `precision` is the number of bits used to represent the number.
-///
-/// This is a lossy, affine transformation. All provided values must be finite and non-NaN.
-/// Care should be taken to not provide values which would cause loss of precision such as values
-/// low enough to get sub-normal numbers, value high enough rounding would cause ±Inf to appear, or
-/// a precision larger than 50b.
-///
-/// # See also
-/// The reverse mapping is [`fixed_point_to_f64()`].
-#[inline]
-pub fn f64_to_fixed_point(val: f64, min: f64, max: f64, precision: u8) -> u64 {
-    debug_assert!((1..53).contains(&precision));
-    debug_assert!(min < max);
-
-    let delta = max - min;
-    let mult = (1u64 << precision) as f64;
-    let bucket_size = delta / mult;
-    let upper_bound = f64_next_down(max).min(max - bucket_size);
-
-    // due to different cases of rounding error, we need to enforce upper_bound to be
-    // max-bucket_size, but also that upper_bound < max, which is not given for small enough
-    // bucket_size.
-    let val = val.clamp(min, upper_bound);
-
-    let res = (val - min) / bucket_size;
-    if res.fract() == 0.5 {
-        res as u64
-    } else {
-        // round down when getting x.5
-        res.round() as u64
-    }
-}
-
-/// Reverse the mapping given by [`f64_to_fixed_point()`].
-#[inline]
-pub fn fixed_point_to_f64(val: u64, min: f64, max: f64, precision: u8) -> f64 {
-    let delta = max - min;
-    let mult = (1u64 << precision) as f64;
-    let bucket_size = delta / mult;
-
-    bucket_size.mul_add(val as f64, min)
-}
-
-// taken from rfc/3173-float-next-up-down, commented out part about nan in infinity as it is not
-// needed.
-fn f64_next_down(this: f64) -> f64 {
-    const NEG_TINY_BITS: u64 = 0x8000_0000_0000_0001;
-    const CLEAR_SIGN_MASK: u64 = 0x7fff_ffff_ffff_ffff;
-
-    let bits = this.to_bits();
-    // if this.is_nan() || bits == f64::NEG_INFINITY.to_bits() {
-    // return this;
-    // }
-    let abs = bits & CLEAR_SIGN_MASK;
-    let next_bits = if abs == 0 {
-        NEG_TINY_BITS
-    } else if bits == abs {
-        bits - 1
-    } else {
-        bits + 1
-    };
-    f64::from_bits(next_bits)
-}
-
 #[cfg(test)]
 pub mod test {
-    use std::cmp::Ordering;
 
     use proptest::prelude::*;
 
-    use super::{
-        f32_to_u64, f64_to_fixed_point, f64_to_u64, fixed_point_to_f64, i64_to_u64, u64_to_f32,
-        u64_to_f64, u64_to_i64, BinarySerializable, FixedSize,
-    };
+    use super::{f64_to_u64, i64_to_u64, u64_to_f64, u64_to_i64, BinarySerializable, FixedSize};
 
     fn test_i64_converter_helper(val: i64) {
         assert_eq!(u64_to_i64(i64_to_u64(val)), val);
     }
 
     fn test_f64_converter_helper(val: f64) {
-        assert_eq!(u64_to_f64(f64_to_u64(val)).total_cmp(&val), Ordering::Equal);
-    }
-
-    fn test_f32_converter_helper(val: f32) {
-        assert_eq!(u64_to_f32(f32_to_u64(val)).total_cmp(&val), Ordering::Equal);
-    }
-
-    fn test_fixed_point_converter_helper(val: f64, min: f64, max: f64, precision: u8) {
-        let bucket_count = 1 << precision;
-
-        let packed = f64_to_fixed_point(val, min, max, precision);
-
-        assert!(packed < bucket_count, "used to much bits");
-
-        let depacked = fixed_point_to_f64(packed, min, max, precision);
-        let repacked = f64_to_fixed_point(depacked, min, max, precision);
-
-        assert_eq!(packed, repacked, "generational loss");
-
-        let error = (val.clamp(min, crate::f64_next_down(max)) - depacked).abs();
-
-        let expected = (max - min) / (bucket_count as f64);
-        assert!(
-            error <= (max - min) / (bucket_count as f64) * 2.0,
-            "error larger than expected"
-        );
+        assert_eq!(u64_to_f64(f64_to_u64(val)), val);
     }
 
     pub fn fixed_size_test<O: BinarySerializable + FixedSize + Default>() {
@@ -248,75 +125,12 @@ pub mod test {
         assert_eq!(buffer.len(), O::SIZE_IN_BYTES);
     }
 
-    fn fixed_point_bound() -> proptest::num::f64::Any {
-        proptest::num::f64::POSITIVE
-            | proptest::num::f64::NEGATIVE
-            | proptest::num::f64::NORMAL
-            | proptest::num::f64::ZERO
-    }
-
     proptest! {
         #[test]
-        fn test_f64_converter_monotonicity_proptest((left, right) in (proptest::num::f64::ANY, proptest::num::f64::ANY)) {
-            test_f64_converter_helper(left);
-            test_f64_converter_helper(right);
-
+        fn test_f64_converter_monotonicity_proptest((left, right) in (proptest::num::f64::NORMAL, proptest::num::f64::NORMAL)) {
             let left_u64 = f64_to_u64(left);
             let right_u64 = f64_to_u64(right);
-
-            assert_eq!(left_u64.cmp(&right_u64),  left.total_cmp(&right));
-        }
-
-        #[test]
-        fn test_f32_converter_monotonicity_proptest((left, right) in (proptest::num::f32::ANY, proptest::num::f32::ANY)) {
-            test_f32_converter_helper(left);
-            test_f32_converter_helper(right);
-
-            let left_u64 = f32_to_u64(left);
-            let right_u64 = f32_to_u64(right);
-            assert_eq!(left_u64.cmp(&right_u64),  left.total_cmp(&right));
-        }
-
-        #[test]
-        fn test_fixed_point_converter_proptest((left, right, min, max, precision) in
-                (fixed_point_bound(), fixed_point_bound(),
-                fixed_point_bound(), fixed_point_bound(),
-                proptest::num::u8::ANY)) {
-            // convert so all input are legal
-            let (min, max) = if min < max {
-                (min, max)
-            } else if min > max {
-                (max, min)
-            } else {
-                return Ok(()); // equals
-            };
-            if 1 > precision || precision >= 50 {
-                return Ok(());
-            }
-
-            let max_full_precision = 53.0 - precision as f64;
-            if (max / min).abs().log2().abs() > max_full_precision {
-                return Ok(());
-            }
-            // we will go in subnormal territories => loss of precision
-            if (((max - min).log2() - precision as f64) as i32) < f64::MIN_EXP {
-                return Ok(());
-            }
-
-            if (max - min).is_infinite() {
-                return Ok(());
-            }
-
-            test_fixed_point_converter_helper(left, min, max, precision);
-            test_fixed_point_converter_helper(right, min, max, precision);
-
-            let left_u64 = f64_to_fixed_point(left, min, max, precision);
-            let right_u64 = f64_to_fixed_point(right, min, max, precision);
-            if left < right {
-                assert!(left_u64 <= right_u64);
-            } else if left > right {
-                assert!(left_u64 >= right_u64)
-            }
+            assert_eq!(left_u64 < right_u64,  left < right);
         }
     }
 
@@ -354,27 +168,4 @@ pub mod test {
         assert!(f64_to_u64(-2.0) < f64_to_u64(1.0));
         assert!(f64_to_u64(-2.0) < f64_to_u64(-1.5));
     }
-
-    #[test]
-    fn test_f32_converter() {
-        test_f32_converter_helper(f32::INFINITY);
-        test_f32_converter_helper(f32::NEG_INFINITY);
-        test_f32_converter_helper(0.0);
-        test_f32_converter_helper(-0.0);
-        test_f32_converter_helper(1.0);
-        test_f32_converter_helper(-1.0);
-    }
-
-    #[test]
-    fn test_f32_order() {
-        assert!(!(f32_to_u64(f32::NEG_INFINITY)..f32_to_u64(f32::INFINITY))
-            .contains(&f32_to_u64(f32::NAN))); // nan is not a number
-        assert!(f32_to_u64(1.5) > f32_to_u64(1.0)); // same exponent, different mantissa
-        assert!(f32_to_u64(2.0) > f32_to_u64(1.0)); // same mantissa, different exponent
-        assert!(f32_to_u64(2.0) > f32_to_u64(1.5)); // different exponent and mantissa
-        assert!(f32_to_u64(1.0) > f32_to_u64(-1.0)); // pos > neg
-        assert!(f32_to_u64(-1.5) < f32_to_u64(-1.0));
-        assert!(f32_to_u64(-2.0) < f32_to_u64(1.0));
-        assert!(f32_to_u64(-2.0) < f32_to_u64(-1.5));
-    }
 }

fastfield_codecs/Cargo.toml

@@ -18,6 +18,7 @@ fastdivide = "0.4"
 log = "0.4"
 itertools = { version = "0.10.3" }
 measure_time = { version="0.8.2", optional=true}
+roaring = "0.10.1"
 
 [dev-dependencies]
 more-asserts = "0.3.0"

fastfield_codecs/benches/bench.rs

@@ -44,6 +44,15 @@ mod tests {
         open(OwnedBytes::new(buffer)).unwrap()
     }
 
+    pub fn serialize_and_load_dense<T: MonotonicallyMappableToU64 + Ord + Default>(
+        column: &[T],
+        fill_ratio: u32,
+    ) -> Arc<dyn Column<T>> {
+        let mut buffer = Vec::new();
+        serialize(VecColumn::from(&column), &mut buffer, &ALL_CODEC_TYPES).unwrap();
+        open_dense(OwnedBytes::new(buffer), fill_ratio).unwrap()
+    }
+
     #[bench]
     fn bench_intfastfield_jumpy_veclookup(b: &mut Bencher) {
         let permutation = generate_permutation();
@@ -183,6 +192,67 @@ mod tests {
         });
     }
 
+    #[bench]
+    fn bench_intfastfield_stride7_fflookup_sparse_roaring(b: &mut Bencher) {
+        let permutation = generate_permutation();
+        let n = permutation.len();
+        let column: Arc<dyn Column<u64>> = serialize_and_load(&permutation);
+        let column = SparseCodecRoaringBitmap::with_full(column);
+        b.iter(|| {
+            let mut a = 0u64;
+            for i in (0..n / 7).map(|val| val * 7) {
+                a += column.get_val(i as u64);
+            }
+            a
+        });
+    }
+
+    #[bench]
+    fn bench_intfastfield_stride7_fflookup_dense_bitmap_with_offset(b: &mut Bencher) {
+        let permutation = generate_permutation();
+        let n = permutation.len();
+        let column: Arc<dyn Column<u64>> = serialize_and_load_dense(&permutation, 1000);
+        b.iter(|| {
+            let mut a = 0u64;
+            for i in (0..n / 7).map(|val| val * 7) {
+                a += column.get_val(i as u64);
+            }
+            a
+        });
+    }
+
+    #[bench]
+    fn bench_intfastfield_stride7_fflookup_dense_bitmap_with_offset_70percent_dense(
+        b: &mut Bencher,
+    ) {
+        let permutation = generate_permutation();
+        let n = permutation.len();
+        let column: Arc<dyn Column<u64>> = serialize_and_load_dense(&permutation, 700);
+        b.iter(|| {
+            let mut a = 0u64;
+            for i in (0..n / 7).map(|val| val * 7) {
+                a += column.get_val(i as u64);
+            }
+            a
+        });
+    }
+
+    #[bench]
+    fn bench_intfastfield_stride7_fflookup_dense_bitmap_with_offset_20percent_dense(
+        b: &mut Bencher,
+    ) {
+        let permutation = generate_permutation();
+        let n = permutation.len();
+        let column: Arc<dyn Column<u64>> = serialize_and_load_dense(&permutation, 200);
+        b.iter(|| {
+            let mut a = 0u64;
+            for i in (0..n / 7).map(|val| val * 7) {
+                a += column.get_val(i as u64);
+            }
+            a
+        });
+    }
+
     #[bench]
     fn bench_intfastfield_scan_all_fflookup(b: &mut Bencher) {
         let permutation = generate_permutation();

fastfield_codecs/src/lib.rs

@@ -22,6 +22,7 @@ mod compact_space;
 mod line;
 mod linear;
 mod monotonic_mapping;
+mod sparse_codec_wrapper;
 
 mod column;
 mod gcd;
@@ -35,6 +36,8 @@ pub use self::monotonic_mapping::MonotonicallyMappableToU64;
 pub use self::serialize::{
     estimate, serialize, serialize_and_load, serialize_u128, NormalizedHeader,
 };
+pub use sparse_codec_wrapper::DenseCodec;
+pub use sparse_codec_wrapper::SparseCodecRoaringBitmap;
 
 #[derive(PartialEq, Eq, PartialOrd, Ord, Debug, Clone, Copy)]
 #[repr(u8)]
@@ -76,6 +79,44 @@ impl FastFieldCodecType {
 pub fn open_u128(bytes: OwnedBytes) -> io::Result<Arc<dyn Column<u128>>> {
     Ok(Arc::new(CompactSpaceDecompressor::open(bytes)?))
 }
+//DenseCodec
+//
+/// Returns the correct codec reader wrapped in the `Arc` for the data.
+pub fn open_dense<T: MonotonicallyMappableToU64>(
+    mut bytes: OwnedBytes,
+    fill_ratio: u32,
+) -> io::Result<Arc<dyn Column<T>>> {
+    let header = Header::deserialize(&mut bytes)?;
+    match header.codec_type {
+        FastFieldCodecType::Bitpacked => {
+            open_specific_codec_dense::<BitpackedCodec, _>(bytes, &header, fill_ratio)
+        }
+        FastFieldCodecType::Linear => {
+            open_specific_codec_dense::<LinearCodec, _>(bytes, &header, fill_ratio)
+        }
+        FastFieldCodecType::BlockwiseLinear => {
+            open_specific_codec_dense::<BlockwiseLinearCodec, _>(bytes, &header, fill_ratio)
+        }
+    }
+}
+
+fn open_specific_codec_dense<C: FastFieldCodec, Item: MonotonicallyMappableToU64>(
+    bytes: OwnedBytes,
+    header: &Header,
+    fill_ratio: u32,
+) -> io::Result<Arc<dyn Column<Item>>> {
+    let normalized_header = header.normalized();
+    let reader = C::open_from_bytes(bytes, normalized_header)?;
+    let reader = DenseCodec::with_fill_ratio(reader, fill_ratio);
+    let min_value = header.min_value;
+    if let Some(gcd) = header.gcd {
+        let monotonic_mapping = move |val: u64| Item::from_u64(min_value + val * gcd.get());
+        Ok(Arc::new(monotonic_map_column(reader, monotonic_mapping)))
+    } else {
+        let monotonic_mapping = move |val: u64| Item::from_u64(min_value + val);
+        Ok(Arc::new(monotonic_map_column(reader, monotonic_mapping)))
+    }
+}
 
 /// Returns the correct codec reader wrapped in the `Arc` for the data.
 pub fn open<T: MonotonicallyMappableToU64>(
@@ -312,7 +353,7 @@ mod tests {
 
     #[test]
     fn estimation_test_bad_interpolation_case_monotonically_increasing() {
-        let mut data: Vec<u64> = (201..=20000_u64).collect();
+        let mut data: Vec<u64> = (200..=20000_u64).collect();
         data.push(1_000_000);
         let data: VecColumn = data.as_slice().into();
 
@@ -440,6 +481,7 @@ mod bench {
         let data: Vec<_> = get_data();
         bench_get::<BitpackedCodec>(b, &data);
     }
+
     #[bench]
     fn bench_fastfield_bitpack_get_dynamic(b: &mut Bencher) {
         let data: Vec<_> = get_data();

fastfield_codecs/src/line.rs

@@ -68,37 +68,29 @@ impl Line {
     }
 
     // Same as train, but the intercept is only estimated from provided sample positions
-    pub fn estimate(sample_positions_and_values: &[(u64, u64)]) -> Self {
-        let first_val = sample_positions_and_values[0].1;
-        let last_val = sample_positions_and_values[sample_positions_and_values.len() - 1].1;
-        let num_vals = sample_positions_and_values[sample_positions_and_values.len() - 1].0 + 1;
+    pub fn estimate(ys: &dyn Column, sample_positions: &[u64]) -> Self {
         Self::train_from(
-            first_val,
-            last_val,
-            num_vals,
-            sample_positions_and_values.iter().cloned(),
+            ys,
+            sample_positions
+                .iter()
+                .cloned()
+                .map(|pos| (pos, ys.get_val(pos))),
         )
     }
 
     // Intercept is only computed from provided positions
-    fn train_from(
-        first_val: u64,
-        last_val: u64,
-        num_vals: u64,
-        positions_and_values: impl Iterator<Item = (u64, u64)>,
-    ) -> Self {
-        // TODO replace with let else
-        let idx_last_val = if let Some(idx_last_val) = NonZeroU64::new(num_vals - 1) {
-            idx_last_val
+    fn train_from(ys: &dyn Column, positions_and_values: impl Iterator<Item = (u64, u64)>) -> Self {
+        let num_vals = if let Some(num_vals) = NonZeroU64::new(ys.num_vals() - 1) {
+            num_vals
         } else {
             return Line::default();
         };
 
-        let y0 = first_val;
-        let y1 = last_val;
+        let y0 = ys.get_val(0);
+        let y1 = ys.get_val(num_vals.get());
 
         // We first independently pick our slope.
-        let slope = compute_slope(y0, y1, idx_last_val);
+        let slope = compute_slope(y0, y1, num_vals);
 
         // We picked our slope. Note that it does not have to be perfect.
         // Now we need to compute the best intercept.
@@ -146,12 +138,8 @@ impl Line {
     /// This function is only invariable by translation if all of the
     /// `ys` are packaged into half of the space. (See heuristic below)
     pub fn train(ys: &dyn Column) -> Self {
-        let first_val = ys.iter().next().unwrap();
-        let last_val = ys.iter().nth(ys.num_vals() as usize - 1).unwrap();
         Self::train_from(
-            first_val,
-            last_val,
-            ys.num_vals(),
+            ys,
             ys.iter().enumerate().map(|(pos, val)| (pos as u64, val)),
         )
     }

fastfield_codecs/src/linear.rs

@@ -126,20 +126,18 @@ impl FastFieldCodec for LinearCodec {
             return None; // disable compressor for this case
         }
 
-        let limit_num_vals = column.num_vals().min(100_000);
+        // let's sample at 0%, 5%, 10% .. 95%, 100%
+        let num_vals = column.num_vals() as f32 / 100.0;
+        let sample_positions = (0..20)
+            .map(|pos| (num_vals * pos as f32 * 5.0) as u64)
+            .collect::<Vec<_>>();
 
-        let num_samples = 100;
-        let step_size = (limit_num_vals / num_samples).max(1); // 20 samples
-        let mut sample_positions_and_values: Vec<_> = Vec::new();
-        for (pos, val) in column.iter().enumerate().step_by(step_size as usize) {
-            sample_positions_and_values.push((pos as u64, val));
-        }
+        let line = Line::estimate(column, &sample_positions);
 
-        let line = Line::estimate(&sample_positions_and_values);
-
-        let estimated_bit_width = sample_positions_and_values
+        let estimated_bit_width = sample_positions
             .into_iter()
-            .map(|(pos, actual_value)| {
+            .map(|pos| {
+                let actual_value = column.get_val(pos);
                 let interpolated_val = line.eval(pos as u64);
                 actual_value.wrapping_sub(interpolated_val)
             })
@@ -148,7 +146,6 @@ impl FastFieldCodec for LinearCodec {
             .max()
             .unwrap_or(0);
 
-        // Extrapolate to whole column
         let num_bits = (estimated_bit_width as u64 * column.num_vals() as u64) + 64;
         let num_bits_uncompressed = 64 * column.num_vals();
         Some(num_bits as f32 / num_bits_uncompressed as f32)

fastfield_codecs/src/sparse_codec_wrapper.rs

@@ -0,0 +1,157 @@
+use std::sync::Arc;
+
+use rand::{thread_rng, Rng};
+use roaring::RoaringBitmap;
+
+use crate::Column;
+
+pub struct SparseCodecRoaringBitmap {
+    null: RoaringBitmap,
+    column: Arc<dyn Column<u64>>, // column: C,
+}
+
+impl SparseCodecRoaringBitmap {
+    pub fn with_full(column: Arc<dyn Column<u64>>) -> Self {
+        let mut rb = RoaringBitmap::new();
+        rb.insert_range(0..column.num_vals() as u32 + 1);
+        Self { null: rb, column }
+    }
+}
+
+impl Column for SparseCodecRoaringBitmap {
+    fn get_val(&self, idx: u64) -> u64 {
+        let position_of_val = self.null.rank(idx as u32);
+        self.column.get_val(position_of_val) // TODO this does not handle null!
+                                             // self.null.select(num_vals)
+    }
+
+    fn min_value(&self) -> u64 {
+        todo!()
+    }
+
+    fn max_value(&self) -> u64 {
+        todo!()
+    }
+
+    fn num_vals(&self) -> u64 {
+        todo!()
+    }
+}
+
+pub struct DenseCodec<C> {
+    // the bitmap blocks of length 64 bit each
+    blocks: Vec<u64>,
+    // the offset for each block
+    offsets: Vec<u32>,
+    column: C, // column: C,
+}
+
+impl<C: Column> DenseCodec<C> {
+    // fill ratio valid range 0..1000 1000 == all elements, 1 == every 1000th element
+    pub fn with_fill_ratio(column: C, fill_ratio: u32) -> Self {
+        let mut rng = thread_rng();
+        let num_blocks = (column.num_vals() as usize / 64) + 1;
+        let mut blocks = Vec::with_capacity(num_blocks);
+        let mut offsets = Vec::with_capacity(num_blocks);
+        // fill all blocks
+        let mut offset = 0;
+        for _block_num in 0..num_blocks {
+            let mut block = 0;
+            for n in 0..64 {
+                if rng.gen_range(0..=1000) <= fill_ratio {
+                    set_bit_at(&mut block, n);
+                }
+            }
+            blocks.push(block);
+            offsets.push(offset);
+            offset += block.count_ones();
+        }
+
+        Self {
+            blocks,
+            offsets,
+            column,
+        }
+    }
+
+    pub fn with_full(column: C) -> Self {
+        let num_blocks = (column.num_vals() as usize / 64) + 1;
+        let mut blocks = Vec::with_capacity(num_blocks);
+        let mut offsets = Vec::with_capacity(num_blocks);
+        // fill all blocks
+        let mut offset = 0;
+        for _block_num in 0..num_blocks {
+            let block = u64::MAX;
+            blocks.push(block);
+            offsets.push(offset);
+            offset += block.count_ones();
+        }
+
+        Self {
+            blocks,
+            offsets,
+            column,
+        }
+    }
+}
+
+fn gen_mask(msb: u64) -> u64 {
+    let src = 1 << msb;
+    src - 1
+}
+
+fn get_bit_at(input: u64, n: u64) -> bool {
+    input & (1 << n) != 0
+}
+
+fn set_bit_at(input: &mut u64, n: u64) {
+    *input |= 1 << n;
+}
+
+impl<C: Column> Column for DenseCodec<C> {
+    fn get_val(&self, idx: u64) -> u64 {
+        let block_pos = idx / 64;
+        let pos_in_block = idx % 64;
+        let offset = self.offsets[block_pos as usize];
+        let bitvec = self.blocks[block_pos as usize];
+        let offset_in_block = (bitvec & gen_mask(pos_in_block)).count_ones();
+        let dense_idx = offset as u64 + offset_in_block as u64;
+        if get_bit_at(bitvec, pos_in_block) {
+            self.column.get_val(dense_idx)
+        } else {
+            0 // TODO null
+        }
+    }
+
+    fn min_value(&self) -> u64 {
+        todo!()
+    }
+
+    fn max_value(&self) -> u64 {
+        todo!()
+    }
+
+    fn num_vals(&self) -> u64 {
+        todo!()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    //use itertools::Itertools;
+
+    //use super::*;
+    //use crate::serialize_and_load;
+
+    //#[test]
+    //fn dense_test() {
+    //let data = (0..100u64).collect_vec();
+    //{
+    //let column = serialize_and_load(&data);
+    //let dense = DenseCodec::with_full(column);
+    //for i in 0..100 {
+    //dense.get_val(i);
+    //}
+    //}
+    //}
+}

src/aggregation/bucket/histogram/histogram.rs

@@ -452,7 +452,7 @@ fn intermediate_buckets_to_final_buckets_fill_gaps(
     histogram_req: &HistogramAggregation,
     sub_aggregation: &AggregationsInternal,
 ) -> crate::Result<Vec<BucketEntry>> {
-    // Generate the full list of buckets without gaps.
+    // Generate the the full list of buckets without gaps.
     //
     // The bounds are the min max from the current buckets, optionally extended by
     // extended_bounds from the request

src/aggregation/bucket/range.rs

@@ -323,8 +323,8 @@ impl SegmentRangeCollector {
 /// Converts the user provided f64 range value to fast field value space.
 ///
 /// Internally fast field values are always stored as u64.
-/// If the fast field has u64 `[1, 2, 5]`, these values are stored as is in the fast field.
-/// A fast field with f64 `[1.0, 2.0, 5.0]` is converted to u64 space, using a
+/// If the fast field has u64 [1,2,5], these values are stored as is in the fast field.
+/// A fast field with f64 [1.0, 2.0, 5.0] is converted to u64 space, using a
 /// monotonic mapping function, so the order is preserved.
 ///
 /// Consequently, a f64 user range 1.0..3.0 needs to be converted to fast field value space using

src/collector/facet_collector.rs

@@ -338,7 +338,11 @@ impl SegmentCollector for FacetSegmentCollector {
         let mut previous_collapsed_ord: usize = usize::MAX;
         for &facet_ord in &self.facet_ords_buf {
             let collapsed_ord = self.collapse_mapping[facet_ord as usize];
-            self.counts[collapsed_ord] += u64::from(collapsed_ord != previous_collapsed_ord);
+            self.counts[collapsed_ord] += if collapsed_ord == previous_collapsed_ord {
+                0
+            } else {
+                1
+            };
             previous_collapsed_ord = collapsed_ord;
         }
     }

src/core/index.rs

@@ -19,7 +19,7 @@ use crate::error::{DataCorruption, TantivyError};
 use crate::indexer::index_writer::{MAX_NUM_THREAD, MEMORY_ARENA_NUM_BYTES_MIN};
 use crate::indexer::segment_updater::save_metas;
 use crate::reader::{IndexReader, IndexReaderBuilder};
-use crate::schema::{Cardinality, Field, FieldType, Schema};
+use crate::schema::{Field, FieldType, Schema};
 use crate::tokenizer::{TextAnalyzer, TokenizerManager};
 use crate::IndexWriter;
 
@@ -152,7 +152,9 @@ impl IndexBuilder {
     /// This should only be used for unit tests.
     pub fn create_in_ram(self) -> Result<Index, TantivyError> {
         let ram_directory = RamDirectory::create();
-        self.create(ram_directory)
+        Ok(self
+            .create(ram_directory)
+            .expect("Creating a RamDirectory should never fail"))
     }
 
     /// Creates a new index in a given filepath.
@@ -226,44 +228,10 @@ impl IndexBuilder {
             ))
         }
     }
-
-    fn validate(&self) -> crate::Result<()> {
-        if let Some(schema) = self.schema.as_ref() {
-            if let Some(sort_by_field) = self.index_settings.sort_by_field.as_ref() {
-                let schema_field = schema.get_field(&sort_by_field.field).ok_or_else(|| {
-                    TantivyError::InvalidArgument(format!(
-                        "Field to sort index {} not found in schema",
-                        sort_by_field.field
-                    ))
-                })?;
-                let entry = schema.get_field_entry(schema_field);
-                if !entry.is_fast() {
-                    return Err(TantivyError::InvalidArgument(format!(
-                        "Field {} is no fast field. Field needs to be a single value fast field \
-                         to be used to sort an index",
-                        sort_by_field.field
-                    )));
-                }
-                if entry.field_type().fastfield_cardinality() != Some(Cardinality::SingleValue) {
-                    return Err(TantivyError::InvalidArgument(format!(
-                        "Only single value fast field Cardinality supported for sorting index {}",
-                        sort_by_field.field
-                    )));
-                }
-            }
-            Ok(())
-        } else {
-            Err(TantivyError::InvalidArgument(
-                "no schema passed".to_string(),
-            ))
-        }
-    }
-
     /// Creates a new index given an implementation of the trait `Directory`.
     ///
     /// If a directory previously existed, it will be erased.
     fn create<T: Into<Box<dyn Directory>>>(self, dir: T) -> crate::Result<Index> {
-        self.validate()?;
         let dir = dir.into();
         let directory = ManagedDirectory::wrap(dir)?;
         save_new_metas(

src/core/inverted_index_reader.rs

@@ -15,11 +15,12 @@ use crate::termdict::TermDictionary;
 ///
 /// It is safe to delete the segment associated with
 /// an `InvertedIndexReader`. As long as it is open,
-/// the [`FileSlice`] it is relying on should
+/// the `FileSlice` it is relying on should
 /// stay available.
 ///
+///
 /// `InvertedIndexReader` are created by calling
-/// [`SegmentReader::inverted_index()`](crate::SegmentReader::inverted_index).
+/// the `SegmentReader`'s [`.inverted_index(...)`] method
 pub struct InvertedIndexReader {
     termdict: TermDictionary,
     postings_file_slice: FileSlice,
@@ -74,7 +75,7 @@ impl InvertedIndexReader {
     ///
     /// This is useful for enumerating through a list of terms,
     /// and consuming the associated posting lists while avoiding
-    /// reallocating a [`BlockSegmentPostings`].
+    /// reallocating a `BlockSegmentPostings`.
     ///
     /// # Warning
     ///
@@ -95,7 +96,7 @@ impl InvertedIndexReader {
     /// Returns a block postings given a `Term`.
     /// This method is for an advanced usage only.
     ///
-    /// Most users should prefer using [`Self::read_postings()`] instead.
+    /// Most user should prefer using `read_postings` instead.
     pub fn read_block_postings(
         &self,
         term: &Term,
@@ -109,7 +110,7 @@ impl InvertedIndexReader {
     /// Returns a block postings given a `term_info`.
     /// This method is for an advanced usage only.
     ///
-    /// Most users should prefer using [`Self::read_postings()`] instead.
+    /// Most user should prefer using `read_postings` instead.
     pub fn read_block_postings_from_terminfo(
         &self,
         term_info: &TermInfo,
@@ -129,7 +130,7 @@ impl InvertedIndexReader {
     /// Returns a posting object given a `term_info`.
     /// This method is for an advanced usage only.
     ///
-    /// Most users should prefer using [`Self::read_postings()`] instead.
+    /// Most user should prefer using `read_postings` instead.
     pub fn read_postings_from_terminfo(
         &self,
         term_info: &TermInfo,
@@ -163,12 +164,12 @@ impl InvertedIndexReader {
     /// or `None` if the term has never been encountered and indexed.
     ///
     /// If the field was not indexed with the indexing options that cover
-    /// the requested options, the returned [`SegmentPostings`] the method does not fail
+    /// the requested options, the returned `SegmentPostings` the method does not fail
     /// and returns a `SegmentPostings` with as much information as possible.
     ///
-    /// For instance, requesting [`IndexRecordOption::WithFreqs`] for a
-    /// [`TextOptions`](crate::schema::TextOptions) that does not index position
-    /// will return a [`SegmentPostings`] with `DocId`s and frequencies.
+    /// For instance, requesting `IndexRecordOption::Freq` for a
+    /// `TextIndexingOptions` that does not index position will return a `SegmentPostings`
+    /// with `DocId`s and frequencies.
     pub fn read_postings(
         &self,
         term: &Term,
@@ -210,7 +211,7 @@ impl InvertedIndexReader {
     /// Returns a block postings given a `Term`.
     /// This method is for an advanced usage only.
     ///
-    /// Most users should prefer using [`Self::read_postings()`] instead.
+    /// Most user should prefer using `read_postings` instead.
     pub async fn warm_postings(
         &self,
         term: &Term,

src/core/segment_reader.rs

@@ -216,10 +216,10 @@ impl SegmentReader {
     /// term dictionary associated with a specific field,
     /// and opening the posting list associated with any term.
     ///
-    /// If the field is not marked as index, a warning is logged and an empty `InvertedIndexReader`
+    /// If the field is not marked as index, a warn is logged and an empty `InvertedIndexReader`
     /// is returned.
-    /// Similarly, if the field is marked as indexed but no term has been indexed for the given
-    /// index, an empty `InvertedIndexReader` is returned (but no warning is logged).
+    /// Similarly if the field is marked as indexed but no term has been indexed for the given
+    /// index. an empty `InvertedIndexReader` is returned (but no warning is logged).
     pub fn inverted_index(&self, field: Field) -> crate::Result<Arc<InvertedIndexReader>> {
         if let Some(inv_idx_reader) = self
             .inv_idx_reader_cache

src/directory/file_slice.rs

@@ -13,8 +13,8 @@ use crate::directory::OwnedBytes;
 /// By contract, whatever happens to the directory file, as long as a FileHandle
 /// is alive, the data associated with it cannot be altered or destroyed.
 ///
-/// The underlying behavior is therefore specific to the [`Directory`](crate::Directory) that
-/// created it. Despite its name, a [`FileSlice`] may or may not directly map to an actual file
+/// The underlying behavior is therefore specific to the `Directory` that created it.
+/// Despite its name, a `FileSlice` may or may not directly map to an actual file
 /// on the filesystem.
 
 #[async_trait]

src/directory/mmap_directory.rs

@@ -304,7 +304,7 @@ pub(crate) fn atomic_write(path: &Path, content: &[u8]) -> io::Result<()> {
             "Path {:?} does not have parent directory.",
         )
     })?;
-    let mut tempfile = tempfile::Builder::new().tempfile_in(parent_path)?;
+    let mut tempfile = tempfile::Builder::new().tempfile_in(&parent_path)?;
     tempfile.write_all(content)?;
     tempfile.flush()?;
     tempfile.as_file_mut().sync_data()?;
@@ -610,11 +610,9 @@ mod tests {
             assert!(num_segments <= 4);
             let num_components_except_deletes_and_tempstore =
                 crate::core::SegmentComponent::iterator().len() - 2;
-            let num_mmapped = mmap_directory.get_cache_info().mmapped.len();
-            assert!(
-                num_mmapped <= num_segments * num_components_except_deletes_and_tempstore,
-                "Expected at most {} mmapped files, got {num_mmapped}",
-                num_segments * num_components_except_deletes_and_tempstore
+            assert_eq!(
+                num_segments * num_components_except_deletes_and_tempstore,
+                mmap_directory.get_cache_info().mmapped.len()
             );
         }
         // This test failed on CI. The last Mmap is dropped from the merging thread so there might

src/fastfield/mod.rs

@@ -26,7 +26,7 @@ pub use self::alive_bitset::{intersect_alive_bitsets, write_alive_bitset, AliveB
 pub use self::bytes::{BytesFastFieldReader, BytesFastFieldWriter};
 pub use self::error::{FastFieldNotAvailableError, Result};
 pub use self::facet_reader::FacetReader;
-pub(crate) use self::multivalued::{get_fastfield_codecs_for_multivalue, MultivalueStartIndex};
+pub(crate) use self::multivalued::MultivalueStartIndex;
 pub use self::multivalued::{MultiValuedFastFieldReader, MultiValuedFastFieldWriter};
 pub use self::readers::FastFieldReaders;
 pub(crate) use self::readers::{type_and_cardinality, FastType};

src/fastfield/multivalued/mod.rs

@@ -1,22 +1,10 @@
 mod reader;
 mod writer;
 
-use fastfield_codecs::FastFieldCodecType;
-
 pub use self::reader::MultiValuedFastFieldReader;
 pub use self::writer::MultiValuedFastFieldWriter;
 pub(crate) use self::writer::MultivalueStartIndex;
 
-/// The valid codecs for multivalue values excludes the linear interpolation codec.
-///
-/// This limitation is only valid for the values, not the offset index of the multivalue index.
-pub(crate) fn get_fastfield_codecs_for_multivalue() -> [FastFieldCodecType; 2] {
-    [
-        FastFieldCodecType::Bitpacked,
-        FastFieldCodecType::BlockwiseLinear,
-    ]
-}
-
 #[cfg(test)]
 mod tests {
     use proptest::strategy::Strategy;

src/fastfield/multivalued/writer.rs

@@ -1,9 +1,9 @@
 use std::io;
+use std::sync::Mutex;
 
 use fastfield_codecs::{Column, MonotonicallyMappableToU64, VecColumn};
 use fnv::FnvHashMap;
 
-use super::get_fastfield_codecs_for_multivalue;
 use crate::fastfield::{value_to_u64, CompositeFastFieldSerializer, FastFieldType};
 use crate::indexer::doc_id_mapping::DocIdMapping;
 use crate::postings::UnorderedTermId;
@@ -195,12 +195,7 @@ impl MultiValuedFastFieldWriter {
                 }
             }
             let col = VecColumn::from(&values[..]);
-            serializer.create_auto_detect_u64_fast_field_with_idx_and_codecs(
-                self.field,
-                col,
-                1,
-                &get_fastfield_codecs_for_multivalue(),
-            )?;
+            serializer.create_auto_detect_u64_fast_field_with_idx(self.field, col, 1)?;
         }
         Ok(())
     }
@@ -209,59 +204,112 @@ impl MultiValuedFastFieldWriter {
 pub(crate) struct MultivalueStartIndex<'a, C: Column> {
     column: &'a C,
     doc_id_map: &'a DocIdMapping,
-    min: u64,
-    max: u64,
+    min_max_opt: Mutex<Option<(u64, u64)>>,
+    random_seeker: Mutex<MultivalueStartIndexRandomSeeker<'a, C>>,
+}
+
+struct MultivalueStartIndexRandomSeeker<'a, C: Column> {
+    seek_head: MultivalueStartIndexIter<'a, C>,
+    seek_next_id: u64,
+}
+impl<'a, C: Column> MultivalueStartIndexRandomSeeker<'a, C> {
+    fn new(column: &'a C, doc_id_map: &'a DocIdMapping) -> Self {
+        Self {
+            seek_head: MultivalueStartIndexIter {
+                column,
+                doc_id_map,
+                new_doc_id: 0,
+                offset: 0u64,
+            },
+            seek_next_id: 0u64,
+        }
+    }
 }
 
 impl<'a, C: Column> MultivalueStartIndex<'a, C> {
     pub fn new(column: &'a C, doc_id_map: &'a DocIdMapping) -> Self {
         assert_eq!(column.num_vals(), doc_id_map.num_old_doc_ids() as u64 + 1);
-        let (min, max) =
-            tantivy_bitpacker::minmax(iter_remapped_multivalue_index(doc_id_map, column))
-                .unwrap_or((0u64, 0u64));
         MultivalueStartIndex {
             column,
             doc_id_map,
-            min,
-            max,
+            min_max_opt: Mutex::default(),
+            random_seeker: Mutex::new(MultivalueStartIndexRandomSeeker::new(column, doc_id_map)),
         }
     }
+
+    fn minmax(&self) -> (u64, u64) {
+        if let Some((min, max)) = *self.min_max_opt.lock().unwrap() {
+            return (min, max);
+        }
+        let (min, max) = tantivy_bitpacker::minmax(self.iter()).unwrap_or((0u64, 0u64));
+        *self.min_max_opt.lock().unwrap() = Some((min, max));
+        (min, max)
+    }
 }
 impl<'a, C: Column> Column for MultivalueStartIndex<'a, C> {
-    fn get_val(&self, _idx: u64) -> u64 {
-        unimplemented!()
+    fn get_val(&self, idx: u64) -> u64 {
+        let mut random_seeker_lock = self.random_seeker.lock().unwrap();
+        if random_seeker_lock.seek_next_id > idx {
+            *random_seeker_lock =
+                MultivalueStartIndexRandomSeeker::new(self.column, self.doc_id_map);
+        }
+        let to_skip = idx - random_seeker_lock.seek_next_id;
+        random_seeker_lock.seek_next_id = idx + 1;
+        random_seeker_lock.seek_head.nth(to_skip as usize).unwrap()
     }
 
     fn min_value(&self) -> u64 {
-        self.min
+        self.minmax().0
     }
 
     fn max_value(&self) -> u64 {
-        self.max
+        self.minmax().1
     }
 
     fn num_vals(&self) -> u64 {
         (self.doc_id_map.num_new_doc_ids() + 1) as u64
     }
 
-    fn iter(&self) -> Box<dyn Iterator<Item = u64> + '_> {
-        Box::new(iter_remapped_multivalue_index(
-            self.doc_id_map,
-            &self.column,
-        ))
+    fn iter<'b>(&'b self) -> Box<dyn Iterator<Item = u64> + 'b> {
+        Box::new(MultivalueStartIndexIter::new(self.column, self.doc_id_map))
     }
 }
 
-fn iter_remapped_multivalue_index<'a, C: Column>(
-    doc_id_map: &'a DocIdMapping,
-    column: &'a C,
-) -> impl Iterator<Item = u64> + 'a {
-    let mut offset = 0;
-    std::iter::once(0).chain(doc_id_map.iter_old_doc_ids().map(move |old_doc| {
-        let num_vals_for_doc = column.get_val(old_doc as u64 + 1) - column.get_val(old_doc as u64);
-        offset += num_vals_for_doc;
-        offset as u64
-    }))
+struct MultivalueStartIndexIter<'a, C: Column> {
+    pub column: &'a C,
+    pub doc_id_map: &'a DocIdMapping,
+    pub new_doc_id: usize,
+    pub offset: u64,
+}
+
+impl<'a, C: Column> MultivalueStartIndexIter<'a, C> {
+    fn new(column: &'a C, doc_id_map: &'a DocIdMapping) -> Self {
+        Self {
+            column,
+            doc_id_map,
+            new_doc_id: 0,
+            offset: 0,
+        }
+    }
+}
+
+impl<'a, C: Column> Iterator for MultivalueStartIndexIter<'a, C> {
+    type Item = u64;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        if self.new_doc_id > self.doc_id_map.num_new_doc_ids() {
+            return None;
+        }
+        let new_doc_id = self.new_doc_id;
+        self.new_doc_id += 1;
+        let start_offset = self.offset;
+        if new_doc_id < self.doc_id_map.num_new_doc_ids() {
+            let old_doc = self.doc_id_map.get_old_doc_id(new_doc_id as u32) as u64;
+            let num_vals_for_doc = self.column.get_val(old_doc + 1) - self.column.get_val(old_doc);
+            self.offset += num_vals_for_doc;
+        }
+        Some(start_offset)
+    }
 }
 
 #[cfg(test)]
@@ -296,5 +344,11 @@ mod tests {
             vec![0, 1, 1, 2, 3, 5, 8, 13, 21, 34, 55]
         );
         assert_eq!(multivalue_start_index.num_vals(), 11);
+        assert_eq!(multivalue_start_index.get_val(3), 2);
+        assert_eq!(multivalue_start_index.get_val(5), 5);
+        assert_eq!(multivalue_start_index.get_val(8), 21);
+        assert_eq!(multivalue_start_index.get_val(4), 3);
+        assert_eq!(multivalue_start_index.get_val(0), 0);
+        assert_eq!(multivalue_start_index.get_val(10), 55);
     }
 }

src/fastfield/serializer/mod.rs

@@ -70,20 +70,6 @@ impl CompositeFastFieldSerializer {
         Ok(())
     }
 
-    /// Serialize data into a new u64 fast field. The best compression codec of the the provided
-    /// will be chosen.
-    pub fn create_auto_detect_u64_fast_field_with_idx_and_codecs<T: MonotonicallyMappableToU64>(
-        &mut self,
-        field: Field,
-        fastfield_accessor: impl Column<T>,
-        idx: usize,
-        codec_types: &[FastFieldCodecType],
-    ) -> io::Result<()> {
-        let field_write = self.composite_write.for_field_with_idx(field, idx);
-        fastfield_codecs::serialize(fastfield_accessor, field_write, codec_types)?;
-        Ok(())
-    }
-
     /// Start serializing a new [u8] fast field. Use the returned writer to write data into the
     /// bytes field. To associate the bytes with documents a seperate index must be created on
     /// index 0. See bytes/writer.rs::serialize for an example.

src/fastfield/writer.rs

@@ -391,8 +391,15 @@ impl<'map, 'bitp> Column for WriterFastFieldAccessProvider<'map, 'bitp> {
     /// # Panics
     ///
     /// May panic if `doc` is greater than the index.
-    fn get_val(&self, _doc: u64) -> u64 {
-        unimplemented!()
+    fn get_val(&self, doc: u64) -> u64 {
+        if let Some(doc_id_map) = self.doc_id_map {
+            self.vals
+                .get(doc_id_map.get_old_doc_id(doc as u32) as usize) // consider extra
+                                                                     // FastFieldReader wrapper for
+                                                                     // non doc_id_map
+        } else {
+            self.vals.get(doc as usize)
+        }
     }
 
     fn iter(&self) -> Box<dyn Iterator<Item = u64> + '_> {

src/indexer/doc_id_mapping.rs

@@ -34,6 +34,10 @@ impl SegmentDocIdMapping {
         self.new_doc_id_to_old_doc_addr.len()
     }
 
+    pub(crate) fn get_old_doc_addr(&self, new_doc_id: DocId) -> DocAddress {
+        self.new_doc_id_to_old_doc_addr[new_doc_id as usize]
+    }
+
     /// This flags means the segments are simply stacked in the order of their ordinal.
     /// e.g. [(0, 1), .. (n, 1), (0, 2)..., (m, 2)]
     ///

src/indexer/flat_map_with_buffer.rs

@@ -1,69 +0,0 @@
-pub struct FlatMapWithBuffer<T, F, Iter> {
-    buffer: Vec<T>,
-    fill_buffer: F,
-    underlying_it: Iter,
-}
-
-impl<T, F, Iter, I> Iterator for FlatMapWithBuffer<T, F, Iter>
-where
-    Iter: Iterator<Item = I>,
-    F: Fn(I, &mut Vec<T>),
-{
-    type Item = T;
-
-    fn next(&mut self) -> Option<Self::Item> {
-        while self.buffer.is_empty() {
-            let next_el = self.underlying_it.next()?;
-            (self.fill_buffer)(next_el, &mut self.buffer);
-            // We will pop elements, so we reverse the buffer first.
-            self.buffer.reverse();
-        }
-        self.buffer.pop()
-    }
-}
-
-pub trait FlatMapWithBufferIter: Iterator {
-    /// Function similar to `flat_map`, but allows reusing a shared `Vec`.
-    fn flat_map_with_buffer<F, T>(self, fill_buffer: F) -> FlatMapWithBuffer<T, F, Self>
-    where
-        F: Fn(Self::Item, &mut Vec<T>),
-        Self: Sized,
-    {
-        FlatMapWithBuffer {
-            buffer: Vec::with_capacity(10),
-            fill_buffer,
-            underlying_it: self,
-        }
-    }
-}
-
-impl<T: ?Sized> FlatMapWithBufferIter for T where T: Iterator {}
-
-#[cfg(test)]
-mod tests {
-    use crate::indexer::flat_map_with_buffer::FlatMapWithBufferIter;
-
-    #[test]
-    fn test_flat_map_with_buffer_empty() {
-        let mut empty_iter = std::iter::empty::<usize>()
-            .flat_map_with_buffer(|_val: usize, _buffer: &mut Vec<usize>| {});
-        assert!(empty_iter.next().is_none());
-    }
-
-    #[test]
-    fn test_flat_map_with_buffer_simple() {
-        let vals: Vec<usize> = (1..5)
-            .flat_map_with_buffer(|val: usize, buffer: &mut Vec<usize>| buffer.extend(0..val))
-            .collect();
-        assert_eq!(&[0, 0, 1, 0, 1, 2, 0, 1, 2, 3], &vals[..]);
-    }
-
-    #[test]
-    fn test_flat_map_filling_no_elements_does_not_stop_iterator() {
-        let vals: Vec<usize> = [2, 0, 0, 3]
-            .into_iter()
-            .flat_map_with_buffer(|val: usize, buffer: &mut Vec<usize>| buffer.extend(0..val))
-            .collect();
-        assert_eq!(&[0, 1, 0, 1, 2], &vals[..]);
-    }
-}

src/indexer/index_writer.rs

@@ -1395,35 +1395,6 @@ mod tests {
         assert!(commit_again.is_ok());
     }
 
-    #[test]
-    fn test_sort_by_multivalue_field_error() -> crate::Result<()> {
-        let mut schema_builder = schema::Schema::builder();
-        let options = NumericOptions::default().set_fast(Cardinality::MultiValues);
-        schema_builder.add_u64_field("id", options);
-        let schema = schema_builder.build();
-
-        let settings = IndexSettings {
-            sort_by_field: Some(IndexSortByField {
-                field: "id".to_string(),
-                order: Order::Desc,
-            }),
-            ..Default::default()
-        };
-
-        let err = Index::builder()
-            .schema(schema)
-            .settings(settings)
-            .create_in_ram()
-            .unwrap_err();
-        assert_eq!(
-            err.to_string(),
-            "An invalid argument was passed: 'Only single value fast field Cardinality supported \
-             for sorting index id'"
-        );
-
-        Ok(())
-    }
-
     #[test]
     fn test_delete_with_sort_by_field() -> crate::Result<()> {
         let mut schema_builder = schema::Schema::builder();

src/indexer/merger.rs

@@ -6,18 +6,16 @@ use fastfield_codecs::VecColumn;
 use itertools::Itertools;
 use measure_time::debug_time;
 
-use super::sorted_doc_id_multivalue_column::RemappedDocIdMultiValueIndexColumn;
 use crate::core::{Segment, SegmentReader};
 use crate::docset::{DocSet, TERMINATED};
 use crate::error::DataCorruption;
 use crate::fastfield::{
-    get_fastfield_codecs_for_multivalue, AliveBitSet, Column, CompositeFastFieldSerializer,
-    MultiValueLength, MultiValuedFastFieldReader,
+    AliveBitSet, Column, CompositeFastFieldSerializer, MultiValueLength, MultiValuedFastFieldReader,
 };
 use crate::fieldnorm::{FieldNormReader, FieldNormReaders, FieldNormsSerializer, FieldNormsWriter};
 use crate::indexer::doc_id_mapping::{expect_field_id_for_sort_field, SegmentDocIdMapping};
-use crate::indexer::sorted_doc_id_column::RemappedDocIdColumn;
-use crate::indexer::sorted_doc_id_multivalue_column::RemappedDocIdMultiValueColumn;
+use crate::indexer::sorted_doc_id_column::SortedDocIdColumn;
+use crate::indexer::sorted_doc_id_multivalue_column::SortedDocIdMultiValueColumn;
 use crate::indexer::SegmentSerializer;
 use crate::postings::{InvertedIndexSerializer, Postings, SegmentPostings};
 use crate::schema::{Cardinality, Field, FieldType, Schema};
@@ -312,7 +310,7 @@ impl IndexMerger {
         fast_field_serializer: &mut CompositeFastFieldSerializer,
         doc_id_mapping: &SegmentDocIdMapping,
     ) -> crate::Result<()> {
-        let fast_field_accessor = RemappedDocIdColumn::new(&self.readers, doc_id_mapping, field);
+        let fast_field_accessor = SortedDocIdColumn::new(&self.readers, doc_id_mapping, field);
         fast_field_serializer.create_auto_detect_u64_fast_field(field, fast_field_accessor)?;
 
         Ok(())
@@ -425,17 +423,33 @@ impl IndexMerger {
     // Creating the index file to point into the data, generic over `BytesFastFieldReader` and
     // `MultiValuedFastFieldReader`
     //
-    fn write_1_n_fast_field_idx_generic<T: MultiValueLength + Send + Sync>(
+    fn write_1_n_fast_field_idx_generic<T: MultiValueLength>(
         field: Field,
         fast_field_serializer: &mut CompositeFastFieldSerializer,
         doc_id_mapping: &SegmentDocIdMapping,
-        segment_and_ff_readers: &[(&SegmentReader, T)],
-    ) -> crate::Result<()> {
-        let column =
-            RemappedDocIdMultiValueIndexColumn::new(segment_and_ff_readers, doc_id_mapping);
+        reader_and_field_accessors: &[(&SegmentReader, T)],
+    ) -> crate::Result<Vec<u64>> {
+        // We can now create our `idx` serializer, and in a second pass,
+        // can effectively push the different indexes.
 
-        fast_field_serializer.create_auto_detect_u64_fast_field(field, column)?;
-        Ok(())
+        // copying into a temp vec is not ideal, but the fast field codec api requires random
+        // access, which is used in the estimation. It's possible to 1. calculate random
+        // access on the fly or 2. change the codec api to make random access optional, but
+        // they both have also major drawbacks.
+
+        let mut offsets = Vec::with_capacity(doc_id_mapping.len());
+        let mut offset = 0;
+        for old_doc_addr in doc_id_mapping.iter_old_doc_addrs() {
+            let reader = &reader_and_field_accessors[old_doc_addr.segment_ord as usize].1;
+            offsets.push(offset);
+            offset += reader.get_len(old_doc_addr.doc_id) as u64;
+        }
+        offsets.push(offset);
+
+        let fastfield_accessor = VecColumn::from(&offsets[..]);
+
+        fast_field_serializer.create_auto_detect_u64_fast_field(field, fastfield_accessor)?;
+        Ok(offsets)
     }
     /// Returns the fastfield index (index for the data, not the data).
     fn write_multi_value_fast_field_idx(
@@ -443,8 +457,8 @@ impl IndexMerger {
         field: Field,
         fast_field_serializer: &mut CompositeFastFieldSerializer,
         doc_id_mapping: &SegmentDocIdMapping,
-    ) -> crate::Result<()> {
-        let segment_and_ff_readers = self
+    ) -> crate::Result<Vec<u64>> {
+        let reader_ordinal_and_field_accessors = self
             .readers
             .iter()
             .map(|reader| {
@@ -463,7 +477,7 @@ impl IndexMerger {
             field,
             fast_field_serializer,
             doc_id_mapping,
-            &segment_and_ff_readers,
+            &reader_ordinal_and_field_accessors,
         )
     }
 
@@ -512,12 +526,7 @@ impl IndexMerger {
             }
 
             let col = VecColumn::from(&vals[..]);
-            fast_field_serializer.create_auto_detect_u64_fast_field_with_idx_and_codecs(
-                field,
-                col,
-                1,
-                &get_fastfield_codecs_for_multivalue(),
-            )?;
+            fast_field_serializer.create_auto_detect_u64_fast_field_with_idx(field, col, 1)?;
         }
         Ok(())
     }
@@ -552,21 +561,20 @@ impl IndexMerger {
         fast_field_serializer: &mut CompositeFastFieldSerializer,
         doc_id_mapping: &SegmentDocIdMapping,
     ) -> crate::Result<()> {
-        // Multifastfield consists of 2 fastfields.
+        // Multifastfield consists in 2 fastfields.
         // The first serves as an index into the second one and is strictly increasing.
         // The second contains the actual values.
 
         // First we merge the idx fast field.
-
-        self.write_multi_value_fast_field_idx(field, fast_field_serializer, doc_id_mapping)?;
+        let offsets =
+            self.write_multi_value_fast_field_idx(field, fast_field_serializer, doc_id_mapping)?;
 
         let fastfield_accessor =
-            RemappedDocIdMultiValueColumn::new(&self.readers, doc_id_mapping, field);
-        fast_field_serializer.create_auto_detect_u64_fast_field_with_idx_and_codecs(
+            SortedDocIdMultiValueColumn::new(&self.readers, doc_id_mapping, &offsets, field);
+        fast_field_serializer.create_auto_detect_u64_fast_field_with_idx(
             field,
             fastfield_accessor,
             1,
-            &get_fastfield_codecs_for_multivalue(),
         )?;
 
         Ok(())
@@ -578,7 +586,7 @@ impl IndexMerger {
         fast_field_serializer: &mut CompositeFastFieldSerializer,
         doc_id_mapping: &SegmentDocIdMapping,
     ) -> crate::Result<()> {
-        let segment_and_ff_readers = self
+        let reader_and_field_accessors = self
             .readers
             .iter()
             .map(|reader| {
@@ -589,17 +597,17 @@ impl IndexMerger {
                 (reader, bytes_reader)
             })
             .collect::<Vec<_>>();
+
         Self::write_1_n_fast_field_idx_generic(
             field,
             fast_field_serializer,
             doc_id_mapping,
-            &segment_and_ff_readers,
+            &reader_and_field_accessors,
         )?;
-
         let mut serialize_vals = fast_field_serializer.new_bytes_fast_field(field);
 
         for old_doc_addr in doc_id_mapping.iter_old_doc_addrs() {
-            let bytes_reader = &segment_and_ff_readers[old_doc_addr.segment_ord as usize].1;
+            let bytes_reader = &reader_and_field_accessors[old_doc_addr.segment_ord as usize].1;
             let val = bytes_reader.get_bytes(old_doc_addr.doc_id);
             serialize_vals.write_all(val)?;
         }

src/indexer/mod.rs

@@ -3,7 +3,6 @@ pub mod delete_queue;
 pub mod demuxer;
 pub mod doc_id_mapping;
 mod doc_opstamp_mapping;
-mod flat_map_with_buffer;
 pub mod index_writer;
 mod index_writer_status;
 mod json_term_writer;

src/indexer/segment_serializer.rs

@@ -24,25 +24,12 @@ impl SegmentSerializer {
         // In the merge case this is not necessary because we can kmerge the already sorted
         // segments
         let remapping_required = segment.index().settings().sort_by_field.is_some() && !is_in_merge;
-        let settings = segment.index().settings().clone();
-        let store_writer = if remapping_required {
-            let store_write = segment.open_write(SegmentComponent::TempStore)?;
-            StoreWriter::new(
-                store_write,
-                crate::store::Compressor::None,
-                0, // we want random access on the docs, so we choose a minimal block size. Every
-                // doc will get its own block.
-                settings.docstore_compress_dedicated_thread,
-            )?
+        let store_component = if remapping_required {
+            SegmentComponent::TempStore
         } else {
-            let store_write = segment.open_write(SegmentComponent::Store)?;
-            StoreWriter::new(
-                store_write,
-                settings.docstore_compression,
-                settings.docstore_blocksize,
-                settings.docstore_compress_dedicated_thread,
-            )?
+            SegmentComponent::Store
         };
+        let store_write = segment.open_write(store_component)?;
 
         let fast_field_write = segment.open_write(SegmentComponent::FastFields)?;
         let fast_field_serializer = CompositeFastFieldSerializer::from_write(fast_field_write)?;
@@ -51,6 +38,13 @@ impl SegmentSerializer {
         let fieldnorms_serializer = FieldNormsSerializer::from_write(fieldnorms_write)?;
 
         let postings_serializer = InvertedIndexSerializer::open(&mut segment)?;
+        let settings = segment.index().settings();
+        let store_writer = StoreWriter::new(
+            store_write,
+            settings.docstore_compression,
+            settings.docstore_blocksize,
+            settings.docstore_compress_dedicated_thread,
+        )?;
         Ok(SegmentSerializer {
             segment,
             store_writer,

src/indexer/segment_updater.rs

@@ -133,15 +133,15 @@ fn merge(
 
 /// Advanced: Merges a list of segments from different indices in a new index.
 ///
-/// Returns `TantivyError` if the indices list is empty or their
+/// Returns `TantivyError` if the the indices list is empty or their
 /// schemas don't match.
 ///
 /// `output_directory`: is assumed to be empty.
 ///
 /// # Warning
 /// This function does NOT check or take the `IndexWriter` is running. It is not
-/// meant to work if you have an `IndexWriter` running for the origin indices, or
-/// the destination `Index`.
+/// meant to work if you have an IndexWriter running for the origin indices, or
+/// the destination Index.
 #[doc(hidden)]
 pub fn merge_indices<T: Into<Box<dyn Directory>>>(
     indices: &[Index],
@@ -179,15 +179,15 @@ pub fn merge_indices<T: Into<Box<dyn Directory>>>(
 /// Advanced: Merges a list of segments from different indices in a new index.
 /// Additional you can provide a delete bitset for each segment to ignore doc_ids.
 ///
-/// Returns `TantivyError` if the indices list is empty or their
+/// Returns `TantivyError` if the the indices list is empty or their
 /// schemas don't match.
 ///
 /// `output_directory`: is assumed to be empty.
 ///
 /// # Warning
 /// This function does NOT check or take the `IndexWriter` is running. It is not
-/// meant to work if you have an `IndexWriter` running for the origin indices, or
-/// the destination `Index`.
+/// meant to work if you have an IndexWriter running for the origin indices, or
+/// the destination Index.
 #[doc(hidden)]
 pub fn merge_filtered_segments<T: Into<Box<dyn Directory>>>(
     segments: &[Segment],

src/indexer/segment_writer.rs

@@ -1,5 +1,4 @@
 use fastfield_codecs::MonotonicallyMappableToU64;
-use itertools::Itertools;
 
 use super::doc_id_mapping::{get_doc_id_mapping_from_field, DocIdMapping};
 use super::operation::AddOperation;
@@ -158,13 +157,7 @@ impl SegmentWriter {
 
     fn index_document(&mut self, doc: &Document) -> crate::Result<()> {
         let doc_id = self.max_doc;
-        let vals_grouped_by_field = doc
-            .field_values()
-            .iter()
-            .sorted_by_key(|el| el.field())
-            .group_by(|el| el.field());
-        for (field, field_values) in &vals_grouped_by_field {
-            let values = field_values.map(|field_value| field_value.value());
+        for (field, values) in doc.get_sorted_field_values() {
             let field_entry = self.schema.get_field_entry(field);
             let make_schema_error = || {
                 crate::TantivyError::SchemaError(format!(
@@ -205,16 +198,24 @@ impl SegmentWriter {
                 }
                 FieldType::Str(_) => {
                     let mut token_streams: Vec<BoxTokenStream> = vec![];
+                    let mut offsets = vec![];
+                    let mut total_offset = 0;
 
                     for value in values {
                         match value {
                             Value::PreTokStr(tok_str) => {
+                                offsets.push(total_offset);
+                                if let Some(last_token) = tok_str.tokens.last() {
+                                    total_offset += last_token.offset_to;
+                                }
                                 token_streams
                                     .push(PreTokenizedStream::from(tok_str.clone()).into());
                             }
                             Value::Str(ref text) => {
                                 let text_analyzer =
                                     &self.per_field_text_analyzers[field.field_id() as usize];
+                                offsets.push(total_offset);
+                                total_offset += text.len();
                                 token_streams.push(text_analyzer.token_stream(text));
                             }
                             _ => (),
@@ -283,8 +284,9 @@ impl SegmentWriter {
                 }
                 FieldType::JsonObject(_) => {
                     let text_analyzer = &self.per_field_text_analyzers[field.field_id() as usize];
-                    let json_values_it =
-                        values.map(|value| value.as_json().ok_or_else(make_schema_error));
+                    let json_values_it = values
+                        .iter()
+                        .map(|value| value.as_json().ok_or_else(make_schema_error));
                     index_json_values(
                         doc_id,
                         json_values_it,
@@ -372,9 +374,9 @@ fn remap_and_write(
         doc_id_map,
     )?;
 
+    debug!("resort-docstore");
     // finalize temp docstore and create version, which reflects the doc_id_map
     if let Some(doc_id_map) = doc_id_map {
-        debug!("resort-docstore");
         let store_write = serializer
             .segment_mut()
             .open_write(SegmentComponent::Store)?;
@@ -391,8 +393,7 @@ fn remap_and_write(
             serializer
                 .segment()
                 .open_read(SegmentComponent::TempStore)?,
-            1, /* The docstore is configured to have one doc per block, and each doc is accessed
-                * only once: we don't need caching. */
+            50,
         )?;
         for old_doc_id in doc_id_map.iter_old_doc_ids() {
             let doc_bytes = store_read.get_document_bytes(old_doc_id)?;

src/indexer/sorted_doc_id_column.rs

@@ -5,9 +5,9 @@ use itertools::Itertools;
 
 use crate::indexer::doc_id_mapping::SegmentDocIdMapping;
 use crate::schema::Field;
-use crate::SegmentReader;
+use crate::{DocAddress, SegmentReader};
 
-pub(crate) struct RemappedDocIdColumn<'a> {
+pub(crate) struct SortedDocIdColumn<'a> {
     doc_id_mapping: &'a SegmentDocIdMapping,
     fast_field_readers: Vec<Arc<dyn Column<u64>>>,
     min_value: u64,
@@ -37,7 +37,7 @@ fn compute_min_max_val(
         .into_option()
 }
 
-impl<'a> RemappedDocIdColumn<'a> {
+impl<'a> SortedDocIdColumn<'a> {
     pub(crate) fn new(
         readers: &'a [SegmentReader],
         doc_id_mapping: &'a SegmentDocIdMapping,
@@ -68,7 +68,7 @@ impl<'a> RemappedDocIdColumn<'a> {
             })
             .collect::<Vec<_>>();
 
-        RemappedDocIdColumn {
+        SortedDocIdColumn {
             doc_id_mapping,
             fast_field_readers,
             min_value,
@@ -78,9 +78,13 @@ impl<'a> RemappedDocIdColumn<'a> {
     }
 }
 
-impl<'a> Column for RemappedDocIdColumn<'a> {
-    fn get_val(&self, _doc: u64) -> u64 {
-        unimplemented!()
+impl<'a> Column for SortedDocIdColumn<'a> {
+    fn get_val(&self, doc: u64) -> u64 {
+        let DocAddress {
+            doc_id,
+            segment_ord,
+        } = self.doc_id_mapping.get_old_doc_addr(doc as u32);
+        self.fast_field_readers[segment_ord as usize].get_val(doc_id as u64)
     }
 
     fn iter(&self) -> Box<dyn Iterator<Item = u64> + '_> {

src/indexer/sorted_doc_id_multivalue_column.rs

@@ -2,24 +2,26 @@ use std::cmp;
 
 use fastfield_codecs::Column;
 
-use super::flat_map_with_buffer::FlatMapWithBufferIter;
 use crate::fastfield::{MultiValueLength, MultiValuedFastFieldReader};
 use crate::indexer::doc_id_mapping::SegmentDocIdMapping;
 use crate::schema::Field;
-use crate::{DocAddress, SegmentReader};
+use crate::{DocId, SegmentReader};
 
-pub(crate) struct RemappedDocIdMultiValueColumn<'a> {
+// We can now initialize our serializer, and push it the different values
+pub(crate) struct SortedDocIdMultiValueColumn<'a> {
     doc_id_mapping: &'a SegmentDocIdMapping,
     fast_field_readers: Vec<MultiValuedFastFieldReader<u64>>,
+    offsets: &'a [u64],
     min_value: u64,
     max_value: u64,
     num_vals: u64,
 }
 
-impl<'a> RemappedDocIdMultiValueColumn<'a> {
+impl<'a> SortedDocIdMultiValueColumn<'a> {
     pub(crate) fn new(
         readers: &'a [SegmentReader],
         doc_id_mapping: &'a SegmentDocIdMapping,
+        offsets: &'a [u64],
         field: Field,
     ) -> Self {
         // Our values are bitpacked and we need to know what should be
@@ -56,9 +58,10 @@ impl<'a> RemappedDocIdMultiValueColumn<'a> {
             min_value = 0;
             max_value = 0;
         }
-        RemappedDocIdMultiValueColumn {
+        SortedDocIdMultiValueColumn {
             doc_id_mapping,
             fast_field_readers,
+            offsets,
             min_value,
             max_value,
             num_vals: num_vals as u64,
@@ -66,18 +69,37 @@ impl<'a> RemappedDocIdMultiValueColumn<'a> {
     }
 }
 
-impl<'a> Column for RemappedDocIdMultiValueColumn<'a> {
-    fn get_val(&self, _pos: u64) -> u64 {
-        unimplemented!()
+impl<'a> Column for SortedDocIdMultiValueColumn<'a> {
+    fn get_val(&self, pos: u64) -> u64 {
+        // use the offsets index to find the doc_id which will contain the position.
+        // the offsets are strictly increasing so we can do a binary search on it.
+
+        let new_doc_id: DocId = self.offsets.partition_point(|&offset| offset <= pos) as DocId - 1; // Offsets start at 0, so -1 is safe
+
+        // now we need to find the position of `pos` in the multivalued bucket
+        let num_pos_covered_until_now = self.offsets[new_doc_id as usize];
+        let pos_in_values = pos - num_pos_covered_until_now;
+
+        let old_doc_addr = self.doc_id_mapping.get_old_doc_addr(new_doc_id);
+        let num_vals =
+            self.fast_field_readers[old_doc_addr.segment_ord as usize].get_len(old_doc_addr.doc_id);
+        assert!(num_vals >= pos_in_values);
+        let mut vals = Vec::new();
+        self.fast_field_readers[old_doc_addr.segment_ord as usize]
+            .get_vals(old_doc_addr.doc_id, &mut vals);
+
+        vals[pos_in_values as usize]
     }
 
     fn iter(&self) -> Box<dyn Iterator<Item = u64> + '_> {
         Box::new(
             self.doc_id_mapping
                 .iter_old_doc_addrs()
-                .flat_map_with_buffer(|old_doc_addr: DocAddress, buffer| {
+                .flat_map(|old_doc_addr| {
                     let ff_reader = &self.fast_field_readers[old_doc_addr.segment_ord as usize];
-                    ff_reader.get_vals(old_doc_addr.doc_id, buffer);
+                    let mut vals = Vec::new();
+                    ff_reader.get_vals(old_doc_addr.doc_id, &mut vals);
+                    vals.into_iter()
                 }),
         )
     }
@@ -93,76 +115,3 @@ impl<'a> Column for RemappedDocIdMultiValueColumn<'a> {
         self.num_vals
     }
 }
-
-pub(crate) struct RemappedDocIdMultiValueIndexColumn<'a, T: MultiValueLength> {
-    doc_id_mapping: &'a SegmentDocIdMapping,
-    multi_value_length_readers: Vec<&'a T>,
-    min_value: u64,
-    max_value: u64,
-    num_vals: u64,
-}
-
-impl<'a, T: MultiValueLength> RemappedDocIdMultiValueIndexColumn<'a, T> {
-    pub(crate) fn new(
-        segment_and_ff_readers: &'a [(&'a SegmentReader, T)],
-        doc_id_mapping: &'a SegmentDocIdMapping,
-    ) -> Self {
-        // We go through a complete first pass to compute the minimum and the
-        // maximum value and initialize our Column.
-        let mut num_vals = 0;
-        let min_value = 0;
-        let mut max_value = 0;
-        let mut multi_value_length_readers = Vec::with_capacity(segment_and_ff_readers.len());
-        for segment_and_ff_reader in segment_and_ff_readers {
-            let segment_reader = segment_and_ff_reader.0;
-            let multi_value_length_reader = &segment_and_ff_reader.1;
-            if !segment_reader.has_deletes() {
-                max_value += multi_value_length_reader.get_total_len();
-            } else {
-                for doc in segment_reader.doc_ids_alive() {
-                    max_value += multi_value_length_reader.get_len(doc);
-                }
-            }
-            num_vals += segment_reader.num_docs() as u64;
-            multi_value_length_readers.push(multi_value_length_reader);
-        }
-        Self {
-            doc_id_mapping,
-            multi_value_length_readers,
-            min_value,
-            max_value,
-            num_vals,
-        }
-    }
-}
-
-impl<'a, T: MultiValueLength + Send + Sync> Column for RemappedDocIdMultiValueIndexColumn<'a, T> {
-    fn get_val(&self, _pos: u64) -> u64 {
-        unimplemented!()
-    }
-
-    fn iter(&self) -> Box<dyn Iterator<Item = u64> + '_> {
-        let mut offset = 0;
-        Box::new(
-            std::iter::once(0).chain(self.doc_id_mapping.iter_old_doc_addrs().map(
-                move |old_doc_addr| {
-                    let ff_reader =
-                        &self.multi_value_length_readers[old_doc_addr.segment_ord as usize];
-                    offset += ff_reader.get_len(old_doc_addr.doc_id);
-                    offset
-                },
-            )),
-        )
-    }
-    fn min_value(&self) -> u64 {
-        self.min_value
-    }
-
-    fn max_value(&self) -> u64 {
-        self.max_value
-    }
-
-    fn num_vals(&self) -> u64 {
-        self.num_vals
-    }
-}

src/postings/compression/mod.rs

@@ -106,7 +106,7 @@ impl BlockDecoder {
 
 pub trait VIntEncoder {
     /// Compresses an array of `u32` integers,
-    /// using [delta-encoding](https://en.wikipedia.org/wiki/Delta_encoding)
+    /// using [delta-encoding](https://en.wikipedia.org/wiki/Delta_ encoding)
     /// and variable bytes encoding.
     ///
     /// The method takes an array of ints to compress, and returns

src/postings/recorder.rs

@@ -47,11 +47,11 @@ impl<'a> Iterator for VInt32Reader<'a> {
     }
 }
 
-/// `Recorder` is in charge of recording relevant information about
+/// Recorder is in charge of recording relevant information about
 /// the presence of a term in a document.
 ///
-/// Depending on the [`TextOptions`](crate::schema::TextOptions) associated
-/// with the field, the recorder may record:
+/// Depending on the `TextIndexingOptions` associated with the
+/// field, the recorder may records
 ///   * the document frequency
 ///   * the document id
 ///   * the term frequency

src/postings/stacker/term_hashmap.rs

@@ -199,7 +199,7 @@ impl TermHashMap {
     /// `update` create a new entry for a given key if it does not exists
     /// or updates the existing entry.
     ///
-    /// The actual logic for this update is define in the `updater`
+    /// The actual logic for this update is define in the the `updater`
     /// argument.
     ///
     /// If the key is not present, `updater` will receive `None` and

src/query/boolean_query/block_wand.rs

@@ -144,7 +144,7 @@ fn advance_all_scorers_on_pivot(term_scorers: &mut Vec<TermScorerWithMaxScore>,
 
 /// Implements the WAND (Weak AND) algorithm for dynamic pruning
 /// described in the paper "Faster Top-k Document Retrieval Using Block-Max Indexes".
-/// Link: <http://engineering.nyu.edu/~suel/papers/bmw.pdf>
+/// Link: http://engineering.nyu.edu/~suel/papers/bmw.pdf
 pub fn block_wand(
     mut scorers: Vec<TermScorer>,
     mut threshold: Score,

src/query/boolean_query/boolean_weight.rs

@@ -174,9 +174,9 @@ impl<TScoreCombiner: ScoreCombiner + Sync> Weight for BooleanWeight<TScoreCombin
                     into_box_scorer(specialized_scorer, &self.score_combiner_fn)
                 })
         } else {
-            self.complex_scorer(reader, boost, DoNothingCombiner::default)
+            self.complex_scorer(reader, boost, &DoNothingCombiner::default)
                 .map(|specialized_scorer| {
-                    into_box_scorer(specialized_scorer, DoNothingCombiner::default)
+                    into_box_scorer(specialized_scorer, &DoNothingCombiner::default)
                 })
         }
     }

src/query/mod.rs

@@ -21,7 +21,6 @@ mod range_query;
 mod regex_query;
 mod reqopt_scorer;
 mod scorer;
-mod set_query;
 mod term_query;
 mod union;
 mod weight;
@@ -59,7 +58,6 @@ pub use self::score_combiner::{
     DisjunctionMaxCombiner, ScoreCombiner, SumCombiner, SumWithCoordsCombiner,
 };
 pub use self::scorer::Scorer;
-pub use self::set_query::TermSetQuery;
 pub use self::term_query::TermQuery;
 pub use self::union::Union;
 #[cfg(test)]

src/query/more_like_this/more_like_this.rs

@@ -33,9 +33,9 @@ impl Ord for ScoreTerm {
     }
 }
 
-/// A struct used as helper to build [`MoreLikeThisQuery`](crate::query::MoreLikeThisQuery)
-/// This more-like-this implementation is inspired by the Apache Lucene
-/// and closely follows the same implementation with adaptation to Tantivy vocabulary and API.
+/// A struct used as helper to build [`MoreLikeThisQuery`]
+/// This more-like-this implementation is inspired by the Appache Lucene
+/// amd closely follows the same implementation with adaptabtion to Tantivy vocabulary and API.
 ///
 /// [MoreLikeThis](https://github.com/apache/lucene/blob/main/lucene/queries/src/java/org/apache/lucene/queries/mlt/MoreLikeThis.java#L147)
 /// [MoreLikeThisQuery](https://github.com/apache/lucene/blob/main/lucene/queries/src/java/org/apache/lucene/queries/mlt/MoreLikeThisQuery.java#L36)

src/query/set_query.rs

@@ -1,222 +0,0 @@
-use std::collections::HashMap;
-
-use tantivy_fst::raw::CompiledAddr;
-use tantivy_fst::{Automaton, Map};
-
-use crate::query::score_combiner::DoNothingCombiner;
-use crate::query::{AutomatonWeight, BooleanWeight, Occur, Query, Weight};
-use crate::schema::Field;
-use crate::{Searcher, Term};
-
-/// A Term Set Query matches all of the documents containing any of the Term provided
-#[derive(Debug, Clone)]
-pub struct TermSetQuery {
-    terms_map: HashMap<Field, Vec<Term>>,
-}
-
-impl TermSetQuery {
-    /// Create a Term Set Query
-    pub fn new<T: IntoIterator<Item = Term>>(terms: T) -> Self {
-        let mut terms_map: HashMap<_, Vec<_>> = HashMap::new();
-        for term in terms {
-            terms_map.entry(term.field()).or_default().push(term);
-        }
-
-        for terms in terms_map.values_mut() {
-            terms.sort_unstable();
-            terms.dedup();
-        }
-
-        TermSetQuery { terms_map }
-    }
-
-    fn specialized_weight(
-        &self,
-        searcher: &Searcher,
-    ) -> crate::Result<BooleanWeight<DoNothingCombiner>> {
-        let mut sub_queries: Vec<(_, Box<dyn Weight>)> = Vec::with_capacity(self.terms_map.len());
-
-        for (&field, sorted_terms) in self.terms_map.iter() {
-            let field_entry = searcher.schema().get_field_entry(field);
-            let field_type = field_entry.field_type();
-            if !field_type.is_indexed() {
-                let error_msg = format!("Field {:?} is not indexed.", field_entry.name());
-                return Err(crate::TantivyError::SchemaError(error_msg));
-            }
-
-            // In practice this won't fail because:
-            // - we are writing to memory, so no IoError
-            // - Terms are ordered
-            let map = Map::from_iter(sorted_terms.iter().map(|key| (key.value_bytes(), 0)))
-                .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))?;
-
-            sub_queries.push((
-                Occur::Should,
-                Box::new(AutomatonWeight::new(field, SetDfaWrapper(map))),
-            ));
-        }
-
-        Ok(BooleanWeight::new(
-            sub_queries,
-            false,
-            Box::new(|| DoNothingCombiner),
-        ))
-    }
-}
-
-impl Query for TermSetQuery {
-    fn weight(
-        &self,
-        searcher: &Searcher,
-        _scoring_enabled: bool,
-    ) -> crate::Result<Box<dyn Weight>> {
-        Ok(Box::new(self.specialized_weight(searcher)?))
-    }
-}
-
-struct SetDfaWrapper(Map<Vec<u8>>);
-
-impl Automaton for SetDfaWrapper {
-    type State = Option<CompiledAddr>;
-
-    fn start(&self) -> Option<CompiledAddr> {
-        Some(self.0.as_ref().root().addr())
-    }
-
-    fn is_match(&self, state_opt: &Option<CompiledAddr>) -> bool {
-        if let Some(state) = state_opt {
-            self.0.as_ref().node(*state).is_final()
-        } else {
-            false
-        }
-    }
-
-    fn accept(&self, state_opt: &Option<CompiledAddr>, byte: u8) -> Option<CompiledAddr> {
-        let state = state_opt.as_ref()?;
-        let node = self.0.as_ref().node(*state);
-        let transition = node.find_input(byte)?;
-        Some(node.transition_addr(transition))
-    }
-
-    fn can_match(&self, state: &Self::State) -> bool {
-        state.is_some()
-    }
-}
-
-#[cfg(test)]
-mod tests {
-
-    use crate::collector::TopDocs;
-    use crate::query::TermSetQuery;
-    use crate::schema::{Schema, TEXT};
-    use crate::{assert_nearly_equals, Index, Term};
-
-    #[test]
-    pub fn test_term_set_query() -> crate::Result<()> {
-        let mut schema_builder = Schema::builder();
-        let field1 = schema_builder.add_text_field("field1", TEXT);
-        let field2 = schema_builder.add_text_field("field1", TEXT);
-        let schema = schema_builder.build();
-        let index = Index::create_in_ram(schema);
-        {
-            let mut index_writer = index.writer_for_tests()?;
-            index_writer.add_document(doc!(
-                field1 => "doc1",
-                field2 => "val1",
-            ))?;
-            index_writer.add_document(doc!(
-                field1 => "doc2",
-                field2 => "val2",
-            ))?;
-            index_writer.add_document(doc!(
-                field1 => "doc3",
-                field2 => "val3",
-            ))?;
-            index_writer.add_document(doc!(
-                field1 => "val3",
-                field2 => "doc3",
-            ))?;
-            index_writer.commit()?;
-        }
-        let reader = index.reader()?;
-        let searcher = reader.searcher();
-
-        {
-            // single element
-            let terms = vec![Term::from_field_text(field1, "doc1")];
-
-            let term_set_query = TermSetQuery::new(terms);
-            let top_docs = searcher.search(&term_set_query, &TopDocs::with_limit(2))?;
-            assert_eq!(top_docs.len(), 1, "Expected 1 document");
-            let (score, _) = top_docs[0];
-            assert_nearly_equals!(1.0, score);
-        }
-
-        {
-            // single element, absent
-            let terms = vec![Term::from_field_text(field1, "doc4")];
-
-            let term_set_query = TermSetQuery::new(terms);
-            let top_docs = searcher.search(&term_set_query, &TopDocs::with_limit(1))?;
-            assert!(top_docs.is_empty(), "Expected 0 document");
-        }
-
-        {
-            // multiple elements
-            let terms = vec![
-                Term::from_field_text(field1, "doc1"),
-                Term::from_field_text(field1, "doc2"),
-            ];
-
-            let term_set_query = TermSetQuery::new(terms);
-            let top_docs = searcher.search(&term_set_query, &TopDocs::with_limit(2))?;
-            assert_eq!(top_docs.len(), 2, "Expected 2 documents");
-            for (score, _) in top_docs {
-                assert_nearly_equals!(1.0, score);
-            }
-        }
-
-        {
-            // multiple elements, mixed fields
-            let terms = vec![
-                Term::from_field_text(field1, "doc1"),
-                Term::from_field_text(field1, "doc1"),
-                Term::from_field_text(field2, "val2"),
-            ];
-
-            let term_set_query = TermSetQuery::new(terms);
-            let top_docs = searcher.search(&term_set_query, &TopDocs::with_limit(3))?;
-
-            assert_eq!(top_docs.len(), 2, "Expected 2 document");
-            for (score, _) in top_docs {
-                assert_nearly_equals!(1.0, score);
-            }
-        }
-
-        {
-            // no field crosstalk
-            let terms = vec![Term::from_field_text(field1, "doc3")];
-
-            let term_set_query = TermSetQuery::new(terms);
-            let top_docs = searcher.search(&term_set_query, &TopDocs::with_limit(3))?;
-            assert_eq!(top_docs.len(), 1, "Expected 1 document");
-
-            let terms = vec![Term::from_field_text(field2, "doc3")];
-
-            let term_set_query = TermSetQuery::new(terms);
-            let top_docs = searcher.search(&term_set_query, &TopDocs::with_limit(3))?;
-            assert_eq!(top_docs.len(), 1, "Expected 1 document");
-
-            let terms = vec![
-                Term::from_field_text(field1, "doc3"),
-                Term::from_field_text(field2, "doc3"),
-            ];
-
-            let term_set_query = TermSetQuery::new(terms);
-            let top_docs = searcher.search(&term_set_query, &TopDocs::with_limit(3))?;
-            assert_eq!(top_docs.len(), 2, "Expected 2 document");
-        }
-
-        Ok(())
-    }
-}

src/reader/mod.rs

@@ -263,7 +263,8 @@ impl InnerIndexReader {
 /// It controls when a new version of the index should be loaded and lends
 /// you instances of `Searcher` for the last loaded version.
 ///
-/// `IndexReader` just wraps an `Arc`.
+/// `Clone` does not clone the different pool of searcher. `IndexReader`
+/// just wraps an `Arc`.
 #[derive(Clone)]
 pub struct IndexReader {
     inner: Arc<InnerIndexReader>,
@@ -293,6 +294,9 @@ impl IndexReader {
     ///
     /// This method should be called every single time a search
     /// query is performed.
+    /// The searchers are taken from a pool of `num_searchers` searchers.
+    /// If no searcher is available
+    /// this may block.
     ///
     /// The same searcher must be used for a given query, as it ensures
     /// the use of a consistent segment set.

src/schema/bytes_options.rs

@@ -3,7 +3,7 @@ use std::ops::BitOr;
 use serde::{Deserialize, Serialize};
 
 use super::flags::{FastFlag, IndexedFlag, SchemaFlagList, StoredFlag};
-/// Define how a bytes field should be handled by tantivy.
+/// Define how an a bytes field should be handled by tantivy.
 #[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
 #[serde(from = "BytesOptionsDeser")]
 pub struct BytesOptions {

src/schema/field_type.rs

@@ -2,7 +2,6 @@ use serde::{Deserialize, Serialize};
 use serde_json::Value as JsonValue;
 use thiserror::Error;
 
-use super::Cardinality;
 use crate::schema::bytes_options::BytesOptions;
 use crate::schema::facet_options::FacetOptions;
 use crate::schema::{
@@ -215,26 +214,6 @@ impl FieldType {
         }
     }
 
-    /// returns true if the field is fast.
-    pub fn fastfield_cardinality(&self) -> Option<Cardinality> {
-        match *self {
-            FieldType::Bytes(ref bytes_options) if bytes_options.is_fast() => {
-                Some(Cardinality::SingleValue)
-            }
-            FieldType::Str(ref text_options) if text_options.is_fast() => {
-                Some(Cardinality::MultiValues)
-            }
-            FieldType::U64(ref int_options)
-            | FieldType::I64(ref int_options)
-            | FieldType::F64(ref int_options)
-            | FieldType::Bool(ref int_options) => int_options.get_fastfield_cardinality(),
-            FieldType::Date(ref date_options) => date_options.get_fastfield_cardinality(),
-            FieldType::Facet(_) => Some(Cardinality::MultiValues),
-            FieldType::JsonObject(_) => None,
-            _ => None,
-        }
-    }
-
     /// returns true if the field is normed (see [fieldnorms](crate::fieldnorm)).
     pub fn has_fieldnorms(&self) -> bool {
         match *self {

src/schema/mod.rs

@@ -28,10 +28,10 @@
 //! use tantivy::schema::*;
 //! let mut schema_builder = Schema::builder();
 //! let title_options = TextOptions::default()
-//!     .set_stored()
-//!     .set_indexing_options(TextFieldIndexing::default()
-//!     .set_tokenizer("default")
-//!     .set_index_option(IndexRecordOption::WithFreqsAndPositions));
+//! .set_stored()
+//! .set_indexing_options(TextFieldIndexing::default()
+//! .set_tokenizer("default")
+//! .set_index_option(IndexRecordOption::WithFreqsAndPositions));
 //! schema_builder.add_text_field("title", title_options);
 //! let schema = schema_builder.build();
 //! ```
@@ -45,7 +45,8 @@
 //! In the first phase, the ability to search for documents by the given field is determined by the
 //! [`IndexRecordOption`] of our [`TextOptions`].
 //!
-//! The effect of each possible setting is described more in detail in [`TextOptions`].
+//! The effect of each possible setting is described more in detail
+//! [`TextIndexingOptions`](enum.TextIndexingOptions.html).
 //!
 //! On the other hand setting the field as stored or not determines whether the field should be
 //! returned when [`Searcher::doc()`](crate::Searcher::doc) is called.
@@ -59,8 +60,8 @@
 //! use tantivy::schema::*;
 //! let mut schema_builder = Schema::builder();
 //! let num_stars_options = NumericOptions::default()
-//!     .set_stored()
-//!     .set_indexed();
+//! .set_stored()
+//! .set_indexed();
 //! schema_builder.add_u64_field("num_stars", num_stars_options);
 //! let schema = schema_builder.build();
 //! ```
@@ -78,8 +79,8 @@
 //! For convenience, it is possible to define your field indexing options by combining different
 //! flags using the  `|` operator.
 //!
-//! For instance, a schema containing the two fields defined in the example above could be
-//! rewritten:
+//! For instance, a schema containing the two fields defined in the example above could be rewritten
+//! :
 //!
 //! ```
 //! use tantivy::schema::*;

src/schema/term.rs

@@ -54,27 +54,27 @@ impl Term {
         term
     }
 
-    /// Builds a term given a field, and a `u64`-value
+    /// Builds a term given a field, and a u64-value
     pub fn from_field_u64(field: Field, val: u64) -> Term {
         Term::from_fast_value(field, &val)
     }
 
-    /// Builds a term given a field, and a `i64`-value
+    /// Builds a term given a field, and a i64-value
     pub fn from_field_i64(field: Field, val: i64) -> Term {
         Term::from_fast_value(field, &val)
     }
 
-    /// Builds a term given a field, and a `f64`-value
+    /// Builds a term given a field, and a f64-value
     pub fn from_field_f64(field: Field, val: f64) -> Term {
         Term::from_fast_value(field, &val)
     }
 
-    /// Builds a term given a field, and a `bool`-value
+    /// Builds a term given a field, and a f64-value
     pub fn from_field_bool(field: Field, val: bool) -> Term {
         Term::from_fast_value(field, &val)
     }
 
-    /// Builds a term given a field, and a `DateTime` value
+    /// Builds a term given a field, and a DateTime value
     pub fn from_field_date(field: Field, val: DateTime) -> Term {
         Term::from_fast_value(field, &val.truncate(DatePrecision::Seconds))
     }
@@ -130,7 +130,7 @@ impl Term {
         self.set_fast_value(val);
     }
 
-    /// Sets a `DateTime` value in the term.
+    /// Sets a `i64` value in the term.
     pub fn set_date(&mut self, date: DateTime) {
         self.set_fast_value(date);
     }

src/schema/text_options.rs

@@ -47,9 +47,7 @@ impl TextOptions {
     /// unchanged. The "default" tokenizer will store the terms as lower case and this will be
     /// reflected in the dictionary.
     ///
-    /// The original text can be retrieved via
-    /// [`TermDictionary::ord_to_term()`](crate::termdict::TermDictionary::ord_to_term)
-    /// from the dictionary.
+    /// The original text can be retrieved via `ord_to_term` from the dictionary.
     #[must_use]
     pub fn set_fast(mut self) -> TextOptions {
         self.fast = true;

src/store/compressors.rs

@@ -104,13 +104,6 @@ impl ZstdCompressor {
                             value, opt_name, err
                         )
                     })?;
-                    if value >= 15 {
-                        warn!(
-                            "High zstd compression level detected: {:?}. High compression levels \
-                             (>=15) are slow and will limit indexing speed.",
-                            value
-                        )
-                    }
                     compressor.compression_level = Some(value);
                 }
                 _ => {

src/store/reader.rs

@@ -1,10 +1,10 @@
 use std::io;
 use std::iter::Sum;
-use std::ops::{AddAssign, Range};
+use std::ops::AddAssign;
 use std::sync::atomic::{AtomicUsize, Ordering};
 use std::sync::{Arc, Mutex};
 
-use common::{BinarySerializable, HasLen};
+use common::{BinarySerializable, HasLen, VInt};
 use lru::LruCache;
 use ownedbytes::OwnedBytes;
 
@@ -140,10 +140,10 @@ impl StoreReader {
         self.cache.stats()
     }
 
-    /// Get checkpoint for `DocId`. The checkpoint can be used to load a block containing the
+    /// Get checkpoint for DocId. The checkpoint can be used to load a block containing the
     /// document.
     ///
-    /// Advanced API. In most cases use [`get`](Self::get).
+    /// Advanced API. In most cases use [get](Self::get).
     fn block_checkpoint(&self, doc_id: DocId) -> crate::Result<Checkpoint> {
         self.skip_index.seek(doc_id).ok_or_else(|| {
             crate::TantivyError::InvalidArgument(format!("Failed to lookup Doc #{}.", doc_id))
@@ -160,7 +160,7 @@ impl StoreReader {
 
     /// Loads and decompresses a block.
     ///
-    /// Advanced API. In most cases use [`get`](Self::get).
+    /// Advanced API. In most cases use [get](Self::get).
     fn read_block(&self, checkpoint: &Checkpoint) -> io::Result<Block> {
         let cache_key = checkpoint.byte_range.start;
         if let Some(block) = self.cache.get_from_cache(cache_key) {
@@ -205,21 +205,28 @@ impl StoreReader {
 
     /// Advanced API.
     ///
-    /// In most cases use [`get_document_bytes`](Self::get_document_bytes).
+    /// In most cases use [get_document_bytes](Self::get_document_bytes).
     fn get_document_bytes_from_block(
         block: OwnedBytes,
         doc_id: DocId,
         checkpoint: &Checkpoint,
     ) -> crate::Result<OwnedBytes> {
-        let doc_pos = doc_id - checkpoint.doc_range.start;
+        let mut cursor = &block[..];
+        let cursor_len_before = cursor.len();
+        for _ in checkpoint.doc_range.start..doc_id {
+            let doc_length = VInt::deserialize(&mut cursor)?.val() as usize;
+            cursor = &cursor[doc_length..];
+        }
 
-        let range = block_read_index(&block, doc_pos)?;
-        Ok(block.slice(range))
+        let doc_length = VInt::deserialize(&mut cursor)?.val() as usize;
+        let start_pos = cursor_len_before - cursor.len();
+        let end_pos = cursor_len_before - cursor.len() + doc_length;
+        Ok(block.slice(start_pos..end_pos))
     }
 
     /// Iterator over all Documents in their order as they are stored in the doc store.
     /// Use this, if you want to extract all Documents from the doc store.
-    /// The `alive_bitset` has to be forwarded from the `SegmentReader` or the results may be wrong.
+    /// The alive_bitset has to be forwarded from the `SegmentReader` or the results maybe wrong.
     pub fn iter<'a: 'b, 'b>(
         &'b self,
         alive_bitset: Option<&'a AliveBitSet>,
@@ -230,9 +237,9 @@ impl StoreReader {
         })
     }
 
-    /// Iterator over all raw Documents in their order as they are stored in the doc store.
+    /// Iterator over all RawDocuments in their order as they are stored in the doc store.
     /// Use this, if you want to extract all Documents from the doc store.
-    /// The `alive_bitset` has to be forwarded from the `SegmentReader` or the results may be wrong.
+    /// The alive_bitset has to be forwarded from the `SegmentReader` or the results maybe wrong.
     pub(crate) fn iter_raw<'a: 'b, 'b>(
         &'b self,
         alive_bitset: Option<&'a AliveBitSet>,
@@ -247,7 +254,9 @@ impl StoreReader {
         let mut curr_block = curr_checkpoint
             .as_ref()
             .map(|checkpoint| self.read_block(checkpoint).map_err(|e| e.kind())); // map error in order to enable cloning
-        let mut doc_pos = 0;
+        let mut block_start_pos = 0;
+        let mut num_skipped = 0;
+        let mut reset_block_pos = false;
         (0..last_doc_id)
             .filter_map(move |doc_id| {
                 // filter_map is only used to resolve lifetime issues between the two closures on
@@ -259,19 +268,24 @@ impl StoreReader {
                     curr_block = curr_checkpoint
                         .as_ref()
                         .map(|checkpoint| self.read_block(checkpoint).map_err(|e| e.kind()));
-                    doc_pos = 0;
+                    reset_block_pos = true;
+                    num_skipped = 0;
                 }
 
                 let alive = alive_bitset.map_or(true, |bitset| bitset.is_alive(doc_id));
-                let res = if alive {
-                    Some((curr_block.clone(), doc_pos))
+                if alive {
+                    let ret = Some((curr_block.clone(), num_skipped, reset_block_pos));
+                    // the map block will move over the num_skipped, so we reset to 0
+                    num_skipped = 0;
+                    reset_block_pos = false;
+                    ret
                 } else {
+                    // we keep the number of skipped documents to move forward in the map block
+                    num_skipped += 1;
                     None
-                };
-                doc_pos += 1;
-                res
+                }
             })
-            .map(move |(block, doc_pos)| {
+            .map(move |(block, num_skipped, reset_block_pos)| {
                 let block = block
                     .ok_or_else(|| {
                         DataCorruption::comment_only(
@@ -282,9 +296,30 @@ impl StoreReader {
                     .map_err(|error_kind| {
                         std::io::Error::new(error_kind, "error when reading block in doc store")
                     })?;
+                // this flag is set, when filter_map moved to the next block
+                if reset_block_pos {
+                    block_start_pos = 0;
+                }
+                let mut cursor = &block[block_start_pos..];
+                let mut pos = 0;
+                // move forward 1 doc + num_skipped in block and return length of current doc
+                let doc_length = loop {
+                    let doc_length = VInt::deserialize(&mut cursor)?.val() as usize;
+                    let num_bytes_read = block[block_start_pos..].len() - cursor.len();
+                    block_start_pos += num_bytes_read;
 
-                let range = block_read_index(&block, doc_pos)?;
-                Ok(block.slice(range))
+                    pos += 1;
+                    if pos == num_skipped + 1 {
+                        break doc_length;
+                    } else {
+                        block_start_pos += doc_length;
+                        cursor = &block[block_start_pos..];
+                    }
+                };
+                let end_pos = block_start_pos + doc_length;
+                let doc_bytes = block.slice(block_start_pos..end_pos);
+                block_start_pos = end_pos;
+                Ok(doc_bytes)
             })
     }
 
@@ -294,33 +329,11 @@ impl StoreReader {
     }
 }
 
-fn block_read_index(block: &[u8], doc_pos: u32) -> crate::Result<Range<usize>> {
-    let doc_pos = doc_pos as usize;
-    let size_of_u32 = std::mem::size_of::<u32>();
-
-    let index_len_pos = block.len() - size_of_u32;
-    let index_len = u32::deserialize(&mut &block[index_len_pos..])? as usize;
-
-    if doc_pos > index_len {
-        return Err(crate::TantivyError::InternalError(
-            "Attempted to read doc from wrong block".to_owned(),
-        ));
-    }
-
-    let index_start = block.len() - (index_len + 1) * size_of_u32;
-    let index = &block[index_start..index_start + index_len * size_of_u32];
-
-    let start_offset = u32::deserialize(&mut &index[doc_pos * size_of_u32..])? as usize;
-    let end_offset = u32::deserialize(&mut &index[(doc_pos + 1) * size_of_u32..])
-        .unwrap_or(index_start as u32) as usize;
-    Ok(start_offset..end_offset)
-}
-
 #[cfg(feature = "quickwit")]
 impl StoreReader {
     /// Advanced API.
     ///
-    /// In most cases use [`get_async`](Self::get_async)
+    /// In most cases use [get_async](Self::get_async)
     ///
     /// Loads and decompresses a block asynchronously.
     async fn read_block_async(&self, checkpoint: &Checkpoint) -> crate::AsyncIoResult<Block> {
@@ -344,14 +357,14 @@ impl StoreReader {
         Ok(decompressed_block)
     }
 
-    /// Reads raw bytes of a given document asynchronously.
+    /// Fetches a document asynchronously.
     pub async fn get_document_bytes_async(&self, doc_id: DocId) -> crate::Result<OwnedBytes> {
         let checkpoint = self.block_checkpoint(doc_id)?;
         let block = self.read_block_async(&checkpoint).await?;
         Self::get_document_bytes_from_block(block, doc_id, &checkpoint)
     }
 
-    /// Fetches a document asynchronously. Async version of [`get`](Self::get).
+    /// Reads raw bytes of a given document. Async version of [get](Self::get).
     pub async fn get_async(&self, doc_id: DocId) -> crate::Result<Document> {
         let mut doc_bytes = self.get_document_bytes_async(doc_id).await?;
         Ok(Document::deserialize(&mut doc_bytes)?)
@@ -414,7 +427,7 @@ mod tests {
         assert_eq!(store.cache_stats().cache_hits, 1);
         assert_eq!(store.cache_stats().cache_misses, 2);
 
-        assert_eq!(store.cache.peek_lru(), Some(11163));
+        assert_eq!(store.cache.peek_lru(), Some(9210));
 
         Ok(())
     }

src/store/writer.rs

@@ -1,6 +1,6 @@
 use std::io::{self, Write};
 
-use common::BinarySerializable;
+use common::{BinarySerializable, VInt};
 
 use super::compressors::Compressor;
 use super::StoreReader;
@@ -22,7 +22,6 @@ pub struct StoreWriter {
     num_docs_in_current_block: DocId,
     intermediary_buffer: Vec<u8>,
     current_block: Vec<u8>,
-    doc_pos: Vec<u32>,
     block_compressor: BlockCompressor,
 }
 
@@ -43,7 +42,6 @@ impl StoreWriter {
             block_size,
             num_docs_in_current_block: 0,
             intermediary_buffer: Vec::new(),
-            doc_pos: Vec::new(),
             current_block: Vec::new(),
             block_compressor,
         })
@@ -55,17 +53,12 @@ impl StoreWriter {
 
     /// The memory used (inclusive childs)
     pub fn mem_usage(&self) -> usize {
-        self.intermediary_buffer.capacity()
-            + self.current_block.capacity()
-            + self.doc_pos.capacity() * std::mem::size_of::<u32>()
+        self.intermediary_buffer.capacity() + self.current_block.capacity()
     }
 
     /// Checks if the current block is full, and if so, compresses and flushes it.
     fn check_flush_block(&mut self) -> io::Result<()> {
-        // this does not count the VInt storing the index lenght itself, but it is negligible in
-        // front of everything else.
-        let index_len = self.doc_pos.len() * std::mem::size_of::<usize>();
-        if self.current_block.len() + index_len > self.block_size {
+        if self.current_block.len() > self.block_size {
             self.send_current_block_to_compressor()?;
         }
         Ok(())
@@ -77,19 +70,8 @@ impl StoreWriter {
         if self.current_block.is_empty() {
             return Ok(());
         }
-
-        let size_of_u32 = std::mem::size_of::<u32>();
-        self.current_block
-            .reserve((self.doc_pos.len() + 1) * size_of_u32);
-
-        for pos in self.doc_pos.iter() {
-            pos.serialize(&mut self.current_block)?;
-        }
-        (self.doc_pos.len() as u32).serialize(&mut self.current_block)?;
-
         self.block_compressor
             .compress_block_and_write(&self.current_block, self.num_docs_in_current_block)?;
-        self.doc_pos.clear();
         self.current_block.clear();
         self.num_docs_in_current_block = 0;
         Ok(())
@@ -105,7 +87,8 @@ impl StoreWriter {
         // calling store bytes would be preferable for code reuse, but then we can't use
         // intermediary_buffer due to the borrow checker
         // a new buffer costs ~1% indexing performance
-        self.doc_pos.push(self.current_block.len() as u32);
+        let doc_num_bytes = self.intermediary_buffer.len();
+        VInt(doc_num_bytes as u64).serialize_into_vec(&mut self.current_block);
         self.current_block
             .write_all(&self.intermediary_buffer[..])?;
         self.num_docs_in_current_block += 1;
@@ -118,7 +101,8 @@ impl StoreWriter {
     /// The document id is implicitly the current number
     /// of documents.
     pub fn store_bytes(&mut self, serialized_document: &[u8]) -> io::Result<()> {
-        self.doc_pos.push(self.current_block.len() as u32);
+        let doc_num_bytes = serialized_document.len();
+        VInt(doc_num_bytes as u64).serialize_into_vec(&mut self.current_block);
         self.current_block.extend_from_slice(serialized_document);
         self.num_docs_in_current_block += 1;
         self.check_flush_block()?;

src/tokenizer/ngram_tokenizer.rs

@@ -255,7 +255,7 @@ where T: Iterator<Item = usize>
 /// Emits all of the offsets where a codepoint starts
 /// or a codepoint ends.
 ///
-/// By convention, we emit `[0]` for the empty string.
+/// By convention, we emit [0] for the empty string.
 struct CodepointFrontiers<'a> {
     s: &'a str,
     next_el: Option<usize>,