blop

.github/workflows/coverage.yml

@@ -12,14 +12,12 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Install Rust
-        run: rustup toolchain install nightly --profile minimal --component llvm-tools-preview
-      - uses: Swatinem/rust-cache@v2
+        run: rustup toolchain install nightly --component llvm-tools-preview
       - uses: taiki-e/install-action@cargo-llvm-cov
       - name: Generate code coverage
         run: cargo +nightly llvm-cov --all-features --workspace --lcov --output-path lcov.info
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v3
-        continue-on-error: true
         with:
           token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos
           files: lcov.info

.github/workflows/long_running.yml

@@ -19,10 +19,11 @@ jobs:
       uses: actions-rs/toolchain@v1
       with:
           toolchain: stable
-          profile: minimal
           override: true
+          components: rustfmt, clippy
 
     - name: Run indexing_unsorted
       run: cargo test indexing_unsorted -- --ignored
     - name: Run indexing_sorted
       run: cargo test indexing_sorted -- --ignored
+

.github/workflows/test.yml

@@ -10,27 +10,34 @@ env:
   CARGO_TERM_COLOR: always
 
 jobs:
-  check:
+  test:
 
     runs-on: ubuntu-latest
 
     steps:
     - uses: actions/checkout@v3
-
-    - name: Install nightly
+    - name: Install latest nightly to test also against unstable feature flag
       uses: actions-rs/toolchain@v1
       with:
             toolchain: nightly
-            profile: minimal
+            override: true
             components: rustfmt
+
     - name: Install stable
       uses: actions-rs/toolchain@v1
       with:
             toolchain: stable
-            profile: minimal
-            components: clippy
+            override: true
+            components: rustfmt, clippy
 
-    - uses: Swatinem/rust-cache@v2
+    - name: Build
+      run: cargo build --verbose --workspace
+
+    - name: Run tests
+      run: cargo +stable test --features mmap,brotli-compression,lz4-compression,snappy-compression,zstd-compression,failpoints --verbose --workspace
+
+    - name: Run tests quickwit feature
+      run: cargo +stable test --features mmap,quickwit,failpoints --verbose --workspace
 
     - name: Check Formatting
       run: cargo +nightly fmt --all -- --check
@@ -41,34 +48,3 @@ jobs:
         token: ${{ secrets.GITHUB_TOKEN }}
         args: --tests
 
-  test:
-
-    runs-on: ubuntu-latest
-
-    strategy:
-      matrix:
-        features: [
-            { label: "all", flags: "mmap,brotli-compression,lz4-compression,snappy-compression,zstd-compression,failpoints" },
-            { label: "quickwit", flags: "mmap,quickwit,failpoints" }
-        ]
-
-    name: test-${{ matrix.features.label}}
-
-    steps:
-    - uses: actions/checkout@v3
-
-    - name: Install stable
-      uses: actions-rs/toolchain@v1
-      with:
-            toolchain: stable
-            profile: minimal
-            override: true
-
-    - uses: taiki-e/install-action@nextest
-    - uses: Swatinem/rust-cache@v2
-
-    - name: Run tests
-      run: cargo +stable nextest run --features ${{ matrix.features.flags }} --verbose --workspace
-
-    - name: Run doctests
-      run: cargo +stable test --doc --features ${{ matrix.features.flags }} --verbose --workspace

ARCHITECTURE.md

@@ -95,7 +95,7 @@ called [`Directory`](src/directory/directory.rs).
 Contrary to Lucene however, "files" are quite different from some kind of `io::Read` object.
 Check out [`src/directory/directory.rs`](src/directory/directory.rs) trait for more details.
 
-Tantivy ships two main directory implementation: the `MmapDirectory` and the `RamDirectory`,
+Tantivy ships two main directory implementation: the `MMapDirectory` and the `RAMDirectory`,
 but users can extend tantivy with their own implementation.
 
 ## [schema/](src/schema): What are documents?

Cargo.toml

@@ -30,7 +30,7 @@ log = "0.4.16"
 serde = { version = "1.0.136", features = ["derive"] }
 serde_json = "1.0.79"
 num_cpus = "1.13.1"
-fs2 = { version = "0.4.3", optional = true }
+fs2={ version = "0.4.3", optional = true }
 levenshtein_automata = "0.2.1"
 uuid = { version = "1.0.0", features = ["v4", "serde"] }
 crossbeam-channel = "0.5.4"
@@ -56,6 +56,7 @@ lru = "0.7.5"
 fastdivide = "0.4.0"
 itertools = "0.10.3"
 measure_time = "0.8.2"
+pretty_assertions = "1.2.1"
 serde_cbor = { version = "0.11.2", optional = true }
 async-trait = "0.1.53"
 arc-swap = "1.5.0"
@@ -67,7 +68,6 @@ winapi = "0.3.9"
 rand = "0.8.5"
 maplit = "1.0.2"
 matches = "0.1.9"
-pretty_assertions = "1.2.1"
 proptest = "1.0.0"
 criterion = "0.3.5"
 test-log = "0.2.10"

README.md

@@ -127,7 +127,6 @@ $ gdb run
 # Companies Using Tantivy
 
 <p align="left">
-<img align="center" src="doc/assets/images/etsy.png" alt="Etsy" height="25" width="auto" />&nbsp;
 <img align="center" src="doc/assets/images/Nuclia.png#gh-light-mode-only" alt="Nuclia" height="25" width="auto" /> &nbsp;
 <img align="center" src="doc/assets/images/humanfirst.png#gh-light-mode-only" alt="Humanfirst.ai" height="30" width="auto" />
 <img align="center" src="doc/assets/images/element.io.svg#gh-light-mode-only" alt="Element.io" height="25" width="auto" />

bitpacker/src/bitpacker.rs

@@ -82,16 +82,14 @@ impl BitUnpacker {
         }
     }
 
-    pub fn bit_width(&self) -> u8 {
-        self.num_bits as u8
-    }
-
     #[inline]
     pub fn get(&self, idx: u64, data: &[u8]) -> u64 {
         if self.num_bits == 0 {
             return 0u64;
         }
-        let addr_in_bits = idx * self.num_bits;
+        let num_bits = self.num_bits;
+        let mask = self.mask;
+        let addr_in_bits = idx * num_bits;
         let addr = addr_in_bits >> 3;
         let bit_shift = addr_in_bits & 7;
         debug_assert!(
@@ -103,7 +101,7 @@ impl BitUnpacker {
             .unwrap();
         let val_unshifted_unmasked: u64 = u64::from_le_bytes(bytes);
         let val_shifted = (val_unshifted_unmasked >> bit_shift) as u64;
-        val_shifted & self.mask
+        val_shifted & mask
     }
 }
 

bitpacker/src/blocked_bitpacker.rs

@@ -14,6 +14,7 @@ pub struct BlockedBitpacker {
     buffer: Vec<u64>,
     offset_and_bits: Vec<BlockedBitpackerEntryMetaData>,
 }
+
 impl Default for BlockedBitpacker {
     fn default() -> Self {
         BlockedBitpacker::new()
@@ -58,18 +59,13 @@ fn metadata_test() {
     assert_eq!(meta.num_bits(), 6);
 }
 
-fn mem_usage<T>(items: &Vec<T>) -> usize {
-    items.capacity() * std::mem::size_of::<T>()
-}
-
 impl BlockedBitpacker {
     pub fn new() -> Self {
-        let mut compressed_blocks = vec![];
-        compressed_blocks.resize(8, 0);
+        let compressed_blocks = vec![0u8; 8];
         Self {
             compressed_blocks,
-            buffer: vec![],
-            offset_and_bits: vec![],
+            buffer: Vec::new(),
+            offset_and_bits: Vec::new(),
         }
     }
 
@@ -77,8 +73,10 @@ impl BlockedBitpacker {
     pub fn mem_usage(&self) -> usize {
         std::mem::size_of::<BlockedBitpacker>()
             + self.compressed_blocks.capacity()
-            + mem_usage(&self.offset_and_bits)
-            + mem_usage(&self.buffer)
+            + self.offset_and_bits.capacity()
+                * std::mem::size_of_val(&self.offset_and_bits.get(0).cloned().unwrap_or_default())
+            + self.buffer.capacity()
+                * std::mem::size_of_val(&self.buffer.get(0).cloned().unwrap_or_default())
     }
 
     #[inline]

common/src/bitset.rs

@@ -259,7 +259,11 @@ impl BitSet {
         // we do not check saturated els.
         let higher = el / 64u32;
         let lower = el % 64u32;
-        self.len += u64::from(self.tinysets[higher as usize].insert_mut(lower));
+        self.len += if self.tinysets[higher as usize].insert_mut(lower) {
+            1
+        } else {
+            0
+        };
     }
 
     /// Inserts an element in the `BitSet`
@@ -268,7 +272,11 @@ impl BitSet {
         // we do not check saturated els.
         let higher = el / 64u32;
         let lower = el % 64u32;
-        self.len -= u64::from(self.tinysets[higher as usize].remove_mut(lower));
+        self.len -= if self.tinysets[higher as usize].remove_mut(lower) {
+            1
+        } else {
+            0
+        };
     }
 
     /// Returns true iff the elements is in the `BitSet`.

common/src/lib.rs

@@ -11,10 +11,7 @@ mod writer;
 
 pub use bitset::*;
 pub use serialize::{BinarySerializable, DeserializeFrom, FixedSize};
-pub use vint::{
-    deserialize_vint_u128, read_u32_vint, read_u32_vint_no_advance, serialize_vint_u128,
-    serialize_vint_u32, write_u32_vint, VInt, VIntU128,
-};
+pub use vint::{read_u32_vint, read_u32_vint_no_advance, serialize_vint_u32, write_u32_vint, VInt};
 pub use writer::{AntiCallToken, CountingWriter, TerminatingWrite};
 
 /// Has length trait
@@ -55,13 +52,13 @@ const HIGHEST_BIT: u64 = 1 << 63;
 /// to values over 2^63, and all values end up requiring 64 bits.
 ///
 /// # See also
-/// The reverse mapping is [`u64_to_i64()`].
+/// The [reverse mapping is `u64_to_i64`](./fn.u64_to_i64.html).
 #[inline]
 pub fn i64_to_u64(val: i64) -> u64 {
     (val as u64) ^ HIGHEST_BIT
 }
 
-/// Reverse the mapping given by [`i64_to_u64()`].
+/// Reverse the mapping given by [`i64_to_u64`](./fn.i64_to_u64.html).
 #[inline]
 pub fn u64_to_i64(val: u64) -> i64 {
     (val ^ HIGHEST_BIT) as i64
@@ -83,7 +80,7 @@ pub fn u64_to_i64(val: u64) -> i64 {
 /// explains the mapping in a clear manner.
 ///
 /// # See also
-/// The reverse mapping is [`u64_to_f64()`].
+/// The [reverse mapping is `u64_to_f64`](./fn.u64_to_f64.html).
 #[inline]
 pub fn f64_to_u64(val: f64) -> u64 {
     let bits = val.to_bits();
@@ -94,7 +91,7 @@ pub fn f64_to_u64(val: f64) -> u64 {
     }
 }
 
-/// Reverse the mapping given by [`f64_to_u64()`].
+/// Reverse the mapping given by [`i64_to_u64`](./fn.i64_to_u64.html).
 #[inline]
 pub fn u64_to_f64(val: u64) -> f64 {
     f64::from_bits(if val & HIGHEST_BIT != 0 {

common/src/serialize.rs

@@ -161,7 +161,8 @@ impl FixedSize for u8 {
 
 impl BinarySerializable for bool {
     fn serialize<W: Write>(&self, writer: &mut W) -> io::Result<()> {
-        writer.write_u8(u8::from(*self))
+        let val = if *self { 1 } else { 0 };
+        writer.write_u8(val)
     }
     fn deserialize<R: Read>(reader: &mut R) -> io::Result<bool> {
         let val = reader.read_u8()?;

common/src/vint.rs

@@ -5,75 +5,6 @@ use byteorder::{ByteOrder, LittleEndian};
 
 use super::BinarySerializable;
 
-/// Variable int serializes a u128 number
-pub fn serialize_vint_u128(mut val: u128, output: &mut Vec<u8>) {
-    loop {
-        let next_byte: u8 = (val % 128u128) as u8;
-        val /= 128u128;
-        if val == 0 {
-            output.push(next_byte | STOP_BIT);
-            return;
-        } else {
-            output.push(next_byte);
-        }
-    }
-}
-
-/// Deserializes a u128 number
-///
-/// Returns the number and the slice after the vint
-pub fn deserialize_vint_u128(data: &[u8]) -> io::Result<(u128, &[u8])> {
-    let mut result = 0u128;
-    let mut shift = 0u64;
-    for i in 0..19 {
-        let b = data[i];
-        result |= u128::from(b % 128u8) << shift;
-        if b >= STOP_BIT {
-            return Ok((result, &data[i + 1..]));
-        }
-        shift += 7;
-    }
-    Err(io::Error::new(
-        io::ErrorKind::InvalidData,
-        "Failed to deserialize u128 vint",
-    ))
-}
-
-///   Wrapper over a `u128` that serializes as a variable int.
-#[derive(Clone, Copy, Debug, Eq, PartialEq)]
-pub struct VIntU128(pub u128);
-
-impl BinarySerializable for VIntU128 {
-    fn serialize<W: Write>(&self, writer: &mut W) -> io::Result<()> {
-        let mut buffer = vec![];
-        serialize_vint_u128(self.0, &mut buffer);
-        writer.write_all(&buffer)
-    }
-
-    fn deserialize<R: Read>(reader: &mut R) -> io::Result<Self> {
-        let mut bytes = reader.bytes();
-        let mut result = 0u128;
-        let mut shift = 0u64;
-        loop {
-            match bytes.next() {
-                Some(Ok(b)) => {
-                    result |= u128::from(b % 128u8) << shift;
-                    if b >= STOP_BIT {
-                        return Ok(VIntU128(result));
-                    }
-                    shift += 7;
-                }
-                _ => {
-                    return Err(io::Error::new(
-                        io::ErrorKind::InvalidData,
-                        "Reach end of buffer while reading VInt",
-                    ));
-                }
-            }
-        }
-    }
-}
-
 ///   Wrapper over a `u64` that serializes as a variable int.
 #[derive(Clone, Copy, Debug, Eq, PartialEq)]
 pub struct VInt(pub u64);
@@ -245,7 +176,6 @@ impl BinarySerializable for VInt {
 mod tests {
 
     use super::{serialize_vint_u32, BinarySerializable, VInt};
-    use crate::vint::{deserialize_vint_u128, serialize_vint_u128, VIntU128};
 
     fn aux_test_vint(val: u64) {
         let mut v = [14u8; 10];
@@ -287,26 +217,6 @@ mod tests {
         assert_eq!(&buffer[..len_vint], res2, "array wrong for {}", val);
     }
 
-    fn aux_test_vint_u128(val: u128) {
-        let mut data = vec![];
-        serialize_vint_u128(val, &mut data);
-        let (deser_val, _data) = deserialize_vint_u128(&data).unwrap();
-        assert_eq!(val, deser_val);
-
-        let mut out = vec![];
-        VIntU128(val).serialize(&mut out).unwrap();
-        let deser_val = VIntU128::deserialize(&mut &out[..]).unwrap();
-        assert_eq!(val, deser_val.0);
-    }
-
-    #[test]
-    fn test_vint_u128() {
-        aux_test_vint_u128(0);
-        aux_test_vint_u128(1);
-        aux_test_vint_u128(u128::MAX / 3);
-        aux_test_vint_u128(u128::MAX);
-    }
-
     #[test]
     fn test_vint_u32() {
         aux_test_serialize_vint_u32(0);

common/src/writer.rs

@@ -55,14 +55,14 @@ impl<W: TerminatingWrite> TerminatingWrite for CountingWriter<W> {
 }
 
 /// Struct used to prevent from calling
-/// [`terminate_ref`](TerminatingWrite::terminate_ref) directly
+/// [`terminate_ref`](trait.TerminatingWrite.html#tymethod.terminate_ref) directly
 ///
 /// The point is that while the type is public, it cannot be built by anyone
 /// outside of this module.
 pub struct AntiCallToken(());
 
 /// Trait used to indicate when no more write need to be done on a writer
-pub trait TerminatingWrite: Write + Send + Sync {
+pub trait TerminatingWrite: Write + Send {
     /// Indicate that the writer will no longer be used. Internally call terminate_ref.
     fn terminate(mut self) -> io::Result<()>
     where Self: Sized {

examples/custom_collector.rs

@@ -7,12 +7,10 @@
 // Of course, you can have a look at the tantivy's built-in collectors
 // such as the `CountCollector` for more examples.
 
-use std::sync::Arc;
-
-use fastfield_codecs::Column;
 // ---
 // Importing tantivy...
 use tantivy::collector::{Collector, SegmentCollector};
+use tantivy::fastfield::{FastFieldReader, FastFieldReaderImpl};
 use tantivy::query::QueryParser;
 use tantivy::schema::{Field, Schema, FAST, INDEXED, TEXT};
 use tantivy::{doc, Index, Score, SegmentReader};
@@ -97,7 +95,7 @@ impl Collector for StatsCollector {
 }
 
 struct StatsSegmentCollector {
-    fast_field_reader: Arc<dyn Column<u64>>,
+    fast_field_reader: FastFieldReaderImpl<u64>,
     stats: Stats,
 }
 
@@ -105,7 +103,7 @@ impl SegmentCollector for StatsSegmentCollector {
     type Fruit = Option<Stats>;
 
     fn collect(&mut self, doc: u32, _score: Score) {
-        let value = self.fast_field_reader.get_val(doc as u64) as f64;
+        let value = self.fast_field_reader.get(doc) as f64;
         self.stats.count += 1;
         self.stats.sum += value;
         self.stats.squared_sum += value * value;

examples/custom_tokenizer.rs

@@ -36,7 +36,8 @@ fn main() -> tantivy::Result<()> {
     // need to be able to be able to retrieve it
     // for our application.
     //
-    // We can make our index lighter by omitting the `STORED` flag.
+    // We can make our index lighter and
+    // by omitting `STORED` flag.
     let body = schema_builder.add_text_field("body", TEXT);
 
     let schema = schema_builder.build();

examples/warmer.rs

@@ -3,6 +3,7 @@ use std::collections::{HashMap, HashSet};
 use std::sync::{Arc, RwLock, Weak};
 
 use tantivy::collector::TopDocs;
+use tantivy::fastfield::FastFieldReader;
 use tantivy::query::QueryParser;
 use tantivy::schema::{Field, Schema, FAST, TEXT};
 use tantivy::{
@@ -51,7 +52,7 @@ impl Warmer for DynamicPriceColumn {
             let product_id_reader = segment.fast_fields().u64(self.field)?;
             let product_ids: Vec<ProductId> = segment
                 .doc_ids_alive()
-                .map(|doc| product_id_reader.get_val(doc as u64))
+                .map(|doc| product_id_reader.get(doc))
                 .collect();
             let mut prices_it = self.price_fetcher.fetch_prices(&product_ids).into_iter();
             let mut price_vals: Vec<Price> = Vec::new();

fastfield_codecs/Cargo.toml

@@ -15,17 +15,12 @@ ownedbytes = { version = "0.3.0", path = "../ownedbytes" }
 prettytable-rs = {version="0.9.0", optional= true}
 rand = {version="0.8.3", optional= true}
 fastdivide = "0.4"
-log = "0.4"
-itertools = { version = "0.10.3" }
-measure_time = { version="0.8.2", optional=true}
 
 [dev-dependencies]
 more-asserts = "0.3.0"
-proptest = "1.0.0"
 rand = "0.8.3"
 
 [features]
-bin = ["prettytable-rs", "rand", "measure_time"]
+bin = ["prettytable-rs", "rand"]
 default = ["bin"]
-unstable = []
 

fastfield_codecs/benches/bench.rs

@@ -4,222 +4,100 @@ extern crate test;
 
 #[cfg(test)]
 mod tests {
-    use std::iter;
-    use std::sync::Arc;
-
+    use fastfield_codecs::bitpacked::{BitpackedFastFieldCodec, BitpackedFastFieldReader};
+    use fastfield_codecs::linearinterpol::{LinearInterpolCodec, LinearInterpolFastFieldReader};
+    use fastfield_codecs::multilinearinterpol::{
+        MultiLinearInterpolFastFieldCodec, MultiLinearInterpolFastFieldReader,
+    };
     use fastfield_codecs::*;
-    use ownedbytes::OwnedBytes;
-    use rand::prelude::*;
-    use test::Bencher;
 
-    use super::*;
-
-    // Warning: this generates the same permutation at each call
-    fn generate_permutation() -> Vec<u64> {
-        let mut permutation: Vec<u64> = (0u64..100_000u64).collect();
-        permutation.shuffle(&mut StdRng::from_seed([1u8; 32]));
-        permutation
-    }
-
-    fn generate_random() -> Vec<u64> {
-        let mut permutation: Vec<u64> = (0u64..100_000u64)
-            .map(|el| el + random::<u16>() as u64)
+    fn get_data() -> Vec<u64> {
+        let mut data: Vec<_> = (100..55000_u64)
+            .map(|num| num + rand::random::<u8>() as u64)
             .collect();
-        permutation.shuffle(&mut StdRng::from_seed([1u8; 32]));
-        permutation
-    }
-
-    // Warning: this generates the same permutation at each call
-    fn generate_permutation_gcd() -> Vec<u64> {
-        let mut permutation: Vec<u64> = (1u64..100_000u64).map(|el| el * 1000).collect();
-        permutation.shuffle(&mut StdRng::from_seed([1u8; 32]));
-        permutation
-    }
-
-    pub fn serialize_and_load<T: MonotonicallyMappableToU64 + Ord + Default>(
-        column: &[T],
-    ) -> Arc<dyn Column<T>> {
-        let mut buffer = Vec::new();
-        serialize(VecColumn::from(&column), &mut buffer, &ALL_CODEC_TYPES).unwrap();
-        open(OwnedBytes::new(buffer)).unwrap()
-    }
-
-    #[bench]
-    fn bench_intfastfield_jumpy_veclookup(b: &mut Bencher) {
-        let permutation = generate_permutation();
-        let n = permutation.len();
-        b.iter(|| {
-            let mut a = 0u64;
-            for _ in 0..n {
-                a = permutation[a as usize];
-            }
-            a
-        });
-    }
-
-    #[bench]
-    fn bench_intfastfield_jumpy_fflookup(b: &mut Bencher) {
-        let permutation = generate_permutation();
-        let n = permutation.len();
-        let column: Arc<dyn Column<u64>> = serialize_and_load(&permutation);
-        b.iter(|| {
-            let mut a = 0u64;
-            for _ in 0..n {
-                a = column.get_val(a as u64);
-            }
-            a
-        });
-    }
-
-    fn get_exp_data() -> Vec<u64> {
-        let mut data = vec![];
-        for i in 0..100 {
-            let num = i * i;
-            data.extend(iter::repeat(i as u64).take(num));
-        }
-        data.shuffle(&mut StdRng::from_seed([1u8; 32]));
-
-        // lengt = 328350
+        data.push(99_000);
+        data.insert(1000, 2000);
+        data.insert(2000, 100);
+        data.insert(3000, 4100);
+        data.insert(4000, 100);
+        data.insert(5000, 800);
         data
     }
 
-    fn get_data_50percent_item() -> (u128, u128, Vec<u128>) {
-        let mut permutation = get_exp_data();
-        let major_item = 20;
-        let minor_item = 10;
-        permutation.extend(iter::repeat(major_item).take(permutation.len()));
-        permutation.shuffle(&mut StdRng::from_seed([1u8; 32]));
-        let permutation = permutation.iter().map(|el| *el as u128).collect::<Vec<_>>();
-        (major_item as u128, minor_item as u128, permutation)
+    fn value_iter() -> impl Iterator<Item = u64> {
+        0..20_000
     }
-    fn get_u128_column_random() -> Arc<dyn Column<u128>> {
-        let permutation = generate_random();
-        let permutation = permutation.iter().map(|el| *el as u128).collect::<Vec<_>>();
-        get_u128_column_from_data(&permutation)
-    }
-
-    fn get_u128_column_from_data(data: &[u128]) -> Arc<dyn Column<u128>> {
-        let mut out = vec![];
-        serialize_u128(VecColumn::from(&data), &mut out).unwrap();
-        let out = OwnedBytes::new(out);
-        open_u128(out).unwrap()
-    }
-
-    #[bench]
-    fn bench_intfastfield_getrange_u128_50percent_hit(b: &mut Bencher) {
-        let (major_item, _minor_item, data) = get_data_50percent_item();
-        let column = get_u128_column_from_data(&data);
-
-        b.iter(|| column.get_between_vals(major_item..=major_item));
-    }
-
-    #[bench]
-    fn bench_intfastfield_getrange_u128_single_hit(b: &mut Bencher) {
-        let (_major_item, minor_item, data) = get_data_50percent_item();
-        let column = get_u128_column_from_data(&data);
-
-        b.iter(|| column.get_between_vals(minor_item..=minor_item));
-    }
-
-    #[bench]
-    fn bench_intfastfield_getrange_u128_hit_all(b: &mut Bencher) {
-        let (_major_item, _minor_item, data) = get_data_50percent_item();
-        let column = get_u128_column_from_data(&data);
-
-        b.iter(|| column.get_between_vals(0..=u128::MAX));
-    }
-
-    #[bench]
-    fn bench_intfastfield_scan_all_fflookup_u128(b: &mut Bencher) {
-        let column = get_u128_column_random();
-
+    fn bench_get<S: FastFieldCodec, R: FastFieldCodecReader>(b: &mut Bencher, data: &[u64]) {
+        let mut bytes = vec![];
+        S::serialize(
+            &mut bytes,
+            &data,
+            stats_from_vec(data),
+            data.iter().cloned(),
+            data.iter().cloned(),
+        )
+        .unwrap();
+        let reader = R::open_from_bytes(&bytes).unwrap();
         b.iter(|| {
-            let mut a = 0u128;
-            for i in 0u64..column.num_vals() as u64 {
-                a += column.get_val(i);
+            for pos in value_iter() {
+                reader.get_u64(pos as u64, &bytes);
             }
-            a
+        });
+    }
+    fn bench_create<S: FastFieldCodec>(b: &mut Bencher, data: &[u64]) {
+        let mut bytes = vec![];
+        b.iter(|| {
+            S::serialize(
+                &mut bytes,
+                &data,
+                stats_from_vec(data),
+                data.iter().cloned(),
+                data.iter().cloned(),
+            )
+            .unwrap();
         });
     }
 
+    use test::Bencher;
     #[bench]
-    fn bench_intfastfield_jumpy_stride5_u128(b: &mut Bencher) {
-        let column = get_u128_column_random();
-
-        b.iter(|| {
-            let n = column.num_vals();
-            let mut a = 0u128;
-            for i in (0..n / 5).map(|val| val * 5) {
-                a += column.get_val(i as u64);
-            }
-            a
-        });
+    fn bench_fastfield_bitpack_create(b: &mut Bencher) {
+        let data: Vec<_> = get_data();
+        bench_create::<BitpackedFastFieldCodec>(b, &data);
     }
-
     #[bench]
-    fn bench_intfastfield_stride7_vec(b: &mut Bencher) {
-        let permutation = generate_permutation();
-        let n = permutation.len();
-        b.iter(|| {
-            let mut a = 0u64;
-            for i in (0..n / 7).map(|val| val * 7) {
-                a += permutation[i as usize];
-            }
-            a
-        });
+    fn bench_fastfield_linearinterpol_create(b: &mut Bencher) {
+        let data: Vec<_> = get_data();
+        bench_create::<LinearInterpolCodec>(b, &data);
     }
-
     #[bench]
-    fn bench_intfastfield_stride7_fflookup(b: &mut Bencher) {
-        let permutation = generate_permutation();
-        let n = permutation.len();
-        let column: Arc<dyn Column<u64>> = serialize_and_load(&permutation);
-        b.iter(|| {
-            let mut a = 0u64;
-            for i in (0..n / 7).map(|val| val * 7) {
-                a += column.get_val(i as u64);
-            }
-            a
-        });
+    fn bench_fastfield_multilinearinterpol_create(b: &mut Bencher) {
+        let data: Vec<_> = get_data();
+        bench_create::<MultiLinearInterpolFastFieldCodec>(b, &data);
     }
-
     #[bench]
-    fn bench_intfastfield_scan_all_fflookup(b: &mut Bencher) {
-        let permutation = generate_permutation();
-        let n = permutation.len();
-        let column: Arc<dyn Column<u64>> = serialize_and_load(&permutation);
-        b.iter(|| {
-            let mut a = 0u64;
-            for i in 0u64..n as u64 {
-                a += column.get_val(i);
-            }
-            a
-        });
+    fn bench_fastfield_bitpack_get(b: &mut Bencher) {
+        let data: Vec<_> = get_data();
+        bench_get::<BitpackedFastFieldCodec, BitpackedFastFieldReader>(b, &data);
     }
-
     #[bench]
-    fn bench_intfastfield_scan_all_fflookup_gcd(b: &mut Bencher) {
-        let permutation = generate_permutation_gcd();
-        let n = permutation.len();
-        let column: Arc<dyn Column<u64>> = serialize_and_load(&permutation);
-        b.iter(|| {
-            let mut a = 0u64;
-            for i in 0..n as u64 {
-                a += column.get_val(i);
-            }
-            a
-        });
+    fn bench_fastfield_linearinterpol_get(b: &mut Bencher) {
+        let data: Vec<_> = get_data();
+        bench_get::<LinearInterpolCodec, LinearInterpolFastFieldReader>(b, &data);
     }
-
     #[bench]
-    fn bench_intfastfield_scan_all_vec(b: &mut Bencher) {
-        let permutation = generate_permutation();
-        b.iter(|| {
-            let mut a = 0u64;
-            for i in 0..permutation.len() {
-                a += permutation[i as usize] as u64;
-            }
-            a
-        });
+    fn bench_fastfield_multilinearinterpol_get(b: &mut Bencher) {
+        let data: Vec<_> = get_data();
+        bench_get::<MultiLinearInterpolFastFieldCodec, MultiLinearInterpolFastFieldReader>(
+            b, &data,
+        );
+    }
+    pub fn stats_from_vec(data: &[u64]) -> FastFieldStats {
+        let min_value = data.iter().cloned().min().unwrap_or(0);
+        let max_value = data.iter().cloned().max().unwrap_or(0);
+        FastFieldStats {
+            min_value,
+            max_value,
+            num_vals: data.len() as u64,
+        }
     }
 }

fastfield_codecs/src/bitpacked.rs

@@ -1,100 +1,156 @@
 use std::io::{self, Write};
 
+use common::BinarySerializable;
 use ownedbytes::OwnedBytes;
 use tantivy_bitpacker::{compute_num_bits, BitPacker, BitUnpacker};
 
-use crate::column::EstimateColumn;
-use crate::serialize::NormalizedHeader;
-use crate::{Column, FastFieldCodec, FastFieldCodecType};
+use crate::{FastFieldCodec, FastFieldCodecReader, FastFieldStats};
 
 /// Depending on the field type, a different
 /// fast field is required.
 #[derive(Clone)]
-pub struct BitpackedReader {
+pub struct BitpackedFastFieldReader {
     data: OwnedBytes,
     bit_unpacker: BitUnpacker,
-    normalized_header: NormalizedHeader,
+    pub min_value_u64: u64,
+    pub max_value_u64: u64,
 }
 
-impl Column for BitpackedReader {
+impl FastFieldCodecReader for BitpackedFastFieldReader {
     #[inline]
-    fn get_val(&self, doc: u64) -> u64 {
-        self.bit_unpacker.get(doc, &self.data)
+    fn get_u64(&self, doc: u64) -> u64 {
+        self.min_value_u64 + self.bit_unpacker.get(doc, &self.data)
     }
     #[inline]
     fn min_value(&self) -> u64 {
-        // The BitpackedReader assumes a normalized vector.
-        0
+        self.min_value_u64
     }
     #[inline]
     fn max_value(&self) -> u64 {
-        self.normalized_header.max_value
+        self.max_value_u64
     }
+}
+pub struct BitpackedFastFieldSerializerLegacy<'a, W: 'a + Write> {
+    bit_packer: BitPacker,
+    write: &'a mut W,
+    min_value: u64,
+    amplitude: u64,
+    num_bits: u8,
+}
+
+impl<'a, W: Write> BitpackedFastFieldSerializerLegacy<'a, W> {
+    /// Creates a new fast field serializer.
+    ///
+    /// The serializer in fact encode the values by bitpacking
+    /// `(val - min_value)`.
+    ///
+    /// It requires a `min_value` and a `max_value` to compute
+    /// compute the minimum number of bits required to encode
+    /// values.
+    pub fn open(
+        write: &'a mut W,
+        min_value: u64,
+        max_value: u64,
+    ) -> io::Result<BitpackedFastFieldSerializerLegacy<'a, W>> {
+        assert!(min_value <= max_value);
+        let amplitude = max_value - min_value;
+        let num_bits = compute_num_bits(amplitude);
+        let bit_packer = BitPacker::new();
+        Ok(BitpackedFastFieldSerializerLegacy {
+            bit_packer,
+            write,
+            min_value,
+            amplitude,
+            num_bits,
+        })
+    }
+    /// Pushes a new value to the currently open u64 fast field.
     #[inline]
-    fn num_vals(&self) -> u64 {
-        self.normalized_header.num_vals
+    pub fn add_val(&mut self, val: u64) -> io::Result<()> {
+        let val_to_write: u64 = val - self.min_value;
+        self.bit_packer
+            .write(val_to_write, self.num_bits, &mut self.write)?;
+        Ok(())
+    }
+    pub fn close_field(mut self) -> io::Result<()> {
+        self.bit_packer.close(&mut self.write)?;
+        self.min_value.serialize(&mut self.write)?;
+        self.amplitude.serialize(&mut self.write)?;
+        Ok(())
     }
 }
 
-pub struct BitpackedCodec;
+pub struct BitpackedFastFieldCodec;
 
-impl FastFieldCodec for BitpackedCodec {
-    /// The CODEC_TYPE is an enum value used for serialization.
-    const CODEC_TYPE: FastFieldCodecType = FastFieldCodecType::Bitpacked;
+impl FastFieldCodec for BitpackedFastFieldCodec {
+    const NAME: &'static str = "Bitpacked";
 
-    type Reader = BitpackedReader;
+    type Reader = BitpackedFastFieldReader;
 
     /// Opens a fast field given a file.
-    fn open_from_bytes(
-        data: OwnedBytes,
-        normalized_header: NormalizedHeader,
-    ) -> io::Result<Self::Reader> {
-        let num_bits = compute_num_bits(normalized_header.max_value);
+    fn open_from_bytes(bytes: OwnedBytes) -> io::Result<Self::Reader> {
+        let footer_offset = bytes.len() - 16;
+        let (data, mut footer) = bytes.split(footer_offset);
+        let min_value = u64::deserialize(&mut footer)?;
+        let amplitude = u64::deserialize(&mut footer)?;
+        let max_value = min_value + amplitude;
+        let num_bits = compute_num_bits(amplitude);
         let bit_unpacker = BitUnpacker::new(num_bits);
-        Ok(BitpackedReader {
+        Ok(BitpackedFastFieldReader {
             data,
+            min_value_u64: min_value,
+            max_value_u64: max_value,
             bit_unpacker,
-            normalized_header,
         })
     }
 
     /// Serializes data with the BitpackedFastFieldSerializer.
     ///
-    /// The bitpacker assumes that the column has been normalized.
-    /// i.e. It has already been shifted by its minimum value, so that its
-    /// current minimum value is 0.
+    /// The serializer in fact encode the values by bitpacking
+    /// `(val - min_value)`.
     ///
-    /// Ideally, we made a shift upstream on the column so that `col.min_value() == 0`.
-    fn serialize(column: &dyn Column, write: &mut impl Write) -> io::Result<()> {
-        assert_eq!(column.min_value(), 0u64);
-        let num_bits = compute_num_bits(column.max_value());
-        let mut bit_packer = BitPacker::new();
-        for val in column.iter() {
-            bit_packer.write(val, num_bits, write)?;
+    /// It requires a `min_value` and a `max_value` to compute
+    /// compute the minimum number of bits required to encode
+    /// values.
+    fn serialize(
+        &self,
+        write: &mut impl io::Write,
+        vals: &[u64],
+        stats: FastFieldStats,
+    ) -> io::Result<()> {
+        let mut serializer =
+            BitpackedFastFieldSerializerLegacy::open(write, stats.min_value, stats.max_value)?;
+
+        for &val in vals {
+            serializer.add_val(val)?;
         }
-        bit_packer.close(write)?;
+        serializer.close_field()?;
+
         Ok(())
     }
-
-    fn estimate(column: &EstimateColumn) -> Option<f32> {
-        let num_bits = compute_num_bits(column.max_value());
+    fn is_applicable(_vals: &[u64], _stats: FastFieldStats) -> bool {
+        true
+    }
+    fn estimate(_vals: &[u64], stats: FastFieldStats) -> f32 {
+        let amplitude = stats.max_value - stats.min_value;
+        let num_bits = compute_num_bits(amplitude);
         let num_bits_uncompressed = 64;
-        Some(num_bits as f32 / num_bits_uncompressed as f32)
+        num_bits as f32 / num_bits_uncompressed as f32
     }
 }
 
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::tests::get_codec_test_datasets;
+    use crate::tests::get_codec_test_data_sets;
 
     fn create_and_validate(data: &[u64], name: &str) {
-        crate::tests::create_and_validate::<BitpackedCodec>(data, name);
+        crate::tests::create_and_validate(&BitpackedFastFieldCodec, data, name);
     }
 
     #[test]
     fn test_with_codec_data_sets() {
-        let data_sets = get_codec_test_datasets();
+        let data_sets = get_codec_test_data_sets();
         for (mut data, name) in data_sets {
             create_and_validate(&data, name);
             data.reverse();

fastfield_codecs/src/blockwise_linear.rs

@@ -1,187 +0,0 @@
-use std::sync::Arc;
-use std::{io, iter};
-
-use common::{BinarySerializable, CountingWriter, DeserializeFrom};
-use ownedbytes::OwnedBytes;
-use tantivy_bitpacker::{compute_num_bits, BitPacker, BitUnpacker};
-
-use crate::column::EstimateColumn;
-use crate::line::Line;
-use crate::serialize::NormalizedHeader;
-use crate::{Column, FastFieldCodec, FastFieldCodecType, VecColumn};
-
-const CHUNK_SIZE: usize = 512;
-
-#[derive(Debug, Default)]
-struct Block {
-    line: Line,
-    bit_unpacker: BitUnpacker,
-    data_start_offset: usize,
-}
-
-impl BinarySerializable for Block {
-    fn serialize<W: io::Write>(&self, writer: &mut W) -> io::Result<()> {
-        self.line.serialize(writer)?;
-        self.bit_unpacker.bit_width().serialize(writer)?;
-        Ok(())
-    }
-
-    fn deserialize<R: io::Read>(reader: &mut R) -> io::Result<Self> {
-        let line = Line::deserialize(reader)?;
-        let bit_width = u8::deserialize(reader)?;
-        Ok(Block {
-            line,
-            bit_unpacker: BitUnpacker::new(bit_width),
-            data_start_offset: 0,
-        })
-    }
-}
-
-fn compute_num_blocks(num_vals: u64) -> usize {
-    (num_vals as usize + CHUNK_SIZE - 1) / CHUNK_SIZE
-}
-
-pub struct BlockwiseLinearCodec;
-
-impl FastFieldCodec for BlockwiseLinearCodec {
-    const CODEC_TYPE: crate::FastFieldCodecType = FastFieldCodecType::BlockwiseLinear;
-    type Reader = BlockwiseLinearReader;
-
-    fn open_from_bytes(
-        bytes: ownedbytes::OwnedBytes,
-        normalized_header: NormalizedHeader,
-    ) -> io::Result<Self::Reader> {
-        let footer_len: u32 = (&bytes[bytes.len() - 4..]).deserialize()?;
-        let footer_offset = bytes.len() - 4 - footer_len as usize;
-        let (data, mut footer) = bytes.split(footer_offset);
-        let num_blocks = compute_num_blocks(normalized_header.num_vals);
-        let mut blocks: Vec<Block> = iter::repeat_with(|| Block::deserialize(&mut footer))
-            .take(num_blocks)
-            .collect::<io::Result<_>>()?;
-
-        let mut start_offset = 0;
-        for block in &mut blocks {
-            block.data_start_offset = start_offset;
-            start_offset += (block.bit_unpacker.bit_width() as usize) * CHUNK_SIZE / 8;
-        }
-        Ok(BlockwiseLinearReader {
-            blocks: Arc::new(blocks),
-            data,
-            normalized_header,
-        })
-    }
-
-    // Estimate first_chunk and extrapolate
-    fn estimate(column: &EstimateColumn) -> Option<f32> {
-        if column.num_vals() < 10 * CHUNK_SIZE as u64 {
-            return None;
-        }
-        let mut first_chunk: Vec<u64> = column.iter().take(CHUNK_SIZE as usize).collect();
-        let line = Line::train(&VecColumn::from(&first_chunk));
-        for (i, buffer_val) in first_chunk.iter_mut().enumerate() {
-            let interpolated_val = line.eval(i as u64);
-            *buffer_val = buffer_val.wrapping_sub(interpolated_val);
-        }
-        let estimated_bit_width = first_chunk
-            .iter()
-            .map(|el| ((el + 1) as f32 * 3.0) as u64)
-            .map(compute_num_bits)
-            .max()
-            .unwrap();
-
-        let metadata_per_block = {
-            let mut out = vec![];
-            Block::default().serialize(&mut out).unwrap();
-            out.len()
-        };
-        let num_bits = estimated_bit_width as u64 * column.num_vals() as u64
-            // function metadata per block
-            + metadata_per_block as u64 * (column.num_vals() / CHUNK_SIZE as u64);
-        let num_bits_uncompressed = 64 * column.num_vals();
-        Some(num_bits as f32 / num_bits_uncompressed as f32)
-    }
-
-    fn serialize(column: &dyn Column, wrt: &mut impl io::Write) -> io::Result<()> {
-        // The BitpackedReader assumes a normalized vector.
-        assert_eq!(column.min_value(), 0);
-        let mut buffer = Vec::with_capacity(CHUNK_SIZE);
-        let num_vals = column.num_vals();
-
-        let num_blocks = compute_num_blocks(num_vals);
-        let mut blocks = Vec::with_capacity(num_blocks);
-
-        let mut vals = column.iter();
-
-        let mut bit_packer = BitPacker::new();
-
-        for _ in 0..num_blocks {
-            buffer.clear();
-            buffer.extend((&mut vals).take(CHUNK_SIZE));
-            let line = Line::train(&VecColumn::from(&buffer));
-
-            assert!(!buffer.is_empty());
-
-            for (i, buffer_val) in buffer.iter_mut().enumerate() {
-                let interpolated_val = line.eval(i as u64);
-                *buffer_val = buffer_val.wrapping_sub(interpolated_val);
-            }
-            let bit_width = buffer.iter().copied().map(compute_num_bits).max().unwrap();
-
-            for &buffer_val in &buffer {
-                bit_packer.write(buffer_val, bit_width, wrt)?;
-            }
-
-            blocks.push(Block {
-                line,
-                bit_unpacker: BitUnpacker::new(bit_width),
-                data_start_offset: 0,
-            });
-        }
-
-        bit_packer.close(wrt)?;
-
-        assert_eq!(blocks.len(), compute_num_blocks(num_vals));
-
-        let mut counting_wrt = CountingWriter::wrap(wrt);
-        for block in &blocks {
-            block.serialize(&mut counting_wrt)?;
-        }
-        let footer_len = counting_wrt.written_bytes();
-        (footer_len as u32).serialize(&mut counting_wrt)?;
-
-        Ok(())
-    }
-}
-
-#[derive(Clone)]
-pub struct BlockwiseLinearReader {
-    blocks: Arc<Vec<Block>>,
-    normalized_header: NormalizedHeader,
-    data: OwnedBytes,
-}
-
-impl Column for BlockwiseLinearReader {
-    #[inline(always)]
-    fn get_val(&self, idx: u64) -> u64 {
-        let block_id = (idx / CHUNK_SIZE as u64) as usize;
-        let idx_within_block = idx % (CHUNK_SIZE as u64);
-        let block = &self.blocks[block_id];
-        let interpoled_val: u64 = block.line.eval(idx_within_block);
-        let block_bytes = &self.data[block.data_start_offset..];
-        let bitpacked_diff = block.bit_unpacker.get(idx_within_block, block_bytes);
-        interpoled_val.wrapping_add(bitpacked_diff)
-    }
-
-    fn min_value(&self) -> u64 {
-        // The BlockwiseLinearReader assumes a normalized vector.
-        0u64
-    }
-
-    fn max_value(&self) -> u64 {
-        self.normalized_header.max_value
-    }
-
-    fn num_vals(&self) -> u64 {
-        self.normalized_header.num_vals
-    }
-}

fastfield_codecs/src/column.rs

@@ -1,341 +0,0 @@
-use std::marker::PhantomData;
-use std::ops::RangeInclusive;
-
-use tantivy_bitpacker::minmax;
-
-pub trait Column<T: PartialOrd = u64>: Send + Sync {
-    /// Return the value associated to the given idx.
-    ///
-    /// This accessor should return as fast as possible.
-    ///
-    /// # Panics
-    ///
-    /// May panic if `idx` is greater than the column length.
-    fn get_val(&self, idx: u64) -> T;
-
-    /// Fills an output buffer with the fast field values
-    /// associated with the `DocId` going from
-    /// `start` to `start + output.len()`.
-    ///
-    /// # Panics
-    ///
-    /// Must panic if `start + output.len()` is greater than
-    /// the segment's `maxdoc`.
-    #[inline]
-    fn get_range(&self, start: u64, output: &mut [T]) {
-        for (out, idx) in output.iter_mut().zip(start..) {
-            *out = self.get_val(idx);
-        }
-    }
-
-    /// Return the positions of values which are in the provided range.
-    #[inline]
-    fn get_between_vals(&self, range: RangeInclusive<T>) -> Vec<u64> {
-        let mut vals = Vec::new();
-        for idx in 0..self.num_vals() {
-            let val = self.get_val(idx);
-            if range.contains(&val) {
-                vals.push(idx);
-            }
-        }
-        vals
-    }
-
-    /// Returns the minimum value for this fast field.
-    ///
-    /// This min_value may not be exact.
-    /// For instance, the min value does not take in account of possible
-    /// deleted document. All values are however guaranteed to be higher than
-    /// `.min_value()`.
-    fn min_value(&self) -> T;
-
-    /// Returns the maximum value for this fast field.
-    ///
-    /// This max_value may not be exact.
-    /// For instance, the max value does not take in account of possible
-    /// deleted document. All values are however guaranteed to be higher than
-    /// `.max_value()`.
-    fn max_value(&self) -> T;
-
-    fn num_vals(&self) -> u64;
-
-    /// Returns a iterator over the data
-    fn iter<'a>(&'a self) -> Box<dyn Iterator<Item = T> + 'a> {
-        Box::new((0..self.num_vals()).map(|idx| self.get_val(idx)))
-    }
-}
-
-pub struct VecColumn<'a, T = u64> {
-    values: &'a [T],
-    min_value: T,
-    max_value: T,
-}
-
-impl<'a, C: Column<T>, T: Copy + PartialOrd> Column<T> for &'a C {
-    fn get_val(&self, idx: u64) -> T {
-        (*self).get_val(idx)
-    }
-
-    fn min_value(&self) -> T {
-        (*self).min_value()
-    }
-
-    fn max_value(&self) -> T {
-        (*self).max_value()
-    }
-
-    fn num_vals(&self) -> u64 {
-        (*self).num_vals()
-    }
-
-    fn iter<'b>(&'b self) -> Box<dyn Iterator<Item = T> + 'b> {
-        (*self).iter()
-    }
-
-    fn get_range(&self, start: u64, output: &mut [T]) {
-        (*self).get_range(start, output)
-    }
-}
-
-impl<'a, T: Copy + PartialOrd + Send + Sync> Column<T> for VecColumn<'a, T> {
-    fn get_val(&self, position: u64) -> T {
-        self.values[position as usize]
-    }
-
-    fn iter(&self) -> Box<dyn Iterator<Item = T> + '_> {
-        Box::new(self.values.iter().copied())
-    }
-
-    fn min_value(&self) -> T {
-        self.min_value
-    }
-
-    fn max_value(&self) -> T {
-        self.max_value
-    }
-
-    fn num_vals(&self) -> u64 {
-        self.values.len() as u64
-    }
-
-    fn get_range(&self, start: u64, output: &mut [T]) {
-        output.copy_from_slice(&self.values[start as usize..][..output.len()])
-    }
-}
-
-impl<'a, T: Copy + Ord + Default, V> From<&'a V> for VecColumn<'a, T>
-where V: AsRef<[T]> + ?Sized
-{
-    fn from(values: &'a V) -> Self {
-        let values = values.as_ref();
-        let (min_value, max_value) = minmax(values.iter().copied()).unwrap_or_default();
-        Self {
-            values,
-            min_value,
-            max_value,
-        }
-    }
-}
-
-// Creates a view over a Column with a limited number of vals. Stats like min max are unchanged
-pub struct EstimateColumn<'a> {
-    column: &'a dyn Column,
-    num_vals: u64,
-}
-impl<'a> EstimateColumn<'a> {
-    pub(crate) fn new(column: &'a dyn Column) -> Self {
-        let limit_num_vals = column.num_vals().min(100_000);
-        Self {
-            column,
-            num_vals: limit_num_vals,
-        }
-    }
-}
-
-impl<'a> Column for EstimateColumn<'a> {
-    fn get_val(&self, idx: u64) -> u64 {
-        (*self.column).get_val(idx)
-    }
-
-    fn min_value(&self) -> u64 {
-        (*self.column).min_value()
-    }
-
-    fn max_value(&self) -> u64 {
-        (*self.column).max_value()
-    }
-
-    fn num_vals(&self) -> u64 {
-        self.num_vals
-    }
-
-    fn iter<'b>(&'b self) -> Box<dyn Iterator<Item = u64> + 'b> {
-        Box::new((*self.column).iter().take(self.num_vals as usize))
-    }
-
-    fn get_range(&self, start: u64, output: &mut [u64]) {
-        (*self.column).get_range(start, output)
-    }
-}
-
-impl<'a> From<&'a dyn Column> for EstimateColumn<'a> {
-    fn from(column: &'a dyn Column) -> Self {
-        let limit_num_vals = column.num_vals().min(100_000);
-        Self {
-            column,
-            num_vals: limit_num_vals,
-        }
-    }
-}
-
-struct MonotonicMappingColumn<C, T, Input> {
-    from_column: C,
-    monotonic_mapping: T,
-    _phantom: PhantomData<Input>,
-}
-
-/// Creates a view of a column transformed by a monotonic mapping.
-pub fn monotonic_map_column<C, T, Input: PartialOrd, Output: PartialOrd>(
-    from_column: C,
-    monotonic_mapping: T,
-) -> impl Column<Output>
-where
-    C: Column<Input>,
-    T: Fn(Input) -> Output + Send + Sync,
-    Input: Send + Sync,
-    Output: Send + Sync,
-{
-    MonotonicMappingColumn {
-        from_column,
-        monotonic_mapping,
-        _phantom: PhantomData,
-    }
-}
-
-impl<C, T, Input: PartialOrd, Output: PartialOrd> Column<Output>
-    for MonotonicMappingColumn<C, T, Input>
-where
-    C: Column<Input>,
-    T: Fn(Input) -> Output + Send + Sync,
-    Input: Send + Sync,
-    Output: Send + Sync,
-{
-    #[inline]
-    fn get_val(&self, idx: u64) -> Output {
-        let from_val = self.from_column.get_val(idx);
-        (self.monotonic_mapping)(from_val)
-    }
-
-    fn min_value(&self) -> Output {
-        let from_min_value = self.from_column.min_value();
-        (self.monotonic_mapping)(from_min_value)
-    }
-
-    fn max_value(&self) -> Output {
-        let from_max_value = self.from_column.max_value();
-        (self.monotonic_mapping)(from_max_value)
-    }
-
-    fn num_vals(&self) -> u64 {
-        self.from_column.num_vals()
-    }
-
-    fn iter(&self) -> Box<dyn Iterator<Item = Output> + '_> {
-        Box::new(self.from_column.iter().map(&self.monotonic_mapping))
-    }
-
-    // We voluntarily do not implement get_range as it yields a regression,
-    // and we do not have any specialized implementation anyway.
-}
-
-pub struct IterColumn<T>(T);
-
-impl<T> From<T> for IterColumn<T>
-where T: Iterator + Clone + ExactSizeIterator
-{
-    fn from(iter: T) -> Self {
-        IterColumn(iter)
-    }
-}
-
-impl<T> Column<T::Item> for IterColumn<T>
-where
-    T: Iterator + Clone + ExactSizeIterator + Send + Sync,
-    T::Item: PartialOrd,
-{
-    fn get_val(&self, idx: u64) -> T::Item {
-        self.0.clone().nth(idx as usize).unwrap()
-    }
-
-    fn min_value(&self) -> T::Item {
-        self.0.clone().next().unwrap()
-    }
-
-    fn max_value(&self) -> T::Item {
-        self.0.clone().last().unwrap()
-    }
-
-    fn num_vals(&self) -> u64 {
-        self.0.len() as u64
-    }
-
-    fn iter(&self) -> Box<dyn Iterator<Item = T::Item> + '_> {
-        Box::new(self.0.clone())
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use crate::MonotonicallyMappableToU64;
-
-    #[test]
-    fn test_monotonic_mapping() {
-        let vals = &[1u64, 3u64][..];
-        let col = VecColumn::from(vals);
-        let mapped = monotonic_map_column(col, |el| el + 4);
-        assert_eq!(mapped.min_value(), 5u64);
-        assert_eq!(mapped.max_value(), 7u64);
-        assert_eq!(mapped.num_vals(), 2);
-        assert_eq!(mapped.num_vals(), 2);
-        assert_eq!(mapped.get_val(0), 5);
-        assert_eq!(mapped.get_val(1), 7);
-    }
-
-    #[test]
-    fn test_range_as_col() {
-        let col = IterColumn::from(10..100);
-        assert_eq!(col.num_vals(), 90);
-        assert_eq!(col.max_value(), 99);
-    }
-
-    #[test]
-    fn test_monotonic_mapping_iter() {
-        let vals: Vec<u64> = (-1..99).map(i64::to_u64).collect();
-        let col = VecColumn::from(&vals);
-        let mapped = monotonic_map_column(col, |el| i64::from_u64(el) * 10i64);
-        let val_i64s: Vec<i64> = mapped.iter().collect();
-        for i in 0..100 {
-            assert_eq!(val_i64s[i as usize], mapped.get_val(i));
-        }
-    }
-
-    #[test]
-    fn test_monotonic_mapping_get_range() {
-        let vals: Vec<u64> = (-1..99).map(i64::to_u64).collect();
-        let col = VecColumn::from(&vals);
-        let mapped = monotonic_map_column(col, |el| i64::from_u64(el) * 10i64);
-        assert_eq!(mapped.min_value(), -10i64);
-        assert_eq!(mapped.max_value(), 980i64);
-        assert_eq!(mapped.num_vals(), 100);
-        let val_i64s: Vec<i64> = mapped.iter().collect();
-        assert_eq!(val_i64s.len(), 100);
-        for i in 0..100 {
-            assert_eq!(val_i64s[i as usize], mapped.get_val(i));
-            assert_eq!(val_i64s[i as usize], i64::from_u64(vals[i as usize]) * 10);
-        }
-        let mut buf = [0i64; 20];
-        mapped.get_range(7, &mut buf[..]);
-        assert_eq!(&val_i64s[7..][..20], &buf);
-    }
-}

fastfield_codecs/src/compact_space/blank_range.rs

@@ -1,43 +0,0 @@
-use std::ops::RangeInclusive;
-
-/// The range of a blank in value space.
-///
-/// A blank is an unoccupied space in the data.
-/// Use try_into() to construct.
-/// A range has to have at least length of 3. Invalid ranges will be rejected.
-///
-/// Ordered by range length.
-#[derive(Debug, Eq, PartialEq, Clone)]
-pub(crate) struct BlankRange {
-    blank_range: RangeInclusive<u128>,
-}
-impl TryFrom<RangeInclusive<u128>> for BlankRange {
-    type Error = &'static str;
-    fn try_from(range: RangeInclusive<u128>) -> Result<Self, Self::Error> {
-        let blank_size = range.end().saturating_sub(*range.start());
-        if blank_size < 2 {
-            Err("invalid range")
-        } else {
-            Ok(BlankRange { blank_range: range })
-        }
-    }
-}
-impl BlankRange {
-    pub(crate) fn blank_size(&self) -> u128 {
-        self.blank_range.end() - self.blank_range.start() + 1
-    }
-    pub(crate) fn blank_range(&self) -> RangeInclusive<u128> {
-        self.blank_range.clone()
-    }
-}
-
-impl Ord for BlankRange {
-    fn cmp(&self, other: &Self) -> std::cmp::Ordering {
-        self.blank_size().cmp(&other.blank_size())
-    }
-}
-impl PartialOrd for BlankRange {
-    fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
-        Some(self.blank_size().cmp(&other.blank_size()))
-    }
-}

fastfield_codecs/src/compact_space/build_compact_space.rs

@@ -1,231 +0,0 @@
-use std::collections::{BTreeSet, BinaryHeap};
-use std::iter;
-use std::ops::RangeInclusive;
-
-use itertools::Itertools;
-
-use super::blank_range::BlankRange;
-use super::{CompactSpace, RangeMapping};
-
-/// Put the blanks for the sorted values into a binary heap
-fn get_blanks(values_sorted: &BTreeSet<u128>) -> BinaryHeap<BlankRange> {
-    let mut blanks: BinaryHeap<BlankRange> = BinaryHeap::new();
-    for (first, second) in values_sorted.iter().tuple_windows() {
-        // Correctness Overflow: the values are deduped and sorted (BTreeSet property), that means
-        // there's always space between two values.
-        let blank_range = first + 1..=second - 1;
-        let blank_range: Result<BlankRange, _> = blank_range.try_into();
-        if let Ok(blank_range) = blank_range {
-            blanks.push(blank_range);
-        }
-    }
-
-    blanks
-}
-
-struct BlankCollector {
-    blanks: Vec<BlankRange>,
-    staged_blanks_sum: u128,
-}
-impl BlankCollector {
-    fn new() -> Self {
-        Self {
-            blanks: vec![],
-            staged_blanks_sum: 0,
-        }
-    }
-    fn stage_blank(&mut self, blank: BlankRange) {
-        self.staged_blanks_sum += blank.blank_size();
-        self.blanks.push(blank);
-    }
-    fn drain(&mut self) -> impl Iterator<Item = BlankRange> + '_ {
-        self.staged_blanks_sum = 0;
-        self.blanks.drain(..)
-    }
-    fn staged_blanks_sum(&self) -> u128 {
-        self.staged_blanks_sum
-    }
-    fn num_staged_blanks(&self) -> usize {
-        self.blanks.len()
-    }
-}
-fn num_bits(val: u128) -> u8 {
-    (128u32 - val.leading_zeros()) as u8
-}
-
-/// Will collect blanks and add them to compact space if more bits are saved than cost from
-/// metadata.
-pub fn get_compact_space(
-    values_deduped_sorted: &BTreeSet<u128>,
-    total_num_values: u64,
-    cost_per_blank: usize,
-) -> CompactSpace {
-    let mut compact_space_builder = CompactSpaceBuilder::new();
-    if values_deduped_sorted.is_empty() {
-        return compact_space_builder.finish();
-    }
-
-    let mut blanks: BinaryHeap<BlankRange> = get_blanks(values_deduped_sorted);
-    // Replace after stabilization of https://github.com/rust-lang/rust/issues/62924
-
-    // We start by space that's limited to min_value..=max_value
-    let min_value = *values_deduped_sorted.iter().next().unwrap_or(&0);
-    let max_value = *values_deduped_sorted.iter().last().unwrap_or(&0);
-
-    // +1 for null, in case min and max covers the whole space, we are off by one.
-    let mut amplitude_compact_space = (max_value - min_value).saturating_add(1);
-    if min_value != 0 {
-        compact_space_builder.add_blanks(iter::once(0..=min_value - 1));
-    }
-    if max_value != u128::MAX {
-        compact_space_builder.add_blanks(iter::once(max_value + 1..=u128::MAX));
-    }
-
-    let mut amplitude_bits: u8 = num_bits(amplitude_compact_space);
-
-    let mut blank_collector = BlankCollector::new();
-    // We will stage blanks until they reduce the compact space by at least 1 bit and then flush
-    // them if the metadata cost is lower than the total number of saved bits.
-    // Binary heap to process the gaps by their size
-    while let Some(blank_range) = blanks.pop() {
-        blank_collector.stage_blank(blank_range);
-
-        let staged_spaces_sum: u128 = blank_collector.staged_blanks_sum();
-        let amplitude_new_compact_space = amplitude_compact_space - staged_spaces_sum;
-        let amplitude_new_bits = num_bits(amplitude_new_compact_space);
-        if amplitude_bits == amplitude_new_bits {
-            continue;
-        }
-        let saved_bits = (amplitude_bits - amplitude_new_bits) as usize * total_num_values as usize;
-        // TODO: Maybe calculate exact cost of blanks and run this more expensive computation only,
-        // when amplitude_new_bits changes
-        let cost = blank_collector.num_staged_blanks() * cost_per_blank;
-        if cost >= saved_bits {
-            // Continue here, since although we walk over the blanks by size,
-            // we can potentially save a lot at the last bits, which are smaller blanks
-            //
-            // E.g. if the first range reduces the compact space by 1000 from 2000 to 1000, which
-            // saves 11-10=1 bit and the next range reduces the compact space by 950 to
-            // 50, which saves 10-6=4 bit
-            continue;
-        }
-
-        amplitude_compact_space = amplitude_new_compact_space;
-        amplitude_bits = amplitude_new_bits;
-        compact_space_builder.add_blanks(blank_collector.drain().map(|blank| blank.blank_range()));
-    }
-
-    // special case, when we don't collected any blanks because:
-    // * the data is empty (early exit)
-    // * the algorithm did decide it's not worth the cost, which can be the case for single values
-    //
-    // We drain one collected blank unconditionally, so the empty case is reserved for empty
-    // data, and therefore empty compact_space means the data is empty and no data is covered
-    // (conversely to all data) and we can assign null to it.
-    if compact_space_builder.is_empty() {
-        compact_space_builder.add_blanks(
-            blank_collector
-                .drain()
-                .map(|blank| blank.blank_range())
-                .take(1),
-        );
-    }
-
-    let compact_space = compact_space_builder.finish();
-    if max_value - min_value != u128::MAX {
-        debug_assert_eq!(
-            compact_space.amplitude_compact_space(),
-            amplitude_compact_space
-        );
-    }
-    compact_space
-}
-
-#[derive(Debug, Clone, Eq, PartialEq)]
-struct CompactSpaceBuilder {
-    blanks: Vec<RangeInclusive<u128>>,
-}
-
-impl CompactSpaceBuilder {
-    /// Creates a new compact space builder which will initially cover the whole space.
-    fn new() -> Self {
-        Self { blanks: Vec::new() }
-    }
-
-    /// Assumes that repeated add_blank calls don't overlap and are not adjacent,
-    /// e.g. [3..=5, 5..=10] is not allowed
-    ///
-    /// Both of those assumptions are true when blanks are produced from sorted values.
-    fn add_blanks(&mut self, blank: impl Iterator<Item = RangeInclusive<u128>>) {
-        self.blanks.extend(blank);
-    }
-
-    fn is_empty(&self) -> bool {
-        self.blanks.is_empty()
-    }
-
-    /// Convert blanks to covered space and assign null value
-    fn finish(mut self) -> CompactSpace {
-        // sort by start. ranges are not allowed to overlap
-        self.blanks.sort_unstable_by_key(|blank| *blank.start());
-
-        let mut covered_space = Vec::with_capacity(self.blanks.len());
-
-        // begining of the blanks
-        if let Some(first_blank_start) = self.blanks.first().map(RangeInclusive::start) {
-            if *first_blank_start != 0 {
-                covered_space.push(0..=first_blank_start - 1);
-            }
-        }
-
-        // Between the blanks
-        let between_blanks = self.blanks.iter().tuple_windows().map(|(left, right)| {
-            assert!(
-                left.end() < right.start(),
-                "overlapping or adjacent ranges detected"
-            );
-            *left.end() + 1..=*right.start() - 1
-        });
-        covered_space.extend(between_blanks);
-
-        // end of the blanks
-        if let Some(last_blank_end) = self.blanks.last().map(RangeInclusive::end) {
-            if *last_blank_end != u128::MAX {
-                covered_space.push(last_blank_end + 1..=u128::MAX);
-            }
-        }
-
-        if covered_space.is_empty() {
-            covered_space.push(0..=0); // empty data case
-        };
-
-        let mut compact_start: u64 = 1; // 0 is reserved for `null`
-        let mut ranges_mapping: Vec<RangeMapping> = Vec::with_capacity(covered_space.len());
-        for cov in covered_space {
-            let range_mapping = super::RangeMapping {
-                value_range: cov,
-                compact_start,
-            };
-            let covered_range_len = range_mapping.range_length();
-            ranges_mapping.push(range_mapping);
-            compact_start += covered_range_len as u64;
-        }
-        // println!("num ranges {}", ranges_mapping.len());
-        CompactSpace { ranges_mapping }
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn test_binary_heap_pop_order() {
-        let mut blanks: BinaryHeap<BlankRange> = BinaryHeap::new();
-        blanks.push((0..=10).try_into().unwrap());
-        blanks.push((100..=200).try_into().unwrap());
-        blanks.push((100..=110).try_into().unwrap());
-        assert_eq!(blanks.pop().unwrap().blank_size(), 101);
-        assert_eq!(blanks.pop().unwrap().blank_size(), 11);
-    }
-}

fastfield_codecs/src/compact_space/mod.rs

@@ -1,666 +0,0 @@
-/// This codec takes a large number space (u128) and reduces it to a compact number space.
-///
-/// It will find spaces in the number range. For example:
-///
-/// 100, 101, 102, 103, 104, 50000, 50001
-/// could be mapped to
-/// 100..104 -> 0..4
-/// 50000..50001 -> 5..6
-///
-/// Compact space 0..=6 requires much less bits than 100..=50001
-///
-/// The codec is created to compress ip addresses, but may be employed in other use cases.
-use std::{
-    cmp::Ordering,
-    collections::BTreeSet,
-    io::{self, Write},
-    ops::RangeInclusive,
-};
-
-use common::{BinarySerializable, CountingWriter, VInt, VIntU128};
-use ownedbytes::OwnedBytes;
-use tantivy_bitpacker::{self, BitPacker, BitUnpacker};
-
-use crate::compact_space::build_compact_space::get_compact_space;
-use crate::Column;
-
-mod blank_range;
-mod build_compact_space;
-
-/// The cost per blank is quite hard actually, since blanks are delta encoded, the actual cost of
-/// blanks depends on the number of blanks.
-///
-/// The number is taken by looking at a real dataset. It is optimized for larger datasets.
-const COST_PER_BLANK_IN_BITS: usize = 36;
-
-#[derive(Debug, Clone, Eq, PartialEq)]
-pub struct CompactSpace {
-    ranges_mapping: Vec<RangeMapping>,
-}
-
-/// Maps the range from the original space to compact_start + range.len()
-#[derive(Debug, Clone, Eq, PartialEq)]
-struct RangeMapping {
-    value_range: RangeInclusive<u128>,
-    compact_start: u64,
-}
-impl RangeMapping {
-    fn range_length(&self) -> u64 {
-        (self.value_range.end() - self.value_range.start()) as u64 + 1
-    }
-
-    // The last value of the compact space in this range
-    fn compact_end(&self) -> u64 {
-        self.compact_start + self.range_length() - 1
-    }
-}
-
-impl BinarySerializable for CompactSpace {
-    fn serialize<W: io::Write>(&self, writer: &mut W) -> io::Result<()> {
-        VInt(self.ranges_mapping.len() as u64).serialize(writer)?;
-
-        let mut prev_value = 0;
-        for value_range in self
-            .ranges_mapping
-            .iter()
-            .map(|range_mapping| &range_mapping.value_range)
-        {
-            let blank_delta_start = value_range.start() - prev_value;
-            VIntU128(blank_delta_start).serialize(writer)?;
-            prev_value = *value_range.start();
-
-            let blank_delta_end = value_range.end() - prev_value;
-            VIntU128(blank_delta_end).serialize(writer)?;
-            prev_value = *value_range.end();
-        }
-
-        Ok(())
-    }
-
-    fn deserialize<R: io::Read>(reader: &mut R) -> io::Result<Self> {
-        let num_ranges = VInt::deserialize(reader)?.0;
-        let mut ranges_mapping: Vec<RangeMapping> = vec![];
-        let mut value = 0u128;
-        let mut compact_start = 1u64; // 0 is reserved for `null`
-        for _ in 0..num_ranges {
-            let blank_delta_start = VIntU128::deserialize(reader)?.0;
-            value += blank_delta_start;
-            let blank_start = value;
-
-            let blank_delta_end = VIntU128::deserialize(reader)?.0;
-            value += blank_delta_end;
-            let blank_end = value;
-
-            let range_mapping = RangeMapping {
-                value_range: blank_start..=blank_end,
-                compact_start,
-            };
-            let range_length = range_mapping.range_length();
-            ranges_mapping.push(range_mapping);
-            compact_start += range_length as u64;
-        }
-
-        Ok(Self { ranges_mapping })
-    }
-}
-
-impl CompactSpace {
-    /// Amplitude is the value range of the compact space including the sentinel value used to
-    /// identify null values. The compact space is 0..=amplitude .
-    ///
-    /// It's only used to verify we don't exceed u64 number space, which would indicate a bug.
-    fn amplitude_compact_space(&self) -> u128 {
-        self.ranges_mapping
-            .last()
-            .map(|last_range| last_range.compact_end() as u128)
-            .unwrap_or(1) // compact space starts at 1, 0 == null
-    }
-
-    fn get_range_mapping(&self, pos: usize) -> &RangeMapping {
-        &self.ranges_mapping[pos]
-    }
-
-    /// Returns either Ok(the value in the compact space) or if it is outside the compact space the
-    /// Err(position where it would be inserted)
-    fn u128_to_compact(&self, value: u128) -> Result<u64, usize> {
-        self.ranges_mapping
-            .binary_search_by(|probe| {
-                let value_range = &probe.value_range;
-                if value < *value_range.start() {
-                    Ordering::Greater
-                } else if value > *value_range.end() {
-                    Ordering::Less
-                } else {
-                    Ordering::Equal
-                }
-            })
-            .map(|pos| {
-                let range_mapping = &self.ranges_mapping[pos];
-                let pos_in_range = (value - range_mapping.value_range.start()) as u64;
-                range_mapping.compact_start + pos_in_range
-            })
-    }
-
-    /// Unpacks a value from compact space u64 to u128 space
-    fn compact_to_u128(&self, compact: u64) -> u128 {
-        let pos = self
-            .ranges_mapping
-            .binary_search_by_key(&compact, |range_mapping| range_mapping.compact_start)
-            // Correctness: Overflow. The first range starts at compact space 0, the error from
-            // binary search can never be 0
-            .map_or_else(|e| e - 1, |v| v);
-
-        let range_mapping = &self.ranges_mapping[pos];
-        let diff = compact - range_mapping.compact_start;
-        range_mapping.value_range.start() + diff as u128
-    }
-}
-
-pub struct CompactSpaceCompressor {
-    params: IPCodecParams,
-}
-#[derive(Debug, Clone)]
-pub struct IPCodecParams {
-    compact_space: CompactSpace,
-    bit_unpacker: BitUnpacker,
-    min_value: u128,
-    max_value: u128,
-    num_vals: u64,
-    num_bits: u8,
-}
-
-impl CompactSpaceCompressor {
-    /// Taking the vals as Vec may cost a lot of memory. It is used to sort the vals.
-    pub fn train_from(column: &impl Column<u128>) -> Self {
-        let mut values_sorted = BTreeSet::new();
-        values_sorted.extend(column.iter());
-        let total_num_values = column.num_vals();
-
-        let compact_space =
-            get_compact_space(&values_sorted, total_num_values, COST_PER_BLANK_IN_BITS);
-        let amplitude_compact_space = compact_space.amplitude_compact_space();
-
-        assert!(
-            amplitude_compact_space <= u64::MAX as u128,
-            "case unsupported."
-        );
-
-        let num_bits = tantivy_bitpacker::compute_num_bits(amplitude_compact_space as u64);
-        let min_value = *values_sorted.iter().next().unwrap_or(&0);
-        let max_value = *values_sorted.iter().last().unwrap_or(&0);
-        assert_eq!(
-            compact_space
-                .u128_to_compact(max_value)
-                .expect("could not convert max value to compact space"),
-            amplitude_compact_space as u64
-        );
-        CompactSpaceCompressor {
-            params: IPCodecParams {
-                compact_space,
-                bit_unpacker: BitUnpacker::new(num_bits),
-                min_value,
-                max_value,
-                num_vals: total_num_values as u64,
-                num_bits,
-            },
-        }
-    }
-
-    fn write_footer(self, writer: &mut impl Write) -> io::Result<()> {
-        let writer = &mut CountingWriter::wrap(writer);
-        self.params.serialize(writer)?;
-
-        let footer_len = writer.written_bytes() as u32;
-        footer_len.serialize(writer)?;
-
-        Ok(())
-    }
-
-    pub fn compress_into(
-        self,
-        vals: impl Iterator<Item = u128>,
-        write: &mut impl Write,
-    ) -> io::Result<()> {
-        let mut bitpacker = BitPacker::default();
-        for val in vals {
-            let compact = self
-                .params
-                .compact_space
-                .u128_to_compact(val)
-                .map_err(|_| {
-                    io::Error::new(
-                        io::ErrorKind::InvalidData,
-                        "Could not convert value to compact_space. This is a bug.",
-                    )
-                })?;
-            bitpacker.write(compact, self.params.num_bits, write)?;
-        }
-        bitpacker.close(write)?;
-        self.write_footer(write)?;
-        Ok(())
-    }
-}
-
-#[derive(Debug, Clone)]
-pub struct CompactSpaceDecompressor {
-    data: OwnedBytes,
-    params: IPCodecParams,
-}
-
-impl BinarySerializable for IPCodecParams {
-    fn serialize<W: io::Write>(&self, writer: &mut W) -> io::Result<()> {
-        // header flags for future optional dictionary encoding
-        let footer_flags = 0u64;
-        footer_flags.serialize(writer)?;
-
-        VIntU128(self.min_value).serialize(writer)?;
-        VIntU128(self.max_value).serialize(writer)?;
-        VIntU128(self.num_vals as u128).serialize(writer)?;
-        self.num_bits.serialize(writer)?;
-
-        self.compact_space.serialize(writer)?;
-
-        Ok(())
-    }
-
-    fn deserialize<R: io::Read>(reader: &mut R) -> io::Result<Self> {
-        let _header_flags = u64::deserialize(reader)?;
-        let min_value = VIntU128::deserialize(reader)?.0;
-        let max_value = VIntU128::deserialize(reader)?.0;
-        let num_vals = VIntU128::deserialize(reader)?.0 as u64;
-        let num_bits = u8::deserialize(reader)?;
-        let compact_space = CompactSpace::deserialize(reader)?;
-
-        Ok(Self {
-            compact_space,
-            bit_unpacker: BitUnpacker::new(num_bits),
-            min_value,
-            max_value,
-            num_vals,
-            num_bits,
-        })
-    }
-}
-
-impl Column<u128> for CompactSpaceDecompressor {
-    #[inline]
-    fn get_val(&self, doc: u64) -> u128 {
-        self.get(doc)
-    }
-
-    fn min_value(&self) -> u128 {
-        self.min_value()
-    }
-
-    fn max_value(&self) -> u128 {
-        self.max_value()
-    }
-
-    fn num_vals(&self) -> u64 {
-        self.params.num_vals
-    }
-
-    #[inline]
-    fn iter(&self) -> Box<dyn Iterator<Item = u128> + '_> {
-        Box::new(self.iter())
-    }
-    fn get_between_vals(&self, range: RangeInclusive<u128>) -> Vec<u64> {
-        self.get_between_vals(range)
-    }
-}
-
-impl CompactSpaceDecompressor {
-    pub fn open(data: OwnedBytes) -> io::Result<CompactSpaceDecompressor> {
-        let (data_slice, footer_len_bytes) = data.split_at(data.len() - 4);
-        let footer_len = u32::deserialize(&mut &footer_len_bytes[..])?;
-
-        let data_footer = &data_slice[data_slice.len() - footer_len as usize..];
-        let params = IPCodecParams::deserialize(&mut &data_footer[..])?;
-        let decompressor = CompactSpaceDecompressor { data, params };
-
-        Ok(decompressor)
-    }
-
-    /// Converting to compact space for the decompressor is more complex, since we may get values
-    /// which are outside the compact space. e.g. if we map
-    /// 1000 => 5
-    /// 2000 => 6
-    ///
-    /// and we want a mapping for 1005, there is no equivalent compact space. We instead return an
-    /// error with the index of the next range.
-    fn u128_to_compact(&self, value: u128) -> Result<u64, usize> {
-        self.params.compact_space.u128_to_compact(value)
-    }
-
-    fn compact_to_u128(&self, compact: u64) -> u128 {
-        self.params.compact_space.compact_to_u128(compact)
-    }
-
-    /// Comparing on compact space: Random dataset 0,24 (50% random hit) - 1.05 GElements/s
-    /// Comparing on compact space: Real dataset 1.08 GElements/s
-    ///
-    /// Comparing on original space: Real dataset .06 GElements/s (not completely optimized)
-    pub fn get_between_vals(&self, range: RangeInclusive<u128>) -> Vec<u64> {
-        if range.start() > range.end() {
-            return Vec::new();
-        }
-        let from_value = *range.start();
-        let to_value = *range.end();
-        assert!(to_value >= from_value);
-        let compact_from = self.u128_to_compact(from_value);
-        let compact_to = self.u128_to_compact(to_value);
-
-        // Quick return, if both ranges fall into the same non-mapped space, the range can't cover
-        // any values, so we can early exit
-        match (compact_to, compact_from) {
-            (Err(pos1), Err(pos2)) if pos1 == pos2 => return Vec::new(),
-            _ => {}
-        }
-
-        let compact_from = compact_from.unwrap_or_else(|pos| {
-            // Correctness: Out of bounds, if this value is Err(last_index + 1), we early exit,
-            // since the to_value also mapps into the same non-mapped space
-            let range_mapping = self.params.compact_space.get_range_mapping(pos);
-            range_mapping.compact_start
-        });
-        // If there is no compact space, we go to the closest upperbound compact space
-        let compact_to = compact_to.unwrap_or_else(|pos| {
-            // Correctness: Overflow, if this value is Err(0), we early exit,
-            // since the from_value also mapps into the same non-mapped space
-
-            // Get end of previous range
-            let pos = pos - 1;
-            let range_mapping = self.params.compact_space.get_range_mapping(pos);
-            range_mapping.compact_end()
-        });
-
-        let range = compact_from..=compact_to;
-        let mut positions = Vec::new();
-
-        let step_size = 4;
-        let cutoff = self.params.num_vals - self.params.num_vals % step_size;
-
-        let mut push_if_in_range = |idx, val| {
-            if range.contains(&val) {
-                positions.push(idx);
-            }
-        };
-        let get_val = |idx| self.params.bit_unpacker.get(idx as u64, &self.data);
-        // unrolled loop
-        for idx in (0..cutoff).step_by(step_size as usize) {
-            let idx1 = idx;
-            let idx2 = idx + 1;
-            let idx3 = idx + 2;
-            let idx4 = idx + 3;
-            let val1 = get_val(idx1);
-            let val2 = get_val(idx2);
-            let val3 = get_val(idx3);
-            let val4 = get_val(idx4);
-            push_if_in_range(idx1, val1);
-            push_if_in_range(idx2, val2);
-            push_if_in_range(idx3, val3);
-            push_if_in_range(idx4, val4);
-        }
-
-        // handle rest
-        for idx in cutoff..self.params.num_vals {
-            push_if_in_range(idx, get_val(idx));
-        }
-
-        positions
-    }
-
-    #[inline]
-    fn iter_compact(&self) -> impl Iterator<Item = u64> + '_ {
-        (0..self.params.num_vals)
-            .map(move |idx| self.params.bit_unpacker.get(idx as u64, &self.data) as u64)
-    }
-
-    #[inline]
-    fn iter(&self) -> impl Iterator<Item = u128> + '_ {
-        // TODO: Performance. It would be better to iterate on the ranges and check existence via
-        // the bit_unpacker.
-        self.iter_compact()
-            .map(|compact| self.compact_to_u128(compact))
-    }
-
-    #[inline]
-    pub fn get(&self, idx: u64) -> u128 {
-        let compact = self.params.bit_unpacker.get(idx, &self.data);
-        self.compact_to_u128(compact)
-    }
-
-    pub fn min_value(&self) -> u128 {
-        self.params.min_value
-    }
-
-    pub fn max_value(&self) -> u128 {
-        self.params.max_value
-    }
-}
-
-#[cfg(test)]
-mod tests {
-
-    use super::*;
-    use crate::{open_u128, serialize_u128, VecColumn};
-
-    #[test]
-    fn compact_space_test() {
-        let ips = &[
-            2u128, 4u128, 1000, 1001, 1002, 1003, 1004, 1005, 1008, 1010, 1012, 1260,
-        ]
-        .into_iter()
-        .collect();
-        let compact_space = get_compact_space(ips, ips.len() as u64, 11);
-        let amplitude = compact_space.amplitude_compact_space();
-        assert_eq!(amplitude, 17);
-        assert_eq!(1, compact_space.u128_to_compact(2).unwrap());
-        assert_eq!(2, compact_space.u128_to_compact(3).unwrap());
-        assert_eq!(compact_space.u128_to_compact(100).unwrap_err(), 1);
-
-        for (num1, num2) in (0..3).tuple_windows() {
-            assert_eq!(
-                compact_space.get_range_mapping(num1).compact_end() + 1,
-                compact_space.get_range_mapping(num2).compact_start
-            );
-        }
-
-        let mut output: Vec<u8> = Vec::new();
-        compact_space.serialize(&mut output).unwrap();
-
-        assert_eq!(
-            compact_space,
-            CompactSpace::deserialize(&mut &output[..]).unwrap()
-        );
-
-        for ip in ips {
-            let compact = compact_space.u128_to_compact(*ip).unwrap();
-            assert_eq!(compact_space.compact_to_u128(compact), *ip);
-        }
-    }
-
-    #[test]
-    fn compact_space_amplitude_test() {
-        let ips = &[100000u128, 1000000].into_iter().collect();
-        let compact_space = get_compact_space(ips, ips.len() as u64, 1);
-        let amplitude = compact_space.amplitude_compact_space();
-        assert_eq!(amplitude, 2);
-    }
-
-    fn test_all(data: OwnedBytes, expected: &[u128]) {
-        let decompressor = CompactSpaceDecompressor::open(data).unwrap();
-        for (idx, expected_val) in expected.iter().cloned().enumerate() {
-            let val = decompressor.get(idx as u64);
-            assert_eq!(val, expected_val);
-
-            let test_range = |range: RangeInclusive<u128>| {
-                let expected_positions = expected
-                    .iter()
-                    .positions(|val| range.contains(val))
-                    .map(|pos| pos as u64)
-                    .collect::<Vec<_>>();
-                let positions = decompressor.get_between_vals(range);
-                assert_eq!(positions, expected_positions);
-            };
-
-            test_range(expected_val.saturating_sub(1)..=expected_val);
-            test_range(expected_val..=expected_val);
-            test_range(expected_val..=expected_val.saturating_add(1));
-            test_range(expected_val.saturating_sub(1)..=expected_val.saturating_add(1));
-        }
-    }
-
-    fn test_aux_vals(u128_vals: &[u128]) -> OwnedBytes {
-        let mut out = Vec::new();
-        serialize_u128(VecColumn::from(u128_vals), &mut out).unwrap();
-
-        let data = OwnedBytes::new(out);
-        test_all(data.clone(), u128_vals);
-        data
-    }
-
-    #[test]
-    fn test_range_1() {
-        let vals = &[
-            1u128,
-            100u128,
-            3u128,
-            99999u128,
-            100000u128,
-            100001u128,
-            4_000_211_221u128,
-            4_000_211_222u128,
-            333u128,
-        ];
-        let data = test_aux_vals(vals);
-        let decomp = CompactSpaceDecompressor::open(data).unwrap();
-        let positions = decomp.get_between_vals(0..=1);
-        assert_eq!(positions, vec![0]);
-        let positions = decomp.get_between_vals(0..=2);
-        assert_eq!(positions, vec![0]);
-        let positions = decomp.get_between_vals(0..=3);
-        assert_eq!(positions, vec![0, 2]);
-        assert_eq!(decomp.get_between_vals(99999u128..=99999u128), vec![3]);
-        assert_eq!(decomp.get_between_vals(99999u128..=100000u128), vec![3, 4]);
-        assert_eq!(decomp.get_between_vals(99998u128..=100000u128), vec![3, 4]);
-        assert_eq!(decomp.get_between_vals(99998u128..=99999u128), vec![3]);
-        assert_eq!(decomp.get_between_vals(99998u128..=99998u128), vec![]);
-        assert_eq!(decomp.get_between_vals(333u128..=333u128), vec![8]);
-        assert_eq!(decomp.get_between_vals(332u128..=333u128), vec![8]);
-        assert_eq!(decomp.get_between_vals(332u128..=334u128), vec![8]);
-        assert_eq!(decomp.get_between_vals(333u128..=334u128), vec![8]);
-
-        assert_eq!(
-            decomp.get_between_vals(4_000_211_221u128..=5_000_000_000u128),
-            vec![6, 7]
-        );
-    }
-
-    #[test]
-    fn test_empty() {
-        let vals = &[];
-        let data = test_aux_vals(vals);
-        let _decomp = CompactSpaceDecompressor::open(data).unwrap();
-    }
-
-    #[test]
-    fn test_range_2() {
-        let vals = &[
-            100u128,
-            99999u128,
-            100000u128,
-            100001u128,
-            4_000_211_221u128,
-            4_000_211_222u128,
-            333u128,
-        ];
-        let data = test_aux_vals(vals);
-        let decomp = CompactSpaceDecompressor::open(data).unwrap();
-        let positions = decomp.get_between_vals(0..=5);
-        assert_eq!(positions, vec![]);
-        let positions = decomp.get_between_vals(0..=100);
-        assert_eq!(positions, vec![0]);
-        let positions = decomp.get_between_vals(0..=105);
-        assert_eq!(positions, vec![0]);
-    }
-
-    #[test]
-    fn test_range_3() {
-        let vals = &[
-            200u128,
-            201,
-            202,
-            203,
-            204,
-            204,
-            206,
-            207,
-            208,
-            209,
-            210,
-            1_000_000,
-            5_000_000_000,
-        ];
-        let mut out = Vec::new();
-        serialize_u128(VecColumn::from(vals), &mut out).unwrap();
-        let decomp = open_u128(OwnedBytes::new(out)).unwrap();
-
-        assert_eq!(decomp.get_between_vals(199..=200), vec![0]);
-        assert_eq!(decomp.get_between_vals(199..=201), vec![0, 1]);
-        assert_eq!(decomp.get_between_vals(200..=200), vec![0]);
-        assert_eq!(decomp.get_between_vals(1_000_000..=1_000_000), vec![11]);
-    }
-
-    #[test]
-    fn test_bug1() {
-        let vals = &[9223372036854775806];
-        let _data = test_aux_vals(vals);
-    }
-
-    #[test]
-    fn test_bug2() {
-        let vals = &[340282366920938463463374607431768211455u128];
-        let _data = test_aux_vals(vals);
-    }
-
-    #[test]
-    fn test_bug3() {
-        let vals = &[340282366920938463463374607431768211454];
-        let _data = test_aux_vals(vals);
-    }
-
-    #[test]
-    fn test_bug4() {
-        let vals = &[340282366920938463463374607431768211455, 0];
-        let _data = test_aux_vals(vals);
-    }
-
-    #[test]
-    fn test_first_large_gaps() {
-        let vals = &[1_000_000_000u128; 100];
-        let _data = test_aux_vals(vals);
-    }
-    use itertools::Itertools;
-    use proptest::prelude::*;
-
-    fn num_strategy() -> impl Strategy<Value = u128> {
-        prop_oneof![
-            1 => prop::num::u128::ANY.prop_map(|num| u128::MAX - (num % 10) ),
-            1 => prop::num::u128::ANY.prop_map(|num| i64::MAX as u128 + 5 - (num % 10) ),
-            1 => prop::num::u128::ANY.prop_map(|num| i128::MAX as u128 + 5 - (num % 10) ),
-            1 => prop::num::u128::ANY.prop_map(|num| num % 10 ),
-            20 => prop::num::u128::ANY,
-        ]
-    }
-
-    proptest! {
-        #![proptest_config(ProptestConfig::with_cases(10))]
-
-            #[test]
-            fn compress_decompress_random(vals in proptest::collection::vec(num_strategy()
-    , 1..1000)) {
-                let _data = test_aux_vals(&vals);
-            }
-        }
-}

fastfield_codecs/src/dynamic.rs

@@ -0,0 +1,254 @@
+// Copyright (C) 2022 Quickwit, Inc.
+//
+// Quickwit is offered under the AGPL v3.0 and as commercial software.
+// For commercial licensing, contact us at hello@quickwit.io.
+//
+// AGPL:
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+//
+
+use std::io;
+use std::num::NonZeroU64;
+use std::sync::Arc;
+
+use common::BinarySerializable;
+use fastdivide::DividerU64;
+use ownedbytes::OwnedBytes;
+
+use crate::bitpacked::BitpackedFastFieldCodec;
+use crate::gcd::{find_gcd, GCDFastFieldCodecReader, GCDParams};
+use crate::linearinterpol::LinearInterpolCodec;
+use crate::multilinearinterpol::MultiLinearInterpolFastFieldCodec;
+use crate::{FastFieldCodec, FastFieldCodecReader, FastFieldStats};
+
+pub struct DynamicFastFieldCodec;
+
+impl FastFieldCodec for DynamicFastFieldCodec {
+    const NAME: &'static str = "dynamic";
+
+    type Reader = DynamicFastFieldReader;
+
+    fn is_applicable(_vals: &[u64], _stats: crate::FastFieldStats) -> bool {
+        true
+    }
+
+    fn estimate(_vals: &[u64], _stats: crate::FastFieldStats) -> f32 {
+        0f32
+    }
+
+    fn serialize(
+        &self,
+        wrt: &mut impl io::Write,
+        vals: &[u64],
+        stats: crate::FastFieldStats,
+    ) -> io::Result<()> {
+        let gcd: NonZeroU64 = find_gcd(vals.iter().copied().map(|val| val - stats.min_value))
+            .unwrap_or(unsafe { NonZeroU64::new_unchecked(1) });
+        if gcd.get() > 1 {
+            let gcd_divider = DividerU64::divide_by(gcd.get());
+            let scaled_vals: Vec<u64> = vals
+                .iter()
+                .copied()
+                .map(|val| gcd_divider.divide(val - stats.min_value))
+                .collect();
+            <CodecType as BinarySerializable>::serialize(&CodecType::Gcd, wrt)?;
+            let gcd_params = GCDParams {
+                min_value: stats.min_value,
+                gcd,
+            };
+            gcd_params.serialize(wrt)?;
+            let codec_type = choose_codec(stats, &scaled_vals);
+            <CodecType as BinarySerializable>::serialize(&codec_type, wrt)?;
+            let scaled_stats = FastFieldStats::compute(&scaled_vals);
+            codec_type.serialize(wrt, &scaled_vals, scaled_stats)?;
+        } else {
+            let codec_type = choose_codec(stats, vals);
+            wrt.write_all(&[codec_type.to_code()])?;
+            codec_type.serialize(wrt, vals, stats)?;
+        }
+        Ok(())
+    }
+
+    fn open_from_bytes(mut bytes: OwnedBytes) -> io::Result<Self::Reader> {
+        let codec_code = bytes.read_u8();
+        let codec_type = CodecType::from_code(codec_code).ok_or_else(|| {
+            io::Error::new(
+                io::ErrorKind::InvalidData,
+                format!("Unknown codec code `{codec_code}`"),
+            )
+        })?;
+        let fast_field_reader: Arc<dyn FastFieldCodecReader> = match codec_type {
+            CodecType::Bitpacked => Arc::new(BitpackedFastFieldCodec::open_from_bytes(bytes)?),
+            CodecType::LinearInterpol => Arc::new(LinearInterpolCodec::open_from_bytes(bytes)?),
+            CodecType::MultiLinearInterpol => {
+                Arc::new(MultiLinearInterpolFastFieldCodec::open_from_bytes(bytes)?)
+            }
+            CodecType::Gcd => {
+                let gcd_params = GCDParams::deserialize(&mut bytes)?;
+                let inner_codec_type = <CodecType as BinarySerializable>::deserialize(&mut bytes)?;
+                match inner_codec_type {
+                    CodecType::Bitpacked => Arc::new(GCDFastFieldCodecReader {
+                        params: gcd_params,
+                        reader: BitpackedFastFieldCodec::open_from_bytes(bytes)?,
+                    }),
+                    CodecType::LinearInterpol => Arc::new(GCDFastFieldCodecReader {
+                        params: gcd_params,
+                        reader: LinearInterpolCodec::open_from_bytes(bytes)?,
+                    }),
+                    CodecType::MultiLinearInterpol => Arc::new(GCDFastFieldCodecReader {
+                        params: gcd_params,
+                        reader: MultiLinearInterpolFastFieldCodec::open_from_bytes(bytes)?,
+                    }),
+                    CodecType::Gcd => {
+                        return Err(io::Error::new(
+                            io::ErrorKind::InvalidData,
+                            "A GCD codec may not wrap another GCD codec.",
+                        ));
+                    }
+                }
+            }
+        };
+        Ok(DynamicFastFieldReader(fast_field_reader))
+    }
+}
+
+#[derive(Clone)]
+/// DynamicFastFieldReader wraps different readers to access
+/// the various encoded fastfield data
+pub struct DynamicFastFieldReader(Arc<dyn FastFieldCodecReader>);
+
+#[repr(u8)]
+#[derive(Debug, Clone, Copy)]
+pub enum CodecType {
+    Bitpacked = 0,
+    LinearInterpol = 1,
+    MultiLinearInterpol = 2,
+    Gcd = 3,
+}
+
+impl BinarySerializable for CodecType {
+    fn serialize<W: io::Write>(&self, wrt: &mut W) -> io::Result<()> {
+        wrt.write_all(&[self.to_code()])?;
+        Ok(())
+    }
+
+    fn deserialize<R: io::Read>(reader: &mut R) -> io::Result<Self> {
+        let codec_code = u8::deserialize(reader)?;
+        let codec_type = CodecType::from_code(codec_code).ok_or_else(|| {
+            io::Error::new(
+                io::ErrorKind::InvalidData,
+                format!("Invalid codec type code {codec_code}"),
+            )
+        })?;
+        Ok(codec_type)
+    }
+}
+
+impl CodecType {
+    pub fn from_code(code: u8) -> Option<Self> {
+        match code {
+            0 => Some(CodecType::Bitpacked),
+            1 => Some(CodecType::LinearInterpol),
+            2 => Some(CodecType::MultiLinearInterpol),
+            3 => Some(CodecType::Gcd),
+            _ => None,
+        }
+    }
+
+    pub fn to_code(self) -> u8 {
+        self as u8
+    }
+
+    fn codec_estimation(
+        &self,
+        stats: FastFieldStats,
+        vals: &[u64],
+        estimations: &mut Vec<(f32, CodecType)>,
+    ) {
+        let estimate_opt: Option<f32> = match self {
+            CodecType::Bitpacked => codec_estimation::<BitpackedFastFieldCodec>(stats, vals),
+            CodecType::LinearInterpol => codec_estimation::<LinearInterpolCodec>(stats, vals),
+            CodecType::MultiLinearInterpol => {
+                codec_estimation::<MultiLinearInterpolFastFieldCodec>(stats, vals)
+            }
+            CodecType::Gcd => None,
+        };
+        if let Some(estimate) = estimate_opt {
+            if !estimate.is_nan() && estimate.is_finite() {
+                estimations.push((estimate, *self));
+            }
+        }
+    }
+
+    fn serialize(
+        &self,
+        wrt: &mut impl io::Write,
+        fastfield_accessor: &[u64],
+        stats: FastFieldStats,
+    ) -> io::Result<()> {
+        match self {
+            CodecType::Bitpacked => {
+                BitpackedFastFieldCodec.serialize(wrt, fastfield_accessor, stats)?;
+            }
+            CodecType::LinearInterpol => {
+                LinearInterpolCodec.serialize(wrt, fastfield_accessor, stats)?;
+            }
+            CodecType::MultiLinearInterpol => {
+                MultiLinearInterpolFastFieldCodec.serialize(wrt, fastfield_accessor, stats)?;
+            }
+            CodecType::Gcd => {
+                panic!("GCD should never be called that way.");
+            }
+        }
+        Ok(())
+    }
+}
+
+impl FastFieldCodecReader for DynamicFastFieldReader {
+    fn get_u64(&self, doc: u64) -> u64 {
+        self.0.get_u64(doc)
+    }
+
+    fn min_value(&self) -> u64 {
+        self.0.min_value()
+    }
+
+    fn max_value(&self) -> u64 {
+        self.0.max_value()
+    }
+}
+
+fn codec_estimation<T: FastFieldCodec>(stats: FastFieldStats, vals: &[u64]) -> Option<f32> {
+    if !T::is_applicable(vals, stats.clone()) {
+        return None;
+    }
+    let ratio = T::estimate(vals, stats);
+    Some(ratio)
+}
+
+const CODEC_TYPES: [CodecType; 3] = [
+    CodecType::Bitpacked,
+    CodecType::LinearInterpol,
+    CodecType::MultiLinearInterpol,
+];
+
+fn choose_codec(stats: FastFieldStats, vals: &[u64]) -> CodecType {
+    let mut estimations = Vec::new();
+    for codec_type in &CODEC_TYPES {
+        codec_type.codec_estimation(stats, vals, &mut estimations);
+    }
+    estimations.sort_by(|a, b| a.0.partial_cmp(&b.0).unwrap());
+    let (_ratio, codec_type) = estimations[0];
+    codec_type
+}

fastfield_codecs/src/gcd.rs

@@ -1,159 +1,236 @@
+use std::io::{self, Write};
 use std::num::NonZeroU64;
 
+use common::BinarySerializable;
 use fastdivide::DividerU64;
 
-/// Compute the gcd of two non null numbers.
+use crate::FastFieldCodecReader;
+
+/// Wrapper for accessing a fastfield.
 ///
-/// It is recommended, but not required, to feed values such that `large >= small`.
-fn compute_gcd(mut large: NonZeroU64, mut small: NonZeroU64) -> NonZeroU64 {
-    loop {
-        let rem: u64 = large.get() % small;
-        if let Some(new_small) = NonZeroU64::new(rem) {
-            (large, small) = (small, new_small);
-        } else {
-            return small;
-        }
+/// Holds the data and the codec to the read the data.
+#[derive(Clone)]
+pub struct GCDFastFieldCodecReader<CodecReader> {
+    pub params: GCDParams,
+    pub reader: CodecReader,
+}
+
+impl<C: FastFieldCodecReader> FastFieldCodecReader for GCDFastFieldCodecReader<C> {
+    #[inline]
+    fn get_u64(&self, doc: u64) -> u64 {
+        self.params.min_value + self.params.gcd.get() * self.reader.get_u64(doc)
+    }
+
+    fn min_value(&self) -> u64 {
+        self.params.min_value + self.params.gcd.get() * self.reader.min_value()
+    }
+
+    fn max_value(&self) -> u64 {
+        self.params.min_value + self.params.gcd.get() * self.reader.max_value()
     }
 }
 
-// Find GCD for iterator of numbers
-pub fn find_gcd(numbers: impl Iterator<Item = u64>) -> Option<NonZeroU64> {
-    let mut numbers = numbers.flat_map(NonZeroU64::new);
-    let mut gcd: NonZeroU64 = numbers.next()?;
-    if gcd.get() == 1 {
-        return Some(gcd);
+#[derive(Debug, Copy, Clone)]
+pub struct GCDParams {
+    pub min_value: u64,
+    pub gcd: NonZeroU64,
+}
+
+impl BinarySerializable for GCDParams {
+    fn serialize<W: Write>(&self, wrt: &mut W) -> io::Result<()> {
+        self.gcd.get().serialize(wrt)?;
+        self.min_value.serialize(wrt)?;
+        Ok(())
     }
 
-    let mut gcd_divider = DividerU64::divide_by(gcd.get());
+    fn deserialize<R: io::Read>(reader: &mut R) -> io::Result<Self> {
+        let gcd = NonZeroU64::new(u64::deserialize(reader)?)
+            .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "GCD=0 is invalid."))?;
+        let min_value = u64::deserialize(reader)?;
+        Ok(GCDParams { min_value, gcd })
+    }
+}
+
+fn compute_gcd(mut left: u64, mut right: u64) -> u64 {
+    while right != 0 {
+        (left, right) = (right, left % right);
+    }
+    left
+}
+
+// Find GCD for iterator of numbers
+//
+// If all numbers are '0' (or if there are not numbers, return None).
+pub fn find_gcd(numbers: impl Iterator<Item = u64>) -> Option<NonZeroU64> {
+    let mut numbers = numbers.filter(|n| *n != 0);
+    let mut gcd = numbers.next()?;
+    if gcd == 1 {
+        return NonZeroU64::new(gcd);
+    }
+
+    let mut gcd_divider = DividerU64::divide_by(gcd);
     for val in numbers {
-        let remainder = val.get() - (gcd_divider.divide(val.get())) * gcd.get();
+        let remainder = val - gcd_divider.divide(val) * gcd;
         if remainder == 0 {
             continue;
         }
-        gcd = compute_gcd(val, gcd);
-        if gcd.get() == 1 {
-            return Some(gcd);
+        gcd = compute_gcd(gcd, val);
+        if gcd == 1 {
+            return NonZeroU64::new(1);
         }
-
-        gcd_divider = DividerU64::divide_by(gcd.get());
+        gcd_divider = DividerU64::divide_by(gcd);
     }
-    Some(gcd)
+    NonZeroU64::new(gcd)
 }
 
 #[cfg(test)]
 mod tests {
-    use std::io;
+
+    // TODO Move test
+    //
+    // use std::collections::HashMap;
+    // use std::path::Path;
+    //
+    // use crate::directory::{CompositeFile, RamDirectory, WritePtr};
+    // use crate::fastfield::serializer::FastFieldCodecEnableCheck;
+    // use crate::fastfield::tests::{FIELD, FIELDI64, SCHEMA, SCHEMAI64};
+    // use super::{
+    // find_gcd, CompositeFastFieldSerializer, DynamicFastFieldReader, FastFieldCodecName,
+    // FastFieldReader, FastFieldsWriter, ALL_CODECS,
+    // };
+    // use crate::schema::Schema;
+    // use crate::Directory;
+    //
+    // fn get_index(
+    // docs: &[crate::Document],
+    // schema: &Schema,
+    // codec_enable_checker: FastFieldCodecEnableCheck,
+    // ) -> crate::Result<RamDirectory> {
+    // let directory: RamDirectory = RamDirectory::create();
+    // {
+    // let write: WritePtr = directory.open_write(Path::new("test")).unwrap();
+    // let mut serializer =
+    // CompositeFastFieldSerializer::from_write_with_codec(write, codec_enable_checker)
+    // .unwrap();
+    // let mut fast_field_writers = FastFieldsWriter::from_schema(schema);
+    // for doc in docs {
+    // fast_field_writers.add_document(doc);
+    // }
+    // fast_field_writers
+    // .serialize(&mut serializer, &HashMap::new(), None)
+    // .unwrap();
+    // serializer.close().unwrap();
+    // }
+    // Ok(directory)
+    // }
+    //
+    // fn test_fastfield_gcd_i64_with_codec(
+    // codec_name: FastFieldCodecName,
+    // num_vals: usize,
+    // ) -> crate::Result<()> {
+    // let path = Path::new("test");
+    // let mut docs = vec![];
+    // for i in 1..=num_vals {
+    // let val = i as i64 * 1000i64;
+    // docs.push(doc!(*FIELDI64=>val));
+    // }
+    // let directory = get_index(&docs, &SCHEMAI64, codec_name.clone().into())?;
+    // let file = directory.open_read(path).unwrap();
+    // assert_eq!(file.len(), 118);
+    // let composite_file = CompositeFile::open(&file)?;
+    // let file = composite_file.open_read(*FIELD).unwrap();
+    // let fast_field_reader = DynamicFastFieldReader::<i64>::open(file)?;
+    // assert_eq!(fast_field_reader.get(0), 1000i64);
+    // assert_eq!(fast_field_reader.get(1), 2000i64);
+    // assert_eq!(fast_field_reader.get(2), 3000i64);
+    // assert_eq!(fast_field_reader.max_value(), num_vals as i64 * 1000);
+    // assert_eq!(fast_field_reader.min_value(), 1000i64);
+    // let file = directory.open_read(path).unwrap();
+    //
+    // Can't apply gcd
+    // let path = Path::new("test");
+    // docs.pop();
+    // docs.push(doc!(*FIELDI64=>2001i64));
+    // let directory = get_index(&docs, &SCHEMAI64, codec_name.into())?;
+    // let file2 = directory.open_read(path).unwrap();
+    // assert!(file2.len() > file.len());
+    //
+    // Ok(())
+    // }
+    //
+    // #[test]
+    // fn test_fastfield_gcd_i64() -> crate::Result<()> {
+    // for codec_name in ALL_CODECS {
+    // test_fastfield_gcd_i64_with_codec(codec_name.clone(), 5005)?;
+    // }
+    // Ok(())
+    // }
+    //
+    // fn test_fastfield_gcd_u64_with_codec(
+    // codec_name: FastFieldCodecName,
+    // num_vals: usize,
+    // ) -> crate::Result<()> {
+    // let path = Path::new("test");
+    // let mut docs = vec![];
+    // for i in 1..=num_vals {
+    // let val = i as u64 * 1000u64;
+    // docs.push(doc!(*FIELD=>val));
+    // }
+    // let directory = get_index(&docs, &SCHEMA, codec_name.clone().into())?;
+    // let file = directory.open_read(path).unwrap();
+    // assert_eq!(file.len(), 118);
+    // let composite_file = CompositeFile::open(&file)?;
+    // let file = composite_file.open_read(*FIELD).unwrap();
+    // let fast_field_reader = DynamicFastFieldReader::<u64>::open(file)?;
+    // assert_eq!(fast_field_reader.get(0), 1000u64);
+    // assert_eq!(fast_field_reader.get(1), 2000u64);
+    // assert_eq!(fast_field_reader.get(2), 3000u64);
+    // assert_eq!(fast_field_reader.max_value(), num_vals as u64 * 1000);
+    // assert_eq!(fast_field_reader.min_value(), 1000u64);
+    // let file = directory.open_read(path).unwrap();
+    //
+    // Can't apply gcd
+    // let path = Path::new("test");
+    // docs.pop();
+    // docs.push(doc!(*FIELDI64=>2001u64));
+    // let directory = get_index(&docs, &SCHEMA, codec_name.into())?;
+    // let file2 = directory.open_read(path).unwrap();
+    // assert!(file2.len() > file.len());
+    //
+    // Ok(())
+    // }
+    //
+    // #[test]
+    // fn test_fastfield_gcd_u64() -> crate::Result<()> {
+    // for codec_name in ALL_CODECS {
+    // test_fastfield_gcd_u64_with_codec(codec_name.clone(), 5005)?;
+    // }
+    // Ok(())
+    // }
+    //
+    // #[test]
+    // pub fn test_fastfield2() {
+    // let test_fastfield = DynamicFastFieldReader::<u64>::from(vec![100, 200, 300]);
+    // assert_eq!(test_fastfield.get(0), 100);
+    // assert_eq!(test_fastfield.get(1), 200);
+    // assert_eq!(test_fastfield.get(2), 300);
+    // }
+
     use std::num::NonZeroU64;
 
-    use ownedbytes::OwnedBytes;
-
     use crate::gcd::{compute_gcd, find_gcd};
-    use crate::{FastFieldCodecType, VecColumn};
-
-    fn test_fastfield_gcd_i64_with_codec(
-        codec_type: FastFieldCodecType,
-        num_vals: usize,
-    ) -> io::Result<()> {
-        let mut vals: Vec<i64> = (-4..=(num_vals as i64) - 5).map(|val| val * 1000).collect();
-        let mut buffer: Vec<u8> = Vec::new();
-        crate::serialize(VecColumn::from(&vals), &mut buffer, &[codec_type])?;
-        let buffer = OwnedBytes::new(buffer);
-        let column = crate::open::<i64>(buffer.clone())?;
-        assert_eq!(column.get_val(0), -4000i64);
-        assert_eq!(column.get_val(1), -3000i64);
-        assert_eq!(column.get_val(2), -2000i64);
-        assert_eq!(column.max_value(), (num_vals as i64 - 5) * 1000);
-        assert_eq!(column.min_value(), -4000i64);
-
-        // Can't apply gcd
-        let mut buffer_without_gcd = Vec::new();
-        vals.pop();
-        vals.push(1001i64);
-        crate::serialize(
-            VecColumn::from(&vals),
-            &mut buffer_without_gcd,
-            &[codec_type],
-        )?;
-        let buffer_without_gcd = OwnedBytes::new(buffer_without_gcd);
-        assert!(buffer_without_gcd.len() > buffer.len());
-
-        Ok(())
-    }
-
-    #[test]
-    fn test_fastfield_gcd_i64() -> io::Result<()> {
-        for &codec_type in &[
-            FastFieldCodecType::Bitpacked,
-            FastFieldCodecType::BlockwiseLinear,
-            FastFieldCodecType::Linear,
-        ] {
-            test_fastfield_gcd_i64_with_codec(codec_type, 5500)?;
-        }
-        Ok(())
-    }
-
-    fn test_fastfield_gcd_u64_with_codec(
-        codec_type: FastFieldCodecType,
-        num_vals: usize,
-    ) -> io::Result<()> {
-        let mut vals: Vec<u64> = (1..=num_vals).map(|i| i as u64 * 1000u64).collect();
-        let mut buffer: Vec<u8> = Vec::new();
-        crate::serialize(VecColumn::from(&vals), &mut buffer, &[codec_type])?;
-        let buffer = OwnedBytes::new(buffer);
-        let column = crate::open::<u64>(buffer.clone())?;
-        assert_eq!(column.get_val(0), 1000u64);
-        assert_eq!(column.get_val(1), 2000u64);
-        assert_eq!(column.get_val(2), 3000u64);
-        assert_eq!(column.max_value(), num_vals as u64 * 1000);
-        assert_eq!(column.min_value(), 1000u64);
-
-        // Can't apply gcd
-        let mut buffer_without_gcd = Vec::new();
-        vals.pop();
-        vals.push(1001u64);
-        crate::serialize(
-            VecColumn::from(&vals),
-            &mut buffer_without_gcd,
-            &[codec_type],
-        )?;
-        let buffer_without_gcd = OwnedBytes::new(buffer_without_gcd);
-        assert!(buffer_without_gcd.len() > buffer.len());
-        Ok(())
-    }
-
-    #[test]
-    fn test_fastfield_gcd_u64() -> io::Result<()> {
-        for &codec_type in &[
-            FastFieldCodecType::Bitpacked,
-            FastFieldCodecType::BlockwiseLinear,
-            FastFieldCodecType::Linear,
-        ] {
-            test_fastfield_gcd_u64_with_codec(codec_type, 5500)?;
-        }
-        Ok(())
-    }
-
-    #[test]
-    pub fn test_fastfield2() {
-        let test_fastfield = crate::serialize_and_load(&[100u64, 200u64, 300u64]);
-        assert_eq!(test_fastfield.get_val(0), 100);
-        assert_eq!(test_fastfield.get_val(1), 200);
-        assert_eq!(test_fastfield.get_val(2), 300);
-    }
 
     #[test]
     fn test_compute_gcd() {
-        let test_compute_gcd_aux = |large, small, expected| {
-            let large = NonZeroU64::new(large).unwrap();
-            let small = NonZeroU64::new(small).unwrap();
-            let expected = NonZeroU64::new(expected).unwrap();
-            assert_eq!(compute_gcd(small, large), expected);
-            assert_eq!(compute_gcd(large, small), expected);
-        };
-        test_compute_gcd_aux(1, 4, 1);
-        test_compute_gcd_aux(2, 4, 2);
-        test_compute_gcd_aux(10, 25, 5);
-        test_compute_gcd_aux(25, 25, 25);
+        assert_eq!(compute_gcd(0, 0), 0);
+        assert_eq!(compute_gcd(4, 0), 4);
+        assert_eq!(compute_gcd(0, 4), 4);
+        assert_eq!(compute_gcd(1, 4), 1);
+        assert_eq!(compute_gcd(4, 1), 1);
+        assert_eq!(compute_gcd(4, 2), 2);
+        assert_eq!(compute_gcd(10, 25), 5);
+        assert_eq!(compute_gcd(25, 10), 5);
+        assert_eq!(compute_gcd(25, 25), 25);
     }
 
     #[test]

fastfield_codecs/src/lib.rs

@@ -1,239 +1,127 @@
-#![cfg_attr(all(feature = "unstable", test), feature(test))]
-
 #[cfg(test)]
 #[macro_use]
 extern crate more_asserts;
 
-#[cfg(all(test, feature = "unstable"))]
-extern crate test;
-
 use std::io;
-use std::io::Write;
-use std::sync::Arc;
 
-use column::EstimateColumn;
-use common::BinarySerializable;
-use compact_space::CompactSpaceDecompressor;
 use ownedbytes::OwnedBytes;
-use serialize::Header;
 
-mod bitpacked;
-mod blockwise_linear;
-mod compact_space;
-mod line;
-mod linear;
-mod monotonic_mapping;
+pub mod bitpacked;
+pub mod dynamic;
+pub mod gcd;
+pub mod linearinterpol;
+pub mod multilinearinterpol;
 
-mod column;
-mod gcd;
-mod serialize;
+// Unify with FastFieldReader
 
-use self::bitpacked::BitpackedCodec;
-use self::blockwise_linear::BlockwiseLinearCodec;
-pub use self::column::{monotonic_map_column, Column, VecColumn};
-use self::linear::LinearCodec;
-pub use self::monotonic_mapping::MonotonicallyMappableToU64;
-pub use self::serialize::{
-    estimate, serialize, serialize_and_load, serialize_u128, NormalizedHeader,
-};
-
-#[derive(PartialEq, Eq, PartialOrd, Ord, Debug, Clone, Copy)]
-#[repr(u8)]
-pub enum FastFieldCodecType {
-    Bitpacked = 1,
-    Linear = 2,
-    BlockwiseLinear = 3,
-}
-
-impl BinarySerializable for FastFieldCodecType {
-    fn serialize<W: Write>(&self, wrt: &mut W) -> io::Result<()> {
-        self.to_code().serialize(wrt)
-    }
-
-    fn deserialize<R: io::Read>(reader: &mut R) -> io::Result<Self> {
-        let code = u8::deserialize(reader)?;
-        let codec_type: Self = Self::from_code(code)
-            .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "Unknown code `{code}.`"))?;
-        Ok(codec_type)
-    }
-}
-
-impl FastFieldCodecType {
-    pub fn to_code(self) -> u8 {
-        self as u8
-    }
-
-    pub fn from_code(code: u8) -> Option<Self> {
-        match code {
-            1 => Some(Self::Bitpacked),
-            2 => Some(Self::Linear),
-            3 => Some(Self::BlockwiseLinear),
-            _ => None,
-        }
-    }
-}
-
-/// Returns the correct codec reader wrapped in the `Arc` for the data.
-pub fn open_u128(bytes: OwnedBytes) -> io::Result<Arc<dyn Column<u128>>> {
-    Ok(Arc::new(CompactSpaceDecompressor::open(bytes)?))
-}
-
-/// Returns the correct codec reader wrapped in the `Arc` for the data.
-pub fn open<T: MonotonicallyMappableToU64>(
-    mut bytes: OwnedBytes,
-) -> io::Result<Arc<dyn Column<T>>> {
-    let header = Header::deserialize(&mut bytes)?;
-    match header.codec_type {
-        FastFieldCodecType::Bitpacked => open_specific_codec::<BitpackedCodec, _>(bytes, &header),
-        FastFieldCodecType::Linear => open_specific_codec::<LinearCodec, _>(bytes, &header),
-        FastFieldCodecType::BlockwiseLinear => {
-            open_specific_codec::<BlockwiseLinearCodec, _>(bytes, &header)
-        }
-    }
-}
-
-fn open_specific_codec<C: FastFieldCodec, Item: MonotonicallyMappableToU64>(
-    bytes: OwnedBytes,
-    header: &Header,
-) -> io::Result<Arc<dyn Column<Item>>> {
-    let normalized_header = header.normalized();
-    let reader = C::open_from_bytes(bytes, normalized_header)?;
-    let min_value = header.min_value;
-    if let Some(gcd) = header.gcd {
-        let monotonic_mapping = move |val: u64| Item::from_u64(min_value + val * gcd.get());
-        Ok(Arc::new(monotonic_map_column(reader, monotonic_mapping)))
-    } else {
-        let monotonic_mapping = move |val: u64| Item::from_u64(min_value + val);
-        Ok(Arc::new(monotonic_map_column(reader, monotonic_mapping)))
-    }
+pub trait FastFieldCodecReader {
+    /// reads the metadata and returns the CodecReader
+    fn get_u64(&self, doc: u64) -> u64;
+    fn min_value(&self) -> u64;
+    fn max_value(&self) -> u64;
 }
 
 /// The FastFieldSerializerEstimate trait is required on all variants
 /// of fast field compressions, to decide which one to choose.
-trait FastFieldCodec: 'static {
-    /// A codex needs to provide a unique name and id, which is
-    /// used for debugging and de/serialization.
-    const CODEC_TYPE: FastFieldCodecType;
+pub trait FastFieldCodec {
+    /// A codex needs to provide a unique name used for debugging.
+    const NAME: &'static str;
 
-    type Reader: Column<u64> + 'static;
+    type Reader: FastFieldCodecReader;
 
-    /// Reads the metadata and returns the CodecReader
-    fn open_from_bytes(bytes: OwnedBytes, header: NormalizedHeader) -> io::Result<Self::Reader>;
-
-    /// Serializes the data using the serializer into write.
-    ///
-    /// The column iterator should be preferred over using column `get_val` method for
-    /// performance reasons.
-    fn serialize(column: &dyn Column, write: &mut impl Write) -> io::Result<()>;
+    /// Check if the Codec is able to compress the data
+    fn is_applicable(vals: &[u64], stats: FastFieldStats) -> bool;
 
     /// Returns an estimate of the compression ratio.
-    /// If the codec is not applicable, returns `None`.
-    ///
     /// The baseline is uncompressed 64bit data.
     ///
     /// It could make sense to also return a value representing
     /// computational complexity.
-    fn estimate(column: &EstimateColumn) -> Option<f32>;
+    fn estimate(vals: &[u64], stats: FastFieldStats) -> f32;
+
+    /// Serializes the data using the serializer into write.
+    /// There are multiple iterators, in case the codec needs to read the data multiple times.
+    /// The iterators should be preferred over using fastfield_accessor for performance reasons.
+    fn serialize(
+        &self,
+        write: &mut impl io::Write,
+        vals: &[u64],
+        stats: FastFieldStats,
+    ) -> io::Result<()>;
+
+    fn open_from_bytes(bytes: OwnedBytes) -> io::Result<Self::Reader>;
 }
 
-pub const ALL_CODEC_TYPES: [FastFieldCodecType; 3] = [
-    FastFieldCodecType::Bitpacked,
-    FastFieldCodecType::BlockwiseLinear,
-    FastFieldCodecType::Linear,
-];
+/// Statistics are used in codec detection and stored in the fast field footer.
+#[derive(Clone, Copy, Default, Debug)]
+pub struct FastFieldStats {
+    pub min_value: u64,
+    pub max_value: u64,
+    pub num_vals: u64,
+}
+
+impl FastFieldStats {
+    pub fn compute(vals: &[u64]) -> Self {
+        if vals.is_empty() {
+            return FastFieldStats::default();
+        }
+        let first_val = vals[0];
+        let mut fast_field_stats = FastFieldStats {
+            min_value: first_val,
+            max_value: first_val,
+            num_vals: 1,
+        };
+        for &val in &vals[1..] {
+            fast_field_stats.record(val);
+        }
+        fast_field_stats
+    }
+
+    pub fn record(&mut self, val: u64) {
+        self.num_vals += 1;
+        self.min_value = self.min_value.min(val);
+        self.max_value = self.max_value.max(val);
+    }
+}
 
 #[cfg(test)]
 mod tests {
-    use proptest::prelude::*;
-    use proptest::strategy::Strategy;
-    use proptest::{prop_oneof, proptest};
+    use crate::bitpacked::BitpackedFastFieldCodec;
+    use crate::linearinterpol::LinearInterpolCodec;
+    use crate::multilinearinterpol::MultiLinearInterpolFastFieldCodec;
 
-    use crate::bitpacked::BitpackedCodec;
-    use crate::blockwise_linear::BlockwiseLinearCodec;
-    use crate::column::EstimateColumn;
-    use crate::linear::LinearCodec;
-    use crate::serialize::Header;
-
-    pub(crate) fn create_and_validate<Codec: FastFieldCodec>(
+    pub fn create_and_validate<S: FastFieldCodec>(
+        codec: &S,
         data: &[u64],
         name: &str,
-    ) -> Option<(f32, f32)> {
-        let col = &VecColumn::from(data);
-        let header = Header::compute_header(col, &[Codec::CODEC_TYPE])?;
-        let normalized_col = header.normalize_column(col);
-
-        let limited_column = EstimateColumn::new(&normalized_col);
-        let estimation = Codec::estimate(&limited_column)?;
-
-        let mut out = Vec::new();
-        let col = VecColumn::from(data);
-        serialize(col, &mut out, &[Codec::CODEC_TYPE]).unwrap();
+    ) -> (f32, f32) {
+        if !S::is_applicable(&data, crate::tests::stats_from_vec(data)) {
+            return (f32::MAX, 0.0);
+        }
+        let estimation = S::estimate(&data, crate::tests::stats_from_vec(data));
+        let mut out: Vec<u8> = Vec::new();
+        codec
+            .serialize(&mut out, &data, crate::tests::stats_from_vec(data))
+            .unwrap();
 
         let actual_compression = out.len() as f32 / (data.len() as f32 * 8.0);
 
-        let reader = crate::open::<u64>(OwnedBytes::new(out)).unwrap();
-        assert_eq!(reader.num_vals(), data.len() as u64);
-        for (doc, orig_val) in data.iter().copied().enumerate() {
-            let val = reader.get_val(doc as u64);
-            assert_eq!(
-                val, orig_val,
-                "val `{val}` does not match orig_val {orig_val:?}, in data set {name}, data \
-                 `{data:?}`",
-            );
+        let reader = S::open_from_bytes(OwnedBytes::new(out)).unwrap();
+        for (doc, orig_val) in data.iter().enumerate() {
+            let val = reader.get_u64(doc as u64);
+            if val != *orig_val {
+                panic!(
+                    "val {:?} does not match orig_val {:?}, in data set {}, data {:?}",
+                    val, orig_val, name, data
+                );
+            }
         }
-        Some((estimation, actual_compression))
+        (estimation, actual_compression)
     }
-
-    proptest! {
-        #![proptest_config(ProptestConfig::with_cases(100))]
-
-        #[test]
-        fn test_proptest_small_bitpacked(data in proptest::collection::vec(num_strategy(), 1..10)) {
-            create_and_validate::<BitpackedCodec>(&data, "proptest bitpacked");
-        }
-
-        #[test]
-        fn test_proptest_small_linear(data in proptest::collection::vec(num_strategy(), 1..10)) {
-            create_and_validate::<LinearCodec>(&data, "proptest linearinterpol");
-        }
-
-        #[test]
-        fn test_proptest_small_blockwise_linear(data in proptest::collection::vec(num_strategy(), 1..10)) {
-            create_and_validate::<BlockwiseLinearCodec>(&data, "proptest multilinearinterpol");
-        }
-    }
-
-    proptest! {
-        #![proptest_config(ProptestConfig::with_cases(10))]
-
-        #[test]
-        fn test_proptest_large_bitpacked(data in proptest::collection::vec(num_strategy(), 1..6000)) {
-            create_and_validate::<BitpackedCodec>(&data, "proptest bitpacked");
-        }
-
-        #[test]
-        fn test_proptest_large_linear(data in proptest::collection::vec(num_strategy(), 1..6000)) {
-            create_and_validate::<LinearCodec>(&data, "proptest linearinterpol");
-        }
-
-        #[test]
-        fn test_proptest_large_blockwise_linear(data in proptest::collection::vec(num_strategy(), 1..6000)) {
-            create_and_validate::<BlockwiseLinearCodec>(&data, "proptest multilinearinterpol");
-        }
-    }
-
-    fn num_strategy() -> impl Strategy<Value = u64> {
-        prop_oneof![
-            1 => prop::num::u64::ANY.prop_map(|num| u64::MAX - (num % 10) ),
-            1 => prop::num::u64::ANY.prop_map(|num| num % 10 ),
-            20 => prop::num::u64::ANY,
-        ]
-    }
-
-    pub fn get_codec_test_datasets() -> Vec<(Vec<u64>, &'static str)> {
+    pub fn get_codec_test_data_sets() -> Vec<(Vec<u64>, &'static str)> {
         let mut data_and_names = vec![];
 
-        let data = (10..=10_000_u64).collect::<Vec<_>>();
+        let data = (10..=20_u64).collect::<Vec<_>>();
         data_and_names.push((data, "simple monotonically increasing"));
 
         data_and_names.push((
@@ -243,235 +131,88 @@ mod tests {
         data_and_names.push((vec![5, 50, 3, 13, 1, 1000, 35], "rand small"));
         data_and_names.push((vec![10], "single value"));
 
-        data_and_names.push((
-            vec![1572656989877777, 1170935903116329, 720575940379279, 0],
-            "overflow error",
-        ));
-
         data_and_names
     }
 
-    fn test_codec<C: FastFieldCodec>() {
-        let codec_name = format!("{:?}", C::CODEC_TYPE);
-        for (data, dataset_name) in get_codec_test_datasets() {
-            let estimate_actual_opt: Option<(f32, f32)> =
-                crate::tests::create_and_validate::<C>(&data, dataset_name);
-            let result = if let Some((estimate, actual)) = estimate_actual_opt {
-                format!("Estimate `{estimate}` Actual `{actual}`")
-            } else {
+    fn test_codec<C: FastFieldCodec>(codec: &C) {
+        let codec_name = C::NAME;
+        for (data, data_set_name) in get_codec_test_data_sets() {
+            let (estimate, actual) = crate::tests::create_and_validate(codec, &data, data_set_name);
+            let result = if estimate == f32::MAX {
                 "Disabled".to_string()
+            } else {
+                format!("Estimate {:?} Actual {:?} ", estimate, actual)
             };
-            println!("Codec {codec_name}, DataSet {dataset_name}, {result}");
+            println!(
+                "Codec {}, DataSet {}, {}",
+                codec_name, data_set_name, result
+            );
         }
     }
     #[test]
     fn test_codec_bitpacking() {
-        test_codec::<BitpackedCodec>();
+        test_codec(&BitpackedFastFieldCodec);
     }
     #[test]
     fn test_codec_interpolation() {
-        test_codec::<LinearCodec>();
+        test_codec(&LinearInterpolCodec);
     }
     #[test]
     fn test_codec_multi_interpolation() {
-        test_codec::<BlockwiseLinearCodec>();
+        test_codec(&MultiLinearInterpolFastFieldCodec);
     }
 
     use super::*;
+    pub fn stats_from_vec(data: &[u64]) -> FastFieldStats {
+        let min_value = data.iter().cloned().min().unwrap_or(0);
+        let max_value = data.iter().cloned().max().unwrap_or(0);
+        FastFieldStats {
+            min_value,
+            max_value,
+            num_vals: data.len() as u64,
+        }
+    }
 
     #[test]
     fn estimation_good_interpolation_case() {
         let data = (10..=20000_u64).collect::<Vec<_>>();
-        let data: VecColumn = data.as_slice().into();
 
         let linear_interpol_estimation =
-            LinearCodec::estimate(&EstimateColumn::new(&data)).unwrap();
+            LinearInterpolCodec::estimate(&data, stats_from_vec(&data));
         assert_le!(linear_interpol_estimation, 0.01);
 
         let multi_linear_interpol_estimation =
-            BlockwiseLinearCodec::estimate(&EstimateColumn::new(&data)).unwrap();
+            MultiLinearInterpolFastFieldCodec::estimate(&&data[..], stats_from_vec(&data));
         assert_le!(multi_linear_interpol_estimation, 0.2);
-        assert_lt!(linear_interpol_estimation, multi_linear_interpol_estimation);
+        assert_le!(linear_interpol_estimation, multi_linear_interpol_estimation);
 
-        let bitpacked_estimation = BitpackedCodec::estimate(&EstimateColumn::new(&data)).unwrap();
-        assert_lt!(linear_interpol_estimation, bitpacked_estimation);
+        let bitpacked_estimation = BitpackedFastFieldCodec::estimate(&data, stats_from_vec(&data));
+        assert_le!(linear_interpol_estimation, bitpacked_estimation);
     }
     #[test]
     fn estimation_test_bad_interpolation_case() {
-        let data: &[u64] = &[200, 10, 10, 10, 10, 1000, 20];
+        let data = vec![200, 10, 10, 10, 10, 1000, 20];
 
-        let data: VecColumn = data.into();
         let linear_interpol_estimation =
-            LinearCodec::estimate(&EstimateColumn::new(&data)).unwrap();
-        assert_le!(linear_interpol_estimation, 0.34);
+            LinearInterpolCodec::estimate(&data, stats_from_vec(&data));
+        assert_le!(linear_interpol_estimation, 0.32);
 
-        let bitpacked_estimation = BitpackedCodec::estimate(&EstimateColumn::new(&data)).unwrap();
-        assert_lt!(bitpacked_estimation, linear_interpol_estimation);
+        let bitpacked_estimation = BitpackedFastFieldCodec::estimate(&data, stats_from_vec(&data));
+        assert_le!(bitpacked_estimation, linear_interpol_estimation);
     }
-
-    #[test]
-    fn estimation_prefer_bitpacked() {
-        let data = VecColumn::from(&[10, 10, 10, 10]);
-        let linear_interpol_estimation =
-            LinearCodec::estimate(&EstimateColumn::new(&data)).unwrap();
-        let bitpacked_estimation = BitpackedCodec::estimate(&EstimateColumn::new(&data)).unwrap();
-        assert_lt!(bitpacked_estimation, linear_interpol_estimation);
-    }
-
     #[test]
     fn estimation_test_bad_interpolation_case_monotonically_increasing() {
-        let mut data: Vec<u64> = (200..=20000_u64).collect();
+        let mut data = (200..=20000_u64).collect::<Vec<_>>();
         data.push(1_000_000);
-        let data: VecColumn = data.as_slice().into();
 
         // in this case the linear interpolation can't in fact not be worse than bitpacking,
         // but the estimator adds some threshold, which leads to estimated worse behavior
         let linear_interpol_estimation =
-            LinearCodec::estimate(&EstimateColumn::new(&data)).unwrap();
+            LinearInterpolCodec::estimate(&data, stats_from_vec(&data));
         assert_le!(linear_interpol_estimation, 0.35);
 
-        let bitpacked_estimation = BitpackedCodec::estimate(&EstimateColumn::new(&data)).unwrap();
+        let bitpacked_estimation = BitpackedFastFieldCodec::estimate(&data, stats_from_vec(&data));
         assert_le!(bitpacked_estimation, 0.32);
         assert_le!(bitpacked_estimation, linear_interpol_estimation);
     }
-
-    #[test]
-    fn test_fast_field_codec_type_to_code() {
-        let mut count_codec = 0;
-        for code in 0..=255 {
-            if let Some(codec_type) = FastFieldCodecType::from_code(code) {
-                assert_eq!(codec_type.to_code(), code);
-                count_codec += 1;
-            }
-        }
-        assert_eq!(count_codec, 3);
-    }
-}
-
-#[cfg(all(test, feature = "unstable"))]
-mod bench {
-    use std::sync::Arc;
-
-    use ownedbytes::OwnedBytes;
-    use rand::rngs::StdRng;
-    use rand::{Rng, SeedableRng};
-    use test::{self, Bencher};
-
-    use super::*;
-    use crate::Column;
-
-    fn get_data() -> Vec<u64> {
-        let mut rng = StdRng::seed_from_u64(2u64);
-        let mut data: Vec<_> = (100..55000_u64)
-            .map(|num| num + rng.gen::<u8>() as u64)
-            .collect();
-        data.push(99_000);
-        data.insert(1000, 2000);
-        data.insert(2000, 100);
-        data.insert(3000, 4100);
-        data.insert(4000, 100);
-        data.insert(5000, 800);
-        data
-    }
-
-    #[inline(never)]
-    fn value_iter() -> impl Iterator<Item = u64> {
-        0..20_000
-    }
-    fn get_reader_for_bench<Codec: FastFieldCodec>(data: &[u64]) -> Codec::Reader {
-        let mut bytes = Vec::new();
-        let min_value = *data.iter().min().unwrap();
-        let data = data.iter().map(|el| *el - min_value).collect::<Vec<_>>();
-        let col = VecColumn::from(&data);
-        let normalized_header = crate::NormalizedHeader {
-            num_vals: col.num_vals(),
-            max_value: col.max_value(),
-        };
-        Codec::serialize(&VecColumn::from(&data), &mut bytes).unwrap();
-        Codec::open_from_bytes(OwnedBytes::new(bytes), normalized_header).unwrap()
-    }
-    fn bench_get<Codec: FastFieldCodec>(b: &mut Bencher, data: &[u64]) {
-        let col = get_reader_for_bench::<Codec>(data);
-        b.iter(|| {
-            let mut sum = 0u64;
-            for pos in value_iter() {
-                let val = col.get_val(pos as u64);
-                sum = sum.wrapping_add(val);
-            }
-            sum
-        });
-    }
-
-    #[inline(never)]
-    fn bench_get_dynamic_helper(b: &mut Bencher, col: Arc<dyn Column>) {
-        b.iter(|| {
-            let mut sum = 0u64;
-            for pos in value_iter() {
-                let val = col.get_val(pos as u64);
-                sum = sum.wrapping_add(val);
-            }
-            sum
-        });
-    }
-
-    fn bench_get_dynamic<Codec: FastFieldCodec>(b: &mut Bencher, data: &[u64]) {
-        let col = Arc::new(get_reader_for_bench::<Codec>(data));
-        bench_get_dynamic_helper(b, col);
-    }
-    fn bench_create<Codec: FastFieldCodec>(b: &mut Bencher, data: &[u64]) {
-        let min_value = *data.iter().min().unwrap();
-        let data = data.iter().map(|el| *el - min_value).collect::<Vec<_>>();
-
-        let mut bytes = Vec::new();
-        b.iter(|| {
-            bytes.clear();
-            Codec::serialize(&VecColumn::from(&data), &mut bytes).unwrap();
-        });
-    }
-
-    #[bench]
-    fn bench_fastfield_bitpack_create(b: &mut Bencher) {
-        let data: Vec<_> = get_data();
-        bench_create::<BitpackedCodec>(b, &data);
-    }
-    #[bench]
-    fn bench_fastfield_linearinterpol_create(b: &mut Bencher) {
-        let data: Vec<_> = get_data();
-        bench_create::<LinearCodec>(b, &data);
-    }
-    #[bench]
-    fn bench_fastfield_multilinearinterpol_create(b: &mut Bencher) {
-        let data: Vec<_> = get_data();
-        bench_create::<BlockwiseLinearCodec>(b, &data);
-    }
-    #[bench]
-    fn bench_fastfield_bitpack_get(b: &mut Bencher) {
-        let data: Vec<_> = get_data();
-        bench_get::<BitpackedCodec>(b, &data);
-    }
-    #[bench]
-    fn bench_fastfield_bitpack_get_dynamic(b: &mut Bencher) {
-        let data: Vec<_> = get_data();
-        bench_get_dynamic::<BitpackedCodec>(b, &data);
-    }
-    #[bench]
-    fn bench_fastfield_linearinterpol_get(b: &mut Bencher) {
-        let data: Vec<_> = get_data();
-        bench_get::<LinearCodec>(b, &data);
-    }
-    #[bench]
-    fn bench_fastfield_linearinterpol_get_dynamic(b: &mut Bencher) {
-        let data: Vec<_> = get_data();
-        bench_get_dynamic::<LinearCodec>(b, &data);
-    }
-    #[bench]
-    fn bench_fastfield_multilinearinterpol_get(b: &mut Bencher) {
-        let data: Vec<_> = get_data();
-        bench_get::<BlockwiseLinearCodec>(b, &data);
-    }
-    #[bench]
-    fn bench_fastfield_multilinearinterpol_get_dynamic(b: &mut Bencher) {
-        let data: Vec<_> = get_data();
-        bench_get_dynamic::<BlockwiseLinearCodec>(b, &data);
-    }
 }

fastfield_codecs/src/line.rs

@@ -1,202 +0,0 @@
-use std::io;
-use std::num::NonZeroU64;
-
-use common::{BinarySerializable, VInt};
-
-use crate::Column;
-
-const MID_POINT: u64 = (1u64 << 32) - 1u64;
-
-/// `Line` describes a line function `y: ax + b` using integer
-/// arithmetics.
-///
-/// The slope is in fact a decimal split into a 32 bit integer value,
-/// and a 32-bit decimal value.
-///
-/// The multiplication then becomes.
-/// `y = m * x >> 32 + b`
-#[derive(Debug, Clone, Copy, Default)]
-pub struct Line {
-    slope: u64,
-    intercept: u64,
-}
-
-/// Compute the line slope.
-///
-/// This function has the nice property of being
-/// invariant by translation.
-/// `
-///   compute_slope(y0, y1)
-/// = compute_slope(y0 + X % 2^64, y1 + X % 2^64)
-/// `
-fn compute_slope(y0: u64, y1: u64, num_vals: NonZeroU64) -> u64 {
-    let dy = y1.wrapping_sub(y0);
-    let sign = dy <= (1 << 63);
-    let abs_dy = if sign {
-        y1.wrapping_sub(y0)
-    } else {
-        y0.wrapping_sub(y1)
-    };
-    if abs_dy >= 1 << 32 {
-        // This is outside of realm we handle.
-        // Let's just bail.
-        return 0u64;
-    }
-
-    let abs_slope = (abs_dy << 32) / num_vals.get();
-    if sign {
-        abs_slope
-    } else {
-        // The complement does indeed create the
-        // opposite decreasing slope...
-        //
-        // Intuitively (without the bitshifts and % u64::MAX)
-        // ```
-        //    (x + shift)*(u64::MAX - abs_slope)
-        // -  (x * (u64::MAX - abs_slope))
-        // = - shift * abs_slope
-        // ```
-        u64::MAX - abs_slope
-    }
-}
-
-impl Line {
-    #[inline(always)]
-    pub fn eval(&self, x: u64) -> u64 {
-        let linear_part = (x.wrapping_mul(self.slope) >> 32) as i32 as u64;
-        self.intercept.wrapping_add(linear_part)
-    }
-
-    // Intercept is only computed from provided positions
-    pub fn train_from(
-        ys: &dyn Column,
-        positions_and_values: impl Iterator<Item = (u64, u64)>,
-    ) -> Self {
-        let num_vals = if let Some(num_vals) = NonZeroU64::new(ys.num_vals() - 1) {
-            num_vals
-        } else {
-            return Line::default();
-        };
-
-        let y0 = ys.get_val(0);
-        let y1 = ys.get_val(num_vals.get());
-
-        // We first independently pick our slope.
-        let slope = compute_slope(y0, y1, num_vals);
-
-        // We picked our slope. Note that it does not have to be perfect.
-        // Now we need to compute the best intercept.
-        //
-        // Intuitively, the best intercept is such that line passes through one of the
-        // `(i, ys[])`.
-        //
-        // The best intercept therefore has the form
-        // `y[i] - line.eval(i)` (using wrapping arithmetics).
-        // In other words, the best intercept is one of the `y - Line::eval(ys[i])`
-        // and our task is just to pick the one that minimizes our error.
-        //
-        // Without sorting our values, this is a difficult problem.
-        // We however rely on the following trick...
-        //
-        // We only focus on the case where the interpolation is half decent.
-        // If the line interpolation is doing its job on a dataset suited for it,
-        // we can hope that the maximum error won't be larger than `u64::MAX / 2`.
-        //
-        // In other words, even without the intercept the values `y - Line::eval(ys[i])` will all be
-        // within an interval that takes less than half of the modulo space of `u64`.
-        //
-        // Our task is therefore to identify this interval.
-        // Here we simply translate all of our values by `y0 - 2^63` and pick the min.
-        let mut line = Line {
-            slope,
-            intercept: 0,
-        };
-        let heuristic_shift = y0.wrapping_sub(MID_POINT);
-        line.intercept = positions_and_values
-            .map(|(pos, y)| y.wrapping_sub(line.eval(pos)))
-            .min_by_key(|&val| val.wrapping_sub(heuristic_shift))
-            .unwrap_or(0u64); //< Never happens.
-        line
-    }
-
-    /// Returns a line that attemps to approximate a function
-    /// f: i in 0..[ys.num_vals()) -> ys[i].
-    ///
-    /// - The approximation is always lower than the actual value.
-    /// Or more rigorously, formally `f(i).wrapping_sub(ys[i])` is small
-    /// for any i in [0..ys.len()).
-    /// - It computes without panicking for any value of it.
-    ///
-    /// This function is only invariable by translation if all of the
-    /// `ys` are packaged into half of the space. (See heuristic below)
-    pub fn train(ys: &dyn Column) -> Self {
-        Self::train_from(
-            ys,
-            ys.iter().enumerate().map(|(pos, val)| (pos as u64, val)),
-        )
-    }
-}
-
-impl BinarySerializable for Line {
-    fn serialize<W: io::Write>(&self, writer: &mut W) -> io::Result<()> {
-        VInt(self.slope).serialize(writer)?;
-        VInt(self.intercept).serialize(writer)?;
-        Ok(())
-    }
-
-    fn deserialize<R: io::Read>(reader: &mut R) -> io::Result<Self> {
-        let slope = VInt::deserialize(reader)?.0;
-        let intercept = VInt::deserialize(reader)?.0;
-        Ok(Line { slope, intercept })
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use crate::VecColumn;
-
-    /// Test training a line and ensuring that the maximum difference between
-    /// the data points and the line is `expected`.
-    ///
-    /// This function operates translation over the data for better coverage.
-    #[track_caller]
-    fn test_line_interpol_with_translation(ys: &[u64], expected: Option<u64>) {
-        let mut translations = vec![0, 100, u64::MAX / 2, u64::MAX, u64::MAX - 1];
-        translations.extend_from_slice(ys);
-        for translation in translations {
-            let translated_ys: Vec<u64> = ys
-                .iter()
-                .copied()
-                .map(|y| y.wrapping_add(translation))
-                .collect();
-            let largest_err = test_eval_max_err(&translated_ys);
-            assert_eq!(largest_err, expected);
-        }
-    }
-
-    fn test_eval_max_err(ys: &[u64]) -> Option<u64> {
-        let line = Line::train(&VecColumn::from(&ys));
-        ys.iter()
-            .enumerate()
-            .map(|(x, y)| y.wrapping_sub(line.eval(x as u64)))
-            .max()
-    }
-
-    #[test]
-    fn test_train() {
-        test_line_interpol_with_translation(&[11, 11, 11, 12, 12, 13], Some(1));
-        test_line_interpol_with_translation(&[13, 12, 12, 11, 11, 11], Some(1));
-        test_line_interpol_with_translation(&[13, 13, 12, 11, 11, 11], Some(1));
-        test_line_interpol_with_translation(&[13, 13, 12, 11, 11, 11], Some(1));
-        test_line_interpol_with_translation(&[u64::MAX - 1, 0, 0, 1], Some(1));
-        test_line_interpol_with_translation(&[u64::MAX - 1, u64::MAX, 0, 1], Some(0));
-        test_line_interpol_with_translation(&[0, 1, 2, 3, 5], Some(0));
-        test_line_interpol_with_translation(&[1, 2, 3, 4], Some(0));
-
-        let data: Vec<u64> = (0..255).collect();
-        test_line_interpol_with_translation(&data, Some(0));
-        let data: Vec<u64> = (0..255).map(|el| el * 2).collect();
-        test_line_interpol_with_translation(&data, Some(0));
-    }
-}

fastfield_codecs/src/linear.rs

@@ -1,229 +0,0 @@
-use std::io::{self, Write};
-
-use common::BinarySerializable;
-use ownedbytes::OwnedBytes;
-use tantivy_bitpacker::{compute_num_bits, BitPacker, BitUnpacker};
-
-use crate::column::EstimateColumn;
-use crate::line::Line;
-use crate::serialize::NormalizedHeader;
-use crate::{Column, FastFieldCodec, FastFieldCodecType};
-
-/// Depending on the field type, a different
-/// fast field is required.
-#[derive(Clone)]
-pub struct LinearReader {
-    data: OwnedBytes,
-    linear_params: LinearParams,
-    header: NormalizedHeader,
-}
-
-impl Column for LinearReader {
-    #[inline]
-    fn get_val(&self, doc: u64) -> u64 {
-        let interpoled_val: u64 = self.linear_params.line.eval(doc);
-        let bitpacked_diff = self.linear_params.bit_unpacker.get(doc, &self.data);
-        interpoled_val.wrapping_add(bitpacked_diff)
-    }
-
-    #[inline]
-    fn min_value(&self) -> u64 {
-        // The LinearReader assumes a normalized vector.
-        0u64
-    }
-
-    #[inline]
-    fn max_value(&self) -> u64 {
-        self.header.max_value
-    }
-
-    #[inline]
-    fn num_vals(&self) -> u64 {
-        self.header.num_vals
-    }
-}
-
-/// Fastfield serializer, which tries to guess values by linear interpolation
-/// and stores the difference bitpacked.
-pub struct LinearCodec;
-
-#[derive(Debug, Clone)]
-struct LinearParams {
-    line: Line,
-    bit_unpacker: BitUnpacker,
-}
-
-impl BinarySerializable for LinearParams {
-    fn serialize<W: io::Write>(&self, writer: &mut W) -> io::Result<()> {
-        self.line.serialize(writer)?;
-        self.bit_unpacker.bit_width().serialize(writer)?;
-        Ok(())
-    }
-
-    fn deserialize<R: io::Read>(reader: &mut R) -> io::Result<Self> {
-        let line = Line::deserialize(reader)?;
-        let bit_width = u8::deserialize(reader)?;
-        Ok(Self {
-            line,
-            bit_unpacker: BitUnpacker::new(bit_width),
-        })
-    }
-}
-
-impl FastFieldCodec for LinearCodec {
-    const CODEC_TYPE: FastFieldCodecType = FastFieldCodecType::Linear;
-
-    type Reader = LinearReader;
-
-    /// Opens a fast field given a file.
-    fn open_from_bytes(mut data: OwnedBytes, header: NormalizedHeader) -> io::Result<Self::Reader> {
-        let linear_params = LinearParams::deserialize(&mut data)?;
-        Ok(LinearReader {
-            data,
-            linear_params,
-            header,
-        })
-    }
-
-    /// Creates a new fast field serializer.
-    fn serialize(column: &dyn Column, write: &mut impl Write) -> io::Result<()> {
-        assert_eq!(column.min_value(), 0);
-        let line = Line::train(column);
-
-        let max_offset_from_line = column
-            .iter()
-            .enumerate()
-            .map(|(pos, actual_value)| {
-                let calculated_value = line.eval(pos as u64);
-                actual_value.wrapping_sub(calculated_value)
-            })
-            .max()
-            .unwrap();
-
-        let num_bits = compute_num_bits(max_offset_from_line);
-        let linear_params = LinearParams {
-            line,
-            bit_unpacker: BitUnpacker::new(num_bits),
-        };
-        linear_params.serialize(write)?;
-
-        let mut bit_packer = BitPacker::new();
-        for (pos, actual_value) in column.iter().enumerate() {
-            let calculated_value = line.eval(pos as u64);
-            let offset = actual_value.wrapping_sub(calculated_value);
-            bit_packer.write(offset, num_bits, write)?;
-        }
-        bit_packer.close(write)?;
-
-        Ok(())
-    }
-
-    /// estimation for linear interpolation is hard because, you don't know
-    /// where the local maxima for the deviation of the calculated value are and
-    /// the offset to shift all values to >=0 is also unknown.
-    #[allow(clippy::question_mark)]
-    fn estimate(column: &EstimateColumn) -> Option<f32> {
-        if column.num_vals() < 3 {
-            return None; // disable compressor for this case
-        }
-
-        // let's sample at 0%, 5%, 10% .. 95%, 100%
-        let num_vals = column.num_vals() as f32 / 100.0;
-        let sample_positions_and_values = (0..20)
-            .map(|pos| (num_vals * pos as f32 * 5.0) as u64)
-            .map(|pos| (pos, column.get_val(pos)))
-            .collect::<Vec<_>>();
-
-        let line = { Line::train_from(column, sample_positions_and_values.iter().cloned()) };
-
-        let estimated_bit_width = sample_positions_and_values
-            .into_iter()
-            .map(|(pos, actual_value)| {
-                let interpolated_val = line.eval(pos as u64);
-                actual_value.wrapping_sub(interpolated_val)
-            })
-            .map(|diff| ((diff as f32 * 1.5) * 2.0) as u64)
-            .map(compute_num_bits)
-            .max()
-            .unwrap_or(0);
-
-        let num_bits = (estimated_bit_width as u64 * column.num_vals() as u64) + 64;
-        let num_bits_uncompressed = 64 * column.num_vals();
-        Some(num_bits as f32 / num_bits_uncompressed as f32)
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use rand::RngCore;
-
-    use super::*;
-    use crate::tests::get_codec_test_datasets;
-
-    fn create_and_validate(data: &[u64], name: &str) -> Option<(f32, f32)> {
-        crate::tests::create_and_validate::<LinearCodec>(data, name)
-    }
-
-    #[test]
-    fn test_compression() {
-        let data = (10..=6_000_u64).collect::<Vec<_>>();
-        let (estimate, actual_compression) =
-            create_and_validate(&data, "simple monotonically large").unwrap();
-
-        assert_le!(actual_compression, 0.001);
-        assert_le!(estimate, 0.02);
-    }
-
-    #[test]
-    fn test_with_codec_datasets() {
-        let data_sets = get_codec_test_datasets();
-        for (mut data, name) in data_sets {
-            create_and_validate(&data, name);
-            data.reverse();
-            create_and_validate(&data, name);
-        }
-    }
-    #[test]
-    fn linear_interpol_fast_field_test_large_amplitude() {
-        let data = vec![
-            i64::MAX as u64 / 2,
-            i64::MAX as u64 / 3,
-            i64::MAX as u64 / 2,
-        ];
-
-        create_and_validate(&data, "large amplitude");
-    }
-
-    #[test]
-    fn overflow_error_test() {
-        let data = vec![1572656989877777, 1170935903116329, 720575940379279, 0];
-        create_and_validate(&data, "overflow test");
-    }
-
-    #[test]
-    fn linear_interpol_fast_concave_data() {
-        let data = vec![0, 1, 2, 5, 8, 10, 20, 50];
-        create_and_validate(&data, "concave data");
-    }
-    #[test]
-    fn linear_interpol_fast_convex_data() {
-        let data = vec![0, 40, 60, 70, 75, 77];
-        create_and_validate(&data, "convex data");
-    }
-    #[test]
-    fn linear_interpol_fast_field_test_simple() {
-        let data = (10..=20_u64).collect::<Vec<_>>();
-        create_and_validate(&data, "simple monotonically");
-    }
-
-    #[test]
-    fn linear_interpol_fast_field_rand() {
-        let mut rng = rand::thread_rng();
-        for _ in 0..50 {
-            let mut data = (0..10_000).map(|_| rng.next_u64()).collect::<Vec<_>>();
-            create_and_validate(&data, "random");
-            data.reverse();
-            create_and_validate(&data, "random");
-        }
-    }
-}

fastfield_codecs/src/linearinterpol.rs

@@ -0,0 +1,300 @@
+use std::io::{self, Read, Write};
+use std::ops::Sub;
+
+use common::{BinarySerializable, FixedSize};
+use ownedbytes::OwnedBytes;
+use tantivy_bitpacker::{compute_num_bits, BitPacker, BitUnpacker};
+
+use crate::{FastFieldCodec, FastFieldCodecReader, FastFieldStats};
+
+/// Depending on the field type, a different
+/// fast field is required.
+#[derive(Clone)]
+pub struct LinearInterpolFastFieldReader {
+    data: OwnedBytes,
+    bit_unpacker: BitUnpacker,
+    pub footer: LinearInterpolFooter,
+    pub slope: f32,
+}
+
+#[derive(Clone, Debug)]
+pub struct LinearInterpolFooter {
+    pub relative_max_value: u64,
+    pub offset: u64,
+    pub first_val: u64,
+    pub last_val: u64,
+    pub num_vals: u64,
+    pub min_value: u64,
+    pub max_value: u64,
+}
+
+impl BinarySerializable for LinearInterpolFooter {
+    fn serialize<W: Write>(&self, write: &mut W) -> io::Result<()> {
+        self.relative_max_value.serialize(write)?;
+        self.offset.serialize(write)?;
+        self.first_val.serialize(write)?;
+        self.last_val.serialize(write)?;
+        self.num_vals.serialize(write)?;
+        self.min_value.serialize(write)?;
+        self.max_value.serialize(write)?;
+        Ok(())
+    }
+
+    fn deserialize<R: Read>(reader: &mut R) -> io::Result<LinearInterpolFooter> {
+        Ok(LinearInterpolFooter {
+            relative_max_value: u64::deserialize(reader)?,
+            offset: u64::deserialize(reader)?,
+            first_val: u64::deserialize(reader)?,
+            last_val: u64::deserialize(reader)?,
+            num_vals: u64::deserialize(reader)?,
+            min_value: u64::deserialize(reader)?,
+            max_value: u64::deserialize(reader)?,
+        })
+    }
+}
+
+impl FixedSize for LinearInterpolFooter {
+    const SIZE_IN_BYTES: usize = 56;
+}
+
+impl FastFieldCodecReader for LinearInterpolFastFieldReader {
+    #[inline]
+    fn get_u64(&self, doc: u64) -> u64 {
+        let calculated_value = get_calculated_value(self.footer.first_val, doc, self.slope);
+        (calculated_value + self.bit_unpacker.get(doc, &self.data)) - self.footer.offset
+    }
+
+    #[inline]
+    fn min_value(&self) -> u64 {
+        self.footer.min_value
+    }
+    #[inline]
+    fn max_value(&self) -> u64 {
+        self.footer.max_value
+    }
+}
+
+/// Fastfield serializer, which tries to guess values by linear interpolation
+/// and stores the difference bitpacked.
+pub struct LinearInterpolCodec;
+
+#[inline]
+fn get_slope(first_val: u64, last_val: u64, num_vals: u64) -> f32 {
+    if num_vals <= 1 {
+        return 0.0;
+    }
+    //  We calculate the slope with f64 high precision and use the result in lower precision f32
+    //  This is done in order to handle estimations for very large values like i64::MAX
+    ((last_val as f64 - first_val as f64) / (num_vals as u64 - 1) as f64) as f32
+}
+
+#[inline]
+fn get_calculated_value(first_val: u64, pos: u64, slope: f32) -> u64 {
+    first_val + (pos as f32 * slope) as u64
+}
+
+impl FastFieldCodec for LinearInterpolCodec {
+    const NAME: &'static str = "LinearInterpol";
+
+    type Reader = LinearInterpolFastFieldReader;
+
+    /// Opens a fast field given a file.
+    fn open_from_bytes(bytes: OwnedBytes) -> io::Result<Self::Reader> {
+        let footer_offset = bytes.len() - LinearInterpolFooter::SIZE_IN_BYTES;
+        let (data, mut footer) = bytes.split(footer_offset);
+        let footer = LinearInterpolFooter::deserialize(&mut footer)?;
+        let slope = get_slope(footer.first_val, footer.last_val, footer.num_vals);
+        let num_bits = compute_num_bits(footer.relative_max_value);
+        let bit_unpacker = BitUnpacker::new(num_bits);
+        Ok(LinearInterpolFastFieldReader {
+            data,
+            bit_unpacker,
+            footer,
+            slope,
+        })
+    }
+
+    /// Creates a new fast field serializer.
+    fn serialize(
+        &self,
+        write: &mut impl Write,
+        vals: &[u64],
+        stats: FastFieldStats,
+    ) -> io::Result<()> {
+        assert!(stats.min_value <= stats.max_value);
+
+        let first_val = vals[0];
+        let last_val = vals[vals.len() - 1];
+
+        let slope = get_slope(first_val, last_val, stats.num_vals);
+        // calculate offset to ensure all values are positive
+        let mut offset = 0;
+        let mut rel_positive_max = 0;
+        for (pos, actual_value) in vals.iter().copied().enumerate() {
+            let calculated_value = get_calculated_value(first_val, pos as u64, slope);
+            if calculated_value > actual_value {
+                // negative value we need to apply an offset
+                // we ignore negative values in the max value calculation, because negative values
+                // will be offset to 0
+                offset = offset.max(calculated_value - actual_value);
+            } else {
+                // positive value no offset reuqired
+                rel_positive_max = rel_positive_max.max(actual_value - calculated_value);
+            }
+        }
+
+        // rel_positive_max will be adjusted by offset
+        let relative_max_value = rel_positive_max + offset;
+
+        let num_bits = compute_num_bits(relative_max_value);
+        let mut bit_packer = BitPacker::new();
+        for (pos, val) in vals.iter().copied().enumerate() {
+            let calculated_value = get_calculated_value(first_val, pos as u64, slope);
+            let diff = (val + offset) - calculated_value;
+            bit_packer.write(diff, num_bits, write)?;
+        }
+        bit_packer.close(write)?;
+
+        let footer = LinearInterpolFooter {
+            relative_max_value,
+            offset,
+            first_val,
+            last_val,
+            num_vals: stats.num_vals,
+            min_value: stats.min_value,
+            max_value: stats.max_value,
+        };
+        footer.serialize(write)?;
+        Ok(())
+    }
+    fn is_applicable(_vals: &[u64], stats: FastFieldStats) -> bool {
+        if stats.num_vals < 3 {
+            return false; // disable compressor for this case
+        }
+        // On serialisation the offset is added to the actual value.
+        // We need to make sure this won't run into overflow calculation issues.
+        // For this we take the maximum theroretical offset and add this to the max value.
+        // If this doesn't overflow the algorithm should be fine
+        let theorethical_maximum_offset = stats.max_value - stats.min_value;
+        if stats
+            .max_value
+            .checked_add(theorethical_maximum_offset)
+            .is_none()
+        {
+            return false;
+        }
+        true
+    }
+    /// estimation for linear interpolation is hard because, you don't know
+    /// where the local maxima for the deviation of the calculated value are and
+    /// the offset to shift all values to >=0 is also unknown.
+    fn estimate(vals: &[u64], stats: FastFieldStats) -> f32 {
+        let first_val = vals[0];
+        let last_val = vals[vals.len() - 1];
+        let slope = get_slope(first_val, last_val, stats.num_vals);
+
+        // let's sample at 0%, 5%, 10% .. 95%, 100%
+        let num_vals = stats.num_vals as f32 / 100.0;
+        let sample_positions: Vec<usize> = (0..20)
+            .map(|pos| (num_vals * pos as f32 * 5.0) as usize)
+            .collect::<Vec<_>>();
+
+        let max_distance = sample_positions
+            .into_iter()
+            .map(|pos| {
+                let calculated_value = get_calculated_value(first_val, pos as u64, slope);
+                let actual_value = vals[pos];
+                distance(calculated_value, actual_value)
+            })
+            .max()
+            .unwrap_or(0);
+
+        // the theory would be that we don't have the actual max_distance, but we are close within
+        // 50% threshold.
+        // It is multiplied by 2 because in a log case scenario the line would be as much above as
+        // below. So the offset would = max_distance
+        //
+        let relative_max_value = (max_distance as f32 * 1.5) * 2.0;
+
+        let num_bits = compute_num_bits(relative_max_value as u64) as u64 * stats.num_vals as u64
+            + LinearInterpolFooter::SIZE_IN_BYTES as u64;
+        let num_bits_uncompressed = 64 * stats.num_vals;
+        num_bits as f32 / num_bits_uncompressed as f32
+    }
+}
+
+#[inline]
+fn distance<T: Sub<Output = T> + Ord>(x: T, y: T) -> T {
+    if x < y {
+        y - x
+    } else {
+        x - y
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::tests::get_codec_test_data_sets;
+
+    fn create_and_validate(data: &[u64], name: &str) -> (f32, f32) {
+        crate::tests::create_and_validate(&LinearInterpolCodec, data, name)
+    }
+
+    #[test]
+    fn test_compression() {
+        let data = (10..=6_000_u64).collect::<Vec<_>>();
+        let (estimate, actual_compression) =
+            create_and_validate(&data, "simple monotonically large");
+
+        assert!(actual_compression < 0.01);
+        assert!(estimate < 0.01);
+    }
+
+    #[test]
+    fn test_with_codec_data_sets() {
+        let data_sets = get_codec_test_data_sets();
+        for (mut data, name) in data_sets {
+            create_and_validate(&data, name);
+            data.reverse();
+            create_and_validate(&data, name);
+        }
+    }
+    #[test]
+    fn linear_interpol_fast_field_test_large_amplitude() {
+        let data = vec![
+            i64::MAX as u64 / 2,
+            i64::MAX as u64 / 3,
+            i64::MAX as u64 / 2,
+        ];
+
+        create_and_validate(&data, "large amplitude");
+    }
+    #[test]
+    fn linear_interpol_fast_concave_data() {
+        let data = vec![0, 1, 2, 5, 8, 10, 20, 50];
+        create_and_validate(&data, "concave data");
+    }
+    #[test]
+    fn linear_interpol_fast_convex_data() {
+        let data = vec![0, 40, 60, 70, 75, 77];
+        create_and_validate(&data, "convex data");
+    }
+    #[test]
+    fn linear_interpol_fast_field_test_simple() {
+        let data = (10..=20_u64).collect::<Vec<_>>();
+
+        create_and_validate(&data, "simple monotonically");
+    }
+
+    #[test]
+    fn linear_interpol_fast_field_rand() {
+        for _ in 0..5000 {
+            let mut data = (0..50).map(|_| rand::random::<u64>()).collect::<Vec<_>>();
+            create_and_validate(&data, "random");
+
+            data.reverse();
+            create_and_validate(&data, "random");
+        }
+    }
+}

fastfield_codecs/src/main.rs

@@ -1,161 +1,51 @@
 #[macro_use]
 extern crate prettytable;
-use std::collections::HashSet;
-use std::env;
-use std::io::BufRead;
-use std::net::{IpAddr, Ipv6Addr};
-use std::str::FromStr;
-
-use fastfield_codecs::{open_u128, serialize_u128, Column, FastFieldCodecType, VecColumn};
-use itertools::Itertools;
-use measure_time::print_time;
-use ownedbytes::OwnedBytes;
+// use fastfield_codecs::linearinterpol::LinearInterpolFastFieldSerializer;
+// use fastfield_codecs::multilinearinterpol::MultiLinearInterpolFastFieldSerializer;
+use fastfield_codecs::bitpacked::BitpackedFastFieldCodec;
+use fastfield_codecs::{FastFieldCodec, FastFieldStats};
 use prettytable::{Cell, Row, Table};
 
-fn print_set_stats(ip_addrs: &[u128]) {
-    println!("NumIps\t{}", ip_addrs.len());
-    let ip_addr_set: HashSet<u128> = ip_addrs.iter().cloned().collect();
-    println!("NumUniqueIps\t{}", ip_addr_set.len());
-    let ratio_unique = ip_addr_set.len() as f64 / ip_addrs.len() as f64;
-    println!("RatioUniqueOverTotal\t{ratio_unique:.4}");
-
-    // histogram
-    let mut ip_addrs = ip_addrs.to_vec();
-    ip_addrs.sort();
-    let mut cnts: Vec<usize> = ip_addrs
-        .into_iter()
-        .dedup_with_count()
-        .map(|(cnt, _)| cnt)
-        .collect();
-    cnts.sort();
-
-    let top_256_cnt: usize = cnts.iter().rev().take(256).sum();
-    let top_128_cnt: usize = cnts.iter().rev().take(128).sum();
-    let top_64_cnt: usize = cnts.iter().rev().take(64).sum();
-    let top_8_cnt: usize = cnts.iter().rev().take(8).sum();
-    let total: usize = cnts.iter().sum();
-
-    println!("{}", total);
-    println!("{}", top_256_cnt);
-    println!("{}", top_128_cnt);
-    println!("Percentage Top8 {:02}", top_8_cnt as f32 / total as f32);
-    println!("Percentage Top64 {:02}", top_64_cnt as f32 / total as f32);
-    println!("Percentage Top128 {:02}", top_128_cnt as f32 / total as f32);
-    println!("Percentage Top256 {:02}", top_256_cnt as f32 / total as f32);
-
-    let mut cnts: Vec<(usize, usize)> = cnts.into_iter().dedup_with_count().collect();
-    cnts.sort_by(|a, b| {
-        if a.1 == b.1 {
-            a.0.cmp(&b.0)
-        } else {
-            b.1.cmp(&a.1)
-        }
-    });
-}
-
-fn ip_dataset() -> Vec<u128> {
-    let mut ip_addr_v4 = 0;
-
-    let stdin = std::io::stdin();
-    let ip_addrs: Vec<u128> = stdin
-        .lock()
-        .lines()
-        .flat_map(|line| {
-            let line = line.unwrap();
-            let line = line.trim();
-            let ip_addr = IpAddr::from_str(line.trim()).ok()?;
-            if ip_addr.is_ipv4() {
-                ip_addr_v4 += 1;
-            }
-            let ip_addr_v6: Ipv6Addr = match ip_addr {
-                IpAddr::V4(v4) => v4.to_ipv6_mapped(),
-                IpAddr::V6(v6) => v6,
-            };
-            Some(ip_addr_v6)
-        })
-        .map(|ip_v6| u128::from_be_bytes(ip_v6.octets()))
-        .collect();
-
-    println!("IpAddrsAny\t{}", ip_addrs.len());
-    println!("IpAddrsV4\t{}", ip_addr_v4);
-
-    ip_addrs
-}
-
-fn bench_ip() {
-    let dataset = ip_dataset();
-    print_set_stats(&dataset);
-
-    // Chunks
-    {
-        let mut data = vec![];
-        for dataset in dataset.chunks(500_000) {
-            serialize_u128(VecColumn::from(dataset), &mut data).unwrap();
-        }
-        let compression = data.len() as f64 / (dataset.len() * 16) as f64;
-        println!("Compression 50_000 chunks {:.4}", compression);
-        println!(
-            "Num Bits per elem {:.2}",
-            (data.len() * 8) as f32 / dataset.len() as f32
-        );
-    }
-
-    let mut data = vec![];
-    serialize_u128(VecColumn::from(&dataset), &mut data).unwrap();
-
-    let compression = data.len() as f64 / (dataset.len() * 16) as f64;
-    println!("Compression {:.2}", compression);
-    println!(
-        "Num Bits per elem {:.2}",
-        (data.len() * 8) as f32 / dataset.len() as f32
-    );
-
-    let decompressor = open_u128(OwnedBytes::new(data)).unwrap();
-    // Sample some ranges
-    for value in dataset.iter().take(1110).skip(1100).cloned() {
-        print_time!("get range");
-        let doc_values = decompressor.get_between_vals(value..=value);
-        println!("{:?}", doc_values.len());
-    }
-}
-
 fn main() {
-    if env::args().nth(1).unwrap() == "bench_ip" {
-        bench_ip();
-        return;
-    }
-
     let mut table = Table::new();
 
     // Add a row per time
     table.add_row(row!["", "Compression Ratio", "Compression Estimation"]);
 
     for (data, data_set_name) in get_codec_test_data_sets() {
-        let results: Vec<(f32, f32, FastFieldCodecType)> = [
-            serialize_with_codec(&data, FastFieldCodecType::Bitpacked),
-            serialize_with_codec(&data, FastFieldCodecType::Linear),
-            serialize_with_codec(&data, FastFieldCodecType::BlockwiseLinear),
-        ]
-        .into_iter()
-        .flatten()
-        .collect();
+        let mut results = Vec::new();
+        // let res = serialize_with_codec::<LinearInterpolFastFieldSerializer>(&data);
+        // results.push(res);
+        // let res = serialize_with_codec::<MultiLinearInterpolFastFieldSerializer>(&data);
+        // results.push(res);
+        let res = serialize_with_codec(&BitpackedFastFieldCodec, &data);
+        results.push(res);
+
+        // let best_estimation_codec = results
+        //.iter()
+        //.min_by(|res1, res2| res1.partial_cmp(&res2).unwrap())
+        //.unwrap();
         let best_compression_ratio_codec = results
             .iter()
-            .min_by(|&res1, &res2| res1.partial_cmp(res2).unwrap())
+            .min_by(|res1, res2| res1.partial_cmp(res2).unwrap())
             .cloned()
             .unwrap();
 
         table.add_row(Row::new(vec![Cell::new(data_set_name).style_spec("Bbb")]));
-        for (est, comp, codec_type) in results {
-            let est_cell = est.to_string();
-            let ratio_cell = comp.to_string();
+        for (is_applicable, est, comp, name) in results {
+            let (est_cell, ratio_cell) = if !is_applicable {
+                ("Codec Disabled".to_string(), "".to_string())
+            } else {
+                (est.to_string(), comp.to_string())
+            };
             let style = if comp == best_compression_ratio_codec.1 {
                 "Fb"
             } else {
                 ""
             };
+
             table.add_row(Row::new(vec![
-                Cell::new(&format!("{codec_type:?}")).style_spec("bFg"),
+                Cell::new(name).style_spec("bFg"),
                 Cell::new(&ratio_cell).style_spec(style),
                 Cell::new(&est_cell).style_spec(""),
             ]));
@@ -200,14 +90,30 @@ pub fn get_codec_test_data_sets() -> Vec<(Vec<u64>, &'static str)> {
     data_and_names
 }
 
-pub fn serialize_with_codec(
+pub fn serialize_with_codec<S: FastFieldCodec>(
+    codec: &S,
     data: &[u64],
-    codec_type: FastFieldCodecType,
-) -> Option<(f32, f32, FastFieldCodecType)> {
-    let col = VecColumn::from(data);
-    let estimation = fastfield_codecs::estimate(&col, codec_type)?;
-    let mut out = Vec::new();
-    fastfield_codecs::serialize(&col, &mut out, &[codec_type]).ok()?;
-    let actual_compression = out.len() as f32 / (col.num_vals() * 8) as f32;
-    Some((estimation, actual_compression, codec_type))
+) -> (bool, f32, f32, &'static str) {
+    let is_applicable = S::is_applicable(&data, stats_from_vec(data));
+    if !is_applicable {
+        return (false, 0.0, 0.0, S::NAME);
+    }
+    let estimation = S::estimate(&data, stats_from_vec(data));
+    let mut out = vec![];
+    codec
+        .serialize(&mut out, &data, stats_from_vec(data))
+        .unwrap();
+
+    let actual_compression = out.len() as f32 / (data.len() * 8) as f32;
+    (true, estimation, actual_compression, S::NAME)
+}
+
+pub fn stats_from_vec(data: &[u64]) -> FastFieldStats {
+    let min_value = data.iter().cloned().min().unwrap_or(0);
+    let max_value = data.iter().cloned().max().unwrap_or(0);
+    FastFieldStats {
+        min_value,
+        max_value,
+        num_vals: data.len() as u64,
+    }
 }

fastfield_codecs/src/monotonic_mapping.rs

@@ -1,56 +0,0 @@
-pub trait MonotonicallyMappableToU64: 'static + PartialOrd + Copy + Send + Sync {
-    /// Converts a value to u64.
-    ///
-    /// Internally all fast field values are encoded as u64.
-    fn to_u64(self) -> u64;
-
-    /// Converts a value from u64
-    ///
-    /// Internally all fast field values are encoded as u64.
-    /// **Note: To be used for converting encoded Term, Posting values.**
-    fn from_u64(val: u64) -> Self;
-}
-
-impl MonotonicallyMappableToU64 for u64 {
-    fn to_u64(self) -> u64 {
-        self
-    }
-
-    fn from_u64(val: u64) -> Self {
-        val
-    }
-}
-
-impl MonotonicallyMappableToU64 for i64 {
-    #[inline(always)]
-    fn to_u64(self) -> u64 {
-        common::i64_to_u64(self)
-    }
-
-    #[inline(always)]
-    fn from_u64(val: u64) -> Self {
-        common::u64_to_i64(val)
-    }
-}
-
-impl MonotonicallyMappableToU64 for bool {
-    #[inline(always)]
-    fn to_u64(self) -> u64 {
-        u64::from(self)
-    }
-
-    #[inline(always)]
-    fn from_u64(val: u64) -> Self {
-        val > 0
-    }
-}
-
-impl MonotonicallyMappableToU64 for f64 {
-    fn to_u64(self) -> u64 {
-        common::f64_to_u64(self)
-    }
-
-    fn from_u64(val: u64) -> Self {
-        common::u64_to_f64(val)
-    }
-}

fastfield_codecs/src/multilinearinterpol.rs

@@ -0,0 +1,413 @@
+//! MultiLinearInterpol compressor uses linear interpolation to guess a values and stores the
+//! offset, but in blocks of 512.
+//!
+//! With a CHUNK_SIZE of 512 and 29 byte metadata per block, we get a overhead for metadata of 232 /
+//! 512 = 0,45 bits per element. The additional space required per element in a block is the the
+//! maximum deviation of the linear interpolation estimation function.
+//!
+//! E.g. if the maximum deviation of an element is 12, all elements cost 4bits.
+//!
+//! Size per block:
+//! Num Elements * Maximum Deviation from Interpolation + 29 Byte Metadata
+
+use std::io::{self, Read, Write};
+use std::ops::Sub;
+
+use common::{BinarySerializable, CountingWriter, DeserializeFrom};
+use ownedbytes::OwnedBytes;
+use tantivy_bitpacker::{compute_num_bits, BitPacker, BitUnpacker};
+
+use crate::{FastFieldCodec, FastFieldCodecReader, FastFieldStats};
+
+const CHUNK_SIZE: usize = 512;
+
+/// Depending on the field type, a different
+/// fast field is required.
+#[derive(Clone)]
+pub struct MultiLinearInterpolFastFieldReader {
+    data: OwnedBytes,
+    pub footer: MultiLinearInterpolFooter,
+}
+
+#[derive(Clone, Debug, Default)]
+struct Function {
+    // The offset in the data is required, because we have different bit_widths per block
+    data_start_offset: u64,
+    // start_pos in the block will be CHUNK_SIZE * BLOCK_NUM
+    start_pos: u64,
+    // only used during serialization, 0 after deserialization
+    end_pos: u64,
+    // only used during serialization, 0 after deserialization
+    value_start_pos: u64,
+    // only used during serialization, 0 after deserialization
+    value_end_pos: u64,
+    slope: f32,
+    // The offset so that all values are positive when writing them
+    positive_val_offset: u64,
+    num_bits: u8,
+    bit_unpacker: BitUnpacker,
+}
+
+impl Function {
+    fn calc_slope(&mut self) {
+        let num_vals = self.end_pos - self.start_pos;
+        self.slope = get_slope(self.value_start_pos, self.value_end_pos, num_vals);
+    }
+    // split the interpolation into two function, change self and return the second split
+    fn split(&mut self, split_pos: u64, split_pos_value: u64) -> Function {
+        let mut new_function = Function {
+            start_pos: split_pos,
+            end_pos: self.end_pos,
+            value_start_pos: split_pos_value,
+            value_end_pos: self.value_end_pos,
+            ..Default::default()
+        };
+        new_function.calc_slope();
+        self.end_pos = split_pos;
+        self.value_end_pos = split_pos_value;
+        self.calc_slope();
+        new_function
+    }
+}
+
+impl BinarySerializable for Function {
+    fn serialize<W: Write>(&self, write: &mut W) -> io::Result<()> {
+        self.data_start_offset.serialize(write)?;
+        self.value_start_pos.serialize(write)?;
+        self.positive_val_offset.serialize(write)?;
+        self.slope.serialize(write)?;
+        self.num_bits.serialize(write)?;
+        Ok(())
+    }
+
+    fn deserialize<R: Read>(reader: &mut R) -> io::Result<Function> {
+        let data_start_offset = u64::deserialize(reader)?;
+        let value_start_pos = u64::deserialize(reader)?;
+        let offset = u64::deserialize(reader)?;
+        let slope = f32::deserialize(reader)?;
+        let num_bits = u8::deserialize(reader)?;
+        let interpolation = Function {
+            data_start_offset,
+            value_start_pos,
+            positive_val_offset: offset,
+            num_bits,
+            bit_unpacker: BitUnpacker::new(num_bits),
+            slope,
+            ..Default::default()
+        };
+
+        Ok(interpolation)
+    }
+}
+
+#[derive(Clone, Debug)]
+pub struct MultiLinearInterpolFooter {
+    pub num_vals: u64,
+    pub min_value: u64,
+    pub max_value: u64,
+    interpolations: Vec<Function>,
+}
+
+impl BinarySerializable for MultiLinearInterpolFooter {
+    fn serialize<W: Write>(&self, write: &mut W) -> io::Result<()> {
+        let mut out = vec![];
+        self.num_vals.serialize(&mut out)?;
+        self.min_value.serialize(&mut out)?;
+        self.max_value.serialize(&mut out)?;
+        self.interpolations.serialize(&mut out)?;
+        write.write_all(&out)?;
+        (out.len() as u32).serialize(write)?;
+        Ok(())
+    }
+
+    fn deserialize<R: Read>(reader: &mut R) -> io::Result<MultiLinearInterpolFooter> {
+        let mut footer = MultiLinearInterpolFooter {
+            num_vals: u64::deserialize(reader)?,
+            min_value: u64::deserialize(reader)?,
+            max_value: u64::deserialize(reader)?,
+            interpolations: Vec::<Function>::deserialize(reader)?,
+        };
+        for (num, interpol) in footer.interpolations.iter_mut().enumerate() {
+            interpol.start_pos = (CHUNK_SIZE * num) as u64;
+        }
+        Ok(footer)
+    }
+}
+
+#[inline]
+fn get_interpolation_function(doc: u64, interpolations: &[Function]) -> &Function {
+    &interpolations[doc as usize / CHUNK_SIZE]
+}
+
+impl FastFieldCodecReader for MultiLinearInterpolFastFieldReader {
+    #[inline]
+    fn get_u64(&self, doc: u64) -> u64 {
+        let interpolation = get_interpolation_function(doc, &self.footer.interpolations);
+        let doc = doc - interpolation.start_pos;
+        let calculated_value =
+            get_calculated_value(interpolation.value_start_pos, doc, interpolation.slope);
+        let diff = interpolation
+            .bit_unpacker
+            .get(doc, &self.data[interpolation.data_start_offset as usize..]);
+        (calculated_value + diff) - interpolation.positive_val_offset
+    }
+
+    #[inline]
+    fn min_value(&self) -> u64 {
+        self.footer.min_value
+    }
+    #[inline]
+    fn max_value(&self) -> u64 {
+        self.footer.max_value
+    }
+}
+
+#[inline]
+fn get_slope(first_val: u64, last_val: u64, num_vals: u64) -> f32 {
+    ((last_val as f64 - first_val as f64) / (num_vals as u64 - 1) as f64) as f32
+}
+
+#[inline]
+fn get_calculated_value(first_val: u64, pos: u64, slope: f32) -> u64 {
+    (first_val as i64 + (pos as f32 * slope) as i64) as u64
+}
+
+/// Same as LinearInterpolFastFieldSerializer, but working on chunks of CHUNK_SIZE elements.
+pub struct MultiLinearInterpolFastFieldCodec;
+
+impl FastFieldCodec for MultiLinearInterpolFastFieldCodec {
+    const NAME: &'static str = "MultiLinearInterpol";
+
+    type Reader = MultiLinearInterpolFastFieldReader;
+
+    /// Opens a fast field given a file.
+    fn open_from_bytes(bytes: OwnedBytes) -> io::Result<Self::Reader> {
+        let footer_len: u32 = (&bytes[bytes.len() - 4..]).deserialize()?;
+        let footer_offset = bytes.len() - 4 - footer_len as usize;
+        let (data, mut footer) = bytes.split(footer_offset);
+        let footer = MultiLinearInterpolFooter::deserialize(&mut footer)?;
+        Ok(MultiLinearInterpolFastFieldReader { data, footer })
+    }
+
+    /// Creates a new fast field serializer.
+    fn serialize(
+        &self,
+        write: &mut impl io::Write,
+        vals: &[u64],
+        stats: FastFieldStats,
+    ) -> io::Result<()> {
+        assert!(stats.min_value <= stats.max_value);
+
+        let first_val = vals[0];
+        let last_val = vals[vals.len() - 1];
+
+        let mut first_function = Function {
+            end_pos: stats.num_vals,
+            value_start_pos: first_val,
+            value_end_pos: last_val,
+            ..Default::default()
+        };
+        first_function.calc_slope();
+        let mut interpolations = vec![first_function];
+
+        //// let's split this into chunks of CHUNK_SIZE
+        for vals_pos in (0..vals.len()).step_by(CHUNK_SIZE).skip(1) {
+            let new_fun = {
+                let current_interpolation = interpolations.last_mut().unwrap();
+                current_interpolation.split(vals_pos as u64, vals[vals_pos])
+            };
+            interpolations.push(new_fun);
+        }
+        // calculate offset and max (-> numbits) for each function
+        for interpolation in &mut interpolations {
+            let mut offset = 0;
+            let mut rel_positive_max = 0;
+            for (pos, actual_value) in vals
+                [interpolation.start_pos as usize..interpolation.end_pos as usize]
+                .iter()
+                .cloned()
+                .enumerate()
+            {
+                let calculated_value = get_calculated_value(
+                    interpolation.value_start_pos,
+                    pos as u64,
+                    interpolation.slope,
+                );
+                if calculated_value > actual_value {
+                    // negative value we need to apply an offset
+                    // we ignore negative values in the max value calculation, because negative
+                    // values will be offset to 0
+                    offset = offset.max(calculated_value - actual_value);
+                } else {
+                    // positive value no offset reuqired
+                    rel_positive_max = rel_positive_max.max(actual_value - calculated_value);
+                }
+            }
+
+            interpolation.positive_val_offset = offset;
+            interpolation.num_bits = compute_num_bits(rel_positive_max + offset);
+        }
+        let mut bit_packer = BitPacker::new();
+
+        let write = &mut CountingWriter::wrap(write);
+        for interpolation in &mut interpolations {
+            interpolation.data_start_offset = write.written_bytes();
+            let num_bits = interpolation.num_bits;
+            for (pos, actual_value) in vals
+                [interpolation.start_pos as usize..interpolation.end_pos as usize]
+                .iter()
+                .cloned()
+                .enumerate()
+            {
+                let calculated_value = get_calculated_value(
+                    interpolation.value_start_pos,
+                    pos as u64,
+                    interpolation.slope,
+                );
+                let diff = (actual_value + interpolation.positive_val_offset) - calculated_value;
+                bit_packer.write(diff, num_bits, write)?;
+            }
+            bit_packer.flush(write)?;
+        }
+        bit_packer.close(write)?;
+
+        let footer = MultiLinearInterpolFooter {
+            num_vals: stats.num_vals,
+            min_value: stats.min_value,
+            max_value: stats.max_value,
+            interpolations,
+        };
+        footer.serialize(write)?;
+        Ok(())
+    }
+
+    fn is_applicable(_vals: &[u64], stats: FastFieldStats) -> bool {
+        if stats.num_vals < 5_000 {
+            return false;
+        }
+        // On serialization the offset is added to the actual value.
+        // We need to make sure this won't run into overflow calculation issues.
+        // For this we take the maximum theroretical offset and add this to the max value.
+        // If this doesn't overflow the algorithm should be fine
+        let theorethical_maximum_offset = stats.max_value - stats.min_value;
+        if stats
+            .max_value
+            .checked_add(theorethical_maximum_offset)
+            .is_none()
+        {
+            return false;
+        }
+        true
+    }
+    /// estimation for linear interpolation is hard because, you don't know
+    /// where the local maxima are for the deviation of the calculated value and
+    /// the offset is also unknown.
+    fn estimate(vals: &[u64], stats: FastFieldStats) -> f32 {
+        // TODO simplify now that we have a vals array.
+        let first_val_in_first_block = vals[0];
+        let last_elem_in_first_chunk = CHUNK_SIZE.min(vals.len());
+        let last_val_in_first_block = vals[last_elem_in_first_chunk - 1];
+        let slope = get_slope(
+            first_val_in_first_block,
+            last_val_in_first_block,
+            stats.num_vals,
+        );
+
+        // let's sample at 0%, 5%, 10% .. 95%, 100%, but for the first block only
+        let sample_positions = (0..20)
+            .map(|pos| (last_elem_in_first_chunk as f32 / 100.0 * pos as f32 * 5.0) as usize)
+            .collect::<Vec<_>>();
+
+        let max_distance = sample_positions
+            .iter()
+            .copied()
+            .map(|pos| {
+                let calculated_value =
+                    get_calculated_value(first_val_in_first_block, pos as u64, slope);
+                let actual_value = vals[pos];
+                distance(calculated_value, actual_value)
+            })
+            .max()
+            .unwrap();
+
+        // Estimate one block and extrapolate the cost to all blocks.
+        // the theory would be that we don't have the actual max_distance, but we are close within
+        // 50% threshold.
+        // It is multiplied by 2 because in a log case scenario the line would be as much above as
+        // below. So the offset would = max_distance
+        //
+        let relative_max_value = (max_distance as f32 * 1.5) * 2.0;
+
+        let num_bits = compute_num_bits(relative_max_value as u64) as u64 * stats.num_vals as u64
+            // function metadata per block
+            + 29 * (stats.num_vals / CHUNK_SIZE as u64);
+        let num_bits_uncompressed = 64 * stats.num_vals;
+        num_bits as f32 / num_bits_uncompressed as f32
+    }
+}
+
+fn distance<T: Sub<Output = T> + Ord>(x: T, y: T) -> T {
+    if x < y {
+        y - x
+    } else {
+        x - y
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::tests::get_codec_test_data_sets;
+
+    fn create_and_validate(data: &[u64], name: &str) -> (f32, f32) {
+        crate::tests::create_and_validate(&MultiLinearInterpolFastFieldCodec, data, name)
+    }
+
+    #[test]
+    fn test_compression() {
+        let data = (10..=6_000_u64).collect::<Vec<_>>();
+        let (estimate, actual_compression) =
+            create_and_validate(&data, "simple monotonically large");
+        assert!(actual_compression < 0.2);
+        assert!(estimate < 0.20);
+        assert!(estimate > 0.15);
+        assert!(actual_compression > 0.01);
+    }
+
+    #[test]
+    fn test_with_codec_data_sets() {
+        let data_sets = get_codec_test_data_sets();
+        for (mut data, name) in data_sets {
+            create_and_validate(&data, name);
+            data.reverse();
+            create_and_validate(&data, name);
+        }
+    }
+    #[test]
+    fn test_simple() {
+        let data = (10..=20_u64).collect::<Vec<_>>();
+        create_and_validate(&data, "simple monotonically");
+    }
+
+    #[test]
+    fn border_cases_1() {
+        let data = (0..1024).collect::<Vec<_>>();
+        create_and_validate(&data, "border case");
+    }
+    #[test]
+    fn border_case_2() {
+        let data = (0..1025).collect::<Vec<_>>();
+        create_and_validate(&data, "border case");
+    }
+    #[test]
+    fn rand() {
+        for _ in 0..10 {
+            let mut data = (5_000..20_000)
+                .map(|_| rand::random::<u32>() as u64)
+                .collect::<Vec<_>>();
+            let _ = create_and_validate(&data, "random");
+            data.reverse();
+            create_and_validate(&data, "random");
+        }
+    }
+}

fastfield_codecs/src/serialize.rs

@@ -1,277 +0,0 @@
-// Copyright (C) 2022 Quickwit, Inc.
-//
-// Quickwit is offered under the AGPL v3.0 and as commercial software.
-// For commercial licensing, contact us at hello@quickwit.io.
-//
-// AGPL:
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as
-// published by the Free Software Foundation, either version 3 of the
-// License, or (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-use std::io;
-use std::num::NonZeroU64;
-use std::sync::Arc;
-
-use common::{BinarySerializable, VInt};
-use fastdivide::DividerU64;
-use log::warn;
-use ownedbytes::OwnedBytes;
-
-use crate::bitpacked::BitpackedCodec;
-use crate::blockwise_linear::BlockwiseLinearCodec;
-use crate::column::EstimateColumn;
-use crate::compact_space::CompactSpaceCompressor;
-use crate::linear::LinearCodec;
-use crate::{
-    monotonic_map_column, Column, FastFieldCodec, FastFieldCodecType, MonotonicallyMappableToU64,
-    VecColumn, ALL_CODEC_TYPES,
-};
-
-/// The normalized header gives some parameters after applying the following
-/// normalization of the vector:
-/// val -> (val - min_value) / gcd
-///
-/// By design, after normalization, `min_value = 0` and `gcd = 1`.
-#[derive(Debug, Copy, Clone)]
-pub struct NormalizedHeader {
-    pub num_vals: u64,
-    pub max_value: u64,
-}
-
-#[derive(Debug, Copy, Clone)]
-pub(crate) struct Header {
-    pub num_vals: u64,
-    pub min_value: u64,
-    pub max_value: u64,
-    pub gcd: Option<NonZeroU64>,
-    pub codec_type: FastFieldCodecType,
-}
-
-impl Header {
-    pub fn normalized(self) -> NormalizedHeader {
-        let max_value =
-            (self.max_value - self.min_value) / self.gcd.map(|gcd| gcd.get()).unwrap_or(1);
-        NormalizedHeader {
-            num_vals: self.num_vals,
-            max_value,
-        }
-    }
-
-    pub fn normalize_column<C: Column>(&self, from_column: C) -> impl Column {
-        let min_value = self.min_value;
-        let gcd = self.gcd.map(|gcd| gcd.get()).unwrap_or(1);
-        let divider = DividerU64::divide_by(gcd);
-        monotonic_map_column(from_column, move |val| divider.divide(val - min_value))
-    }
-
-    pub fn compute_header(
-        column: impl Column<u64>,
-        codecs: &[FastFieldCodecType],
-    ) -> Option<Header> {
-        let num_vals = column.num_vals();
-        let min_value = column.min_value();
-        let max_value = column.max_value();
-        let gcd = crate::gcd::find_gcd(column.iter().map(|val| val - min_value))
-            .filter(|gcd| gcd.get() > 1u64);
-        let divider = DividerU64::divide_by(gcd.map(|gcd| gcd.get()).unwrap_or(1u64));
-        let shifted_column = monotonic_map_column(&column, |val| divider.divide(val - min_value));
-        let codec_type = detect_codec(shifted_column, codecs)?;
-        Some(Header {
-            num_vals,
-            min_value,
-            max_value,
-            gcd,
-            codec_type,
-        })
-    }
-}
-
-impl BinarySerializable for Header {
-    fn serialize<W: io::Write>(&self, writer: &mut W) -> io::Result<()> {
-        VInt(self.num_vals).serialize(writer)?;
-        VInt(self.min_value).serialize(writer)?;
-        VInt(self.max_value - self.min_value).serialize(writer)?;
-        if let Some(gcd) = self.gcd {
-            VInt(gcd.get()).serialize(writer)?;
-        } else {
-            VInt(0u64).serialize(writer)?;
-        }
-        self.codec_type.serialize(writer)?;
-        Ok(())
-    }
-
-    fn deserialize<R: io::Read>(reader: &mut R) -> io::Result<Self> {
-        let num_vals = VInt::deserialize(reader)?.0;
-        let min_value = VInt::deserialize(reader)?.0;
-        let amplitude = VInt::deserialize(reader)?.0;
-        let max_value = min_value + amplitude;
-        let gcd_u64 = VInt::deserialize(reader)?.0;
-        let codec_type = FastFieldCodecType::deserialize(reader)?;
-        Ok(Header {
-            num_vals,
-            min_value,
-            max_value,
-            gcd: NonZeroU64::new(gcd_u64),
-            codec_type,
-        })
-    }
-}
-
-pub fn serialize_u128(
-    typed_column: impl Column<u128>,
-    output: &mut impl io::Write,
-) -> io::Result<()> {
-    // TODO write header, to later support more codecs
-    let compressor = CompactSpaceCompressor::train_from(&typed_column);
-    compressor
-        .compress_into(typed_column.iter(), output)
-        .unwrap();
-
-    Ok(())
-}
-
-pub fn serialize<T: MonotonicallyMappableToU64>(
-    typed_column: impl Column<T>,
-    output: &mut impl io::Write,
-    codecs: &[FastFieldCodecType],
-) -> io::Result<()> {
-    let column = monotonic_map_column(typed_column, T::to_u64);
-    let header = Header::compute_header(&column, codecs).ok_or_else(|| {
-        io::Error::new(
-            io::ErrorKind::InvalidInput,
-            format!(
-                "Data cannot be serialized with this list of codec. {:?}",
-                codecs
-            ),
-        )
-    })?;
-    header.serialize(output)?;
-    let normalized_column = header.normalize_column(column);
-    assert_eq!(normalized_column.min_value(), 0u64);
-    serialize_given_codec(normalized_column, header.codec_type, output)?;
-    Ok(())
-}
-
-pub fn estimate<T: MonotonicallyMappableToU64>(
-    typed_column: impl Column<T>,
-    codec_type: FastFieldCodecType,
-) -> Option<f32> {
-    let column = monotonic_map_column(typed_column, T::to_u64);
-    let min_value = column.min_value();
-    let gcd = crate::gcd::find_gcd(column.iter().map(|val| val - min_value))
-        .filter(|gcd| gcd.get() > 1u64);
-    let divider = DividerU64::divide_by(gcd.map(|gcd| gcd.get()).unwrap_or(1u64));
-    let normalized_column = monotonic_map_column(&column, |val| divider.divide(val - min_value));
-    let estimate_column = EstimateColumn::new(&normalized_column);
-    match codec_type {
-        FastFieldCodecType::Bitpacked => BitpackedCodec::estimate(&estimate_column),
-        FastFieldCodecType::Linear => LinearCodec::estimate(&estimate_column),
-        FastFieldCodecType::BlockwiseLinear => BlockwiseLinearCodec::estimate(&estimate_column),
-    }
-}
-
-fn detect_codec(
-    column: impl Column<u64>,
-    codecs: &[FastFieldCodecType],
-) -> Option<FastFieldCodecType> {
-    let column: EstimateColumn = EstimateColumn::new(&column);
-    let mut estimations = Vec::new();
-    for &codec in codecs {
-        let estimation_opt = match codec {
-            FastFieldCodecType::Bitpacked => BitpackedCodec::estimate(&column),
-            FastFieldCodecType::Linear => LinearCodec::estimate(&column),
-            FastFieldCodecType::BlockwiseLinear => BlockwiseLinearCodec::estimate(&column),
-        };
-        if let Some(estimation) = estimation_opt {
-            estimations.push((estimation, codec));
-        }
-    }
-    if let Some(broken_estimation) = estimations.iter().find(|estimation| estimation.0.is_nan()) {
-        warn!(
-            "broken estimation for fast field codec {:?}",
-            broken_estimation.1
-        );
-    }
-    // removing nan values for codecs with broken calculations, and max values which disables
-    // codecs
-    estimations.retain(|estimation| !estimation.0.is_nan() && estimation.0 != f32::MAX);
-    estimations.sort_by(|(score_left, _), (score_right, _)| score_left.total_cmp(score_right));
-    Some(estimations.first()?.1)
-}
-
-fn serialize_given_codec(
-    column: impl Column<u64>,
-    codec_type: FastFieldCodecType,
-    output: &mut impl io::Write,
-) -> io::Result<()> {
-    match codec_type {
-        FastFieldCodecType::Bitpacked => {
-            BitpackedCodec::serialize(&column, output)?;
-        }
-        FastFieldCodecType::Linear => {
-            LinearCodec::serialize(&column, output)?;
-        }
-        FastFieldCodecType::BlockwiseLinear => {
-            BlockwiseLinearCodec::serialize(&column, output)?;
-        }
-    }
-    output.flush()?;
-    Ok(())
-}
-
-pub fn serialize_and_load<T: MonotonicallyMappableToU64 + Ord + Default>(
-    column: &[T],
-) -> Arc<dyn Column<T>> {
-    let mut buffer = Vec::new();
-    super::serialize(VecColumn::from(&column), &mut buffer, &ALL_CODEC_TYPES).unwrap();
-    super::open(OwnedBytes::new(buffer)).unwrap()
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn test_serialize_deserialize() {
-        let original = [1u64, 5u64, 10u64];
-        let restored: Vec<u64> = serialize_and_load(&original[..]).iter().collect();
-        assert_eq!(&restored, &original[..]);
-    }
-
-    #[test]
-    fn test_fastfield_bool_size_bitwidth_1() {
-        let mut buffer = Vec::new();
-        let col = VecColumn::from(&[false, true][..]);
-        serialize(col, &mut buffer, &ALL_CODEC_TYPES).unwrap();
-        // 5 bytes of header, 1 byte of value, 7 bytes of padding.
-        assert_eq!(buffer.len(), 5 + 8);
-    }
-
-    #[test]
-    fn test_fastfield_bool_bit_size_bitwidth_0() {
-        let mut buffer = Vec::new();
-        let col = VecColumn::from(&[true][..]);
-        serialize(col, &mut buffer, &ALL_CODEC_TYPES).unwrap();
-        // 5 bytes of header, 0 bytes of value, 7 bytes of padding.
-        assert_eq!(buffer.len(), 5 + 7);
-    }
-
-    #[test]
-    fn test_fastfield_gcd() {
-        let mut buffer = Vec::new();
-        let vals: Vec<u64> = (0..80).map(|val| (val % 7) * 1_000u64).collect();
-        let col = VecColumn::from(&vals[..]);
-        serialize(col, &mut buffer, &[FastFieldCodecType::Bitpacked]).unwrap();
-        // Values are stored over 3 bits.
-        assert_eq!(buffer.len(), 7 + (3 * 80 / 8) + 7);
-    }
-}

ownedbytes/src/lib.rs

@@ -6,7 +6,7 @@ use std::{fmt, io, mem};
 use stable_deref_trait::StableDeref;
 
 /// An OwnedBytes simply wraps an object that owns a slice of data and exposes
-/// this data as a slice.
+/// this data as a static slice.
 ///
 /// The backing object is required to be `StableDeref`.
 #[derive(Clone)]

query-grammar/src/lib.rs

@@ -1,5 +1,3 @@
-#![allow(clippy::derive_partial_eq_without_eq)]
-
 mod occur;
 mod query_grammar;
 mod user_input_ast;

query-grammar/src/query_grammar.rs

@@ -23,7 +23,7 @@ const ESCAPED_SPECIAL_CHARS_PATTERN: &str = r#"\\(\+|\^|`|:|\{|\}|"|\[|\]|\(|\)|
 /// Parses a field_name
 /// A field name must have at least one character and be followed by a colon.
 /// All characters are allowed including special characters `SPECIAL_CHARS`, but these
-/// need to be escaped with a backslash character '\'.
+/// need to be escaped with a backslack character '\'.
 fn field_name<'a>() -> impl Parser<&'a str, Output = String> {
     static ESCAPED_SPECIAL_CHARS_RE: Lazy<Regex> =
         Lazy::new(|| Regex::new(ESCAPED_SPECIAL_CHARS_PATTERN).unwrap());
@@ -68,7 +68,7 @@ fn word<'a>() -> impl Parser<&'a str, Output = String> {
 ///
 /// NOTE: also accepts 999999-99-99T99:99:99.266051969+99:99
 /// We delegate rejecting such invalid dates to the logical AST computation code
-/// which invokes `time::OffsetDateTime::parse(..., &Rfc3339)` on the value to actually parse
+/// which invokes time::OffsetDateTime::parse(..., &Rfc3339) on the value to actually parse
 /// it (instead of merely extracting the datetime value as string as done here).
 fn date_time<'a>() -> impl Parser<&'a str, Output = String> {
     let two_digits = || recognize::<String, _, _>((digit(), digit()));

src/aggregation/agg_req.rs

@@ -1,7 +1,7 @@
 //! Contains the aggregation request tree. Used to build an
-//! [`AggregationCollector`](super::AggregationCollector).
+//! [AggregationCollector](super::AggregationCollector).
 //!
-//! [`Aggregations`] is the top level entry point to create a request, which is a `HashMap<String,
+//! [Aggregations] is the top level entry point to create a request, which is a `HashMap<String,
 //! Aggregation>`.
 //!
 //! Requests are compatible with the json format of elasticsearch.
@@ -54,8 +54,8 @@ use super::bucket::{HistogramAggregation, TermsAggregation};
 use super::metric::{AverageAggregation, StatsAggregation};
 use super::VecWithNames;
 
-/// The top-level aggregation request structure, which contains [`Aggregation`] and their user
-/// defined names. It is also used in [buckets](BucketAggregation) to define sub-aggregations.
+/// The top-level aggregation request structure, which contains [Aggregation] and their user defined
+/// names. It is also used in [buckets](BucketAggregation) to define sub-aggregations.
 ///
 /// The key is the user defined name of the aggregation.
 pub type Aggregations = HashMap<String, Aggregation>;
@@ -139,15 +139,15 @@ pub fn get_fast_field_names(aggs: &Aggregations) -> HashSet<String> {
     fast_field_names
 }
 
-/// Aggregation request of [`BucketAggregation`] or [`MetricAggregation`].
+/// Aggregation request of [BucketAggregation] or [MetricAggregation].
 ///
 /// An aggregation is either a bucket or a metric.
 #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
 #[serde(untagged)]
 pub enum Aggregation {
-    /// Bucket aggregation, see [`BucketAggregation`] for details.
+    /// Bucket aggregation, see [BucketAggregation] for details.
     Bucket(BucketAggregation),
-    /// Metric aggregation, see [`MetricAggregation`] for details.
+    /// Metric aggregation, see [MetricAggregation] for details.
     Metric(MetricAggregation),
 }
 

src/aggregation/agg_req_with_accessor.rs

@@ -4,14 +4,14 @@ use std::rc::Rc;
 use std::sync::atomic::AtomicU32;
 use std::sync::Arc;
 
-use fastfield_codecs::Column;
-
 use super::agg_req::{Aggregation, Aggregations, BucketAggregationType, MetricAggregation};
 use super::bucket::{HistogramAggregation, RangeAggregation, TermsAggregation};
 use super::metric::{AverageAggregation, StatsAggregation};
 use super::segment_agg_result::BucketCount;
 use super::VecWithNames;
-use crate::fastfield::{type_and_cardinality, FastType, MultiValuedFastFieldReader};
+use crate::fastfield::{
+    type_and_cardinality, FastFieldReaderImpl, FastType, MultiValuedFastFieldReader,
+};
 use crate::schema::{Cardinality, Type};
 use crate::{InvertedIndexReader, SegmentReader, TantivyError};
 
@@ -37,16 +37,10 @@ impl AggregationsWithAccessor {
 #[derive(Clone)]
 pub(crate) enum FastFieldAccessor {
     Multi(MultiValuedFastFieldReader<u64>),
-    Single(Arc<dyn Column<u64>>),
+    Single(FastFieldReaderImpl<u64>),
 }
 impl FastFieldAccessor {
-    pub fn as_single(&self) -> Option<&dyn Column<u64>> {
-        match self {
-            FastFieldAccessor::Multi(_) => None,
-            FastFieldAccessor::Single(reader) => Some(&**reader),
-        }
-    }
-    pub fn into_single(self) -> Option<Arc<dyn Column<u64>>> {
+    pub fn as_single(&self) -> Option<&FastFieldReaderImpl<u64>> {
         match self {
             FastFieldAccessor::Multi(_) => None,
             FastFieldAccessor::Single(reader) => Some(reader),
@@ -124,7 +118,7 @@ impl BucketAggregationWithAccessor {
 pub struct MetricAggregationWithAccessor {
     pub metric: MetricAggregation,
     pub field_type: Type,
-    pub accessor: Arc<dyn Column>,
+    pub accessor: FastFieldReaderImpl<u64>,
 }
 
 impl MetricAggregationWithAccessor {
@@ -140,8 +134,9 @@ impl MetricAggregationWithAccessor {
 
                 Ok(MetricAggregationWithAccessor {
                     accessor: accessor
-                        .into_single()
-                        .expect("unexpected fast field cardinality"),
+                        .as_single()
+                        .expect("unexpected fast field cardinality")
+                        .clone(),
                     field_type,
                     metric: metric.clone(),
                 })

src/aggregation/agg_result.rs

@@ -113,14 +113,14 @@ pub enum BucketResult {
         ///
         /// If there are holes depends on the request, if min_doc_count is 0, then there are no
         /// holes between the first and last bucket.
-        /// See [`HistogramAggregation`](super::bucket::HistogramAggregation)
+        /// See [HistogramAggregation](super::bucket::HistogramAggregation)
         buckets: BucketEntries<BucketEntry>,
     },
     /// This is the term result
     Terms {
         /// The buckets.
         ///
-        /// See [`TermsAggregation`](super::bucket::TermsAggregation)
+        /// See [TermsAggregation](super::bucket::TermsAggregation)
         buckets: Vec<BucketEntry>,
         /// The number of documents that didn’t make it into to TOP N due to shard_size or size
         sum_other_doc_count: u64,
@@ -234,10 +234,10 @@ pub struct RangeBucketEntry {
     #[serde(flatten)]
     /// sub-aggregations in this bucket.
     pub sub_aggregation: AggregationResults,
-    /// The from range of the bucket. Equals `f64::MIN` when `None`.
+    /// The from range of the bucket. Equals f64::MIN when None.
     #[serde(skip_serializing_if = "Option::is_none")]
     pub from: Option<f64>,
-    /// The to range of the bucket. Equals `f64::MAX` when `None`.
+    /// The to range of the bucket. Equals f64::MAX when None.
     #[serde(skip_serializing_if = "Option::is_none")]
     pub to: Option<f64>,
 }

src/aggregation/bucket/histogram/histogram.rs

@@ -1,7 +1,6 @@
 use std::cmp::Ordering;
 use std::fmt::Display;
 
-use fastfield_codecs::Column;
 use itertools::Itertools;
 use serde::{Deserialize, Serialize};
 
@@ -15,6 +14,7 @@ use crate::aggregation::intermediate_agg_result::{
     IntermediateAggregationResults, IntermediateBucketResult, IntermediateHistogramBucketEntry,
 };
 use crate::aggregation::segment_agg_result::SegmentAggregationResultsCollector;
+use crate::fastfield::{FastFieldReader, FastFieldReaderImpl};
 use crate::schema::Type;
 use crate::{DocId, TantivyError};
 
@@ -37,14 +37,14 @@ use crate::{DocId, TantivyError};
 /// [hard_bounds](HistogramAggregation::hard_bounds).
 ///
 /// # Result
-/// Result type is [`BucketResult`](crate::aggregation::agg_result::BucketResult) with
-/// [`BucketEntry`](crate::aggregation::agg_result::BucketEntry) on the
-/// `AggregationCollector`.
+/// Result type is [BucketResult](crate::aggregation::agg_result::BucketResult) with
+/// [BucketEntry](crate::aggregation::agg_result::BucketEntry) on the
+/// AggregationCollector.
 ///
 /// Result type is
-/// [`IntermediateBucketResult`](crate::aggregation::intermediate_agg_result::IntermediateBucketResult) with
-/// [`IntermediateHistogramBucketEntry`](crate::aggregation::intermediate_agg_result::IntermediateHistogramBucketEntry) on the
-/// `DistributedAggregationCollector`.
+/// [crate::aggregation::intermediate_agg_result::IntermediateBucketResult] with
+/// [crate::aggregation::intermediate_agg_result::IntermediateHistogramBucketEntry] on the
+/// DistributedAggregationCollector.
 ///
 /// # Limitations/Compatibility
 ///
@@ -61,7 +61,7 @@ use crate::{DocId, TantivyError};
 /// ```
 ///
 /// Response
-/// See [`BucketEntry`](crate::aggregation::agg_result::BucketEntry)
+/// See [BucketEntry](crate::aggregation::agg_result::BucketEntry)
 
 #[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
 pub struct HistogramAggregation {
@@ -263,7 +263,7 @@ impl SegmentHistogramCollector {
         req: &HistogramAggregation,
         sub_aggregation: &AggregationsWithAccessor,
         field_type: Type,
-        accessor: &dyn Column<u64>,
+        accessor: &FastFieldReaderImpl<u64>,
     ) -> crate::Result<Self> {
         req.validate()?;
         let min = f64_from_fastfield_u64(accessor.min_value(), &field_type);
@@ -331,10 +331,10 @@ impl SegmentHistogramCollector {
             .expect("unexpected fast field cardinatility");
         let mut iter = doc.chunks_exact(4);
         for docs in iter.by_ref() {
-            let val0 = self.f64_from_fastfield_u64(accessor.get_val(docs[0] as u64));
-            let val1 = self.f64_from_fastfield_u64(accessor.get_val(docs[1] as u64));
-            let val2 = self.f64_from_fastfield_u64(accessor.get_val(docs[2] as u64));
-            let val3 = self.f64_from_fastfield_u64(accessor.get_val(docs[3] as u64));
+            let val0 = self.f64_from_fastfield_u64(accessor.get(docs[0]));
+            let val1 = self.f64_from_fastfield_u64(accessor.get(docs[1]));
+            let val2 = self.f64_from_fastfield_u64(accessor.get(docs[2]));
+            let val3 = self.f64_from_fastfield_u64(accessor.get(docs[3]));
 
             let bucket_pos0 = get_bucket_num(val0);
             let bucket_pos1 = get_bucket_num(val1);
@@ -370,8 +370,8 @@ impl SegmentHistogramCollector {
                 &bucket_with_accessor.sub_aggregation,
             )?;
         }
-        for &doc in iter.remainder() {
-            let val = f64_from_fastfield_u64(accessor.get_val(doc as u64), &self.field_type);
+        for doc in iter.remainder() {
+            let val = f64_from_fastfield_u64(accessor.get(*doc), &self.field_type);
             if !bounds.contains(val) {
                 continue;
             }
@@ -382,7 +382,7 @@ impl SegmentHistogramCollector {
                 self.buckets[bucket_pos].key,
                 get_bucket_val(val, self.interval, self.offset) as f64
             );
-            self.increment_bucket(bucket_pos, doc, &bucket_with_accessor.sub_aggregation)?;
+            self.increment_bucket(bucket_pos, *doc, &bucket_with_accessor.sub_aggregation)?;
         }
         if force_flush {
             if let Some(sub_aggregations) = self.sub_aggregations.as_mut() {
@@ -425,7 +425,7 @@ impl SegmentHistogramCollector {
         let bucket = &mut self.buckets[bucket_pos];
         bucket.doc_count += 1;
         if let Some(sub_aggregation) = self.sub_aggregations.as_mut() {
-            sub_aggregation[bucket_pos].collect(doc, bucket_with_accessor)?;
+            (&mut sub_aggregation[bucket_pos]).collect(doc, bucket_with_accessor)?;
         }
         Ok(())
     }
@@ -518,7 +518,7 @@ pub(crate) fn intermediate_histogram_buckets_to_final_buckets(
 
 /// Applies req extended_bounds/hard_bounds on the min_max value
 ///
-/// May return `(f64::MAX, f64::MIN)`, if there is no range.
+/// May return (f64::MAX, f64::MIN), if there is no range.
 fn get_req_min_max(req: &HistogramAggregation, min_max: Option<(f64, f64)>) -> (f64, f64) {
     let (mut min, mut max) = min_max.unwrap_or((f64::MAX, f64::MIN));
 

src/aggregation/bucket/mod.rs

@@ -1,11 +1,11 @@
 //! Module for all bucket aggregations.
 //!
 //! BucketAggregations create buckets of documents
-//! [`BucketAggregation`](super::agg_req::BucketAggregation).
+//! [BucketAggregation](super::agg_req::BucketAggregation).
 //!
-//! Results of final buckets are [`BucketResult`](super::agg_result::BucketResult).
+//! Results of final buckets are [BucketResult](super::agg_result::BucketResult).
 //! Results of intermediate buckets are
-//! [`IntermediateBucketResult`](super::intermediate_agg_result::IntermediateBucketResult)
+//! [IntermediateBucketResult](super::intermediate_agg_result::IntermediateBucketResult)
 
 mod histogram;
 mod range;

src/aggregation/bucket/range.rs

@@ -12,6 +12,7 @@ use crate::aggregation::intermediate_agg_result::{
 };
 use crate::aggregation::segment_agg_result::{BucketCount, SegmentAggregationResultsCollector};
 use crate::aggregation::{f64_from_fastfield_u64, f64_to_fastfield_u64, Key, SerializedKey};
+use crate::fastfield::FastFieldReader;
 use crate::schema::Type;
 use crate::{DocId, TantivyError};
 
@@ -22,14 +23,14 @@ use crate::{DocId, TantivyError};
 /// against each bucket range. Note that this aggregation includes the from value and excludes the
 /// to value for each range.
 ///
-/// Result type is [`BucketResult`](crate::aggregation::agg_result::BucketResult) with
-/// [`RangeBucketEntry`](crate::aggregation::agg_result::RangeBucketEntry) on the
-/// `AggregationCollector`.
+/// Result type is [BucketResult](crate::aggregation::agg_result::BucketResult) with
+/// [RangeBucketEntry](crate::aggregation::agg_result::RangeBucketEntry) on the
+/// AggregationCollector.
 ///
 /// Result type is
-/// [`IntermediateBucketResult`](crate::aggregation::intermediate_agg_result::IntermediateBucketResult) with
-/// [`IntermediateRangeBucketEntry`](crate::aggregation::intermediate_agg_result::IntermediateRangeBucketEntry) on the
-/// `DistributedAggregationCollector`.
+/// [crate::aggregation::intermediate_agg_result::IntermediateBucketResult] with
+/// [crate::aggregation::intermediate_agg_result::IntermediateRangeBucketEntry] on the
+/// DistributedAggregationCollector.
 ///
 /// # Limitations/Compatibility
 /// Overlapping ranges are not yet supported.
@@ -67,11 +68,11 @@ pub struct RangeAggregationRange {
     #[serde(skip_serializing_if = "Option::is_none", default)]
     pub key: Option<String>,
     /// The from range value, which is inclusive in the range.
-    /// `None` equals to an open ended interval.
+    /// None equals to an open ended interval.
     #[serde(skip_serializing_if = "Option::is_none", default)]
     pub from: Option<f64>,
     /// The to range value, which is not inclusive in the range.
-    /// `None` equals to an open ended interval.
+    /// None equals to an open ended interval.
     #[serde(skip_serializing_if = "Option::is_none", default)]
     pub to: Option<f64>,
 }
@@ -101,7 +102,7 @@ impl From<Range<f64>> for RangeAggregationRange {
 pub(crate) struct InternalRangeAggregationRange {
     /// Custom key for the range bucket
     key: Option<String>,
-    /// `u64` range value
+    /// u64 range value
     range: Range<u64>,
 }
 
@@ -131,9 +132,9 @@ pub(crate) struct SegmentRangeBucketEntry {
     pub key: Key,
     pub doc_count: u64,
     pub sub_aggregation: Option<SegmentAggregationResultsCollector>,
-    /// The from range of the bucket. Equals `f64::MIN` when `None`.
+    /// The from range of the bucket. Equals f64::MIN when None.
     pub from: Option<f64>,
-    /// The to range of the bucket. Equals `f64::MAX` when `None`. Open interval, `to` is not
+    /// The to range of the bucket. Equals f64::MAX when None. Open interval, `to` is not
     /// inclusive.
     pub to: Option<f64>,
 }
@@ -261,12 +262,12 @@ impl SegmentRangeCollector {
         let accessor = bucket_with_accessor
             .accessor
             .as_single()
-            .expect("unexpected fast field cardinality");
+            .expect("unexpected fast field cardinatility");
         for docs in iter.by_ref() {
-            let val1 = accessor.get_val(docs[0] as u64);
-            let val2 = accessor.get_val(docs[1] as u64);
-            let val3 = accessor.get_val(docs[2] as u64);
-            let val4 = accessor.get_val(docs[3] as u64);
+            let val1 = accessor.get(docs[0]);
+            let val2 = accessor.get(docs[1]);
+            let val3 = accessor.get(docs[2]);
+            let val4 = accessor.get(docs[3]);
             let bucket_pos1 = self.get_bucket_pos(val1);
             let bucket_pos2 = self.get_bucket_pos(val2);
             let bucket_pos3 = self.get_bucket_pos(val3);
@@ -277,10 +278,10 @@ impl SegmentRangeCollector {
             self.increment_bucket(bucket_pos3, docs[2], &bucket_with_accessor.sub_aggregation)?;
             self.increment_bucket(bucket_pos4, docs[3], &bucket_with_accessor.sub_aggregation)?;
         }
-        for &doc in iter.remainder() {
-            let val = accessor.get_val(doc as u64);
+        for doc in iter.remainder() {
+            let val = accessor.get(*doc);
             let bucket_pos = self.get_bucket_pos(val);
-            self.increment_bucket(bucket_pos, doc, &bucket_with_accessor.sub_aggregation)?;
+            self.increment_bucket(bucket_pos, *doc, &bucket_with_accessor.sub_aggregation)?;
         }
         if force_flush {
             for bucket in &mut self.buckets {
@@ -423,13 +424,12 @@ pub(crate) fn range_to_key(range: &Range<u64>, field_type: &Type) -> Key {
 #[cfg(test)]
 mod tests {
 
-    use fastfield_codecs::MonotonicallyMappableToU64;
-
     use super::*;
     use crate::aggregation::agg_req::{
         Aggregation, Aggregations, BucketAggregation, BucketAggregationType,
     };
     use crate::aggregation::tests::{exec_request_with_query, get_test_index_with_num_docs};
+    use crate::fastfield::FastValue;
 
     pub fn get_collector_from_ranges(
         ranges: Vec<RangeAggregationRange>,

src/aggregation/bucket/term_agg.rs

@@ -31,7 +31,7 @@ use crate::{DocId, TantivyError};
 ///
 /// Even with a larger `segment_size` value, doc_count values for a terms aggregation may be
 /// approximate. As a result, any sub-aggregations on the terms aggregation may also be approximate.
-/// `sum_other_doc_count` is the number of documents that didn’t make it into the top size
+/// `sum_other_doc_count` is the number of documents that didn’t make it into the the top size
 /// terms. If this is greater than 0, you can be sure that the terms agg had to throw away some
 /// buckets, either because they didn’t fit into size on the root node or they didn’t fit into
 /// `segment_size` on the segment node.
@@ -42,14 +42,14 @@ use crate::{DocId, TantivyError};
 /// each segment. It’s the sum of the size of the largest bucket on each segment that didn’t fit
 /// into segment_size.
 ///
-/// Result type is [`BucketResult`](crate::aggregation::agg_result::BucketResult) with
-/// [`TermBucketEntry`](crate::aggregation::agg_result::BucketEntry) on the
-/// `AggregationCollector`.
+/// Result type is [BucketResult](crate::aggregation::agg_result::BucketResult) with
+/// [TermBucketEntry](crate::aggregation::agg_result::BucketEntry) on the
+/// AggregationCollector.
 ///
 /// Result type is
-/// [`IntermediateBucketResult`](crate::aggregation::intermediate_agg_result::IntermediateBucketResult) with
-/// [`IntermediateTermBucketEntry`](crate::aggregation::intermediate_agg_result::IntermediateTermBucketEntry) on the
-/// `DistributedAggregationCollector`.
+/// [crate::aggregation::intermediate_agg_result::IntermediateBucketResult] with
+/// [crate::aggregation::intermediate_agg_result::IntermediateTermBucketEntry] on the
+/// DistributedAggregationCollector.
 ///
 /// # Limitations/Compatibility
 ///

src/aggregation/collector.rs

@@ -131,7 +131,7 @@ fn merge_fruits(
     }
 }
 
-/// `AggregationSegmentCollector` does the aggregation collection on a segment.
+/// AggregationSegmentCollector does the aggregation collection on a segment.
 pub struct AggregationSegmentCollector {
     aggs_with_accessor: AggregationsWithAccessor,
     result: SegmentAggregationResultsCollector,
@@ -139,8 +139,8 @@ pub struct AggregationSegmentCollector {
 }
 
 impl AggregationSegmentCollector {
-    /// Creates an `AggregationSegmentCollector from` an [`Aggregations`] request and a segment
-    /// reader. Also includes validation, e.g. checking field types and existence.
+    /// Creates an AggregationSegmentCollector from an [Aggregations] request and a segment reader.
+    /// Also includes validation, e.g. checking field types and existence.
     pub fn from_agg_req_and_reader(
         agg: &Aggregations,
         reader: &SegmentReader,

src/aggregation/intermediate_agg_result.rs

@@ -108,10 +108,10 @@ impl IntermediateAggregationResults {
         Self { metrics, buckets }
     }
 
-    /// Merge another intermediate aggregation result into this result.
+    /// Merge an other intermediate aggregation result into this result.
     ///
     /// The order of the values need to be the same on both results. This is ensured when the same
-    /// (key values) are present on the underlying `VecWithNames` struct.
+    /// (key values) are present on the underlying VecWithNames struct.
     pub fn merge_fruits(&mut self, other: IntermediateAggregationResults) {
         if let (Some(buckets_left), Some(buckets_right)) = (&mut self.buckets, other.buckets) {
             for (bucket_left, bucket_right) in
@@ -560,10 +560,10 @@ pub struct IntermediateRangeBucketEntry {
     pub doc_count: u64,
     /// The sub_aggregation in this bucket.
     pub sub_aggregation: IntermediateAggregationResults,
-    /// The from range of the bucket. Equals `f64::MIN` when `None`.
+    /// The from range of the bucket. Equals f64::MIN when None.
     #[serde(skip_serializing_if = "Option::is_none")]
     pub from: Option<f64>,
-    /// The to range of the bucket. Equals `f64::MAX` when `None`.
+    /// The to range of the bucket. Equals f64::MAX when None.
     #[serde(skip_serializing_if = "Option::is_none")]
     pub to: Option<f64>,
 }

src/aggregation/metric/average.rs

@@ -1,9 +1,9 @@
 use std::fmt::Debug;
 
-use fastfield_codecs::Column;
 use serde::{Deserialize, Serialize};
 
 use crate::aggregation::f64_from_fastfield_u64;
+use crate::fastfield::{FastFieldReader, FastFieldReaderImpl};
 use crate::schema::Type;
 use crate::DocId;
 
@@ -43,7 +43,7 @@ pub(crate) struct SegmentAverageCollector {
 }
 
 impl Debug for SegmentAverageCollector {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
         f.debug_struct("AverageCollector")
             .field("data", &self.data)
             .finish()
@@ -57,13 +57,13 @@ impl SegmentAverageCollector {
             data: Default::default(),
         }
     }
-    pub(crate) fn collect_block(&mut self, doc: &[DocId], field: &dyn Column<u64>) {
+    pub(crate) fn collect_block(&mut self, doc: &[DocId], field: &FastFieldReaderImpl<u64>) {
         let mut iter = doc.chunks_exact(4);
         for docs in iter.by_ref() {
-            let val1 = field.get_val(docs[0] as u64);
-            let val2 = field.get_val(docs[1] as u64);
-            let val3 = field.get_val(docs[2] as u64);
-            let val4 = field.get_val(docs[3] as u64);
+            let val1 = field.get(docs[0]);
+            let val2 = field.get(docs[1]);
+            let val3 = field.get(docs[2]);
+            let val4 = field.get(docs[3]);
             let val1 = f64_from_fastfield_u64(val1, &self.field_type);
             let val2 = f64_from_fastfield_u64(val2, &self.field_type);
             let val3 = f64_from_fastfield_u64(val3, &self.field_type);
@@ -73,8 +73,8 @@ impl SegmentAverageCollector {
             self.data.collect(val3);
             self.data.collect(val4);
         }
-        for &doc in iter.remainder() {
-            let val = field.get_val(doc as u64);
+        for doc in iter.remainder() {
+            let val = field.get(*doc);
             let val = f64_from_fastfield_u64(val, &self.field_type);
             self.data.collect(val);
         }

src/aggregation/metric/stats.rs

@@ -1,14 +1,14 @@
-use fastfield_codecs::Column;
 use serde::{Deserialize, Serialize};
 
 use crate::aggregation::f64_from_fastfield_u64;
+use crate::fastfield::{FastFieldReader, FastFieldReaderImpl};
 use crate::schema::Type;
 use crate::{DocId, TantivyError};
 
 /// A multi-value metric aggregation that computes stats of numeric values that are
 /// extracted from the aggregated documents.
-/// Supported field types are `u64`, `i64`, and `f64`.
-/// See [`Stats`] for returned statistics.
+/// Supported field types are u64, i64, and f64.
+/// See [Stats] for returned statistics.
 ///
 /// # JSON Format
 /// ```json
@@ -43,13 +43,13 @@ pub struct Stats {
     pub count: usize,
     /// The sum of the fast field values.
     pub sum: f64,
-    /// The standard deviation of the fast field values. `None` for count == 0.
+    /// The standard deviation of the fast field values. None for count == 0.
     pub standard_deviation: Option<f64>,
     /// The min value of the fast field values.
     pub min: Option<f64>,
     /// The max value of the fast field values.
     pub max: Option<f64>,
-    /// The average of the values. `None` for count == 0.
+    /// The average of the values. None for count == 0.
     pub avg: Option<f64>,
 }
 
@@ -70,7 +70,7 @@ impl Stats {
     }
 }
 
-/// `IntermediateStats` contains the mergeable version for stats.
+/// IntermediateStats contains the mergeable version for stats.
 #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
 pub struct IntermediateStats {
     count: usize,
@@ -163,13 +163,13 @@ impl SegmentStatsCollector {
             stats: IntermediateStats::default(),
         }
     }
-    pub(crate) fn collect_block(&mut self, doc: &[DocId], field: &dyn Column<u64>) {
+    pub(crate) fn collect_block(&mut self, doc: &[DocId], field: &FastFieldReaderImpl<u64>) {
         let mut iter = doc.chunks_exact(4);
         for docs in iter.by_ref() {
-            let val1 = field.get_val(docs[0] as u64);
-            let val2 = field.get_val(docs[1] as u64);
-            let val3 = field.get_val(docs[2] as u64);
-            let val4 = field.get_val(docs[3] as u64);
+            let val1 = field.get(docs[0]);
+            let val2 = field.get(docs[1]);
+            let val3 = field.get(docs[2]);
+            let val4 = field.get(docs[3]);
             let val1 = f64_from_fastfield_u64(val1, &self.field_type);
             let val2 = f64_from_fastfield_u64(val2, &self.field_type);
             let val3 = f64_from_fastfield_u64(val3, &self.field_type);
@@ -179,8 +179,8 @@ impl SegmentStatsCollector {
             self.stats.collect(val3);
             self.stats.collect(val4);
         }
-        for &doc in iter.remainder() {
-            let val = field.get_val(doc as u64);
+        for doc in iter.remainder() {
+            let val = field.get(*doc);
             let val = f64_from_fastfield_u64(val, &self.field_type);
             self.stats.collect(val);
         }

src/aggregation/mod.rs

@@ -14,14 +14,13 @@
 //!
 //!
 //! To use aggregations, build an aggregation request by constructing
-//! [`Aggregations`](agg_req::Aggregations).
-//! Create an [`AggregationCollector`] from this request. `AggregationCollector` implements the
-//! [`Collector`](crate::collector::Collector) trait and can be passed as collector into
-//! [`Searcher::search()`](crate::Searcher::search).
+//! [Aggregations](agg_req::Aggregations).
+//! Create an [AggregationCollector] from this request. AggregationCollector implements the
+//! `Collector` trait and can be passed as collector into `searcher.search()`.
 //!
 //! #### Limitations
 //!
-//! Currently aggregations work only on single value fast fields of type `u64`, `f64`, `i64` and
+//! Currently aggregations work only on single value fast fields of type u64, f64, i64 and
 //! fast fields on text fields.
 //!
 //! # JSON Format
@@ -45,8 +44,8 @@
 //!     - [Stats](metric::StatsAggregation)
 //!
 //! # Example
-//! Compute the average metric, by building [`agg_req::Aggregations`], which is built from an
-//! `(String, agg_req::Aggregation)` iterator.
+//! Compute the average metric, by building [agg_req::Aggregations], which is built from an (String,
+//! [agg_req::Aggregation]) iterator.
 //!
 //! ```
 //! use tantivy::aggregation::agg_req::{Aggregations, Aggregation, MetricAggregation};
@@ -144,15 +143,15 @@
 //! ```
 //!
 //! # Distributed Aggregation
-//! When the data is distributed on different [`Index`](crate::Index) instances, the
-//! [`DistributedAggregationCollector`] provides functionality to merge data between independent
+//! When the data is distributed on different [crate::Index] instances, the
+//! [DistributedAggregationCollector] provides functionality to merge data between independent
 //! search calls by returning
-//! [`IntermediateAggregationResults`](intermediate_agg_result::IntermediateAggregationResults).
-//! `IntermediateAggregationResults` provides the
-//! [`merge_fruits`](intermediate_agg_result::IntermediateAggregationResults::merge_fruits) method
-//! to merge multiple results. The merged result can then be converted into
-//! [`AggregationResults`](agg_result::AggregationResults) via the
-//! [`into_final_bucket_result`](intermediate_agg_result::IntermediateAggregationResults::into_final_bucket_result) method.
+//! [IntermediateAggregationResults](intermediate_agg_result::IntermediateAggregationResults).
+//! IntermediateAggregationResults provides the
+//! [merge_fruits](intermediate_agg_result::IntermediateAggregationResults::merge_fruits) method to
+//! merge multiple results. The merged result can then be converted into
+//! [agg_result::AggregationResults] via the
+//! [agg_result::AggregationResults::from_intermediate_and_req] method.
 
 pub mod agg_req;
 mod agg_req_with_accessor;
@@ -162,6 +161,7 @@ mod collector;
 pub mod intermediate_agg_result;
 pub mod metric;
 mod segment_agg_result;
+
 use std::collections::HashMap;
 use std::fmt::Display;
 
@@ -169,10 +169,10 @@ pub use collector::{
     AggregationCollector, AggregationSegmentCollector, DistributedAggregationCollector,
     MAX_BUCKET_COUNT,
 };
-use fastfield_codecs::MonotonicallyMappableToU64;
 use itertools::Itertools;
 use serde::{Deserialize, Serialize};
 
+use crate::fastfield::FastValue;
 use crate::schema::Type;
 
 /// Represents an associative array `(key => values)` in a very efficient manner.
@@ -260,7 +260,7 @@ impl<T: Clone> VecWithNames<T> {
     }
 }
 
-/// The serialized key is used in a `HashMap`.
+/// The serialized key is used in a HashMap.
 pub type SerializedKey = String;
 
 #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, PartialOrd)]
@@ -269,7 +269,7 @@ pub type SerializedKey = String;
 pub enum Key {
     /// String key
     Str(String),
-    /// `f64` key
+    /// f64 key
     F64(f64),
 }
 
@@ -282,10 +282,10 @@ impl Display for Key {
     }
 }
 
-/// Inverse of `to_fastfield_u64`. Used to convert to `f64` for metrics.
+/// Invert of to_fastfield_u64. Used to convert to f64 for metrics.
 ///
 /// # Panics
-/// Only `u64`, `f64`, and `i64` are supported.
+/// Only u64, f64, i64 is supported
 pub(crate) fn f64_from_fastfield_u64(val: u64, field_type: &Type) -> f64 {
     match field_type {
         Type::U64 => val as f64,
@@ -297,15 +297,15 @@ pub(crate) fn f64_from_fastfield_u64(val: u64, field_type: &Type) -> f64 {
     }
 }
 
-/// Converts the `f64` value to fast field value space.
+/// Converts the f64 value to fast field value space.
 ///
-/// If the fast field has `u64`, values are stored as `u64` in the fast field.
-/// A `f64` value of e.g. `2.0` therefore needs to be converted to `1u64`.
+/// If the fast field has u64, values are stored as u64 in the fast field.
+/// A f64 value of e.g. 2.0 therefore needs to be converted to 1u64
 ///
-/// If the fast field has `f64` values are converted and stored to `u64` using a
+/// If the fast field has f64 values are converted and stored to u64 using a
 /// monotonic mapping.
-/// A `f64` value of e.g. `2.0` needs to be converted using the same monotonic
-/// conversion function, so that the value matches the `u64` value stored in the fast
+/// A f64 value of e.g. 2.0 needs to be converted using the same monotonic
+/// conversion function, so that the value matches the u64 value stored in the fast
 /// field.
 pub(crate) fn f64_to_fastfield_u64(val: f64, field_type: &Type) -> Option<u64> {
     match field_type {

src/aggregation/segment_agg_result.rs

@@ -185,10 +185,10 @@ impl SegmentMetricResultCollector {
     pub(crate) fn collect_block(&mut self, doc: &[DocId], metric: &MetricAggregationWithAccessor) {
         match self {
             SegmentMetricResultCollector::Average(avg_collector) => {
-                avg_collector.collect_block(doc, &*metric.accessor);
+                avg_collector.collect_block(doc, &metric.accessor);
             }
             SegmentMetricResultCollector::Stats(stats_collector) => {
-                stats_collector.collect_block(doc, &*metric.accessor);
+                stats_collector.collect_block(doc, &metric.accessor);
             }
         }
     }

src/collector/custom_score_top_collector.rs

@@ -24,7 +24,7 @@ where TScore: Clone + PartialOrd
 /// A custom segment scorer makes it possible to define any kind of score
 /// for a given document belonging to a specific segment.
 ///
-/// It is the segment local version of the [`CustomScorer`].
+/// It is the segment local version of the [`CustomScorer`](./trait.CustomScorer.html).
 pub trait CustomSegmentScorer<TScore>: 'static {
     /// Computes the score of a specific `doc`.
     fn score(&mut self, doc: DocId) -> TScore;
@@ -36,7 +36,7 @@ pub trait CustomSegmentScorer<TScore>: 'static {
 /// Instead, it helps constructing `Self::Child` instances that will compute
 /// the score at a segment scale.
 pub trait CustomScorer<TScore>: Sync {
-    /// Type of the associated [`CustomSegmentScorer`].
+    /// Type of the associated [`CustomSegmentScorer`](./trait.CustomSegmentScorer.html).
     type Child: CustomSegmentScorer<TScore>;
     /// Builds a child scorer for a specific segment. The child scorer is associated to
     /// a specific segment.

src/collector/facet_collector.rs

@@ -67,10 +67,10 @@ fn facet_depth(facet_bytes: &[u8]) -> usize {
 /// (e.g. `/category/fiction`, `/category/biography`, `/category/personal_development`).
 ///
 /// Once collection is finished, you can harvest its results in the form
-/// of a [`FacetCounts`] object, and extract your facet counts from it.
+/// of a `FacetCounts` object, and extract your face                t counts from it.
 ///
 /// This implementation assumes you are working with a number of facets that
-/// is many hundreds of times smaller than your number of documents.
+/// is much hundreds of time lower than your number of documents.
 ///
 ///
 /// ```rust
@@ -231,7 +231,7 @@ impl FacetCollector {
     ///
     /// Adding two facets within which one is the prefix of the other is forbidden.
     /// If you need the correct number of unique documents for two such facets,
-    /// just add them in a separate `FacetCollector`.
+    /// just add them in separate `FacetCollector`.
     pub fn add_facet<T>(&mut self, facet_from: T)
     where Facet: From<T> {
         let facet = Facet::from(facet_from);
@@ -391,7 +391,7 @@ impl<'a> Iterator for FacetChildIterator<'a> {
 
 impl FacetCounts {
     /// Returns an iterator over all of the facet count pairs inside this result.
-    /// See the documentation for [`FacetCollector`] for a usage example.
+    /// See the documentation for [FacetCollector] for a usage example.
     pub fn get<T>(&self, facet_from: T) -> FacetChildIterator<'_>
     where Facet: From<T> {
         let facet = Facet::from(facet_from);
@@ -410,7 +410,7 @@ impl FacetCounts {
     }
 
     /// Returns a vector of top `k` facets with their counts, sorted highest-to-lowest by counts.
-    /// See the documentation for [`FacetCollector`] for a usage example.
+    /// See the documentation for [FacetCollector] for a usage example.
     pub fn top_k<T>(&self, facet: T, k: usize) -> Vec<(&Facet, u64)>
     where Facet: From<T> {
         let mut heap = BinaryHeap::with_capacity(k);

src/collector/filter_collector_wrapper.rs

@@ -10,12 +10,9 @@
 // ---
 // Importing tantivy...
 use std::marker::PhantomData;
-use std::sync::Arc;
-
-use fastfield_codecs::Column;
 
 use crate::collector::{Collector, SegmentCollector};
-use crate::fastfield::FastValue;
+use crate::fastfield::{FastFieldReader, FastFieldReaderImpl, FastValue};
 use crate::schema::Field;
 use crate::{Score, SegmentReader, TantivyError};
 
@@ -161,7 +158,7 @@ where
     TPredicate: 'static,
     TPredicateValue: FastValue,
 {
-    fast_field_reader: Arc<dyn Column<TPredicateValue>>,
+    fast_field_reader: FastFieldReaderImpl<TPredicateValue>,
     segment_collector: TSegmentCollector,
     predicate: TPredicate,
     t_predicate_value: PhantomData<TPredicateValue>,
@@ -177,7 +174,7 @@ where
     type Fruit = TSegmentCollector::Fruit;
 
     fn collect(&mut self, doc: u32, score: Score) {
-        let value = self.fast_field_reader.get_val(doc as u64);
+        let value = self.fast_field_reader.get(doc);
         if (self.predicate)(value) {
             self.segment_collector.collect(doc, score)
         }

src/collector/histogram_collector.rs

@@ -1,10 +1,7 @@
-use std::sync::Arc;
-
 use fastdivide::DividerU64;
-use fastfield_codecs::Column;
 
 use crate::collector::{Collector, SegmentCollector};
-use crate::fastfield::FastValue;
+use crate::fastfield::{FastFieldReader, FastFieldReaderImpl, FastValue};
 use crate::schema::{Field, Type};
 use crate::{DocId, Score};
 
@@ -87,14 +84,14 @@ impl HistogramComputer {
 }
 pub struct SegmentHistogramCollector {
     histogram_computer: HistogramComputer,
-    ff_reader: Arc<dyn Column<u64>>,
+    ff_reader: FastFieldReaderImpl<u64>,
 }
 
 impl SegmentCollector for SegmentHistogramCollector {
     type Fruit = Vec<u64>;
 
     fn collect(&mut self, doc: DocId, _score: Score) {
-        let value = self.ff_reader.get_val(doc as u64);
+        let value = self.ff_reader.get(doc);
         self.histogram_computer.add_value(value);
     }
 

src/collector/mod.rs

@@ -4,13 +4,13 @@
 //! In tantivy jargon, we call this information your search "fruit".
 //!
 //! Your fruit could for instance be :
-//! - [the count of matching documents](crate::collector::Count)
-//! - [the top 10 documents, by relevancy or by a fast field](crate::collector::TopDocs)
-//! - [facet counts](FacetCollector)
+//! - [the count of matching documents](./struct.Count.html)
+//! - [the top 10 documents, by relevancy or by a fast field](./struct.TopDocs.html)
+//! - [facet counts](./struct.FacetCollector.html)
 //!
-//! At some point in your code, you will trigger the actual search operation by calling
-//! [`Searcher::search()`](crate::Searcher::search).
-//! This call will look like this:
+//! At one point in your code, you will trigger the actual search operation by calling
+//! [the `search(...)` method of your `Searcher` object](../struct.Searcher.html#method.search).
+//! This call will look like this.
 //!
 //! ```verbatim
 //! let fruit = searcher.search(&query, &collector)?;
@@ -64,7 +64,7 @@
 //!
 //! The `Collector` trait is implemented for up to 4 collectors.
 //! If you have more than 4 collectors, you can either group them into
-//! tuples of tuples `(a,(b,(c,d)))`, or rely on [`MultiCollector`].
+//! tuples of tuples `(a,(b,(c,d)))`, or rely on [`MultiCollector`](./struct.MultiCollector.html).
 //!
 //! # Combining several collectors dynamically
 //!
@@ -74,7 +74,7 @@
 //!
 //! Unfortunately it requires you to know at compile time your collector types.
 //! If on the other hand, the collectors depend on some query parameter,
-//! you can rely on [`MultiCollector`]'s.
+//! you can rely on `MultiCollector`'s.
 //!
 //!
 //! # Implementing your own collectors.

src/collector/tests.rs

@@ -1,11 +1,7 @@
-use std::sync::Arc;
-
-use fastfield_codecs::Column;
-
 use super::*;
 use crate::collector::{Count, FilterCollector, TopDocs};
 use crate::core::SegmentReader;
-use crate::fastfield::BytesFastFieldReader;
+use crate::fastfield::{BytesFastFieldReader, FastFieldReader, FastFieldReaderImpl};
 use crate::query::{AllQuery, QueryParser};
 use crate::schema::{Field, Schema, FAST, TEXT};
 use crate::time::format_description::well_known::Rfc3339;
@@ -160,7 +156,7 @@ pub struct FastFieldTestCollector {
 
 pub struct FastFieldSegmentCollector {
     vals: Vec<u64>,
-    reader: Arc<dyn Column<u64>>,
+    reader: FastFieldReaderImpl<u64>,
 }
 
 impl FastFieldTestCollector {
@@ -201,7 +197,7 @@ impl SegmentCollector for FastFieldSegmentCollector {
     type Fruit = Vec<u64>;
 
     fn collect(&mut self, doc: DocId, _score: Score) {
-        let val = self.reader.get_val(doc as u64);
+        let val = self.reader.get(doc);
         self.vals.push(val);
     }
 

src/collector/top_score_collector.rs

@@ -1,9 +1,6 @@
 use std::collections::BinaryHeap;
 use std::fmt;
 use std::marker::PhantomData;
-use std::sync::Arc;
-
-use fastfield_codecs::Column;
 
 use super::Collector;
 use crate::collector::custom_score_top_collector::CustomScoreTopCollector;
@@ -12,7 +9,7 @@ use crate::collector::tweak_score_top_collector::TweakedScoreTopCollector;
 use crate::collector::{
     CustomScorer, CustomSegmentScorer, ScoreSegmentTweaker, ScoreTweaker, SegmentCollector,
 };
-use crate::fastfield::FastValue;
+use crate::fastfield::{FastFieldReader, FastFieldReaderImpl, FastValue};
 use crate::query::Weight;
 use crate::schema::Field;
 use crate::{DocAddress, DocId, Score, SegmentOrdinal, SegmentReader, TantivyError};
@@ -132,12 +129,12 @@ impl fmt::Debug for TopDocs {
 }
 
 struct ScorerByFastFieldReader {
-    ff_reader: Arc<dyn Column<u64>>,
+    ff_reader: FastFieldReaderImpl<u64>,
 }
 
 impl CustomSegmentScorer<u64> for ScorerByFastFieldReader {
     fn score(&mut self, doc: DocId) -> u64 {
-        self.ff_reader.get_val(doc as u64)
+        self.ff_reader.get(doc)
     }
 }
 
@@ -287,7 +284,7 @@ impl TopDocs {
     /// # See also
     ///
     /// To comfortably work with `u64`s, `i64`s, `f64`s, or `date`s, please refer to
-    /// the [.order_by_fast_field(...)](TopDocs::order_by_fast_field) method.
+    /// [.order_by_fast_field(...)](#method.order_by_fast_field) method.
     pub fn order_by_u64_field(
         self,
         field: Field,
@@ -384,7 +381,7 @@ impl TopDocs {
     ///
     /// This method offers a convenient way to tweak or replace
     /// the documents score. As suggested by the prototype you can
-    /// manually define your own [`ScoreTweaker`]
+    /// manually define your own [`ScoreTweaker`](./trait.ScoreTweaker.html)
     /// and pass it as an argument, but there is a much simpler way to
     /// tweak your score: you can use a closure as in the following
     /// example.
@@ -401,7 +398,7 @@ impl TopDocs {
     /// In the following example will will tweak our ranking a bit by
     /// boosting popular products a notch.
     ///
-    /// In more serious application, this tweaking could involve running a
+    /// In more serious application, this tweaking could involved running a
     /// learning-to-rank model over various features
     ///
     /// ```rust
@@ -410,6 +407,7 @@ impl TopDocs {
     /// # use tantivy::query::QueryParser;
     /// use tantivy::SegmentReader;
     /// use tantivy::collector::TopDocs;
+    /// use tantivy::fastfield::FastFieldReader;
     /// use tantivy::schema::Field;
     ///
     /// fn create_schema() -> Schema {
@@ -458,7 +456,7 @@ impl TopDocs {
     ///
     ///             // We can now define our actual scoring function
     ///             move |doc: DocId, original_score: Score| {
-    ///                 let popularity: u64 = popularity_reader.get_val(doc as u64);
+    ///                 let popularity: u64 = popularity_reader.get(doc);
     ///                 // Well.. For the sake of the example we use a simple logarithm
     ///                 // function.
     ///                 let popularity_boost_score = ((2u64 + popularity) as Score).log2();
@@ -474,7 +472,7 @@ impl TopDocs {
     /// ```
     ///
     /// # See also
-    /// - [custom_score(...)](TopDocs::custom_score)
+    /// [custom_score(...)](#method.custom_score).
     pub fn tweak_score<TScore, TScoreSegmentTweaker, TScoreTweaker>(
         self,
         score_tweaker: TScoreTweaker,
@@ -491,7 +489,8 @@ impl TopDocs {
     ///
     /// This method offers a convenient way to use a different score.
     ///
-    /// As suggested by the prototype you can manually define your own [`CustomScorer`]
+    /// As suggested by the prototype you can manually define your
+    /// own [`CustomScorer`](./trait.CustomScorer.html)
     /// and pass it as an argument, but there is a much simpler way to
     /// tweak your score: you can use a closure as in the following
     /// example.
@@ -516,6 +515,7 @@ impl TopDocs {
     /// use tantivy::SegmentReader;
     /// use tantivy::collector::TopDocs;
     /// use tantivy::schema::Field;
+    /// use tantivy::fastfield::FastFieldReader;
     ///
     /// # fn create_schema() -> Schema {
     /// #    let mut schema_builder = Schema::builder();
@@ -567,8 +567,8 @@ impl TopDocs {
     ///
     ///             // We can now define our actual scoring function
     ///             move |doc: DocId| {
-    ///                 let popularity: u64 = popularity_reader.get_val(doc as u64);
-    ///                 let boosted: u64 = boosted_reader.get_val(doc as u64);
+    ///                 let popularity: u64 = popularity_reader.get(doc);
+    ///                 let boosted: u64 = boosted_reader.get(doc);
     ///                 // Score do not have to be `f64` in tantivy.
     ///                 // Here we return a couple to get lexicographical order
     ///                 // for free.
@@ -587,7 +587,7 @@ impl TopDocs {
     /// ```
     ///
     /// # See also
-    /// - [tweak_score(...)](TopDocs::tweak_score)
+    /// [tweak_score(...)](#method.tweak_score).
     pub fn custom_score<TScore, TCustomSegmentScorer, TCustomScorer>(
         self,
         custom_score: TCustomScorer,

src/collector/tweak_score_top_collector.rs

@@ -24,7 +24,7 @@ where TScore: Clone + PartialOrd
 /// A `ScoreSegmentTweaker` makes it possible to modify the default score
 /// for a given document belonging to a specific segment.
 ///
-/// It is the segment local version of the [`ScoreTweaker`].
+/// It is the segment local version of the [`ScoreTweaker`](./trait.ScoreTweaker.html).
 pub trait ScoreSegmentTweaker<TScore>: 'static {
     /// Tweak the given `score` for the document `doc`.
     fn score(&mut self, doc: DocId, score: Score) -> TScore;
@@ -37,7 +37,7 @@ pub trait ScoreSegmentTweaker<TScore>: 'static {
 /// Instead, it helps constructing `Self::Child` instances that will compute
 /// the score at a segment scale.
 pub trait ScoreTweaker<TScore>: Sync {
-    /// Type of the associated [`ScoreSegmentTweaker`].
+    /// Type of the associated [`ScoreSegmentTweaker`](./trait.ScoreSegmentTweaker.html).
     type Child: ScoreSegmentTweaker<TScore>;
 
     /// Builds a child tweaker for a specific segment. The child scorer is associated to

src/core/index.rs

@@ -7,7 +7,6 @@ use std::sync::Arc;
 
 use super::segment::Segment;
 use super::IndexSettings;
-use crate::core::single_segment_index_writer::SingleSegmentIndexWriter;
 use crate::core::{
     Executor, IndexMeta, SegmentId, SegmentMeta, SegmentMetaInventory, META_FILEPATH,
 };
@@ -17,7 +16,7 @@ use crate::directory::MmapDirectory;
 use crate::directory::{Directory, ManagedDirectory, RamDirectory, INDEX_WRITER_LOCK};
 use crate::error::{DataCorruption, TantivyError};
 use crate::indexer::index_writer::{MAX_NUM_THREAD, MEMORY_ARENA_NUM_BYTES_MIN};
-use crate::indexer::segment_updater::save_metas;
+use crate::indexer::segment_updater::save_new_metas;
 use crate::reader::{IndexReader, IndexReaderBuilder};
 use crate::schema::{Field, FieldType, Schema};
 use crate::tokenizer::{TextAnalyzer, TokenizerManager};
@@ -48,38 +47,10 @@ fn load_metas(
         .map_err(From::from)
 }
 
-/// Save the index meta file.
-/// This operation is atomic :
-/// Either
-///  - it fails, in which case an error is returned,
-/// and the `meta.json` remains untouched,
-/// - it succeeds, and `meta.json` is written
-/// and flushed.
-///
-/// This method is not part of tantivy's public API
-fn save_new_metas(
-    schema: Schema,
-    index_settings: IndexSettings,
-    directory: &dyn Directory,
-) -> crate::Result<()> {
-    save_metas(
-        &IndexMeta {
-            index_settings,
-            segments: Vec::new(),
-            schema,
-            opstamp: 0u64,
-            payload: None,
-        },
-        directory,
-    )?;
-    directory.sync_directory()?;
-    Ok(())
-}
-
 /// IndexBuilder can be used to create an index.
 ///
-/// Use in conjunction with [`SchemaBuilder`][crate::schema::SchemaBuilder].
-/// Global index settings can be configured with [`IndexSettings`].
+/// Use in conjunction with `SchemaBuilder`. Global index settings
+/// can be configured with `IndexSettings`
 ///
 /// # Examples
 ///
@@ -97,13 +68,7 @@ fn save_new_metas(
 /// );
 ///
 /// let schema = schema_builder.build();
-/// let settings = IndexSettings{
-///     sort_by_field: Some(IndexSortByField{
-///         field: "number".to_string(),
-///         order: Order::Asc
-///     }),
-///     ..Default::default()
-/// };
+/// let settings = IndexSettings{sort_by_field: Some(IndexSortByField{field:"number".to_string(), order:Order::Asc}), ..Default::default()};
 /// let index = Index::builder().schema(schema).settings(settings).create_in_ram();
 /// ```
 pub struct IndexBuilder {
@@ -146,7 +111,7 @@ impl IndexBuilder {
         self
     }
 
-    /// Creates a new index using the [`RamDirectory`].
+    /// Creates a new index using the `RAMDirectory`.
     ///
     /// The index will be allocated in anonymous memory.
     /// This should only be used for unit tests.
@@ -154,14 +119,13 @@ impl IndexBuilder {
         let ram_directory = RamDirectory::create();
         Ok(self
             .create(ram_directory)
-            .expect("Creating a RamDirectory should never fail"))
+            .expect("Creating a RAMDirectory should never fail"))
     }
 
     /// Creates a new index in a given filepath.
-    /// The index will use the [`MmapDirectory`].
+    /// The index will use the `MMapDirectory`.
     ///
-    /// If a previous index was in this directory, it returns an
-    /// [`TantivyError::IndexAlreadyExists`] error.
+    /// If a previous index was in this directory, it returns an `IndexAlreadyExists` error.
     #[cfg(feature = "mmap")]
     pub fn create_in_dir<P: AsRef<Path>>(self, directory_path: P) -> crate::Result<Index> {
         let mmap_directory: Box<dyn Directory> = Box::new(MmapDirectory::open(directory_path)?);
@@ -171,34 +135,14 @@ impl IndexBuilder {
         self.create(mmap_directory)
     }
 
-    /// Dragons ahead!!!
-    ///
-    /// The point of this API is to let users create a simple index with a single segment
-    /// and without starting any thread.
-    ///
-    /// Do not use this method if you are not sure what you are doing.
-    ///
-    /// It expects an originally empty directory, and will not run any GC operation.
-    #[doc(hidden)]
-    pub fn single_segment_index_writer(
-        self,
-        dir: impl Into<Box<dyn Directory>>,
-        mem_budget: usize,
-    ) -> crate::Result<SingleSegmentIndexWriter> {
-        let index = self.create(dir)?;
-        let index_simple_writer = SingleSegmentIndexWriter::new(index, mem_budget)?;
-        Ok(index_simple_writer)
-    }
-
     /// Creates a new index in a temp directory.
     ///
-    /// The index will use the [`MmapDirectory`] in a newly created directory.
-    /// The temp directory will be destroyed automatically when the [`Index`] object
+    /// The index will use the `MMapDirectory` in a newly created directory.
+    /// The temp directory will be destroyed automatically when the `Index` object
     /// is destroyed.
     ///
-    /// The temp directory is only used for testing the [`MmapDirectory`].
-    /// For other unit tests, prefer the [`RamDirectory`], see:
-    /// [`IndexBuilder::create_in_ram()`].
+    /// The temp directory is only used for testing the `MmapDirectory`.
+    /// For other unit tests, prefer the `RAMDirectory`, see: `create_in_ram`.
     #[cfg(feature = "mmap")]
     pub fn create_from_tempdir(self) -> crate::Result<Index> {
         let mmap_directory: Box<dyn Directory> = Box::new(MmapDirectory::create_from_tempdir()?);
@@ -294,7 +238,7 @@ impl Index {
         self.set_multithread_executor(default_num_threads)
     }
 
-    /// Creates a new index using the [`RamDirectory`].
+    /// Creates a new index using the `RamDirectory`.
     ///
     /// The index will be allocated in anonymous memory.
     /// This is useful for indexing small set of documents
@@ -304,10 +248,9 @@ impl Index {
     }
 
     /// Creates a new index in a given filepath.
-    /// The index will use the [`MmapDirectory`].
+    /// The index will use the `MMapDirectory`.
     ///
-    /// If a previous index was in this directory, then it returns
-    /// a [`TantivyError::IndexAlreadyExists`] error.
+    /// If a previous index was in this directory, then it returns  an `IndexAlreadyExists` error.
     #[cfg(feature = "mmap")]
     pub fn create_in_dir<P: AsRef<Path>>(
         directory_path: P,
@@ -329,13 +272,12 @@ impl Index {
 
     /// Creates a new index in a temp directory.
     ///
-    /// The index will use the [`MmapDirectory`] in a newly created directory.
-    /// The temp directory will be destroyed automatically when the [`Index`] object
+    /// The index will use the `MMapDirectory` in a newly created directory.
+    /// The temp directory will be destroyed automatically when the `Index` object
     /// is destroyed.
     ///
-    /// The temp directory is only used for testing the [`MmapDirectory`].
-    /// For other unit tests, prefer the [`RamDirectory`],
-    /// see: [`IndexBuilder::create_in_ram()`].
+    /// The temp directory is only used for testing the `MmapDirectory`.
+    /// For other unit tests, prefer the `RamDirectory`, see: `create_in_ram`.
     #[cfg(feature = "mmap")]
     pub fn create_from_tempdir(schema: Schema) -> crate::Result<Index> {
         IndexBuilder::new().schema(schema).create_from_tempdir()
@@ -355,7 +297,7 @@ impl Index {
         builder.create(dir)
     }
 
-    /// Creates a new index given a directory and an [`IndexMeta`].
+    /// Creates a new index given a directory and an `IndexMeta`.
     fn open_from_metas(
         directory: ManagedDirectory,
         metas: &IndexMeta,
@@ -382,7 +324,7 @@ impl Index {
         &self.tokenizers
     }
 
-    /// Get the tokenizer associated with a specific field.
+    /// Helper to access the tokenizer associated to a specific field.
     pub fn tokenizer_for_field(&self, field: Field) -> crate::Result<TextAnalyzer> {
         let field_entry = self.schema.get_field_entry(field);
         let field_type = field_entry.field_type();
@@ -414,14 +356,14 @@ impl Index {
             })
     }
 
-    /// Create a default [`IndexReader`] for the given index.
+    /// Create a default `IndexReader` for the given index.
     ///
-    /// See [`Index.reader_builder()`].
+    /// See [`Index.reader_builder()`](#method.reader_builder).
     pub fn reader(&self) -> crate::Result<IndexReader> {
         self.reader_builder().try_into()
     }
 
-    /// Create a [`IndexReader`] for the given index.
+    /// Create a `IndexReader` for the given index.
     ///
     /// Most project should create at most one reader for a given index.
     /// This method is typically called only once per `Index` instance.
@@ -638,12 +580,10 @@ impl fmt::Debug for Index {
 
 #[cfg(test)]
 mod tests {
-    use crate::collector::Count;
     use crate::directory::{RamDirectory, WatchCallback};
-    use crate::query::TermQuery;
-    use crate::schema::{Field, IndexRecordOption, Schema, INDEXED, TEXT};
+    use crate::schema::{Field, Schema, INDEXED, TEXT};
     use crate::tokenizer::TokenizerManager;
-    use crate::{Directory, Index, IndexBuilder, IndexReader, IndexSettings, ReloadPolicy, Term};
+    use crate::{Directory, Index, IndexBuilder, IndexReader, IndexSettings, ReloadPolicy};
 
     #[test]
     fn test_indexer_for_field() {
@@ -909,28 +849,4 @@ mod tests {
         );
         Ok(())
     }
-
-    #[test]
-    fn test_single_segment_index_writer() -> crate::Result<()> {
-        let mut schema_builder = Schema::builder();
-        let text_field = schema_builder.add_text_field("text", TEXT);
-        let schema = schema_builder.build();
-        let directory = RamDirectory::default();
-        let mut single_segment_index_writer = Index::builder()
-            .schema(schema)
-            .single_segment_index_writer(directory, 10_000_000)?;
-        for _ in 0..10 {
-            let doc = doc!(text_field=>"hello");
-            single_segment_index_writer.add_document(doc)?;
-        }
-        let index = single_segment_index_writer.finalize()?;
-        let searcher = index.reader()?.searcher();
-        let term_query = TermQuery::new(
-            Term::from_field_text(text_field, "hello"),
-            IndexRecordOption::Basic,
-        );
-        let count = searcher.search(&term_query, &Count)?;
-        assert_eq!(count, 10);
-        Ok(())
-    }
 }

src/core/index_meta.rs

@@ -235,14 +235,6 @@ impl InnerSegmentMeta {
     }
 }
 
-fn return_true() -> bool {
-    true
-}
-
-fn is_true(val: &bool) -> bool {
-    *val
-}
-
 /// Search Index Settings.
 ///
 /// Contains settings which are applied on the whole
@@ -256,12 +248,6 @@ pub struct IndexSettings {
     /// The `Compressor` used to compress the doc store.
     #[serde(default)]
     pub docstore_compression: Compressor,
-    /// If set to true, docstore compression will happen on a dedicated thread.
-    /// (defaults: true)
-    #[doc(hidden)]
-    #[serde(default = "return_true")]
-    #[serde(skip_serializing_if = "is_true")]
-    pub docstore_compress_dedicated_thread: bool,
     #[serde(default = "default_docstore_blocksize")]
     /// The size of each block that will be compressed and written to disk
     pub docstore_blocksize: usize,
@@ -278,7 +264,6 @@ impl Default for IndexSettings {
             sort_by_field: None,
             docstore_compression: Compressor::default(),
             docstore_blocksize: default_docstore_blocksize(),
-            docstore_compress_dedicated_thread: true,
         }
     }
 }
@@ -410,7 +395,7 @@ mod tests {
     use super::IndexMeta;
     use crate::core::index_meta::UntrackedIndexMeta;
     use crate::schema::{Schema, TEXT};
-    use crate::store::{Compressor, ZstdCompressor};
+    use crate::store::ZstdCompressor;
     use crate::{IndexSettings, IndexSortByField, Order};
 
     #[test]
@@ -462,7 +447,6 @@ mod tests {
                     compression_level: Some(4),
                 }),
                 docstore_blocksize: 1_000_000,
-                docstore_compress_dedicated_thread: true,
             },
             segments: Vec::new(),
             schema,
@@ -501,47 +485,4 @@ mod tests {
             "unknown zstd option \"bla\" at line 1 column 103".to_string()
         );
     }
-
-    #[test]
-    #[cfg(feature = "lz4-compression")]
-    fn test_index_settings_default() {
-        let mut index_settings = IndexSettings::default();
-        assert_eq!(
-            index_settings,
-            IndexSettings {
-                sort_by_field: None,
-                docstore_compression: Compressor::default(),
-                docstore_compress_dedicated_thread: true,
-                docstore_blocksize: 16_384
-            }
-        );
-        {
-            let index_settings_json = serde_json::to_value(&index_settings).unwrap();
-            assert_eq!(
-                index_settings_json,
-                serde_json::json!({
-                    "docstore_compression": "lz4",
-                    "docstore_blocksize": 16384
-                })
-            );
-            let index_settings_deser: IndexSettings =
-                serde_json::from_value(index_settings_json).unwrap();
-            assert_eq!(index_settings_deser, index_settings);
-        }
-        {
-            index_settings.docstore_compress_dedicated_thread = false;
-            let index_settings_json = serde_json::to_value(&index_settings).unwrap();
-            assert_eq!(
-                index_settings_json,
-                serde_json::json!({
-                    "docstore_compression": "lz4",
-                    "docstore_blocksize": 16384,
-                    "docstore_compress_dedicated_thread": false,
-                })
-            );
-            let index_settings_deser: IndexSettings =
-                serde_json::from_value(index_settings_json).unwrap();
-            assert_eq!(index_settings_deser, index_settings);
-        }
-    }
 }

src/core/mod.rs

@@ -7,7 +7,6 @@ mod segment;
 mod segment_component;
 mod segment_id;
 mod segment_reader;
-mod single_segment_index_writer;
 
 use std::path::Path;
 
@@ -24,7 +23,6 @@ pub use self::segment::Segment;
 pub use self::segment_component::SegmentComponent;
 pub use self::segment_id::SegmentId;
 pub use self::segment_reader::SegmentReader;
-pub use self::single_segment_index_writer::SingleSegmentIndexWriter;
 
 /// The meta file contains all the information about the list of segments and the schema
 /// of the index.

src/core/searcher.rs

@@ -10,12 +10,12 @@ use crate::space_usage::SearcherSpaceUsage;
 use crate::store::{CacheStats, StoreReader};
 use crate::{DocAddress, Index, Opstamp, SegmentId, TrackedObject};
 
-/// Identifies the searcher generation accessed by a [`Searcher`].
+/// Identifies the searcher generation accessed by a [Searcher].
 ///
-/// While this might seem redundant, a [`SearcherGeneration`] contains
+/// While this might seem redundant, a [SearcherGeneration] contains
 /// both a `generation_id` AND a list of `(SegmentId, DeleteOpstamp)`.
 ///
-/// This is on purpose. This object is used by the [`Warmer`](crate::reader::Warmer) API.
+/// This is on purpose. This object is used by the `Warmer` API.
 /// Having both information makes it possible to identify which
 /// artifact should be refreshed or garbage collected.
 ///
@@ -74,15 +74,15 @@ impl Searcher {
         &self.inner.index
     }
 
-    /// [`SearcherGeneration`] which identifies the version of the snapshot held by this `Searcher`.
+    /// [SearcherGeneration] which identifies the version of the snapshot held by this `Searcher`.
     pub fn generation(&self) -> &SearcherGeneration {
         self.inner.generation.as_ref()
     }
 
-    /// Fetches a document from tantivy's store given a [`DocAddress`].
+    /// Fetches a document from tantivy's store given a `DocAddress`.
     ///
     /// The searcher uses the segment ordinal to route the
-    /// request to the right `Segment`.
+    /// the request to the right `Segment`.
     pub fn doc(&self, doc_address: DocAddress) -> crate::Result<Document> {
         let store_reader = &self.inner.store_readers[doc_address.segment_ord as usize];
         store_reader.get(doc_address.doc_id)
@@ -180,7 +180,7 @@ impl Searcher {
         self.search_with_executor(query, collector, executor)
     }
 
-    /// Same as [`search(...)`](Searcher::search) but multithreaded.
+    /// Same as [`search(...)`](#method.search) but multithreaded.
     ///
     /// The current implementation is rather naive :
     /// multithreading is by splitting search into as many task
@@ -247,14 +247,6 @@ impl SearcherInner {
         generation: TrackedObject<SearcherGeneration>,
         doc_store_cache_size: usize,
     ) -> io::Result<SearcherInner> {
-        assert_eq!(
-            &segment_readers
-                .iter()
-                .map(|reader| (reader.segment_id(), reader.delete_opstamp()))
-                .collect::<BTreeMap<_, _>>(),
-            generation.segments(),
-            "Set of segments referenced by this Searcher and its SearcherGeneration must match"
-        );
         let store_readers: Vec<StoreReader> = segment_readers
             .iter()
             .map(|segment_reader| segment_reader.get_store_reader(doc_store_cache_size))

src/core/segment_id.rs

@@ -57,7 +57,7 @@ impl SegmentId {
     /// Picking the first 8 chars is ok to identify
     /// segments in a display message (e.g. a5c4dfcb).
     pub fn short_uuid_string(&self) -> String {
-        self.0.as_simple().to_string()[..8].to_string()
+        (&self.0.as_simple().to_string()[..8]).to_string()
     }
 
     /// Returns a segment uuid string.

src/core/single_segment_index_writer.rs

@@ -1,51 +0,0 @@
-use crate::indexer::operation::AddOperation;
-use crate::indexer::segment_updater::save_metas;
-use crate::indexer::SegmentWriter;
-use crate::{Directory, Document, Index, IndexMeta, Opstamp, Segment};
-
-#[doc(hidden)]
-pub struct SingleSegmentIndexWriter {
-    segment_writer: SegmentWriter,
-    segment: Segment,
-    opstamp: Opstamp,
-}
-
-impl SingleSegmentIndexWriter {
-    pub fn new(index: Index, mem_budget: usize) -> crate::Result<Self> {
-        let segment = index.new_segment();
-        let segment_writer = SegmentWriter::for_segment(mem_budget, segment.clone())?;
-        Ok(Self {
-            segment_writer,
-            segment,
-            opstamp: 0,
-        })
-    }
-
-    pub fn mem_usage(&self) -> usize {
-        self.segment_writer.mem_usage()
-    }
-
-    pub fn add_document(&mut self, document: Document) -> crate::Result<()> {
-        let opstamp = self.opstamp;
-        self.opstamp += 1;
-        self.segment_writer
-            .add_document(AddOperation { opstamp, document })
-    }
-
-    pub fn finalize(self) -> crate::Result<Index> {
-        let max_doc = self.segment_writer.max_doc();
-        self.segment_writer.finalize()?;
-        let segment: Segment = self.segment.with_max_doc(max_doc);
-        let index = segment.index();
-        let index_meta = IndexMeta {
-            index_settings: index.settings().clone(),
-            segments: vec![segment.meta().clone()],
-            schema: index.schema(),
-            opstamp: 0,
-            payload: None,
-        };
-        save_metas(&index_meta, index.directory())?;
-        index.directory().sync_directory()?;
-        Ok(segment.index().clone())
-    }
-}

src/directory/composite_file.rs

@@ -38,7 +38,7 @@ impl BinarySerializable for FileAddr {
 /// A `CompositeWrite` is used to write a `CompositeFile`.
 pub struct CompositeWrite<W = WritePtr> {
     write: CountingWriter<W>,
-    offsets: Vec<(FileAddr, u64)>,
+    offsets: HashMap<FileAddr, u64>,
 }
 
 impl<W: TerminatingWrite + Write> CompositeWrite<W> {
@@ -47,7 +47,7 @@ impl<W: TerminatingWrite + Write> CompositeWrite<W> {
     pub fn wrap(w: W) -> CompositeWrite<W> {
         CompositeWrite {
             write: CountingWriter::wrap(w),
-            offsets: Vec::new(),
+            offsets: HashMap::new(),
         }
     }
 
@@ -60,8 +60,8 @@ impl<W: TerminatingWrite + Write> CompositeWrite<W> {
     pub fn for_field_with_idx(&mut self, field: Field, idx: usize) -> &mut CountingWriter<W> {
         let offset = self.write.written_bytes();
         let file_addr = FileAddr::new(field, idx);
-        assert!(!self.offsets.iter().any(|el| el.0 == file_addr));
-        self.offsets.push((file_addr, offset));
+        assert!(!self.offsets.contains_key(&file_addr));
+        self.offsets.insert(file_addr, offset);
         &mut self.write
     }
 
@@ -73,8 +73,16 @@ impl<W: TerminatingWrite + Write> CompositeWrite<W> {
         let footer_offset = self.write.written_bytes();
         VInt(self.offsets.len() as u64).serialize(&mut self.write)?;
 
+        let mut offset_fields: Vec<_> = self
+            .offsets
+            .iter()
+            .map(|(file_addr, offset)| (*offset, *file_addr))
+            .collect();
+
+        offset_fields.sort();
+
         let mut prev_offset = 0;
-        for (file_addr, offset) in self.offsets {
+        for (offset, file_addr) in offset_fields {
             VInt((offset - prev_offset) as u64).serialize(&mut self.write)?;
             file_addr.serialize(&mut self.write)?;
             prev_offset = offset;
@@ -98,14 +106,6 @@ pub struct CompositeFile {
     offsets_index: HashMap<FileAddr, Range<usize>>,
 }
 
-impl std::fmt::Debug for CompositeFile {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.debug_struct("CompositeFile")
-            .field("offsets_index", &self.offsets_index)
-            .finish()
-    }
-}
-
 impl CompositeFile {
     /// Opens a composite file stored in a given
     /// `FileSlice`.
@@ -233,56 +233,4 @@ mod test {
         }
         Ok(())
     }
-
-    #[test]
-    fn test_composite_file_bug() -> crate::Result<()> {
-        let path = Path::new("test_path");
-        let directory = RamDirectory::create();
-        {
-            let w = directory.open_write(path).unwrap();
-            let mut composite_write = CompositeWrite::wrap(w);
-            let mut write = composite_write.for_field_with_idx(Field::from_field_id(1u32), 0);
-            VInt(32431123u64).serialize(&mut write)?;
-            write.flush()?;
-            let write = composite_write.for_field_with_idx(Field::from_field_id(1u32), 1);
-            write.flush()?;
-
-            let mut write = composite_write.for_field_with_idx(Field::from_field_id(0u32), 0);
-            VInt(1_000_000).serialize(&mut write)?;
-            write.flush()?;
-
-            composite_write.close()?;
-        }
-        {
-            let r = directory.open_read(path)?;
-            let composite_file = CompositeFile::open(&r)?;
-            {
-                let file = composite_file
-                    .open_read_with_idx(Field::from_field_id(1u32), 0)
-                    .unwrap()
-                    .read_bytes()?;
-                let mut file0_buf = file.as_slice();
-                let payload_0 = VInt::deserialize(&mut file0_buf)?.0;
-                assert_eq!(file0_buf.len(), 0);
-                assert_eq!(payload_0, 32431123u64);
-            }
-            {
-                let file = composite_file
-                    .open_read_with_idx(Field::from_field_id(1u32), 1)
-                    .unwrap()
-                    .read_bytes()?;
-                let file = file.as_slice();
-                assert_eq!(file.len(), 0);
-            }
-            {
-                let file = composite_file
-                    .open_read_with_idx(Field::from_field_id(0u32), 0)
-                    .unwrap()
-                    .read_bytes()?;
-                let file = file.as_slice();
-                assert_eq!(file.len(), 3);
-            }
-        }
-        Ok(())
-    }
 }

src/directory/directory.rs

@@ -117,9 +117,9 @@ pub trait Directory: DirectoryClone + fmt::Debug + Send + Sync + 'static {
     /// change.
     ///
     /// Specifically, subsequent writes or flushes should
-    /// have no effect on the returned [`FileSlice`] object.
+    /// have no effect on the returned `FileSlice` object.
     ///
-    /// You should only use this to read files create with [`Directory::open_write()`].
+    /// You should only use this to read files create with [Directory::open_write].
     fn open_read(&self, path: &Path) -> Result<FileSlice, OpenReadError> {
         let file_handle = self.get_file_handle(path)?;
         Ok(FileSlice::new(file_handle))
@@ -128,28 +128,27 @@ pub trait Directory: DirectoryClone + fmt::Debug + Send + Sync + 'static {
     /// Removes a file
     ///
     /// Removing a file will not affect an eventual
-    /// existing [`FileSlice`] pointing to it.
+    /// existing FileSlice pointing to it.
     ///
-    /// Removing a nonexistent file, returns a
-    /// [`DeleteError::FileDoesNotExist`].
+    /// Removing a nonexistent file, yields a
+    /// `DeleteError::DoesNotExist`.
     fn delete(&self, path: &Path) -> Result<(), DeleteError>;
 
     /// Returns true if and only if the file exists
     fn exists(&self, path: &Path) -> Result<bool, OpenReadError>;
 
     /// Opens a writer for the *virtual file* associated with
-    /// a [`Path`].
+    /// a Path.
     ///
     /// Right after this call, for the span of the execution of the program
-    /// the file should be created and any subsequent call to
-    /// [`Directory::open_read()`] for the same path should return
-    /// a [`FileSlice`].
+    /// the file should be created and any subsequent call to `open_read` for the
+    /// same path should return a `FileSlice`.
     ///
     /// However, depending on the directory implementation,
-    /// it might be required to call [`Directory::sync_directory()`] to ensure
+    /// it might be required to call `sync_directory` to ensure
     /// that the file is durably created.
     /// (The semantics here are the same when dealing with
-    /// a POSIX filesystem.)
+    /// a posix filesystem.)
     ///
     /// Write operations may be aggressively buffered.
     /// The client of this trait is responsible for calling flush
@@ -158,19 +157,19 @@ pub trait Directory: DirectoryClone + fmt::Debug + Send + Sync + 'static {
     ///
     /// Flush operation should also be persistent.
     ///
-    /// The user shall not rely on [`Drop`] triggering `flush`.
-    /// Note that [`RamDirectory`][crate::directory::RamDirectory] will
-    /// panic! if `flush` was not called.
+    /// The user shall not rely on `Drop` triggering `flush`.
+    /// Note that `RamDirectory` will panic! if `flush`
+    /// was not called.
     ///
     /// The file may not previously exist.
     fn open_write(&self, path: &Path) -> Result<WritePtr, OpenWriteError>;
 
     /// Reads the full content file that has been written using
-    /// [`Directory::atomic_write()`].
+    /// atomic_write.
     ///
     /// This should only be used for small files.
     ///
-    /// You should only use this to read files create with [`Directory::atomic_write()`].
+    /// You should only use this to read files create with [Directory::atomic_write].
     fn atomic_read(&self, path: &Path) -> Result<Vec<u8>, OpenReadError>;
 
     /// Atomically replace the content of a file with data.
@@ -187,9 +186,9 @@ pub trait Directory: DirectoryClone + fmt::Debug + Send + Sync + 'static {
     /// effectively stored durably.
     fn sync_directory(&self) -> io::Result<()>;
 
-    /// Acquire a lock in the directory given in the [`Lock`].
+    /// Acquire a lock in the given directory.
     ///
-    /// The method is blocking or not depending on the [`Lock`] object.
+    /// The method is blocking or not depending on the `Lock` object.
     fn acquire_lock(&self, lock: &Lock) -> Result<DirectoryLock, LockError> {
         let mut box_directory = self.box_clone();
         let mut retry_policy = retry_policy(lock.is_blocking);
@@ -211,15 +210,15 @@ pub trait Directory: DirectoryClone + fmt::Debug + Send + Sync + 'static {
     }
 
     /// Registers a callback that will be called whenever a change on the `meta.json`
-    /// using the [`Directory::atomic_write()`] API is detected.
+    /// using the `atomic_write` API is detected.
     ///
-    /// The behavior when using `.watch()` on a file using [`Directory::open_write()`] is, on the
-    /// other hand, undefined.
+    /// The behavior when using `.watch()` on a file using [Directory::open_write] is, on the other
+    /// hand, undefined.
     ///
     /// The file will be watched for the lifetime of the returned `WatchHandle`. The caller is
     /// required to keep it.
     /// It does not override previous callbacks. When the file is modified, all callback that are
-    /// registered (and whose [`WatchHandle`] is still alive) are triggered.
+    /// registered (and whose `WatchHandle` is still alive) are triggered.
     ///
     /// Internally, tantivy only uses this API to detect new commits to implement the
     /// `OnCommit` `ReloadPolicy`. Not implementing watch in a `Directory` only prevents the

src/directory/directory_lock.rs

@@ -4,14 +4,12 @@ use once_cell::sync::Lazy;
 
 /// A directory lock.
 ///
-/// A lock is associated with a specific path.
-///
-/// The lock will be passed to [`Directory::acquire_lock`](crate::Directory::acquire_lock).
-///
+/// A lock is associated to a specific path and some
+/// [`LockParams`](./enum.LockParams.html).
 /// Tantivy itself uses only two locks but client application
 /// can use the directory facility to define their own locks.
-/// - [`INDEX_WRITER_LOCK`]
-/// - [`META_LOCK`]
+/// - [INDEX_WRITER_LOCK]
+/// - [META_LOCK]
 ///
 /// Check out these locks documentation for more information.
 #[derive(Debug)]
@@ -20,21 +18,19 @@ pub struct Lock {
     /// Depending on the platform, the lock might rely on the creation
     /// and deletion of this filepath.
     pub filepath: PathBuf,
-    /// `is_blocking` describes whether acquiring the lock is meant
+    /// `lock_params` describes whether acquiring the lock is meant
     /// to be a blocking operation or a non-blocking.
     ///
     /// Acquiring a blocking lock blocks until the lock is
     /// available.
-    ///
-    /// Acquiring a non-blocking lock returns rapidly, either successfully
+    /// Acquiring a blocking lock returns rapidly, either successfully
     /// or with an error signifying that someone is already holding
     /// the lock.
     pub is_blocking: bool,
 }
 
 /// Only one process should be able to write tantivy's index at a time.
-/// This lock file, when present, is in charge of preventing other processes to open an
-/// `IndexWriter`.
+/// This lock file, when present, is in charge of preventing other processes to open an IndexWriter.
 ///
 /// If the process is killed and this file remains, it is safe to remove it manually.
 ///

src/directory/error.rs

@@ -4,9 +4,7 @@ use std::{fmt, io};
 
 use crate::Version;
 
-/// Error while trying to acquire a directory [lock](crate::directory::Lock).
-///
-/// This is returned from [`Directory::acquire_lock`](crate::Directory::acquire_lock).
+/// Error while trying to acquire a directory lock.
 #[derive(Debug, Clone, Error)]
 pub enum LockError {
     /// Failed to acquired a lock as it is already held by another

src/directory/mmap_directory.rs

@@ -27,7 +27,7 @@ pub(crate) fn make_io_err(msg: String) -> io::Error {
     io::Error::new(io::ErrorKind::Other, msg)
 }
 
-/// Returns `None` iff the file exists, can be read, but is empty (and hence
+/// Returns None iff the file exists, can be read, but is empty (and hence
 /// cannot be mmapped)
 fn open_mmap(full_path: &Path) -> result::Result<Option<Mmap>, OpenReadError> {
     let file = File::open(full_path).map_err(|io_err| {
@@ -56,10 +56,10 @@ fn open_mmap(full_path: &Path) -> result::Result<Option<Mmap>, OpenReadError> {
 
 #[derive(Default, Clone, Debug, Serialize, Deserialize)]
 pub struct CacheCounters {
-    /// Number of time the cache prevents to call `mmap`
+    // Number of time the cache prevents to call `mmap`
     pub hit: usize,
-    /// Number of time tantivy had to call `mmap`
-    /// as no entry was in the cache.
+    // Number of time tantivy had to call `mmap`
+    // as no entry was in the cache.
     pub miss: usize,
 }
 
@@ -472,8 +472,6 @@ mod tests {
     // There are more tests in directory/mod.rs
     // The following tests are specific to the MmapDirectory
 
-    use std::time::Duration;
-
     use common::HasLen;
 
     use super::*;
@@ -612,14 +610,7 @@ mod tests {
                 mmap_directory.get_cache_info().mmapped.len()
             );
         }
-        // This test failed on CI. The last Mmap is dropped from the merging thread so there might
-        // be a race condition indeed.
-        for _ in 0..10 {
-            if mmap_directory.get_cache_info().mmapped.is_empty() {
-                return Ok(());
-            }
-            std::thread::sleep(Duration::from_millis(200));
-        }
-        panic!("The cache still contains information. One of the Mmap has not been dropped.");
+        assert!(mmap_directory.get_cache_info().mmapped.is_empty());
+        Ok(())
     }
 }

src/directory/ram_directory.rs

@@ -15,7 +15,7 @@ use crate::directory::{
     WatchHandle, WritePtr,
 };
 
-/// Writer associated with the [`RamDirectory`].
+/// Writer associated with the `RamDirectory`
 ///
 /// The Writer just writes a buffer.
 struct VecWriter {
@@ -136,32 +136,18 @@ impl RamDirectory {
         Self::default()
     }
 
-    /// Deep clones the directory.
-    ///
-    /// Ulterior writes on one of the copy
-    /// will not affect the other copy.
-    pub fn deep_clone(&self) -> RamDirectory {
-        let inner_clone = InnerDirectory {
-            fs: self.fs.read().unwrap().fs.clone(),
-            watch_router: Default::default(),
-        };
-        RamDirectory {
-            fs: Arc::new(RwLock::new(inner_clone)),
-        }
-    }
-
     /// Returns the sum of the size of the different files
-    /// in the [`RamDirectory`].
+    /// in the RamDirectory.
     pub fn total_mem_usage(&self) -> usize {
         self.fs.read().unwrap().total_mem_usage()
     }
 
-    /// Write a copy of all of the files saved in the [`RamDirectory`] in the target [`Directory`].
+    /// Write a copy of all of the files saved in the RamDirectory in the target `Directory`.
     ///
-    /// Files are all written using the [`Directory::open_write()`] meaning, even if they were
-    /// written using the [`Directory::atomic_write()`] api.
+    /// Files are all written using the `Directory::write` meaning, even if they were
+    /// written using the `atomic_write` api.
     ///
-    /// If an error is encountered, files may be persisted partially.
+    /// If an error is encounterred, files may be persisted partially.
     pub fn persist(&self, dest: &dyn Directory) -> crate::Result<()> {
         let wlock = self.fs.write().unwrap();
         for (path, file) in wlock.fs.iter() {
@@ -270,23 +256,4 @@ mod tests {
         assert_eq!(directory_copy.atomic_read(path_atomic).unwrap(), msg_atomic);
         assert_eq!(directory_copy.atomic_read(path_seq).unwrap(), msg_seq);
     }
-
-    #[test]
-    fn test_ram_directory_deep_clone() {
-        let dir = RamDirectory::default();
-        let test = Path::new("test");
-        let test2 = Path::new("test2");
-        dir.atomic_write(test, b"firstwrite").unwrap();
-        let dir_clone = dir.deep_clone();
-        assert_eq!(
-            dir_clone.atomic_read(test).unwrap(),
-            dir.atomic_read(test).unwrap()
-        );
-        dir.atomic_write(test, b"original").unwrap();
-        dir_clone.atomic_write(test, b"clone").unwrap();
-        dir_clone.atomic_write(test2, b"clone2").unwrap();
-        assert_eq!(dir.atomic_read(test).unwrap(), b"original");
-        assert_eq!(&dir_clone.atomic_read(test).unwrap(), b"clone");
-        assert_eq!(&dir_clone.atomic_read(test2).unwrap(), b"clone2");
-    }
 }

src/docset.rs

@@ -3,10 +3,10 @@ use std::borrow::{Borrow, BorrowMut};
 use crate::fastfield::AliveBitSet;
 use crate::DocId;
 
-/// Sentinel value returned when a [`DocSet`] has been entirely consumed.
+/// Sentinel value returned when a DocSet has been entirely consumed.
 ///
-/// This is not `u32::MAX` as one would have expected, due to the lack of SSE2 instructions
-/// to compare `[u32; 4]`.
+/// This is not u32::MAX as one would have expected, due to the lack of SSE2 instructions
+/// to compare [u32; 4].
 pub const TERMINATED: DocId = i32::MAX as u32;
 
 /// Represents an iterable set of sorted doc ids.
@@ -20,21 +20,21 @@ pub trait DocSet: Send {
     /// assert_eq!(doc, docset.doc());
     /// ```
     ///
-    /// If we reached the end of the `DocSet`, [`TERMINATED`] should be returned.
+    /// If we reached the end of the DocSet, TERMINATED should be returned.
     ///
-    /// Calling `.advance()` on a terminated `DocSet` should be supported, and [`TERMINATED`] should
+    /// Calling `.advance()` on a terminated DocSet should be supported, and TERMINATED should
     /// be returned.
     fn advance(&mut self) -> DocId;
 
-    /// Advances the `DocSet` forward until reaching the target, or going to the
-    /// lowest [`DocId`] greater than the target.
+    /// Advances the DocSet forward until reaching the target, or going to the
+    /// lowest DocId greater than the target.
     ///
-    /// If the end of the `DocSet` is reached, [`TERMINATED`] is returned.
+    /// If the end of the DocSet is reached, TERMINATED is returned.
     ///
-    /// Calling `.seek(target)` on a terminated `DocSet` is legal. Implementation
-    /// of `DocSet` should support it.
+    /// Calling `.seek(target)` on a terminated DocSet is legal. Implementation
+    /// of DocSet should support it.
     ///
-    /// Calling `seek(TERMINATED)` is also legal and is the normal way to consume a `DocSet`.
+    /// Calling `seek(TERMINATED)` is also legal and is the normal way to consume a DocSet.
     fn seek(&mut self, target: DocId) -> DocId {
         let mut doc = self.doc();
         debug_assert!(doc <= target);
@@ -73,9 +73,9 @@ pub trait DocSet: Send {
     }
 
     /// Returns the current document
-    /// Right after creating a new `DocSet`, the docset points to the first document.
+    /// Right after creating a new DocSet, the docset points to the first document.
     ///
-    /// If the `DocSet` is empty, `.doc()` should return [`TERMINATED`].
+    /// If the DocSet is empty, .doc() should return `TERMINATED`.
     fn doc(&self) -> DocId;
 
     /// Returns a best-effort hint of the

src/fastfield/bytes/reader.rs

@@ -1,9 +1,5 @@
-use std::sync::Arc;
-
-use fastfield_codecs::Column;
-
 use crate::directory::{FileSlice, OwnedBytes};
-use crate::fastfield::MultiValueLength;
+use crate::fastfield::{FastFieldReader, FastFieldReaderImpl, MultiValueLength};
 use crate::DocId;
 
 /// Reader for byte array fast fields
@@ -18,13 +14,13 @@ use crate::DocId;
 /// and the start index for the next document, and keeping the bytes in between.
 #[derive(Clone)]
 pub struct BytesFastFieldReader {
-    idx_reader: Arc<dyn Column<u64>>,
+    idx_reader: FastFieldReaderImpl<u64>,
     values: OwnedBytes,
 }
 
 impl BytesFastFieldReader {
     pub(crate) fn open(
-        idx_reader: Arc<dyn Column<u64>>,
+        idx_reader: FastFieldReaderImpl<u64>,
         values_file: FileSlice,
     ) -> crate::Result<BytesFastFieldReader> {
         let values = values_file.read_bytes()?;
@@ -32,9 +28,8 @@ impl BytesFastFieldReader {
     }
 
     fn range(&self, doc: DocId) -> (usize, usize) {
-        let idx = doc as u64;
-        let start = self.idx_reader.get_val(idx) as usize;
-        let stop = self.idx_reader.get_val(idx + 1) as usize;
+        let start = self.idx_reader.get(doc) as usize;
+        let stop = self.idx_reader.get(doc + 1) as usize;
         (start, stop)
     }
 

src/fastfield/bytes/writer.rs

@@ -1,9 +1,6 @@
-use std::io::{self, Write};
-
-use fastfield_codecs::VecColumn;
+use std::io;
 
 use crate::fastfield::serializer::CompositeFastFieldSerializer;
-use crate::fastfield::MultivalueStartIndex;
 use crate::indexer::doc_id_mapping::DocIdMapping;
 use crate::schema::{Document, Field, Value};
 use crate::DocId;
@@ -13,17 +10,15 @@ use crate::DocId;
 /// This `BytesFastFieldWriter` is only useful for advanced users.
 /// The normal way to get your associated bytes in your index
 /// is to
-/// - declare your field with fast set to
-/// [`Cardinality::SingleValue`](crate::schema::Cardinality::SingleValue)
+/// - declare your field with fast set to `Cardinality::SingleValue`
 /// in your schema
 /// - add your document simply by calling `.add_document(...)` with associating bytes to the field.
 ///
 /// The `BytesFastFieldWriter` can be acquired from the
 /// fast field writer by calling
-/// [`.get_bytes_writer_mut(...)`](crate::fastfield::FastFieldsWriter::get_bytes_writer_mut).
+/// [`.get_bytes_writer(...)`](./struct.FastFieldsWriter.html#method.get_bytes_writer).
 ///
-/// Once acquired, writing is done by calling
-/// [`.add_document_val(&[u8])`](BytesFastFieldWriter::add_document_val)
+/// Once acquired, writing is done by calling `.add_document_val(&[u8])`
 /// once per document, even if there are no bytes associated to it.
 pub struct BytesFastFieldWriter {
     field: Field,
@@ -109,27 +104,22 @@ impl BytesFastFieldWriter {
 
     /// Serializes the fast field values by pushing them to the `FastFieldSerializer`.
     pub fn serialize(
-        mut self,
+        &self,
         serializer: &mut CompositeFastFieldSerializer,
         doc_id_map: Option<&DocIdMapping>,
     ) -> io::Result<()> {
         // writing the offset index
-        {
-            self.doc_index.push(self.vals.len() as u64);
-            let col = VecColumn::from(&self.doc_index[..]);
-            if let Some(doc_id_map) = doc_id_map {
-                let multi_value_start_index = MultivalueStartIndex::new(&col, doc_id_map);
-                serializer.create_auto_detect_u64_fast_field_with_idx(
-                    self.field,
-                    multi_value_start_index,
-                    0,
-                )?;
-            } else {
-                serializer.create_auto_detect_u64_fast_field_with_idx(self.field, col, 0)?;
-            }
+        let mut doc_index_serializer =
+            serializer.new_u64_fast_field_with_idx(self.field, 0, self.vals.len() as u64, 0)?;
+        let mut offset = 0;
+        for vals in self.get_ordered_values(doc_id_map) {
+            doc_index_serializer.add_val(offset)?;
+            offset += vals.len() as u64;
         }
+        doc_index_serializer.add_val(self.vals.len() as u64)?;
+        doc_index_serializer.close_field()?;
         // writing the values themselves
-        let mut value_serializer = serializer.new_bytes_fast_field(self.field);
+        let mut value_serializer = serializer.new_bytes_fast_field_with_idx(self.field, 1);
         // the else could be removed, but this is faster (difference not benchmarked)
         if let Some(doc_id_map) = doc_id_map {
             for vals in self.get_ordered_values(Some(doc_id_map)) {

src/fastfield/mod.rs

@@ -20,19 +20,20 @@
 //!
 //! Read access performance is comparable to that of an array lookup.
 
-use fastfield_codecs::MonotonicallyMappableToU64;
+use fastfield_codecs::dynamic::DynamicFastFieldCodec;
 
 pub use self::alive_bitset::{intersect_alive_bitsets, write_alive_bitset, AliveBitSet};
 pub use self::bytes::{BytesFastFieldReader, BytesFastFieldWriter};
 pub use self::error::{FastFieldNotAvailableError, Result};
 pub use self::facet_reader::FacetReader;
-pub(crate) use self::multivalued::MultivalueStartIndex;
 pub use self::multivalued::{MultiValuedFastFieldReader, MultiValuedFastFieldWriter};
+pub use self::reader::FastFieldReader;
 pub use self::readers::FastFieldReaders;
 pub(crate) use self::readers::{type_and_cardinality, FastType};
-pub use self::serializer::{Column, CompositeFastFieldSerializer};
+pub use self::serializer::{CompositeFastFieldSerializer, FastFieldStats};
+pub use self::wrapper::FastFieldReaderWrapper;
 pub use self::writer::{FastFieldsWriter, IntFastFieldWriter};
-use crate::schema::{Type, Value};
+use crate::schema::{Cardinality, FieldType, Type, Value};
 use crate::{DateTime, DocId};
 
 mod alive_bitset;
@@ -40,8 +41,10 @@ mod bytes;
 mod error;
 mod facet_reader;
 mod multivalued;
+mod reader;
 mod readers;
 mod serializer;
+mod wrapper;
 mod writer;
 
 /// Trait for `BytesFastFieldReader` and `MultiValuedFastFieldReader` to return the length of data
@@ -55,64 +58,172 @@ pub trait MultiValueLength {
 
 /// Trait for types that are allowed for fast fields:
 /// (u64, i64 and f64, bool, DateTime).
-pub trait FastValue:
-    MonotonicallyMappableToU64 + Copy + Send + Sync + PartialOrd + 'static
-{
-    /// Returns the `schema::Type` for this FastValue.
-    fn to_type() -> Type;
+pub trait FastValue: Clone + Copy + Send + Sync + PartialOrd + 'static {
+    /// Converts a value from u64
+    ///
+    /// Internally all fast field values are encoded as u64.
+    /// **Note: To be used for converting encoded Term, Posting values.**
+    fn from_u64(val: u64) -> Self;
+
+    /// Converts a value to u64.
+    ///
+    /// Internally all fast field values are encoded as u64.
+    fn to_u64(&self) -> u64;
+
+    /// Returns the fast field cardinality that can be extracted from the given
+    /// `FieldType`.
+    ///
+    /// If the type is not a fast field, `None` is returned.
+    fn fast_field_cardinality(field_type: &FieldType) -> Option<Cardinality>;
+
+    /// Cast value to `u64`.
+    /// The value is just reinterpreted in memory.
+    fn as_u64(&self) -> u64;
 
     /// Build a default value. This default value is never used, so the value does not
     /// really matter.
     fn make_zero() -> Self {
-        Self::from_u64(0u64)
+        Self::from_u64(0i64.to_u64())
     }
+
+    /// Returns the `schema::Type` for this FastValue.
+    fn to_type() -> Type;
 }
 
 impl FastValue for u64 {
+    fn from_u64(val: u64) -> Self {
+        val
+    }
+
+    fn to_u64(&self) -> u64 {
+        *self
+    }
+
+    fn fast_field_cardinality(field_type: &FieldType) -> Option<Cardinality> {
+        match *field_type {
+            FieldType::U64(ref integer_options) => integer_options.get_fastfield_cardinality(),
+            FieldType::Facet(_) => Some(Cardinality::MultiValues),
+            _ => None,
+        }
+    }
+
+    fn as_u64(&self) -> u64 {
+        *self
+    }
+
     fn to_type() -> Type {
         Type::U64
     }
 }
 
+// TODO rename
+pub type FastFieldReaderImpl<V> = FastFieldReaderWrapper<V, DynamicFastFieldCodec>;
+
 impl FastValue for i64 {
+    fn from_u64(val: u64) -> Self {
+        common::u64_to_i64(val)
+    }
+
+    fn to_u64(&self) -> u64 {
+        common::i64_to_u64(*self)
+    }
+
+    fn fast_field_cardinality(field_type: &FieldType) -> Option<Cardinality> {
+        match *field_type {
+            FieldType::I64(ref integer_options) => integer_options.get_fastfield_cardinality(),
+            _ => None,
+        }
+    }
+
+    fn as_u64(&self) -> u64 {
+        *self as u64
+    }
+
     fn to_type() -> Type {
         Type::I64
     }
 }
 
 impl FastValue for f64 {
+    fn from_u64(val: u64) -> Self {
+        common::u64_to_f64(val)
+    }
+
+    fn to_u64(&self) -> u64 {
+        common::f64_to_u64(*self)
+    }
+
+    fn fast_field_cardinality(field_type: &FieldType) -> Option<Cardinality> {
+        match *field_type {
+            FieldType::F64(ref integer_options) => integer_options.get_fastfield_cardinality(),
+            _ => None,
+        }
+    }
+
+    fn as_u64(&self) -> u64 {
+        self.to_bits()
+    }
+
     fn to_type() -> Type {
         Type::F64
     }
 }
 
 impl FastValue for bool {
+    fn from_u64(val: u64) -> Self {
+        val != 0u64
+    }
+
+    fn to_u64(&self) -> u64 {
+        match self {
+            false => 0,
+            true => 1,
+        }
+    }
+
+    fn fast_field_cardinality(field_type: &FieldType) -> Option<Cardinality> {
+        match *field_type {
+            FieldType::Bool(ref integer_options) => integer_options.get_fastfield_cardinality(),
+            _ => None,
+        }
+    }
+
+    fn as_u64(&self) -> u64 {
+        *self as u64
+    }
+
     fn to_type() -> Type {
         Type::Bool
     }
 }
 
-impl MonotonicallyMappableToU64 for DateTime {
-    fn to_u64(self) -> u64 {
-        self.timestamp_micros.to_u64()
-    }
-
-    fn from_u64(val: u64) -> Self {
-        let timestamp_micros = i64::from_u64(val);
-        DateTime { timestamp_micros }
-    }
-}
-
 impl FastValue for DateTime {
+    /// Converts a timestamp microseconds into DateTime.
+    ///
+    /// **Note the timestamps is expected to be in microseconds.**
+    fn from_u64(timestamp_micros_u64: u64) -> Self {
+        let timestamp_micros = i64::from_u64(timestamp_micros_u64);
+        Self::from_timestamp_micros(timestamp_micros)
+    }
+
+    fn to_u64(&self) -> u64 {
+        common::i64_to_u64(self.into_timestamp_micros())
+    }
+
+    fn fast_field_cardinality(field_type: &FieldType) -> Option<Cardinality> {
+        match *field_type {
+            FieldType::Date(ref options) => options.get_fastfield_cardinality(),
+            _ => None,
+        }
+    }
+
+    fn as_u64(&self) -> u64 {
+        self.into_timestamp_micros().as_u64()
+    }
+
     fn to_type() -> Type {
         Type::Date
     }
-
-    fn make_zero() -> Self {
-        DateTime {
-            timestamp_micros: 0,
-        }
-    }
 }
 
 fn value_to_u64(value: &Value) -> u64 {
@@ -152,19 +263,17 @@ mod tests {
     use std::collections::HashMap;
     use std::ops::Range;
     use std::path::Path;
-    use std::sync::Arc;
 
     use common::HasLen;
-    use fastfield_codecs::{open, FastFieldCodecType};
     use once_cell::sync::Lazy;
     use rand::prelude::SliceRandom;
     use rand::rngs::StdRng;
-    use rand::{Rng, SeedableRng};
+    use rand::SeedableRng;
 
     use super::*;
     use crate::directory::{CompositeFile, Directory, RamDirectory, WritePtr};
     use crate::merge_policy::NoMergePolicy;
-    use crate::schema::{Cardinality, Document, Field, Schema, SchemaBuilder, FAST, STRING, TEXT};
+    use crate::schema::{Document, Field, Schema, FAST, STRING, TEXT};
     use crate::time::OffsetDateTime;
     use crate::{DateOptions, DatePrecision, Index, SegmentId, SegmentReader};
 
@@ -173,14 +282,15 @@ mod tests {
         schema_builder.add_u64_field("field", FAST);
         schema_builder.build()
     });
+
     pub static FIELD: Lazy<Field> = Lazy::new(|| SCHEMA.get_field("field").unwrap());
 
     #[test]
     pub fn test_fastfield() {
-        let test_fastfield = fastfield_codecs::serialize_and_load(&[100u64, 200u64, 300u64][..]);
-        assert_eq!(test_fastfield.get_val(0u64), 100);
-        assert_eq!(test_fastfield.get_val(1u64), 200);
-        assert_eq!(test_fastfield.get_val(2u64), 300);
+        let test_fastfield = FastFieldReaderImpl::<u64>::from(&[100, 200, 300]);
+        assert_eq!(test_fastfield.get(0), 100);
+        assert_eq!(test_fastfield.get(1), 200);
+        assert_eq!(test_fastfield.get(2), 300);
     }
 
     #[test]
@@ -206,13 +316,13 @@ mod tests {
             serializer.close().unwrap();
         }
         let file = directory.open_read(path).unwrap();
-        assert_eq!(file.len(), 25);
+        assert_eq!(file.len(), 37);
         let composite_file = CompositeFile::open(&file)?;
-        let fast_field_bytes = composite_file.open_read(*FIELD).unwrap().read_bytes()?;
-        let fast_field_reader = open::<u64>(fast_field_bytes)?;
-        assert_eq!(fast_field_reader.get_val(0), 13u64);
-        assert_eq!(fast_field_reader.get_val(1), 14u64);
-        assert_eq!(fast_field_reader.get_val(2), 2u64);
+        let file = composite_file.open_read(*FIELD).unwrap();
+        let fast_field_reader = FastFieldReaderImpl::<u64>::open(file)?;
+        assert_eq!(fast_field_reader.get(0), 13u64);
+        assert_eq!(fast_field_reader.get(1), 14u64);
+        assert_eq!(fast_field_reader.get(2), 2u64);
         Ok(())
     }
 
@@ -237,23 +347,20 @@ mod tests {
             serializer.close()?;
         }
         let file = directory.open_read(path)?;
-        assert_eq!(file.len(), 53);
+        assert_eq!(file.len(), 62);
         {
             let fast_fields_composite = CompositeFile::open(&file)?;
-            let data = fast_fields_composite
-                .open_read(*FIELD)
-                .unwrap()
-                .read_bytes()?;
-            let fast_field_reader = open::<u64>(data)?;
-            assert_eq!(fast_field_reader.get_val(0), 4u64);
-            assert_eq!(fast_field_reader.get_val(1), 14_082_001u64);
-            assert_eq!(fast_field_reader.get_val(2), 3_052u64);
-            assert_eq!(fast_field_reader.get_val(3), 9002u64);
-            assert_eq!(fast_field_reader.get_val(4), 15_001u64);
-            assert_eq!(fast_field_reader.get_val(5), 777u64);
-            assert_eq!(fast_field_reader.get_val(6), 1_002u64);
-            assert_eq!(fast_field_reader.get_val(7), 1_501u64);
-            assert_eq!(fast_field_reader.get_val(8), 215u64);
+            let data = fast_fields_composite.open_read(*FIELD).unwrap();
+            let fast_field_reader = FastFieldReaderImpl::<u64>::open(data)?;
+            assert_eq!(fast_field_reader.get(0), 4u64);
+            assert_eq!(fast_field_reader.get(1), 14_082_001u64);
+            assert_eq!(fast_field_reader.get(2), 3_052u64);
+            assert_eq!(fast_field_reader.get(3), 9002u64);
+            assert_eq!(fast_field_reader.get(4), 15_001u64);
+            assert_eq!(fast_field_reader.get(5), 777u64);
+            assert_eq!(fast_field_reader.get(6), 1_002u64);
+            assert_eq!(fast_field_reader.get(7), 1_501u64);
+            assert_eq!(fast_field_reader.get(8), 215u64);
         }
         Ok(())
     }
@@ -276,16 +383,13 @@ mod tests {
             serializer.close().unwrap();
         }
         let file = directory.open_read(path).unwrap();
-        assert_eq!(file.len(), 26);
+        assert_eq!(file.len(), 35);
         {
             let fast_fields_composite = CompositeFile::open(&file).unwrap();
-            let data = fast_fields_composite
-                .open_read(*FIELD)
-                .unwrap()
-                .read_bytes()?;
-            let fast_field_reader = open::<u64>(data)?;
+            let data = fast_fields_composite.open_read(*FIELD).unwrap();
+            let fast_field_reader = FastFieldReaderImpl::<u64>::open(data)?;
             for doc in 0..10_000 {
-                assert_eq!(fast_field_reader.get_val(doc), 100_000u64);
+                assert_eq!(fast_field_reader.get(doc), 100_000u64);
             }
         }
         Ok(())
@@ -311,18 +415,15 @@ mod tests {
             serializer.close().unwrap();
         }
         let file = directory.open_read(path).unwrap();
-        assert_eq!(file.len(), 80040);
+        assert_eq!(file.len(), 80043);
         {
             let fast_fields_composite = CompositeFile::open(&file)?;
-            let data = fast_fields_composite
-                .open_read(*FIELD)
-                .unwrap()
-                .read_bytes()?;
-            let fast_field_reader = open::<u64>(data)?;
-            assert_eq!(fast_field_reader.get_val(0), 0u64);
+            let data = fast_fields_composite.open_read(*FIELD).unwrap();
+            let fast_field_reader = FastFieldReaderImpl::<u64>::open(data)?;
+            assert_eq!(fast_field_reader.get(0), 0u64);
             for doc in 1..10_001 {
                 assert_eq!(
-                    fast_field_reader.get_val(doc),
+                    fast_field_reader.get(doc),
                     5_000_000_000_000_000_000u64 + doc as u64 - 1u64
                 );
             }
@@ -353,20 +454,17 @@ mod tests {
             serializer.close().unwrap();
         }
         let file = directory.open_read(path).unwrap();
-        assert_eq!(file.len(), 40_usize);
-
+        // assert_eq!(file.len(), 17710 as usize); //bitpacked size
+        assert_eq!(file.len(), 10175_usize); // linear interpol size
         {
             let fast_fields_composite = CompositeFile::open(&file)?;
-            let data = fast_fields_composite
-                .open_read(i64_field)
-                .unwrap()
-                .read_bytes()?;
-            let fast_field_reader = open::<i64>(data)?;
+            let data = fast_fields_composite.open_read(i64_field).unwrap();
+            let fast_field_reader = FastFieldReaderImpl::<i64>::open(data)?;
 
             assert_eq!(fast_field_reader.min_value(), -100i64);
             assert_eq!(fast_field_reader.max_value(), 9_999i64);
             for (doc, i) in (-100i64..10_000i64).enumerate() {
-                assert_eq!(fast_field_reader.get_val(doc as u64), i);
+                assert_eq!(fast_field_reader.get(doc as u32), i);
             }
             let mut buffer = vec![0i64; 100];
             fast_field_reader.get_range(53, &mut buffer[..]);
@@ -400,12 +498,9 @@ mod tests {
         let file = directory.open_read(path).unwrap();
         {
             let fast_fields_composite = CompositeFile::open(&file).unwrap();
-            let data = fast_fields_composite
-                .open_read(i64_field)
-                .unwrap()
-                .read_bytes()?;
-            let fast_field_reader = open::<i64>(data)?;
-            assert_eq!(fast_field_reader.get_val(0), 0i64);
+            let data = fast_fields_composite.open_read(i64_field).unwrap();
+            let fast_field_reader = FastFieldReaderImpl::<i64>::open(data)?;
+            assert_eq!(fast_field_reader.get(0u32), 0i64);
         }
         Ok(())
     }
@@ -441,14 +536,11 @@ mod tests {
         let file = directory.open_read(path)?;
         {
             let fast_fields_composite = CompositeFile::open(&file)?;
-            let data = fast_fields_composite
-                .open_read(*FIELD)
-                .unwrap()
-                .read_bytes()?;
-            let fast_field_reader = open::<u64>(data)?;
+            let data = fast_fields_composite.open_read(*FIELD).unwrap();
+            let fast_field_reader = FastFieldReaderImpl::<u64>::open(data)?;
 
             for a in 0..n {
-                assert_eq!(fast_field_reader.get_val(a as u64), permutation[a as usize]);
+                assert_eq!(fast_field_reader.get(a as u32), permutation[a as usize]);
             }
         }
         Ok(())
@@ -504,7 +596,7 @@ mod tests {
         let mut all = vec![];
 
         for doc in docs {
-            let mut out: Vec<u64> = vec![];
+            let mut out = vec![];
             ff.get_vals(doc, &mut out);
             all.extend(out);
         }
@@ -701,6 +793,7 @@ mod tests {
 
     #[test]
     fn test_datefastfield() -> crate::Result<()> {
+        use crate::fastfield::FastValue;
         let mut schema_builder = Schema::builder();
         let date_field = schema_builder.add_date_field(
             "date",
@@ -738,19 +831,19 @@ mod tests {
         let dates_fast_field = fast_fields.dates(multi_date_field).unwrap();
         let mut dates = vec![];
         {
-            assert_eq!(date_fast_field.get_val(0).into_timestamp_micros(), 1i64);
+            assert_eq!(date_fast_field.get(0u32).into_timestamp_micros(), 1i64);
             dates_fast_field.get_vals(0u32, &mut dates);
             assert_eq!(dates.len(), 2);
             assert_eq!(dates[0].into_timestamp_micros(), 2i64);
             assert_eq!(dates[1].into_timestamp_micros(), 3i64);
         }
         {
-            assert_eq!(date_fast_field.get_val(1).into_timestamp_micros(), 4i64);
+            assert_eq!(date_fast_field.get(1u32).into_timestamp_micros(), 4i64);
             dates_fast_field.get_vals(1u32, &mut dates);
             assert!(dates.is_empty());
         }
         {
-            assert_eq!(date_fast_field.get_val(2).into_timestamp_micros(), 0i64);
+            assert_eq!(date_fast_field.get(2u32).into_timestamp_micros(), 0i64);
             dates_fast_field.get_vals(2u32, &mut dates);
             assert_eq!(dates.len(), 2);
             assert_eq!(dates[0].into_timestamp_micros(), 5i64);
@@ -761,12 +854,11 @@ mod tests {
 
     #[test]
     pub fn test_fastfield_bool() {
-        let test_fastfield: Arc<dyn Column<bool>> =
-            fastfield_codecs::serialize_and_load::<bool>(&[true, false, true, false]);
-        assert_eq!(test_fastfield.get_val(0), true);
-        assert_eq!(test_fastfield.get_val(1), false);
-        assert_eq!(test_fastfield.get_val(2), true);
-        assert_eq!(test_fastfield.get_val(3), false);
+        let test_fastfield = FastFieldReaderImpl::<bool>::from(&[true, false, true, false]);
+        assert_eq!(test_fastfield.get(0), true);
+        assert_eq!(test_fastfield.get(1), false);
+        assert_eq!(test_fastfield.get(2), true);
+        assert_eq!(test_fastfield.get(3), false);
     }
 
     #[test]
@@ -793,14 +885,14 @@ mod tests {
             serializer.close().unwrap();
         }
         let file = directory.open_read(path).unwrap();
-        assert_eq!(file.len(), 24);
+        assert_eq!(file.len(), 36);
         let composite_file = CompositeFile::open(&file)?;
-        let data = composite_file.open_read(field).unwrap().read_bytes()?;
-        let fast_field_reader = open::<bool>(data)?;
-        assert_eq!(fast_field_reader.get_val(0), true);
-        assert_eq!(fast_field_reader.get_val(1), false);
-        assert_eq!(fast_field_reader.get_val(2), true);
-        assert_eq!(fast_field_reader.get_val(3), false);
+        let file = composite_file.open_read(field).unwrap();
+        let fast_field_reader = FastFieldReaderImpl::<bool>::open(file)?;
+        assert_eq!(fast_field_reader.get(0), true);
+        assert_eq!(fast_field_reader.get(1), false);
+        assert_eq!(fast_field_reader.get(2), true);
+        assert_eq!(fast_field_reader.get(3), false);
 
         Ok(())
     }
@@ -829,13 +921,13 @@ mod tests {
             serializer.close().unwrap();
         }
         let file = directory.open_read(path).unwrap();
-        assert_eq!(file.len(), 36);
+        assert_eq!(file.len(), 48);
         let composite_file = CompositeFile::open(&file)?;
-        let data = composite_file.open_read(field).unwrap().read_bytes()?;
-        let fast_field_reader = open::<bool>(data)?;
+        let file = composite_file.open_read(field).unwrap();
+        let fast_field_reader = FastFieldReaderImpl::<bool>::open(file)?;
         for i in 0..25 {
-            assert_eq!(fast_field_reader.get_val(i * 2), true);
-            assert_eq!(fast_field_reader.get_val(i * 2 + 1), false);
+            assert_eq!(fast_field_reader.get(i * 2), true);
+            assert_eq!(fast_field_reader.get(i * 2 + 1), false);
         }
 
         Ok(())
@@ -847,95 +939,168 @@ mod tests {
         let directory: RamDirectory = RamDirectory::create();
 
         let mut schema_builder = Schema::builder();
-        let field = schema_builder.add_bool_field("field_bool", FAST);
+        schema_builder.add_bool_field("field_bool", FAST);
         let schema = schema_builder.build();
+        let field = schema.get_field("field_bool").unwrap();
 
         {
             let write: WritePtr = directory.open_write(path).unwrap();
-            let mut serializer = CompositeFastFieldSerializer::from_write(write)?;
+            let mut serializer = CompositeFastFieldSerializer::from_write(write).unwrap();
             let mut fast_field_writers = FastFieldsWriter::from_schema(&schema);
             let doc = Document::default();
             fast_field_writers.add_document(&doc);
-            fast_field_writers.serialize(&mut serializer, &HashMap::new(), None)?;
-            serializer.close()?;
+            fast_field_writers
+                .serialize(&mut serializer, &HashMap::new(), None)
+                .unwrap();
+            serializer.close().unwrap();
         }
         let file = directory.open_read(path).unwrap();
+        assert_eq!(file.len(), 35);
         let composite_file = CompositeFile::open(&file)?;
-        assert_eq!(file.len(), 23);
-        let data = composite_file.open_read(field).unwrap().read_bytes()?;
-        let fast_field_reader = open::<bool>(data)?;
-        assert_eq!(fast_field_reader.get_val(0), false);
+        let file = composite_file.open_read(field).unwrap();
+        let fast_field_reader = FastFieldReaderImpl::<bool>::open(file)?;
+        assert_eq!(fast_field_reader.get(0), false);
 
         Ok(())
     }
+}
 
-    fn get_index(
-        docs: &[crate::Document],
-        schema: &Schema,
-        codec_types: &[FastFieldCodecType],
-    ) -> crate::Result<RamDirectory> {
+#[cfg(all(test, feature = "unstable"))]
+mod bench {
+    use std::collections::HashMap;
+    use std::path::Path;
+
+    use test::{self, Bencher};
+
+    use super::tests::{generate_permutation, FIELD, SCHEMA};
+    use super::*;
+    use crate::directory::{CompositeFile, Directory, RamDirectory, WritePtr};
+    use crate::fastfield::tests::generate_permutation_gcd;
+    use crate::fastfield::FastFieldReader;
+
+    #[bench]
+    fn bench_intfastfield_linear_veclookup(b: &mut Bencher) {
+        let permutation = generate_permutation();
+        b.iter(|| {
+            let n = test::black_box(7000u32);
+            let mut a = 0u64;
+            for i in (0u32..n / 7).map(|v| v * 7) {
+                a ^= permutation[i as usize];
+            }
+            a
+        });
+    }
+
+    #[bench]
+    fn bench_intfastfield_veclookup(b: &mut Bencher) {
+        let permutation = generate_permutation();
+        b.iter(|| {
+            let n = test::black_box(1000u32);
+            let mut a = 0u64;
+            for _ in 0u32..n {
+                a = permutation[a as usize];
+            }
+            a
+        });
+    }
+
+    #[bench]
+    fn bench_intfastfield_linear_fflookup(b: &mut Bencher) {
+        let path = Path::new("test");
+        let permutation = generate_permutation();
         let directory: RamDirectory = RamDirectory::create();
         {
             let write: WritePtr = directory.open_write(Path::new("test")).unwrap();
-            let mut serializer =
-                CompositeFastFieldSerializer::from_write_with_codec(write, codec_types).unwrap();
-            let mut fast_field_writers = FastFieldsWriter::from_schema(schema);
-            for doc in docs {
-                fast_field_writers.add_document(doc);
+            let mut serializer = CompositeFastFieldSerializer::from_write(write).unwrap();
+            let mut fast_field_writers = FastFieldsWriter::from_schema(&SCHEMA);
+            for &x in &permutation {
+                fast_field_writers.add_document(&doc!(*FIELD=>x));
             }
             fast_field_writers
                 .serialize(&mut serializer, &HashMap::new(), None)
                 .unwrap();
             serializer.close().unwrap();
         }
-        Ok(directory)
-    }
+        let file = directory.open_read(&path).unwrap();
+        {
+            let fast_fields_composite = CompositeFile::open(&file).unwrap();
+            let data = fast_fields_composite.open_read(*FIELD).unwrap();
+            let fast_field_reader = DynamicFastFieldReader::<u64>::open(data).unwrap();
 
-    #[test]
-    pub fn test_gcd_date() -> crate::Result<()> {
-        let size_prec_sec =
-            test_gcd_date_with_codec(FastFieldCodecType::Bitpacked, DatePrecision::Seconds)?;
-        assert_eq!(size_prec_sec, 28 + (1_000 * 13) / 8); // 13 bits per val = ceil(log_2(number of seconds in 2hours);
-        let size_prec_micro =
-            test_gcd_date_with_codec(FastFieldCodecType::Bitpacked, DatePrecision::Microseconds)?;
-        assert_eq!(size_prec_micro, 26 + (1_000 * 33) / 8); // 33 bits per val = ceil(log_2(number of microsecsseconds in 2hours);
-        Ok(())
-    }
-
-    fn test_gcd_date_with_codec(
-        codec_type: FastFieldCodecType,
-        precision: DatePrecision,
-    ) -> crate::Result<usize> {
-        let mut rng = StdRng::seed_from_u64(2u64);
-        const T0: i64 = 1_662_345_825_012_529i64;
-        const ONE_HOUR_IN_MICROSECS: i64 = 3_600 * 1_000_000;
-        let times: Vec<DateTime> = std::iter::repeat_with(|| {
-            // +- One hour.
-            let t = T0 + rng.gen_range(-ONE_HOUR_IN_MICROSECS..ONE_HOUR_IN_MICROSECS);
-            DateTime::from_timestamp_micros(t)
-        })
-        .take(1_000)
-        .collect();
-        let date_options = DateOptions::default()
-            .set_fast(Cardinality::SingleValue)
-            .set_precision(precision);
-        let mut schema_builder = SchemaBuilder::default();
-        let field = schema_builder.add_date_field("field", date_options);
-        let schema = schema_builder.build();
-
-        let docs: Vec<Document> = times.iter().map(|time| doc!(field=>*time)).collect();
-
-        let directory = get_index(&docs[..], &schema, &[codec_type])?;
-        let path = Path::new("test");
-        let file = directory.open_read(path).unwrap();
-        let composite_file = CompositeFile::open(&file)?;
-        let file = composite_file.open_read(*FIELD).unwrap();
-        let len = file.len();
-        let test_fastfield = open::<DateTime>(file.read_bytes()?)?;
-
-        for (i, time) in times.iter().enumerate() {
-            assert_eq!(test_fastfield.get_val(i as u64), time.truncate(precision));
+            b.iter(|| {
+                let n = test::black_box(7000u32);
+                let mut a = 0u64;
+                for i in (0u32..n / 7).map(|val| val * 7) {
+                    a ^= fast_field_reader.get(i);
+                }
+                a
+            });
+        }
+    }
+
+    #[bench]
+    fn bench_intfastfield_fflookup(b: &mut Bencher) {
+        let path = Path::new("test");
+        let permutation = generate_permutation();
+        let directory: RamDirectory = RamDirectory::create();
+        {
+            let write: WritePtr = directory.open_write(Path::new("test")).unwrap();
+            let mut serializer = CompositeFastFieldSerializer::from_write(write).unwrap();
+            let mut fast_field_writers = FastFieldsWriter::from_schema(&SCHEMA);
+            for &x in &permutation {
+                fast_field_writers.add_document(&doc!(*FIELD=>x));
+            }
+            fast_field_writers
+                .serialize(&mut serializer, &HashMap::new(), None)
+                .unwrap();
+            serializer.close().unwrap();
+        }
+        let file = directory.open_read(&path).unwrap();
+        {
+            let fast_fields_composite = CompositeFile::open(&file).unwrap();
+            let data = fast_fields_composite.open_read(*FIELD).unwrap();
+            let fast_field_reader = DynamicFastFieldReader::<u64>::open(data).unwrap();
+
+            b.iter(|| {
+                let mut a = 0u32;
+                for i in 0u32..permutation.len() as u32 {
+                    a = fast_field_reader.get(i) as u32;
+                }
+                a
+            });
+        }
+    }
+
+    #[bench]
+    fn bench_intfastfield_fflookup_gcd(b: &mut Bencher) {
+        let path = Path::new("test");
+        let permutation = generate_permutation_gcd();
+        let directory: RamDirectory = RamDirectory::create();
+        {
+            let write: WritePtr = directory.open_write(Path::new("test")).unwrap();
+            let mut serializer = CompositeFastFieldSerializer::from_write(write).unwrap();
+            let mut fast_field_writers = FastFieldsWriter::from_schema(&SCHEMA);
+            for &x in &permutation {
+                fast_field_writers.add_document(&doc!(*FIELD=>x));
+            }
+            fast_field_writers
+                .serialize(&mut serializer, &HashMap::new(), None)
+                .unwrap();
+            serializer.close().unwrap();
+        }
+        let file = directory.open_read(&path).unwrap();
+        {
+            let fast_fields_composite = CompositeFile::open(&file).unwrap();
+            let data = fast_fields_composite.open_read(*FIELD).unwrap();
+            let fast_field_reader = DynamicFastFieldReader::<u64>::open(data).unwrap();
+
+            b.iter(|| {
+                let mut a = 0u32;
+                for i in 0u32..permutation.len() as u32 {
+                    a = fast_field_reader.get(i) as u32;
+                }
+                a
+            });
         }
-        Ok(len)
     }
 }

src/fastfield/multivalued/mod.rs

@@ -3,7 +3,6 @@ mod writer;
 
 pub use self::reader::MultiValuedFastFieldReader;
 pub use self::writer::MultiValuedFastFieldWriter;
-pub(crate) use self::writer::MultivalueStartIndex;
 
 #[cfg(test)]
 mod tests {
@@ -342,33 +341,37 @@ mod tests {
     }
 
     proptest! {
-        #![proptest_config(proptest::prelude::ProptestConfig::with_cases(5))]
         #[test]
         fn test_multivalued_proptest(ops in proptest::collection::vec(operation_strategy(), 1..10)) {
             assert!(test_multivalued_no_panic(&ops[..]).is_ok());
         }
     }
 
+    #[test]
+    fn test_proptest_merge_multivalued_bug() {
+        use IndexingOp::*;
+        let ops = &[AddDoc { id: 7 }, AddDoc { id: 4 }, Merge];
+        assert!(test_multivalued_no_panic(ops).is_ok());
+    }
+
     #[test]
     fn test_multivalued_proptest_gcd() {
         use IndexingOp::*;
-        let ops = [AddDoc { id: 9 }, AddDoc { id: 9 }, Merge];
-
-        assert!(test_multivalued_no_panic(&ops[..]).is_ok());
+        let ops = &[AddDoc { id: 9 }, AddDoc { id: 9 }, Merge];
+        assert!(test_multivalued_no_panic(ops).is_ok());
     }
 
     #[test]
     fn test_multivalued_proptest_off_by_one_bug_1151() {
         use IndexingOp::*;
-        let ops = [
+        let ops = &[
             AddDoc { id: 3 },
             AddDoc { id: 1 },
             AddDoc { id: 3 },
             Commit,
             Merge,
         ];
-
-        assert!(test_multivalued_no_panic(&ops[..]).is_ok());
+        assert!(test_multivalued_no_panic(ops).is_ok());
     }
 
     #[test]
@@ -387,219 +390,3 @@ mod tests {
         Ok(())
     }
 }
-
-#[cfg(all(test, feature = "unstable"))]
-mod bench {
-    use std::collections::HashMap;
-    use std::path::Path;
-
-    use test::{self, Bencher};
-
-    use super::*;
-    use crate::directory::{CompositeFile, Directory, RamDirectory, WritePtr};
-    use crate::fastfield::{CompositeFastFieldSerializer, FastFieldsWriter};
-    use crate::indexer::doc_id_mapping::DocIdMapping;
-    use crate::schema::{Cardinality, NumericOptions, Schema};
-    use crate::Document;
-
-    fn bench_multi_value_ff_merge_opt(
-        num_docs: usize,
-        segments_every_n_docs: usize,
-        merge_policy: impl crate::indexer::MergePolicy + 'static,
-    ) {
-        let mut builder = crate::schema::SchemaBuilder::new();
-
-        let fast_multi =
-            crate::schema::NumericOptions::default().set_fast(Cardinality::MultiValues);
-        let multi_field = builder.add_f64_field("f64s", fast_multi);
-
-        let index = crate::Index::create_in_ram(builder.build());
-
-        let mut writer = index.writer_for_tests().unwrap();
-        writer.set_merge_policy(Box::new(merge_policy));
-
-        for i in 0..num_docs {
-            let mut doc = crate::Document::new();
-            doc.add_f64(multi_field, 0.24);
-            doc.add_f64(multi_field, 0.27);
-            doc.add_f64(multi_field, 0.37);
-            if i % 3 == 0 {
-                doc.add_f64(multi_field, 0.44);
-            }
-
-            writer.add_document(doc).unwrap();
-            if i % segments_every_n_docs == 0 {
-                writer.commit().unwrap();
-            }
-        }
-
-        {
-            writer.wait_merging_threads().unwrap();
-            let mut writer = index.writer_for_tests().unwrap();
-            let segment_ids = index.searchable_segment_ids().unwrap();
-            writer.merge(&segment_ids).wait().unwrap();
-        }
-
-        // If a merging thread fails, we should end up with more
-        // than one segment here
-        assert_eq!(1, index.searchable_segments().unwrap().len());
-    }
-
-    #[bench]
-    fn bench_multi_value_ff_merge_many_segments(b: &mut Bencher) {
-        let num_docs = 100_000;
-        b.iter(|| {
-            bench_multi_value_ff_merge_opt(num_docs, 1_000, crate::indexer::NoMergePolicy);
-        });
-    }
-
-    #[bench]
-    fn bench_multi_value_ff_merge_many_segments_log_merge(b: &mut Bencher) {
-        let num_docs = 100_000;
-        b.iter(|| {
-            let merge_policy = crate::indexer::LogMergePolicy::default();
-            bench_multi_value_ff_merge_opt(num_docs, 1_000, merge_policy);
-        });
-    }
-
-    #[bench]
-    fn bench_multi_value_ff_merge_few_segments(b: &mut Bencher) {
-        let num_docs = 100_000;
-        b.iter(|| {
-            bench_multi_value_ff_merge_opt(num_docs, 33_000, crate::indexer::NoMergePolicy);
-        });
-    }
-
-    fn multi_values(num_docs: usize, vals_per_doc: usize) -> Vec<Vec<u64>> {
-        let mut vals = vec![];
-        for _i in 0..num_docs {
-            let mut block = vec![];
-            for j in 0..vals_per_doc {
-                block.push(j as u64);
-            }
-            vals.push(block);
-        }
-
-        vals
-    }
-
-    #[bench]
-    fn bench_multi_value_fflookup(b: &mut Bencher) {
-        let num_docs = 100_000;
-
-        let path = Path::new("test");
-        let directory: RamDirectory = RamDirectory::create();
-        let field = {
-            let options = NumericOptions::default().set_fast(Cardinality::MultiValues);
-            let mut schema_builder = Schema::builder();
-            let field = schema_builder.add_u64_field("field", options);
-            let schema = schema_builder.build();
-
-            let write: WritePtr = directory.open_write(Path::new("test")).unwrap();
-            let mut serializer = CompositeFastFieldSerializer::from_write(write).unwrap();
-            let mut fast_field_writers = FastFieldsWriter::from_schema(&schema);
-            for block in &multi_values(num_docs, 3) {
-                let mut doc = Document::new();
-                for val in block {
-                    doc.add_u64(field, *val);
-                }
-                fast_field_writers.add_document(&doc);
-            }
-            fast_field_writers
-                .serialize(&mut serializer, &HashMap::new(), None)
-                .unwrap();
-            serializer.close().unwrap();
-            field
-        };
-        let file = directory.open_read(&path).unwrap();
-        {
-            let fast_fields_composite = CompositeFile::open(&file).unwrap();
-            let data_idx = fast_fields_composite
-                .open_read_with_idx(field, 0)
-                .unwrap()
-                .read_bytes()
-                .unwrap();
-            let idx_reader = fastfield_codecs::open(data_idx).unwrap();
-
-            let data_vals = fast_fields_composite
-                .open_read_with_idx(field, 1)
-                .unwrap()
-                .read_bytes()
-                .unwrap();
-            let vals_reader = fastfield_codecs::open(data_vals).unwrap();
-            let fast_field_reader = MultiValuedFastFieldReader::open(idx_reader, vals_reader);
-            b.iter(|| {
-                let mut sum = 0u64;
-                let mut data = Vec::with_capacity(10);
-                for i in 0u32..num_docs as u32 {
-                    fast_field_reader.get_vals(i, &mut data);
-                    sum += data.iter().sum::<u64>();
-                }
-                sum
-            });
-        }
-    }
-
-    #[bench]
-    fn bench_multi_value_ff_creation(b: &mut Bencher) {
-        // 3 million ff entries
-        let num_docs = 1_000_000;
-        let multi_values = multi_values(num_docs, 3);
-
-        b.iter(|| {
-            let directory: RamDirectory = RamDirectory::create();
-            let options = NumericOptions::default().set_fast(Cardinality::MultiValues);
-            let mut schema_builder = Schema::builder();
-            let field = schema_builder.add_u64_field("field", options);
-            let schema = schema_builder.build();
-
-            let write: WritePtr = directory.open_write(Path::new("test")).unwrap();
-            let mut serializer = CompositeFastFieldSerializer::from_write(write).unwrap();
-            let mut fast_field_writers = FastFieldsWriter::from_schema(&schema);
-            for block in &multi_values {
-                let mut doc = Document::new();
-                for val in block {
-                    doc.add_u64(field, *val);
-                }
-                fast_field_writers.add_document(&doc);
-            }
-            fast_field_writers
-                .serialize(&mut serializer, &HashMap::new(), None)
-                .unwrap();
-            serializer.close().unwrap();
-        });
-    }
-
-    #[bench]
-    fn bench_multi_value_ff_creation_with_sorting(b: &mut Bencher) {
-        // 3 million ff entries
-        let num_docs = 1_000_000;
-        let multi_values = multi_values(num_docs, 3);
-
-        let doc_id_mapping =
-            DocIdMapping::from_new_id_to_old_id((0..1_000_000).collect::<Vec<_>>());
-
-        b.iter(|| {
-            let directory: RamDirectory = RamDirectory::create();
-            let options = NumericOptions::default().set_fast(Cardinality::MultiValues);
-            let mut schema_builder = Schema::builder();
-            let field = schema_builder.add_u64_field("field", options);
-            let schema = schema_builder.build();
-
-            let write: WritePtr = directory.open_write(Path::new("test")).unwrap();
-            let mut serializer = CompositeFastFieldSerializer::from_write(write).unwrap();
-            let mut fast_field_writers = FastFieldsWriter::from_schema(&schema);
-            for block in &multi_values {
-                let mut doc = Document::new();
-                for val in block {
-                    doc.add_u64(field, *val);
-                }
-                fast_field_writers.add_document(&doc);
-            }
-            fast_field_writers
-                .serialize(&mut serializer, &HashMap::new(), Some(&doc_id_mapping))
-                .unwrap();
-            serializer.close().unwrap();
-        });
-    }
-}

src/fastfield/multivalued/reader.rs

@@ -1,9 +1,6 @@
 use std::ops::Range;
-use std::sync::Arc;
 
-use fastfield_codecs::Column;
-
-use crate::fastfield::{FastValue, MultiValueLength};
+use crate::fastfield::{FastFieldReader, FastFieldReaderImpl, FastValue, MultiValueLength};
 use crate::DocId;
 
 /// Reader for a multivalued `u64` fast field.
@@ -15,14 +12,14 @@ use crate::DocId;
 /// The `idx_reader` associated, for each document, the index of its first value.
 #[derive(Clone)]
 pub struct MultiValuedFastFieldReader<Item: FastValue> {
-    idx_reader: Arc<dyn Column<u64>>,
-    vals_reader: Arc<dyn Column<Item>>,
+    idx_reader: FastFieldReaderImpl<u64>,
+    vals_reader: FastFieldReaderImpl<Item>,
 }
 
 impl<Item: FastValue> MultiValuedFastFieldReader<Item> {
     pub(crate) fn open(
-        idx_reader: Arc<dyn Column<u64>>,
-        vals_reader: Arc<dyn Column<Item>>,
+        idx_reader: FastFieldReaderImpl<u64>,
+        vals_reader: FastFieldReaderImpl<Item>,
     ) -> MultiValuedFastFieldReader<Item> {
         MultiValuedFastFieldReader {
             idx_reader,
@@ -34,9 +31,8 @@ impl<Item: FastValue> MultiValuedFastFieldReader<Item> {
     /// to the given document are `start..end`.
     #[inline]
     fn range(&self, doc: DocId) -> Range<u64> {
-        let idx = doc as u64;
-        let start = self.idx_reader.get_val(idx);
-        let end = self.idx_reader.get_val(idx + 1);
+        let start = self.idx_reader.get(doc);
+        let end = self.idx_reader.get(doc + 1);
         start..end
     }
 

src/fastfield/multivalued/writer.rs

@@ -1,10 +1,10 @@
 use std::io;
-use std::sync::Mutex;
 
-use fastfield_codecs::{Column, MonotonicallyMappableToU64, VecColumn};
 use fnv::FnvHashMap;
+use tantivy_bitpacker::minmax;
 
-use crate::fastfield::{value_to_u64, CompositeFastFieldSerializer, FastFieldType};
+use crate::fastfield::serializer::BitpackedFastFieldSerializerLegacy;
+use crate::fastfield::{value_to_u64, CompositeFastFieldSerializer, FastFieldType, FastValue};
 use crate::indexer::doc_id_mapping::DocIdMapping;
 use crate::postings::UnorderedTermId;
 use crate::schema::{Document, Field, Value};
@@ -17,15 +17,17 @@ use crate::{DatePrecision, DocId};
 /// This `Writer` is only useful for advanced users.
 /// The normal way to get your multivalued int in your index
 /// is to
-/// - declare your field with fast set to
-///   [`Cardinality::MultiValues`](crate::schema::Cardinality::MultiValues) in your schema
+/// - declare your field with fast set to `Cardinality::MultiValues`
+/// in your schema
 /// - add your document simply by calling `.add_document(...)`.
 ///
-/// The `MultiValuedFastFieldWriter` can be acquired from the fastfield writer, by calling
-/// [`FastFieldWriter::get_multivalue_writer_mut()`](crate::fastfield::FastFieldsWriter::get_multivalue_writer_mut).
+/// The `MultiValuedFastFieldWriter` can be acquired from the
+/// fastfield writer, by calling
+/// [`.get_multivalue_writer_mut(...)`](./struct.FastFieldsWriter.html#method.
+/// get_multivalue_writer_mut).
 ///
 /// Once acquired, writing is done by calling
-/// [`.add_document(&Document)`](MultiValuedFastFieldWriter::add_document) once per value.
+/// [`.add_document_vals(&[u64])`](MultiValuedFastFieldWriter::add_document_vals) once per document.
 ///
 /// The serializer makes it possible to remap all of the values
 /// that were pushed to the writer using a mapping.
@@ -99,6 +101,16 @@ impl MultiValuedFastFieldWriter {
         }
     }
 
+    /// Register all of the values associated to a document.
+    ///
+    /// The method returns the `DocId` of the document that was
+    /// just written.
+    pub fn add_document_vals(&mut self, vals: &[UnorderedTermId]) -> DocId {
+        let doc = self.doc_index.len() as DocId;
+        self.next_doc();
+        self.vals.extend_from_slice(vals);
+        doc
+    }
     /// Returns an iterator over values per doc_id in ascending doc_id order.
     ///
     /// Normally the order is simply iterating self.doc_id_index.
@@ -138,217 +150,73 @@ impl MultiValuedFastFieldWriter {
     /// `tantivy` builds a mapping to convert this `UnorderedTermId` into
     /// term ordinals.
     pub fn serialize(
-        mut self,
+        &self,
         serializer: &mut CompositeFastFieldSerializer,
-        term_mapping_opt: Option<&FnvHashMap<UnorderedTermId, TermOrdinal>>,
+        mapping_opt: Option<&FnvHashMap<UnorderedTermId, TermOrdinal>>,
         doc_id_map: Option<&DocIdMapping>,
     ) -> io::Result<()> {
         {
-            self.doc_index.push(self.vals.len() as u64);
-            let col = VecColumn::from(&self.doc_index[..]);
-            if let Some(doc_id_map) = doc_id_map {
-                let multi_value_start_index = MultivalueStartIndex::new(&col, doc_id_map);
-                serializer.create_auto_detect_u64_fast_field_with_idx(
-                    self.field,
-                    multi_value_start_index,
-                    0,
-                )?;
-            } else {
-                serializer.create_auto_detect_u64_fast_field_with_idx(self.field, col, 0)?;
+            // writing the offset index
+            let mut doc_index_serializer =
+                serializer.new_u64_fast_field_with_idx(self.field, 0, self.vals.len() as u64, 0)?;
+
+            let mut offset = 0;
+            for vals in self.get_ordered_values(doc_id_map) {
+                doc_index_serializer.add_val(offset)?;
+                offset += vals.len() as u64;
             }
+            doc_index_serializer.add_val(self.vals.len() as u64)?;
+
+            doc_index_serializer.close_field()?;
         }
         {
-            // Writing the values themselves.
-            // TODO FIXME: Use less memory.
-            let mut values: Vec<u64> = Vec::new();
-            if let Some(term_mapping) = term_mapping_opt {
+            // writing the values themselves.
+            let mut value_serializer: BitpackedFastFieldSerializerLegacy<'_, _>;
+            if let Some(mapping) = mapping_opt {
+                value_serializer = serializer.new_u64_fast_field_with_idx(
+                    self.field,
+                    0u64,
+                    mapping.len() as u64,
+                    1,
+                )?;
+
                 if self.fast_field_type.is_facet() {
                     let mut doc_vals: Vec<u64> = Vec::with_capacity(100);
                     for vals in self.get_ordered_values(doc_id_map) {
-                        // In the case of facets, we want a vec of facet ord that is sorted.
                         doc_vals.clear();
                         let remapped_vals = vals
                             .iter()
-                            .map(|val| *term_mapping.get(val).expect("Missing term ordinal"));
+                            .map(|val| *mapping.get(val).expect("Missing term ordinal"));
                         doc_vals.extend(remapped_vals);
                         doc_vals.sort_unstable();
                         for &val in &doc_vals {
-                            values.push(val);
+                            value_serializer.add_val(val)?;
                         }
                     }
                 } else {
                     for vals in self.get_ordered_values(doc_id_map) {
                         let remapped_vals = vals
                             .iter()
-                            .map(|val| *term_mapping.get(val).expect("Missing term ordinal"));
+                            .map(|val| *mapping.get(val).expect("Missing term ordinal"));
                         for val in remapped_vals {
-                            values.push(val);
+                            value_serializer.add_val(val)?;
                         }
                     }
                 }
             } else {
+                let val_min_max = minmax(self.vals.iter().cloned());
+                let (val_min, val_max) = val_min_max.unwrap_or((0u64, 0u64));
+                value_serializer =
+                    serializer.new_u64_fast_field_with_idx(self.field, val_min, val_max, 1)?;
                 for vals in self.get_ordered_values(doc_id_map) {
                     // sort values in case of remapped doc_ids?
                     for &val in vals {
-                        values.push(val);
+                        value_serializer.add_val(val)?;
                     }
                 }
             }
-            let col = VecColumn::from(&values[..]);
-            serializer.create_auto_detect_u64_fast_field_with_idx(self.field, col, 1)?;
+            value_serializer.close_field()?;
         }
         Ok(())
     }
 }
-
-pub(crate) struct MultivalueStartIndex<'a, C: Column> {
-    column: &'a C,
-    doc_id_map: &'a DocIdMapping,
-    min_max_opt: Mutex<Option<(u64, u64)>>,
-    random_seeker: Mutex<MultivalueStartIndexRandomSeeker<'a, C>>,
-}
-
-struct MultivalueStartIndexRandomSeeker<'a, C: Column> {
-    seek_head: MultivalueStartIndexIter<'a, C>,
-    seek_next_id: u64,
-}
-impl<'a, C: Column> MultivalueStartIndexRandomSeeker<'a, C> {
-    fn new(column: &'a C, doc_id_map: &'a DocIdMapping) -> Self {
-        Self {
-            seek_head: MultivalueStartIndexIter {
-                column,
-                doc_id_map,
-                new_doc_id: 0,
-                offset: 0u64,
-            },
-            seek_next_id: 0u64,
-        }
-    }
-}
-
-impl<'a, C: Column> MultivalueStartIndex<'a, C> {
-    pub fn new(column: &'a C, doc_id_map: &'a DocIdMapping) -> Self {
-        assert_eq!(column.num_vals(), doc_id_map.num_old_doc_ids() as u64 + 1);
-        MultivalueStartIndex {
-            column,
-            doc_id_map,
-            min_max_opt: Mutex::default(),
-            random_seeker: Mutex::new(MultivalueStartIndexRandomSeeker::new(column, doc_id_map)),
-        }
-    }
-
-    fn minmax(&self) -> (u64, u64) {
-        if let Some((min, max)) = *self.min_max_opt.lock().unwrap() {
-            return (min, max);
-        }
-        let (min, max) = tantivy_bitpacker::minmax(self.iter()).unwrap_or((0u64, 0u64));
-        *self.min_max_opt.lock().unwrap() = Some((min, max));
-        (min, max)
-    }
-}
-impl<'a, C: Column> Column for MultivalueStartIndex<'a, C> {
-    fn get_val(&self, idx: u64) -> u64 {
-        let mut random_seeker_lock = self.random_seeker.lock().unwrap();
-        if random_seeker_lock.seek_next_id > idx {
-            *random_seeker_lock =
-                MultivalueStartIndexRandomSeeker::new(self.column, self.doc_id_map);
-        }
-        let to_skip = idx - random_seeker_lock.seek_next_id;
-        random_seeker_lock.seek_next_id = idx + 1;
-        random_seeker_lock.seek_head.nth(to_skip as usize).unwrap()
-    }
-
-    fn min_value(&self) -> u64 {
-        self.minmax().0
-    }
-
-    fn max_value(&self) -> u64 {
-        self.minmax().1
-    }
-
-    fn num_vals(&self) -> u64 {
-        (self.doc_id_map.num_new_doc_ids() + 1) as u64
-    }
-
-    fn iter<'b>(&'b self) -> Box<dyn Iterator<Item = u64> + 'b> {
-        Box::new(MultivalueStartIndexIter::new(self.column, self.doc_id_map))
-    }
-}
-
-struct MultivalueStartIndexIter<'a, C: Column> {
-    pub column: &'a C,
-    pub doc_id_map: &'a DocIdMapping,
-    pub new_doc_id: usize,
-    pub offset: u64,
-}
-
-impl<'a, C: Column> MultivalueStartIndexIter<'a, C> {
-    fn new(column: &'a C, doc_id_map: &'a DocIdMapping) -> Self {
-        Self {
-            column,
-            doc_id_map,
-            new_doc_id: 0,
-            offset: 0,
-        }
-    }
-}
-
-impl<'a, C: Column> Iterator for MultivalueStartIndexIter<'a, C> {
-    type Item = u64;
-
-    fn next(&mut self) -> Option<Self::Item> {
-        if self.new_doc_id > self.doc_id_map.num_new_doc_ids() {
-            return None;
-        }
-        let new_doc_id = self.new_doc_id;
-        self.new_doc_id += 1;
-        let start_offset = self.offset;
-        if new_doc_id < self.doc_id_map.num_new_doc_ids() {
-            let old_doc = self.doc_id_map.get_old_doc_id(new_doc_id as u32) as u64;
-            let num_vals_for_doc = self.column.get_val(old_doc + 1) - self.column.get_val(old_doc);
-            self.offset += num_vals_for_doc;
-        }
-        Some(start_offset)
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn test_multivalue_start_index() {
-        let doc_id_mapping = DocIdMapping::from_new_id_to_old_id(vec![4, 1, 2]);
-        assert_eq!(doc_id_mapping.num_old_doc_ids(), 5);
-        let col = VecColumn::from(&[0u64, 3, 5, 10, 12, 16][..]);
-        let multivalue_start_index = MultivalueStartIndex::new(
-            &col, // 3, 2, 5, 2, 4
-            &doc_id_mapping,
-        );
-        assert_eq!(multivalue_start_index.num_vals(), 4);
-        assert_eq!(
-            multivalue_start_index.iter().collect::<Vec<u64>>(),
-            vec![0, 4, 6, 11]
-        ); // 4, 2, 5
-    }
-
-    #[test]
-    fn test_multivalue_get_vals() {
-        let doc_id_mapping =
-            DocIdMapping::from_new_id_to_old_id(vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9]);
-        assert_eq!(doc_id_mapping.num_old_doc_ids(), 10);
-        let col = VecColumn::from(&[0, 1, 1, 2, 3, 5, 8, 13, 21, 34, 55][..]);
-        let multivalue_start_index = MultivalueStartIndex::new(&col, &doc_id_mapping);
-        assert_eq!(
-            multivalue_start_index.iter().collect::<Vec<u64>>(),
-            vec![0, 1, 1, 2, 3, 5, 8, 13, 21, 34, 55]
-        );
-        assert_eq!(multivalue_start_index.num_vals(), 11);
-        assert_eq!(multivalue_start_index.get_val(3), 2);
-        assert_eq!(multivalue_start_index.get_val(5), 5);
-        assert_eq!(multivalue_start_index.get_val(8), 21);
-        assert_eq!(multivalue_start_index.get_val(4), 3);
-        assert_eq!(multivalue_start_index.get_val(0), 0);
-        assert_eq!(multivalue_start_index.get_val(10), 55);
-    }
-}

src/fastfield/reader.rs

@@ -0,0 +1,43 @@
+use super::FastValue;
+use crate::DocId;
+
+/// FastFieldReader is the trait to access fast field data.
+pub trait FastFieldReader<Item: FastValue> {
+    /// Return the value associated to the given document.
+    ///
+    /// This accessor should return as fast as possible.
+    ///
+    /// # Panics
+    ///
+    /// May panic if `doc` is greater than the segment
+    fn get(&self, doc: DocId) -> Item;
+
+    /// Fills an output buffer with the fast field values
+    /// associated with the `DocId` going from
+    /// `start` to `start + output.len()`.
+    ///
+    /// Regardless of the type of `Item`, this method works
+    /// - transmuting the output array
+    /// - extracting the `Item`s as if they were `u64`
+    /// - possibly converting the `u64` value to the right type.
+    ///
+    /// # Panics
+    ///
+    /// May panic if `start + output.len()` is greater than
+    /// the segment's `maxdoc`.
+    fn get_range(&self, start: u64, output: &mut [Item]);
+
+    /// Returns the minimum value for this fast field.
+    ///
+    /// The min value does not take in account of possible
+    /// deleted document, and should be considered as a lower bound
+    /// of the actual minimum value.
+    fn min_value(&self) -> Item;
+
+    /// Returns the maximum value for this fast field.
+    ///
+    /// The max value does not take in account of possible
+    /// deleted document, and should be considered as an upper bound
+    /// of the actual maximum value.
+    fn max_value(&self) -> Item;
+}

src/fastfield/readers.rs

@@ -1,10 +1,7 @@
-use std::sync::Arc;
-
-use fastfield_codecs::{open, Column};
-
 use crate::directory::{CompositeFile, FileSlice};
 use crate::fastfield::{
-    BytesFastFieldReader, FastFieldNotAvailableError, FastValue, MultiValuedFastFieldReader,
+    BytesFastFieldReader, FastFieldNotAvailableError, FastFieldReaderImpl, FastValue,
+    MultiValuedFastFieldReader,
 };
 use crate::schema::{Cardinality, Field, FieldType, Schema};
 use crate::space_usage::PerFieldSpaceUsage;
@@ -112,17 +109,15 @@ impl FastFieldReaders {
         &self,
         field: Field,
         index: usize,
-    ) -> crate::Result<Arc<dyn Column<TFastValue>>> {
+    ) -> crate::Result<FastFieldReaderImpl<TFastValue>> {
         let fast_field_slice = self.fast_field_data(field, index)?;
-        let bytes = fast_field_slice.read_bytes()?;
-        let column = fastfield_codecs::open(bytes)?;
-        Ok(column)
+        let fast_field_data = fast_field_slice.read_bytes()?;
+        FastFieldReaderImpl::open_from_bytes(fast_field_data)
     }
-
     pub(crate) fn typed_fast_field_reader<TFastValue: FastValue>(
         &self,
         field: Field,
-    ) -> crate::Result<Arc<dyn Column<TFastValue>>> {
+    ) -> crate::Result<FastFieldReaderImpl<TFastValue>> {
         self.typed_fast_field_reader_with_idx(field, 0)
     }
 
@@ -138,7 +133,7 @@ impl FastFieldReaders {
     /// Returns the `u64` fast field reader reader associated to `field`.
     ///
     /// If `field` is not a u64 fast field, this method returns an Error.
-    pub fn u64(&self, field: Field) -> crate::Result<Arc<dyn Column<u64>>> {
+    pub fn u64(&self, field: Field) -> crate::Result<FastFieldReaderImpl<u64>> {
         self.check_type(field, FastType::U64, Cardinality::SingleValue)?;
         self.typed_fast_field_reader(field)
     }
@@ -148,14 +143,14 @@ impl FastFieldReaders {
     ///
     /// If not, the fastfield reader will returns the u64-value associated to the original
     /// FastValue.
-    pub fn u64_lenient(&self, field: Field) -> crate::Result<Arc<dyn Column<u64>>> {
+    pub fn u64_lenient(&self, field: Field) -> crate::Result<FastFieldReaderImpl<u64>> {
         self.typed_fast_field_reader(field)
     }
 
     /// Returns the `i64` fast field reader reader associated to `field`.
     ///
     /// If `field` is not a i64 fast field, this method returns an Error.
-    pub fn i64(&self, field: Field) -> crate::Result<Arc<dyn Column<i64>>> {
+    pub fn i64(&self, field: Field) -> crate::Result<FastFieldReaderImpl<i64>> {
         self.check_type(field, FastType::I64, Cardinality::SingleValue)?;
         self.typed_fast_field_reader(field)
     }
@@ -163,7 +158,7 @@ impl FastFieldReaders {
     /// Returns the `date` fast field reader reader associated to `field`.
     ///
     /// If `field` is not a date fast field, this method returns an Error.
-    pub fn date(&self, field: Field) -> crate::Result<Arc<dyn Column<DateTime>>> {
+    pub fn date(&self, field: Field) -> crate::Result<FastFieldReaderImpl<DateTime>> {
         self.check_type(field, FastType::Date, Cardinality::SingleValue)?;
         self.typed_fast_field_reader(field)
     }
@@ -171,7 +166,7 @@ impl FastFieldReaders {
     /// Returns the `f64` fast field reader reader associated to `field`.
     ///
     /// If `field` is not a f64 fast field, this method returns an Error.
-    pub fn f64(&self, field: Field) -> crate::Result<Arc<dyn Column<f64>>> {
+    pub fn f64(&self, field: Field) -> crate::Result<FastFieldReaderImpl<f64>> {
         self.check_type(field, FastType::F64, Cardinality::SingleValue)?;
         self.typed_fast_field_reader(field)
     }
@@ -179,7 +174,7 @@ impl FastFieldReaders {
     /// Returns the `bool` fast field reader reader associated to `field`.
     ///
     /// If `field` is not a bool fast field, this method returns an Error.
-    pub fn bool(&self, field: Field) -> crate::Result<Arc<dyn Column<bool>>> {
+    pub fn bool(&self, field: Field) -> crate::Result<FastFieldReaderImpl<bool>> {
         self.check_type(field, FastType::Bool, Cardinality::SingleValue)?;
         self.typed_fast_field_reader(field)
     }
@@ -248,7 +243,7 @@ impl FastFieldReaders {
             }
             let fast_field_idx_file = self.fast_field_data(field, 0)?;
             let fast_field_idx_bytes = fast_field_idx_file.read_bytes()?;
-            let idx_reader = open(fast_field_idx_bytes)?;
+            let idx_reader = FastFieldReaderImpl::open_from_bytes(fast_field_idx_bytes)?;
             let data = self.fast_field_data(field, 1)?;
             BytesFastFieldReader::open(idx_reader, data)
         } else {

src/fastfield/serializer/mod.rs

@@ -1,7 +1,11 @@
 use std::io::{self, Write};
 
-pub use fastfield_codecs::Column;
-use fastfield_codecs::{FastFieldCodecType, MonotonicallyMappableToU64, ALL_CODEC_TYPES};
+use common::{BinarySerializable, CountingWriter};
+pub use fastfield_codecs::bitpacked::{
+    BitpackedFastFieldCodec, BitpackedFastFieldSerializerLegacy,
+};
+use fastfield_codecs::dynamic::{CodecType, DynamicFastFieldCodec};
+pub use fastfield_codecs::{FastFieldCodec, FastFieldStats};
 
 use crate::directory::{CompositeWrite, WritePtr};
 use crate::schema::Field;
@@ -15,68 +19,95 @@ use crate::schema::Field;
 /// the serializer.
 /// The serializer expects to receive the following calls.
 ///
-/// * `create_auto_detect_u64_fast_field(...)`
-/// * `create_auto_detect_u64_fast_field(...)`
+/// * `new_u64_fast_field(...)`
+/// * `add_val(...)`
+/// * `add_val(...)`
+/// * `add_val(...)`
 /// * ...
-/// * `let bytes_fastfield = new_bytes_fast_field(...)`
-/// * `bytes_fastfield.write_all(...)`
-/// * `bytes_fastfield.write_all(...)`
-/// * `bytes_fastfield.flush()`
+/// * `close_field()`
+/// * `new_u64_fast_field(...)`
+/// * `add_val(...)`
 /// * ...
+/// * `close_field()`
 /// * `close()`
 pub struct CompositeFastFieldSerializer {
     composite_write: CompositeWrite<WritePtr>,
-    codec_types: Vec<FastFieldCodecType>,
 }
 
 impl CompositeFastFieldSerializer {
-    /// New fast field serializer with all codec types
+    /// Constructor
     pub fn from_write(write: WritePtr) -> io::Result<CompositeFastFieldSerializer> {
-        Self::from_write_with_codec(write, &ALL_CODEC_TYPES)
-    }
-
-    /// New fast field serializer with allowed codec types
-    pub fn from_write_with_codec(
-        write: WritePtr,
-        codec_types: &[FastFieldCodecType],
-    ) -> io::Result<CompositeFastFieldSerializer> {
         let composite_write = CompositeWrite::wrap(write);
-        Ok(CompositeFastFieldSerializer {
-            composite_write,
-            codec_types: codec_types.to_vec(),
-        })
+        Ok(CompositeFastFieldSerializer { composite_write })
     }
 
     /// Serialize data into a new u64 fast field. The best compression codec will be chosen
     /// automatically.
-    pub fn create_auto_detect_u64_fast_field<T: MonotonicallyMappableToU64>(
+    pub fn create_auto_detect_u64_fast_field(
         &mut self,
         field: Field,
-        fastfield_accessor: impl Column<T>,
+        stats: FastFieldStats,
+        vals: &[u64],
     ) -> io::Result<()> {
-        self.create_auto_detect_u64_fast_field_with_idx(field, fastfield_accessor, 0)
+        self.create_auto_detect_u64_fast_field_with_idx(field, stats, vals, 0)
     }
 
     /// Serialize data into a new u64 fast field. The best compression codec will be chosen
     /// automatically.
-    pub fn create_auto_detect_u64_fast_field_with_idx<T: MonotonicallyMappableToU64>(
+    pub fn create_auto_detect_u64_fast_field_with_idx(
         &mut self,
         field: Field,
-        fastfield_accessor: impl Column<T>,
+        stats: FastFieldStats,
+        vals: &[u64],
         idx: usize,
     ) -> io::Result<()> {
         let field_write = self.composite_write.for_field_with_idx(field, idx);
-        fastfield_codecs::serialize(fastfield_accessor, field_write, &self.codec_types)?;
+        DynamicFastFieldCodec.serialize(field_write, vals, stats)?;
         Ok(())
     }
 
-    /// Start serializing a new [u8] fast field. Use the returned writer to write data into the
-    /// bytes field. To associate the bytes with documents a seperate index must be created on
-    /// index 0. See bytes/writer.rs::serialize for an example.
-    ///
-    /// The bytes will be stored as is, no compression will be applied.
-    pub fn new_bytes_fast_field(&mut self, field: Field) -> impl Write + '_ {
-        self.composite_write.for_field_with_idx(field, 1)
+    /// Start serializing a new u64 fast field
+    pub fn serialize_into(
+        &mut self,
+        field: Field,
+        min_value: u64,
+        max_value: u64,
+    ) -> io::Result<BitpackedFastFieldSerializerLegacy<'_, CountingWriter<WritePtr>>> {
+        self.new_u64_fast_field_with_idx(field, min_value, max_value, 0)
+    }
+
+    /// Start serializing a new u64 fast field
+    pub fn new_u64_fast_field(
+        &mut self,
+        field: Field,
+        min_value: u64,
+        max_value: u64,
+    ) -> io::Result<BitpackedFastFieldSerializerLegacy<'_, CountingWriter<WritePtr>>> {
+        self.new_u64_fast_field_with_idx(field, min_value, max_value, 0)
+    }
+
+    /// Start serializing a new u64 fast field
+    pub fn new_u64_fast_field_with_idx(
+        &mut self,
+        field: Field,
+        min_value: u64,
+        max_value: u64,
+        idx: usize,
+    ) -> io::Result<BitpackedFastFieldSerializerLegacy<'_, CountingWriter<WritePtr>>> {
+        let field_write = self.composite_write.for_field_with_idx(field, idx);
+        // Prepend codec id to field data for compatibility with DynamicFastFieldReader.
+        CodecType::Bitpacked.serialize(field_write)?;
+        BitpackedFastFieldSerializerLegacy::open(field_write, min_value, max_value)
+    }
+
+    /// Start serializing a new [u8] fast field
+    pub fn new_bytes_fast_field_with_idx(
+        &mut self,
+        field: Field,
+        idx: usize,
+    ) -> FastBytesFieldSerializer<'_, CountingWriter<WritePtr>> {
+        let field_write = self.composite_write.for_field_with_idx(field, idx);
+        FastBytesFieldSerializer { write: field_write }
     }
 
     /// Closes the serializer
@@ -86,3 +117,17 @@ impl CompositeFastFieldSerializer {
         self.composite_write.close()
     }
 }
+
+pub struct FastBytesFieldSerializer<'a, W: Write> {
+    write: &'a mut W,
+}
+
+impl<'a, W: Write> FastBytesFieldSerializer<'a, W> {
+    pub fn write_all(&mut self, vals: &[u8]) -> io::Result<()> {
+        self.write.write_all(vals)
+    }
+
+    pub fn flush(&mut self) -> io::Result<()> {
+        self.write.flush()
+    }
+}

src/fastfield/wrapper.rs

@@ -0,0 +1,184 @@
+// Copyright (C) 2022 Quickwit, Inc.
+//
+// Quickwit is offered under the AGPL v3.0 and as commercial software.
+// For commercial licensing, contact us at hello@quickwit.io.
+//
+// AGPL:
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+//
+
+use std::marker::PhantomData;
+
+use fastfield_codecs::dynamic::DynamicFastFieldCodec;
+use fastfield_codecs::{FastFieldCodec, FastFieldCodecReader, FastFieldStats};
+use ownedbytes::OwnedBytes;
+
+use crate::directory::FileSlice;
+use crate::fastfield::{FastFieldReader, FastFieldReaderImpl, FastValue};
+use crate::DocId;
+
+/// Wrapper for accessing a fastfield.
+///
+/// Holds the data and the codec to the read the data.
+pub struct FastFieldReaderWrapper<Item: FastValue, Codec: FastFieldCodec> {
+    reader: Codec::Reader,
+    _phantom: PhantomData<Item>,
+    _codec: PhantomData<Codec>,
+}
+
+impl<Item: FastValue, Codec: FastFieldCodec> FastFieldReaderWrapper<Item, Codec> {
+    fn new(reader: Codec::Reader) -> Self {
+        Self {
+            reader,
+            _phantom: PhantomData,
+            _codec: PhantomData,
+        }
+    }
+}
+
+impl<Item: FastValue, Codec: FastFieldCodec> Clone for FastFieldReaderWrapper<Item, Codec>
+where Codec::Reader: Clone
+{
+    fn clone(&self) -> Self {
+        Self {
+            reader: self.reader.clone(),
+            _phantom: PhantomData,
+            _codec: PhantomData,
+        }
+    }
+}
+
+impl<Item: FastValue, C: FastFieldCodec> FastFieldReader<Item> for FastFieldReaderWrapper<Item, C> {
+    /// Return the value associated to the given document.
+    ///
+    /// This accessor should return as fast as possible.
+    ///
+    /// # Panics
+    ///
+    /// May panic if `doc` is greater than the segment
+    // `maxdoc`.
+    fn get(&self, doc: DocId) -> Item {
+        self.get_u64(u64::from(doc))
+    }
+
+    /// Fills an output buffer with the fast field values
+    /// associated with the `DocId` going from
+    /// `start` to `start + output.len()`.
+    ///
+    /// Regardless of the type of `Item`, this method works
+    /// - transmuting the output array
+    /// - extracting the `Item`s as if they were `u64`
+    /// - possibly converting the `u64` value to the right type.
+    ///
+    /// # Panics
+    ///
+    /// May panic if `start + output.len()` is greater than
+    /// the segment's `maxdoc`.
+    fn get_range(&self, start: u64, output: &mut [Item]) {
+        self.get_range_u64(start, output);
+    }
+
+    /// Returns the minimum value for this fast field.
+    ///
+    /// The max value does not take in account of possible
+    /// deleted document, and should be considered as an upper bound
+    /// of the actual maximum value.
+    fn min_value(&self) -> Item {
+        Item::from_u64(self.reader.min_value())
+    }
+
+    /// Returns the maximum value for this fast field.
+    ///
+    /// The max value does not take in account of possible
+    /// deleted document, and should be considered as an upper bound
+    /// of the actual maximum value.
+    fn max_value(&self) -> Item {
+        Item::from_u64(self.reader.max_value())
+    }
+}
+
+impl<Item: FastValue, Codec: FastFieldCodec> FastFieldReaderWrapper<Item, Codec> {
+    /// Opens a fast field given a file.
+    pub fn open(file: FileSlice) -> crate::Result<Self> {
+        let mut bytes = file.read_bytes()?;
+        // TODO
+        // let codec_id = bytes.read_u8();
+        // assert_eq!(
+        //     0u8, codec_id,
+        //     "Tried to open fast field as bitpacked encoded (id=1), but got serializer with \
+        //      different id"
+        // );
+        Self::open_from_bytes(bytes)
+    }
+
+    /// Opens a fast field given the bytes.
+    pub fn open_from_bytes(bytes: OwnedBytes) -> crate::Result<Self> {
+        let reader = Codec::open_from_bytes(bytes)?;
+        Ok(FastFieldReaderWrapper {
+            reader,
+            _codec: PhantomData,
+            _phantom: PhantomData,
+        })
+    }
+
+    #[inline]
+    pub(crate) fn get_u64(&self, doc: u64) -> Item {
+        let data = self.reader.get_u64(doc);
+        Item::from_u64(data)
+    }
+
+    /// Internally `multivalued` also use SingleValue Fast fields.
+    /// It works as follows... A first column contains the list of start index
+    /// for each document, a second column contains the actual values.
+    ///
+    /// The values associated to a given doc, are then
+    ///  `second_column[first_column.get(doc)..first_column.get(doc+1)]`.
+    ///
+    /// Which means single value fast field reader can be indexed internally with
+    /// something different from a `DocId`. For this use case, we want to use `u64`
+    /// values.
+    ///
+    /// See `get_range` for an actual documentation about this method.
+    pub(crate) fn get_range_u64(&self, start: u64, output: &mut [Item]) {
+        for (i, out) in output.iter_mut().enumerate() {
+            *out = self.get_u64(start + (i as u64));
+        }
+    }
+}
+
+use itertools::Itertools;
+
+impl<Item: FastValue, Arr: AsRef<[Item]>> From<Arr> for FastFieldReaderImpl<Item> {
+    fn from(vals: Arr) -> FastFieldReaderImpl<Item> {
+        let mut buffer = Vec::new();
+        let vals_u64: Vec<u64> = vals.as_ref().iter().map(|val| val.to_u64()).collect();
+        let (min_value, max_value) = vals_u64
+            .iter()
+            .copied()
+            .minmax()
+            .into_option()
+            .expect("Expected non empty");
+        let stats = FastFieldStats {
+            min_value,
+            max_value,
+            num_vals: vals_u64.len() as u64,
+        };
+        DynamicFastFieldCodec
+            .serialize(&mut buffer, &vals_u64, stats)
+            .unwrap();
+        let bytes = OwnedBytes::new(buffer);
+        let fast_field_reader = DynamicFastFieldCodec::open_from_bytes(bytes).unwrap();
+        FastFieldReaderImpl::new(fast_field_reader)
+    }
+}

src/fastfield/writer.rs

@@ -2,12 +2,12 @@ use std::collections::HashMap;
 use std::io;
 
 use common;
-use fastfield_codecs::{Column, MonotonicallyMappableToU64};
 use fnv::FnvHashMap;
 use tantivy_bitpacker::BlockedBitpacker;
 
 use super::multivalued::MultiValuedFastFieldWriter;
-use super::FastFieldType;
+use super::serializer::FastFieldStats;
+use super::{FastFieldType, FastValue};
 use crate::fastfield::{BytesFastFieldWriter, CompositeFastFieldSerializer};
 use crate::indexer::doc_id_mapping::DocIdMapping;
 use crate::postings::UnorderedTermId;
@@ -168,7 +168,7 @@ impl FastFieldsWriter {
 
     /// Returns the fast field multi-value writer for the given field.
     ///
-    /// Returns `None` if the field does not exist, or is not
+    /// Returns None if the field does not exist, or is not
     /// configured as a multivalued fastfield in the schema.
     pub fn get_multivalue_writer_mut(
         &mut self,
@@ -182,7 +182,7 @@ impl FastFieldsWriter {
 
     /// Returns the bytes fast field writer for the given field.
     ///
-    /// Returns `None` if the field does not exist, or is not
+    /// Returns None if the field does not exist, or is not
     /// configured as a bytes fastfield in the schema.
     pub fn get_bytes_writer_mut(&mut self, field: Field) -> Option<&mut BytesFastFieldWriter> {
         // TODO optimize
@@ -210,24 +210,25 @@ impl FastFieldsWriter {
     /// Serializes all of the `FastFieldWriter`s by pushing them in
     /// order to the fast field serializer.
     pub fn serialize(
-        self,
+        &self,
         serializer: &mut CompositeFastFieldSerializer,
         mapping: &HashMap<Field, FnvHashMap<UnorderedTermId, TermOrdinal>>,
         doc_id_map: Option<&DocIdMapping>,
     ) -> io::Result<()> {
-        for field_writer in self.term_id_writers {
+        for field_writer in &self.term_id_writers {
             let field = field_writer.field();
+            dbg!("multifield", field);
             field_writer.serialize(serializer, mapping.get(&field), doc_id_map)?;
         }
         for field_writer in &self.single_value_writers {
+            dbg!("singlefield");
             field_writer.serialize(serializer, doc_id_map)?;
         }
-
-        for field_writer in self.multi_values_writers {
+        for field_writer in &self.multi_values_writers {
             let field = field_writer.field();
             field_writer.serialize(serializer, mapping.get(&field), doc_id_map)?;
         }
-        for field_writer in self.bytes_value_writers {
+        for field_writer in &self.bytes_value_writers {
             field_writer.serialize(serializer, doc_id_map)?;
         }
         Ok(())
@@ -359,70 +360,26 @@ impl IntFastFieldWriter {
             (self.val_min, self.val_max)
         };
 
-        let fastfield_accessor = WriterFastFieldAccessProvider {
-            doc_id_map,
-            vals: &self.vals,
+        let vals = compute_fast_field_vals(&self.vals, doc_id_map);
+        let stats = FastFieldStats {
             min_value: min,
             max_value: max,
             num_vals: self.val_count as u64,
         };
-
-        serializer.create_auto_detect_u64_fast_field(self.field, fastfield_accessor)?;
-
+        dbg!(&stats);
+        dbg!(&vals);
+        serializer.create_auto_detect_u64_fast_field(self.field, stats, &vals)?;
         Ok(())
     }
 }
 
-#[derive(Clone)]
-struct WriterFastFieldAccessProvider<'map, 'bitp> {
-    doc_id_map: Option<&'map DocIdMapping>,
-    vals: &'bitp BlockedBitpacker,
-    min_value: u64,
-    max_value: u64,
-    num_vals: u64,
-}
-
-impl<'map, 'bitp> Column for WriterFastFieldAccessProvider<'map, 'bitp> {
-    /// Return the value associated to the given doc.
-    ///
-    /// Whenever possible use the Iterator passed to the fastfield creation instead, for performance
-    /// reasons.
-    ///
-    /// # Panics
-    ///
-    /// May panic if `doc` is greater than the index.
-    fn get_val(&self, doc: u64) -> u64 {
-        if let Some(doc_id_map) = self.doc_id_map {
-            self.vals
-                .get(doc_id_map.get_old_doc_id(doc as u32) as usize) // consider extra
-                                                                     // FastFieldReader wrapper for
-                                                                     // non doc_id_map
-        } else {
-            self.vals.get(doc as usize)
-        }
-    }
-
-    fn iter(&self) -> Box<dyn Iterator<Item = u64> + '_> {
-        if let Some(doc_id_map) = self.doc_id_map {
-            Box::new(
-                doc_id_map
-                    .iter_old_doc_ids()
-                    .map(|doc_id| self.vals.get(doc_id as usize)),
-            )
-        } else {
-            Box::new(self.vals.iter())
-        }
-    }
-
-    fn min_value(&self) -> u64 {
-        self.min_value
-    }
-
-    fn max_value(&self) -> u64 {
-        self.max_value
-    }
-
-    fn num_vals(&self) -> u64 {
-        self.num_vals
+fn compute_fast_field_vals(vals: &BlockedBitpacker, doc_id_map: Option<&DocIdMapping>) -> Vec<u64> {
+    if let Some(doc_id_mapping) = doc_id_map {
+        doc_id_mapping
+            .iter_old_doc_ids()
+            .map(|old_doc_id| vals.get(old_doc_id as usize))
+            .collect()
+    } else {
+        vals.iter().collect()
     }
 }

src/indexer/delete_queue.rs

@@ -178,7 +178,7 @@ pub struct DeleteCursor {
 impl DeleteCursor {
     /// Skips operations and position it so that
     /// - either all of the delete operation currently in the queue are consume and the next get
-    ///   will return `None`.
+    ///   will return None.
     /// - the next get will return the first operation with an
     /// `opstamp >= target_opstamp`.
     pub fn skip_to(&mut self, target_opstamp: Opstamp) {
@@ -246,27 +246,18 @@ impl DeleteCursor {
 mod tests {
 
     use super::{DeleteOperation, DeleteQueue};
-    use crate::query::{Explanation, Scorer, Weight};
-    use crate::{DocId, Score, SegmentReader};
-
-    struct DummyWeight;
-    impl Weight for DummyWeight {
-        fn scorer(&self, _reader: &SegmentReader, _boost: Score) -> crate::Result<Box<dyn Scorer>> {
-            Err(crate::TantivyError::InternalError("dummy impl".to_owned()))
-        }
-
-        fn explain(&self, _reader: &SegmentReader, _doc: DocId) -> crate::Result<Explanation> {
-            Err(crate::TantivyError::InternalError("dummy impl".to_owned()))
-        }
-    }
+    use crate::schema::{Field, Term};
 
     #[test]
     fn test_deletequeue() {
         let delete_queue = DeleteQueue::new();
 
-        let make_op = |i: usize| DeleteOperation {
-            opstamp: i as u64,
-            target: Box::new(DummyWeight),
+        let make_op = |i: usize| {
+            let field = Field::from_field_id(1u32);
+            DeleteOperation {
+                opstamp: i as u64,
+                term: Term::from_field_u64(field, i as u64),
+            }
         };
 
         delete_queue.push(make_op(1));

src/indexer/doc_id_mapping.rs

@@ -2,42 +2,35 @@
 //! to get mappings from old doc_id to new doc_id and vice versa, after sorting
 
 use std::cmp::Reverse;
+use std::ops::Index;
 
 use super::SegmentWriter;
 use crate::schema::{Field, Schema};
-use crate::{DocAddress, DocId, IndexSortByField, Order, TantivyError};
+use crate::{DocId, IndexSortByField, Order, SegmentOrdinal, TantivyError};
 
 /// Struct to provide mapping from new doc_id to old doc_id and segment.
 #[derive(Clone)]
 pub(crate) struct SegmentDocIdMapping {
-    new_doc_id_to_old_doc_addr: Vec<DocAddress>,
+    new_doc_id_to_old_and_segment: Vec<(DocId, SegmentOrdinal)>,
     is_trivial: bool,
 }
 
 impl SegmentDocIdMapping {
-    pub(crate) fn new(new_doc_id_to_old_and_segment: Vec<DocAddress>, is_trivial: bool) -> Self {
+    pub(crate) fn new(
+        new_doc_id_to_old_and_segment: Vec<(DocId, SegmentOrdinal)>,
+        is_trivial: bool,
+    ) -> Self {
         Self {
-            new_doc_id_to_old_doc_addr: new_doc_id_to_old_and_segment,
+            new_doc_id_to_old_and_segment,
             is_trivial,
         }
     }
-
-    /// Returns an iterator over the old document addresses, ordered by the new document ids.
-    ///
-    /// In the returned `DocAddress`, the `segment_ord` is the ordinal of targetted segment
-    /// in the list of merged segments.
-    pub(crate) fn iter_old_doc_addrs(&self) -> impl Iterator<Item = DocAddress> + '_ {
-        self.new_doc_id_to_old_doc_addr.iter().copied()
+    pub(crate) fn iter(&self) -> impl Iterator<Item = &(DocId, SegmentOrdinal)> {
+        self.new_doc_id_to_old_and_segment.iter()
     }
-
     pub(crate) fn len(&self) -> usize {
-        self.new_doc_id_to_old_doc_addr.len()
+        self.new_doc_id_to_old_and_segment.len()
     }
-
-    pub(crate) fn get_old_doc_addr(&self, new_doc_id: DocId) -> DocAddress {
-        self.new_doc_id_to_old_doc_addr[new_doc_id as usize]
-    }
-
     /// This flags means the segments are simply stacked in the order of their ordinal.
     /// e.g. [(0, 1), .. (n, 1), (0, 2)..., (m, 2)]
     ///
@@ -46,6 +39,21 @@ impl SegmentDocIdMapping {
         self.is_trivial
     }
 }
+impl Index<usize> for SegmentDocIdMapping {
+    type Output = (DocId, SegmentOrdinal);
+
+    fn index(&self, idx: usize) -> &Self::Output {
+        &self.new_doc_id_to_old_and_segment[idx]
+    }
+}
+impl IntoIterator for SegmentDocIdMapping {
+    type Item = (DocId, SegmentOrdinal);
+    type IntoIter = std::vec::IntoIter<Self::Item>;
+
+    fn into_iter(self) -> Self::IntoIter {
+        self.new_doc_id_to_old_and_segment.into_iter()
+    }
+}
 
 /// Struct to provide mapping from old doc_id to new doc_id and vice versa within a segment.
 pub struct DocIdMapping {
@@ -91,12 +99,6 @@ impl DocIdMapping {
             .map(|old_doc| els[*old_doc as usize])
             .collect()
     }
-    pub fn num_new_doc_ids(&self) -> usize {
-        self.new_doc_id_to_old.len()
-    }
-    pub fn num_old_doc_ids(&self) -> usize {
-        self.old_doc_id_to_new.len()
-    }
 }
 
 pub(crate) fn expect_field_id_for_sort_field(
@@ -150,6 +152,7 @@ pub(crate) fn get_doc_id_mapping_from_field(
 #[cfg(test)]
 mod tests_indexsorting {
     use crate::collector::TopDocs;
+    use crate::fastfield::FastFieldReader;
     use crate::indexer::doc_id_mapping::DocIdMapping;
     use crate::query::QueryParser;
     use crate::schema::{Schema, *};
@@ -469,9 +472,9 @@ mod tests_indexsorting {
         let my_number = index.schema().get_field("my_number").unwrap();
 
         let fast_field = fast_fields.u64(my_number).unwrap();
-        assert_eq!(fast_field.get_val(0), 10u64);
-        assert_eq!(fast_field.get_val(1), 20u64);
-        assert_eq!(fast_field.get_val(2), 30u64);
+        assert_eq!(fast_field.get(0u32), 10u64);
+        assert_eq!(fast_field.get(1u32), 20u64);
+        assert_eq!(fast_field.get(2u32), 30u64);
 
         let multi_numbers = index.schema().get_field("multi_numbers").unwrap();
         let multifield = fast_fields.u64s(multi_numbers).unwrap();

src/indexer/index_writer.rs

@@ -11,6 +11,7 @@ use super::segment_updater::SegmentUpdater;
 use super::{AddBatch, AddBatchReceiver, AddBatchSender, PreparedCommit};
 use crate::core::{Index, Segment, SegmentComponent, SegmentId, SegmentMeta, SegmentReader};
 use crate::directory::{DirectoryLock, GarbageCollectionResult, TerminatingWrite};
+use crate::docset::{DocSet, TERMINATED};
 use crate::error::TantivyError;
 use crate::fastfield::write_alive_bitset;
 use crate::indexer::delete_queue::{DeleteCursor, DeleteQueue};
@@ -19,9 +20,8 @@ use crate::indexer::index_writer_status::IndexWriterStatus;
 use crate::indexer::operation::DeleteOperation;
 use crate::indexer::stamper::Stamper;
 use crate::indexer::{MergePolicy, SegmentEntry, SegmentWriter};
-use crate::query::{Query, TermQuery};
 use crate::schema::{Document, IndexRecordOption, Term};
-use crate::{FutureResult, IndexReader, Opstamp};
+use crate::{FutureResult, Opstamp};
 
 // Size of the margin for the `memory_arena`. A segment is closed when the remaining memory
 // in the `memory_arena` goes below MARGIN_IN_BYTES.
@@ -31,7 +31,7 @@ pub const MARGIN_IN_BYTES: usize = 1_000_000;
 pub const MEMORY_ARENA_NUM_BYTES_MIN: usize = ((MARGIN_IN_BYTES as u32) * 3u32) as usize;
 pub const MEMORY_ARENA_NUM_BYTES_MAX: usize = u32::MAX as usize - MARGIN_IN_BYTES;
 
-// We impose the number of index writer threads to be at most this.
+// We impose the number of index writer thread to be at most this.
 pub const MAX_NUM_THREAD: usize = 8;
 
 // Add document will block if the number of docs waiting in the queue to be indexed
@@ -40,7 +40,7 @@ const PIPELINE_MAX_SIZE_IN_DOCS: usize = 10_000;
 
 fn error_in_index_worker_thread(context: &str) -> TantivyError {
     TantivyError::ErrorInThread(format!(
-        "{}. A worker thread encountered an error (io::Error most likely) or panicked.",
+        "{}. A worker thread encounterred an error (io::Error most likely) or panicked.",
         context
     ))
 }
@@ -49,7 +49,7 @@ fn error_in_index_worker_thread(context: &str) -> TantivyError {
 ///
 /// It manages a small number of indexing thread, as well as a shared
 /// indexing queue.
-/// Each indexing thread builds its own independent [`Segment`], via
+/// Each indexing thread builds its own independent `Segment`, via
 /// a `SegmentWriter` object.
 pub struct IndexWriter {
     // the lock is just used to bind the
@@ -57,7 +57,6 @@ pub struct IndexWriter {
     _directory_lock: Option<DirectoryLock>,
 
     index: Index,
-    index_reader: IndexReader,
 
     memory_arena_in_bytes_per_thread: usize,
 
@@ -93,14 +92,19 @@ fn compute_deleted_bitset(
 
         // A delete operation should only affect
         // document that were inserted before it.
-        delete_op
-            .target
-            .for_each(segment_reader, &mut |doc_matching_delete_query, _| {
-                if doc_opstamps.is_deleted(doc_matching_delete_query, delete_op.opstamp) {
-                    alive_bitset.remove(doc_matching_delete_query);
+        let inverted_index = segment_reader.inverted_index(delete_op.term.field())?;
+        if let Some(mut docset) =
+            inverted_index.read_postings(&delete_op.term, IndexRecordOption::Basic)?
+        {
+            let mut doc_matching_deleted_term = docset.doc();
+            while doc_matching_deleted_term != TERMINATED {
+                if doc_opstamps.is_deleted(doc_matching_deleted_term, delete_op.opstamp) {
+                    alive_bitset.remove(doc_matching_deleted_term);
                     might_have_changed = true;
                 }
-            })?;
+                doc_matching_deleted_term = docset.advance();
+            }
+        }
         delete_cursor.advance();
     }
     Ok(might_have_changed)
@@ -170,7 +174,9 @@ fn index_documents(
     segment_updater: &mut SegmentUpdater,
     mut delete_cursor: DeleteCursor,
 ) -> crate::Result<()> {
-    let mut segment_writer = SegmentWriter::for_segment(memory_budget, segment.clone())?;
+    let schema = segment.schema();
+
+    let mut segment_writer = SegmentWriter::for_segment(memory_budget, segment.clone(), schema)?;
     for document_group in grouped_document_iterator {
         for doc in document_group {
             segment_writer.add_document(doc)?;
@@ -298,7 +304,6 @@ impl IndexWriter {
 
             memory_arena_in_bytes_per_thread,
             index: index.clone(),
-            index_reader: index.reader()?,
 
             index_writer_status: IndexWriterStatus::from(document_receiver),
             operation_sender: document_sender,
@@ -382,8 +387,8 @@ impl IndexWriter {
             .operation_receiver()
             .ok_or_else(|| {
                 crate::TantivyError::ErrorInThread(
-                    "The index writer was killed. It can happen if an indexing worker encountered \
-                     an Io error for instance."
+                    "The index writer was killed. It can happen if an indexing worker \
+                     encounterred an Io error for instance."
                         .to_string(),
                 )
             })
@@ -507,12 +512,10 @@ impl IndexWriter {
         Ok(self.committed_opstamp)
     }
 
-    /// Merges a given list of segments.
-    ///
-    /// If all segments are empty no new segment will be created.
+    /// Merges a given list of segments
     ///
     /// `segment_ids` is required to be non-empty.
-    pub fn merge(&mut self, segment_ids: &[SegmentId]) -> FutureResult<Option<SegmentMeta>> {
+    pub fn merge(&mut self, segment_ids: &[SegmentId]) -> FutureResult<SegmentMeta> {
         let merge_operation = self.segment_updater.make_merge_operation(segment_ids);
         let segment_updater = self.segment_updater.clone();
         segment_updater.start_merge(merge_operation)
@@ -592,14 +595,14 @@ impl IndexWriter {
     /// * `.commit()`: to accept this commit
     /// * `.abort()`: to cancel this commit.
     ///
-    /// In the current implementation, [`PreparedCommit`] borrows
-    /// the [`IndexWriter`] mutably so we are guaranteed that no new
+    /// In the current implementation, `PreparedCommit` borrows
+    /// the `IndexWriter` mutably so we are guaranteed that no new
     /// document can be added as long as it is committed or is
     /// dropped.
     ///
     /// It is also possible to add a payload to the `commit`
     /// using this API.
-    /// See [`PreparedCommit::set_payload()`].
+    /// See [`PreparedCommit::set_payload()`](PreparedCommit.html)
     pub fn prepare_commit(&mut self) -> crate::Result<PreparedCommit> {
         // Here, because we join all of the worker threads,
         // all of the segment update for this commit have been
@@ -663,33 +666,10 @@ impl IndexWriter {
     /// Like adds, the deletion itself will be visible
     /// only after calling `commit()`.
     pub fn delete_term(&self, term: Term) -> Opstamp {
-        let query = TermQuery::new(term, IndexRecordOption::Basic);
-        // For backward compatibility, if Term is invalid for the index, do nothing but return an
-        // Opstamp
-        self.delete_query(Box::new(query))
-            .unwrap_or_else(|_| self.stamper.stamp())
-    }
-
-    /// Delete all documents matching a given query.
-    /// Returns an `Err` if the query can't be executed.
-    ///
-    /// Delete operation only affects documents that
-    /// were added in previous commits, and documents
-    /// that were added previously in the same commit.
-    ///
-    /// Like adds, the deletion itself will be visible
-    /// only after calling `commit()`.
-    #[doc(hidden)]
-    pub fn delete_query(&self, query: Box<dyn Query>) -> crate::Result<Opstamp> {
-        let weight = query.weight(&self.index_reader.searcher(), false)?;
-
         let opstamp = self.stamper.stamp();
-        let delete_operation = DeleteOperation {
-            opstamp,
-            target: weight,
-        };
+        let delete_operation = DeleteOperation { opstamp, term };
         self.delete_queue.push(delete_operation);
-        Ok(opstamp)
+        opstamp
     }
 
     /// Returns the opstamp of the last successful commit.
@@ -758,17 +738,10 @@ impl IndexWriter {
         let (batch_opstamp, stamps) = self.get_batch_opstamps(count);
 
         let mut adds = AddBatch::default();
-
         for (user_op, opstamp) in user_operations_it.zip(stamps) {
             match user_op {
                 UserOperation::Delete(term) => {
-                    let query = TermQuery::new(term, IndexRecordOption::Basic);
-                    let weight = query.weight(&self.index_reader.searcher(), false)?;
-
-                    let delete_operation = DeleteOperation {
-                        opstamp,
-                        target: weight,
-                    };
+                    let delete_operation = DeleteOperation { opstamp, term };
                     self.delete_queue.push(delete_operation);
                 }
                 UserOperation::Add(document) => {
@@ -812,8 +785,9 @@ mod tests {
     use crate::collector::TopDocs;
     use crate::directory::error::LockError;
     use crate::error::*;
+    use crate::fastfield::FastFieldReader;
     use crate::indexer::NoMergePolicy;
-    use crate::query::{BooleanQuery, Occur, Query, QueryParser, TermQuery};
+    use crate::query::{QueryParser, TermQuery};
     use crate::schema::{
         self, Cardinality, Facet, FacetOptions, IndexRecordOption, NumericOptions,
         TextFieldIndexing, TextOptions, FAST, INDEXED, STORED, STRING, TEXT,
@@ -1040,92 +1014,6 @@ mod tests {
         Ok(())
     }
 
-    #[test]
-    fn test_merge_on_empty_segments_single_segment() -> crate::Result<()> {
-        let mut schema_builder = schema::Schema::builder();
-        let text_field = schema_builder.add_text_field("text", schema::TEXT);
-        let index = Index::create_in_ram(schema_builder.build());
-        let reader = index
-            .reader_builder()
-            .reload_policy(ReloadPolicy::Manual)
-            .try_into()?;
-        let num_docs_containing = |s: &str| {
-            let term_a = Term::from_field_text(text_field, s);
-            reader.searcher().doc_freq(&term_a).unwrap()
-        };
-        // writing the segment
-        let mut index_writer = index.writer(12_000_000).unwrap();
-        index_writer.add_document(doc!(text_field=>"a"))?;
-        index_writer.commit()?;
-        //  this should create 1 segment
-
-        let segments = index.searchable_segment_ids().unwrap();
-        assert_eq!(segments.len(), 1);
-
-        reader.reload().unwrap();
-        assert_eq!(num_docs_containing("a"), 1);
-
-        index_writer.delete_term(Term::from_field_text(text_field, "a"));
-        index_writer.commit()?;
-
-        reader.reload().unwrap();
-        assert_eq!(num_docs_containing("a"), 0);
-
-        index_writer.merge(&segments);
-        index_writer.wait_merging_threads().unwrap();
-
-        let segments = index.searchable_segment_ids().unwrap();
-        assert_eq!(segments.len(), 0);
-
-        Ok(())
-    }
-
-    #[test]
-    fn test_merge_on_empty_segments() -> crate::Result<()> {
-        let mut schema_builder = schema::Schema::builder();
-        let text_field = schema_builder.add_text_field("text", schema::TEXT);
-        let index = Index::create_in_ram(schema_builder.build());
-        let reader = index
-            .reader_builder()
-            .reload_policy(ReloadPolicy::Manual)
-            .try_into()?;
-        let num_docs_containing = |s: &str| {
-            let term_a = Term::from_field_text(text_field, s);
-            reader.searcher().doc_freq(&term_a).unwrap()
-        };
-        // writing the segment
-        let mut index_writer = index.writer(12_000_000).unwrap();
-        index_writer.add_document(doc!(text_field=>"a"))?;
-        index_writer.commit()?;
-        index_writer.add_document(doc!(text_field=>"a"))?;
-        index_writer.commit()?;
-        index_writer.add_document(doc!(text_field=>"a"))?;
-        index_writer.commit()?;
-        index_writer.add_document(doc!(text_field=>"a"))?;
-        index_writer.commit()?;
-        //  this should create 4 segments
-
-        let segments = index.searchable_segment_ids().unwrap();
-        assert_eq!(segments.len(), 4);
-
-        reader.reload().unwrap();
-        assert_eq!(num_docs_containing("a"), 4);
-
-        index_writer.delete_term(Term::from_field_text(text_field, "a"));
-        index_writer.commit()?;
-
-        reader.reload().unwrap();
-        assert_eq!(num_docs_containing("a"), 0);
-
-        index_writer.merge(&segments);
-        index_writer.wait_merging_threads().unwrap();
-
-        let segments = index.searchable_segment_ids().unwrap();
-        assert_eq!(segments.len(), 0);
-
-        Ok(())
-    }
-
     #[test]
     fn test_with_merges() -> crate::Result<()> {
         let mut schema_builder = schema::Schema::builder();
@@ -1439,78 +1327,16 @@ mod tests {
         let fast_field_reader = segment_reader.fast_fields().u64(id_field)?;
         let in_order_alive_ids: Vec<u64> = segment_reader
             .doc_ids_alive()
-            .map(|doc| fast_field_reader.get_val(doc as u64))
+            .map(|doc| fast_field_reader.get(doc))
             .collect();
         assert_eq!(&in_order_alive_ids[..], &[9, 8, 7, 6, 5, 4, 1, 0]);
         Ok(())
     }
 
-    #[test]
-    fn test_delete_query_with_sort_by_field() -> crate::Result<()> {
-        let mut schema_builder = schema::Schema::builder();
-        let id_field =
-            schema_builder.add_u64_field("id", schema::INDEXED | schema::STORED | schema::FAST);
-        let schema = schema_builder.build();
-
-        let settings = IndexSettings {
-            sort_by_field: Some(IndexSortByField {
-                field: "id".to_string(),
-                order: Order::Desc,
-            }),
-            ..Default::default()
-        };
-
-        let index = Index::builder()
-            .schema(schema)
-            .settings(settings)
-            .create_in_ram()?;
-        let index_reader = index.reader()?;
-        let mut index_writer = index.writer_for_tests()?;
-
-        // create and delete docs in same commit
-        for id in 0u64..5u64 {
-            index_writer.add_document(doc!(id_field => id))?;
-        }
-        for id in 1u64..4u64 {
-            let term = Term::from_field_u64(id_field, id);
-            let not_term = Term::from_field_u64(id_field, 2);
-            let term = Box::new(TermQuery::new(term, Default::default()));
-            let not_term = Box::new(TermQuery::new(not_term, Default::default()));
-
-            let query: BooleanQuery = vec![
-                (Occur::Must, term as Box<dyn Query>),
-                (Occur::MustNot, not_term as Box<dyn Query>),
-            ]
-            .into();
-
-            index_writer.delete_query(Box::new(query))?;
-        }
-        for id in 5u64..10u64 {
-            index_writer.add_document(doc!(id_field => id))?;
-        }
-        index_writer.commit()?;
-        index_reader.reload()?;
-
-        let searcher = index_reader.searcher();
-        assert_eq!(searcher.segment_readers().len(), 1);
-
-        let segment_reader = searcher.segment_reader(0);
-        assert_eq!(segment_reader.num_docs(), 8);
-        assert_eq!(segment_reader.max_doc(), 10);
-        let fast_field_reader = segment_reader.fast_fields().u64(id_field)?;
-        let in_order_alive_ids: Vec<u64> = segment_reader
-            .doc_ids_alive()
-            .map(|doc| fast_field_reader.get_val(doc as u64))
-            .collect();
-        assert_eq!(&in_order_alive_ids[..], &[9, 8, 7, 6, 5, 4, 2, 0]);
-        Ok(())
-    }
-
     #[derive(Debug, Clone, Copy)]
     enum IndexingOp {
         AddDoc { id: u64 },
         DeleteDoc { id: u64 },
-        DeleteDocQuery { id: u64 },
         Commit,
         Merge,
     }
@@ -1518,7 +1344,6 @@ mod tests {
     fn balanced_operation_strategy() -> impl Strategy<Value = IndexingOp> {
         prop_oneof![
             (0u64..20u64).prop_map(|id| IndexingOp::DeleteDoc { id }),
-            (0u64..20u64).prop_map(|id| IndexingOp::DeleteDocQuery { id }),
             (0u64..20u64).prop_map(|id| IndexingOp::AddDoc { id }),
             (0u64..1u64).prop_map(|_| IndexingOp::Commit),
             (0u64..1u64).prop_map(|_| IndexingOp::Merge),
@@ -1527,8 +1352,7 @@ mod tests {
 
     fn adding_operation_strategy() -> impl Strategy<Value = IndexingOp> {
         prop_oneof![
-            5 => (0u64..100u64).prop_map(|id| IndexingOp::DeleteDoc { id }),
-            5 => (0u64..100u64).prop_map(|id| IndexingOp::DeleteDocQuery { id }),
+            10 => (0u64..100u64).prop_map(|id| IndexingOp::DeleteDoc { id }),
             50 => (0u64..100u64).prop_map(|id| IndexingOp::AddDoc { id }),
             2 => (0u64..1u64).prop_map(|_| IndexingOp::Commit),
             1 => (0u64..1u64).prop_map(|_| IndexingOp::Merge),
@@ -1548,10 +1372,6 @@ mod tests {
                     existing_ids.remove(&id);
                     deleted_ids.insert(id);
                 }
-                IndexingOp::DeleteDocQuery { id } => {
-                    existing_ids.remove(&id);
-                    deleted_ids.insert(id);
-                }
                 _ => {}
             }
         }
@@ -1634,11 +1454,6 @@ mod tests {
                 IndexingOp::DeleteDoc { id } => {
                     index_writer.delete_term(Term::from_field_u64(id_field, id));
                 }
-                IndexingOp::DeleteDocQuery { id } => {
-                    let term = Term::from_field_u64(id_field, id);
-                    let query = TermQuery::new(term, Default::default());
-                    index_writer.delete_query(Box::new(query))?;
-                }
                 IndexingOp::Commit => {
                     index_writer.commit()?;
                 }
@@ -1678,7 +1493,7 @@ mod tests {
                 let ff_reader = segment_reader.fast_fields().u64(id_field).unwrap();
                 segment_reader
                     .doc_ids_alive()
-                    .map(move |doc| ff_reader.get_val(doc as u64))
+                    .map(move |doc| ff_reader.get(doc))
             })
             .collect();
 
@@ -1689,7 +1504,7 @@ mod tests {
                 let ff_reader = segment_reader.fast_fields().u64(id_field).unwrap();
                 segment_reader
                     .doc_ids_alive()
-                    .map(move |doc| ff_reader.get_val(doc as u64))
+                    .map(move |doc| ff_reader.get(doc))
             })
             .collect();
 
@@ -1717,7 +1532,6 @@ mod tests {
 
         // multivalue fast field tests
         for segment_reader in searcher.segment_readers().iter() {
-            let id_reader = segment_reader.fast_fields().u64(id_field).unwrap();
             let ff_reader = segment_reader.fast_fields().u64s(multi_numbers).unwrap();
             let bool_ff_reader = segment_reader.fast_fields().bools(multi_bools).unwrap();
             for doc in segment_reader.doc_ids_alive() {
@@ -1725,7 +1539,6 @@ mod tests {
                 ff_reader.get_vals(doc, &mut vals);
                 assert_eq!(vals.len(), 2);
                 assert_eq!(vals[0], vals[1]);
-                assert_eq!(id_reader.get_val(doc as u64), vals[0]);
 
                 let mut bool_vals = vec![];
                 bool_ff_reader.get_vals(doc, &mut bool_vals);
@@ -1809,7 +1622,7 @@ mod tests {
                 facet_reader
                     .facet_from_ord(facet_ords[0], &mut facet)
                     .unwrap();
-                let id = ff_reader.get_val(doc_id as u64);
+                let id = ff_reader.get(doc_id);
                 let facet_expected = Facet::from(&("/cola/".to_string() + &id.to_string()));
 
                 assert_eq!(facet, facet_expected);

src/indexer/index_writer_status.rs

@@ -15,7 +15,7 @@ impl IndexWriterStatus {
     }
 
     /// Returns a copy of the operation receiver.
-    /// If the index writer was killed, returns `None`.
+    /// If the index writer was killed, returns None.
     pub fn operation_receiver(&self) -> Option<AddBatchReceiver> {
         let rlock = self
             .inner

src/indexer/json_term_writer.rs

@@ -1,4 +1,3 @@
-use fastfield_codecs::MonotonicallyMappableToU64;
 use fnv::FnvHashMap;
 use murmurhash32::murmurhash2;
 

src/indexer/merger.rs

@@ -1,29 +1,28 @@
+use std::cmp;
 use std::collections::HashMap;
-use std::io::Write;
 use std::sync::Arc;
 
-use fastfield_codecs::VecColumn;
 use itertools::Itertools;
 use measure_time::debug_time;
+use tantivy_bitpacker::minmax;
 
 use crate::core::{Segment, SegmentReader};
 use crate::docset::{DocSet, TERMINATED};
 use crate::error::DataCorruption;
 use crate::fastfield::{
-    AliveBitSet, Column, CompositeFastFieldSerializer, MultiValueLength, MultiValuedFastFieldReader,
+    AliveBitSet, CompositeFastFieldSerializer, FastFieldReader, FastFieldReaderImpl,
+    FastFieldStats, MultiValueLength, MultiValuedFastFieldReader,
 };
 use crate::fieldnorm::{FieldNormReader, FieldNormReaders, FieldNormsSerializer, FieldNormsWriter};
 use crate::indexer::doc_id_mapping::{expect_field_id_for_sort_field, SegmentDocIdMapping};
-use crate::indexer::sorted_doc_id_column::SortedDocIdColumn;
-use crate::indexer::sorted_doc_id_multivalue_column::SortedDocIdMultiValueColumn;
 use crate::indexer::SegmentSerializer;
 use crate::postings::{InvertedIndexSerializer, Postings, SegmentPostings};
 use crate::schema::{Cardinality, Field, FieldType, Schema};
 use crate::store::StoreWriter;
 use crate::termdict::{TermMerger, TermOrdinal};
 use crate::{
-    DocAddress, DocId, IndexSettings, IndexSortByField, InvertedIndexReader, Order,
-    SegmentComponent, SegmentOrdinal,
+    DocId, IndexSettings, IndexSortByField, InvertedIndexReader, Order, SegmentComponent,
+    SegmentOrdinal,
 };
 
 /// Segment's max doc must be `< MAX_DOC_LIMIT`.
@@ -88,6 +87,28 @@ pub struct IndexMerger {
     max_doc: u32,
 }
 
+fn compute_min_max_val(
+    u64_reader: &impl FastFieldReader<u64>,
+    segment_reader: &SegmentReader,
+) -> Option<(u64, u64)> {
+    if segment_reader.max_doc() == 0 {
+        return None;
+    }
+
+    if segment_reader.alive_bitset().is_none() {
+        // no deleted documents,
+        // we can use the previous min_val, max_val.
+        return Some((u64_reader.min_value(), u64_reader.max_value()));
+    }
+    // some deleted documents,
+    // we need to recompute the max / min
+    minmax(
+        segment_reader
+            .doc_ids_alive()
+            .map(|doc_id| u64_reader.get(doc_id)),
+    )
+}
+
 struct TermOrdinalMapping {
     per_segment_new_term_ordinals: Vec<Vec<TermOrdinal>>,
 }
@@ -109,6 +130,14 @@ impl TermOrdinalMapping {
     fn get_segment(&self, segment_ord: usize) -> &[TermOrdinal] {
         &(self.per_segment_new_term_ordinals[segment_ord])[..]
     }
+
+    fn max_term_ord(&self) -> TermOrdinal {
+        self.per_segment_new_term_ordinals
+            .iter()
+            .flat_map(|term_ordinals| term_ordinals.iter().cloned().max())
+            .max()
+            .unwrap_or_default()
+    }
 }
 
 struct DeltaComputer {
@@ -135,6 +164,30 @@ impl DeltaComputer {
     }
 }
 
+fn compute_sorted_multivalued_vals(
+    doc_id_mapping: &SegmentDocIdMapping,
+    fast_field_readers: &Vec<MultiValuedFastFieldReader<u64>>,
+) -> Vec<u64> {
+    let mut vals = Vec::new();
+    let mut buf: Vec<u64> = Vec::new();
+    for &(doc_id, segment_ord) in doc_id_mapping.iter() {
+        fast_field_readers[segment_ord as usize].get_vals(doc_id, &mut buf);
+        vals.extend_from_slice(&buf);
+    }
+    vals
+}
+
+fn compute_vals_sorted(
+    doc_id_mapping: &SegmentDocIdMapping,
+    fast_field_readers: &[FastFieldReaderImpl<u64>],
+) -> Vec<u64> {
+    let mut vals = Vec::with_capacity(doc_id_mapping.len());
+    for &(doc_id, segment_ord) in doc_id_mapping.iter() {
+        vals.push(fast_field_readers[segment_ord as usize].get_u64(doc_id as u64));
+    }
+    vals
+}
+
 impl IndexMerger {
     pub fn open(
         schema: Schema,
@@ -171,7 +224,6 @@ impl IndexMerger {
                 readers.push(reader);
             }
         }
-
         let max_doc = readers.iter().map(|reader| reader.num_docs()).sum();
         if let Some(sort_by_field) = index_settings.sort_by_field.as_ref() {
             readers = Self::sort_readers_by_min_sort_field(readers, sort_by_field)?;
@@ -232,9 +284,9 @@ impl IndexMerger {
                 .iter()
                 .map(|reader| reader.get_fieldnorms_reader(field))
                 .collect::<Result<_, _>>()?;
-            for old_doc_addr in doc_id_mapping.iter_old_doc_addrs() {
-                let fieldnorms_reader = &fieldnorms_readers[old_doc_addr.segment_ord as usize];
-                let fieldnorm_id = fieldnorms_reader.fieldnorm_id(old_doc_addr.doc_id);
+            for (doc_id, reader_ordinal) in doc_id_mapping.iter() {
+                let fieldnorms_reader = &fieldnorms_readers[*reader_ordinal as usize];
+                let fieldnorm_id = fieldnorms_reader.fieldnorm_id(*doc_id);
                 fieldnorms_data.push(fieldnorm_id);
             }
 
@@ -310,8 +362,41 @@ impl IndexMerger {
         fast_field_serializer: &mut CompositeFastFieldSerializer,
         doc_id_mapping: &SegmentDocIdMapping,
     ) -> crate::Result<()> {
-        let fast_field_accessor = SortedDocIdColumn::new(&self.readers, doc_id_mapping, field);
-        fast_field_serializer.create_auto_detect_u64_fast_field(field, fast_field_accessor)?;
+        let (min_value, max_value) = self
+            .readers
+            .iter()
+            .filter_map(|reader| {
+                let u64_reader: FastFieldReaderImpl<u64> =
+                    reader.fast_fields().typed_fast_field_reader(field).expect(
+                        "Failed to find a reader for single fast field. This is a tantivy bug and \
+                         it should never happen.",
+                    );
+                compute_min_max_val(&u64_reader, reader)
+            })
+            .reduce(|a, b| (a.0.min(b.0), a.1.max(b.1)))
+            .expect("Unexpected error, empty readers in IndexMerger");
+
+        let fast_field_readers = self
+            .readers
+            .iter()
+            .map(|reader| {
+                let u64_reader: crate::fastfield::FastFieldReaderImpl<u64> =
+                    reader.fast_fields().typed_fast_field_reader(field).expect(
+                        "Failed to find a reader for single fast field. This is a tantivy bug and \
+                         it should never happen.",
+                    );
+                u64_reader
+            })
+            .collect::<Vec<_>>();
+
+        let stats = FastFieldStats {
+            min_value,
+            max_value,
+            num_vals: doc_id_mapping.len() as u64,
+        };
+
+        let vals = compute_vals_sorted(doc_id_mapping, &fast_field_readers);
+        fast_field_serializer.create_auto_detect_u64_fast_field(field, stats, &vals)?;
 
         Ok(())
     }
@@ -327,7 +412,7 @@ impl IndexMerger {
 
         let everything_is_in_order = reader_ordinal_and_field_accessors
             .into_iter()
-            .map(|(_, col)| Arc::new(col))
+            .map(|reader| reader.1)
             .tuple_windows()
             .all(|(field_accessor1, field_accessor2)| {
                 if sort_by_field.order.is_asc() {
@@ -342,7 +427,7 @@ impl IndexMerger {
     pub(crate) fn get_sort_field_accessor(
         reader: &SegmentReader,
         sort_by_field: &IndexSortByField,
-    ) -> crate::Result<Arc<dyn Column>> {
+    ) -> crate::Result<FastFieldReaderImpl<u64>> {
         let field_id = expect_field_id_for_sort_field(reader.schema(), sort_by_field)?; // for now expect fastfield, but not strictly required
         let value_accessor = reader.fast_fields().u64_lenient(field_id)?;
         Ok(value_accessor)
@@ -351,7 +436,7 @@ impl IndexMerger {
     pub(crate) fn get_reader_with_sort_field_accessor(
         &self,
         sort_by_field: &IndexSortByField,
-    ) -> crate::Result<Vec<(SegmentOrdinal, Arc<dyn Column>)>> {
+    ) -> crate::Result<Vec<(SegmentOrdinal, FastFieldReaderImpl<u64>)>> {
         let reader_ordinal_and_field_accessors = self
             .readers
             .iter()
@@ -384,11 +469,15 @@ impl IndexMerger {
         let doc_id_reader_pair =
             reader_ordinal_and_field_accessors
                 .iter()
-                .map(|(reader_ord, ff_reader)| {
-                    let reader = &self.readers[*reader_ord as usize];
-                    reader
-                        .doc_ids_alive()
-                        .map(move |doc_id| (doc_id, reader_ord, ff_reader))
+                .map(|reader_and_field_accessor| {
+                    let reader = &self.readers[reader_and_field_accessor.0 as usize];
+                    reader.doc_ids_alive().map(move |doc_id| {
+                        (
+                            doc_id,
+                            reader_and_field_accessor.0,
+                            &reader_and_field_accessor.1,
+                        )
+                    })
                 });
 
         let total_num_new_docs = self
@@ -397,25 +486,22 @@ impl IndexMerger {
             .map(|reader| reader.num_docs() as usize)
             .sum();
 
-        let mut sorted_doc_ids: Vec<DocAddress> = Vec::with_capacity(total_num_new_docs);
+        let mut sorted_doc_ids = Vec::with_capacity(total_num_new_docs);
 
         // create iterator tuple of (old doc_id, reader) in order of the new doc_ids
         sorted_doc_ids.extend(
             doc_id_reader_pair
                 .into_iter()
                 .kmerge_by(|a, b| {
-                    let val1 = a.2.get_val(a.0 as u64);
-                    let val2 = b.2.get_val(b.0 as u64);
+                    let val1 = a.2.get(a.0);
+                    let val2 = b.2.get(b.0);
                     if sort_by_field.order == Order::Asc {
                         val1 < val2
                     } else {
                         val1 > val2
                     }
                 })
-                .map(|(doc_id, &segment_ord, _)| DocAddress {
-                    doc_id,
-                    segment_ord,
-                }),
+                .map(|(doc_id, reader_with_id, _)| (doc_id, reader_with_id)),
         );
         Ok(SegmentDocIdMapping::new(sorted_doc_ids, false))
     }
@@ -429,6 +515,31 @@ impl IndexMerger {
         doc_id_mapping: &SegmentDocIdMapping,
         reader_and_field_accessors: &[(&SegmentReader, T)],
     ) -> crate::Result<Vec<u64>> {
+        let mut total_num_vals = 0u64;
+        // In the first pass, we compute the total number of vals.
+        //
+        // This is required by the bitpacker, as it needs to know
+        // what should be the bit length use for bitpacking.
+        let mut num_docs = 0;
+        for (reader, u64s_reader) in reader_and_field_accessors.iter() {
+            if let Some(alive_bitset) = reader.alive_bitset() {
+                num_docs += alive_bitset.num_alive_docs() as u64;
+                for doc in reader.doc_ids_alive() {
+                    let num_vals = u64s_reader.get_len(doc) as u64;
+                    total_num_vals += num_vals;
+                }
+            } else {
+                num_docs += reader.max_doc() as u64;
+                total_num_vals += u64s_reader.get_total_len();
+            }
+        }
+
+        let stats = FastFieldStats {
+            max_value: total_num_vals,
+            // The fastfield offset index contains (num_docs + 1) values.
+            num_vals: num_docs + 1,
+            min_value: 0,
+        };
         // We can now create our `idx` serializer, and in a second pass,
         // can effectively push the different indexes.
 
@@ -437,18 +548,16 @@ impl IndexMerger {
         // access on the fly or 2. change the codec api to make random access optional, but
         // they both have also major drawbacks.
 
-        let mut offsets = Vec::with_capacity(doc_id_mapping.len());
+        let mut offsets: Vec<u64> = Vec::with_capacity(doc_id_mapping.len());
         let mut offset = 0;
-        for old_doc_addr in doc_id_mapping.iter_old_doc_addrs() {
-            let reader = &reader_and_field_accessors[old_doc_addr.segment_ord as usize].1;
+        for (doc_id, reader) in doc_id_mapping.iter() {
+            let reader = &reader_and_field_accessors[*reader as usize].1;
             offsets.push(offset);
-            offset += reader.get_len(old_doc_addr.doc_id) as u64;
+            offset += reader.get_len(*doc_id) as u64;
         }
         offsets.push(offset);
 
-        let fastfield_accessor = VecColumn::from(&offsets[..]);
-
-        fast_field_serializer.create_auto_detect_u64_fast_field(field, fastfield_accessor)?;
+        fast_field_serializer.create_auto_detect_u64_fast_field(field, stats, &offsets[..])?;
         Ok(offsets)
     }
     /// Returns the fastfield index (index for the data, not the data).
@@ -457,14 +566,14 @@ impl IndexMerger {
         field: Field,
         fast_field_serializer: &mut CompositeFastFieldSerializer,
         doc_id_mapping: &SegmentDocIdMapping,
-    ) -> crate::Result<Vec<u64>> {
+    ) -> crate::Result<()> {
         let reader_ordinal_and_field_accessors = self
             .readers
             .iter()
             .map(|reader| {
                 let u64s_reader: MultiValuedFastFieldReader<u64> = reader
                     .fast_fields()
-                    .typed_fast_field_multi_reader::<u64>(field)
+                    .typed_fast_field_multi_reader(field)
                     .expect(
                         "Failed to find index for multivalued field. This is a bug in tantivy, \
                          please report.",
@@ -478,7 +587,8 @@ impl IndexMerger {
             fast_field_serializer,
             doc_id_mapping,
             &reader_ordinal_and_field_accessors,
-        )
+        )?;
+        Ok(())
     }
 
     fn write_term_id_fast_field(
@@ -510,23 +620,25 @@ impl IndexMerger {
             .collect::<Vec<_>>();
         // We can now write the actual fast field values.
         // In the case of hierarchical facets, they are actually term ordinals.
+        let max_term_ord = term_ordinal_mappings.max_term_ord();
         {
-            let mut vals = Vec::new();
-            let mut buffer = Vec::new();
-            for old_doc_addr in doc_id_mapping.iter_old_doc_addrs() {
-                let term_ordinal_mapping: &[TermOrdinal] =
-                    term_ordinal_mappings.get_segment(old_doc_addr.segment_ord as usize);
+            let mut serialize_vals =
+                fast_field_serializer.new_u64_fast_field_with_idx(field, 0u64, max_term_ord, 1)?;
+            let mut vals = Vec::with_capacity(100);
 
-                let ff_reader = &fast_field_reader[old_doc_addr.segment_ord as usize];
-                ff_reader.get_vals(old_doc_addr.doc_id, &mut buffer);
-                for &prev_term_ord in &buffer {
+            for (old_doc_id, reader_ordinal) in doc_id_mapping.iter() {
+                let term_ordinal_mapping: &[TermOrdinal] =
+                    term_ordinal_mappings.get_segment(*reader_ordinal as usize);
+
+                let ff_reader = &fast_field_reader[*reader_ordinal as usize];
+                ff_reader.get_vals(*old_doc_id, &mut vals);
+                for &prev_term_ord in &vals {
                     let new_term_ord = term_ordinal_mapping[prev_term_ord as usize];
-                    vals.push(new_term_ord);
+                    serialize_vals.add_val(new_term_ord)?;
                 }
             }
 
-            let col = VecColumn::from(&vals[..]);
-            fast_field_serializer.create_auto_detect_u64_fast_field_with_idx(field, col, 1)?;
+            serialize_vals.close_field()?;
         }
         Ok(())
     }
@@ -540,17 +652,16 @@ impl IndexMerger {
             .map(|reader| reader.num_docs() as usize)
             .sum();
 
-        let mut mapping: Vec<DocAddress> = Vec::with_capacity(total_num_new_docs);
+        let mut mapping = Vec::with_capacity(total_num_new_docs);
 
         mapping.extend(
             self.readers
                 .iter()
                 .enumerate()
-                .flat_map(|(segment_ord, reader)| {
-                    reader.doc_ids_alive().map(move |doc_id| DocAddress {
-                        segment_ord: segment_ord as u32,
-                        doc_id,
-                    })
+                .flat_map(|(reader_ordinal, reader)| {
+                    reader
+                        .doc_ids_alive()
+                        .map(move |doc_id| (doc_id, reader_ordinal as SegmentOrdinal))
                 }),
         );
         Ok(SegmentDocIdMapping::new(mapping, true))
@@ -566,14 +677,40 @@ impl IndexMerger {
         // The second contains the actual values.
 
         // First we merge the idx fast field.
-        let offsets =
-            self.write_multi_value_fast_field_idx(field, fast_field_serializer, doc_id_mapping)?;
+        self.write_multi_value_fast_field_idx(field, fast_field_serializer, doc_id_mapping)?;
+
+        let mut vals = Vec::with_capacity(100);
+
+        let mut ff_readers = Vec::new();
+
+        // Our values are bitpacked and we need to know what should be
+        // our bitwidth and our minimum value before serializing any values.
+        //
+        // Computing those is non-trivial if some documents are deleted.
+        // We go through a complete first pass to compute the minimum and the
+        // maximum value and initialize our Serializer.
+        for reader in &self.readers {
+            let ff_reader: MultiValuedFastFieldReader<u64> = reader
+                .fast_fields()
+                .typed_fast_field_multi_reader(field)
+                .expect(
+                    "Failed to find multivalued fast field reader. This is a bug in tantivy. \
+                     Please report.",
+                );
+            for doc in reader.doc_ids_alive() {
+                ff_reader.get_vals(doc, &mut vals);
+            }
+            ff_readers.push(ff_reader);
+            // TODO optimize when no deletes
+        }
+
+        let vals = compute_sorted_multivalued_vals(doc_id_mapping, &ff_readers);
+        let stats = FastFieldStats::compute(&vals);
 
-        let fastfield_accessor =
-            SortedDocIdMultiValueColumn::new(&self.readers, doc_id_mapping, &offsets, field);
         fast_field_serializer.create_auto_detect_u64_fast_field_with_idx(
             field,
-            fastfield_accessor,
+            stats,
+            &vals[..],
             1,
         )?;
 
@@ -604,11 +741,11 @@ impl IndexMerger {
             doc_id_mapping,
             &reader_and_field_accessors,
         )?;
-        let mut serialize_vals = fast_field_serializer.new_bytes_fast_field(field);
+        let mut serialize_vals = fast_field_serializer.new_bytes_fast_field_with_idx(field, 1);
 
-        for old_doc_addr in doc_id_mapping.iter_old_doc_addrs() {
-            let bytes_reader = &reader_and_field_accessors[old_doc_addr.segment_ord as usize].1;
-            let val = bytes_reader.get_bytes(old_doc_addr.doc_id);
+        for (doc_id, reader_ordinal) in doc_id_mapping.iter() {
+            let bytes_reader = &reader_and_field_accessors[*reader_ordinal as usize].1;
+            let val = bytes_reader.get_bytes(*doc_id);
             serialize_vals.write_all(val)?;
         }
 
@@ -664,9 +801,9 @@ impl IndexMerger {
                 segment_local_map
             })
             .collect();
-        for (new_doc_id, old_doc_addr) in doc_id_mapping.iter_old_doc_addrs().enumerate() {
-            let segment_map = &mut merged_doc_id_map[old_doc_addr.segment_ord as usize];
-            segment_map[old_doc_addr.doc_id as usize] = Some(new_doc_id as DocId);
+        for (new_doc_id, (old_doc_id, segment_ord)) in doc_id_mapping.iter().enumerate() {
+            let segment_map = &mut merged_doc_id_map[*segment_ord as usize];
+            segment_map[*old_doc_id as usize] = Some(new_doc_id as DocId);
         }
 
         // Note that the total number of tokens is not exact.
@@ -841,15 +978,15 @@ impl IndexMerger {
                 .map(|(i, store)| store.iter_raw(self.readers[i].alive_bitset()))
                 .collect();
 
-            for old_doc_addr in doc_id_mapping.iter_old_doc_addrs() {
-                let doc_bytes_it = &mut document_iterators[old_doc_addr.segment_ord as usize];
+            for (old_doc_id, reader_ordinal) in doc_id_mapping.iter() {
+                let doc_bytes_it = &mut document_iterators[*reader_ordinal as usize];
                 if let Some(doc_bytes_res) = doc_bytes_it.next() {
                     let doc_bytes = doc_bytes_res?;
                     store_writer.store_bytes(&doc_bytes)?;
                 } else {
                     return Err(DataCorruption::comment_only(&format!(
-                        "unexpected missing document in docstore on merge, doc address \
-                         {old_doc_addr:?}",
+                        "unexpected missing document in docstore on merge, doc id {:?}",
+                        old_doc_id
                     ))
                     .into());
                 }
@@ -943,6 +1080,7 @@ mod tests {
     };
     use crate::collector::{Count, FacetCollector};
     use crate::core::Index;
+    use crate::fastfield::FastFieldReader;
     use crate::query::{AllQuery, BooleanQuery, Scorer, TermQuery};
     use crate::schema::{
         Cardinality, Document, Facet, FacetOptions, IndexRecordOption, NumericOptions, Term,

src/indexer/merger_sorted_index_test.rs

@@ -2,7 +2,7 @@
 mod tests {
     use crate::collector::TopDocs;
     use crate::core::Index;
-    use crate::fastfield::{AliveBitSet, MultiValuedFastFieldReader};
+    use crate::fastfield::{AliveBitSet, FastFieldReader, MultiValuedFastFieldReader};
     use crate::query::QueryParser;
     use crate::schema::{
         self, BytesOptions, Cardinality, Facet, FacetOptions, IndexRecordOption, NumericOptions,
@@ -186,17 +186,17 @@ mod tests {
 
         let fast_fields = segment_reader.fast_fields();
         let fast_field = fast_fields.u64(int_field).unwrap();
-        assert_eq!(fast_field.get_val(5), 1u64);
-        assert_eq!(fast_field.get_val(4), 2u64);
-        assert_eq!(fast_field.get_val(3), 3u64);
+        assert_eq!(fast_field.get(5u32), 1u64);
+        assert_eq!(fast_field.get(4u32), 2u64);
+        assert_eq!(fast_field.get(3u32), 3u64);
         if force_disjunct_segment_sort_values {
-            assert_eq!(fast_field.get_val(2u64), 20u64);
-            assert_eq!(fast_field.get_val(1u64), 100u64);
+            assert_eq!(fast_field.get(2u32), 20u64);
+            assert_eq!(fast_field.get(1u32), 100u64);
         } else {
-            assert_eq!(fast_field.get_val(2u64), 10u64);
-            assert_eq!(fast_field.get_val(1u64), 20u64);
+            assert_eq!(fast_field.get(2u32), 10u64);
+            assert_eq!(fast_field.get(1u32), 20u64);
         }
-        assert_eq!(fast_field.get_val(0u64), 1_000u64);
+        assert_eq!(fast_field.get(0u32), 1_000u64);
 
         // test new field norm mapping
         {
@@ -373,12 +373,12 @@ mod tests {
 
         let fast_fields = segment_reader.fast_fields();
         let fast_field = fast_fields.u64(int_field).unwrap();
-        assert_eq!(fast_field.get_val(0), 1u64);
-        assert_eq!(fast_field.get_val(1), 2u64);
-        assert_eq!(fast_field.get_val(2), 3u64);
-        assert_eq!(fast_field.get_val(3), 10u64);
-        assert_eq!(fast_field.get_val(4), 20u64);
-        assert_eq!(fast_field.get_val(5), 1_000u64);
+        assert_eq!(fast_field.get(0u32), 1u64);
+        assert_eq!(fast_field.get(1u32), 2u64);
+        assert_eq!(fast_field.get(2u32), 3u64);
+        assert_eq!(fast_field.get(3u32), 10u64);
+        assert_eq!(fast_field.get(4u32), 20u64);
+        assert_eq!(fast_field.get(5u32), 1_000u64);
 
         let get_vals = |fast_field: &MultiValuedFastFieldReader<u64>, doc_id: u32| -> Vec<u64> {
             let mut vals = vec![];
@@ -478,14 +478,13 @@ mod tests {
 #[cfg(all(test, feature = "unstable"))]
 mod bench_sorted_index_merge {
 
-    use std::sync::Arc;
-
-    use fastfield_codecs::Column;
     use test::{self, Bencher};
 
     use crate::core::Index;
+    // use cratedoc_id, readerdoc_id_mappinglet vals = reader.fate::schema;
+    use crate::fastfield::{DynamicFastFieldReader, FastFieldReader};
     use crate::indexer::merger::IndexMerger;
-    use crate::schema::{Cardinality, NumericOptions, Schema};
+    use crate::schema::{Cardinality, Document, NumericOptions, Schema};
     use crate::{IndexSettings, IndexSortByField, IndexWriter, Order};
     fn create_index(sort_by_field: Option<IndexSortByField>) -> Index {
         let mut schema_builder = Schema::builder();
@@ -504,7 +503,9 @@ mod bench_sorted_index_merge {
         {
             let mut index_writer = index.writer_for_tests().unwrap();
             let index_doc = |index_writer: &mut IndexWriter, val: u64| {
-                index_writer.add_document(doc!(int_field=>val)).unwrap();
+                let mut doc = Document::default();
+                doc.add_u64(int_field, val);
+                index_writer.add_document(doc).unwrap();
             };
             // 3 segments with 10_000 values in the fast fields
             for _ in 0..3 {
@@ -517,7 +518,6 @@ mod bench_sorted_index_merge {
         }
         index
     }
-
     #[bench]
     fn create_sorted_index_walk_overkmerge_on_merge_fastfield(
         b: &mut Bencher,
@@ -533,19 +533,19 @@ mod bench_sorted_index_merge {
             IndexMerger::open(index.schema(), index.settings().clone(), &segments[..])?;
         let doc_id_mapping = merger.generate_doc_id_mapping(&sort_by_field).unwrap();
         b.iter(|| {
-            let sorted_doc_ids = doc_id_mapping.iter_old_doc_addrs().map(|doc_addr| {
-                let reader = &merger.readers[doc_addr.segment_ord as usize];
-                let u64_reader: Arc<dyn Column<u64>> =
+            let sorted_doc_ids = doc_id_mapping.iter().map(|(doc_id, ordinal)| {
+                let reader = &merger.readers[*ordinal as usize];
+                let u64_reader: DynamicFastFieldReader<u64> =
                     reader.fast_fields().typed_fast_field_reader(field).expect(
                         "Failed to find a reader for single fast field. This is a tantivy bug and \
                          it should never happen.",
                     );
-                (doc_addr.doc_id, reader, u64_reader)
+                (doc_id, reader, u64_reader)
             });
             // add values in order of the new doc_ids
             let mut val = 0;
             for (doc_id, _reader, field_reader) in sorted_doc_ids {
-                val = field_reader.get_val(doc_id as u64);
+                val = field_reader.get(*doc_id);
             }
 
             val

src/indexer/mod.rs

@@ -19,8 +19,6 @@ mod segment_register;
 pub mod segment_serializer;
 pub mod segment_updater;
 mod segment_writer;
-mod sorted_doc_id_column;
-mod sorted_doc_id_multivalue_column;
 mod stamper;
 
 use crossbeam_channel as channel;

src/indexer/operation.rs

@@ -1,11 +1,20 @@
-use crate::query::Weight;
 use crate::schema::{Document, Term};
 use crate::Opstamp;
 
 /// Timestamped Delete operation.
+#[derive(Clone, Eq, PartialEq, Debug)]
 pub struct DeleteOperation {
     pub opstamp: Opstamp,
-    pub target: Box<dyn Weight>,
+    pub term: Term,
+}
+
+impl Default for DeleteOperation {
+    fn default() -> Self {
+        DeleteOperation {
+            opstamp: 0u64,
+            term: Term::new(),
+        }
+    }
 }
 
 /// Timestamped Add operation.

src/indexer/segment_manager.rs

@@ -173,7 +173,6 @@ impl SegmentManager {
                 .to_string();
             return Err(TantivyError::InvalidArgument(error_msg));
         }
-
         Ok(segment_entries)
     }
 
@@ -187,7 +186,7 @@ impl SegmentManager {
     pub(crate) fn end_merge(
         &self,
         before_merge_segment_ids: &[SegmentId],
-        after_merge_segment_entry: Option<SegmentEntry>,
+        after_merge_segment_entry: SegmentEntry,
     ) -> crate::Result<SegmentsStatus> {
         let mut registers_lock = self.write();
         let segments_status = registers_lock
@@ -208,9 +207,7 @@ impl SegmentManager {
         for segment_id in before_merge_segment_ids {
             target_register.remove_segment(segment_id);
         }
-        if let Some(entry) = after_merge_segment_entry {
-            target_register.add_segment_entry(entry);
-        }
+        target_register.add_segment_entry(after_merge_segment_entry);
         Ok(segments_status)
     }
 

src/indexer/segment_serializer.rs

@@ -38,16 +38,11 @@ impl SegmentSerializer {
         let fieldnorms_serializer = FieldNormsSerializer::from_write(fieldnorms_write)?;
 
         let postings_serializer = InvertedIndexSerializer::open(&mut segment)?;
-        let settings = segment.index().settings();
-        let store_writer = StoreWriter::new(
-            store_write,
-            settings.docstore_compression,
-            settings.docstore_blocksize,
-            settings.docstore_compress_dedicated_thread,
-        )?;
+        let compressor = segment.index().settings().docstore_compression;
+        let blocksize = segment.index().settings().docstore_blocksize;
         Ok(SegmentSerializer {
             segment,
-            store_writer,
+            store_writer: StoreWriter::new(store_write, compressor, blocksize)?,
             fast_field_serializer,
             fieldnorms_serializer: Some(fieldnorms_serializer),
             postings_serializer,

src/indexer/segment_updater.rs

@@ -25,10 +25,39 @@ use crate::indexer::{
     DefaultMergePolicy, MergeCandidate, MergeOperation, MergePolicy, SegmentEntry,
     SegmentSerializer,
 };
+use crate::schema::Schema;
 use crate::{FutureResult, Opstamp};
 
 const NUM_MERGE_THREADS: usize = 4;
 
+/// Save the index meta file.
+/// This operation is atomic :
+/// Either
+///  - it fails, in which case an error is returned,
+/// and the `meta.json` remains untouched,
+/// - it succeeds, and `meta.json` is written
+/// and flushed.
+///
+/// This method is not part of tantivy's public API
+pub fn save_new_metas(
+    schema: Schema,
+    index_settings: IndexSettings,
+    directory: &dyn Directory,
+) -> crate::Result<()> {
+    save_metas(
+        &IndexMeta {
+            index_settings,
+            segments: Vec::new(),
+            schema,
+            opstamp: 0u64,
+            payload: None,
+        },
+        directory,
+    )?;
+    directory.sync_directory()?;
+    Ok(())
+}
+
 /// Save the index meta file.
 /// This operation is atomic:
 /// Either
@@ -38,7 +67,7 @@ const NUM_MERGE_THREADS: usize = 4;
 /// and flushed.
 ///
 /// This method is not part of tantivy's public API
-pub(crate) fn save_metas(metas: &IndexMeta, directory: &dyn Directory) -> crate::Result<()> {
+fn save_metas(metas: &IndexMeta, directory: &dyn Directory) -> crate::Result<()> {
     info!("save metas");
     let mut buffer = serde_json::to_vec_pretty(metas)?;
     // Just adding a new line at the end of the buffer.
@@ -91,15 +120,7 @@ fn merge(
     index: &Index,
     mut segment_entries: Vec<SegmentEntry>,
     target_opstamp: Opstamp,
-) -> crate::Result<Option<SegmentEntry>> {
-    let num_docs = segment_entries
-        .iter()
-        .map(|segment| segment.meta().num_docs() as u64)
-        .sum::<u64>();
-    if num_docs == 0 {
-        return Ok(None);
-    }
-
+) -> crate::Result<SegmentEntry> {
     // first we need to apply deletes to our segment.
     let merged_segment = index.new_segment();
 
@@ -128,7 +149,7 @@ fn merge(
     let merged_segment_id = merged_segment.id();
 
     let segment_meta = index.new_segment_meta(merged_segment_id, num_docs);
-    Ok(Some(SegmentEntry::new(segment_meta, delete_cursor, None)))
+    Ok(SegmentEntry::new(segment_meta, delete_cursor, None))
 }
 
 /// Advanced: Merges a list of segments from different indices in a new index.
@@ -483,10 +504,7 @@ impl SegmentUpdater {
     // suggested and the moment when it ended up being executed.)
     //
     // `segment_ids` is required to be non-empty.
-    pub fn start_merge(
-        &self,
-        merge_operation: MergeOperation,
-    ) -> FutureResult<Option<SegmentMeta>> {
+    pub fn start_merge(&self, merge_operation: MergeOperation) -> FutureResult<SegmentMeta> {
         assert!(
             !merge_operation.segment_ids().is_empty(),
             "Segment_ids cannot be empty."
@@ -523,8 +541,9 @@ impl SegmentUpdater {
                 merge_operation.target_opstamp(),
             ) {
                 Ok(after_merge_segment_entry) => {
-                    let res = segment_updater.end_merge(merge_operation, after_merge_segment_entry);
-                    let _send_result = merging_future_send.send(res);
+                    let segment_meta_res =
+                        segment_updater.end_merge(merge_operation, after_merge_segment_entry);
+                    let _send_result = merging_future_send.send(segment_meta_res);
                 }
                 Err(merge_error) => {
                     warn!(
@@ -532,10 +551,8 @@ impl SegmentUpdater {
                         merge_operation.segment_ids().to_vec(),
                         merge_error
                     );
-                    if cfg!(test) {
-                        panic!("{:?}", merge_error);
-                    }
                     let _send_result = merging_future_send.send(Err(merge_error));
+                    assert!(!cfg!(test), "Merge failed.");
                 }
             }
         });
@@ -585,42 +602,35 @@ impl SegmentUpdater {
     fn end_merge(
         &self,
         merge_operation: MergeOperation,
-        mut after_merge_segment_entry: Option<SegmentEntry>,
-    ) -> crate::Result<Option<SegmentMeta>> {
+        mut after_merge_segment_entry: SegmentEntry,
+    ) -> crate::Result<SegmentMeta> {
         let segment_updater = self.clone();
-        let after_merge_segment_meta = after_merge_segment_entry
-            .as_ref()
-            .map(|after_merge_segment_entry| after_merge_segment_entry.meta().clone());
+        let after_merge_segment_meta = after_merge_segment_entry.meta().clone();
         self.schedule_task(move || {
-            info!(
-                "End merge {:?}",
-                after_merge_segment_entry.as_ref().map(|entry| entry.meta())
-            );
+            info!("End merge {:?}", after_merge_segment_entry.meta());
             {
-                if let Some(after_merge_segment_entry) = after_merge_segment_entry.as_mut() {
-                    let mut delete_cursor = after_merge_segment_entry.delete_cursor().clone();
-                    if let Some(delete_operation) = delete_cursor.get() {
-                        let committed_opstamp = segment_updater.load_meta().opstamp;
-                        if delete_operation.opstamp < committed_opstamp {
-                            let index = &segment_updater.index;
-                            let segment = index.segment(after_merge_segment_entry.meta().clone());
-                            if let Err(advance_deletes_err) = advance_deletes(
-                                segment,
-                                after_merge_segment_entry,
-                                committed_opstamp,
-                            ) {
-                                error!(
-                                    "Merge of {:?} was cancelled (advancing deletes failed): {:?}",
-                                    merge_operation.segment_ids(),
-                                    advance_deletes_err
-                                );
-                                assert!(!cfg!(test), "Merge failed.");
+                let mut delete_cursor = after_merge_segment_entry.delete_cursor().clone();
+                if let Some(delete_operation) = delete_cursor.get() {
+                    let committed_opstamp = segment_updater.load_meta().opstamp;
+                    if delete_operation.opstamp < committed_opstamp {
+                        let index = &segment_updater.index;
+                        let segment = index.segment(after_merge_segment_entry.meta().clone());
+                        if let Err(advance_deletes_err) = advance_deletes(
+                            segment,
+                            &mut after_merge_segment_entry,
+                            committed_opstamp,
+                        ) {
+                            error!(
+                                "Merge of {:?} was cancelled (advancing deletes failed): {:?}",
+                                merge_operation.segment_ids(),
+                                advance_deletes_err
+                            );
+                            assert!(!cfg!(test), "Merge failed.");
 
-                                // ... cancel merge
-                                // `merge_operations` are tracked. As it is dropped, the
-                                // the segment_ids will be available again for merge.
-                                return Err(advance_deletes_err);
-                            }
+                            // ... cancel merge
+                            // `merge_operations` are tracked. As it is dropped, the
+                            // the segment_ids will be available again for merge.
+                            return Err(advance_deletes_err);
                         }
                     }
                 }

src/indexer/segment_writer.rs

@@ -1,9 +1,7 @@
-use fastfield_codecs::MonotonicallyMappableToU64;
-
 use super::doc_id_mapping::{get_doc_id_mapping_from_field, DocIdMapping};
 use super::operation::AddOperation;
 use crate::core::Segment;
-use crate::fastfield::FastFieldsWriter;
+use crate::fastfield::{FastFieldsWriter, FastValue as _};
 use crate::fieldnorm::{FieldNormReaders, FieldNormsWriter};
 use crate::indexer::json_term_writer::index_json_values;
 use crate::indexer::segment_serializer::SegmentSerializer;
@@ -82,8 +80,8 @@ impl SegmentWriter {
     pub fn for_segment(
         memory_budget_in_bytes: usize,
         segment: Segment,
+        schema: Schema,
     ) -> crate::Result<SegmentWriter> {
-        let schema = segment.schema();
         let tokenizer_manager = segment.index().tokenizers().clone();
         let table_size = compute_initial_table_size(memory_budget_in_bytes)?;
         let segment_serializer = SegmentSerializer::for_segment(segment, false)?;
@@ -138,7 +136,7 @@ impl SegmentWriter {
         remap_and_write(
             &self.per_field_postings_writers,
             self.ctx,
-            self.fast_field_writers,
+            &self.fast_field_writers,
             &self.fieldnorms_writer,
             &self.schema,
             self.segment_serializer,
@@ -345,7 +343,7 @@ impl SegmentWriter {
 fn remap_and_write(
     per_field_postings_writers: &PerFieldPostingsWriter,
     ctx: IndexingContext,
-    fast_field_writers: FastFieldsWriter,
+    fast_field_writers: &FastFieldsWriter,
     fieldnorms_writer: &FieldNormsWriter,
     schema: &Schema,
     mut serializer: SegmentSerializer,
@@ -380,14 +378,12 @@ fn remap_and_write(
         let store_write = serializer
             .segment_mut()
             .open_write(SegmentComponent::Store)?;
-        let settings = serializer.segment().index().settings();
-        let store_writer = StoreWriter::new(
-            store_write,
-            settings.docstore_compression,
-            settings.docstore_blocksize,
-            settings.docstore_compress_dedicated_thread,
-        )?;
-        let old_store_writer = std::mem::replace(&mut serializer.store_writer, store_writer);
+        let compressor = serializer.segment().index().settings().docstore_compression;
+        let block_size = serializer.segment().index().settings().docstore_blocksize;
+        let old_store_writer = std::mem::replace(
+            &mut serializer.store_writer,
+            StoreWriter::new(store_write, compressor, block_size)?,
+        );
         old_store_writer.close()?;
         let store_read = StoreReader::open(
             serializer

src/indexer/sorted_doc_id_column.rs

@@ -1,112 +0,0 @@
-use std::sync::Arc;
-
-use fastfield_codecs::Column;
-use itertools::Itertools;
-
-use crate::indexer::doc_id_mapping::SegmentDocIdMapping;
-use crate::schema::Field;
-use crate::{DocAddress, SegmentReader};
-
-pub(crate) struct SortedDocIdColumn<'a> {
-    doc_id_mapping: &'a SegmentDocIdMapping,
-    fast_field_readers: Vec<Arc<dyn Column<u64>>>,
-    min_value: u64,
-    max_value: u64,
-    num_vals: u64,
-}
-
-fn compute_min_max_val(
-    u64_reader: &dyn Column<u64>,
-    segment_reader: &SegmentReader,
-) -> Option<(u64, u64)> {
-    if segment_reader.max_doc() == 0 {
-        return None;
-    }
-
-    if segment_reader.alive_bitset().is_none() {
-        // no deleted documents,
-        // we can use the previous min_val, max_val.
-        return Some((u64_reader.min_value(), u64_reader.max_value()));
-    }
-    // some deleted documents,
-    // we need to recompute the max / min
-    segment_reader
-        .doc_ids_alive()
-        .map(|doc_id| u64_reader.get_val(doc_id as u64))
-        .minmax()
-        .into_option()
-}
-
-impl<'a> SortedDocIdColumn<'a> {
-    pub(crate) fn new(
-        readers: &'a [SegmentReader],
-        doc_id_mapping: &'a SegmentDocIdMapping,
-        field: Field,
-    ) -> Self {
-        let (min_value, max_value) = readers
-            .iter()
-            .filter_map(|reader| {
-                let u64_reader: Arc<dyn Column<u64>> =
-                    reader.fast_fields().typed_fast_field_reader(field).expect(
-                        "Failed to find a reader for single fast field. This is a tantivy bug and \
-                         it should never happen.",
-                    );
-                compute_min_max_val(&*u64_reader, reader)
-            })
-            .reduce(|a, b| (a.0.min(b.0), a.1.max(b.1)))
-            .expect("Unexpected error, empty readers in IndexMerger");
-
-        let fast_field_readers = readers
-            .iter()
-            .map(|reader| {
-                let u64_reader: Arc<dyn Column<u64>> =
-                    reader.fast_fields().typed_fast_field_reader(field).expect(
-                        "Failed to find a reader for single fast field. This is a tantivy bug and \
-                         it should never happen.",
-                    );
-                u64_reader
-            })
-            .collect::<Vec<_>>();
-
-        SortedDocIdColumn {
-            doc_id_mapping,
-            fast_field_readers,
-            min_value,
-            max_value,
-            num_vals: doc_id_mapping.len() as u64,
-        }
-    }
-}
-
-impl<'a> Column for SortedDocIdColumn<'a> {
-    fn get_val(&self, doc: u64) -> u64 {
-        let DocAddress {
-            doc_id,
-            segment_ord,
-        } = self.doc_id_mapping.get_old_doc_addr(doc as u32);
-        self.fast_field_readers[segment_ord as usize].get_val(doc_id as u64)
-    }
-
-    fn iter(&self) -> Box<dyn Iterator<Item = u64> + '_> {
-        Box::new(
-            self.doc_id_mapping
-                .iter_old_doc_addrs()
-                .map(|old_doc_addr| {
-                    let fast_field_reader =
-                        &self.fast_field_readers[old_doc_addr.segment_ord as usize];
-                    fast_field_reader.get_val(old_doc_addr.doc_id as u64)
-                }),
-        )
-    }
-    fn min_value(&self) -> u64 {
-        self.min_value
-    }
-
-    fn max_value(&self) -> u64 {
-        self.max_value
-    }
-
-    fn num_vals(&self) -> u64 {
-        self.num_vals
-    }
-}

src/indexer/sorted_doc_id_multivalue_column.rs

@@ -1,121 +0,0 @@
-use std::cmp;
-
-use fastfield_codecs::Column;
-
-use crate::fastfield::{MultiValueLength, MultiValuedFastFieldReader};
-use crate::indexer::doc_id_mapping::SegmentDocIdMapping;
-use crate::schema::Field;
-use crate::{DocId, SegmentReader};
-
-// We can now initialize our serializer, and push it the different values
-pub(crate) struct SortedDocIdMultiValueColumn<'a> {
-    doc_id_mapping: &'a SegmentDocIdMapping,
-    fast_field_readers: Vec<MultiValuedFastFieldReader<u64>>,
-    offsets: &'a [u64],
-    min_value: u64,
-    max_value: u64,
-    num_vals: u64,
-}
-
-impl<'a> SortedDocIdMultiValueColumn<'a> {
-    pub(crate) fn new(
-        readers: &'a [SegmentReader],
-        doc_id_mapping: &'a SegmentDocIdMapping,
-        offsets: &'a [u64],
-        field: Field,
-    ) -> Self {
-        // Our values are bitpacked and we need to know what should be
-        // our bitwidth and our minimum value before serializing any values.
-        //
-        // Computing those is non-trivial if some documents are deleted.
-        // We go through a complete first pass to compute the minimum and the
-        // maximum value and initialize our Serializer.
-        let mut num_vals = 0;
-        let mut min_value = u64::MAX;
-        let mut max_value = u64::MIN;
-        let mut vals = Vec::new();
-        let mut fast_field_readers = Vec::with_capacity(readers.len());
-        for reader in readers {
-            let ff_reader: MultiValuedFastFieldReader<u64> = reader
-                .fast_fields()
-                .typed_fast_field_multi_reader::<u64>(field)
-                .expect(
-                    "Failed to find multivalued fast field reader. This is a bug in tantivy. \
-                     Please report.",
-                );
-            for doc in reader.doc_ids_alive() {
-                ff_reader.get_vals(doc, &mut vals);
-                for &val in &vals {
-                    min_value = cmp::min(val, min_value);
-                    max_value = cmp::max(val, max_value);
-                }
-                num_vals += vals.len();
-            }
-            fast_field_readers.push(ff_reader);
-            // TODO optimize when no deletes
-        }
-        if min_value > max_value {
-            min_value = 0;
-            max_value = 0;
-        }
-        SortedDocIdMultiValueColumn {
-            doc_id_mapping,
-            fast_field_readers,
-            offsets,
-            min_value,
-            max_value,
-            num_vals: num_vals as u64,
-        }
-    }
-}
-
-impl<'a> Column for SortedDocIdMultiValueColumn<'a> {
-    fn get_val(&self, pos: u64) -> u64 {
-        // use the offsets index to find the doc_id which will contain the position.
-        // the offsets are strictly increasing so we can do a simple search on it.
-        let new_doc_id: DocId = self
-            .offsets
-            .iter()
-            .position(|&offset| offset > pos)
-            .expect("pos is out of bounds") as DocId
-            - 1u32;
-
-        // now we need to find the position of `pos` in the multivalued bucket
-        let num_pos_covered_until_now = self.offsets[new_doc_id as usize];
-        let pos_in_values = pos - num_pos_covered_until_now;
-
-        let old_doc_addr = self.doc_id_mapping.get_old_doc_addr(new_doc_id);
-        let num_vals =
-            self.fast_field_readers[old_doc_addr.segment_ord as usize].get_len(old_doc_addr.doc_id);
-        assert!(num_vals >= pos_in_values);
-        let mut vals = Vec::new();
-        self.fast_field_readers[old_doc_addr.segment_ord as usize]
-            .get_vals(old_doc_addr.doc_id, &mut vals);
-
-        vals[pos_in_values as usize]
-    }
-
-    fn iter(&self) -> Box<dyn Iterator<Item = u64> + '_> {
-        Box::new(
-            self.doc_id_mapping
-                .iter_old_doc_addrs()
-                .flat_map(|old_doc_addr| {
-                    let ff_reader = &self.fast_field_readers[old_doc_addr.segment_ord as usize];
-                    let mut vals = Vec::new();
-                    ff_reader.get_vals(old_doc_addr.doc_id, &mut vals);
-                    vals.into_iter()
-                }),
-        )
-    }
-    fn min_value(&self) -> u64 {
-        self.min_value
-    }
-
-    fn max_value(&self) -> u64 {
-        self.max_value
-    }
-
-    fn num_vals(&self) -> u64 {
-        self.num_vals
-    }
-}

src/lib.rs

@@ -11,7 +11,6 @@
 #![doc(test(attr(allow(unused_variables), deny(warnings))))]
 #![warn(missing_docs)]
 #![allow(clippy::len_without_is_empty)]
-#![allow(clippy::derive_partial_eq_without_eq)]
 
 //! # `tantivy`
 //!
@@ -301,7 +300,7 @@ pub use self::docset::{DocSet, TERMINATED};
 pub use crate::core::{
     Executor, Index, IndexBuilder, IndexMeta, IndexSettings, IndexSortByField, InvertedIndexReader,
     Order, Searcher, SearcherGeneration, Segment, SegmentComponent, SegmentId, SegmentMeta,
-    SegmentReader, SingleSegmentIndexWriter,
+    SegmentReader,
 };
 pub use crate::directory::Directory;
 pub use crate::indexer::demuxer::*;
@@ -429,6 +428,7 @@ pub mod tests {
     use crate::collector::tests::TEST_COLLECTOR_WITH_SCORE;
     use crate::core::SegmentReader;
     use crate::docset::{DocSet, TERMINATED};
+    use crate::fastfield::FastFieldReader;
     use crate::merge_policy::NoMergePolicy;
     use crate::query::BooleanQuery;
     use crate::schema::*;
@@ -1035,21 +1035,21 @@ pub mod tests {
             let fast_field_reader_opt = segment_reader.fast_fields().u64(fast_field_unsigned);
             assert!(fast_field_reader_opt.is_ok());
             let fast_field_reader = fast_field_reader_opt.unwrap();
-            assert_eq!(fast_field_reader.get_val(0), 4u64)
+            assert_eq!(fast_field_reader.get(0), 4u64)
         }
 
         {
             let fast_field_reader_res = segment_reader.fast_fields().i64(fast_field_signed);
             assert!(fast_field_reader_res.is_ok());
             let fast_field_reader = fast_field_reader_res.unwrap();
-            assert_eq!(fast_field_reader.get_val(0), 4i64)
+            assert_eq!(fast_field_reader.get(0), 4i64)
         }
 
         {
             let fast_field_reader_res = segment_reader.fast_fields().f64(fast_field_float);
             assert!(fast_field_reader_res.is_ok());
             let fast_field_reader = fast_field_reader_res.unwrap();
-            assert_eq!(fast_field_reader.get_val(0), 4f64)
+            assert_eq!(fast_field_reader.get(0), 4f64)
         }
         Ok(())
     }

src/positions/mod.rs

@@ -1,18 +1,18 @@
 //! Tantivy can (if instructed to do so in the schema) store the term positions in a given field.
 //! This position is expressed as token ordinal. For instance,
-//! In "The beauty and the beast", the term "the" appears in position 0 and position 3.
+//! In "The beauty and the beast", the term "the" appears in position 0 and position 4.
 //! This information is useful to run phrase queries.
 //!
-//! The [position](crate::SegmentComponent::Positions) file contains all of the
+//! The [position](../enum.SegmentComponent.html#variant.Positions) file contains all of the
 //! bitpacked positions delta, for all terms of a given field, one term after the other.
 //!
 //! Each term is encoded independently.
-//! Like for posting lists, tantivy relies on simd bitpacking to encode the positions delta in
-//! blocks of 128 deltas. Because we rarely have a multiple of 128, the final block encodes
-//! the remaining values with variable int encoding.
+//! Like for positing lists, tantivy relies on simd bitpacking to encode the positions delta in
+//! blocks of 128 deltas. Because we rarely have a multiple of 128, a final block may encode the
+//! remaining values variable byte encoding.
 //!
-//! In order to make reading possible, the term delta positions first encode the number of
-//! bitpacked blocks, then the bitwidth for each block, then the actual bitpacked blocks and finally
+//! In order to make reading possible, the term delta positions first encodes the number of
+//! bitpacked blocks, then the bitwidth for each blocks, then the actual bitpacked block and finally
 //! the final variable int encoded block.
 //!
 //! Contrary to postings list, the reader does not have access on the number of positions that is