Avoid scoring buffered unions when scores are ignored

BufferedUnionScorer can use score_doc during refill only when the score combiner needs scores. DoNothingCombiner now advertises that scoring is unnecessary, preserving the no-score path for count collectors and avoiding wasted score_doc calls.

Add a regression test that verifies DoNothingCombiner does not invoke score() or score_doc() while counting a buffered union.

.github/workflows/coverage.yml

@@ -20,7 +20,7 @@ jobs:
       contents: read
 
     steps:
-      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
       - name: Install Rust
         run: rustup toolchain install nightly-2025-12-01 --profile minimal --component llvm-tools-preview
       - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1

.github/workflows/long_running.yml

@@ -25,7 +25,7 @@ jobs:
       contents: read
 
     steps:
-    - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
     - name: Install stable
       uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af # v1.0.7
       with:

.github/workflows/scorecard.yml

@@ -22,7 +22,7 @@ jobs:
 
     steps:
       - name: 'Checkout code'
-        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
           persist-credentials: false
 
@@ -36,7 +36,7 @@ jobs:
 
       # Upload the results as artifacts.
       - name: 'Upload artifact'
-        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
         with:
           name: SARIF file
           path: results.sarif
@@ -44,6 +44,6 @@ jobs:
 
       # Upload the results to GitHub's code scanning dashboard.
       - name: 'Upload to code-scanning'
-        uses: github/codeql-action/upload-sarif@87557b9c84dde89fdd9b10e88954ac2f4248e463 # v4.36.1
+        uses: github/codeql-action/upload-sarif@95e58e9a2cdfd71adc6e0353d5c52f41a045d225 # v4.35.2
         with:
           sarif_file: results.sarif

.github/workflows/test.yml

@@ -27,7 +27,7 @@ jobs:
       checks: write
 
     steps:
-    - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
 
     - name: Install nightly
       uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af # v1.0.7
@@ -77,7 +77,7 @@ jobs:
     name: test-${{ matrix.features.label}}
 
     steps:
-    - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
 
     - name: Install stable
       uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af # v1.0.7

bitpacker/Cargo.toml

@@ -18,10 +18,5 @@ homepage = "https://github.com/quickwit-oss/tantivy"
 bitpacking = { version = "0.9.2", default-features = false, features = ["bitpacker1x"] }
 
 [dev-dependencies]
-binggan = "0.17.0"
 rand = "0.9"
 proptest = "1"
-
-[[bench]]
-name = "bench"
-harness = false

bitpacker/benches/bench.rs

@@ -1,110 +1,65 @@
-use std::cell::RefCell;
+#![feature(test)]
 
-use binggan::{BenchRunner, black_box};
-use rand::rng;
-use rand::seq::IteratorRandom;
-use tantivy_bitpacker::{BitPacker, BitUnpacker, BlockedBitpacker};
+extern crate test;
 
-fn create_bitpacked_data(bit_width: u8, num_els: u32) -> Vec<u8> {
-    let mut bitpacker = BitPacker::new();
-    let mut buffer = Vec::new();
-    for _ in 0..num_els {
-        bitpacker.write(0u64, bit_width, &mut buffer).unwrap();
-        bitpacker.flush(&mut buffer).unwrap();
-    }
-    buffer
-}
+#[cfg(test)]
+mod tests {
+    use rand::rng;
+    use rand::seq::IteratorRandom;
+    use tantivy_bitpacker::{BitPacker, BitUnpacker, BlockedBitpacker};
+    use test::Bencher;
 
-const N: usize = 100_000;
-const MAX_VAL: u64 = 1_000;
-const BIT_WIDTH: u8 = 10; // 2^10 = 1024 > MAX_VAL
-
-fn create_packed_data() -> (BitUnpacker, Vec<u8>) {
-    let mut bitpacker = BitPacker::new();
-    let mut data = Vec::new();
-    for i in 0..N as u64 {
-        let val = i * MAX_VAL / N as u64;
-        bitpacker.write(val, BIT_WIDTH, &mut data).unwrap();
-    }
-    bitpacker.close(&mut data).unwrap();
-    (BitUnpacker::new(BIT_WIDTH), data)
-}
-
-fn bench_bitpacking() {
-    let mut runner = BenchRunner::new();
-    let bit_width = 3;
-    let num_els = 1_000_000u32;
-    let bit_unpacker = BitUnpacker::new(bit_width);
-    let data = create_bitpacked_data(bit_width, num_els);
-    let idxs: Vec<u32> = (0..num_els).choose_multiple(&mut rng(), 100_000);
-    runner.bench_function("bitpacking_read", move |_| {
-        let mut out = 0u64;
-        for &idx in &idxs {
-            out = out.wrapping_add(bit_unpacker.get(idx, &data[..]));
+    #[inline(never)]
+    fn create_bitpacked_data(bit_width: u8, num_els: u32) -> Vec<u8> {
+        let mut bitpacker = BitPacker::new();
+        let mut buffer = Vec::new();
+        for _ in 0..num_els {
+            // the values do not matter.
+            bitpacker.write(0u64, bit_width, &mut buffer).unwrap();
+            bitpacker.flush(&mut buffer).unwrap();
         }
-        black_box(out);
-    });
-}
-
-fn bench_blocked_bitpacker() {
-    let mut runner = BenchRunner::new();
-    let mut blocked_bitpacker = BlockedBitpacker::new();
-    for val in 0..=21500 {
-        blocked_bitpacker.add(val * val);
+        buffer
     }
-    runner.bench_function("blockedbitp_read", move |_| {
-        let mut out = 0u64;
-        for val in 0..=21500 {
-            out = out.wrapping_add(blocked_bitpacker.get(val));
-        }
-        black_box(out);
-    });
-    runner.bench_function("blockedbitp_create", |_| {
+
+    #[bench]
+    fn bench_bitpacking_read(b: &mut Bencher) {
+        let bit_width = 3;
+        let num_els = 1_000_000u32;
+        let bit_unpacker = BitUnpacker::new(bit_width);
+        let data = create_bitpacked_data(bit_width, num_els);
+        let idxs: Vec<u32> = (0..num_els).choose_multiple(&mut rng(), 100_000);
+        b.iter(|| {
+            let mut out = 0u64;
+            for &idx in &idxs {
+                out = out.wrapping_add(bit_unpacker.get(idx, &data[..]));
+            }
+            out
+        });
+    }
+
+    #[bench]
+    fn bench_blockedbitp_read(b: &mut Bencher) {
         let mut blocked_bitpacker = BlockedBitpacker::new();
         for val in 0..=21500 {
             blocked_bitpacker.add(val * val);
         }
-        black_box(blocked_bitpacker);
-    });
-}
-
-fn bench_filter_vec() {
-    let mut runner = BenchRunner::new();
-
-    let (unpacker, data) = create_packed_data();
-    let positions = RefCell::new(Vec::with_capacity(N));
-    runner.bench_function("filter_vec_dense", move |_| {
-        unpacker.get_ids_for_value_range(
-            250..=750,
-            0..N as u32,
-            &data,
-            &mut positions.borrow_mut(),
-        );
-        black_box(positions.borrow().len());
-    });
-
-    let (unpacker, data) = create_packed_data();
-    let positions = RefCell::new(Vec::with_capacity(N));
-    runner.bench_function("filter_vec_sparse", move |_| {
-        unpacker.get_ids_for_value_range(0..=50, 0..N as u32, &data, &mut positions.borrow_mut());
-        black_box(positions.borrow().len());
-    });
-
-    let (unpacker, data) = create_packed_data();
-    let positions = RefCell::new(Vec::with_capacity(N));
-    runner.bench_function("filter_vec_full", move |_| {
-        unpacker.get_ids_for_value_range(
-            0..=MAX_VAL,
-            0..N as u32,
-            &data,
-            &mut positions.borrow_mut(),
-        );
-        black_box(positions.borrow().len());
-    });
-}
-
-fn main() {
-    bench_bitpacking();
-    bench_blocked_bitpacker();
-    bench_filter_vec();
+        b.iter(|| {
+            let mut out = 0u64;
+            for val in 0..=21500 {
+                out = out.wrapping_add(blocked_bitpacker.get(val));
+            }
+            out
+        });
+    }
+
+    #[bench]
+    fn bench_blockedbitp_create(b: &mut Bencher) {
+        b.iter(|| {
+            let mut blocked_bitpacker = BlockedBitpacker::new();
+            for val in 0..=21500 {
+                blocked_bitpacker.add(val * val);
+            }
+            blocked_bitpacker
+        });
+    }
 }

bitpacker/src/filter_vec/mod.rs

@@ -1,17 +1,8 @@
-#[cfg(all(target_arch = "aarch64", not(target_vendor = "apple")))]
-use std::arch::is_aarch64_feature_detected;
 use std::ops::RangeInclusive;
 
 #[cfg(target_arch = "x86_64")]
 mod avx2;
 
-#[cfg(target_arch = "aarch64")]
-mod neon;
-
-// SVE intrinsics are not exposed on aarch64-apple-darwin.
-#[cfg(all(target_arch = "aarch64", not(target_vendor = "apple")))]
-mod sve;
-
 mod scalar;
 
 #[derive(Clone, Copy, Eq, PartialEq, Debug)]
@@ -19,10 +10,6 @@ mod scalar;
 enum FilterImplPerInstructionSet {
     #[cfg(target_arch = "x86_64")]
     AVX2 = 0u8,
-    #[cfg(all(target_arch = "aarch64", not(target_vendor = "apple")))]
-    SVE = 3u8,
-    #[cfg(target_arch = "aarch64")]
-    Neon = 2u8,
     Scalar = 1u8,
 }
 
@@ -32,57 +19,29 @@ impl FilterImplPerInstructionSet {
         match *self {
             #[cfg(target_arch = "x86_64")]
             FilterImplPerInstructionSet::AVX2 => is_x86_feature_detected!("avx2"),
-            #[cfg(all(target_arch = "aarch64", not(target_vendor = "apple")))]
-            FilterImplPerInstructionSet::SVE => is_aarch64_feature_detected!("sve"),
-            // TIL Neon is required on aarch 64.
-            #[cfg(target_arch = "aarch64")]
-            FilterImplPerInstructionSet::Neon => true,
             FilterImplPerInstructionSet::Scalar => true,
         }
     }
 }
 
-// List of available implementations in preferred order.
+// List of available implementation in preferred order.
 #[cfg(target_arch = "x86_64")]
 const IMPLS: [FilterImplPerInstructionSet; 2] = [
     FilterImplPerInstructionSet::AVX2,
     FilterImplPerInstructionSet::Scalar,
 ];
 
-// Non-Apple aarch64: try SVE, NEON, Scalar.
-#[cfg(all(target_arch = "aarch64", not(target_vendor = "apple")))]
-const IMPLS: [FilterImplPerInstructionSet; 3] = [
-    FilterImplPerInstructionSet::SVE,
-    FilterImplPerInstructionSet::Neon,
-    FilterImplPerInstructionSet::Scalar,
-];
-
-// Apple aarch64 (M-series): SVE not available; use NEON or Scalar.
-#[cfg(all(target_arch = "aarch64", target_vendor = "apple"))]
-const IMPLS: [FilterImplPerInstructionSet; 2] = [
-    FilterImplPerInstructionSet::Neon,
-    FilterImplPerInstructionSet::Scalar,
-];
-
-#[cfg(not(any(target_arch = "x86_64", target_arch = "aarch64")))]
+#[cfg(not(target_arch = "x86_64"))]
 const IMPLS: [FilterImplPerInstructionSet; 1] = [FilterImplPerInstructionSet::Scalar];
 
 impl FilterImplPerInstructionSet {
     #[inline]
-    #[allow(unused_variables)]
+    #[allow(unused_variables)] // on non-x86_64, code is unused.
     fn from(code: u8) -> FilterImplPerInstructionSet {
         #[cfg(target_arch = "x86_64")]
         if code == FilterImplPerInstructionSet::AVX2 as u8 {
             return FilterImplPerInstructionSet::AVX2;
         }
-        #[cfg(all(target_arch = "aarch64", not(target_vendor = "apple")))]
-        if code == FilterImplPerInstructionSet::SVE as u8 {
-            return FilterImplPerInstructionSet::SVE;
-        }
-        #[cfg(target_arch = "aarch64")]
-        if code == FilterImplPerInstructionSet::Neon as u8 {
-            return FilterImplPerInstructionSet::Neon;
-        }
         FilterImplPerInstructionSet::Scalar
     }
 
@@ -91,13 +50,6 @@ impl FilterImplPerInstructionSet {
         match self {
             #[cfg(target_arch = "x86_64")]
             FilterImplPerInstructionSet::AVX2 => avx2::filter_vec_in_place(range, offset, output),
-            #[cfg(all(target_arch = "aarch64", not(target_vendor = "apple")))]
-            // SAFETY: SVE availability was verified by is_available() before selecting this impl.
-            FilterImplPerInstructionSet::SVE => unsafe {
-                sve::filter_vec_in_place(range, offset, output)
-            },
-            #[cfg(target_arch = "aarch64")]
-            FilterImplPerInstructionSet::Neon => neon::filter_vec_in_place(range, offset, output),
             FilterImplPerInstructionSet::Scalar => {
                 scalar::filter_vec_in_place(range, offset, output)
             }
@@ -105,12 +57,6 @@ impl FilterImplPerInstructionSet {
     }
 }
 
-fn available_impls() -> impl Iterator<Item = FilterImplPerInstructionSet> {
-    IMPLS
-        .into_iter()
-        .filter(FilterImplPerInstructionSet::is_available)
-}
-
 #[inline]
 fn get_best_available_instruction_set() -> FilterImplPerInstructionSet {
     use std::sync::atomic::{AtomicU8, Ordering};
@@ -118,7 +64,10 @@ fn get_best_available_instruction_set() -> FilterImplPerInstructionSet {
     let instruction_set_byte: u8 = INSTRUCTION_SET_BYTE.load(Ordering::Relaxed);
     if instruction_set_byte == u8::MAX {
         // Let's initialize the instruction set and cache it.
-        let instruction_set = available_impls().next().unwrap();
+        let instruction_set = IMPLS
+            .into_iter()
+            .find(FilterImplPerInstructionSet::is_available)
+            .unwrap();
         INSTRUCTION_SET_BYTE.store(instruction_set as u8, Ordering::Relaxed);
         return instruction_set;
     }
@@ -131,12 +80,12 @@ pub fn filter_vec_in_place(range: RangeInclusive<u32>, offset: u32, output: &mut
 
 #[cfg(test)]
 mod tests {
-    use proptest::strategy::Strategy;
-
     use super::*;
 
     #[test]
     fn test_get_best_available_instruction_set() {
+        // This does not test much unfortunately.
+        // We just make sure the function returns without crashing and returns the same result.
         let instruction_set = get_best_available_instruction_set();
         assert_eq!(get_best_available_instruction_set(), instruction_set);
     }
@@ -153,31 +102,6 @@ mod tests {
         }
     }
 
-    #[cfg(all(target_arch = "aarch64", not(target_vendor = "apple")))]
-    #[test]
-    fn test_instruction_set_to_code_from_code() {
-        for instruction_set in [
-            FilterImplPerInstructionSet::SVE,
-            FilterImplPerInstructionSet::Neon,
-            FilterImplPerInstructionSet::Scalar,
-        ] {
-            let code = instruction_set as u8;
-            assert_eq!(instruction_set, FilterImplPerInstructionSet::from(code));
-        }
-    }
-
-    #[cfg(all(target_arch = "aarch64", target_vendor = "apple"))]
-    #[test]
-    fn test_instruction_set_to_code_from_code() {
-        for instruction_set in [
-            FilterImplPerInstructionSet::Neon,
-            FilterImplPerInstructionSet::Scalar,
-        ] {
-            let code = instruction_set as u8;
-            assert_eq!(instruction_set, FilterImplPerInstructionSet::from(code));
-        }
-    }
-
     fn test_filter_impl_empty_aux(filter_impl: FilterImplPerInstructionSet) {
         let mut output = vec![];
         filter_impl.filter_vec_in_place(0..=u32::MAX, 0, &mut output);
@@ -202,20 +126,11 @@ mod tests {
         assert_eq!(&output, &[1, 3, 4, 5, 6, 7, 8]);
     }
 
-    fn test_filter_impl_empty_range_aux(filter_impl: FilterImplPerInstructionSet) {
-        // start > end: RangeInclusive::contains always returns false; output must be empty.
-        // The SVE path's wrapping_sub would otherwise produce a huge range_width.
-        let mut output = vec![3, 2, 1, 5, 11, 2, 5, 10, 2];
-        filter_impl.filter_vec_in_place(10..=5, 0, &mut output);
-        assert_eq!(&output, &[]);
-    }
-
     fn test_filter_impl_test_suite(filter_impl: FilterImplPerInstructionSet) {
         test_filter_impl_empty_aux(filter_impl);
         test_filter_impl_simple_aux(filter_impl);
         test_filter_impl_simple_aux_shifted(filter_impl);
         test_filter_impl_simple_outside_i32_range(filter_impl);
-        test_filter_impl_empty_range_aux(filter_impl);
     }
 
     #[test]
@@ -226,60 +141,25 @@ mod tests {
         }
     }
 
-    #[test]
-    #[cfg(all(target_arch = "aarch64", not(target_vendor = "apple")))]
-    fn test_filter_implementation_sve() {
-        if FilterImplPerInstructionSet::SVE.is_available() {
-            test_filter_impl_test_suite(FilterImplPerInstructionSet::SVE);
-        }
-    }
-
-    #[test]
-    #[cfg(target_arch = "aarch64")]
-    fn test_filter_implementation_neon() {
-        test_filter_impl_test_suite(FilterImplPerInstructionSet::Neon);
-    }
-
     #[test]
     fn test_filter_implementation_scalar() {
         test_filter_impl_test_suite(FilterImplPerInstructionSet::Scalar);
     }
 
-    fn max_val_strategy() -> impl proptest::strategy::Strategy<Value = u32> {
-        proptest::prop_oneof![
-            0u32..10u32,
-            255u32..258u32,
-            proptest::prelude::Just(1u32 << 25),
-            proptest::prelude::Just(u32::MAX - 1),
-            proptest::prelude::Just(u32::MAX),
-        ]
-    }
-
-    fn vals_strategy() -> impl proptest::strategy::Strategy<Value = Vec<u32>> {
-        proptest::prop_oneof![
-            proptest::collection::vec(proptest::prelude::any::<u32>(), 0..300),
-            max_val_strategy()
-                .prop_flat_map(|max_val| { proptest::collection::vec(0..=max_val, 0..300) })
-        ]
-    }
-
+    #[cfg(target_arch = "x86_64")]
     proptest::proptest! {
         #[test]
-        fn test_filter_compare_scalar_and_impls_impl_proptest(
-            start in 0u32..400u32,
-            end in 0u32..400u32,
+        fn test_filter_compare_scalar_and_avx2_impl_proptest(
+            start in proptest::prelude::any::<u32>(),
+            end in proptest::prelude::any::<u32>(),
             offset in 0u32..2u32,
-            vals in vals_strategy()) {
-                for implementation in available_impls() {
-                    if implementation == FilterImplPerInstructionSet::Scalar {
-                        continue;
-                    }
-                    let mut impl_output = vals.clone();
-                    let mut scalar_output = vals.clone();
-                    implementation.filter_vec_in_place(start..=end, offset, &mut impl_output);
-                    FilterImplPerInstructionSet::Scalar.filter_vec_in_place(start..=end, offset, &mut scalar_output);
-                    assert_eq!(&impl_output, &scalar_output);
-                }
+            mut vals in proptest::collection::vec(0..u32::MAX, 0..30)) {
+            if FilterImplPerInstructionSet::AVX2.is_available() {
+                let mut vals_clone = vals.clone();
+                FilterImplPerInstructionSet::AVX2.filter_vec_in_place(start..=end, offset, &mut vals);
+                FilterImplPerInstructionSet::Scalar.filter_vec_in_place(start..=end, offset, &mut vals_clone);
+                assert_eq!(&vals, &vals_clone);
+            }
        }
     }
 }

bitpacker/src/filter_vec/neon.rs

@@ -1,118 +0,0 @@
-use std::arch::aarch64::*;
-use std::ops::RangeInclusive;
-
-const NUM_LANES: usize = 4;
-
-// Compacts matching lanes to the front using a byte-level shuffle.
-// `mask` is a 4-bit value: bit k=1 means lane k should appear in the output.
-#[inline]
-#[target_feature(enable = "neon")]
-unsafe fn compact(data: uint32x4_t, mask: u8) -> uint32x4_t {
-    unsafe {
-        // SAFETY: mask is always in [0, 15] by construction (max sum of [1,2,4,8]).
-        // BYTE_SHUFFLE_TABLE has 16 entries, so this is always in bounds.
-        let shuffle = BYTE_SHUFFLE_TABLE.get_unchecked(mask as usize);
-        let shuffle_vec = vld1q_u8(shuffle.as_ptr());
-        vreinterpretq_u32_u8(vqtbl1q_u8(vreinterpretq_u8_u32(data), shuffle_vec))
-    }
-}
-
-// Safe (not unsafe) because NEON is mandatory on aarch64: no runtime feature check needed.
-#[inline(never)]
-pub fn filter_vec_in_place(range: RangeInclusive<u32>, offset: u32, output: &mut Vec<u32>) {
-    let num_words = output.len() / NUM_LANES;
-    let mut output_len = unsafe {
-        filter_vec_neon_aux(
-            output.as_ptr(),
-            range.clone(),
-            output.as_mut_ptr(),
-            offset,
-            num_words,
-        )
-    };
-    let remainder_start = num_words * NUM_LANES;
-    for i in remainder_start..output.len() {
-        let val = output[i];
-        output[output_len] = offset + i as u32;
-        output_len += if range.contains(&val) { 1 } else { 0 };
-    }
-    output.truncate(output_len);
-}
-
-#[target_feature(enable = "neon")]
-unsafe fn filter_vec_neon_aux(
-    input: *const u32,
-    range: RangeInclusive<u32>,
-    output: *mut u32,
-    offset: u32,
-    num_words: usize,
-) -> usize {
-    unsafe {
-        let mut input = input;
-        let mut output_tail = output;
-        let range_start_simd = vdupq_n_u32(*range.start());
-        let range_end_simd = vdupq_n_u32(*range.end());
-        let mut ids = vld1q_u32([offset, offset + 1, offset + 2, offset + 3].as_ptr());
-        let shift = vdupq_n_u32(NUM_LANES as u32);
-        let bit_weights = vld1q_u32([1u32, 2, 4, 8].as_ptr());
-
-        for _ in 0..num_words {
-            let word = vld1q_u32(input);
-
-            // Unsigned compares: CMHS (compare higher or same) tests `word >= start`
-            // and `end >= word`. ANDing both gives the inside-range mask directly,
-            // which is cheaper than computing `outside` and then negating.
-            let ge_start = vcgeq_u32(word, range_start_simd);
-            let le_end = vcleq_u32(word, range_end_simd);
-            // inside[k] = 0xFFFFFFFF if val[k] is in range, 0 otherwise.
-            let inside = vandq_u32(ge_start, le_end);
-
-            // Build the 4-bit mask: AND bit_weights with the inside lane mask, so each
-            // inside lane contributes its bit_weight (1, 2, 4, or 8). Summing yields the
-            // 4-bit mask in one addv.
-            let inside_bits = vandq_u32(bit_weights, inside);
-            let mask = vaddvq_u32(inside_bits) as u8;
-            // mask is mathematically bounded: max value is 1+2+4+8=15 (all lanes match)
-            debug_assert!(mask <= 15, "mask must fit in 4 bits: {}", mask);
-
-            // Count of matching lanes = popcount(mask). Derives the count directly from
-            // the mask instead of running a parallel SIMD reduction over `outside`.
-            let added_len = mask.count_ones() as usize;
-
-            // Safe because mask is guaranteed to be in [0, 15]
-            let filtered_ids = compact(ids, mask);
-            vst1q_u32(output_tail, filtered_ids);
-            output_tail = output_tail.add(added_len);
-            ids = vaddq_u32(ids, shift);
-            input = input.add(NUM_LANES);
-        }
-
-        output_tail.offset_from(output) as usize
-    }
-}
-
-// Byte shuffle patterns to compact matching lanes to the front of the vector.
-// Index is a 4-bit mask: bit k=1 means lane k (bytes 4k..4k+3) is in-range.
-// The j-th set bit determines which input lane goes to output position j.
-const BYTE_SHUFFLE_TABLE: [[u8; 16]; 16] = [
-    [
-        16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16,
-    ], // 0b0000: none
-    [0, 1, 2, 3, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16], // 0b0001: lane 0
-    [4, 5, 6, 7, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16], // 0b0010: lane 1
-    [0, 1, 2, 3, 4, 5, 6, 7, 16, 16, 16, 16, 16, 16, 16, 16],     // 0b0011: lanes 0,1
-    [8, 9, 10, 11, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16], // 0b0100: lane 2
-    [0, 1, 2, 3, 8, 9, 10, 11, 16, 16, 16, 16, 16, 16, 16, 16],   // 0b0101: lanes 0,2
-    [4, 5, 6, 7, 8, 9, 10, 11, 16, 16, 16, 16, 16, 16, 16, 16],   // 0b0110: lanes 1,2
-    [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 16, 16, 16, 16],       // 0b0111: lanes 0,1,2
-    [
-        12, 13, 14, 15, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16,
-    ], // 0b1000: lane 3
-    [0, 1, 2, 3, 12, 13, 14, 15, 16, 16, 16, 16, 16, 16, 16, 16], // 0b1001: lanes 0,3
-    [4, 5, 6, 7, 12, 13, 14, 15, 16, 16, 16, 16, 16, 16, 16, 16], // 0b1010: lanes 1,3
-    [0, 1, 2, 3, 4, 5, 6, 7, 12, 13, 14, 15, 16, 16, 16, 16],     // 0b1011: lanes 0,1,3
-    [8, 9, 10, 11, 12, 13, 14, 15, 16, 16, 16, 16, 16, 16, 16, 16], // 0b1100: lanes 2,3
-    [0, 1, 2, 3, 8, 9, 10, 11, 12, 13, 14, 15, 16, 16, 16, 16],   // 0b1101: lanes 0,2,3
-    [4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 16, 16, 16],   // 0b1110: lanes 1,2,3
-    [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15],       // 0b1111: all lanes
-];

bitpacker/src/filter_vec/sve.rs

@@ -1,260 +0,0 @@
-use std::ops::RangeInclusive;
-
-// SVE vector length (in u32 lanes) is not a compile-time constant; query at runtime.
-// Safe to call only when SVE is confirmed available via is_aarch64_feature_detected!("sve").
-#[target_feature(enable = "sve")]
-unsafe fn num_lanes() -> usize {
-    let vl: usize;
-    unsafe {
-        core::arch::asm!(
-            "cntw {vl}",
-            vl = out(reg) vl,
-            options(nostack, nomem, preserves_flags),
-        );
-    }
-    vl
-}
-
-// SAFETY: caller must ensure SVE is available (checked via is_aarch64_feature_detected!("sve")).
-// Unlike NEON, SVE is optional on aarch64 and not guaranteed by the target architecture.
-pub unsafe fn filter_vec_in_place(range: RangeInclusive<u32>, offset: u32, output: &mut Vec<u32>) {
-    if range.start() > range.end() {
-        output.clear();
-        return;
-    }
-    let vl = unsafe { num_lanes() };
-    let num_words = output.len() / vl;
-    let range_start = *range.start();
-    // Unsigned subtraction trick: val ∈ [lo, hi] ↔ (val - lo) ≤ᵤ (hi - lo).
-    // Values below lo wrap around to large u32, so the single unsigned ≤ excludes them.
-    let range_width = range.end().wrapping_sub(range_start);
-    let mut output_len = unsafe {
-        filter_vec_sve_aux(
-            output.as_ptr(),
-            range_start,
-            range_width,
-            output.as_mut_ptr(),
-            offset,
-            num_words,
-            vl,
-        )
-    };
-    let remainder_start = num_words * vl;
-    for i in remainder_start..output.len() {
-        let val = output[i];
-        output[output_len] = offset + i as u32;
-        output_len += if range.contains(&val) { 1 } else { 0 };
-    }
-    output.truncate(output_len);
-}
-
-// Register allocation for the asm! blocks:
-//   z0        ids_a (index vector for first half of each pair, advances by step2 each iter)
-//   z1        range_width broadcast
-//   z2        range_start broadcast
-//   z3        step2 broadcast (2 * vl)
-//   z4        ids_b (index vector for second half, = ids_a + step, advances by step2)
-//   z5        scratch: loaded word_a, then compacted_a
-//   z6        scratch: loaded word_b, then compacted_b
-//   p0        all-true predicate (ptrue p0.s)
-//   p1        in-range mask for word_a
-//   p2        in-range mask for word_b
-#[target_feature(enable = "sve")]
-unsafe fn filter_vec_sve_aux(
-    input: *const u32,
-    range_start: u32,
-    range_width: u32,
-    output: *mut u32,
-    offset: u32,
-    num_words: usize,
-    vl: usize,
-) -> usize {
-    let num_pairs = num_words / 2;
-    let mut input_ptr = input;
-    let mut output_tail = output;
-
-    if num_pairs > 0 {
-        unsafe {
-            // We rely on asm! because the SVE intrinsics are not available in stable Rust.
-            // The code that follows was generated by Rustc nightly based on the intrinsics version
-            // at the bottom of this file.
-            core::arch::asm!(
-                // --- Setup ---
-                // All-true predicate for 32-bit lanes.
-                "ptrue p0.s",
-                // ids_a = [offset, offset+1, offset+2, ...]
-                "index z0.s, {offset:w}, #1",
-                // Broadcast scalars into SVE vectors.
-                "mov z1.s, {range_width:w}",
-                "mov z2.s, {range_start:w}",
-                // vl_gpr = number of 32-bit lanes (cntw).
-                "cntw {vl_gpr}",
-                // step2_bytes will first hold 2*vl (for the step2 vector), then 2*VL in bytes.
-                "lsl {step2_bytes}, {vl_gpr}, #1",
-                // z4 = step = [vl, vl, ...]; will become ids_b after the add below.
-                "mov z4.s, {vl_gpr:w}",
-                // z3 = step2 = [2*vl, 2*vl, ...], used to advance both id vectors each iter.
-                "mov z3.s, {step2_bytes:w}",
-                // Repurpose step2_bytes to hold the byte stride for advancing the input pointer
-                // by two full SVE vectors per iteration.
-                "rdvl {step2_bytes}, #2",
-                // ids_b = ids_a + step = [offset+vl, offset+vl+1, ...]
-                "add z4.s, z0.s, z4.s",
-
-                // --- Main loop: process two SVE vectors (ids_a and ids_b) per iteration ---
-                "0:",
-                // Load two consecutive SVE vectors from input.
-                "ld1w {{z5.s}}, p0/z, [{input}]",
-                "ld1w {{z6.s}}, p0/z, [{input}, #1, mul vl]",
-                // Advance input pointer by 2 * VL bytes.
-                "add {input}, {input}, {step2_bytes}",
-                // Unsigned shift: subtract range_start so in-range check becomes a single cmpu ≤.
-                "sub z5.s, z5.s, z2.s",
-                "sub z6.s, z6.s, z2.s",
-                // in_range: shifted value ≤ range_width  (unsigned, so values below lo also fail).
-                "cmphs p1.s, p0/z, z1.s, z5.s",
-                "cmphs p2.s, p0/z, z1.s, z6.s",
-                // Count matching lanes; both cntp calls have independent inputs for OOO parallelism.
-                "cntp {cnt_a}, p0, p1.s",
-                "compact z5.s, p1, z0.s",
-                "compact z6.s, p2, z4.s",
-                "cntp {cnt_b}, p0, p2.s",
-                // Advance id vectors for the next iteration.
-                "add z0.s, z0.s, z3.s",
-                "add z4.s, z4.s, z3.s",
-                // Store compacted ids. Only the first cnt_a / cnt_b slots are valid; the rest
-                // will be overwritten by subsequent iterations before the final truncate.
-                "str z5, [{out}]",
-                "st1w {{z6.s}}, p0, [{out}, {cnt_a}, lsl #2]",
-                "add {out}, {out}, {cnt_a}, lsl #2",
-                "add {out}, {out}, {cnt_b}, lsl #2",
-                "subs {pairs}, {pairs}, #1",
-                "b.ne 0b",
-
-                // --- Operands ---
-                input       = inout(reg) input_ptr,
-                out         = inout(reg) output_tail,
-                pairs       = inout(reg) num_pairs => _,
-                offset      = in(reg) offset,
-                range_start = in(reg) range_start,
-                range_width = in(reg) range_width,
-                vl_gpr      = out(reg) _,
-                step2_bytes = out(reg) _,
-                cnt_a       = out(reg) _,
-                cnt_b       = out(reg) _,
-                out("p0") _, out("p1") _, out("p2") _,
-                out("v0") _, out("v1") _, out("v2") _, out("v3") _,
-                out("v4") _, out("v5") _, out("v6") _,
-                options(nostack),
-            );
-        }
-    }
-
-    // Handle an odd trailing vector.
-    if num_words % 2 == 1 {
-        // ids_a for the odd word starts at offset + num_pairs * 2 * vl.
-        // input_ptr was advanced by the main loop and now points at the odd word.
-        let odd_offset =
-            offset.wrapping_add((num_pairs as u32).wrapping_mul(2).wrapping_mul(vl as u32));
-        unsafe {
-            core::arch::asm!(
-                "ptrue p0.s",
-                "index z0.s, {odd_offset:w}, #1",
-                "mov z1.s, {range_width:w}",
-                "mov z2.s, {range_start:w}",
-                "ld1w {{z3.s}}, p0/z, [{input}]",
-                "sub z3.s, z3.s, z2.s",
-                "cmphs p1.s, p0/z, z1.s, z3.s",
-                "cntp {cnt}, p0, p1.s",
-                "compact z0.s, p1, z0.s",
-                "str z0, [{out}]",
-                "add {out}, {out}, {cnt}, lsl #2",
-                odd_offset  = in(reg) odd_offset,
-                range_width = in(reg) range_width,
-                range_start = in(reg) range_start,
-                input       = in(reg) input_ptr,
-                out         = inout(reg) output_tail,
-                cnt         = out(reg) _,
-                out("p0") _, out("p1") _,
-                out("v0") _, out("v1") _, out("v2") _, out("v3") _,
-                options(nostack),
-            );
-        }
-    }
-
-    unsafe { output_tail.offset_from(output) as usize }
-}
-
-// SVE implements with intrinsics.
-//
-// #[target_feature(enable = "sve")]
-// unsafe fn filter_vec_sve_aux(
-//     input: *const u32,
-//     range_start: u32,
-//     range_width: u32,
-//     output: *mut u32,
-//     offset: u32,
-//     num_words: usize,
-//     vl: usize,
-// ) -> usize {
-//     unsafe {
-//         let all_true = svptrue_b32();
-//         let range_start_simd = svdup_n_u32(range_start);
-//         let range_width_simd = svdup_n_u32(range_width);
-//         // ids_a covers [offset .. offset+vl), ids_b covers the next vl ids.
-//         // Keeping them separate breaks the loop-carried dependency through ids so
-//         // both compact/cntp chains are fully independent within each unrolled body.
-//         let mut ids_a = svindex_u32(offset, 1);
-//         let step = svdup_n_u32(vl as u32);
-//         let step2 = svdup_n_u32(2 * vl as u32);
-//         let mut ids_b = svadd_u32_x(all_true, ids_a, step);
-
-//         let mut input = input;
-//         let mut output_tail = output;
-
-//         // Unrolled ×2: both cntp calls have independent inputs and execute in parallel.
-//         // The two output_tail updates are sequential but together cost 4+1+1=6 cy per
-//         // pair vs 5+5=10 cy for two scalar iterations, breaking the cntp latency chain.
-//         let num_pairs = num_words / 2;
-//         for _ in 0..num_pairs {
-//             let word_a = svld1_u32(all_true, input);
-//             let word_b = svld1_u32(all_true, input.add(vl));
-
-//             let shifted_a = svsub_u32_x(all_true, word_a, range_start_simd);
-//             let shifted_b = svsub_u32_x(all_true, word_b, range_start_simd);
-
-//             let in_range_a = svcmple_u32(all_true, shifted_a, range_width_simd);
-//             let in_range_b = svcmple_u32(all_true, shifted_b, range_width_simd);
-
-//             let compacted_a = svcompact_u32(in_range_a, ids_a);
-//             let compacted_b = svcompact_u32(in_range_b, ids_b);
-//             // cntp_a and cntp_b have independent inputs: OOO engine issues them in parallel.
-//             let added_len_a = svcntp_b32(all_true, in_range_a) as usize;
-//             let added_len_b = svcntp_b32(all_true, in_range_b) as usize;
-
-//             // Write the full vector — only the first added_len slots are valid.
-//             // Subsequent iterations overwrite the trailing zeros before truncate.
-//             svst1_u32(all_true, output_tail, compacted_a);
-//             output_tail = output_tail.add(added_len_a);
-//             svst1_u32(all_true, output_tail, compacted_b);
-//             output_tail = output_tail.add(added_len_b);
-
-//             ids_a = svadd_u32_x(all_true, ids_a, step2);
-//             ids_b = svadd_u32_x(all_true, ids_b, step2);
-//             input = input.add(2 * vl);
-//         }
-
-//         // Handle an odd trailing word.
-//         if num_words % 2 == 1 {
-//             let word = svld1_u32(all_true, input);
-//             let shifted = svsub_u32_x(all_true, word, range_start_simd);
-//             let in_range = svcmple_u32(all_true, shifted, range_width_simd);
-//             let added_len = svcntp_b32(all_true, in_range) as usize;
-//             let compacted_ids = svcompact_u32(in_range, ids_a);
-//             svst1_u32(all_true, output_tail, compacted_ids);
-//             output_tail = output_tail.add(added_len);
-//         }
-
-//         output_tail.offset_from(output) as usize
-//     }
-// }

common/src/bitset.rs

@@ -196,11 +196,13 @@ impl TinySet {
 #[derive(Clone)]
 pub struct BitSet {
     tinysets: Box<[TinySet]>,
+    len: u64,
     max_value: u32,
 }
 impl std::fmt::Debug for BitSet {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         f.debug_struct("BitSet")
+            .field("len", &self.len)
             .field("max_value", &self.max_value)
             .finish()
     }
@@ -228,6 +230,7 @@ impl BitSet {
         let tinybitsets = vec![TinySet::empty(); num_buckets as usize].into_boxed_slice();
         BitSet {
             tinysets: tinybitsets,
+            len: 0,
             max_value,
         }
     }
@@ -245,6 +248,7 @@ impl BitSet {
         }
         BitSet {
             tinysets: tinybitsets,
+            len: max_value as u64,
             max_value,
         }
     }
@@ -263,19 +267,17 @@ impl BitSet {
 
     /// Intersect with tinysets
     fn intersect_update_with_iter(&mut self, other: impl Iterator<Item = TinySet>) {
+        self.len = 0;
         for (left, right) in self.tinysets.iter_mut().zip(other) {
             *left = left.intersect(right);
+            self.len += left.len() as u64;
         }
     }
 
     /// Returns the number of elements in the `BitSet`.
     #[inline]
     pub fn len(&self) -> usize {
-        self.tinysets
-            .iter()
-            .copied()
-            .map(|tinyset| tinyset.len())
-            .sum::<u32>() as usize
+        self.len as usize
     }
 
     /// Inserts an element in the `BitSet`
@@ -284,7 +286,7 @@ impl BitSet {
         // we do not check saturated els.
         let higher = el / 64u32;
         let lower = el % 64u32;
-        self.tinysets[higher as usize].insert_mut(lower);
+        self.len += u64::from(self.tinysets[higher as usize].insert_mut(lower));
     }
 
     /// Inserts an element in the `BitSet`
@@ -293,7 +295,7 @@ impl BitSet {
         // we do not check saturated els.
         let higher = el / 64u32;
         let lower = el % 64u32;
-        self.tinysets[higher as usize].remove_mut(lower);
+        self.len -= u64::from(self.tinysets[higher as usize].remove_mut(lower));
     }
 
     /// Returns true iff the elements is in the `BitSet`.
@@ -315,9 +317,6 @@ impl BitSet {
             .map(|delta_bucket| bucket + delta_bucket as u32)
     }
 
-    /// Returns the maximum number of elements in the bitset.
-    ///
-    /// Warning: The largest element the bitset can contain is `max_value - 1`.
     #[inline]
     pub fn max_value(&self) -> u32 {
         self.max_value

common/src/file_slice.rs

@@ -121,7 +121,7 @@ pub struct FileSlice {
 
 impl fmt::Debug for FileSlice {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "FileSlice({:?}, {:?})", self.data, self.range)
+        write!(f, "FileSlice({:?}, {:?})", &self.data, self.range)
     }
 }
 

examples/iterating_docs_and_positions.rs

@@ -91,10 +91,46 @@ fn main() -> tantivy::Result<()> {
         }
     }
 
-    // Some other powerful operations (especially `.seek`) may be useful to consume these
+    // A `Term` is a text token associated with a field.
+    // Let's go through all docs containing the term `title:the` and access their position
+    let term_the = Term::from_field_text(title, "the");
+
+    // Some other powerful operations (especially `.skip_to`) may be useful to consume these
     // posting lists rapidly.
     // You can check for them in the [`DocSet`](https://docs.rs/tantivy/~0/tantivy/trait.DocSet.html) trait
     // and the [`Postings`](https://docs.rs/tantivy/~0/tantivy/trait.Postings.html) trait
 
+    // Also, for some VERY specific high performance use case like an OLAP analysis of logs,
+    // you can get better performance by accessing directly the blocks of doc ids.
+    for segment_reader in searcher.segment_readers() {
+        // A segment contains different data structure.
+        // Inverted index stands for the combination of
+        // - the term dictionary
+        // - the inverted lists associated with each terms and their positions
+        let inverted_index = segment_reader.inverted_index(title)?;
+
+        // This segment posting object is like a cursor over the documents matching the term.
+        // The `IndexRecordOption` arguments tells tantivy we will be interested in both term
+        // frequencies and positions.
+        //
+        // If you don't need all this information, you may get better performance by decompressing
+        // less information.
+        if let Some(mut block_segment_postings) =
+            inverted_index.read_block_postings(&term_the, IndexRecordOption::Basic)?
+        {
+            loop {
+                let docs = block_segment_postings.docs();
+                if docs.is_empty() {
+                    break;
+                }
+                // Once again these docs MAY contains deleted documents as well.
+                let docs = block_segment_postings.docs();
+                // Prints `Docs [0, 2].`
+                println!("Docs {docs:?}");
+                block_segment_postings.advance();
+            }
+        }
+    }
+
     Ok(())
 }

src/codec/mod.rs

@@ -1,237 +0,0 @@
-/// Codec specific to postings data.
-pub mod postings;
-
-/// Codec specific to positions data.
-pub mod positions;
-
-/// Standard tantivy codec. This is the codec you use by default.
-pub mod standard;
-
-use std::io;
-
-pub use standard::StandardCodec;
-
-use crate::codec::positions::PositionsCodec;
-use crate::codec::postings::PostingsCodec;
-use crate::fieldnorm::FieldNormReader;
-use crate::postings::{Postings, TermInfo};
-use crate::query::score_combiner::DoNothingCombiner;
-use crate::query::term_query::TermScorer;
-use crate::query::{box_scorer, Bm25Weight, BufferedUnionScorer, Scorer, SumCombiner};
-use crate::schema::IndexRecordOption;
-use crate::{DocId, InvertedIndexReader, Score};
-
-/// Codecs describes how data is layed out on disk.
-pub trait Codec: Clone + std::fmt::Debug + Send + Sync + 'static {
-    /// The specific postings codec used by this codec.
-    type PostingsCodec: PostingsCodec;
-
-    /// The specific positions codec used by this codec.
-    type PositionsCodec: PositionsCodec;
-
-    /// ID of the codec. It should be unique to your codec.
-    /// Make it human-readable, descriptive, short and unique.
-    const ID: &'static str;
-
-    /// Load codec based on the codec configuration.
-    fn from_json_props(json_value: &serde_json::Value) -> crate::Result<Self>;
-
-    /// Get codec configuration.
-    fn to_json_props(&self) -> serde_json::Value;
-
-    /// Returns the postings codec.
-    fn postings_codec(&self) -> &Self::PostingsCodec;
-
-    /// Returns the positions codec.
-    fn positions_codec(&self) -> &Self::PositionsCodec;
-}
-
-/// Object-safe codec is a Codec that can be used in a trait object.
-///
-/// The point of it is to offer a way to use a codec without a proliferation of generics.
-pub trait ObjectSafeCodec: 'static + Send + Sync {
-    /// Loads a type-erased Postings object for the given term.
-    ///
-    /// If the schema used to build the index did not provide enough
-    /// information to match the requested `option`, a Postings is still
-    /// returned in a best-effort manner.
-    fn load_postings_type_erased(
-        &self,
-        term_info: &TermInfo,
-        option: IndexRecordOption,
-        inverted_index_reader: &InvertedIndexReader,
-    ) -> io::Result<Box<dyn Postings>>;
-
-    /// Loads a type-erased TermScorer object for the given term.
-    ///
-    /// If the schema used to build the index did not provide enough
-    /// information to match the requested `option`, a TermScorer is still
-    /// returned in a best-effort manner.
-    ///
-    /// The point of this contraption is that the return TermScorer is backed,
-    /// not by Box<dyn Postings> but by the codec's concrete Postings type.
-    fn load_term_scorer_type_erased(
-        &self,
-        term_info: &TermInfo,
-        option: IndexRecordOption,
-        inverted_index_reader: &InvertedIndexReader,
-        fieldnorm_reader: FieldNormReader,
-        similarity_weight: Bm25Weight,
-    ) -> io::Result<Box<dyn Scorer>>;
-
-    /// Loads a type-erased PhraseScorer object for the given term.
-    ///
-    /// If the schema used to build the index did not provide enough
-    /// information to match the requested `option`, a TermScorer is still
-    /// returned in a best-effort manner.
-    ///
-    /// The point of this contraption is that the return PhraseScorer is backed,
-    /// not by Box<dyn Postings> but by the codec's concrete Postings type.
-    fn new_phrase_scorer_type_erased(
-        &self,
-        term_infos: &[(usize, TermInfo)],
-        similarity_weight: Option<Bm25Weight>,
-        fieldnorm_reader: FieldNormReader,
-        slop: u32,
-        inverted_index_reader: &InvertedIndexReader,
-    ) -> io::Result<Box<dyn Scorer>>;
-
-    /// Performs a for_each_pruning operation on the given scorer.
-    ///
-    /// The function will go through matching documents and call the callback
-    /// function for all docs with a score exceeding the threshold.
-    ///
-    /// The function itself will return a larger threshold value,
-    /// meant to update the threshold value.
-    ///
-    /// If the codec and the scorer allow it, this function can rely on
-    /// optimizations like the block-max wand.
-    fn for_each_pruning(
-        &self,
-        threshold: Score,
-        scorer: Box<dyn Scorer>,
-        callback: &mut dyn FnMut(DocId, Score) -> Score,
-    );
-
-    /// Builds a union scorer possibly specialized if
-    /// all scorers are `Term<Self::Postings>`.
-    fn build_union_scorer_with_sum_combiner(
-        &self,
-        scorers: Vec<Box<dyn Scorer>>,
-        num_docs: DocId,
-        score_combiner_type: SumOrDoNothingCombiner,
-    ) -> Box<dyn Scorer>;
-}
-
-impl<TCodec: Codec> ObjectSafeCodec for TCodec {
-    fn load_postings_type_erased(
-        &self,
-        term_info: &TermInfo,
-        option: IndexRecordOption,
-        inverted_index_reader: &InvertedIndexReader,
-    ) -> io::Result<Box<dyn Postings>> {
-        let postings = inverted_index_reader
-            .read_postings_from_terminfo_specialized(term_info, option, self)?;
-        Ok(Box::new(postings))
-    }
-
-    fn load_term_scorer_type_erased(
-        &self,
-        term_info: &TermInfo,
-        option: IndexRecordOption,
-        inverted_index_reader: &InvertedIndexReader,
-        fieldnorm_reader: FieldNormReader,
-        similarity_weight: Bm25Weight,
-    ) -> io::Result<Box<dyn Scorer>> {
-        let scorer = inverted_index_reader.new_term_scorer_specialized(
-            term_info,
-            option,
-            fieldnorm_reader,
-            similarity_weight,
-            self,
-        )?;
-        Ok(box_scorer(scorer))
-    }
-
-    fn new_phrase_scorer_type_erased(
-        &self,
-        term_infos: &[(usize, TermInfo)],
-        similarity_weight: Option<Bm25Weight>,
-        fieldnorm_reader: FieldNormReader,
-        slop: u32,
-        inverted_index_reader: &InvertedIndexReader,
-    ) -> io::Result<Box<dyn Scorer>> {
-        let scorer = inverted_index_reader.new_phrase_scorer_type_specialized(
-            term_infos,
-            similarity_weight,
-            fieldnorm_reader,
-            slop,
-            self,
-        )?;
-        Ok(box_scorer(scorer))
-    }
-
-    fn build_union_scorer_with_sum_combiner(
-        &self,
-        scorers: Vec<Box<dyn Scorer>>,
-        num_docs: DocId,
-        sum_or_do_nothing_combiner: SumOrDoNothingCombiner,
-    ) -> Box<dyn Scorer> {
-        if !scorers.iter().all(|scorer| {
-            scorer.is::<TermScorer<<<Self as Codec>::PostingsCodec as PostingsCodec>::Postings>>()
-        }) {
-            return box_scorer(BufferedUnionScorer::build(
-                scorers,
-                SumCombiner::default,
-                num_docs,
-            ));
-        }
-        let specialized_scorers: Vec<
-            TermScorer<<<Self as Codec>::PostingsCodec as PostingsCodec>::Postings>,
-        > = scorers
-            .into_iter()
-            .map(|scorer| {
-                *scorer.downcast::<TermScorer<_>>().ok().expect(
-                    "Downcast failed despite the fact we already checked the type was correct",
-                )
-            })
-            .collect();
-        match sum_or_do_nothing_combiner {
-            SumOrDoNothingCombiner::Sum => box_scorer(BufferedUnionScorer::build(
-                specialized_scorers,
-                SumCombiner::default,
-                num_docs,
-            )),
-            SumOrDoNothingCombiner::DoNothing => box_scorer(BufferedUnionScorer::build(
-                specialized_scorers,
-                DoNothingCombiner::default,
-                num_docs,
-            )),
-        }
-    }
-
-    fn for_each_pruning(
-        &self,
-        threshold: Score,
-        scorer: Box<dyn Scorer>,
-        callback: &mut dyn FnMut(DocId, Score) -> Score,
-    ) {
-        let accerelerated_foreach_pruning_res =
-            <TCodec as Codec>::PostingsCodec::try_accelerated_for_each_pruning(
-                threshold, scorer, callback,
-            );
-        if let Err(mut scorer) = accerelerated_foreach_pruning_res {
-            // No acceleration available. We need to do things manually.
-            scorer.for_each_pruning(threshold, callback);
-        }
-    }
-}
-
-/// SumCombiner or DoNothingCombiner
-#[derive(Copy, Clone)]
-pub enum SumOrDoNothingCombiner {
-    /// Sum scores together
-    Sum,
-    /// Do not track any score.
-    DoNothing,
-}

src/codec/positions/mod.rs

@@ -1,49 +0,0 @@
-use std::io;
-
-use common::OwnedBytes;
-
-/// Codec for the positions file.
-pub trait PositionsCodec: Send + Sync + 'static {
-    /// The serializer type created by this codec.
-    type Serializer<W: io::Write>: PositionsSerializer<W>;
-    /// The reader type created by this codec.
-    type Reader: PositionsReader;
-
-    /// Creates a new positions serializer writing into `writer`.
-    fn new_serializer<W: io::Write>(&self, writer: W) -> Self::Serializer<W>;
-
-    /// Opens a positions reader from the given raw byte slice.
-    fn open_reader(&self, data: OwnedBytes) -> io::Result<Self::Reader>;
-}
-
-/// Serializes delta-encoded positions for all terms in a field.
-///
-/// A single serializer is reused across all terms. Clients must call
-/// `close_term` after each term, then `close` once when the field is done.
-pub trait PositionsSerializer<W: io::Write> {
-    /// Returns the total number of bytes written since this serializer was created.
-    fn written_bytes(&self) -> u64;
-
-    /// Appends delta-encoded positions for the current document.
-    fn write_positions_delta(&mut self, positions_delta: &[u32]);
-
-    /// Finalizes and flushes positions data for the current term.
-    fn close_term(&mut self) -> io::Result<()>;
-
-    /// Flushes the underlying writer. Must be called once after all terms are done.
-    fn close(self) -> io::Result<()>;
-}
-
-/// Reads delta-encoded positions from a byte slice.
-pub trait PositionsReader: Send + 'static {
-    /// Fills `output` with delta-encoded positions starting at `offset`.
-    ///
-    /// Hidden contract: offset values should be non-decreasing for best performance;
-    /// passing a lower offset resets internal state and incurs extra work.
-    fn read(&mut self, offset: u64, output: &mut [u32]);
-
-    /// Returns a heap-allocated clone of this reader.
-    ///
-    /// Needed to clone `SegmentPostings`, which owns a boxed reader.
-    fn clone_box(&self) -> Box<dyn PositionsReader>;
-}

src/codec/postings/mod.rs

@@ -1,139 +0,0 @@
-use std::io;
-
-/// Block-max WAND algorithm.
-pub mod block_wand;
-use common::OwnedBytes;
-
-use crate::codec::positions::PositionsReader;
-use crate::fieldnorm::FieldNormReader;
-use crate::postings::Postings;
-use crate::query::{Bm25Weight, Scorer};
-use crate::schema::IndexRecordOption;
-use crate::{DocId, Score};
-
-/// Postings codec.
-pub trait PostingsCodec: Send + Sync + 'static {
-    /// Serializer type for the postings codec.
-    type PostingsSerializer: PostingsSerializer;
-    /// Postings type for the postings codec.
-    type Postings: Postings + Clone;
-    /// Creates a new postings serializer.
-    fn new_serializer(
-        &self,
-        avg_fieldnorm: Score,
-        mode: IndexRecordOption,
-        fieldnorm_reader: Option<FieldNormReader>,
-    ) -> Self::PostingsSerializer;
-
-    /// Loads postings
-    ///
-    /// Record option is the option that was passed at indexing time.
-    /// Requested option is the option that is requested.
-    ///
-    /// For instance, we may have term_freq in the posting list
-    /// but we can skip decompressing as we read the posting list.
-    ///
-    /// If record option does not support the requested option,
-    /// this method does NOT return an error and will in fact restrict
-    /// requested_option to what is available.
-    ///
-    /// `position_reader` is `Some` iff `requested_option` includes positions.
-    /// It is already opened by the caller via the codec's `PositionsCodec`.
-    fn load_postings(
-        &self,
-        doc_freq: u32,
-        postings_data: OwnedBytes,
-        record_option: IndexRecordOption,
-        requested_option: IndexRecordOption,
-        position_reader: Option<Box<dyn PositionsReader>>,
-    ) -> io::Result<Self::Postings>;
-
-    /// If your codec supports different ways to accelerate `for_each_pruning` that's
-    /// where you should implement it.
-    ///
-    /// Returning `Err(scorer)` without mutating the scorer nor calling the callback function,
-    /// is never "wrong". It just leaves the responsability to the caller to call a fallback
-    /// implementation on the scorer.
-    ///
-    /// If your codec supports BlockMax-Wand, you just need to have your
-    /// postings implement `PostingsWithBlockMax` and copy what is done in the StandardPostings
-    /// codec to enable it.
-    fn try_accelerated_for_each_pruning(
-        _threshold: Score,
-        scorer: Box<dyn Scorer>,
-        _callback: &mut dyn FnMut(DocId, Score) -> Score,
-    ) -> Result<(), Box<dyn Scorer>> {
-        Err(scorer)
-    }
-}
-
-/// A postings serializer is a listener that is in charge of serializing postings
-///
-/// IO is done only once per postings, once all of the data has been received.
-/// A serializer will therefore contain internal buffers.
-///
-/// A serializer is created once and recycled for all postings.
-///
-/// Clients should use PostingsSerializer as follows.
-/// ```text
-/// // First postings list
-/// serializer.new_term(2, true);
-/// serializer.write_doc(2, 1);
-/// serializer.write_doc(6, 2);
-/// serializer.close_term(3, &mut wrt)?;
-/// // Second postings list
-/// serializer.new_term(1, true);
-/// serializer.write_doc(3, 1);
-/// serializer.close_term(1, &mut wrt)?;
-/// ```
-pub trait PostingsSerializer {
-    /// The term_doc_freq here is the number of documents
-    /// in the postings lists.
-    ///
-    /// It can be used to compute the idf that will be used for the
-    /// blockmax parameters.
-    ///
-    /// If not available (e.g. if we do not collect `term_frequencies`
-    /// blockwand is disabled), the term_doc_freq passed will be set 0.
-    fn new_term(&mut self, term_doc_freq: u32, record_term_freq: bool);
-
-    /// Codec-specific per-term payload.
-    ///
-    /// It is supplied right after `new_term` and before any `write_doc`, so the
-    /// codec can let it influence how the postings list is encoded.
-    ///
-    /// Hidden contract: `new_term` MUST reset any per-term payload state to its
-    /// default. This method is only called for terms that actually have a
-    /// payload registered, so a codec cannot rely on it being called for every
-    /// term.
-    ///
-    /// The default implementation ignores the payload.
-    fn set_term_payload(&mut self, _payload: &dyn std::any::Any) {}
-
-    /// Records a new document id for the current term.
-    /// The serializer may ignore it.
-    fn write_doc(&mut self, doc_id: DocId, term_freq: u32);
-
-    /// Closes the current term and writes the postings list associated.
-    fn close_term(&mut self, doc_freq: u32, wrt: &mut impl io::Write) -> io::Result<()>;
-}
-
-/// A light complement interface to Postings to allow block-max wand acceleration.
-pub trait PostingsWithBlockMax: Postings {
-    /// Moves the postings to the block containign `target_doc` and returns
-    /// an upperbound of the score for documents in the block.
-    ///
-    /// `Warning`: Calling this method may leave the postings in an invalid state.
-    /// callers are required to call seek before calling any other of the
-    /// `Postings` method (like doc / advance etc.).
-    fn seek_block_max(
-        &mut self,
-        target_doc: crate::DocId,
-        fieldnorm_reader: &FieldNormReader,
-        similarity_weight: &Bm25Weight,
-    ) -> Score;
-
-    /// Returns the last document in the current block (or Terminated if this
-    /// is the last block).
-    fn last_doc_in_block(&self) -> crate::DocId;
-}

src/codec/standard/mod.rs

@@ -1,44 +0,0 @@
-use serde::{Deserialize, Serialize};
-
-use crate::codec::standard::positions::StandardPositionsCodec;
-use crate::codec::standard::postings::StandardPostingsCodec;
-use crate::codec::Codec;
-
-/// Tantivy's default postings codec.
-pub mod postings;
-
-/// Tantivy's default positions codec.
-pub mod positions;
-
-/// Tantivy's default codec.
-#[derive(Debug, Default, Clone, Serialize, Deserialize)]
-pub struct StandardCodec;
-
-impl Codec for StandardCodec {
-    type PostingsCodec = StandardPostingsCodec;
-    type PositionsCodec = StandardPositionsCodec;
-
-    const ID: &'static str = "tantivy-default";
-
-    fn from_json_props(json_value: &serde_json::Value) -> crate::Result<Self> {
-        if !json_value.is_null() {
-            return Err(crate::TantivyError::InvalidArgument(format!(
-                "Codec property for the StandardCodec are unexpected. expected null, got {}",
-                json_value.as_str().unwrap_or("null")
-            )));
-        }
-        Ok(StandardCodec)
-    }
-
-    fn to_json_props(&self) -> serde_json::Value {
-        serde_json::Value::Null
-    }
-
-    fn postings_codec(&self) -> &Self::PostingsCodec {
-        &StandardPostingsCodec
-    }
-
-    fn positions_codec(&self) -> &Self::PositionsCodec {
-        &StandardPositionsCodec
-    }
-}

src/codec/standard/positions/mod.rs

@@ -1,50 +0,0 @@
-use std::io;
-
-use common::OwnedBytes;
-
-use crate::codec::positions::{PositionsCodec, PositionsReader, PositionsSerializer};
-use crate::positions::{PositionReader, PositionSerializer};
-
-/// The default positions codec for tantivy.
-pub struct StandardPositionsCodec;
-
-impl PositionsCodec for StandardPositionsCodec {
-    type Serializer<W: io::Write> = PositionSerializer<W>;
-    type Reader = PositionReader;
-
-    fn new_serializer<W: io::Write>(&self, writer: W) -> Self::Serializer<W> {
-        PositionSerializer::new(writer)
-    }
-
-    fn open_reader(&self, data: OwnedBytes) -> io::Result<Self::Reader> {
-        PositionReader::open(data)
-    }
-}
-
-impl<W: io::Write> PositionsSerializer<W> for PositionSerializer<W> {
-    fn written_bytes(&self) -> u64 {
-        PositionSerializer::written_bytes(self)
-    }
-
-    fn write_positions_delta(&mut self, positions_delta: &[u32]) {
-        PositionSerializer::write_positions_delta(self, positions_delta);
-    }
-
-    fn close_term(&mut self) -> io::Result<()> {
-        PositionSerializer::close_term(self)
-    }
-
-    fn close(self) -> io::Result<()> {
-        PositionSerializer::close(self)
-    }
-}
-
-impl PositionsReader for PositionReader {
-    fn read(&mut self, offset: u64, output: &mut [u32]) {
-        PositionReader::read(self, offset, output);
-    }
-
-    fn clone_box(&self) -> Box<dyn PositionsReader> {
-        Box::new(self.clone())
-    }
-}

src/codec/standard/postings/block.rs

@@ -1,50 +0,0 @@
-use crate::postings::compression::COMPRESSION_BLOCK_SIZE;
-use crate::DocId;
-
-pub struct Block {
-    doc_ids: [DocId; COMPRESSION_BLOCK_SIZE],
-    term_freqs: [u32; COMPRESSION_BLOCK_SIZE],
-    len: usize,
-}
-
-impl Block {
-    pub fn new() -> Self {
-        Block {
-            doc_ids: [0u32; COMPRESSION_BLOCK_SIZE],
-            term_freqs: [0u32; COMPRESSION_BLOCK_SIZE],
-            len: 0,
-        }
-    }
-
-    pub fn doc_ids(&self) -> &[DocId] {
-        &self.doc_ids[..self.len]
-    }
-
-    pub fn term_freqs(&self) -> &[u32] {
-        &self.term_freqs[..self.len]
-    }
-
-    pub fn clear(&mut self) {
-        self.len = 0;
-    }
-
-    pub fn append_doc(&mut self, doc: DocId, term_freq: u32) {
-        let len = self.len;
-        self.doc_ids[len] = doc;
-        self.term_freqs[len] = term_freq;
-        self.len = len + 1;
-    }
-
-    pub fn is_full(&self) -> bool {
-        self.len == COMPRESSION_BLOCK_SIZE
-    }
-
-    pub fn is_empty(&self) -> bool {
-        self.len == 0
-    }
-
-    pub fn last_doc(&self) -> DocId {
-        assert_eq!(self.len, COMPRESSION_BLOCK_SIZE);
-        self.doc_ids[COMPRESSION_BLOCK_SIZE - 1]
-    }
-}

src/codec/standard/postings/mod.rs

@@ -1,163 +0,0 @@
-use std::io;
-
-use crate::codec::positions::PositionsReader;
-use crate::codec::postings::block_wand::{block_wand, block_wand_single_scorer};
-use crate::codec::postings::PostingsCodec;
-use crate::codec::standard::postings::block_segment_postings::BlockSegmentPostings;
-pub use crate::codec::standard::postings::segment_postings::SegmentPostings;
-use crate::fieldnorm::FieldNormReader;
-use crate::query::term_query::TermScorer;
-use crate::query::{BufferedUnionScorer, Scorer, SumCombiner};
-use crate::schema::IndexRecordOption;
-use crate::{DocSet as _, Score, TERMINATED};
-
-mod block;
-mod block_segment_postings;
-mod segment_postings;
-mod skip;
-mod standard_postings_serializer;
-
-pub use segment_postings::SegmentPostings as StandardPostings;
-pub use standard_postings_serializer::StandardPostingsSerializer;
-
-/// The default postings codec for tantivy.
-pub struct StandardPostingsCodec;
-
-#[expect(clippy::enum_variant_names)]
-#[derive(Debug, PartialEq, Clone, Copy, Eq)]
-pub(crate) enum FreqReadingOption {
-    NoFreq,
-    SkipFreq,
-    ReadFreq,
-}
-
-impl PostingsCodec for StandardPostingsCodec {
-    type PostingsSerializer = StandardPostingsSerializer;
-    type Postings = SegmentPostings;
-
-    fn new_serializer(
-        &self,
-        avg_fieldnorm: Score,
-        mode: IndexRecordOption,
-        fieldnorm_reader: Option<FieldNormReader>,
-    ) -> Self::PostingsSerializer {
-        StandardPostingsSerializer::new(avg_fieldnorm, mode, fieldnorm_reader)
-    }
-
-    fn load_postings(
-        &self,
-        doc_freq: u32,
-        postings_data: common::OwnedBytes,
-        record_option: IndexRecordOption,
-        requested_option: IndexRecordOption,
-        position_reader: Option<Box<dyn PositionsReader>>,
-    ) -> io::Result<Self::Postings> {
-        // Rationalize record_option/requested_option.
-        let requested_option = requested_option.downgrade(record_option);
-        let block_segment_postings =
-            BlockSegmentPostings::open(doc_freq, postings_data, record_option, requested_option)?;
-        Ok(SegmentPostings::from_block_postings(
-            block_segment_postings,
-            position_reader,
-        ))
-    }
-
-    fn try_accelerated_for_each_pruning(
-        mut threshold: Score,
-        mut scorer: Box<dyn Scorer>,
-        callback: &mut dyn FnMut(crate::DocId, Score) -> Score,
-    ) -> Result<(), Box<dyn Scorer>> {
-        scorer = match scorer.downcast::<TermScorer<Self::Postings>>() {
-            Ok(term_scorer) => {
-                block_wand_single_scorer(*term_scorer, threshold, callback);
-                return Ok(());
-            }
-            Err(scorer) => scorer,
-        };
-        let mut union_scorer =
-            scorer.downcast::<BufferedUnionScorer<Box<dyn Scorer>, SumCombiner>>()?;
-        if !union_scorer
-            .scorers()
-            .iter()
-            .all(|scorer| scorer.is::<TermScorer<Self::Postings>>())
-        {
-            return Err(union_scorer);
-        }
-        let doc = union_scorer.doc();
-        if doc == TERMINATED {
-            return Ok(());
-        }
-        let score = union_scorer.score();
-        if score > threshold {
-            threshold = callback(doc, score);
-        }
-        let boxed_scorers: Vec<Box<dyn Scorer>> = union_scorer.into_scorers();
-        let scorers: Vec<TermScorer<Self::Postings>> = boxed_scorers
-            .into_iter()
-            .map(|scorer| {
-                *scorer.downcast::<TermScorer<Self::Postings>>().ok().expect(
-                    "Downcast failed despite the fact we already checked the type was correct",
-                )
-            })
-            .collect();
-        block_wand(scorers, threshold, callback);
-        Ok(())
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use common::OwnedBytes;
-
-    use super::*;
-    use crate::codec::postings::PostingsSerializer as _;
-    use crate::postings::Postings as _;
-
-    fn test_segment_postings_tf_aux(num_docs: u32, include_term_freq: bool) -> SegmentPostings {
-        let mut postings_serializer =
-            StandardPostingsCodec.new_serializer(1.0f32, IndexRecordOption::WithFreqs, None);
-        let mut buffer = Vec::new();
-        postings_serializer.new_term(num_docs, include_term_freq);
-        for i in 0..num_docs {
-            postings_serializer.write_doc(i, 2);
-        }
-        postings_serializer
-            .close_term(num_docs, &mut buffer)
-            .unwrap();
-        StandardPostingsCodec
-            .load_postings(
-                num_docs,
-                OwnedBytes::new(buffer),
-                IndexRecordOption::WithFreqs,
-                IndexRecordOption::WithFreqs,
-                None,
-            )
-            .unwrap()
-    }
-
-    #[test]
-    fn test_segment_postings_small_block_with_and_without_freq() {
-        let small_block_without_term_freq = test_segment_postings_tf_aux(1, false);
-        assert!(!small_block_without_term_freq.has_freq());
-        assert_eq!(small_block_without_term_freq.doc(), 0);
-        assert_eq!(small_block_without_term_freq.term_freq(), 1);
-
-        let small_block_with_term_freq = test_segment_postings_tf_aux(1, true);
-        assert!(small_block_with_term_freq.has_freq());
-        assert_eq!(small_block_with_term_freq.doc(), 0);
-        assert_eq!(small_block_with_term_freq.term_freq(), 2);
-    }
-
-    #[test]
-    fn test_segment_postings_large_block_with_and_without_freq() {
-        let large_block_without_term_freq = test_segment_postings_tf_aux(128, false);
-        assert!(!large_block_without_term_freq.has_freq());
-        assert_eq!(large_block_without_term_freq.doc(), 0);
-        assert_eq!(large_block_without_term_freq.term_freq(), 1);
-
-        let large_block_with_term_freq = test_segment_postings_tf_aux(128, true);
-        assert!(large_block_with_term_freq.has_freq());
-        assert_eq!(large_block_with_term_freq.doc(), 0);
-        assert_eq!(large_block_with_term_freq.term_freq(), 2);
-    }
-}

src/codec/standard/postings/standard_postings_serializer.rs

@@ -1,184 +0,0 @@
-use std::cmp::Ordering;
-use std::io::{self, Write as _};
-
-use common::{BinarySerializable as _, VInt};
-
-use crate::codec::postings::PostingsSerializer;
-use crate::codec::standard::postings::block::Block;
-use crate::codec::standard::postings::skip::SkipSerializer;
-use crate::fieldnorm::FieldNormReader;
-use crate::postings::compression::{BlockEncoder, VIntEncoder as _, COMPRESSION_BLOCK_SIZE};
-use crate::query::Bm25Weight;
-use crate::schema::IndexRecordOption;
-use crate::{DocId, Score};
-
-/// Serializer object for tantivy's default postings format.
-pub struct StandardPostingsSerializer {
-    last_doc_id_encoded: u32,
-
-    block_encoder: BlockEncoder,
-    block: Box<Block>,
-
-    postings_write: Vec<u8>,
-    skip_write: SkipSerializer,
-
-    mode: IndexRecordOption,
-    fieldnorm_reader: Option<FieldNormReader>,
-
-    bm25_weight: Option<Bm25Weight>,
-    avg_fieldnorm: Score, /* Average number of term in the field for that segment.
-                           * this value is used to compute the block wand information. */
-    term_has_freq: bool,
-}
-
-impl StandardPostingsSerializer {
-    pub(crate) fn new(
-        avg_fieldnorm: Score,
-        mode: IndexRecordOption,
-        fieldnorm_reader: Option<FieldNormReader>,
-    ) -> StandardPostingsSerializer {
-        Self {
-            last_doc_id_encoded: 0,
-            block_encoder: BlockEncoder::new(),
-            block: Box::new(Block::new()),
-            postings_write: Vec::new(),
-            skip_write: SkipSerializer::new(),
-            mode,
-            fieldnorm_reader,
-            bm25_weight: None,
-            avg_fieldnorm,
-            term_has_freq: false,
-        }
-    }
-}
-
-impl PostingsSerializer for StandardPostingsSerializer {
-    fn new_term(&mut self, term_doc_freq: u32, record_term_freq: bool) {
-        self.clear();
-
-        self.term_has_freq = self.mode.has_freq() && record_term_freq;
-        if !self.term_has_freq {
-            return;
-        }
-
-        let num_docs_in_segment: u64 =
-            if let Some(fieldnorm_reader) = self.fieldnorm_reader.as_ref() {
-                fieldnorm_reader.num_docs() as u64
-            } else {
-                return;
-            };
-
-        if num_docs_in_segment == 0 {
-            return;
-        }
-
-        self.bm25_weight = Some(Bm25Weight::for_one_term_without_explain(
-            term_doc_freq as u64,
-            num_docs_in_segment,
-            self.avg_fieldnorm,
-        ));
-    }
-
-    fn write_doc(&mut self, doc_id: DocId, term_freq: u32) {
-        self.block.append_doc(doc_id, term_freq);
-        if self.block.is_full() {
-            self.write_block();
-        }
-    }
-
-    fn close_term(&mut self, doc_freq: u32, output_write: &mut impl io::Write) -> io::Result<()> {
-        if !self.block.is_empty() {
-            // we have doc ids waiting to be written
-            // this happens when the number of doc ids is
-            // not a perfect multiple of our block size.
-            //
-            // In that case, the remaining part is encoded
-            // using variable int encoding.
-            {
-                let block_encoded = self
-                    .block_encoder
-                    .compress_vint_sorted(self.block.doc_ids(), self.last_doc_id_encoded);
-                self.postings_write.write_all(block_encoded)?;
-            }
-            // ... Idem for term frequencies
-            if self.term_has_freq {
-                let block_encoded = self
-                    .block_encoder
-                    .compress_vint_unsorted(self.block.term_freqs());
-                self.postings_write.write_all(block_encoded)?;
-            }
-            self.block.clear();
-        }
-        if doc_freq >= COMPRESSION_BLOCK_SIZE as u32 {
-            let skip_data = self.skip_write.data();
-            VInt(skip_data.len() as u64).serialize(output_write)?;
-            output_write.write_all(skip_data)?;
-        }
-        output_write.write_all(&self.postings_write[..])?;
-        self.skip_write.clear();
-        self.postings_write.clear();
-        self.bm25_weight = None;
-        Ok(())
-    }
-}
-
-impl StandardPostingsSerializer {
-    fn clear(&mut self) {
-        self.bm25_weight = None;
-        self.block.clear();
-        self.last_doc_id_encoded = 0;
-    }
-
-    fn write_block(&mut self) {
-        {
-            // encode the doc ids
-            let (num_bits, block_encoded): (u8, &[u8]) = self
-                .block_encoder
-                .compress_block_sorted(self.block.doc_ids(), self.last_doc_id_encoded);
-            self.last_doc_id_encoded = self.block.last_doc();
-            self.skip_write
-                .write_doc(self.last_doc_id_encoded, num_bits);
-            // last el block 0, offset block 1,
-            self.postings_write.extend(block_encoded);
-        }
-        if self.term_has_freq {
-            let (num_bits, block_encoded): (u8, &[u8]) = self
-                .block_encoder
-                .compress_block_unsorted(self.block.term_freqs(), true);
-            self.postings_write.extend(block_encoded);
-            self.skip_write.write_term_freq(num_bits);
-            if self.mode.has_positions() {
-                // We serialize the sum of term freqs within the skip information
-                // in order to navigate through positions.
-                let sum_freq = self.block.term_freqs().iter().cloned().sum();
-                self.skip_write.write_total_term_freq(sum_freq);
-            }
-            let mut blockwand_params = (0u8, 0u32);
-            if let Some(bm25_weight) = self.bm25_weight.as_ref() {
-                if let Some(fieldnorm_reader) = self.fieldnorm_reader.as_ref() {
-                    let docs = self.block.doc_ids().iter().cloned();
-                    let term_freqs = self.block.term_freqs().iter().cloned();
-                    let fieldnorms = docs.map(|doc| fieldnorm_reader.fieldnorm_id(doc));
-                    blockwand_params = fieldnorms
-                        .zip(term_freqs)
-                        .max_by(
-                            |(left_fieldnorm_id, left_term_freq),
-                             (right_fieldnorm_id, right_term_freq)| {
-                                let left_score =
-                                    bm25_weight.tf_factor(*left_fieldnorm_id, *left_term_freq);
-                                let right_score =
-                                    bm25_weight.tf_factor(*right_fieldnorm_id, *right_term_freq);
-                                left_score
-                                    .partial_cmp(&right_score)
-                                    .unwrap_or(Ordering::Equal)
-                            },
-                        )
-                        .unwrap();
-                }
-            }
-            let (fieldnorm_id, term_freq) = blockwand_params;
-            self.skip_write.write_blockwand_max(fieldnorm_id, term_freq);
-        }
-        self.block.clear();
-    }
-}

src/collector/sort_key/sort_by_static_fast_value.rs

@@ -52,7 +52,7 @@ impl<T: FastValue> SortKeyComputer for SortByStaticFastValue<T> {
         if schema_type != T::to_type() {
             return Err(crate::TantivyError::SchemaError(format!(
                 "Field `{}` is of type {schema_type:?}, not of the type {:?}.",
-                self.field,
+                &self.field,
                 T::to_type()
             )));
         }

src/core/json_utils.rs

@@ -4,7 +4,7 @@ use common::{replace_in_place, JsonPathWriter};
 use rustc_hash::FxHashMap;
 
 use crate::indexer::indexing_term::IndexingTerm;
-use crate::postings::{IndexingContext, IndexingPosition, PostingsWriter as _, PostingsWriterEnum};
+use crate::postings::{IndexingContext, IndexingPosition, PostingsWriter};
 use crate::schema::document::{ReferenceValue, ReferenceValueLeaf, Value};
 use crate::schema::{Type, DATE_TIME_PRECISION_INDEXED};
 use crate::time::format_description::well_known::Rfc3339;
@@ -52,8 +52,7 @@ use crate::{DateTime, DocId, Term};
 /// We can therefore afford working with a map that is not imperfect. It is fine if several
 /// path map to the same index position as long as the probability is relatively low.
 #[derive(Default)]
-#[doc(hidden)]
-pub struct IndexingPositionsPerPath {
+pub(crate) struct IndexingPositionsPerPath {
     positions_per_path: FxHashMap<u32, IndexingPosition>,
 }
 
@@ -81,7 +80,7 @@ fn index_json_object<'a, V: Value<'a>>(
     text_analyzer: &mut TextAnalyzer,
     term_buffer: &mut IndexingTerm,
     json_path_writer: &mut JsonPathWriter,
-    postings_writer: &mut PostingsWriterEnum,
+    postings_writer: &mut dyn PostingsWriter,
     ctx: &mut IndexingContext,
     positions_per_path: &mut IndexingPositionsPerPath,
 ) {
@@ -105,14 +104,13 @@ fn index_json_object<'a, V: Value<'a>>(
 }
 
 #[expect(clippy::too_many_arguments)]
-#[doc(hidden)]
-pub fn index_json_value<'a, V: Value<'a>>(
+pub(crate) fn index_json_value<'a, V: Value<'a>>(
     doc: DocId,
     json_value: V,
     text_analyzer: &mut TextAnalyzer,
     term_buffer: &mut IndexingTerm,
     json_path_writer: &mut JsonPathWriter,
-    postings_writer: &mut PostingsWriterEnum,
+    postings_writer: &mut dyn PostingsWriter,
     ctx: &mut IndexingContext,
     positions_per_path: &mut IndexingPositionsPerPath,
 ) {

src/docset.rs

@@ -1,6 +1,6 @@
-use std::ops::{Deref as _, DerefMut as _};
+use std::borrow::{Borrow, BorrowMut};
 
-use common::{BitSet, TinySet};
+use common::TinySet;
 
 use crate::fastfield::AliveBitSet;
 use crate::DocId;
@@ -138,17 +138,29 @@ pub trait DocSet: Send {
         buffer.len()
     }
 
-    /// Fills the given bitset with the documents in the docset.
+    /// Fills a given mutable buffer with the next doc ids smaller than `horizon`.
     ///
-    /// If the docset max_doc is smaller than the largest doc, this function might not consume the
-    /// docset entirely.
-    fn fill_bitset(&mut self, bitset: &mut BitSet) {
-        let bitset_max_value: u32 = bitset.max_value();
-        let mut doc = self.doc();
-        while doc < bitset_max_value {
-            bitset.insert(doc);
-            doc = self.advance();
+    /// Unlike [`DocSet::fill_buffer`], this method must not advance past a doc id greater than or
+    /// equal to `horizon`.
+    fn fill_buffer_up_to(
+        &mut self,
+        horizon: DocId,
+        buffer: &mut [DocId; COLLECT_BLOCK_BUFFER_LEN],
+    ) -> usize {
+        if self.doc() == TERMINATED {
+            return 0;
         }
+        for (pos, buffer_val) in buffer.iter_mut().enumerate() {
+            let doc = self.doc();
+            if doc >= horizon {
+                return pos;
+            }
+            *buffer_val = doc;
+            if self.advance() == TERMINATED {
+                return pos + 1;
+            }
+        }
+        buffer.len()
     }
 
     /// Returns the current document
@@ -264,6 +276,14 @@ impl DocSet for &mut dyn DocSet {
         (**self).fill_buffer(buffer)
     }
 
+    fn fill_buffer_up_to(
+        &mut self,
+        horizon: DocId,
+        buffer: &mut [DocId; COLLECT_BLOCK_BUFFER_LEN],
+    ) -> usize {
+        (**self).fill_buffer_up_to(horizon, buffer)
+    }
+
     fn fill_bitset_block(
         &mut self,
         min_doc: DocId,
@@ -291,30 +311,27 @@ impl DocSet for &mut dyn DocSet {
     fn count_including_deleted(&mut self) -> u32 {
         (**self).count_including_deleted()
     }
-
-    fn fill_bitset(&mut self, bitset: &mut BitSet) {
-        (**self).fill_bitset(bitset);
-    }
 }
 
 impl<TDocSet: DocSet + ?Sized> DocSet for Box<TDocSet> {
-    #[inline]
     fn advance(&mut self) -> DocId {
-        self.deref_mut().advance()
+        let unboxed: &mut TDocSet = self.borrow_mut();
+        unboxed.advance()
     }
 
-    #[inline]
     fn seek(&mut self, target: DocId) -> DocId {
-        self.deref_mut().seek(target)
+        let unboxed: &mut TDocSet = self.borrow_mut();
+        unboxed.seek(target)
     }
 
     fn seek_danger(&mut self, target: DocId) -> SeekDangerResult {
-        self.deref_mut().seek_danger(target)
+        let unboxed: &mut TDocSet = self.borrow_mut();
+        unboxed.seek_danger(target)
     }
 
-    #[inline]
     fn fill_buffer(&mut self, buffer: &mut [DocId; COLLECT_BLOCK_BUFFER_LEN]) -> usize {
-        self.deref_mut().fill_buffer(buffer)
+        let unboxed: &mut TDocSet = self.borrow_mut();
+        unboxed.fill_buffer(buffer)
     }
 
     fn fill_bitset_block(
@@ -322,35 +339,32 @@ impl<TDocSet: DocSet + ?Sized> DocSet for Box<TDocSet> {
         min_doc: DocId,
         mask: &mut [TinySet; BLOCK_NUM_TINYBITSETS],
     ) -> DocId {
-        let unboxed: &mut TDocSet = &mut **self;
+        let unboxed: &mut TDocSet = self.borrow_mut();
         unboxed.fill_bitset_block(min_doc, mask)
     }
 
-    #[inline]
     fn doc(&self) -> DocId {
-        self.deref().doc()
+        let unboxed: &TDocSet = self.borrow();
+        unboxed.doc()
     }
 
-    #[inline]
     fn size_hint(&self) -> u32 {
-        self.deref().size_hint()
+        let unboxed: &TDocSet = self.borrow();
+        unboxed.size_hint()
     }
 
-    #[inline]
     fn cost(&self) -> u64 {
-        self.deref().cost()
+        let unboxed: &TDocSet = self.borrow();
+        unboxed.cost()
     }
 
-    #[inline]
     fn count(&mut self, alive_bitset: &AliveBitSet) -> u32 {
-        self.deref_mut().count(alive_bitset)
+        let unboxed: &mut TDocSet = self.borrow_mut();
+        unboxed.count(alive_bitset)
     }
 
     fn count_including_deleted(&mut self) -> u32 {
-        self.deref_mut().count_including_deleted()
-    }
-
-    fn fill_bitset(&mut self, bitset: &mut BitSet) {
-        self.deref_mut().fill_bitset(bitset);
+        let unboxed: &mut TDocSet = self.borrow_mut();
+        unboxed.count_including_deleted()
     }
 }

src/fastfield/writer.rs

@@ -117,24 +117,6 @@ impl FastFieldsWriter {
         Ok(())
     }
 
-    /// Indexes the fast fields of a new document from its `(field, value)` pairs directly.
-    ///
-    /// This is like [`add_document`](Self::add_document), but for documents that cannot
-    /// satisfy the `Document` trait's `'static` bound (e.g. a value borrowing from a batch
-    /// being indexed). The caller supplies the document's field/value pairs; like
-    /// `add_document` it advances `num_docs` by exactly one.
-    pub fn add_document_from_values<'a, V: Value<'a>>(
-        &mut self,
-        fields_and_values: impl Iterator<Item = (Field, V)>,
-    ) -> crate::Result<()> {
-        let doc_id = self.num_docs;
-        for (field, value) in fields_and_values {
-            self.add_doc_value(doc_id, field, value)?;
-        }
-        self.num_docs += 1;
-        Ok(())
-    }
-
     fn add_doc_value<'a, V: Value<'a>>(
         &mut self,
         doc_id: DocId,

src/index/codec_configuration.rs

@@ -1,49 +0,0 @@
-use std::borrow::Cow;
-
-use serde::{Deserialize, Serialize};
-
-use crate::codec::{Codec, StandardCodec};
-
-/// A Codec configuration is just a serializable object.
-#[derive(Serialize, Deserialize, Clone, Debug)]
-pub struct CodecConfiguration {
-    codec_id: Cow<'static, str>,
-    #[serde(default, skip_serializing_if = "serde_json::Value::is_null")]
-    props: serde_json::Value,
-}
-
-impl CodecConfiguration {
-    /// Returns true if the codec is the standard codec.
-    pub fn is_standard(&self) -> bool {
-        self.codec_id == StandardCodec::ID && self.props.is_null()
-    }
-
-    /// Creates a codec instance from the configuration.
-    ///
-    /// If the codec id does not match the code's name, an error is returned.
-    pub fn to_codec<C: Codec>(&self) -> crate::Result<C> {
-        if self.codec_id != C::ID {
-            return Err(crate::TantivyError::InvalidArgument(format!(
-                "Codec id mismatch: expected {}, got {}",
-                C::ID,
-                self.codec_id
-            )));
-        }
-        C::from_json_props(&self.props)
-    }
-}
-
-impl<'a, C: Codec> From<&'a C> for CodecConfiguration {
-    fn from(codec: &'a C) -> Self {
-        CodecConfiguration {
-            codec_id: Cow::Borrowed(C::ID),
-            props: codec.to_json_props(),
-        }
-    }
-}
-
-impl Default for CodecConfiguration {
-    fn default() -> Self {
-        CodecConfiguration::from(&StandardCodec)
-    }
-}

src/index/index.rs

@@ -8,14 +8,12 @@ use std::thread::available_parallelism;
 use super::segment::Segment;
 use super::segment_reader::merge_field_meta_data;
 use super::{FieldMetadata, IndexSettings};
-use crate::codec::StandardCodec;
 use crate::core::{Executor, META_FILEPATH};
 use crate::directory::error::OpenReadError;
 #[cfg(feature = "mmap")]
 use crate::directory::MmapDirectory;
 use crate::directory::{Directory, ManagedDirectory, RamDirectory, INDEX_WRITER_LOCK};
 use crate::error::{DataCorruption, TantivyError};
-use crate::index::codec_configuration::CodecConfiguration;
 use crate::index::{IndexMeta, SegmentId, SegmentMeta, SegmentMetaInventory};
 use crate::indexer::index_writer::{
     IndexWriterOptions, MAX_NUM_THREAD, MEMORY_BUDGET_NUM_BYTES_MIN,
@@ -61,7 +59,6 @@ fn save_new_metas(
     schema: Schema,
     index_settings: IndexSettings,
     directory: &dyn Directory,
-    codec: CodecConfiguration,
 ) -> crate::Result<()> {
     save_metas(
         &IndexMeta {
@@ -70,7 +67,6 @@ fn save_new_metas(
             schema,
             opstamp: 0u64,
             payload: None,
-            codec,
         },
         directory,
     )?;
@@ -105,21 +101,18 @@ fn save_new_metas(
 /// };
 /// let index = Index::builder().schema(schema).settings(settings).create_in_ram();
 /// ```
-pub struct IndexBuilder<Codec: crate::codec::Codec = StandardCodec> {
+pub struct IndexBuilder {
     schema: Option<Schema>,
     index_settings: IndexSettings,
     tokenizer_manager: TokenizerManager,
     fast_field_tokenizer_manager: TokenizerManager,
-    codec: Codec,
 }
-
-impl Default for IndexBuilder<StandardCodec> {
+impl Default for IndexBuilder {
     fn default() -> Self {
         IndexBuilder::new()
     }
 }
-
-impl IndexBuilder<StandardCodec> {
+impl IndexBuilder {
     /// Creates a new `IndexBuilder`
     pub fn new() -> Self {
         Self {
@@ -127,21 +120,6 @@ impl IndexBuilder<StandardCodec> {
             index_settings: IndexSettings::default(),
             tokenizer_manager: TokenizerManager::default(),
             fast_field_tokenizer_manager: TokenizerManager::default(),
-            codec: StandardCodec,
-        }
-    }
-}
-
-impl<Codec: crate::codec::Codec> IndexBuilder<Codec> {
-    /// Set the codec
-    #[must_use]
-    pub fn codec<NewCodec: crate::codec::Codec>(self, codec: NewCodec) -> IndexBuilder<NewCodec> {
-        IndexBuilder {
-            schema: self.schema,
-            index_settings: self.index_settings,
-            tokenizer_manager: self.tokenizer_manager,
-            fast_field_tokenizer_manager: self.fast_field_tokenizer_manager,
-            codec,
         }
     }
 
@@ -176,7 +154,7 @@ impl<Codec: crate::codec::Codec> IndexBuilder<Codec> {
     /// The index will be allocated in anonymous memory.
     /// This is useful for indexing small set of documents
     /// for instances like unit test or temporary in memory index.
-    pub fn create_in_ram(self) -> Result<Index<Codec>, TantivyError> {
+    pub fn create_in_ram(self) -> Result<Index, TantivyError> {
         let ram_directory = RamDirectory::create();
         self.create(ram_directory)
     }
@@ -187,7 +165,7 @@ impl<Codec: crate::codec::Codec> IndexBuilder<Codec> {
     /// If a previous index was in this directory, it returns an
     /// [`TantivyError::IndexAlreadyExists`] error.
     #[cfg(feature = "mmap")]
-    pub fn create_in_dir<P: AsRef<Path>>(self, directory_path: P) -> crate::Result<Index<Codec>> {
+    pub fn create_in_dir<P: AsRef<Path>>(self, directory_path: P) -> crate::Result<Index> {
         let mmap_directory: Box<dyn Directory> = Box::new(MmapDirectory::open(directory_path)?);
         if Index::exists(&*mmap_directory)? {
             return Err(TantivyError::IndexAlreadyExists);
@@ -208,7 +186,7 @@ impl<Codec: crate::codec::Codec> IndexBuilder<Codec> {
         self,
         dir: impl Into<Box<dyn Directory>>,
         mem_budget: usize,
-    ) -> crate::Result<SingleSegmentIndexWriter<Codec, D>> {
+    ) -> crate::Result<SingleSegmentIndexWriter<D>> {
         let index = self.create(dir)?;
         let index_simple_writer = SingleSegmentIndexWriter::new(index, mem_budget)?;
         Ok(index_simple_writer)
@@ -224,7 +202,7 @@ impl<Codec: crate::codec::Codec> IndexBuilder<Codec> {
     /// For other unit tests, prefer the [`RamDirectory`], see:
     /// [`IndexBuilder::create_in_ram()`].
     #[cfg(feature = "mmap")]
-    pub fn create_from_tempdir(self) -> crate::Result<Index<Codec>> {
+    pub fn create_from_tempdir(self) -> crate::Result<Index> {
         let mmap_directory: Box<dyn Directory> = Box::new(MmapDirectory::create_from_tempdir()?);
         self.create(mmap_directory)
     }
@@ -237,15 +215,12 @@ impl<Codec: crate::codec::Codec> IndexBuilder<Codec> {
     }
 
     /// Opens or creates a new index in the provided directory
-    pub fn open_or_create<T: Into<Box<dyn Directory>>>(
-        self,
-        dir: T,
-    ) -> crate::Result<Index<Codec>> {
+    pub fn open_or_create<T: Into<Box<dyn Directory>>>(self, dir: T) -> crate::Result<Index> {
         let dir: Box<dyn Directory> = dir.into();
         if !Index::exists(&*dir)? {
             return self.create(dir);
         }
-        let mut index: Index<Codec> = Index::<Codec>::open_with_codec(dir)?;
+        let mut index = Index::open(dir)?;
         index.set_tokenizers(self.tokenizer_manager.clone());
         if index.schema() == self.get_expect_schema()? {
             Ok(index)
@@ -269,25 +244,18 @@ impl<Codec: crate::codec::Codec> IndexBuilder<Codec> {
     /// Creates a new index given an implementation of the trait `Directory`.
     ///
     /// If a directory previously existed, it will be erased.
-    pub fn create<T: Into<Box<dyn Directory>>>(self, dir: T) -> crate::Result<Index<Codec>> {
-        self.create_avoid_monomorphization(dir.into())
-    }
-
-    fn create_avoid_monomorphization(self, dir: Box<dyn Directory>) -> crate::Result<Index<Codec>> {
+    fn create<T: Into<Box<dyn Directory>>>(self, dir: T) -> crate::Result<Index> {
         self.validate()?;
+        let dir = dir.into();
         let directory = ManagedDirectory::wrap(dir)?;
-        let codec: CodecConfiguration = CodecConfiguration::from(&self.codec);
         save_new_metas(
             self.get_expect_schema()?,
             self.index_settings.clone(),
             &directory,
-            codec,
         )?;
-        let schema = self.get_expect_schema()?;
-        let mut metas = IndexMeta::with_schema_and_codec(schema, &self.codec);
+        let mut metas = IndexMeta::with_schema(self.get_expect_schema()?);
         metas.index_settings = self.index_settings;
-        let mut index: Index<Codec> =
-            Index::<Codec>::open_from_metas(directory, &metas, SegmentMetaInventory::default())?;
+        let mut index = Index::open_from_metas(directory, &metas, SegmentMetaInventory::default());
         index.set_tokenizers(self.tokenizer_manager);
         index.set_fast_field_tokenizers(self.fast_field_tokenizer_manager);
         Ok(index)
@@ -296,7 +264,7 @@ impl<Codec: crate::codec::Codec> IndexBuilder<Codec> {
 
 /// Search Index
 #[derive(Clone)]
-pub struct Index<Codec: crate::codec::Codec = crate::codec::StandardCodec> {
+pub struct Index {
     directory: ManagedDirectory,
     schema: Schema,
     settings: IndexSettings,
@@ -304,7 +272,6 @@ pub struct Index<Codec: crate::codec::Codec = crate::codec::StandardCodec> {
     tokenizers: TokenizerManager,
     fast_field_tokenizers: TokenizerManager,
     inventory: SegmentMetaInventory,
-    codec: Codec,
 }
 
 impl Index {
@@ -312,6 +279,41 @@ impl Index {
     pub fn builder() -> IndexBuilder {
         IndexBuilder::new()
     }
+    /// Examines the directory to see if it contains an index.
+    ///
+    /// Effectively, it only checks for the presence of the `meta.json` file.
+    pub fn exists(dir: &dyn Directory) -> Result<bool, OpenReadError> {
+        dir.exists(&META_FILEPATH)
+    }
+
+    /// Accessor to the search executor.
+    ///
+    /// This pool is used by default when calling `searcher.search(...)`
+    /// to perform search on the individual segments.
+    ///
+    /// By default the executor is single thread, and simply runs in the calling thread.
+    pub fn search_executor(&self) -> &Executor {
+        &self.executor
+    }
+
+    /// Replace the default single thread search executor pool
+    /// by a thread pool with a given number of threads.
+    pub fn set_multithread_executor(&mut self, num_threads: usize) -> crate::Result<()> {
+        self.executor = Executor::multi_thread(num_threads, "tantivy-search-")?;
+        Ok(())
+    }
+
+    /// Custom thread pool by a outer thread pool.
+    pub fn set_executor(&mut self, executor: Executor) {
+        self.executor = executor;
+    }
+
+    /// Replace the default single thread search executor pool
+    /// by a thread pool with as many threads as there are CPUs on the system.
+    pub fn set_default_multithread_executor(&mut self) -> crate::Result<()> {
+        let default_num_threads = available_parallelism()?.get();
+        self.set_multithread_executor(default_num_threads)
+    }
 
     /// Creates a new index using the [`RamDirectory`].
     ///
@@ -322,13 +324,6 @@ impl Index {
         IndexBuilder::new().schema(schema).create_in_ram().unwrap()
     }
 
-    /// Examines the directory to see if it contains an index.
-    ///
-    /// Effectively, it only checks for the presence of the `meta.json` file.
-    pub fn exists(directory: &dyn Directory) -> Result<bool, OpenReadError> {
-        directory.exists(&META_FILEPATH)
-    }
-
     /// Creates a new index in a given filepath.
     /// The index will use the [`MmapDirectory`].
     ///
@@ -375,108 +370,20 @@ impl Index {
         schema: Schema,
         settings: IndexSettings,
     ) -> crate::Result<Index> {
-        Self::create_to_avoid_monomorphization(dir.into(), schema, settings)
-    }
-
-    fn create_to_avoid_monomorphization(
-        dir: Box<dyn Directory>,
-        schema: Schema,
-        settings: IndexSettings,
-    ) -> crate::Result<Index> {
+        let dir: Box<dyn Directory> = dir.into();
         let mut builder = IndexBuilder::new().schema(schema);
         builder = builder.settings(settings);
         builder.create(dir)
     }
 
-    /// Opens a new directory from an index path.
-    #[cfg(feature = "mmap")]
-    pub fn open_in_dir<P: AsRef<Path>>(directory_path: P) -> crate::Result<Index> {
-        Self::open_in_dir_to_avoid_monomorphization(directory_path.as_ref())
-    }
-
-    #[cfg(feature = "mmap")]
-    #[inline(never)]
-    fn open_in_dir_to_avoid_monomorphization(directory_path: &Path) -> crate::Result<Index> {
-        let mmap_directory = MmapDirectory::open(directory_path)?;
-        Index::open(mmap_directory)
-    }
-
-    /// Open the index using the provided directory
-    pub fn open<T: Into<Box<dyn Directory>>>(directory: T) -> crate::Result<Index> {
-        Index::<StandardCodec>::open_with_codec(directory.into())
-    }
-}
-
-impl<Codec: crate::codec::Codec> Index<Codec> {
-    /// Returns a version of this index with the standard codec.
-    /// This is useful when you need to pass the index to APIs that
-    /// don't care about the codec (e.g., for reading).
-    pub(crate) fn with_standard_codec(&self) -> Index<StandardCodec> {
-        Index {
-            directory: self.directory.clone(),
-            schema: self.schema.clone(),
-            settings: self.settings.clone(),
-            executor: self.executor.clone(),
-            tokenizers: self.tokenizers.clone(),
-            fast_field_tokenizers: self.fast_field_tokenizers.clone(),
-            inventory: self.inventory.clone(),
-            codec: StandardCodec,
-        }
-    }
-
-    /// Open the index using the provided directory
-    #[inline(never)]
-    pub fn open_with_codec(directory: Box<dyn Directory>) -> crate::Result<Index<Codec>> {
-        let directory = ManagedDirectory::wrap(directory)?;
-        let inventory = SegmentMetaInventory::default();
-        let metas = load_metas(&directory, &inventory)?;
-        let index: Index<Codec> = Index::<Codec>::open_from_metas(directory, &metas, inventory)?;
-        Ok(index)
-    }
-
-    /// Accessor to the codec.
-    pub fn codec(&self) -> &Codec {
-        &self.codec
-    }
-
-    /// Accessor to the search executor.
-    ///
-    /// This pool is used by default when calling `searcher.search(...)`
-    /// to perform search on the individual segments.
-    ///
-    /// By default the executor is single thread, and simply runs in the calling thread.
-    pub fn search_executor(&self) -> &Executor {
-        &self.executor
-    }
-
-    /// Replace the default single thread search executor pool
-    /// by a thread pool with a given number of threads.
-    pub fn set_multithread_executor(&mut self, num_threads: usize) -> crate::Result<()> {
-        self.executor = Executor::multi_thread(num_threads, "tantivy-search-")?;
-        Ok(())
-    }
-
-    /// Custom thread pool by a outer thread pool.
-    pub fn set_executor(&mut self, executor: Executor) {
-        self.executor = executor;
-    }
-
-    /// Replace the default single thread search executor pool
-    /// by a thread pool with as many threads as there are CPUs on the system.
-    pub fn set_default_multithread_executor(&mut self) -> crate::Result<()> {
-        let default_num_threads = available_parallelism()?.get();
-        self.set_multithread_executor(default_num_threads)
-    }
-
     /// Creates a new index given a directory and an [`IndexMeta`].
-    fn open_from_metas<C: crate::codec::Codec>(
+    fn open_from_metas(
         directory: ManagedDirectory,
         metas: &IndexMeta,
         inventory: SegmentMetaInventory,
-    ) -> crate::Result<Index<C>> {
+    ) -> Index {
         let schema = metas.schema.clone();
-        let codec = metas.codec.to_codec::<C>()?;
-        Ok(Index {
+        Index {
             settings: metas.index_settings.clone(),
             directory,
             schema,
@@ -484,8 +391,7 @@ impl<Codec: crate::codec::Codec> Index<Codec> {
             fast_field_tokenizers: TokenizerManager::default(),
             executor: Executor::single_thread(),
             inventory,
-            codec,
-        })
+        }
     }
 
     /// Setter for the tokenizer manager.
@@ -541,7 +447,7 @@ impl<Codec: crate::codec::Codec> Index<Codec> {
     /// Create a default [`IndexReader`] for the given index.
     ///
     /// See [`Index.reader_builder()`].
-    pub fn reader(&self) -> crate::Result<IndexReader<Codec>> {
+    pub fn reader(&self) -> crate::Result<IndexReader> {
         self.reader_builder().try_into()
     }
 
@@ -549,10 +455,17 @@ impl<Codec: crate::codec::Codec> Index<Codec> {
     ///
     /// Most project should create at most one reader for a given index.
     /// This method is typically called only once per `Index` instance.
-    pub fn reader_builder(&self) -> IndexReaderBuilder<Codec> {
+    pub fn reader_builder(&self) -> IndexReaderBuilder {
         IndexReaderBuilder::new(self.clone())
     }
 
+    /// Opens a new directory from an index path.
+    #[cfg(feature = "mmap")]
+    pub fn open_in_dir<P: AsRef<Path>>(directory_path: P) -> crate::Result<Index> {
+        let mmap_directory = MmapDirectory::open(directory_path)?;
+        Index::open(mmap_directory)
+    }
+
     /// Returns the list of the segment metas tracked by the index.
     ///
     /// Such segments can of course be part of the index,
@@ -593,6 +506,16 @@ impl<Codec: crate::codec::Codec> Index<Codec> {
         self.inventory.new_segment_meta(segment_id, max_doc)
     }
 
+    /// Open the index using the provided directory
+    pub fn open<T: Into<Box<dyn Directory>>>(directory: T) -> crate::Result<Index> {
+        let directory = directory.into();
+        let directory = ManagedDirectory::wrap(directory)?;
+        let inventory = SegmentMetaInventory::default();
+        let metas = load_metas(&directory, &inventory)?;
+        let index = Index::open_from_metas(directory, &metas, inventory);
+        Ok(index)
+    }
+
     /// Reads the index meta file from the directory.
     pub fn load_metas(&self) -> crate::Result<IndexMeta> {
         load_metas(self.directory(), &self.inventory)
@@ -616,7 +539,7 @@ impl<Codec: crate::codec::Codec> Index<Codec> {
     pub fn writer_with_options<D: Document>(
         &self,
         options: IndexWriterOptions,
-    ) -> crate::Result<IndexWriter<Codec, D>> {
+    ) -> crate::Result<IndexWriter<D>> {
         let directory_lock = self
             .directory
             .acquire_lock(&INDEX_WRITER_LOCK)
@@ -658,7 +581,7 @@ impl<Codec: crate::codec::Codec> Index<Codec> {
         &self,
         num_threads: usize,
         overall_memory_budget_in_bytes: usize,
-    ) -> crate::Result<IndexWriter<Codec, D>> {
+    ) -> crate::Result<IndexWriter<D>> {
         let memory_arena_in_bytes_per_thread = overall_memory_budget_in_bytes / num_threads;
         let options = IndexWriterOptions::builder()
             .num_worker_threads(num_threads)
@@ -672,7 +595,7 @@ impl<Codec: crate::codec::Codec> Index<Codec> {
     /// That index writer only simply has a single thread and a memory budget of 15 MB.
     /// Using a single thread gives us a deterministic allocation of DocId.
     #[cfg(test)]
-    pub fn writer_for_tests<D: Document>(&self) -> crate::Result<IndexWriter<Codec, D>> {
+    pub fn writer_for_tests<D: Document>(&self) -> crate::Result<IndexWriter<D>> {
         self.writer_with_num_threads(1, MEMORY_BUDGET_NUM_BYTES_MIN)
     }
 
@@ -690,7 +613,7 @@ impl<Codec: crate::codec::Codec> Index<Codec> {
     pub fn writer<D: Document>(
         &self,
         memory_budget_in_bytes: usize,
-    ) -> crate::Result<IndexWriter<Codec, D>> {
+    ) -> crate::Result<IndexWriter<D>> {
         let mut num_threads = std::cmp::min(available_parallelism()?.get(), MAX_NUM_THREAD);
         let memory_budget_num_bytes_per_thread = memory_budget_in_bytes / num_threads;
         if memory_budget_num_bytes_per_thread < MEMORY_BUDGET_NUM_BYTES_MIN {
@@ -717,7 +640,7 @@ impl<Codec: crate::codec::Codec> Index<Codec> {
     }
 
     /// Returns the list of segments that are searchable
-    pub fn searchable_segments(&self) -> crate::Result<Vec<Segment<Codec>>> {
+    pub fn searchable_segments(&self) -> crate::Result<Vec<Segment>> {
         Ok(self
             .searchable_segment_metas()?
             .into_iter()
@@ -726,12 +649,12 @@ impl<Codec: crate::codec::Codec> Index<Codec> {
     }
 
     #[doc(hidden)]
-    pub fn segment(&self, segment_meta: SegmentMeta) -> Segment<Codec> {
+    pub fn segment(&self, segment_meta: SegmentMeta) -> Segment {
         Segment::for_index(self.clone(), segment_meta)
     }
 
     /// Creates a new segment.
-    pub fn new_segment(&self) -> Segment<Codec> {
+    pub fn new_segment(&self) -> Segment {
         let segment_meta = self
             .inventory
             .new_segment_meta(SegmentId::generate_random(), 0);
@@ -785,7 +708,7 @@ impl<Codec: crate::codec::Codec> Index<Codec> {
 }
 
 impl fmt::Debug for Index {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         write!(f, "Index({:?})", self.directory)
     }
 }

src/index/index_meta.rs

@@ -5,8 +5,7 @@ use std::path::PathBuf;
 use serde::{Deserialize, Serialize};
 
 use super::SegmentComponent;
-use crate::codec::Codec;
-use crate::index::{CodecConfiguration, SegmentId};
+use crate::index::SegmentId;
 use crate::schema::Schema;
 use crate::store::Compressor;
 use crate::{Inventory, Opstamp, TrackedObject};
@@ -287,10 +286,8 @@ pub struct IndexMeta {
     /// This payload is entirely unused by tantivy.
     #[serde(skip_serializing_if = "Option::is_none")]
     pub payload: Option<String>,
-    /// Codec configuration for the index.
-    #[serde(skip_serializing_if = "CodecConfiguration::is_standard")]
-    pub codec: CodecConfiguration,
 }
+
 #[derive(Deserialize, Debug)]
 struct UntrackedIndexMeta {
     pub segments: Vec<InnerSegmentMeta>,
@@ -300,8 +297,6 @@ struct UntrackedIndexMeta {
     pub opstamp: Opstamp,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub payload: Option<String>,
-    #[serde(default)]
-    pub codec: CodecConfiguration,
 }
 
 impl UntrackedIndexMeta {
@@ -316,7 +311,6 @@ impl UntrackedIndexMeta {
             schema: self.schema,
             opstamp: self.opstamp,
             payload: self.payload,
-            codec: self.codec,
         }
     }
 }
@@ -327,14 +321,13 @@ impl IndexMeta {
     ///
     /// This new index does not contains any segments.
     /// Opstamp will the value `0u64`.
-    pub fn with_schema_and_codec<C: Codec>(schema: Schema, codec: &C) -> IndexMeta {
+    pub fn with_schema(schema: Schema) -> IndexMeta {
         IndexMeta {
             index_settings: IndexSettings::default(),
             segments: vec![],
             schema,
             opstamp: 0u64,
             payload: None,
-            codec: CodecConfiguration::from(codec),
         }
     }
 
@@ -385,38 +378,14 @@ mod tests {
             schema,
             opstamp: 0u64,
             payload: None,
-            codec: Default::default(),
         };
-        let json_value: serde_json::Value =
-            serde_json::to_value(&index_metas).expect("serialization failed");
+        let json = serde_json::ser::to_string(&index_metas).expect("serialization failed");
         assert_eq!(
-            &json_value,
-            &serde_json::json!(
-            {
-              "index_settings": {
-                "docstore_compression": "none",
-                "docstore_blocksize": 16384
-              },
-              "segments": [],
-              "schema": [
-                {
-                  "name": "text",
-                  "type": "text",
-                  "options": {
-                    "indexing": {
-                      "record": "position",
-                      "fieldnorms": true,
-                      "tokenizer": "default"
-                    },
-                    "stored": false,
-                    "fast": false
-                  }
-                }
-              ],
-              "opstamp": 0
-            })
+            json,
+            r#"{"index_settings":{"docstore_compression":"none","docstore_blocksize":16384},"segments":[],"schema":[{"name":"text","type":"text","options":{"indexing":{"record":"position","fieldnorms":true,"tokenizer":"default"},"stored":false,"fast":false}}],"opstamp":0}"#
         );
-        let deser_meta: UntrackedIndexMeta = serde_json::from_value(json_value).unwrap();
+
+        let deser_meta: UntrackedIndexMeta = serde_json::from_str(&json).unwrap();
         assert_eq!(index_metas.index_settings, deser_meta.index_settings);
         assert_eq!(index_metas.schema, deser_meta.schema);
         assert_eq!(index_metas.opstamp, deser_meta.opstamp);
@@ -442,39 +411,14 @@ mod tests {
             schema,
             opstamp: 0u64,
             payload: None,
-            codec: Default::default(),
         };
-        let json_value = serde_json::to_value(&index_metas).expect("serialization failed");
+        let json = serde_json::ser::to_string(&index_metas).expect("serialization failed");
         assert_eq!(
-            &json_value,
-            &serde_json::json!(
-                {
-                  "index_settings": {
-                    "docstore_compression": "zstd(compression_level=4)",
-                    "docstore_blocksize": 1000000
-                  },
-                  "segments": [],
-                  "schema": [
-                    {
-                      "name": "text",
-                      "type": "text",
-                      "options": {
-                        "indexing": {
-                          "record": "position",
-                          "fieldnorms": true,
-                          "tokenizer": "default"
-                        },
-                        "stored": false,
-                        "fast": false
-                      }
-                    }
-                  ],
-                  "opstamp": 0
-                }
-            )
+            json,
+            r#"{"index_settings":{"docstore_compression":"zstd(compression_level=4)","docstore_blocksize":1000000},"segments":[],"schema":[{"name":"text","type":"text","options":{"indexing":{"record":"position","fieldnorms":true,"tokenizer":"default"},"stored":false,"fast":false}}],"opstamp":0}"#
         );
 
-        let deser_meta: UntrackedIndexMeta = serde_json::from_value(json_value).unwrap();
+        let deser_meta: UntrackedIndexMeta = serde_json::from_str(&json).unwrap();
         assert_eq!(index_metas.index_settings, deser_meta.index_settings);
         assert_eq!(index_metas.schema, deser_meta.schema);
         assert_eq!(index_metas.opstamp, deser_meta.opstamp);

src/index/inverted_index_reader.rs

@@ -1,5 +1,4 @@
 use std::io;
-use std::sync::Arc;
 
 use common::json_path_writer::JSON_END_OF_PATH;
 use common::{BinarySerializable, ByteCount};
@@ -10,14 +9,9 @@ use itertools::Itertools;
 #[cfg(feature = "quickwit")]
 use tantivy_fst::automaton::{AlwaysMatch, Automaton};
 
-use crate::codec::positions::PositionsCodec;
-use crate::codec::postings::PostingsCodec;
-use crate::codec::{Codec, ObjectSafeCodec, StandardCodec};
 use crate::directory::FileSlice;
-use crate::fieldnorm::FieldNormReader;
-use crate::postings::{Postings, TermInfo};
-use crate::query::term_query::TermScorer;
-use crate::query::{Bm25Weight, PhraseScorer, Scorer};
+use crate::positions::PositionReader;
+use crate::postings::{BlockSegmentPostings, SegmentPostings, TermInfo};
 use crate::schema::{IndexRecordOption, Term, Type};
 use crate::termdict::TermDictionary;
 
@@ -39,7 +33,6 @@ pub struct InvertedIndexReader {
     positions_file_slice: FileSlice,
     record_option: IndexRecordOption,
     total_num_tokens: u64,
-    codec: Arc<dyn ObjectSafeCodec>,
 }
 
 /// Object that records the amount of space used by a field in an inverted index.
@@ -75,7 +68,6 @@ impl InvertedIndexReader {
         postings_file_slice: FileSlice,
         positions_file_slice: FileSlice,
         record_option: IndexRecordOption,
-        codec: Arc<dyn ObjectSafeCodec>,
     ) -> io::Result<InvertedIndexReader> {
         let (total_num_tokens_slice, postings_body) = postings_file_slice.split(8);
         let total_num_tokens = u64::deserialize(&mut total_num_tokens_slice.read_bytes()?)?;
@@ -85,7 +77,6 @@ impl InvertedIndexReader {
             positions_file_slice,
             record_option,
             total_num_tokens,
-            codec,
         })
     }
 
@@ -98,7 +89,6 @@ impl InvertedIndexReader {
             positions_file_slice: FileSlice::empty(),
             record_option,
             total_num_tokens: 0u64,
-            codec: Arc::new(StandardCodec),
         }
     }
 
@@ -170,99 +160,61 @@ impl InvertedIndexReader {
         Ok(fields)
     }
 
-    pub(crate) fn new_term_scorer_specialized<C: Codec>(
-        &self,
-        term_info: &TermInfo,
-        option: IndexRecordOption,
-        fieldnorm_reader: FieldNormReader,
-        similarity_weight: Bm25Weight,
-        codec: &C,
-    ) -> io::Result<TermScorer<<<C as Codec>::PostingsCodec as PostingsCodec>::Postings>> {
-        let postings = self.read_postings_from_terminfo_specialized(term_info, option, codec)?;
-        let term_scorer = TermScorer::new(postings, fieldnorm_reader, similarity_weight);
-        Ok(term_scorer)
-    }
-
-    pub(crate) fn new_phrase_scorer_type_specialized<C: Codec>(
-        &self,
-        term_infos: &[(usize, TermInfo)],
-        similarity_weight_opt: Option<Bm25Weight>,
-        fieldnorm_reader: FieldNormReader,
-        slop: u32,
-        codec: &C,
-    ) -> io::Result<PhraseScorer<<<C as Codec>::PostingsCodec as PostingsCodec>::Postings>> {
-        let mut offset_and_term_postings: Vec<(
-            usize,
-            <<C as Codec>::PostingsCodec as PostingsCodec>::Postings,
-        )> = Vec::with_capacity(term_infos.len());
-        for (offset, term_info) in term_infos {
-            let postings = self.read_postings_from_terminfo_specialized(
-                term_info,
-                IndexRecordOption::WithFreqsAndPositions,
-                codec,
-            )?;
-            offset_and_term_postings.push((*offset, postings));
-        }
-        let phrase_scorer = PhraseScorer::new(
-            offset_and_term_postings,
-            similarity_weight_opt,
-            fieldnorm_reader,
-            slop,
-        );
-        Ok(phrase_scorer)
-    }
-
-    /// Build a new term scorer.
-    pub fn new_term_scorer(
-        &self,
-        term_info: &TermInfo,
-        option: IndexRecordOption,
-        fieldnorm_reader: FieldNormReader,
-        similarity_weight: Bm25Weight,
-    ) -> io::Result<Box<dyn Scorer>> {
-        let term_scorer = self.codec.load_term_scorer_type_erased(
-            term_info,
-            option,
-            self,
-            fieldnorm_reader,
-            similarity_weight,
-        )?;
-        Ok(term_scorer)
-    }
-
-    /// Returns a postings object specific with a concrete type.
+    /// Resets the block segment to another position of the postings
+    /// file.
     ///
-    /// This requires you to provied the actual codec.
-    pub fn read_postings_from_terminfo_specialized<C: Codec>(
+    /// This is useful for enumerating through a list of terms,
+    /// and consuming the associated posting lists while avoiding
+    /// reallocating a [`BlockSegmentPostings`].
+    ///
+    /// # Warning
+    ///
+    /// This does not reset the positions list.
+    pub fn reset_block_postings_from_terminfo(
         &self,
         term_info: &TermInfo,
+        block_postings: &mut BlockSegmentPostings,
+    ) -> io::Result<()> {
+        let postings_slice = self
+            .postings_file_slice
+            .slice(term_info.postings_range.clone());
+        let postings_bytes = postings_slice.read_bytes()?;
+        block_postings.reset(term_info.doc_freq, postings_bytes)?;
+        Ok(())
+    }
+
+    /// Returns a block postings given a `Term`.
+    /// This method is for an advanced usage only.
+    ///
+    /// Most users should prefer using [`Self::read_postings()`] instead.
+    pub fn read_block_postings(
+        &self,
+        term: &Term,
         option: IndexRecordOption,
-        codec: &C,
-    ) -> io::Result<<<C as Codec>::PostingsCodec as PostingsCodec>::Postings> {
-        let option = option.downgrade(self.record_option);
+    ) -> io::Result<Option<BlockSegmentPostings>> {
+        self.get_term_info(term)?
+            .map(move |term_info| self.read_block_postings_from_terminfo(&term_info, option))
+            .transpose()
+    }
+
+    /// Returns a block postings given a `term_info`.
+    /// This method is for an advanced usage only.
+    ///
+    /// Most users should prefer using [`Self::read_postings()`] instead.
+    pub fn read_block_postings_from_terminfo(
+        &self,
+        term_info: &TermInfo,
+        requested_option: IndexRecordOption,
+    ) -> io::Result<BlockSegmentPostings> {
         let postings_data = self
             .postings_file_slice
-            .slice(term_info.postings_range.clone())
-            .read_bytes()?;
-        let position_reader = if option.has_positions() {
-            let positions_data = self
-                .positions_file_slice
-                .slice(term_info.positions_range.clone())
-                .read_bytes()?;
-            let reader = codec.positions_codec().open_reader(positions_data)?;
-            Some(Box::new(reader) as Box<dyn crate::codec::positions::PositionsReader>)
-        } else {
-            None
-        };
-        let postings: <<C as Codec>::PostingsCodec as PostingsCodec>::Postings =
-            codec.postings_codec().load_postings(
-                term_info.doc_freq,
-                postings_data,
-                self.record_option,
-                option,
-                position_reader,
-            )?;
-        Ok(postings)
+            .slice(term_info.postings_range.clone());
+        BlockSegmentPostings::open(
+            term_info.doc_freq,
+            postings_data,
+            self.record_option,
+            requested_option,
+        )
     }
 
     /// Returns a posting object given a `term_info`.
@@ -273,9 +225,25 @@ impl InvertedIndexReader {
         &self,
         term_info: &TermInfo,
         option: IndexRecordOption,
-    ) -> io::Result<Box<dyn Postings>> {
-        self.codec
-            .load_postings_type_erased(term_info, option, self)
+    ) -> io::Result<SegmentPostings> {
+        let option = option.downgrade(self.record_option);
+
+        let block_postings = self.read_block_postings_from_terminfo(term_info, option)?;
+        let position_reader = {
+            if option.has_positions() {
+                let positions_data = self
+                    .positions_file_slice
+                    .read_bytes_slice(term_info.positions_range.clone())?;
+                let position_reader = PositionReader::open(positions_data)?;
+                Some(position_reader)
+            } else {
+                None
+            }
+        };
+        Ok(SegmentPostings::from_block_postings(
+            block_postings,
+            position_reader,
+        ))
     }
 
     /// Returns the total number of tokens recorded for all documents
@@ -298,7 +266,7 @@ impl InvertedIndexReader {
         &self,
         term: &Term,
         option: IndexRecordOption,
-    ) -> io::Result<Option<Box<dyn Postings>>> {
+    ) -> io::Result<Option<SegmentPostings>> {
         self.get_term_info(term)?
             .map(move |term_info| self.read_postings_from_terminfo(&term_info, option))
             .transpose()

src/index/mod.rs

@@ -2,7 +2,6 @@
 //!
 //! It contains `Index` and `Segment`, where a `Index` consists of one or more `Segment`s.
 
-mod codec_configuration;
 mod index;
 mod index_meta;
 mod inverted_index_reader;
@@ -11,7 +10,6 @@ mod segment_component;
 mod segment_id;
 mod segment_reader;
 
-pub use self::codec_configuration::CodecConfiguration;
 pub use self::index::{Index, IndexBuilder};
 pub(crate) use self::index_meta::SegmentMetaInventory;
 pub use self::index_meta::{IndexMeta, IndexSettings, Order, SegmentMeta};

src/index/segment.rs

@@ -2,7 +2,6 @@ use std::fmt;
 use std::path::PathBuf;
 
 use super::SegmentComponent;
-use crate::codec::StandardCodec;
 use crate::directory::error::{OpenReadError, OpenWriteError};
 use crate::directory::{Directory, FileSlice, WritePtr};
 use crate::index::{Index, SegmentId, SegmentMeta};
@@ -11,25 +10,25 @@ use crate::Opstamp;
 
 /// A segment is a piece of the index.
 #[derive(Clone)]
-pub struct Segment<C: crate::codec::Codec = StandardCodec> {
-    index: Index<C>,
+pub struct Segment {
+    index: Index,
     meta: SegmentMeta,
 }
 
-impl<C: crate::codec::Codec> fmt::Debug for Segment<C> {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+impl fmt::Debug for Segment {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         write!(f, "Segment({:?})", self.id().uuid_string())
     }
 }
 
-impl<C: crate::codec::Codec> Segment<C> {
+impl Segment {
     /// Creates a new segment given an `Index` and a `SegmentId`
-    pub(crate) fn for_index(index: Index<C>, meta: SegmentMeta) -> Segment<C> {
+    pub(crate) fn for_index(index: Index, meta: SegmentMeta) -> Segment {
         Segment { index, meta }
     }
 
     /// Returns the index the segment belongs to.
-    pub fn index(&self) -> &Index<C> {
+    pub fn index(&self) -> &Index {
         &self.index
     }
 
@@ -47,7 +46,7 @@ impl<C: crate::codec::Codec> Segment<C> {
     ///
     /// This method is only used when updating `max_doc` from 0
     /// as we finalize a fresh new segment.
-    pub fn with_max_doc(self, max_doc: u32) -> Segment<C> {
+    pub fn with_max_doc(self, max_doc: u32) -> Segment {
         Segment {
             index: self.index,
             meta: self.meta.with_max_doc(max_doc),
@@ -56,7 +55,7 @@ impl<C: crate::codec::Codec> Segment<C> {
 
     #[doc(hidden)]
     #[must_use]
-    pub fn with_delete_meta(self, num_deleted_docs: u32, opstamp: Opstamp) -> Segment<C> {
+    pub fn with_delete_meta(self, num_deleted_docs: u32, opstamp: Opstamp) -> Segment {
         Segment {
             index: self.index,
             meta: self.meta.with_delete_meta(num_deleted_docs, opstamp),

src/index/segment_reader.rs

@@ -6,7 +6,6 @@ use common::{ByteCount, HasLen};
 use fnv::FnvHashMap;
 use itertools::Itertools;
 
-use crate::codec::ObjectSafeCodec;
 use crate::directory::error::OpenReadError;
 use crate::directory::{CompositeFile, FileSlice};
 use crate::error::DataCorruption;
@@ -49,7 +48,6 @@ pub struct SegmentReader {
     store_file: FileSlice,
     alive_bitset_opt: Option<AliveBitSet>,
     schema: Schema,
-    codec: Arc<dyn ObjectSafeCodec>,
 }
 
 impl SegmentReader {
@@ -70,11 +68,6 @@ impl SegmentReader {
         &self.schema
     }
 
-    /// Returns the index codec.
-    pub fn codec(&self) -> &dyn ObjectSafeCodec {
-        &*self.codec
-    }
-
     /// Return the number of documents that have been
     /// deleted in the segment.
     pub fn num_deleted_docs(&self) -> DocId {
@@ -148,16 +141,15 @@ impl SegmentReader {
     }
 
     /// Open a new segment for reading.
-    pub fn open<C: crate::codec::Codec>(segment: &Segment<C>) -> crate::Result<SegmentReader> {
+    pub fn open(segment: &Segment) -> crate::Result<SegmentReader> {
         Self::open_with_custom_alive_set(segment, None)
     }
 
     /// Open a new segment for reading.
-    pub fn open_with_custom_alive_set<C: crate::codec::Codec>(
-        segment: &Segment<C>,
+    pub fn open_with_custom_alive_set(
+        segment: &Segment,
         custom_bitset: Option<AliveBitSet>,
     ) -> crate::Result<SegmentReader> {
-        let codec: Arc<dyn ObjectSafeCodec> = Arc::new(segment.index().codec().clone());
         let termdict_file = segment.open_read(SegmentComponent::Terms)?;
         let termdict_composite = CompositeFile::open(&termdict_file)?;
 
@@ -211,7 +203,6 @@ impl SegmentReader {
             alive_bitset_opt,
             positions_composite,
             schema,
-            codec,
         })
     }
 
@@ -281,7 +272,6 @@ impl SegmentReader {
             postings_file,
             positions_file,
             record_option,
-            self.codec.clone(),
         )?);
 
         // by releasing the lock in between, we may end up opening the inverting index
@@ -332,7 +322,7 @@ impl SegmentReader {
                             // Without expand dots enabled dots need to be escaped.
                             let escaped_json_path = json_path.replace('.', "\\.");
                             let full_path = format!("{field_name}.{escaped_json_path}");
-                            let full_path_unescaped = format!("{}.{}", field_name, json_path);
+                            let full_path_unescaped = format!("{}.{}", field_name, &json_path);
                             map_to_canonical.insert(full_path_unescaped, full_path.to_string());
                             full_path
                         } else {

src/indexer/index_writer.rs

@@ -9,7 +9,6 @@ use smallvec::smallvec;
 use super::operation::{AddOperation, UserOperation};
 use super::segment_updater::SegmentUpdater;
 use super::{AddBatch, AddBatchReceiver, AddBatchSender, PreparedCommit};
-use crate::codec::{Codec, StandardCodec};
 use crate::directory::{DirectoryLock, GarbageCollectionResult, TerminatingWrite};
 use crate::error::TantivyError;
 use crate::fastfield::write_alive_bitset;
@@ -69,12 +68,12 @@ pub struct IndexWriterOptions {
 /// indexing queue.
 /// Each indexing thread builds its own independent [`Segment`], via
 /// a `SegmentWriter` object.
-pub struct IndexWriter<C: Codec = StandardCodec, D: Document = TantivyDocument> {
+pub struct IndexWriter<D: Document = TantivyDocument> {
     // the lock is just used to bind the
     // lifetime of the lock with that of the IndexWriter.
     _directory_lock: Option<DirectoryLock>,
 
-    index: Index<C>,
+    index: Index,
 
     options: IndexWriterOptions,
 
@@ -83,7 +82,7 @@ pub struct IndexWriter<C: Codec = StandardCodec, D: Document = TantivyDocument>
     index_writer_status: IndexWriterStatus<D>,
     operation_sender: AddBatchSender<D>,
 
-    segment_updater: SegmentUpdater<C>,
+    segment_updater: SegmentUpdater,
 
     worker_id: usize,
 
@@ -129,8 +128,8 @@ fn compute_deleted_bitset(
 /// is `==` target_opstamp.
 /// For instance, there was no delete operation between the state of the `segment_entry` and
 /// the `target_opstamp`, `segment_entry` is not updated.
-pub fn advance_deletes<C: Codec>(
-    mut segment: Segment<C>,
+pub fn advance_deletes(
+    mut segment: Segment,
     segment_entry: &mut SegmentEntry,
     target_opstamp: Opstamp,
 ) -> crate::Result<()> {
@@ -180,11 +179,11 @@ pub fn advance_deletes<C: Codec>(
     Ok(())
 }
 
-fn index_documents<C: crate::codec::Codec, D: Document>(
+fn index_documents<D: Document>(
     memory_budget: usize,
-    segment: Segment<C>,
+    segment: Segment,
     grouped_document_iterator: &mut dyn Iterator<Item = AddBatch<D>>,
-    segment_updater: &SegmentUpdater<C>,
+    segment_updater: &SegmentUpdater,
     mut delete_cursor: DeleteCursor,
 ) -> crate::Result<()> {
     let mut segment_writer = SegmentWriter::for_segment(memory_budget, segment.clone())?;
@@ -227,8 +226,8 @@ fn index_documents<C: crate::codec::Codec, D: Document>(
 }
 
 /// `doc_opstamps` is required to be non-empty.
-fn apply_deletes<C: crate::codec::Codec>(
-    segment: &Segment<C>,
+fn apply_deletes(
+    segment: &Segment,
     delete_cursor: &mut DeleteCursor,
     doc_opstamps: &[Opstamp],
 ) -> crate::Result<Option<BitSet>> {
@@ -263,7 +262,7 @@ fn apply_deletes<C: crate::codec::Codec>(
     })
 }
 
-impl<C: Codec, D: Document> IndexWriter<C, D> {
+impl<D: Document> IndexWriter<D> {
     /// Create a new index writer. Attempts to acquire a lockfile.
     ///
     /// The lockfile should be deleted on drop, but it is possible
@@ -279,7 +278,7 @@ impl<C: Codec, D: Document> IndexWriter<C, D> {
     /// If the memory arena per thread is too small or too big, returns
     /// `TantivyError::InvalidArgument`
     pub(crate) fn new(
-        index: &Index<C>,
+        index: &Index,
         options: IndexWriterOptions,
         directory_lock: DirectoryLock,
     ) -> crate::Result<Self> {
@@ -346,7 +345,7 @@ impl<C: Codec, D: Document> IndexWriter<C, D> {
     }
 
     /// Accessor to the index.
-    pub fn index(&self) -> &Index<C> {
+    pub fn index(&self) -> &Index {
         &self.index
     }
 
@@ -394,7 +393,7 @@ impl<C: Codec, D: Document> IndexWriter<C, D> {
     /// It is safe to start writing file associated with the new `Segment`.
     /// These will not be garbage collected as long as an instance object of
     /// `SegmentMeta` object associated with the new `Segment` is "alive".
-    pub fn new_segment(&self) -> Segment<C> {
+    pub fn new_segment(&self) -> Segment {
         self.index.new_segment()
     }
 
@@ -616,7 +615,7 @@ impl<C: Codec, D: Document> IndexWriter<C, D> {
     /// It is also possible to add a payload to the `commit`
     /// using this API.
     /// See [`PreparedCommit::set_payload()`].
-    pub fn prepare_commit(&mut self) -> crate::Result<PreparedCommit<'_, C, D>> {
+    pub fn prepare_commit(&mut self) -> crate::Result<PreparedCommit<'_, D>> {
         // Here, because we join all of the worker threads,
         // all of the segment update for this commit have been
         // sent.
@@ -666,7 +665,7 @@ impl<C: Codec, D: Document> IndexWriter<C, D> {
         self.prepare_commit()?.commit()
     }
 
-    pub(crate) fn segment_updater(&self) -> &SegmentUpdater<C> {
+    pub(crate) fn segment_updater(&self) -> &SegmentUpdater {
         &self.segment_updater
     }
 
@@ -805,7 +804,7 @@ impl<C: Codec, D: Document> IndexWriter<C, D> {
     }
 }
 
-impl<C: Codec, D: Document> Drop for IndexWriter<C, D> {
+impl<D: Document> Drop for IndexWriter<D> {
     fn drop(&mut self) {
         self.segment_updater.kill();
         self.drop_sender();

src/indexer/indexing_term.rs

@@ -13,8 +13,7 @@ use crate::schema::Field;
 /// We serialize the field, because we index everything in a single
 /// global term dictionary during indexing.
 #[derive(Clone)]
-#[doc(hidden)]
-pub struct IndexingTerm<B = Vec<u8>>(B)
+pub(crate) struct IndexingTerm<B = Vec<u8>>(B)
 where B: AsRef<[u8]>;
 
 /// The number of bytes used as metadata by `Term`.
@@ -43,7 +42,7 @@ impl IndexingTerm {
     }
 
     /// Removes the value_bytes and set the field
-    pub fn clear_with_field(&mut self, field: Field) {
+    pub(crate) fn clear_with_field(&mut self, field: Field) {
         self.truncate_value_bytes(0);
         self.set_field(field);
     }

src/indexer/merge_index_test.rs

@@ -1,10 +1,9 @@
 #[cfg(test)]
 mod tests {
-    use crate::codec::StandardCodec;
     use crate::collector::TopDocs;
     use crate::fastfield::AliveBitSet;
     use crate::index::Index;
-    use crate::postings::{DocFreq, Postings};
+    use crate::postings::Postings;
     use crate::query::QueryParser;
     use crate::schema::{
         self, BytesOptions, Facet, FacetOptions, IndexRecordOption, NumericOptions,
@@ -122,26 +121,21 @@ mod tests {
             let my_text_field = index.schema().get_field("text_field").unwrap();
             let term_a = Term::from_field_text(my_text_field, "text");
             let inverted_index = segment_reader.inverted_index(my_text_field).unwrap();
-            let term_info = inverted_index.get_term_info(&term_a).unwrap().unwrap();
             let mut postings = inverted_index
-                .read_postings_from_terminfo_specialized(
-                    &term_info,
-                    IndexRecordOption::WithFreqsAndPositions,
-                    &StandardCodec,
-                )
+                .read_postings(&term_a, IndexRecordOption::WithFreqsAndPositions)
+                .unwrap()
                 .unwrap();
-            assert_eq!(postings.doc_freq(), DocFreq::Exact(2));
+            assert_eq!(postings.doc_freq(), 2);
             let fallback_bitset = AliveBitSet::for_test_from_deleted_docs(&[0], 100);
             assert_eq!(
-                crate::indexer::merger::doc_freq_given_deletes(
-                    &postings,
+                postings.doc_freq_given_deletes(
                     segment_reader.alive_bitset().unwrap_or(&fallback_bitset)
                 ),
                 2
             );
 
             assert_eq!(postings.term_freq(), 1);
-            let mut output = Vec::new();
+            let mut output = vec![];
             postings.positions(&mut output);
             assert_eq!(output, vec![1]);
             postings.advance();

src/indexer/merger.rs

@@ -7,8 +7,6 @@ use common::ReadOnlyBitSet;
 use itertools::Itertools;
 use measure_time::debug_time;
 
-use crate::codec::postings::PostingsCodec;
-use crate::codec::{Codec, StandardCodec};
 use crate::directory::WritePtr;
 use crate::docset::{DocSet, TERMINATED};
 use crate::error::DataCorruption;
@@ -17,7 +15,7 @@ use crate::fieldnorm::{FieldNormReader, FieldNormReaders, FieldNormsSerializer,
 use crate::index::{Segment, SegmentComponent, SegmentReader};
 use crate::indexer::doc_id_mapping::{MappingType, SegmentDocIdMapping};
 use crate::indexer::SegmentSerializer;
-use crate::postings::{InvertedIndexSerializer, Postings};
+use crate::postings::{InvertedIndexSerializer, Postings, SegmentPostings};
 use crate::schema::{value_type_to_column_type, Field, FieldType, Schema};
 use crate::store::StoreWriter;
 use crate::termdict::{TermMerger, TermOrdinal};
@@ -78,11 +76,10 @@ fn estimate_total_num_tokens(readers: &[SegmentReader], field: Field) -> crate::
     Ok(total_num_tokens)
 }
 
-pub struct IndexMerger<C: Codec = StandardCodec> {
+pub struct IndexMerger {
     schema: Schema,
     pub(crate) readers: Vec<SegmentReader>,
     max_doc: u32,
-    codec: C,
 }
 
 struct DeltaComputer {
@@ -147,8 +144,8 @@ fn extract_fast_field_required_columns(schema: &Schema) -> Vec<(String, ColumnTy
         .collect()
 }
 
-impl<C: Codec> IndexMerger<C> {
-    pub fn open(schema: Schema, segments: &[Segment<C>]) -> crate::Result<IndexMerger<C>> {
+impl IndexMerger {
+    pub fn open(schema: Schema, segments: &[Segment]) -> crate::Result<IndexMerger> {
         let alive_bitset = segments.iter().map(|_| None).collect_vec();
         Self::open_with_custom_alive_set(schema, segments, alive_bitset)
     }
@@ -165,15 +162,11 @@ impl<C: Codec> IndexMerger<C> {
     // This can be used to merge but also apply an additional filter.
     // One use case is demux, which is basically taking a list of
     // segments and partitions them e.g. by a value in a field.
-    //
-    // # Panics if segments is empty.
     pub fn open_with_custom_alive_set(
         schema: Schema,
-        segments: &[Segment<C>],
+        segments: &[Segment],
         alive_bitset_opt: Vec<Option<AliveBitSet>>,
-    ) -> crate::Result<IndexMerger<C>> {
-        assert!(!segments.is_empty());
-        let codec = segments[0].index().codec().clone();
+    ) -> crate::Result<IndexMerger> {
         let mut readers = vec![];
         for (segment, new_alive_bitset_opt) in segments.iter().zip(alive_bitset_opt) {
             if segment.meta().num_docs() > 0 {
@@ -196,7 +189,6 @@ impl<C: Codec> IndexMerger<C> {
             schema,
             readers,
             max_doc,
-            codec,
         })
     }
 
@@ -295,7 +287,7 @@ impl<C: Codec> IndexMerger<C> {
         &self,
         indexed_field: Field,
         _field_type: &FieldType,
-        serializer: &mut InvertedIndexSerializer<C>,
+        serializer: &mut InvertedIndexSerializer,
         fieldnorm_reader: Option<FieldNormReader>,
         doc_id_mapping: &SegmentDocIdMapping,
     ) -> crate::Result<()> {
@@ -363,10 +355,7 @@ impl<C: Codec> IndexMerger<C> {
                          indexed. Have you modified the schema?",
         );
 
-        let mut segment_postings_containing_the_term: Vec<(
-            usize,
-            <C::PostingsCodec as PostingsCodec>::Postings,
-        )> = Vec::with_capacity(self.readers.len());
+        let mut segment_postings_containing_the_term: Vec<(usize, SegmentPostings)> = vec![];
 
         while merged_terms.advance() {
             segment_postings_containing_the_term.clear();
@@ -378,24 +367,17 @@ impl<C: Codec> IndexMerger<C> {
             for (segment_ord, term_info) in merged_terms.current_segment_ords_and_term_infos() {
                 let segment_reader = &self.readers[segment_ord];
                 let inverted_index: &InvertedIndexReader = &field_readers[segment_ord];
-                let postings = inverted_index.read_postings_from_terminfo_specialized(
-                    &term_info,
-                    segment_postings_option,
-                    &self.codec,
-                )?;
+                let segment_postings = inverted_index
+                    .read_postings_from_terminfo(&term_info, segment_postings_option)?;
                 let alive_bitset_opt = segment_reader.alive_bitset();
                 let doc_freq = if let Some(alive_bitset) = alive_bitset_opt {
-                    doc_freq_given_deletes(&postings, alive_bitset)
+                    segment_postings.doc_freq_given_deletes(alive_bitset)
                 } else {
-                    // We do not an exact document frequency here.
-                    match postings.doc_freq() {
-                        crate::postings::DocFreq::Approximate(_) => exact_doc_freq(&postings),
-                        crate::postings::DocFreq::Exact(doc_freq) => doc_freq,
-                    }
+                    segment_postings.doc_freq()
                 };
                 if doc_freq > 0u32 {
                     total_doc_freq += doc_freq;
-                    segment_postings_containing_the_term.push((segment_ord, postings));
+                    segment_postings_containing_the_term.push((segment_ord, segment_postings));
                 }
             }
 
@@ -413,7 +395,11 @@ impl<C: Codec> IndexMerger<C> {
             assert!(!segment_postings_containing_the_term.is_empty());
 
             let has_term_freq = {
-                let has_term_freq = segment_postings_containing_the_term[0].1.has_freq();
+                let has_term_freq = !segment_postings_containing_the_term[0]
+                    .1
+                    .block_cursor
+                    .freqs()
+                    .is_empty();
                 for (_, postings) in &segment_postings_containing_the_term[1..] {
                     // This may look at a strange way to test whether we have term freq or not.
                     // With JSON object, the schema is not sufficient to know whether a term
@@ -429,7 +415,7 @@ impl<C: Codec> IndexMerger<C> {
                     //
                     // Overall the reliable way to know if we have actual frequencies loaded or not
                     // is to check whether the actual decoded array is empty or not.
-                    if postings.has_freq() != has_term_freq {
+                    if has_term_freq == postings.block_cursor.freqs().is_empty() {
                         return Err(DataCorruption::comment_only(
                             "Term freqs are inconsistent across segments",
                         )
@@ -481,7 +467,7 @@ impl<C: Codec> IndexMerger<C> {
 
     fn write_postings(
         &self,
-        serializer: &mut InvertedIndexSerializer<C>,
+        serializer: &mut InvertedIndexSerializer,
         fieldnorm_readers: FieldNormReaders,
         doc_id_mapping: &SegmentDocIdMapping,
     ) -> crate::Result<()> {
@@ -539,7 +525,7 @@ impl<C: Codec> IndexMerger<C> {
     ///
     /// # Returns
     /// The number of documents in the resulting segment.
-    pub fn write(&self, mut serializer: SegmentSerializer<C>) -> crate::Result<u32> {
+    pub fn write(&self, mut serializer: SegmentSerializer) -> crate::Result<u32> {
         let doc_id_mapping = self.get_doc_id_from_concatenated_data()?;
         debug!("write-fieldnorms");
         if let Some(fieldnorms_serializer) = serializer.extract_fieldnorms_serializer() {
@@ -567,43 +553,6 @@ impl<C: Codec> IndexMerger<C> {
     }
 }
 
-/// Compute the number of non-deleted documents.
-///
-/// This method will clone and scan through the posting lists.
-/// (this is a rather expensive operation).
-pub(crate) fn doc_freq_given_deletes<P: Postings + Clone>(
-    postings: &P,
-    alive_bitset: &AliveBitSet,
-) -> u32 {
-    let mut docset = postings.clone();
-    let mut doc_freq = 0;
-    loop {
-        let doc = docset.doc();
-        if doc == TERMINATED {
-            return doc_freq;
-        }
-        if alive_bitset.is_alive(doc) {
-            doc_freq += 1u32;
-        }
-        docset.advance();
-    }
-}
-
-/// If the postings is not able to inform us of the document frequency,
-/// we just scan through it.
-pub(crate) fn exact_doc_freq<P: Postings + Clone>(postings: &P) -> u32 {
-    let mut docset = postings.clone();
-    let mut doc_freq = 0;
-    loop {
-        let doc = docset.doc();
-        if doc == TERMINATED {
-            return doc_freq;
-        }
-        doc_freq += 1u32;
-        docset.advance();
-    }
-}
-
 #[cfg(test)]
 mod tests {
 
@@ -612,16 +561,12 @@ mod tests {
     use proptest::strategy::Strategy;
     use schema::FAST;
 
-    use crate::codec::postings::PostingsCodec;
-    use crate::codec::standard::postings::StandardPostingsCodec;
     use crate::collector::tests::{
         BytesFastFieldTestCollector, FastFieldTestCollector, TEST_COLLECTOR_WITH_SCORE,
     };
     use crate::collector::{Count, FacetCollector};
-    use crate::fastfield::AliveBitSet;
     use crate::index::{Index, SegmentId};
     use crate::indexer::NoMergePolicy;
-    use crate::postings::{DocFreq, Postings as _};
     use crate::query::{AllQuery, BooleanQuery, EnableScoring, Scorer, TermQuery};
     use crate::schema::{
         Facet, FacetOptions, IndexRecordOption, NumericOptions, TantivyDocument, Term,
@@ -1573,10 +1518,10 @@ mod tests {
         let searcher = reader.searcher();
         let mut term_scorer = term_query
             .specialized_weight(EnableScoring::enabled_from_searcher(&searcher))?
-            .term_scorer_for_test(searcher.segment_reader(0u32), 1.0)
+            .term_scorer_for_test(searcher.segment_reader(0u32), 1.0)?
             .unwrap();
         assert_eq!(term_scorer.doc(), 0);
-        assert_nearly_equals!(term_scorer.seek_block_max(0), 0.0079681855);
+        assert_nearly_equals!(term_scorer.block_max_score(), 0.0079681855);
         assert_nearly_equals!(term_scorer.score(), 0.0079681855);
         for _ in 0..81 {
             writer.add_document(doc!(text=>"hello happy tax payer"))?;
@@ -1589,13 +1534,13 @@ mod tests {
         for segment_reader in searcher.segment_readers() {
             let mut term_scorer = term_query
                 .specialized_weight(EnableScoring::enabled_from_searcher(&searcher))?
-                .term_scorer_for_test(segment_reader, 1.0)
+                .term_scorer_for_test(segment_reader, 1.0)?
                 .unwrap();
             // the difference compared to before is intrinsic to the bm25 formula. no worries
             // there.
             for doc in segment_reader.doc_ids_alive() {
                 assert_eq!(term_scorer.doc(), doc);
-                assert_nearly_equals!(term_scorer.seek_block_max(doc), 0.003478312);
+                assert_nearly_equals!(term_scorer.block_max_score(), 0.003478312);
                 assert_nearly_equals!(term_scorer.score(), 0.003478312);
                 term_scorer.advance();
             }
@@ -1615,12 +1560,12 @@ mod tests {
         let segment_reader = searcher.segment_reader(0u32);
         let mut term_scorer = term_query
             .specialized_weight(EnableScoring::enabled_from_searcher(&searcher))?
-            .term_scorer_for_test(segment_reader, 1.0)
+            .term_scorer_for_test(segment_reader, 1.0)?
             .unwrap();
         // the difference compared to before is intrinsic to the bm25 formula. no worries there.
         for doc in segment_reader.doc_ids_alive() {
             assert_eq!(term_scorer.doc(), doc);
-            assert_nearly_equals!(term_scorer.seek_block_max(doc), 0.003478312);
+            assert_nearly_equals!(term_scorer.block_max_score(), 0.003478312);
             assert_nearly_equals!(term_scorer.score(), 0.003478312);
             term_scorer.advance();
         }
@@ -1634,16 +1579,4 @@ mod tests {
         assert!(((super::MAX_DOC_LIMIT - 1) as i32) >= 0);
         assert!((super::MAX_DOC_LIMIT as i32) < 0);
     }
-
-    #[test]
-    fn test_doc_freq_given_delete() {
-        let docs =
-            <StandardPostingsCodec as PostingsCodec>::Postings::create_from_docs(&[0, 2, 10]);
-        assert_eq!(docs.doc_freq(), DocFreq::Exact(3));
-        let alive_bitset = AliveBitSet::for_test_from_deleted_docs(&[2], 12);
-        assert_eq!(super::doc_freq_given_deletes(&docs, &alive_bitset), 2);
-        let all_deleted =
-            AliveBitSet::for_test_from_deleted_docs(&[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], 12);
-        assert_eq!(super::doc_freq_given_deletes(&docs, &all_deleted), 0);
-    }
 }

src/indexer/mod.rs

@@ -34,8 +34,6 @@ use crossbeam_channel as channel;
 use smallvec::SmallVec;
 
 pub use self::index_writer::{advance_deletes, IndexWriter, IndexWriterOptions};
-#[doc(hidden)]
-pub use self::indexing_term::IndexingTerm;
 pub use self::log_merge_policy::LogMergePolicy;
 pub use self::merge_operation::MergeOperation;
 pub use self::merge_policy::{MergeCandidate, MergePolicy, NoMergePolicy};

src/indexer/prepared_commit.rs

@@ -1,17 +1,16 @@
 use super::IndexWriter;
-use crate::codec::Codec;
 use crate::schema::document::Document;
 use crate::{FutureResult, Opstamp, TantivyDocument};
 
 /// A prepared commit
-pub struct PreparedCommit<'a, C: Codec, D: Document = TantivyDocument> {
-    index_writer: &'a mut IndexWriter<C, D>,
+pub struct PreparedCommit<'a, D: Document = TantivyDocument> {
+    index_writer: &'a mut IndexWriter<D>,
     payload: Option<String>,
     opstamp: Opstamp,
 }
 
-impl<'a, C: Codec, D: Document> PreparedCommit<'a, C, D> {
-    pub(crate) fn new(index_writer: &'a mut IndexWriter<C, D>, opstamp: Opstamp) -> Self {
+impl<'a, D: Document> PreparedCommit<'a, D> {
+    pub(crate) fn new(index_writer: &'a mut IndexWriter<D>, opstamp: Opstamp) -> Self {
         Self {
             index_writer,
             payload: None,

src/indexer/segment_serializer.rs

@@ -8,17 +8,17 @@ use crate::store::StoreWriter;
 
 /// Segment serializer is in charge of laying out on disk
 /// the data accumulated and sorted by the `SegmentWriter`.
-pub struct SegmentSerializer<C: crate::codec::Codec> {
-    segment: Segment<C>,
+pub struct SegmentSerializer {
+    segment: Segment,
     pub(crate) store_writer: StoreWriter,
     fast_field_write: WritePtr,
     fieldnorms_serializer: Option<FieldNormsSerializer>,
-    postings_serializer: InvertedIndexSerializer<C>,
+    postings_serializer: InvertedIndexSerializer,
 }
 
-impl<C: crate::codec::Codec> SegmentSerializer<C> {
+impl SegmentSerializer {
     /// Creates a new `SegmentSerializer`.
-    pub fn for_segment(mut segment: Segment<C>) -> crate::Result<SegmentSerializer<C>> {
+    pub fn for_segment(mut segment: Segment) -> crate::Result<SegmentSerializer> {
         let settings = segment.index().settings().clone();
         let store_writer = {
             let store_write = segment.open_write(SegmentComponent::Store)?;
@@ -50,12 +50,12 @@ impl<C: crate::codec::Codec> SegmentSerializer<C> {
         self.store_writer.mem_usage()
     }
 
-    pub fn segment(&self) -> &Segment<C> {
+    pub fn segment(&self) -> &Segment {
         &self.segment
     }
 
     /// Accessor to the `PostingsSerializer`.
-    pub fn get_postings_serializer(&mut self) -> &mut InvertedIndexSerializer<C> {
+    pub fn get_postings_serializer(&mut self) -> &mut InvertedIndexSerializer {
         &mut self.postings_serializer
     }
 

src/indexer/segment_updater.rs

@@ -10,13 +10,10 @@ use std::sync::{Arc, RwLock};
 use rayon::{ThreadPool, ThreadPoolBuilder};
 
 use super::segment_manager::SegmentManager;
-use crate::codec::Codec;
 use crate::core::META_FILEPATH;
 use crate::directory::{Directory, DirectoryClone, GarbageCollectionResult};
 use crate::fastfield::AliveBitSet;
-use crate::index::{
-    CodecConfiguration, Index, IndexMeta, IndexSettings, Segment, SegmentId, SegmentMeta,
-};
+use crate::index::{Index, IndexMeta, IndexSettings, Segment, SegmentId, SegmentMeta};
 use crate::indexer::delete_queue::DeleteCursor;
 use crate::indexer::index_writer::advance_deletes;
 use crate::indexer::merge_operation::MergeOperationInventory;
@@ -64,10 +61,10 @@ pub(crate) fn save_metas(metas: &IndexMeta, directory: &dyn Directory) -> crate:
 // We voluntarily pass a merge_operation ref to guarantee that
 // the merge_operation is alive during the process
 #[derive(Clone)]
-pub(crate) struct SegmentUpdater<C: Codec>(Arc<InnerSegmentUpdater<C>>);
+pub(crate) struct SegmentUpdater(Arc<InnerSegmentUpdater>);
 
-impl<C: Codec> Deref for SegmentUpdater<C> {
-    type Target = InnerSegmentUpdater<C>;
+impl Deref for SegmentUpdater {
+    type Target = InnerSegmentUpdater;
 
     #[inline]
     fn deref(&self) -> &Self::Target {
@@ -75,8 +72,8 @@ impl<C: Codec> Deref for SegmentUpdater<C> {
     }
 }
 
-fn garbage_collect_files<C: Codec>(
-    segment_updater: SegmentUpdater<C>,
+fn garbage_collect_files(
+    segment_updater: SegmentUpdater,
 ) -> crate::Result<GarbageCollectionResult> {
     info!("Running garbage collection");
     let mut index = segment_updater.index.clone();
@@ -87,8 +84,8 @@ fn garbage_collect_files<C: Codec>(
 
 /// Merges a list of segments the list of segment givens in the `segment_entries`.
 /// This function happens in the calling thread and is computationally expensive.
-fn merge<Codec: crate::codec::Codec>(
-    index: &Index<Codec>,
+fn merge(
+    index: &Index,
     mut segment_entries: Vec<SegmentEntry>,
     target_opstamp: Opstamp,
 ) -> crate::Result<Option<SegmentEntry>> {
@@ -111,13 +108,13 @@ fn merge<Codec: crate::codec::Codec>(
 
     let delete_cursor = segment_entries[0].delete_cursor().clone();
 
-    let segments: Vec<Segment<Codec>> = segment_entries
+    let segments: Vec<Segment> = segment_entries
         .iter()
         .map(|segment_entry| index.segment(segment_entry.meta().clone()))
         .collect();
 
     // An IndexMerger is like a "view" of our merged segments.
-    let merger: IndexMerger<Codec> = IndexMerger::open(index.schema(), &segments[..])?;
+    let merger: IndexMerger = IndexMerger::open(index.schema(), &segments[..])?;
 
     // ... we just serialize this index merger in our new segment to merge the segments.
     let segment_serializer = SegmentSerializer::for_segment(merged_segment.clone())?;
@@ -142,10 +139,10 @@ fn merge<Codec: crate::codec::Codec>(
 /// meant to work if you have an `IndexWriter` running for the origin indices, or
 /// the destination `Index`.
 #[doc(hidden)]
-pub fn merge_indices<Codec: crate::codec::Codec>(
-    indices: &[Index<Codec>],
-    output_directory: Box<dyn Directory>,
-) -> crate::Result<Index<Codec>> {
+pub fn merge_indices<T: Into<Box<dyn Directory>>>(
+    indices: &[Index],
+    output_directory: T,
+) -> crate::Result<Index> {
     if indices.is_empty() {
         // If there are no indices to merge, there is no need to do anything.
         return Err(crate::TantivyError::InvalidArgument(
@@ -166,7 +163,7 @@ pub fn merge_indices<Codec: crate::codec::Codec>(
         ));
     }
 
-    let mut segments: Vec<Segment<Codec>> = Vec::new();
+    let mut segments: Vec<Segment> = Vec::new();
     for index in indices {
         segments.extend(index.searchable_segments()?);
     }
@@ -188,12 +185,12 @@ pub fn merge_indices<Codec: crate::codec::Codec>(
 /// meant to work if you have an `IndexWriter` running for the origin indices, or
 /// the destination `Index`.
 #[doc(hidden)]
-pub fn merge_filtered_segments<C: crate::codec::Codec, T: Into<Box<dyn Directory>>>(
-    segments: &[Segment<C>],
+pub fn merge_filtered_segments<T: Into<Box<dyn Directory>>>(
+    segments: &[Segment],
     target_settings: IndexSettings,
     filter_doc_ids: Vec<Option<AliveBitSet>>,
     output_directory: T,
-) -> crate::Result<Index<C>> {
+) -> crate::Result<Index> {
     if segments.is_empty() {
         // If there are no indices to merge, there is no need to do anything.
         return Err(crate::TantivyError::InvalidArgument(
@@ -214,15 +211,14 @@ pub fn merge_filtered_segments<C: crate::codec::Codec, T: Into<Box<dyn Directory
         ));
     }
 
-    let mut merged_index: Index<C> = Index::builder()
-        .schema(target_schema.clone())
-        .codec(segments[0].index().codec().clone())
-        .settings(target_settings.clone())
-        .create(output_directory.into())?;
-
+    let mut merged_index = Index::create(
+        output_directory,
+        target_schema.clone(),
+        target_settings.clone(),
+    )?;
     let merged_segment = merged_index.new_segment();
     let merged_segment_id = merged_segment.id();
-    let merger: IndexMerger<C> =
+    let merger: IndexMerger =
         IndexMerger::open_with_custom_alive_set(merged_index.schema(), segments, filter_doc_ids)?;
     let segment_serializer = SegmentSerializer::for_segment(merged_segment)?;
     let num_docs = merger.write(segment_serializer)?;
@@ -239,7 +235,6 @@ pub fn merge_filtered_segments<C: crate::codec::Codec, T: Into<Box<dyn Directory
             ))
             .trim_end()
     );
-    let codec_configuration = CodecConfiguration::from(segments[0].index().codec());
 
     let index_meta = IndexMeta {
         index_settings: target_settings, // index_settings of all segments should be the same
@@ -247,7 +242,6 @@ pub fn merge_filtered_segments<C: crate::codec::Codec, T: Into<Box<dyn Directory
         schema: target_schema,
         opstamp: 0u64,
         payload: Some(stats),
-        codec: codec_configuration,
     };
 
     // save the meta.json
@@ -256,7 +250,7 @@ pub fn merge_filtered_segments<C: crate::codec::Codec, T: Into<Box<dyn Directory
     Ok(merged_index)
 }
 
-pub(crate) struct InnerSegmentUpdater<C: Codec> {
+pub(crate) struct InnerSegmentUpdater {
     // we keep a copy of the current active IndexMeta to
     // avoid loading the file every time we need it in the
     // `SegmentUpdater`.
@@ -267,7 +261,7 @@ pub(crate) struct InnerSegmentUpdater<C: Codec> {
     pool: ThreadPool,
     merge_thread_pool: ThreadPool,
 
-    index: Index<C>,
+    index: Index,
     segment_manager: SegmentManager,
     merge_policy: RwLock<Arc<dyn MergePolicy>>,
     killed: AtomicBool,
@@ -275,13 +269,13 @@ pub(crate) struct InnerSegmentUpdater<C: Codec> {
     merge_operations: MergeOperationInventory,
 }
 
-impl<Codec: crate::codec::Codec> SegmentUpdater<Codec> {
+impl SegmentUpdater {
     pub fn create(
-        index: Index<Codec>,
+        index: Index,
         stamper: Stamper,
         delete_cursor: &DeleteCursor,
         num_merge_threads: usize,
-    ) -> crate::Result<Self> {
+    ) -> crate::Result<SegmentUpdater> {
         let segments = index.searchable_segment_metas()?;
         let segment_manager = SegmentManager::from_segments(segments, delete_cursor);
         let pool = ThreadPoolBuilder::new()
@@ -411,14 +405,12 @@ impl<Codec: crate::codec::Codec> SegmentUpdater<Codec> {
             // Segment 1 from disk 1, Segment 1 from disk 2, etc.
             committed_segment_metas
                 .sort_by_key(|segment_meta| std::cmp::Reverse(segment_meta.max_doc()));
-            let codec = CodecConfiguration::from(index.codec());
             let index_meta = IndexMeta {
                 index_settings: index.settings().clone(),
                 segments: committed_segment_metas,
                 schema: index.schema(),
                 opstamp,
                 payload: commit_message,
-                codec,
             };
             // TODO add context to the error.
             save_metas(&index_meta, directory.box_clone().borrow_mut())?;
@@ -452,7 +444,7 @@ impl<Codec: crate::codec::Codec> SegmentUpdater<Codec> {
         opstamp: Opstamp,
         payload: Option<String>,
     ) -> FutureResult<Opstamp> {
-        let segment_updater: SegmentUpdater<Codec> = self.clone();
+        let segment_updater: SegmentUpdater = self.clone();
         self.schedule_task(move || {
             let segment_entries = segment_updater.purge_deletes(opstamp)?;
             segment_updater.segment_manager.commit(segment_entries);
@@ -708,7 +700,6 @@ impl<Codec: crate::codec::Codec> SegmentUpdater<Codec> {
 #[cfg(test)]
 mod tests {
     use super::merge_indices;
-    use crate::codec::StandardCodec;
     use crate::collector::TopDocs;
     use crate::directory::RamDirectory;
     use crate::fastfield::AliveBitSet;
@@ -939,7 +930,7 @@ mod tests {
 
     #[test]
     fn test_merge_empty_indices_array() {
-        let merge_result = merge_indices::<StandardCodec>(&[], Box::new(RamDirectory::default()));
+        let merge_result = merge_indices(&[], RamDirectory::default());
         assert!(merge_result.is_err());
     }
 
@@ -966,10 +957,7 @@ mod tests {
         };
 
         // mismatched schema index list
-        let result = merge_indices(
-            &[first_index, second_index],
-            Box::new(RamDirectory::default()),
-        );
+        let result = merge_indices(&[first_index, second_index], RamDirectory::default());
         assert!(result.is_err());
 
         Ok(())

src/indexer/segment_writer.rs

@@ -1,12 +1,9 @@
-use std::any::Any;
-
 use columnar::MonotonicallyMappableToU64;
 use common::JsonPathWriter;
 use itertools::Itertools;
 use tokenizer_api::BoxTokenStream;
 
 use super::operation::AddOperation;
-use crate::codec::Codec;
 use crate::fastfield::FastFieldsWriter;
 use crate::fieldnorm::{FieldNormReaders, FieldNormsWriter};
 use crate::index::{Segment, SegmentComponent};
@@ -15,10 +12,10 @@ use crate::indexer::segment_serializer::SegmentSerializer;
 use crate::json_utils::{index_json_value, IndexingPositionsPerPath};
 use crate::postings::{
     compute_table_memory_size, serialize_postings, IndexingContext, IndexingPosition,
-    PerFieldPostingsWriter, PostingsWriter, PostingsWriterEnum,
+    PerFieldPostingsWriter, PostingsWriter,
 };
 use crate::schema::document::{Document, Value};
-use crate::schema::{Field, FieldEntry, FieldType, Schema, DATE_TIME_PRECISION_INDEXED};
+use crate::schema::{FieldEntry, FieldType, Schema, DATE_TIME_PRECISION_INDEXED};
 use crate::tokenizer::{FacetTokenizer, PreTokenizedStream, TextAnalyzer, Tokenizer};
 use crate::{DocId, Opstamp, TantivyError};
 
@@ -48,22 +45,22 @@ fn compute_initial_table_size(per_thread_memory_budget: usize) -> crate::Result<
 ///
 /// They creates the postings list in anonymous memory.
 /// The segment is laid on disk when the segment gets `finalized`.
-pub struct SegmentWriter<Codec: crate::codec::Codec> {
+pub struct SegmentWriter {
     pub(crate) max_doc: DocId,
     pub(crate) ctx: IndexingContext,
-    pub per_field_postings_writers: PerFieldPostingsWriter,
-    pub(crate) segment_serializer: SegmentSerializer<Codec>,
+    pub(crate) per_field_postings_writers: PerFieldPostingsWriter,
+    pub(crate) segment_serializer: SegmentSerializer,
     pub(crate) fast_field_writers: FastFieldsWriter,
     pub(crate) fieldnorms_writer: FieldNormsWriter,
     pub(crate) json_path_writer: JsonPathWriter,
     pub(crate) json_positions_per_path: IndexingPositionsPerPath,
     pub(crate) doc_opstamps: Vec<Opstamp>,
-    schema: Schema,
     per_field_text_analyzers: Vec<TextAnalyzer>,
     term_buffer: IndexingTerm,
+    schema: Schema,
 }
 
-impl<Codec: crate::codec::Codec> SegmentWriter<Codec> {
+impl SegmentWriter {
     /// Creates a new `SegmentWriter`
     ///
     /// The arguments are defined as follows
@@ -73,10 +70,7 @@ impl<Codec: crate::codec::Codec> SegmentWriter<Codec> {
     ///   behavior as a memory limit.
     /// - segment: The segment being written
     /// - schema
-    pub fn for_segment(
-        memory_budget_in_bytes: usize,
-        segment: Segment<Codec>,
-    ) -> crate::Result<Self> {
+    pub fn for_segment(memory_budget_in_bytes: usize, segment: Segment) -> crate::Result<Self> {
         let schema = segment.schema();
         let tokenizer_manager = segment.index().tokenizers().clone();
         let tokenizer_manager_fast_field = segment.index().fast_field_tokenizer().clone();
@@ -150,111 +144,6 @@ impl<Codec: crate::codec::Codec> SegmentWriter<Codec> {
             + self.segment_serializer.mem_usage()
     }
 
-    /// Attaches or updates a codec-specific payload on a term of a regular
-    /// (non-JSON) field.
-    ///
-    /// `value_bytes` is the serialized term value, i.e. exactly what would be
-    /// appended after the field id (the raw text bytes for a str field, or the
-    /// big-endian bytes for a numeric field).
-    ///
-    /// If the term does not exist yet, it is inserted with an empty recorder so
-    /// that it still gets serialized even though it belongs to no document.
-    /// `updater` receives the previously registered payload (`None` if absent)
-    /// and returns the payload to store. The payload is handed to the codec's
-    /// postings serializer (via `set_term_payload`) at the beginning of the
-    /// term during serialization.
-    pub(crate) fn update_term_payload(
-        &mut self,
-        field: Field,
-        value_bytes: &[u8],
-        updater: impl FnOnce(Option<Box<dyn Any + Send>>) -> Box<dyn Any + Send>,
-    ) {
-        let mut term = IndexingTerm::with_capacity(value_bytes.len());
-        term.set_field(field);
-        term.append_bytes(value_bytes);
-        self.update_term_payload_for_serialized_term(field, term.serialized_term(), updater);
-    }
-
-    /// Same as [`Self::update_term_payload`] for a JSON field.
-    ///
-    /// `value_bytes` must be the type-tagged value (`[type code][value]`), the
-    /// representation that follows the path within a JSON term.
-    pub(crate) fn update_json_term_payload(
-        &mut self,
-        field: Field,
-        json_path: &str,
-        value_bytes: &[u8],
-        updater: impl FnOnce(Option<Box<dyn Any + Send>>) -> Box<dyn Any + Send>,
-    ) {
-        let unordered_id = self
-            .ctx
-            .path_to_unordered_id
-            .get_or_allocate_unordered_id(json_path);
-        // JSON term key layout: `[field:4][unordered_path_id:4][type code][value]`.
-        let mut serialized_term = Vec::with_capacity(8 + value_bytes.len());
-        serialized_term.extend_from_slice(&field.field_id().to_be_bytes());
-        serialized_term.extend_from_slice(&unordered_id.to_be_bytes());
-        serialized_term.extend_from_slice(value_bytes);
-        self.update_term_payload_for_serialized_term(field, &serialized_term, updater);
-    }
-
-    fn update_term_payload_for_serialized_term(
-        &mut self,
-        field: Field,
-        serialized_term: &[u8],
-        updater: impl FnOnce(Option<Box<dyn Any + Send>>) -> Box<dyn Any + Send>,
-    ) {
-        let postings_writer = self.per_field_postings_writers.get_for_field(field);
-        let addr = postings_writer.ensure_term(serialized_term, &mut self.ctx);
-        let previous_payload = self.ctx.codec_term_payloads.remove(&addr);
-        let new_payload = updater(previous_payload);
-        self.ctx.codec_term_payloads.insert(addr, new_payload);
-    }
-
-    /// Returns disjoint mutable borrows of the pieces needed to index field
-    /// values outside of `index_document` (e.g. moshiki's placeholder
-    /// routines): the per-field postings writers, the indexing context
-    /// (memory arena + term hashmap), the shared term buffer, and the
-    /// per-field text analyzers (indexed by `Field::field_id`).
-    ///
-    /// The text analyzers are exactly the ones `index_document` uses, so
-    /// indexing a value through them yields identical postings.
-    #[doc(hidden)]
-    pub fn indexing_parts(
-        &mut self,
-    ) -> (
-        &mut PerFieldPostingsWriter,
-        &mut IndexingContext,
-        &mut IndexingTerm,
-        &mut [TextAnalyzer],
-    ) {
-        (
-            &mut self.per_field_postings_writers,
-            &mut self.ctx,
-            &mut self.term_buffer,
-            &mut self.per_field_text_analyzers,
-        )
-    }
-
-    /// Indexes the fast fields of one document from its `(field, value)` pairs, and
-    /// advances `max_doc` by one.
-    ///
-    /// This is for callers (e.g. moshiki) that drive the postings/positions through
-    /// [`indexing_parts`](Self::indexing_parts) with explicit doc ids and need a matching
-    /// fast-field + doc-count pass. It is the document-creating step: it keeps the
-    /// fast-field writer's `num_docs` and `max_doc` in lockstep, so it must be called
-    /// exactly once per document, in doc order.
-    #[doc(hidden)]
-    pub fn add_fast_field_document<'a, V: Value<'a>>(
-        &mut self,
-        fields_and_values: impl Iterator<Item = (Field, V)>,
-    ) -> crate::Result<()> {
-        self.fast_field_writers
-            .add_document_from_values(fields_and_values)?;
-        self.max_doc += 1;
-        Ok(())
-    }
-
     fn index_document<D: Document>(&mut self, doc: &D) -> crate::Result<()> {
         let doc_id = self.max_doc;
 
@@ -280,7 +169,7 @@ impl<Codec: crate::codec::Codec> SegmentWriter<Codec> {
             }
 
             let (term_buffer, ctx) = (&mut self.term_buffer, &mut self.ctx);
-            let postings_writer: &mut PostingsWriterEnum =
+            let postings_writer: &mut dyn PostingsWriter =
                 self.per_field_postings_writers.get_for_field_mut(field);
             term_buffer.clear_with_field(field);
 
@@ -497,13 +386,13 @@ impl<Codec: crate::codec::Codec> SegmentWriter<Codec> {
 /// to the `SegmentSerializer`.
 ///
 /// `doc_id_map` is used to map to the new doc_id order.
-fn remap_and_write<C: Codec>(
+fn remap_and_write(
     schema: Schema,
     per_field_postings_writers: &PerFieldPostingsWriter,
     ctx: IndexingContext,
     fast_field_writers: FastFieldsWriter,
     fieldnorms_writer: &FieldNormsWriter,
-    mut serializer: SegmentSerializer<C>,
+    mut serializer: SegmentSerializer,
 ) -> crate::Result<()> {
     debug!("remap-and-write");
     if let Some(fieldnorms_serializer) = serializer.extract_fieldnorms_serializer() {

src/indexer/single_segment_index_writer.rs

@@ -1,35 +1,28 @@
-use std::any::Any;
 use std::marker::PhantomData;
 
-use crate::codec::StandardCodec;
-use crate::index::CodecConfiguration;
 use crate::indexer::operation::AddOperation;
 use crate::indexer::segment_updater::save_metas;
 use crate::indexer::SegmentWriter;
 use crate::schema::document::Document;
-use crate::schema::{Field, Schema};
 use crate::{Directory, Index, IndexMeta, Opstamp, Segment, TantivyDocument};
 
 #[doc(hidden)]
-pub struct SingleSegmentIndexWriter<
-    Codec: crate::codec::Codec = StandardCodec,
-    D: Document = TantivyDocument,
-> {
-    pub segment_writer: SegmentWriter<Codec>,
-    segment: Segment<Codec>,
+pub struct SingleSegmentIndexWriter<D: Document = TantivyDocument> {
+    segment_writer: SegmentWriter,
+    segment: Segment,
     opstamp: Opstamp,
-    _doc: PhantomData<D>,
+    _phantom: PhantomData<D>,
 }
 
-impl<Codec: crate::codec::Codec, D: Document> SingleSegmentIndexWriter<Codec, D> {
-    pub fn new(index: Index<Codec>, mem_budget: usize) -> crate::Result<Self> {
+impl<D: Document> SingleSegmentIndexWriter<D> {
+    pub fn new(index: Index, mem_budget: usize) -> crate::Result<Self> {
         let segment = index.new_segment();
         let segment_writer = SegmentWriter::for_segment(mem_budget, segment.clone())?;
         Ok(Self {
             segment_writer,
             segment,
             opstamp: 0,
-            _doc: PhantomData,
+            _phantom: PhantomData,
         })
     }
 
@@ -44,51 +37,10 @@ impl<Codec: crate::codec::Codec, D: Document> SingleSegmentIndexWriter<Codec, D>
             .add_document(AddOperation { opstamp, document })
     }
 
-    pub fn schema(&self) -> Schema {
-        self.segment.schema()
-    }
-
-    /// Attaches or updates a codec-specific payload on a term of a regular
-    /// (non-JSON) field.
-    ///
-    /// `value_bytes` is the serialized term value, i.e. exactly what would be
-    /// appended after the field id (the raw text bytes for a str field, or the
-    /// big-endian bytes for a numeric field).
-    ///
-    /// The term does not need to belong to any document: if it does not exist
-    /// yet, it is created with an empty recorder so it still gets serialized.
-    /// `updater` receives the previously registered payload (`None` if absent)
-    /// and returns the payload to store. The payload is handed to the codec at
-    /// the beginning of the term during serialization.
-    pub fn update_term_payload(
-        &mut self,
-        field: Field,
-        value_bytes: &[u8],
-        updater: impl FnOnce(Option<Box<dyn Any + Send>>) -> Box<dyn Any + Send>,
-    ) {
-        self.segment_writer
-            .update_term_payload(field, value_bytes, updater);
-    }
-
-    /// Same as [`Self::update_term_payload`] for a JSON field.
-    ///
-    /// `value_bytes` must be the type-tagged value (`[type code][value]`), the
-    /// representation that follows the path within a JSON term.
-    pub fn update_json_term_payload(
-        &mut self,
-        field: Field,
-        json_path: &str,
-        value_bytes: &[u8],
-        updater: impl FnOnce(Option<Box<dyn Any + Send>>) -> Box<dyn Any + Send>,
-    ) {
-        self.segment_writer
-            .update_json_term_payload(field, json_path, value_bytes, updater);
-    }
-
-    pub fn finalize(self) -> crate::Result<Index<Codec>> {
+    pub fn finalize(self) -> crate::Result<Index> {
         let max_doc = self.segment_writer.max_doc();
         self.segment_writer.finalize()?;
-        let segment: Segment<Codec> = self.segment.with_max_doc(max_doc);
+        let segment: Segment = self.segment.with_max_doc(max_doc);
         let index = segment.index();
         let index_meta = IndexMeta {
             index_settings: index.settings().clone(),
@@ -96,245 +48,9 @@ impl<Codec: crate::codec::Codec, D: Document> SingleSegmentIndexWriter<Codec, D>
             schema: index.schema(),
             opstamp: 0,
             payload: None,
-            codec: CodecConfiguration::from(index.codec()),
         };
         save_metas(&index_meta, index.directory())?;
         index.directory().sync_directory()?;
         Ok(segment.index().clone())
     }
 }
-
-#[cfg(test)]
-mod tests {
-    use std::any::Any;
-    use std::cell::RefCell;
-    use std::io;
-
-    use super::SingleSegmentIndexWriter;
-    use crate::codec::positions::PositionsReader;
-    use crate::codec::postings::{PostingsCodec, PostingsSerializer};
-    use crate::codec::standard::positions::StandardPositionsCodec;
-    use crate::codec::standard::postings::{
-        SegmentPostings, StandardPostingsCodec, StandardPostingsSerializer,
-    };
-    use crate::codec::Codec;
-    use crate::fieldnorm::FieldNormReader;
-    use crate::schema::{IndexRecordOption, Schema, Type, STRING};
-    use crate::{DocId, Score, Term};
-
-    // The codec is round-tripped through `from_json_props` when the index is
-    // opened, so it cannot carry the capture sink itself. We use a thread-local
-    // sink instead: the `SingleSegmentIndexWriter` is single-threaded, so
-    // serialization runs on the test thread, and each test owns its own
-    // thread-local (clear it at the start of the test).
-    thread_local! {
-        static CAPTURED_PAYLOADS: RefCell<Vec<u64>> = const { RefCell::new(Vec::new()) };
-    }
-
-    fn reset_captured() {
-        CAPTURED_PAYLOADS.with(|captured| captured.borrow_mut().clear());
-    }
-
-    fn captured_payloads() -> Vec<u64> {
-        CAPTURED_PAYLOADS.with(|captured| captured.borrow().clone())
-    }
-
-    /// A postings serializer that delegates to the standard one, but records
-    /// the `u64` payload value of every term that carries a codec payload.
-    struct CapturingPostingsSerializer {
-        inner: StandardPostingsSerializer,
-    }
-
-    impl PostingsSerializer for CapturingPostingsSerializer {
-        fn new_term(&mut self, term_doc_freq: u32, record_term_freq: bool) {
-            self.inner.new_term(term_doc_freq, record_term_freq);
-        }
-
-        fn set_term_payload(&mut self, payload: &dyn Any) {
-            let value = *payload
-                .downcast_ref::<u64>()
-                .expect("payload should be a u64");
-            CAPTURED_PAYLOADS.with(|captured| captured.borrow_mut().push(value));
-        }
-
-        fn write_doc(&mut self, doc_id: DocId, term_freq: u32) {
-            self.inner.write_doc(doc_id, term_freq);
-        }
-
-        fn close_term(&mut self, doc_freq: u32, wrt: &mut impl io::Write) -> io::Result<()> {
-            self.inner.close_term(doc_freq, wrt)
-        }
-    }
-
-    #[derive(Clone, Debug)]
-    struct CapturingPostingsCodec;
-
-    impl PostingsCodec for CapturingPostingsCodec {
-        type PostingsSerializer = CapturingPostingsSerializer;
-        type Postings = SegmentPostings;
-
-        fn new_serializer(
-            &self,
-            avg_fieldnorm: Score,
-            mode: IndexRecordOption,
-            fieldnorm_reader: Option<FieldNormReader>,
-        ) -> Self::PostingsSerializer {
-            CapturingPostingsSerializer {
-                inner: StandardPostingsCodec.new_serializer(avg_fieldnorm, mode, fieldnorm_reader),
-            }
-        }
-
-        fn load_postings(
-            &self,
-            doc_freq: u32,
-            postings_data: common::OwnedBytes,
-            record_option: IndexRecordOption,
-            requested_option: IndexRecordOption,
-            position_reader: Option<Box<dyn PositionsReader>>,
-        ) -> io::Result<Self::Postings> {
-            StandardPostingsCodec.load_postings(
-                doc_freq,
-                postings_data,
-                record_option,
-                requested_option,
-                position_reader,
-            )
-        }
-    }
-
-    #[derive(Clone, Debug, Default)]
-    struct CapturingCodec;
-
-    impl Codec for CapturingCodec {
-        type PostingsCodec = CapturingPostingsCodec;
-        type PositionsCodec = StandardPositionsCodec;
-
-        const ID: &'static str = "test-capturing-codec";
-
-        fn from_json_props(_json_value: &serde_json::Value) -> crate::Result<Self> {
-            Ok(CapturingCodec)
-        }
-
-        fn to_json_props(&self) -> serde_json::Value {
-            serde_json::Value::Null
-        }
-
-        fn postings_codec(&self) -> &Self::PostingsCodec {
-            &CapturingPostingsCodec
-        }
-
-        fn positions_codec(&self) -> &Self::PositionsCodec {
-            &StandardPositionsCodec
-        }
-    }
-
-    fn build_writer(schema: Schema) -> SingleSegmentIndexWriter<CapturingCodec> {
-        let index = crate::IndexBuilder::default()
-            .codec(CapturingCodec)
-            .schema(schema)
-            .create_in_ram()
-            .unwrap();
-        SingleSegmentIndexWriter::new(index, 15_000_000).unwrap()
-    }
-
-    #[test]
-    fn test_update_term_payload_regular_field() {
-        reset_captured();
-        let mut schema_builder = Schema::builder();
-        let text = schema_builder.add_text_field("text", STRING);
-        let schema = schema_builder.build();
-        let mut writer = build_writer(schema);
-
-        writer.add_document(crate::doc!(text => "alpha")).unwrap();
-        writer.add_document(crate::doc!(text => "beta")).unwrap();
-        writer.add_document(crate::doc!(text => "gamma")).unwrap();
-
-        // Existing term that belongs to a document.
-        writer.update_term_payload(text, b"beta", |previous_payload| {
-            assert!(previous_payload.is_none());
-            Box::new(100u64)
-        });
-        // Updating the same term: the previous payload is handed back.
-        writer.update_term_payload(text, b"beta", |previous_payload| {
-            let previous = previous_payload.expect("expected the previous payload");
-            assert_eq!(*previous.downcast::<u64>().unwrap(), 100u64);
-            Box::new(101u64)
-        });
-        // Brand-new term that belongs to no document: an empty recorder is
-        // created so it still lands in the term dictionary.
-        writer.update_term_payload(text, b"zeta", |previous_payload| {
-            assert!(previous_payload.is_none());
-            Box::new(200u64)
-        });
-
-        let index = writer.finalize().unwrap();
-
-        // Terms are serialized in sorted order: alpha, beta, gamma, zeta.
-        // Only beta and zeta carry a payload.
-        assert_eq!(captured_payloads(), vec![101u64, 200u64]);
-
-        let searcher = index.reader().unwrap().searcher();
-        let segment_reader = searcher.segment_reader(0);
-        let inverted_index = segment_reader.inverted_index(text).unwrap();
-
-        let beta_info = inverted_index
-            .get_term_info(&Term::from_field_text(text, "beta"))
-            .unwrap()
-            .expect("beta should be in the dictionary");
-        assert_eq!(beta_info.doc_freq, 1);
-
-        let zeta_info = inverted_index
-            .get_term_info(&Term::from_field_text(text, "zeta"))
-            .unwrap()
-            .expect("zeta (no document) should still be in the dictionary");
-        assert_eq!(zeta_info.doc_freq, 0);
-    }
-
-    #[test]
-    fn test_update_json_term_payload() {
-        reset_captured();
-        let mut schema_builder = Schema::builder();
-        let json_field = schema_builder.add_json_field("json", STRING);
-        let schema = schema_builder.build();
-        let mut writer = build_writer(schema);
-
-        writer
-            .add_document(crate::doc!(json_field => serde_json::json!({"name": "hello"})))
-            .unwrap();
-
-        let str_value = |value: &str| {
-            let mut bytes = vec![Type::Str.to_code()];
-            bytes.extend_from_slice(value.as_bytes());
-            bytes
-        };
-
-        // Existing str JSON term (path "name", value "hello").
-        writer.update_json_term_payload(json_field, "name", &str_value("hello"), |previous| {
-            assert!(previous.is_none());
-            Box::new(1u64)
-        });
-        // Brand-new str JSON term with no document.
-        writer.update_json_term_payload(json_field, "name", &str_value("world"), |previous| {
-            assert!(previous.is_none());
-            Box::new(2u64)
-        });
-        // Brand-new non-str (numeric) JSON term with no document: exercises the
-        // DocIdRecorder branch of `ensure_term`.
-        let numeric_value = {
-            let mut bytes = vec![Type::I64.to_code()];
-            bytes.extend_from_slice(&[0u8; 8]);
-            bytes
-        };
-        writer.update_json_term_payload(json_field, "count", &numeric_value, |previous| {
-            assert!(previous.is_none());
-            Box::new(3u64)
-        });
-
-        // Should not panic and should serialize cleanly.
-        let _index = writer.finalize().unwrap();
-
-        let mut got = captured_payloads();
-        got.sort_unstable();
-        assert_eq!(got, vec![1u64, 2u64, 3u64]);
-    }
-}

src/lib.rs

@@ -166,9 +166,6 @@ mod functional_test;
 
 #[macro_use]
 mod macros;
-
-/// Tantivy codecs describes how data is layed out on disk.
-pub mod codec;
 mod future_result;
 
 // Re-exports
@@ -221,7 +218,7 @@ pub mod snippet;
 use std::fmt;
 
 pub use census::{Inventory, TrackedObject};
-pub use common::{self, f64_to_u64, i64_to_u64, u64_to_f64, u64_to_i64, HasLen};
+pub use common::{f64_to_u64, i64_to_u64, u64_to_f64, u64_to_i64, HasLen};
 use once_cell::sync::Lazy;
 use serde::{Deserialize, Serialize};
 

src/postings/block_segment_postings.rs

@@ -1,19 +1,28 @@
 use std::io;
 
-use common::{OwnedBytes, VInt};
+use common::VInt;
 
-use crate::codec::standard::postings::skip::{BlockInfo, SkipReader};
-use crate::codec::standard::postings::FreqReadingOption;
+use crate::directory::{FileSlice, OwnedBytes};
 use crate::fieldnorm::FieldNormReader;
-use crate::postings::compression::{BlockDecoder, VIntDecoder as _, COMPRESSION_BLOCK_SIZE};
+use crate::postings::compression::{BlockDecoder, VIntDecoder, COMPRESSION_BLOCK_SIZE};
+use crate::postings::{BlockInfo, FreqReadingOption, SkipReader};
 use crate::query::Bm25Weight;
 use crate::schema::IndexRecordOption;
 use crate::{DocId, Score, TERMINATED};
 
+fn max_score<I: Iterator<Item = Score>>(mut it: I) -> Option<Score> {
+    it.next().map(|first| it.fold(first, Score::max))
+}
+
 /// `BlockSegmentPostings` is a cursor iterating over blocks
 /// of documents.
+///
+/// # Warning
+///
+/// While it is useful for some very specific high-performance
+/// use cases, you should prefer using `SegmentPostings` for most usage.
 #[derive(Clone)]
-pub(crate) struct BlockSegmentPostings {
+pub struct BlockSegmentPostings {
     pub(crate) doc_decoder: BlockDecoder,
     block_loaded: bool,
     freq_decoder: BlockDecoder,
@@ -79,7 +88,7 @@ fn split_into_skips_and_postings(
 }
 
 impl BlockSegmentPostings {
-    /// Opens a `StandardPostingsReader`.
+    /// Opens a `BlockSegmentPostings`.
     /// `doc_freq` is the number of documents in the posting list.
     /// `record_option` represents the amount of data available according to the schema.
     /// `requested_option` is the amount of data requested by the user.
@@ -87,10 +96,11 @@ impl BlockSegmentPostings {
     /// term frequency blocks.
     pub(crate) fn open(
         doc_freq: u32,
-        bytes: OwnedBytes,
+        data: FileSlice,
         mut record_option: IndexRecordOption,
         requested_option: IndexRecordOption,
     ) -> io::Result<BlockSegmentPostings> {
+        let bytes = data.read_bytes()?;
         let (skip_data_opt, postings_data) = split_into_skips_and_postings(doc_freq, bytes)?;
         let skip_reader = match skip_data_opt {
             Some(skip_data) => {
@@ -128,86 +138,6 @@ impl BlockSegmentPostings {
         block_segment_postings.load_block();
         Ok(block_segment_postings)
     }
-}
-
-fn max_score<I: Iterator<Item = Score>>(mut it: I) -> Option<Score> {
-    it.next().map(|first| it.fold(first, Score::max))
-}
-
-impl BlockSegmentPostings {
-    /// Returns the overall number of documents in the block postings.
-    /// It does not take in account whether documents are deleted or not.
-    ///
-    /// This `doc_freq` is simply the sum of the length of all of the blocks
-    /// length, and it does not take in account deleted documents.
-    pub fn doc_freq(&self) -> u32 {
-        self.doc_freq
-    }
-
-    /// Returns the array of docs in the current block.
-    ///
-    /// Before the first call to `.advance()`, the block
-    /// returned by `.docs()` is empty.
-    #[inline]
-    pub fn docs(&self) -> &[DocId] {
-        debug_assert!(self.block_loaded);
-        self.doc_decoder.output_array()
-    }
-
-    /// Return the document at index `idx` of the block.
-    #[inline]
-    pub fn doc(&self, idx: usize) -> u32 {
-        self.doc_decoder.output(idx)
-    }
-
-    /// Return the array of `term freq` in the block.
-    #[inline]
-    pub fn freqs(&self) -> &[u32] {
-        debug_assert!(self.block_loaded);
-        self.freq_decoder.output_array()
-    }
-
-    /// Return the frequency at index `idx` of the block.
-    #[inline]
-    pub fn freq(&self, idx: usize) -> u32 {
-        debug_assert!(self.block_loaded);
-        self.freq_decoder.output(idx)
-    }
-
-    /// Position on a block that may contains `target_doc`.
-    ///
-    /// If all docs are smaller than target, the block loaded may be empty,
-    /// or be the last an incomplete VInt block.
-    pub fn seek(&mut self, target_doc: DocId) -> usize {
-        // Move to the block that might contain our document.
-        self.seek_block_without_loading(target_doc);
-        self.load_block();
-
-        // At this point we are on the block that might contain our document.
-        let doc = self.doc_decoder.seek_within_block(target_doc);
-
-        // The last block is not full and padded with TERMINATED,
-        // so we are guaranteed to have at least one value (real or padding)
-        // that is >= target_doc.
-        debug_assert!(doc < COMPRESSION_BLOCK_SIZE);
-
-        // `doc` is now the first element >= `target_doc`.
-        // If all docs are smaller than target, the current block is incomplete and padded
-        // with TERMINATED. After the search, the cursor points to the first TERMINATED.
-        doc
-    }
-
-    pub fn position_offset(&self) -> u64 {
-        self.skip_reader.position_offset()
-    }
-
-    /// Advance to the next block.
-    pub fn advance(&mut self) {
-        self.skip_reader.advance();
-        self.block_loaded = false;
-        self.block_max_score_cache = None;
-        self.load_block();
-    }
 
     /// Returns the block_max_score for the current block.
     /// It does not require the block to be loaded. For instance, it is ok to call this method
@@ -230,7 +160,7 @@ impl BlockSegmentPostings {
         }
         // this is the last block of the segment posting list.
         // If it is actually loaded, we can compute block max manually.
-        if self.block_loaded {
+        if self.block_is_loaded() {
             let docs = self.doc_decoder.output_array().iter().cloned();
             let freqs = self.freq_decoder.output_array().iter().cloned();
             let bm25_scores = docs.zip(freqs).map(|(doc, term_freq)| {
@@ -247,25 +177,154 @@ impl BlockSegmentPostings {
         // We do not cache it however, so that it gets computed when once block is loaded.
         bm25_weight.max_score()
     }
-}
 
-impl BlockSegmentPostings {
-    /// Returns an empty segment postings object
-    pub fn empty() -> BlockSegmentPostings {
-        BlockSegmentPostings {
-            doc_decoder: BlockDecoder::with_val(TERMINATED),
-            block_loaded: true,
-            freq_decoder: BlockDecoder::with_val(1),
-            freq_reading_option: FreqReadingOption::NoFreq,
-            block_max_score_cache: None,
-            doc_freq: 0,
-            data: OwnedBytes::empty(),
-            skip_reader: SkipReader::new(OwnedBytes::empty(), 0, IndexRecordOption::Basic),
-        }
+    pub(crate) fn freq_reading_option(&self) -> FreqReadingOption {
+        self.freq_reading_option
     }
 
-    pub(crate) fn skip_reader(&self) -> &SkipReader {
-        &self.skip_reader
+    // Resets the block segment postings on another position
+    // in the postings file.
+    //
+    // This is useful for enumerating through a list of terms,
+    // and consuming the associated posting lists while avoiding
+    // reallocating a `BlockSegmentPostings`.
+    //
+    // # Warning
+    //
+    // This does not reset the positions list.
+    pub(crate) fn reset(&mut self, doc_freq: u32, postings_data: OwnedBytes) -> io::Result<()> {
+        let (skip_data_opt, postings_data) =
+            split_into_skips_and_postings(doc_freq, postings_data)?;
+        self.data = postings_data;
+        self.block_max_score_cache = None;
+        self.block_loaded = false;
+        if let Some(skip_data) = skip_data_opt {
+            self.skip_reader.reset(skip_data, doc_freq);
+        } else {
+            self.skip_reader.reset(OwnedBytes::empty(), doc_freq);
+        }
+        self.doc_freq = doc_freq;
+        self.load_block();
+        Ok(())
+    }
+
+    /// Returns the overall number of documents in the block postings.
+    /// It does not take in account whether documents are deleted or not.
+    ///
+    /// This `doc_freq` is simply the sum of the length of all of the blocks
+    /// length, and it does not take in account deleted documents.
+    pub fn doc_freq(&self) -> u32 {
+        self.doc_freq
+    }
+
+    /// Returns the array of docs in the current block.
+    ///
+    /// Before the first call to `.advance()`, the block
+    /// returned by `.docs()` is empty.
+    #[inline]
+    pub fn docs(&self) -> &[DocId] {
+        debug_assert!(self.block_is_loaded());
+        self.doc_decoder.output_array()
+    }
+
+    /// Return the document at index `idx` of the block.
+    #[inline]
+    pub fn doc(&self, idx: usize) -> u32 {
+        self.doc_decoder.output(idx)
+    }
+
+    /// Return the array of `term freq` in the block.
+    #[inline]
+    pub fn freqs(&self) -> &[u32] {
+        debug_assert!(self.block_is_loaded());
+        self.freq_decoder.output_array()
+    }
+
+    pub(crate) fn copy_docs_and_term_freqs(
+        &self,
+        block_offset: usize,
+        horizon: DocId,
+        docs: &mut [DocId],
+        term_freqs: &mut [u32],
+    ) -> usize {
+        debug_assert_eq!(docs.len(), term_freqs.len());
+        let block_docs = self.docs();
+        let remaining_docs_in_block = block_docs.len().saturating_sub(block_offset);
+        let max_len = remaining_docs_in_block.min(docs.len());
+        if max_len == 0 {
+            return 0;
+        }
+
+        let source_docs = &block_docs[block_offset..block_offset + max_len];
+        let len = if source_docs[max_len - 1] < horizon {
+            max_len
+        } else {
+            source_docs
+                .iter()
+                .position(|&doc| doc >= horizon)
+                .unwrap_or(max_len)
+        };
+
+        docs[..len].copy_from_slice(&source_docs[..len]);
+
+        let block_freqs = self.freq_output_array();
+        if block_freqs.len() >= block_offset + len {
+            term_freqs[..len].copy_from_slice(&block_freqs[block_offset..block_offset + len]);
+        } else {
+            term_freqs[..len].fill(1);
+        }
+        len
+    }
+
+    /// Return the frequency at index `idx` of the block.
+    #[inline]
+    pub fn freq(&self, idx: usize) -> u32 {
+        debug_assert!(self.block_is_loaded());
+        self.freq_decoder.output(idx)
+    }
+
+    /// Returns the length of the current block.
+    ///
+    /// Returns the decoded term-frequency buffer for the current block.
+    #[inline]
+    pub(crate) fn freq_output_array(&self) -> &[u32] {
+        self.freq_decoder.output_array()
+    }
+
+    /// All blocks have a length of `NUM_DOCS_PER_BLOCK`,
+    /// except the last block that may have a length
+    /// of any number between 1 and `NUM_DOCS_PER_BLOCK - 1`
+    #[inline]
+    pub fn block_len(&self) -> usize {
+        debug_assert!(self.block_is_loaded());
+        self.doc_decoder.output_len
+    }
+
+    /// Position on a block that may contains `target_doc`.
+    ///
+    /// If all docs are smaller than target, the block loaded may be empty,
+    /// or be the last an incomplete VInt block.
+    pub fn seek(&mut self, target_doc: DocId) -> usize {
+        // Move to the block that might contain our document.
+        self.seek_block(target_doc);
+        self.load_block();
+
+        // At this point we are on the block that might contain our document.
+        let doc = self.doc_decoder.seek_within_block(target_doc);
+
+        // The last block is not full and padded with TERMINATED,
+        // so we are guaranteed to have at least one value (real or padding)
+        // that is >= target_doc.
+        debug_assert!(doc < COMPRESSION_BLOCK_SIZE);
+
+        // `doc` is now the first element >= `target_doc`.
+        // If all docs are smaller than target, the current block is incomplete and padded
+        // with TERMINATED. After the search, the cursor points to the first TERMINATED.
+        doc
+    }
+
+    pub(crate) fn position_offset(&self) -> u64 {
+        self.skip_reader.position_offset()
     }
 
     /// Dangerous API! This calls seeks the next block on the skip list,
@@ -274,15 +333,24 @@ impl BlockSegmentPostings {
     /// `.load_block()` needs to be called manually afterwards.
     /// If all docs are smaller than target, the block loaded may be empty,
     /// or be the last an incomplete VInt block.
-    pub(crate) fn seek_block_without_loading(&mut self, target_doc: DocId) {
+    pub(crate) fn seek_block(&mut self, target_doc: DocId) {
         if self.skip_reader.seek(target_doc) {
             self.block_max_score_cache = None;
             self.block_loaded = false;
         }
     }
 
+    #[inline]
+    pub(crate) fn has_remaining_docs(&self) -> bool {
+        self.skip_reader.has_remaining_docs()
+    }
+
+    pub(crate) fn block_is_loaded(&self) -> bool {
+        self.block_loaded
+    }
+
     pub(crate) fn load_block(&mut self) {
-        if self.block_loaded {
+        if self.block_is_loaded() {
             return;
         }
         let offset = self.skip_reader.byte_offset();
@@ -330,40 +398,68 @@ impl BlockSegmentPostings {
         }
         self.block_loaded = true;
     }
+
+    /// Advance to the next block.
+    pub fn advance(&mut self) {
+        self.skip_reader.advance();
+        self.block_loaded = false;
+        self.block_max_score_cache = None;
+        self.load_block();
+    }
+
+    /// Returns an empty segment postings object
+    pub fn empty() -> BlockSegmentPostings {
+        BlockSegmentPostings {
+            doc_decoder: BlockDecoder::with_val(TERMINATED),
+            block_loaded: true,
+            freq_decoder: BlockDecoder::with_val(1),
+            freq_reading_option: FreqReadingOption::NoFreq,
+            block_max_score_cache: None,
+            doc_freq: 0,
+            data: OwnedBytes::empty(),
+            skip_reader: SkipReader::new(OwnedBytes::empty(), 0, IndexRecordOption::Basic),
+        }
+    }
+
+    pub(crate) fn skip_reader(&self) -> &SkipReader {
+        &self.skip_reader
+    }
 }
 
 #[cfg(test)]
 mod tests {
-    use common::OwnedBytes;
+    use common::HasLen;
 
     use super::BlockSegmentPostings;
-    use crate::codec::postings::PostingsSerializer;
-    use crate::codec::standard::postings::segment_postings::SegmentPostings;
-    use crate::codec::standard::postings::StandardPostingsSerializer;
     use crate::docset::{DocSet, TERMINATED};
+    use crate::index::Index;
     use crate::postings::compression::COMPRESSION_BLOCK_SIZE;
-    use crate::schema::IndexRecordOption;
+    use crate::postings::postings::Postings;
+    use crate::postings::SegmentPostings;
+    use crate::schema::{IndexRecordOption, Schema, Term, INDEXED};
+    use crate::DocId;
 
-    #[cfg(test)]
-    fn build_block_postings(docs: &[u32]) -> BlockSegmentPostings {
-        let doc_freq = docs.len() as u32;
-        let mut postings_serializer =
-            StandardPostingsSerializer::new(1.0f32, IndexRecordOption::Basic, None);
-        postings_serializer.new_term(docs.len() as u32, false);
-        for doc in docs {
-            postings_serializer.write_doc(*doc, 1u32);
-        }
-        let mut buffer: Vec<u8> = Vec::new();
-        postings_serializer
-            .close_term(doc_freq, &mut buffer)
-            .unwrap();
-        BlockSegmentPostings::open(
-            doc_freq,
-            OwnedBytes::new(buffer),
-            IndexRecordOption::Basic,
-            IndexRecordOption::Basic,
-        )
-        .unwrap()
+    #[test]
+    fn test_empty_segment_postings() {
+        let mut postings = SegmentPostings::empty();
+        assert_eq!(postings.doc(), TERMINATED);
+        assert_eq!(postings.advance(), TERMINATED);
+        assert_eq!(postings.advance(), TERMINATED);
+        assert_eq!(postings.doc_freq(), 0);
+        assert_eq!(postings.len(), 0);
+    }
+
+    #[test]
+    fn test_empty_postings_doc_returns_terminated() {
+        let mut postings = SegmentPostings::empty();
+        assert_eq!(postings.doc(), TERMINATED);
+        assert_eq!(postings.advance(), TERMINATED);
+    }
+
+    #[test]
+    fn test_empty_postings_doc_term_freq_returns_0() {
+        let postings = SegmentPostings::empty();
+        assert_eq!(postings.term_freq(), 1);
     }
 
     #[test]
@@ -378,7 +474,7 @@ mod tests {
 
     #[test]
     fn test_block_segment_postings() -> crate::Result<()> {
-        let mut block_segments = build_block_postings(&(0..100_000).collect::<Vec<u32>>());
+        let mut block_segments = build_block_postings(&(0..100_000).collect::<Vec<u32>>())?;
         let mut offset: u32 = 0u32;
         // checking that the `doc_freq` is correct
         assert_eq!(block_segments.doc_freq(), 100_000);
@@ -403,7 +499,7 @@ mod tests {
         doc_ids.push(129);
         doc_ids.push(130);
         {
-            let block_segments = build_block_postings(&doc_ids);
+            let block_segments = build_block_postings(&doc_ids)?;
             let mut docset = SegmentPostings::from_block_postings(block_segments, None);
             assert_eq!(docset.seek(128), 129);
             assert_eq!(docset.doc(), 129);
@@ -412,7 +508,7 @@ mod tests {
             assert_eq!(docset.advance(), TERMINATED);
         }
         {
-            let block_segments = build_block_postings(&doc_ids);
+            let block_segments = build_block_postings(&doc_ids).unwrap();
             let mut docset = SegmentPostings::from_block_postings(block_segments, None);
             assert_eq!(docset.seek(129), 129);
             assert_eq!(docset.doc(), 129);
@@ -421,7 +517,7 @@ mod tests {
             assert_eq!(docset.advance(), TERMINATED);
         }
         {
-            let block_segments = build_block_postings(&doc_ids);
+            let block_segments = build_block_postings(&doc_ids)?;
             let mut docset = SegmentPostings::from_block_postings(block_segments, None);
             assert_eq!(docset.doc(), 0);
             assert_eq!(docset.seek(131), TERMINATED);
@@ -430,13 +526,38 @@ mod tests {
         Ok(())
     }
 
+    fn build_block_postings(docs: &[DocId]) -> crate::Result<BlockSegmentPostings> {
+        let mut schema_builder = Schema::builder();
+        let int_field = schema_builder.add_u64_field("id", INDEXED);
+        let schema = schema_builder.build();
+        let index = Index::create_in_ram(schema);
+        let mut index_writer = index.writer_for_tests()?;
+        let mut last_doc = 0u32;
+        for &doc in docs {
+            for _ in last_doc..doc {
+                index_writer.add_document(doc!(int_field=>1u64))?;
+            }
+            index_writer.add_document(doc!(int_field=>0u64))?;
+            last_doc = doc + 1;
+        }
+        index_writer.commit()?;
+        let searcher = index.reader()?.searcher();
+        let segment_reader = searcher.segment_reader(0);
+        let inverted_index = segment_reader.inverted_index(int_field).unwrap();
+        let term = Term::from_field_u64(int_field, 0u64);
+        let term_info = inverted_index.get_term_info(&term)?.unwrap();
+        let block_postings = inverted_index
+            .read_block_postings_from_terminfo(&term_info, IndexRecordOption::Basic)?;
+        Ok(block_postings)
+    }
+
     #[test]
     fn test_block_segment_postings_seek() -> crate::Result<()> {
-        let mut docs = Vec::new();
+        let mut docs = vec![0];
         for i in 0..1300 {
             docs.push((i * i / 100) + i);
         }
-        let mut block_postings = build_block_postings(&docs[..]);
+        let mut block_postings = build_block_postings(&docs[..])?;
         for i in &[0, 424, 10000] {
             block_postings.seek(*i);
             let docs = block_postings.docs();
@@ -447,4 +568,40 @@ mod tests {
         assert_eq!(block_postings.doc(COMPRESSION_BLOCK_SIZE - 1), TERMINATED);
         Ok(())
     }
+
+    #[test]
+    fn test_reset_block_segment_postings() -> crate::Result<()> {
+        let mut schema_builder = Schema::builder();
+        let int_field = schema_builder.add_u64_field("id", INDEXED);
+        let schema = schema_builder.build();
+        let index = Index::create_in_ram(schema);
+        let mut index_writer = index.writer_for_tests()?;
+        // create two postings list, one containing even number,
+        // the other containing odd numbers.
+        for i in 0..6 {
+            let doc = doc!(int_field=> (i % 2) as u64);
+            index_writer.add_document(doc)?;
+        }
+        index_writer.commit()?;
+        let searcher = index.reader()?.searcher();
+        let segment_reader = searcher.segment_reader(0);
+
+        let mut block_segments;
+        {
+            let term = Term::from_field_u64(int_field, 0u64);
+            let inverted_index = segment_reader.inverted_index(int_field)?;
+            let term_info = inverted_index.get_term_info(&term)?.unwrap();
+            block_segments = inverted_index
+                .read_block_postings_from_terminfo(&term_info, IndexRecordOption::Basic)?;
+        }
+        assert_eq!(block_segments.docs(), &[0, 2, 4]);
+        {
+            let term = Term::from_field_u64(int_field, 1u64);
+            let inverted_index = segment_reader.inverted_index(int_field)?;
+            let term_info = inverted_index.get_term_info(&term)?.unwrap();
+            inverted_index.reset_block_postings_from_terminfo(&term_info, &mut block_segments)?;
+        }
+        assert_eq!(block_segments.docs(), &[1, 3, 5]);
+        Ok(())
+    }
 }

src/postings/indexing_context.rs

@@ -1,29 +1,16 @@
-use std::any::Any;
-
-use fnv::FnvHashMap;
-use stacker::{Addr, ArenaHashMap, MemoryArena};
+use stacker::{ArenaHashMap, MemoryArena};
 
 use crate::indexer::path_to_unordered_id::PathToUnorderedId;
 
 /// IndexingContext contains all of the transient memory arenas
 /// required for building the inverted index.
-#[doc(hidden)]
-pub struct IndexingContext {
+pub(crate) struct IndexingContext {
     /// The term index is an adhoc hashmap,
     /// itself backed by a dedicated memory arena.
-    pub(crate) term_index: ArenaHashMap,
+    pub term_index: ArenaHashMap,
     /// Arena is a memory arena that stores posting lists / term frequencies / positions.
-    pub(crate) arena: MemoryArena,
-    pub(crate) path_to_unordered_id: PathToUnorderedId,
-    /// Optional codec-specific payload attached to a term, keyed by the value
-    /// `Addr` of the term's recorder in `term_index`.
-    ///
-    /// Hidden contract: keying on `Addr` is sound because a term's recorder
-    /// address never changes once allocated (the arena only appends, and
-    /// `subscribe` updates the recorder in place). The payload is therefore
-    /// looked up by `Addr` at serialization time and fed to the codec's
-    /// postings serializer at the beginning of the term.
-    pub(crate) codec_term_payloads: FnvHashMap<Addr, Box<dyn Any + Send>>,
+    pub arena: MemoryArena,
+    pub path_to_unordered_id: PathToUnorderedId,
 }
 
 impl IndexingContext {
@@ -34,7 +21,6 @@ impl IndexingContext {
             arena: MemoryArena::default(),
             term_index,
             path_to_unordered_id: PathToUnorderedId::default(),
-            codec_term_payloads: FnvHashMap::default(),
         }
     }
 

src/postings/json_postings_writer.rs

@@ -3,7 +3,6 @@ use std::io;
 use common::json_path_writer::JSON_END_OF_PATH;
 use stacker::Addr;
 
-use crate::codec::Codec;
 use crate::indexer::indexing_term::IndexingTerm;
 use crate::indexer::path_to_unordered_id::OrderedPathId;
 use crate::postings::postings_writer::SpecializedPostingsWriter;
@@ -18,11 +17,17 @@ use crate::DocId;
 /// `subscribe` is called directly to index non-text tokens, while
 /// `index_text` is used to index text.
 #[derive(Default)]
-pub struct JsonPostingsWriter<Rec: Recorder> {
+pub(crate) struct JsonPostingsWriter<Rec: Recorder> {
     str_posting_writer: SpecializedPostingsWriter<Rec>,
     non_str_posting_writer: SpecializedPostingsWriter<DocIdRecorder>,
 }
 
+impl<Rec: Recorder> From<JsonPostingsWriter<Rec>> for Box<dyn PostingsWriter> {
+    fn from(json_postings_writer: JsonPostingsWriter<Rec>) -> Box<dyn PostingsWriter> {
+        Box::new(json_postings_writer)
+    }
+}
+
 impl<Rec: Recorder> PostingsWriter for JsonPostingsWriter<Rec> {
     #[inline]
     fn subscribe(
@@ -53,12 +58,12 @@ impl<Rec: Recorder> PostingsWriter for JsonPostingsWriter<Rec> {
     }
 
     /// The actual serialization format is handled by the `PostingsSerializer`.
-    fn serialize<C: Codec>(
+    fn serialize(
         &self,
         ordered_term_addrs: &[(Field, OrderedPathId, &[u8], Addr)],
         ordered_id_to_path: &[&str],
         ctx: &IndexingContext,
-        serializer: &mut FieldSerializer<C>,
+        serializer: &mut FieldSerializer,
     ) -> io::Result<()> {
         let mut term_buffer = JsonTermSerializer(Vec::with_capacity(48));
         let mut buffer_lender = BufferLender::default();
@@ -96,20 +101,6 @@ impl<Rec: Recorder> PostingsWriter for JsonPostingsWriter<Rec> {
         Ok(())
     }
 
-    fn ensure_term(&self, serialized_term: &[u8], ctx: &mut IndexingContext) -> Addr {
-        // JSON term key layout: `[field:4][unordered_path_id:4][type code][value]`.
-        // Str values are recorded with `Rec`, all other types with `DocIdRecorder`
-        // (mirroring the dispatch in `serialize`).
-        let typ = Type::from_code(serialized_term[8]).expect("Invalid type code in JSON term");
-        if typ == Type::Str {
-            ctx.term_index
-                .get_or_create_value_addr::<Rec>(serialized_term, Rec::default)
-        } else {
-            ctx.term_index
-                .get_or_create_value_addr::<DocIdRecorder>(serialized_term, DocIdRecorder::default)
-        }
-    }
-
     fn total_num_tokens(&self) -> u64 {
         self.str_posting_writer.total_num_tokens() + self.non_str_posting_writer.total_num_tokens()
     }

src/postings/loaded_postings.rs

@@ -1,5 +1,5 @@
 use crate::docset::{DocSet, TERMINATED};
-use crate::postings::{DocFreq, Postings};
+use crate::postings::{Postings, SegmentPostings};
 use crate::DocId;
 
 /// `LoadedPostings` is a `DocSet` and `Postings` implementation.
@@ -25,16 +25,16 @@ impl LoadedPostings {
     /// Creates a new `LoadedPostings` from a `SegmentPostings`.
     ///
     /// It will also preload positions, if positions are available in the SegmentPostings.
-    pub fn load(postings: &mut Box<dyn Postings>) -> LoadedPostings {
-        let num_docs: usize = u32::from(postings.doc_freq()) as usize;
+    pub fn load(segment_postings: &mut SegmentPostings) -> LoadedPostings {
+        let num_docs = segment_postings.doc_freq() as usize;
         let mut doc_ids = Vec::with_capacity(num_docs);
         let mut positions = Vec::with_capacity(num_docs);
         let mut position_offsets = Vec::with_capacity(num_docs);
-        while postings.doc() != TERMINATED {
+        while segment_postings.doc() != TERMINATED {
             position_offsets.push(positions.len() as u32);
-            doc_ids.push(postings.doc());
-            postings.append_positions_with_offset(0, &mut positions);
-            postings.advance();
+            doc_ids.push(segment_postings.doc());
+            segment_postings.append_positions_with_offset(0, &mut positions);
+            segment_postings.advance();
         }
         position_offsets.push(positions.len() as u32);
         LoadedPostings {
@@ -101,14 +101,6 @@ impl Postings for LoadedPostings {
             output.push(*pos + offset);
         }
     }
-
-    fn has_freq(&self) -> bool {
-        true
-    }
-
-    fn doc_freq(&self) -> DocFreq {
-        DocFreq::Exact(self.doc_ids.len() as u32)
-    }
 }
 
 #[cfg(test)]

src/postings/mod.rs

@@ -4,6 +4,7 @@ mod block_search;
 
 pub(crate) use self::block_search::branchless_binary_search;
 
+mod block_segment_postings;
 pub(crate) mod compression;
 mod indexing_context;
 mod json_postings_writer;
@@ -12,29 +13,33 @@ mod per_field_postings_writer;
 mod postings;
 mod postings_writer;
 mod recorder;
+mod segment_postings;
 /// Serializer module for the inverted index
 pub mod serializer;
+mod skip;
 mod term_info;
 
 pub(crate) use loaded_postings::LoadedPostings;
-pub use postings::DocFreq;
 pub(crate) use stacker::compute_table_memory_size;
 
-#[doc(hidden)]
-pub use self::indexing_context::IndexingContext;
-#[doc(hidden)]
-pub use self::per_field_postings_writer::PerFieldPostingsWriter;
+pub use self::block_segment_postings::BlockSegmentPostings;
+pub(crate) use self::indexing_context::IndexingContext;
+pub(crate) use self::per_field_postings_writer::PerFieldPostingsWriter;
 pub use self::postings::Postings;
-#[doc(hidden)]
-pub use self::postings_writer::IndexingPosition;
-pub use self::postings_writer::PostingsWriterEnum;
-pub(crate) use self::postings_writer::{serialize_postings, PostingsWriter};
-pub use self::recorder::{
-    BufferLender, DocIdRecorder, Recorder, TermFrequencyRecorder, TfAndPositionRecorder,
-};
+pub(crate) use self::postings_writer::{serialize_postings, IndexingPosition, PostingsWriter};
+pub use self::segment_postings::SegmentPostings;
 pub use self::serializer::{FieldSerializer, InvertedIndexSerializer};
+pub(crate) use self::skip::{BlockInfo, SkipReader};
 pub use self::term_info::TermInfo;
 
+#[expect(clippy::enum_variant_names)]
+#[derive(Debug, PartialEq, Clone, Copy, Eq)]
+pub(crate) enum FreqReadingOption {
+    NoFreq,
+    SkipFreq,
+    ReadFreq,
+}
+
 #[cfg(test)]
 pub(crate) mod tests {
     use std::mem;
@@ -45,7 +50,6 @@ pub(crate) mod tests {
     use crate::index::{Index, SegmentComponent, SegmentReader};
     use crate::indexer::operation::AddOperation;
     use crate::indexer::SegmentWriter;
-    use crate::postings::DocFreq;
     use crate::query::Scorer;
     use crate::schema::{
         Field, IndexRecordOption, Schema, Term, TextFieldIndexing, TextOptions, INDEXED, TEXT,
@@ -276,11 +280,11 @@ pub(crate) mod tests {
             }
             {
                 let term_a = Term::from_field_text(text_field, "a");
-                let mut postings_a: Box<dyn Postings> = segment_reader
+                let mut postings_a = segment_reader
                     .inverted_index(term_a.field())?
                     .read_postings(&term_a, IndexRecordOption::WithFreqsAndPositions)?
                     .unwrap();
-                assert_eq!(postings_a.doc_freq(), DocFreq::Exact(1000));
+                assert_eq!(postings_a.len(), 1000);
                 assert_eq!(postings_a.doc(), 0);
                 assert_eq!(postings_a.term_freq(), 6);
                 postings_a.positions(&mut positions);
@@ -303,7 +307,7 @@ pub(crate) mod tests {
                     .inverted_index(term_e.field())?
                     .read_postings(&term_e, IndexRecordOption::WithFreqsAndPositions)?
                     .unwrap();
-                assert_eq!(postings_e.doc_freq(), DocFreq::Exact(1000 - 2));
+                assert_eq!(postings_e.len(), 1000 - 2);
                 for i in 2u32..1000u32 {
                     assert_eq!(postings_e.term_freq(), i);
                     postings_e.positions(&mut positions);
@@ -528,6 +532,16 @@ pub(crate) mod tests {
         fn score(&mut self) -> Score {
             self.0.score()
         }
+
+        #[inline]
+        fn can_score_doc(&self) -> bool {
+            self.0.can_score_doc()
+        }
+
+        #[inline]
+        fn score_doc(&mut self, doc: DocId, term_freq: u32) -> Score {
+            self.0.score_doc(doc, term_freq)
+        }
     }
 
     pub fn test_skip_against_unoptimized<F: Fn() -> Box<dyn DocSet>>(

src/postings/per_field_postings_writer.rs

@@ -1,15 +1,16 @@
 use crate::postings::json_postings_writer::JsonPostingsWriter;
-use crate::postings::postings_writer::{PostingsWriterEnum, SpecializedPostingsWriter};
+use crate::postings::postings_writer::SpecializedPostingsWriter;
 use crate::postings::recorder::{DocIdRecorder, TermFrequencyRecorder, TfAndPositionRecorder};
+use crate::postings::PostingsWriter;
 use crate::schema::{Field, FieldEntry, FieldType, IndexRecordOption, Schema};
 
-pub struct PerFieldPostingsWriter {
-    per_field_postings_writers: Vec<PostingsWriterEnum>,
+pub(crate) struct PerFieldPostingsWriter {
+    per_field_postings_writers: Vec<Box<dyn PostingsWriter>>,
 }
 
 impl PerFieldPostingsWriter {
     pub fn for_schema(schema: &Schema) -> Self {
-        let per_field_postings_writers: Vec<PostingsWriterEnum> = schema
+        let per_field_postings_writers = schema
             .fields()
             .map(|(_, field_entry)| posting_writer_from_field_entry(field_entry))
             .collect();
@@ -18,16 +19,16 @@ impl PerFieldPostingsWriter {
         }
     }
 
-    pub(crate) fn get_for_field(&self, field: Field) -> &PostingsWriterEnum {
-        &self.per_field_postings_writers[field.field_id() as usize]
+    pub(crate) fn get_for_field(&self, field: Field) -> &dyn PostingsWriter {
+        self.per_field_postings_writers[field.field_id() as usize].as_ref()
     }
 
-    pub fn get_for_field_mut(&mut self, field: Field) -> &mut PostingsWriterEnum {
-        &mut self.per_field_postings_writers[field.field_id() as usize]
+    pub(crate) fn get_for_field_mut(&mut self, field: Field) -> &mut dyn PostingsWriter {
+        self.per_field_postings_writers[field.field_id() as usize].as_mut()
     }
 }
 
-fn posting_writer_from_field_entry(field_entry: &FieldEntry) -> PostingsWriterEnum {
+fn posting_writer_from_field_entry(field_entry: &FieldEntry) -> Box<dyn PostingsWriter> {
     match *field_entry.field_type() {
         FieldType::Str(ref text_options) => text_options
             .get_indexing_options()
@@ -50,7 +51,7 @@ fn posting_writer_from_field_entry(field_entry: &FieldEntry) -> PostingsWriterEn
         | FieldType::Date(_)
         | FieldType::Bytes(_)
         | FieldType::IpAddr(_)
-        | FieldType::Facet(_) => <SpecializedPostingsWriter<DocIdRecorder>>::default().into(),
+        | FieldType::Facet(_) => Box::<SpecializedPostingsWriter<DocIdRecorder>>::default(),
         FieldType::JsonObject(ref json_object_options) => {
             if let Some(text_indexing_option) = json_object_options.get_text_indexing_options() {
                 match text_indexing_option.index_option() {

src/postings/postings.rs

@@ -1,25 +1,5 @@
 use crate::docset::DocSet;
 
-/// Result of the doc_freq method.
-///
-/// Postings can inform us that the document frequency is approximate.
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub enum DocFreq {
-    /// The document frequency is approximate.
-    Approximate(u32),
-    /// The document frequency is exact.
-    Exact(u32),
-}
-
-impl From<DocFreq> for u32 {
-    fn from(doc_freq: DocFreq) -> Self {
-        match doc_freq {
-            DocFreq::Approximate(approximate_doc_freq) => approximate_doc_freq,
-            DocFreq::Exact(doc_freq) => doc_freq,
-        }
-    }
-}
-
 /// Postings (also called inverted list)
 ///
 /// For a given term, it is the list of doc ids of the doc
@@ -34,9 +14,6 @@ pub trait Postings: DocSet + 'static {
     /// The number of times the term appears in the document.
     fn term_freq(&self) -> u32;
 
-    /// Returns the number of documents containing the term in the segment.
-    fn doc_freq(&self) -> DocFreq;
-
     /// Returns the positions offsetted with a given value.
     /// It is not necessary to clear the `output` before calling this method.
     /// The output vector will be resized to the `term_freq`.
@@ -54,16 +31,6 @@ pub trait Postings: DocSet + 'static {
     fn positions(&mut self, output: &mut Vec<u32>) {
         self.positions_with_offset(0u32, output);
     }
-
-    /// Returns true if the term_frequency is available.
-    ///
-    /// This is a tricky question, because on JSON fields, it is possible
-    /// for a text term to have term freq, whereas a number term in the field has none.
-    ///
-    /// This function returns whether the actual term has term frequencies or not.
-    /// In this above JSON field example, `has_freq` should return true for the
-    /// earlier and false for the latter.
-    fn has_freq(&self) -> bool;
 }
 
 impl Postings for Box<dyn Postings> {
@@ -74,12 +41,4 @@ impl Postings for Box<dyn Postings> {
     fn append_positions_with_offset(&mut self, offset: u32, output: &mut Vec<u32>) {
         (**self).append_positions_with_offset(offset, output);
     }
-
-    fn has_freq(&self) -> bool {
-        (**self).has_freq()
-    }
-
-    fn doc_freq(&self) -> DocFreq {
-        (**self).doc_freq()
-    }
 }

src/postings/postings_writer.rs

@@ -4,15 +4,10 @@ use std::ops::Range;
 
 use stacker::Addr;
 
-use crate::codec::Codec;
 use crate::fieldnorm::FieldNormReaders;
 use crate::indexer::indexing_term::IndexingTerm;
 use crate::indexer::path_to_unordered_id::OrderedPathId;
-use crate::postings::json_postings_writer::JsonPostingsWriter;
-use crate::postings::recorder::{
-    BufferLender, DocIdRecorder, Recorder, TermFrequencyRecorder, TfAndPositionRecorder,
-    UNINITIALIZED_DOC,
-};
+use crate::postings::recorder::{BufferLender, Recorder};
 use crate::postings::{
     FieldSerializer, IndexingContext, InvertedIndexSerializer, PerFieldPostingsWriter,
 };
@@ -50,12 +45,12 @@ fn make_field_partition(
 /// Serialize the inverted index.
 /// It pushes all term, one field at a time, towards the
 /// postings serializer.
-pub(crate) fn serialize_postings<C: Codec>(
+pub(crate) fn serialize_postings(
     ctx: IndexingContext,
     schema: Schema,
     per_field_postings_writers: &PerFieldPostingsWriter,
     fieldnorm_readers: FieldNormReaders,
-    serializer: &mut InvertedIndexSerializer<C>,
+    serializer: &mut InvertedIndexSerializer,
 ) -> crate::Result<()> {
     // Replace unordered ids by ordered ids to be able to sort
     let unordered_id_to_ordered_id: Vec<OrderedPathId> =
@@ -100,190 +95,11 @@ pub(crate) fn serialize_postings<C: Codec>(
 }
 
 #[derive(Default, Debug)]
-#[doc(hidden)]
-pub struct IndexingPosition {
+pub(crate) struct IndexingPosition {
     pub num_tokens: u32,
     pub end_position: u32,
 }
 
-pub enum PostingsWriterEnum {
-    DocId(SpecializedPostingsWriter<DocIdRecorder>),
-    DocIdTf(SpecializedPostingsWriter<TermFrequencyRecorder>),
-    DocTfAndPosition(SpecializedPostingsWriter<TfAndPositionRecorder>),
-    JsonDocId(JsonPostingsWriter<DocIdRecorder>),
-    JsonDocIdTf(JsonPostingsWriter<TermFrequencyRecorder>),
-    JsonDocTfAndPosition(JsonPostingsWriter<TfAndPositionRecorder>),
-}
-
-impl From<SpecializedPostingsWriter<DocIdRecorder>> for PostingsWriterEnum {
-    fn from(doc_id_recorder_writer: SpecializedPostingsWriter<DocIdRecorder>) -> Self {
-        PostingsWriterEnum::DocId(doc_id_recorder_writer)
-    }
-}
-
-impl From<SpecializedPostingsWriter<TermFrequencyRecorder>> for PostingsWriterEnum {
-    fn from(doc_id_tf_recorder_writer: SpecializedPostingsWriter<TermFrequencyRecorder>) -> Self {
-        PostingsWriterEnum::DocIdTf(doc_id_tf_recorder_writer)
-    }
-}
-
-impl From<SpecializedPostingsWriter<TfAndPositionRecorder>> for PostingsWriterEnum {
-    fn from(
-        doc_id_tf_and_positions_recorder_writer: SpecializedPostingsWriter<TfAndPositionRecorder>,
-    ) -> Self {
-        PostingsWriterEnum::DocTfAndPosition(doc_id_tf_and_positions_recorder_writer)
-    }
-}
-
-impl From<JsonPostingsWriter<DocIdRecorder>> for PostingsWriterEnum {
-    fn from(doc_id_recorder_writer: JsonPostingsWriter<DocIdRecorder>) -> Self {
-        PostingsWriterEnum::JsonDocId(doc_id_recorder_writer)
-    }
-}
-
-impl From<JsonPostingsWriter<TermFrequencyRecorder>> for PostingsWriterEnum {
-    fn from(doc_id_tf_recorder_writer: JsonPostingsWriter<TermFrequencyRecorder>) -> Self {
-        PostingsWriterEnum::JsonDocIdTf(doc_id_tf_recorder_writer)
-    }
-}
-
-impl From<JsonPostingsWriter<TfAndPositionRecorder>> for PostingsWriterEnum {
-    fn from(
-        doc_id_tf_and_positions_recorder_writer: JsonPostingsWriter<TfAndPositionRecorder>,
-    ) -> Self {
-        PostingsWriterEnum::JsonDocTfAndPosition(doc_id_tf_and_positions_recorder_writer)
-    }
-}
-
-impl PostingsWriterEnum {
-    /// Public, codec-agnostic entry point that tokenizes `token_stream` and
-    /// records every token for `doc_id`, mirroring what `SegmentWriter` does
-    /// for a text field.
-    ///
-    /// `indexing_position.end_position` offsets the token positions (set it to
-    /// shift the tokens, e.g. to place a placeholder's tokens after the static
-    /// tokens that precede it) and is advanced as tokens are consumed.
-    #[doc(hidden)]
-    pub fn index_text(
-        &mut self,
-        doc_id: DocId,
-        token_stream: &mut dyn TokenStream,
-        term_buffer: &mut IndexingTerm,
-        ctx: &mut IndexingContext,
-        indexing_position: &mut IndexingPosition,
-    ) {
-        <Self as PostingsWriter>::index_text(
-            self,
-            doc_id,
-            token_stream,
-            term_buffer,
-            ctx,
-            indexing_position,
-        )
-    }
-}
-
-impl PostingsWriter for PostingsWriterEnum {
-    fn subscribe(&mut self, doc: DocId, pos: u32, term: &IndexingTerm, ctx: &mut IndexingContext) {
-        match self {
-            PostingsWriterEnum::DocId(writer) => writer.subscribe(doc, pos, term, ctx),
-            PostingsWriterEnum::DocIdTf(writer) => writer.subscribe(doc, pos, term, ctx),
-            PostingsWriterEnum::DocTfAndPosition(writer) => writer.subscribe(doc, pos, term, ctx),
-            PostingsWriterEnum::JsonDocId(writer) => writer.subscribe(doc, pos, term, ctx),
-            PostingsWriterEnum::JsonDocIdTf(writer) => writer.subscribe(doc, pos, term, ctx),
-            PostingsWriterEnum::JsonDocTfAndPosition(writer) => {
-                writer.subscribe(doc, pos, term, ctx)
-            }
-        }
-    }
-
-    fn serialize<C: Codec>(
-        &self,
-        term_addrs: &[(Field, OrderedPathId, &[u8], Addr)],
-        ordered_id_to_path: &[&str],
-        ctx: &IndexingContext,
-        serializer: &mut FieldSerializer<C>,
-    ) -> io::Result<()> {
-        match self {
-            PostingsWriterEnum::DocId(writer) => {
-                writer.serialize(term_addrs, ordered_id_to_path, ctx, serializer)
-            }
-            PostingsWriterEnum::DocIdTf(writer) => {
-                writer.serialize(term_addrs, ordered_id_to_path, ctx, serializer)
-            }
-            PostingsWriterEnum::DocTfAndPosition(writer) => {
-                writer.serialize(term_addrs, ordered_id_to_path, ctx, serializer)
-            }
-            PostingsWriterEnum::JsonDocId(writer) => {
-                writer.serialize(term_addrs, ordered_id_to_path, ctx, serializer)
-            }
-            PostingsWriterEnum::JsonDocIdTf(writer) => {
-                writer.serialize(term_addrs, ordered_id_to_path, ctx, serializer)
-            }
-            PostingsWriterEnum::JsonDocTfAndPosition(writer) => {
-                writer.serialize(term_addrs, ordered_id_to_path, ctx, serializer)
-            }
-        }
-    }
-
-    fn ensure_term(&self, serialized_term: &[u8], ctx: &mut IndexingContext) -> Addr {
-        match self {
-            PostingsWriterEnum::DocId(writer) => writer.ensure_term(serialized_term, ctx),
-            PostingsWriterEnum::DocIdTf(writer) => writer.ensure_term(serialized_term, ctx),
-            PostingsWriterEnum::DocTfAndPosition(writer) => {
-                writer.ensure_term(serialized_term, ctx)
-            }
-            PostingsWriterEnum::JsonDocId(writer) => writer.ensure_term(serialized_term, ctx),
-            PostingsWriterEnum::JsonDocIdTf(writer) => writer.ensure_term(serialized_term, ctx),
-            PostingsWriterEnum::JsonDocTfAndPosition(writer) => {
-                writer.ensure_term(serialized_term, ctx)
-            }
-        }
-    }
-
-    /// Tokenize a text and subscribe all of its token.
-    fn index_text(
-        &mut self,
-        doc_id: DocId,
-        token_stream: &mut dyn TokenStream,
-        term_buffer: &mut IndexingTerm,
-        ctx: &mut IndexingContext,
-        indexing_position: &mut IndexingPosition,
-    ) {
-        match self {
-            PostingsWriterEnum::DocId(writer) => {
-                writer.index_text(doc_id, token_stream, term_buffer, ctx, indexing_position)
-            }
-            PostingsWriterEnum::DocIdTf(writer) => {
-                writer.index_text(doc_id, token_stream, term_buffer, ctx, indexing_position)
-            }
-            PostingsWriterEnum::DocTfAndPosition(writer) => {
-                writer.index_text(doc_id, token_stream, term_buffer, ctx, indexing_position)
-            }
-            PostingsWriterEnum::JsonDocId(writer) => {
-                writer.index_text(doc_id, token_stream, term_buffer, ctx, indexing_position)
-            }
-            PostingsWriterEnum::JsonDocIdTf(writer) => {
-                writer.index_text(doc_id, token_stream, term_buffer, ctx, indexing_position)
-            }
-            PostingsWriterEnum::JsonDocTfAndPosition(writer) => {
-                writer.index_text(doc_id, token_stream, term_buffer, ctx, indexing_position)
-            }
-        }
-    }
-
-    fn total_num_tokens(&self) -> u64 {
-        match self {
-            PostingsWriterEnum::DocId(writer) => writer.total_num_tokens(),
-            PostingsWriterEnum::DocIdTf(writer) => writer.total_num_tokens(),
-            PostingsWriterEnum::DocTfAndPosition(writer) => writer.total_num_tokens(),
-            PostingsWriterEnum::JsonDocId(writer) => writer.total_num_tokens(),
-            PostingsWriterEnum::JsonDocIdTf(writer) => writer.total_num_tokens(),
-            PostingsWriterEnum::JsonDocTfAndPosition(writer) => writer.total_num_tokens(),
-        }
-    }
-}
-
 /// The `PostingsWriter` is in charge of receiving documenting
 /// and building a `Segment` in anonymous memory.
 ///
@@ -300,23 +116,14 @@ pub(crate) trait PostingsWriter: Send + Sync {
 
     /// Serializes the postings on disk.
     /// The actual serialization format is handled by the `PostingsSerializer`.
-    fn serialize<C: Codec>(
+    fn serialize(
         &self,
         term_addrs: &[(Field, OrderedPathId, &[u8], Addr)],
         ordered_id_to_path: &[&str],
         ctx: &IndexingContext,
-        serializer: &mut FieldSerializer<C>,
+        serializer: &mut FieldSerializer,
     ) -> io::Result<()>;
 
-    /// Ensures `serialized_term` has an entry in the term index, creating an
-    /// empty recorder (matching this writer's indexing option) if the term is
-    /// not present yet, and returns the value `Addr` of its recorder.
-    ///
-    /// An existing recorder is never overwritten, so the term keeps any
-    /// posting data already recorded for it. This is used to attach a
-    /// codec-specific payload to a term that may belong to no document.
-    fn ensure_term(&self, serialized_term: &[u8], ctx: &mut IndexingContext) -> Addr;
-
     /// Tokenize a text and subscribe all of its token.
     fn index_text(
         &mut self,
@@ -359,27 +166,31 @@ pub(crate) trait PostingsWriter: Send + Sync {
 /// The `SpecializedPostingsWriter` is just here to remove dynamic
 /// dispatch to the recorder information.
 #[derive(Default)]
-pub struct SpecializedPostingsWriter<Rec: Recorder> {
+pub(crate) struct SpecializedPostingsWriter<Rec: Recorder> {
     total_num_tokens: u64,
     _recorder_type: PhantomData<Rec>,
 }
 
+impl<Rec: Recorder> From<SpecializedPostingsWriter<Rec>> for Box<dyn PostingsWriter> {
+    fn from(
+        specialized_postings_writer: SpecializedPostingsWriter<Rec>,
+    ) -> Box<dyn PostingsWriter> {
+        Box::new(specialized_postings_writer)
+    }
+}
+
 impl<Rec: Recorder> SpecializedPostingsWriter<Rec> {
     #[inline]
-    pub(crate) fn serialize_one_term<C: Codec>(
+    pub(crate) fn serialize_one_term(
         term: &[u8],
         addr: Addr,
         buffer_lender: &mut BufferLender,
         ctx: &IndexingContext,
-        serializer: &mut FieldSerializer<C>,
+        serializer: &mut FieldSerializer,
     ) -> io::Result<()> {
         let recorder: Rec = ctx.term_index.read(addr);
         let term_doc_freq = recorder.term_doc_freq().unwrap_or(0u32);
         serializer.new_term(term, term_doc_freq, recorder.has_term_freq())?;
-        if let Some(payload) = ctx.codec_term_payloads.get(&addr) {
-            // `&(dyn Any + Send)` upcasts to `&dyn Any`.
-            serializer.set_term_payload(payload.as_ref());
-        }
         recorder.serialize(&ctx.arena, serializer, buffer_lender);
         serializer.close_term()?;
         Ok(())
@@ -399,31 +210,29 @@ impl<Rec: Recorder> PostingsWriter for SpecializedPostingsWriter<Rec> {
         self.total_num_tokens += 1;
         let (term_index, arena) = (&mut ctx.term_index, &mut ctx.arena);
         term_index.mutate_or_create(term.serialized_term(), |opt_recorder: Option<Rec>| {
-            // A recorder may already exist without having started any document: the codec
-            // payload mechanism (`ensure_term`) pre-creates one to attach a payload to a
-            // term (e.g. a static template token in the moshiki codec). Such a recorder has
-            // `current_doc == UNINITIALIZED_DOC`. We must NOT `close_doc` it on the first
-            // real occurrence — that would emit a spurious doc terminator and desync the
-            // posting/position stream. `new_doc` writes the first doc id as an absolute delta.
-            let mut recorder = opt_recorder.unwrap_or_default();
-            let current_doc = recorder.current_doc();
-            if current_doc != doc {
-                if current_doc != UNINITIALIZED_DOC {
+            if let Some(mut recorder) = opt_recorder {
+                let current_doc = recorder.current_doc();
+                if current_doc != doc {
                     recorder.close_doc(arena);
+                    recorder.new_doc(doc, arena);
                 }
+                recorder.record_position(position, arena);
+                recorder
+            } else {
+                let mut recorder = Rec::default();
                 recorder.new_doc(doc, arena);
+                recorder.record_position(position, arena);
+                recorder
             }
-            recorder.record_position(position, arena);
-            recorder
         });
     }
 
-    fn serialize<C: Codec>(
+    fn serialize(
         &self,
         term_addrs: &[(Field, OrderedPathId, &[u8], Addr)],
         _ordered_id_to_path: &[&str],
         ctx: &IndexingContext,
-        serializer: &mut FieldSerializer<C>,
+        serializer: &mut FieldSerializer,
     ) -> io::Result<()> {
         let mut buffer_lender = BufferLender::default();
         for (_field, _path_id, term, addr) in term_addrs {
@@ -432,11 +241,6 @@ impl<Rec: Recorder> PostingsWriter for SpecializedPostingsWriter<Rec> {
         Ok(())
     }
 
-    fn ensure_term(&self, serialized_term: &[u8], ctx: &mut IndexingContext) -> Addr {
-        ctx.term_index
-            .get_or_create_value_addr::<Rec>(serialized_term, Rec::default)
-    }
-
     fn total_num_tokens(&self) -> u64 {
         self.total_num_tokens
     }

src/postings/recorder.rs

@@ -1,37 +1,13 @@
 use common::read_u32_vint;
 use stacker::{ExpUnrolledLinkedList, MemoryArena};
 
-use crate::codec::Codec;
 use crate::postings::FieldSerializer;
 use crate::DocId;
 
 const POSITION_END: u32 = 0;
 
-/// Sentinel `current_doc` for a recorder that has not yet started any document.
-///
-/// A recorder can exist before its first `new_doc` because the codec payload
-/// mechanism (`ensure_term`) pre-creates a recorder to attach a payload to a term
-/// — e.g. a static template token in the moshiki codec. `DocId::MAX` is never a
-/// real document id (it is `TERMINATED`), so it unambiguously marks "no document
-/// started yet": `subscribe` uses it to skip the spurious `close_doc` that would
-/// otherwise desync the position stream, and `new_doc` uses it to write the first
-/// doc id as an absolute delta.
-pub(crate) const UNINITIALIZED_DOC: DocId = DocId::MAX;
-
-/// Doc-id delta to vint-encode in `new_doc`. The first document of a term is stored
-/// as an absolute id (the recorder's `current_doc` is still `UNINITIALIZED_DOC`),
-/// matching the decoder which seeds `prev_doc = 0`.
-#[inline]
-fn doc_delta(current_doc: DocId, doc: DocId) -> u32 {
-    if current_doc == UNINITIALIZED_DOC {
-        doc
-    } else {
-        doc - current_doc
-    }
-}
-
 #[derive(Default)]
-pub struct BufferLender {
+pub(crate) struct BufferLender {
     buffer_u8: Vec<u8>,
     buffer_u32: Vec<u32>,
 }
@@ -79,7 +55,7 @@ impl Iterator for VInt32Reader<'_> {
 ///   * the document id
 ///   * the term frequency
 ///   * the term positions
-pub trait Recorder: Copy + Default + Send + Sync + 'static {
+pub(crate) trait Recorder: Copy + Default + Send + Sync + 'static {
     /// Returns the current document
     fn current_doc(&self) -> u32;
     /// Starts recording information about a new document
@@ -91,10 +67,10 @@ pub trait Recorder: Copy + Default + Send + Sync + 'static {
     /// Close the document. It will help record the term frequency.
     fn close_doc(&mut self, arena: &mut MemoryArena);
     /// Pushes the postings information to the serializer.
-    fn serialize<C: Codec>(
+    fn serialize(
         &self,
         arena: &MemoryArena,
-        serializer: &mut FieldSerializer<C>,
+        serializer: &mut FieldSerializer<'_>,
         buffer_lender: &mut BufferLender,
     );
     /// Returns the number of document containing this term.
@@ -109,21 +85,12 @@ pub trait Recorder: Copy + Default + Send + Sync + 'static {
 }
 
 /// Only records the doc ids
-#[derive(Clone, Copy)]
+#[derive(Clone, Copy, Default)]
 pub struct DocIdRecorder {
     stack: ExpUnrolledLinkedList,
     current_doc: DocId,
 }
 
-impl Default for DocIdRecorder {
-    fn default() -> Self {
-        DocIdRecorder {
-            stack: ExpUnrolledLinkedList::default(),
-            current_doc: UNINITIALIZED_DOC,
-        }
-    }
-}
-
 impl Recorder for DocIdRecorder {
     #[inline]
     fn current_doc(&self) -> DocId {
@@ -132,7 +99,7 @@ impl Recorder for DocIdRecorder {
 
     #[inline]
     fn new_doc(&mut self, doc: DocId, arena: &mut MemoryArena) {
-        let delta = doc_delta(self.current_doc, doc);
+        let delta = doc - self.current_doc;
         self.current_doc = doc;
         self.stack.writer(arena).write_u32_vint(delta);
     }
@@ -143,10 +110,10 @@ impl Recorder for DocIdRecorder {
     #[inline]
     fn close_doc(&mut self, _arena: &mut MemoryArena) {}
 
-    fn serialize<C: Codec>(
+    fn serialize(
         &self,
         arena: &MemoryArena,
-        serializer: &mut FieldSerializer<C>,
+        serializer: &mut FieldSerializer<'_>,
         buffer_lender: &mut BufferLender,
     ) {
         let buffer = buffer_lender.lend_u8();
@@ -177,7 +144,7 @@ fn get_sum_reader(iter: impl Iterator<Item = u32>) -> impl Iterator<Item = u32>
 }
 
 /// Recorder encoding document ids, and term frequencies
-#[derive(Clone, Copy)]
+#[derive(Clone, Copy, Default)]
 pub struct TermFrequencyRecorder {
     stack: ExpUnrolledLinkedList,
     current_doc: DocId,
@@ -185,17 +152,6 @@ pub struct TermFrequencyRecorder {
     term_doc_freq: u32,
 }
 
-impl Default for TermFrequencyRecorder {
-    fn default() -> Self {
-        TermFrequencyRecorder {
-            stack: ExpUnrolledLinkedList::default(),
-            current_doc: UNINITIALIZED_DOC,
-            current_tf: 0,
-            term_doc_freq: 0,
-        }
-    }
-}
-
 impl Recorder for TermFrequencyRecorder {
     #[inline]
     fn current_doc(&self) -> DocId {
@@ -204,7 +160,7 @@ impl Recorder for TermFrequencyRecorder {
 
     #[inline]
     fn new_doc(&mut self, doc: DocId, arena: &mut MemoryArena) {
-        let delta = doc_delta(self.current_doc, doc);
+        let delta = doc - self.current_doc;
         self.term_doc_freq += 1;
         self.current_doc = doc;
         self.stack.writer(arena).write_u32_vint(delta);
@@ -222,10 +178,10 @@ impl Recorder for TermFrequencyRecorder {
         self.current_tf = 0;
     }
 
-    fn serialize<C: Codec>(
+    fn serialize(
         &self,
         arena: &MemoryArena,
-        serializer: &mut FieldSerializer<C>,
+        serializer: &mut FieldSerializer<'_>,
         buffer_lender: &mut BufferLender,
     ) {
         let buffer = buffer_lender.lend_u8();
@@ -246,23 +202,13 @@ impl Recorder for TermFrequencyRecorder {
 }
 
 /// Recorder encoding term frequencies as well as positions.
-#[derive(Clone, Copy)]
+#[derive(Clone, Copy, Default)]
 pub struct TfAndPositionRecorder {
     stack: ExpUnrolledLinkedList,
     current_doc: DocId,
     term_doc_freq: u32,
 }
 
-impl Default for TfAndPositionRecorder {
-    fn default() -> Self {
-        TfAndPositionRecorder {
-            stack: ExpUnrolledLinkedList::default(),
-            current_doc: UNINITIALIZED_DOC,
-            term_doc_freq: 0,
-        }
-    }
-}
-
 impl Recorder for TfAndPositionRecorder {
     #[inline]
     fn current_doc(&self) -> DocId {
@@ -271,7 +217,7 @@ impl Recorder for TfAndPositionRecorder {
 
     #[inline]
     fn new_doc(&mut self, doc: DocId, arena: &mut MemoryArena) {
-        let delta = doc_delta(self.current_doc, doc);
+        let delta = doc - self.current_doc;
         self.current_doc = doc;
         self.term_doc_freq += 1u32;
         self.stack.writer(arena).write_u32_vint(delta);
@@ -289,10 +235,10 @@ impl Recorder for TfAndPositionRecorder {
         self.stack.writer(arena).write_u32_vint(POSITION_END);
     }
 
-    fn serialize<C: Codec>(
+    fn serialize(
         &self,
         arena: &MemoryArena,
-        serializer: &mut FieldSerializer<C>,
+        serializer: &mut FieldSerializer<'_>,
         buffer_lender: &mut BufferLender,
     ) {
         let (buffer_u8, buffer_positions) = buffer_lender.lend_all();
@@ -310,11 +256,6 @@ impl Recorder for TfAndPositionRecorder {
                         break;
                     }
                     Some(position_plus_one) => {
-                        debug_assert!(
-                            position_plus_one >= prev_position_plus_one,
-                            "positions for a (term, doc) must be recorded non-decreasing (got \
-                             {position_plus_one} after {prev_position_plus_one})",
-                        );
                         let delta_position = position_plus_one - prev_position_plus_one;
                         buffer_positions.push(delta_position);
                         prev_position_plus_one = position_plus_one;
@@ -334,9 +275,8 @@ impl Recorder for TfAndPositionRecorder {
 mod tests {
 
     use common::write_u32_vint;
-    use stacker::MemoryArena;
 
-    use super::{BufferLender, Recorder, TermFrequencyRecorder, VInt32Reader};
+    use super::{BufferLender, VInt32Reader};
 
     #[test]
     fn test_buffer_lender() {
@@ -374,98 +314,4 @@ mod tests {
         let res: Vec<u32> = VInt32Reader::new(&buffer[..]).collect();
         assert_eq!(&res[..], &vals[..]);
     }
-
-    // ── TermFrequencyRecorder ─────────────────────────────────────────────────
-
-    #[test]
-    fn term_frequency_recorder_has_term_freq() {
-        let rec = TermFrequencyRecorder::default();
-        assert!(
-            rec.has_term_freq(),
-            "TermFrequencyRecorder must advertise term-frequency support"
-        );
-    }
-
-    #[test]
-    fn term_frequency_recorder_term_doc_freq_single_doc() {
-        let mut arena = MemoryArena::default();
-        let mut rec = TermFrequencyRecorder::default();
-
-        // Record one document with two term occurrences.
-        rec.new_doc(0, &mut arena);
-        rec.record_position(0, &mut arena);
-        rec.record_position(1, &mut arena);
-        rec.close_doc(&mut arena);
-
-        assert_eq!(
-            rec.term_doc_freq(),
-            Some(1),
-            "term_doc_freq should be 1 after recording one document"
-        );
-    }
-
-    #[test]
-    fn term_frequency_recorder_term_doc_freq_multiple_docs() {
-        let mut arena = MemoryArena::default();
-        let mut rec = TermFrequencyRecorder::default();
-
-        // Three documents with 1, 3, and 2 occurrences respectively.
-        for (doc, tf) in [(0u32, 1u32), (5, 3), (10, 2)] {
-            rec.new_doc(doc, &mut arena);
-            for pos in 0..tf {
-                rec.record_position(pos, &mut arena);
-            }
-            rec.close_doc(&mut arena);
-        }
-
-        assert_eq!(
-            rec.term_doc_freq(),
-            Some(3),
-            "term_doc_freq should equal the number of documents recorded"
-        );
-    }
-
-    #[test]
-    fn term_frequency_recorder_zero_docs() {
-        let rec = TermFrequencyRecorder::default();
-        assert_eq!(
-            rec.term_doc_freq(),
-            Some(0),
-            "term_doc_freq should be 0 before any document is recorded"
-        );
-    }
-
-    #[test]
-    fn term_frequency_recorder_single_occurrence_per_doc() {
-        let mut arena = MemoryArena::default();
-        let mut rec = TermFrequencyRecorder::default();
-
-        // Each document has exactly one occurrence — the minimum non-trivial case.
-        for doc in [1u32, 2, 100] {
-            rec.new_doc(doc, &mut arena);
-            rec.record_position(0, &mut arena);
-            rec.close_doc(&mut arena);
-        }
-
-        assert_eq!(rec.term_doc_freq(), Some(3));
-    }
-
-    #[test]
-    fn term_frequency_recorder_high_frequency_doc() {
-        let mut arena = MemoryArena::default();
-        let mut rec = TermFrequencyRecorder::default();
-
-        // A document where the term appears many times.
-        rec.new_doc(42, &mut arena);
-        for pos in 0..1000 {
-            rec.record_position(pos, &mut arena);
-        }
-        rec.close_doc(&mut arena);
-
-        assert_eq!(
-            rec.term_doc_freq(),
-            Some(1),
-            "term_doc_freq counts documents, not occurrences"
-        );
-    }
 }

src/postings/segment_postings.rs

@@ -1,34 +1,22 @@
-use common::BitSet;
+use common::HasLen;
 
-use super::BlockSegmentPostings;
-use crate::codec::positions::PositionsReader;
-use crate::codec::postings::PostingsWithBlockMax;
-use crate::docset::DocSet;
-use crate::fieldnorm::FieldNormReader;
+use crate::docset::{DocSet, COLLECT_BLOCK_BUFFER_LEN};
+use crate::fastfield::AliveBitSet;
+use crate::positions::PositionReader;
 use crate::postings::compression::COMPRESSION_BLOCK_SIZE;
-use crate::postings::{DocFreq, Postings};
-use crate::query::Bm25Weight;
-use crate::{DocId, Score};
+use crate::postings::{BlockSegmentPostings, Postings};
+use crate::{DocId, TERMINATED};
 
 /// `SegmentPostings` represents the inverted list or postings associated with
 /// a term in a `Segment`.
 ///
 /// As we iterate through the `SegmentPostings`, the frequencies are optionally decoded.
 /// Positions on the other hand, are optionally entirely decoded upfront.
+#[derive(Clone)]
 pub struct SegmentPostings {
     pub(crate) block_cursor: BlockSegmentPostings,
     cur: usize,
-    position_reader: Option<Box<dyn PositionsReader>>,
-}
-
-impl Clone for SegmentPostings {
-    fn clone(&self) -> Self {
-        SegmentPostings {
-            block_cursor: self.block_cursor.clone(),
-            cur: self.cur,
-            position_reader: self.position_reader.as_ref().map(|r| r.clone_box()),
-        }
-    }
+    position_reader: Option<PositionReader>,
 }
 
 impl SegmentPostings {
@@ -41,6 +29,31 @@ impl SegmentPostings {
         }
     }
 
+    /// Compute the number of non-deleted documents.
+    ///
+    /// This method will clone and scan through the posting lists.
+    /// (this is a rather expensive operation).
+    pub fn doc_freq_given_deletes(&self, alive_bitset: &AliveBitSet) -> u32 {
+        let mut docset = self.clone();
+        let mut doc_freq = 0;
+        loop {
+            let doc = docset.doc();
+            if doc == TERMINATED {
+                return doc_freq;
+            }
+            if alive_bitset.is_alive(doc) {
+                doc_freq += 1u32;
+            }
+            docset.advance();
+        }
+    }
+
+    /// Returns the overall number of documents in the block postings.
+    /// It does not take in account whether documents are deleted or not.
+    pub fn doc_freq(&self) -> u32 {
+        self.block_cursor.doc_freq()
+    }
+
     /// Creates a segment postings object with the given documents
     /// and no frequency encoded.
     ///
@@ -51,19 +64,13 @@ impl SegmentPostings {
     /// buffer with the serialized data.
     #[cfg(test)]
     pub fn create_from_docs(docs: &[u32]) -> SegmentPostings {
-        use common::OwnedBytes;
-
+        use crate::directory::FileSlice;
+        use crate::postings::serializer::PostingsSerializer;
         use crate::schema::IndexRecordOption;
         let mut buffer = Vec::new();
         {
-            use crate::codec::postings::PostingsSerializer;
-
             let mut postings_serializer =
-                crate::codec::standard::postings::StandardPostingsSerializer::new(
-                    0.0,
-                    IndexRecordOption::Basic,
-                    None,
-                );
+                PostingsSerializer::new(0.0, IndexRecordOption::Basic, None);
             postings_serializer.new_term(docs.len() as u32, false);
             for &doc in docs {
                 postings_serializer.write_doc(doc, 1u32);
@@ -74,7 +81,7 @@ impl SegmentPostings {
         }
         let block_segment_postings = BlockSegmentPostings::open(
             docs.len() as u32,
-            OwnedBytes::new(buffer),
+            FileSlice::from(buffer),
             IndexRecordOption::Basic,
             IndexRecordOption::Basic,
         )
@@ -88,11 +95,9 @@ impl SegmentPostings {
         doc_and_tfs: &[(u32, u32)],
         fieldnorms: Option<&[u32]>,
     ) -> SegmentPostings {
-        use common::OwnedBytes;
-
-        use crate::codec::postings::PostingsSerializer as _;
-        use crate::codec::standard::postings::StandardPostingsSerializer;
+        use crate::directory::FileSlice;
         use crate::fieldnorm::FieldNormReader;
+        use crate::postings::serializer::PostingsSerializer;
         use crate::schema::IndexRecordOption;
         use crate::Score;
         let mut buffer: Vec<u8> = Vec::new();
@@ -109,7 +114,7 @@ impl SegmentPostings {
                 total_num_tokens as Score / fieldnorms.len() as Score
             })
             .unwrap_or(0.0);
-        let mut postings_serializer = StandardPostingsSerializer::new(
+        let mut postings_serializer = PostingsSerializer::new(
             average_field_norm,
             IndexRecordOption::WithFreqs,
             fieldnorm_reader,
@@ -123,7 +128,7 @@ impl SegmentPostings {
             .unwrap();
         let block_segment_postings = BlockSegmentPostings::open(
             doc_and_tfs.len() as u32,
-            OwnedBytes::new(buffer),
+            FileSlice::from(buffer),
             IndexRecordOption::WithFreqs,
             IndexRecordOption::WithFreqs,
         )
@@ -138,7 +143,7 @@ impl SegmentPostings {
     /// * `freq_handler` - the freq handler is in charge of decoding frequencies and/or positions
     pub(crate) fn from_block_postings(
         segment_block_postings: BlockSegmentPostings,
-        position_reader: Option<Box<dyn PositionsReader>>,
+        position_reader: Option<PositionReader>,
     ) -> SegmentPostings {
         SegmentPostings {
             block_cursor: segment_block_postings,
@@ -146,6 +151,34 @@ impl SegmentPostings {
             position_reader,
         }
     }
+
+    pub(crate) fn fill_buffer_up_to_with_term_freqs(
+        &mut self,
+        horizon: DocId,
+        docs: &mut [DocId; COLLECT_BLOCK_BUFFER_LEN],
+        term_freqs: &mut [u32; COLLECT_BLOCK_BUFFER_LEN],
+    ) -> usize {
+        let mut num_elems = 0;
+        while num_elems < COLLECT_BLOCK_BUFFER_LEN && self.doc() < horizon {
+            let copied = self.block_cursor.copy_docs_and_term_freqs(
+                self.cur,
+                horizon,
+                &mut docs[num_elems..],
+                &mut term_freqs[num_elems..],
+            );
+            if copied == 0 {
+                break;
+            }
+            num_elems += copied;
+            self.cur += copied;
+
+            if self.cur == COMPRESSION_BLOCK_SIZE {
+                self.cur = 0;
+                self.block_cursor.advance();
+            }
+        }
+        num_elems
+    }
 }
 
 impl DocSet for SegmentPostings {
@@ -153,6 +186,7 @@ impl DocSet for SegmentPostings {
     // next needs to be called a first time to point to the correct element.
     #[inline]
     fn advance(&mut self) -> DocId {
+        debug_assert!(self.block_cursor.block_is_loaded());
         if self.cur == COMPRESSION_BLOCK_SIZE - 1 {
             self.cur = 0;
             self.block_cursor.advance();
@@ -191,31 +225,13 @@ impl DocSet for SegmentPostings {
     }
 
     fn size_hint(&self) -> u32 {
-        self.doc_freq().into()
+        self.len() as u32
     }
+}
 
-    fn fill_bitset(&mut self, bitset: &mut BitSet) {
-        let bitset_max_value: DocId = bitset.max_value();
-        loop {
-            let docs = self.block_cursor.docs();
-            let Some(&last_doc) = docs.last() else {
-                break;
-            };
-            if last_doc < bitset_max_value {
-                // All docs are within the range of the bitset
-                for &doc in docs {
-                    bitset.insert(doc);
-                }
-            } else {
-                for &doc in docs {
-                    if doc < bitset_max_value {
-                        bitset.insert(doc);
-                    }
-                }
-                break;
-            }
-            self.block_cursor.advance();
-        }
+impl HasLen for SegmentPostings {
+    fn len(&self) -> usize {
+        self.block_cursor.doc_freq() as usize
     }
 }
 
@@ -241,13 +257,6 @@ impl Postings for SegmentPostings {
         self.block_cursor.freq(self.cur)
     }
 
-    /// Returns the overall number of documents in the block postings.
-    /// It does not take in account whether documents are deleted or not.
-    #[inline(always)]
-    fn doc_freq(&self) -> DocFreq {
-        DocFreq::Exact(self.block_cursor.doc_freq())
-    }
-
     fn append_positions_with_offset(&mut self, offset: u32, output: &mut Vec<u32>) {
         let term_freq = self.term_freq();
         let prev_len = output.len();
@@ -271,42 +280,24 @@ impl Postings for SegmentPostings {
             }
         }
     }
-
-    fn has_freq(&self) -> bool {
-        !self.block_cursor.freqs().is_empty()
-    }
-}
-
-impl PostingsWithBlockMax for SegmentPostings {
-    fn seek_block_max(
-        &mut self,
-        target_doc: crate::DocId,
-        fieldnorm_reader: &FieldNormReader,
-        similarity_weight: &Bm25Weight,
-    ) -> Score {
-        self.block_cursor.seek_block_without_loading(target_doc);
-        self.block_cursor
-            .block_max_score(fieldnorm_reader, similarity_weight)
-    }
-
-    fn last_doc_in_block(&self) -> crate::DocId {
-        self.block_cursor.skip_reader().last_doc_in_block()
-    }
 }
 
 #[cfg(test)]
 mod tests {
+
+    use common::HasLen;
+
     use super::SegmentPostings;
     use crate::docset::{DocSet, TERMINATED};
-    use crate::postings::Postings;
+    use crate::fastfield::AliveBitSet;
+    use crate::postings::postings::Postings;
 
     #[test]
     fn test_empty_segment_postings() {
         let mut postings = SegmentPostings::empty();
-        assert_eq!(postings.doc(), TERMINATED);
         assert_eq!(postings.advance(), TERMINATED);
         assert_eq!(postings.advance(), TERMINATED);
-        assert_eq!(postings.doc_freq(), crate::postings::DocFreq::Exact(0));
+        assert_eq!(postings.len(), 0);
     }
 
     #[test]
@@ -321,4 +312,15 @@ mod tests {
         let postings = SegmentPostings::empty();
         assert_eq!(postings.term_freq(), 1);
     }
+
+    #[test]
+    fn test_doc_freq() {
+        let docs = SegmentPostings::create_from_docs(&[0, 2, 10]);
+        assert_eq!(docs.doc_freq(), 3);
+        let alive_bitset = AliveBitSet::for_test_from_deleted_docs(&[2], 12);
+        assert_eq!(docs.doc_freq_given_deletes(&alive_bitset), 2);
+        let all_deleted =
+            AliveBitSet::for_test_from_deleted_docs(&[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], 12);
+        assert_eq!(docs.doc_freq_given_deletes(&all_deleted), 0);
+    }
 }

src/postings/serializer.rs

@@ -1,15 +1,17 @@
+use std::cmp::Ordering;
 use std::io::{self, Write};
 
-use common::{BinarySerializable, CountingWriter};
+use common::{BinarySerializable, CountingWriter, VInt};
 
 use super::TermInfo;
-use crate::codec::positions::{PositionsCodec, PositionsSerializer};
-use crate::codec::postings::PostingsSerializer;
-use crate::codec::Codec;
 use crate::directory::{CompositeWrite, WritePtr};
 use crate::fieldnorm::FieldNormReader;
 use crate::index::Segment;
-use crate::schema::{Field, FieldEntry, FieldType, IndexRecordOption, Schema};
+use crate::positions::PositionSerializer;
+use crate::postings::compression::{BlockEncoder, VIntEncoder, COMPRESSION_BLOCK_SIZE};
+use crate::postings::skip::SkipSerializer;
+use crate::query::Bm25Weight;
+use crate::schema::{Field, FieldEntry, IndexRecordOption, Schema};
 use crate::termdict::TermDictionaryBuilder;
 use crate::{DocId, Score};
 
@@ -44,27 +46,22 @@ use crate::{DocId, Score};
 ///
 /// A description of the serialization format is
 /// [available here](https://fulmicoton.gitbooks.io/tantivy-doc/content/inverted-index.html).
-pub struct InvertedIndexSerializer<C: Codec> {
+pub struct InvertedIndexSerializer {
     terms_write: CompositeWrite<WritePtr>,
     postings_write: CompositeWrite<WritePtr>,
     positions_write: CompositeWrite<WritePtr>,
     schema: Schema,
-    codec: C,
 }
 
-use crate::codec::postings::PostingsCodec;
-
-impl<C: Codec> InvertedIndexSerializer<C> {
+impl InvertedIndexSerializer {
     /// Open a new `InvertedIndexSerializer` for the given segment
-    pub fn open(segment: &mut Segment<C>) -> crate::Result<InvertedIndexSerializer<C>> {
+    pub fn open(segment: &mut Segment) -> crate::Result<InvertedIndexSerializer> {
         use crate::index::SegmentComponent::{Positions, Postings, Terms};
-        let codec = segment.index().codec().clone();
         let inv_index_serializer = InvertedIndexSerializer {
             terms_write: CompositeWrite::wrap(segment.open_write(Terms)?),
             postings_write: CompositeWrite::wrap(segment.open_write(Postings)?),
             positions_write: CompositeWrite::wrap(segment.open_write(Positions)?),
             schema: segment.schema(),
-            codec,
         };
         Ok(inv_index_serializer)
     }
@@ -78,19 +75,22 @@ impl<C: Codec> InvertedIndexSerializer<C> {
         field: Field,
         total_num_tokens: u64,
         fieldnorm_reader: Option<FieldNormReader>,
-    ) -> io::Result<FieldSerializer<'_, C>> {
+    ) -> io::Result<FieldSerializer<'_>> {
         let field_entry: &FieldEntry = self.schema.get_field_entry(field);
         let term_dictionary_write = self.terms_write.for_field(field);
         let postings_write = self.postings_write.for_field(field);
         let positions_write = self.positions_write.for_field(field);
+        let index_record_option = field_entry
+            .field_type()
+            .index_record_option()
+            .unwrap_or(IndexRecordOption::Basic);
         FieldSerializer::create(
-            field_entry.field_type(),
+            index_record_option,
             total_num_tokens,
             term_dictionary_write,
             postings_write,
             positions_write,
             fieldnorm_reader,
-            &self.codec,
         )
     }
 
@@ -105,43 +105,36 @@ impl<C: Codec> InvertedIndexSerializer<C> {
 
 /// The field serializer is in charge of
 /// the serialization of a specific field.
-pub struct FieldSerializer<'a, C: Codec> {
-    term_dictionary_builder: TermDictionaryBuilder<&'a mut CountingWriter<WritePtr>>,
-    postings_serializer: <C::PostingsCodec as PostingsCodec>::PostingsSerializer,
-    positions_serializer_opt:
-        Option<<C::PositionsCodec as PositionsCodec>::Serializer<&'a mut CountingWriter<WritePtr>>>,
+pub struct FieldSerializer<'a, W: Write = WritePtr> {
+    term_dictionary_builder: TermDictionaryBuilder<&'a mut CountingWriter<W>>,
+    postings_serializer: PostingsSerializer,
+    positions_serializer_opt: Option<PositionSerializer<&'a mut CountingWriter<W>>>,
     current_term_info: TermInfo,
     term_open: bool,
-    postings_write: &'a mut CountingWriter<WritePtr>,
+    postings_write: &'a mut CountingWriter<W>,
     postings_start_offset: u64,
 }
 
-impl<'a, C: Codec> FieldSerializer<'a, C> {
-    fn create(
-        field_type: &FieldType,
+impl<'a, W: Write> FieldSerializer<'a, W> {
+    /// Creates a new `FieldSerializer` for the given field type.
+    pub fn create(
+        index_record_option: IndexRecordOption,
         total_num_tokens: u64,
-        term_dictionary_write: &'a mut CountingWriter<WritePtr>,
-        postings_write: &'a mut CountingWriter<WritePtr>,
-        positions_write: &'a mut CountingWriter<WritePtr>,
+        term_dictionary_write: &'a mut CountingWriter<W>,
+        postings_write: &'a mut CountingWriter<W>,
+        positions_write: &'a mut CountingWriter<W>,
         fieldnorm_reader: Option<FieldNormReader>,
-        codec: &C,
-    ) -> io::Result<FieldSerializer<'a, C>> {
-        let index_record_option = field_type
-            .index_record_option()
-            .unwrap_or(IndexRecordOption::Basic);
+    ) -> io::Result<FieldSerializer<'a, W>> {
         total_num_tokens.serialize(postings_write)?;
         let term_dictionary_builder = TermDictionaryBuilder::create(term_dictionary_write)?;
         let average_fieldnorm = fieldnorm_reader
             .as_ref()
             .map(|ff_reader| total_num_tokens as Score / ff_reader.num_docs() as Score)
             .unwrap_or(0.0);
-        let postings_serializer = codec.postings_codec().new_serializer(
-            average_fieldnorm,
-            index_record_option,
-            fieldnorm_reader,
-        );
+        let postings_serializer =
+            PostingsSerializer::new(average_fieldnorm, index_record_option, fieldnorm_reader);
         let positions_serializer_opt = if index_record_option.has_positions() {
-            Some(codec.positions_codec().new_serializer(positions_write))
+            Some(PositionSerializer::new(positions_write))
         } else {
             None
         };
@@ -192,6 +185,7 @@ impl<'a, C: Codec> FieldSerializer<'a, C> {
             "Called new_term, while the previous term was not closed."
         );
         self.term_open = true;
+        self.postings_serializer.clear();
         self.current_term_info = self.current_term_info();
         self.term_dictionary_builder.insert_key(term)?;
         self.postings_serializer
@@ -204,13 +198,6 @@ impl<'a, C: Codec> FieldSerializer<'a, C> {
         self.new_term(term, 0, false)
     }
 
-    /// Forwards a codec-specific per-term payload to the postings serializer.
-    ///
-    /// Must be called after `new_term` and before any `write_doc`.
-    pub fn set_term_payload(&mut self, payload: &dyn std::any::Any) {
-        self.postings_serializer.set_term_payload(payload);
-    }
-
     /// Serialize the information that a document contains for the current term:
     /// its term frequency, and the position deltas.
     ///
@@ -267,3 +254,234 @@ impl<'a, C: Codec> FieldSerializer<'a, C> {
         Ok(())
     }
 }
+
+struct Block {
+    doc_ids: [DocId; COMPRESSION_BLOCK_SIZE],
+    term_freqs: [u32; COMPRESSION_BLOCK_SIZE],
+    len: usize,
+}
+
+impl Block {
+    fn new() -> Self {
+        Block {
+            doc_ids: [0u32; COMPRESSION_BLOCK_SIZE],
+            term_freqs: [0u32; COMPRESSION_BLOCK_SIZE],
+            len: 0,
+        }
+    }
+
+    fn doc_ids(&self) -> &[DocId] {
+        &self.doc_ids[..self.len]
+    }
+
+    fn term_freqs(&self) -> &[u32] {
+        &self.term_freqs[..self.len]
+    }
+
+    fn clear(&mut self) {
+        self.len = 0;
+    }
+
+    fn append_doc(&mut self, doc: DocId, term_freq: u32) {
+        let len = self.len;
+        self.doc_ids[len] = doc;
+        self.term_freqs[len] = term_freq;
+        self.len = len + 1;
+    }
+
+    fn is_full(&self) -> bool {
+        self.len == COMPRESSION_BLOCK_SIZE
+    }
+
+    fn is_empty(&self) -> bool {
+        self.len == 0
+    }
+
+    fn last_doc(&self) -> DocId {
+        assert_eq!(self.len, COMPRESSION_BLOCK_SIZE);
+        self.doc_ids[COMPRESSION_BLOCK_SIZE - 1]
+    }
+}
+
+/// Serializer for postings lists.
+pub struct PostingsSerializer {
+    last_doc_id_encoded: u32,
+
+    block_encoder: BlockEncoder,
+    block: Box<Block>,
+
+    postings_write: Vec<u8>,
+    skip_write: SkipSerializer,
+
+    mode: IndexRecordOption,
+    fieldnorm_reader: Option<FieldNormReader>,
+
+    bm25_weight: Option<Bm25Weight>,
+    avg_fieldnorm: Score, /* Average number of term in the field for that segment.
+                           * this value is used to compute the block wand information. */
+    term_has_freq: bool,
+}
+
+impl PostingsSerializer {
+    /// Creates a new `PostingsSerializer`.
+    /// * avg_fieldnorm - average field norm for the field being serialized.
+    /// * mode - indexing options for the field being serialized.
+    pub fn new(
+        avg_fieldnorm: Score,
+        mode: IndexRecordOption,
+        fieldnorm_reader: Option<FieldNormReader>,
+    ) -> PostingsSerializer {
+        PostingsSerializer {
+            block_encoder: BlockEncoder::new(),
+            block: Box::new(Block::new()),
+
+            postings_write: Vec::new(),
+            skip_write: SkipSerializer::new(),
+
+            last_doc_id_encoded: 0u32,
+            mode,
+
+            fieldnorm_reader,
+            bm25_weight: None,
+            avg_fieldnorm,
+            term_has_freq: false,
+        }
+    }
+
+    /// Starts the serialization for a new term.
+    /// * term_doc_freq - the number of documents containing the term.
+    pub fn new_term(&mut self, term_doc_freq: u32, record_term_freq: bool) {
+        self.bm25_weight = None;
+
+        self.term_has_freq = self.mode.has_freq() && record_term_freq;
+        if !self.term_has_freq {
+            return;
+        }
+
+        let num_docs_in_segment: u64 =
+            if let Some(fieldnorm_reader) = self.fieldnorm_reader.as_ref() {
+                fieldnorm_reader.num_docs() as u64
+            } else {
+                return;
+            };
+
+        if num_docs_in_segment == 0 {
+            return;
+        }
+
+        self.bm25_weight = Some(Bm25Weight::for_one_term_without_explain(
+            term_doc_freq as u64,
+            num_docs_in_segment,
+            self.avg_fieldnorm,
+        ));
+    }
+
+    fn write_block(&mut self) {
+        {
+            // encode the doc ids
+            let (num_bits, block_encoded): (u8, &[u8]) = self
+                .block_encoder
+                .compress_block_sorted(self.block.doc_ids(), self.last_doc_id_encoded);
+            self.last_doc_id_encoded = self.block.last_doc();
+            self.skip_write
+                .write_doc(self.last_doc_id_encoded, num_bits);
+            // last el block 0, offset block 1,
+            self.postings_write.extend(block_encoded);
+        }
+        if self.term_has_freq {
+            // encode the term frequencies
+            let (num_bits, block_encoded): (u8, &[u8]) = self
+                .block_encoder
+                .compress_block_unsorted(self.block.term_freqs(), true);
+            self.postings_write.extend(block_encoded);
+            self.skip_write.write_term_freq(num_bits);
+            if self.mode.has_positions() {
+                // We serialize the sum of term freqs within the skip information
+                // in order to navigate through positions.
+                let sum_freq = self.block.term_freqs().iter().cloned().sum();
+                self.skip_write.write_total_term_freq(sum_freq);
+            }
+            let mut blockwand_params = (0u8, 0u32);
+            if let Some(bm25_weight) = self.bm25_weight.as_ref() {
+                if let Some(fieldnorm_reader) = self.fieldnorm_reader.as_ref() {
+                    let docs = self.block.doc_ids().iter().cloned();
+                    let term_freqs = self.block.term_freqs().iter().cloned();
+                    let fieldnorms = docs.map(|doc| fieldnorm_reader.fieldnorm_id(doc));
+                    blockwand_params = fieldnorms
+                        .zip(term_freqs)
+                        .max_by(
+                            |(left_fieldnorm_id, left_term_freq),
+                             (right_fieldnorm_id, right_term_freq)| {
+                                let left_score =
+                                    bm25_weight.tf_factor(*left_fieldnorm_id, *left_term_freq);
+                                let right_score =
+                                    bm25_weight.tf_factor(*right_fieldnorm_id, *right_term_freq);
+                                left_score
+                                    .partial_cmp(&right_score)
+                                    .unwrap_or(Ordering::Equal)
+                            },
+                        )
+                        .unwrap();
+                }
+            }
+            let (fieldnorm_id, term_freq) = blockwand_params;
+            self.skip_write.write_blockwand_max(fieldnorm_id, term_freq);
+        }
+        self.block.clear();
+    }
+
+    /// Register that the given document contains the current term.
+    /// * doc_id - the document id.
+    /// * term_freq - the term frequency within the document.
+    pub fn write_doc(&mut self, doc_id: DocId, term_freq: u32) {
+        self.block.append_doc(doc_id, term_freq);
+        if self.block.is_full() {
+            self.write_block();
+        }
+    }
+
+    /// Finish the serialization for this term.
+    pub fn close_term(
+        &mut self,
+        doc_freq: u32,
+        output_write: &mut impl std::io::Write,
+    ) -> io::Result<()> {
+        if !self.block.is_empty() {
+            // we have doc ids waiting to be written
+            // this happens when the number of doc ids is
+            // not a perfect multiple of our block size.
+            //
+            // In that case, the remaining part is encoded
+            // using variable int encoding.
+            {
+                let block_encoded = self
+                    .block_encoder
+                    .compress_vint_sorted(self.block.doc_ids(), self.last_doc_id_encoded);
+                self.postings_write.write_all(block_encoded)?;
+            }
+            // ... Idem for term frequencies
+            if self.term_has_freq {
+                let block_encoded = self
+                    .block_encoder
+                    .compress_vint_unsorted(self.block.term_freqs());
+                self.postings_write.write_all(block_encoded)?;
+            }
+            self.block.clear();
+        }
+        if doc_freq >= COMPRESSION_BLOCK_SIZE as u32 {
+            let skip_data = self.skip_write.data();
+            VInt(skip_data.len() as u64).serialize(output_write)?;
+            output_write.write_all(skip_data)?;
+        }
+        output_write.write_all(&self.postings_write[..])?;
+        self.skip_write.clear();
+        self.postings_write.clear();
+        self.bm25_weight = None;
+        Ok(())
+    }
+
+    fn clear(&mut self) {
+        self.block.clear();
+        self.last_doc_id_encoded = 0;
+    }
+}

src/postings/skip.rs

@@ -146,6 +146,28 @@ impl SkipReader {
         skip_reader
     }
 
+    #[inline(always)]
+    pub fn has_remaining_docs(&self) -> bool {
+        self.remaining_docs != 0
+    }
+
+    pub fn reset(&mut self, data: OwnedBytes, doc_freq: u32) {
+        self.last_doc_in_block = if doc_freq >= COMPRESSION_BLOCK_SIZE as u32 {
+            0
+        } else {
+            TERMINATED
+        };
+        self.last_doc_in_previous_block = 0u32;
+        self.owned_read = data;
+        self.block_info = BlockInfo::VInt { num_docs: doc_freq };
+        self.byte_offset = 0;
+        self.remaining_docs = doc_freq;
+        self.position_offset = 0u64;
+        if doc_freq >= COMPRESSION_BLOCK_SIZE as u32 {
+            self.read_block_info();
+        }
+    }
+
     // Returns the block max score for this block if available.
     //
     // The block max score is available for all full bitpacked block,

src/query/all_query.rs

@@ -2,7 +2,7 @@ use crate::docset::{DocSet, COLLECT_BLOCK_BUFFER_LEN, TERMINATED};
 use crate::index::SegmentReader;
 use crate::query::boost_query::BoostScorer;
 use crate::query::explanation::does_not_match;
-use crate::query::{box_scorer, EnableScoring, Explanation, Query, Scorer, Weight};
+use crate::query::{EnableScoring, Explanation, Query, Scorer, Weight};
 use crate::{DocId, Score};
 
 /// Query that matches all of the documents.
@@ -24,9 +24,9 @@ impl Weight for AllWeight {
     fn scorer(&self, reader: &SegmentReader, boost: Score) -> crate::Result<Box<dyn Scorer>> {
         let all_scorer = AllScorer::new(reader.max_doc());
         if boost != 1.0 {
-            Ok(box_scorer(BoostScorer::new(all_scorer, boost)))
+            Ok(Box::new(BoostScorer::new(all_scorer, boost)))
         } else {
-            Ok(box_scorer(all_scorer))
+            Ok(Box::new(all_scorer))
         }
     }
 
@@ -109,6 +109,16 @@ impl Scorer for AllScorer {
     fn score(&mut self) -> Score {
         1.0
     }
+
+    #[inline]
+    fn can_score_doc(&self) -> bool {
+        true
+    }
+
+    #[inline]
+    fn score_doc(&mut self, _doc: DocId, _term_freq: u32) -> Score {
+        1.0
+    }
 }
 
 #[cfg(test)]

src/query/automaton_weight.rs

@@ -10,7 +10,7 @@ use crate::postings::TermInfo;
 use crate::query::{BitSetDocSet, ConstScorer, Explanation, Scorer, Weight};
 use crate::schema::{Field, IndexRecordOption};
 use crate::termdict::{TermDictionary, TermStreamer};
-use crate::{DocId, DocSet, Score, TantivyError};
+use crate::{DocId, Score, TantivyError};
 
 /// A weight struct for Fuzzy Term and Regex Queries
 pub struct AutomatonWeight<A> {
@@ -92,9 +92,18 @@ where
         let mut term_stream = self.automaton_stream(term_dict)?;
         while term_stream.advance() {
             let term_info = term_stream.value();
-            let mut block_segment_postings =
-                inverted_index.read_postings_from_terminfo(term_info, IndexRecordOption::Basic)?;
-            block_segment_postings.fill_bitset(&mut doc_bitset);
+            let mut block_segment_postings = inverted_index
+                .read_block_postings_from_terminfo(term_info, IndexRecordOption::Basic)?;
+            loop {
+                let docs = block_segment_postings.docs();
+                if docs.is_empty() {
+                    break;
+                }
+                for &doc in docs {
+                    doc_bitset.insert(doc);
+                }
+                block_segment_postings.advance();
+            }
         }
         let doc_bitset = BitSetDocSet::from(doc_bitset);
         let const_scorer = ConstScorer::new(doc_bitset, boost);

src/query/bitset/mod.rs

@@ -24,13 +24,6 @@ impl BitSetDocSet {
         self.cursor_bucket = bucket_addr;
         self.cursor_tinybitset = self.docs.tinyset(bucket_addr);
     }
-
-    /// Returns the number of documents in the bitset.
-    ///
-    /// This call is not free: it will bitcount the number of bits in the bitset.
-    pub fn doc_freq(&self) -> u32 {
-        self.docs.len() as u32
-    }
 }
 
 impl From<BitSet> for BitSetDocSet {

src/query/bm25.rs

@@ -1,5 +1,9 @@
+use std::cell::RefCell;
+use std::num::NonZeroUsize;
 use std::sync::Arc;
 
+use lru::LruCache;
+
 use crate::fieldnorm::FieldNormReader;
 use crate::query::Explanation;
 use crate::schema::Field;
@@ -59,7 +63,9 @@ fn cached_tf_component(fieldnorm: u32, average_fieldnorm: Score) -> Score {
     K1 * (1.0 - B + B * fieldnorm as Score / average_fieldnorm)
 }
 
-fn compute_tf_cache(average_fieldnorm: Score) -> Arc<[Score; 256]> {
+const BM25_TF_CACHE_CAPACITY: usize = 64;
+
+fn compute_tf_cache_uncached(average_fieldnorm: Score) -> Arc<[Score; 256]> {
     let mut cache: [Score; 256] = [0.0; 256];
     for (fieldnorm_id, cache_mut) in cache.iter_mut().enumerate() {
         let fieldnorm = FieldNormReader::id_to_fieldnorm(fieldnorm_id as u8);
@@ -68,6 +74,36 @@ fn compute_tf_cache(average_fieldnorm: Score) -> Arc<[Score; 256]> {
     Arc::new(cache)
 }
 
+thread_local! {
+    static TF_CACHES: RefCell<LruCache<u32, Arc<[Score; 256]>>> = RefCell::new(LruCache::new(
+        NonZeroUsize::new(BM25_TF_CACHE_CAPACITY).unwrap(),
+    ));
+}
+
+/// The cache is shared across all [Bm25Weight] with the same average fieldnorm on the same thread.
+/// It is stored in a thread local LRU cache.
+///
+/// On one query all terms on the same field will share the same average fieldnorm, and thus the
+/// same cache. This will lower cache pressure.
+///
+/// Even between queries (on the same thread), the cache will be reused, which allows the cache to
+/// better learn the memory address of the cache and access patterns.
+///
+/// Thread local is used in order to be defensive about potential contention on the cache.
+fn compute_tf_cache(average_fieldnorm: Score) -> Arc<[Score; 256]> {
+    let cache_key = average_fieldnorm.to_bits();
+    TF_CACHES.with(|cache_by_average_fieldnorm| {
+        let mut cache_by_average_fieldnorm = cache_by_average_fieldnorm.borrow_mut();
+        if let Some(cache) = cache_by_average_fieldnorm.get(&cache_key) {
+            return cache.clone();
+        }
+
+        let cache = compute_tf_cache_uncached(average_fieldnorm);
+        cache_by_average_fieldnorm.put(cache_key, cache.clone());
+        cache
+    })
+}
+
 /// A struct used for computing BM25 scores.
 #[derive(Clone)]
 pub struct Bm25Weight {
@@ -229,7 +265,7 @@ impl Bm25Weight {
 #[cfg(test)]
 mod tests {
 
-    use super::idf;
+    use super::{idf, Bm25Weight};
     use crate::{assert_nearly_equals, Score};
 
     #[test]
@@ -237,4 +273,12 @@ mod tests {
         let score: Score = 2.0;
         assert_nearly_equals!(idf(1, 2), score.ln());
     }
+
+    #[test]
+    fn test_bm25_tf_cache_is_shared_for_same_average_fieldnorm() {
+        let weight1 = Bm25Weight::for_one_term(1, 10, 3.0);
+        let weight2 = Bm25Weight::for_one_term(2, 10, 3.0);
+
+        assert!(std::sync::Arc::ptr_eq(&weight1.cache, &weight2.cache));
+    }
 }

src/query/boolean_query/block_wand_union.rs

@@ -1,6 +1,5 @@
 use std::ops::{Deref, DerefMut};
 
-use crate::codec::postings::PostingsWithBlockMax;
 use crate::query::term_query::TermScorer;
 use crate::query::Scorer;
 use crate::{DocId, DocSet, Score, TERMINATED};
@@ -14,8 +13,8 @@ use crate::{DocId, DocSet, Score, TERMINATED};
 /// We always have `before_pivot_len` < `pivot_len`.
 ///
 /// `None` is returned if we establish that no document can exceed the threshold.
-fn find_pivot_doc<TPostings: PostingsWithBlockMax>(
-    term_scorers: &[TermScorerWithMaxScore<TPostings>],
+fn find_pivot_doc(
+    term_scorers: &[TermScorerWithMaxScore],
     threshold: Score,
 ) -> Option<(usize, usize, DocId)> {
     let mut max_score = 0.0;
@@ -47,8 +46,8 @@ fn find_pivot_doc<TPostings: PostingsWithBlockMax>(
 /// the next doc candidate defined by the min of `last_doc_in_block + 1` for
 /// scorer in scorers[..pivot_len] and `scorer.doc()` for scorer in scorers[pivot_len..].
 /// Note: before and after calling this method, scorers need to be sorted by their `.doc()`.
-fn block_max_was_too_low_advance_one_scorer<TPostings: PostingsWithBlockMax>(
-    scorers: &mut [TermScorerWithMaxScore<TPostings>],
+fn block_max_was_too_low_advance_one_scorer(
+    scorers: &mut [TermScorerWithMaxScore],
     pivot_len: usize,
 ) {
     debug_assert!(scorers.iter().map(|scorer| scorer.doc()).is_sorted());
@@ -83,10 +82,7 @@ fn block_max_was_too_low_advance_one_scorer<TPostings: PostingsWithBlockMax>(
 // Given a list of term_scorers and a `ord` and assuming that `term_scorers[ord]` is sorted
 // except term_scorers[ord] that might be in advance compared to its ranks,
 // bubble up term_scorers[ord] in order to restore the ordering.
-fn restore_ordering<TPostings: PostingsWithBlockMax>(
-    term_scorers: &mut [TermScorerWithMaxScore<TPostings>],
-    ord: usize,
-) {
+fn restore_ordering(term_scorers: &mut [TermScorerWithMaxScore], ord: usize) {
     let doc = term_scorers[ord].doc();
     for i in ord + 1..term_scorers.len() {
         if term_scorers[i].doc() >= doc {
@@ -101,10 +97,9 @@ fn restore_ordering<TPostings: PostingsWithBlockMax>(
 // If this works, return true.
 // If this fails (ie: one of the term_scorer does not contain `pivot_doc` and seek goes past the
 // pivot), reorder the term_scorers to ensure the list is still sorted and returns `false`.
-// If a term_scorer reach TERMINATED in the process return false remove the term_scorer and
-// return.
-fn align_scorers<TPostings: PostingsWithBlockMax>(
-    term_scorers: &mut Vec<TermScorerWithMaxScore<TPostings>>,
+// If a term_scorer reach TERMINATED in the process return false remove the term_scorer and return.
+fn align_scorers(
+    term_scorers: &mut Vec<TermScorerWithMaxScore>,
     pivot_doc: DocId,
     before_pivot_len: usize,
 ) -> bool {
@@ -131,10 +126,7 @@ fn align_scorers<TPostings: PostingsWithBlockMax>(
 // Assumes terms_scorers[..pivot_len] are positioned on the same doc (pivot_doc).
 // Advance term_scorers[..pivot_len] and out of these removes the terminated scores.
 // Restores the ordering of term_scorers.
-fn advance_all_scorers_on_pivot<TPostings: PostingsWithBlockMax>(
-    term_scorers: &mut Vec<TermScorerWithMaxScore<TPostings>>,
-    pivot_len: usize,
-) {
+fn advance_all_scorers_on_pivot(term_scorers: &mut Vec<TermScorerWithMaxScore>, pivot_len: usize) {
     for term_scorer in &mut term_scorers[..pivot_len] {
         term_scorer.advance();
     }
@@ -153,8 +145,8 @@ fn advance_all_scorers_on_pivot<TPostings: PostingsWithBlockMax>(
 /// Implements the WAND (Weak AND) algorithm for dynamic pruning
 /// described in the paper "Faster Top-k Document Retrieval Using Block-Max Indexes".
 /// Link: <http://engineering.nyu.edu/~suel/papers/bmw.pdf>
-pub fn block_wand<TPostings: PostingsWithBlockMax>(
-    mut scorers: Vec<TermScorer<TPostings>>,
+pub fn block_wand(
+    mut scorers: Vec<TermScorer>,
     mut threshold: Score,
     callback: &mut dyn FnMut(u32, Score) -> Score,
 ) {
@@ -163,7 +155,7 @@ pub fn block_wand<TPostings: PostingsWithBlockMax>(
         let scorer = scorers.pop().unwrap();
         return block_wand_single_scorer(scorer, threshold, callback);
     }
-    let mut scorers: Vec<TermScorerWithMaxScore<TPostings>> = scorers
+    let mut scorers: Vec<TermScorerWithMaxScore> = scorers
         .iter_mut()
         .map(TermScorerWithMaxScore::from)
         .collect();
@@ -178,7 +170,10 @@ pub fn block_wand<TPostings: PostingsWithBlockMax>(
 
         let block_max_score_upperbound: Score = scorers[..pivot_len]
             .iter_mut()
-            .map(|scorer| scorer.seek_block_max(pivot_doc))
+            .map(|scorer| {
+                scorer.seek_block(pivot_doc);
+                scorer.block_max_score()
+            })
             .sum();
 
         // Beware after shallow advance, skip readers can be in advance compared to
@@ -229,22 +224,21 @@ pub fn block_wand<TPostings: PostingsWithBlockMax>(
 ///   - On a block, advance until the end and execute `callback` when the doc score is greater or
 ///     equal to the `threshold`.
 pub fn block_wand_single_scorer(
-    mut scorer: TermScorer<impl PostingsWithBlockMax>,
+    mut scorer: TermScorer,
     mut threshold: Score,
     callback: &mut dyn FnMut(u32, Score) -> Score,
 ) {
     let mut doc = scorer.doc();
-    let mut block_max_score = scorer.seek_block_max(doc);
     loop {
         // We position the scorer on a block that can reach
         // the threshold.
-        while block_max_score < threshold {
+        while scorer.block_max_score() <= threshold {
             let last_doc_in_block = scorer.last_doc_in_block();
             if last_doc_in_block == TERMINATED {
                 return;
             }
             doc = last_doc_in_block + 1;
-            block_max_score = scorer.seek_block_max(doc);
+            scorer.seek_block(doc);
         }
         // Seek will effectively load that block.
         doc = scorer.seek(doc);
@@ -266,33 +260,31 @@ pub fn block_wand_single_scorer(
             }
         }
         doc += 1;
-        block_max_score = scorer.seek_block_max(doc);
+        scorer.seek_block(doc);
     }
 }
 
-struct TermScorerWithMaxScore<'a, TPostings: PostingsWithBlockMax> {
-    scorer: &'a mut TermScorer<TPostings>,
+struct TermScorerWithMaxScore<'a> {
+    scorer: &'a mut TermScorer,
     max_score: Score,
 }
 
-impl<'a, TPostings: PostingsWithBlockMax> From<&'a mut TermScorer<TPostings>>
-    for TermScorerWithMaxScore<'a, TPostings>
-{
-    fn from(scorer: &'a mut TermScorer<TPostings>) -> Self {
+impl<'a> From<&'a mut TermScorer> for TermScorerWithMaxScore<'a> {
+    fn from(scorer: &'a mut TermScorer) -> Self {
         let max_score = scorer.max_score();
         TermScorerWithMaxScore { scorer, max_score }
     }
 }
 
-impl<TPostings: PostingsWithBlockMax> Deref for TermScorerWithMaxScore<'_, TPostings> {
-    type Target = TermScorer<TPostings>;
+impl Deref for TermScorerWithMaxScore<'_> {
+    type Target = TermScorer;
 
     fn deref(&self) -> &Self::Target {
         self.scorer
     }
 }
 
-impl<TPostings: PostingsWithBlockMax> DerefMut for TermScorerWithMaxScore<'_, TPostings> {
+impl DerefMut for TermScorerWithMaxScore<'_> {
     fn deref_mut(&mut self) -> &mut Self::Target {
         self.scorer
     }

src/query/boolean_query/boolean_weight.rs

@@ -1,18 +1,25 @@
 use std::collections::HashMap;
 
-use crate::codec::{ObjectSafeCodec, SumOrDoNothingCombiner};
 use crate::docset::COLLECT_BLOCK_BUFFER_LEN;
 use crate::index::SegmentReader;
+use crate::postings::FreqReadingOption;
 use crate::query::disjunction::Disjunction;
 use crate::query::explanation::does_not_match;
 use crate::query::score_combiner::{DoNothingCombiner, ScoreCombiner};
-use crate::query::weight::for_each_docset_buffered;
+use crate::query::term_query::TermScorer;
+use crate::query::weight::{for_each_docset_buffered, for_each_pruning_scorer, for_each_scorer};
 use crate::query::{
-    box_scorer, intersect_scorers, AllScorer, BufferedUnionScorer, EmptyScorer, Exclude,
-    Explanation, Occur, RequiredOptionalScorer, Scorer, SumCombiner, Weight,
+    intersect_scorers, AllScorer, BufferedUnionScorer, EmptyScorer, Exclude, Explanation, Occur,
+    RequiredOptionalScorer, Scorer, Weight,
 };
 use crate::{DocId, Score};
 
+enum SpecializedScorer {
+    TermUnion(Vec<TermScorer>),
+    TermIntersection(Vec<TermScorer>),
+    Other(Box<dyn Scorer>),
+}
+
 fn scorer_disjunction<TScoreCombiner>(
     scorers: Vec<Box<dyn Scorer>>,
     score_combiner: TScoreCombiner,
@@ -26,7 +33,7 @@ where
     if scorers.len() == 1 {
         return scorers.into_iter().next().unwrap(); // Safe unwrap.
     }
-    box_scorer(Disjunction::new(
+    Box::new(Disjunction::new(
         scorers,
         score_combiner,
         minimum_match_required,
@@ -38,41 +45,70 @@ fn scorer_union<TScoreCombiner>(
     scorers: Vec<Box<dyn Scorer>>,
     score_combiner_fn: impl Fn() -> TScoreCombiner,
     num_docs: u32,
-    codec: &dyn ObjectSafeCodec,
-) -> Box<dyn Scorer>
+) -> SpecializedScorer
 where
     TScoreCombiner: ScoreCombiner,
 {
-    match scorers.len() {
-        0 => box_scorer(EmptyScorer),
-        1 => scorers.into_iter().next().unwrap(),
-        _ => {
-            let combiner_opt: Option<SumOrDoNothingCombiner> = if std::any::TypeId::of::<
-                TScoreCombiner,
-            >() == std::any::TypeId::of::<
-                SumCombiner,
-            >() {
-                Some(SumOrDoNothingCombiner::Sum)
-            } else if std::any::TypeId::of::<TScoreCombiner>()
-                == std::any::TypeId::of::<DoNothingCombiner>()
+    assert!(!scorers.is_empty());
+    if scorers.len() == 1 && !scorers[0].is::<TermScorer>() {
+        return SpecializedScorer::Other(scorers.into_iter().next().unwrap()); //< we checked the size beforehand
+    }
+    {
+        let is_all_term_queries = scorers.iter().all(|scorer| scorer.is::<TermScorer>());
+        if is_all_term_queries {
+            let scorers: Vec<TermScorer> = scorers
+                .into_iter()
+                .map(|scorer| *(scorer.downcast::<TermScorer>().map_err(|_| ()).unwrap()))
+                .collect();
+            if scorers
+                .iter()
+                .all(|scorer| scorer.freq_reading_option() == FreqReadingOption::ReadFreq)
             {
-                Some(SumOrDoNothingCombiner::DoNothing)
+                // Block wand is only available if we read frequencies.
+                return SpecializedScorer::TermUnion(scorers);
+            } else if scorers.len() == 1 {
+                // Single TermScorer without freq reading — unwrap directly.
+                return SpecializedScorer::Other(Box::new(scorers.into_iter().next().unwrap()));
             } else {
-                None
-            };
-            if let Some(combiner) = combiner_opt {
-                let scorer =
-                    codec.build_union_scorer_with_sum_combiner(scorers, num_docs, combiner);
-                scorer
-            } else {
-                box_scorer(BufferedUnionScorer::build(
+                return SpecializedScorer::Other(Box::new(BufferedUnionScorer::build(
                     scorers,
                     score_combiner_fn,
                     num_docs,
-                ))
+                )));
             }
         }
     }
+    SpecializedScorer::Other(Box::new(BufferedUnionScorer::build(
+        scorers,
+        score_combiner_fn,
+        num_docs,
+    )))
+}
+
+fn into_box_scorer<TScoreCombiner: ScoreCombiner>(
+    scorer: SpecializedScorer,
+    score_combiner_fn: impl Fn() -> TScoreCombiner,
+    num_docs: u32,
+) -> Box<dyn Scorer> {
+    match scorer {
+        SpecializedScorer::TermUnion(mut term_scorers) => {
+            if term_scorers.len() == 1 {
+                Box::new(term_scorers.pop().unwrap())
+            } else {
+                let union_scorer =
+                    BufferedUnionScorer::build(term_scorers, score_combiner_fn, num_docs);
+                Box::new(union_scorer)
+            }
+        }
+        SpecializedScorer::TermIntersection(term_scorers) => {
+            let boxed_scorers: Vec<Box<dyn Scorer>> = term_scorers
+                .into_iter()
+                .map(|s| Box::new(s) as Box<dyn Scorer>)
+                .collect();
+            intersect_scorers(boxed_scorers, num_docs)
+        }
+        SpecializedScorer::Other(scorer) => scorer,
+    }
 }
 
 /// Returns the effective MUST scorer, accounting for removed AllScorers.
@@ -88,7 +124,7 @@ fn effective_must_scorer(
     if must_scorers.is_empty() {
         if removed_all_scorer_count > 0 {
             // Had AllScorer(s) only - all docs match
-            Some(box_scorer(AllScorer::new(max_doc)))
+            Some(Box::new(AllScorer::new(max_doc)))
         } else {
             // No MUST constraint at all
             None
@@ -106,26 +142,28 @@ fn effective_must_scorer(
 /// When `scoring_enabled` is false, we can just return AllScorer alone since
 /// we don't need score contributions from the should_scorer.
 fn effective_should_scorer_for_union<TScoreCombiner: ScoreCombiner>(
-    should_scorer: Box<dyn Scorer>,
+    should_scorer: SpecializedScorer,
     removed_all_scorer_count: usize,
     max_doc: DocId,
     num_docs: u32,
     score_combiner_fn: impl Fn() -> TScoreCombiner,
     scoring_enabled: bool,
-) -> Box<dyn Scorer> {
+) -> SpecializedScorer {
     if removed_all_scorer_count > 0 {
         if scoring_enabled {
             // Need to union to get score contributions from both
-            let all_scorers: Vec<Box<dyn Scorer>> =
-                vec![should_scorer, box_scorer(AllScorer::new(max_doc))];
-            box_scorer(BufferedUnionScorer::build(
+            let all_scorers: Vec<Box<dyn Scorer>> = vec![
+                into_box_scorer(should_scorer, &score_combiner_fn, num_docs),
+                Box::new(AllScorer::new(max_doc)),
+            ];
+            SpecializedScorer::Other(Box::new(BufferedUnionScorer::build(
                 all_scorers,
                 score_combiner_fn,
                 num_docs,
-            ))
+            )))
         } else {
             // Scoring disabled - AllScorer alone is sufficient
-            box_scorer(AllScorer::new(max_doc))
+            SpecializedScorer::Other(Box::new(AllScorer::new(max_doc)))
         }
     } else {
         should_scorer
@@ -136,9 +174,9 @@ enum ShouldScorersCombinationMethod {
     // Should scorers are irrelevant.
     Ignored,
     // Only contributes to final score.
-    Optional(Box<dyn Scorer>),
+    Optional(SpecializedScorer),
     // Regardless of score, the should scorers may impact whether a document is matching or not.
-    Required(Box<dyn Scorer>),
+    Required(SpecializedScorer),
 }
 
 /// Weight associated to the `BoolQuery`.
@@ -200,7 +238,7 @@ impl<TScoreCombiner: ScoreCombiner> BooleanWeight<TScoreCombiner> {
         reader: &SegmentReader,
         boost: Score,
         score_combiner_fn: impl Fn() -> TComplexScoreCombiner,
-    ) -> crate::Result<Box<dyn Scorer>> {
+    ) -> crate::Result<SpecializedScorer> {
         let num_docs = reader.num_docs();
         let mut per_occur_scorers = self.per_occur_scorers(reader, boost)?;
 
@@ -210,7 +248,7 @@ impl<TScoreCombiner: ScoreCombiner> BooleanWeight<TScoreCombiner> {
         let must_special_scorer_counts = remove_and_count_all_and_empty_scorers(&mut must_scorers);
 
         if must_special_scorer_counts.num_empty_scorers > 0 {
-            return Ok(box_scorer(EmptyScorer));
+            return Ok(SpecializedScorer::Other(Box::new(EmptyScorer)));
         }
 
         let mut should_scorers = per_occur_scorers.remove(&Occur::Should).unwrap_or_default();
@@ -225,7 +263,7 @@ impl<TScoreCombiner: ScoreCombiner> BooleanWeight<TScoreCombiner> {
 
         if exclude_special_scorer_counts.num_all_scorers > 0 {
             // We exclude all documents at one point.
-            return Ok(box_scorer(EmptyScorer));
+            return Ok(SpecializedScorer::Other(Box::new(EmptyScorer)));
         }
 
         let effective_minimum_number_should_match = self
@@ -237,7 +275,7 @@ impl<TScoreCombiner: ScoreCombiner> BooleanWeight<TScoreCombiner> {
             if effective_minimum_number_should_match > num_of_should_scorers {
                 // We don't have enough scorers to satisfy the minimum number of should matches.
                 // The request will match no documents.
-                return Ok(box_scorer(EmptyScorer));
+                return Ok(SpecializedScorer::Other(Box::new(EmptyScorer)));
             }
             match effective_minimum_number_should_match {
                 0 if num_of_should_scorers == 0 => ShouldScorersCombinationMethod::Ignored,
@@ -245,13 +283,11 @@ impl<TScoreCombiner: ScoreCombiner> BooleanWeight<TScoreCombiner> {
                     should_scorers,
                     &score_combiner_fn,
                     num_docs,
-                    reader.codec(),
                 )),
                 1 => ShouldScorersCombinationMethod::Required(scorer_union(
                     should_scorers,
                     &score_combiner_fn,
                     num_docs,
-                    reader.codec(),
                 )),
                 n if num_of_should_scorers == n => {
                     // When num_of_should_scorers equals the number of should clauses,
@@ -259,40 +295,59 @@ impl<TScoreCombiner: ScoreCombiner> BooleanWeight<TScoreCombiner> {
                     must_scorers.append(&mut should_scorers);
                     ShouldScorersCombinationMethod::Ignored
                 }
-                _ => ShouldScorersCombinationMethod::Required(scorer_disjunction(
-                    should_scorers,
-                    score_combiner_fn(),
-                    effective_minimum_number_should_match,
+                _ => ShouldScorersCombinationMethod::Required(SpecializedScorer::Other(
+                    scorer_disjunction(
+                        should_scorers,
+                        score_combiner_fn(),
+                        effective_minimum_number_should_match,
+                    ),
                 )),
             }
         };
 
-        let exclude_scorer_opt: Option<Box<dyn Scorer>> = if exclude_scorers.is_empty() {
-            None
-        } else {
-            let exclude_scorers_union: Box<dyn Scorer> = scorer_union(
-                exclude_scorers,
-                DoNothingCombiner::default,
-                num_docs,
-                reader.codec(),
-            );
-            Some(exclude_scorers_union)
-        };
-
         let include_scorer = match (should_scorers, must_scorers) {
             (ShouldScorersCombinationMethod::Ignored, must_scorers) => {
                 // No SHOULD clauses (or they were absorbed into MUST).
                 // Result depends entirely on MUST + any removed AllScorers.
                 let combined_all_scorer_count = must_special_scorer_counts.num_all_scorers
                     + should_special_scorer_counts.num_all_scorers;
-                let boxed_scorer: Box<dyn Scorer> = effective_must_scorer(
-                    must_scorers,
-                    combined_all_scorer_count,
-                    reader.max_doc(),
-                    num_docs,
-                )
-                .unwrap_or_else(|| box_scorer(EmptyScorer));
-                boxed_scorer
+
+                // Try to detect a pure TermScorer intersection for block-max optimization.
+                // Preconditions: no removed AllScorers, at least 2 scorers, all TermScorer
+                // with frequency reading enabled.
+                if combined_all_scorer_count == 0
+                    && must_scorers.len() >= 2
+                    && must_scorers.iter().all(|s| s.is::<TermScorer>())
+                {
+                    let term_scorers: Vec<TermScorer> = must_scorers
+                        .into_iter()
+                        .map(|s| *(s.downcast::<TermScorer>().map_err(|_| ()).unwrap()))
+                        .collect();
+                    if term_scorers
+                        .iter()
+                        .all(|s| s.freq_reading_option() == FreqReadingOption::ReadFreq)
+                    {
+                        SpecializedScorer::TermIntersection(term_scorers)
+                    } else {
+                        let must_scorers: Vec<Box<dyn Scorer>> = term_scorers
+                            .into_iter()
+                            .map(|s| Box::new(s) as Box<dyn Scorer>)
+                            .collect();
+                        let boxed_scorer: Box<dyn Scorer> =
+                            effective_must_scorer(must_scorers, 0, reader.max_doc(), num_docs)
+                                .unwrap_or_else(|| Box::new(EmptyScorer));
+                        SpecializedScorer::Other(boxed_scorer)
+                    }
+                } else {
+                    let boxed_scorer: Box<dyn Scorer> = effective_must_scorer(
+                        must_scorers,
+                        combined_all_scorer_count,
+                        reader.max_doc(),
+                        num_docs,
+                    )
+                    .unwrap_or_else(|| Box::new(EmptyScorer));
+                    SpecializedScorer::Other(boxed_scorer)
+                }
             }
             (ShouldScorersCombinationMethod::Optional(should_scorer), must_scorers) => {
                 // Optional SHOULD: contributes to scoring but not required for matching.
@@ -317,12 +372,16 @@ impl<TScoreCombiner: ScoreCombiner> BooleanWeight<TScoreCombiner> {
                     Some(must_scorer) => {
                         // Has MUST constraint: SHOULD only affects scoring.
                         if self.scoring_enabled {
-                            box_scorer(RequiredOptionalScorer::<_, _, TScoreCombiner>::new(
+                            SpecializedScorer::Other(Box::new(RequiredOptionalScorer::<
+                                _,
+                                _,
+                                TScoreCombiner,
+                            >::new(
                                 must_scorer,
-                                should_scorer,
-                            ))
+                                into_box_scorer(should_scorer, &score_combiner_fn, num_docs),
+                            )))
                         } else {
-                            must_scorer
+                            SpecializedScorer::Other(must_scorer)
                         }
                     }
                 }
@@ -342,16 +401,33 @@ impl<TScoreCombiner: ScoreCombiner> BooleanWeight<TScoreCombiner> {
                     }
                     Some(must_scorer) => {
                         // Has MUST constraint: intersect MUST with SHOULD.
-                        intersect_scorers(vec![must_scorer, should_scorer], num_docs)
+                        let should_boxed =
+                            into_box_scorer(should_scorer, &score_combiner_fn, num_docs);
+                        SpecializedScorer::Other(intersect_scorers(
+                            vec![must_scorer, should_boxed],
+                            num_docs,
+                        ))
                     }
                 }
             }
         };
-        if let Some(exclude_scorer) = exclude_scorer_opt {
-            Ok(box_scorer(Exclude::new(include_scorer, exclude_scorer)))
-        } else {
-            Ok(include_scorer)
+        if exclude_scorers.is_empty() {
+            return Ok(include_scorer);
         }
+
+        let include_scorer_boxed = into_box_scorer(include_scorer, &score_combiner_fn, num_docs);
+        let scorer: Box<dyn Scorer> = if exclude_scorers.len() == 1 {
+            let exclude_scorer = exclude_scorers.pop().unwrap();
+            match exclude_scorer.downcast::<TermScorer>() {
+                // Cast to TermScorer succeeded
+                Ok(exclude_scorer) => Box::new(Exclude::new(include_scorer_boxed, *exclude_scorer)),
+                // We get back the original Box<dyn Scorer>
+                Err(exclude_scorer) => Box::new(Exclude::new(include_scorer_boxed, exclude_scorer)),
+            }
+        } else {
+            Box::new(Exclude::new(include_scorer_boxed, exclude_scorers))
+        };
+        Ok(SpecializedScorer::Other(scorer))
     }
 }
 
@@ -381,6 +457,7 @@ fn remove_and_count_all_and_empty_scorers(
 
 impl<TScoreCombiner: ScoreCombiner + Sync> Weight for BooleanWeight<TScoreCombiner> {
     fn scorer(&self, reader: &SegmentReader, boost: Score) -> crate::Result<Box<dyn Scorer>> {
+        let num_docs = reader.num_docs();
         if self.weights.is_empty() {
             Ok(Box::new(EmptyScorer))
         } else if self.weights.len() == 1 {
@@ -392,8 +469,14 @@ impl<TScoreCombiner: ScoreCombiner + Sync> Weight for BooleanWeight<TScoreCombin
             }
         } else if self.scoring_enabled {
             self.complex_scorer(reader, boost, &self.score_combiner_fn)
+                .map(|specialized_scorer| {
+                    into_box_scorer(specialized_scorer, &self.score_combiner_fn, num_docs)
+                })
         } else {
             self.complex_scorer(reader, boost, DoNothingCombiner::default)
+                .map(|specialized_scorer| {
+                    into_box_scorer(specialized_scorer, DoNothingCombiner::default, num_docs)
+                })
         }
     }
 
@@ -422,8 +505,26 @@ impl<TScoreCombiner: ScoreCombiner + Sync> Weight for BooleanWeight<TScoreCombin
         reader: &SegmentReader,
         callback: &mut dyn FnMut(DocId, Score),
     ) -> crate::Result<()> {
-        let mut scorer = self.complex_scorer(reader, 1.0, &self.score_combiner_fn)?;
-        scorer.for_each(callback);
+        let scorer = self.complex_scorer(reader, 1.0, &self.score_combiner_fn)?;
+        let num_docs = reader.num_docs();
+        match scorer {
+            SpecializedScorer::TermUnion(term_scorers) => {
+                let mut union_scorer =
+                    BufferedUnionScorer::build(term_scorers, &self.score_combiner_fn, num_docs);
+                for_each_scorer(&mut union_scorer, callback);
+            }
+            SpecializedScorer::TermIntersection(term_scorers) => {
+                let boxed_scorers: Vec<Box<dyn Scorer>> = term_scorers
+                    .into_iter()
+                    .map(|term_scorer| Box::new(term_scorer) as Box<dyn Scorer>)
+                    .collect();
+                let mut intersection = intersect_scorers(boxed_scorers, num_docs);
+                for_each_scorer(intersection.as_mut(), callback);
+            }
+            SpecializedScorer::Other(mut scorer) => {
+                for_each_scorer(scorer.as_mut(), callback);
+            }
+        }
         Ok(())
     }
 
@@ -432,9 +533,28 @@ impl<TScoreCombiner: ScoreCombiner + Sync> Weight for BooleanWeight<TScoreCombin
         reader: &SegmentReader,
         callback: &mut dyn FnMut(&[DocId]),
     ) -> crate::Result<()> {
-        let mut scorer = self.complex_scorer(reader, 1.0, || DoNothingCombiner)?;
+        let scorer = self.complex_scorer(reader, 1.0, || DoNothingCombiner)?;
+        let num_docs = reader.num_docs();
         let mut buffer = [0u32; COLLECT_BLOCK_BUFFER_LEN];
-        for_each_docset_buffered(scorer.as_mut(), &mut buffer, callback);
+
+        match scorer {
+            SpecializedScorer::TermUnion(term_scorers) => {
+                let mut union_scorer =
+                    BufferedUnionScorer::build(term_scorers, &self.score_combiner_fn, num_docs);
+                for_each_docset_buffered(&mut union_scorer, &mut buffer, callback);
+            }
+            SpecializedScorer::TermIntersection(term_scorers) => {
+                let boxed_scorers: Vec<Box<dyn Scorer>> = term_scorers
+                    .into_iter()
+                    .map(|term_scorer| Box::new(term_scorer) as Box<dyn Scorer>)
+                    .collect();
+                let mut intersection = intersect_scorers(boxed_scorers, num_docs);
+                for_each_docset_buffered(intersection.as_mut(), &mut buffer, callback);
+            }
+            SpecializedScorer::Other(mut scorer) => {
+                for_each_docset_buffered(scorer.as_mut(), &mut buffer, callback);
+            }
+        }
         Ok(())
     }
 
@@ -455,7 +575,17 @@ impl<TScoreCombiner: ScoreCombiner + Sync> Weight for BooleanWeight<TScoreCombin
         callback: &mut dyn FnMut(DocId, Score) -> Score,
     ) -> crate::Result<()> {
         let scorer = self.complex_scorer(reader, 1.0, &self.score_combiner_fn)?;
-        reader.codec().for_each_pruning(threshold, scorer, callback);
+        match scorer {
+            SpecializedScorer::TermUnion(term_scorers) => {
+                super::block_wand(term_scorers, threshold, callback);
+            }
+            SpecializedScorer::TermIntersection(term_scorers) => {
+                super::block_wand_intersection(term_scorers, threshold, callback);
+            }
+            SpecializedScorer::Other(mut scorer) => {
+                for_each_pruning_scorer(scorer.as_mut(), threshold, callback);
+            }
+        }
         Ok(())
     }
 }

src/query/boolean_query/mod.rs

@@ -1,6 +1,10 @@
+mod block_wand_intersection;
+mod block_wand_union;
 mod boolean_query;
 mod boolean_weight;
 
+pub(crate) use self::block_wand_intersection::block_wand_intersection;
+pub(crate) use self::block_wand_union::{block_wand, block_wand_single_scorer};
 pub use self::boolean_query::BooleanQuery;
 pub use self::boolean_weight::BooleanWeight;
 

src/query/boost_query.rs

@@ -112,6 +112,14 @@ impl<S: Scorer> DocSet for BoostScorer<S> {
         self.underlying.fill_buffer(buffer)
     }
 
+    fn fill_buffer_up_to(
+        &mut self,
+        horizon: DocId,
+        buffer: &mut [DocId; COLLECT_BLOCK_BUFFER_LEN],
+    ) -> usize {
+        self.underlying.fill_buffer_up_to(horizon, buffer)
+    }
+
     fn doc(&self) -> u32 {
         self.underlying.doc()
     }
@@ -138,6 +146,27 @@ impl<S: Scorer> Scorer for BoostScorer<S> {
     fn score(&mut self) -> Score {
         self.underlying.score() * self.boost
     }
+
+    #[inline]
+    fn can_score_doc(&self) -> bool {
+        self.underlying.can_score_doc()
+    }
+
+    #[inline]
+    fn score_doc(&mut self, doc: DocId, term_freq: u32) -> Score {
+        self.underlying.score_doc(doc, term_freq) * self.boost
+    }
+
+    #[inline]
+    fn fill_buffer_up_to_with_term_freqs(
+        &mut self,
+        horizon: DocId,
+        docs: &mut [DocId; COLLECT_BLOCK_BUFFER_LEN],
+        term_freqs: &mut [u32; COLLECT_BLOCK_BUFFER_LEN],
+    ) -> usize {
+        self.underlying
+            .fill_buffer_up_to_with_term_freqs(horizon, docs, term_freqs)
+    }
 }
 
 #[cfg(test)]

src/query/const_score_query.rs

@@ -1,7 +1,7 @@
 use std::fmt;
 
 use crate::docset::COLLECT_BLOCK_BUFFER_LEN;
-use crate::query::{box_scorer, EnableScoring, Explanation, Query, Scorer, Weight};
+use crate::query::{EnableScoring, Explanation, Query, Scorer, Weight};
 use crate::{DocId, DocSet, Score, SegmentReader, TantivyError, Term};
 
 /// `ConstScoreQuery` is a wrapper over a query to provide a constant score.
@@ -65,10 +65,7 @@ impl ConstWeight {
 impl Weight for ConstWeight {
     fn scorer(&self, reader: &SegmentReader, boost: Score) -> crate::Result<Box<dyn Scorer>> {
         let inner_scorer = self.weight.scorer(reader, boost)?;
-        Ok(box_scorer(ConstScorer::new(
-            inner_scorer,
-            boost * self.score,
-        )))
+        Ok(Box::new(ConstScorer::new(inner_scorer, boost * self.score)))
     }
 
     fn explain(&self, reader: &SegmentReader, doc: u32) -> crate::Result<Explanation> {
@@ -144,6 +141,16 @@ impl<TDocSet: DocSet + 'static> Scorer for ConstScorer<TDocSet> {
     fn score(&mut self) -> Score {
         self.score
     }
+
+    #[inline]
+    fn can_score_doc(&self) -> bool {
+        true
+    }
+
+    #[inline]
+    fn score_doc(&mut self, _doc: DocId, _term_freq: u32) -> Score {
+        self.score
+    }
 }
 
 #[cfg(test)]

src/query/disjunction.rs

@@ -315,6 +315,20 @@ mod tests {
         fn score(&mut self) -> Score {
             self.foo.get(self.cursor).map(|x| x.1).unwrap_or(0.0)
         }
+
+        #[inline]
+        fn can_score_doc(&self) -> bool {
+            true
+        }
+
+        #[inline]
+        fn score_doc(&mut self, doc: DocId, _term_freq: u32) -> Score {
+            self.foo
+                .iter()
+                .find(|(candidate_doc, _)| *candidate_doc == doc)
+                .map(|(_, score)| *score)
+                .unwrap_or(0.0)
+        }
     }
 
     #[test]

src/query/empty_query.rs

@@ -2,7 +2,7 @@ use super::Scorer;
 use crate::docset::TERMINATED;
 use crate::index::SegmentReader;
 use crate::query::explanation::does_not_match;
-use crate::query::{box_scorer, EnableScoring, Explanation, Query, Weight};
+use crate::query::{EnableScoring, Explanation, Query, Weight};
 use crate::{DocId, DocSet, Score, Searcher};
 
 /// `EmptyQuery` is a dummy `Query` in which no document matches.
@@ -27,7 +27,7 @@ impl Query for EmptyQuery {
 pub struct EmptyWeight;
 impl Weight for EmptyWeight {
     fn scorer(&self, _reader: &SegmentReader, _boost: Score) -> crate::Result<Box<dyn Scorer>> {
-        Ok(box_scorer(EmptyScorer))
+        Ok(Box::new(EmptyScorer))
     }
 
     fn explain(&self, _reader: &SegmentReader, doc: DocId) -> crate::Result<Explanation> {
@@ -59,6 +59,16 @@ impl Scorer for EmptyScorer {
     fn score(&mut self) -> Score {
         0.0
     }
+
+    #[inline]
+    fn can_score_doc(&self) -> bool {
+        true
+    }
+
+    #[inline]
+    fn score_doc(&mut self, _doc: DocId, _term_freq: u32) -> Score {
+        0.0
+    }
 }
 
 #[cfg(test)]

src/query/exist_query.rs

@@ -3,7 +3,7 @@ use core::fmt::Debug;
 use columnar::{ColumnIndex, DynamicColumn};
 use common::BitSet;
 
-use super::{box_scorer, ConstScorer, EmptyScorer};
+use super::{ConstScorer, EmptyScorer};
 use crate::docset::{DocSet, TERMINATED};
 use crate::index::SegmentReader;
 use crate::query::all_query::AllScorer;
@@ -117,7 +117,7 @@ impl Weight for ExistsWeight {
             }
         }
         if non_empty_columns.is_empty() {
-            return Ok(box_scorer(EmptyScorer));
+            return Ok(Box::new(EmptyScorer));
         }
 
         // If any column is full, all docs match.
@@ -128,9 +128,9 @@ impl Weight for ExistsWeight {
         {
             let all_scorer = AllScorer::new(max_doc);
             if boost != 1.0f32 {
-                return Ok(box_scorer(BoostScorer::new(all_scorer, boost)));
+                return Ok(Box::new(BoostScorer::new(all_scorer, boost)));
             } else {
-                return Ok(box_scorer(all_scorer));
+                return Ok(Box::new(all_scorer));
             }
         }
 
@@ -138,7 +138,7 @@ impl Weight for ExistsWeight {
         // NOTE: A lower number may be better for very sparse columns
         if non_empty_columns.len() < 4 {
             let docset = ExistsDocSet::new(non_empty_columns, reader.max_doc());
-            return Ok(box_scorer(ConstScorer::new(docset, boost)));
+            return Ok(Box::new(ConstScorer::new(docset, boost)));
         }
 
         // If we have many dynamic columns, precompute a bitset of matching docs
@@ -162,7 +162,7 @@ impl Weight for ExistsWeight {
             }
         }
         let docset = BitSetDocSet::from(doc_bitset);
-        Ok(box_scorer(ConstScorer::new(docset, boost)))
+        Ok(Box::new(ConstScorer::new(docset, boost)))
     }
 
     fn explain(&self, reader: &SegmentReader, doc: DocId) -> crate::Result<Explanation> {

src/query/intersection.rs

@@ -3,7 +3,7 @@ use common::TinySet;
 use super::size_hint::estimate_intersection;
 use crate::docset::{DocSet, SeekDangerResult, BLOCK_NUM_TINYBITSETS, TERMINATED};
 use crate::query::term_query::TermScorer;
-use crate::query::{box_scorer, EmptyScorer, Scorer};
+use crate::query::{EmptyScorer, Scorer};
 use crate::{DocId, Score};
 
 /// Returns the intersection scorer.
@@ -22,7 +22,7 @@ pub fn intersect_scorers(
     segment_num_docs: u32,
 ) -> Box<dyn Scorer> {
     if scorers.is_empty() {
-        return box_scorer(EmptyScorer);
+        return Box::new(EmptyScorer);
     }
     if scorers.len() == 1 {
         return scorers.pop().unwrap();
@@ -31,7 +31,7 @@ pub fn intersect_scorers(
     scorers.sort_by_key(|scorer| scorer.cost());
     let doc = go_to_first_doc(&mut scorers[..]);
     if doc == TERMINATED {
-        return box_scorer(EmptyScorer);
+        return Box::new(EmptyScorer);
     }
     // We know that we have at least 2 elements.
     let left = scorers.remove(0);
@@ -40,14 +40,14 @@ pub fn intersect_scorers(
         .iter()
         .all(|&scorer| scorer.is::<TermScorer>());
     if all_term_scorers {
-        return box_scorer(Intersection {
+        return Box::new(Intersection {
             left: *(left.downcast::<TermScorer>().map_err(|_| ()).unwrap()),
             right: *(right.downcast::<TermScorer>().map_err(|_| ()).unwrap()),
             others: scorers,
             segment_num_docs,
         });
     }
-    box_scorer(Intersection {
+    Box::new(Intersection {
         left,
         right,
         others: scorers,

src/query/mod.rs

@@ -24,7 +24,7 @@ mod reqopt_scorer;
 mod scorer;
 mod set_query;
 mod size_hint;
-pub(crate) mod term_query;
+mod term_query;
 mod union;
 mod weight;
 
@@ -54,14 +54,13 @@ pub use self::more_like_this::{MoreLikeThisQuery, MoreLikeThisQueryBuilder};
 pub use self::phrase_prefix_query::PhrasePrefixQuery;
 pub use self::phrase_query::regex_phrase_query::{wildcard_query_to_regex_str, RegexPhraseQuery};
 pub use self::phrase_query::PhraseQuery;
-pub(crate) use self::phrase_query::PhraseScorer;
 pub use self::query::{EnableScoring, Query, QueryClone};
 pub use self::query_parser::{QueryParser, QueryParserError};
 pub use self::range_query::*;
 pub use self::regex_query::RegexQuery;
 pub use self::reqopt_scorer::RequiredOptionalScorer;
 pub use self::score_combiner::{DisjunctionMaxCombiner, ScoreCombiner, SumCombiner};
-pub use self::scorer::{box_scorer, Scorer};
+pub use self::scorer::Scorer;
 pub use self::set_query::TermSetQuery;
 pub use self::term_query::TermQuery;
 pub use self::union::BufferedUnionScorer;

src/query/phrase_prefix_query/phrase_prefix_scorer.rs

@@ -2,7 +2,7 @@ use crate::docset::{DocSet, SeekDangerResult, TERMINATED};
 use crate::fieldnorm::FieldNormReader;
 use crate::postings::Postings;
 use crate::query::bm25::Bm25Weight;
-use crate::query::phrase_query::{intersection_exists, PhraseScorer};
+use crate::query::phrase_query::{intersection_count, PhraseScorer};
 use crate::query::Scorer;
 use crate::{DocId, Score};
 
@@ -100,6 +100,7 @@ pub struct PhrasePrefixScorer<TPostings: Postings> {
     phrase_scorer: PhraseKind<TPostings>,
     suffixes: Vec<TPostings>,
     suffix_offset: u32,
+    phrase_count: u32,
     suffix_position_buffer: Vec<u32>,
 }
 
@@ -143,6 +144,7 @@ impl<TPostings: Postings> PhrasePrefixScorer<TPostings> {
             phrase_scorer,
             suffixes,
             suffix_offset: (max_offset - suffix_pos) as u32,
+            phrase_count: 0,
             suffix_position_buffer: Vec::with_capacity(100),
         };
         if phrase_prefix_scorer.doc() != TERMINATED && !phrase_prefix_scorer.matches_prefix() {
@@ -151,7 +153,12 @@ impl<TPostings: Postings> PhrasePrefixScorer<TPostings> {
         phrase_prefix_scorer
     }
 
+    pub fn phrase_count(&self) -> u32 {
+        self.phrase_count
+    }
+
     fn matches_prefix(&mut self) -> bool {
+        let mut count = 0;
         let current_doc = self.doc();
         let pos_matching = self.phrase_scorer.get_intersection();
         for suffix in &mut self.suffixes {
@@ -161,12 +168,11 @@ impl<TPostings: Postings> PhrasePrefixScorer<TPostings> {
             let doc = suffix.seek(current_doc);
             if doc == current_doc {
                 suffix.positions_with_offset(self.suffix_offset, &mut self.suffix_position_buffer);
-                if intersection_exists(pos_matching, &self.suffix_position_buffer) {
-                    return true;
-                }
+                count += intersection_count(pos_matching, &self.suffix_position_buffer);
             }
         }
-        false
+        self.phrase_count = count as u32;
+        count != 0
     }
 }
 

src/query/phrase_prefix_query/phrase_prefix_weight.rs

@@ -1,11 +1,12 @@
 use super::{prefix_end, PhrasePrefixScorer};
 use crate::fieldnorm::FieldNormReader;
 use crate::index::SegmentReader;
-use crate::postings::Postings;
+use crate::postings::SegmentPostings;
 use crate::query::bm25::Bm25Weight;
-use crate::query::{box_scorer, EmptyScorer, Scorer, Weight};
+use crate::query::explanation::does_not_match;
+use crate::query::{EmptyScorer, Explanation, Scorer, Weight};
 use crate::schema::{IndexRecordOption, Term};
-use crate::Score;
+use crate::{DocId, DocSet, Score};
 
 pub struct PhrasePrefixWeight {
     phrase_terms: Vec<(usize, Term)>,
@@ -45,13 +46,13 @@ impl PhrasePrefixWeight {
         &self,
         reader: &SegmentReader,
         boost: Score,
-    ) -> crate::Result<Option<Box<dyn Scorer>>> {
+    ) -> crate::Result<Option<PhrasePrefixScorer<SegmentPostings>>> {
         let similarity_weight_opt = self
             .similarity_weight_opt
             .as_ref()
             .map(|similarity_weight| similarity_weight.boost_by(boost));
         let fieldnorm_reader = self.fieldnorm_reader(reader)?;
-        let mut term_postings_list: Vec<(usize, Box<dyn Postings>)> = Vec::new();
+        let mut term_postings_list = Vec::new();
         for &(offset, ref term) in &self.phrase_terms {
             if let Some(postings) = reader
                 .inverted_index(term.field())?
@@ -102,32 +103,49 @@ impl PhrasePrefixWeight {
             }
         }
 
-        Ok(Some(box_scorer(PhrasePrefixScorer::new(
+        Ok(Some(PhrasePrefixScorer::new(
             term_postings_list,
             similarity_weight_opt,
             fieldnorm_reader,
             suffixes,
             self.prefix.0,
-        ))))
+        )))
     }
 }
 
 impl Weight for PhrasePrefixWeight {
     fn scorer(&self, reader: &SegmentReader, boost: Score) -> crate::Result<Box<dyn Scorer>> {
         if let Some(scorer) = self.phrase_scorer(reader, boost)? {
-            Ok(scorer)
+            Ok(Box::new(scorer))
         } else {
-            Ok(box_scorer(EmptyScorer))
+            Ok(Box::new(EmptyScorer))
         }
     }
+
+    fn explain(&self, reader: &SegmentReader, doc: DocId) -> crate::Result<Explanation> {
+        let scorer_opt = self.phrase_scorer(reader, 1.0)?;
+        if scorer_opt.is_none() {
+            return Err(does_not_match(doc));
+        }
+        let mut scorer = scorer_opt.unwrap();
+        if scorer.seek(doc) != doc {
+            return Err(does_not_match(doc));
+        }
+        let fieldnorm_reader = self.fieldnorm_reader(reader)?;
+        let fieldnorm_id = fieldnorm_reader.fieldnorm_id(doc);
+        let phrase_count = scorer.phrase_count();
+        let mut explanation = Explanation::new("Phrase Prefix Scorer", scorer.score());
+        if let Some(similarity_weight) = self.similarity_weight_opt.as_ref() {
+            explanation.add_detail(similarity_weight.explain(fieldnorm_id, phrase_count));
+        }
+        Ok(explanation)
+    }
 }
 
 #[cfg(test)]
 mod tests {
     use crate::docset::TERMINATED;
     use crate::index::Index;
-    use crate::postings::Postings;
-    use crate::query::phrase_prefix_query::PhrasePrefixScorer;
     use crate::query::{EnableScoring, PhrasePrefixQuery, Query};
     use crate::schema::{Schema, TEXT};
     use crate::{DocSet, IndexWriter, Term};
@@ -168,14 +186,14 @@ mod tests {
             .phrase_prefix_query_weight(enable_scoring)
             .unwrap()
             .unwrap();
-        let mut phrase_scorer_boxed = phrase_weight
+        let mut phrase_scorer = phrase_weight
             .phrase_scorer(searcher.segment_reader(0u32), 1.0)?
             .unwrap();
-        let phrase_scorer: &mut PhrasePrefixScorer<Box<dyn Postings>> =
-            phrase_scorer_boxed.as_any_mut().downcast_mut().unwrap();
         assert_eq!(phrase_scorer.doc(), 1);
+        assert_eq!(phrase_scorer.phrase_count(), 2);
         assert_eq!(phrase_scorer.advance(), 2);
         assert_eq!(phrase_scorer.doc(), 2);
+        assert_eq!(phrase_scorer.phrase_count(), 1);
         assert_eq!(phrase_scorer.advance(), TERMINATED);
         Ok(())
     }
@@ -195,15 +213,14 @@ mod tests {
             .phrase_prefix_query_weight(enable_scoring)
             .unwrap()
             .unwrap();
-        let mut phrase_scorer_boxed = phrase_weight
+        let mut phrase_scorer = phrase_weight
             .phrase_scorer(searcher.segment_reader(0u32), 1.0)?
             .unwrap();
-        let phrase_scorer = phrase_scorer_boxed
-            .downcast_mut::<PhrasePrefixScorer<Box<dyn Postings>>>()
-            .unwrap();
         assert_eq!(phrase_scorer.doc(), 1);
+        assert_eq!(phrase_scorer.phrase_count(), 2);
         assert_eq!(phrase_scorer.advance(), 2);
         assert_eq!(phrase_scorer.doc(), 2);
+        assert_eq!(phrase_scorer.phrase_count(), 1);
         assert_eq!(phrase_scorer.advance(), TERMINATED);
         Ok(())
     }

src/query/phrase_query/mod.rs

@@ -5,7 +5,7 @@ pub mod regex_phrase_query;
 mod regex_phrase_weight;
 
 pub use self::phrase_query::PhraseQuery;
-pub(crate) use self::phrase_scorer::intersection_exists;
+pub(crate) use self::phrase_scorer::intersection_count;
 pub use self::phrase_scorer::PhraseScorer;
 pub use self::phrase_weight::PhraseWeight;
 

src/query/phrase_query/phrase_query.rs

@@ -126,7 +126,7 @@ impl PhraseQuery {
         };
         let mut weight = PhraseWeight::new(self.phrase_terms.clone(), bm25_weight_opt);
         if self.slop > 0 {
-            weight.set_slop(self.slop);
+            weight.slop(self.slop);
         }
         Ok(weight)
     }

src/query/phrase_query/phrase_scorer.rs

@@ -1,11 +1,10 @@
 use std::cmp::Ordering;
 
-use crate::codec::standard::postings::StandardPostings;
 use crate::docset::{DocSet, SeekDangerResult, TERMINATED};
 use crate::fieldnorm::FieldNormReader;
 use crate::postings::Postings;
 use crate::query::bm25::Bm25Weight;
-use crate::query::{Explanation, Intersection, Scorer};
+use crate::query::{Intersection, Scorer};
 use crate::{DocId, Score};
 
 struct PostingsWithOffset<TPostings> {
@@ -44,7 +43,7 @@ impl<TPostings: Postings> DocSet for PostingsWithOffset<TPostings> {
     }
 }
 
-pub struct PhraseScorer<TPostings: Postings = StandardPostings> {
+pub struct PhraseScorer<TPostings: Postings> {
     intersection_docset: Intersection<PostingsWithOffset<TPostings>, PostingsWithOffset<TPostings>>,
     num_terms: usize,
     left_positions: Vec<u32>,
@@ -59,7 +58,7 @@ pub struct PhraseScorer<TPostings: Postings = StandardPostings> {
 }
 
 /// Returns true if and only if the two sorted arrays contain a common element
-pub(crate) fn intersection_exists(left: &[u32], right: &[u32]) -> bool {
+fn intersection_exists(left: &[u32], right: &[u32]) -> bool {
     let mut left_index = 0;
     let mut right_index = 0;
     while left_index < left.len() && right_index < right.len() {
@@ -80,7 +79,7 @@ pub(crate) fn intersection_exists(left: &[u32], right: &[u32]) -> bool {
     false
 }
 
-fn intersection_count(left: &[u32], right: &[u32]) -> usize {
+pub(crate) fn intersection_count(left: &[u32], right: &[u32]) -> usize {
     let mut left_index = 0;
     let mut right_index = 0;
     let mut count = 0;
@@ -403,7 +402,6 @@ impl<TPostings: Postings> PhraseScorer<TPostings> {
         scorer
     }
 
-    /// Returns the number of phrases identified in the current matching doc.
     pub fn phrase_count(&self) -> u32 {
         self.phrase_count
     }
@@ -586,17 +584,6 @@ impl<TPostings: Postings> Scorer for PhraseScorer<TPostings> {
             1.0f32
         }
     }
-
-    fn explain(&mut self) -> Explanation {
-        let doc = self.doc();
-        let phrase_count = self.phrase_count();
-        let fieldnorm_id = self.fieldnorm_reader.fieldnorm_id(doc);
-        let mut explanation = Explanation::new("Phrase Scorer", self.score());
-        if let Some(similarity_weight) = self.similarity_weight_opt.as_ref() {
-            explanation.add_detail(similarity_weight.explain(fieldnorm_id, phrase_count));
-        }
-        explanation
-    }
 }
 
 #[cfg(test)]

src/query/phrase_query/phrase_weight.rs

@@ -1,10 +1,11 @@
+use super::PhraseScorer;
 use crate::fieldnorm::FieldNormReader;
 use crate::index::SegmentReader;
-use crate::postings::TermInfo;
+use crate::postings::SegmentPostings;
 use crate::query::bm25::Bm25Weight;
 use crate::query::explanation::does_not_match;
-use crate::query::{box_scorer, EmptyScorer, Explanation, Scorer, Weight};
-use crate::schema::Term;
+use crate::query::{EmptyScorer, Explanation, Scorer, Weight};
+use crate::schema::{IndexRecordOption, Term};
 use crate::{DocId, DocSet, Score};
 
 pub struct PhraseWeight {
@@ -20,10 +21,11 @@ impl PhraseWeight {
         phrase_terms: Vec<(usize, Term)>,
         similarity_weight_opt: Option<Bm25Weight>,
     ) -> PhraseWeight {
+        let slop = 0;
         PhraseWeight {
             phrase_terms,
             similarity_weight_opt,
-            slop: 0,
+            slop,
         }
     }
 
@@ -41,52 +43,32 @@ impl PhraseWeight {
         &self,
         reader: &SegmentReader,
         boost: Score,
-    ) -> crate::Result<Option<Box<dyn Scorer>>> {
+    ) -> crate::Result<Option<PhraseScorer<SegmentPostings>>> {
         let similarity_weight_opt = self
             .similarity_weight_opt
             .as_ref()
             .map(|similarity_weight| similarity_weight.boost_by(boost));
         let fieldnorm_reader = self.fieldnorm_reader(reader)?;
-
-        if self.phrase_terms.is_empty() {
-            return Ok(None);
-        }
-        let field = self.phrase_terms[0].1.field();
-
-        if !self
-            .phrase_terms
-            .iter()
-            .all(|(_offset, term)| term.field() == field)
-        {
-            return Err(crate::TantivyError::InvalidArgument(
-                "All terms in a phrase query must belong to the same field".to_string(),
-            ));
-        }
-
-        let inverted_index_reader = reader.inverted_index(field)?;
-
-        let mut term_infos: Vec<(usize, TermInfo)> = Vec::with_capacity(self.phrase_terms.len());
-
+        let mut term_postings_list = Vec::new();
         for &(offset, ref term) in &self.phrase_terms {
-            let Some(term_info) = inverted_index_reader.get_term_info(term)? else {
+            if let Some(postings) = reader
+                .inverted_index(term.field())?
+                .read_postings(term, IndexRecordOption::WithFreqsAndPositions)?
+            {
+                term_postings_list.push((offset, postings));
+            } else {
                 return Ok(None);
-            };
-            term_infos.push((offset, term_info));
+            }
         }
-
-        let scorer = reader.codec().new_phrase_scorer_type_erased(
-            &term_infos[..],
+        Ok(Some(PhraseScorer::new(
+            term_postings_list,
             similarity_weight_opt,
             fieldnorm_reader,
             self.slop,
-            &inverted_index_reader,
-        )?;
-
-        Ok(Some(scorer))
+        )))
     }
 
-    /// Sets the slop for the given PhraseWeight.
-    pub fn set_slop(&mut self, slop: u32) {
+    pub fn slop(&mut self, slop: u32) {
         self.slop = slop;
     }
 }
@@ -94,9 +76,9 @@ impl PhraseWeight {
 impl Weight for PhraseWeight {
     fn scorer(&self, reader: &SegmentReader, boost: Score) -> crate::Result<Box<dyn Scorer>> {
         if let Some(scorer) = self.phrase_scorer(reader, boost)? {
-            Ok(scorer)
+            Ok(Box::new(scorer))
         } else {
-            Ok(box_scorer(EmptyScorer))
+            Ok(Box::new(EmptyScorer))
         }
     }
 
@@ -109,7 +91,14 @@ impl Weight for PhraseWeight {
         if scorer.seek(doc) != doc {
             return Err(does_not_match(doc));
         }
-        Ok(scorer.explain())
+        let fieldnorm_reader = self.fieldnorm_reader(reader)?;
+        let fieldnorm_id = fieldnorm_reader.fieldnorm_id(doc);
+        let phrase_count = scorer.phrase_count();
+        let mut explanation = Explanation::new("Phrase Scorer", scorer.score());
+        if let Some(similarity_weight) = self.similarity_weight_opt.as_ref() {
+            explanation.add_detail(similarity_weight.explain(fieldnorm_id, phrase_count));
+        }
+        Ok(explanation)
     }
 }
 
@@ -117,8 +106,7 @@ impl Weight for PhraseWeight {
 mod tests {
     use super::super::tests::create_index;
     use crate::docset::TERMINATED;
-    use crate::query::phrase_query::PhraseScorer;
-    use crate::query::{EnableScoring, PhraseQuery, Scorer};
+    use crate::query::{EnableScoring, PhraseQuery};
     use crate::{DocSet, Term};
 
     #[test]
@@ -133,11 +121,9 @@ mod tests {
         ]);
         let enable_scoring = EnableScoring::enabled_from_searcher(&searcher);
         let phrase_weight = phrase_query.phrase_weight(enable_scoring).unwrap();
-        let phrase_scorer_boxed: Box<dyn Scorer> = phrase_weight
+        let mut phrase_scorer = phrase_weight
             .phrase_scorer(searcher.segment_reader(0u32), 1.0)?
             .unwrap();
-        let mut phrase_scorer: Box<PhraseScorer> =
-            phrase_scorer_boxed.downcast::<PhraseScorer>().ok().unwrap();
         assert_eq!(phrase_scorer.doc(), 1);
         assert_eq!(phrase_scorer.phrase_count(), 2);
         assert_eq!(phrase_scorer.advance(), 2);

src/query/phrase_query/regex_phrase_weight.rs

@@ -6,13 +6,11 @@ use tantivy_fst::Regex;
 use super::PhraseScorer;
 use crate::fieldnorm::FieldNormReader;
 use crate::index::SegmentReader;
-use crate::postings::{LoadedPostings, Postings, TermInfo};
+use crate::postings::{LoadedPostings, Postings, SegmentPostings, TermInfo};
 use crate::query::bm25::Bm25Weight;
 use crate::query::explanation::does_not_match;
 use crate::query::union::{BitSetPostingUnion, SimpleUnion};
-use crate::query::{
-    box_scorer, AutomatonWeight, BitSetDocSet, EmptyScorer, Explanation, Scorer, Weight,
-};
+use crate::query::{AutomatonWeight, BitSetDocSet, EmptyScorer, Explanation, Scorer, Weight};
 use crate::schema::{Field, IndexRecordOption};
 use crate::{DocId, DocSet, InvertedIndexReader, Score};
 
@@ -105,9 +103,18 @@ impl RegexPhraseWeight {
         term_info: &TermInfo,
         doc_bitset: &mut BitSet,
     ) -> crate::Result<()> {
-        let mut segment_postings =
-            inverted_index.read_postings_from_terminfo(term_info, IndexRecordOption::Basic)?;
-        segment_postings.fill_bitset(doc_bitset);
+        let mut block_segment_postings = inverted_index
+            .read_block_postings_from_terminfo(term_info, IndexRecordOption::Basic)?;
+        loop {
+            let docs = block_segment_postings.docs();
+            if docs.is_empty() {
+                break;
+            }
+            for &doc in docs {
+                doc_bitset.insert(doc);
+            }
+            block_segment_postings.advance();
+        }
         Ok(())
     }
 
@@ -181,7 +188,7 @@ impl RegexPhraseWeight {
         // - Bucket 1: Terms appearing in 0.1% to 1% of documents
         // - Bucket 2: Terms appearing in 1% to 10% of documents
         // - Bucket 3: Terms appearing in more than 10% of documents
-        let mut buckets: Vec<(BitSet, Vec<Box<dyn Postings>>)> = (0..4)
+        let mut buckets: Vec<(BitSet, Vec<SegmentPostings>)> = (0..4)
             .map(|_| (BitSet::with_max_value(max_doc), Vec::new()))
             .collect();
 
@@ -190,7 +197,7 @@ impl RegexPhraseWeight {
         for term_info in term_infos {
             let mut term_posting = inverted_index
                 .read_postings_from_terminfo(term_info, IndexRecordOption::WithFreqsAndPositions)?;
-            let num_docs = u32::from(term_posting.doc_freq());
+            let num_docs = term_posting.doc_freq();
 
             if num_docs < SPARSE_TERM_DOC_THRESHOLD {
                 let current_bucket = &mut sparse_buckets[0];
@@ -264,9 +271,9 @@ impl RegexPhraseWeight {
 impl Weight for RegexPhraseWeight {
     fn scorer(&self, reader: &SegmentReader, boost: Score) -> crate::Result<Box<dyn Scorer>> {
         if let Some(scorer) = self.phrase_scorer(reader, boost)? {
-            Ok(box_scorer(scorer))
+            Ok(Box::new(scorer))
         } else {
-            Ok(box_scorer(EmptyScorer))
+            Ok(Box::new(EmptyScorer))
         }
     }
 

src/query/range_query/range_query.rs

@@ -8,12 +8,10 @@ use super::range_query_fastfield::FastFieldRangeWeight;
 use crate::index::SegmentReader;
 use crate::query::explanation::does_not_match;
 use crate::query::range_query::is_type_valid_for_fastfield_range_query;
-use crate::query::{
-    box_scorer, BitSetDocSet, ConstScorer, EnableScoring, Explanation, Query, Scorer, Weight,
-};
+use crate::query::{BitSetDocSet, ConstScorer, EnableScoring, Explanation, Query, Scorer, Weight};
 use crate::schema::{Field, IndexRecordOption, Term, Type};
 use crate::termdict::{TermDictionary, TermStreamer};
-use crate::{DocId, DocSet, Score};
+use crate::{DocId, Score};
 
 /// `RangeQuery` matches all documents that have at least one term within a defined range.
 ///
@@ -230,12 +228,21 @@ impl Weight for InvertedIndexRangeWeight {
             }
             processed_count += 1;
             let term_info = term_range.value();
-            let mut postings =
-                inverted_index.read_postings_from_terminfo(term_info, IndexRecordOption::Basic)?;
-            postings.fill_bitset(&mut doc_bitset);
+            let mut block_segment_postings = inverted_index
+                .read_block_postings_from_terminfo(term_info, IndexRecordOption::Basic)?;
+            loop {
+                let docs = block_segment_postings.docs();
+                if docs.is_empty() {
+                    break;
+                }
+                for &doc in block_segment_postings.docs() {
+                    doc_bitset.insert(doc);
+                }
+                block_segment_postings.advance();
+            }
         }
         let doc_bitset = BitSetDocSet::from(doc_bitset);
-        Ok(box_scorer(ConstScorer::new(doc_bitset, boost)))
+        Ok(Box::new(ConstScorer::new(doc_bitset, boost)))
     }
 
     fn explain(&self, reader: &SegmentReader, doc: DocId) -> crate::Result<Explanation> {

src/query/range_query/range_query_fastfield.rs

@@ -13,8 +13,7 @@ use common::bounds::{BoundsRange, TransformBound};
 
 use super::fast_field_range_doc_set::RangeDocSet;
 use crate::query::{
-    box_scorer, AllScorer, ConstScorer, EmptyScorer, EnableScoring, Explanation, Query, Scorer,
-    Weight,
+    AllScorer, ConstScorer, EmptyScorer, EnableScoring, Explanation, Query, Scorer, Weight,
 };
 use crate::schema::{Type, ValueBytes};
 use crate::{DocId, DocSet, Score, SegmentReader, TantivyError, Term};
@@ -56,7 +55,7 @@ impl Weight for FastFieldRangeWeight {
     fn scorer(&self, reader: &SegmentReader, boost: Score) -> crate::Result<Box<dyn Scorer>> {
         // Check if both bounds are Bound::Unbounded
         if self.bounds.is_unbounded() {
-            return Ok(box_scorer(AllScorer::new(reader.max_doc())));
+            return Ok(Box::new(AllScorer::new(reader.max_doc())));
         }
 
         let term = self
@@ -96,7 +95,7 @@ impl Weight for FastFieldRangeWeight {
                     let Some(str_dict_column): Option<StrColumn> =
                         reader.fast_fields().str(&field_name)?
                     else {
-                        return Ok(box_scorer(EmptyScorer));
+                        return Ok(Box::new(EmptyScorer));
                     };
                     let dict = str_dict_column.dictionary();
 
@@ -108,7 +107,7 @@ impl Weight for FastFieldRangeWeight {
                     let Some((column, _col_type)) = fast_field_reader
                         .u64_lenient_for_type(Some(&[ColumnType::Str]), &field_name)?
                     else {
-                        return Ok(box_scorer(EmptyScorer));
+                        return Ok(Box::new(EmptyScorer));
                     };
                     search_on_u64_ff(column, boost, BoundsRange::new(lower_bound, upper_bound))
                 }
@@ -120,7 +119,7 @@ impl Weight for FastFieldRangeWeight {
                     let Some((column, _col_type)) = fast_field_reader
                         .u64_lenient_for_type(Some(&[ColumnType::DateTime]), &field_name)?
                     else {
-                        return Ok(box_scorer(EmptyScorer));
+                        return Ok(Box::new(EmptyScorer));
                     };
                     let bounds = bounds.map_bound(|term| term.as_date().unwrap().to_u64());
                     search_on_u64_ff(
@@ -147,7 +146,7 @@ impl Weight for FastFieldRangeWeight {
             let Some(ip_addr_column): Option<Column<Ipv6Addr>> =
                 reader.fast_fields().column_opt(&field_name)?
             else {
-                return Ok(box_scorer(EmptyScorer));
+                return Ok(Box::new(EmptyScorer));
             };
             let value_range = bound_range_inclusive_ip(
                 &bounds.lower_bound,
@@ -156,11 +155,11 @@ impl Weight for FastFieldRangeWeight {
                 ip_addr_column.max_value(),
             );
             let docset = RangeDocSet::new(value_range, ip_addr_column);
-            Ok(box_scorer(ConstScorer::new(docset, boost)))
+            Ok(Box::new(ConstScorer::new(docset, boost)))
         } else if field_type.is_str() {
             let Some(str_dict_column): Option<StrColumn> = reader.fast_fields().str(&field_name)?
             else {
-                return Ok(box_scorer(EmptyScorer));
+                return Ok(Box::new(EmptyScorer));
             };
             let dict = str_dict_column.dictionary();
 
@@ -172,7 +171,7 @@ impl Weight for FastFieldRangeWeight {
             let Some((column, _col_type)) =
                 fast_field_reader.u64_lenient_for_type(None, &field_name)?
             else {
-                return Ok(box_scorer(EmptyScorer));
+                return Ok(Box::new(EmptyScorer));
             };
             search_on_u64_ff(column, boost, BoundsRange::new(lower_bound, upper_bound))
         } else if field_type.is_bytes() {
@@ -229,7 +228,7 @@ impl Weight for FastFieldRangeWeight {
                 &field_name,
             )?
             else {
-                return Ok(box_scorer(EmptyScorer));
+                return Ok(Box::new(EmptyScorer));
             };
             search_on_u64_ff(
                 column,
@@ -270,7 +269,7 @@ fn search_on_json_numerical_field(
     let Some((column, col_type)) =
         fast_field_reader.u64_lenient_for_type(allowed_column_types, field_name)?
     else {
-        return Ok(box_scorer(EmptyScorer));
+        return Ok(Box::new(EmptyScorer));
     };
     let actual_column_type: NumericalType = col_type
         .numerical_type()
@@ -428,18 +427,18 @@ fn search_on_u64_ff(
     )
     .unwrap_or(1..=0); // empty range
     if value_range.is_empty() {
-        return Ok(box_scorer(EmptyScorer));
+        return Ok(Box::new(EmptyScorer));
     }
     if col_min_value >= *value_range.start() && col_max_value <= *value_range.end() {
         // all values in the column are within the range.
         if column.index.get_cardinality() == Cardinality::Full {
             if boost != 1.0f32 {
-                return Ok(box_scorer(ConstScorer::new(
+                return Ok(Box::new(ConstScorer::new(
                     AllScorer::new(column.num_docs()),
                     boost,
                 )));
             } else {
-                return Ok(box_scorer(AllScorer::new(column.num_docs())));
+                return Ok(Box::new(AllScorer::new(column.num_docs())));
             }
         } else {
             // TODO Make it a field presence request for that specific column
@@ -447,7 +446,7 @@ fn search_on_u64_ff(
     }
 
     let docset = RangeDocSet::new(value_range, column);
-    Ok(box_scorer(ConstScorer::new(docset, boost)))
+    Ok(Box::new(ConstScorer::new(docset, boost)))
 }
 
 /// Returns true if the type maps to a u64 fast field

src/query/score_combiner.rs

@@ -1,5 +1,40 @@
+use crate::docset::{DocSet, TERMINATED};
 use crate::query::Scorer;
-use crate::Score;
+use crate::{DocId, Score};
+
+struct ScoreOnlyScorer {
+    doc: DocId,
+    score: Score,
+}
+
+impl DocSet for ScoreOnlyScorer {
+    fn advance(&mut self) -> DocId {
+        self.doc = TERMINATED;
+        TERMINATED
+    }
+
+    fn doc(&self) -> DocId {
+        self.doc
+    }
+
+    fn size_hint(&self) -> u32 {
+        1
+    }
+}
+
+impl Scorer for ScoreOnlyScorer {
+    fn score(&mut self) -> Score {
+        self.score
+    }
+
+    fn can_score_doc(&self) -> bool {
+        true
+    }
+
+    fn score_doc(&mut self, _doc: DocId, _term_freq: u32) -> Score {
+        self.score
+    }
+}
 
 /// The `ScoreCombiner` trait defines how to compute
 /// an overall score given a list of scores.
@@ -10,6 +45,17 @@ pub trait ScoreCombiner: Default + Clone + Send + Copy + 'static {
     /// or not.
     fn update<TScorer: Scorer>(&mut self, scorer: &mut TScorer);
 
+    /// Aggregates the score combiner with an already computed score.
+    fn update_score(&mut self, doc: DocId, score: Score) {
+        let mut scorer = ScoreOnlyScorer { doc, score };
+        self.update(&mut scorer);
+    }
+
+    /// Returns true if this combiner needs scorer scores to compute its state.
+    fn requires_scoring() -> bool {
+        true
+    }
+
     /// Clears the score combiner state back to its initial state.
     fn clear(&mut self);
 
@@ -27,6 +73,12 @@ pub struct DoNothingCombiner;
 impl ScoreCombiner for DoNothingCombiner {
     fn update<TScorer: Scorer>(&mut self, _scorer: &mut TScorer) {}
 
+    fn update_score(&mut self, _doc: DocId, _score: Score) {}
+
+    fn requires_scoring() -> bool {
+        false
+    }
+
     fn clear(&mut self) {}
 
     #[inline]
@@ -42,10 +94,16 @@ pub struct SumCombiner {
 }
 
 impl ScoreCombiner for SumCombiner {
+    #[inline]
     fn update<TScorer: Scorer>(&mut self, scorer: &mut TScorer) {
         self.score += scorer.score();
     }
 
+    #[inline]
+    fn update_score(&mut self, _doc: DocId, score: Score) {
+        self.score += score;
+    }
+
     fn clear(&mut self) {
         self.score = 0.0;
     }
@@ -77,12 +135,19 @@ impl DisjunctionMaxCombiner {
 }
 
 impl ScoreCombiner for DisjunctionMaxCombiner {
+    #[inline]
     fn update<TScorer: Scorer>(&mut self, scorer: &mut TScorer) {
         let score = scorer.score();
         self.max = Score::max(score, self.max);
         self.sum += score;
     }
 
+    #[inline]
+    fn update_score(&mut self, _doc: DocId, score: Score) {
+        self.max = Score::max(score, self.max);
+        self.sum += score;
+    }
+
     fn clear(&mut self) {
         self.max = 0.0;
         self.sum = 0.0;

src/query/scorer.rs

@@ -1,11 +1,9 @@
-use std::mem::{transmute_copy, ManuallyDrop};
 use std::ops::DerefMut;
 
 use downcast_rs::impl_downcast;
 
-use crate::docset::DocSet;
-use crate::query::Explanation;
-use crate::{DocId, Score, TERMINATED};
+use crate::docset::{DocSet, COLLECT_BLOCK_BUFFER_LEN};
+use crate::{DocId, Score};
 
 /// Scored set of documents matching a query within a specific segment.
 ///
@@ -16,51 +14,34 @@ pub trait Scorer: downcast_rs::Downcast + DocSet + 'static {
     /// This method will perform a bit of computation and is not cached.
     fn score(&mut self) -> Score;
 
-    /// Calls `callback` with all of the `(doc, score)` for which score
-    /// is exceeding a given threshold.
+    /// Returns true if [`Scorer::score_doc`] can score buffered docs without
+    /// repositioning the scorer.
     ///
-    /// This method is useful for the TopDocs collector.
-    /// For all docsets, the blanket implementation has the benefit
-    /// of prefiltering (doc, score) pairs, avoiding the
-    /// virtual dispatch cost.
+    /// Scorers whose [`Scorer::score_doc`] needs term frequencies must also override
+    /// [`Scorer::fill_buffer_up_to_with_term_freqs`].
+    fn can_score_doc(&self) -> bool {
+        false
+    }
+
+    /// Returns the score for `doc` with its term frequency.
+    fn score_doc(&mut self, _doc: DocId, _term_freq: u32) -> Score {
+        panic!(
+            "score_doc is not supported by this scorer. You need check can_score_doc() before \
+             calling this method."
+        )
+    }
+
+    /// Fills docs up to `horizon`.
     ///
-    /// More importantly, it makes it possible for scorers to implement
-    /// important optimization (e.g. BlockWAND for union).
-    fn for_each_pruning(
+    /// The default implementation does not fill `term_freqs`. Scorers whose
+    /// [`Scorer::score_doc`] reads term frequencies must override this method.
+    fn fill_buffer_up_to_with_term_freqs(
         &mut self,
-        threshold: Score,
-        callback: &mut dyn FnMut(DocId, Score) -> Score,
-    ) {
-        for_each_pruning_scorer_default_impl(self, threshold, callback);
-    }
-
-    /// Calls `callback` with all of the `(doc, score)` in the scorer.
-    fn for_each(&mut self, callback: &mut dyn FnMut(DocId, Score)) {
-        let mut doc = self.doc();
-        while doc != TERMINATED {
-            callback(doc, self.score());
-            doc = self.advance();
-        }
-    }
-
-    /// Returns an explanation for the score of the current document.
-    fn explain(&mut self) -> Explanation {
-        let score = self.score();
-        let name = std::any::type_name_of_val(self);
-        Explanation::new(name, score)
-    }
-}
-
-/// Boxes a scorer. Prefer this to Box::new as it avoids double boxing
-/// when TScorer is already a Box<dyn Scorer>.
-pub fn box_scorer<TScorer: Scorer>(scorer: TScorer) -> Box<dyn Scorer> {
-    if std::any::TypeId::of::<TScorer>() == std::any::TypeId::of::<Box<dyn Scorer>>() {
-        unsafe {
-            let forget_me = ManuallyDrop::new(scorer);
-            transmute_copy::<TScorer, Box<dyn Scorer>>(&forget_me)
-        }
-    } else {
-        Box::new(scorer)
+        horizon: DocId,
+        docs: &mut [DocId; COLLECT_BLOCK_BUFFER_LEN],
+        _term_freqs: &mut [u32; COLLECT_BLOCK_BUFFER_LEN],
+    ) -> usize {
+        DocSet::fill_buffer_up_to(self, horizon, docs)
     }
 }
 
@@ -72,40 +53,24 @@ impl Scorer for Box<dyn Scorer> {
         self.deref_mut().score()
     }
 
-    fn for_each_pruning(
+    #[inline]
+    fn can_score_doc(&self) -> bool {
+        self.as_ref().can_score_doc()
+    }
+
+    #[inline]
+    fn score_doc(&mut self, doc: DocId, term_freq: u32) -> Score {
+        self.deref_mut().score_doc(doc, term_freq)
+    }
+
+    #[inline]
+    fn fill_buffer_up_to_with_term_freqs(
         &mut self,
-        threshold: Score,
-        callback: &mut dyn FnMut(DocId, Score) -> Score,
-    ) {
-        self.deref_mut().for_each_pruning(threshold, callback);
-    }
-
-    fn for_each(&mut self, callback: &mut dyn FnMut(DocId, Score)) {
-        self.deref_mut().for_each(callback);
-    }
-}
-
-/// Calls `callback` with all of the `(doc, score)` for which score
-/// is exceeding a given threshold.
-///
-/// This method is useful for the [`TopDocs`](crate::collector::TopDocs) collector.
-/// For all docsets, the blanket implementation has the benefit
-/// of prefiltering (doc, score) pairs, avoiding the
-/// virtual dispatch cost.
-///
-/// More importantly, it makes it possible for scorers to implement
-/// important optimization (e.g. BlockWAND for union).
-pub(crate) fn for_each_pruning_scorer_default_impl<TScorer: Scorer + ?Sized>(
-    scorer: &mut TScorer,
-    mut threshold: Score,
-    callback: &mut dyn FnMut(DocId, Score) -> Score,
-) {
-    let mut doc = scorer.doc();
-    while doc != TERMINATED {
-        let score = scorer.score();
-        if score > threshold {
-            threshold = callback(doc, score);
-        }
-        doc = scorer.advance();
+        horizon: DocId,
+        docs: &mut [DocId; COLLECT_BLOCK_BUFFER_LEN],
+        term_freqs: &mut [u32; COLLECT_BLOCK_BUFFER_LEN],
+    ) -> usize {
+        self.deref_mut()
+            .fill_buffer_up_to_with_term_freqs(horizon, docs, term_freqs)
     }
 }

src/query/term_query/mod.rs

@@ -4,9 +4,9 @@ mod term_weight;
 
 pub use self::term_query::TermQuery;
 pub use self::term_scorer::TermScorer;
-
 #[cfg(test)]
 mod tests {
+
     use crate::collector::TopDocs;
     use crate::docset::DocSet;
     use crate::postings::compression::COMPRESSION_BLOCK_SIZE;

src/query/term_query/term_scorer.rs

@@ -1,27 +1,23 @@
-use crate::codec::postings::{PostingsCodec, PostingsWithBlockMax};
-use crate::codec::{Codec, StandardCodec};
-use crate::docset::DocSet;
+use crate::docset::{DocSet, COLLECT_BLOCK_BUFFER_LEN};
 use crate::fieldnorm::FieldNormReader;
-use crate::postings::Postings;
+use crate::postings::{BlockSegmentPostings, FreqReadingOption, Postings, SegmentPostings};
 use crate::query::bm25::Bm25Weight;
 use crate::query::{Explanation, Scorer};
 use crate::{DocId, Score};
 
 #[derive(Clone)]
-pub struct TermScorer<
-    TPostings: Postings = <<StandardCodec as Codec>::PostingsCodec as PostingsCodec>::Postings,
-> {
-    postings: TPostings,
+pub struct TermScorer {
+    postings: SegmentPostings,
     fieldnorm_reader: FieldNormReader,
     similarity_weight: Bm25Weight,
 }
 
-impl<TPostings: Postings> TermScorer<TPostings> {
+impl TermScorer {
     pub fn new(
-        postings: TPostings,
+        postings: SegmentPostings,
         fieldnorm_reader: FieldNormReader,
         similarity_weight: Bm25Weight,
-    ) -> TermScorer<TPostings> {
+    ) -> TermScorer {
         TermScorer {
             postings,
             fieldnorm_reader,
@@ -29,35 +25,10 @@ impl<TPostings: Postings> TermScorer<TPostings> {
         }
     }
 
-    pub fn term_freq(&self) -> u32 {
-        self.postings.term_freq()
+    pub(crate) fn seek_block(&mut self, target_doc: DocId) {
+        self.postings.block_cursor.seek_block(target_doc);
     }
 
-    pub fn fieldnorm_id(&self) -> u8 {
-        self.fieldnorm_reader.fieldnorm_id(self.doc())
-    }
-
-    pub fn max_score(&self) -> Score {
-        self.similarity_weight.max_score()
-    }
-}
-
-impl<TPostingsWithBlockMax: PostingsWithBlockMax> TermScorer<TPostingsWithBlockMax> {
-    pub(crate) fn last_doc_in_block(&self) -> DocId {
-        self.postings.last_doc_in_block()
-    }
-
-    /// Advances the term scorer to the block containing target_doc and returns
-    /// an upperbound for the score all of the documents in the block.
-    /// (BlockMax). This score is not guaranteed to be the
-    /// effective maximum score of the block.
-    pub(crate) fn seek_block_max(&mut self, target_doc: DocId) -> Score {
-        self.postings
-            .seek_block_max(target_doc, &self.fieldnorm_reader, &self.similarity_weight)
-    }
-}
-
-impl TermScorer {
     #[cfg(test)]
     pub fn create_for_test(
         doc_and_tfs: &[(DocId, u32)],
@@ -73,15 +44,75 @@ impl TermScorer {
                 .unwrap_or(0u32)
                 < fieldnorms.len() as u32
         );
-        type SegmentPostings = <<StandardCodec as Codec>::PostingsCodec as PostingsCodec>::Postings;
-        let segment_postings: SegmentPostings =
+        let segment_postings =
             SegmentPostings::create_from_docs_and_tfs(doc_and_tfs, Some(fieldnorms));
         let fieldnorm_reader = FieldNormReader::for_test(fieldnorms);
         TermScorer::new(segment_postings, fieldnorm_reader, similarity_weight)
     }
+
+    /// See `FreqReadingOption`.
+    pub(crate) fn freq_reading_option(&self) -> FreqReadingOption {
+        self.postings.block_cursor.freq_reading_option()
+    }
+
+    /// Returns the maximum score for the current block.
+    ///
+    /// In some rare case, the result may not be exact. In this case a lower value is returned,
+    /// (and may lead us to return a lesser document).
+    ///
+    /// At index time, we store the (fieldnorm_id, term frequency) pair that maximizes the
+    /// score assuming the average fieldnorm computed on this segment.
+    ///
+    /// Though extremely rare, it is theoretically possible that the actual average fieldnorm
+    /// is different enough from the current segment average fieldnorm that the maximum over a
+    /// specific is achieved on a different document.
+    ///
+    /// (The result is on the other hand guaranteed to be correct if there is only one segment).
+    pub fn block_max_score(&mut self) -> Score {
+        self.postings
+            .block_cursor
+            .block_max_score(&self.fieldnorm_reader, &self.similarity_weight)
+    }
+
+    pub fn term_freq(&self) -> u32 {
+        self.postings.term_freq()
+    }
+
+    pub fn fieldnorm_id(&self) -> u8 {
+        self.fieldnorm_reader.fieldnorm_id(self.doc())
+    }
+
+    pub fn explain(&self) -> Explanation {
+        let fieldnorm_id = self.fieldnorm_id();
+        let term_freq = self.term_freq();
+        self.similarity_weight.explain(fieldnorm_id, term_freq)
+    }
+
+    pub fn max_score(&self) -> Score {
+        self.similarity_weight.max_score()
+    }
+
+    pub fn last_doc_in_block(&self) -> DocId {
+        self.postings.block_cursor.skip_reader().last_doc_in_block()
+    }
+
+    /// Returns a mutable reference to the underlying block cursor.
+    pub(crate) fn block_cursor(&mut self) -> &mut BlockSegmentPostings {
+        &mut self.postings.block_cursor
+    }
+
+    /// Returns a reference to the fieldnorm reader for batch lookups.
+    pub(crate) fn fieldnorm_reader(&self) -> &FieldNormReader {
+        &self.fieldnorm_reader
+    }
+
+    /// Returns a reference to the BM25 weight for batch score computation.
+    pub(crate) fn bm25_weight(&self) -> &Bm25Weight {
+        &self.similarity_weight
+    }
 }
 
-impl<TPostings: Postings> DocSet for TermScorer<TPostings> {
+impl DocSet for TermScorer {
     #[inline]
     fn advance(&mut self) -> DocId {
         self.postings.advance()
@@ -109,7 +140,7 @@ impl<TPostings: Postings> DocSet for TermScorer<TPostings> {
     // and do not have access to x86 to investigate.
 }
 
-impl<TPostings: Postings> Scorer for TermScorer<TPostings> {
+impl Scorer for TermScorer {
     #[inline]
     fn score(&mut self) -> Score {
         let fieldnorm_id = self.fieldnorm_id();
@@ -117,10 +148,25 @@ impl<TPostings: Postings> Scorer for TermScorer<TPostings> {
         self.similarity_weight.score(fieldnorm_id, term_freq)
     }
 
-    fn explain(&mut self) -> Explanation {
-        let fieldnorm_id = self.fieldnorm_id();
-        let term_freq = self.term_freq();
-        self.similarity_weight.explain(fieldnorm_id, term_freq)
+    #[inline]
+    fn can_score_doc(&self) -> bool {
+        true
+    }
+
+    #[inline]
+    fn score_doc(&mut self, doc: DocId, term_freq: u32) -> Score {
+        let fieldnorm_id = self.fieldnorm_reader.fieldnorm_id(doc);
+        self.similarity_weight.score(fieldnorm_id, term_freq)
+    }
+
+    fn fill_buffer_up_to_with_term_freqs(
+        &mut self,
+        horizon: DocId,
+        docs: &mut [DocId; COLLECT_BLOCK_BUFFER_LEN],
+        term_freqs: &mut [u32; COLLECT_BLOCK_BUFFER_LEN],
+    ) -> usize {
+        self.postings
+            .fill_buffer_up_to_with_term_freqs(horizon, docs, term_freqs)
     }
 }
 
@@ -130,7 +176,7 @@ mod tests {
 
     use crate::index::SegmentId;
     use crate::indexer::index_writer::MEMORY_BUDGET_NUM_BYTES_MIN;
-    use crate::indexer::NoMergePolicy;
+    use crate::merge_policy::NoMergePolicy;
     use crate::postings::compression::COMPRESSION_BLOCK_SIZE;
     use crate::query::term_query::TermScorer;
     use crate::query::{Bm25Weight, EnableScoring, Scorer, TermQuery};
@@ -151,7 +197,7 @@ mod tests {
         crate::assert_nearly_equals!(max_scorer, 1.3990127);
         assert_eq!(term_scorer.doc(), 2);
         assert_eq!(term_scorer.term_freq(), 3);
-        assert_nearly_equals!(term_scorer.seek_block_max(2), 1.3676447);
+        assert_nearly_equals!(term_scorer.block_max_score(), 1.3676447);
         assert_nearly_equals!(term_scorer.score(), 1.0892314);
         assert_eq!(term_scorer.advance(), 3);
         assert_eq!(term_scorer.doc(), 3);
@@ -166,9 +212,9 @@ mod tests {
     }
 
     #[test]
-    fn test_term_scorer_shallow_advance() {
+    fn test_term_scorer_shallow_advance() -> crate::Result<()> {
         let bm25_weight = Bm25Weight::for_one_term(300, 1024, 10.0);
-        let mut doc_and_tfs = Vec::new();
+        let mut doc_and_tfs = vec![];
         for i in 0u32..300u32 {
             let doc = i * 10;
             doc_and_tfs.push((doc, 1u32 + doc % 3u32));
@@ -176,10 +222,11 @@ mod tests {
         let fieldnorms: Vec<u32> = std::iter::repeat_n(10u32, 3_000).collect();
         let mut term_scorer = TermScorer::create_for_test(&doc_and_tfs, &fieldnorms, bm25_weight);
         assert_eq!(term_scorer.doc(), 0u32);
-        term_scorer.seek_block_max(1289);
+        term_scorer.seek_block(1289);
         assert_eq!(term_scorer.doc(), 0u32);
         term_scorer.seek(1289);
         assert_eq!(term_scorer.doc(), 1290);
+        Ok(())
     }
 
     proptest! {
@@ -213,7 +260,7 @@ mod tests {
 
          let docs: Vec<DocId> = (0..term_doc_freq).map(|doc| doc as DocId).collect();
          for block in docs.chunks(COMPRESSION_BLOCK_SIZE) {
-             let block_max_score: Score = term_scorer.seek_block_max(0);
+             let block_max_score: Score = term_scorer.block_max_score();
              let mut block_max_score_computed: Score = 0.0;
              for &doc in block {
                 assert_eq!(term_scorer.doc(), doc);
@@ -241,24 +288,25 @@ mod tests {
         let fieldnorms: Vec<u32> = std::iter::repeat_n(20u32, 300).collect();
         let bm25_weight = Bm25Weight::for_one_term(10, 129, 20.0);
         let mut docs = TermScorer::create_for_test(&doc_tfs[..], &fieldnorms[..], bm25_weight);
-        assert_nearly_equals!(docs.seek_block_max(0), 2.5161593);
-        assert_nearly_equals!(docs.seek_block_max(135), 3.4597192);
+        assert_nearly_equals!(docs.block_max_score(), 2.5161593);
+        docs.seek_block(135);
+        assert_nearly_equals!(docs.block_max_score(), 3.4597192);
+        docs.seek_block(256);
         // the block is not loaded yet.
-        assert_nearly_equals!(docs.seek_block_max(256), 5.2971773);
+        assert_nearly_equals!(docs.block_max_score(), 5.2971773);
         assert_eq!(256, docs.seek(256));
-        assert_nearly_equals!(docs.seek_block_max(256), 3.9539647);
+        assert_nearly_equals!(docs.block_max_score(), 3.9539647);
     }
 
-    fn test_block_wand_aux(term_query: &TermQuery, searcher: &Searcher) {
-        let term_weight = term_query
-            .specialized_weight(EnableScoring::enabled_from_searcher(searcher))
-            .unwrap();
+    fn test_block_wand_aux(term_query: &TermQuery, searcher: &Searcher) -> crate::Result<()> {
+        let term_weight =
+            term_query.specialized_weight(EnableScoring::enabled_from_searcher(searcher))?;
         for reader in searcher.segment_readers() {
             let mut block_max_scores = vec![];
             let mut block_max_scores_b = vec![];
             let mut docs = vec![];
             {
-                let mut term_scorer = term_weight.term_scorer_for_test(reader, 1.0).unwrap();
+                let mut term_scorer = term_weight.term_scorer_for_test(reader, 1.0)?.unwrap();
                 while term_scorer.doc() != TERMINATED {
                     let mut score = term_scorer.score();
                     docs.push(term_scorer.doc());
@@ -272,10 +320,10 @@ mod tests {
                 }
             }
             {
-                let mut term_scorer = term_weight.term_scorer_for_test(reader, 1.0).unwrap();
+                let mut term_scorer = term_weight.term_scorer_for_test(reader, 1.0)?.unwrap();
                 for d in docs {
-                    let block_max_score = term_scorer.seek_block_max(d);
-                    block_max_scores_b.push(block_max_score);
+                    term_scorer.seek_block(d);
+                    block_max_scores_b.push(term_scorer.block_max_score());
                 }
             }
             for (l, r) in block_max_scores
@@ -286,18 +334,18 @@ mod tests {
                 assert_nearly_equals!(l, r);
             }
         }
+        Ok(())
     }
 
     #[ignore]
     #[test]
-    fn test_block_wand_long_test() {
+    fn test_block_wand_long_test() -> crate::Result<()> {
         let mut schema_builder = Schema::builder();
         let text_field = schema_builder.add_text_field("text", TEXT);
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
-        let mut writer: IndexWriter = index
-            .writer_with_num_threads(3, 3 * MEMORY_BUDGET_NUM_BYTES_MIN)
-            .unwrap();
+        let mut writer: IndexWriter =
+            index.writer_with_num_threads(3, 3 * MEMORY_BUDGET_NUM_BYTES_MIN)?;
         use rand::Rng;
         let mut rng = rand::rng();
         writer.set_merge_policy(Box::new(NoMergePolicy));
@@ -305,15 +353,15 @@ mod tests {
             let term_freq = rng.random_range(1..10000);
             let words: Vec<&str> = std::iter::repeat_n("bbbb", term_freq).collect();
             let text = words.join(" ");
-            writer.add_document(doc!(text_field=>text)).unwrap();
+            writer.add_document(doc!(text_field=>text))?;
         }
-        writer.commit().unwrap();
+        writer.commit()?;
         let term_query = TermQuery::new(
             Term::from_field_text(text_field, "bbbb"),
             IndexRecordOption::WithFreqs,
         );
         let segment_ids: Vec<SegmentId>;
-        let reader = index.reader().unwrap();
+        let reader = index.reader()?;
         {
             let searcher = reader.searcher();
             segment_ids = searcher
@@ -321,14 +369,15 @@ mod tests {
                 .iter()
                 .map(|segment| segment.segment_id())
                 .collect();
-            test_block_wand_aux(&term_query, &searcher);
+            test_block_wand_aux(&term_query, &searcher)?;
         }
         writer.merge(&segment_ids[..]).wait().unwrap();
         {
-            reader.reload().unwrap();
+            reader.reload()?;
             let searcher = reader.searcher();
             assert_eq!(searcher.segment_readers().len(), 1);
-            test_block_wand_aux(&term_query, &searcher);
+            test_block_wand_aux(&term_query, &searcher)?;
         }
+        Ok(())
     }
 }

src/query/term_query/term_weight.rs

@@ -1,11 +1,12 @@
+use super::term_scorer::TermScorer;
 use crate::docset::{DocSet, COLLECT_BLOCK_BUFFER_LEN};
 use crate::fieldnorm::FieldNormReader;
 use crate::index::SegmentReader;
+use crate::postings::SegmentPostings;
 use crate::query::bm25::Bm25Weight;
 use crate::query::explanation::does_not_match;
-use crate::query::term_query::TermScorer;
-use crate::query::weight::for_each_docset_buffered;
-use crate::query::{box_scorer, AllScorer, AllWeight, EmptyScorer, Explanation, Scorer, Weight};
+use crate::query::weight::{for_each_docset_buffered, for_each_scorer};
+use crate::query::{AllScorer, AllWeight, EmptyScorer, Explanation, Scorer, Weight};
 use crate::schema::IndexRecordOption;
 use crate::{DocId, Score, TantivyError, Term};
 
@@ -17,7 +18,7 @@ pub struct TermWeight {
 }
 
 enum TermOrEmptyOrAllScorer {
-    TermScorer(Box<dyn Scorer>),
+    TermScorer(Box<TermScorer>),
     Empty,
     AllMatch(AllScorer),
 }
@@ -26,8 +27,8 @@ impl TermOrEmptyOrAllScorer {
     pub fn into_boxed_scorer(self) -> Box<dyn Scorer> {
         match self {
             TermOrEmptyOrAllScorer::TermScorer(scorer) => scorer,
-            TermOrEmptyOrAllScorer::Empty => box_scorer(EmptyScorer),
-            TermOrEmptyOrAllScorer::AllMatch(scorer) => box_scorer(scorer),
+            TermOrEmptyOrAllScorer::Empty => Box::new(EmptyScorer),
+            TermOrEmptyOrAllScorer::AllMatch(scorer) => Box::new(scorer),
         }
     }
 }
@@ -43,7 +44,6 @@ impl Weight for TermWeight {
                 if term_scorer.doc() > doc || term_scorer.seek(doc) != doc {
                     return Err(does_not_match(doc));
                 }
-                let mut term_scorer = term_scorer.downcast::<TermScorer>().ok().unwrap();
                 let mut explanation = term_scorer.explain();
                 explanation.add_context(format!("Term={:?}", self.term,));
                 Ok(explanation)
@@ -73,11 +73,11 @@ impl Weight for TermWeight {
     ) -> crate::Result<()> {
         match self.specialized_scorer(reader, 1.0)? {
             TermOrEmptyOrAllScorer::TermScorer(mut term_scorer) => {
-                term_scorer.for_each(callback);
+                for_each_scorer(&mut *term_scorer, callback);
             }
             TermOrEmptyOrAllScorer::Empty => {}
             TermOrEmptyOrAllScorer::AllMatch(mut all_scorer) => {
-                all_scorer.for_each(callback);
+                for_each_scorer(&mut all_scorer, callback);
             }
         }
         Ok(())
@@ -124,9 +124,11 @@ impl Weight for TermWeight {
         let specialized_scorer = self.specialized_scorer(reader, 1.0)?;
         match specialized_scorer {
             TermOrEmptyOrAllScorer::TermScorer(term_scorer) => {
-                reader
-                    .codec()
-                    .for_each_pruning(threshold, term_scorer, callback);
+                crate::query::boolean_query::block_wand_single_scorer(
+                    *term_scorer,
+                    threshold,
+                    callback,
+                );
             }
             TermOrEmptyOrAllScorer::Empty => {}
             TermOrEmptyOrAllScorer::AllMatch(_) => {
@@ -166,15 +168,12 @@ impl TermWeight {
         &self,
         reader: &SegmentReader,
         boost: Score,
-    ) -> Option<super::TermScorer> {
-        let scorer = self.specialized_scorer(reader, boost).unwrap();
-        match scorer {
-            TermOrEmptyOrAllScorer::TermScorer(scorer) => {
-                let term_scorer = scorer.downcast::<super::TermScorer>().ok()?;
-                Some(*term_scorer)
-            }
+    ) -> crate::Result<Option<TermScorer>> {
+        let scorer = self.specialized_scorer(reader, boost)?;
+        Ok(match scorer {
+            TermOrEmptyOrAllScorer::TermScorer(scorer) => Some(*scorer),
             _ => None,
-        }
+        })
     }
 
     fn specialized_scorer(
@@ -197,16 +196,14 @@ impl TermWeight {
             )));
         }
 
+        let segment_postings: SegmentPostings =
+            inverted_index.read_postings_from_terminfo(&term_info, self.index_record_option)?;
+
         let fieldnorm_reader = self.fieldnorm_reader(reader)?;
         let similarity_weight = self.similarity_weight.boost_by(boost);
-        let term_scorer = inverted_index.new_term_scorer(
-            &term_info,
-            self.index_record_option,
-            fieldnorm_reader,
-            similarity_weight,
-        )?;
-
-        Ok(TermOrEmptyOrAllScorer::TermScorer(term_scorer))
+        Ok(TermOrEmptyOrAllScorer::TermScorer(Box::new(
+            TermScorer::new(segment_postings, fieldnorm_reader, similarity_weight),
+        )))
     }
 
     fn fieldnorm_reader(&self, segment_reader: &SegmentReader) -> crate::Result<FieldNormReader> {

src/query/union/bitset_union.rs

@@ -1,7 +1,7 @@
 use std::cell::RefCell;
 
 use crate::docset::DocSet;
-use crate::postings::{DocFreq, Postings};
+use crate::postings::Postings;
 use crate::query::BitSetDocSet;
 use crate::DocId;
 
@@ -16,9 +16,6 @@ pub struct BitSetPostingUnion<TDocSet> {
     docsets: RefCell<Vec<TDocSet>>,
     /// The already unionized BitSet of the docsets
     bitset: BitSetDocSet,
-    /// The total number of documents in the union (regardless of the position we are in the
-    /// bitset).
-    doc_freq: u32,
 }
 
 impl<TDocSet: DocSet> BitSetPostingUnion<TDocSet> {
@@ -26,11 +23,9 @@ impl<TDocSet: DocSet> BitSetPostingUnion<TDocSet> {
         docsets: Vec<TDocSet>,
         bitset: BitSetDocSet,
     ) -> BitSetPostingUnion<TDocSet> {
-        let doc_freq = bitset.doc_freq();
         BitSetPostingUnion {
             docsets: RefCell::new(docsets),
             bitset,
-            doc_freq,
         }
     }
 }
@@ -51,10 +46,6 @@ impl<TDocSet: Postings> Postings for BitSetPostingUnion<TDocSet> {
         term_freq
     }
 
-    fn has_freq(&self) -> bool {
-        true
-    }
-
     fn append_positions_with_offset(&mut self, offset: u32, output: &mut Vec<u32>) {
         let curr_doc = self.bitset.doc();
         let mut docsets = self.docsets.borrow_mut();
@@ -73,10 +64,6 @@ impl<TDocSet: Postings> Postings for BitSetPostingUnion<TDocSet> {
         output.sort_unstable();
         output.dedup();
     }
-
-    fn doc_freq(&self) -> DocFreq {
-        DocFreq::Exact(self.doc_freq)
-    }
 }
 
 impl<TDocSet: DocSet> DocSet for BitSetPostingUnion<TDocSet> {

src/query/union/buffered_union.rs

@@ -10,28 +10,12 @@ use crate::{DocId, Score};
 // of upcoming document IDs (the "horizon").
 const HORIZON_NUM_TINYBITSETS: usize = HORIZON as usize / 64;
 const HORIZON: u32 = 64u32 * 64u32;
-
-// `drain_filter` is not stable yet.
-// This function is similar except that it does is not unstable, and
-// it does not keep the original vector ordering.
-//
-// Elements are dropped and not yielded.
-fn unordered_drain_filter<T, P>(v: &mut Vec<T>, mut predicate: P)
-where P: FnMut(&mut T) -> bool {
-    let mut i = 0;
-    while i < v.len() {
-        if predicate(&mut v[i]) {
-            v.swap_remove(i);
-        } else {
-            i += 1;
-        }
-    }
-}
+const GROUPED_INSERT_MAX_BUCKET_SPAN: u32 = 2;
 
 /// Creates a `DocSet` that iterate through the union of two or more `DocSet`s.
 pub struct BufferedUnionScorer<TScorer, TScoreCombiner = DoNothingCombiner> {
     /// Active scorers (already filtered of `TERMINATED`).
-    scorers: Vec<TScorer>,
+    docsets: Vec<TScorer>,
     /// Sliding window presence map for upcoming docs.
     ///
     /// There are `HORIZON_NUM_TINYBITSETS` buckets, each covering
@@ -46,8 +30,6 @@ pub struct BufferedUnionScorer<TScorer, TScoreCombiner = DoNothingCombiner> {
     /// hit the same doc within the buffered window.
     scores: Box<[TScoreCombiner; HORIZON as usize]>,
     /// Start doc ID (inclusive) of the current sliding window.
-    /// None if the window is not loaded yet. This is true for a freshly created
-    /// BufferedUnionScorer.
     window_start_doc: DocId,
     /// Current doc ID of the union.
     doc: DocId,
@@ -55,109 +37,269 @@ pub struct BufferedUnionScorer<TScorer, TScoreCombiner = DoNothingCombiner> {
     score: Score,
     /// Number of documents in the segment.
     num_docs: u32,
+    /// Scratch buffer for block-based refill.
+    refill_docs: [DocId; COLLECT_BLOCK_BUFFER_LEN],
+    /// Scratch buffer for term frequencies matching `refill_docs`.
+    refill_term_freqs: [u32; COLLECT_BLOCK_BUFFER_LEN],
+    /// Whether all children support scoring buffered docs after advancing.
+    use_score_doc_refill: bool,
+}
+
+#[inline]
+fn union_bucket(
+    bitsets: &mut [TinySet; HORIZON_NUM_TINYBITSETS],
+    bucket_pos: u32,
+    tinyset: TinySet,
+) {
+    debug_assert!((bucket_pos as usize) < HORIZON_NUM_TINYBITSETS);
+    // `bucket` comes from a doc delta below `HORIZON`; there are exactly
+    // `HORIZON / 64` buckets in the refill window.
+    bitsets[bucket_pos as usize] = bitsets[bucket_pos as usize].union(tinyset);
+}
+
+#[inline]
+fn insert_delta(bitsets: &mut [TinySet; HORIZON_NUM_TINYBITSETS], delta: DocId) {
+    debug_assert!(delta < HORIZON);
+    // `delta < HORIZON`, so `delta / 64` is in the bitset array. The bit
+    // offset is reduced modulo 64 before being inserted in the TinySet.
+    bitsets[delta as usize / 64].insert_mut(delta % 64u32);
+}
+
+fn insert_and_score_full_buffer<TScorer: Scorer, TScoreCombiner: ScoreCombiner>(
+    scorer: &mut TScorer,
+    docs: &[DocId; COLLECT_BLOCK_BUFFER_LEN],
+    term_freqs: &[u32; COLLECT_BLOCK_BUFFER_LEN],
+    bitsets: &mut [TinySet; HORIZON_NUM_TINYBITSETS],
+    score_combiner: &mut [TScoreCombiner; HORIZON as usize],
+    min_doc: DocId,
+) {
+    debug_assert!(docs.windows(2).all(|pair| pair[0] < pair[1]));
+    debug_assert!(docs[COLLECT_BLOCK_BUFFER_LEN - 1] - min_doc < HORIZON);
+
+    let first_delta = docs[0] - min_doc;
+    let last_delta = docs[COLLECT_BLOCK_BUFFER_LEN - 1] - min_doc;
+    let first_bucket = first_delta / 64;
+    let last_bucket = last_delta / 64;
+
+    // Common for very dense scorers: 64 distinct doc ids in one 64-doc bucket
+    // means all bits in that bucket are present.
+    if first_bucket == last_bucket {
+        union_bucket(bitsets, first_bucket, TinySet::full());
+        score_full_buffer(scorer, docs, term_freqs, score_combiner, min_doc);
+        return;
+    }
+
+    // 64 sorted distinct integers spanning exactly 64 values are consecutive.
+    // If they cross a TinySet boundary, this is just the suffix of the first
+    // bucket plus the prefix of the second bucket.
+    if last_delta - first_delta == COLLECT_BLOCK_BUFFER_LEN as u32 - 1 {
+        union_bucket(
+            bitsets,
+            first_bucket,
+            TinySet::range_greater_or_equal(first_delta % 64u32),
+        );
+        union_bucket(
+            bitsets,
+            last_bucket,
+            TinySet::range_lower((last_delta + 1) % 64u32),
+        );
+        score_full_buffer(scorer, docs, term_freqs, score_combiner, min_doc);
+        return;
+    }
+
+    // Grouping wins only for very dense buffers that hit the same TinySet many
+    // times. Once the 64 docs are spread farther, a straight pass is cheaper.
+    if last_bucket - first_bucket <= GROUPED_INSERT_MAX_BUCKET_SPAN {
+        let mut bucket = first_bucket;
+        let mut tinyset = TinySet::empty();
+        for (&doc, &term_freq) in docs.iter().zip(term_freqs.iter()) {
+            let delta = doc - min_doc;
+            let delta_bucket = delta / 64;
+            if delta_bucket != bucket {
+                union_bucket(bitsets, bucket, tinyset);
+                bucket = delta_bucket;
+                tinyset = TinySet::empty();
+            }
+            tinyset.insert_mut(delta % 64u32);
+            let score = scorer.score_doc(doc, term_freq);
+            update_score_combiner(score_combiner, delta, doc, score);
+        }
+        union_bucket(bitsets, bucket, tinyset);
+    } else {
+        for (&doc, &term_freq) in docs.iter().zip(term_freqs.iter()) {
+            let delta = doc - min_doc;
+            insert_delta(bitsets, delta);
+            // TODO: score_doc access the field_norm reader for each _term_, instead of once per
+            // doc. We could optimize this by caching the field norm for the doc, and
+            // reusing it for all terms in the doc.
+            let score = scorer.score_doc(doc, term_freq);
+            update_score_combiner(score_combiner, delta, doc, score);
+        }
+    }
+}
+
+#[inline]
+fn update_score_combiner<TScoreCombiner: ScoreCombiner>(
+    score_combiner: &mut [TScoreCombiner; HORIZON as usize],
+    delta: DocId,
+    doc: DocId,
+    score: Score,
+) {
+    debug_assert!(delta < HORIZON);
+    // Full and partial refill only buffer docs below `horizon`, so their
+    // deltas are always in the score-combiner window.
+    score_combiner[delta as usize].update_score(doc, score);
+}
+
+fn score_full_buffer<TScorer: Scorer, TScoreCombiner: ScoreCombiner>(
+    scorer: &mut TScorer,
+    docs: &[DocId; COLLECT_BLOCK_BUFFER_LEN],
+    term_freqs: &[u32; COLLECT_BLOCK_BUFFER_LEN],
+    score_combiner: &mut [TScoreCombiner; HORIZON as usize],
+    min_doc: DocId,
+) {
+    for (&doc, &term_freq) in docs.iter().zip(term_freqs.iter()) {
+        let score = scorer.score_doc(doc, term_freq);
+        update_score_combiner(score_combiner, doc - min_doc, doc, score);
+    }
+}
+
+fn refill_scorer_with_score_docs<TScorer: Scorer, TScoreCombiner: ScoreCombiner>(
+    scorer: &mut TScorer,
+    bitsets: &mut [TinySet; HORIZON_NUM_TINYBITSETS],
+    score_combiner: &mut [TScoreCombiner; HORIZON as usize],
+    docs: &mut [DocId; COLLECT_BLOCK_BUFFER_LEN],
+    term_freqs: &mut [u32; COLLECT_BLOCK_BUFFER_LEN],
+    min_doc: DocId,
+    horizon: DocId,
+) {
+    loop {
+        let len = scorer.fill_buffer_up_to_with_term_freqs(horizon, docs, term_freqs);
+        if len == COLLECT_BLOCK_BUFFER_LEN {
+            debug_assert!(docs[COLLECT_BLOCK_BUFFER_LEN - 1] != TERMINATED);
+            debug_assert!(docs[COLLECT_BLOCK_BUFFER_LEN - 1] < horizon);
+            insert_and_score_full_buffer(
+                scorer,
+                docs,
+                term_freqs,
+                bitsets,
+                score_combiner,
+                min_doc,
+            );
+        } else {
+            for (&doc, &term_freq) in docs[..len].iter().zip(term_freqs[..len].iter()) {
+                let delta = doc - min_doc;
+                insert_delta(bitsets, delta);
+                let score = scorer.score_doc(doc, term_freq);
+                update_score_combiner(score_combiner, delta, doc, score);
+            }
+            break;
+        }
+    }
+}
+
+fn refill_scorer_from_current_doc<TScorer: Scorer, TScoreCombiner: ScoreCombiner>(
+    scorer: &mut TScorer,
+    bitsets: &mut [TinySet; HORIZON_NUM_TINYBITSETS],
+    score_combiner: &mut [TScoreCombiner; HORIZON as usize],
+    min_doc: DocId,
+    horizon: DocId,
+) {
+    loop {
+        let doc = scorer.doc();
+        if doc >= horizon {
+            break;
+        }
+        let delta = doc - min_doc;
+        insert_delta(bitsets, delta);
+        debug_assert!(delta < HORIZON);
+        score_combiner[delta as usize].update(scorer);
+        scorer.advance();
+    }
 }
 
 fn refill<TScorer: Scorer, TScoreCombiner: ScoreCombiner>(
     scorers: &mut Vec<TScorer>,
     bitsets: &mut [TinySet; HORIZON_NUM_TINYBITSETS],
     score_combiner: &mut [TScoreCombiner; HORIZON as usize],
+    docs: &mut [DocId; COLLECT_BLOCK_BUFFER_LEN],
+    term_freqs: &mut [u32; COLLECT_BLOCK_BUFFER_LEN],
     min_doc: DocId,
+    use_score_doc_refill: bool,
 ) {
-    unordered_drain_filter(scorers, |scorer| {
-        let horizon = min_doc + HORIZON;
-        loop {
-            let doc = scorer.doc();
-            if doc >= horizon {
-                return false;
-            }
-            // add this document
-            let delta = doc - min_doc;
-            bitsets[(delta / 64) as usize].insert_mut(delta % 64u32);
-            score_combiner[delta as usize].update(scorer);
-            if scorer.advance() == TERMINATED {
-                // remove the docset, it has been entirely consumed.
-                return true;
-            }
+    let horizon = min_doc + HORIZON;
+    for scorer in scorers.iter_mut() {
+        if use_score_doc_refill {
+            refill_scorer_with_score_docs(
+                scorer,
+                bitsets,
+                score_combiner,
+                docs,
+                term_freqs,
+                min_doc,
+                horizon,
+            );
+        } else {
+            refill_scorer_from_current_doc(scorer, bitsets, score_combiner, min_doc, horizon);
         }
-    });
+    }
+    scorers.retain(|scorer| scorer.doc() != TERMINATED);
 }
 
 impl<TScorer: Scorer, TScoreCombiner: ScoreCombiner> BufferedUnionScorer<TScorer, TScoreCombiner> {
-    /// Returns the underlying scorers in the union.
-    pub fn into_scorers(self) -> Vec<TScorer> {
-        self.scorers
-    }
-
-    /// Accessor for the underlying scorers in the union.
-    pub fn scorers(&self) -> &[TScorer] {
-        &self.scorers[..]
-    }
-
     /// num_docs is the number of documents in the segment.
     pub(crate) fn build(
         docsets: Vec<TScorer>,
         score_combiner_fn: impl FnOnce() -> TScoreCombiner,
         num_docs: u32,
     ) -> BufferedUnionScorer<TScorer, TScoreCombiner> {
-        let score_combiner = score_combiner_fn();
-        let mut non_empty_docsets: Vec<TScorer> = docsets
+        let use_score_doc_refill =
+            TScoreCombiner::requires_scoring() && docsets.iter().all(Scorer::can_score_doc);
+        let non_empty_docsets: Vec<TScorer> = docsets
             .into_iter()
             .filter(|docset| docset.doc() != TERMINATED)
             .collect();
-
-        let first_doc: DocId = non_empty_docsets
-            .iter()
-            .map(|docset| docset.doc())
-            .min()
-            .unwrap_or(TERMINATED);
-        let mut score_combiner_cloned = score_combiner;
-        let mut i = 0;
-        while i < non_empty_docsets.len() {
-            let should_remove_docset: bool = {
-                let non_empty_docset = &mut non_empty_docsets[i];
-                if non_empty_docset.doc() != first_doc {
-                    false
-                } else {
-                    score_combiner_cloned.update(non_empty_docset);
-                    non_empty_docsets[i].advance() == TERMINATED
-                }
-            };
-            if should_remove_docset {
-                non_empty_docsets.swap_remove(i);
-            } else {
-                i += 1;
-            }
-        }
-        let first_score: Score = score_combiner_cloned.score();
-        BufferedUnionScorer {
-            scorers: non_empty_docsets,
+        let mut union = BufferedUnionScorer {
+            docsets: non_empty_docsets,
             bitsets: Box::new([TinySet::empty(); HORIZON_NUM_TINYBITSETS]),
-            scores: Box::new([score_combiner; HORIZON as usize]),
+            scores: Box::new([score_combiner_fn(); HORIZON as usize]),
             bucket_idx: HORIZON_NUM_TINYBITSETS,
-            // That way we will be detected as outside the window,
-            window_start_doc: u32::MAX - HORIZON,
-            doc: first_doc,
-            score: first_score,
+            window_start_doc: 0,
+            doc: 0,
+            score: 0.0,
             num_docs,
+            refill_docs: [TERMINATED; COLLECT_BLOCK_BUFFER_LEN],
+            refill_term_freqs: [1u32; COLLECT_BLOCK_BUFFER_LEN],
+            use_score_doc_refill,
+        };
+        if union.refill() {
+            union.advance();
+        } else {
+            union.doc = TERMINATED;
         }
+        union
     }
 
     fn refill(&mut self) -> bool {
-        let Some(min_doc) = self.scorers.iter().map(DocSet::doc).min() else {
-            return false;
-        };
-        // Reset the sliding window to start at the smallest doc
-        // across all scorers and prebuffer within the horizon.
-        self.window_start_doc = min_doc;
-        self.bucket_idx = 0;
-        self.doc = min_doc;
-        refill(
-            &mut self.scorers,
-            &mut self.bitsets,
-            &mut self.scores,
-            min_doc,
-        );
-        true
+        if let Some(min_doc) = self.docsets.iter().map(DocSet::doc).min() {
+            // Reset the sliding window to start at the smallest doc
+            // across all scorers and prebuffer within the horizon.
+            self.window_start_doc = min_doc;
+            self.bucket_idx = 0;
+            self.doc = min_doc;
+            refill(
+                &mut self.docsets,
+                &mut self.bitsets,
+                &mut self.scores,
+                &mut self.refill_docs,
+                &mut self.refill_term_freqs,
+                min_doc,
+                self.use_score_doc_refill,
+            );
+            true
+        } else {
+            false
+        }
     }
 
     #[inline]
@@ -179,7 +321,6 @@ impl<TScorer: Scorer, TScoreCombiner: ScoreCombiner> BufferedUnionScorer<TScorer
 
     fn is_in_horizon(&self, target: DocId) -> bool {
         // wrapping_sub, because target may be < window_start_doc
-        // in particular during initialization.
         let gap = target.wrapping_sub(self.window_start_doc);
         gap < HORIZON
     }
@@ -249,10 +390,11 @@ where
         if self.doc >= target {
             return self.doc;
         }
-        if self.is_in_horizon(target) {
+        let gap = target - self.window_start_doc;
+        if gap < HORIZON {
             // Our value is within the buffered horizon.
+
             // Skipping to corresponding bucket.
-            let gap = target.wrapping_sub(self.window_start_doc);
             let new_bucket_idx = gap as usize / 64;
             for obsolete_tinyset in &mut self.bitsets[self.bucket_idx..new_bucket_idx] {
                 obsolete_tinyset.clear();
@@ -271,19 +413,21 @@ where
             doc
         } else {
             // clear the buffered info.
-            self.bitsets.fill(TinySet::empty());
+            for obsolete_tinyset in self.bitsets.iter_mut() {
+                *obsolete_tinyset = TinySet::empty();
+            }
             for score_combiner in self.scores.iter_mut() {
                 score_combiner.clear();
             }
 
             // The target is outside of the buffered horizon.
             // advance all docsets to a doc >= to the target.
-            unordered_drain_filter(&mut self.scorers, |docset| {
+            for docset in &mut self.docsets {
                 if docset.doc() < target {
                     docset.seek(target);
                 }
-                docset.doc() == TERMINATED
-            });
+            }
+            self.docsets.retain(|docset| docset.doc() != TERMINATED);
 
             // at this point all of the docsets
             // are positioned on a doc >= to the target.
@@ -315,8 +459,8 @@ where
         let mut is_hit = false;
         let mut min_new_target = TERMINATED;
 
-        for scorer in self.scorers.iter_mut() {
-            match scorer.seek_danger(target) {
+        for docset in self.docsets.iter_mut() {
+            match docset.seek_danger(target) {
                 SeekDangerResult::Found => {
                     is_hit = true;
                     break;
@@ -345,11 +489,11 @@ where
     }
 
     fn size_hint(&self) -> u32 {
-        estimate_union(self.scorers.iter().map(DocSet::size_hint), self.num_docs)
+        estimate_union(self.docsets.iter().map(DocSet::size_hint), self.num_docs)
     }
 
     fn cost(&self) -> u64 {
-        self.scorers.iter().map(|docset| docset.cost()).sum()
+        self.docsets.iter().map(|docset| docset.cost()).sum()
     }
 
     // TODO Also implement `count` with deletes efficiently.
@@ -357,17 +501,21 @@ where
         if self.doc == TERMINATED {
             return 0;
         }
-        let mut count = 1 + self.bitsets[self.bucket_idx..HORIZON_NUM_TINYBITSETS]
+        let mut count = self.bitsets[self.bucket_idx..HORIZON_NUM_TINYBITSETS]
             .iter()
-            .copied()
-            .map(TinySet::len)
-            .sum::<u32>();
+            .map(|bitset| bitset.len())
+            .sum::<u32>()
+            + 1;
+        for bitset in self.bitsets.iter_mut() {
+            bitset.clear();
+        }
         while self.refill() {
-            count += self.bitsets.iter().copied().map(TinySet::len).sum::<u32>();
-            self.bitsets.fill(TinySet::empty());
+            count += self.bitsets.iter().map(|bitset| bitset.len()).sum::<u32>();
+            for bitset in self.bitsets.iter_mut() {
+                bitset.clear();
+            }
         }
         self.bucket_idx = HORIZON_NUM_TINYBITSETS;
-        self.doc = TERMINATED;
         count
     }
 }

src/query/union/mod.rs

@@ -10,6 +10,8 @@ pub use simple_union::SimpleUnion;
 mod tests {
 
     use std::collections::BTreeSet;
+    use std::sync::atomic::{AtomicUsize, Ordering};
+    use std::sync::Arc;
 
     use common::BitSet;
 
@@ -18,8 +20,8 @@ mod tests {
     use crate::postings::tests::test_skip_against_unoptimized;
     use crate::query::score_combiner::DoNothingCombiner;
     use crate::query::union::bitset_union::BitSetPostingUnion;
-    use crate::query::{BitSetDocSet, ConstScorer, VecDocSet};
-    use crate::{tests, DocId};
+    use crate::query::{BitSetDocSet, ConstScorer, Scorer, VecDocSet};
+    use crate::{tests, DocId, Score};
 
     fn vec_doc_set_from_docs_list(
         docs_list: &[Vec<DocId>],
@@ -66,6 +68,61 @@ mod tests {
         }
         BitSetDocSet::from(doc_bitset)
     }
+
+    struct CountingScorer {
+        docset: VecDocSet,
+        score_calls: Arc<AtomicUsize>,
+        score_doc_calls: Arc<AtomicUsize>,
+    }
+
+    impl CountingScorer {
+        fn new(
+            doc_ids: Vec<DocId>,
+            score_calls: Arc<AtomicUsize>,
+            score_doc_calls: Arc<AtomicUsize>,
+        ) -> Self {
+            CountingScorer {
+                docset: VecDocSet::from(doc_ids),
+                score_calls,
+                score_doc_calls,
+            }
+        }
+    }
+
+    impl DocSet for CountingScorer {
+        fn advance(&mut self) -> DocId {
+            self.docset.advance()
+        }
+
+        fn seek(&mut self, target: DocId) -> DocId {
+            self.docset.seek(target)
+        }
+
+        fn doc(&self) -> DocId {
+            self.docset.doc()
+        }
+
+        fn size_hint(&self) -> u32 {
+            self.docset.size_hint()
+        }
+    }
+
+    impl Scorer for CountingScorer {
+        fn score(&mut self) -> Score {
+            self.score_calls.fetch_add(1, Ordering::SeqCst);
+            1.0
+        }
+
+        fn can_score_doc(&self) -> bool {
+            true
+        }
+
+        fn score_doc(&mut self, _doc: DocId, _term_freq: u32) -> Score {
+            self.score_doc_calls.fetch_add(1, Ordering::SeqCst);
+            1.0
+        }
+    }
+
     fn aux_test_union(docs_list: &[Vec<DocId>]) {
         for constructor in [
             posting_list_union_from_docs_list,
@@ -168,6 +225,22 @@ mod tests {
         ]);
     }
 
+    #[test]
+    fn test_do_nothing_combiner_does_not_score_buffered_docs() {
+        let score_calls = Arc::new(AtomicUsize::new(0));
+        let score_doc_calls = Arc::new(AtomicUsize::new(0));
+        let scorers = vec![
+            CountingScorer::new(vec![1, 3, 5], score_calls.clone(), score_doc_calls.clone()),
+            CountingScorer::new(vec![2, 3, 6], score_calls.clone(), score_doc_calls.clone()),
+        ];
+
+        let mut union = BufferedUnionScorer::build(scorers, DoNothingCombiner::default, 10);
+
+        assert_eq!(union.count_including_deleted(), 5);
+        assert_eq!(score_calls.load(Ordering::SeqCst), 0);
+        assert_eq!(score_doc_calls.load(Ordering::SeqCst), 0);
+    }
+
     fn test_aux_union_skip(docs_list: &[Vec<DocId>], skip_targets: Vec<DocId>) {
         for constructor in [
             posting_list_union_from_docs_list,

src/query/union/simple_union.rs

@@ -1,5 +1,5 @@
 use crate::docset::{DocSet, TERMINATED};
-use crate::postings::{DocFreq, Postings};
+use crate::postings::Postings;
 use crate::DocId;
 
 /// A `SimpleUnion` is a `DocSet` that is the union of multiple `DocSet`.
@@ -56,22 +56,6 @@ impl<TDocSet: Postings> Postings for SimpleUnion<TDocSet> {
         term_freq
     }
 
-    fn has_freq(&self) -> bool {
-        true
-    }
-
-    /// We do not know the actual document frequency, so we return
-    /// the maximum document frequency of the docsets.
-    fn doc_freq(&self) -> DocFreq {
-        let approximate_doc_freq = self
-            .docsets
-            .iter()
-            .map(|docset| u32::from(docset.doc_freq()))
-            .max()
-            .unwrap_or(0u32);
-        DocFreq::Approximate(approximate_doc_freq)
-    }
-
     fn append_positions_with_offset(&mut self, offset: u32, output: &mut Vec<u32>) {
         for docset in &mut self.docsets {
             let doc = docset.doc();

src/query/weight.rs

@@ -1,9 +1,21 @@
 use super::Scorer;
 use crate::docset::COLLECT_BLOCK_BUFFER_LEN;
 use crate::index::SegmentReader;
-use crate::query::explanation::does_not_match;
 use crate::query::Explanation;
-use crate::{DocId, DocSet, Score};
+use crate::{DocId, DocSet, Score, TERMINATED};
+
+/// Iterates through all of the documents and scores matched by the DocSet
+/// `DocSet`.
+pub(crate) fn for_each_scorer<TScorer: Scorer + ?Sized>(
+    scorer: &mut TScorer,
+    callback: &mut dyn FnMut(DocId, Score),
+) {
+    let mut doc = scorer.doc();
+    while doc != TERMINATED {
+        callback(doc, scorer.score());
+        doc = scorer.advance();
+    }
+}
 
 /// Iterates through all of the documents matched by the DocSet
 /// `DocSet`.
@@ -22,6 +34,31 @@ pub(crate) fn for_each_docset_buffered<T: DocSet + ?Sized>(
     }
 }
 
+/// Calls `callback` with all of the `(doc, score)` for which score
+/// is exceeding a given threshold.
+///
+/// This method is useful for the [`TopDocs`](crate::collector::TopDocs) collector.
+/// For all docsets, the blanket implementation has the benefit
+/// of prefiltering (doc, score) pairs, avoiding the
+/// virtual dispatch cost.
+///
+/// More importantly, it makes it possible for scorers to implement
+/// important optimization (e.g. BlockWAND for union).
+pub(crate) fn for_each_pruning_scorer<TScorer: Scorer + ?Sized>(
+    scorer: &mut TScorer,
+    mut threshold: Score,
+    callback: &mut dyn FnMut(DocId, Score) -> Score,
+) {
+    let mut doc = scorer.doc();
+    while doc != TERMINATED {
+        let score = scorer.score();
+        if score > threshold {
+            threshold = callback(doc, score);
+        }
+        doc = scorer.advance();
+    }
+}
+
 /// A Weight is the specialization of a `Query`
 /// for a given set of segments.
 ///
@@ -35,13 +72,7 @@ pub trait Weight: Send + Sync + 'static {
     fn scorer(&self, reader: &SegmentReader, boost: Score) -> crate::Result<Box<dyn Scorer>>;
 
     /// Returns an [`Explanation`] for the given document.
-    fn explain(&self, reader: &SegmentReader, doc: DocId) -> crate::Result<Explanation> {
-        let mut scorer = self.scorer(reader, 1.0)?;
-        if scorer.doc() > doc || scorer.seek(doc) != doc {
-            return Err(does_not_match(doc));
-        }
-        Ok(scorer.explain())
-    }
+    fn explain(&self, reader: &SegmentReader, doc: DocId) -> crate::Result<Explanation>;
 
     /// Returns the number documents within the given [`SegmentReader`].
     fn count(&self, reader: &SegmentReader) -> crate::Result<u32> {
@@ -61,7 +92,7 @@ pub trait Weight: Send + Sync + 'static {
         callback: &mut dyn FnMut(DocId, Score),
     ) -> crate::Result<()> {
         let mut scorer = self.scorer(reader, 1.0)?;
-        scorer.for_each(callback);
+        for_each_scorer(scorer.as_mut(), callback);
         Ok(())
     }
 
@@ -96,7 +127,7 @@ pub trait Weight: Send + Sync + 'static {
         callback: &mut dyn FnMut(DocId, Score) -> Score,
     ) -> crate::Result<()> {
         let mut scorer = self.scorer(reader, 1.0)?;
-        scorer.for_each_pruning(threshold, callback);
+        for_each_pruning_scorer(scorer.as_mut(), threshold, callback);
         Ok(())
     }
 }

src/reader/mod.rs

@@ -7,7 +7,6 @@ use arc_swap::ArcSwap;
 pub use warming::Warmer;
 
 use self::warming::WarmingState;
-use crate::codec::Codec;
 use crate::core::searcher::{SearcherGeneration, SearcherInner};
 use crate::directory::{Directory, WatchCallback, WatchHandle, META_LOCK};
 use crate::store::DOCSTORE_CACHE_CAPACITY;
@@ -39,17 +38,17 @@ pub enum ReloadPolicy {
 /// - number of warming threads, for parallelizing warming work
 /// - The cache size of the underlying doc store readers.
 #[derive(Clone)]
-pub struct IndexReaderBuilder<C: Codec = crate::codec::StandardCodec> {
+pub struct IndexReaderBuilder {
     reload_policy: ReloadPolicy,
-    index: Index<C>,
+    index: Index,
     warmers: Vec<Weak<dyn Warmer>>,
     num_warming_threads: usize,
     doc_store_cache_num_blocks: usize,
 }
 
-impl<C: Codec> IndexReaderBuilder<C> {
+impl IndexReaderBuilder {
     #[must_use]
-    pub(crate) fn new(index: Index<C>) -> IndexReaderBuilder<C> {
+    pub(crate) fn new(index: Index) -> IndexReaderBuilder {
         IndexReaderBuilder {
             reload_policy: ReloadPolicy::OnCommitWithDelay,
             index,
@@ -64,7 +63,7 @@ impl<C: Codec> IndexReaderBuilder<C> {
     /// Building the reader is a non-trivial operation that requires
     /// to open different segment readers. It may take hundreds of milliseconds
     /// of time and it may return an error.
-    pub fn try_into(self) -> crate::Result<IndexReader<C>> {
+    pub fn try_into(self) -> crate::Result<IndexReader> {
         let searcher_generation_inventory = Inventory::default();
         let warming_state = WarmingState::new(
             self.num_warming_threads,
@@ -107,7 +106,7 @@ impl<C: Codec> IndexReaderBuilder<C> {
     ///
     /// See [`ReloadPolicy`] for more details.
     #[must_use]
-    pub fn reload_policy(mut self, reload_policy: ReloadPolicy) -> IndexReaderBuilder<C> {
+    pub fn reload_policy(mut self, reload_policy: ReloadPolicy) -> IndexReaderBuilder {
         self.reload_policy = reload_policy;
         self
     }
@@ -119,14 +118,14 @@ impl<C: Codec> IndexReaderBuilder<C> {
     pub fn doc_store_cache_num_blocks(
         mut self,
         doc_store_cache_num_blocks: usize,
-    ) -> IndexReaderBuilder<C> {
+    ) -> IndexReaderBuilder {
         self.doc_store_cache_num_blocks = doc_store_cache_num_blocks;
         self
     }
 
     /// Set the [`Warmer`]s that are invoked when reloading searchable segments.
     #[must_use]
-    pub fn warmers(mut self, warmers: Vec<Weak<dyn Warmer>>) -> IndexReaderBuilder<C> {
+    pub fn warmers(mut self, warmers: Vec<Weak<dyn Warmer>>) -> IndexReaderBuilder {
         self.warmers = warmers;
         self
     }
@@ -136,33 +135,33 @@ impl<C: Codec> IndexReaderBuilder<C> {
     /// This allows parallelizing warming work when there are multiple [`Warmer`] registered with
     /// the [`IndexReader`].
     #[must_use]
-    pub fn num_warming_threads(mut self, num_warming_threads: usize) -> IndexReaderBuilder<C> {
+    pub fn num_warming_threads(mut self, num_warming_threads: usize) -> IndexReaderBuilder {
         self.num_warming_threads = num_warming_threads;
         self
     }
 }
 
-impl<C: Codec> TryInto<IndexReader<C>> for IndexReaderBuilder<C> {
+impl TryInto<IndexReader> for IndexReaderBuilder {
     type Error = crate::TantivyError;
 
-    fn try_into(self) -> crate::Result<IndexReader<C>> {
+    fn try_into(self) -> crate::Result<IndexReader> {
         IndexReaderBuilder::try_into(self)
     }
 }
 
-struct InnerIndexReader<C: Codec> {
+struct InnerIndexReader {
     doc_store_cache_num_blocks: usize,
-    index: Index<C>,
+    index: Index,
     warming_state: WarmingState,
     searcher: arc_swap::ArcSwap<SearcherInner>,
     searcher_generation_counter: Arc<AtomicU64>,
     searcher_generation_inventory: Inventory<SearcherGeneration>,
 }
 
-impl<C: Codec> InnerIndexReader<C> {
+impl InnerIndexReader {
     fn new(
         doc_store_cache_num_blocks: usize,
-        index: Index<C>,
+        index: Index,
         warming_state: WarmingState,
         // The searcher_generation_inventory is not used as source, but as target to track the
         // loaded segments.
@@ -190,7 +189,7 @@ impl<C: Codec> InnerIndexReader<C> {
     ///
     /// This function acquires a lock to prevent GC from removing files
     /// as we are opening our index.
-    fn open_segment_readers(index: &Index<C>) -> crate::Result<Vec<SegmentReader>> {
+    fn open_segment_readers(index: &Index) -> crate::Result<Vec<SegmentReader>> {
         // Prevents segment files from getting deleted while we are in the process of opening them
         let _meta_lock = index.directory().acquire_lock(&META_LOCK)?;
         let searchable_segments = index.searchable_segments()?;
@@ -213,7 +212,7 @@ impl<C: Codec> InnerIndexReader<C> {
     }
 
     fn create_searcher(
-        index: &Index<C>,
+        index: &Index,
         doc_store_cache_num_blocks: usize,
         warming_state: &WarmingState,
         searcher_generation_counter: &Arc<AtomicU64>,
@@ -227,10 +226,9 @@ impl<C: Codec> InnerIndexReader<C> {
         );
 
         let schema = index.schema();
-        // SearcherInner uses Index<StandardCodec> since the codec doesn't affect reading
         let searcher = Arc::new(SearcherInner::new(
             schema,
-            index.with_standard_codec(),
+            index.clone(),
             segment_readers,
             searcher_generation,
             doc_store_cache_num_blocks,
@@ -266,14 +264,14 @@ impl<C: Codec> InnerIndexReader<C> {
 ///
 /// `IndexReader` just wraps an `Arc`.
 #[derive(Clone)]
-pub struct IndexReader<C: Codec = crate::codec::StandardCodec> {
-    inner: Arc<InnerIndexReader<C>>,
+pub struct IndexReader {
+    inner: Arc<InnerIndexReader>,
     _watch_handle_opt: Option<WatchHandle>,
 }
 
-impl<C: Codec> IndexReader<C> {
+impl IndexReader {
     #[cfg(test)]
-    pub(crate) fn index(&self) -> Index<C> {
+    pub(crate) fn index(&self) -> Index {
         self.inner.index.clone()
     }
 

src/schema/document/existing_type_impls.rs

@@ -24,24 +24,13 @@ use crate::tokenizer::PreTokenizedString;
 
 // Serde compatibility support.
 pub fn can_be_rfc3339_date_time(text: &str) -> bool {
-    // DISABLED: JSON string values that look like RFC3339 dates are NOT coerced to
-    // `Date` during document parsing — they are kept as plain strings. This keeps the
-    // standard-codec index consistent with the moshiki path (which never coerces) and
-    // avoids collapsing distinct date strings into second-truncated `Date` terms.
-    // Restore the original heuristic below to re-enable date detection.
-    let _ = text;
-    return false;
-
-    #[allow(unreachable_code)]
-    {
-        if let Some(&first_byte) = text.as_bytes().first() {
-            if first_byte.is_ascii_digit() {
-                return true;
-            }
+    if let Some(&first_byte) = text.as_bytes().first() {
+        if first_byte.is_ascii_digit() {
+            return true;
         }
-
-        false
     }
+
+    false
 }
 
 impl<'a> Value<'a> for &'a serde_json::Value {

src/store/index/skip_index.rs

@@ -94,7 +94,13 @@ impl SkipIndex {
             byte_range: 0..first_layer_len,
         };
         for layer in &self.layers {
-            cur_checkpoint = layer.seek_start_at_offset(target, cur_checkpoint.byte_range.start)?;
+            if let Some(checkpoint) =
+                layer.seek_start_at_offset(target, cur_checkpoint.byte_range.start)
+            {
+                cur_checkpoint = checkpoint;
+            } else {
+                return None;
+            }
         }
         Some(cur_checkpoint)
     }

src/store/reader.rs

@@ -251,7 +251,7 @@ impl StoreReader {
     /// decompressing a compressed block. The store utilizes a LRU cache,
     /// so accessing docs from the same compressed block should be faster.
     /// For that reason a store reader should be kept and reused.
-    fn get_document_bytes(&self, doc_id: DocId) -> crate::Result<OwnedBytes> {
+    pub fn get_document_bytes(&self, doc_id: DocId) -> crate::Result<OwnedBytes> {
         let checkpoint = self.block_checkpoint(doc_id)?;
         let block = self.read_block(&checkpoint)?;
         Self::get_document_bytes_from_block(block, doc_id, &checkpoint)

stacker/Cargo.toml

@@ -8,7 +8,7 @@ repository = "https://github.com/quickwit-oss/tantivy"
 description = "term hashmap used for indexing"
 
 [dependencies]
-murmurhash32 = "0.4"
+murmurhash32 = "0.3"
 common = { version = "0.11", path = "../common/", package = "tantivy-common" }
 ahash = { version = "0.8.11", default-features = false, optional = true }
 

stacker/src/arena_hashmap.rs

@@ -83,28 +83,6 @@ impl ArenaHashMap {
         self.shared_arena_hashmap
             .mutate_or_create(key, &mut self.memory_arena, updater);
     }
-
-    /// Returns the address of the value associated to `key`, creating an entry
-    /// with `make_default()` if the key is not present yet (an existing value
-    /// is left untouched).
-    ///
-    /// See [`SharedArenaHashMap::get_or_create_value_addr`] for the address
-    /// stability guarantees.
-    #[inline]
-    pub fn get_or_create_value_addr<V>(
-        &mut self,
-        key: &[u8],
-        make_default: impl FnOnce() -> V,
-    ) -> Addr
-    where
-        V: Copy + 'static,
-    {
-        self.shared_arena_hashmap.get_or_create_value_addr(
-            key,
-            &mut self.memory_arena,
-            make_default,
-        )
-    }
 }
 
 #[cfg(test)]
@@ -143,28 +121,6 @@ mod tests {
         assert_eq!(hash_map.get::<u32>(b"abc"), None);
     }
 
-    #[test]
-    fn test_get_or_create_value_addr() {
-        let mut hash_map: ArenaHashMap = ArenaHashMap::default();
-        // Creates the entry with the default value.
-        let addr_abc = hash_map.get_or_create_value_addr(b"abc", || 7u32);
-        assert_eq!(hash_map.read::<u32>(addr_abc), 7u32);
-        // Returns the same address and does NOT overwrite an existing value.
-        let addr_abc_again = hash_map.get_or_create_value_addr(b"abc", || 99u32);
-        assert_eq!(addr_abc_again, addr_abc);
-        assert_eq!(hash_map.read::<u32>(addr_abc), 7u32);
-        // A different key gets its own entry.
-        let addr_def = hash_map.get_or_create_value_addr(b"def", || 5u32);
-        assert_ne!(addr_def, addr_abc);
-        assert_eq!(hash_map.read::<u32>(addr_def), 5u32);
-        // The address matches the one yielded by `iter`.
-        for (key, addr) in hash_map.iter() {
-            if key == b"abc" {
-                assert_eq!(addr, addr_abc);
-            }
-        }
-    }
-
     #[test]
     fn test_many_terms() {
         let mut terms: Vec<String> = (0..20_000).map(|val| val.to_string()).collect();

stacker/src/memory_arena.rs

@@ -36,7 +36,7 @@ const PAGE_SIZE: usize = 1 << NUM_BITS_PAGE_ADDR; // pages are 1 MB large
 /// page of memory.
 ///
 /// The last 20 bits are an address within this page of memory.
-#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
+#[derive(Copy, Clone, Debug)]
 pub struct Addr(u32);
 
 impl Addr {

stacker/src/shared_arena_hashmap.rs

@@ -347,67 +347,6 @@ impl SharedArenaHashMap {
             kv = self.table[bucket];
         }
     }
-
-    /// Returns the address of the value associated to `key`.
-    ///
-    /// If the key is not present yet, a new entry is created with the value
-    /// returned by `make_default()`. If the key is already present, the stored
-    /// value is left untouched and its address is returned.
-    ///
-    /// The returned `Addr` is the value address, i.e. the same address yielded
-    /// by [`Self::iter`] and consumed by [`MemoryArena::read`]. It remains valid
-    /// for the lifetime of the arena: arena allocations only ever append, and a
-    /// table resize relocates buckets, not the arena-backed key/value data.
-    ///
-    /// The key will be truncated to `u16::MAX` bytes.
-    #[inline]
-    pub fn get_or_create_value_addr<V>(
-        &mut self,
-        key: &[u8],
-        memory_arena: &mut MemoryArena,
-        make_default: impl FnOnce() -> V,
-    ) -> Addr
-    where
-        V: Copy + 'static,
-    {
-        if self.is_saturated() {
-            self.resize();
-        }
-        // Limit the key size to u16::MAX
-        let key = &key[..std::cmp::min(key.len(), u16::MAX as usize)];
-        let hash = self.get_hash(key);
-        let mut probe = self.probe(hash);
-        let mut bucket = probe.next_probe();
-        let mut kv: KeyValue = self.table[bucket];
-        loop {
-            if kv.is_empty() {
-                // The key does not exist yet: create it with the default value.
-                let val = make_default();
-                let num_bytes = std::mem::size_of::<u16>() + key.len() + std::mem::size_of::<V>();
-                let key_addr = memory_arena.allocate_space(num_bytes);
-                {
-                    let data = memory_arena.slice_mut(key_addr, num_bytes);
-                    let key_len_bytes: [u8; 2] = (key.len() as u16).to_le_bytes();
-                    data[..2].copy_from_slice(&key_len_bytes);
-                    let stop = 2 + key.len();
-                    fast_short_slice_copy(key, &mut data[2..stop]);
-                    store(&mut data[stop..], val);
-                }
-                self.set_bucket(hash, key_addr, bucket);
-                return key_addr.offset(2 + key.len() as u32);
-            }
-            if kv.hash == hash
-                && let Some(val_addr) =
-                    self.get_value_addr_if_key_match(key, kv.key_value_addr, memory_arena)
-            {
-                // The key already exists: leave its value untouched.
-                return val_addr;
-            }
-            // This allows fetching the next bucket before the loop jmp
-            bucket = probe.next_probe();
-            kv = self.table[bucket];
-        }
-    }
 }
 
 #[cfg(test)]