renaming

Removes the Write generics argument in PostingsSerializer.
This removes useless generic.
Prepares the path for codecs.
Removes one useless CountingWrite layer.
etc.

Cargo.toml

@@ -27,7 +27,7 @@ regex = { version = "1.5.5", default-features = false, features = [
 aho-corasick = "1.0"
 tantivy-fst = "0.5"
 memmap2 = { version = "0.9.0", optional = true }
-lz4_flex = { version = "0.12", default-features = false, optional = true }
+lz4_flex = { version = "0.11", default-features = false, optional = true }
 zstd = { version = "0.13", optional = true, default-features = false }
 tempfile = { version = "3.12.0", optional = true }
 log = "0.4.16"
@@ -50,7 +50,7 @@ fail = { version = "0.5.0", optional = true }
 time = { version = "0.3.35", features = ["serde-well-known"] }
 smallvec = "1.8.0"
 rayon = "1.5.2"
-lru = "0.16.3"
+lru = "0.12.0"
 fastdivide = "0.4.0"
 itertools = "0.14.0"
 measure_time = "0.9.0"
@@ -76,7 +76,7 @@ winapi = "0.3.9"
 
 [dev-dependencies]
 binggan = "0.14.2"
-rand = "0.9"
+rand = "0.8.5"
 maplit = "1.0.2"
 matches = "0.1.9"
 pretty_assertions = "1.2.1"
@@ -85,7 +85,7 @@ test-log = "0.2.10"
 futures = "0.3.21"
 paste = "1.0.11"
 more-asserts = "0.3.1"
-rand_distr = "0.5"
+rand_distr = "0.4.3"
 time = { version = "0.3.10", features = ["serde-well-known", "macros"] }
 postcard = { version = "1.0.4", features = [
     "use-std",

benches/agg_bench.rs

@@ -1,8 +1,8 @@
 use binggan::plugins::PeakMemAllocPlugin;
 use binggan::{black_box, InputGroup, PeakMemAlloc, INSTRUMENTED_SYSTEM};
-use rand::distr::weighted::WeightedIndex;
+use rand::distributions::WeightedIndex;
+use rand::prelude::SliceRandom;
 use rand::rngs::StdRng;
-use rand::seq::IndexedRandom;
 use rand::{Rng, SeedableRng};
 use rand_distr::Distribution;
 use serde_json::json;
@@ -532,7 +532,7 @@ fn get_test_index_bench(cardinality: Cardinality) -> tantivy::Result<Index> {
     // Prepare 1000 unique terms sampled using a Zipf distribution.
     // Exponent ~1.1 approximates top-20 terms covering around ~20%.
     let terms_1000: Vec<String> = (1..=1000).map(|i| format!("term_{i}")).collect();
-    let zipf_1000 = rand_distr::Zipf::new(1000.0, 1.1f64).unwrap();
+    let zipf_1000 = rand_distr::Zipf::new(1000, 1.1f64).unwrap();
 
     {
         let mut rng = StdRng::from_seed([1u8; 32]);
@@ -576,8 +576,8 @@ fn get_test_index_bench(cardinality: Cardinality) -> tantivy::Result<Index> {
         }
         let _val_max = 1_000_000.0;
         for _ in 0..doc_with_value {
-            let val: f64 = rng.random_range(0.0..1_000_000.0);
-            let json = if rng.random_bool(0.1) {
+            let val: f64 = rng.gen_range(0.0..1_000_000.0);
+            let json = if rng.gen_bool(0.1) {
                 // 10% are numeric values
                 json!({ "mixed_type": val })
             } else {
@@ -586,7 +586,7 @@ fn get_test_index_bench(cardinality: Cardinality) -> tantivy::Result<Index> {
             index_writer.add_document(doc!(
                 text_field => "cool",
                 json_field => json,
-                text_field_all_unique_terms => format!("unique_term_{}", rng.random::<u64>()),
+                text_field_all_unique_terms => format!("unique_term_{}", rng.gen::<u64>()),
                 text_field_many_terms => many_terms_data.choose(&mut rng).unwrap().to_string(),
                 text_field_few_terms_status => status_field_data[log_level_distribution.sample(&mut rng)].0,
                 text_field_1000_terms_zipf => terms_1000[zipf_1000.sample(&mut rng) as usize - 1].as_str(),

benches/and_or_queries.rs

@@ -55,29 +55,29 @@ fn build_shared_indices(num_docs: usize, p_a: f32, p_b: f32, p_c: f32) -> (Bench
     {
         let mut writer = index.writer_with_num_threads(1, 500_000_000).unwrap();
         for _ in 0..num_docs {
-            let has_a = rng.random_bool(p_a as f64);
-            let has_b = rng.random_bool(p_b as f64);
-            let has_c = rng.random_bool(p_c as f64);
-            let score = rng.random_range(0u64..100u64);
-            let score2 = rng.random_range(0u64..100_000u64);
+            let has_a = rng.gen_bool(p_a as f64);
+            let has_b = rng.gen_bool(p_b as f64);
+            let has_c = rng.gen_bool(p_c as f64);
+            let score = rng.gen_range(0u64..100u64);
+            let score2 = rng.gen_range(0u64..100_000u64);
             let mut title_tokens: Vec<&str> = Vec::new();
             let mut body_tokens: Vec<&str> = Vec::new();
             if has_a {
-                if rng.random_bool(0.1) {
+                if rng.gen_bool(0.1) {
                     title_tokens.push("a");
                 } else {
                     body_tokens.push("a");
                 }
             }
             if has_b {
-                if rng.random_bool(0.1) {
+                if rng.gen_bool(0.1) {
                     title_tokens.push("b");
                 } else {
                     body_tokens.push("b");
                 }
             }
             if has_c {
-                if rng.random_bool(0.1) {
+                if rng.gen_bool(0.1) {
                     title_tokens.push("c");
                 } else {
                     body_tokens.push("c");

benches/bool_queries_with_range.rs

@@ -36,13 +36,13 @@ fn build_shared_indices(num_docs: usize, p_title_a: f32, distribution: &str) ->
             "dense" => {
                 for doc_id in 0..num_docs {
                     // Always add title to avoid empty documents
-                    let title_token = if rng.random_bool(p_title_a as f64) {
+                    let title_token = if rng.gen_bool(p_title_a as f64) {
                         "a"
                     } else {
                         "b"
                     };
 
-                    let num_rand = rng.random_range(0u64..1000u64);
+                    let num_rand = rng.gen_range(0u64..1000u64);
 
                     let num_asc = (doc_id / 10000) as u64;
 
@@ -60,13 +60,13 @@ fn build_shared_indices(num_docs: usize, p_title_a: f32, distribution: &str) ->
             "sparse" => {
                 for doc_id in 0..num_docs {
                     // Always add title to avoid empty documents
-                    let title_token = if rng.random_bool(p_title_a as f64) {
+                    let title_token = if rng.gen_bool(p_title_a as f64) {
                         "a"
                     } else {
                         "b"
                     };
 
-                    let num_rand = rng.random_range(0u64..10000000u64);
+                    let num_rand = rng.gen_range(0u64..10000000u64);
 
                     let num_asc = doc_id as u64;
 

benches/range_queries.rs

@@ -33,7 +33,7 @@ fn build_shared_indices(num_docs: usize, distribution: &str) -> BenchIndex {
         match distribution {
             "dense" => {
                 for doc_id in 0..num_docs {
-                    let num_rand = rng.random_range(0u64..1000u64);
+                    let num_rand = rng.gen_range(0u64..1000u64);
                     let num_asc = (doc_id / 10000) as u64;
 
                     writer
@@ -46,7 +46,7 @@ fn build_shared_indices(num_docs: usize, distribution: &str) -> BenchIndex {
             }
             "sparse" => {
                 for doc_id in 0..num_docs {
-                    let num_rand = rng.random_range(0u64..10000000u64);
+                    let num_rand = rng.gen_range(0u64..10000000u64);
                     let num_asc = doc_id as u64;
 
                     writer

benches/range_query.rs

@@ -97,20 +97,20 @@ fn get_index_0_to_100() -> Index {
     let num_vals = 100_000;
     let docs: Vec<_> = (0..num_vals)
         .map(|_i| {
-            let id_name = if rng.random_bool(0.01) {
+            let id_name = if rng.gen_bool(0.01) {
                 "veryfew".to_string() // 1%
-            } else if rng.random_bool(0.1) {
+            } else if rng.gen_bool(0.1) {
                 "few".to_string() // 9%
             } else {
                 "most".to_string() // 90%
             };
             Doc {
                 id_name,
-                id: rng.random_range(0..100),
+                id: rng.gen_range(0..100),
                 // Multiply by 1000, so that we create most buckets in the compact space
                 // The benches depend on this range to select n-percent of elements with the
                 // methods below.
-                ip: Ipv6Addr::from_u128(rng.random_range(0..100) * 1000),
+                ip: Ipv6Addr::from_u128(rng.gen_range(0..100) * 1000),
             }
         })
         .collect();

bitpacker/Cargo.toml

@@ -18,5 +18,5 @@ homepage = "https://github.com/quickwit-oss/tantivy"
 bitpacking = { version = "0.9.2", default-features = false, features = ["bitpacker1x"] }
 
 [dev-dependencies]
-rand = "0.9"
+rand = "0.8"
 proptest = "1"

bitpacker/benches/bench.rs

@@ -4,8 +4,8 @@ extern crate test;
 
 #[cfg(test)]
 mod tests {
-    use rand::rng;
     use rand::seq::IteratorRandom;
+    use rand::thread_rng;
     use tantivy_bitpacker::{BitPacker, BitUnpacker, BlockedBitpacker};
     use test::Bencher;
 
@@ -27,7 +27,7 @@ mod tests {
         let num_els = 1_000_000u32;
         let bit_unpacker = BitUnpacker::new(bit_width);
         let data = create_bitpacked_data(bit_width, num_els);
-        let idxs: Vec<u32> = (0..num_els).choose_multiple(&mut rng(), 100_000);
+        let idxs: Vec<u32> = (0..num_els).choose_multiple(&mut thread_rng(), 100_000);
         b.iter(|| {
             let mut out = 0u64;
             for &idx in &idxs {

columnar/Cargo.toml

@@ -22,7 +22,7 @@ downcast-rs = "2.0.1"
 [dev-dependencies]
 proptest = "1"
 more-asserts = "0.3.1"
-rand = "0.9"
+rand = "0.8"
 binggan = "0.14.0"
 
 [[bench]]

columnar/benches/bench_column_values_get.rs

@@ -9,7 +9,7 @@ use tantivy_columnar::column_values::{CodecType, serialize_and_load_u64_based_co
 fn get_data() -> Vec<u64> {
     let mut rng = StdRng::seed_from_u64(2u64);
     let mut data: Vec<_> = (100..55_000_u64)
-        .map(|num| num + rng.random::<u8>() as u64)
+        .map(|num| num + rng.r#gen::<u8>() as u64)
         .collect();
     data.push(99_000);
     data.insert(1000, 2000);

columnar/benches/bench_create_column_values.rs

@@ -6,7 +6,7 @@ use tantivy_columnar::column_values::{CodecType, serialize_u64_based_column_valu
 fn get_data() -> Vec<u64> {
     let mut rng = StdRng::seed_from_u64(2u64);
     let mut data: Vec<_> = (100..55_000_u64)
-        .map(|num| num + rng.random::<u8>() as u64)
+        .map(|num| num + rng.r#gen::<u8>() as u64)
         .collect();
     data.push(99_000);
     data.insert(1000, 2000);

columnar/benches/bench_optional_index.rs

@@ -8,7 +8,7 @@ const TOTAL_NUM_VALUES: u32 = 1_000_000;
 fn gen_optional_index(fill_ratio: f64) -> OptionalIndex {
     let mut rng: StdRng = StdRng::from_seed([1u8; 32]);
     let vals: Vec<u32> = (0..TOTAL_NUM_VALUES)
-        .map(|_| rng.random_bool(fill_ratio))
+        .map(|_| rng.gen_bool(fill_ratio))
         .enumerate()
         .filter(|(_pos, val)| *val)
         .map(|(pos, _)| pos as u32)
@@ -25,7 +25,7 @@ fn random_range_iterator(
     let mut rng: StdRng = StdRng::from_seed([1u8; 32]);
     let mut current = start;
     std::iter::from_fn(move || {
-        current += rng.random_range(avg_step_size - avg_deviation..=avg_step_size + avg_deviation);
+        current += rng.gen_range(avg_step_size - avg_deviation..=avg_step_size + avg_deviation);
         if current >= end { None } else { Some(current) }
     })
 }

columnar/benches/bench_values_u128.rs

@@ -39,7 +39,7 @@ fn get_data_50percent_item() -> Vec<u128> {
 
     let mut data = vec![];
     for _ in 0..300_000 {
-        let val = rng.random_range(1..=100);
+        let val = rng.gen_range(1..=100);
         data.push(val);
     }
     data.push(SINGLE_ITEM);

columnar/benches/bench_values_u64.rs

@@ -34,7 +34,7 @@ fn get_data_50percent_item() -> Vec<u128> {
 
     let mut data = vec![];
     for _ in 0..300_000 {
-        let val = rng.random_range(1..=100);
+        let val = rng.gen_range(1..=100);
         data.push(val);
     }
     data.push(SINGLE_ITEM);

columnar/src/column_values/u64_based/linear.rs

@@ -268,7 +268,7 @@ mod tests {
 
     #[test]
     fn linear_interpol_fast_field_rand() {
-        let mut rng = rand::rng();
+        let mut rng = rand::thread_rng();
         for _ in 0..50 {
             let mut data = (0..10_000).map(|_| rng.next_u64()).collect::<Vec<_>>();
             create_and_validate::<LinearCodec>(&data, "random");

columnar/src/column_values/u64_based/tests.rs

@@ -122,7 +122,7 @@ pub(crate) fn create_and_validate<TColumnCodec: ColumnCodec>(
     assert_eq!(vals, buffer);
 
     if !vals.is_empty() {
-        let test_rand_idx = rand::rng().random_range(0..=vals.len() - 1);
+        let test_rand_idx = rand::thread_rng().gen_range(0..=vals.len() - 1);
         let expected_positions: Vec<u32> = vals
             .iter()
             .enumerate()

common/Cargo.toml

@@ -21,5 +21,5 @@ serde = { version = "1.0.136", features = ["derive"] }
 [dev-dependencies]
 binggan = "0.14.0"
 proptest = "1.0.0"
-rand = "0.9"
+rand = "0.8.4"
 

common/benches/bench.rs

@@ -1,6 +1,6 @@
 use binggan::{BenchRunner, black_box};
-use rand::rng;
 use rand::seq::IteratorRandom;
+use rand::thread_rng;
 use tantivy_common::{BitSet, TinySet, serialize_vint_u32};
 
 fn bench_vint() {
@@ -17,7 +17,7 @@ fn bench_vint() {
         black_box(out);
     });
 
-    let vals: Vec<u32> = (0..20_000).choose_multiple(&mut rng(), 100_000);
+    let vals: Vec<u32> = (0..20_000).choose_multiple(&mut thread_rng(), 100_000);
     runner.bench_function("bench_vint_rand", move |_| {
         let mut out = 0u64;
         for val in vals.iter().cloned() {

common/src/bitset.rs

@@ -416,7 +416,7 @@ mod tests {
     use std::collections::HashSet;
 
     use ownedbytes::OwnedBytes;
-    use rand::distr::Bernoulli;
+    use rand::distributions::Bernoulli;
     use rand::rngs::StdRng;
     use rand::{Rng, SeedableRng};
 

examples/iterating_docs_and_positions.rs

@@ -91,46 +91,10 @@ fn main() -> tantivy::Result<()> {
         }
     }
 
-    // A `Term` is a text token associated with a field.
-    // Let's go through all docs containing the term `title:the` and access their position
-    let term_the = Term::from_field_text(title, "the");
-
-    // Some other powerful operations (especially `.skip_to`) may be useful to consume these
+    // Some other powerful operations (especially `.seek`) may be useful to consume these
     // posting lists rapidly.
     // You can check for them in the [`DocSet`](https://docs.rs/tantivy/~0/tantivy/trait.DocSet.html) trait
     // and the [`Postings`](https://docs.rs/tantivy/~0/tantivy/trait.Postings.html) trait
 
-    // Also, for some VERY specific high performance use case like an OLAP analysis of logs,
-    // you can get better performance by accessing directly the blocks of doc ids.
-    for segment_reader in searcher.segment_readers() {
-        // A segment contains different data structure.
-        // Inverted index stands for the combination of
-        // - the term dictionary
-        // - the inverted lists associated with each terms and their positions
-        let inverted_index = segment_reader.inverted_index(title)?;
-
-        // This segment posting object is like a cursor over the documents matching the term.
-        // The `IndexRecordOption` arguments tells tantivy we will be interested in both term
-        // frequencies and positions.
-        //
-        // If you don't need all this information, you may get better performance by decompressing
-        // less information.
-        if let Some(mut block_segment_postings) =
-            inverted_index.read_block_postings(&term_the, IndexRecordOption::Basic)?
-        {
-            loop {
-                let docs = block_segment_postings.docs();
-                if docs.is_empty() {
-                    break;
-                }
-                // Once again these docs MAY contains deleted documents as well.
-                let docs = block_segment_postings.docs();
-                // Prints `Docs [0, 2].`
-                println!("Docs {docs:?}");
-                block_segment_postings.advance();
-            }
-        }
-    }
-
     Ok(())
 }

src/aggregation/agg_req.rs

@@ -115,66 +115,6 @@ pub fn get_fast_field_names(aggs: &Aggregations) -> HashSet<String> {
     fast_field_names
 }
 
-/// Validates that all fields referenced in the aggregation request exist in the schema
-/// and are configured as fast fields.
-///
-/// This is a convenience function for upfront validation before executing aggregations.
-/// Returns an error if any field doesn't exist or is not a fast field.
-///
-/// # Example
-/// ```
-/// use tantivy::aggregation::agg_req::{Aggregations, validate_aggregation_fields};
-/// use tantivy::schema::{Schema, FAST};
-/// use tantivy::Index;
-///
-/// # fn main() -> tantivy::Result<()> {
-/// // Create a simple index
-/// let mut schema_builder = Schema::builder();
-/// schema_builder.add_f64_field("price", FAST);
-/// let schema = schema_builder.build();
-/// let index = Index::create_in_ram(schema);
-///
-/// // Parse aggregation request
-/// let agg_req: Aggregations = serde_json::from_str(r#"{
-///     "avg_price": { "avg": { "field": "price" } }
-/// }"#)?;
-///
-/// let reader = index.reader()?;
-/// let searcher = reader.searcher();
-///
-/// // Validate fields before executing
-/// for segment_reader in searcher.segment_readers() {
-///     validate_aggregation_fields(&agg_req, segment_reader)?;
-/// }
-/// # Ok(())
-/// # }
-/// ```
-pub fn validate_aggregation_fields(
-    aggs: &Aggregations,
-    reader: &crate::SegmentReader,
-) -> crate::Result<()> {
-    let field_names = get_fast_field_names(aggs);
-    let schema = reader.schema();
-
-    for field_name in field_names {
-        // Check if the field is either directly in the schema or could be part of a json field
-        // present in the schema, and verify it's a fast field.
-        if let Some((field, _path)) = schema.find_field(&field_name) {
-            let field_type = schema.get_field_entry(field).field_type();
-            if !field_type.is_fast() {
-                return Err(crate::TantivyError::SchemaError(format!(
-                    "Field '{}' is not a fast field. Aggregations require fast fields.",
-                    field_name
-                )));
-            }
-        } else {
-            return Err(crate::TantivyError::FieldNotFound(field_name));
-        }
-    }
-
-    Ok(())
-}
-
 #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
 /// All aggregation types.
 pub enum AggregationVariants {

src/aggregation/agg_tests.rs

@@ -1436,44 +1436,3 @@ fn test_aggregation_on_json_object_mixed_numerical_segments() {
         )
     );
 }
-
-#[test]
-fn test_aggregation_field_validation_helper() {
-    // Test the standalone validation helper function for field validation
-    let index = get_test_index_2_segments(false).unwrap();
-    let reader = index.reader().unwrap();
-    let searcher = reader.searcher();
-    let segment_reader = searcher.segment_reader(0);
-
-    // Test with invalid field
-    let agg_req: Aggregations = serde_json::from_str(
-        r#"{
-        "avg_test": {
-            "avg": { "field": "nonexistent_field" }
-        }
-    }"#,
-    )
-    .unwrap();
-
-    let result = crate::aggregation::agg_req::validate_aggregation_fields(&agg_req, segment_reader);
-    assert!(result.is_err());
-    match result {
-        Err(crate::TantivyError::FieldNotFound(field_name)) => {
-            assert_eq!(field_name, "nonexistent_field");
-        }
-        _ => panic!("Expected FieldNotFound error, got: {:?}", result),
-    }
-
-    // Test with valid field
-    let agg_req: Aggregations = serde_json::from_str(
-        r#"{
-        "avg_test": {
-            "avg": { "field": "score" }
-        }
-    }"#,
-    )
-    .unwrap();
-
-    let result = crate::aggregation::agg_req::validate_aggregation_fields(&agg_req, segment_reader);
-    assert!(result.is_ok());
-}

src/codec/mod.rs

@@ -0,0 +1,90 @@
+pub mod postings;
+pub mod standard;
+
+use std::borrow::Cow;
+use std::io;
+
+use common::OwnedBytes;
+use serde::{Deserialize, Serialize};
+pub use standard::StandardCodec;
+
+use crate::codec::postings::PostingsCodec;
+use crate::postings::Postings;
+use crate::schema::IndexRecordOption;
+
+pub trait Codec: Clone + std::fmt::Debug + Send + Sync + 'static {
+    type PostingsCodec: PostingsCodec;
+
+    const NAME: &'static str;
+
+    fn from_json_props(json_value: &serde_json::Value) -> crate::Result<Self>;
+    fn to_json_props(&self) -> serde_json::Value;
+
+    fn postings_codec(&self) -> &Self::PostingsCodec;
+}
+
+#[derive(Serialize, Deserialize, Clone, Debug)]
+pub struct CodecConfiguration {
+    name: Cow<'static, str>,
+    #[serde(default, skip_serializing_if = "serde_json::Value::is_null")]
+    props: serde_json::Value,
+}
+
+impl CodecConfiguration {
+    pub fn from_codec<C: Codec>(codec: &C) -> Self {
+        CodecConfiguration {
+            name: Cow::Borrowed(C::NAME),
+            props: codec.to_json_props(),
+        }
+    }
+
+    pub fn to_codec<C: Codec>(&self) -> crate::Result<C> {
+        if self.name != C::NAME {
+            return Err(crate::TantivyError::InvalidArgument(format!(
+                "Codec name mismatch: expected {}, got {}",
+                C::NAME,
+                self.name
+            )));
+        }
+        C::from_json_props(&self.props)
+    }
+}
+
+impl Default for CodecConfiguration {
+    fn default() -> Self {
+        CodecConfiguration::from_codec(&StandardCodec)
+    }
+}
+
+pub trait ObjectSafeCodec: 'static + Send + Sync {
+    fn load_postings_type_erased(
+        &self,
+        doc_freq: u32,
+        postings_data: OwnedBytes,
+        record_option: IndexRecordOption,
+        requested_option: IndexRecordOption,
+        positions_data: Option<OwnedBytes>,
+    ) -> io::Result<Box<dyn Postings>>;
+}
+
+impl<TCodec: Codec> ObjectSafeCodec for TCodec {
+    fn load_postings_type_erased(
+        &self,
+        doc_freq: u32,
+        postings_data: OwnedBytes,
+        record_option: IndexRecordOption,
+        requested_option: IndexRecordOption,
+        positions_data: Option<OwnedBytes>,
+    ) -> io::Result<Box<dyn Postings>> {
+        let postings: <<Self as Codec>::PostingsCodec as PostingsCodec>::Postings =
+            self.postings_codec().load_postings(
+                doc_freq,
+                postings_data,
+                record_option,
+                requested_option,
+                positions_data,
+            )?;
+        let boxed_postings: Box<dyn Postings> = Box::new(postings);
+        Ok(boxed_postings)
+    }
+}

src/codec/postings/mod.rs

@@ -0,0 +1,40 @@
+use std::io;
+
+use common::OwnedBytes;
+
+use crate::fieldnorm::FieldNormReader;
+use crate::postings::{FreqReadingOption, Postings};
+use crate::query::Bm25Weight;
+use crate::schema::IndexRecordOption;
+use crate::{DocId, Score};
+
+pub trait PostingsCodec: Send + Sync + 'static {
+    type PostingsSerializer: PostingsSerializer;
+    type Postings: Postings;
+
+    fn new_serializer(
+        &self,
+        avg_fieldnorm: Score,
+        mode: IndexRecordOption,
+        fieldnorm_reader: Option<FieldNormReader>,
+    ) -> Self::PostingsSerializer;
+
+    fn load_postings(
+        &self,
+        doc_freq: u32,
+        postings_data: OwnedBytes,
+        record_option: IndexRecordOption,
+        requested_option: IndexRecordOption,
+        positions_data: Option<OwnedBytes>,
+    ) -> io::Result<Self::Postings>;
+}
+
+pub trait PostingsSerializer {
+    fn new_term(&mut self, term_doc_freq: u32, record_term_freq: bool);
+
+    fn write_doc(&mut self, doc_id: DocId, term_freq: u32);
+
+    fn close_term(&mut self, doc_freq: u32, wrt: &mut impl io::Write) -> io::Result<()>;
+
+    fn clear(&mut self);
+}

src/codec/standard/mod.rs

@@ -0,0 +1,33 @@
+use serde::{Deserialize, Serialize};
+
+use crate::codec::standard::postings::StandardPostingsCodec;
+use crate::codec::Codec;
+
+pub mod postings;
+
+#[derive(Debug, Default, Clone, Serialize, Deserialize)]
+pub struct StandardCodec;
+
+impl Codec for StandardCodec {
+    type PostingsCodec = StandardPostingsCodec;
+
+    const NAME: &'static str = "standard";
+
+    fn from_json_props(json_value: &serde_json::Value) -> crate::Result<Self> {
+        if !json_value.is_null() {
+            return Err(crate::TantivyError::InvalidArgument(format!(
+                "Codec property for the StandardCodec are unexpected. expected null, got {}",
+                json_value.as_str().unwrap_or("null")
+            )));
+        }
+        Ok(StandardCodec)
+    }
+
+    fn to_json_props(&self) -> serde_json::Value {
+        serde_json::Value::Null
+    }
+
+    fn postings_codec(&self) -> &Self::PostingsCodec {
+        &StandardPostingsCodec
+    }
+}

src/codec/standard/postings/block.rs

@@ -0,0 +1,50 @@
+use crate::postings::compression::COMPRESSION_BLOCK_SIZE;
+use crate::DocId;
+
+pub struct Block {
+    doc_ids: [DocId; COMPRESSION_BLOCK_SIZE],
+    term_freqs: [u32; COMPRESSION_BLOCK_SIZE],
+    len: usize,
+}
+
+impl Block {
+    pub fn new() -> Self {
+        Block {
+            doc_ids: [0u32; COMPRESSION_BLOCK_SIZE],
+            term_freqs: [0u32; COMPRESSION_BLOCK_SIZE],
+            len: 0,
+        }
+    }
+
+    pub fn doc_ids(&self) -> &[DocId] {
+        &self.doc_ids[..self.len]
+    }
+
+    pub fn term_freqs(&self) -> &[u32] {
+        &self.term_freqs[..self.len]
+    }
+
+    pub fn clear(&mut self) {
+        self.len = 0;
+    }
+
+    pub fn append_doc(&mut self, doc: DocId, term_freq: u32) {
+        let len = self.len;
+        self.doc_ids[len] = doc;
+        self.term_freqs[len] = term_freq;
+        self.len = len + 1;
+    }
+
+    pub fn is_full(&self) -> bool {
+        self.len == COMPRESSION_BLOCK_SIZE
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.len == 0
+    }
+
+    pub fn last_doc(&self) -> DocId {
+        assert_eq!(self.len, COMPRESSION_BLOCK_SIZE);
+        self.doc_ids[COMPRESSION_BLOCK_SIZE - 1]
+    }
+}

src/codec/standard/postings/block_segment_postings.rs

@@ -1,11 +1,11 @@
 use std::io;
 
-use common::VInt;
+use common::{OwnedBytes, VInt};
 
-use crate::directory::{FileSlice, OwnedBytes};
+use crate::codec::standard::postings::skip::{BlockInfo, SkipReader};
 use crate::fieldnorm::FieldNormReader;
-use crate::postings::compression::{BlockDecoder, VIntDecoder, COMPRESSION_BLOCK_SIZE};
-use crate::postings::{BlockInfo, FreqReadingOption, SkipReader};
+use crate::postings::compression::{BlockDecoder, VIntDecoder as _, COMPRESSION_BLOCK_SIZE};
+use crate::postings::FreqReadingOption;
 use crate::query::Bm25Weight;
 use crate::schema::IndexRecordOption;
 use crate::{DocId, Score, TERMINATED};
@@ -16,13 +16,8 @@ fn max_score<I: Iterator<Item = Score>>(mut it: I) -> Option<Score> {
 
 /// `BlockSegmentPostings` is a cursor iterating over blocks
 /// of documents.
-///
-/// # Warning
-///
-/// While it is useful for some very specific high-performance
-/// use cases, you should prefer using `SegmentPostings` for most usage.
 #[derive(Clone)]
-pub struct BlockSegmentPostings {
+pub(crate) struct BlockSegmentPostings {
     pub(crate) doc_decoder: BlockDecoder,
     block_loaded: bool,
     freq_decoder: BlockDecoder,
@@ -88,7 +83,7 @@ fn split_into_skips_and_postings(
 }
 
 impl BlockSegmentPostings {
-    /// Opens a `BlockSegmentPostings`.
+    /// Opens a `StandardPostingsReader`.
     /// `doc_freq` is the number of documents in the posting list.
     /// `record_option` represents the amount of data available according to the schema.
     /// `requested_option` is the amount of data requested by the user.
@@ -96,11 +91,10 @@ impl BlockSegmentPostings {
     /// term frequency blocks.
     pub(crate) fn open(
         doc_freq: u32,
-        data: FileSlice,
+        bytes: OwnedBytes,
         mut record_option: IndexRecordOption,
         requested_option: IndexRecordOption,
     ) -> io::Result<BlockSegmentPostings> {
-        let bytes = data.read_bytes()?;
         let (skip_data_opt, postings_data) = split_into_skips_and_postings(doc_freq, bytes)?;
         let skip_reader = match skip_data_opt {
             Some(skip_data) => {
@@ -138,76 +132,13 @@ impl BlockSegmentPostings {
         block_segment_postings.load_block();
         Ok(block_segment_postings)
     }
+}
 
-    /// Returns the block_max_score for the current block.
-    /// It does not require the block to be loaded. For instance, it is ok to call this method
-    /// after having called `.shallow_advance(..)`.
-    ///
-    /// See `TermScorer::block_max_score(..)` for more information.
-    pub fn block_max_score(
-        &mut self,
-        fieldnorm_reader: &FieldNormReader,
-        bm25_weight: &Bm25Weight,
-    ) -> Score {
-        if let Some(score) = self.block_max_score_cache {
-            return score;
-        }
-        if let Some(skip_reader_max_score) = self.skip_reader.block_max_score(bm25_weight) {
-            // if we are on a full block, the skip reader should have the block max information
-            // for us
-            self.block_max_score_cache = Some(skip_reader_max_score);
-            return skip_reader_max_score;
-        }
-        // this is the last block of the segment posting list.
-        // If it is actually loaded, we can compute block max manually.
-        if self.block_is_loaded() {
-            let docs = self.doc_decoder.output_array().iter().cloned();
-            let freqs = self.freq_decoder.output_array().iter().cloned();
-            let bm25_scores = docs.zip(freqs).map(|(doc, term_freq)| {
-                let fieldnorm_id = fieldnorm_reader.fieldnorm_id(doc);
-                bm25_weight.score(fieldnorm_id, term_freq)
-            });
-            let block_max_score = max_score(bm25_scores).unwrap_or(0.0);
-            self.block_max_score_cache = Some(block_max_score);
-            return block_max_score;
-        }
-        // We do not have access to any good block max value. We return bm25_weight.max_score()
-        // as it is a valid upperbound.
-        //
-        // We do not cache it however, so that it gets computed when once block is loaded.
-        bm25_weight.max_score()
-    }
-
-    pub(crate) fn freq_reading_option(&self) -> FreqReadingOption {
+impl BlockSegmentPostings {
+    pub fn freq_reading_option(&self) -> FreqReadingOption {
         self.freq_reading_option
     }
 
-    // Resets the block segment postings on another position
-    // in the postings file.
-    //
-    // This is useful for enumerating through a list of terms,
-    // and consuming the associated posting lists while avoiding
-    // reallocating a `BlockSegmentPostings`.
-    //
-    // # Warning
-    //
-    // This does not reset the positions list.
-    pub(crate) fn reset(&mut self, doc_freq: u32, postings_data: OwnedBytes) -> io::Result<()> {
-        let (skip_data_opt, postings_data) =
-            split_into_skips_and_postings(doc_freq, postings_data)?;
-        self.data = postings_data;
-        self.block_max_score_cache = None;
-        self.block_loaded = false;
-        if let Some(skip_data) = skip_data_opt {
-            self.skip_reader.reset(skip_data, doc_freq);
-        } else {
-            self.skip_reader.reset(OwnedBytes::empty(), doc_freq);
-        }
-        self.doc_freq = doc_freq;
-        self.load_block();
-        Ok(())
-    }
-
     /// Returns the overall number of documents in the block postings.
     /// It does not take in account whether documents are deleted or not.
     ///
@@ -223,41 +154,31 @@ impl BlockSegmentPostings {
     /// returned by `.docs()` is empty.
     #[inline]
     pub fn docs(&self) -> &[DocId] {
-        debug_assert!(self.block_is_loaded());
+        debug_assert!(self.block_loaded);
         self.doc_decoder.output_array()
     }
 
     /// Return the document at index `idx` of the block.
     #[inline]
     pub fn doc(&self, idx: usize) -> u32 {
+        debug_assert!(self.block_loaded);
         self.doc_decoder.output(idx)
     }
 
     /// Return the array of `term freq` in the block.
     #[inline]
     pub fn freqs(&self) -> &[u32] {
-        debug_assert!(self.block_is_loaded());
+        debug_assert!(self.block_loaded);
         self.freq_decoder.output_array()
     }
 
     /// Return the frequency at index `idx` of the block.
     #[inline]
     pub fn freq(&self, idx: usize) -> u32 {
-        debug_assert!(self.block_is_loaded());
+        debug_assert!(self.block_loaded);
         self.freq_decoder.output(idx)
     }
 
-    /// Returns the length of the current block.
-    ///
-    /// All blocks have a length of `NUM_DOCS_PER_BLOCK`,
-    /// except the last block that may have a length
-    /// of any number between 1 and `NUM_DOCS_PER_BLOCK - 1`
-    #[inline]
-    pub fn block_len(&self) -> usize {
-        debug_assert!(self.block_is_loaded());
-        self.doc_decoder.output_len
-    }
-
     /// Position on a block that may contains `target_doc`.
     ///
     /// If all docs are smaller than target, the block loaded may be empty,
@@ -281,10 +202,77 @@ impl BlockSegmentPostings {
         doc
     }
 
-    pub(crate) fn position_offset(&self) -> u64 {
+    pub fn position_offset(&self) -> u64 {
         self.skip_reader.position_offset()
     }
 
+    /// Advance to the next block.
+    pub fn advance(&mut self) {
+        self.skip_reader.advance();
+        self.block_loaded = false;
+        self.block_max_score_cache = None;
+        self.load_block();
+    }
+
+    /// Returns the block_max_score for the current block.
+    /// It does not require the block to be loaded. For instance, it is ok to call this method
+    /// after having called `.shallow_advance(..)`.
+    ///
+    /// See `TermScorer::block_max_score(..)` for more information.
+    pub fn block_max_score(
+        &mut self,
+        fieldnorm_reader: &FieldNormReader,
+        bm25_weight: &Bm25Weight,
+    ) -> Score {
+        if let Some(score) = self.block_max_score_cache {
+            return score;
+        }
+        if let Some(skip_reader_max_score) = self.skip_reader.block_max_score(bm25_weight) {
+            // if we are on a full block, the skip reader should have the block max information
+            // for us
+            self.block_max_score_cache = Some(skip_reader_max_score);
+            return skip_reader_max_score;
+        }
+        // this is the last block of the segment posting list.
+        // If it is actually loaded, we can compute block max manually.
+        if self.block_loaded {
+            let docs = self.doc_decoder.output_array().iter().cloned();
+            let freqs = self.freq_decoder.output_array().iter().cloned();
+            let bm25_scores = docs.zip(freqs).map(|(doc, term_freq)| {
+                let fieldnorm_id = fieldnorm_reader.fieldnorm_id(doc);
+                bm25_weight.score(fieldnorm_id, term_freq)
+            });
+            let block_max_score = max_score(bm25_scores).unwrap_or(0.0);
+            self.block_max_score_cache = Some(block_max_score);
+            return block_max_score;
+        }
+        // We do not have access to any good block max value. We return bm25_weight.max_score()
+        // as it is a valid upperbound.
+        //
+        // We do not cache it however, so that it gets computed when once block is loaded.
+        bm25_weight.max_score()
+    }
+}
+
+impl BlockSegmentPostings {
+    /// Returns an empty segment postings object
+    pub fn empty() -> BlockSegmentPostings {
+        BlockSegmentPostings {
+            doc_decoder: BlockDecoder::with_val(TERMINATED),
+            block_loaded: true,
+            freq_decoder: BlockDecoder::with_val(1),
+            freq_reading_option: FreqReadingOption::NoFreq,
+            block_max_score_cache: None,
+            doc_freq: 0,
+            data: OwnedBytes::empty(),
+            skip_reader: SkipReader::new(OwnedBytes::empty(), 0, IndexRecordOption::Basic),
+        }
+    }
+
+    pub(crate) fn skip_reader(&self) -> &SkipReader {
+        &self.skip_reader
+    }
+
     /// Dangerous API! This calls seeks the next block on the skip list,
     /// but does not `.load_block()` afterwards.
     ///
@@ -298,15 +286,11 @@ impl BlockSegmentPostings {
         }
     }
 
-    pub(crate) fn block_is_loaded(&self) -> bool {
-        self.block_loaded
-    }
-
     pub(crate) fn load_block(&mut self) {
-        let offset = self.skip_reader.byte_offset();
-        if self.block_is_loaded() {
+        if self.block_loaded {
             return;
         }
+        let offset = self.skip_reader.byte_offset();
         match self.skip_reader.block_info() {
             BlockInfo::BitPacked {
                 doc_num_bits,
@@ -351,32 +335,6 @@ impl BlockSegmentPostings {
         }
         self.block_loaded = true;
     }
-
-    /// Advance to the next block.
-    pub fn advance(&mut self) {
-        self.skip_reader.advance();
-        self.block_loaded = false;
-        self.block_max_score_cache = None;
-        self.load_block();
-    }
-
-    /// Returns an empty segment postings object
-    pub fn empty() -> BlockSegmentPostings {
-        BlockSegmentPostings {
-            doc_decoder: BlockDecoder::with_val(TERMINATED),
-            block_loaded: true,
-            freq_decoder: BlockDecoder::with_val(1),
-            freq_reading_option: FreqReadingOption::NoFreq,
-            block_max_score_cache: None,
-            doc_freq: 0,
-            data: OwnedBytes::empty(),
-            skip_reader: SkipReader::new(OwnedBytes::empty(), 0, IndexRecordOption::Basic),
-        }
-    }
-
-    pub(crate) fn skip_reader(&self) -> &SkipReader {
-        &self.skip_reader
-    }
 }
 
 #[cfg(test)]
@@ -387,10 +345,8 @@ mod tests {
     use crate::docset::{DocSet, TERMINATED};
     use crate::index::Index;
     use crate::postings::compression::COMPRESSION_BLOCK_SIZE;
-    use crate::postings::postings::Postings;
-    use crate::postings::SegmentPostings;
+    use crate::postings::{Postings as _, SegmentPostings};
     use crate::schema::{IndexRecordOption, Schema, Term, INDEXED};
-    use crate::DocId;
 
     #[test]
     fn test_empty_segment_postings() {
@@ -479,31 +435,6 @@ mod tests {
         Ok(())
     }
 
-    fn build_block_postings(docs: &[DocId]) -> crate::Result<BlockSegmentPostings> {
-        let mut schema_builder = Schema::builder();
-        let int_field = schema_builder.add_u64_field("id", INDEXED);
-        let schema = schema_builder.build();
-        let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_for_tests()?;
-        let mut last_doc = 0u32;
-        for &doc in docs {
-            for _ in last_doc..doc {
-                index_writer.add_document(doc!(int_field=>1u64))?;
-            }
-            index_writer.add_document(doc!(int_field=>0u64))?;
-            last_doc = doc + 1;
-        }
-        index_writer.commit()?;
-        let searcher = index.reader()?.searcher();
-        let segment_reader = searcher.segment_reader(0);
-        let inverted_index = segment_reader.inverted_index(int_field).unwrap();
-        let term = Term::from_field_u64(int_field, 0u64);
-        let term_info = inverted_index.get_term_info(&term)?.unwrap();
-        let block_postings = inverted_index
-            .read_block_postings_from_terminfo(&term_info, IndexRecordOption::Basic)?;
-        Ok(block_postings)
-    }
-
     #[test]
     fn test_block_segment_postings_seek() -> crate::Result<()> {
         let mut docs = vec![0];
@@ -539,7 +470,7 @@ mod tests {
         let searcher = index.reader()?.searcher();
         let segment_reader = searcher.segment_reader(0);
 
-        let mut block_segments;
+        let block_segments;
         {
             let term = Term::from_field_u64(int_field, 0u64);
             let inverted_index = segment_reader.inverted_index(int_field)?;
@@ -548,13 +479,6 @@ mod tests {
                 .read_block_postings_from_terminfo(&term_info, IndexRecordOption::Basic)?;
         }
         assert_eq!(block_segments.docs(), &[0, 2, 4]);
-        {
-            let term = Term::from_field_u64(int_field, 1u64);
-            let inverted_index = segment_reader.inverted_index(int_field)?;
-            let term_info = inverted_index.get_term_info(&term)?.unwrap();
-            inverted_index.reset_block_postings_from_terminfo(&term_info, &mut block_segments)?;
-        }
-        assert_eq!(block_segments.docs(), &[1, 3, 5]);
         Ok(())
     }
 }

src/codec/standard/postings/mod.rs

@@ -0,0 +1,52 @@
+use std::io;
+
+use crate::codec::postings::PostingsCodec;
+use crate::codec::standard::postings::block_segment_postings::BlockSegmentPostings;
+use crate::codec::standard::postings::segment_postings::SegmentPostings;
+use crate::fieldnorm::FieldNormReader;
+use crate::positions::PositionReader;
+use crate::schema::IndexRecordOption;
+use crate::Score;
+
+mod block;
+mod block_segment_postings;
+mod segment_postings;
+mod skip;
+mod standard_postings_serializer;
+
+pub use standard_postings_serializer::StandardPostingsSerializer;
+
+pub struct StandardPostingsCodec;
+
+impl PostingsCodec for StandardPostingsCodec {
+    type PostingsSerializer = StandardPostingsSerializer;
+    type Postings = SegmentPostings;
+
+    fn new_serializer(
+        &self,
+        avg_fieldnorm: Score,
+        mode: IndexRecordOption,
+        fieldnorm_reader: Option<FieldNormReader>,
+    ) -> Self::PostingsSerializer {
+        StandardPostingsSerializer::new(avg_fieldnorm, mode, fieldnorm_reader)
+    }
+
+    fn load_postings(
+        &self,
+        doc_freq: u32,
+        postings_data: common::OwnedBytes,
+        record_option: IndexRecordOption,
+        requested_option: IndexRecordOption,
+        positions_data_opt: Option<common::OwnedBytes>,
+    ) -> io::Result<Self::Postings> {
+        // Rationalize record_option/requested_option.
+        let record_option = requested_option.downgrade(record_option);
+        let block_segment_postings =
+            BlockSegmentPostings::open(doc_freq, postings_data, record_option, requested_option)?;
+        let position_reader = positions_data_opt.map(PositionReader::open).transpose()?;
+        Ok(SegmentPostings::from_block_postings(
+            block_segment_postings,
+            position_reader,
+        ))
+    }
+}

src/codec/standard/postings/segment_postings.rs

@@ -1,11 +1,13 @@
-use common::HasLen;
+use common::{BitSet, HasLen};
 
+use super::BlockSegmentPostings;
 use crate::docset::DocSet;
-use crate::fastfield::AliveBitSet;
+use crate::fieldnorm::FieldNormReader;
 use crate::positions::PositionReader;
 use crate::postings::compression::COMPRESSION_BLOCK_SIZE;
-use crate::postings::{BlockSegmentPostings, Postings};
-use crate::{DocId, TERMINATED};
+use crate::postings::{FreqReadingOption, Postings};
+use crate::query::Bm25Weight;
+use crate::{DocId, Score, TERMINATED};
 
 /// `SegmentPostings` represents the inverted list or postings associated with
 /// a term in a `Segment`.
@@ -29,31 +31,6 @@ impl SegmentPostings {
         }
     }
 
-    /// Compute the number of non-deleted documents.
-    ///
-    /// This method will clone and scan through the posting lists.
-    /// (this is a rather expensive operation).
-    pub fn doc_freq_given_deletes(&self, alive_bitset: &AliveBitSet) -> u32 {
-        let mut docset = self.clone();
-        let mut doc_freq = 0;
-        loop {
-            let doc = docset.doc();
-            if doc == TERMINATED {
-                return doc_freq;
-            }
-            if alive_bitset.is_alive(doc) {
-                doc_freq += 1u32;
-            }
-            docset.advance();
-        }
-    }
-
-    /// Returns the overall number of documents in the block postings.
-    /// It does not take in account whether documents are deleted or not.
-    pub fn doc_freq(&self) -> u32 {
-        self.block_cursor.doc_freq()
-    }
-
     /// Creates a segment postings object with the given documents
     /// and no frequency encoded.
     ///
@@ -64,13 +41,19 @@ impl SegmentPostings {
     /// buffer with the serialized data.
     #[cfg(test)]
     pub fn create_from_docs(docs: &[u32]) -> SegmentPostings {
-        use crate::directory::FileSlice;
-        use crate::postings::serializer::PostingsSerializer;
+        use common::OwnedBytes;
+
         use crate::schema::IndexRecordOption;
         let mut buffer = Vec::new();
         {
+            use crate::codec::postings::PostingsSerializer;
+
             let mut postings_serializer =
-                PostingsSerializer::new(0.0, IndexRecordOption::Basic, None);
+                crate::codec::standard::postings::StandardPostingsSerializer::new(
+                    0.0,
+                    IndexRecordOption::Basic,
+                    None,
+                );
             postings_serializer.new_term(docs.len() as u32, false);
             for &doc in docs {
                 postings_serializer.write_doc(doc, 1u32);
@@ -81,7 +64,7 @@ impl SegmentPostings {
         }
         let block_segment_postings = BlockSegmentPostings::open(
             docs.len() as u32,
-            FileSlice::from(buffer),
+            OwnedBytes::new(buffer),
             IndexRecordOption::Basic,
             IndexRecordOption::Basic,
         )
@@ -95,9 +78,11 @@ impl SegmentPostings {
         doc_and_tfs: &[(u32, u32)],
         fieldnorms: Option<&[u32]>,
     ) -> SegmentPostings {
-        use crate::directory::FileSlice;
+        use common::OwnedBytes;
+
+        use crate::codec::postings::PostingsSerializer as _;
+        use crate::codec::standard::postings::StandardPostingsSerializer;
         use crate::fieldnorm::FieldNormReader;
-        use crate::postings::serializer::PostingsSerializer;
         use crate::schema::IndexRecordOption;
         use crate::Score;
         let mut buffer: Vec<u8> = Vec::new();
@@ -114,7 +99,7 @@ impl SegmentPostings {
                 total_num_tokens as Score / fieldnorms.len() as Score
             })
             .unwrap_or(0.0);
-        let mut postings_serializer = PostingsSerializer::new(
+        let mut postings_serializer = StandardPostingsSerializer::new(
             average_field_norm,
             IndexRecordOption::WithFreqs,
             fieldnorm_reader,
@@ -128,7 +113,7 @@ impl SegmentPostings {
             .unwrap();
         let block_segment_postings = BlockSegmentPostings::open(
             doc_and_tfs.len() as u32,
-            FileSlice::from(buffer),
+            OwnedBytes::new(buffer),
             IndexRecordOption::WithFreqs,
             IndexRecordOption::WithFreqs,
         )
@@ -158,7 +143,6 @@ impl DocSet for SegmentPostings {
     // next needs to be called a first time to point to the correct element.
     #[inline]
     fn advance(&mut self) -> DocId {
-        debug_assert!(self.block_cursor.block_is_loaded());
         if self.cur == COMPRESSION_BLOCK_SIZE - 1 {
             self.cur = 0;
             self.block_cursor.advance();
@@ -191,6 +175,19 @@ impl DocSet for SegmentPostings {
     fn size_hint(&self) -> u32 {
         self.len() as u32
     }
+
+    fn fill_bitset(&mut self, bitset: &mut BitSet) {
+        loop {
+            let docs = self.block_cursor.docs();
+            if docs.is_empty() {
+                break;
+            }
+            for &doc in docs {
+                bitset.insert(doc);
+            }
+            self.block_cursor.advance();
+        }
+    }
 }
 
 impl HasLen for SegmentPostings {
@@ -200,6 +197,15 @@ impl HasLen for SegmentPostings {
 }
 
 impl Postings for SegmentPostings {
+    fn new_term_scorer(
+        self: Box<Self>,
+        fieldnorm_reader: FieldNormReader,
+        similarity_weight: Bm25Weight,
+    ) -> Box<dyn crate::query::Scorer> {
+        use crate::query::term_query::TermScorer;
+        Box::new(TermScorer::new(*self, fieldnorm_reader, similarity_weight))
+    }
+
     /// Returns the frequency associated with the current document.
     /// If the schema is set up so that no frequency have been encoded,
     /// this method should always return 1.
@@ -221,6 +227,12 @@ impl Postings for SegmentPostings {
         self.block_cursor.freq(self.cur)
     }
 
+    /// Returns the overall number of documents in the block postings.
+    /// It does not take in account whether documents are deleted or not.
+    fn doc_freq(&self) -> u32 {
+        self.block_cursor.doc_freq()
+    }
+
     fn append_positions_with_offset(&mut self, offset: u32, output: &mut Vec<u32>) {
         let term_freq = self.term_freq();
         let prev_len = output.len();
@@ -244,6 +256,29 @@ impl Postings for SegmentPostings {
             }
         }
     }
+
+    fn supports_block_max(&self) -> bool {
+        true
+    }
+
+    fn seek_block(
+        &mut self,
+        target_doc: crate::DocId,
+        fieldnorm_reader: &FieldNormReader,
+        similarity_weight: &Bm25Weight,
+    ) -> Score {
+        self.block_cursor.seek_block(target_doc);
+        self.block_cursor
+            .block_max_score(&fieldnorm_reader, &similarity_weight)
+    }
+
+    fn last_doc_in_block(&self) -> crate::DocId {
+        self.block_cursor.skip_reader().last_doc_in_block()
+    }
+
+    fn freq_reading_option(&self) -> FreqReadingOption {
+        self.block_cursor.freq_reading_option()
+    }
 }
 
 #[cfg(test)]

src/codec/standard/postings/standard_postings_serializer.rs

@@ -0,0 +1,186 @@
+use std::cmp::Ordering;
+use std::io::{self, Write as _};
+
+use common::{BinarySerializable as _, VInt};
+
+use crate::codec::postings::PostingsSerializer;
+use crate::codec::standard::postings::block::Block;
+use crate::codec::standard::postings::skip::SkipSerializer;
+use crate::fieldnorm::FieldNormReader;
+use crate::postings::compression::{BlockEncoder, VIntEncoder as _, COMPRESSION_BLOCK_SIZE};
+use crate::query::Bm25Weight;
+use crate::schema::IndexRecordOption;
+use crate::{DocId, Score};
+
+pub struct StandardPostingsSerializer {
+    last_doc_id_encoded: u32,
+
+    block_encoder: BlockEncoder,
+    block: Box<Block>,
+
+    postings_write: Vec<u8>,
+    skip_write: SkipSerializer,
+
+    mode: IndexRecordOption,
+    fieldnorm_reader: Option<FieldNormReader>,
+
+    bm25_weight: Option<Bm25Weight>,
+    avg_fieldnorm: Score, /* Average number of term in the field for that segment.
+                           * this value is used to compute the block wand information. */
+    term_has_freq: bool,
+}
+
+impl StandardPostingsSerializer {
+    pub fn new(
+        avg_fieldnorm: Score,
+        mode: IndexRecordOption,
+        fieldnorm_reader: Option<FieldNormReader>,
+    ) -> StandardPostingsSerializer {
+        Self {
+            last_doc_id_encoded: 0,
+            block_encoder: BlockEncoder::new(),
+            block: Box::new(Block::new()),
+            postings_write: Vec::new(),
+            skip_write: SkipSerializer::new(),
+            mode,
+            fieldnorm_reader,
+            bm25_weight: None,
+            avg_fieldnorm,
+            term_has_freq: false,
+        }
+    }
+}
+
+impl PostingsSerializer for StandardPostingsSerializer {
+    fn new_term(&mut self, term_doc_freq: u32, record_term_freq: bool) {
+        self.bm25_weight = None;
+
+        self.term_has_freq = self.mode.has_freq() && record_term_freq;
+        if !self.term_has_freq {
+            return;
+        }
+
+        let num_docs_in_segment: u64 =
+            if let Some(fieldnorm_reader) = self.fieldnorm_reader.as_ref() {
+                fieldnorm_reader.num_docs() as u64
+            } else {
+                return;
+            };
+
+        if num_docs_in_segment == 0 {
+            return;
+        }
+
+        self.bm25_weight = Some(Bm25Weight::for_one_term_without_explain(
+            term_doc_freq as u64,
+            num_docs_in_segment,
+            self.avg_fieldnorm,
+        ));
+    }
+
+    fn write_doc(&mut self, doc_id: DocId, term_freq: u32) {
+        self.block.append_doc(doc_id, term_freq);
+        if self.block.is_full() {
+            self.write_block();
+        }
+    }
+
+    fn close_term(
+        &mut self,
+        doc_freq: u32,
+        output_write: &mut impl std::io::Write,
+    ) -> io::Result<()> {
+        if !self.block.is_empty() {
+            // we have doc ids waiting to be written
+            // this happens when the number of doc ids is
+            // not a perfect multiple of our block size.
+            //
+            // In that case, the remaining part is encoded
+            // using variable int encoding.
+            {
+                let block_encoded = self
+                    .block_encoder
+                    .compress_vint_sorted(self.block.doc_ids(), self.last_doc_id_encoded);
+                self.postings_write.write_all(block_encoded)?;
+            }
+            // ... Idem for term frequencies
+            if self.term_has_freq {
+                let block_encoded = self
+                    .block_encoder
+                    .compress_vint_unsorted(self.block.term_freqs());
+                self.postings_write.write_all(block_encoded)?;
+            }
+            self.block.clear();
+        }
+        if doc_freq >= COMPRESSION_BLOCK_SIZE as u32 {
+            let skip_data = self.skip_write.data();
+            VInt(skip_data.len() as u64).serialize(output_write)?;
+            output_write.write_all(skip_data)?;
+        }
+        output_write.write_all(&self.postings_write[..])?;
+        self.skip_write.clear();
+        self.postings_write.clear();
+        self.bm25_weight = None;
+        Ok(())
+    }
+
+    fn clear(&mut self) {
+        self.block.clear();
+        self.last_doc_id_encoded = 0;
+    }
+}
+
+impl StandardPostingsSerializer {
+    fn write_block(&mut self) {
+        {
+            // encode the doc ids
+            let (num_bits, block_encoded): (u8, &[u8]) = self
+                .block_encoder
+                .compress_block_sorted(self.block.doc_ids(), self.last_doc_id_encoded);
+            self.last_doc_id_encoded = self.block.last_doc();
+            self.skip_write
+                .write_doc(self.last_doc_id_encoded, num_bits);
+            // last el block 0, offset block 1,
+            self.postings_write.extend(block_encoded);
+        }
+        if self.term_has_freq {
+            let (num_bits, block_encoded): (u8, &[u8]) = self
+                .block_encoder
+                .compress_block_unsorted(self.block.term_freqs(), true);
+            self.postings_write.extend(block_encoded);
+            self.skip_write.write_term_freq(num_bits);
+            if self.mode.has_positions() {
+                // We serialize the sum of term freqs within the skip information
+                // in order to navigate through positions.
+                let sum_freq = self.block.term_freqs().iter().cloned().sum();
+                self.skip_write.write_total_term_freq(sum_freq);
+            }
+            let mut blockwand_params = (0u8, 0u32);
+            if let Some(bm25_weight) = self.bm25_weight.as_ref() {
+                if let Some(fieldnorm_reader) = self.fieldnorm_reader.as_ref() {
+                    let docs = self.block.doc_ids().iter().cloned();
+                    let term_freqs = self.block.term_freqs().iter().cloned();
+                    let fieldnorms = docs.map(|doc| fieldnorm_reader.fieldnorm_id(doc));
+                    blockwand_params = fieldnorms
+                        .zip(term_freqs)
+                        .max_by(
+                            |(left_fieldnorm_id, left_term_freq),
+                             (right_fieldnorm_id, right_term_freq)| {
+                                let left_score =
+                                    bm25_weight.tf_factor(*left_fieldnorm_id, *left_term_freq);
+                                let right_score =
+                                    bm25_weight.tf_factor(*right_fieldnorm_id, *right_term_freq);
+                                left_score
+                                    .partial_cmp(&right_score)
+                                    .unwrap_or(Ordering::Equal)
+                            },
+                        )
+                        .unwrap();
+                }
+            }
+            let (fieldnorm_id, term_freq) = blockwand_params;
+            self.skip_write.write_blockwand_max(fieldnorm_id, term_freq);
+        }
+        self.block.clear();
+    }
+}

src/collector/facet_collector.rs

@@ -486,9 +486,9 @@ mod tests {
     use std::collections::BTreeSet;
 
     use columnar::Dictionary;
-    use rand::distr::Uniform;
+    use rand::distributions::Uniform;
     use rand::prelude::SliceRandom;
-    use rand::{rng, Rng};
+    use rand::{thread_rng, Rng};
 
     use super::{FacetCollector, FacetCounts};
     use crate::collector::facet_collector::compress_mapping;
@@ -731,7 +731,7 @@ mod tests {
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
 
-        let uniform = Uniform::new_inclusive(1, 100_000).unwrap();
+        let uniform = Uniform::new_inclusive(1, 100_000);
         let mut docs: Vec<TantivyDocument> =
             vec![("a", 10), ("b", 100), ("c", 7), ("d", 12), ("e", 21)]
                 .into_iter()
@@ -741,11 +741,14 @@ mod tests {
                     std::iter::repeat_n(doc, count)
                 })
                 .map(|mut doc| {
-                    doc.add_facet(facet_field, &format!("/facet/{}", rng().sample(uniform)));
+                    doc.add_facet(
+                        facet_field,
+                        &format!("/facet/{}", thread_rng().sample(uniform)),
+                    );
                     doc
                 })
                 .collect();
-        docs[..].shuffle(&mut rng());
+        docs[..].shuffle(&mut thread_rng());
 
         let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
         for doc in docs {
@@ -819,8 +822,8 @@ mod tests {
 #[cfg(all(test, feature = "unstable"))]
 mod bench {
 
-    use rand::rng;
     use rand::seq::SliceRandom;
+    use rand::thread_rng;
     use test::Bencher;
 
     use crate::collector::FacetCollector;
@@ -843,7 +846,7 @@ mod bench {
             }
         }
         // 40425 docs
-        docs[..].shuffle(&mut rng());
+        docs[..].shuffle(&mut thread_rng());
 
         let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
         for doc in docs {

src/collector/sort_key_top_collector.rs

@@ -160,7 +160,7 @@ mod tests {
         expected: &[(crate::Score, usize)],
     ) {
         let mut vals: Vec<(crate::Score, usize)> = (0..10).map(|val| (val as f32, val)).collect();
-        vals.shuffle(&mut rand::rng());
+        vals.shuffle(&mut rand::thread_rng());
         let vals_merged = merge_top_k(vals.into_iter(), doc_range, ComparatorEnum::from(order));
         assert_eq!(&vals_merged, expected);
     }

src/core/json_utils.rs

@@ -4,7 +4,7 @@ use common::{replace_in_place, JsonPathWriter};
 use rustc_hash::FxHashMap;
 
 use crate::indexer::indexing_term::IndexingTerm;
-use crate::postings::{IndexingContext, IndexingPosition, PostingsWriter};
+use crate::postings::{IndexingContext, IndexingPosition, PostingsWriter as _, PostingsWriterEnum};
 use crate::schema::document::{ReferenceValue, ReferenceValueLeaf, Value};
 use crate::schema::{Type, DATE_TIME_PRECISION_INDEXED};
 use crate::time::format_description::well_known::Rfc3339;
@@ -80,7 +80,7 @@ fn index_json_object<'a, V: Value<'a>>(
     text_analyzer: &mut TextAnalyzer,
     term_buffer: &mut IndexingTerm,
     json_path_writer: &mut JsonPathWriter,
-    postings_writer: &mut dyn PostingsWriter,
+    postings_writer: &mut PostingsWriterEnum,
     ctx: &mut IndexingContext,
     positions_per_path: &mut IndexingPositionsPerPath,
 ) {
@@ -110,7 +110,7 @@ pub(crate) fn index_json_value<'a, V: Value<'a>>(
     text_analyzer: &mut TextAnalyzer,
     term_buffer: &mut IndexingTerm,
     json_path_writer: &mut JsonPathWriter,
-    postings_writer: &mut dyn PostingsWriter,
+    postings_writer: &mut PostingsWriterEnum,
     ctx: &mut IndexingContext,
     positions_per_path: &mut IndexingPositionsPerPath,
 ) {

src/docset.rs

@@ -1,5 +1,7 @@
 use std::borrow::{Borrow, BorrowMut};
 
+use common::BitSet;
+
 use crate::fastfield::AliveBitSet;
 use crate::DocId;
 
@@ -106,6 +108,15 @@ pub trait DocSet: Send {
         buffer.len()
     }
 
+    /// TODO comment on the size of the bitset
+    fn fill_bitset(&mut self, bitset: &mut BitSet) {
+        let mut doc = self.doc();
+        while doc != TERMINATED {
+            bitset.insert(doc);
+            doc = self.advance();
+        }
+    }
+
     /// Returns the current document
     /// Right after creating a new `DocSet`, the docset points to the first document.
     ///

src/fastfield/alive_bitset.rs

@@ -162,7 +162,7 @@ mod tests {
 mod bench {
 
     use rand::prelude::IteratorRandom;
-    use rand::rng;
+    use rand::thread_rng;
     use test::Bencher;
 
     use super::AliveBitSet;
@@ -176,7 +176,7 @@ mod bench {
     }
 
     fn remove_rand(raw: &mut Vec<u32>) {
-        let i = (0..raw.len()).choose(&mut rng()).unwrap();
+        let i = (0..raw.len()).choose(&mut thread_rng()).unwrap();
         raw.remove(i);
     }
 

src/fastfield/mod.rs

@@ -879,7 +879,7 @@ mod tests {
         const ONE_HOUR_IN_MICROSECS: i64 = 3_600 * 1_000_000;
         let times: Vec<DateTime> = std::iter::repeat_with(|| {
             // +- One hour.
-            let t = T0 + rng.random_range(-ONE_HOUR_IN_MICROSECS..ONE_HOUR_IN_MICROSECS);
+            let t = T0 + rng.gen_range(-ONE_HOUR_IN_MICROSECS..ONE_HOUR_IN_MICROSECS);
             DateTime::from_timestamp_micros(t)
         })
         .take(1_000)

src/functional_test.rs

@@ -1,6 +1,6 @@
 use std::collections::HashSet;
 
-use rand::{rng, Rng};
+use rand::{thread_rng, Rng};
 
 use crate::indexer::index_writer::MEMORY_BUDGET_NUM_BYTES_MIN;
 use crate::schema::*;
@@ -29,7 +29,7 @@ fn test_functional_store() -> crate::Result<()> {
     let index = Index::create_in_ram(schema);
     let reader = index.reader()?;
 
-    let mut rng = rng();
+    let mut rng = thread_rng();
 
     let mut index_writer: IndexWriter =
         index.writer_with_num_threads(3, 3 * MEMORY_BUDGET_NUM_BYTES_MIN)?;
@@ -38,9 +38,9 @@ fn test_functional_store() -> crate::Result<()> {
 
     let mut doc_id = 0u64;
     for _iteration in 0..get_num_iterations() {
-        let num_docs: usize = rng.random_range(0..4);
+        let num_docs: usize = rng.gen_range(0..4);
         if !doc_set.is_empty() {
-            let doc_to_remove_id = rng.random_range(0..doc_set.len());
+            let doc_to_remove_id = rng.gen_range(0..doc_set.len());
             let removed_doc_id = doc_set.swap_remove(doc_to_remove_id);
             index_writer.delete_term(Term::from_field_u64(id_field, removed_doc_id));
         }
@@ -70,10 +70,10 @@ const LOREM: &str = "Doc Lorem ipsum dolor sit amet, consectetur adipiscing elit
                      cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat \
                      non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.";
 fn get_text() -> String {
-    use rand::seq::IndexedRandom;
-    let mut rng = rng();
+    use rand::seq::SliceRandom;
+    let mut rng = thread_rng();
     let tokens: Vec<_> = LOREM.split(' ').collect();
-    let random_val = rng.random_range(0..20);
+    let random_val = rng.gen_range(0..20);
 
     (0..random_val)
         .map(|_| tokens.choose(&mut rng).unwrap())
@@ -101,7 +101,7 @@ fn test_functional_indexing_unsorted() -> crate::Result<()> {
     let index = Index::create_from_tempdir(schema)?;
     let reader = index.reader()?;
 
-    let mut rng = rng();
+    let mut rng = thread_rng();
 
     let mut index_writer: IndexWriter =
         index.writer_with_num_threads(3, 3 * MEMORY_BUDGET_NUM_BYTES_MIN)?;
@@ -110,7 +110,7 @@ fn test_functional_indexing_unsorted() -> crate::Result<()> {
     let mut uncommitted_docs: HashSet<u64> = HashSet::new();
 
     for _ in 0..get_num_iterations() {
-        let random_val = rng.random_range(0..20);
+        let random_val = rng.gen_range(0..20);
         if random_val == 0 {
             index_writer.commit()?;
             committed_docs.extend(&uncommitted_docs);

src/index/index.rs

@@ -8,6 +8,7 @@ use std::thread::available_parallelism;
 use super::segment::Segment;
 use super::segment_reader::merge_field_meta_data;
 use super::{FieldMetadata, IndexSettings};
+use crate::codec::{CodecConfiguration, StandardCodec};
 use crate::core::{Executor, META_FILEPATH};
 use crate::directory::error::OpenReadError;
 #[cfg(feature = "mmap")]
@@ -59,6 +60,7 @@ fn save_new_metas(
     schema: Schema,
     index_settings: IndexSettings,
     directory: &dyn Directory,
+    codec: CodecConfiguration,
 ) -> crate::Result<()> {
     save_metas(
         &IndexMeta {
@@ -67,6 +69,7 @@ fn save_new_metas(
             schema,
             opstamp: 0u64,
             payload: None,
+            codec,
         },
         directory,
     )?;
@@ -101,18 +104,21 @@ fn save_new_metas(
 /// };
 /// let index = Index::builder().schema(schema).settings(settings).create_in_ram();
 /// ```
-pub struct IndexBuilder {
+pub struct IndexBuilder<Codec: crate::codec::Codec = StandardCodec> {
     schema: Option<Schema>,
     index_settings: IndexSettings,
     tokenizer_manager: TokenizerManager,
     fast_field_tokenizer_manager: TokenizerManager,
+    codec: Codec,
 }
-impl Default for IndexBuilder {
+
+impl Default for IndexBuilder<StandardCodec> {
     fn default() -> Self {
         IndexBuilder::new()
     }
 }
-impl IndexBuilder {
+
+impl IndexBuilder<StandardCodec> {
     /// Creates a new `IndexBuilder`
     pub fn new() -> Self {
         Self {
@@ -120,6 +126,21 @@ impl IndexBuilder {
             index_settings: IndexSettings::default(),
             tokenizer_manager: TokenizerManager::default(),
             fast_field_tokenizer_manager: TokenizerManager::default(),
+            codec: StandardCodec,
+        }
+    }
+}
+
+impl<Codec: crate::codec::Codec> IndexBuilder<Codec> {
+    /// Set the codec
+    #[must_use]
+    pub fn codec<NewCodec: crate::codec::Codec>(self, codec: NewCodec) -> IndexBuilder<NewCodec> {
+        IndexBuilder {
+            schema: self.schema,
+            index_settings: self.index_settings,
+            tokenizer_manager: self.tokenizer_manager,
+            fast_field_tokenizer_manager: self.fast_field_tokenizer_manager,
+            codec,
         }
     }
 
@@ -154,7 +175,7 @@ impl IndexBuilder {
     /// The index will be allocated in anonymous memory.
     /// This is useful for indexing small set of documents
     /// for instances like unit test or temporary in memory index.
-    pub fn create_in_ram(self) -> Result<Index, TantivyError> {
+    pub fn create_in_ram(self) -> Result<Index<Codec>, TantivyError> {
         let ram_directory = RamDirectory::create();
         self.create(ram_directory)
     }
@@ -165,7 +186,7 @@ impl IndexBuilder {
     /// If a previous index was in this directory, it returns an
     /// [`TantivyError::IndexAlreadyExists`] error.
     #[cfg(feature = "mmap")]
-    pub fn create_in_dir<P: AsRef<Path>>(self, directory_path: P) -> crate::Result<Index> {
+    pub fn create_in_dir<P: AsRef<Path>>(self, directory_path: P) -> crate::Result<Index<Codec>> {
         let mmap_directory: Box<dyn Directory> = Box::new(MmapDirectory::open(directory_path)?);
         if Index::exists(&*mmap_directory)? {
             return Err(TantivyError::IndexAlreadyExists);
@@ -186,7 +207,7 @@ impl IndexBuilder {
         self,
         dir: impl Into<Box<dyn Directory>>,
         mem_budget: usize,
-    ) -> crate::Result<SingleSegmentIndexWriter<D>> {
+    ) -> crate::Result<SingleSegmentIndexWriter<Codec, D>> {
         let index = self.create(dir)?;
         let index_simple_writer = SingleSegmentIndexWriter::new(index, mem_budget)?;
         Ok(index_simple_writer)
@@ -202,7 +223,7 @@ impl IndexBuilder {
     /// For other unit tests, prefer the [`RamDirectory`], see:
     /// [`IndexBuilder::create_in_ram()`].
     #[cfg(feature = "mmap")]
-    pub fn create_from_tempdir(self) -> crate::Result<Index> {
+    pub fn create_from_tempdir(self) -> crate::Result<Index<Codec>> {
         let mmap_directory: Box<dyn Directory> = Box::new(MmapDirectory::create_from_tempdir()?);
         self.create(mmap_directory)
     }
@@ -215,12 +236,15 @@ impl IndexBuilder {
     }
 
     /// Opens or creates a new index in the provided directory
-    pub fn open_or_create<T: Into<Box<dyn Directory>>>(self, dir: T) -> crate::Result<Index> {
+    pub fn open_or_create<T: Into<Box<dyn Directory>>>(
+        self,
+        dir: T,
+    ) -> crate::Result<Index<Codec>> {
         let dir: Box<dyn Directory> = dir.into();
         if !Index::exists(&*dir)? {
             return self.create(dir);
         }
-        let mut index = Index::open(dir)?;
+        let mut index: Index<Codec> = Index::<Codec>::open_with_codec(dir)?;
         index.set_tokenizers(self.tokenizer_manager.clone());
         if index.schema() == self.get_expect_schema()? {
             Ok(index)
@@ -244,18 +268,26 @@ impl IndexBuilder {
     /// Creates a new index given an implementation of the trait `Directory`.
     ///
     /// If a directory previously existed, it will be erased.
-    fn create<T: Into<Box<dyn Directory>>>(self, dir: T) -> crate::Result<Index> {
+    pub fn create<T: Into<Box<dyn Directory>>>(self, dir: T) -> crate::Result<Index<Codec>> {
+        self.create_avoid_monomorphization(dir.into())
+    }
+
+    fn create_avoid_monomorphization(self, dir: Box<dyn Directory>) -> crate::Result<Index<Codec>> {
         self.validate()?;
         let dir = dir.into();
         let directory = ManagedDirectory::wrap(dir)?;
+        let codec: CodecConfiguration = CodecConfiguration::from_codec(&self.codec);
         save_new_metas(
             self.get_expect_schema()?,
             self.index_settings.clone(),
             &directory,
+            codec,
         )?;
-        let mut metas = IndexMeta::with_schema(self.get_expect_schema()?);
+        let schema = self.get_expect_schema()?;
+        let mut metas = IndexMeta::with_schema_and_codec(schema, &self.codec);
         metas.index_settings = self.index_settings;
-        let mut index = Index::open_from_metas(directory, &metas, SegmentMetaInventory::default());
+        let mut index: Index<Codec> =
+            Index::<Codec>::open_from_metas(directory, &metas, SegmentMetaInventory::default())?;
         index.set_tokenizers(self.tokenizer_manager);
         index.set_fast_field_tokenizers(self.fast_field_tokenizer_manager);
         Ok(index)
@@ -264,7 +296,7 @@ impl IndexBuilder {
 
 /// Search Index
 #[derive(Clone)]
-pub struct Index {
+pub struct Index<Codec: crate::codec::Codec = crate::codec::StandardCodec> {
     directory: ManagedDirectory,
     schema: Schema,
     settings: IndexSettings,
@@ -272,6 +304,7 @@ pub struct Index {
     tokenizers: TokenizerManager,
     fast_field_tokenizers: TokenizerManager,
     inventory: SegmentMetaInventory,
+    codec: Codec,
 }
 
 impl Index {
@@ -279,41 +312,6 @@ impl Index {
     pub fn builder() -> IndexBuilder {
         IndexBuilder::new()
     }
-    /// Examines the directory to see if it contains an index.
-    ///
-    /// Effectively, it only checks for the presence of the `meta.json` file.
-    pub fn exists(dir: &dyn Directory) -> Result<bool, OpenReadError> {
-        dir.exists(&META_FILEPATH)
-    }
-
-    /// Accessor to the search executor.
-    ///
-    /// This pool is used by default when calling `searcher.search(...)`
-    /// to perform search on the individual segments.
-    ///
-    /// By default the executor is single thread, and simply runs in the calling thread.
-    pub fn search_executor(&self) -> &Executor {
-        &self.executor
-    }
-
-    /// Replace the default single thread search executor pool
-    /// by a thread pool with a given number of threads.
-    pub fn set_multithread_executor(&mut self, num_threads: usize) -> crate::Result<()> {
-        self.executor = Executor::multi_thread(num_threads, "tantivy-search-")?;
-        Ok(())
-    }
-
-    /// Custom thread pool by a outer thread pool.
-    pub fn set_executor(&mut self, executor: Executor) {
-        self.executor = executor;
-    }
-
-    /// Replace the default single thread search executor pool
-    /// by a thread pool with as many threads as there are CPUs on the system.
-    pub fn set_default_multithread_executor(&mut self) -> crate::Result<()> {
-        let default_num_threads = available_parallelism()?.get();
-        self.set_multithread_executor(default_num_threads)
-    }
 
     /// Creates a new index using the [`RamDirectory`].
     ///
@@ -324,6 +322,13 @@ impl Index {
         IndexBuilder::new().schema(schema).create_in_ram().unwrap()
     }
 
+    /// Examines the directory to see if it contains an index.
+    ///
+    /// Effectively, it only checks for the presence of the `meta.json` file.
+    pub fn exists(directory: &dyn Directory) -> Result<bool, OpenReadError> {
+        directory.exists(&META_FILEPATH)
+    }
+
     /// Creates a new index in a given filepath.
     /// The index will use the [`MmapDirectory`].
     ///
@@ -370,20 +375,107 @@ impl Index {
         schema: Schema,
         settings: IndexSettings,
     ) -> crate::Result<Index> {
-        let dir: Box<dyn Directory> = dir.into();
+        Self::create_to_avoid_monomorphization(dir.into(), schema, settings)
+    }
+
+    fn create_to_avoid_monomorphization(
+        dir: Box<dyn Directory>,
+        schema: Schema,
+        settings: IndexSettings,
+    ) -> crate::Result<Index> {
         let mut builder = IndexBuilder::new().schema(schema);
         builder = builder.settings(settings);
         builder.create(dir)
     }
 
+    /// Opens a new directory from an index path.
+    #[cfg(feature = "mmap")]
+    pub fn open_in_dir<P: AsRef<Path>>(directory_path: P) -> crate::Result<Index> {
+        Self::open_in_dir_to_avoid_monomorphization(directory_path.as_ref())
+    }
+
+    #[inline(never)]
+    fn open_in_dir_to_avoid_monomorphization(directory_path: &Path) -> crate::Result<Index> {
+        let mmap_directory = MmapDirectory::open(directory_path)?;
+        Index::open(mmap_directory)
+    }
+
+    /// Open the index using the provided directory
+    pub fn open<T: Into<Box<dyn Directory>>>(directory: T) -> crate::Result<Index> {
+        Index::<StandardCodec>::open_with_codec(directory.into())
+    }
+}
+
+impl<Codec: crate::codec::Codec> Index<Codec> {
+    /// Returns a version of this index with the standard codec.
+    /// This is useful when you need to pass the index to APIs that
+    /// don't care about the codec (e.g., for reading).
+    pub(crate) fn with_standard_codec(&self) -> Index<StandardCodec> {
+        Index {
+            directory: self.directory.clone(),
+            schema: self.schema.clone(),
+            settings: self.settings.clone(),
+            executor: self.executor.clone(),
+            tokenizers: self.tokenizers.clone(),
+            fast_field_tokenizers: self.fast_field_tokenizers.clone(),
+            inventory: self.inventory.clone(),
+            codec: StandardCodec::default(),
+        }
+    }
+
+    /// Open the index using the provided directory
+    #[inline(never)]
+    pub fn open_with_codec(directory: Box<dyn Directory>) -> crate::Result<Index<Codec>> {
+        let directory = ManagedDirectory::wrap(directory)?;
+        let inventory = SegmentMetaInventory::default();
+        let metas = load_metas(&directory, &inventory)?;
+        let index: Index<Codec> = Index::<Codec>::open_from_metas(directory, &metas, inventory)?;
+        Ok(index)
+    }
+
+    /// Accessor to the codec.
+    pub fn codec(&self) -> &Codec {
+        &self.codec
+    }
+
+    /// Accessor to the search executor.
+    ///
+    /// This pool is used by default when calling `searcher.search(...)`
+    /// to perform search on the individual segments.
+    ///
+    /// By default the executor is single thread, and simply runs in the calling thread.
+    pub fn search_executor(&self) -> &Executor {
+        &self.executor
+    }
+
+    /// Replace the default single thread search executor pool
+    /// by a thread pool with a given number of threads.
+    pub fn set_multithread_executor(&mut self, num_threads: usize) -> crate::Result<()> {
+        self.executor = Executor::multi_thread(num_threads, "tantivy-search-")?;
+        Ok(())
+    }
+
+    /// Custom thread pool by a outer thread pool.
+    pub fn set_executor(&mut self, executor: Executor) {
+        self.executor = executor;
+    }
+
+    /// Replace the default single thread search executor pool
+    /// by a thread pool with as many threads as there are CPUs on the system.
+    pub fn set_default_multithread_executor(&mut self) -> crate::Result<()> {
+        let default_num_threads = available_parallelism()?.get();
+        self.set_multithread_executor(default_num_threads)
+    }
+
     /// Creates a new index given a directory and an [`IndexMeta`].
-    fn open_from_metas(
+    fn open_from_metas<C: crate::codec::Codec>(
         directory: ManagedDirectory,
         metas: &IndexMeta,
         inventory: SegmentMetaInventory,
-    ) -> Index {
+    ) -> crate::Result<Index<C>> {
         let schema = metas.schema.clone();
-        Index {
+        let codec = metas.codec.to_codec::<C>()?;
+        Ok(Index {
             settings: metas.index_settings.clone(),
             directory,
             schema,
@@ -391,7 +483,8 @@ impl Index {
             fast_field_tokenizers: TokenizerManager::default(),
             executor: Executor::single_thread(),
             inventory,
-        }
+            codec,
+        })
     }
 
     /// Setter for the tokenizer manager.
@@ -447,7 +540,7 @@ impl Index {
     /// Create a default [`IndexReader`] for the given index.
     ///
     /// See [`Index.reader_builder()`].
-    pub fn reader(&self) -> crate::Result<IndexReader> {
+    pub fn reader(&self) -> crate::Result<IndexReader<Codec>> {
         self.reader_builder().try_into()
     }
 
@@ -455,17 +548,10 @@ impl Index {
     ///
     /// Most project should create at most one reader for a given index.
     /// This method is typically called only once per `Index` instance.
-    pub fn reader_builder(&self) -> IndexReaderBuilder {
+    pub fn reader_builder(&self) -> IndexReaderBuilder<Codec> {
         IndexReaderBuilder::new(self.clone())
     }
 
-    /// Opens a new directory from an index path.
-    #[cfg(feature = "mmap")]
-    pub fn open_in_dir<P: AsRef<Path>>(directory_path: P) -> crate::Result<Index> {
-        let mmap_directory = MmapDirectory::open(directory_path)?;
-        Index::open(mmap_directory)
-    }
-
     /// Returns the list of the segment metas tracked by the index.
     ///
     /// Such segments can of course be part of the index,
@@ -506,16 +592,6 @@ impl Index {
         self.inventory.new_segment_meta(segment_id, max_doc)
     }
 
-    /// Open the index using the provided directory
-    pub fn open<T: Into<Box<dyn Directory>>>(directory: T) -> crate::Result<Index> {
-        let directory = directory.into();
-        let directory = ManagedDirectory::wrap(directory)?;
-        let inventory = SegmentMetaInventory::default();
-        let metas = load_metas(&directory, &inventory)?;
-        let index = Index::open_from_metas(directory, &metas, inventory);
-        Ok(index)
-    }
-
     /// Reads the index meta file from the directory.
     pub fn load_metas(&self) -> crate::Result<IndexMeta> {
         load_metas(self.directory(), &self.inventory)
@@ -539,7 +615,7 @@ impl Index {
     pub fn writer_with_options<D: Document>(
         &self,
         options: IndexWriterOptions,
-    ) -> crate::Result<IndexWriter<D>> {
+    ) -> crate::Result<IndexWriter<Codec, D>> {
         let directory_lock = self
             .directory
             .acquire_lock(&INDEX_WRITER_LOCK)
@@ -581,7 +657,7 @@ impl Index {
         &self,
         num_threads: usize,
         overall_memory_budget_in_bytes: usize,
-    ) -> crate::Result<IndexWriter<D>> {
+    ) -> crate::Result<IndexWriter<Codec, D>> {
         let memory_arena_in_bytes_per_thread = overall_memory_budget_in_bytes / num_threads;
         let options = IndexWriterOptions::builder()
             .num_worker_threads(num_threads)
@@ -595,7 +671,7 @@ impl Index {
     /// That index writer only simply has a single thread and a memory budget of 15 MB.
     /// Using a single thread gives us a deterministic allocation of DocId.
     #[cfg(test)]
-    pub fn writer_for_tests<D: Document>(&self) -> crate::Result<IndexWriter<D>> {
+    pub fn writer_for_tests<D: Document>(&self) -> crate::Result<IndexWriter<Codec, D>> {
         self.writer_with_num_threads(1, MEMORY_BUDGET_NUM_BYTES_MIN)
     }
 
@@ -613,7 +689,7 @@ impl Index {
     pub fn writer<D: Document>(
         &self,
         memory_budget_in_bytes: usize,
-    ) -> crate::Result<IndexWriter<D>> {
+    ) -> crate::Result<IndexWriter<Codec, D>> {
         let mut num_threads = std::cmp::min(available_parallelism()?.get(), MAX_NUM_THREAD);
         let memory_budget_num_bytes_per_thread = memory_budget_in_bytes / num_threads;
         if memory_budget_num_bytes_per_thread < MEMORY_BUDGET_NUM_BYTES_MIN {
@@ -640,7 +716,7 @@ impl Index {
     }
 
     /// Returns the list of segments that are searchable
-    pub fn searchable_segments(&self) -> crate::Result<Vec<Segment>> {
+    pub fn searchable_segments(&self) -> crate::Result<Vec<Segment<Codec>>> {
         Ok(self
             .searchable_segment_metas()?
             .into_iter()
@@ -649,12 +725,12 @@ impl Index {
     }
 
     #[doc(hidden)]
-    pub fn segment(&self, segment_meta: SegmentMeta) -> Segment {
+    pub fn segment(&self, segment_meta: SegmentMeta) -> Segment<Codec> {
         Segment::for_index(self.clone(), segment_meta)
     }
 
     /// Creates a new segment.
-    pub fn new_segment(&self) -> Segment {
+    pub fn new_segment(&self) -> Segment<Codec> {
         let segment_meta = self
             .inventory
             .new_segment_meta(SegmentId::generate_random(), 0);

src/index/index_meta.rs

@@ -7,6 +7,7 @@ use std::sync::Arc;
 use serde::{Deserialize, Serialize};
 
 use super::SegmentComponent;
+use crate::codec::{Codec, CodecConfiguration};
 use crate::index::SegmentId;
 use crate::schema::Schema;
 use crate::store::Compressor;
@@ -320,6 +321,7 @@ pub struct IndexMeta {
     /// This payload is entirely unused by tantivy.
     #[serde(skip_serializing_if = "Option::is_none")]
     pub payload: Option<String>,
+    pub codec: CodecConfiguration,
 }
 
 #[derive(Deserialize, Debug)]
@@ -331,6 +333,8 @@ struct UntrackedIndexMeta {
     pub opstamp: Opstamp,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub payload: Option<String>,
+    #[serde(default)]
+    pub codec: CodecConfiguration,
 }
 
 impl UntrackedIndexMeta {
@@ -345,6 +349,7 @@ impl UntrackedIndexMeta {
             schema: self.schema,
             opstamp: self.opstamp,
             payload: self.payload,
+            codec: self.codec,
         }
     }
 }
@@ -355,13 +360,14 @@ impl IndexMeta {
     ///
     /// This new index does not contains any segments.
     /// Opstamp will the value `0u64`.
-    pub fn with_schema(schema: Schema) -> IndexMeta {
+    pub fn with_schema_and_codec<C: Codec>(schema: Schema, codec: &C) -> IndexMeta {
         IndexMeta {
             index_settings: IndexSettings::default(),
             segments: vec![],
             schema,
             opstamp: 0u64,
             payload: None,
+            codec: CodecConfiguration::from_codec(codec),
         }
     }
 
@@ -412,11 +418,12 @@ mod tests {
             schema,
             opstamp: 0u64,
             payload: None,
+            codec: Default::default(),
         };
         let json = serde_json::ser::to_string(&index_metas).expect("serialization failed");
         assert_eq!(
             json,
-            r#"{"index_settings":{"docstore_compression":"none","docstore_blocksize":16384},"segments":[],"schema":[{"name":"text","type":"text","options":{"indexing":{"record":"position","fieldnorms":true,"tokenizer":"default"},"stored":false,"fast":false}}],"opstamp":0}"#
+            r#"{"index_settings":{"docstore_compression":"none","docstore_blocksize":16384},"segments":[],"schema":[{"name":"text","type":"text","options":{"indexing":{"record":"position","fieldnorms":true,"tokenizer":"default"},"stored":false,"fast":false}}],"opstamp":0,"codec":{"name":"standard"}}"#
         );
 
         let deser_meta: UntrackedIndexMeta = serde_json::from_str(&json).unwrap();

src/index/inverted_index_reader.rs

@@ -1,4 +1,5 @@
 use std::io;
+use std::sync::Arc;
 
 use common::json_path_writer::JSON_END_OF_PATH;
 use common::{BinarySerializable, ByteCount};
@@ -9,9 +10,9 @@ use itertools::Itertools;
 #[cfg(feature = "quickwit")]
 use tantivy_fst::automaton::{AlwaysMatch, Automaton};
 
+use crate::codec::{ObjectSafeCodec, StandardCodec};
 use crate::directory::FileSlice;
-use crate::positions::PositionReader;
-use crate::postings::{BlockSegmentPostings, SegmentPostings, TermInfo};
+use crate::postings::{Postings, TermInfo};
 use crate::schema::{IndexRecordOption, Term, Type};
 use crate::termdict::TermDictionary;
 
@@ -33,6 +34,7 @@ pub struct InvertedIndexReader {
     positions_file_slice: FileSlice,
     record_option: IndexRecordOption,
     total_num_tokens: u64,
+    codec: Arc<dyn ObjectSafeCodec>,
 }
 
 /// Object that records the amount of space used by a field in an inverted index.
@@ -68,6 +70,7 @@ impl InvertedIndexReader {
         postings_file_slice: FileSlice,
         positions_file_slice: FileSlice,
         record_option: IndexRecordOption,
+        codec: Arc<dyn ObjectSafeCodec>,
     ) -> io::Result<InvertedIndexReader> {
         let (total_num_tokens_slice, postings_body) = postings_file_slice.split(8);
         let total_num_tokens = u64::deserialize(&mut total_num_tokens_slice.read_bytes()?)?;
@@ -77,6 +80,7 @@ impl InvertedIndexReader {
             positions_file_slice,
             record_option,
             total_num_tokens,
+            codec,
         })
     }
 
@@ -89,6 +93,7 @@ impl InvertedIndexReader {
             positions_file_slice: FileSlice::empty(),
             record_option,
             total_num_tokens: 0u64,
+            codec: Arc::new(StandardCodec),
         }
     }
 
@@ -160,62 +165,26 @@ impl InvertedIndexReader {
         Ok(fields)
     }
 
-    /// Resets the block segment to another position of the postings
-    /// file.
-    ///
-    /// This is useful for enumerating through a list of terms,
-    /// and consuming the associated posting lists while avoiding
-    /// reallocating a [`BlockSegmentPostings`].
-    ///
-    /// # Warning
-    ///
-    /// This does not reset the positions list.
-    pub fn reset_block_postings_from_terminfo(
-        &self,
-        term_info: &TermInfo,
-        block_postings: &mut BlockSegmentPostings,
-    ) -> io::Result<()> {
-        let postings_slice = self
-            .postings_file_slice
-            .slice(term_info.postings_range.clone());
-        let postings_bytes = postings_slice.read_bytes()?;
-        block_postings.reset(term_info.doc_freq, postings_bytes)?;
-        Ok(())
-    }
-
-    /// Returns a block postings given a `Term`.
-    /// This method is for an advanced usage only.
-    ///
-    /// Most users should prefer using [`Self::read_postings()`] instead.
-    pub fn read_block_postings(
-        &self,
-        term: &Term,
-        option: IndexRecordOption,
-    ) -> io::Result<Option<BlockSegmentPostings>> {
-        self.get_term_info(term)?
-            .map(move |term_info| self.read_block_postings_from_terminfo(&term_info, option))
-            .transpose()
-    }
-
-    /// Returns a block postings given a `term_info`.
-    /// This method is for an advanced usage only.
-    ///
-    /// Most users should prefer using [`Self::read_postings()`] instead.
-    pub fn read_block_postings_from_terminfo(
-        &self,
-        term_info: &TermInfo,
-        requested_option: IndexRecordOption,
-    ) -> io::Result<BlockSegmentPostings> {
-        let postings_data = self
-            .postings_file_slice
-            .slice(term_info.postings_range.clone());
-        BlockSegmentPostings::open(
-            term_info.doc_freq,
-            postings_data,
-            self.record_option,
-            requested_option,
-        )
-    }
+    // /// Returns a block postings given a `term_info`.
+    // /// This method is for an advanced usage only.
+    // ///
+    // /// Most users should prefer using [`Self::read_postings()`] instead.
+    // pub fn read_block_postings_from_terminfo(
+    //     &self,
+    //     term_info: &TermInfo,
+    //     requested_option: IndexRecordOption,
+    // ) -> io::Result<BlockSegmentPostings> {
+    //     let postings_data = self
+    //         .postings_file_slice
+    //         .slice(term_info.postings_range.clone())
+    //         .read_bytes()?;
+    //     BlockSegmentPostings::open(
+    //         term_info.doc_freq,
+    //         postings_data,
+    //         self.record_option,
+    //         requested_option,
+    //     )
+    // }
 
     /// Returns a posting object given a `term_info`.
     /// This method is for an advanced usage only.
@@ -225,25 +194,32 @@ impl InvertedIndexReader {
         &self,
         term_info: &TermInfo,
         option: IndexRecordOption,
-    ) -> io::Result<SegmentPostings> {
+    ) -> io::Result<Box<dyn Postings>> {
         let option = option.downgrade(self.record_option);
 
-        let block_postings = self.read_block_postings_from_terminfo(term_info, option)?;
-        let position_reader = {
+        let postings_data = self
+            .postings_file_slice
+            .slice(term_info.postings_range.clone())
+            .read_bytes()?;
+        let positions_data = {
             if option.has_positions() {
                 let positions_data = self
                     .positions_file_slice
-                    .read_bytes_slice(term_info.positions_range.clone())?;
-                let position_reader = PositionReader::open(positions_data)?;
-                Some(position_reader)
+                    .slice(term_info.positions_range.clone())
+                    .read_bytes()?;
+                Some(positions_data)
             } else {
                 None
             }
         };
-        Ok(SegmentPostings::from_block_postings(
-            block_postings,
-            position_reader,
-        ))
+        let postings = self.codec.load_postings_type_erased(
+            term_info.doc_freq,
+            postings_data,
+            self.record_option,
+            option,
+            positions_data,
+        )?;
+        Ok(postings)
     }
 
     /// Returns the total number of tokens recorded for all documents
@@ -266,7 +242,7 @@ impl InvertedIndexReader {
         &self,
         term: &Term,
         option: IndexRecordOption,
-    ) -> io::Result<Option<SegmentPostings>> {
+    ) -> io::Result<Option<Box<dyn Postings>>> {
         self.get_term_info(term)?
             .map(move |term_info| self.read_postings_from_terminfo(&term_info, option))
             .transpose()

src/index/segment.rs

@@ -2,6 +2,7 @@ use std::fmt;
 use std::path::PathBuf;
 
 use super::SegmentComponent;
+use crate::codec::StandardCodec;
 use crate::directory::error::{OpenReadError, OpenWriteError};
 use crate::directory::{Directory, FileSlice, WritePtr};
 use crate::index::{Index, SegmentId, SegmentMeta};
@@ -10,25 +11,25 @@ use crate::Opstamp;
 
 /// A segment is a piece of the index.
 #[derive(Clone)]
-pub struct Segment {
-    index: Index,
+pub struct Segment<C: crate::codec::Codec = StandardCodec> {
+    index: Index<C>,
     meta: SegmentMeta,
 }
 
-impl fmt::Debug for Segment {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+impl<C: crate::codec::Codec> fmt::Debug for Segment<C> {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         write!(f, "Segment({:?})", self.id().uuid_string())
     }
 }
 
-impl Segment {
+impl<C: crate::codec::Codec> Segment<C> {
     /// Creates a new segment given an `Index` and a `SegmentId`
-    pub(crate) fn for_index(index: Index, meta: SegmentMeta) -> Segment {
+    pub(crate) fn for_index(index: Index<C>, meta: SegmentMeta) -> Segment<C> {
         Segment { index, meta }
     }
 
     /// Returns the index the segment belongs to.
-    pub fn index(&self) -> &Index {
+    pub fn index(&self) -> &Index<C> {
         &self.index
     }
 
@@ -46,7 +47,7 @@ impl Segment {
     ///
     /// This method is only used when updating `max_doc` from 0
     /// as we finalize a fresh new segment.
-    pub fn with_max_doc(self, max_doc: u32) -> Segment {
+    pub fn with_max_doc(self, max_doc: u32) -> Segment<C> {
         Segment {
             index: self.index,
             meta: self.meta.with_max_doc(max_doc),
@@ -55,7 +56,7 @@ impl Segment {
 
     #[doc(hidden)]
     #[must_use]
-    pub fn with_delete_meta(self, num_deleted_docs: u32, opstamp: Opstamp) -> Segment {
+    pub fn with_delete_meta(self, num_deleted_docs: u32, opstamp: Opstamp) -> Segment<C> {
         Segment {
             index: self.index,
             meta: self.meta.with_delete_meta(num_deleted_docs, opstamp),

src/index/segment_reader.rs

@@ -6,6 +6,7 @@ use common::{ByteCount, HasLen};
 use fnv::FnvHashMap;
 use itertools::Itertools;
 
+use crate::codec::ObjectSafeCodec;
 use crate::directory::{CompositeFile, FileSlice};
 use crate::error::DataCorruption;
 use crate::fastfield::{intersect_alive_bitsets, AliveBitSet, FacetReader, FastFieldReaders};
@@ -47,6 +48,8 @@ pub struct SegmentReader {
     store_file: FileSlice,
     alive_bitset_opt: Option<AliveBitSet>,
     schema: Schema,
+
+    codec: Arc<dyn ObjectSafeCodec>,
 }
 
 impl SegmentReader {
@@ -140,15 +143,16 @@ impl SegmentReader {
     }
 
     /// Open a new segment for reading.
-    pub fn open(segment: &Segment) -> crate::Result<SegmentReader> {
+    pub fn open<C: crate::codec::Codec>(segment: &Segment<C>) -> crate::Result<SegmentReader> {
         Self::open_with_custom_alive_set(segment, None)
     }
 
     /// Open a new segment for reading.
-    pub fn open_with_custom_alive_set(
-        segment: &Segment,
+    pub fn open_with_custom_alive_set<C: crate::codec::Codec>(
+        segment: &Segment<C>,
         custom_bitset: Option<AliveBitSet>,
     ) -> crate::Result<SegmentReader> {
+        let codec: Arc<dyn ObjectSafeCodec> = Arc::new(segment.index().codec().clone());
         let termdict_file = segment.open_read(SegmentComponent::Terms)?;
         let termdict_composite = CompositeFile::open(&termdict_file)?;
 
@@ -204,6 +208,7 @@ impl SegmentReader {
             alive_bitset_opt,
             positions_composite,
             schema,
+            codec,
         })
     }
 
@@ -273,6 +278,7 @@ impl SegmentReader {
             postings_file,
             positions_file,
             record_option,
+            self.codec.clone(),
         )?);
 
         // by releasing the lock in between, we may end up opening the inverting index

src/indexer/index_writer.rs

@@ -9,6 +9,7 @@ use smallvec::smallvec;
 use super::operation::{AddOperation, UserOperation};
 use super::segment_updater::SegmentUpdater;
 use super::{AddBatch, AddBatchReceiver, AddBatchSender, PreparedCommit};
+use crate::codec::{Codec, StandardCodec};
 use crate::directory::{DirectoryLock, GarbageCollectionResult, TerminatingWrite};
 use crate::error::TantivyError;
 use crate::fastfield::write_alive_bitset;
@@ -68,12 +69,12 @@ pub struct IndexWriterOptions {
 /// indexing queue.
 /// Each indexing thread builds its own independent [`Segment`], via
 /// a `SegmentWriter` object.
-pub struct IndexWriter<D: Document = TantivyDocument> {
+pub struct IndexWriter<C: Codec = StandardCodec, D: Document = TantivyDocument> {
     // the lock is just used to bind the
     // lifetime of the lock with that of the IndexWriter.
     _directory_lock: Option<DirectoryLock>,
 
-    index: Index,
+    index: Index<C>,
 
     options: IndexWriterOptions,
 
@@ -82,7 +83,7 @@ pub struct IndexWriter<D: Document = TantivyDocument> {
     index_writer_status: IndexWriterStatus<D>,
     operation_sender: AddBatchSender<D>,
 
-    segment_updater: SegmentUpdater,
+    segment_updater: SegmentUpdater<C>,
 
     worker_id: usize,
 
@@ -128,8 +129,8 @@ fn compute_deleted_bitset(
 /// is `==` target_opstamp.
 /// For instance, there was no delete operation between the state of the `segment_entry` and
 /// the `target_opstamp`, `segment_entry` is not updated.
-pub fn advance_deletes(
-    mut segment: Segment,
+pub fn advance_deletes<C: Codec>(
+    mut segment: Segment<C>,
     segment_entry: &mut SegmentEntry,
     target_opstamp: Opstamp,
 ) -> crate::Result<()> {
@@ -179,11 +180,11 @@ pub fn advance_deletes(
     Ok(())
 }
 
-fn index_documents<D: Document>(
+fn index_documents<C: crate::codec::Codec, D: Document>(
     memory_budget: usize,
-    segment: Segment,
+    segment: Segment<C>,
     grouped_document_iterator: &mut dyn Iterator<Item = AddBatch<D>>,
-    segment_updater: &SegmentUpdater,
+    segment_updater: &SegmentUpdater<C>,
     mut delete_cursor: DeleteCursor,
 ) -> crate::Result<()> {
     let mut segment_writer = SegmentWriter::for_segment(memory_budget, segment.clone())?;
@@ -226,8 +227,8 @@ fn index_documents<D: Document>(
 }
 
 /// `doc_opstamps` is required to be non-empty.
-fn apply_deletes(
-    segment: &Segment,
+fn apply_deletes<C: crate::codec::Codec>(
+    segment: &Segment<C>,
     delete_cursor: &mut DeleteCursor,
     doc_opstamps: &[Opstamp],
 ) -> crate::Result<Option<BitSet>> {
@@ -262,7 +263,7 @@ fn apply_deletes(
     })
 }
 
-impl<D: Document> IndexWriter<D> {
+impl<C: Codec, D: Document> IndexWriter<C, D> {
     /// Create a new index writer. Attempts to acquire a lockfile.
     ///
     /// The lockfile should be deleted on drop, but it is possible
@@ -278,7 +279,7 @@ impl<D: Document> IndexWriter<D> {
     /// If the memory arena per thread is too small or too big, returns
     /// `TantivyError::InvalidArgument`
     pub(crate) fn new(
-        index: &Index,
+        index: &Index<C>,
         options: IndexWriterOptions,
         directory_lock: DirectoryLock,
     ) -> crate::Result<Self> {
@@ -345,7 +346,7 @@ impl<D: Document> IndexWriter<D> {
     }
 
     /// Accessor to the index.
-    pub fn index(&self) -> &Index {
+    pub fn index(&self) -> &Index<C> {
         &self.index
     }
 
@@ -393,7 +394,7 @@ impl<D: Document> IndexWriter<D> {
     /// It is safe to start writing file associated with the new `Segment`.
     /// These will not be garbage collected as long as an instance object of
     /// `SegmentMeta` object associated with the new `Segment` is "alive".
-    pub fn new_segment(&self) -> Segment {
+    pub fn new_segment(&self) -> Segment<C> {
         self.index.new_segment()
     }
 
@@ -615,7 +616,7 @@ impl<D: Document> IndexWriter<D> {
     /// It is also possible to add a payload to the `commit`
     /// using this API.
     /// See [`PreparedCommit::set_payload()`].
-    pub fn prepare_commit(&mut self) -> crate::Result<PreparedCommit<'_, D>> {
+    pub fn prepare_commit(&mut self) -> crate::Result<PreparedCommit<'_, C, D>> {
         // Here, because we join all of the worker threads,
         // all of the segment update for this commit have been
         // sent.
@@ -665,7 +666,7 @@ impl<D: Document> IndexWriter<D> {
         self.prepare_commit()?.commit()
     }
 
-    pub(crate) fn segment_updater(&self) -> &SegmentUpdater {
+    pub(crate) fn segment_updater(&self) -> &SegmentUpdater<C> {
         &self.segment_updater
     }
 
@@ -804,7 +805,7 @@ impl<D: Document> IndexWriter<D> {
     }
 }
 
-impl<D: Document> Drop for IndexWriter<D> {
+impl<C: Codec, D: Document> Drop for IndexWriter<C, D> {
     fn drop(&mut self) {
         self.segment_updater.kill();
         self.drop_sender();

src/indexer/merger.rs

@@ -7,6 +7,7 @@ use common::ReadOnlyBitSet;
 use itertools::Itertools;
 use measure_time::debug_time;
 
+use crate::codec::{Codec, StandardCodec};
 use crate::directory::WritePtr;
 use crate::docset::{DocSet, TERMINATED};
 use crate::error::DataCorruption;
@@ -15,7 +16,7 @@ use crate::fieldnorm::{FieldNormReader, FieldNormReaders, FieldNormsSerializer,
 use crate::index::{Segment, SegmentComponent, SegmentReader};
 use crate::indexer::doc_id_mapping::{MappingType, SegmentDocIdMapping};
 use crate::indexer::SegmentSerializer;
-use crate::postings::{InvertedIndexSerializer, Postings, SegmentPostings};
+use crate::postings::{FreqReadingOption, InvertedIndexSerializer, Postings};
 use crate::schema::{value_type_to_column_type, Field, FieldType, Schema};
 use crate::store::StoreWriter;
 use crate::termdict::{TermMerger, TermOrdinal};
@@ -76,10 +77,11 @@ fn estimate_total_num_tokens(readers: &[SegmentReader], field: Field) -> crate::
     Ok(total_num_tokens)
 }
 
-pub struct IndexMerger {
+pub struct IndexMerger<C: Codec = StandardCodec> {
     schema: Schema,
     pub(crate) readers: Vec<SegmentReader>,
     max_doc: u32,
+    codec: C,
 }
 
 struct DeltaComputer {
@@ -144,8 +146,8 @@ fn extract_fast_field_required_columns(schema: &Schema) -> Vec<(String, ColumnTy
         .collect()
 }
 
-impl IndexMerger {
-    pub fn open(schema: Schema, segments: &[Segment]) -> crate::Result<IndexMerger> {
+impl<C: Codec> IndexMerger<C> {
+    pub fn open(schema: Schema, segments: &[Segment<C>]) -> crate::Result<IndexMerger<C>> {
         let alive_bitset = segments.iter().map(|_| None).collect_vec();
         Self::open_with_custom_alive_set(schema, segments, alive_bitset)
     }
@@ -162,11 +164,15 @@ impl IndexMerger {
     // This can be used to merge but also apply an additional filter.
     // One use case is demux, which is basically taking a list of
     // segments and partitions them e.g. by a value in a field.
+    //
+    // # Panics if segments is empty.
     pub fn open_with_custom_alive_set(
         schema: Schema,
-        segments: &[Segment],
+        segments: &[Segment<C>],
         alive_bitset_opt: Vec<Option<AliveBitSet>>,
-    ) -> crate::Result<IndexMerger> {
+    ) -> crate::Result<IndexMerger<C>> {
+        assert!(!segments.is_empty());
+        let codec = segments[0].index().codec().clone();
         let mut readers = vec![];
         for (segment, new_alive_bitset_opt) in segments.iter().zip(alive_bitset_opt) {
             if segment.meta().num_docs() > 0 {
@@ -189,6 +195,7 @@ impl IndexMerger {
             schema,
             readers,
             max_doc,
+            codec,
         })
     }
 
@@ -287,7 +294,7 @@ impl IndexMerger {
         &self,
         indexed_field: Field,
         _field_type: &FieldType,
-        serializer: &mut InvertedIndexSerializer,
+        serializer: &mut InvertedIndexSerializer<C>,
         fieldnorm_reader: Option<FieldNormReader>,
         doc_id_mapping: &SegmentDocIdMapping,
     ) -> crate::Result<()> {
@@ -355,7 +362,7 @@ impl IndexMerger {
                          indexed. Have you modified the schema?",
         );
 
-        let mut segment_postings_containing_the_term: Vec<(usize, SegmentPostings)> = vec![];
+        let mut segment_postings_containing_the_term: Vec<(usize, Box<dyn Postings>)> = vec![];
 
         while merged_terms.advance() {
             segment_postings_containing_the_term.clear();
@@ -395,11 +402,10 @@ impl IndexMerger {
             assert!(!segment_postings_containing_the_term.is_empty());
 
             let has_term_freq = {
-                let has_term_freq = !segment_postings_containing_the_term[0]
+                let has_term_freq = segment_postings_containing_the_term[0]
                     .1
-                    .block_cursor
-                    .freqs()
-                    .is_empty();
+                    .freq_reading_option()
+                    == FreqReadingOption::ReadFreq;
                 for (_, postings) in &segment_postings_containing_the_term[1..] {
                     // This may look at a strange way to test whether we have term freq or not.
                     // With JSON object, the schema is not sufficient to know whether a term
@@ -415,7 +421,7 @@ impl IndexMerger {
                     //
                     // Overall the reliable way to know if we have actual frequencies loaded or not
                     // is to check whether the actual decoded array is empty or not.
-                    if has_term_freq == postings.block_cursor.freqs().is_empty() {
+                    if postings.freq_reading_option() != FreqReadingOption::ReadFreq {
                         return Err(DataCorruption::comment_only(
                             "Term freqs are inconsistent across segments",
                         )
@@ -467,7 +473,7 @@ impl IndexMerger {
 
     fn write_postings(
         &self,
-        serializer: &mut InvertedIndexSerializer,
+        serializer: &mut InvertedIndexSerializer<C>,
         fieldnorm_readers: FieldNormReaders,
         doc_id_mapping: &SegmentDocIdMapping,
     ) -> crate::Result<()> {
@@ -525,7 +531,7 @@ impl IndexMerger {
     ///
     /// # Returns
     /// The number of documents in the resulting segment.
-    pub fn write(&self, mut serializer: SegmentSerializer) -> crate::Result<u32> {
+    pub fn write(&self, mut serializer: SegmentSerializer<C>) -> crate::Result<u32> {
         let doc_id_mapping = self.get_doc_id_from_concatenated_data()?;
         debug!("write-fieldnorms");
         if let Some(fieldnorms_serializer) = serializer.extract_fieldnorms_serializer() {
@@ -1521,7 +1527,7 @@ mod tests {
             .term_scorer_for_test(searcher.segment_reader(0u32), 1.0)?
             .unwrap();
         assert_eq!(term_scorer.doc(), 0);
-        assert_nearly_equals!(term_scorer.block_max_score(), 0.0079681855);
+        assert_nearly_equals!(term_scorer.seek_block(0), 0.0079681855);
         assert_nearly_equals!(term_scorer.score(), 0.0079681855);
         for _ in 0..81 {
             writer.add_document(doc!(text=>"hello happy tax payer"))?;
@@ -1540,7 +1546,7 @@ mod tests {
             // there.
             for doc in segment_reader.doc_ids_alive() {
                 assert_eq!(term_scorer.doc(), doc);
-                assert_nearly_equals!(term_scorer.block_max_score(), 0.003478312);
+                assert_nearly_equals!(term_scorer.seek_block(doc), 0.003478312);
                 assert_nearly_equals!(term_scorer.score(), 0.003478312);
                 term_scorer.advance();
             }
@@ -1565,7 +1571,7 @@ mod tests {
         // the difference compared to before is intrinsic to the bm25 formula. no worries there.
         for doc in segment_reader.doc_ids_alive() {
             assert_eq!(term_scorer.doc(), doc);
-            assert_nearly_equals!(term_scorer.block_max_score(), 0.003478312);
+            assert_nearly_equals!(term_scorer.seek_block(doc), 0.003478312);
             assert_nearly_equals!(term_scorer.score(), 0.003478312);
             term_scorer.advance();
         }

src/indexer/prepared_commit.rs

@@ -1,16 +1,17 @@
 use super::IndexWriter;
+use crate::codec::Codec;
 use crate::schema::document::Document;
 use crate::{FutureResult, Opstamp, TantivyDocument};
 
 /// A prepared commit
-pub struct PreparedCommit<'a, D: Document = TantivyDocument> {
-    index_writer: &'a mut IndexWriter<D>,
+pub struct PreparedCommit<'a, C: Codec, D: Document = TantivyDocument> {
+    index_writer: &'a mut IndexWriter<C, D>,
     payload: Option<String>,
     opstamp: Opstamp,
 }
 
-impl<'a, D: Document> PreparedCommit<'a, D> {
-    pub(crate) fn new(index_writer: &'a mut IndexWriter<D>, opstamp: Opstamp) -> Self {
+impl<'a, C: Codec, D: Document> PreparedCommit<'a, C, D> {
+    pub(crate) fn new(index_writer: &'a mut IndexWriter<C, D>, opstamp: Opstamp) -> Self {
         Self {
             index_writer,
             payload: None,

src/indexer/segment_serializer.rs

@@ -8,17 +8,17 @@ use crate::store::StoreWriter;
 
 /// Segment serializer is in charge of laying out on disk
 /// the data accumulated and sorted by the `SegmentWriter`.
-pub struct SegmentSerializer {
-    segment: Segment,
+pub struct SegmentSerializer<C: crate::codec::Codec> {
+    segment: Segment<C>,
     pub(crate) store_writer: StoreWriter,
     fast_field_write: WritePtr,
     fieldnorms_serializer: Option<FieldNormsSerializer>,
-    postings_serializer: InvertedIndexSerializer,
+    postings_serializer: InvertedIndexSerializer<C>,
 }
 
-impl SegmentSerializer {
+impl<C: crate::codec::Codec> SegmentSerializer<C> {
     /// Creates a new `SegmentSerializer`.
-    pub fn for_segment(mut segment: Segment) -> crate::Result<SegmentSerializer> {
+    pub fn for_segment(mut segment: Segment<C>) -> crate::Result<SegmentSerializer<C>> {
         let settings = segment.index().settings().clone();
         let store_writer = {
             let store_write = segment.open_write(SegmentComponent::Store)?;
@@ -50,12 +50,12 @@ impl SegmentSerializer {
         self.store_writer.mem_usage()
     }
 
-    pub fn segment(&self) -> &Segment {
+    pub fn segment(&self) -> &Segment<C> {
         &self.segment
     }
 
     /// Accessor to the `PostingsSerializer`.
-    pub fn get_postings_serializer(&mut self) -> &mut InvertedIndexSerializer {
+    pub fn get_postings_serializer(&mut self) -> &mut InvertedIndexSerializer<C> {
         &mut self.postings_serializer
     }
 

src/indexer/segment_updater.rs

@@ -10,6 +10,7 @@ use std::sync::{Arc, RwLock};
 use rayon::{ThreadPool, ThreadPoolBuilder};
 
 use super::segment_manager::SegmentManager;
+use crate::codec::{Codec, CodecConfiguration};
 use crate::core::META_FILEPATH;
 use crate::directory::{Directory, DirectoryClone, GarbageCollectionResult};
 use crate::fastfield::AliveBitSet;
@@ -61,10 +62,10 @@ pub(crate) fn save_metas(metas: &IndexMeta, directory: &dyn Directory) -> crate:
 // We voluntarily pass a merge_operation ref to guarantee that
 // the merge_operation is alive during the process
 #[derive(Clone)]
-pub(crate) struct SegmentUpdater(Arc<InnerSegmentUpdater>);
+pub(crate) struct SegmentUpdater<C: Codec>(Arc<InnerSegmentUpdater<C>>);
 
-impl Deref for SegmentUpdater {
-    type Target = InnerSegmentUpdater;
+impl<C: Codec> Deref for SegmentUpdater<C> {
+    type Target = InnerSegmentUpdater<C>;
 
     #[inline]
     fn deref(&self) -> &Self::Target {
@@ -72,8 +73,8 @@ impl Deref for SegmentUpdater {
     }
 }
 
-fn garbage_collect_files(
-    segment_updater: SegmentUpdater,
+fn garbage_collect_files<C: Codec>(
+    segment_updater: SegmentUpdater<C>,
 ) -> crate::Result<GarbageCollectionResult> {
     info!("Running garbage collection");
     let mut index = segment_updater.index.clone();
@@ -84,8 +85,8 @@ fn garbage_collect_files(
 
 /// Merges a list of segments the list of segment givens in the `segment_entries`.
 /// This function happens in the calling thread and is computationally expensive.
-fn merge(
-    index: &Index,
+fn merge<Codec: crate::codec::Codec>(
+    index: &Index<Codec>,
     mut segment_entries: Vec<SegmentEntry>,
     target_opstamp: Opstamp,
 ) -> crate::Result<Option<SegmentEntry>> {
@@ -108,13 +109,13 @@ fn merge(
 
     let delete_cursor = segment_entries[0].delete_cursor().clone();
 
-    let segments: Vec<Segment> = segment_entries
+    let segments: Vec<Segment<Codec>> = segment_entries
         .iter()
         .map(|segment_entry| index.segment(segment_entry.meta().clone()))
         .collect();
 
     // An IndexMerger is like a "view" of our merged segments.
-    let merger: IndexMerger = IndexMerger::open(index.schema(), &segments[..])?;
+    let merger: IndexMerger<Codec> = IndexMerger::open(index.schema(), &segments[..])?;
 
     // ... we just serialize this index merger in our new segment to merge the segments.
     let segment_serializer = SegmentSerializer::for_segment(merged_segment.clone())?;
@@ -139,10 +140,10 @@ fn merge(
 /// meant to work if you have an `IndexWriter` running for the origin indices, or
 /// the destination `Index`.
 #[doc(hidden)]
-pub fn merge_indices<T: Into<Box<dyn Directory>>>(
-    indices: &[Index],
-    output_directory: T,
-) -> crate::Result<Index> {
+pub fn merge_indices<Codec: crate::codec::Codec>(
+    indices: &[Index<Codec>],
+    output_directory: Box<dyn Directory>,
+) -> crate::Result<Index<Codec>> {
     if indices.is_empty() {
         // If there are no indices to merge, there is no need to do anything.
         return Err(crate::TantivyError::InvalidArgument(
@@ -163,7 +164,7 @@ pub fn merge_indices<T: Into<Box<dyn Directory>>>(
         ));
     }
 
-    let mut segments: Vec<Segment> = Vec::new();
+    let mut segments: Vec<Segment<Codec>> = Vec::new();
     for index in indices {
         segments.extend(index.searchable_segments()?);
     }
@@ -185,12 +186,12 @@ pub fn merge_indices<T: Into<Box<dyn Directory>>>(
 /// meant to work if you have an `IndexWriter` running for the origin indices, or
 /// the destination `Index`.
 #[doc(hidden)]
-pub fn merge_filtered_segments<T: Into<Box<dyn Directory>>>(
-    segments: &[Segment],
+pub fn merge_filtered_segments<C: crate::codec::Codec, T: Into<Box<dyn Directory>>>(
+    segments: &[Segment<C>],
     target_settings: IndexSettings,
     filter_doc_ids: Vec<Option<AliveBitSet>>,
     output_directory: T,
-) -> crate::Result<Index> {
+) -> crate::Result<Index<C>> {
     if segments.is_empty() {
         // If there are no indices to merge, there is no need to do anything.
         return Err(crate::TantivyError::InvalidArgument(
@@ -211,14 +212,15 @@ pub fn merge_filtered_segments<T: Into<Box<dyn Directory>>>(
         ));
     }
 
-    let mut merged_index = Index::create(
-        output_directory,
-        target_schema.clone(),
-        target_settings.clone(),
-    )?;
+    let mut merged_index: Index<C> = Index::builder()
+        .schema(target_schema.clone())
+        .codec(segments[0].index().codec().clone())
+        .settings(target_settings.clone())
+        .create(output_directory.into())?;
+
     let merged_segment = merged_index.new_segment();
     let merged_segment_id = merged_segment.id();
-    let merger: IndexMerger =
+    let merger: IndexMerger<C> =
         IndexMerger::open_with_custom_alive_set(merged_index.schema(), segments, filter_doc_ids)?;
     let segment_serializer = SegmentSerializer::for_segment(merged_segment)?;
     let num_docs = merger.write(segment_serializer)?;
@@ -235,6 +237,7 @@ pub fn merge_filtered_segments<T: Into<Box<dyn Directory>>>(
             ))
             .trim_end()
     );
+    let codec_configuration = CodecConfiguration::from_codec(segments[0].index().codec());
 
     let index_meta = IndexMeta {
         index_settings: target_settings, // index_settings of all segments should be the same
@@ -242,6 +245,7 @@ pub fn merge_filtered_segments<T: Into<Box<dyn Directory>>>(
         schema: target_schema,
         opstamp: 0u64,
         payload: Some(stats),
+        codec: codec_configuration,
     };
 
     // save the meta.json
@@ -250,7 +254,7 @@ pub fn merge_filtered_segments<T: Into<Box<dyn Directory>>>(
     Ok(merged_index)
 }
 
-pub(crate) struct InnerSegmentUpdater {
+pub(crate) struct InnerSegmentUpdater<C: Codec> {
     // we keep a copy of the current active IndexMeta to
     // avoid loading the file every time we need it in the
     // `SegmentUpdater`.
@@ -261,7 +265,7 @@ pub(crate) struct InnerSegmentUpdater {
     pool: ThreadPool,
     merge_thread_pool: ThreadPool,
 
-    index: Index,
+    index: Index<C>,
     segment_manager: SegmentManager,
     merge_policy: RwLock<Arc<dyn MergePolicy>>,
     killed: AtomicBool,
@@ -269,13 +273,13 @@ pub(crate) struct InnerSegmentUpdater {
     merge_operations: MergeOperationInventory,
 }
 
-impl SegmentUpdater {
+impl<Codec: crate::codec::Codec> SegmentUpdater<Codec> {
     pub fn create(
-        index: Index,
+        index: Index<Codec>,
         stamper: Stamper,
         delete_cursor: &DeleteCursor,
         num_merge_threads: usize,
-    ) -> crate::Result<SegmentUpdater> {
+    ) -> crate::Result<Self> {
         let segments = index.searchable_segment_metas()?;
         let segment_manager = SegmentManager::from_segments(segments, delete_cursor);
         let pool = ThreadPoolBuilder::new()
@@ -404,12 +408,14 @@ impl SegmentUpdater {
             //
             // Segment 1 from disk 1, Segment 1 from disk 2, etc.
             committed_segment_metas.sort_by_key(|segment_meta| -(segment_meta.max_doc() as i32));
+            let codec = CodecConfiguration::from_codec(index.codec());
             let index_meta = IndexMeta {
                 index_settings: index.settings().clone(),
                 segments: committed_segment_metas,
                 schema: index.schema(),
                 opstamp,
                 payload: commit_message,
+                codec,
             };
             // TODO add context to the error.
             save_metas(&index_meta, directory.box_clone().borrow_mut())?;
@@ -443,7 +449,7 @@ impl SegmentUpdater {
         opstamp: Opstamp,
         payload: Option<String>,
     ) -> FutureResult<Opstamp> {
-        let segment_updater: SegmentUpdater = self.clone();
+        let segment_updater: SegmentUpdater<Codec> = self.clone();
         self.schedule_task(move || {
             let segment_entries = segment_updater.purge_deletes(opstamp)?;
             segment_updater.segment_manager.commit(segment_entries);
@@ -702,6 +708,7 @@ impl SegmentUpdater {
 #[cfg(test)]
 mod tests {
     use super::merge_indices;
+    use crate::codec::StandardCodec;
     use crate::collector::TopDocs;
     use crate::directory::RamDirectory;
     use crate::fastfield::AliveBitSet;
@@ -915,7 +922,7 @@ mod tests {
 
     #[test]
     fn test_merge_empty_indices_array() {
-        let merge_result = merge_indices(&[], RamDirectory::default());
+        let merge_result = merge_indices::<StandardCodec>(&[], Box::new(RamDirectory::default()));
         assert!(merge_result.is_err());
     }
 
@@ -942,7 +949,10 @@ mod tests {
         };
 
         // mismatched schema index list
-        let result = merge_indices(&[first_index, second_index], RamDirectory::default());
+        let result = merge_indices(
+            &[first_index, second_index],
+            Box::new(RamDirectory::default()),
+        );
         assert!(result.is_err());
 
         Ok(())

src/indexer/segment_writer.rs

@@ -4,6 +4,7 @@ use itertools::Itertools;
 use tokenizer_api::BoxTokenStream;
 
 use super::operation::AddOperation;
+use crate::codec::Codec;
 use crate::fastfield::FastFieldsWriter;
 use crate::fieldnorm::{FieldNormReaders, FieldNormsWriter};
 use crate::index::{Segment, SegmentComponent};
@@ -12,7 +13,7 @@ use crate::indexer::segment_serializer::SegmentSerializer;
 use crate::json_utils::{index_json_value, IndexingPositionsPerPath};
 use crate::postings::{
     compute_table_memory_size, serialize_postings, IndexingContext, IndexingPosition,
-    PerFieldPostingsWriter, PostingsWriter,
+    PerFieldPostingsWriter, PostingsWriter, PostingsWriterEnum,
 };
 use crate::schema::document::{Document, Value};
 use crate::schema::{FieldEntry, FieldType, Schema, DATE_TIME_PRECISION_INDEXED};
@@ -45,11 +46,11 @@ fn compute_initial_table_size(per_thread_memory_budget: usize) -> crate::Result<
 ///
 /// They creates the postings list in anonymous memory.
 /// The segment is laid on disk when the segment gets `finalized`.
-pub struct SegmentWriter {
+pub struct SegmentWriter<Codec: crate::codec::Codec> {
     pub(crate) max_doc: DocId,
     pub(crate) ctx: IndexingContext,
     pub(crate) per_field_postings_writers: PerFieldPostingsWriter,
-    pub(crate) segment_serializer: SegmentSerializer,
+    pub(crate) segment_serializer: SegmentSerializer<Codec>,
     pub(crate) fast_field_writers: FastFieldsWriter,
     pub(crate) fieldnorms_writer: FieldNormsWriter,
     pub(crate) json_path_writer: JsonPathWriter,
@@ -60,7 +61,7 @@ pub struct SegmentWriter {
     schema: Schema,
 }
 
-impl SegmentWriter {
+impl<Codec: crate::codec::Codec> SegmentWriter<Codec> {
     /// Creates a new `SegmentWriter`
     ///
     /// The arguments are defined as follows
@@ -70,7 +71,10 @@ impl SegmentWriter {
     ///   behavior as a memory limit.
     /// - segment: The segment being written
     /// - schema
-    pub fn for_segment(memory_budget_in_bytes: usize, segment: Segment) -> crate::Result<Self> {
+    pub fn for_segment(
+        memory_budget_in_bytes: usize,
+        segment: Segment<Codec>,
+    ) -> crate::Result<Self> {
         let schema = segment.schema();
         let tokenizer_manager = segment.index().tokenizers().clone();
         let tokenizer_manager_fast_field = segment.index().fast_field_tokenizer().clone();
@@ -169,7 +173,7 @@ impl SegmentWriter {
             }
 
             let (term_buffer, ctx) = (&mut self.term_buffer, &mut self.ctx);
-            let postings_writer: &mut dyn PostingsWriter =
+            let postings_writer: &mut PostingsWriterEnum =
                 self.per_field_postings_writers.get_for_field_mut(field);
             term_buffer.clear_with_field(field);
 
@@ -386,13 +390,13 @@ impl SegmentWriter {
 /// to the `SegmentSerializer`.
 ///
 /// `doc_id_map` is used to map to the new doc_id order.
-fn remap_and_write(
+fn remap_and_write<C: Codec>(
     schema: Schema,
     per_field_postings_writers: &PerFieldPostingsWriter,
     ctx: IndexingContext,
     fast_field_writers: FastFieldsWriter,
     fieldnorms_writer: &FieldNormsWriter,
-    mut serializer: SegmentSerializer,
+    mut serializer: SegmentSerializer<C>,
 ) -> crate::Result<()> {
     debug!("remap-and-write");
     if let Some(fieldnorms_serializer) = serializer.extract_fieldnorms_serializer() {

src/indexer/single_segment_index_writer.rs

@@ -1,5 +1,6 @@
 use std::marker::PhantomData;
 
+use crate::codec::CodecConfiguration;
 use crate::indexer::operation::AddOperation;
 use crate::indexer::segment_updater::save_metas;
 use crate::indexer::SegmentWriter;
@@ -7,22 +8,22 @@ use crate::schema::document::Document;
 use crate::{Directory, Index, IndexMeta, Opstamp, Segment, TantivyDocument};
 
 #[doc(hidden)]
-pub struct SingleSegmentIndexWriter<D: Document = TantivyDocument> {
-    segment_writer: SegmentWriter,
-    segment: Segment,
+pub struct SingleSegmentIndexWriter<Codec: crate::codec::Codec, D: Document = TantivyDocument> {
+    segment_writer: SegmentWriter<Codec>,
+    segment: Segment<Codec>,
     opstamp: Opstamp,
-    _phantom: PhantomData<D>,
+    _doc: PhantomData<D>,
 }
 
-impl<D: Document> SingleSegmentIndexWriter<D> {
-    pub fn new(index: Index, mem_budget: usize) -> crate::Result<Self> {
+impl<Codec: crate::codec::Codec, D: Document> SingleSegmentIndexWriter<Codec, D> {
+    pub fn new(index: Index<Codec>, mem_budget: usize) -> crate::Result<Self> {
         let segment = index.new_segment();
         let segment_writer = SegmentWriter::for_segment(mem_budget, segment.clone())?;
         Ok(Self {
             segment_writer,
             segment,
             opstamp: 0,
-            _phantom: PhantomData,
+            _doc: PhantomData,
         })
     }
 
@@ -37,10 +38,10 @@ impl<D: Document> SingleSegmentIndexWriter<D> {
             .add_document(AddOperation { opstamp, document })
     }
 
-    pub fn finalize(self) -> crate::Result<Index> {
+    pub fn finalize(self) -> crate::Result<Index<Codec>> {
         let max_doc = self.segment_writer.max_doc();
         self.segment_writer.finalize()?;
-        let segment: Segment = self.segment.with_max_doc(max_doc);
+        let segment: Segment<Codec> = self.segment.with_max_doc(max_doc);
         let index = segment.index();
         let index_meta = IndexMeta {
             index_settings: index.settings().clone(),
@@ -48,6 +49,7 @@ impl<D: Document> SingleSegmentIndexWriter<D> {
             schema: index.schema(),
             opstamp: 0,
             payload: None,
+            codec: CodecConfiguration::from_codec(index.codec()),
         };
         save_metas(&index_meta, index.directory())?;
         index.directory().sync_directory()?;

src/lib.rs

@@ -166,6 +166,7 @@ mod functional_test;
 
 #[macro_use]
 mod macros;
+pub mod codec;
 mod future_result;
 
 // Re-exports
@@ -377,7 +378,7 @@ pub mod tests {
 
     use common::{BinarySerializable, FixedSize};
     use query_grammar::{UserInputAst, UserInputLeaf, UserInputLiteral};
-    use rand::distr::{Bernoulli, Uniform};
+    use rand::distributions::{Bernoulli, Uniform};
     use rand::rngs::StdRng;
     use rand::{Rng, SeedableRng};
     use time::OffsetDateTime;
@@ -428,7 +429,7 @@ pub mod tests {
     pub fn generate_nonunique_unsorted(max_value: u32, n_elems: usize) -> Vec<u32> {
         let seed: [u8; 32] = [1; 32];
         StdRng::from_seed(seed)
-            .sample_iter(&Uniform::new(0u32, max_value).unwrap())
+            .sample_iter(&Uniform::new(0u32, max_value))
             .take(n_elems)
             .collect::<Vec<u32>>()
     }

src/postings/compression/mod.rs

@@ -397,10 +397,7 @@ mod bench {
         let mut seed: [u8; 32] = [0; 32];
         seed[31] = seed_val;
         let mut rng = StdRng::from_seed(seed);
-        (0u32..)
-            .filter(|_| rng.random_bool(ratio))
-            .take(n)
-            .collect()
+        (0u32..).filter(|_| rng.gen_bool(ratio)).take(n).collect()
     }
 
     pub fn generate_array(n: usize, ratio: f64) -> Vec<u32> {

src/postings/json_postings_writer.rs

@@ -3,6 +3,7 @@ use std::io;
 use common::json_path_writer::JSON_END_OF_PATH;
 use stacker::Addr;
 
+use crate::codec::Codec;
 use crate::indexer::indexing_term::IndexingTerm;
 use crate::indexer::path_to_unordered_id::OrderedPathId;
 use crate::postings::postings_writer::SpecializedPostingsWriter;
@@ -22,12 +23,6 @@ pub(crate) struct JsonPostingsWriter<Rec: Recorder> {
     non_str_posting_writer: SpecializedPostingsWriter<DocIdRecorder>,
 }
 
-impl<Rec: Recorder> From<JsonPostingsWriter<Rec>> for Box<dyn PostingsWriter> {
-    fn from(json_postings_writer: JsonPostingsWriter<Rec>) -> Box<dyn PostingsWriter> {
-        Box::new(json_postings_writer)
-    }
-}
-
 impl<Rec: Recorder> PostingsWriter for JsonPostingsWriter<Rec> {
     #[inline]
     fn subscribe(
@@ -58,12 +53,12 @@ impl<Rec: Recorder> PostingsWriter for JsonPostingsWriter<Rec> {
     }
 
     /// The actual serialization format is handled by the `PostingsSerializer`.
-    fn serialize(
+    fn serialize<C: Codec>(
         &self,
         ordered_term_addrs: &[(Field, OrderedPathId, &[u8], Addr)],
         ordered_id_to_path: &[&str],
         ctx: &IndexingContext,
-        serializer: &mut FieldSerializer,
+        serializer: &mut FieldSerializer<C>,
     ) -> io::Result<()> {
         let mut term_buffer = JsonTermSerializer(Vec::with_capacity(48));
         let mut buffer_lender = BufferLender::default();

src/postings/loaded_postings.rs

@@ -1,5 +1,8 @@
 use crate::docset::{DocSet, TERMINATED};
-use crate::postings::{Postings, SegmentPostings};
+use crate::fieldnorm::FieldNormReader;
+use crate::postings::Postings;
+use crate::query::term_query::TermScorer;
+use crate::query::{Bm25Weight, Scorer};
 use crate::DocId;
 
 /// `LoadedPostings` is a `DocSet` and `Postings` implementation.
@@ -22,19 +25,27 @@ pub struct LoadedPostings {
 }
 
 impl LoadedPostings {
+    fn new_term_scorer(
+        self: Box<Self>,
+        fieldnorm_reader: FieldNormReader,
+        similarity_weight: Bm25Weight,
+    ) -> Box<dyn Scorer> {
+        Box::new(TermScorer::new(*self, fieldnorm_reader, similarity_weight))
+    }
+
     /// Creates a new `LoadedPostings` from a `SegmentPostings`.
     ///
     /// It will also preload positions, if positions are available in the SegmentPostings.
-    pub fn load(segment_postings: &mut SegmentPostings) -> LoadedPostings {
-        let num_docs = segment_postings.doc_freq() as usize;
+    pub fn load(postings: &mut Box<dyn Postings>) -> LoadedPostings {
+        let num_docs = postings.doc_freq() as usize;
         let mut doc_ids = Vec::with_capacity(num_docs);
         let mut positions = Vec::with_capacity(num_docs);
         let mut position_offsets = Vec::with_capacity(num_docs);
-        while segment_postings.doc() != TERMINATED {
+        while postings.doc() != TERMINATED {
             position_offsets.push(positions.len() as u32);
-            doc_ids.push(segment_postings.doc());
-            segment_postings.append_positions_with_offset(0, &mut positions);
-            segment_postings.advance();
+            doc_ids.push(postings.doc());
+            postings.append_positions_with_offset(0, &mut positions);
+            postings.advance();
         }
         position_offsets.push(positions.len() as u32);
         LoadedPostings {
@@ -88,6 +99,14 @@ impl DocSet for LoadedPostings {
     }
 }
 impl Postings for LoadedPostings {
+    fn new_term_scorer(
+        self: Box<Self>,
+        fieldnorm_reader: FieldNormReader,
+        similarity_weight: Bm25Weight,
+    ) -> Box<dyn Scorer> {
+        Box::new(TermScorer::new(*self, fieldnorm_reader, similarity_weight))
+    }
+
     fn term_freq(&self) -> u32 {
         let start = self.position_offsets[self.cursor] as usize;
         let end = self.position_offsets[self.cursor + 1] as usize;
@@ -101,6 +120,10 @@ impl Postings for LoadedPostings {
             output.push(*pos + offset);
         }
     }
+
+    fn freq_reading_option(&self) -> super::FreqReadingOption {
+        super::FreqReadingOption::ReadFreq
+    }
 }
 
 #[cfg(test)]

src/postings/mod.rs

@@ -4,7 +4,7 @@ mod block_search;
 
 pub(crate) use self::block_search::branchless_binary_search;
 
-mod block_segment_postings;
+// mod block_segment_postings;
 pub(crate) mod compression;
 mod indexing_context;
 mod json_postings_writer;
@@ -13,27 +13,24 @@ mod per_field_postings_writer;
 mod postings;
 mod postings_writer;
 mod recorder;
-mod segment_postings;
 mod serializer;
-mod skip;
 mod term_info;
 
 pub(crate) use loaded_postings::LoadedPostings;
 pub(crate) use stacker::compute_table_memory_size;
 
-pub use self::block_segment_postings::BlockSegmentPostings;
 pub(crate) use self::indexing_context::IndexingContext;
 pub(crate) use self::per_field_postings_writer::PerFieldPostingsWriter;
 pub use self::postings::Postings;
-pub(crate) use self::postings_writer::{serialize_postings, IndexingPosition, PostingsWriter};
-pub use self::segment_postings::SegmentPostings;
+pub(crate) use self::postings_writer::{
+    serialize_postings, IndexingPosition, PostingsWriter, PostingsWriterEnum,
+};
 pub use self::serializer::{FieldSerializer, InvertedIndexSerializer};
-pub(crate) use self::skip::{BlockInfo, SkipReader};
 pub use self::term_info::TermInfo;
 
 #[expect(clippy::enum_variant_names)]
 #[derive(Debug, PartialEq, Clone, Copy, Eq)]
-pub(crate) enum FreqReadingOption {
+pub enum FreqReadingOption {
     NoFreq,
     SkipFreq,
     ReadFreq,
@@ -604,13 +601,13 @@ mod bench {
             let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
             for _ in 0..posting_list_size {
                 let mut doc = TantivyDocument::default();
-                if rng.random_bool(1f64 / 15f64) {
+                if rng.gen_bool(1f64 / 15f64) {
                     doc.add_text(text_field, "a");
                 }
-                if rng.random_bool(1f64 / 10f64) {
+                if rng.gen_bool(1f64 / 10f64) {
                     doc.add_text(text_field, "b");
                 }
-                if rng.random_bool(1f64 / 5f64) {
+                if rng.gen_bool(1f64 / 5f64) {
                     doc.add_text(text_field, "c");
                 }
                 doc.add_text(text_field, "d");

src/postings/per_field_postings_writer.rs

@@ -1,16 +1,15 @@
 use crate::postings::json_postings_writer::JsonPostingsWriter;
-use crate::postings::postings_writer::SpecializedPostingsWriter;
+use crate::postings::postings_writer::{PostingsWriterEnum, SpecializedPostingsWriter};
 use crate::postings::recorder::{DocIdRecorder, TermFrequencyRecorder, TfAndPositionRecorder};
-use crate::postings::PostingsWriter;
 use crate::schema::{Field, FieldEntry, FieldType, IndexRecordOption, Schema};
 
 pub(crate) struct PerFieldPostingsWriter {
-    per_field_postings_writers: Vec<Box<dyn PostingsWriter>>,
+    per_field_postings_writers: Vec<PostingsWriterEnum>,
 }
 
 impl PerFieldPostingsWriter {
     pub fn for_schema(schema: &Schema) -> Self {
-        let per_field_postings_writers = schema
+        let per_field_postings_writers: Vec<PostingsWriterEnum> = schema
             .fields()
             .map(|(_, field_entry)| posting_writer_from_field_entry(field_entry))
             .collect();
@@ -19,16 +18,16 @@ impl PerFieldPostingsWriter {
         }
     }
 
-    pub(crate) fn get_for_field(&self, field: Field) -> &dyn PostingsWriter {
-        self.per_field_postings_writers[field.field_id() as usize].as_ref()
+    pub(crate) fn get_for_field(&self, field: Field) -> &PostingsWriterEnum {
+        &self.per_field_postings_writers[field.field_id() as usize]
     }
 
-    pub(crate) fn get_for_field_mut(&mut self, field: Field) -> &mut dyn PostingsWriter {
-        self.per_field_postings_writers[field.field_id() as usize].as_mut()
+    pub(crate) fn get_for_field_mut(&mut self, field: Field) -> &mut PostingsWriterEnum {
+        &mut self.per_field_postings_writers[field.field_id() as usize]
     }
 }
 
-fn posting_writer_from_field_entry(field_entry: &FieldEntry) -> Box<dyn PostingsWriter> {
+fn posting_writer_from_field_entry(field_entry: &FieldEntry) -> PostingsWriterEnum {
     match *field_entry.field_type() {
         FieldType::Str(ref text_options) => text_options
             .get_indexing_options()
@@ -51,7 +50,7 @@ fn posting_writer_from_field_entry(field_entry: &FieldEntry) -> Box<dyn Postings
         | FieldType::Date(_)
         | FieldType::Bytes(_)
         | FieldType::IpAddr(_)
-        | FieldType::Facet(_) => Box::<SpecializedPostingsWriter<DocIdRecorder>>::default(),
+        | FieldType::Facet(_) => <SpecializedPostingsWriter<DocIdRecorder>>::default().into(),
         FieldType::JsonObject(ref json_object_options) => {
             if let Some(text_indexing_option) = json_object_options.get_text_indexing_options() {
                 match text_indexing_option.index_option() {

src/postings/postings.rs

@@ -1,4 +1,9 @@
 use crate::docset::DocSet;
+use crate::fastfield::AliveBitSet;
+use crate::fieldnorm::FieldNormReader;
+use crate::postings::FreqReadingOption;
+use crate::query::{Bm25Weight, Scorer};
+use crate::{Score, TERMINATED};
 
 /// Postings (also called inverted list)
 ///
@@ -11,9 +16,42 @@ use crate::docset::DocSet;
 /// but other implementations mocking `SegmentPostings` exist,
 /// for merging segments or for testing.
 pub trait Postings: DocSet + 'static {
+    fn new_term_scorer(
+        self: Box<Self>,
+        fieldnorm_reader: FieldNormReader,
+        similarity_weight: Bm25Weight,
+    ) -> Box<dyn Scorer> {
+        // let self_dyn: Box<dyn Postings>  = self;
+        todo!();
+    }
+
     /// The number of times the term appears in the document.
     fn term_freq(&self) -> u32;
 
+    fn doc_freq(&self) -> u32 {
+        todo!();
+    }
+
+    /// Compute the number of non-deleted documents.
+    ///
+    /// This method will clone and scan through the posting lists.
+    /// (this is a rather expensive operation).
+    fn doc_freq_given_deletes(&self, alive_bitset: &AliveBitSet) -> u32 {
+        todo!();
+        // let mut docset = self.clone();
+        // let mut doc_freq = 0;
+        // loop {
+        //     let doc = docset.doc();
+        //     if doc == TERMINATED {
+        //         return doc_freq;
+        //     }
+        //     if alive_bitset.is_alive(doc) {
+        //         doc_freq += 1u32;
+        //     }
+        //     docset.advance();
+        // }
+    }
+
     /// Returns the positions offsetted with a given value.
     /// It is not necessary to clear the `output` before calling this method.
     /// The output vector will be resized to the `term_freq`.
@@ -31,9 +69,41 @@ pub trait Postings: DocSet + 'static {
     fn positions(&mut self, output: &mut Vec<u32>) {
         self.positions_with_offset(0u32, output);
     }
+
+    fn freq_reading_option(&self) -> FreqReadingOption;
+
+    // TODO see if we can put that in a lift to PostingsWithBlockMax trait.
+    // supports Block-Wand
+    fn supports_block_max(&self) -> bool {
+        false
+    }
+    // TODO document
+    // Only allowed for block max.
+    fn seek_block(
+        &mut self,
+        target_doc: crate::DocId,
+        fieldnorm_reader: &FieldNormReader,
+        similarity_weight: &Bm25Weight,
+    ) -> Score {
+        unimplemented!()
+    }
+
+    // TODO
+    // Only allowed for block max.
+    fn last_doc_in_block(&self) -> crate::DocId {
+        unimplemented!()
+    }
 }
 
 impl Postings for Box<dyn Postings> {
+    fn new_term_scorer(
+        self: Box<Self>,
+        fieldnorm_reader: FieldNormReader,
+        similarity_weight: Bm25Weight,
+    ) -> Box<dyn Scorer> {
+        (*self).new_term_scorer(fieldnorm_reader, similarity_weight)
+    }
+
     fn term_freq(&self) -> u32 {
         (**self).term_freq()
     }
@@ -41,4 +111,25 @@ impl Postings for Box<dyn Postings> {
     fn append_positions_with_offset(&mut self, offset: u32, output: &mut Vec<u32>) {
         (**self).append_positions_with_offset(offset, output);
     }
+
+    fn supports_block_max(&self) -> bool {
+        (**self).supports_block_max()
+    }
+
+    fn seek_block(
+        &mut self,
+        target_doc: crate::DocId,
+        fieldnorm_reader: &FieldNormReader,
+        similarity_weight: &Bm25Weight,
+    ) -> Score {
+        (**self).seek_block(target_doc, fieldnorm_reader, similarity_weight)
+    }
+
+    fn last_doc_in_block(&self) -> crate::DocId {
+        (**self).last_doc_in_block()
+    }
+
+    fn freq_reading_option(&self) -> FreqReadingOption {
+        (**self).freq_reading_option()
+    }
 }

src/postings/postings_writer.rs

@@ -4,10 +4,14 @@ use std::ops::Range;
 
 use stacker::Addr;
 
+use crate::codec::Codec;
 use crate::fieldnorm::FieldNormReaders;
 use crate::indexer::indexing_term::IndexingTerm;
 use crate::indexer::path_to_unordered_id::OrderedPathId;
-use crate::postings::recorder::{BufferLender, Recorder};
+use crate::postings::json_postings_writer::JsonPostingsWriter;
+use crate::postings::recorder::{
+    BufferLender, DocIdRecorder, Recorder, TermFrequencyRecorder, TfAndPositionRecorder,
+};
 use crate::postings::{
     FieldSerializer, IndexingContext, InvertedIndexSerializer, PerFieldPostingsWriter,
 };
@@ -45,12 +49,12 @@ fn make_field_partition(
 /// Serialize the inverted index.
 /// It pushes all term, one field at a time, towards the
 /// postings serializer.
-pub(crate) fn serialize_postings(
+pub(crate) fn serialize_postings<C: Codec>(
     ctx: IndexingContext,
     schema: Schema,
     per_field_postings_writers: &PerFieldPostingsWriter,
     fieldnorm_readers: FieldNormReaders,
-    serializer: &mut InvertedIndexSerializer,
+    serializer: &mut InvertedIndexSerializer<C>,
 ) -> crate::Result<()> {
     // Replace unordered ids by ordered ids to be able to sort
     let unordered_id_to_ordered_id: Vec<OrderedPathId> =
@@ -100,6 +104,141 @@ pub(crate) struct IndexingPosition {
     pub end_position: u32,
 }
 
+pub enum PostingsWriterEnum {
+    DocId(SpecializedPostingsWriter<DocIdRecorder>),
+    DocIdTf(SpecializedPostingsWriter<TermFrequencyRecorder>),
+    DocTfAndPosition(SpecializedPostingsWriter<TfAndPositionRecorder>),
+    JsonDocId(JsonPostingsWriter<DocIdRecorder>),
+    JsonDocIdTf(JsonPostingsWriter<TermFrequencyRecorder>),
+    JsonDocTfAndPosition(JsonPostingsWriter<TfAndPositionRecorder>),
+}
+
+impl From<SpecializedPostingsWriter<DocIdRecorder>> for PostingsWriterEnum {
+    fn from(doc_id_recorder_writer: SpecializedPostingsWriter<DocIdRecorder>) -> Self {
+        PostingsWriterEnum::DocId(doc_id_recorder_writer)
+    }
+}
+
+impl From<SpecializedPostingsWriter<TermFrequencyRecorder>> for PostingsWriterEnum {
+    fn from(doc_id_tf_recorder_writer: SpecializedPostingsWriter<TermFrequencyRecorder>) -> Self {
+        PostingsWriterEnum::DocIdTf(doc_id_tf_recorder_writer)
+    }
+}
+
+impl From<SpecializedPostingsWriter<TfAndPositionRecorder>> for PostingsWriterEnum {
+    fn from(
+        doc_id_tf_and_positions_recorder_writer: SpecializedPostingsWriter<TfAndPositionRecorder>,
+    ) -> Self {
+        PostingsWriterEnum::DocTfAndPosition(doc_id_tf_and_positions_recorder_writer)
+    }
+}
+
+impl From<JsonPostingsWriter<DocIdRecorder>> for PostingsWriterEnum {
+    fn from(doc_id_recorder_writer: JsonPostingsWriter<DocIdRecorder>) -> Self {
+        PostingsWriterEnum::JsonDocId(doc_id_recorder_writer)
+    }
+}
+
+impl From<JsonPostingsWriter<TermFrequencyRecorder>> for PostingsWriterEnum {
+    fn from(doc_id_tf_recorder_writer: JsonPostingsWriter<TermFrequencyRecorder>) -> Self {
+        PostingsWriterEnum::JsonDocIdTf(doc_id_tf_recorder_writer)
+    }
+}
+
+impl From<JsonPostingsWriter<TfAndPositionRecorder>> for PostingsWriterEnum {
+    fn from(
+        doc_id_tf_and_positions_recorder_writer: JsonPostingsWriter<TfAndPositionRecorder>,
+    ) -> Self {
+        PostingsWriterEnum::JsonDocTfAndPosition(doc_id_tf_and_positions_recorder_writer)
+    }
+}
+
+impl PostingsWriter for PostingsWriterEnum {
+    fn subscribe(&mut self, doc: DocId, pos: u32, term: &IndexingTerm, ctx: &mut IndexingContext) {
+        match self {
+            PostingsWriterEnum::DocId(writer) => writer.subscribe(doc, pos, term, ctx),
+            PostingsWriterEnum::DocIdTf(writer) => writer.subscribe(doc, pos, term, ctx),
+            PostingsWriterEnum::DocTfAndPosition(writer) => writer.subscribe(doc, pos, term, ctx),
+            PostingsWriterEnum::JsonDocId(writer) => writer.subscribe(doc, pos, term, ctx),
+            PostingsWriterEnum::JsonDocIdTf(writer) => writer.subscribe(doc, pos, term, ctx),
+            PostingsWriterEnum::JsonDocTfAndPosition(writer) => {
+                writer.subscribe(doc, pos, term, ctx)
+            }
+        }
+    }
+
+    fn serialize<C: Codec>(
+        &self,
+        term_addrs: &[(Field, OrderedPathId, &[u8], Addr)],
+        ordered_id_to_path: &[&str],
+        ctx: &IndexingContext,
+        serializer: &mut FieldSerializer<C>,
+    ) -> io::Result<()> {
+        match self {
+            PostingsWriterEnum::DocId(writer) => {
+                writer.serialize(term_addrs, ordered_id_to_path, ctx, serializer)
+            }
+            PostingsWriterEnum::DocIdTf(writer) => {
+                writer.serialize(term_addrs, ordered_id_to_path, ctx, serializer)
+            }
+            PostingsWriterEnum::DocTfAndPosition(writer) => {
+                writer.serialize(term_addrs, ordered_id_to_path, ctx, serializer)
+            }
+            PostingsWriterEnum::JsonDocId(writer) => {
+                writer.serialize(term_addrs, ordered_id_to_path, ctx, serializer)
+            }
+            PostingsWriterEnum::JsonDocIdTf(writer) => {
+                writer.serialize(term_addrs, ordered_id_to_path, ctx, serializer)
+            }
+            PostingsWriterEnum::JsonDocTfAndPosition(writer) => {
+                writer.serialize(term_addrs, ordered_id_to_path, ctx, serializer)
+            }
+        }
+    }
+
+    /// Tokenize a text and subscribe all of its token.
+    fn index_text(
+        &mut self,
+        doc_id: DocId,
+        token_stream: &mut dyn TokenStream,
+        term_buffer: &mut IndexingTerm,
+        ctx: &mut IndexingContext,
+        indexing_position: &mut IndexingPosition,
+    ) {
+        match self {
+            PostingsWriterEnum::DocId(writer) => {
+                writer.index_text(doc_id, token_stream, term_buffer, ctx, indexing_position)
+            }
+            PostingsWriterEnum::DocIdTf(writer) => {
+                writer.index_text(doc_id, token_stream, term_buffer, ctx, indexing_position)
+            }
+            PostingsWriterEnum::DocTfAndPosition(writer) => {
+                writer.index_text(doc_id, token_stream, term_buffer, ctx, indexing_position)
+            }
+            PostingsWriterEnum::JsonDocId(writer) => {
+                writer.index_text(doc_id, token_stream, term_buffer, ctx, indexing_position)
+            }
+            PostingsWriterEnum::JsonDocIdTf(writer) => {
+                writer.index_text(doc_id, token_stream, term_buffer, ctx, indexing_position)
+            }
+            PostingsWriterEnum::JsonDocTfAndPosition(writer) => {
+                writer.index_text(doc_id, token_stream, term_buffer, ctx, indexing_position)
+            }
+        }
+    }
+
+    fn total_num_tokens(&self) -> u64 {
+        match self {
+            PostingsWriterEnum::DocId(writer) => writer.total_num_tokens(),
+            PostingsWriterEnum::DocIdTf(writer) => writer.total_num_tokens(),
+            PostingsWriterEnum::DocTfAndPosition(writer) => writer.total_num_tokens(),
+            PostingsWriterEnum::JsonDocId(writer) => writer.total_num_tokens(),
+            PostingsWriterEnum::JsonDocIdTf(writer) => writer.total_num_tokens(),
+            PostingsWriterEnum::JsonDocTfAndPosition(writer) => writer.total_num_tokens(),
+        }
+    }
+}
+
 /// The `PostingsWriter` is in charge of receiving documenting
 /// and building a `Segment` in anonymous memory.
 ///
@@ -116,12 +255,12 @@ pub(crate) trait PostingsWriter: Send + Sync {
 
     /// Serializes the postings on disk.
     /// The actual serialization format is handled by the `PostingsSerializer`.
-    fn serialize(
+    fn serialize<C: Codec>(
         &self,
         term_addrs: &[(Field, OrderedPathId, &[u8], Addr)],
         ordered_id_to_path: &[&str],
         ctx: &IndexingContext,
-        serializer: &mut FieldSerializer,
+        serializer: &mut FieldSerializer<C>,
     ) -> io::Result<()>;
 
     /// Tokenize a text and subscribe all of its token.
@@ -171,22 +310,14 @@ pub(crate) struct SpecializedPostingsWriter<Rec: Recorder> {
     _recorder_type: PhantomData<Rec>,
 }
 
-impl<Rec: Recorder> From<SpecializedPostingsWriter<Rec>> for Box<dyn PostingsWriter> {
-    fn from(
-        specialized_postings_writer: SpecializedPostingsWriter<Rec>,
-    ) -> Box<dyn PostingsWriter> {
-        Box::new(specialized_postings_writer)
-    }
-}
-
 impl<Rec: Recorder> SpecializedPostingsWriter<Rec> {
     #[inline]
-    pub(crate) fn serialize_one_term(
+    pub(crate) fn serialize_one_term<C: Codec>(
         term: &[u8],
         addr: Addr,
         buffer_lender: &mut BufferLender,
         ctx: &IndexingContext,
-        serializer: &mut FieldSerializer,
+        serializer: &mut FieldSerializer<C>,
     ) -> io::Result<()> {
         let recorder: Rec = ctx.term_index.read(addr);
         let term_doc_freq = recorder.term_doc_freq().unwrap_or(0u32);
@@ -227,12 +358,12 @@ impl<Rec: Recorder> PostingsWriter for SpecializedPostingsWriter<Rec> {
         });
     }
 
-    fn serialize(
+    fn serialize<C: Codec>(
         &self,
         term_addrs: &[(Field, OrderedPathId, &[u8], Addr)],
         _ordered_id_to_path: &[&str],
         ctx: &IndexingContext,
-        serializer: &mut FieldSerializer,
+        serializer: &mut FieldSerializer<C>,
     ) -> io::Result<()> {
         let mut buffer_lender = BufferLender::default();
         for (_field, _path_id, term, addr) in term_addrs {

src/postings/recorder.rs

@@ -1,6 +1,7 @@
 use common::read_u32_vint;
 use stacker::{ExpUnrolledLinkedList, MemoryArena};
 
+use crate::codec::Codec;
 use crate::postings::FieldSerializer;
 use crate::DocId;
 
@@ -67,10 +68,10 @@ pub(crate) trait Recorder: Copy + Default + Send + Sync + 'static {
     /// Close the document. It will help record the term frequency.
     fn close_doc(&mut self, arena: &mut MemoryArena);
     /// Pushes the postings information to the serializer.
-    fn serialize(
+    fn serialize<C: Codec>(
         &self,
         arena: &MemoryArena,
-        serializer: &mut FieldSerializer<'_>,
+        serializer: &mut FieldSerializer<C>,
         buffer_lender: &mut BufferLender,
     );
     /// Returns the number of document containing this term.
@@ -110,10 +111,10 @@ impl Recorder for DocIdRecorder {
     #[inline]
     fn close_doc(&mut self, _arena: &mut MemoryArena) {}
 
-    fn serialize(
+    fn serialize<C: Codec>(
         &self,
         arena: &MemoryArena,
-        serializer: &mut FieldSerializer<'_>,
+        serializer: &mut FieldSerializer<C>,
         buffer_lender: &mut BufferLender,
     ) {
         let buffer = buffer_lender.lend_u8();
@@ -178,10 +179,10 @@ impl Recorder for TermFrequencyRecorder {
         self.current_tf = 0;
     }
 
-    fn serialize(
+    fn serialize<C: Codec>(
         &self,
         arena: &MemoryArena,
-        serializer: &mut FieldSerializer<'_>,
+        serializer: &mut FieldSerializer<C>,
         buffer_lender: &mut BufferLender,
     ) {
         let buffer = buffer_lender.lend_u8();
@@ -235,10 +236,10 @@ impl Recorder for TfAndPositionRecorder {
         self.stack.writer(arena).write_u32_vint(POSITION_END);
     }
 
-    fn serialize(
+    fn serialize<C: Codec>(
         &self,
         arena: &MemoryArena,
-        serializer: &mut FieldSerializer<'_>,
+        serializer: &mut FieldSerializer<C>,
         buffer_lender: &mut BufferLender,
     ) {
         let (buffer_u8, buffer_positions) = buffer_lender.lend_all();

src/postings/serializer.rs

@@ -1,16 +1,14 @@
-use std::cmp::Ordering;
 use std::io::{self, Write};
 
-use common::{BinarySerializable, CountingWriter, VInt};
+use common::{BinarySerializable, CountingWriter};
 
 use super::TermInfo;
+use crate::codec::postings::PostingsSerializer;
+use crate::codec::Codec;
 use crate::directory::{CompositeWrite, WritePtr};
 use crate::fieldnorm::FieldNormReader;
 use crate::index::Segment;
 use crate::positions::PositionSerializer;
-use crate::postings::compression::{BlockEncoder, VIntEncoder, COMPRESSION_BLOCK_SIZE};
-use crate::postings::skip::SkipSerializer;
-use crate::query::Bm25Weight;
 use crate::schema::{Field, FieldEntry, FieldType, IndexRecordOption, Schema};
 use crate::termdict::TermDictionaryBuilder;
 use crate::{DocId, Score};
@@ -46,22 +44,27 @@ use crate::{DocId, Score};
 ///
 /// A description of the serialization format is
 /// [available here](https://fulmicoton.gitbooks.io/tantivy-doc/content/inverted-index.html).
-pub struct InvertedIndexSerializer {
+pub struct InvertedIndexSerializer<C: Codec> {
     terms_write: CompositeWrite<WritePtr>,
     postings_write: CompositeWrite<WritePtr>,
     positions_write: CompositeWrite<WritePtr>,
     schema: Schema,
+    codec: C,
 }
 
-impl InvertedIndexSerializer {
+use crate::codec::postings::PostingsCodec;
+
+impl<C: Codec> InvertedIndexSerializer<C> {
     /// Open a new `InvertedIndexSerializer` for the given segment
-    pub fn open(segment: &mut Segment) -> crate::Result<InvertedIndexSerializer> {
+    pub fn open(segment: &mut Segment<C>) -> crate::Result<InvertedIndexSerializer<C>> {
         use crate::index::SegmentComponent::{Positions, Postings, Terms};
+        let codec = segment.index().codec().clone();
         let inv_index_serializer = InvertedIndexSerializer {
             terms_write: CompositeWrite::wrap(segment.open_write(Terms)?),
             postings_write: CompositeWrite::wrap(segment.open_write(Postings)?),
             positions_write: CompositeWrite::wrap(segment.open_write(Positions)?),
             schema: segment.schema(),
+            codec,
         };
         Ok(inv_index_serializer)
     }
@@ -75,7 +78,7 @@ impl InvertedIndexSerializer {
         field: Field,
         total_num_tokens: u64,
         fieldnorm_reader: Option<FieldNormReader>,
-    ) -> io::Result<FieldSerializer<'_>> {
+    ) -> io::Result<FieldSerializer<'_, C>> {
         let field_entry: &FieldEntry = self.schema.get_field_entry(field);
         let term_dictionary_write = self.terms_write.for_field(field);
         let postings_write = self.postings_write.for_field(field);
@@ -88,6 +91,7 @@ impl InvertedIndexSerializer {
             postings_write,
             positions_write,
             fieldnorm_reader,
+            &self.codec,
         )
     }
 
@@ -102,9 +106,9 @@ impl InvertedIndexSerializer {
 
 /// The field serializer is in charge of
 /// the serialization of a specific field.
-pub struct FieldSerializer<'a> {
+pub struct FieldSerializer<'a, C: Codec> {
     term_dictionary_builder: TermDictionaryBuilder<&'a mut CountingWriter<WritePtr>>,
-    postings_serializer: PostingsSerializer,
+    postings_serializer: <C::PostingsCodec as PostingsCodec>::PostingsSerializer,
     positions_serializer_opt: Option<PositionSerializer<&'a mut CountingWriter<WritePtr>>>,
     current_term_info: TermInfo,
     term_open: bool,
@@ -112,7 +116,7 @@ pub struct FieldSerializer<'a> {
     postings_start_offset: u64,
 }
 
-impl<'a> FieldSerializer<'a> {
+impl<'a, C: Codec> FieldSerializer<'a, C> {
     fn create(
         field_type: &FieldType,
         total_num_tokens: u64,
@@ -120,7 +124,8 @@ impl<'a> FieldSerializer<'a> {
         postings_write: &'a mut CountingWriter<WritePtr>,
         positions_write: &'a mut CountingWriter<WritePtr>,
         fieldnorm_reader: Option<FieldNormReader>,
-    ) -> io::Result<FieldSerializer<'a>> {
+        codec: &C,
+    ) -> io::Result<FieldSerializer<'a, C>> {
         total_num_tokens.serialize(postings_write)?;
         let index_record_option = field_type
             .index_record_option()
@@ -130,8 +135,11 @@ impl<'a> FieldSerializer<'a> {
             .as_ref()
             .map(|ff_reader| total_num_tokens as Score / ff_reader.num_docs() as Score)
             .unwrap_or(0.0);
-        let postings_serializer =
-            PostingsSerializer::new(average_fieldnorm, index_record_option, fieldnorm_reader);
+        let postings_serializer = codec.postings_codec().new_serializer(
+            average_fieldnorm,
+            index_record_option,
+            fieldnorm_reader,
+        );
         let positions_serializer_opt = if index_record_option.has_positions() {
             Some(PositionSerializer::new(positions_write))
         } else {
@@ -248,223 +256,3 @@ impl<'a> FieldSerializer<'a> {
         Ok(())
     }
 }
-
-struct Block {
-    doc_ids: [DocId; COMPRESSION_BLOCK_SIZE],
-    term_freqs: [u32; COMPRESSION_BLOCK_SIZE],
-    len: usize,
-}
-
-impl Block {
-    fn new() -> Self {
-        Block {
-            doc_ids: [0u32; COMPRESSION_BLOCK_SIZE],
-            term_freqs: [0u32; COMPRESSION_BLOCK_SIZE],
-            len: 0,
-        }
-    }
-
-    fn doc_ids(&self) -> &[DocId] {
-        &self.doc_ids[..self.len]
-    }
-
-    fn term_freqs(&self) -> &[u32] {
-        &self.term_freqs[..self.len]
-    }
-
-    fn clear(&mut self) {
-        self.len = 0;
-    }
-
-    fn append_doc(&mut self, doc: DocId, term_freq: u32) {
-        let len = self.len;
-        self.doc_ids[len] = doc;
-        self.term_freqs[len] = term_freq;
-        self.len = len + 1;
-    }
-
-    fn is_full(&self) -> bool {
-        self.len == COMPRESSION_BLOCK_SIZE
-    }
-
-    fn is_empty(&self) -> bool {
-        self.len == 0
-    }
-
-    fn last_doc(&self) -> DocId {
-        assert_eq!(self.len, COMPRESSION_BLOCK_SIZE);
-        self.doc_ids[COMPRESSION_BLOCK_SIZE - 1]
-    }
-}
-
-pub struct PostingsSerializer {
-    last_doc_id_encoded: u32,
-
-    block_encoder: BlockEncoder,
-    block: Box<Block>,
-
-    postings_write: Vec<u8>,
-    skip_write: SkipSerializer,
-
-    mode: IndexRecordOption,
-    fieldnorm_reader: Option<FieldNormReader>,
-
-    bm25_weight: Option<Bm25Weight>,
-    avg_fieldnorm: Score, /* Average number of term in the field for that segment.
-                           * this value is used to compute the block wand information. */
-    term_has_freq: bool,
-}
-
-impl PostingsSerializer {
-    pub fn new(
-        avg_fieldnorm: Score,
-        mode: IndexRecordOption,
-        fieldnorm_reader: Option<FieldNormReader>,
-    ) -> PostingsSerializer {
-        PostingsSerializer {
-            block_encoder: BlockEncoder::new(),
-            block: Box::new(Block::new()),
-
-            postings_write: Vec::new(),
-            skip_write: SkipSerializer::new(),
-
-            last_doc_id_encoded: 0u32,
-            mode,
-
-            fieldnorm_reader,
-            bm25_weight: None,
-            avg_fieldnorm,
-            term_has_freq: false,
-        }
-    }
-
-    pub fn new_term(&mut self, term_doc_freq: u32, record_term_freq: bool) {
-        self.bm25_weight = None;
-
-        self.term_has_freq = self.mode.has_freq() && record_term_freq;
-        if !self.term_has_freq {
-            return;
-        }
-
-        let num_docs_in_segment: u64 =
-            if let Some(fieldnorm_reader) = self.fieldnorm_reader.as_ref() {
-                fieldnorm_reader.num_docs() as u64
-            } else {
-                return;
-            };
-
-        if num_docs_in_segment == 0 {
-            return;
-        }
-
-        self.bm25_weight = Some(Bm25Weight::for_one_term_without_explain(
-            term_doc_freq as u64,
-            num_docs_in_segment,
-            self.avg_fieldnorm,
-        ));
-    }
-
-    fn write_block(&mut self) {
-        {
-            // encode the doc ids
-            let (num_bits, block_encoded): (u8, &[u8]) = self
-                .block_encoder
-                .compress_block_sorted(self.block.doc_ids(), self.last_doc_id_encoded);
-            self.last_doc_id_encoded = self.block.last_doc();
-            self.skip_write
-                .write_doc(self.last_doc_id_encoded, num_bits);
-            // last el block 0, offset block 1,
-            self.postings_write.extend(block_encoded);
-        }
-        if self.term_has_freq {
-            let (num_bits, block_encoded): (u8, &[u8]) = self
-                .block_encoder
-                .compress_block_unsorted(self.block.term_freqs(), true);
-            self.postings_write.extend(block_encoded);
-            self.skip_write.write_term_freq(num_bits);
-            if self.mode.has_positions() {
-                // We serialize the sum of term freqs within the skip information
-                // in order to navigate through positions.
-                let sum_freq = self.block.term_freqs().iter().cloned().sum();
-                self.skip_write.write_total_term_freq(sum_freq);
-            }
-            let mut blockwand_params = (0u8, 0u32);
-            if let Some(bm25_weight) = self.bm25_weight.as_ref() {
-                if let Some(fieldnorm_reader) = self.fieldnorm_reader.as_ref() {
-                    let docs = self.block.doc_ids().iter().cloned();
-                    let term_freqs = self.block.term_freqs().iter().cloned();
-                    let fieldnorms = docs.map(|doc| fieldnorm_reader.fieldnorm_id(doc));
-                    blockwand_params = fieldnorms
-                        .zip(term_freqs)
-                        .max_by(
-                            |(left_fieldnorm_id, left_term_freq),
-                             (right_fieldnorm_id, right_term_freq)| {
-                                let left_score =
-                                    bm25_weight.tf_factor(*left_fieldnorm_id, *left_term_freq);
-                                let right_score =
-                                    bm25_weight.tf_factor(*right_fieldnorm_id, *right_term_freq);
-                                left_score
-                                    .partial_cmp(&right_score)
-                                    .unwrap_or(Ordering::Equal)
-                            },
-                        )
-                        .unwrap();
-                }
-            }
-            let (fieldnorm_id, term_freq) = blockwand_params;
-            self.skip_write.write_blockwand_max(fieldnorm_id, term_freq);
-        }
-        self.block.clear();
-    }
-
-    pub fn write_doc(&mut self, doc_id: DocId, term_freq: u32) {
-        self.block.append_doc(doc_id, term_freq);
-        if self.block.is_full() {
-            self.write_block();
-        }
-    }
-
-    pub fn close_term(
-        &mut self,
-        doc_freq: u32,
-        output_write: &mut impl std::io::Write,
-    ) -> io::Result<()> {
-        if !self.block.is_empty() {
-            // we have doc ids waiting to be written
-            // this happens when the number of doc ids is
-            // not a perfect multiple of our block size.
-            //
-            // In that case, the remaining part is encoded
-            // using variable int encoding.
-            {
-                let block_encoded = self
-                    .block_encoder
-                    .compress_vint_sorted(self.block.doc_ids(), self.last_doc_id_encoded);
-                self.postings_write.write_all(block_encoded)?;
-            }
-            // ... Idem for term frequencies
-            if self.term_has_freq {
-                let block_encoded = self
-                    .block_encoder
-                    .compress_vint_unsorted(self.block.term_freqs());
-                self.postings_write.write_all(block_encoded)?;
-            }
-            self.block.clear();
-        }
-        if doc_freq >= COMPRESSION_BLOCK_SIZE as u32 {
-            let skip_data = self.skip_write.data();
-            VInt(skip_data.len() as u64).serialize(output_write)?;
-            output_write.write_all(skip_data)?;
-        }
-        output_write.write_all(&self.postings_write[..])?;
-        self.skip_write.clear();
-        self.postings_write.clear();
-        self.bm25_weight = None;
-        Ok(())
-    }
-
-    fn clear(&mut self) {
-        self.block.clear();
-        self.last_doc_id_encoded = 0;
-    }
-}

src/query/automaton_weight.rs

@@ -10,7 +10,7 @@ use crate::postings::TermInfo;
 use crate::query::{BitSetDocSet, ConstScorer, Explanation, Scorer, Weight};
 use crate::schema::{Field, IndexRecordOption};
 use crate::termdict::{TermDictionary, TermStreamer};
-use crate::{DocId, Score, TantivyError};
+use crate::{DocId, DocSet, Score, TantivyError};
 
 /// A weight struct for Fuzzy Term and Regex Queries
 pub struct AutomatonWeight<A> {
@@ -92,18 +92,9 @@ where
         let mut term_stream = self.automaton_stream(term_dict)?;
         while term_stream.advance() {
             let term_info = term_stream.value();
-            let mut block_segment_postings = inverted_index
-                .read_block_postings_from_terminfo(term_info, IndexRecordOption::Basic)?;
-            loop {
-                let docs = block_segment_postings.docs();
-                if docs.is_empty() {
-                    break;
-                }
-                for &doc in docs {
-                    doc_bitset.insert(doc);
-                }
-                block_segment_postings.advance();
-            }
+            let mut block_segment_postings =
+                inverted_index.read_postings_from_terminfo(term_info, IndexRecordOption::Basic)?;
+            block_segment_postings.fill_bitset(&mut doc_bitset);
         }
         let doc_bitset = BitSetDocSet::from(doc_bitset);
         let const_scorer = ConstScorer::new(doc_bitset, boost);

src/query/boolean_query/block_wand.rs

@@ -4,143 +4,143 @@ use crate::query::term_query::TermScorer;
 use crate::query::Scorer;
 use crate::{DocId, DocSet, Score, TERMINATED};
 
-/// Takes a term_scorers sorted by their current doc() and a threshold and returns
-/// Returns (pivot_len, pivot_ord) defined as follows:
-/// - `pivot_doc` lowest document that has a chance of exceeding (>) the threshold score.
-/// - `before_pivot_len` number of term_scorers such that term_scorer.doc() < pivot.
-/// - `pivot_len` number of term_scorers such that term_scorer.doc() <= pivot.
-///
-/// We always have `before_pivot_len` < `pivot_len`.
-///
-/// `None` is returned if we establish that no document can exceed the threshold.
-fn find_pivot_doc(
-    term_scorers: &[TermScorerWithMaxScore],
-    threshold: Score,
-) -> Option<(usize, usize, DocId)> {
-    let mut max_score = 0.0;
-    let mut before_pivot_len = 0;
-    let mut pivot_doc = TERMINATED;
-    while before_pivot_len < term_scorers.len() {
-        let term_scorer = &term_scorers[before_pivot_len];
-        max_score += term_scorer.max_score;
-        if max_score > threshold {
-            pivot_doc = term_scorer.doc();
-            break;
-        }
-        before_pivot_len += 1;
-    }
-    if pivot_doc == TERMINATED {
-        return None;
-    }
-    // Right now i is an ordinal, we want a len.
-    let mut pivot_len = before_pivot_len + 1;
-    // Some other term_scorer may be positioned on the same document.
-    pivot_len += term_scorers[pivot_len..]
-        .iter()
-        .take_while(|term_scorer| term_scorer.doc() == pivot_doc)
-        .count();
-    Some((before_pivot_len, pivot_len, pivot_doc))
-}
+// /// Takes a term_scorers sorted by their current doc() and a threshold and returns
+// /// Returns (pivot_len, pivot_ord) defined as follows:
+// /// - `pivot_doc` lowest document that has a chance of exceeding (>) the threshold score.
+// /// - `before_pivot_len` number of term_scorers such that term_scorer.doc() < pivot.
+// /// - `pivot_len` number of term_scorers such that term_scorer.doc() <= pivot.
+// ///
+// /// We always have `before_pivot_len` < `pivot_len`.
+// ///
+// /// `None` is returned if we establish that no document can exceed the threshold.
+// fn find_pivot_doc(
+//     term_scorers: &[TermScorerWithMaxScore],
+//     threshold: Score,
+// ) -> Option<(usize, usize, DocId)> {
+//     let mut max_score = 0.0;
+//     let mut before_pivot_len = 0;
+//     let mut pivot_doc = TERMINATED;
+//     while before_pivot_len < term_scorers.len() {
+//         let term_scorer = &term_scorers[before_pivot_len];
+//         max_score += term_scorer.max_score;
+//         if max_score > threshold {
+//             pivot_doc = term_scorer.doc();
+//             break;
+//         }
+//         before_pivot_len += 1;
+//     }
+//     if pivot_doc == TERMINATED {
+//         return None;
+//     }
+//     // Right now i is an ordinal, we want a len.
+//     let mut pivot_len = before_pivot_len + 1;
+//     // Some other term_scorer may be positioned on the same document.
+//     pivot_len += term_scorers[pivot_len..]
+//         .iter()
+//         .take_while(|term_scorer| term_scorer.doc() == pivot_doc)
+//         .count();
+//     Some((before_pivot_len, pivot_len, pivot_doc))
+// }
 
-/// Advance the scorer with best score among the scorers[..pivot_len] to
-/// the next doc candidate defined by the min of `last_doc_in_block + 1` for
-/// scorer in scorers[..pivot_len] and `scorer.doc()` for scorer in scorers[pivot_len..].
-/// Note: before and after calling this method, scorers need to be sorted by their `.doc()`.
-fn block_max_was_too_low_advance_one_scorer(
-    scorers: &mut [TermScorerWithMaxScore],
-    pivot_len: usize,
-) {
-    debug_assert!(is_sorted(scorers.iter().map(|scorer| scorer.doc())));
-    let mut scorer_to_seek = pivot_len - 1;
-    let mut global_max_score = scorers[scorer_to_seek].max_score;
-    let mut doc_to_seek_after = scorers[scorer_to_seek].last_doc_in_block();
-    for scorer_ord in (0..pivot_len - 1).rev() {
-        let scorer = &scorers[scorer_ord];
-        if scorer.last_doc_in_block() <= doc_to_seek_after {
-            doc_to_seek_after = scorer.last_doc_in_block();
-        }
-        if scorers[scorer_ord].max_score > global_max_score {
-            global_max_score = scorers[scorer_ord].max_score;
-            scorer_to_seek = scorer_ord;
-        }
-    }
-    // Add +1 to go to the next block unless we are already at the end.
-    if doc_to_seek_after != TERMINATED {
-        doc_to_seek_after += 1;
-    }
-    for scorer in &scorers[pivot_len..] {
-        if scorer.doc() <= doc_to_seek_after {
-            doc_to_seek_after = scorer.doc();
-        }
-    }
-    scorers[scorer_to_seek].seek(doc_to_seek_after);
+// /// Advance the scorer with best score among the scorers[..pivot_len] to
+// /// the next doc candidate defined by the min of `last_doc_in_block + 1` for
+// /// scorer in scorers[..pivot_len] and `scorer.doc()` for scorer in scorers[pivot_len..].
+// /// Note: before and after calling this method, scorers need to be sorted by their `.doc()`.
+// fn block_max_was_too_low_advance_one_scorer(
+//     scorers: &mut [TermScorerWithMaxScore],
+//     pivot_len: usize,
+// ) {
+//     debug_assert!(is_sorted(scorers.iter().map(|scorer| scorer.doc())));
+//     let mut scorer_to_seek = pivot_len - 1;
+//     let mut global_max_score = scorers[scorer_to_seek].max_score;
+//     let mut doc_to_seek_after = scorers[scorer_to_seek].last_doc_in_block();
+//     for scorer_ord in (0..pivot_len - 1).rev() {
+//         let scorer = &scorers[scorer_ord];
+//         if scorer.last_doc_in_block() <= doc_to_seek_after {
+//             doc_to_seek_after = scorer.last_doc_in_block();
+//         }
+//         if scorers[scorer_ord].max_score > global_max_score {
+//             global_max_score = scorers[scorer_ord].max_score;
+//             scorer_to_seek = scorer_ord;
+//         }
+//     }
+//     // Add +1 to go to the next block unless we are already at the end.
+//     if doc_to_seek_after != TERMINATED {
+//         doc_to_seek_after += 1;
+//     }
+//     for scorer in &scorers[pivot_len..] {
+//         if scorer.doc() <= doc_to_seek_after {
+//             doc_to_seek_after = scorer.doc();
+//         }
+//     }
+//     scorers[scorer_to_seek].seek(doc_to_seek_after);
 
-    restore_ordering(scorers, scorer_to_seek);
-    debug_assert!(is_sorted(scorers.iter().map(|scorer| scorer.doc())));
-}
+//     restore_ordering(scorers, scorer_to_seek);
+//     debug_assert!(is_sorted(scorers.iter().map(|scorer| scorer.doc())));
+// }
 
-// Given a list of term_scorers and a `ord` and assuming that `term_scorers[ord]` is sorted
-// except term_scorers[ord] that might be in advance compared to its ranks,
-// bubble up term_scorers[ord] in order to restore the ordering.
-fn restore_ordering(term_scorers: &mut [TermScorerWithMaxScore], ord: usize) {
-    let doc = term_scorers[ord].doc();
-    for i in ord + 1..term_scorers.len() {
-        if term_scorers[i].doc() >= doc {
-            break;
-        }
-        term_scorers.swap(i, i - 1);
-    }
-    debug_assert!(is_sorted(term_scorers.iter().map(|scorer| scorer.doc())));
-}
+// // Given a list of term_scorers and a `ord` and assuming that `term_scorers[ord]` is sorted
+// // except term_scorers[ord] that might be in advance compared to its ranks,
+// // bubble up term_scorers[ord] in order to restore the ordering.
+// fn restore_ordering(term_scorers: &mut [TermScorerWithMaxScore], ord: usize) {
+//     let doc = term_scorers[ord].doc();
+//     for i in ord + 1..term_scorers.len() {
+//         if term_scorers[i].doc() >= doc {
+//             break;
+//         }
+//         term_scorers.swap(i, i - 1);
+//     }
+//     debug_assert!(is_sorted(term_scorers.iter().map(|scorer| scorer.doc())));
+// }
 
-// Attempts to advance all term_scorers between `&term_scorers[0..before_len]` to the pivot.
-// If this works, return true.
-// If this fails (ie: one of the term_scorer does not contain `pivot_doc` and seek goes past the
-// pivot), reorder the term_scorers to ensure the list is still sorted and returns `false`.
-// If a term_scorer reach TERMINATED in the process return false remove the term_scorer and return.
-fn align_scorers(
-    term_scorers: &mut Vec<TermScorerWithMaxScore>,
-    pivot_doc: DocId,
-    before_pivot_len: usize,
-) -> bool {
-    debug_assert_ne!(pivot_doc, TERMINATED);
-    for i in (0..before_pivot_len).rev() {
-        let new_doc = term_scorers[i].seek(pivot_doc);
-        if new_doc != pivot_doc {
-            if new_doc == TERMINATED {
-                term_scorers.swap_remove(i);
-            }
-            // We went past the pivot.
-            // We just go through the outer loop mechanic (Note that pivot is
-            // still a possible candidate).
-            //
-            // Termination is still guaranteed since we can only consider the same
-            // pivot at most term_scorers.len() - 1 times.
-            restore_ordering(term_scorers, i);
-            return false;
-        }
-    }
-    true
-}
+// // Attempts to advance all term_scorers between `&term_scorers[0..before_len]` to the pivot.
+// // If this works, return true.
+// // If this fails (ie: one of the term_scorer does not contain `pivot_doc` and seek goes past the
+// // pivot), reorder the term_scorers to ensure the list is still sorted and returns `false`.
+// // If a term_scorer reach TERMINATED in the process return false remove the term_scorer and
+// return. fn align_scorers(
+//     term_scorers: &mut Vec<TermScorerWithMaxScore>,
+//     pivot_doc: DocId,
+//     before_pivot_len: usize,
+// ) -> bool {
+//     debug_assert_ne!(pivot_doc, TERMINATED);
+//     for i in (0..before_pivot_len).rev() {
+//         let new_doc = term_scorers[i].seek(pivot_doc);
+//         if new_doc != pivot_doc {
+//             if new_doc == TERMINATED {
+//                 term_scorers.swap_remove(i);
+//             }
+//             // We went past the pivot.
+//             // We just go through the outer loop mechanic (Note that pivot is
+//             // still a possible candidate).
+//             //
+//             // Termination is still guaranteed since we can only consider the same
+//             // pivot at most term_scorers.len() - 1 times.
+//             restore_ordering(term_scorers, i);
+//             return false;
+//         }
+//     }
+//     true
+// }
 
-// Assumes terms_scorers[..pivot_len] are positioned on the same doc (pivot_doc).
-// Advance term_scorers[..pivot_len] and out of these removes the terminated scores.
-// Restores the ordering of term_scorers.
-fn advance_all_scorers_on_pivot(term_scorers: &mut Vec<TermScorerWithMaxScore>, pivot_len: usize) {
-    for term_scorer in &mut term_scorers[..pivot_len] {
-        term_scorer.advance();
-    }
-    // TODO use drain_filter when available.
-    let mut i = 0;
-    while i != term_scorers.len() {
-        if term_scorers[i].doc() == TERMINATED {
-            term_scorers.swap_remove(i);
-        } else {
-            i += 1;
-        }
-    }
-    term_scorers.sort_by_key(|scorer| scorer.doc());
-}
+// // Assumes terms_scorers[..pivot_len] are positioned on the same doc (pivot_doc).
+// // Advance term_scorers[..pivot_len] and out of these removes the terminated scores.
+// // Restores the ordering of term_scorers.
+// fn advance_all_scorers_on_pivot(term_scorers: &mut Vec<TermScorerWithMaxScore>, pivot_len: usize)
+// {     for term_scorer in &mut term_scorers[..pivot_len] {
+//         term_scorer.advance();
+//     }
+//     // TODO use drain_filter when available.
+//     let mut i = 0;
+//     while i != term_scorers.len() {
+//         if term_scorers[i].doc() == TERMINATED {
+//             term_scorers.swap_remove(i);
+//         } else {
+//             i += 1;
+//         }
+//     }
+//     term_scorers.sort_by_key(|scorer| scorer.doc());
+// }
 
 /// Implements the WAND (Weak AND) algorithm for dynamic pruning
 /// described in the paper "Faster Top-k Document Retrieval Using Block-Max Indexes".
@@ -150,65 +150,63 @@ pub fn block_wand(
     mut threshold: Score,
     callback: &mut dyn FnMut(u32, Score) -> Score,
 ) {
-    let mut scorers: Vec<TermScorerWithMaxScore> = scorers
-        .iter_mut()
-        .map(TermScorerWithMaxScore::from)
-        .collect();
-    scorers.sort_by_key(|scorer| scorer.doc());
-    // At this point we need to ensure that the scorers are sorted!
-    debug_assert!(is_sorted(scorers.iter().map(|scorer| scorer.doc())));
-    while let Some((before_pivot_len, pivot_len, pivot_doc)) =
-        find_pivot_doc(&scorers[..], threshold)
-    {
-        debug_assert!(is_sorted(scorers.iter().map(|scorer| scorer.doc())));
-        debug_assert_ne!(pivot_doc, TERMINATED);
-        debug_assert!(before_pivot_len < pivot_len);
+    // let mut scorers: Vec<TermScorerWithMaxScore> = scorers
+    //     .iter_mut()
+    //     .map(TermScorerWithMaxScore::from)
+    //     .collect();
+    // scorers.sort_by_key(|scorer| scorer.doc());
+    // // At this point we need to ensure that the scorers are sorted!
+    // debug_assert!(is_sorted(scorers.iter().map(|scorer| scorer.doc())));
+    // while let Some((before_pivot_len, pivot_len, pivot_doc)) =
+    //     find_pivot_doc(&scorers[..], threshold)
+    // {
+    //     debug_assert!(is_sorted(scorers.iter().map(|scorer| scorer.doc())));
+    //     debug_assert_ne!(pivot_doc, TERMINATED);
+    //     debug_assert!(before_pivot_len < pivot_len);
 
-        let block_max_score_upperbound: Score = scorers[..pivot_len]
-            .iter_mut()
-            .map(|scorer| {
-                scorer.seek_block(pivot_doc);
-                scorer.block_max_score()
-            })
-            .sum();
+    //     let block_max_score_upperbound: Score = scorers[..pivot_len]
+    //         .iter_mut()
+    //         .map(|scorer| scorer.seek_block(pivot_doc))
+    //         .sum();
 
-        // Beware after shallow advance, skip readers can be in advance compared to
-        // the segment posting lists.
-        //
-        // `block_segment_postings.load_block()` need to be called separately.
-        if block_max_score_upperbound <= threshold {
-            // Block max condition was not reached
-            // We could get away by simply advancing the scorers to DocId + 1 but it would
-            // be inefficient. The optimization requires proper explanation and was
-            // isolated in a different function.
-            block_max_was_too_low_advance_one_scorer(&mut scorers, pivot_len);
-            continue;
-        }
+    //     // Beware after shallow advance, skip readers can be in advance compared to
+    //     // the segment posting lists.
+    //     //
+    //     // `block_segment_postings.load_block()` need to be called separately.
+    //     if block_max_score_upperbound <= threshold {
+    //         // Block max condition was not reached
+    //         // We could get away by simply advancing the scorers to DocId + 1 but it would
+    //         // be inefficient. The optimization requires proper explanation and was
+    //         // isolated in a different function.
+    //         block_max_was_too_low_advance_one_scorer(&mut scorers, pivot_len);
+    //         continue;
+    //     }
 
-        // Block max condition is observed.
-        //
-        // Let's try and advance all scorers before the pivot to the pivot.
-        if !align_scorers(&mut scorers, pivot_doc, before_pivot_len) {
-            // At least of the scorer does not contain the pivot.
-            //
-            // Let's stop scoring this pivot and go through the pivot selection again.
-            // Note that the current pivot is not necessarily a bad candidate and it
-            // may be picked again.
-            continue;
-        }
+    //     // Block max condition is observed.
+    //     //
+    //     // Let's try and advance all scorers before the pivot to the pivot.
+    //     if !align_scorers(&mut scorers, pivot_doc, before_pivot_len) {
+    //         // At least of the scorer does not contain the pivot.
+    //         //
+    //         // Let's stop scoring this pivot and go through the pivot selection again.
+    //         // Note that the current pivot is not necessarily a bad candidate and it
+    //         // may be picked again.
+    //         continue;
+    //     }
 
-        // At this point, all scorers are positioned on the doc.
-        let score = scorers[..pivot_len]
-            .iter_mut()
-            .map(|scorer| scorer.score())
-            .sum();
+    //     // At this point, all scorers are positioned on the doc.
+    //     let score = scorers[..pivot_len]
+    //         .iter_mut()
+    //         .map(|scorer| scorer.score())
+    //         .sum();
 
-        if score > threshold {
-            threshold = callback(pivot_doc, score);
-        }
-        // let's advance all of the scorers that are currently positioned on the pivot.
-        advance_all_scorers_on_pivot(&mut scorers, pivot_len);
-    }
+    //     if score > threshold {
+    //         threshold = callback(pivot_doc, score);
+    //     }
+    //     // let's advance all of the scorers that are currently positioned on the pivot.
+    //     advance_all_scorers_on_pivot(&mut scorers, pivot_len);
+    // }
+    todo!();
 }
 
 /// Specialized version of [`block_wand`] for a single scorer.
@@ -224,40 +222,42 @@ pub fn block_wand_single_scorer(
     mut threshold: Score,
     callback: &mut dyn FnMut(u32, Score) -> Score,
 ) {
-    let mut doc = scorer.doc();
-    loop {
-        // We position the scorer on a block that can reach
-        // the threshold.
-        while scorer.block_max_score() < threshold {
-            let last_doc_in_block = scorer.last_doc_in_block();
-            if last_doc_in_block == TERMINATED {
-                return;
-            }
-            doc = last_doc_in_block + 1;
-            scorer.seek_block(doc);
-        }
-        // Seek will effectively load that block.
-        doc = scorer.seek(doc);
-        if doc == TERMINATED {
-            break;
-        }
-        loop {
-            let score = scorer.score();
-            if score > threshold {
-                threshold = callback(doc, score);
-            }
-            debug_assert!(doc <= scorer.last_doc_in_block());
-            if doc == scorer.last_doc_in_block() {
-                break;
-            }
-            doc = scorer.advance();
-            if doc == TERMINATED {
-                return;
-            }
-        }
-        doc += 1;
-        scorer.seek_block(doc);
-    }
+    todo!();
+    // let mut doc = scorer.doc();
+    // let mut block_max_score = scorer.seek_block(doc);
+    // loop {
+    //     // We position the scorer on a block that can reach
+    //     // the threshold.
+    //     while block_max_score < threshold {
+    //         let last_doc_in_block = scorer.last_doc_in_block();
+    //         if last_doc_in_block == TERMINATED {
+    //             return;
+    //         }
+    //         doc = last_doc_in_block + 1;
+    //         block_max_score = scorer.seek_block(doc);
+    //     }
+    //     // Seek will effectively load that block.
+    //     doc = scorer.seek(doc);
+    //     if doc == TERMINATED {
+    //         break;
+    //     }
+    //     loop {
+    //         let score = scorer.score();
+    //         if score > threshold {
+    //             threshold = callback(doc, score);
+    //         }
+    //         debug_assert!(doc <= scorer.last_doc_in_block());
+    //         if doc == scorer.last_doc_in_block() {
+    //             break;
+    //         }
+    //         doc = scorer.advance();
+    //         if doc == TERMINATED {
+    //             return;
+    //         }
+    //     }
+    //     doc += 1;
+    //     block_max_score = scorer.seek_block(doc);
+    // }
 }
 
 struct TermScorerWithMaxScore<'a> {
@@ -305,6 +305,7 @@ mod tests {
 
     use proptest::prelude::*;
 
+    use crate::postings::SegmentPostings;
     use crate::query::score_combiner::SumCombiner;
     use crate::query::term_query::TermScorer;
     use crate::query::{Bm25Weight, BufferedUnionScorer, Scorer};

src/query/mod.rs

@@ -24,7 +24,7 @@ mod reqopt_scorer;
 mod scorer;
 mod set_query;
 mod size_hint;
-mod term_query;
+pub(crate) mod term_query;
 mod union;
 mod weight;
 

src/query/phrase_prefix_query/phrase_prefix_weight.rs

@@ -1,7 +1,7 @@
 use super::{prefix_end, PhrasePrefixScorer};
 use crate::fieldnorm::FieldNormReader;
 use crate::index::SegmentReader;
-use crate::postings::SegmentPostings;
+use crate::postings::Postings;
 use crate::query::bm25::Bm25Weight;
 use crate::query::explanation::does_not_match;
 use crate::query::{EmptyScorer, Explanation, Scorer, Weight};
@@ -46,13 +46,13 @@ impl PhrasePrefixWeight {
         &self,
         reader: &SegmentReader,
         boost: Score,
-    ) -> crate::Result<Option<PhrasePrefixScorer<SegmentPostings>>> {
+    ) -> crate::Result<Option<Box<dyn Scorer>>> {
         let similarity_weight_opt = self
             .similarity_weight_opt
             .as_ref()
             .map(|similarity_weight| similarity_weight.boost_by(boost));
         let fieldnorm_reader = self.fieldnorm_reader(reader)?;
-        let mut term_postings_list = Vec::new();
+        let mut term_postings_list: Vec<(usize, Box<dyn Postings>)> = Vec::new();
         for &(offset, ref term) in &self.phrase_terms {
             if let Some(postings) = reader
                 .inverted_index(term.field())?
@@ -103,42 +103,44 @@ impl PhrasePrefixWeight {
             }
         }
 
-        Ok(Some(PhrasePrefixScorer::new(
+        // TODO make this specialized.
+        Ok(Some(Box::new(PhrasePrefixScorer::new(
             term_postings_list,
             similarity_weight_opt,
             fieldnorm_reader,
             suffixes,
             self.prefix.0,
-        )))
+        ))))
     }
 }
 
 impl Weight for PhrasePrefixWeight {
     fn scorer(&self, reader: &SegmentReader, boost: Score) -> crate::Result<Box<dyn Scorer>> {
         if let Some(scorer) = self.phrase_scorer(reader, boost)? {
-            Ok(Box::new(scorer))
+            Ok(scorer)
         } else {
             Ok(Box::new(EmptyScorer))
         }
     }
 
     fn explain(&self, reader: &SegmentReader, doc: DocId) -> crate::Result<Explanation> {
-        let scorer_opt = self.phrase_scorer(reader, 1.0)?;
-        if scorer_opt.is_none() {
-            return Err(does_not_match(doc));
-        }
-        let mut scorer = scorer_opt.unwrap();
-        if scorer.seek(doc) != doc {
-            return Err(does_not_match(doc));
-        }
-        let fieldnorm_reader = self.fieldnorm_reader(reader)?;
-        let fieldnorm_id = fieldnorm_reader.fieldnorm_id(doc);
-        let phrase_count = scorer.phrase_count();
-        let mut explanation = Explanation::new("Phrase Prefix Scorer", scorer.score());
-        if let Some(similarity_weight) = self.similarity_weight_opt.as_ref() {
-            explanation.add_detail(similarity_weight.explain(fieldnorm_id, phrase_count));
-        }
-        Ok(explanation)
+        todo!();
+        // let scorer_opt = self.phrase_scorer(reader, 1.0)?;
+        // if scorer_opt.is_none() {
+        //     return Err(does_not_match(doc));
+        // }
+        // let mut scorer = scorer_opt.unwrap();
+        // if scorer.seek(doc) != doc {
+        //     return Err(does_not_match(doc));
+        // }
+        // let fieldnorm_reader = self.fieldnorm_reader(reader)?;
+        // let fieldnorm_id = fieldnorm_reader.fieldnorm_id(doc);
+        // let phrase_count = scorer.phrase_count();
+        // let mut explanation = Explanation::new("Phrase Prefix Scorer", scorer.score());
+        // if let Some(similarity_weight) = self.similarity_weight_opt.as_ref() {
+        //     explanation.add_detail(similarity_weight.explain(fieldnorm_id, phrase_count));
+        // }
+        // Ok(explanation)
     }
 }
 

src/query/phrase_query/phrase_weight.rs

@@ -1,7 +1,6 @@
 use super::PhraseScorer;
 use crate::fieldnorm::FieldNormReader;
 use crate::index::SegmentReader;
-use crate::postings::SegmentPostings;
 use crate::query::bm25::Bm25Weight;
 use crate::query::explanation::does_not_match;
 use crate::query::{EmptyScorer, Explanation, Scorer, Weight};
@@ -43,13 +42,14 @@ impl PhraseWeight {
         &self,
         reader: &SegmentReader,
         boost: Score,
-    ) -> crate::Result<Option<PhraseScorer<SegmentPostings>>> {
+    ) -> crate::Result<Option<Box<dyn Scorer>>> {
         let similarity_weight_opt = self
             .similarity_weight_opt
             .as_ref()
             .map(|similarity_weight| similarity_weight.boost_by(boost));
         let fieldnorm_reader = self.fieldnorm_reader(reader)?;
         let mut term_postings_list = Vec::new();
+        // TODO make it specialized
         for &(offset, ref term) in &self.phrase_terms {
             if let Some(postings) = reader
                 .inverted_index(term.field())?
@@ -60,12 +60,12 @@ impl PhraseWeight {
                 return Ok(None);
             }
         }
-        Ok(Some(PhraseScorer::new(
+        Ok(Some(Box::new(PhraseScorer::new(
             term_postings_list,
             similarity_weight_opt,
             fieldnorm_reader,
             self.slop,
-        )))
+        ))))
     }
 
     pub fn slop(&mut self, slop: u32) {
@@ -76,29 +76,30 @@ impl PhraseWeight {
 impl Weight for PhraseWeight {
     fn scorer(&self, reader: &SegmentReader, boost: Score) -> crate::Result<Box<dyn Scorer>> {
         if let Some(scorer) = self.phrase_scorer(reader, boost)? {
-            Ok(Box::new(scorer))
+            Ok(scorer)
         } else {
             Ok(Box::new(EmptyScorer))
         }
     }
 
     fn explain(&self, reader: &SegmentReader, doc: DocId) -> crate::Result<Explanation> {
-        let scorer_opt = self.phrase_scorer(reader, 1.0)?;
-        if scorer_opt.is_none() {
-            return Err(does_not_match(doc));
-        }
-        let mut scorer = scorer_opt.unwrap();
-        if scorer.seek(doc) != doc {
-            return Err(does_not_match(doc));
-        }
-        let fieldnorm_reader = self.fieldnorm_reader(reader)?;
-        let fieldnorm_id = fieldnorm_reader.fieldnorm_id(doc);
-        let phrase_count = scorer.phrase_count();
-        let mut explanation = Explanation::new("Phrase Scorer", scorer.score());
-        if let Some(similarity_weight) = self.similarity_weight_opt.as_ref() {
-            explanation.add_detail(similarity_weight.explain(fieldnorm_id, phrase_count));
-        }
-        Ok(explanation)
+        todo!();
+        // let scorer_opt = self.phrase_scorer(reader, 1.0)?;
+        // if scorer_opt.is_none() {
+        //     return Err(does_not_match(doc));
+        // }
+        // let mut scorer = scorer_opt.unwrap();
+        // if scorer.seek(doc) != doc {
+        //     return Err(does_not_match(doc));
+        // }
+        // let fieldnorm_reader = self.fieldnorm_reader(reader)?;
+        // let fieldnorm_id = fieldnorm_reader.fieldnorm_id(doc);
+        // let phrase_count = scorer.phrase_count();
+        // let mut explanation = Explanation::new("Phrase Scorer", scorer.score());
+        // if let Some(similarity_weight) = self.similarity_weight_opt.as_ref() {
+        //     explanation.add_detail(similarity_weight.explain(fieldnorm_id, phrase_count));
+        // }
+        // Ok(explanation)
     }
 }
 

src/query/phrase_query/regex_phrase_weight.rs

@@ -6,7 +6,7 @@ use tantivy_fst::Regex;
 use super::PhraseScorer;
 use crate::fieldnorm::FieldNormReader;
 use crate::index::SegmentReader;
-use crate::postings::{LoadedPostings, Postings, SegmentPostings, TermInfo};
+use crate::postings::{LoadedPostings, Postings, TermInfo};
 use crate::query::bm25::Bm25Weight;
 use crate::query::explanation::does_not_match;
 use crate::query::union::{BitSetPostingUnion, SimpleUnion};
@@ -103,18 +103,9 @@ impl RegexPhraseWeight {
         term_info: &TermInfo,
         doc_bitset: &mut BitSet,
     ) -> crate::Result<()> {
-        let mut block_segment_postings = inverted_index
-            .read_block_postings_from_terminfo(term_info, IndexRecordOption::Basic)?;
-        loop {
-            let docs = block_segment_postings.docs();
-            if docs.is_empty() {
-                break;
-            }
-            for &doc in docs {
-                doc_bitset.insert(doc);
-            }
-            block_segment_postings.advance();
-        }
+        let mut segment_postings =
+            inverted_index.read_postings_from_terminfo(term_info, IndexRecordOption::Basic)?;
+        segment_postings.fill_bitset(doc_bitset);
         Ok(())
     }
 
@@ -188,7 +179,7 @@ impl RegexPhraseWeight {
         // - Bucket 1: Terms appearing in 0.1% to 1% of documents
         // - Bucket 2: Terms appearing in 1% to 10% of documents
         // - Bucket 3: Terms appearing in more than 10% of documents
-        let mut buckets: Vec<(BitSet, Vec<SegmentPostings>)> = (0..4)
+        let mut buckets: Vec<(BitSet, Vec<Box<dyn Postings>>)> = (0..4)
             .map(|_| (BitSet::with_max_value(max_doc), Vec::new()))
             .collect();
 
@@ -311,7 +302,7 @@ mod tests {
         #![proptest_config(ProptestConfig::with_cases(50))]
         #[test]
         fn test_phrase_regex_with_random_strings(mut random_strings in proptest::collection::vec("[c-z ]{0,10}", 1..100), num_occurrences in 1..150_usize) {
-            let mut rng = rand::rng();
+            let mut rng = rand::thread_rng();
 
             // Insert "aaa ccc" the specified number of times into the list
             for _ in 0..num_occurrences {

src/query/range_query/range_query.rs

@@ -11,7 +11,7 @@ use crate::query::range_query::is_type_valid_for_fastfield_range_query;
 use crate::query::{BitSetDocSet, ConstScorer, EnableScoring, Explanation, Query, Scorer, Weight};
 use crate::schema::{Field, IndexRecordOption, Term, Type};
 use crate::termdict::{TermDictionary, TermStreamer};
-use crate::{DocId, Score};
+use crate::{DocId, DocSet, Score};
 
 /// `RangeQuery` matches all documents that have at least one term within a defined range.
 ///
@@ -228,18 +228,9 @@ impl Weight for InvertedIndexRangeWeight {
             }
             processed_count += 1;
             let term_info = term_range.value();
-            let mut block_segment_postings = inverted_index
-                .read_block_postings_from_terminfo(term_info, IndexRecordOption::Basic)?;
-            loop {
-                let docs = block_segment_postings.docs();
-                if docs.is_empty() {
-                    break;
-                }
-                for &doc in block_segment_postings.docs() {
-                    doc_bitset.insert(doc);
-                }
-                block_segment_postings.advance();
-            }
+            let mut postings =
+                inverted_index.read_postings_from_terminfo(term_info, IndexRecordOption::Basic)?;
+            postings.fill_bitset(&mut doc_bitset);
         }
         let doc_bitset = BitSetDocSet::from(doc_bitset);
         Ok(Box::new(ConstScorer::new(doc_bitset, boost)))
@@ -429,7 +420,7 @@ mod tests {
                 docs.push(doc);
             }
 
-            docs.shuffle(&mut rand::rng());
+            docs.shuffle(&mut rand::thread_rng());
             let mut docs_it = docs.into_iter();
             for doc in (&mut docs_it).take(50) {
                 index_writer.add_document(doc)?;

src/query/range_query/range_query_fastfield.rs

@@ -491,7 +491,7 @@ mod tests {
     use common::DateTime;
     use proptest::prelude::*;
     use rand::rngs::StdRng;
-    use rand::seq::IndexedRandom;
+    use rand::seq::SliceRandom;
     use rand::SeedableRng;
     use time::format_description::well_known::Rfc3339;
     use time::OffsetDateTime;

src/query/term_query/mod.rs

@@ -4,6 +4,7 @@ mod term_weight;
 
 pub use self::term_query::TermQuery;
 pub use self::term_scorer::TermScorer;
+
 #[cfg(test)]
 mod tests {
 

src/query/term_query/term_scorer.rs

@@ -1,23 +1,27 @@
+use crate::codec::postings::PostingsCodec;
+use crate::codec::{Codec, StandardCodec};
 use crate::docset::DocSet;
 use crate::fieldnorm::FieldNormReader;
-use crate::postings::{FreqReadingOption, Postings, SegmentPostings};
+use crate::postings::{FreqReadingOption, Postings};
 use crate::query::bm25::Bm25Weight;
 use crate::query::{Explanation, Scorer};
 use crate::{DocId, Score};
 
 #[derive(Clone)]
-pub struct TermScorer {
-    postings: SegmentPostings,
+pub struct TermScorer<
+    TPostings: Postings = <<StandardCodec as Codec>::PostingsCodec as PostingsCodec>::Postings,
+> {
+    postings: TPostings,
     fieldnorm_reader: FieldNormReader,
     similarity_weight: Bm25Weight,
 }
 
-impl TermScorer {
+impl<TPostings: Postings> TermScorer<TPostings> {
     pub fn new(
-        postings: SegmentPostings,
+        postings: TPostings,
         fieldnorm_reader: FieldNormReader,
         similarity_weight: Bm25Weight,
-    ) -> TermScorer {
+    ) -> TermScorer<TPostings> {
         TermScorer {
             postings,
             fieldnorm_reader,
@@ -25,53 +29,9 @@ impl TermScorer {
         }
     }
 
-    pub(crate) fn seek_block(&mut self, target_doc: DocId) {
-        self.postings.block_cursor.seek_block(target_doc);
-    }
-
-    #[cfg(test)]
-    pub fn create_for_test(
-        doc_and_tfs: &[(DocId, u32)],
-        fieldnorms: &[u32],
-        similarity_weight: Bm25Weight,
-    ) -> TermScorer {
-        assert!(!doc_and_tfs.is_empty());
-        assert!(
-            doc_and_tfs
-                .iter()
-                .map(|(doc, _tf)| *doc)
-                .max()
-                .unwrap_or(0u32)
-                < fieldnorms.len() as u32
-        );
-        let segment_postings =
-            SegmentPostings::create_from_docs_and_tfs(doc_and_tfs, Some(fieldnorms));
-        let fieldnorm_reader = FieldNormReader::for_test(fieldnorms);
-        TermScorer::new(segment_postings, fieldnorm_reader, similarity_weight)
-    }
-
     /// See `FreqReadingOption`.
     pub(crate) fn freq_reading_option(&self) -> FreqReadingOption {
-        self.postings.block_cursor.freq_reading_option()
-    }
-
-    /// Returns the maximum score for the current block.
-    ///
-    /// In some rare case, the result may not be exact. In this case a lower value is returned,
-    /// (and may lead us to return a lesser document).
-    ///
-    /// At index time, we store the (fieldnorm_id, term frequency) pair that maximizes the
-    /// score assuming the average fieldnorm computed on this segment.
-    ///
-    /// Though extremely rare, it is theoretically possible that the actual average fieldnorm
-    /// is different enough from the current segment average fieldnorm that the maximum over a
-    /// specific is achieved on a different document.
-    ///
-    /// (The result is on the other hand guaranteed to be correct if there is only one segment).
-    pub fn block_max_score(&mut self) -> Score {
-        self.postings
-            .block_cursor
-            .block_max_score(&self.fieldnorm_reader, &self.similarity_weight)
+        self.postings.freq_reading_option()
     }
 
     pub fn term_freq(&self) -> u32 {
@@ -93,11 +53,39 @@ impl TermScorer {
     }
 
     pub fn last_doc_in_block(&self) -> DocId {
-        self.postings.block_cursor.skip_reader().last_doc_in_block()
+        self.postings.last_doc_in_block()
+    }
+
+    pub(crate) fn seek_block(&mut self, target_doc: DocId) -> Score {
+        self.postings
+            .seek_block(target_doc, &self.fieldnorm_reader, &self.similarity_weight)
     }
 }
 
-impl DocSet for TermScorer {
+impl TermScorer {
+    #[cfg(test)]
+    pub fn create_for_test(
+        doc_and_tfs: &[(DocId, u32)],
+        fieldnorms: &[u32],
+        similarity_weight: Bm25Weight,
+    ) -> TermScorer {
+        assert!(!doc_and_tfs.is_empty());
+        assert!(
+            doc_and_tfs
+                .iter()
+                .map(|(doc, _tf)| *doc)
+                .max()
+                .unwrap_or(0u32)
+                < fieldnorms.len() as u32
+        );
+        let segment_postings: SegmentPostings =
+            SegmentPostings::create_from_docs_and_tfs(doc_and_tfs, Some(fieldnorms));
+        let fieldnorm_reader = FieldNormReader::for_test(fieldnorms);
+        TermScorer::new(segment_postings, fieldnorm_reader, similarity_weight)
+    }
+}
+
+impl<TPostings: Postings> DocSet for TermScorer<TPostings> {
     #[inline]
     fn advance(&mut self) -> DocId {
         self.postings.advance()
@@ -118,7 +106,7 @@ impl DocSet for TermScorer {
     }
 }
 
-impl Scorer for TermScorer {
+impl<TPostings: Postings> Scorer for TermScorer<TPostings> {
     #[inline]
     fn score(&mut self) -> Score {
         let fieldnorm_id = self.fieldnorm_id();
@@ -154,7 +142,7 @@ mod tests {
         crate::assert_nearly_equals!(max_scorer, 1.3990127);
         assert_eq!(term_scorer.doc(), 2);
         assert_eq!(term_scorer.term_freq(), 3);
-        assert_nearly_equals!(term_scorer.block_max_score(), 1.3676447);
+        assert_nearly_equals!(term_scorer.seek_block(2), 1.3676447);
         assert_nearly_equals!(term_scorer.score(), 1.0892314);
         assert_eq!(term_scorer.advance(), 3);
         assert_eq!(term_scorer.doc(), 3);
@@ -217,7 +205,7 @@ mod tests {
 
          let docs: Vec<DocId> = (0..term_doc_freq).map(|doc| doc as DocId).collect();
          for block in docs.chunks(COMPRESSION_BLOCK_SIZE) {
-             let block_max_score: Score = term_scorer.block_max_score();
+             let block_max_score: Score = term_scorer.seek_block(0);
              let mut block_max_score_computed: Score = 0.0;
              for &doc in block {
                 assert_eq!(term_scorer.doc(), doc);
@@ -245,14 +233,12 @@ mod tests {
         let fieldnorms: Vec<u32> = std::iter::repeat_n(20u32, 300).collect();
         let bm25_weight = Bm25Weight::for_one_term(10, 129, 20.0);
         let mut docs = TermScorer::create_for_test(&doc_tfs[..], &fieldnorms[..], bm25_weight);
-        assert_nearly_equals!(docs.block_max_score(), 2.5161593);
-        docs.seek_block(135);
-        assert_nearly_equals!(docs.block_max_score(), 3.4597192);
-        docs.seek_block(256);
+        assert_nearly_equals!(docs.seek_block(0), 2.5161593);
+        assert_nearly_equals!(docs.seek_block(135), 3.4597192);
         // the block is not loaded yet.
-        assert_nearly_equals!(docs.block_max_score(), 5.2971773);
+        assert_nearly_equals!(docs.seek_block(256), 5.2971773);
         assert_eq!(256, docs.seek(256));
-        assert_nearly_equals!(docs.block_max_score(), 3.9539647);
+        assert_nearly_equals!(docs.seek_block(256), 3.9539647);
     }
 
     fn test_block_wand_aux(term_query: &TermQuery, searcher: &Searcher) -> crate::Result<()> {
@@ -279,8 +265,8 @@ mod tests {
             {
                 let mut term_scorer = term_weight.term_scorer_for_test(reader, 1.0)?.unwrap();
                 for d in docs {
-                    term_scorer.seek_block(d);
-                    block_max_scores_b.push(term_scorer.block_max_score());
+                    let block_max_score = term_scorer.seek_block(d);
+                    block_max_scores_b.push(block_max_score);
                 }
             }
             for (l, r) in block_max_scores
@@ -304,10 +290,10 @@ mod tests {
         let mut writer: IndexWriter =
             index.writer_with_num_threads(3, 3 * MEMORY_BUDGET_NUM_BYTES_MIN)?;
         use rand::Rng;
-        let mut rng = rand::rng();
+        let mut rng = rand::thread_rng();
         writer.set_merge_policy(Box::new(NoMergePolicy));
         for _ in 0..3_000 {
-            let term_freq = rng.random_range(1..10000);
+            let term_freq = rng.gen_range(1..10000);
             let words: Vec<&str> = std::iter::repeat_n("bbbb", term_freq).collect();
             let text = words.join(" ");
             writer.add_document(doc!(text_field=>text))?;

src/query/term_query/term_weight.rs

@@ -2,7 +2,7 @@ use super::term_scorer::TermScorer;
 use crate::docset::{DocSet, COLLECT_BLOCK_BUFFER_LEN};
 use crate::fieldnorm::FieldNormReader;
 use crate::index::SegmentReader;
-use crate::postings::SegmentPostings;
+use crate::postings::Postings;
 use crate::query::bm25::Bm25Weight;
 use crate::query::explanation::does_not_match;
 use crate::query::weight::{for_each_docset_buffered, for_each_scorer};
@@ -18,7 +18,7 @@ pub struct TermWeight {
 }
 
 enum TermOrEmptyOrAllScorer {
-    TermScorer(Box<TermScorer>),
+    TermScorer(Box<dyn Scorer>),
     Empty,
     AllMatch(AllScorer),
 }
@@ -39,18 +39,19 @@ impl Weight for TermWeight {
     }
 
     fn explain(&self, reader: &SegmentReader, doc: DocId) -> crate::Result<Explanation> {
-        match self.specialized_scorer(reader, 1.0)? {
-            TermOrEmptyOrAllScorer::TermScorer(mut term_scorer) => {
-                if term_scorer.doc() > doc || term_scorer.seek(doc) != doc {
-                    return Err(does_not_match(doc));
-                }
-                let mut explanation = term_scorer.explain();
-                explanation.add_context(format!("Term={:?}", self.term,));
-                Ok(explanation)
-            }
-            TermOrEmptyOrAllScorer::Empty => Err(does_not_match(doc)),
-            TermOrEmptyOrAllScorer::AllMatch(_) => AllWeight.explain(reader, doc),
-        }
+        todo!();
+        // match self.specialized_scorer(reader, 1.0)? {
+        //     TermOrEmptyOrAllScorer::TermScorer(mut term_scorer) => {
+        //         if term_scorer.doc() > doc || term_scorer.seek(doc) != doc {
+        //             return Err(does_not_match(doc));
+        //         }
+        //         let mut explanation = term_scorer.explain();
+        //         explanation.add_context(format!("Term={:?}", self.term,));
+        //         Ok(explanation)
+        //     }
+        //     TermOrEmptyOrAllScorer::Empty => Err(does_not_match(doc)),
+        //     TermOrEmptyOrAllScorer::AllMatch(_) => AllWeight.explain(reader, doc),
+        // }
     }
 
     fn count(&self, reader: &SegmentReader) -> crate::Result<u32> {
@@ -124,11 +125,12 @@ impl Weight for TermWeight {
         let specialized_scorer = self.specialized_scorer(reader, 1.0)?;
         match specialized_scorer {
             TermOrEmptyOrAllScorer::TermScorer(term_scorer) => {
-                crate::query::boolean_query::block_wand_single_scorer(
-                    *term_scorer,
-                    threshold,
-                    callback,
-                );
+                todo!();
+                // crate::query::boolean_query::block_wand_single_scorer(
+                //     *term_scorer,
+                //     threshold,
+                //     callback,
+                // );
             }
             TermOrEmptyOrAllScorer::Empty => {}
             TermOrEmptyOrAllScorer::AllMatch(_) => {
@@ -168,10 +170,10 @@ impl TermWeight {
         &self,
         reader: &SegmentReader,
         boost: Score,
-    ) -> crate::Result<Option<TermScorer>> {
+    ) -> crate::Result<Option<Box<dyn Scorer>>> {
         let scorer = self.specialized_scorer(reader, boost)?;
         Ok(match scorer {
-            TermOrEmptyOrAllScorer::TermScorer(scorer) => Some(*scorer),
+            TermOrEmptyOrAllScorer::TermScorer(scorer) => Some(scorer),
             _ => None,
         })
     }
@@ -196,14 +198,14 @@ impl TermWeight {
             )));
         }
 
-        let segment_postings: SegmentPostings =
+        let segment_postings: Box<dyn Postings> =
             inverted_index.read_postings_from_terminfo(&term_info, self.index_record_option)?;
 
         let fieldnorm_reader = self.fieldnorm_reader(reader)?;
         let similarity_weight = self.similarity_weight.boost_by(boost);
-        Ok(TermOrEmptyOrAllScorer::TermScorer(Box::new(
-            TermScorer::new(segment_postings, fieldnorm_reader, similarity_weight),
-        )))
+        Ok(TermOrEmptyOrAllScorer::TermScorer(
+            segment_postings.new_term_scorer(fieldnorm_reader, similarity_weight),
+        ))
     }
 
     fn fieldnorm_reader(&self, segment_reader: &SegmentReader) -> crate::Result<FieldNormReader> {

src/query/union/bitset_union.rs

@@ -1,7 +1,7 @@
 use std::cell::RefCell;
 
 use crate::docset::DocSet;
-use crate::postings::Postings;
+use crate::postings::{FreqReadingOption, Postings};
 use crate::query::BitSetDocSet;
 use crate::DocId;
 
@@ -31,6 +31,15 @@ impl<TDocSet: DocSet> BitSetPostingUnion<TDocSet> {
 }
 
 impl<TDocSet: Postings> Postings for BitSetPostingUnion<TDocSet> {
+    fn new_term_scorer(
+        self: Box<Self>,
+        fieldnorm_reader: crate::fieldnorm::FieldNormReader,
+        similarity_weight: crate::query::Bm25Weight,
+    ) -> Box<dyn crate::query::Scorer> {
+        use crate::query::term_query::TermScorer;
+        Box::new(TermScorer::new(*self, fieldnorm_reader, similarity_weight))
+    }
+
     fn term_freq(&self) -> u32 {
         let curr_doc = self.bitset.doc();
         let mut term_freq = 0;
@@ -46,6 +55,10 @@ impl<TDocSet: Postings> Postings for BitSetPostingUnion<TDocSet> {
         term_freq
     }
 
+    fn freq_reading_option(&self) -> FreqReadingOption {
+        FreqReadingOption::ReadFreq
+    }
+
     fn append_positions_with_offset(&mut self, offset: u32, output: &mut Vec<u32>) {
         let curr_doc = self.bitset.doc();
         let mut docsets = self.docsets.borrow_mut();

src/query/union/simple_union.rs

@@ -1,5 +1,5 @@
 use crate::docset::{DocSet, TERMINATED};
-use crate::postings::Postings;
+use crate::postings::{FreqReadingOption, Postings};
 use crate::DocId;
 
 /// A `SimpleUnion` is a `DocSet` that is the union of multiple `DocSet`.
@@ -45,6 +45,15 @@ impl<TDocSet: DocSet> SimpleUnion<TDocSet> {
 }
 
 impl<TDocSet: Postings> Postings for SimpleUnion<TDocSet> {
+    fn new_term_scorer(
+        self: Box<Self>,
+        fieldnorm_reader: crate::fieldnorm::FieldNormReader,
+        similarity_weight: crate::query::Bm25Weight,
+    ) -> Box<dyn crate::query::Scorer> {
+        use crate::query::term_query::TermScorer;
+        Box::new(TermScorer::new(*self, fieldnorm_reader, similarity_weight))
+    }
+
     fn term_freq(&self) -> u32 {
         let mut term_freq = 0;
         for docset in &self.docsets {
@@ -56,6 +65,10 @@ impl<TDocSet: Postings> Postings for SimpleUnion<TDocSet> {
         term_freq
     }
 
+    fn freq_reading_option(&self) -> FreqReadingOption {
+        FreqReadingOption::ReadFreq
+    }
+
     fn append_positions_with_offset(&mut self, offset: u32, output: &mut Vec<u32>) {
         for docset in &mut self.docsets {
             let doc = docset.doc();

src/reader/mod.rs

@@ -7,6 +7,7 @@ use arc_swap::ArcSwap;
 pub use warming::Warmer;
 
 use self::warming::WarmingState;
+use crate::codec::Codec;
 use crate::core::searcher::{SearcherGeneration, SearcherInner};
 use crate::directory::{Directory, WatchCallback, WatchHandle, META_LOCK};
 use crate::store::DOCSTORE_CACHE_CAPACITY;
@@ -38,17 +39,17 @@ pub enum ReloadPolicy {
 /// - number of warming threads, for parallelizing warming work
 /// - The cache size of the underlying doc store readers.
 #[derive(Clone)]
-pub struct IndexReaderBuilder {
+pub struct IndexReaderBuilder<C: Codec = crate::codec::StandardCodec> {
     reload_policy: ReloadPolicy,
-    index: Index,
+    index: Index<C>,
     warmers: Vec<Weak<dyn Warmer>>,
     num_warming_threads: usize,
     doc_store_cache_num_blocks: usize,
 }
 
-impl IndexReaderBuilder {
+impl<C: Codec> IndexReaderBuilder<C> {
     #[must_use]
-    pub(crate) fn new(index: Index) -> IndexReaderBuilder {
+    pub(crate) fn new(index: Index<C>) -> IndexReaderBuilder<C> {
         IndexReaderBuilder {
             reload_policy: ReloadPolicy::OnCommitWithDelay,
             index,
@@ -63,7 +64,7 @@ impl IndexReaderBuilder {
     /// Building the reader is a non-trivial operation that requires
     /// to open different segment readers. It may take hundreds of milliseconds
     /// of time and it may return an error.
-    pub fn try_into(self) -> crate::Result<IndexReader> {
+    pub fn try_into(self) -> crate::Result<IndexReader<C>> {
         let searcher_generation_inventory = Inventory::default();
         let warming_state = WarmingState::new(
             self.num_warming_threads,
@@ -106,7 +107,7 @@ impl IndexReaderBuilder {
     ///
     /// See [`ReloadPolicy`] for more details.
     #[must_use]
-    pub fn reload_policy(mut self, reload_policy: ReloadPolicy) -> IndexReaderBuilder {
+    pub fn reload_policy(mut self, reload_policy: ReloadPolicy) -> IndexReaderBuilder<C> {
         self.reload_policy = reload_policy;
         self
     }
@@ -118,14 +119,14 @@ impl IndexReaderBuilder {
     pub fn doc_store_cache_num_blocks(
         mut self,
         doc_store_cache_num_blocks: usize,
-    ) -> IndexReaderBuilder {
+    ) -> IndexReaderBuilder<C> {
         self.doc_store_cache_num_blocks = doc_store_cache_num_blocks;
         self
     }
 
     /// Set the [`Warmer`]s that are invoked when reloading searchable segments.
     #[must_use]
-    pub fn warmers(mut self, warmers: Vec<Weak<dyn Warmer>>) -> IndexReaderBuilder {
+    pub fn warmers(mut self, warmers: Vec<Weak<dyn Warmer>>) -> IndexReaderBuilder<C> {
         self.warmers = warmers;
         self
     }
@@ -135,33 +136,33 @@ impl IndexReaderBuilder {
     /// This allows parallelizing warming work when there are multiple [`Warmer`] registered with
     /// the [`IndexReader`].
     #[must_use]
-    pub fn num_warming_threads(mut self, num_warming_threads: usize) -> IndexReaderBuilder {
+    pub fn num_warming_threads(mut self, num_warming_threads: usize) -> IndexReaderBuilder<C> {
         self.num_warming_threads = num_warming_threads;
         self
     }
 }
 
-impl TryInto<IndexReader> for IndexReaderBuilder {
+impl<C: Codec> TryInto<IndexReader<C>> for IndexReaderBuilder<C> {
     type Error = crate::TantivyError;
 
-    fn try_into(self) -> crate::Result<IndexReader> {
+    fn try_into(self) -> crate::Result<IndexReader<C>> {
         IndexReaderBuilder::try_into(self)
     }
 }
 
-struct InnerIndexReader {
+struct InnerIndexReader<C: Codec> {
     doc_store_cache_num_blocks: usize,
-    index: Index,
+    index: Index<C>,
     warming_state: WarmingState,
     searcher: arc_swap::ArcSwap<SearcherInner>,
     searcher_generation_counter: Arc<AtomicU64>,
     searcher_generation_inventory: Inventory<SearcherGeneration>,
 }
 
-impl InnerIndexReader {
+impl<C: Codec> InnerIndexReader<C> {
     fn new(
         doc_store_cache_num_blocks: usize,
-        index: Index,
+        index: Index<C>,
         warming_state: WarmingState,
         // The searcher_generation_inventory is not used as source, but as target to track the
         // loaded segments.
@@ -189,7 +190,7 @@ impl InnerIndexReader {
     ///
     /// This function acquires a lock to prevent GC from removing files
     /// as we are opening our index.
-    fn open_segment_readers(index: &Index) -> crate::Result<Vec<SegmentReader>> {
+    fn open_segment_readers(index: &Index<C>) -> crate::Result<Vec<SegmentReader>> {
         // Prevents segment files from getting deleted while we are in the process of opening them
         let _meta_lock = index.directory().acquire_lock(&META_LOCK)?;
         let searchable_segments = index.searchable_segments()?;
@@ -212,7 +213,7 @@ impl InnerIndexReader {
     }
 
     fn create_searcher(
-        index: &Index,
+        index: &Index<C>,
         doc_store_cache_num_blocks: usize,
         warming_state: &WarmingState,
         searcher_generation_counter: &Arc<AtomicU64>,
@@ -226,9 +227,10 @@ impl InnerIndexReader {
         );
 
         let schema = index.schema();
+        // SearcherInner uses Index<StandardCodec> since the codec doesn't affect reading
         let searcher = Arc::new(SearcherInner::new(
             schema,
-            index.clone(),
+            index.with_standard_codec(),
             segment_readers,
             searcher_generation,
             doc_store_cache_num_blocks,
@@ -264,14 +266,14 @@ impl InnerIndexReader {
 ///
 /// `IndexReader` just wraps an `Arc`.
 #[derive(Clone)]
-pub struct IndexReader {
-    inner: Arc<InnerIndexReader>,
+pub struct IndexReader<C: Codec = crate::codec::StandardCodec> {
+    inner: Arc<InnerIndexReader<C>>,
     _watch_handle_opt: Option<WatchHandle>,
 }
 
-impl IndexReader {
+impl<C: Codec> IndexReader<C> {
     #[cfg(test)]
-    pub(crate) fn index(&self) -> Index {
+    pub(crate) fn index(&self) -> Index<C> {
         self.inner.index.clone()
     }
 

src/termdict/fst_termdict/merger.rs

@@ -95,7 +95,7 @@ impl<'a> TermMerger<'a> {
 #[cfg(all(test, feature = "unstable"))]
 mod bench {
     use rand::distributions::Alphanumeric;
-    use rand::{rng, Rng};
+    use rand::{thread_rng, Rng};
     use test::{self, Bencher};
 
     use super::TermMerger;
@@ -117,9 +117,9 @@ mod bench {
         let buffer: Vec<u8> = {
             let mut terms = vec![];
             for _i in 0..num_terms {
-                let rand_string: String = rng()
+                let rand_string: String = thread_rng()
                     .sample_iter(&Alphanumeric)
-                    .take(rng().random_range(30..42))
+                    .take(thread_rng().gen_range(30..42))
                     .map(char::from)
                     .collect();
                 terms.push(rand_string);

sstable/Cargo.toml

@@ -25,7 +25,7 @@ zstd-compression = ["zstd"]
 proptest = "1"
 criterion = { version = "0.5", default-features = false }
 names = "0.14"
-rand = "0.9"
+rand = "0.8"
 
 [[bench]]
 name = "stream_bench"

sstable/benches/stream_bench.rs

@@ -10,9 +10,9 @@ use tantivy_sstable::{Dictionary, MonotonicU64SSTable};
 const CHARSET: &[u8] = b"abcdefghij";
 
 fn generate_key(rng: &mut impl Rng) -> String {
-    let len = rng.random_range(3..12);
+    let len = rng.gen_range(3..12);
     std::iter::from_fn(|| {
-        let idx = rng.random_range(0..CHARSET.len());
+        let idx = rng.gen_range(0..CHARSET.len());
         Some(CHARSET[idx] as char)
     })
     .take(len)

stacker/Cargo.toml

@@ -23,12 +23,12 @@ name = "hashmap"
 path = "example/hashmap.rs"
 
 [dev-dependencies]
-rand = "0.9"
+rand = "0.8.5"
 zipf = "7.0.0"
 rustc-hash = "2.1.0"
 proptest = "1.2.0"
 binggan = { version = "0.14.0" }
-rand_distr = "0.5"
+rand_distr = "0.4.3"
 
 [features]
 compare_hash_only = ["ahash"] # Compare hash only, not the key in the Hashmap

stacker/benches/bench.rs

@@ -90,10 +90,10 @@ fn bench_vint() {
             }
             // benchmark zipfs distribution numbers
             {
-                use rand::distr::Distribution;
+                use rand::distributions::Distribution;
                 use rand::rngs::StdRng;
                 let mut rng = StdRng::from_seed([3u8; 32]);
-                let zipf = rand_distr::Zipf::new(10_000.0f64, 1.03).unwrap();
+                let zipf = zipf::ZipfDistribution::new(10_000, 1.03).unwrap();
                 let numbers: Vec<[u8; 8]> = (0..num_numbers)
                     .map(|_| zipf.sample(&mut rng).to_le_bytes())
                     .collect();

stacker/fuzz_test/Cargo.toml

@@ -7,8 +7,8 @@ edition = "2021"
 
 [dependencies]
 ahash = "0.8.7"
-rand = "0.9"
-rand_distr = "0.5"
+rand = "0.8.5"
+rand_distr = "0.4.3"
 tantivy-stacker = { version = "0.2.0", path = ".." }
 
 [workspace]

stacker/fuzz_test/src/main.rs

@@ -14,7 +14,7 @@ fn test_with_seed(seed: u64) {
     let mut hash_map = AHashMap::new();
     let mut arena_hashmap = ArenaHashMap::default();
     let mut rng = StdRng::seed_from_u64(seed);
-    let key_count = rng.random_range(1_000..=1_000_000);
+    let key_count = rng.gen_range(1_000..=1_000_000);
     let exp = Exp::new(0.05).unwrap();
 
     for _ in 0..key_count {