Fixed agg validation

Co-authored-by: Ming <ming.ying.nyc@gmail.com>

benches/agg_bench.rs

@@ -54,33 +54,33 @@ fn bench_agg(mut group: InputGroup<Index>) {
     register!(group, stats_f64);
     register!(group, extendedstats_f64);
     register!(group, percentiles_f64);
-    register!(group, terms_7);
+    register!(group, terms_few);
     register!(group, terms_all_unique);
-    register!(group, terms_150_000);
+    register!(group, terms_many);
     register!(group, terms_many_top_1000);
     register!(group, terms_many_order_by_term);
     register!(group, terms_many_with_top_hits);
     register!(group, terms_all_unique_with_avg_sub_agg);
     register!(group, terms_many_with_avg_sub_agg);
+    register!(group, terms_few_with_avg_sub_agg);
     register!(group, terms_status_with_avg_sub_agg);
+    register!(group, terms_status);
+    register!(group, terms_few_with_histogram);
     register!(group, terms_status_with_histogram);
-    register!(group, terms_zipf_1000);
-    register!(group, terms_zipf_1000_with_histogram);
-    register!(group, terms_zipf_1000_with_avg_sub_agg);
 
     register!(group, terms_many_json_mixed_type_with_avg_sub_agg);
 
     register!(group, cardinality_agg);
-    register!(group, terms_status_with_cardinality_agg);
+    register!(group, terms_few_with_cardinality_agg);
 
     register!(group, range_agg);
     register!(group, range_agg_with_avg_sub_agg);
-    register!(group, range_agg_with_term_agg_status);
+    register!(group, range_agg_with_term_agg_few);
     register!(group, range_agg_with_term_agg_many);
     register!(group, histogram);
     register!(group, histogram_hard_bounds);
     register!(group, histogram_with_avg_sub_agg);
-    register!(group, histogram_with_term_agg_status);
+    register!(group, histogram_with_term_agg_few);
     register!(group, avg_and_range_with_avg_sub_agg);
 
     // Filter aggregation benchmarks
@@ -159,10 +159,10 @@ fn cardinality_agg(index: &Index) {
     });
     execute_agg(index, agg_req);
 }
-fn terms_status_with_cardinality_agg(index: &Index) {
+fn terms_few_with_cardinality_agg(index: &Index) {
     let agg_req = json!({
         "my_texts": {
-            "terms": { "field": "text_few_terms_status" },
+            "terms": { "field": "text_few_terms" },
             "aggs": {
                 "cardinality": {
                     "cardinality": {
@@ -175,7 +175,13 @@ fn terms_status_with_cardinality_agg(index: &Index) {
     execute_agg(index, agg_req);
 }
 
-fn terms_7(index: &Index) {
+fn terms_few(index: &Index) {
+    let agg_req = json!({
+        "my_texts": { "terms": { "field": "text_few_terms" } },
+    });
+    execute_agg(index, agg_req);
+}
+fn terms_status(index: &Index) {
     let agg_req = json!({
         "my_texts": { "terms": { "field": "text_few_terms_status" } },
     });
@@ -188,7 +194,7 @@ fn terms_all_unique(index: &Index) {
     execute_agg(index, agg_req);
 }
 
-fn terms_150_000(index: &Index) {
+fn terms_many(index: &Index) {
     let agg_req = json!({
         "my_texts": { "terms": { "field": "text_many_terms" } },
     });
@@ -247,6 +253,17 @@ fn terms_all_unique_with_avg_sub_agg(index: &Index) {
     });
     execute_agg(index, agg_req);
 }
+fn terms_few_with_histogram(index: &Index) {
+    let agg_req = json!({
+        "my_texts": {
+            "terms": { "field": "text_few_terms" },
+            "aggs": {
+                "histo": {"histogram": { "field": "score_f64", "interval": 10 }}
+            }
+        }
+    });
+    execute_agg(index, agg_req);
+}
 fn terms_status_with_histogram(index: &Index) {
     let agg_req = json!({
         "my_texts": {
@@ -259,18 +276,17 @@ fn terms_status_with_histogram(index: &Index) {
     execute_agg(index, agg_req);
 }
 
-fn terms_zipf_1000_with_histogram(index: &Index) {
+fn terms_few_with_avg_sub_agg(index: &Index) {
     let agg_req = json!({
         "my_texts": {
-            "terms": { "field": "text_1000_terms_zipf" },
+            "terms": { "field": "text_few_terms" },
             "aggs": {
-                "histo": {"histogram": { "field": "score_f64", "interval": 10 }}
+                "average_f64": { "avg": { "field": "score_f64" } }
             }
-        }
+        },
     });
     execute_agg(index, agg_req);
 }
-
 fn terms_status_with_avg_sub_agg(index: &Index) {
     let agg_req = json!({
         "my_texts": {
@@ -283,25 +299,6 @@ fn terms_status_with_avg_sub_agg(index: &Index) {
     execute_agg(index, agg_req);
 }
 
-fn terms_zipf_1000_with_avg_sub_agg(index: &Index) {
-    let agg_req = json!({
-        "my_texts": {
-            "terms": { "field": "text_1000_terms_zipf" },
-            "aggs": {
-                "average_f64": { "avg": { "field": "score_f64" } }
-            }
-        },
-    });
-    execute_agg(index, agg_req);
-}
-
-fn terms_zipf_1000(index: &Index) {
-    let agg_req = json!({
-        "my_texts": { "terms": { "field": "text_1000_terms_zipf" } },
-    });
-    execute_agg(index, agg_req);
-}
-
 fn terms_many_json_mixed_type_with_avg_sub_agg(index: &Index) {
     let agg_req = json!({
         "my_texts": {
@@ -357,7 +354,7 @@ fn range_agg_with_avg_sub_agg(index: &Index) {
     execute_agg(index, agg_req);
 }
 
-fn range_agg_with_term_agg_status(index: &Index) {
+fn range_agg_with_term_agg_few(index: &Index) {
     let agg_req = json!({
         "rangef64": {
             "range": {
@@ -372,7 +369,7 @@ fn range_agg_with_term_agg_status(index: &Index) {
                 ]
             },
             "aggs": {
-                "my_texts": { "terms": { "field": "text_few_terms_status" } },
+                "my_texts": { "terms": { "field": "text_few_terms" } },
             }
         },
     });
@@ -428,12 +425,12 @@ fn histogram_with_avg_sub_agg(index: &Index) {
     });
     execute_agg(index, agg_req);
 }
-fn histogram_with_term_agg_status(index: &Index) {
+fn histogram_with_term_agg_few(index: &Index) {
     let agg_req = json!({
         "rangef64": {
             "histogram": { "field": "score_f64", "interval": 10 },
             "aggs": {
-                "my_texts": { "terms": { "field": "text_few_terms_status" } }
+                "my_texts": { "terms": { "field": "text_few_terms" } }
             }
         }
     });
@@ -478,13 +475,6 @@ fn get_collector(agg_req: Aggregations) -> AggregationCollector {
 }
 
 fn get_test_index_bench(cardinality: Cardinality) -> tantivy::Result<Index> {
-    // Flag to use existing index
-    let reuse_index = std::env::var("REUSE_AGG_BENCH_INDEX").is_ok();
-    if reuse_index && std::path::Path::new("agg_bench").exists() {
-        return Index::open_in_dir("agg_bench");
-    }
-    // crreate dir
-    std::fs::create_dir_all("agg_bench")?;
     let mut schema_builder = Schema::builder();
     let text_fieldtype = tantivy::schema::TextOptions::default()
         .set_indexing_options(
@@ -496,44 +486,24 @@ fn get_test_index_bench(cardinality: Cardinality) -> tantivy::Result<Index> {
     let text_field_all_unique_terms =
         schema_builder.add_text_field("text_all_unique_terms", STRING | FAST);
     let text_field_many_terms = schema_builder.add_text_field("text_many_terms", STRING | FAST);
+    let text_field_many_terms = schema_builder.add_text_field("text_many_terms", STRING | FAST);
+    let text_field_few_terms = schema_builder.add_text_field("text_few_terms", STRING | FAST);
     let text_field_few_terms_status =
         schema_builder.add_text_field("text_few_terms_status", STRING | FAST);
-    let text_field_1000_terms_zipf =
-        schema_builder.add_text_field("text_1000_terms_zipf", STRING | FAST);
     let score_fieldtype = tantivy::schema::NumericOptions::default().set_fast();
     let score_field = schema_builder.add_u64_field("score", score_fieldtype.clone());
     let score_field_f64 = schema_builder.add_f64_field("score_f64", score_fieldtype.clone());
     let score_field_i64 = schema_builder.add_i64_field("score_i64", score_fieldtype);
-    // use tmp dir
-    let index = if reuse_index {
-        Index::create_in_dir("agg_bench", schema_builder.build())?
-    } else {
-        Index::create_from_tempdir(schema_builder.build())?
-    };
-    // Approximate log proportions
-    let status_field_data = [
-        ("INFO", 8000),
-        ("ERROR", 300),
-        ("WARN", 1200),
-        ("DEBUG", 500),
-        ("OK", 500),
-        ("CRITICAL", 20),
-        ("EMERGENCY", 1),
-    ];
-    let log_level_distribution =
-        WeightedIndex::new(status_field_data.iter().map(|item| item.1)).unwrap();
+    let index = Index::create_from_tempdir(schema_builder.build())?;
+    let few_terms_data = ["INFO", "ERROR", "WARN", "DEBUG"];
+    // Approximate production log proportions: INFO dominant, WARN and DEBUG occasional, ERROR rare.
+    let log_level_distribution = WeightedIndex::new([80u32, 3, 12, 5]).unwrap();
 
     let lg_norm = rand_distr::LogNormal::new(2.996f64, 0.979f64).unwrap();
 
     let many_terms_data = (0..150_000)
         .map(|num| format!("author{num}"))
         .collect::<Vec<_>>();
-
-    // Prepare 1000 unique terms sampled using a Zipf distribution.
-    // Exponent ~1.1 approximates top-20 terms covering around ~20%.
-    let terms_1000: Vec<String> = (1..=1000).map(|i| format!("term_{i}")).collect();
-    let zipf_1000 = rand_distr::Zipf::new(1000, 1.1f64).unwrap();
-
     {
         let mut rng = StdRng::from_seed([1u8; 32]);
         let mut index_writer = index.writer_with_num_threads(1, 200_000_000)?;
@@ -543,12 +513,8 @@ fn get_test_index_bench(cardinality: Cardinality) -> tantivy::Result<Index> {
             index_writer.add_document(doc!())?;
         }
         if cardinality == Cardinality::Multivalued {
-            let log_level_sample_a = status_field_data[log_level_distribution.sample(&mut rng)].0;
-            let log_level_sample_b = status_field_data[log_level_distribution.sample(&mut rng)].0;
-            let idx_a = zipf_1000.sample(&mut rng) as usize - 1;
-            let idx_b = zipf_1000.sample(&mut rng) as usize - 1;
-            let term_1000_a = &terms_1000[idx_a];
-            let term_1000_b = &terms_1000[idx_b];
+            let log_level_sample_a = few_terms_data[log_level_distribution.sample(&mut rng)];
+            let log_level_sample_b = few_terms_data[log_level_distribution.sample(&mut rng)];
             index_writer.add_document(doc!(
                 json_field => json!({"mixed_type": 10.0}),
                 json_field => json!({"mixed_type": 10.0}),
@@ -558,10 +524,10 @@ fn get_test_index_bench(cardinality: Cardinality) -> tantivy::Result<Index> {
                 text_field_all_unique_terms => "coolo",
                 text_field_many_terms => "cool",
                 text_field_many_terms => "cool",
+                text_field_few_terms => "cool",
+                text_field_few_terms => "cool",
                 text_field_few_terms_status => log_level_sample_a,
                 text_field_few_terms_status => log_level_sample_b,
-                text_field_1000_terms_zipf => term_1000_a.as_str(),
-                text_field_1000_terms_zipf => term_1000_b.as_str(),
                 score_field => 1u64,
                 score_field => 1u64,
                 score_field_f64 => lg_norm.sample(&mut rng),
@@ -588,8 +554,8 @@ fn get_test_index_bench(cardinality: Cardinality) -> tantivy::Result<Index> {
                 json_field => json,
                 text_field_all_unique_terms => format!("unique_term_{}", rng.gen::<u64>()),
                 text_field_many_terms => many_terms_data.choose(&mut rng).unwrap().to_string(),
-                text_field_few_terms_status => status_field_data[log_level_distribution.sample(&mut rng)].0,
-                text_field_1000_terms_zipf => terms_1000[zipf_1000.sample(&mut rng) as usize - 1].as_str(),
+                text_field_few_terms => few_terms_data.choose(&mut rng).unwrap().to_string(),
+                text_field_few_terms_status => few_terms_data[log_level_distribution.sample(&mut rng)],
                 score_field => val as u64,
                 score_field_f64 => lg_norm.sample(&mut rng),
                 score_field_i64 => val as i64,
@@ -641,7 +607,7 @@ fn filter_agg_all_query_with_sub_aggs(index: &Index) {
                 "avg_score": { "avg": { "field": "score" } },
                 "stats_score": { "stats": { "field": "score_f64" } },
                 "terms_text": {
-                    "terms": { "field": "text_few_terms_status" }
+                    "terms": { "field": "text_few_terms" }
                 }
             }
         }
@@ -657,7 +623,7 @@ fn filter_agg_term_query_with_sub_aggs(index: &Index) {
                 "avg_score": { "avg": { "field": "score" } },
                 "stats_score": { "stats": { "field": "score_f64" } },
                 "terms_text": {
-                    "terms": { "field": "text_few_terms_status" }
+                    "terms": { "field": "text_few_terms" }
                 }
             }
         }

columnar/src/block_accessor.rs

@@ -29,20 +29,12 @@ impl<T: PartialOrd + Copy + std::fmt::Debug + Send + Sync + 'static + Default>
         }
     }
     #[inline]
-    pub fn fetch_block_with_missing(
-        &mut self,
-        docs: &[u32],
-        accessor: &Column<T>,
-        missing: Option<T>,
-    ) {
+    pub fn fetch_block_with_missing(&mut self, docs: &[u32], accessor: &Column<T>, missing: T) {
         self.fetch_block(docs, accessor);
         // no missing values
         if accessor.index.get_cardinality().is_full() {
             return;
         }
-        let Some(missing) = missing else {
-            return;
-        };
 
         // We can compare docid_cache length with docs to find missing docs
         // For multi value columns we can't rely on the length and always need to scan

columnar/src/dynamic_column.rs

@@ -3,7 +3,8 @@ use std::sync::Arc;
 use std::{fmt, io};
 
 use common::file_slice::FileSlice;
-use common::{ByteCount, DateTime, HasLen, OwnedBytes};
+use common::{ByteCount, DateTime, OwnedBytes};
+use serde::{Deserialize, Serialize};
 
 use crate::column::{BytesColumn, Column, StrColumn};
 use crate::column_values::{StrictlyMonotonicFn, monotonic_map_column};
@@ -317,10 +318,89 @@ impl DynamicColumnHandle {
     }
 
     pub fn num_bytes(&self) -> ByteCount {
-        self.file_slice.len().into()
+        self.file_slice.num_bytes()
+    }
+
+    /// Legacy helper returning the column space usage.
+    pub fn column_and_dictionary_num_bytes(&self) -> io::Result<ColumnSpaceUsage> {
+        self.space_usage()
+    }
+
+    /// Return the space usage of the column, optionally broken down by dictionary and column
+    /// values.
+    ///
+    /// For dictionary encoded columns (strings and bytes), this splits the total footprint into
+    /// the dictionary and the remaining column data (including index and values).
+    /// For all other column types, the dictionary size is `None` and the column size
+    /// equals the total bytes.
+    pub fn space_usage(&self) -> io::Result<ColumnSpaceUsage> {
+        let total_num_bytes = self.num_bytes();
+        let dynamic_column = self.open()?;
+        let dictionary_num_bytes = match &dynamic_column {
+            DynamicColumn::Bytes(bytes_column) => bytes_column.dictionary().num_bytes(),
+            DynamicColumn::Str(str_column) => str_column.dictionary().num_bytes(),
+            _ => {
+                return Ok(ColumnSpaceUsage::new(self.num_bytes(), None));
+            }
+        };
+        assert!(dictionary_num_bytes <= total_num_bytes);
+        let column_num_bytes =
+            ByteCount::from(total_num_bytes.get_bytes() - dictionary_num_bytes.get_bytes());
+        Ok(ColumnSpaceUsage::new(
+            column_num_bytes,
+            Some(dictionary_num_bytes),
+        ))
     }
 
     pub fn column_type(&self) -> ColumnType {
         self.column_type
     }
 }
+
+/// Represents space usage of a column.
+///
+/// `column_num_bytes` tracks the column payload (index, values and footer).
+/// For dictionary encoded columns, `dictionary_num_bytes` captures the dictionary footprint.
+/// [`ColumnSpaceUsage::total_num_bytes`] returns the sum of both parts.
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct ColumnSpaceUsage {
+    column_num_bytes: ByteCount,
+    dictionary_num_bytes: Option<ByteCount>,
+}
+
+impl ColumnSpaceUsage {
+    pub(crate) fn new(
+        column_num_bytes: ByteCount,
+        dictionary_num_bytes: Option<ByteCount>,
+    ) -> Self {
+        ColumnSpaceUsage {
+            column_num_bytes,
+            dictionary_num_bytes,
+        }
+    }
+
+    pub fn column_num_bytes(&self) -> ByteCount {
+        self.column_num_bytes
+    }
+
+    pub fn dictionary_num_bytes(&self) -> Option<ByteCount> {
+        self.dictionary_num_bytes
+    }
+
+    pub fn total_num_bytes(&self) -> ByteCount {
+        self.column_num_bytes + self.dictionary_num_bytes.unwrap_or_default()
+    }
+
+    /// Merge two space usage values by summing their components.
+    pub fn merge(&self, other: &ColumnSpaceUsage) -> ColumnSpaceUsage {
+        let dictionary_num_bytes = match (self.dictionary_num_bytes, other.dictionary_num_bytes) {
+            (Some(lhs), Some(rhs)) => Some(lhs + rhs),
+            (Some(val), None) | (None, Some(val)) => Some(val),
+            (None, None) => None,
+        };
+        ColumnSpaceUsage {
+            column_num_bytes: self.column_num_bytes + other.column_num_bytes,
+            dictionary_num_bytes,
+        }
+    }
+}

columnar/src/lib.rs

@@ -48,7 +48,7 @@ pub use columnar::{
 use sstable::VoidSSTable;
 pub use value::{NumericalType, NumericalValue};
 
-pub use self::dynamic_column::{DynamicColumn, DynamicColumnHandle};
+pub use self::dynamic_column::{ColumnSpaceUsage, DynamicColumn, DynamicColumnHandle};
 
 pub type RowId = u32;
 pub type DocId = u32;

common/src/bitset.rs

@@ -181,14 +181,6 @@ pub struct BitSet {
     len: u64,
     max_value: u32,
 }
-impl std::fmt::Debug for BitSet {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        f.debug_struct("BitSet")
-            .field("len", &self.len)
-            .field("max_value", &self.max_value)
-            .finish()
-    }
-}
 
 fn num_buckets(max_val: u32) -> u32 {
     max_val.div_ceil(64u32)

src/aggregation/accessor_helpers.rs

@@ -55,22 +55,44 @@ pub(crate) fn get_numeric_or_date_column_types() -> &'static [ColumnType] {
     ]
 }
 
-/// Get fast field reader or empty as default.
+/// Get fast field reader or return an error if the field doesn't exist.
 pub(crate) fn get_ff_reader(
     reader: &SegmentReader,
     field_name: &str,
     allowed_column_types: Option<&[ColumnType]>,
 ) -> crate::Result<(columnar::Column<u64>, ColumnType)> {
     let ff_fields = reader.fast_fields();
-    let ff_field_with_type = ff_fields
-        .u64_lenient_for_type(allowed_column_types, field_name)?
-        .unwrap_or_else(|| {
-            (
+    let ff_field_with_type = ff_fields.u64_lenient_for_type(allowed_column_types, field_name)?;
+
+    match ff_field_with_type {
+        Some(field) => Ok(field),
+        None => {
+            // Check if the field exists in the schema but is not a fast field
+            let schema = reader.schema();
+            if let Some((field, _path)) = schema.find_field(field_name) {
+                let field_type = schema.get_field_entry(field).field_type();
+                if !field_type.is_fast() {
+                    return Err(crate::TantivyError::SchemaError(format!(
+                        "Field '{}' is not a fast field. Aggregations require fast fields.",
+                        field_name
+                    )));
+                }
+            }
+
+            // Field doesn't exist at all or has no values in this segment
+            // Check if it exists in schema to provide a better error message
+            if schema.find_field(field_name).is_none() {
+                return Err(crate::TantivyError::FieldNotFound(field_name.to_string()));
+            }
+
+            // Field exists in schema and is a fast field, but has no values in this segment
+            // This is acceptable - return an empty column
+            Ok((
                 Column::build_empty_column(reader.num_docs()),
                 ColumnType::U64,
-            )
-        });
-    Ok(ff_field_with_type)
+            ))
+        }
+    }
 }
 
 pub(crate) fn get_dynamic_columns(
@@ -89,6 +111,7 @@ pub(crate) fn get_dynamic_columns(
 /// Get all fast field reader or empty as default.
 ///
 /// Is guaranteed to return at least one column.
+/// Returns an error if the field doesn't exist in the schema or is not a fast field.
 pub(crate) fn get_all_ff_reader_or_empty(
     reader: &SegmentReader,
     field_name: &str,
@@ -98,7 +121,25 @@ pub(crate) fn get_all_ff_reader_or_empty(
     let ff_fields = reader.fast_fields();
     let mut ff_field_with_type =
         ff_fields.u64_lenient_for_type_all(allowed_column_types, field_name)?;
+
     if ff_field_with_type.is_empty() {
+        // Check if the field exists in the schema but is not a fast field
+        let schema = reader.schema();
+        if let Some((field, _path)) = schema.find_field(field_name) {
+            let field_type = schema.get_field_entry(field).field_type();
+            if !field_type.is_fast() {
+                return Err(crate::TantivyError::SchemaError(format!(
+                    "Field '{}' is not a fast field. Aggregations require fast fields.",
+                    field_name
+                )));
+            }
+        } else {
+            // Field doesn't exist in the schema at all
+            return Err(crate::TantivyError::FieldNotFound(field_name.to_string()));
+        }
+
+        // Field exists in schema and is a fast field, but has no values in this segment
+        // This is acceptable - return an empty column
         ff_field_with_type.push((Column::build_empty_column(reader.num_docs()), fallback_type));
     }
     Ok(ff_field_with_type)

src/aggregation/agg_data.rs

@@ -1,4 +1,4 @@
-use columnar::{Column, ColumnBlockAccessor, ColumnType, StrColumn};
+use columnar::{Column, ColumnType, StrColumn};
 use common::BitSet;
 use rustc_hash::FxHashSet;
 use serde::Serialize;
@@ -10,16 +10,16 @@ use crate::aggregation::accessor_helpers::{
 };
 use crate::aggregation::agg_req::{Aggregation, AggregationVariants, Aggregations};
 use crate::aggregation::bucket::{
-    build_segment_range_collector, FilterAggReqData, HistogramAggReqData, HistogramBounds,
-    IncludeExcludeParam, MissingTermAggReqData, RangeAggReqData, SegmentFilterCollector,
-    SegmentHistogramCollector, TermMissingAgg, TermsAggReqData, TermsAggregation,
+    FilterAggReqData, HistogramAggReqData, HistogramBounds, IncludeExcludeParam,
+    MissingTermAggReqData, RangeAggReqData, SegmentFilterCollector, SegmentHistogramCollector,
+    SegmentRangeCollector, TermMissingAgg, TermsAggReqData, TermsAggregation,
     TermsAggregationInternal,
 };
 use crate::aggregation::metric::{
-    build_segment_stats_collector, AverageAggregation, CardinalityAggReqData,
-    CardinalityAggregationReq, CountAggregation, ExtendedStatsAggregation, MaxAggregation,
-    MetricAggReqData, MinAggregation, SegmentCardinalityCollector, SegmentExtendedStatsCollector,
-    SegmentPercentilesCollector, StatsAggregation, StatsType, SumAggregation, TopHitsAggReqData,
+    AverageAggregation, CardinalityAggReqData, CardinalityAggregationReq, CountAggregation,
+    ExtendedStatsAggregation, MaxAggregation, MetricAggReqData, MinAggregation,
+    SegmentCardinalityCollector, SegmentExtendedStatsCollector, SegmentPercentilesCollector,
+    SegmentStatsCollector, StatsAggregation, StatsType, SumAggregation, TopHitsAggReqData,
     TopHitsSegmentCollector,
 };
 use crate::aggregation::segment_agg_result::{
@@ -35,7 +35,6 @@ pub struct AggregationsSegmentCtx {
     /// Request data for each aggregation type.
     pub per_request: PerRequestAggSegCtx,
     pub context: AggContextParams,
-    pub column_block_accessor: ColumnBlockAccessor<u64>,
 }
 
 impl AggregationsSegmentCtx {
@@ -108,14 +107,21 @@ impl AggregationsSegmentCtx {
             .as_deref()
             .expect("range_req_data slot is empty (taken)")
     }
+    #[inline]
+    pub(crate) fn get_filter_req_data(&self, idx: usize) -> &FilterAggReqData {
+        self.per_request.filter_req_data[idx]
+            .as_deref()
+            .expect("filter_req_data slot is empty (taken)")
+    }
 
     // ---------- mutable getters ----------
 
     #[inline]
-    pub(crate) fn get_metric_req_data_mut(&mut self, idx: usize) -> &mut MetricAggReqData {
-        &mut self.per_request.stats_metric_req_data[idx]
+    pub(crate) fn get_term_req_data_mut(&mut self, idx: usize) -> &mut TermsAggReqData {
+        self.per_request.term_req_data[idx]
+            .as_deref_mut()
+            .expect("term_req_data slot is empty (taken)")
     }
-
     #[inline]
     pub(crate) fn get_cardinality_req_data_mut(
         &mut self,
@@ -123,7 +129,10 @@ impl AggregationsSegmentCtx {
     ) -> &mut CardinalityAggReqData {
         &mut self.per_request.cardinality_req_data[idx]
     }
-
+    #[inline]
+    pub(crate) fn get_metric_req_data_mut(&mut self, idx: usize) -> &mut MetricAggReqData {
+        &mut self.per_request.stats_metric_req_data[idx]
+    }
     #[inline]
     pub(crate) fn get_histogram_req_data_mut(&mut self, idx: usize) -> &mut HistogramAggReqData {
         self.per_request.histogram_req_data[idx]
@@ -133,6 +142,21 @@ impl AggregationsSegmentCtx {
 
     // ---------- take / put (terms, histogram, range) ----------
 
+    /// Move out the boxed Terms request at `idx`, leaving `None`.
+    #[inline]
+    pub(crate) fn take_term_req_data(&mut self, idx: usize) -> Box<TermsAggReqData> {
+        self.per_request.term_req_data[idx]
+            .take()
+            .expect("term_req_data slot is empty (taken)")
+    }
+
+    /// Put back a Terms request into an empty slot at `idx`.
+    #[inline]
+    pub(crate) fn put_back_term_req_data(&mut self, idx: usize, value: Box<TermsAggReqData>) {
+        debug_assert!(self.per_request.term_req_data[idx].is_none());
+        self.per_request.term_req_data[idx] = Some(value);
+    }
+
     /// Move out the boxed Histogram request at `idx`, leaving `None`.
     #[inline]
     pub(crate) fn take_histogram_req_data(&mut self, idx: usize) -> Box<HistogramAggReqData> {
@@ -296,7 +320,6 @@ impl PerRequestAggSegCtx {
 
     /// Convert the aggregation tree into a serializable struct representation.
     /// Each node contains: { name, kind, children }.
-    #[allow(dead_code)]
     pub fn get_view_tree(&self) -> Vec<AggTreeViewNode> {
         fn node_to_view(node: &AggRefNode, pr: &PerRequestAggSegCtx) -> AggTreeViewNode {
             let mut children: Vec<AggTreeViewNode> =
@@ -322,19 +345,12 @@ impl PerRequestAggSegCtx {
 pub(crate) fn build_segment_agg_collectors_root(
     req: &mut AggregationsSegmentCtx,
 ) -> crate::Result<Box<dyn SegmentAggregationCollector>> {
-    build_segment_agg_collectors_generic(req, &req.per_request.agg_tree.clone())
+    build_segment_agg_collectors(req, &req.per_request.agg_tree.clone())
 }
 
 pub(crate) fn build_segment_agg_collectors(
     req: &mut AggregationsSegmentCtx,
     nodes: &[AggRefNode],
-) -> crate::Result<Box<dyn SegmentAggregationCollector>> {
-    build_segment_agg_collectors_generic(req, nodes)
-}
-
-fn build_segment_agg_collectors_generic(
-    req: &mut AggregationsSegmentCtx,
-    nodes: &[AggRefNode],
 ) -> crate::Result<Box<dyn SegmentAggregationCollector>> {
     let mut collectors = Vec::new();
     for node in nodes.iter() {
@@ -372,8 +388,6 @@ pub(crate) fn build_segment_agg_collector(
             Ok(Box::new(SegmentCardinalityCollector::from_req(
                 req_data.column_type,
                 node.idx_in_req_data,
-                req_data.accessor.clone(),
-                req_data.missing_value_for_accessor,
             )))
         }
         AggKind::StatsKind(stats_type) => {
@@ -384,21 +398,20 @@ pub(crate) fn build_segment_agg_collector(
                 | StatsType::Count
                 | StatsType::Max
                 | StatsType::Min
-                | StatsType::Stats => build_segment_stats_collector(req_data),
-                StatsType::ExtendedStats(sigma) => Ok(Box::new(
-                    SegmentExtendedStatsCollector::from_req(req_data, sigma),
-                )),
-                StatsType::Percentiles => {
-                    let req_data = req.get_metric_req_data_mut(node.idx_in_req_data);
-                    Ok(Box::new(
-                        SegmentPercentilesCollector::from_req_and_validate(
-                            req_data.field_type,
-                            req_data.missing_u64,
-                            req_data.accessor.clone(),
-                            node.idx_in_req_data,
-                        ),
-                    ))
+                | StatsType::Stats => Ok(Box::new(SegmentStatsCollector::from_req(
+                    node.idx_in_req_data,
+                ))),
+                StatsType::ExtendedStats(sigma) => {
+                    Ok(Box::new(SegmentExtendedStatsCollector::from_req(
+                        req_data.field_type,
+                        sigma,
+                        node.idx_in_req_data,
+                        req_data.missing,
+                    )))
                 }
+                StatsType::Percentiles => Ok(Box::new(
+                    SegmentPercentilesCollector::from_req_and_validate(node.idx_in_req_data)?,
+                )),
             }
         }
         AggKind::TopHits => {
@@ -415,7 +428,9 @@ pub(crate) fn build_segment_agg_collector(
         AggKind::DateHistogram => Ok(Box::new(SegmentHistogramCollector::from_req_and_validate(
             req, node,
         )?)),
-        AggKind::Range => Ok(build_segment_range_collector(req, node)?),
+        AggKind::Range => Ok(Box::new(SegmentRangeCollector::from_req_and_validate(
+            req, node,
+        )?)),
         AggKind::Filter => Ok(Box::new(SegmentFilterCollector::from_req_and_validate(
             req, node,
         )?)),
@@ -478,7 +493,6 @@ pub(crate) fn build_aggregations_data_from_req(
     let mut data = AggregationsSegmentCtx {
         per_request: Default::default(),
         context,
-        column_block_accessor: ColumnBlockAccessor::default(),
     };
 
     for (name, agg) in aggs.iter() {
@@ -507,9 +521,9 @@ fn build_nodes(
             let idx_in_req_data = data.push_range_req_data(RangeAggReqData {
                 accessor,
                 field_type,
+                column_block_accessor: Default::default(),
                 name: agg_name.to_string(),
                 req: range_req.clone(),
-                is_top_level,
             });
             let children = build_children(&req.sub_aggregation, reader, segment_ordinal, data)?;
             Ok(vec![AggRefNode {
@@ -527,7 +541,9 @@ fn build_nodes(
             let idx_in_req_data = data.push_histogram_req_data(HistogramAggReqData {
                 accessor,
                 field_type,
+                column_block_accessor: Default::default(),
                 name: agg_name.to_string(),
+                sub_aggregation_blueprint: None,
                 req: histo_req.clone(),
                 is_date_histogram: false,
                 bounds: HistogramBounds {
@@ -552,7 +568,9 @@ fn build_nodes(
             let idx_in_req_data = data.push_histogram_req_data(HistogramAggReqData {
                 accessor,
                 field_type,
+                column_block_accessor: Default::default(),
                 name: agg_name.to_string(),
+                sub_aggregation_blueprint: None,
                 req: histo_req,
                 is_date_histogram: true,
                 bounds: HistogramBounds {
@@ -632,6 +650,7 @@ fn build_nodes(
             let idx_in_req_data = data.push_metric_req_data(MetricAggReqData {
                 accessor,
                 field_type,
+                column_block_accessor: Default::default(),
                 name: agg_name.to_string(),
                 collecting_for,
                 missing: *missing,
@@ -659,6 +678,7 @@ fn build_nodes(
             let idx_in_req_data = data.push_metric_req_data(MetricAggReqData {
                 accessor,
                 field_type,
+                column_block_accessor: Default::default(),
                 name: agg_name.to_string(),
                 collecting_for: StatsType::Percentiles,
                 missing: percentiles_req.missing,
@@ -875,7 +895,7 @@ fn build_terms_or_cardinality_nodes(
         });
     }
 
-    // Add one node per accessor
+    // Add one node per accessor to mirror previous behavior and allow per-type missing handling.
     for (accessor, column_type) in column_and_types {
         let missing_value_for_accessor = if use_special_missing_agg {
             None
@@ -906,8 +926,11 @@ fn build_terms_or_cardinality_nodes(
                     column_type,
                     str_dict_column: str_dict_column.clone(),
                     missing_value_for_accessor,
+                    column_block_accessor: Default::default(),
                     name: agg_name.to_string(),
                     req: TermsAggregationInternal::from_req(req),
+                    // Will be filled later when building collectors
+                    sub_aggregation_blueprint: None,
                     sug_aggregations: sub_aggs.clone(),
                     allowed_term_ids,
                     is_top_level,
@@ -920,6 +943,7 @@ fn build_terms_or_cardinality_nodes(
                     column_type,
                     str_dict_column: str_dict_column.clone(),
                     missing_value_for_accessor,
+                    column_block_accessor: Default::default(),
                     name: agg_name.to_string(),
                     req: req.clone(),
                 });
@@ -1033,7 +1057,7 @@ mod tests {
             "avg": {"field": "score"}
         }));
         let terms_string_with_child = agg_from_json(json!({
-            "terms": {"field": "string_id"},
+            "terms": {"field": "text"},
             "aggs": {
                 "histo": {"histogram": {"field": "score", "interval": 10.0}}
             }

src/aggregation/agg_tests.rs

@@ -2,441 +2,15 @@ use serde_json::Value;
 
 use crate::aggregation::agg_req::{Aggregation, Aggregations};
 use crate::aggregation::agg_result::AggregationResults;
+use crate::aggregation::buf_collector::DOC_BLOCK_SIZE;
 use crate::aggregation::collector::AggregationCollector;
 use crate::aggregation::intermediate_agg_result::IntermediateAggregationResults;
 use crate::aggregation::tests::{get_test_index_2_segments, get_test_index_from_values_and_terms};
 use crate::aggregation::DistributedAggregationCollector;
-use crate::docset::COLLECT_BLOCK_BUFFER_LEN;
 use crate::query::{AllQuery, TermQuery};
 use crate::schema::{IndexRecordOption, Schema, FAST};
 use crate::{Index, IndexWriter, Term};
 
-// The following tests ensure that each bucket aggregation type correctly functions as a
-// sub-aggregation of another bucket aggregation in two scenarios:
-// 1) The parent has more buckets than the child sub-aggregation
-// 2) The child sub-aggregation has more buckets than the parent
-//
-// These scenarios exercise the bucket id mapping and sub-aggregation routing logic.
-
-#[test]
-fn test_terms_as_subagg_parent_more_vs_child_more() -> crate::Result<()> {
-    let index = get_test_index_2_segments(false)?;
-
-    // Case A: parent has more buckets than child
-    // Parent: range with 4 buckets
-    // Child: terms on text -> 2 buckets
-    let agg_parent_more: Aggregations = serde_json::from_value(json!({
-        "parent_range": {
-            "range": {
-                "field": "score",
-                "ranges": [
-                    {"to": 3.0},
-                    {"from": 3.0, "to": 7.0},
-                    {"from": 7.0, "to": 20.0},
-                    {"from": 20.0}
-                ]
-            },
-            "aggs": {
-                "child_terms": {"terms": {"field": "text", "order": {"_key": "asc"}}}
-            }
-        }
-    }))
-    .unwrap();
-
-    let res = crate::aggregation::tests::exec_request(agg_parent_more, &index)?;
-    // Exact expected structure and counts
-    assert_eq!(
-        res["parent_range"]["buckets"],
-        json!([
-            {
-                "key": "*-3",
-                "doc_count": 1,
-                "to": 3.0,
-                "child_terms": {
-                    "buckets": [
-                        {"doc_count": 1, "key": "cool"}
-                    ],
-                    "sum_other_doc_count": 0
-                }
-            },
-            {
-                "key": "3-7",
-                "doc_count": 3,
-                "from": 3.0,
-                "to": 7.0,
-                "child_terms": {
-                    "buckets": [
-                        {"doc_count": 2, "key": "cool"},
-                        {"doc_count": 1, "key": "nohit"}
-                    ],
-                    "sum_other_doc_count": 0
-                }
-            },
-            {
-                "key": "7-20",
-                "doc_count": 3,
-                "from": 7.0,
-                "to": 20.0,
-                "child_terms": {
-                    "buckets": [
-                        {"doc_count": 3, "key": "cool"}
-                    ],
-                    "sum_other_doc_count": 0
-                }
-            },
-            {
-                "key": "20-*",
-                "doc_count": 2,
-                "from": 20.0,
-                "child_terms": {
-                    "buckets": [
-                        {"doc_count": 1, "key": "cool"},
-                        {"doc_count": 1, "key": "nohit"}
-                    ],
-                    "sum_other_doc_count": 0
-                }
-            }
-        ])
-    );
-
-    // Case B: child has more buckets than parent
-    // Parent: histogram on score with large interval -> 1 bucket
-    // Child: terms on text -> 2 buckets (cool/nohit)
-    let agg_child_more: Aggregations = serde_json::from_value(json!({
-        "parent_hist": {
-            "histogram": {"field": "score", "interval": 100.0},
-            "aggs": {
-                "child_terms": {"terms": {"field": "text", "order": {"_key": "asc"}}}
-            }
-        }
-    }))
-    .unwrap();
-
-    let res = crate::aggregation::tests::exec_request(agg_child_more, &index)?;
-    assert_eq!(
-        res["parent_hist"],
-        json!({
-            "buckets": [
-                {
-                    "key": 0.0,
-                    "doc_count": 9,
-                    "child_terms": {
-                        "buckets": [
-                            {"doc_count": 7, "key": "cool"},
-                            {"doc_count": 2, "key": "nohit"}
-                        ],
-                        "sum_other_doc_count": 0
-                    }
-                }
-            ]
-        })
-    );
-
-    Ok(())
-}
-
-#[test]
-fn test_range_as_subagg_parent_more_vs_child_more() -> crate::Result<()> {
-    let index = get_test_index_2_segments(false)?;
-
-    // Case A: parent has more buckets than child
-    // Parent: range with 5 buckets
-    // Child: coarse range with 3 buckets
-    let agg_parent_more: Aggregations = serde_json::from_value(json!({
-        "parent_range": {
-            "range": {
-                "field": "score",
-                "ranges": [
-                    {"to": 3.0},
-                    {"from": 3.0, "to": 7.0},
-                    {"from": 7.0, "to": 11.0},
-                    {"from": 11.0, "to": 20.0},
-                    {"from": 20.0}
-                ]
-            },
-            "aggs": {
-                "child_range": {
-                    "range": {
-                        "field": "score",
-                        "ranges": [
-                            {"to": 3.0},
-                            {"from": 3.0, "to": 20.0}
-                        ]
-                    }
-                }
-            }
-        }
-    }))
-    .unwrap();
-    let res = crate::aggregation::tests::exec_request(agg_parent_more, &index)?;
-    assert_eq!(
-        res["parent_range"]["buckets"],
-        json!([
-            {"key": "*-3", "doc_count": 1, "to": 3.0,
-                "child_range": {"buckets": [
-                    {"key": "*-3", "doc_count": 1, "to": 3.0},
-                    {"key": "3-20", "doc_count": 0, "from": 3.0, "to": 20.0},
-                    {"key": "20-*", "doc_count": 0, "from": 20.0}
-                ]}
-            },
-            {"key": "3-7", "doc_count": 3, "from": 3.0, "to": 7.0,
-                "child_range": {"buckets": [
-                    {"key": "*-3", "doc_count": 0, "to": 3.0},
-                    {"key": "3-20", "doc_count": 3, "from": 3.0, "to": 20.0},
-                    {"key": "20-*", "doc_count": 0, "from": 20.0}
-                ]}
-            },
-            {"key": "7-11", "doc_count": 1, "from": 7.0, "to": 11.0,
-                "child_range": {"buckets": [
-                    {"key": "*-3", "doc_count": 0, "to": 3.0},
-                    {"key": "3-20", "doc_count": 1, "from": 3.0, "to": 20.0},
-                    {"key": "20-*", "doc_count": 0, "from": 20.0}
-                ]}
-            },
-            {"key": "11-20", "doc_count": 2, "from": 11.0, "to": 20.0,
-                "child_range": {"buckets": [
-                    {"key": "*-3", "doc_count": 0, "to": 3.0},
-                    {"key": "3-20", "doc_count": 2, "from": 3.0, "to": 20.0},
-                    {"key": "20-*", "doc_count": 0, "from": 20.0}
-                ]}
-            },
-            {"key": "20-*", "doc_count": 2, "from": 20.0,
-                "child_range": {"buckets": [
-                    {"key": "*-3", "doc_count": 0, "to": 3.0},
-                    {"key": "3-20", "doc_count": 0, "from": 3.0, "to": 20.0},
-                    {"key": "20-*", "doc_count": 2, "from": 20.0}
-                ]}
-            }
-        ])
-    );
-
-    // Case B: child has more buckets than parent
-    // Parent: terms on text (2 buckets)
-    // Child: range with 4 buckets
-    let agg_child_more: Aggregations = serde_json::from_value(json!({
-        "parent_terms": {
-            "terms": {"field": "text"},
-            "aggs": {
-                "child_range": {
-                    "range": {
-                        "field": "score",
-                        "ranges": [
-                            {"to": 3.0},
-                            {"from": 3.0, "to": 7.0},
-                            {"from": 7.0, "to": 20.0}
-                        ]
-                    }
-                }
-            }
-        }
-    }))
-    .unwrap();
-    let res = crate::aggregation::tests::exec_request(agg_child_more, &index)?;
-
-    assert_eq!(
-        res["parent_terms"],
-        json!({
-            "buckets": [
-                {
-                    "key": "cool",
-                    "doc_count": 7,
-                    "child_range": {
-                        "buckets": [
-                            {"key": "*-3", "doc_count": 1, "to": 3.0},
-                            {"key": "3-7", "doc_count": 2, "from": 3.0, "to": 7.0},
-                            {"key": "7-20", "doc_count": 3, "from": 7.0, "to": 20.0},
-                            {"key": "20-*", "doc_count": 1, "from": 20.0}
-                        ]
-                    }
-                },
-                {
-                    "key": "nohit",
-                    "doc_count": 2,
-                    "child_range": {
-                        "buckets": [
-                            {"key": "*-3", "doc_count": 0, "to": 3.0},
-                            {"key": "3-7", "doc_count": 1, "from": 3.0, "to": 7.0},
-                            {"key": "7-20", "doc_count": 0, "from": 7.0, "to": 20.0},
-                            {"key": "20-*", "doc_count": 1, "from": 20.0}
-                        ]
-                    }
-                }
-            ],
-            "doc_count_error_upper_bound": 0,
-            "sum_other_doc_count": 0
-        })
-    );
-
-    Ok(())
-}
-
-#[test]
-fn test_histogram_as_subagg_parent_more_vs_child_more() -> crate::Result<()> {
-    let index = get_test_index_2_segments(false)?;
-
-    // Case A: parent has more buckets than child
-    // Parent: range with several ranges
-    // Child: histogram with large interval (single bucket per parent)
-    let agg_parent_more: Aggregations = serde_json::from_value(json!({
-        "parent_range": {
-            "range": {
-                "field": "score",
-                "ranges": [
-                    {"to": 3.0},
-                    {"from": 3.0, "to": 7.0},
-                    {"from": 7.0, "to": 11.0},
-                    {"from": 11.0, "to": 20.0},
-                    {"from": 20.0}
-                ]
-            },
-            "aggs": {
-                "child_hist": {"histogram": {"field": "score", "interval": 100.0}}
-            }
-        }
-    }))
-    .unwrap();
-    let res = crate::aggregation::tests::exec_request(agg_parent_more, &index)?;
-    assert_eq!(
-        res["parent_range"]["buckets"],
-        json!([
-            {"key": "*-3", "doc_count": 1, "to": 3.0,
-                "child_hist": {"buckets": [ {"key": 0.0, "doc_count": 1} ]}
-            },
-            {"key": "3-7", "doc_count": 3, "from": 3.0, "to": 7.0,
-                "child_hist": {"buckets": [ {"key": 0.0, "doc_count": 3} ]}
-            },
-            {"key": "7-11", "doc_count": 1, "from": 7.0, "to": 11.0,
-                "child_hist": {"buckets": [ {"key": 0.0, "doc_count": 1} ]}
-            },
-            {"key": "11-20", "doc_count": 2, "from": 11.0, "to": 20.0,
-                "child_hist": {"buckets": [ {"key": 0.0, "doc_count": 2} ]}
-            },
-            {"key": "20-*", "doc_count": 2, "from": 20.0,
-                "child_hist": {"buckets": [ {"key": 0.0, "doc_count": 2} ]}
-            }
-        ])
-    );
-
-    // Case B: child has more buckets than parent
-    // Parent: terms on text -> 2 buckets
-    // Child: histogram with small interval -> multiple buckets including empties
-    let agg_child_more: Aggregations = serde_json::from_value(json!({
-        "parent_terms": {
-            "terms": {"field": "text"},
-            "aggs": {
-                "child_hist": {"histogram": {"field": "score", "interval": 10.0}}
-            }
-        }
-    }))
-    .unwrap();
-    let res = crate::aggregation::tests::exec_request(agg_child_more, &index)?;
-    assert_eq!(
-        res["parent_terms"],
-        json!({
-            "buckets": [
-                {
-                    "key": "cool",
-                    "doc_count": 7,
-                    "child_hist": {
-                        "buckets": [
-                            {"key": 0.0, "doc_count": 4},
-                            {"key": 10.0, "doc_count": 2},
-                            {"key": 20.0, "doc_count": 0},
-                            {"key": 30.0, "doc_count": 0},
-                            {"key": 40.0, "doc_count": 1}
-                        ]
-                    }
-                },
-                {
-                    "key": "nohit",
-                    "doc_count": 2,
-                    "child_hist": {
-                        "buckets": [
-                            {"key": 0.0, "doc_count": 1},
-                            {"key": 10.0, "doc_count": 0},
-                            {"key": 20.0, "doc_count": 0},
-                            {"key": 30.0, "doc_count": 0},
-                            {"key": 40.0, "doc_count": 1}
-                        ]
-                    }
-                }
-            ],
-            "doc_count_error_upper_bound": 0,
-            "sum_other_doc_count": 0
-        })
-    );
-
-    Ok(())
-}
-
-#[test]
-fn test_date_histogram_as_subagg_parent_more_vs_child_more() -> crate::Result<()> {
-    let index = get_test_index_2_segments(false)?;
-
-    // Case A: parent has more buckets than child
-    // Parent: range with several buckets
-    // Child: date_histogram with 30d -> single bucket per parent
-    let agg_parent_more: Aggregations = serde_json::from_value(json!({
-        "parent_range": {
-            "range": {
-                "field": "score",
-                "ranges": [
-                    {"to": 3.0},
-                    {"from": 3.0, "to": 7.0},
-                    {"from": 7.0, "to": 11.0},
-                    {"from": 11.0, "to": 20.0},
-                    {"from": 20.0}
-                ]
-            },
-            "aggs": {
-                "child_date_hist": {"date_histogram": {"field": "date", "fixed_interval": "30d"}}
-            }
-        }
-    }))
-    .unwrap();
-    let res = crate::aggregation::tests::exec_request(agg_parent_more, &index)?;
-    let buckets = res["parent_range"]["buckets"].as_array().unwrap();
-    // Verify each parent bucket has exactly one child date bucket with matching doc_count
-    for bucket in buckets {
-        let parent_count = bucket["doc_count"].as_u64().unwrap();
-        let child_buckets = bucket["child_date_hist"]["buckets"].as_array().unwrap();
-        assert_eq!(child_buckets.len(), 1);
-        assert_eq!(child_buckets[0]["doc_count"], parent_count);
-    }
-
-    // Case B: child has more buckets than parent
-    // Parent: terms on text (2 buckets)
-    // Child: date_histogram with 1d -> multiple buckets
-    let agg_child_more: Aggregations = serde_json::from_value(json!({
-        "parent_terms": {
-            "terms": {"field": "text"},
-            "aggs": {
-                "child_date_hist": {"date_histogram": {"field": "date", "fixed_interval": "1d"}}
-            }
-        }
-    }))
-    .unwrap();
-    let res = crate::aggregation::tests::exec_request(agg_child_more, &index)?;
-    let buckets = res["parent_terms"]["buckets"].as_array().unwrap();
-
-    // cool bucket
-    assert_eq!(buckets[0]["key"], "cool");
-    let cool_buckets = buckets[0]["child_date_hist"]["buckets"].as_array().unwrap();
-    assert_eq!(cool_buckets.len(), 3);
-    assert_eq!(cool_buckets[0]["doc_count"], 1); // day 0
-    assert_eq!(cool_buckets[1]["doc_count"], 4); // day 1
-    assert_eq!(cool_buckets[2]["doc_count"], 2); // day 2
-
-    // nohit bucket
-    assert_eq!(buckets[1]["key"], "nohit");
-    let nohit_buckets = buckets[1]["child_date_hist"]["buckets"].as_array().unwrap();
-    assert_eq!(nohit_buckets.len(), 2);
-    assert_eq!(nohit_buckets[0]["doc_count"], 1); // day 1
-    assert_eq!(nohit_buckets[1]["doc_count"], 1); // day 2
-
-    Ok(())
-}
-
 fn get_avg_req(field_name: &str) -> Aggregation {
     serde_json::from_value(json!({
         "avg": {
@@ -451,10 +25,6 @@ fn get_collector(agg_req: Aggregations) -> AggregationCollector {
 }
 
 // *** EVERY BUCKET-TYPE SHOULD BE TESTED HERE ***
-// Note: The flushng part of these  tests are outdated, since the buffering change after converting
-// the collection into one collector per request instead of per bucket.
-//
-// However they are useful as they test a complex aggregation requests.
 fn test_aggregation_flushing(
     merge_segments: bool,
     use_distributed_collector: bool,
@@ -467,9 +37,8 @@ fn test_aggregation_flushing(
 
     let reader = index.reader()?;
 
-    assert_eq!(COLLECT_BLOCK_BUFFER_LEN, 64);
-    // In the tree we cache documents of COLLECT_BLOCK_BUFFER_LEN before passing them down as one
-    // block.
+    assert_eq!(DOC_BLOCK_SIZE, 64);
+    // In the tree we cache Documents of DOC_BLOCK_SIZE, before passing them down as one block.
     //
     // Build a request so that on the first level we have one full cache, which is then flushed.
     // The same cache should have some residue docs at the end, which are flushed (Range 0-70)
@@ -1436,3 +1005,123 @@ fn test_aggregation_on_json_object_mixed_numerical_segments() {
         )
     );
 }
+
+#[test]
+fn test_aggregation_invalid_field_returns_error() {
+    // Test that aggregations return an error when given an invalid field name
+    let index = get_test_index_2_segments(false).unwrap();
+    let reader = index.reader().unwrap();
+    let searcher = reader.searcher();
+
+    // Test with a field that doesn't exist at all
+    let agg_req_str = r#"
+    {
+        "date_histogram_test": {
+            "date_histogram": {
+                "field": "not_valid_field",
+                "fixed_interval": "30d"
+            }
+        }
+    }"#;
+    let agg: Aggregations = serde_json::from_str(agg_req_str).unwrap();
+    let collector = get_collector(agg);
+    let result = searcher.search(&AllQuery, &collector);
+
+    assert!(result.is_err());
+    match result {
+        Err(crate::TantivyError::FieldNotFound(field_name)) => {
+            assert_eq!(field_name, "not_valid_field");
+        }
+        _ => panic!("Expected FieldNotFound error, got: {:?}", result),
+    }
+
+    // Test with histogram aggregation on invalid field
+    let agg_req_str = r#"
+    {
+        "histogram_test": {
+            "histogram": {
+                "field": "invalid_histogram_field",
+                "interval": 10.0
+            }
+        }
+    }"#;
+    let agg: Aggregations = serde_json::from_str(agg_req_str).unwrap();
+    let collector = get_collector(agg);
+    let result = searcher.search(&AllQuery, &collector);
+
+    assert!(result.is_err());
+    match result {
+        Err(crate::TantivyError::FieldNotFound(field_name)) => {
+            assert_eq!(field_name, "invalid_histogram_field");
+        }
+        _ => panic!("Expected FieldNotFound error, got: {:?}", result),
+    }
+
+    // Test with terms aggregation on invalid field
+    let agg_req_str = r#"
+    {
+        "terms_test": {
+            "terms": {
+                "field": "invalid_terms_field"
+            }
+        }
+    }"#;
+    let agg: Aggregations = serde_json::from_str(agg_req_str).unwrap();
+    let collector = get_collector(agg);
+    let result = searcher.search(&AllQuery, &collector);
+
+    assert!(result.is_err());
+    match result {
+        Err(crate::TantivyError::FieldNotFound(field_name)) => {
+            assert_eq!(field_name, "invalid_terms_field");
+        }
+        _ => panic!("Expected FieldNotFound error, got: {:?}", result),
+    }
+
+    // Test with avg metric aggregation on invalid field
+    let agg_req_str = r#"
+    {
+        "avg_test": {
+            "avg": {
+                "field": "invalid_avg_field"
+            }
+        }
+    }"#;
+    let agg: Aggregations = serde_json::from_str(agg_req_str).unwrap();
+    let collector = get_collector(agg);
+    let result = searcher.search(&AllQuery, &collector);
+
+    assert!(result.is_err());
+    match result {
+        Err(crate::TantivyError::FieldNotFound(field_name)) => {
+            assert_eq!(field_name, "invalid_avg_field");
+        }
+        _ => panic!("Expected FieldNotFound error, got: {:?}", result),
+    }
+
+    // Test with range aggregation on invalid field
+    let agg_req_str = r#"
+    {
+        "range_test": {
+            "range": {
+                "field": "invalid_range_field",
+                "ranges": [
+                    { "to": 10.0 },
+                    { "from": 10.0, "to": 20.0 },
+                    { "from": 20.0 }
+                ]
+            }
+        }
+    }"#;
+    let agg: Aggregations = serde_json::from_str(agg_req_str).unwrap();
+    let collector = get_collector(agg);
+    let result = searcher.search(&AllQuery, &collector);
+
+    assert!(result.is_err());
+    match result {
+        Err(crate::TantivyError::FieldNotFound(field_name)) => {
+            assert_eq!(field_name, "invalid_range_field");
+        }
+        _ => panic!("Expected FieldNotFound error, got: {:?}", result),
+    }
+}

src/aggregation/bucket/filter.rs

@@ -6,12 +6,10 @@ use serde::{Deserialize, Deserializer, Serialize, Serializer};
 use crate::aggregation::agg_data::{
     build_segment_agg_collectors, AggRefNode, AggregationsSegmentCtx,
 };
-use crate::aggregation::cached_sub_aggs::CachedSubAggs;
 use crate::aggregation::intermediate_agg_result::{
     IntermediateAggregationResult, IntermediateAggregationResults, IntermediateBucketResult,
 };
-use crate::aggregation::segment_agg_result::{BucketIdProvider, SegmentAggregationCollector};
-use crate::aggregation::BucketId;
+use crate::aggregation::segment_agg_result::{CollectorClone, SegmentAggregationCollector};
 use crate::docset::DocSet;
 use crate::query::{AllQuery, EnableScoring, Query, QueryParser};
 use crate::schema::Schema;
@@ -412,9 +410,9 @@ impl FilterAggReqData {
     pub(crate) fn get_memory_consumption(&self) -> usize {
         // Estimate: name + segment reader reference + bitset + buffer capacity
         self.name.len()
-        + std::mem::size_of::<SegmentReader>()
-        + self.evaluator.bitset.len() / 8 // BitSet memory (bits to bytes)
-        + self.matching_docs_buffer.capacity() * std::mem::size_of::<DocId>()
+            + std::mem::size_of::<SegmentReader>()
+            + self.evaluator.bitset.len() / 8 // BitSet memory (bits to bytes)
+            + self.matching_docs_buffer.capacity() * std::mem::size_of::<DocId>()
     }
 }
 
@@ -491,19 +489,12 @@ impl Debug for DocumentQueryEvaluator {
     }
 }
 
-#[derive(Debug, Clone, PartialEq, Copy)]
-struct DocCount {
-    doc_count: u64,
-    bucket_id: BucketId,
-}
-
 /// Segment collector for filter aggregation
 pub struct SegmentFilterCollector {
-    /// Document counts per parent bucket
-    parent_buckets: Vec<DocCount>,
+    /// Document count in this bucket
+    doc_count: u64,
     /// Sub-aggregation collectors
-    sub_aggregations: Option<CachedSubAggs<true>>,
-    bucket_id_provider: BucketIdProvider,
+    sub_aggregations: Option<Box<dyn SegmentAggregationCollector>>,
     /// Accessor index for this filter aggregation (to access FilterAggReqData)
     accessor_idx: usize,
 }
@@ -520,13 +511,11 @@ impl SegmentFilterCollector {
         } else {
             None
         };
-        let sub_agg_collector = sub_agg_collector.map(CachedSubAggs::new);
 
         Ok(SegmentFilterCollector {
-            parent_buckets: Vec::new(),
+            doc_count: 0,
             sub_aggregations: sub_agg_collector,
             accessor_idx: node.idx_in_req_data,
-            bucket_id_provider: BucketIdProvider::default(),
         })
     }
 }
@@ -534,41 +523,35 @@ impl SegmentFilterCollector {
 impl Debug for SegmentFilterCollector {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         f.debug_struct("SegmentFilterCollector")
-            .field("buckets", &self.parent_buckets)
+            .field("doc_count", &self.doc_count)
             .field("has_sub_aggs", &self.sub_aggregations.is_some())
             .field("accessor_idx", &self.accessor_idx)
             .finish()
     }
 }
 
+impl CollectorClone for SegmentFilterCollector {
+    fn clone_box(&self) -> Box<dyn SegmentAggregationCollector> {
+        // For now, panic - this needs proper implementation with weight recreation
+        panic!("SegmentFilterCollector cloning not yet implemented - requires weight recreation")
+    }
+}
+
 impl SegmentAggregationCollector for SegmentFilterCollector {
     fn add_intermediate_aggregation_result(
-        &mut self,
+        self: Box<Self>,
         agg_data: &AggregationsSegmentCtx,
         results: &mut IntermediateAggregationResults,
-        parent_bucket_id: BucketId,
     ) -> crate::Result<()> {
         let mut sub_results = IntermediateAggregationResults::default();
-        let bucket_opt = self.parent_buckets.get(parent_bucket_id as usize);
 
-        if let Some(sub_aggs) = &mut self.sub_aggregations {
-            sub_aggs
-                .get_sub_agg_collector()
-                .add_intermediate_aggregation_result(
-                    agg_data,
-                    &mut sub_results,
-                    // Here we create a new bucket ID for sub-aggregations if the bucket doesn't
-                    // exist, so that sub-aggregations can still produce results (e.g., zero doc
-                    // count)
-                    bucket_opt
-                        .map(|bucket| bucket.bucket_id)
-                        .unwrap_or(self.bucket_id_provider.next_bucket_id()),
-                )?;
+        if let Some(sub_aggs) = self.sub_aggregations {
+            sub_aggs.add_intermediate_aggregation_result(agg_data, &mut sub_results)?;
         }
 
         // Create the filter bucket result
         let filter_bucket_result = IntermediateBucketResult::Filter {
-            doc_count: bucket_opt.map(|b| b.doc_count).unwrap_or(0),
+            doc_count: self.doc_count,
             sub_aggregations: sub_results,
         };
 
@@ -587,17 +570,32 @@ impl SegmentAggregationCollector for SegmentFilterCollector {
         Ok(())
     }
 
-    fn collect(
+    fn collect(&mut self, doc: DocId, agg_data: &mut AggregationsSegmentCtx) -> crate::Result<()> {
+        // Access the evaluator from FilterAggReqData
+        let req_data = agg_data.get_filter_req_data(self.accessor_idx);
+
+        // O(1) BitSet lookup to check if document matches filter
+        if req_data.evaluator.matches_document(doc) {
+            self.doc_count += 1;
+
+            // If we have sub-aggregations, collect on them for this filtered document
+            if let Some(sub_aggs) = &mut self.sub_aggregations {
+                sub_aggs.collect(doc, agg_data)?;
+            }
+        }
+        Ok(())
+    }
+
+    #[inline]
+    fn collect_block(
         &mut self,
-        parent_bucket_id: BucketId,
-        docs: &[crate::DocId],
+        docs: &[DocId],
         agg_data: &mut AggregationsSegmentCtx,
     ) -> crate::Result<()> {
         if docs.is_empty() {
             return Ok(());
         }
 
-        let mut bucket = self.parent_buckets[parent_bucket_id as usize];
         // Take the request data to avoid borrow checker issues with sub-aggregations
         let mut req = agg_data.take_filter_req_data(self.accessor_idx);
 
@@ -606,24 +604,18 @@ impl SegmentAggregationCollector for SegmentFilterCollector {
         req.evaluator
             .filter_batch(docs, &mut req.matching_docs_buffer);
 
-        bucket.doc_count += req.matching_docs_buffer.len() as u64;
+        self.doc_count += req.matching_docs_buffer.len() as u64;
 
         // Batch process sub-aggregations if we have matches
         if !req.matching_docs_buffer.is_empty() {
             if let Some(sub_aggs) = &mut self.sub_aggregations {
-                for &doc_id in &req.matching_docs_buffer {
-                    sub_aggs.push(bucket.bucket_id, doc_id);
-                }
+                // Use collect_block for better sub-aggregation performance
+                sub_aggs.collect_block(&req.matching_docs_buffer, agg_data)?;
             }
         }
 
         // Put the request data back
         agg_data.put_back_filter_req_data(self.accessor_idx, req);
-        if let Some(sub_aggs) = &mut self.sub_aggregations {
-            sub_aggs.check_flush_local(agg_data)?;
-        }
-        // put back bucket
-        self.parent_buckets[parent_bucket_id as usize] = bucket;
 
         Ok(())
     }
@@ -634,21 +626,6 @@ impl SegmentAggregationCollector for SegmentFilterCollector {
         }
         Ok(())
     }
-
-    fn prepare_max_bucket(
-        &mut self,
-        max_bucket: BucketId,
-        _agg_data: &AggregationsSegmentCtx,
-    ) -> crate::Result<()> {
-        while self.parent_buckets.len() <= max_bucket as usize {
-            let bucket_id = self.bucket_id_provider.next_bucket_id();
-            self.parent_buckets.push(DocCount {
-                doc_count: 0,
-                bucket_id,
-            });
-        }
-        Ok(())
-    }
 }
 
 /// Intermediate result for filter aggregation
@@ -1542,9 +1519,9 @@ mod tests {
         let searcher = reader.searcher();
 
         let agg = json!({
-                "test": {
-                    "filter": deserialized,
-                    "aggs": { "count": { "value_count": { "field": "brand" } } }
+            "test": {
+                "filter": deserialized,
+                "aggs": { "count": { "value_count": { "field": "brand" } } }
             }
         });
 

src/aggregation/bucket/histogram/histogram.rs

@@ -1,6 +1,6 @@
 use std::cmp::Ordering;
 
-use columnar::{Column, ColumnType};
+use columnar::{Column, ColumnBlockAccessor, ColumnType};
 use rustc_hash::FxHashMap;
 use serde::{Deserialize, Serialize};
 use tantivy_bitpacker::minmax;
@@ -8,14 +8,14 @@ use tantivy_bitpacker::minmax;
 use crate::aggregation::agg_data::{
     build_segment_agg_collectors, AggRefNode, AggregationsSegmentCtx,
 };
+use crate::aggregation::agg_limits::MemoryConsumption;
 use crate::aggregation::agg_req::Aggregations;
 use crate::aggregation::agg_result::BucketEntry;
-use crate::aggregation::cached_sub_aggs::CachedSubAggs;
 use crate::aggregation::intermediate_agg_result::{
     IntermediateAggregationResult, IntermediateAggregationResults, IntermediateBucketResult,
     IntermediateHistogramBucketEntry,
 };
-use crate::aggregation::segment_agg_result::{BucketIdProvider, SegmentAggregationCollector};
+use crate::aggregation::segment_agg_result::SegmentAggregationCollector;
 use crate::aggregation::*;
 use crate::TantivyError;
 
@@ -26,8 +26,13 @@ pub struct HistogramAggReqData {
     pub accessor: Column<u64>,
     /// The field type of the fast field.
     pub field_type: ColumnType,
+    /// The column block accessor to access the fast field values.
+    pub column_block_accessor: ColumnBlockAccessor<u64>,
     /// The name of the aggregation.
     pub name: String,
+    /// The sub aggregation blueprint, used to create sub aggregations for each bucket.
+    /// Will be filled during initialization of the collector.
+    pub sub_aggregation_blueprint: Option<Box<dyn SegmentAggregationCollector>>,
     /// The histogram aggregation request.
     pub req: HistogramAggregation,
     /// True if this is a date_histogram aggregation.
@@ -252,24 +257,18 @@ impl HistogramBounds {
 pub(crate) struct SegmentHistogramBucketEntry {
     pub key: f64,
     pub doc_count: u64,
-    pub bucket_id: BucketId,
 }
 
 impl SegmentHistogramBucketEntry {
     pub(crate) fn into_intermediate_bucket_entry(
         self,
-        sub_aggregation: &mut Option<CachedSubAggs>,
+        sub_aggregation: Option<Box<dyn SegmentAggregationCollector>>,
         agg_data: &AggregationsSegmentCtx,
     ) -> crate::Result<IntermediateHistogramBucketEntry> {
         let mut sub_aggregation_res = IntermediateAggregationResults::default();
         if let Some(sub_aggregation) = sub_aggregation {
             sub_aggregation
-                .get_sub_agg_collector()
-                .add_intermediate_aggregation_result(
-                    agg_data,
-                    &mut sub_aggregation_res,
-                    self.bucket_id,
-                )?;
+                .add_intermediate_aggregation_result(agg_data, &mut sub_aggregation_res)?;
         }
         Ok(IntermediateHistogramBucketEntry {
             key: self.key,
@@ -279,38 +278,27 @@ impl SegmentHistogramBucketEntry {
     }
 }
 
-#[derive(Clone, Debug, Default)]
-struct HistogramBuckets {
-    pub buckets: FxHashMap<i64, SegmentHistogramBucketEntry>,
-}
-
 /// The collector puts values from the fast field into the correct buckets and does a conversion to
 /// the correct datatype.
-#[derive(Debug)]
+#[derive(Clone, Debug)]
 pub struct SegmentHistogramCollector {
     /// The buckets containing the aggregation data.
-    /// One Histogram bucket per parent bucket id.
-    parent_buckets: Vec<HistogramBuckets>,
-    sub_agg: Option<CachedSubAggs>,
+    buckets: FxHashMap<i64, SegmentHistogramBucketEntry>,
+    sub_aggregations: FxHashMap<i64, Box<dyn SegmentAggregationCollector>>,
     accessor_idx: usize,
-    bucket_id_provider: BucketIdProvider,
 }
 
 impl SegmentAggregationCollector for SegmentHistogramCollector {
     fn add_intermediate_aggregation_result(
-        &mut self,
+        self: Box<Self>,
         agg_data: &AggregationsSegmentCtx,
         results: &mut IntermediateAggregationResults,
-        parent_bucket_id: BucketId,
     ) -> crate::Result<()> {
         let name = agg_data
             .get_histogram_req_data(self.accessor_idx)
             .name
             .clone();
-        // TODO: avoid prepare_max_bucket here and handle empty buckets.
-        self.prepare_max_bucket(parent_bucket_id, agg_data)?;
-        let histogram = std::mem::take(&mut self.parent_buckets[parent_bucket_id as usize]);
-        let bucket = self.add_intermediate_bucket_result(agg_data, histogram)?;
+        let bucket = self.into_intermediate_bucket_result(agg_data)?;
         results.push(name, IntermediateAggregationResult::Bucket(bucket))?;
 
         Ok(())
@@ -319,40 +307,44 @@ impl SegmentAggregationCollector for SegmentHistogramCollector {
     #[inline]
     fn collect(
         &mut self,
-        parent_bucket_id: BucketId,
+        doc: crate::DocId,
+        agg_data: &mut AggregationsSegmentCtx,
+    ) -> crate::Result<()> {
+        self.collect_block(&[doc], agg_data)
+    }
+
+    #[inline]
+    fn collect_block(
+        &mut self,
         docs: &[crate::DocId],
         agg_data: &mut AggregationsSegmentCtx,
     ) -> crate::Result<()> {
-        let req = agg_data.take_histogram_req_data(self.accessor_idx);
+        let mut req = agg_data.take_histogram_req_data(self.accessor_idx);
         let mem_pre = self.get_memory_consumption();
-        let buckets = &mut self.parent_buckets[parent_bucket_id as usize].buckets;
 
         let bounds = req.bounds;
         let interval = req.req.interval;
         let offset = req.offset;
         let get_bucket_pos = |val| get_bucket_pos_f64(val, interval, offset) as i64;
 
-        agg_data
-            .column_block_accessor
-            .fetch_block(docs, &req.accessor);
-        for (doc, val) in agg_data
+        req.column_block_accessor.fetch_block(docs, &req.accessor);
+        for (doc, val) in req
             .column_block_accessor
             .iter_docid_vals(docs, &req.accessor)
         {
-            let val = f64_from_fastfield_u64(val, req.field_type);
+            let val = f64_from_fastfield_u64(val, &req.field_type);
             let bucket_pos = get_bucket_pos(val);
             if bounds.contains(val) {
-                let bucket = buckets.entry(bucket_pos).or_insert_with(|| {
+                let bucket = self.buckets.entry(bucket_pos).or_insert_with(|| {
                     let key = get_bucket_key_from_pos(bucket_pos as f64, interval, offset);
-                    SegmentHistogramBucketEntry {
-                        key,
-                        doc_count: 0,
-                        bucket_id: self.bucket_id_provider.next_bucket_id(),
-                    }
+                    SegmentHistogramBucketEntry { key, doc_count: 0 }
                 });
                 bucket.doc_count += 1;
-                if let Some(sub_agg) = &mut self.sub_agg {
-                    sub_agg.push(bucket.bucket_id, doc);
+                if let Some(sub_aggregation_blueprint) = req.sub_aggregation_blueprint.as_ref() {
+                    self.sub_aggregations
+                        .entry(bucket_pos)
+                        .or_insert_with(|| sub_aggregation_blueprint.clone())
+                        .collect(doc, agg_data)?;
                 }
             }
         }
@@ -366,30 +358,14 @@ impl SegmentAggregationCollector for SegmentHistogramCollector {
                 .add_memory_consumed(mem_delta as u64)?;
         }
 
-        if let Some(sub_agg) = &mut self.sub_agg {
-            sub_agg.check_flush_local(agg_data)?;
-        }
-
         Ok(())
     }
 
     fn flush(&mut self, agg_data: &mut AggregationsSegmentCtx) -> crate::Result<()> {
-        if let Some(sub_aggregation) = &mut self.sub_agg {
+        for sub_aggregation in self.sub_aggregations.values_mut() {
             sub_aggregation.flush(agg_data)?;
         }
-        Ok(())
-    }
 
-    fn prepare_max_bucket(
-        &mut self,
-        max_bucket: BucketId,
-        _agg_data: &AggregationsSegmentCtx,
-    ) -> crate::Result<()> {
-        while self.parent_buckets.len() <= max_bucket as usize {
-            self.parent_buckets.push(HistogramBuckets {
-                buckets: FxHashMap::default(),
-            });
-        }
         Ok(())
     }
 }
@@ -397,19 +373,22 @@ impl SegmentAggregationCollector for SegmentHistogramCollector {
 impl SegmentHistogramCollector {
     fn get_memory_consumption(&self) -> usize {
         let self_mem = std::mem::size_of::<Self>();
-        let buckets_mem = self.parent_buckets.len() * std::mem::size_of::<HistogramBuckets>();
-        self_mem + buckets_mem
+        let sub_aggs_mem = self.sub_aggregations.memory_consumption();
+        let buckets_mem = self.buckets.memory_consumption();
+        self_mem + sub_aggs_mem + buckets_mem
     }
     /// Converts the collector result into a intermediate bucket result.
-    fn add_intermediate_bucket_result(
-        &mut self,
+    pub fn into_intermediate_bucket_result(
+        self,
         agg_data: &AggregationsSegmentCtx,
-        histogram: HistogramBuckets,
     ) -> crate::Result<IntermediateBucketResult> {
-        let mut buckets = Vec::with_capacity(histogram.buckets.len());
+        let mut buckets = Vec::with_capacity(self.buckets.len());
 
-        for bucket in histogram.buckets.into_values() {
-            let bucket_res = bucket.into_intermediate_bucket_entry(&mut self.sub_agg, agg_data);
+        for (bucket_pos, bucket) in self.buckets {
+            let bucket_res = bucket.into_intermediate_bucket_entry(
+                self.sub_aggregations.get(&bucket_pos).cloned(),
+                agg_data,
+            );
 
             buckets.push(bucket_res?);
         }
@@ -429,7 +408,7 @@ impl SegmentHistogramCollector {
         agg_data: &mut AggregationsSegmentCtx,
         node: &AggRefNode,
     ) -> crate::Result<Self> {
-        let sub_agg = if !node.children.is_empty() {
+        let blueprint = if !node.children.is_empty() {
             Some(build_segment_agg_collectors(agg_data, &node.children)?)
         } else {
             None
@@ -444,13 +423,13 @@ impl SegmentHistogramCollector {
             max: f64::MAX,
         });
         req_data.offset = req_data.req.offset.unwrap_or(0.0);
-        let sub_agg = sub_agg.map(CachedSubAggs::new);
+
+        req_data.sub_aggregation_blueprint = blueprint;
 
         Ok(Self {
-            parent_buckets: Default::default(),
-            sub_agg,
+            buckets: Default::default(),
+            sub_aggregations: Default::default(),
             accessor_idx: node.idx_in_req_data,
-            bucket_id_provider: BucketIdProvider::default(),
         })
     }
 }

src/aggregation/bucket/range.rs

@@ -1,20 +1,18 @@
 use std::fmt::Debug;
 use std::ops::Range;
 
-use columnar::{Column, ColumnType};
+use columnar::{Column, ColumnBlockAccessor, ColumnType};
 use rustc_hash::FxHashMap;
 use serde::{Deserialize, Serialize};
 
 use crate::aggregation::agg_data::{
     build_segment_agg_collectors, AggRefNode, AggregationsSegmentCtx,
 };
-use crate::aggregation::agg_limits::AggregationLimitsGuard;
-use crate::aggregation::cached_sub_aggs::CachedSubAggs;
 use crate::aggregation::intermediate_agg_result::{
     IntermediateAggregationResult, IntermediateAggregationResults, IntermediateBucketResult,
     IntermediateRangeBucketEntry, IntermediateRangeBucketResult,
 };
-use crate::aggregation::segment_agg_result::{BucketIdProvider, SegmentAggregationCollector};
+use crate::aggregation::segment_agg_result::SegmentAggregationCollector;
 use crate::aggregation::*;
 use crate::TantivyError;
 
@@ -25,12 +23,12 @@ pub struct RangeAggReqData {
     pub accessor: Column<u64>,
     /// The type of the fast field.
     pub field_type: ColumnType,
+    /// The column block accessor to access the fast field values.
+    pub column_block_accessor: ColumnBlockAccessor<u64>,
     /// The range aggregation request.
     pub req: RangeAggregation,
     /// The name of the aggregation.
     pub name: String,
-    /// Whether this is a top-level aggregation.
-    pub is_top_level: bool,
 }
 
 impl RangeAggReqData {
@@ -153,47 +151,19 @@ pub(crate) struct SegmentRangeAndBucketEntry {
 
 /// The collector puts values from the fast field into the correct buckets and does a conversion to
 /// the correct datatype.
-pub struct SegmentRangeCollector<const LOWCARD: bool = false> {
+#[derive(Clone, Debug)]
+pub struct SegmentRangeCollector {
     /// The buckets containing the aggregation data.
-    /// One for each ParentBucketId
-    parent_buckets: Vec<Vec<SegmentRangeAndBucketEntry>>,
+    buckets: Vec<SegmentRangeAndBucketEntry>,
     column_type: ColumnType,
     pub(crate) accessor_idx: usize,
-    sub_agg: Option<CachedSubAggs<LOWCARD>>,
-    /// Here things get a bit weird. We need to assign unique bucket ids across all
-    /// parent buckets. So we keep track of the next available bucket id here.
-    /// This allows a kind of flattening of the bucket ids across all parent buckets.
-    /// E.g. in nested aggregations:
-    /// Term Agg -> Range aggregation -> Stats aggregation
-    /// E.g. the Term Agg creates 3 buckets ["INFO", "ERROR", "WARN"], each of these has a Range
-    /// aggregation with 4 buckets. The Range aggregation will create buckets with ids:
-    /// - INFO: 0,1,2,3
-    /// - ERROR: 4,5,6,7
-    /// - WARN: 8,9,10,11
-    ///
-    /// This allows the Stats aggregation to have unique bucket ids to refer to.
-    bucket_id_provider: BucketIdProvider,
-    limits: AggregationLimitsGuard,
 }
 
-impl<const LOWCARD: bool> Debug for SegmentRangeCollector<LOWCARD> {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.debug_struct("SegmentRangeCollector")
-            .field("parent_buckets_len", &self.parent_buckets.len())
-            .field("column_type", &self.column_type)
-            .field("accessor_idx", &self.accessor_idx)
-            .field("has_sub_agg", &self.sub_agg.is_some())
-            .finish()
-    }
-}
-
-/// TODO: Bad naming, there's also SegmentRangeAndBucketEntry
 #[derive(Clone)]
 pub(crate) struct SegmentRangeBucketEntry {
     pub key: Key,
     pub doc_count: u64,
-    // pub sub_aggregation: Option<Box<dyn SegmentAggregationCollector>>,
-    pub bucket_id: BucketId,
+    pub sub_aggregation: Option<Box<dyn SegmentAggregationCollector>>,
     /// The from range of the bucket. Equals `f64::MIN` when `None`.
     pub from: Option<f64>,
     /// The to range of the bucket. Equals `f64::MAX` when `None`. Open interval, `to` is not
@@ -214,50 +184,48 @@ impl Debug for SegmentRangeBucketEntry {
 impl SegmentRangeBucketEntry {
     pub(crate) fn into_intermediate_bucket_entry(
         self,
+        agg_data: &AggregationsSegmentCtx,
     ) -> crate::Result<IntermediateRangeBucketEntry> {
-        let sub_aggregation = IntermediateAggregationResults::default();
+        let mut sub_aggregation_res = IntermediateAggregationResults::default();
+        if let Some(sub_aggregation) = self.sub_aggregation {
+            sub_aggregation
+                .add_intermediate_aggregation_result(agg_data, &mut sub_aggregation_res)?
+        } else {
+            Default::default()
+        };
 
         Ok(IntermediateRangeBucketEntry {
             key: self.key.into(),
             doc_count: self.doc_count,
-            sub_aggregation_res: sub_aggregation,
+            sub_aggregation: sub_aggregation_res,
             from: self.from,
             to: self.to,
         })
     }
 }
 
-impl<const LOWCARD: bool> SegmentAggregationCollector for SegmentRangeCollector<LOWCARD> {
+impl SegmentAggregationCollector for SegmentRangeCollector {
     fn add_intermediate_aggregation_result(
-        &mut self,
+        self: Box<Self>,
         agg_data: &AggregationsSegmentCtx,
         results: &mut IntermediateAggregationResults,
-        parent_bucket_id: BucketId,
     ) -> crate::Result<()> {
-        self.prepare_max_bucket(parent_bucket_id, agg_data)?;
         let field_type = self.column_type;
         let name = agg_data
             .get_range_req_data(self.accessor_idx)
             .name
             .to_string();
 
-        let buckets = std::mem::take(&mut self.parent_buckets[parent_bucket_id as usize]);
-
-        let buckets: FxHashMap<SerializedKey, IntermediateRangeBucketEntry> = buckets
+        let buckets: FxHashMap<SerializedKey, IntermediateRangeBucketEntry> = self
+            .buckets
             .into_iter()
-            .map(|range_bucket| {
-                let bucket_id = range_bucket.bucket.bucket_id;
-                let mut agg = range_bucket.bucket.into_intermediate_bucket_entry()?;
-                if let Some(sub_aggregation) = &mut self.sub_agg {
-                    sub_aggregation
-                        .get_sub_agg_collector()
-                        .add_intermediate_aggregation_result(
-                            agg_data,
-                            &mut agg.sub_aggregation_res,
-                            bucket_id,
-                        )?;
-                }
-                Ok((range_to_string(&range_bucket.range, &field_type)?, agg))
+            .map(move |range_bucket| {
+                Ok((
+                    range_to_string(&range_bucket.range, &field_type)?,
+                    range_bucket
+                        .bucket
+                        .into_intermediate_bucket_entry(agg_data)?,
+                ))
             })
             .collect::<crate::Result<_>>()?;
 
@@ -274,114 +242,73 @@ impl<const LOWCARD: bool> SegmentAggregationCollector for SegmentRangeCollector<
     #[inline]
     fn collect(
         &mut self,
-        parent_bucket_id: BucketId,
+        doc: crate::DocId,
+        agg_data: &mut AggregationsSegmentCtx,
+    ) -> crate::Result<()> {
+        self.collect_block(&[doc], agg_data)
+    }
+
+    #[inline]
+    fn collect_block(
+        &mut self,
         docs: &[crate::DocId],
         agg_data: &mut AggregationsSegmentCtx,
     ) -> crate::Result<()> {
-        let req = agg_data.take_range_req_data(self.accessor_idx);
+        // Take request data to avoid borrow conflicts during sub-aggregation
+        let mut req = agg_data.take_range_req_data(self.accessor_idx);
 
-        agg_data
-            .column_block_accessor
-            .fetch_block(docs, &req.accessor);
+        req.column_block_accessor.fetch_block(docs, &req.accessor);
 
-        let buckets = &mut self.parent_buckets[parent_bucket_id as usize];
-
-        for (doc, val) in agg_data
+        for (doc, val) in req
             .column_block_accessor
             .iter_docid_vals(docs, &req.accessor)
         {
-            let bucket_pos = get_bucket_pos(val, buckets);
-            let bucket = &mut buckets[bucket_pos];
+            let bucket_pos = self.get_bucket_pos(val);
+            let bucket = &mut self.buckets[bucket_pos];
             bucket.bucket.doc_count += 1;
-            if let Some(sub_agg) = self.sub_agg.as_mut() {
-                sub_agg.push(bucket.bucket.bucket_id, doc);
+            if let Some(sub_agg) = bucket.bucket.sub_aggregation.as_mut() {
+                sub_agg.collect(doc, agg_data)?;
             }
         }
 
         agg_data.put_back_range_req_data(self.accessor_idx, req);
-        if let Some(sub_agg) = self.sub_agg.as_mut() {
-            sub_agg.check_flush_local(agg_data)?;
-        }
 
         Ok(())
     }
 
     fn flush(&mut self, agg_data: &mut AggregationsSegmentCtx) -> crate::Result<()> {
-        if let Some(sub_agg) = self.sub_agg.as_mut() {
-            sub_agg.flush(agg_data)?;
+        for bucket in self.buckets.iter_mut() {
+            if let Some(sub_agg) = bucket.bucket.sub_aggregation.as_mut() {
+                sub_agg.flush(agg_data)?;
+            }
         }
         Ok(())
     }
-
-    fn prepare_max_bucket(
-        &mut self,
-        max_bucket: BucketId,
-        agg_data: &AggregationsSegmentCtx,
-    ) -> crate::Result<()> {
-        while self.parent_buckets.len() <= max_bucket as usize {
-            let new_buckets = self.create_new_buckets(agg_data)?;
-            self.parent_buckets.push(new_buckets);
-        }
-
-        Ok(())
-    }
-}
-/// Build a concrete `SegmentRangeCollector` with either a Vec- or HashMap-backed
-/// bucket storage, depending on the column type and aggregation level.
-pub(crate) fn build_segment_range_collector(
-    agg_data: &mut AggregationsSegmentCtx,
-    node: &AggRefNode,
-) -> crate::Result<Box<dyn SegmentAggregationCollector>> {
-    let accessor_idx = node.idx_in_req_data;
-    let req_data = agg_data.get_range_req_data(node.idx_in_req_data);
-    let field_type = req_data.field_type;
-
-    // TODO: A better metric instead of is_top_level would be the number of buckets expected.
-    // E.g. If range agg is not top level, but the parent is a bucket agg with less than 10 buckets,
-    // we can are still in low cardinality territory.
-    let is_low_card = req_data.is_top_level && req_data.req.ranges.len() <= 64;
-
-    let sub_agg = if !node.children.is_empty() {
-        Some(build_segment_agg_collectors(agg_data, &node.children)?)
-    } else {
-        None
-    };
-
-    if is_low_card {
-        Ok(Box::new(SegmentRangeCollector {
-            sub_agg: sub_agg.map(CachedSubAggs::<true>::new),
-            column_type: field_type,
-            accessor_idx,
-            parent_buckets: Vec::new(),
-            bucket_id_provider: BucketIdProvider::default(),
-            limits: agg_data.context.limits.clone(),
-        }))
-    } else {
-        Ok(Box::new(SegmentRangeCollector {
-            sub_agg: sub_agg.map(CachedSubAggs::<false>::new),
-            column_type: field_type,
-            accessor_idx,
-            parent_buckets: Vec::new(),
-            bucket_id_provider: BucketIdProvider::default(),
-            limits: agg_data.context.limits.clone(),
-        }))
-    }
 }
 
-impl<const LOWCARD: bool> SegmentRangeCollector<LOWCARD> {
-    pub(crate) fn create_new_buckets(
-        &mut self,
-        agg_data: &AggregationsSegmentCtx,
-    ) -> crate::Result<Vec<SegmentRangeAndBucketEntry>> {
-        let field_type = self.column_type;
-        let req_data = agg_data.get_range_req_data(self.accessor_idx);
+impl SegmentRangeCollector {
+    pub(crate) fn from_req_and_validate(
+        req_data: &mut AggregationsSegmentCtx,
+        node: &AggRefNode,
+    ) -> crate::Result<Self> {
+        let accessor_idx = node.idx_in_req_data;
+        let (field_type, ranges) = {
+            let req_view = req_data.get_range_req_data(node.idx_in_req_data);
+            (req_view.field_type, req_view.req.ranges.clone())
+        };
+
         // The range input on the request is f64.
         // We need to convert to u64 ranges, because we read the values as u64.
         // The mapping from the conversion is monotonic so ordering is preserved.
-        let buckets: Vec<_> = extend_validate_ranges(&req_data.req.ranges, &field_type)?
+        let sub_agg_prototype = if !node.children.is_empty() {
+            Some(build_segment_agg_collectors(req_data, &node.children)?)
+        } else {
+            None
+        };
+
+        let buckets: Vec<_> = extend_validate_ranges(&ranges, &field_type)?
             .iter()
             .map(|range| {
-                let bucket_id = self.bucket_id_provider.next_bucket_id();
                 let key = range
                     .key
                     .clone()
@@ -390,20 +317,20 @@ impl<const LOWCARD: bool> SegmentRangeCollector<LOWCARD> {
                 let to = if range.range.end == u64::MAX {
                     None
                 } else {
-                    Some(f64_from_fastfield_u64(range.range.end, field_type))
+                    Some(f64_from_fastfield_u64(range.range.end, &field_type))
                 };
                 let from = if range.range.start == u64::MIN {
                     None
                 } else {
-                    Some(f64_from_fastfield_u64(range.range.start, field_type))
+                    Some(f64_from_fastfield_u64(range.range.start, &field_type))
                 };
-                // let sub_aggregation = sub_agg_prototype.clone();
+                let sub_aggregation = sub_agg_prototype.clone();
 
                 Ok(SegmentRangeAndBucketEntry {
                     range: range.range.clone(),
                     bucket: SegmentRangeBucketEntry {
                         doc_count: 0,
-                        bucket_id,
+                        sub_aggregation,
                         key,
                         from,
                         to,
@@ -412,19 +339,26 @@ impl<const LOWCARD: bool> SegmentRangeCollector<LOWCARD> {
             })
             .collect::<crate::Result<_>>()?;
 
-        self.limits.add_memory_consumed(
+        req_data.context.limits.add_memory_consumed(
             buckets.len() as u64 * std::mem::size_of::<SegmentRangeAndBucketEntry>() as u64,
         )?;
-        Ok(buckets)
+
+        Ok(SegmentRangeCollector {
+            buckets,
+            column_type: field_type,
+            accessor_idx,
+        })
+    }
+
+    #[inline]
+    fn get_bucket_pos(&self, val: u64) -> usize {
+        let pos = self
+            .buckets
+            .binary_search_by_key(&val, |probe| probe.range.start)
+            .unwrap_or_else(|pos| pos - 1);
+        debug_assert!(self.buckets[pos].range.contains(&val));
+        pos
     }
-}
-#[inline]
-fn get_bucket_pos(val: u64, buckets: &[SegmentRangeAndBucketEntry]) -> usize {
-    let pos = buckets
-        .binary_search_by_key(&val, |probe| probe.range.start)
-        .unwrap_or_else(|pos| pos - 1);
-    debug_assert!(buckets[pos].range.contains(&val));
-    pos
 }
 
 /// Converts the user provided f64 range value to fast field value space.
@@ -522,7 +456,7 @@ pub(crate) fn range_to_string(
             let val = i64::from_u64(val);
             format_date(val)
         } else {
-            Ok(f64_from_fastfield_u64(val, *field_type).to_string())
+            Ok(f64_from_fastfield_u64(val, field_type).to_string())
         }
     };
 
@@ -572,33 +506,30 @@ mod tests {
                 let to = if range.range.end == u64::MAX {
                     None
                 } else {
-                    Some(f64_from_fastfield_u64(range.range.end, field_type))
+                    Some(f64_from_fastfield_u64(range.range.end, &field_type))
                 };
                 let from = if range.range.start == u64::MIN {
                     None
                 } else {
-                    Some(f64_from_fastfield_u64(range.range.start, field_type))
+                    Some(f64_from_fastfield_u64(range.range.start, &field_type))
                 };
                 SegmentRangeAndBucketEntry {
                     range: range.range.clone(),
                     bucket: SegmentRangeBucketEntry {
                         doc_count: 0,
+                        sub_aggregation: None,
                         key,
                         from,
                         to,
-                        bucket_id: 0,
                     },
                 }
             })
             .collect();
 
         SegmentRangeCollector {
-            parent_buckets: vec![buckets],
+            buckets,
             column_type: field_type,
             accessor_idx: 0,
-            sub_agg: None,
-            bucket_id_provider: Default::default(),
-            limits: AggregationLimitsGuard::default(),
         }
     }
 
@@ -845,7 +776,7 @@ mod tests {
         let buckets = vec![(10f64..20f64).into(), (30f64..40f64).into()];
         let collector = get_collector_from_ranges(buckets, ColumnType::F64);
 
-        let buckets = collector.parent_buckets[0].clone();
+        let buckets = collector.buckets;
         assert_eq!(buckets[0].range.start, u64::MIN);
         assert_eq!(buckets[0].range.end, 10f64.to_u64());
         assert_eq!(buckets[1].range.start, 10f64.to_u64());
@@ -868,7 +799,7 @@ mod tests {
         ];
         let collector = get_collector_from_ranges(buckets, ColumnType::F64);
 
-        let buckets = collector.parent_buckets[0].clone();
+        let buckets = collector.buckets;
         assert_eq!(buckets[0].range.start, u64::MIN);
         assert_eq!(buckets[0].range.end, 10f64.to_u64());
         assert_eq!(buckets[1].range.start, 10f64.to_u64());
@@ -883,7 +814,7 @@ mod tests {
         let buckets = vec![(-10f64..-1f64).into()];
         let collector = get_collector_from_ranges(buckets, ColumnType::F64);
 
-        let buckets = collector.parent_buckets[0].clone();
+        let buckets = collector.buckets;
         assert_eq!(&buckets[0].bucket.key.to_string(), "*--10");
         assert_eq!(&buckets[buckets.len() - 1].bucket.key.to_string(), "-1-*");
     }
@@ -892,7 +823,7 @@ mod tests {
         let buckets = vec![(0f64..10f64).into()];
         let collector = get_collector_from_ranges(buckets, ColumnType::F64);
 
-        let buckets = collector.parent_buckets[0].clone();
+        let buckets = collector.buckets;
         assert_eq!(&buckets[0].bucket.key.to_string(), "*-0");
         assert_eq!(&buckets[buckets.len() - 1].bucket.key.to_string(), "10-*");
     }
@@ -901,7 +832,7 @@ mod tests {
     fn range_binary_search_test_u64() {
         let check_ranges = |ranges: Vec<RangeAggregationRange>| {
             let collector = get_collector_from_ranges(ranges, ColumnType::U64);
-            let search = |val: u64| get_bucket_pos(val, &collector.parent_buckets[0]);
+            let search = |val: u64| collector.get_bucket_pos(val);
 
             assert_eq!(search(u64::MIN), 0);
             assert_eq!(search(9), 0);
@@ -947,7 +878,7 @@ mod tests {
         let ranges = vec![(10.0..100.0).into()];
 
         let collector = get_collector_from_ranges(ranges, ColumnType::F64);
-        let search = |val: u64| get_bucket_pos(val, &collector.parent_buckets[0]);
+        let search = |val: u64| collector.get_bucket_pos(val);
 
         assert_eq!(search(u64::MIN), 0);
         assert_eq!(search(9f64.to_u64()), 0);
@@ -959,3 +890,63 @@ mod tests {
                                              // the max value
     }
 }
+
+#[cfg(all(test, feature = "unstable"))]
+mod bench {
+
+    use itertools::Itertools;
+    use rand::seq::SliceRandom;
+    use rand::thread_rng;
+
+    use super::*;
+    use crate::aggregation::bucket::range::tests::get_collector_from_ranges;
+
+    const TOTAL_DOCS: u64 = 1_000_000u64;
+    const NUM_DOCS: u64 = 50_000u64;
+
+    fn get_collector_with_buckets(num_buckets: u64, num_docs: u64) -> SegmentRangeCollector {
+        let bucket_size = num_docs / num_buckets;
+        let mut buckets: Vec<RangeAggregationRange> = vec![];
+        for i in 0..num_buckets {
+            let bucket_start = (i * bucket_size) as f64;
+            buckets.push((bucket_start..bucket_start + bucket_size as f64).into())
+        }
+
+        get_collector_from_ranges(buckets, ColumnType::U64)
+    }
+
+    fn get_rand_docs(total_docs: u64, num_docs_returned: u64) -> Vec<u64> {
+        let mut rng = thread_rng();
+
+        let all_docs = (0..total_docs - 1).collect_vec();
+        let mut vals = all_docs
+            .as_slice()
+            .choose_multiple(&mut rng, num_docs_returned as usize)
+            .cloned()
+            .collect_vec();
+        vals.sort();
+        vals
+    }
+
+    fn bench_range_binary_search(b: &mut test::Bencher, num_buckets: u64) {
+        let collector = get_collector_with_buckets(num_buckets, TOTAL_DOCS);
+        let vals = get_rand_docs(TOTAL_DOCS, NUM_DOCS);
+        b.iter(|| {
+            let mut bucket_pos = 0;
+            for val in &vals {
+                bucket_pos = collector.get_bucket_pos(*val);
+            }
+            bucket_pos
+        })
+    }
+
+    #[bench]
+    fn bench_range_100_buckets(b: &mut test::Bencher) {
+        bench_range_binary_search(b, 100)
+    }
+
+    #[bench]
+    fn bench_range_10_buckets(b: &mut test::Bencher) {
+        bench_range_binary_search(b, 10)
+    }
+}

src/aggregation/bucket/term_missing_agg.rs

@@ -5,13 +5,11 @@ use crate::aggregation::agg_data::{
     build_segment_agg_collectors, AggRefNode, AggregationsSegmentCtx,
 };
 use crate::aggregation::bucket::term_agg::TermsAggregation;
-use crate::aggregation::cached_sub_aggs::CachedSubAggs;
 use crate::aggregation::intermediate_agg_result::{
     IntermediateAggregationResult, IntermediateAggregationResults, IntermediateBucketResult,
     IntermediateKey, IntermediateTermBucketEntry, IntermediateTermBucketResult,
 };
-use crate::aggregation::segment_agg_result::{BucketIdProvider, SegmentAggregationCollector};
-use crate::aggregation::BucketId;
+use crate::aggregation::segment_agg_result::SegmentAggregationCollector;
 
 /// Special aggregation to handle missing values for term aggregations.
 /// This missing aggregation will check multiple columns for existence.
@@ -37,55 +35,41 @@ impl MissingTermAggReqData {
     }
 }
 
-#[derive(Default, Debug, Clone)]
-struct MissingCount {
-    missing_count: u32,
-    bucket_id: BucketId,
-}
-
 /// The specialized missing term aggregation.
-#[derive(Default, Debug)]
+#[derive(Default, Debug, Clone)]
 pub struct TermMissingAgg {
+    missing_count: u32,
     accessor_idx: usize,
-    sub_agg: Option<CachedSubAggs>,
-    /// Idx = parent bucket id, Value = missing count for that bucket
-    missing_count_per_bucket: Vec<MissingCount>,
-    bucket_id_provider: BucketIdProvider,
+    sub_agg: Option<Box<dyn SegmentAggregationCollector>>,
 }
 impl TermMissingAgg {
     pub(crate) fn new(
-        agg_data: &mut AggregationsSegmentCtx,
+        req_data: &mut AggregationsSegmentCtx,
         node: &AggRefNode,
     ) -> crate::Result<Self> {
         let has_sub_aggregations = !node.children.is_empty();
         let accessor_idx = node.idx_in_req_data;
         let sub_agg = if has_sub_aggregations {
-            let sub_aggregation = build_segment_agg_collectors(agg_data, &node.children)?;
+            let sub_aggregation = build_segment_agg_collectors(req_data, &node.children)?;
             Some(sub_aggregation)
         } else {
             None
         };
 
-        let sub_agg = sub_agg.map(CachedSubAggs::new);
-        let bucket_id_provider = BucketIdProvider::default();
-
         Ok(Self {
             accessor_idx,
             sub_agg,
-            missing_count_per_bucket: Vec::new(),
-            bucket_id_provider,
+            ..Default::default()
         })
     }
 }
 
 impl SegmentAggregationCollector for TermMissingAgg {
     fn add_intermediate_aggregation_result(
-        &mut self,
+        self: Box<Self>,
         agg_data: &AggregationsSegmentCtx,
         results: &mut IntermediateAggregationResults,
-        parent_bucket_id: BucketId,
     ) -> crate::Result<()> {
-        self.prepare_max_bucket(parent_bucket_id, agg_data)?;
         let req_data = agg_data.get_missing_term_req_data(self.accessor_idx);
         let term_agg = &req_data.req;
         let missing = term_agg
@@ -96,16 +80,13 @@ impl SegmentAggregationCollector for TermMissingAgg {
         let mut entries: FxHashMap<IntermediateKey, IntermediateTermBucketEntry> =
             Default::default();
 
-        let missing_count = &self.missing_count_per_bucket[parent_bucket_id as usize];
         let mut missing_entry = IntermediateTermBucketEntry {
-            doc_count: missing_count.missing_count,
+            doc_count: self.missing_count,
             sub_aggregation: Default::default(),
         };
-        if let Some(sub_agg) = &mut self.sub_agg {
+        if let Some(sub_agg) = self.sub_agg {
             let mut res = IntermediateAggregationResults::default();
-            sub_agg
-                .get_sub_agg_collector()
-                .add_intermediate_aggregation_result(agg_data, &mut res, missing_count.bucket_id)?;
+            sub_agg.add_intermediate_aggregation_result(agg_data, &mut res)?;
             missing_entry.sub_aggregation = res;
         }
         entries.insert(missing.into(), missing_entry);
@@ -128,52 +109,30 @@ impl SegmentAggregationCollector for TermMissingAgg {
 
     fn collect(
         &mut self,
-        parent_bucket_id: BucketId,
+        doc: crate::DocId,
+        agg_data: &mut AggregationsSegmentCtx,
+    ) -> crate::Result<()> {
+        let req_data = agg_data.get_missing_term_req_data(self.accessor_idx);
+        let has_value = req_data
+            .accessors
+            .iter()
+            .any(|(acc, _)| acc.index.has_value(doc));
+        if !has_value {
+            self.missing_count += 1;
+            if let Some(sub_agg) = self.sub_agg.as_mut() {
+                sub_agg.collect(doc, agg_data)?;
+            }
+        }
+        Ok(())
+    }
+
+    fn collect_block(
+        &mut self,
         docs: &[crate::DocId],
         agg_data: &mut AggregationsSegmentCtx,
     ) -> crate::Result<()> {
-        let bucket = &mut self.missing_count_per_bucket[parent_bucket_id as usize];
-        let req_data = agg_data.get_missing_term_req_data(self.accessor_idx);
-
         for doc in docs {
-            let doc = *doc;
-            let has_value = req_data
-                .accessors
-                .iter()
-                .any(|(acc, _)| acc.index.has_value(doc));
-            if !has_value {
-                bucket.missing_count += 1;
-
-                if let Some(sub_agg) = self.sub_agg.as_mut() {
-                    sub_agg.push(bucket.bucket_id, doc);
-                }
-            }
-        }
-
-        if let Some(sub_agg) = self.sub_agg.as_mut() {
-            sub_agg.check_flush_local(agg_data)?;
-        }
-        Ok(())
-    }
-
-    fn prepare_max_bucket(
-        &mut self,
-        max_bucket: BucketId,
-        _agg_data: &AggregationsSegmentCtx,
-    ) -> crate::Result<()> {
-        while self.missing_count_per_bucket.len() <= max_bucket as usize {
-            let bucket_id = self.bucket_id_provider.next_bucket_id();
-            self.missing_count_per_bucket.push(MissingCount {
-                missing_count: 0,
-                bucket_id,
-            });
-        }
-        Ok(())
-    }
-
-    fn flush(&mut self, agg_data: &mut AggregationsSegmentCtx) -> crate::Result<()> {
-        if let Some(sub_agg) = self.sub_agg.as_mut() {
-            sub_agg.flush(agg_data)?;
+            self.collect(*doc, agg_data)?;
         }
         Ok(())
     }
@@ -296,6 +255,7 @@ mod tests {
     fn terms_aggregation_missing_mult_seg_empty() -> crate::Result<()> {
         let mut schema_builder = Schema::builder();
         let score = schema_builder.add_f64_field("score", FAST);
+        schema_builder.add_json_field("json", FAST);
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
         let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
@@ -343,6 +303,7 @@ mod tests {
     fn terms_aggregation_missing_single_seg_empty() -> crate::Result<()> {
         let mut schema_builder = Schema::builder();
         let score = schema_builder.add_f64_field("score", FAST);
+        schema_builder.add_json_field("json", FAST);
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
         let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();

src/aggregation/buf_collector.rs

@@ -0,0 +1,87 @@
+use super::intermediate_agg_result::IntermediateAggregationResults;
+use super::segment_agg_result::SegmentAggregationCollector;
+use crate::aggregation::agg_data::AggregationsSegmentCtx;
+use crate::DocId;
+
+#[cfg(test)]
+pub(crate) const DOC_BLOCK_SIZE: usize = 64;
+
+#[cfg(not(test))]
+pub(crate) const DOC_BLOCK_SIZE: usize = 256;
+
+pub(crate) type DocBlock = [DocId; DOC_BLOCK_SIZE];
+
+/// BufAggregationCollector buffers documents before calling collect_block().
+#[derive(Clone)]
+pub(crate) struct BufAggregationCollector {
+    pub(crate) collector: Box<dyn SegmentAggregationCollector>,
+    staged_docs: DocBlock,
+    num_staged_docs: usize,
+}
+
+impl std::fmt::Debug for BufAggregationCollector {
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        f.debug_struct("SegmentAggregationResultsCollector")
+            .field("staged_docs", &&self.staged_docs[..self.num_staged_docs])
+            .field("num_staged_docs", &self.num_staged_docs)
+            .finish()
+    }
+}
+
+impl BufAggregationCollector {
+    pub fn new(collector: Box<dyn SegmentAggregationCollector>) -> Self {
+        Self {
+            collector,
+            num_staged_docs: 0,
+            staged_docs: [0; DOC_BLOCK_SIZE],
+        }
+    }
+}
+
+impl SegmentAggregationCollector for BufAggregationCollector {
+    #[inline]
+    fn add_intermediate_aggregation_result(
+        self: Box<Self>,
+        agg_data: &AggregationsSegmentCtx,
+        results: &mut IntermediateAggregationResults,
+    ) -> crate::Result<()> {
+        Box::new(self.collector).add_intermediate_aggregation_result(agg_data, results)
+    }
+
+    #[inline]
+    fn collect(
+        &mut self,
+        doc: crate::DocId,
+        agg_data: &mut AggregationsSegmentCtx,
+    ) -> crate::Result<()> {
+        self.staged_docs[self.num_staged_docs] = doc;
+        self.num_staged_docs += 1;
+        if self.num_staged_docs == self.staged_docs.len() {
+            self.collector
+                .collect_block(&self.staged_docs[..self.num_staged_docs], agg_data)?;
+            self.num_staged_docs = 0;
+        }
+        Ok(())
+    }
+
+    #[inline]
+    fn collect_block(
+        &mut self,
+        docs: &[crate::DocId],
+        agg_data: &mut AggregationsSegmentCtx,
+    ) -> crate::Result<()> {
+        self.collector.collect_block(docs, agg_data)?;
+        Ok(())
+    }
+
+    #[inline]
+    fn flush(&mut self, agg_data: &mut AggregationsSegmentCtx) -> crate::Result<()> {
+        self.collector
+            .collect_block(&self.staged_docs[..self.num_staged_docs], agg_data)?;
+        self.num_staged_docs = 0;
+
+        self.collector.flush(agg_data)?;
+
+        Ok(())
+    }
+}

src/aggregation/cached_sub_aggs.rs

@@ -1,185 +0,0 @@
-use super::segment_agg_result::SegmentAggregationCollector;
-use crate::aggregation::agg_data::AggregationsSegmentCtx;
-use crate::aggregation::bucket::MAX_NUM_TERMS_FOR_VEC;
-use crate::aggregation::BucketId;
-use crate::DocId;
-
-#[derive(Debug)]
-/// A cache for sub-aggregations, storing doc ids per bucket id.
-/// Depending on the cardinality of the parent aggregation, we use different
-/// storage strategies.
-///
-/// ## Low Cardinality
-/// Cardinality here refers to the number of unique flattened buckets that can be created
-/// by the parent aggregation.
-/// Flattened buckets are the result of combining all buckets per collector
-/// into a single list of buckets, where each bucket is identified by its BucketId.
-///
-/// ## Usage
-/// Since this is caching for sub-aggregations, it is only used by bucket
-/// aggregations.
-///
-/// TODO: consider using a more advanced data structure for high cardinality
-/// aggregations.
-/// What this datastructure does in general is to group docs by bucket id.
-pub(crate) struct CachedSubAggs<const LOWCARD: bool = false> {
-    /// Only used when LOWCARD is true.
-    /// Cache doc ids per bucket for sub-aggregations.
-    ///
-    /// The outer Vec is indexed by BucketId.
-    per_bucket_docs: Vec<Vec<DocId>>,
-    /// Only used when LOWCARD is false.
-    ///
-    /// This weird partitioning is used to do some cheap grouping on the bucket ids.
-    /// bucket ids are dense, e.g. when we don't detect the cardinality as low cardinality,
-    /// but there are just 16 bucket ids, each bucket id will go to its own partition.
-    ///
-    /// We want to keep this cheap, because high cardinality aggregations can have a lot of
-    /// buckets, and they may be nothing to group.
-    partitions: [PartitionEntry; NUM_PARTITIONS],
-    pub(crate) sub_agg_collector: Box<dyn SegmentAggregationCollector>,
-    num_docs: usize,
-}
-
-const FLUSH_THRESHOLD: usize = 2048;
-const NUM_PARTITIONS: usize = 16;
-
-impl<const LOWCARD: bool> CachedSubAggs<LOWCARD> {
-    pub fn get_sub_agg_collector(&mut self) -> &mut Box<dyn SegmentAggregationCollector> {
-        &mut self.sub_agg_collector
-    }
-
-    pub fn new(sub_agg: Box<dyn SegmentAggregationCollector>) -> Self {
-        Self {
-            per_bucket_docs: Vec::new(),
-            num_docs: 0,
-            sub_agg_collector: sub_agg,
-            partitions: core::array::from_fn(|_| PartitionEntry::default()),
-        }
-    }
-
-    #[inline]
-    pub fn clear(&mut self) {
-        for v in &mut self.per_bucket_docs {
-            v.clear();
-        }
-        for partition in &mut self.partitions {
-            partition.clear();
-        }
-        self.num_docs = 0;
-    }
-
-    #[inline]
-    pub fn push(&mut self, bucket_id: BucketId, doc_id: DocId) {
-        if LOWCARD {
-            // TODO: We could flush single buckets here
-            let idx = bucket_id as usize;
-            if self.per_bucket_docs.len() <= idx {
-                self.per_bucket_docs.resize_with(idx + 1, Vec::new);
-            }
-            self.per_bucket_docs[idx].push(doc_id);
-        } else {
-            let idx = bucket_id % NUM_PARTITIONS as u32;
-            let slot = &mut self.partitions[idx as usize];
-            slot.bucket_ids.push(bucket_id);
-            slot.docs.push(doc_id);
-        }
-        self.num_docs += 1;
-    }
-
-    /// Check if we need to flush based on the number of documents cached.
-    /// If so, flushes the cache to the provided aggregation collector.
-    pub fn check_flush_local(
-        &mut self,
-        agg_data: &mut AggregationsSegmentCtx,
-    ) -> crate::Result<()> {
-        if self.num_docs >= FLUSH_THRESHOLD {
-            self.flush_local(agg_data, false)?;
-        }
-        Ok(())
-    }
-
-    /// Note: this does _not_ flush the sub aggregations
-    fn flush_local(
-        &mut self,
-        agg_data: &mut AggregationsSegmentCtx,
-        force: bool,
-    ) -> crate::Result<()> {
-        if LOWCARD {
-            // Pre-aggregated: call collect per bucket.
-            let max_bucket = (self.per_bucket_docs.len() as BucketId).saturating_sub(1);
-            self.sub_agg_collector
-                .prepare_max_bucket(max_bucket, agg_data)?;
-            // The threshold above which we flush buckets individually.
-            // Note: We need to make sure that we don't lock ourselves into a situation where we hit
-            // the FLUSH_THRESHOLD, but never flush any buckets. (except the final flush)
-            let mut bucket_treshold = FLUSH_THRESHOLD / (self.per_bucket_docs.len().max(1) * 2);
-            const _: () = {
-                // MAX_NUM_TERMS_FOR_VEC == LOWCARD threshold
-                let bucket_treshold = FLUSH_THRESHOLD / (MAX_NUM_TERMS_FOR_VEC as usize * 2);
-                assert!(
-                    bucket_treshold > 0,
-                    "Bucket threshold must be greater than 0"
-                );
-            };
-            if force {
-                bucket_treshold = 0;
-            }
-            for (bucket_id, docs) in self
-                .per_bucket_docs
-                .iter()
-                .enumerate()
-                .filter(|(_, docs)| docs.len() > bucket_treshold)
-            {
-                self.sub_agg_collector
-                    .collect(bucket_id as BucketId, docs, agg_data)?;
-            }
-        } else {
-            let mut max_bucket = 0u32;
-            for partition in &self.partitions {
-                if let Some(&local_max) = partition.bucket_ids.iter().max() {
-                    max_bucket = max_bucket.max(local_max);
-                }
-            }
-
-            self.sub_agg_collector
-                .prepare_max_bucket(max_bucket, agg_data)?;
-
-            for slot in &self.partitions {
-                if !slot.bucket_ids.is_empty() {
-                    // Reduce dynamic dispatch overhead by collecting a full partition in one call.
-                    self.sub_agg_collector.collect_multiple(
-                        &slot.bucket_ids,
-                        &slot.docs,
-                        agg_data,
-                    )?;
-                }
-            }
-        }
-        self.clear();
-        Ok(())
-    }
-
-    /// Note: this _does_ flush the sub aggregations
-    pub fn flush(&mut self, agg_data: &mut AggregationsSegmentCtx) -> crate::Result<()> {
-        if self.num_docs != 0 {
-            self.flush_local(agg_data, true)?;
-        }
-        self.sub_agg_collector.flush(agg_data)?;
-        Ok(())
-    }
-}
-
-#[derive(Debug, Clone, Default)]
-struct PartitionEntry {
-    bucket_ids: Vec<BucketId>,
-    docs: Vec<DocId>,
-}
-
-impl PartitionEntry {
-    #[inline]
-    fn clear(&mut self) {
-        self.bucket_ids.clear();
-        self.docs.clear();
-    }
-}

src/aggregation/collector.rs

@@ -1,9 +1,9 @@
 use super::agg_req::Aggregations;
 use super::agg_result::AggregationResults;
-use super::cached_sub_aggs::CachedSubAggs;
+use super::buf_collector::BufAggregationCollector;
 use super::intermediate_agg_result::IntermediateAggregationResults;
+use super::segment_agg_result::SegmentAggregationCollector;
 use super::AggContextParams;
-// group buffering strategy is chosen explicitly by callers; no need to hash-group on the fly.
 use crate::aggregation::agg_data::{
     build_aggregations_data_from_req, build_segment_agg_collectors_root, AggregationsSegmentCtx,
 };
@@ -136,7 +136,7 @@ fn merge_fruits(
 /// `AggregationSegmentCollector` does the aggregation collection on a segment.
 pub struct AggregationSegmentCollector {
     aggs_with_accessor: AggregationsSegmentCtx,
-    agg_collector: CachedSubAggs<true>,
+    agg_collector: BufAggregationCollector,
     error: Option<TantivyError>,
 }
 
@@ -151,10 +151,8 @@ impl AggregationSegmentCollector {
     ) -> crate::Result<Self> {
         let mut agg_data =
             build_aggregations_data_from_req(agg, reader, segment_ordinal, context.clone())?;
-        let mut result = CachedSubAggs::new(build_segment_agg_collectors_root(&mut agg_data)?);
-        result
-            .get_sub_agg_collector()
-            .prepare_max_bucket(0, &agg_data)?; // prepare for bucket zero
+        let result =
+            BufAggregationCollector::new(build_segment_agg_collectors_root(&mut agg_data)?);
 
         Ok(AggregationSegmentCollector {
             aggs_with_accessor: agg_data,
@@ -172,31 +170,26 @@ impl SegmentCollector for AggregationSegmentCollector {
         if self.error.is_some() {
             return;
         }
-        self.agg_collector.push(0, doc);
-        match self
+        if let Err(err) = self
             .agg_collector
-            .check_flush_local(&mut self.aggs_with_accessor)
+            .collect(doc, &mut self.aggs_with_accessor)
         {
-            Ok(_) => {}
-            Err(e) => {
-                self.error = Some(e);
-            }
+            self.error = Some(err);
         }
     }
+
+    /// The query pushes the documents to the collector via this method.
+    ///
+    /// Only valid for Collectors that ignore docs
     fn collect_block(&mut self, docs: &[DocId]) {
         if self.error.is_some() {
             return;
         }
-
-        match self.agg_collector.get_sub_agg_collector().collect(
-            0,
-            docs,
-            &mut self.aggs_with_accessor,
-        ) {
-            Ok(_) => {}
-            Err(e) => {
-                self.error = Some(e);
-            }
+        if let Err(err) = self
+            .agg_collector
+            .collect_block(docs, &mut self.aggs_with_accessor)
+        {
+            self.error = Some(err);
         }
     }
 
@@ -207,13 +200,10 @@ impl SegmentCollector for AggregationSegmentCollector {
         self.agg_collector.flush(&mut self.aggs_with_accessor)?;
 
         let mut sub_aggregation_res = IntermediateAggregationResults::default();
-        self.agg_collector
-            .get_sub_agg_collector()
-            .add_intermediate_aggregation_result(
-                &self.aggs_with_accessor,
-                &mut sub_aggregation_res,
-                0,
-            )?;
+        Box::new(self.agg_collector).add_intermediate_aggregation_result(
+            &self.aggs_with_accessor,
+            &mut sub_aggregation_res,
+        )?;
 
         Ok(sub_aggregation_res)
     }

src/aggregation/intermediate_agg_result.rs

@@ -792,7 +792,7 @@ pub struct IntermediateRangeBucketEntry {
     /// The number of documents in the bucket.
     pub doc_count: u64,
     /// The sub_aggregation in this bucket.
-    pub sub_aggregation_res: IntermediateAggregationResults,
+    pub sub_aggregation: IntermediateAggregationResults,
     /// The from range of the bucket. Equals `f64::MIN` when `None`.
     pub from: Option<f64>,
     /// The to range of the bucket. Equals `f64::MAX` when `None`.
@@ -811,7 +811,7 @@ impl IntermediateRangeBucketEntry {
             key: self.key.into(),
             doc_count: self.doc_count,
             sub_aggregation: self
-                .sub_aggregation_res
+                .sub_aggregation
                 .into_final_result_internal(req, limits)?,
             to: self.to,
             from: self.from,
@@ -857,8 +857,7 @@ impl MergeFruits for IntermediateTermBucketEntry {
 impl MergeFruits for IntermediateRangeBucketEntry {
     fn merge_fruits(&mut self, other: IntermediateRangeBucketEntry) -> crate::Result<()> {
         self.doc_count += other.doc_count;
-        self.sub_aggregation_res
-            .merge_fruits(other.sub_aggregation_res)?;
+        self.sub_aggregation.merge_fruits(other.sub_aggregation)?;
         Ok(())
     }
 }
@@ -888,7 +887,7 @@ mod tests {
                 IntermediateRangeBucketEntry {
                     key: IntermediateKey::Str(key.to_string()),
                     doc_count: *doc_count,
-                    sub_aggregation_res: Default::default(),
+                    sub_aggregation: Default::default(),
                     from: None,
                     to: None,
                 },
@@ -921,7 +920,7 @@ mod tests {
                     doc_count: *doc_count,
                     from: None,
                     to: None,
-                    sub_aggregation_res: get_sub_test_tree(&[(
+                    sub_aggregation: get_sub_test_tree(&[(
                         sub_aggregation_key.to_string(),
                         *sub_aggregation_count,
                     )]),

src/aggregation/metric/average.rs

@@ -52,8 +52,10 @@ pub struct IntermediateAverage {
 
 impl IntermediateAverage {
     /// Creates a new [`IntermediateAverage`] instance from a [`SegmentStatsCollector`].
-    pub(crate) fn from_stats(stats: IntermediateStats) -> Self {
-        Self { stats }
+    pub(crate) fn from_collector(collector: SegmentStatsCollector) -> Self {
+        Self {
+            stats: collector.stats,
+        }
     }
     /// Merges the other intermediate result into self.
     pub fn merge_fruits(&mut self, other: IntermediateAverage) {

src/aggregation/metric/cardinality.rs

@@ -2,7 +2,7 @@ use std::collections::hash_map::DefaultHasher;
 use std::hash::{BuildHasher, Hasher};
 
 use columnar::column_values::CompactSpaceU64Accessor;
-use columnar::{Column, ColumnType, Dictionary, StrColumn};
+use columnar::{Column, ColumnBlockAccessor, ColumnType, Dictionary, StrColumn};
 use common::f64_to_u64;
 use hyperloglogplus::{HyperLogLog, HyperLogLogPlus};
 use rustc_hash::FxHashSet;
@@ -106,6 +106,8 @@ pub struct CardinalityAggReqData {
     pub str_dict_column: Option<StrColumn>,
     /// The missing value normalized to the internal u64 representation of the field type.
     pub missing_value_for_accessor: Option<u64>,
+    /// The column block accessor to access the fast field values.
+    pub(crate) column_block_accessor: ColumnBlockAccessor<u64>,
     /// The name of the aggregation.
     pub name: String,
     /// The aggregation request.
@@ -133,34 +135,45 @@ impl CardinalityAggregationReq {
     }
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, PartialEq)]
 pub(crate) struct SegmentCardinalityCollector {
-    buckets: Vec<SegmentCardinalityCollectorBucket>,
-    accessor_idx: usize,
-    /// The column accessor to access the fast field values.
-    accessor: Column<u64>,
-    /// The column_type of the field.
-    column_type: ColumnType,
-    /// The missing value normalized to the internal u64 representation of the field type.
-    missing_value_for_accessor: Option<u64>,
-}
-
-#[derive(Clone, Debug, PartialEq, Default)]
-pub(crate) struct SegmentCardinalityCollectorBucket {
     cardinality: CardinalityCollector,
     entries: FxHashSet<u64>,
+    accessor_idx: usize,
 }
-impl SegmentCardinalityCollectorBucket {
-    pub fn new(column_type: ColumnType) -> Self {
+
+impl SegmentCardinalityCollector {
+    pub fn from_req(column_type: ColumnType, accessor_idx: usize) -> Self {
         Self {
             cardinality: CardinalityCollector::new(column_type as u8),
-            entries: FxHashSet::default(),
+            entries: Default::default(),
+            accessor_idx,
         }
     }
+
+    fn fetch_block_with_field(
+        &mut self,
+        docs: &[crate::DocId],
+        agg_data: &mut CardinalityAggReqData,
+    ) {
+        if let Some(missing) = agg_data.missing_value_for_accessor {
+            agg_data.column_block_accessor.fetch_block_with_missing(
+                docs,
+                &agg_data.accessor,
+                missing,
+            );
+        } else {
+            agg_data
+                .column_block_accessor
+                .fetch_block(docs, &agg_data.accessor);
+        }
+    }
+
     fn into_intermediate_metric_result(
         mut self,
-        req_data: &CardinalityAggReqData,
+        agg_data: &AggregationsSegmentCtx,
     ) -> crate::Result<IntermediateMetricResult> {
+        let req_data = &agg_data.get_cardinality_req_data(self.accessor_idx);
         if req_data.column_type == ColumnType::Str {
             let fallback_dict = Dictionary::empty();
             let dict = req_data
@@ -181,7 +194,6 @@ impl SegmentCardinalityCollectorBucket {
                     term_ids.push(term_ord as u32);
                 }
             }
-
             term_ids.sort_unstable();
             dict.sorted_ords_to_term_cb(term_ids.iter().map(|term| *term as u64), |term| {
                 self.cardinality.sketch.insert_any(&term);
@@ -215,49 +227,16 @@ impl SegmentCardinalityCollectorBucket {
     }
 }
 
-impl SegmentCardinalityCollector {
-    pub fn from_req(
-        column_type: ColumnType,
-        accessor_idx: usize,
-        accessor: Column<u64>,
-        missing_value_for_accessor: Option<u64>,
-    ) -> Self {
-        Self {
-            buckets: vec![SegmentCardinalityCollectorBucket::new(column_type); 1],
-            column_type,
-            accessor_idx,
-            accessor,
-            missing_value_for_accessor,
-        }
-    }
-
-    fn fetch_block_with_field(
-        &mut self,
-        docs: &[crate::DocId],
-        agg_data: &mut AggregationsSegmentCtx,
-    ) {
-        agg_data.column_block_accessor.fetch_block_with_missing(
-            docs,
-            &self.accessor,
-            self.missing_value_for_accessor,
-        );
-    }
-}
-
 impl SegmentAggregationCollector for SegmentCardinalityCollector {
     fn add_intermediate_aggregation_result(
-        &mut self,
+        self: Box<Self>,
         agg_data: &AggregationsSegmentCtx,
         results: &mut IntermediateAggregationResults,
-        parent_bucket_id: BucketId,
     ) -> crate::Result<()> {
-        self.prepare_max_bucket(parent_bucket_id, agg_data)?;
         let req_data = &agg_data.get_cardinality_req_data(self.accessor_idx);
         let name = req_data.name.to_string();
-        // take the bucket in buckets and replace it with a new empty one
-        let bucket = std::mem::take(&mut self.buckets[parent_bucket_id as usize]);
 
-        let intermediate_result = bucket.into_intermediate_metric_result(req_data)?;
+        let intermediate_result = self.into_intermediate_metric_result(agg_data)?;
         results.push(
             name,
             IntermediateAggregationResult::Metric(intermediate_result),
@@ -268,20 +247,27 @@ impl SegmentAggregationCollector for SegmentCardinalityCollector {
 
     fn collect(
         &mut self,
-        parent_bucket_id: BucketId,
+        doc: crate::DocId,
+        agg_data: &mut AggregationsSegmentCtx,
+    ) -> crate::Result<()> {
+        self.collect_block(&[doc], agg_data)
+    }
+
+    fn collect_block(
+        &mut self,
         docs: &[crate::DocId],
         agg_data: &mut AggregationsSegmentCtx,
     ) -> crate::Result<()> {
-        self.fetch_block_with_field(docs, agg_data);
-        let bucket = &mut self.buckets[parent_bucket_id as usize];
+        let req_data = agg_data.get_cardinality_req_data_mut(self.accessor_idx);
+        self.fetch_block_with_field(docs, req_data);
 
-        let col_block_accessor = &agg_data.column_block_accessor;
-        if self.column_type == ColumnType::Str {
+        let col_block_accessor = &req_data.column_block_accessor;
+        if req_data.column_type == ColumnType::Str {
             for term_ord in col_block_accessor.iter_vals() {
-                bucket.entries.insert(term_ord);
+                self.entries.insert(term_ord);
             }
-        } else if self.column_type == ColumnType::IpAddr {
-            let compact_space_accessor = self
+        } else if req_data.column_type == ColumnType::IpAddr {
+            let compact_space_accessor = req_data
                 .accessor
                 .values
                 .clone()
@@ -296,29 +282,16 @@ impl SegmentAggregationCollector for SegmentCardinalityCollector {
                 })?;
             for val in col_block_accessor.iter_vals() {
                 let val: u128 = compact_space_accessor.compact_to_u128(val as u32);
-                bucket.cardinality.sketch.insert_any(&val);
+                self.cardinality.sketch.insert_any(&val);
             }
         } else {
             for val in col_block_accessor.iter_vals() {
-                bucket.cardinality.sketch.insert_any(&val);
+                self.cardinality.sketch.insert_any(&val);
             }
         }
 
         Ok(())
     }
-
-    fn prepare_max_bucket(
-        &mut self,
-        max_bucket: BucketId,
-        _agg_data: &AggregationsSegmentCtx,
-    ) -> crate::Result<()> {
-        if max_bucket as usize >= self.buckets.len() {
-            self.buckets.resize_with(max_bucket as usize + 1, || {
-                SegmentCardinalityCollectorBucket::new(self.column_type)
-            });
-        }
-        Ok(())
-    }
 }
 
 #[derive(Clone, Debug, Serialize, Deserialize)]

src/aggregation/metric/count.rs

@@ -52,8 +52,10 @@ pub struct IntermediateCount {
 
 impl IntermediateCount {
     /// Creates a new [`IntermediateCount`] instance from a [`SegmentStatsCollector`].
-    pub(crate) fn from_stats(stats: IntermediateStats) -> Self {
-        Self { stats }
+    pub(crate) fn from_collector(collector: SegmentStatsCollector) -> Self {
+        Self {
+            stats: collector.stats,
+        }
     }
     /// Merges the other intermediate result into self.
     pub fn merge_fruits(&mut self, other: IntermediateCount) {

src/aggregation/metric/extended_stats.rs

@@ -8,9 +8,10 @@ use crate::aggregation::agg_data::AggregationsSegmentCtx;
 use crate::aggregation::intermediate_agg_result::{
     IntermediateAggregationResult, IntermediateAggregationResults, IntermediateMetricResult,
 };
+use crate::aggregation::metric::MetricAggReqData;
 use crate::aggregation::segment_agg_result::SegmentAggregationCollector;
 use crate::aggregation::*;
-use crate::TantivyError;
+use crate::{DocId, TantivyError};
 
 /// A multi-value metric aggregation that computes a collection of extended statistics
 /// on numeric values that are extracted
@@ -317,28 +318,51 @@ impl IntermediateExtendedStats {
     }
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, PartialEq)]
 pub(crate) struct SegmentExtendedStatsCollector {
-    name: String,
     missing: Option<u64>,
     field_type: ColumnType,
-    accessor: columnar::Column<u64>,
-    buckets: Vec<IntermediateExtendedStats>,
-    sigma: Option<f64>,
+    pub(crate) extended_stats: IntermediateExtendedStats,
+    pub(crate) accessor_idx: usize,
+    val_cache: Vec<u64>,
 }
 
 impl SegmentExtendedStatsCollector {
-    pub fn from_req(req: &MetricAggReqData, sigma: Option<f64>) -> Self {
-        let missing = req
-            .missing
-            .and_then(|val| f64_to_fastfield_u64(val, &req.field_type));
+    pub fn from_req(
+        field_type: ColumnType,
+        sigma: Option<f64>,
+        accessor_idx: usize,
+        missing: Option<f64>,
+    ) -> Self {
+        let missing = missing.and_then(|val| f64_to_fastfield_u64(val, &field_type));
         Self {
-            name: req.name.clone(),
-            field_type: req.field_type,
-            accessor: req.accessor.clone(),
+            field_type,
+            extended_stats: IntermediateExtendedStats::with_sigma(sigma),
+            accessor_idx,
             missing,
-            buckets: vec![IntermediateExtendedStats::with_sigma(sigma); 16],
-            sigma,
+            val_cache: Default::default(),
+        }
+    }
+    #[inline]
+    pub(crate) fn collect_block_with_field(
+        &mut self,
+        docs: &[DocId],
+        req_data: &mut MetricAggReqData,
+    ) {
+        if let Some(missing) = self.missing.as_ref() {
+            req_data.column_block_accessor.fetch_block_with_missing(
+                docs,
+                &req_data.accessor,
+                *missing,
+            );
+        } else {
+            req_data
+                .column_block_accessor
+                .fetch_block(docs, &req_data.accessor);
+        }
+        for val in req_data.column_block_accessor.iter_vals() {
+            let val1 = f64_from_fastfield_u64(val, &self.field_type);
+            self.extended_stats.collect(val1);
         }
     }
 }
@@ -346,18 +370,15 @@ impl SegmentExtendedStatsCollector {
 impl SegmentAggregationCollector for SegmentExtendedStatsCollector {
     #[inline]
     fn add_intermediate_aggregation_result(
-        &mut self,
+        self: Box<Self>,
         agg_data: &AggregationsSegmentCtx,
         results: &mut IntermediateAggregationResults,
-        parent_bucket_id: BucketId,
     ) -> crate::Result<()> {
-        let name = self.name.clone();
-        self.prepare_max_bucket(parent_bucket_id, agg_data)?;
-        let extended_stats = std::mem::take(&mut self.buckets[parent_bucket_id as usize]);
+        let name = agg_data.get_metric_req_data(self.accessor_idx).name.clone();
         results.push(
             name,
             IntermediateAggregationResult::Metric(IntermediateMetricResult::ExtendedStats(
-                extended_stats,
+                self.extended_stats,
             )),
         )?;
 
@@ -367,36 +388,39 @@ impl SegmentAggregationCollector for SegmentExtendedStatsCollector {
     #[inline]
     fn collect(
         &mut self,
-        parent_bucket_id: BucketId,
-        docs: &[crate::DocId],
+        doc: crate::DocId,
         agg_data: &mut AggregationsSegmentCtx,
     ) -> crate::Result<()> {
-        let mut extended_stats = self.buckets[parent_bucket_id as usize].clone();
-
-        agg_data
-            .column_block_accessor
-            .fetch_block_with_missing(docs, &self.accessor, self.missing);
-        for val in agg_data.column_block_accessor.iter_vals() {
-            let val1 = f64_from_fastfield_u64(val, self.field_type);
-            extended_stats.collect(val1);
+        let req_data = agg_data.get_metric_req_data(self.accessor_idx);
+        if let Some(missing) = self.missing {
+            let mut has_val = false;
+            for val in req_data.accessor.values_for_doc(doc) {
+                let val1 = f64_from_fastfield_u64(val, &self.field_type);
+                self.extended_stats.collect(val1);
+                has_val = true;
+            }
+            if !has_val {
+                self.extended_stats
+                    .collect(f64_from_fastfield_u64(missing, &self.field_type));
+            }
+        } else {
+            for val in req_data.accessor.values_for_doc(doc) {
+                let val1 = f64_from_fastfield_u64(val, &self.field_type);
+                self.extended_stats.collect(val1);
+            }
         }
 
-        // store back
-        self.buckets[parent_bucket_id as usize] = extended_stats;
-
         Ok(())
     }
 
-    fn prepare_max_bucket(
+    #[inline]
+    fn collect_block(
         &mut self,
-        max_bucket: BucketId,
-        _agg_data: &AggregationsSegmentCtx,
+        docs: &[crate::DocId],
+        agg_data: &mut AggregationsSegmentCtx,
     ) -> crate::Result<()> {
-        if self.buckets.len() <= max_bucket as usize {
-            self.buckets.resize_with(max_bucket as usize + 1, || {
-                IntermediateExtendedStats::with_sigma(self.sigma)
-            });
-        }
+        let req_data = agg_data.get_metric_req_data_mut(self.accessor_idx);
+        self.collect_block_with_field(docs, req_data);
         Ok(())
     }
 }

src/aggregation/metric/max.rs

@@ -52,8 +52,10 @@ pub struct IntermediateMax {
 
 impl IntermediateMax {
     /// Creates a new [`IntermediateMax`] instance from a [`SegmentStatsCollector`].
-    pub(crate) fn from_stats(stats: IntermediateStats) -> Self {
-        Self { stats }
+    pub(crate) fn from_collector(collector: SegmentStatsCollector) -> Self {
+        Self {
+            stats: collector.stats,
+        }
     }
     /// Merges the other intermediate result into self.
     pub fn merge_fruits(&mut self, other: IntermediateMax) {

src/aggregation/metric/min.rs

@@ -52,8 +52,10 @@ pub struct IntermediateMin {
 
 impl IntermediateMin {
     /// Creates a new [`IntermediateMin`] instance from a [`SegmentStatsCollector`].
-    pub(crate) fn from_stats(stats: IntermediateStats) -> Self {
-        Self { stats }
+    pub(crate) fn from_collector(collector: SegmentStatsCollector) -> Self {
+        Self {
+            stats: collector.stats,
+        }
     }
     /// Merges the other intermediate result into self.
     pub fn merge_fruits(&mut self, other: IntermediateMin) {

src/aggregation/metric/mod.rs

@@ -31,7 +31,7 @@ use std::collections::HashMap;
 
 pub use average::*;
 pub use cardinality::*;
-use columnar::{Column, ColumnType};
+use columnar::{Column, ColumnBlockAccessor, ColumnType};
 pub use count::*;
 pub use extended_stats::*;
 pub use max::*;
@@ -55,6 +55,8 @@ pub struct MetricAggReqData {
     pub field_type: ColumnType,
     /// The missing value normalized to the internal u64 representation of the field type.
     pub missing_u64: Option<u64>,
+    /// The column block accessor to access the fast field values.
+    pub column_block_accessor: ColumnBlockAccessor<u64>,
     /// The column accessor to access the fast field values.
     pub accessor: Column<u64>,
     /// Used when converting to intermediate result

src/aggregation/metric/percentiles.rs

@@ -7,9 +7,10 @@ use crate::aggregation::agg_data::AggregationsSegmentCtx;
 use crate::aggregation::intermediate_agg_result::{
     IntermediateAggregationResult, IntermediateAggregationResults, IntermediateMetricResult,
 };
+use crate::aggregation::metric::MetricAggReqData;
 use crate::aggregation::segment_agg_result::SegmentAggregationCollector;
 use crate::aggregation::*;
-use crate::TantivyError;
+use crate::{DocId, TantivyError};
 
 /// # Percentiles
 ///
@@ -130,16 +131,10 @@ impl PercentilesAggregationReq {
     }
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, PartialEq)]
 pub(crate) struct SegmentPercentilesCollector {
-    pub(crate) buckets: Vec<PercentilesCollector>,
+    pub(crate) percentiles: PercentilesCollector,
     pub(crate) accessor_idx: usize,
-    /// The type of the field.
-    pub field_type: ColumnType,
-    /// The missing value normalized to the internal u64 representation of the field type.
-    pub missing_u64: Option<u64>,
-    /// The column accessor to access the fast field values.
-    pub accessor: Column<u64>,
 }
 
 #[derive(Clone, Serialize, Deserialize)]
@@ -234,18 +229,33 @@ impl PercentilesCollector {
 }
 
 impl SegmentPercentilesCollector {
-    pub fn from_req_and_validate(
-        field_type: ColumnType,
-        missing_u64: Option<u64>,
-        accessor: Column<u64>,
-        accessor_idx: usize,
-    ) -> Self {
-        Self {
-            buckets: Vec::with_capacity(64),
-            field_type,
-            missing_u64,
-            accessor,
+    pub fn from_req_and_validate(accessor_idx: usize) -> crate::Result<Self> {
+        Ok(Self {
+            percentiles: PercentilesCollector::new(),
             accessor_idx,
+        })
+    }
+    #[inline]
+    pub(crate) fn collect_block_with_field(
+        &mut self,
+        docs: &[DocId],
+        req_data: &mut MetricAggReqData,
+    ) {
+        if let Some(missing) = req_data.missing_u64.as_ref() {
+            req_data.column_block_accessor.fetch_block_with_missing(
+                docs,
+                &req_data.accessor,
+                *missing,
+            );
+        } else {
+            req_data
+                .column_block_accessor
+                .fetch_block(docs, &req_data.accessor);
+        }
+
+        for val in req_data.column_block_accessor.iter_vals() {
+            let val1 = f64_from_fastfield_u64(val, &req_data.field_type);
+            self.percentiles.collect(val1);
         }
     }
 }
@@ -253,18 +263,12 @@ impl SegmentPercentilesCollector {
 impl SegmentAggregationCollector for SegmentPercentilesCollector {
     #[inline]
     fn add_intermediate_aggregation_result(
-        &mut self,
+        self: Box<Self>,
         agg_data: &AggregationsSegmentCtx,
         results: &mut IntermediateAggregationResults,
-        parent_bucket_id: BucketId,
     ) -> crate::Result<()> {
         let name = agg_data.get_metric_req_data(self.accessor_idx).name.clone();
-        self.prepare_max_bucket(parent_bucket_id, agg_data)?;
-        // Swap collector with an empty one to avoid cloning
-        let percentiles_collector = std::mem::take(&mut self.buckets[parent_bucket_id as usize]);
-
-        let intermediate_metric_result =
-            IntermediateMetricResult::Percentiles(percentiles_collector);
+        let intermediate_metric_result = IntermediateMetricResult::Percentiles(self.percentiles);
 
         results.push(
             name,
@@ -277,33 +281,40 @@ impl SegmentAggregationCollector for SegmentPercentilesCollector {
     #[inline]
     fn collect(
         &mut self,
-        parent_bucket_id: BucketId,
-        docs: &[crate::DocId],
+        doc: crate::DocId,
         agg_data: &mut AggregationsSegmentCtx,
     ) -> crate::Result<()> {
-        let percentiles = &mut self.buckets[parent_bucket_id as usize];
-        agg_data.column_block_accessor.fetch_block_with_missing(
-            docs,
-            &self.accessor,
-            self.missing_u64,
-        );
+        let req_data = agg_data.get_metric_req_data(self.accessor_idx);
 
-        for val in agg_data.column_block_accessor.iter_vals() {
-            let val1 = f64_from_fastfield_u64(val, self.field_type);
-            percentiles.collect(val1);
+        if let Some(missing) = req_data.missing_u64 {
+            let mut has_val = false;
+            for val in req_data.accessor.values_for_doc(doc) {
+                let val1 = f64_from_fastfield_u64(val, &req_data.field_type);
+                self.percentiles.collect(val1);
+                has_val = true;
+            }
+            if !has_val {
+                self.percentiles
+                    .collect(f64_from_fastfield_u64(missing, &req_data.field_type));
+            }
+        } else {
+            for val in req_data.accessor.values_for_doc(doc) {
+                let val1 = f64_from_fastfield_u64(val, &req_data.field_type);
+                self.percentiles.collect(val1);
+            }
         }
 
         Ok(())
     }
 
-    fn prepare_max_bucket(
+    #[inline]
+    fn collect_block(
         &mut self,
-        max_bucket: BucketId,
-        _agg_data: &AggregationsSegmentCtx,
+        docs: &[crate::DocId],
+        agg_data: &mut AggregationsSegmentCtx,
     ) -> crate::Result<()> {
-        while self.buckets.len() <= max_bucket as usize {
-            self.buckets.push(PercentilesCollector::new());
-        }
+        let req_data = agg_data.get_metric_req_data_mut(self.accessor_idx);
+        self.collect_block_with_field(docs, req_data);
         Ok(())
     }
 }

src/aggregation/metric/stats.rs

@@ -1,6 +1,5 @@
 use std::fmt::Debug;
 
-use columnar::{Column, ColumnType};
 use serde::{Deserialize, Serialize};
 
 use super::*;
@@ -8,9 +7,10 @@ use crate::aggregation::agg_data::AggregationsSegmentCtx;
 use crate::aggregation::intermediate_agg_result::{
     IntermediateAggregationResult, IntermediateAggregationResults, IntermediateMetricResult,
 };
+use crate::aggregation::metric::MetricAggReqData;
 use crate::aggregation::segment_agg_result::SegmentAggregationCollector;
 use crate::aggregation::*;
-use crate::TantivyError;
+use crate::{DocId, TantivyError};
 
 /// A multi-value metric aggregation that computes a collection of statistics on numeric values that
 /// are extracted from the aggregated documents.
@@ -83,7 +83,7 @@ impl Stats {
 
 /// Intermediate result of the stats aggregation that can be combined with other intermediate
 /// results.
-#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize)]
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
 pub struct IntermediateStats {
     /// The number of extracted values.
     pub(crate) count: u64,
@@ -187,75 +187,75 @@ pub enum StatsType {
     Percentiles,
 }
 
-fn create_collector<const TYPE_ID: u8>(
-    req: &MetricAggReqData,
-) -> Box<dyn SegmentAggregationCollector> {
-    Box::new(SegmentStatsCollector::<TYPE_ID> {
-        name: req.name.clone(),
-        collecting_for: req.collecting_for,
-        is_number_or_date_type: req.is_number_or_date_type,
-        missing_u64: req.missing_u64,
-        accessor: req.accessor.clone(),
-        buckets: vec![IntermediateStats::default()],
-    })
+#[derive(Clone, Debug)]
+pub(crate) struct SegmentStatsCollector {
+    pub(crate) stats: IntermediateStats,
+    pub(crate) accessor_idx: usize,
 }
 
-/// Build a concrete `SegmentStatsCollector` depending on the column type.
-pub(crate) fn build_segment_stats_collector(
-    req: &MetricAggReqData,
-) -> crate::Result<Box<dyn SegmentAggregationCollector>> {
-    match req.field_type {
-        ColumnType::I64 => Ok(create_collector::<{ ColumnType::I64 as u8 }>(req)),
-        ColumnType::U64 => Ok(create_collector::<{ ColumnType::U64 as u8 }>(req)),
-        ColumnType::F64 => Ok(create_collector::<{ ColumnType::F64 as u8 }>(req)),
-        ColumnType::Bool => Ok(create_collector::<{ ColumnType::Bool as u8 }>(req)),
-        ColumnType::DateTime => Ok(create_collector::<{ ColumnType::DateTime as u8 }>(req)),
-        ColumnType::Bytes => Ok(create_collector::<{ ColumnType::Bytes as u8 }>(req)),
-        ColumnType::Str => Ok(create_collector::<{ ColumnType::Str as u8 }>(req)),
-        ColumnType::IpAddr => Ok(create_collector::<{ ColumnType::IpAddr as u8 }>(req)),
+impl SegmentStatsCollector {
+    pub fn from_req(accessor_idx: usize) -> Self {
+        Self {
+            stats: IntermediateStats::default(),
+            accessor_idx,
+        }
+    }
+    #[inline]
+    pub(crate) fn collect_block_with_field(
+        &mut self,
+        docs: &[DocId],
+        req_data: &mut MetricAggReqData,
+    ) {
+        if let Some(missing) = req_data.missing_u64.as_ref() {
+            req_data.column_block_accessor.fetch_block_with_missing(
+                docs,
+                &req_data.accessor,
+                *missing,
+            );
+        } else {
+            req_data
+                .column_block_accessor
+                .fetch_block(docs, &req_data.accessor);
+        }
+        if req_data.is_number_or_date_type {
+            for val in req_data.column_block_accessor.iter_vals() {
+                let val1 = f64_from_fastfield_u64(val, &req_data.field_type);
+                self.stats.collect(val1);
+            }
+        } else {
+            for _val in req_data.column_block_accessor.iter_vals() {
+                // we ignore the value and simply record that we got something
+                self.stats.collect(0.0);
+            }
+        }
     }
 }
 
-#[repr(C)]
-#[derive(Clone, Debug)]
-pub(crate) struct SegmentStatsCollector<const COLUMN_TYPE_ID: u8> {
-    pub(crate) missing_u64: Option<u64>,
-    pub(crate) accessor: Column<u64>,
-    pub(crate) is_number_or_date_type: bool,
-    pub(crate) buckets: Vec<IntermediateStats>,
-    pub(crate) name: String,
-    pub(crate) collecting_for: StatsType,
-}
-
-impl<const COLUMN_TYPE_ID: u8> SegmentAggregationCollector
-    for SegmentStatsCollector<COLUMN_TYPE_ID>
-{
+impl SegmentAggregationCollector for SegmentStatsCollector {
     #[inline]
     fn add_intermediate_aggregation_result(
-        &mut self,
+        self: Box<Self>,
         agg_data: &AggregationsSegmentCtx,
         results: &mut IntermediateAggregationResults,
-        parent_bucket_id: BucketId,
     ) -> crate::Result<()> {
-        let name = self.name.clone();
+        let req = agg_data.get_metric_req_data(self.accessor_idx);
+        let name = req.name.clone();
 
-        self.prepare_max_bucket(parent_bucket_id, agg_data)?;
-        let stats = self.buckets[parent_bucket_id as usize];
-        let intermediate_metric_result = match self.collecting_for {
+        let intermediate_metric_result = match req.collecting_for {
             StatsType::Average => {
-                IntermediateMetricResult::Average(IntermediateAverage::from_stats(stats))
+                IntermediateMetricResult::Average(IntermediateAverage::from_collector(*self))
             }
             StatsType::Count => {
-                IntermediateMetricResult::Count(IntermediateCount::from_stats(stats))
+                IntermediateMetricResult::Count(IntermediateCount::from_collector(*self))
             }
-            StatsType::Max => IntermediateMetricResult::Max(IntermediateMax::from_stats(stats)),
-            StatsType::Min => IntermediateMetricResult::Min(IntermediateMin::from_stats(stats)),
-            StatsType::Stats => IntermediateMetricResult::Stats(stats),
-            StatsType::Sum => IntermediateMetricResult::Sum(IntermediateSum::from_stats(stats)),
+            StatsType::Max => IntermediateMetricResult::Max(IntermediateMax::from_collector(*self)),
+            StatsType::Min => IntermediateMetricResult::Min(IntermediateMin::from_collector(*self)),
+            StatsType::Stats => IntermediateMetricResult::Stats(self.stats),
+            StatsType::Sum => IntermediateMetricResult::Sum(IntermediateSum::from_collector(*self)),
             _ => {
                 return Err(TantivyError::InvalidArgument(format!(
                     "Unsupported stats type for stats aggregation: {:?}",
-                    self.collecting_for
+                    req.collecting_for
                 )))
             }
         };
@@ -271,67 +271,41 @@ impl<const COLUMN_TYPE_ID: u8> SegmentAggregationCollector
     #[inline]
     fn collect(
         &mut self,
-        parent_bucket_id: BucketId,
+        doc: crate::DocId,
+        agg_data: &mut AggregationsSegmentCtx,
+    ) -> crate::Result<()> {
+        let req_data = agg_data.get_metric_req_data(self.accessor_idx);
+        if let Some(missing) = req_data.missing_u64 {
+            let mut has_val = false;
+            for val in req_data.accessor.values_for_doc(doc) {
+                let val1 = f64_from_fastfield_u64(val, &req_data.field_type);
+                self.stats.collect(val1);
+                has_val = true;
+            }
+            if !has_val {
+                self.stats
+                    .collect(f64_from_fastfield_u64(missing, &req_data.field_type));
+            }
+        } else {
+            for val in req_data.accessor.values_for_doc(doc) {
+                let val1 = f64_from_fastfield_u64(val, &req_data.field_type);
+                self.stats.collect(val1);
+            }
+        }
+
+        Ok(())
+    }
+
+    #[inline]
+    fn collect_block(
+        &mut self,
         docs: &[crate::DocId],
         agg_data: &mut AggregationsSegmentCtx,
     ) -> crate::Result<()> {
-        // TODO: remove once we fetch all values for all bucket ids in one go
-        if docs.len() == 1 && self.missing_u64.is_none() {
-            collect_stats::<COLUMN_TYPE_ID>(
-                &mut self.buckets[parent_bucket_id as usize],
-                self.accessor.values_for_doc(docs[0]),
-                self.is_number_or_date_type,
-            )?;
-
-            return Ok(());
-        }
-        agg_data.column_block_accessor.fetch_block_with_missing(
-            docs,
-            &self.accessor,
-            self.missing_u64,
-        );
-        collect_stats::<COLUMN_TYPE_ID>(
-            &mut self.buckets[parent_bucket_id as usize],
-            agg_data.column_block_accessor.iter_vals(),
-            self.is_number_or_date_type,
-        )?;
-
+        let req_data = agg_data.get_metric_req_data_mut(self.accessor_idx);
+        self.collect_block_with_field(docs, req_data);
         Ok(())
     }
-
-    fn prepare_max_bucket(
-        &mut self,
-        max_bucket: BucketId,
-        _agg_data: &AggregationsSegmentCtx,
-    ) -> crate::Result<()> {
-        let required_buckets = (max_bucket as usize) + 1;
-        if self.buckets.len() < required_buckets {
-            self.buckets
-                .resize_with(required_buckets, IntermediateStats::default);
-        }
-        Ok(())
-    }
-}
-
-#[inline]
-fn collect_stats<const COLUMN_TYPE_ID: u8>(
-    stats: &mut IntermediateStats,
-    vals: impl Iterator<Item = u64>,
-    is_number_or_date_type: bool,
-) -> crate::Result<()> {
-    if is_number_or_date_type {
-        for val in vals {
-            let val1 = convert_to_f64::<COLUMN_TYPE_ID>(val);
-            stats.collect(val1);
-        }
-    } else {
-        for _val in vals {
-            // we ignore the value and simply record that we got something
-            stats.collect(0.0);
-        }
-    }
-
-    Ok(())
 }
 
 #[cfg(test)]

src/aggregation/metric/sum.rs

@@ -52,8 +52,10 @@ pub struct IntermediateSum {
 
 impl IntermediateSum {
     /// Creates a new [`IntermediateSum`] instance from a [`SegmentStatsCollector`].
-    pub(crate) fn from_stats(stats: IntermediateStats) -> Self {
-        Self { stats }
+    pub(crate) fn from_collector(collector: SegmentStatsCollector) -> Self {
+        Self {
+            stats: collector.stats,
+        }
     }
     /// Merges the other intermediate result into self.
     pub fn merge_fruits(&mut self, other: IntermediateSum) {

src/aggregation/metric/top_hits.rs

@@ -15,11 +15,12 @@ use crate::aggregation::intermediate_agg_result::{
     IntermediateAggregationResult, IntermediateMetricResult,
 };
 use crate::aggregation::segment_agg_result::SegmentAggregationCollector;
-use crate::aggregation::{AggregationError, BucketId};
+use crate::aggregation::AggregationError;
 use crate::collector::sort_key::ReverseComparator;
 use crate::collector::TopNComputer;
 use crate::schema::OwnedValue;
 use crate::{DocAddress, DocId, SegmentOrdinal};
+// duplicate import removed; already imported above
 
 /// Contains all information required by the TopHitsSegmentCollector to perform the
 /// top_hits aggregation on a segment.
@@ -471,10 +472,7 @@ impl TopHitsTopNComputer {
     /// Create a new TopHitsCollector
     pub fn new(req: &TopHitsAggregationReq) -> Self {
         Self {
-            top_n: TopNComputer::new_with_comparator(
-                req.size + req.from.unwrap_or(0),
-                ReverseComparator,
-            ),
+            top_n: TopNComputer::new(req.size + req.from.unwrap_or(0)),
             req: req.clone(),
         }
     }
@@ -520,8 +518,7 @@ impl TopHitsTopNComputer {
 pub(crate) struct TopHitsSegmentCollector {
     segment_ordinal: SegmentOrdinal,
     accessor_idx: usize,
-    buckets: Vec<TopNComputer<Vec<DocValueAndOrder>, DocAddress, ReverseComparator>>,
-    num_hits: usize,
+    top_n: TopNComputer<Vec<DocValueAndOrder>, DocAddress, ReverseComparator>,
 }
 
 impl TopHitsSegmentCollector {
@@ -530,29 +527,19 @@ impl TopHitsSegmentCollector {
         accessor_idx: usize,
         segment_ordinal: SegmentOrdinal,
     ) -> Self {
-        let num_hits = req.size + req.from.unwrap_or(0);
         Self {
-            num_hits,
+            top_n: TopNComputer::new(req.size + req.from.unwrap_or(0)),
             segment_ordinal,
             accessor_idx,
-            buckets: vec![TopNComputer::new_with_comparator(num_hits, ReverseComparator); 1],
         }
     }
-    fn get_top_hits_computer(
-        &mut self,
-        parent_bucket_id: BucketId,
+    fn into_top_hits_collector(
+        self,
         value_accessors: &HashMap<String, Vec<DynamicColumn>>,
         req: &TopHitsAggregationReq,
     ) -> TopHitsTopNComputer {
-        if parent_bucket_id as usize >= self.buckets.len() {
-            return TopHitsTopNComputer::new(req);
-        }
-        let top_n = std::mem::replace(
-            &mut self.buckets[parent_bucket_id as usize],
-            TopNComputer::new(0),
-        );
         let mut top_hits_computer = TopHitsTopNComputer::new(req);
-        let top_results = top_n.into_vec();
+        let top_results = self.top_n.into_vec();
 
         for res in top_results {
             let doc_value_fields = req.get_document_field_data(value_accessors, res.doc.doc_id);
@@ -567,24 +554,54 @@ impl TopHitsSegmentCollector {
 
         top_hits_computer
     }
+
+    /// TODO add a specialized variant for a single sort field
+    fn collect_with(
+        &mut self,
+        doc_id: crate::DocId,
+        req: &TopHitsAggregationReq,
+        accessors: &[(Column<u64>, ColumnType)],
+    ) -> crate::Result<()> {
+        let sorts: Vec<DocValueAndOrder> = req
+            .sort
+            .iter()
+            .enumerate()
+            .map(|(idx, KeyOrder { order, .. })| {
+                let order = *order;
+                let value = accessors
+                    .get(idx)
+                    .expect("could not find field in accessors")
+                    .0
+                    .values_for_doc(doc_id)
+                    .next();
+                DocValueAndOrder { value, order }
+            })
+            .collect();
+
+        self.top_n.push(
+            sorts,
+            DocAddress {
+                segment_ord: self.segment_ordinal,
+                doc_id,
+            },
+        );
+        Ok(())
+    }
 }
 
 impl SegmentAggregationCollector for TopHitsSegmentCollector {
     fn add_intermediate_aggregation_result(
-        &mut self,
+        self: Box<Self>,
         agg_data: &AggregationsSegmentCtx,
         results: &mut crate::aggregation::intermediate_agg_result::IntermediateAggregationResults,
-        parent_bucket_id: BucketId,
     ) -> crate::Result<()> {
         let req_data = agg_data.get_top_hits_req_data(self.accessor_idx);
 
         let value_accessors = &req_data.value_accessors;
 
-        let intermediate_result = IntermediateMetricResult::TopHits(self.get_top_hits_computer(
-            parent_bucket_id,
-            value_accessors,
-            &req_data.req,
-        ));
+        let intermediate_result = IntermediateMetricResult::TopHits(
+            self.into_top_hits_collector(value_accessors, &req_data.req),
+        );
         results.push(
             req_data.name.to_string(),
             IntermediateAggregationResult::Metric(intermediate_result),
@@ -594,55 +611,24 @@ impl SegmentAggregationCollector for TopHitsSegmentCollector {
     /// TODO: Consider a caching layer to reduce the call overhead
     fn collect(
         &mut self,
-        parent_bucket_id: BucketId,
-        docs: &[crate::DocId],
+        doc_id: crate::DocId,
         agg_data: &mut AggregationsSegmentCtx,
     ) -> crate::Result<()> {
-        let top_n = &mut self.buckets[parent_bucket_id as usize];
         let req_data = agg_data.get_top_hits_req_data(self.accessor_idx);
-        let req = &req_data.req;
-        let accessors = &req_data.accessors;
-        for doc_id in docs {
-            let doc_id = *doc_id;
-            // TODO: this is terrible, a new vec is allocated for every doc
-            // We can fetch blocks instead
-            // We don't need to store the order for every value
-            let sorts: Vec<DocValueAndOrder> = req
-                .sort
-                .iter()
-                .enumerate()
-                .map(|(idx, KeyOrder { order, .. })| {
-                    let order = *order;
-                    let value = accessors
-                        .get(idx)
-                        .expect("could not find field in accessors")
-                        .0
-                        .values_for_doc(doc_id)
-                        .next();
-                    DocValueAndOrder { value, order }
-                })
-                .collect();
-
-            top_n.push(
-                sorts,
-                DocAddress {
-                    segment_ord: self.segment_ordinal,
-                    doc_id,
-                },
-            );
-        }
+        self.collect_with(doc_id, &req_data.req, &req_data.accessors)?;
         Ok(())
     }
 
-    fn prepare_max_bucket(
+    fn collect_block(
         &mut self,
-        max_bucket: BucketId,
-        _agg_data: &AggregationsSegmentCtx,
+        docs: &[crate::DocId],
+        agg_data: &mut AggregationsSegmentCtx,
     ) -> crate::Result<()> {
-        self.buckets.resize(
-            (max_bucket as usize) + 1,
-            TopNComputer::new_with_comparator(self.num_hits, ReverseComparator),
-        );
+        let req_data = agg_data.get_top_hits_req_data(self.accessor_idx);
+        // TODO: Consider getting fields with the column block accessor.
+        for doc in docs {
+            self.collect_with(*doc, &req_data.req, &req_data.accessors)?;
+        }
         Ok(())
     }
 }
@@ -760,7 +746,7 @@ mod tests {
                     ],
                     "from": 0,
                 }
-            }
+        }
         }))
         .unwrap();
 
@@ -889,7 +875,7 @@ mod tests {
                         "mixed.*",
                     ],
                 }
-            }
+        }
         }))?;
 
         let collector = AggregationCollector::from_aggs(d, Default::default());

src/aggregation/mod.rs

@@ -133,7 +133,7 @@ mod agg_limits;
 pub mod agg_req;
 pub mod agg_result;
 pub mod bucket;
-pub(crate) mod cached_sub_aggs;
+mod buf_collector;
 mod collector;
 mod date;
 mod error;
@@ -162,19 +162,6 @@ use serde::{Deserialize, Deserializer, Serialize};
 
 use crate::tokenizer::TokenizerManager;
 
-/// A bucket id is a dense identifier for a bucket within an aggregation.
-/// It is used to index into a Vec that hold per-bucket data.
-///
-/// For example, in a terms aggregation, each unique term will be assigned a incremental BucketId.
-/// This BucketId will be forwarded to sub-aggregations to identify the parent bucket.
-///
-/// This allows to have a single AggregationCollector instance per aggregation,
-/// that can handle multiple buckets efficiently.
-///
-/// The API to call sub-aggregations is therefore a &[(BucketId, &[DocId])].
-/// For that we'll need a buffer. One Vec per bucket aggregation is needed.
-pub type BucketId = u32;
-
 /// Context parameters for aggregation execution
 ///
 /// This struct holds shared resources needed during aggregation execution:
@@ -348,37 +335,19 @@ impl Display for Key {
     }
 }
 
-pub(crate) fn convert_to_f64<const COLUMN_TYPE_ID: u8>(val: u64) -> f64 {
-    if COLUMN_TYPE_ID == ColumnType::U64 as u8 {
-        val as f64
-    } else if COLUMN_TYPE_ID == ColumnType::I64 as u8
-        || COLUMN_TYPE_ID == ColumnType::DateTime as u8
-    {
-        i64::from_u64(val) as f64
-    } else if COLUMN_TYPE_ID == ColumnType::F64 as u8 {
-        f64::from_u64(val)
-    } else if COLUMN_TYPE_ID == ColumnType::Bool as u8 {
-        val as f64
-    } else {
-        panic!(
-            "ColumnType ID {} cannot be converted to f64 metric",
-            COLUMN_TYPE_ID
-        )
-    }
-}
-
 /// Inverse of `to_fastfield_u64`. Used to convert to `f64` for metrics.
 ///
 /// # Panics
 /// Only `u64`, `f64`, `date`, and `i64` are supported.
-pub(crate) fn f64_from_fastfield_u64(val: u64, field_type: ColumnType) -> f64 {
+pub(crate) fn f64_from_fastfield_u64(val: u64, field_type: &ColumnType) -> f64 {
     match field_type {
-        ColumnType::U64 => convert_to_f64::<{ ColumnType::U64 as u8 }>(val),
-        ColumnType::I64 => convert_to_f64::<{ ColumnType::I64 as u8 }>(val),
-        ColumnType::F64 => convert_to_f64::<{ ColumnType::F64 as u8 }>(val),
-        ColumnType::Bool => convert_to_f64::<{ ColumnType::Bool as u8 }>(val),
-        ColumnType::DateTime => convert_to_f64::<{ ColumnType::DateTime as u8 }>(val),
-        _ => panic!("unexpected type {field_type:?}. This should not happen"),
+        ColumnType::U64 => val as f64,
+        ColumnType::I64 | ColumnType::DateTime => i64::from_u64(val) as f64,
+        ColumnType::F64 => f64::from_u64(val),
+        ColumnType::Bool => val as f64,
+        _ => {
+            panic!("unexpected type {field_type:?}. This should not happen")
+        }
     }
 }
 

src/aggregation/segment_agg_result.rs

@@ -8,67 +8,25 @@ use std::fmt::Debug;
 pub(crate) use super::agg_limits::AggregationLimitsGuard;
 use super::intermediate_agg_result::IntermediateAggregationResults;
 use crate::aggregation::agg_data::AggregationsSegmentCtx;
-use crate::aggregation::BucketId;
-
-/// Monotonically increasing provider of BucketIds.
-#[derive(Debug, Clone, Default)]
-pub struct BucketIdProvider(u32);
-impl BucketIdProvider {
-    /// Get the next BucketId.
-    pub fn next_bucket_id(&mut self) -> BucketId {
-        let bucket_id = self.0;
-        self.0 += 1;
-        bucket_id
-    }
-}
 
 /// A SegmentAggregationCollector is used to collect aggregation results.
-pub trait SegmentAggregationCollector: Debug {
+pub trait SegmentAggregationCollector: CollectorClone + Debug {
     fn add_intermediate_aggregation_result(
-        &mut self,
+        self: Box<Self>,
         agg_data: &AggregationsSegmentCtx,
         results: &mut IntermediateAggregationResults,
-        parent_bucket_id: BucketId,
     ) -> crate::Result<()>;
 
-    /// Note: The caller needs to call `prepare_max_bucket` before calling `collect`.
     fn collect(
         &mut self,
-        parent_bucket_id: BucketId,
-        docs: &[crate::DocId],
+        doc: crate::DocId,
         agg_data: &mut AggregationsSegmentCtx,
     ) -> crate::Result<()>;
 
-    /// Collect docs for multiple buckets in one call.
-    /// Minimizes dynamic dispatch overhead when collecting many buckets.
-    ///
-    /// Note: The caller needs to call `prepare_max_bucket` before calling `collect`.
-    fn collect_multiple(
+    fn collect_block(
         &mut self,
-        bucket_ids: &[BucketId],
         docs: &[crate::DocId],
         agg_data: &mut AggregationsSegmentCtx,
-    ) -> crate::Result<()> {
-        debug_assert_eq!(bucket_ids.len(), docs.len());
-        let mut start = 0;
-        while start < bucket_ids.len() {
-            let bucket_id = bucket_ids[start];
-            let mut end = start + 1;
-            while end < bucket_ids.len() && bucket_ids[end] == bucket_id {
-                end += 1;
-            }
-            self.collect(bucket_id, &docs[start..end], agg_data)?;
-            start = end;
-        }
-        Ok(())
-    }
-
-    /// Prepare the collector for collecting up to BucketId `max_bucket`.
-    /// This is useful so we can split allocation ahead of time of collecting.
-    fn prepare_max_bucket(
-        &mut self,
-        max_bucket: BucketId,
-        agg_data: &AggregationsSegmentCtx,
     ) -> crate::Result<()>;
 
     /// Finalize method. Some Aggregator collect blocks of docs before calling `collect_block`.
@@ -78,7 +36,26 @@ pub trait SegmentAggregationCollector: Debug {
     }
 }
 
-#[derive(Default)]
+/// A helper trait to enable cloning of Box<dyn SegmentAggregationCollector>
+pub trait CollectorClone {
+    fn clone_box(&self) -> Box<dyn SegmentAggregationCollector>;
+}
+
+impl<T> CollectorClone for T
+where T: 'static + SegmentAggregationCollector + Clone
+{
+    fn clone_box(&self) -> Box<dyn SegmentAggregationCollector> {
+        Box::new(self.clone())
+    }
+}
+
+impl Clone for Box<dyn SegmentAggregationCollector> {
+    fn clone(&self) -> Box<dyn SegmentAggregationCollector> {
+        self.clone_box()
+    }
+}
+
+#[derive(Clone, Default)]
 /// The GenericSegmentAggregationResultsCollector is the generic version of the collector, which
 /// can handle arbitrary complexity of  sub-aggregations. Ideally we never have to pick this one
 /// and can provide specialized versions instead, that remove some of its overhead.
@@ -96,13 +73,12 @@ impl Debug for GenericSegmentAggregationResultsCollector {
 
 impl SegmentAggregationCollector for GenericSegmentAggregationResultsCollector {
     fn add_intermediate_aggregation_result(
-        &mut self,
+        self: Box<Self>,
         agg_data: &AggregationsSegmentCtx,
         results: &mut IntermediateAggregationResults,
-        parent_bucket_id: BucketId,
     ) -> crate::Result<()> {
-        for agg in &mut self.aggs {
-            agg.add_intermediate_aggregation_result(agg_data, results, parent_bucket_id)?;
+        for agg in self.aggs {
+            agg.add_intermediate_aggregation_result(agg_data, results)?;
         }
 
         Ok(())
@@ -110,13 +86,23 @@ impl SegmentAggregationCollector for GenericSegmentAggregationResultsCollector {
 
     fn collect(
         &mut self,
-        parent_bucket_id: BucketId,
+        doc: crate::DocId,
+        agg_data: &mut AggregationsSegmentCtx,
+    ) -> crate::Result<()> {
+        self.collect_block(&[doc], agg_data)?;
+
+        Ok(())
+    }
+
+    fn collect_block(
+        &mut self,
         docs: &[crate::DocId],
         agg_data: &mut AggregationsSegmentCtx,
     ) -> crate::Result<()> {
         for collector in &mut self.aggs {
-            collector.collect(parent_bucket_id, docs, agg_data)?;
+            collector.collect_block(docs, agg_data)?;
         }
+
         Ok(())
     }
 
@@ -126,15 +112,4 @@ impl SegmentAggregationCollector for GenericSegmentAggregationResultsCollector {
         }
         Ok(())
     }
-
-    fn prepare_max_bucket(
-        &mut self,
-        max_bucket: BucketId,
-        agg_data: &AggregationsSegmentCtx,
-    ) -> crate::Result<()> {
-        for collector in &mut self.aggs {
-            collector.prepare_max_bucket(max_bucket, agg_data)?;
-        }
-        Ok(())
-    }
 }

src/core/executor.rs

@@ -48,15 +48,7 @@ impl Executor {
         F: Sized + Sync + Fn(A) -> crate::Result<R>,
     {
         match self {
-            Executor::SingleThread => {
-                // Avoid `collect`, since the stacktrace is blown up by it, which makes profiling
-                // harder.
-                let mut result = Vec::with_capacity(args.size_hint().0);
-                for arg in args {
-                    result.push(f(arg)?);
-                }
-                Ok(result)
-            }
+            Executor::SingleThread => args.map(f).collect::<crate::Result<_>>(),
             Executor::ThreadPool(pool) => {
                 let args: Vec<A> = args.collect();
                 let num_fruits = args.len();

src/core/json_utils.rs

@@ -406,7 +406,7 @@ mod tests {
         let mut term = Term::from_field_json_path(field, "color", false);
         term.append_type_and_str("red");
 
-        assert_eq!(term.serialized_term(), b"\x00\x00\x00\x01jcolor\x00sred")
+        assert_eq!(term.serialized_value_bytes(), b"color\x00sred".to_vec())
     }
 
     #[test]
@@ -416,8 +416,8 @@ mod tests {
         term.append_type_and_fast_value(-4i64);
 
         assert_eq!(
-            term.serialized_term(),
-            b"\x00\x00\x00\x01jcolor\x00i\x7f\xff\xff\xff\xff\xff\xff\xfc"
+            term.serialized_value_bytes(),
+            b"color\x00i\x7f\xff\xff\xff\xff\xff\xff\xfc".to_vec()
         )
     }
 
@@ -428,8 +428,8 @@ mod tests {
         term.append_type_and_fast_value(4u64);
 
         assert_eq!(
-            term.serialized_term(),
-            b"\x00\x00\x00\x01jcolor\x00u\x00\x00\x00\x00\x00\x00\x00\x04"
+            term.serialized_value_bytes(),
+            b"color\x00u\x00\x00\x00\x00\x00\x00\x00\x04".to_vec()
         )
     }
 
@@ -439,8 +439,8 @@ mod tests {
         let mut term = Term::from_field_json_path(field, "color", false);
         term.append_type_and_fast_value(4.0f64);
         assert_eq!(
-            term.serialized_term(),
-            b"\x00\x00\x00\x01jcolor\x00f\xc0\x10\x00\x00\x00\x00\x00\x00"
+            term.serialized_value_bytes(),
+            b"color\x00f\xc0\x10\x00\x00\x00\x00\x00\x00".to_vec()
         )
     }
 
@@ -450,8 +450,8 @@ mod tests {
         let mut term = Term::from_field_json_path(field, "color", false);
         term.append_type_and_fast_value(true);
         assert_eq!(
-            term.serialized_term(),
-            b"\x00\x00\x00\x01jcolor\x00o\x00\x00\x00\x00\x00\x00\x00\x01"
+            term.serialized_value_bytes(),
+            b"color\x00o\x00\x00\x00\x00\x00\x00\x00\x01".to_vec()
         )
     }
 

src/directory/composite_file.rs

@@ -5,7 +5,7 @@ use std::ops::Range;
 use common::{BinarySerializable, CountingWriter, HasLen, VInt};
 
 use crate::directory::{FileSlice, TerminatingWrite, WritePtr};
-use crate::schema::Field;
+use crate::schema::{Field, Schema};
 use crate::space_usage::{FieldUsage, PerFieldSpaceUsage};
 
 #[derive(Eq, PartialEq, Hash, Copy, Ord, PartialOrd, Clone, Debug)]
@@ -167,10 +167,11 @@ impl CompositeFile {
             .map(|byte_range| self.data.slice(byte_range.clone()))
     }
 
-    pub fn space_usage(&self) -> PerFieldSpaceUsage {
+    pub fn space_usage(&self, schema: &Schema) -> PerFieldSpaceUsage {
         let mut fields = Vec::new();
         for (&field_addr, byte_range) in &self.offsets_index {
-            let mut field_usage = FieldUsage::empty(field_addr.field);
+            let field_name = schema.get_field_name(field_addr.field).to_string();
+            let mut field_usage = FieldUsage::empty(field_name);
             field_usage.add_field_idx(field_addr.idx, byte_range.len().into());
             fields.push(field_usage);
         }

src/fastfield/readers.rs

@@ -8,7 +8,7 @@ use columnar::{
 };
 use common::ByteCount;
 
-use crate::core::json_utils::encode_column_name;
+use crate::core::json_utils::{encode_column_name, json_path_sep_to_dot};
 use crate::directory::FileSlice;
 use crate::schema::{Field, FieldEntry, FieldType, Schema};
 use crate::space_usage::{FieldUsage, PerFieldSpaceUsage};
@@ -39,19 +39,15 @@ impl FastFieldReaders {
         self.resolve_column_name_given_default_field(column_name, default_field_opt)
     }
 
-    pub(crate) fn space_usage(&self, schema: &Schema) -> io::Result<PerFieldSpaceUsage> {
+    pub(crate) fn space_usage(&self) -> io::Result<PerFieldSpaceUsage> {
         let mut per_field_usages: Vec<FieldUsage> = Default::default();
-        for (field, field_entry) in schema.fields() {
-            let column_handles = self.columnar.read_columns(field_entry.name())?;
-            let num_bytes: ByteCount = column_handles
-                .iter()
-                .map(|column_handle| column_handle.num_bytes())
-                .sum();
-            let mut field_usage = FieldUsage::empty(field);
-            field_usage.add_field_idx(0, num_bytes);
+        for (mut field_name, column_handle) in self.columnar.iter_columns()? {
+            json_path_sep_to_dot(&mut field_name);
+            let space_usage = column_handle.space_usage()?;
+            let mut field_usage = FieldUsage::empty(field_name);
+            field_usage.set_column_usage(space_usage);
             per_field_usages.push(field_usage);
         }
-        // TODO fix space usage for JSON fields.
         Ok(PerFieldSpaceUsage::new(per_field_usages))
     }
 

src/fieldnorm/reader.rs

@@ -2,7 +2,7 @@ use std::sync::Arc;
 
 use super::{fieldnorm_to_id, id_to_fieldnorm};
 use crate::directory::{CompositeFile, FileSlice, OwnedBytes};
-use crate::schema::Field;
+use crate::schema::{Field, Schema};
 use crate::space_usage::PerFieldSpaceUsage;
 use crate::DocId;
 
@@ -37,8 +37,8 @@ impl FieldNormReaders {
     }
 
     /// Return a break down of the space usage per field.
-    pub fn space_usage(&self) -> PerFieldSpaceUsage {
-        self.data.space_usage()
+    pub fn space_usage(&self, schema: &Schema) -> PerFieldSpaceUsage {
+        self.data.space_usage(schema)
     }
 
     /// Returns a handle to inner file

src/index/segment_reader.rs

@@ -455,11 +455,11 @@ impl SegmentReader {
     pub fn space_usage(&self) -> io::Result<SegmentSpaceUsage> {
         Ok(SegmentSpaceUsage::new(
             self.num_docs(),
-            self.termdict_composite.space_usage(),
-            self.postings_composite.space_usage(),
-            self.positions_composite.space_usage(),
-            self.fast_fields_readers.space_usage(self.schema())?,
-            self.fieldnorm_readers.space_usage(),
+            self.termdict_composite.space_usage(self.schema()),
+            self.postings_composite.space_usage(self.schema()),
+            self.positions_composite.space_usage(self.schema()),
+            self.fast_fields_readers.space_usage()?,
+            self.fieldnorm_readers.space_usage(self.schema()),
             self.get_store_reader(0)?.space_usage(),
             self.alive_bitset_opt
                 .as_ref()

src/indexer/indexing_term.rs

@@ -3,21 +3,21 @@ use std::net::Ipv6Addr;
 use columnar::MonotonicallyMappableToU128;
 
 use crate::fastfield::FastValue;
-use crate::schema::{Field, Type};
+use crate::schema::Field;
 
-/// Term represents the value that the token can take.
-/// It's a serialized representation over different types.
+/// IndexingTerm is used to represent a term during indexing.
+/// It's a serialized representation over field and value.
 ///
-/// It actually wraps a `Vec<u8>`. The first 5 bytes are metadata.
-/// 4 bytes are the field id, and the last byte is the type.
+/// It actually wraps a `Vec<u8>`. The first 4 bytes are the field.
 ///
-/// The serialized value `ValueBytes` is considered everything after the 4 first bytes (term id).
+/// We serialize the field, because we index everything in a single
+/// global term dictionary during indexing.
 #[derive(Clone)]
 pub(crate) struct IndexingTerm<B = Vec<u8>>(B)
 where B: AsRef<[u8]>;
 
 /// The number of bytes used as metadata by `Term`.
-const TERM_METADATA_LENGTH: usize = 5;
+const TERM_METADATA_LENGTH: usize = 4;
 
 impl IndexingTerm {
     /// Create a new Term with a buffer with a given capacity.
@@ -31,10 +31,9 @@ impl IndexingTerm {
     /// Use `clear_with_field_and_type` in that case.
     ///
     /// Sets field and the type.
-    pub(crate) fn set_field_and_type(&mut self, field: Field, typ: Type) {
+    pub(crate) fn set_field(&mut self, field: Field) {
         assert!(self.is_empty());
         self.0[0..4].clone_from_slice(field.field_id().to_be_bytes().as_ref());
-        self.0[4] = typ.to_code();
     }
 
     /// Is empty if there are no value bytes.
@@ -42,10 +41,10 @@ impl IndexingTerm {
         self.0.len() == TERM_METADATA_LENGTH
     }
 
-    /// Removes the value_bytes and set the field and type code.
-    pub(crate) fn clear_with_field_and_type(&mut self, typ: Type, field: Field) {
+    /// Removes the value_bytes and set the field
+    pub(crate) fn clear_with_field(&mut self, field: Field) {
         self.truncate_value_bytes(0);
-        self.set_field_and_type(field, typ);
+        self.set_field(field);
     }
 
     /// Sets a u64 value in the term.
@@ -122,6 +121,23 @@ impl IndexingTerm {
 impl<B> IndexingTerm<B>
 where B: AsRef<[u8]>
 {
+    /// Wraps serialized term bytes.
+    ///
+    /// The input buffer is expected to be the concatenation of the big endian encoded field id
+    /// followed by the serialized value bytes (type tag + payload).
+    #[inline]
+    pub fn wrap(serialized_term: B) -> IndexingTerm<B> {
+        debug_assert!(serialized_term.as_ref().len() >= TERM_METADATA_LENGTH);
+        IndexingTerm(serialized_term)
+    }
+
+    /// Returns the field this term belongs to.
+    #[inline]
+    pub fn field(&self) -> Field {
+        let field_id_bytes: [u8; 4] = self.0.as_ref()[..4].try_into().unwrap();
+        Field::from_field_id(u32::from_be_bytes(field_id_bytes))
+    }
+
     /// Returns the serialized representation of Term.
     /// This includes field_id, value type and value.
     ///
@@ -136,6 +152,7 @@ where B: AsRef<[u8]>
 #[cfg(test)]
 mod tests {
 
+    use super::IndexingTerm;
     use crate::schema::*;
 
     #[test]
@@ -143,42 +160,55 @@ mod tests {
         let mut schema_builder = Schema::builder();
         schema_builder.add_text_field("text", STRING);
         let title_field = schema_builder.add_text_field("title", STRING);
-        let term = Term::from_field_text(title_field, "test");
+        let mut term = IndexingTerm::with_capacity(0);
+        term.set_field(title_field);
+        term.set_bytes(b"test");
         assert_eq!(term.field(), title_field);
-        assert_eq!(term.typ(), Type::Str);
-        assert_eq!(term.value().as_str(), Some("test"))
+        assert_eq!(term.serialized_term(), b"\x00\x00\x00\x01test".to_vec())
     }
 
     /// Size (in bytes) of the buffer of a fast value (u64, i64, f64, or date) term.
     /// <field> + <type byte> + <value len>
     ///
     /// - <field> is a big endian encoded u32 field id
-    /// - <type_byte>'s most significant bit expresses whether the term is a json term or not The
-    ///   remaining 7 bits are used to encode the type of the value. If this is a JSON term, the
-    ///   type is the type of the leaf of the json.
     /// - <value> is,  if this is not the json term, a binary representation specific to the type.
     ///   If it is a JSON Term, then it is prepended with the path that leads to this leaf value.
-    const FAST_VALUE_TERM_LEN: usize = 4 + 1 + 8;
+    const FAST_VALUE_TERM_LEN: usize = 4 + 8;
 
     #[test]
     pub fn test_term_u64() {
         let mut schema_builder = Schema::builder();
         let count_field = schema_builder.add_u64_field("count", INDEXED);
-        let term = Term::from_field_u64(count_field, 983u64);
+        let mut term = IndexingTerm::with_capacity(0);
+        term.set_field(count_field);
+        term.set_u64(983u64);
         assert_eq!(term.field(), count_field);
-        assert_eq!(term.typ(), Type::U64);
         assert_eq!(term.serialized_term().len(), FAST_VALUE_TERM_LEN);
-        assert_eq!(term.value().as_u64(), Some(983u64))
     }
 
     #[test]
     pub fn test_term_bool() {
         let mut schema_builder = Schema::builder();
         let bool_field = schema_builder.add_bool_field("bool", INDEXED);
-        let term = Term::from_field_bool(bool_field, true);
+        let term = {
+            let mut term = IndexingTerm::with_capacity(0);
+            term.set_field(bool_field);
+            term.set_bool(true);
+            term
+        };
         assert_eq!(term.field(), bool_field);
-        assert_eq!(term.typ(), Type::Bool);
         assert_eq!(term.serialized_term().len(), FAST_VALUE_TERM_LEN);
-        assert_eq!(term.value().as_bool(), Some(true))
+    }
+
+    #[test]
+    pub fn indexing_term_wrap_extracts_field() {
+        let field = Field::from_field_id(7u32);
+        let mut term = IndexingTerm::with_capacity(0);
+        term.set_field(field);
+        term.append_bytes(b"abc");
+
+        let wrapped = IndexingTerm::wrap(term.serialized_term());
+        assert_eq!(wrapped.field(), field);
+        assert_eq!(wrapped.serialized_term(), term.serialized_term());
     }
 }

src/indexer/segment_writer.rs

@@ -171,7 +171,7 @@ impl SegmentWriter {
             let (term_buffer, ctx) = (&mut self.term_buffer, &mut self.ctx);
             let postings_writer: &mut dyn PostingsWriter =
                 self.per_field_postings_writers.get_for_field_mut(field);
-            term_buffer.clear_with_field_and_type(field_entry.field_type().value_type(), field);
+            term_buffer.clear_with_field(field);
 
             match field_entry.field_type() {
                 FieldType::Facet(_) => {

src/lib.rs

@@ -216,9 +216,7 @@ use once_cell::sync::Lazy;
 use serde::{Deserialize, Serialize};
 
 pub use self::docset::{DocSet, COLLECT_BLOCK_BUFFER_LEN, TERMINATED};
-#[doc(hidden)]
-pub use crate::core::json_utils;
-pub use crate::core::{Executor, Searcher, SearcherGeneration};
+pub use crate::core::{json_utils, Executor, Searcher, SearcherGeneration};
 pub use crate::directory::Directory;
 pub use crate::index::{
     Index, IndexBuilder, IndexMeta, IndexSettings, InvertedIndexReader, Order, Segment,

src/postings/json_postings_writer.rs

@@ -8,7 +8,7 @@ use crate::indexer::path_to_unordered_id::OrderedPathId;
 use crate::postings::postings_writer::SpecializedPostingsWriter;
 use crate::postings::recorder::{BufferLender, DocIdRecorder, Recorder};
 use crate::postings::{FieldSerializer, IndexingContext, IndexingPosition, PostingsWriter};
-use crate::schema::{Field, Type, ValueBytes};
+use crate::schema::{Field, Type};
 use crate::tokenizer::TokenStream;
 use crate::DocId;
 
@@ -79,8 +79,7 @@ impl<Rec: Recorder> PostingsWriter for JsonPostingsWriter<Rec> {
             term_buffer.truncate(term_path_len);
             term_buffer.append_bytes(term);
 
-            let json_value = ValueBytes::wrap(term);
-            let typ = json_value.typ();
+            let typ = Type::from_code(term[0]).expect("Invalid type code in JSON term");
             if typ == Type::Str {
                 SpecializedPostingsWriter::<Rec>::serialize_one_term(
                     term_buffer.as_bytes(),
@@ -107,6 +106,8 @@ impl<Rec: Recorder> PostingsWriter for JsonPostingsWriter<Rec> {
     }
 }
 
+/// Helper to build the JSON term bytes that land in the term dictionary.
+/// Format: `[json path utf8][JSON_END_OF_PATH][type tag][payload]`
 struct JsonTermSerializer(Vec<u8>);
 impl JsonTermSerializer {
     /// Appends a JSON path to the Term.

src/postings/postings_writer.rs

@@ -11,7 +11,7 @@ use crate::postings::recorder::{BufferLender, Recorder};
 use crate::postings::{
     FieldSerializer, IndexingContext, InvertedIndexSerializer, PerFieldPostingsWriter,
 };
-use crate::schema::{Field, Schema, Term, Type};
+use crate::schema::{Field, Schema, Type};
 use crate::tokenizer::{Token, TokenStream, MAX_TOKEN_LEN};
 use crate::DocId;
 
@@ -59,14 +59,14 @@ pub(crate) fn serialize_postings(
     let mut term_offsets: Vec<(Field, OrderedPathId, &[u8], Addr)> =
         Vec::with_capacity(ctx.term_index.len());
     term_offsets.extend(ctx.term_index.iter().map(|(key, addr)| {
-        let field = Term::wrap(key).field();
+        let field = IndexingTerm::wrap(key).field();
         if schema.get_field_entry(field).field_type().value_type() == Type::Json {
-            let byte_range_path = 5..5 + 4;
+            let byte_range_path = 4..4 + 4;
             let unordered_id = u32::from_be_bytes(key[byte_range_path.clone()].try_into().unwrap());
             let path_id = unordered_id_to_ordered_id[unordered_id as usize];
             (field, path_id, &key[byte_range_path.end..], addr)
         } else {
-            (field, 0.into(), &key[5..], addr)
+            (field, 0.into(), &key[4..], addr)
         }
     }));
     // Sort by field, path, and term

src/schema/term.rs

@@ -1,10 +1,11 @@
-use std::hash::{Hash, Hasher};
+use std::hash::Hash;
 use std::net::Ipv6Addr;
 use std::{fmt, str};
 
 use columnar::MonotonicallyMappableToU128;
 use common::json_path_writer::{JSON_END_OF_PATH, JSON_PATH_SEGMENT_SEP_STR};
 use common::JsonPathWriter;
+use serde::{Deserialize, Serialize};
 
 use super::date_time_options::DATE_TIME_PRECISION_INDEXED;
 use super::{Field, Schema};
@@ -16,23 +17,54 @@ use crate::DateTime;
 /// Term represents the value that the token can take.
 /// It's a serialized representation over different types.
 ///
-/// It actually wraps a `Vec<u8>`. The first 5 bytes are metadata.
-/// 4 bytes are the field id, and the last byte is the type.
-///
-/// The serialized value `ValueBytes` is considered everything after the 4 first bytes (term id).
-#[derive(Clone)]
-pub struct Term<B = Vec<u8>>(B)
-where B: AsRef<[u8]>;
+/// A term is composed of Field and the serialized value bytes.
+/// The serialized value bytes themselves start with a one byte type tag followed by the payload.
+#[derive(Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)]
+pub struct Term {
+    field: Field,
+    serialized_value_bytes: Vec<u8>,
+}
 
-/// The number of bytes used as metadata by `Term`.
-const TERM_METADATA_LENGTH: usize = 5;
+/// The number of bytes used as metadata when serializing a term.
+const TERM_TYPE_TAG_LEN: usize = 1;
 
 impl Term {
+    /// Takes a serialized term and wraps it as a Term.
+    /// First 4 bytes are the field id
+    #[deprecated(
+        note = "we want to avoid working on the serialized representation directly, replace with \
+                typed API calls (add more if needed) or use serde to serialize/deserialize"
+    )]
+    pub fn wrap(serialized: &[u8]) -> Term {
+        let field_id_bytes: [u8; 4] = serialized[0..4].try_into().unwrap();
+        let field_id = u32::from_be_bytes(field_id_bytes);
+        Term {
+            field: Field::from_field_id(field_id),
+            serialized_value_bytes: serialized[4..].to_vec(),
+        }
+    }
+
+    /// Returns the serialized representation of the term.
+    /// First 4 bytes are the field id
+    #[deprecated(
+        note = "we want to avoid working on the serialized representation directly, replace with \
+                typed API calls (add more if needed) or use serde to serialize/deserialize"
+    )]
+    pub fn serialized_term(&self) -> Vec<u8> {
+        let mut serialized = Vec::with_capacity(4 + self.serialized_value_bytes.len());
+        serialized.extend(self.field.field_id().to_be_bytes().as_ref());
+        serialized.extend_from_slice(&self.serialized_value_bytes);
+        serialized
+    }
+
     /// Create a new Term with a buffer with a given capacity.
     pub fn with_capacity(capacity: usize) -> Term {
-        let mut data = Vec::with_capacity(TERM_METADATA_LENGTH + capacity);
-        data.resize(TERM_METADATA_LENGTH, 0u8);
-        Term(data)
+        let mut data = Vec::with_capacity(TERM_TYPE_TAG_LEN + capacity);
+        data.resize(TERM_TYPE_TAG_LEN, 0u8);
+        Term {
+            field: Field::from_field_id(0u32),
+            serialized_value_bytes: data,
+        }
     }
 
     /// Creates a term from a json path.
@@ -89,7 +121,7 @@ impl Term {
     fn with_bytes_and_field_and_payload(typ: Type, field: Field, bytes: &[u8]) -> Term {
         let mut term = Self::with_capacity(bytes.len());
         term.set_field_and_type(field, typ);
-        term.0.extend_from_slice(bytes);
+        term.serialized_value_bytes.extend_from_slice(bytes);
         term
     }
 
@@ -105,13 +137,13 @@ impl Term {
     /// Sets field and the type.
     pub(crate) fn set_field_and_type(&mut self, field: Field, typ: Type) {
         assert!(self.is_empty());
-        self.0[0..4].clone_from_slice(field.field_id().to_be_bytes().as_ref());
-        self.0[4] = typ.to_code();
+        self.field = field;
+        self.serialized_value_bytes[0] = typ.to_code();
     }
 
     /// Is empty if there are no value bytes.
     pub fn is_empty(&self) -> bool {
-        self.0.len() == TERM_METADATA_LENGTH
+        self.serialized_value_bytes.len() == TERM_TYPE_TAG_LEN
     }
 
     /// Builds a term given a field, and a `Ipv6Addr`-value
@@ -177,7 +209,7 @@ impl Term {
     /// Removes the value_bytes and set the type code.
     pub fn clear_with_type(&mut self, typ: Type) {
         self.truncate_value_bytes(0);
-        self.0[4] = typ.to_code();
+        self.serialized_value_bytes[0] = typ.to_code();
     }
 
     /// Append a type marker + fast value to a term.
@@ -185,9 +217,10 @@ impl Term {
     ///
     /// It will not clear existing bytes.
     pub fn append_type_and_fast_value<T: FastValue>(&mut self, val: T) {
-        self.0.push(T::to_type().to_code());
+        self.serialized_value_bytes.push(T::to_type().to_code());
         let value = val.to_u64();
-        self.0.extend(value.to_be_bytes().as_ref());
+        self.serialized_value_bytes
+            .extend(value.to_be_bytes().as_ref());
     }
 
     /// Append a string type marker + string to a term.
@@ -195,24 +228,25 @@ impl Term {
     ///
     /// It will not clear existing bytes.
     pub fn append_type_and_str(&mut self, val: &str) {
-        self.0.push(Type::Str.to_code());
-        self.0.extend(val.as_bytes().as_ref());
+        self.serialized_value_bytes.push(Type::Str.to_code());
+        self.serialized_value_bytes.extend(val.as_bytes().as_ref());
     }
 
     /// Sets the value of a `Bytes` field.
     pub fn set_bytes(&mut self, bytes: &[u8]) {
         self.truncate_value_bytes(0);
-        self.0.extend(bytes);
+        self.serialized_value_bytes.extend(bytes);
     }
 
     /// Truncates the value bytes of the term. Value and field type stays the same.
     pub fn truncate_value_bytes(&mut self, len: usize) {
-        self.0.truncate(len + TERM_METADATA_LENGTH);
+        self.serialized_value_bytes
+            .truncate(len + TERM_TYPE_TAG_LEN);
     }
 
     /// The length of the bytes.
     pub fn len_bytes(&self) -> usize {
-        self.0.len() - TERM_METADATA_LENGTH
+        self.serialized_value_bytes.len() - TERM_TYPE_TAG_LEN
     }
 
     /// Appends value bytes to the Term.
@@ -220,18 +254,9 @@ impl Term {
     /// This function returns the segment that has just been added.
     #[inline]
     pub fn append_bytes(&mut self, bytes: &[u8]) -> &mut [u8] {
-        let len_before = self.0.len();
-        self.0.extend_from_slice(bytes);
-        &mut self.0[len_before..]
-    }
-}
-
-impl<B> Term<B>
-where B: AsRef<[u8]>
-{
-    /// Wraps a object holding bytes
-    pub fn wrap(data: B) -> Term<B> {
-        Term(data)
+        let len_before = self.serialized_value_bytes.len();
+        self.serialized_value_bytes.extend_from_slice(bytes);
+        &mut self.serialized_value_bytes[len_before..]
     }
 
     /// Return the type of the term.
@@ -241,8 +266,7 @@ where B: AsRef<[u8]>
 
     /// Returns the field.
     pub fn field(&self) -> Field {
-        let field_id_bytes: [u8; 4] = (&self.0.as_ref()[..4]).try_into().unwrap();
-        Field::from_field_id(u32::from_be_bytes(field_id_bytes))
+        self.field
     }
 
     /// Returns the serialized representation of the value.
@@ -252,23 +276,13 @@ where B: AsRef<[u8]>
     /// If the term is a u64, its value is encoded according
     /// to `byteorder::BigEndian`.
     pub fn serialized_value_bytes(&self) -> &[u8] {
-        &self.0.as_ref()[TERM_METADATA_LENGTH..]
+        &self.serialized_value_bytes[TERM_TYPE_TAG_LEN..]
     }
 
     /// Returns the value of the term.
     /// address or JSON path + value. (this does not include the field.)
     pub fn value(&self) -> ValueBytes<&[u8]> {
-        ValueBytes::wrap(&self.0.as_ref()[4..])
-    }
-
-    /// Returns the serialized representation of Term.
-    /// This includes field_id, value type and value.
-    ///
-    /// Do NOT rely on this byte representation in the index.
-    /// This value is likely to change in the future.
-    #[inline]
-    pub fn serialized_term(&self) -> &[u8] {
-        self.0.as_ref()
+        ValueBytes::wrap(self.serialized_value_bytes.as_ref())
     }
 }
 
@@ -452,10 +466,7 @@ where B: AsRef<[u8]>
         }
     }
 
-    /// Returns the serialized representation of Term.
-    ///
-    /// Do NOT rely on this byte representation in the index.
-    /// This value is likely to change in the future.
+    /// Returns the serialized representation of the value bytes including the type tag.
     pub fn as_serialized(&self) -> &[u8] {
         self.0.as_ref()
     }
@@ -508,40 +519,6 @@ where B: AsRef<[u8]>
     }
 }
 
-impl<B> Ord for Term<B>
-where B: AsRef<[u8]>
-{
-    fn cmp(&self, other: &Self) -> std::cmp::Ordering {
-        self.serialized_term().cmp(other.serialized_term())
-    }
-}
-
-impl<B> PartialOrd for Term<B>
-where B: AsRef<[u8]>
-{
-    fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
-        Some(self.cmp(other))
-    }
-}
-
-impl<B> PartialEq for Term<B>
-where B: AsRef<[u8]>
-{
-    fn eq(&self, other: &Self) -> bool {
-        self.serialized_term() == other.serialized_term()
-    }
-}
-
-impl<B> Eq for Term<B> where B: AsRef<[u8]> {}
-
-impl<B> Hash for Term<B>
-where B: AsRef<[u8]>
-{
-    fn hash<H: Hasher>(&self, state: &mut H) {
-        self.0.as_ref().hash(state)
-    }
-}
-
 fn write_opt<T: std::fmt::Debug>(f: &mut fmt::Formatter, val_opt: Option<T>) -> fmt::Result {
     if let Some(val) = val_opt {
         write!(f, "{val:?}")?;
@@ -549,13 +526,11 @@ fn write_opt<T: std::fmt::Debug>(f: &mut fmt::Formatter, val_opt: Option<T>) ->
     Ok(())
 }
 
-impl<B> fmt::Debug for Term<B>
-where B: AsRef<[u8]>
-{
+impl fmt::Debug for Term {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        let field_id = self.field().field_id();
+        let field_id = self.field.field_id();
         write!(f, "Term(field={field_id}, ")?;
-        let value_bytes = ValueBytes::wrap(&self.0.as_ref()[4..]);
+        let value_bytes = ValueBytes::wrap(&self.serialized_value_bytes);
         value_bytes.debug_value_bytes(f)?;
         write!(f, ")",)?;
         Ok(())
@@ -578,17 +553,6 @@ mod tests {
         assert_eq!(term.value().as_str(), Some("test"))
     }
 
-    /// Size (in bytes) of the buffer of a fast value (u64, i64, f64, or date) term.
-    /// <field> + <type byte> + <value len>
-    ///
-    /// - <field> is a big endian encoded u32 field id
-    /// - <type_byte>'s most significant bit expresses whether the term is a json term or not The
-    ///   remaining 7 bits are used to encode the type of the value. If this is a JSON term, the
-    ///   type is the type of the leaf of the json.
-    /// - <value> is,  if this is not the json term, a binary representation specific to the type.
-    ///   If it is a JSON Term, then it is prepended with the path that leads to this leaf value.
-    const FAST_VALUE_TERM_LEN: usize = 4 + 1 + 8;
-
     #[test]
     pub fn test_term_u64() {
         let mut schema_builder = Schema::builder();
@@ -596,7 +560,7 @@ mod tests {
         let term = Term::from_field_u64(count_field, 983u64);
         assert_eq!(term.field(), count_field);
         assert_eq!(term.typ(), Type::U64);
-        assert_eq!(term.serialized_term().len(), FAST_VALUE_TERM_LEN);
+        assert_eq!(term.serialized_value_bytes().len(), 8);
         assert_eq!(term.value().as_u64(), Some(983u64))
     }
 
@@ -607,7 +571,7 @@ mod tests {
         let term = Term::from_field_bool(bool_field, true);
         assert_eq!(term.field(), bool_field);
         assert_eq!(term.typ(), Type::Bool);
-        assert_eq!(term.serialized_term().len(), FAST_VALUE_TERM_LEN);
+        assert_eq!(term.serialized_value_bytes().len(), 8);
         assert_eq!(term.value().as_bool(), Some(true))
     }
 }

src/space_usage/mod.rs

@@ -7,13 +7,14 @@
 //! storage-level details into consideration. For example, if your file system block size is 4096
 //! bytes, we can under-count actual resultant space usage by up to 4095 bytes per file.
 
-use std::collections::HashMap;
+use std::collections::btree_map::Entry;
+use std::collections::BTreeMap;
 
+use columnar::ColumnSpaceUsage;
 use common::ByteCount;
 use serde::{Deserialize, Serialize};
 
 use crate::index::SegmentComponent;
-use crate::schema::Field;
 
 /// Enum containing any of the possible space usage results for segment components.
 pub enum ComponentSpaceUsage {
@@ -212,17 +213,26 @@ impl StoreSpaceUsage {
 /// Multiple indexes are used to handle variable length things, where
 #[derive(Clone, Debug, Serialize, Deserialize)]
 pub struct PerFieldSpaceUsage {
-    fields: HashMap<Field, FieldUsage>,
+    fields: BTreeMap<String, FieldUsage>,
     total: ByteCount,
 }
 
 impl PerFieldSpaceUsage {
     pub(crate) fn new(fields: Vec<FieldUsage>) -> PerFieldSpaceUsage {
-        let total = fields.iter().map(FieldUsage::total).sum();
-        let field_usage_map: HashMap<Field, FieldUsage> = fields
-            .into_iter()
-            .map(|field_usage| (field_usage.field(), field_usage))
-            .collect();
+        let mut total = ByteCount::default();
+        let mut field_usage_map: BTreeMap<String, FieldUsage> = BTreeMap::new();
+        for field_usage in fields {
+            total += field_usage.total();
+            let field_name = field_usage.field_name().to_string();
+            match field_usage_map.entry(field_name) {
+                Entry::Vacant(entry) => {
+                    entry.insert(field_usage);
+                }
+                Entry::Occupied(mut entry) => {
+                    entry.get_mut().merge(field_usage);
+                }
+            }
+        }
         PerFieldSpaceUsage {
             fields: field_usage_map,
             total,
@@ -230,8 +240,8 @@ impl PerFieldSpaceUsage {
     }
 
     /// Per field space usage
-    pub fn fields(&self) -> impl Iterator<Item = (&Field, &FieldUsage)> {
-        self.fields.iter()
+    pub fn fields(&self) -> impl Iterator<Item = &FieldUsage> {
+        self.fields.values()
     }
 
     /// Bytes used by the represented file
@@ -246,20 +256,23 @@ impl PerFieldSpaceUsage {
 /// See documentation for [`PerFieldSpaceUsage`] for slightly more information.
 #[derive(Clone, Debug, Serialize, Deserialize)]
 pub struct FieldUsage {
-    field: Field,
+    field_name: String,
     num_bytes: ByteCount,
     /// A field can be composed of more than one piece.
     /// These pieces are indexed by arbitrary numbers starting at zero.
     /// `self.num_bytes` includes all of `self.sub_num_bytes`.
     sub_num_bytes: Vec<Option<ByteCount>>,
+    /// Space usage of the column for fast fields, if relevant.
+    column_space_usage: Option<ColumnSpaceUsage>,
 }
 
 impl FieldUsage {
-    pub(crate) fn empty(field: Field) -> FieldUsage {
+    pub(crate) fn empty(field_name: impl Into<String>) -> FieldUsage {
         FieldUsage {
-            field,
+            field_name: field_name.into(),
             num_bytes: Default::default(),
             sub_num_bytes: Vec::new(),
+            column_space_usage: None,
         }
     }
 
@@ -272,9 +285,14 @@ impl FieldUsage {
         self.num_bytes += size
     }
 
+    pub(crate) fn set_column_usage(&mut self, column_space_usage: ColumnSpaceUsage) {
+        self.num_bytes += column_space_usage.total_num_bytes();
+        self.column_space_usage = Some(column_space_usage);
+    }
+
     /// Field
-    pub fn field(&self) -> Field {
-        self.field
+    pub fn field_name(&self) -> &str {
+        &self.field_name
     }
 
     /// Space usage for each index
@@ -282,16 +300,64 @@ impl FieldUsage {
         &self.sub_num_bytes[..]
     }
 
+    /// Returns the number of bytes used by the column payload, if the field is columnar.
+    pub fn column_num_bytes(&self) -> Option<ByteCount> {
+        self.column_space_usage
+            .as_ref()
+            .map(ColumnSpaceUsage::column_num_bytes)
+    }
+
+    /// Returns the number of bytes used by the dictionary for dictionary-encoded columns.
+    pub fn dictionary_num_bytes(&self) -> Option<ByteCount> {
+        self.column_space_usage
+            .as_ref()
+            .and_then(ColumnSpaceUsage::dictionary_num_bytes)
+    }
+
+    /// Returns the space usage of the column, if any.
+    pub fn column_space_usage(&self) -> Option<&ColumnSpaceUsage> {
+        self.column_space_usage.as_ref()
+    }
+
     /// Total bytes used for this field in this context
     pub fn total(&self) -> ByteCount {
         self.num_bytes
     }
+
+    fn merge(&mut self, other: FieldUsage) {
+        assert_eq!(self.field_name, other.field_name);
+        self.num_bytes += other.num_bytes;
+        if other.sub_num_bytes.len() > self.sub_num_bytes.len() {
+            self.sub_num_bytes.resize(other.sub_num_bytes.len(), None);
+        }
+        for (idx, num_bytes_opt) in other.sub_num_bytes.into_iter().enumerate() {
+            if let Some(num_bytes) = num_bytes_opt {
+                match self.sub_num_bytes[idx] {
+                    Some(existing) => self.sub_num_bytes[idx] = Some(existing + num_bytes),
+                    None => self.sub_num_bytes[idx] = Some(num_bytes),
+                }
+            }
+        }
+        self.column_space_usage =
+            merge_column_space_usage(self.column_space_usage.take(), other.column_space_usage);
+    }
+}
+
+fn merge_column_space_usage(
+    left: Option<ColumnSpaceUsage>,
+    right: Option<ColumnSpaceUsage>,
+) -> Option<ColumnSpaceUsage> {
+    match (left, right) {
+        (Some(lhs), Some(rhs)) => Some(lhs.merge(&rhs)),
+        (Some(space), None) | (None, Some(space)) => Some(space),
+        (None, None) => None,
+    }
 }
 
 #[cfg(test)]
 mod test {
     use crate::index::Index;
-    use crate::schema::{Field, Schema, FAST, INDEXED, STORED, TEXT};
+    use crate::schema::{Schema, FAST, INDEXED, STORED, TEXT};
     use crate::space_usage::PerFieldSpaceUsage;
     use crate::{IndexWriter, Term};
 
@@ -307,17 +373,17 @@ mod test {
 
     fn expect_single_field(
         field_space: &PerFieldSpaceUsage,
-        field: &Field,
+        field: &str,
         min_size: u64,
         max_size: u64,
     ) {
         assert!(field_space.total() >= min_size);
         assert!(field_space.total() <= max_size);
         assert_eq!(
-            vec![(field, field_space.total())],
+            vec![(field.to_string(), field_space.total())],
             field_space
                 .fields()
-                .map(|(x, y)| (x, y.total()))
+                .map(|usage| (usage.field_name().to_string(), usage.total()))
                 .collect::<Vec<_>>()
         );
     }
@@ -327,6 +393,7 @@ mod test {
         let mut schema_builder = Schema::builder();
         let name = schema_builder.add_u64_field("name", FAST | INDEXED);
         let schema = schema_builder.build();
+        let field_name = schema.get_field_name(name).to_string();
         let index = Index::create_in_ram(schema);
 
         {
@@ -349,11 +416,11 @@ mod test {
 
         assert_eq!(4, segment.num_docs());
 
-        expect_single_field(segment.termdict(), &name, 1, 512);
-        expect_single_field(segment.postings(), &name, 1, 512);
+        expect_single_field(segment.termdict(), &field_name, 1, 512);
+        expect_single_field(segment.postings(), &field_name, 1, 512);
         assert_eq!(segment.positions().total(), 0);
-        expect_single_field(segment.fast_fields(), &name, 1, 512);
-        expect_single_field(segment.fieldnorms(), &name, 1, 512);
+        expect_single_field(segment.fast_fields(), &field_name, 1, 512);
+        expect_single_field(segment.fieldnorms(), &field_name, 1, 512);
         // TODO: understand why the following fails
         //        assert_eq!(0, segment.store().total());
         assert_eq!(segment.deletes(), 0);
@@ -365,6 +432,7 @@ mod test {
         let mut schema_builder = Schema::builder();
         let name = schema_builder.add_text_field("name", TEXT);
         let schema = schema_builder.build();
+        let field_name = schema.get_field_name(name).to_string();
         let index = Index::create_in_ram(schema);
 
         {
@@ -389,11 +457,11 @@ mod test {
 
         assert_eq!(4, segment.num_docs());
 
-        expect_single_field(segment.termdict(), &name, 1, 512);
-        expect_single_field(segment.postings(), &name, 1, 512);
-        expect_single_field(segment.positions(), &name, 1, 512);
+        expect_single_field(segment.termdict(), &field_name, 1, 512);
+        expect_single_field(segment.postings(), &field_name, 1, 512);
+        expect_single_field(segment.positions(), &field_name, 1, 512);
         assert_eq!(segment.fast_fields().total(), 0);
-        expect_single_field(segment.fieldnorms(), &name, 1, 512);
+        expect_single_field(segment.fieldnorms(), &field_name, 1, 512);
         // TODO: understand why the following fails
         //        assert_eq!(0, segment.store().total());
         assert_eq!(segment.deletes(), 0);
@@ -429,10 +497,15 @@ mod test {
         assert_eq!(4, segment.num_docs());
 
         assert_eq!(segment.termdict().total(), 0);
+        assert!(segment.termdict().fields().next().is_none());
         assert_eq!(segment.postings().total(), 0);
+        assert!(segment.postings().fields().next().is_none());
         assert_eq!(segment.positions().total(), 0);
+        assert!(segment.positions().fields().next().is_none());
         assert_eq!(segment.fast_fields().total(), 0);
+        assert!(segment.fast_fields().fields().next().is_none());
         assert_eq!(segment.fieldnorms().total(), 0);
+        assert!(segment.fieldnorms().fields().next().is_none());
         assert!(segment.store().total() > 0);
         assert!(segment.store().total() < 512);
         assert_eq!(segment.deletes(), 0);
@@ -444,6 +517,7 @@ mod test {
         let mut schema_builder = Schema::builder();
         let name = schema_builder.add_u64_field("name", INDEXED);
         let schema = schema_builder.build();
+        let field_name = schema.get_field_name(name).to_string();
         let index = Index::create_in_ram(schema);
 
         {
@@ -474,11 +548,11 @@ mod test {
 
         assert_eq!(2, segment_space_usage.num_docs());
 
-        expect_single_field(segment_space_usage.termdict(), &name, 1, 512);
-        expect_single_field(segment_space_usage.postings(), &name, 1, 512);
+        expect_single_field(segment_space_usage.termdict(), &field_name, 1, 512);
+        expect_single_field(segment_space_usage.postings(), &field_name, 1, 512);
         assert_eq!(segment_space_usage.positions().total(), 0u64);
         assert_eq!(segment_space_usage.fast_fields().total(), 0u64);
-        expect_single_field(segment_space_usage.fieldnorms(), &name, 1, 512);
+        expect_single_field(segment_space_usage.fieldnorms(), &field_name, 1, 512);
         assert!(segment_space_usage.deletes() > 0);
         Ok(())
     }

src/tokenizer/stemmer.rs

@@ -7,7 +7,7 @@ use serde::{Deserialize, Serialize};
 use super::{Token, TokenFilter, TokenStream, Tokenizer};
 
 /// Available stemmer languages.
-#[derive(Debug, Serialize, Deserialize, Eq, PartialEq, Copy, Clone)]
+#[derive(Debug, Serialize, Deserialize, Eq, PartialEq, Copy, Clone, Hash)]
 #[allow(missing_docs)]
 pub enum Language {
     Arabic,

sstable/src/dictionary.rs

@@ -8,7 +8,7 @@ use std::sync::Arc;
 
 use common::bounds::{TransformBound, transform_bound_inner_res};
 use common::file_slice::FileSlice;
-use common::{BinarySerializable, OwnedBytes};
+use common::{BinarySerializable, ByteCount, OwnedBytes};
 use futures_util::{StreamExt, TryStreamExt, stream};
 use itertools::Itertools;
 use tantivy_fst::Automaton;
@@ -43,6 +43,7 @@ use crate::{
 pub struct Dictionary<TSSTable: SSTable = VoidSSTable> {
     pub sstable_slice: FileSlice,
     pub sstable_index: SSTableIndex,
+    num_bytes: ByteCount,
     num_terms: u64,
     phantom_data: PhantomData<TSSTable>,
 }
@@ -278,6 +279,7 @@ impl<TSSTable: SSTable> Dictionary<TSSTable> {
 
     /// Opens a `TermDictionary`.
     pub fn open(term_dictionary_file: FileSlice) -> io::Result<Self> {
+        let num_bytes = term_dictionary_file.num_bytes();
         let (main_slice, footer_len_slice) = term_dictionary_file.split_from_end(20);
         let mut footer_len_bytes: OwnedBytes = footer_len_slice.read_bytes()?;
         let index_offset = u64::deserialize(&mut footer_len_bytes)?;
@@ -317,6 +319,7 @@ impl<TSSTable: SSTable> Dictionary<TSSTable> {
         Ok(Dictionary {
             sstable_slice,
             sstable_index,
+            num_bytes,
             num_terms,
             phantom_data: PhantomData,
         })
@@ -343,6 +346,11 @@ impl<TSSTable: SSTable> Dictionary<TSSTable> {
         self.num_terms as usize
     }
 
+    /// Returns the total number of bytes used by the dictionary on disk.
+    pub fn num_bytes(&self) -> ByteCount {
+        self.num_bytes
+    }
+
     /// Decode a DeltaReader up to key, returning the number of terms traversed
     ///
     /// If the key was not found, returns Ok(None).