Experimental do not merge

* fix clippy

* fix clippy

* fix duplicate imports

Cargo.toml

@@ -11,7 +11,7 @@ repository = "https://github.com/quickwit-oss/tantivy"
 readme = "README.md"
 keywords = ["search", "information", "retrieval"]
 edition = "2021"
-rust-version = "1.62"
+rust-version = "1.63"
 exclude = ["benches/*.json", "benches/*.txt"]
 
 [dependencies]
@@ -78,6 +78,9 @@ paste = "1.0.11"
 more-asserts = "0.3.1"
 rand_distr = "0.4.3"
 time = { version = "0.3.10", features = ["serde-well-known", "macros"] }
+postcard = { version = "1.0.4", features = [
+  "use-std",
+], default-features = false }
 
 [target.'cfg(not(windows))'.dev-dependencies]
 criterion = { version = "0.5", default-features = false }

columnar/src/column_index/merge/shuffled.rs

@@ -140,7 +140,7 @@ mod tests {
     #[test]
     fn test_merge_column_index_optional_shuffle() {
         let optional_index: ColumnIndex = OptionalIndex::for_test(2, &[0]).into();
-        let column_indexes = vec![optional_index, ColumnIndex::Full];
+        let column_indexes = [optional_index, ColumnIndex::Full];
         let row_addrs = vec![
             RowAddr {
                 segment_ord: 0u32,

columnar/src/column_values/mod.rs

@@ -75,7 +75,7 @@ pub trait ColumnValues<T: PartialOrd = u64>: Send + Sync + DowncastSync {
         let out_and_idx_chunks = output
             .chunks_exact_mut(4)
             .into_remainder()
-            .into_iter()
+            .iter_mut()
             .zip(indexes.chunks_exact(4).remainder());
         for (out, idx) in out_and_idx_chunks {
             *out = self.get_val(*idx);
@@ -102,7 +102,7 @@ pub trait ColumnValues<T: PartialOrd = u64>: Send + Sync + DowncastSync {
         let out_and_idx_chunks = output
             .chunks_exact_mut(4)
             .into_remainder()
-            .into_iter()
+            .iter_mut()
             .zip(indexes.chunks_exact(4).remainder());
         for (out, idx) in out_and_idx_chunks {
             *out = Some(self.get_val(*idx));

columnar/src/column_values/u128_based/compact_space/mod.rs

@@ -148,7 +148,7 @@ impl CompactSpace {
             .binary_search_by_key(&compact, |range_mapping| range_mapping.compact_start)
             // Correctness: Overflow. The first range starts at compact space 0, the error from
             // binary search can never be 0
-            .map_or_else(|e| e - 1, |v| v);
+            .unwrap_or_else(|e| e - 1);
 
         let range_mapping = &self.ranges_mapping[pos];
         let diff = compact - range_mapping.compact_start;

columnar/src/columnar/writer/serializer.rs

@@ -18,7 +18,12 @@ pub struct ColumnarSerializer<W: io::Write> {
 /// code.
 fn prepare_key(key: &[u8], column_type: ColumnType, buffer: &mut Vec<u8>) {
     buffer.clear();
-    buffer.extend_from_slice(key);
+    // Convert 0 bytes to '0' string, as 0 bytes are reserved for the end of the path.
+    if key.contains(&0u8) {
+        buffer.extend(key.iter().map(|&b| if b == 0 { b'0' } else { b }));
+    } else {
+        buffer.extend_from_slice(key);
+    }
     buffer.push(0u8);
     buffer.push(column_type.to_code());
 }
@@ -102,7 +107,7 @@ mod tests {
         let mut buffer: Vec<u8> = b"somegarbage".to_vec();
         prepare_key(b"root\0child", ColumnType::Str, &mut buffer);
         assert_eq!(buffer.len(), 12);
-        assert_eq!(&buffer[..10], b"root\0child");
+        assert_eq!(&buffer[..10], b"root0child");
         assert_eq!(buffer[10], 0u8);
         assert_eq!(buffer[11], ColumnType::Str.to_code());
     }

common/src/datetime.rs

@@ -40,7 +40,7 @@ pub type DatePrecision = DateTimePrecision;
 /// All constructors and conversions are provided as explicit
 /// functions and not by implementing any `From`/`Into` traits
 /// to prevent unintended usage.
-#[derive(Clone, Default, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
+#[derive(Clone, Default, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
 pub struct DateTime {
     // Timestamp in nanoseconds.
     pub(crate) timestamp_nanos: i64,

common/src/serialize.rs

@@ -290,8 +290,7 @@ impl<'a> BinarySerializable for Cow<'a, [u8]> {
 #[cfg(test)]
 pub mod test {
 
-    use super::{VInt, *};
-    use crate::serialize::BinarySerializable;
+    use super::*;
     pub fn fixed_size_test<O: BinarySerializable + FixedSize + Default>() {
         let mut buffer = Vec::new();
         O::default().serialize(&mut buffer).unwrap();

src/aggregation/agg_req_with_accessor.rs

@@ -292,7 +292,7 @@ impl AggregationWithAccessor {
                 add_agg_with_accessor(&agg, accessor, column_type, &mut res)?;
             }
             TopHits(ref mut top_hits) => {
-                top_hits.validate_and_resolve(reader.fast_fields().columnar())?;
+                top_hits.validate_and_resolve_field_names(reader.fast_fields().columnar())?;
                 let accessors: Vec<(Column<u64>, ColumnType)> = top_hits
                     .field_names()
                     .iter()

src/aggregation/agg_tests.rs

@@ -4,6 +4,7 @@ use crate::aggregation::agg_req::{Aggregation, Aggregations};
 use crate::aggregation::agg_result::AggregationResults;
 use crate::aggregation::buf_collector::DOC_BLOCK_SIZE;
 use crate::aggregation::collector::AggregationCollector;
+use crate::aggregation::intermediate_agg_result::IntermediateAggregationResults;
 use crate::aggregation::segment_agg_result::AggregationLimits;
 use crate::aggregation::tests::{get_test_index_2_segments, get_test_index_from_values_and_terms};
 use crate::aggregation::DistributedAggregationCollector;
@@ -66,6 +67,22 @@ fn test_aggregation_flushing(
             }
         }
     },
+    "top_hits_test":{
+        "terms": {
+            "field": "string_id"
+        },
+        "aggs": {
+            "bucketsL2": {
+                "top_hits": {
+                    "size": 2,
+                    "sort": [
+                        { "score": "asc" }
+                    ],
+                    "docvalue_fields": ["score"]
+                }
+            }
+        }
+    },
     "histogram_test":{
         "histogram": {
             "field": "score",
@@ -108,6 +125,16 @@ fn test_aggregation_flushing(
 
         let searcher = reader.searcher();
         let intermediate_agg_result = searcher.search(&AllQuery, &collector).unwrap();
+
+        // Test postcard roundtrip serialization
+        let intermediate_agg_result_bytes = postcard::to_allocvec(&intermediate_agg_result).expect(
+            "Postcard Serialization failed, flatten etc. is not supported in the intermediate \
+             result",
+        );
+        let intermediate_agg_result: IntermediateAggregationResults =
+            postcard::from_bytes(&intermediate_agg_result_bytes)
+                .expect("Post deserialization failed");
+
         intermediate_agg_result
             .into_final_result(agg_req, &Default::default())
             .unwrap()

src/aggregation/bucket/histogram/histogram.rs

@@ -1,7 +1,5 @@
 use std::cmp::Ordering;
 
-use columnar::ColumnType;
-use itertools::Itertools;
 use rustc_hash::FxHashMap;
 use serde::{Deserialize, Serialize};
 use tantivy_bitpacker::minmax;
@@ -17,7 +15,7 @@ use crate::aggregation::intermediate_agg_result::{
     IntermediateHistogramBucketEntry,
 };
 use crate::aggregation::segment_agg_result::{
-    build_segment_agg_collector, AggregationLimits, SegmentAggregationCollector,
+    build_segment_agg_collector, SegmentAggregationCollector,
 };
 use crate::aggregation::*;
 use crate::TantivyError;

src/aggregation/bucket/range.rs

@@ -1,7 +1,6 @@
 use std::fmt::Debug;
 use std::ops::Range;
 
-use columnar::{ColumnType, MonotonicallyMappableToU64};
 use rustc_hash::FxHashMap;
 use serde::{Deserialize, Serialize};
 
@@ -450,7 +449,6 @@ pub(crate) fn range_to_key(range: &Range<u64>, field_type: &ColumnType) -> crate
 #[cfg(test)]
 mod tests {
 
-    use columnar::MonotonicallyMappableToU64;
     use serde_json::Value;
 
     use super::*;
@@ -459,7 +457,6 @@ mod tests {
         exec_request, exec_request_with_query, get_test_index_2_segments,
         get_test_index_with_num_docs,
     };
-    use crate::aggregation::AggregationLimits;
 
     pub fn get_collector_from_ranges(
         ranges: Vec<RangeAggregationRange>,

src/aggregation/intermediate_agg_result.rs

@@ -20,7 +20,7 @@ use super::bucket::{
 };
 use super::metric::{
     IntermediateAverage, IntermediateCount, IntermediateMax, IntermediateMin, IntermediateStats,
-    IntermediateSum, PercentilesCollector, TopHitsCollector,
+    IntermediateSum, PercentilesCollector, TopHitsTopNComputer,
 };
 use super::segment_agg_result::AggregationLimits;
 use super::{format_date, AggregationError, Key, SerializedKey};
@@ -221,9 +221,9 @@ pub(crate) fn empty_from_req(req: &Aggregation) -> IntermediateAggregationResult
         Percentiles(_) => IntermediateAggregationResult::Metric(
             IntermediateMetricResult::Percentiles(PercentilesCollector::default()),
         ),
-        TopHits(_) => IntermediateAggregationResult::Metric(IntermediateMetricResult::TopHits(
-            TopHitsCollector::default(),
-        )),
+        TopHits(ref req) => IntermediateAggregationResult::Metric(
+            IntermediateMetricResult::TopHits(TopHitsTopNComputer::new(req.clone())),
+        ),
     }
 }
 
@@ -285,7 +285,7 @@ pub enum IntermediateMetricResult {
     /// Intermediate sum result.
     Sum(IntermediateSum),
     /// Intermediate top_hits result
-    TopHits(TopHitsCollector),
+    TopHits(TopHitsTopNComputer),
 }
 
 impl IntermediateMetricResult {
@@ -314,7 +314,7 @@ impl IntermediateMetricResult {
                     .into_final_result(req.agg.as_percentile().expect("unexpected metric type")),
             ),
             IntermediateMetricResult::TopHits(top_hits) => {
-                MetricResult::TopHits(top_hits.finalize())
+                MetricResult::TopHits(top_hits.into_final_result())
             }
         }
     }

src/aggregation/metric/mod.rs

@@ -25,6 +25,8 @@ mod stats;
 mod sum;
 mod top_hits;
 
+use std::collections::HashMap;
+
 pub use average::*;
 pub use count::*;
 pub use max::*;
@@ -36,6 +38,8 @@ pub use stats::*;
 pub use sum::*;
 pub use top_hits::*;
 
+use crate::schema::OwnedValue;
+
 /// Single-metric aggregations use this common result structure.
 ///
 /// Main reason to wrap it in value is to match elasticsearch output structure.
@@ -92,8 +96,9 @@ pub struct TopHitsVecEntry {
 
     /// Search results, for queries that include field retrieval requests
     /// (`docvalue_fields`).
-    #[serde(flatten)]
-    pub search_results: FieldRetrivalResult,
+    #[serde(rename = "docvalue_fields")]
+    #[serde(skip_serializing_if = "HashMap::is_empty")]
+    pub doc_value_fields: HashMap<String, OwnedValue>,
 }
 
 /// The top_hits metric aggregation results a list of top hits by sort criteria.

src/aggregation/metric/percentiles.rs

@@ -1,6 +1,5 @@
 use std::fmt::Debug;
 
-use columnar::ColumnType;
 use serde::{Deserialize, Serialize};
 
 use super::*;

src/aggregation/metric/stats.rs

@@ -1,4 +1,3 @@
-use columnar::ColumnType;
 use serde::{Deserialize, Serialize};
 
 use super::*;

src/aggregation/metric/top_hits.rs

@@ -1,7 +1,8 @@
 use std::collections::HashMap;
-use std::fmt::Formatter;
+use std::net::Ipv6Addr;
 
 use columnar::{ColumnarReader, DynamicColumn};
+use common::DateTime;
 use regex::Regex;
 use serde::ser::SerializeMap;
 use serde::{Deserialize, Deserializer, Serialize, Serializer};
@@ -92,53 +93,61 @@ pub struct TopHitsAggregation {
     size: usize,
     from: Option<usize>,
 
-    #[serde(flatten)]
-    retrieval: RetrievalFields,
-}
-
-const fn default_doc_value_fields() -> Vec<String> {
-    Vec::new()
-}
-
-/// Search query spec for each matched document
-/// TODO: move this to a common module
-#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Default)]
-pub struct RetrievalFields {
-    /// The fast fields to return for each hit.
-    /// This is the only variant supported for now.
-    /// TODO: support the {field, format} variant for custom formatting.
     #[serde(rename = "docvalue_fields")]
-    #[serde(default = "default_doc_value_fields")]
-    pub doc_value_fields: Vec<String>,
+    #[serde(default)]
+    doc_value_fields: Vec<String>,
 }
 
-/// Search query result for each matched document
-/// TODO: move this to a common module
-#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Default)]
-pub struct FieldRetrivalResult {
-    /// The fast fields returned for each hit.
-    #[serde(rename = "docvalue_fields")]
-    #[serde(skip_serializing_if = "HashMap::is_empty")]
-    pub doc_value_fields: HashMap<String, OwnedValue>,
+#[derive(Debug, Clone, PartialEq, Default)]
+struct KeyOrder {
+    field: String,
+    order: Order,
 }
 
-impl RetrievalFields {
-    fn get_field_names(&self) -> Vec<&str> {
-        self.doc_value_fields.iter().map(|s| s.as_str()).collect()
+impl Serialize for KeyOrder {
+    fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
+        let KeyOrder { field, order } = self;
+        let mut map = serializer.serialize_map(Some(1))?;
+        map.serialize_entry(field, order)?;
+        map.end()
     }
+}
 
-    fn resolve_field_names(&mut self, reader: &ColumnarReader) -> crate::Result<()> {
-        // Tranform a glob (`pattern*`, for example) into a regex::Regex (`^pattern.*$`)
-        let globbed_string_to_regex = |glob: &str| {
-            // Replace `*` glob with `.*` regex
-            let sanitized = format!("^{}$", regex::escape(glob).replace(r"\*", ".*"));
-            Regex::new(&sanitized.replace('*', ".*")).map_err(|e| {
-                crate::TantivyError::SchemaError(format!(
-                    "Invalid regex '{}' in docvalue_fields: {}",
-                    glob, e
-                ))
-            })
-        };
+impl<'de> Deserialize<'de> for KeyOrder {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where D: Deserializer<'de> {
+        let mut key_order = <HashMap<String, Order>>::deserialize(deserializer)?.into_iter();
+        let (field, order) = key_order.next().ok_or(serde::de::Error::custom(
+            "Expected exactly one key-value pair in sort parameter of top_hits, found none",
+        ))?;
+        if key_order.next().is_some() {
+            return Err(serde::de::Error::custom(format!(
+                "Expected exactly one key-value pair in sort parameter of top_hits, found {:?}",
+                key_order
+            )));
+        }
+        Ok(Self { field, order })
+    }
+}
+
+// Tranform a glob (`pattern*`, for example) into a regex::Regex (`^pattern.*$`)
+fn globbed_string_to_regex(glob: &str) -> Result<Regex, crate::TantivyError> {
+    // Replace `*` glob with `.*` regex
+    let sanitized = format!("^{}$", regex::escape(glob).replace(r"\*", ".*"));
+    Regex::new(&sanitized.replace('*', ".*")).map_err(|e| {
+        crate::TantivyError::SchemaError(format!(
+            "Invalid regex '{}' in docvalue_fields: {}",
+            glob, e
+        ))
+    })
+}
+
+impl TopHitsAggregation {
+    /// Validate and resolve field retrieval parameters
+    pub fn validate_and_resolve_field_names(
+        &mut self,
+        reader: &ColumnarReader,
+    ) -> crate::Result<()> {
         self.doc_value_fields = self
             .doc_value_fields
             .iter()
@@ -175,12 +184,25 @@ impl RetrievalFields {
         Ok(())
     }
 
+    /// Return fields accessed by the aggregator, in order.
+    pub fn field_names(&self) -> Vec<&str> {
+        self.sort
+            .iter()
+            .map(|KeyOrder { field, .. }| field.as_str())
+            .collect()
+    }
+
+    /// Return fields accessed by the aggregator's value retrieval.
+    pub fn value_field_names(&self) -> Vec<&str> {
+        self.doc_value_fields.iter().map(|s| s.as_str()).collect()
+    }
+
     fn get_document_field_data(
         &self,
         accessors: &HashMap<String, Vec<DynamicColumn>>,
         doc_id: DocId,
-    ) -> FieldRetrivalResult {
-        let dvf = self
+    ) -> HashMap<String, FastFieldValue> {
+        let doc_value_fields = self
             .doc_value_fields
             .iter()
             .map(|field| {
@@ -188,20 +210,20 @@ impl RetrievalFields {
                     .get(field)
                     .unwrap_or_else(|| panic!("field '{}' not found in accessors", field));
 
-                let values: Vec<OwnedValue> = accessors
+                let values: Vec<FastFieldValue> = accessors
                     .iter()
                     .flat_map(|accessor| match accessor {
                         DynamicColumn::U64(accessor) => accessor
                             .values_for_doc(doc_id)
-                            .map(OwnedValue::U64)
+                            .map(FastFieldValue::U64)
                             .collect::<Vec<_>>(),
                         DynamicColumn::I64(accessor) => accessor
                             .values_for_doc(doc_id)
-                            .map(OwnedValue::I64)
+                            .map(FastFieldValue::I64)
                             .collect::<Vec<_>>(),
                         DynamicColumn::F64(accessor) => accessor
                             .values_for_doc(doc_id)
-                            .map(OwnedValue::F64)
+                            .map(FastFieldValue::F64)
                             .collect::<Vec<_>>(),
                         DynamicColumn::Bytes(accessor) => accessor
                             .term_ords(doc_id)
@@ -213,7 +235,7 @@ impl RetrievalFields {
                                         .expect("could not read term dictionary"),
                                     "term corresponding to term_ord does not exist"
                                 );
-                                OwnedValue::Bytes(buffer)
+                                FastFieldValue::Bytes(buffer)
                             })
                             .collect::<Vec<_>>(),
                         DynamicColumn::Str(accessor) => accessor
@@ -226,94 +248,82 @@ impl RetrievalFields {
                                         .expect("could not read term dictionary"),
                                     "term corresponding to term_ord does not exist"
                                 );
-                                OwnedValue::Str(String::from_utf8(buffer).unwrap())
+                                FastFieldValue::Str(String::from_utf8(buffer).unwrap())
                             })
                             .collect::<Vec<_>>(),
                         DynamicColumn::Bool(accessor) => accessor
                             .values_for_doc(doc_id)
-                            .map(OwnedValue::Bool)
+                            .map(FastFieldValue::Bool)
                             .collect::<Vec<_>>(),
                         DynamicColumn::IpAddr(accessor) => accessor
                             .values_for_doc(doc_id)
-                            .map(OwnedValue::IpAddr)
+                            .map(FastFieldValue::IpAddr)
                             .collect::<Vec<_>>(),
                         DynamicColumn::DateTime(accessor) => accessor
                             .values_for_doc(doc_id)
-                            .map(OwnedValue::Date)
+                            .map(FastFieldValue::Date)
                             .collect::<Vec<_>>(),
                     })
                     .collect();
 
-                (field.to_owned(), OwnedValue::Array(values))
+                (field.to_owned(), FastFieldValue::Array(values))
             })
             .collect();
-        FieldRetrivalResult {
-            doc_value_fields: dvf,
+        doc_value_fields
+    }
+}
+
+/// A retrieved value from a fast field.
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+pub enum FastFieldValue {
+    /// The str type is used for any text information.
+    Str(String),
+    /// Unsigned 64-bits Integer `u64`
+    U64(u64),
+    /// Signed 64-bits Integer `i64`
+    I64(i64),
+    /// 64-bits Float `f64`
+    F64(f64),
+    /// Bool value
+    Bool(bool),
+    /// Date/time with nanoseconds precision
+    Date(DateTime),
+    /// Arbitrarily sized byte array
+    Bytes(Vec<u8>),
+    /// IpV6 Address. Internally there is no IpV4, it needs to be converted to `Ipv6Addr`.
+    IpAddr(Ipv6Addr),
+    /// A list of values.
+    Array(Vec<Self>),
+}
+
+impl From<FastFieldValue> for OwnedValue {
+    fn from(value: FastFieldValue) -> Self {
+        match value {
+            FastFieldValue::Str(s) => OwnedValue::Str(s),
+            FastFieldValue::U64(u) => OwnedValue::U64(u),
+            FastFieldValue::I64(i) => OwnedValue::I64(i),
+            FastFieldValue::F64(f) => OwnedValue::F64(f),
+            FastFieldValue::Bool(b) => OwnedValue::Bool(b),
+            FastFieldValue::Date(d) => OwnedValue::Date(d),
+            FastFieldValue::Bytes(b) => OwnedValue::Bytes(b),
+            FastFieldValue::IpAddr(ip) => OwnedValue::IpAddr(ip),
+            FastFieldValue::Array(a) => {
+                OwnedValue::Array(a.into_iter().map(OwnedValue::from).collect())
+            }
         }
     }
 }
 
-#[derive(Debug, Clone, PartialEq, Default)]
-struct KeyOrder {
-    field: String,
-    order: Order,
-}
-
-impl Serialize for KeyOrder {
-    fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
-        let KeyOrder { field, order } = self;
-        let mut map = serializer.serialize_map(Some(1))?;
-        map.serialize_entry(field, order)?;
-        map.end()
-    }
-}
-
-impl<'de> Deserialize<'de> for KeyOrder {
-    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
-    where D: Deserializer<'de> {
-        let mut k_o = <HashMap<String, Order>>::deserialize(deserializer)?.into_iter();
-        let (k, v) = k_o.next().ok_or(serde::de::Error::custom(
-            "Expected exactly one key-value pair in KeyOrder, found none",
-        ))?;
-        if k_o.next().is_some() {
-            return Err(serde::de::Error::custom(
-                "Expected exactly one key-value pair in KeyOrder, found more",
-            ));
-        }
-        Ok(Self { field: k, order: v })
-    }
-}
-
-impl TopHitsAggregation {
-    /// Validate and resolve field retrieval parameters
-    pub fn validate_and_resolve(&mut self, reader: &ColumnarReader) -> crate::Result<()> {
-        self.retrieval.resolve_field_names(reader)
-    }
-
-    /// Return fields accessed by the aggregator, in order.
-    pub fn field_names(&self) -> Vec<&str> {
-        self.sort
-            .iter()
-            .map(|KeyOrder { field, .. }| field.as_str())
-            .collect()
-    }
-
-    /// Return fields accessed by the aggregator's value retrieval.
-    pub fn value_field_names(&self) -> Vec<&str> {
-        self.retrieval.get_field_names()
-    }
-}
-
-/// Holds a single comparable doc feature, and the order in which it should be sorted.
+/// Holds a fast field value in its u64 representation, and the order in which it should be sorted.
 #[derive(Clone, Serialize, Deserialize, Debug)]
-struct ComparableDocFeature {
-    /// Stores any u64-mappable feature.
+struct DocValueAndOrder {
+    /// A fast field value in its u64 representation.
     value: Option<u64>,
-    /// Sort order for the doc feature
+    /// Sort order for the value
     order: Order,
 }
 
-impl Ord for ComparableDocFeature {
+impl Ord for DocValueAndOrder {
     fn cmp(&self, other: &Self) -> std::cmp::Ordering {
         let invert = |cmp: std::cmp::Ordering| match self.order {
             Order::Asc => cmp,
@@ -329,26 +339,32 @@ impl Ord for ComparableDocFeature {
     }
 }
 
-impl PartialOrd for ComparableDocFeature {
+impl PartialOrd for DocValueAndOrder {
     fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
         Some(self.cmp(other))
     }
 }
 
-impl PartialEq for ComparableDocFeature {
+impl PartialEq for DocValueAndOrder {
     fn eq(&self, other: &Self) -> bool {
         self.value.cmp(&other.value) == std::cmp::Ordering::Equal
     }
 }
 
-impl Eq for ComparableDocFeature {}
+impl Eq for DocValueAndOrder {}
 
 #[derive(Clone, Serialize, Deserialize, Debug)]
-struct ComparableDocFeatures(Vec<ComparableDocFeature>, FieldRetrivalResult);
+struct DocSortValuesAndFields {
+    sorts: Vec<DocValueAndOrder>,
 
-impl Ord for ComparableDocFeatures {
+    #[serde(rename = "docvalue_fields")]
+    #[serde(skip_serializing_if = "HashMap::is_empty")]
+    doc_value_fields: HashMap<String, FastFieldValue>,
+}
+
+impl Ord for DocSortValuesAndFields {
     fn cmp(&self, other: &Self) -> std::cmp::Ordering {
-        for (self_feature, other_feature) in self.0.iter().zip(other.0.iter()) {
+        for (self_feature, other_feature) in self.sorts.iter().zip(other.sorts.iter()) {
             let cmp = self_feature.cmp(other_feature);
             if cmp != std::cmp::Ordering::Equal {
                 return cmp;
@@ -358,53 +374,43 @@ impl Ord for ComparableDocFeatures {
     }
 }
 
-impl PartialOrd for ComparableDocFeatures {
+impl PartialOrd for DocSortValuesAndFields {
     fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
         Some(self.cmp(other))
     }
 }
 
-impl PartialEq for ComparableDocFeatures {
+impl PartialEq for DocSortValuesAndFields {
     fn eq(&self, other: &Self) -> bool {
         self.cmp(other) == std::cmp::Ordering::Equal
     }
 }
 
-impl Eq for ComparableDocFeatures {}
+impl Eq for DocSortValuesAndFields {}
 
 /// The TopHitsCollector used for collecting over segments and merging results.
-#[derive(Clone, Serialize, Deserialize)]
-pub struct TopHitsCollector {
+#[derive(Clone, Serialize, Deserialize, Debug)]
+pub struct TopHitsTopNComputer {
     req: TopHitsAggregation,
-    top_n: TopNComputer<ComparableDocFeatures, DocAddress, false>,
+    top_n: TopNComputer<DocSortValuesAndFields, DocAddress, false>,
 }
 
-impl Default for TopHitsCollector {
-    fn default() -> Self {
-        Self {
-            req: TopHitsAggregation::default(),
-            top_n: TopNComputer::new(1),
-        }
-    }
-}
-
-impl std::fmt::Debug for TopHitsCollector {
-    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
-        f.debug_struct("TopHitsCollector")
-            .field("req", &self.req)
-            .field("top_n_threshold", &self.top_n.threshold)
-            .finish()
-    }
-}
-
-impl std::cmp::PartialEq for TopHitsCollector {
+impl std::cmp::PartialEq for TopHitsTopNComputer {
     fn eq(&self, _other: &Self) -> bool {
         false
     }
 }
 
-impl TopHitsCollector {
-    fn collect(&mut self, features: ComparableDocFeatures, doc: DocAddress) {
+impl TopHitsTopNComputer {
+    /// Create a new TopHitsCollector
+    pub fn new(req: TopHitsAggregation) -> Self {
+        Self {
+            top_n: TopNComputer::new(req.size + req.from.unwrap_or(0)),
+            req,
+        }
+    }
+
+    fn collect(&mut self, features: DocSortValuesAndFields, doc: DocAddress) {
         self.top_n.push(features, doc);
     }
 
@@ -416,14 +422,19 @@ impl TopHitsCollector {
     }
 
     /// Finalize by converting self into the final result form
-    pub fn finalize(self) -> TopHitsMetricResult {
+    pub fn into_final_result(self) -> TopHitsMetricResult {
         let mut hits: Vec<TopHitsVecEntry> = self
             .top_n
             .into_sorted_vec()
             .into_iter()
             .map(|doc| TopHitsVecEntry {
-                sort: doc.feature.0.iter().map(|f| f.value).collect(),
-                search_results: doc.feature.1,
+                sort: doc.feature.sorts.iter().map(|f| f.value).collect(),
+                doc_value_fields: doc
+                    .feature
+                    .doc_value_fields
+                    .into_iter()
+                    .map(|(k, v)| (k, v.into()))
+                    .collect(),
             })
             .collect();
 
@@ -436,48 +447,63 @@ impl TopHitsCollector {
     }
 }
 
-#[derive(Clone)]
-pub(crate) struct SegmentTopHitsCollector {
+#[derive(Clone, Debug)]
+pub(crate) struct TopHitsSegmentCollector {
     segment_ordinal: SegmentOrdinal,
     accessor_idx: usize,
-    inner_collector: TopHitsCollector,
+    req: TopHitsAggregation,
+    top_n: TopNComputer<Vec<DocValueAndOrder>, DocAddress, false>,
 }
 
-impl SegmentTopHitsCollector {
+impl TopHitsSegmentCollector {
     pub fn from_req(
         req: &TopHitsAggregation,
         accessor_idx: usize,
         segment_ordinal: SegmentOrdinal,
     ) -> Self {
         Self {
-            inner_collector: TopHitsCollector {
-                req: req.clone(),
-                top_n: TopNComputer::new(req.size + req.from.unwrap_or(0)),
-            },
+            req: req.clone(),
+            top_n: TopNComputer::new(req.size + req.from.unwrap_or(0)),
             segment_ordinal,
             accessor_idx,
         }
     }
-}
+    fn into_top_hits_collector(
+        self,
+        value_accessors: &HashMap<String, Vec<DynamicColumn>>,
+    ) -> TopHitsTopNComputer {
+        let mut top_hits_computer = TopHitsTopNComputer::new(self.req.clone());
+        let top_results = self.top_n.into_vec();
 
-impl std::fmt::Debug for SegmentTopHitsCollector {
-    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
-        f.debug_struct("SegmentTopHitsCollector")
-            .field("segment_id", &self.segment_ordinal)
-            .field("accessor_idx", &self.accessor_idx)
-            .field("inner_collector", &self.inner_collector)
-            .finish()
+        for res in top_results {
+            let doc_value_fields = self
+                .req
+                .get_document_field_data(value_accessors, res.doc.doc_id);
+            top_hits_computer.collect(
+                DocSortValuesAndFields {
+                    sorts: res.feature,
+                    doc_value_fields,
+                },
+                res.doc,
+            );
+        }
+
+        top_hits_computer
     }
 }
 
-impl SegmentAggregationCollector for SegmentTopHitsCollector {
+impl SegmentAggregationCollector for TopHitsSegmentCollector {
     fn add_intermediate_aggregation_result(
         self: Box<Self>,
         agg_with_accessor: &crate::aggregation::agg_req_with_accessor::AggregationsWithAccessor,
         results: &mut crate::aggregation::intermediate_agg_result::IntermediateAggregationResults,
     ) -> crate::Result<()> {
         let name = agg_with_accessor.aggs.keys[self.accessor_idx].to_string();
-        let intermediate_result = IntermediateMetricResult::TopHits(self.inner_collector);
+
+        let value_accessors = &agg_with_accessor.aggs.values[self.accessor_idx].value_accessors;
+
+        let intermediate_result =
+            IntermediateMetricResult::TopHits(self.into_top_hits_collector(value_accessors));
         results.push(
             name,
             IntermediateAggregationResult::Metric(intermediate_result),
@@ -490,9 +516,7 @@ impl SegmentAggregationCollector for SegmentTopHitsCollector {
         agg_with_accessor: &mut crate::aggregation::agg_req_with_accessor::AggregationsWithAccessor,
     ) -> crate::Result<()> {
         let accessors = &agg_with_accessor.aggs.values[self.accessor_idx].accessors;
-        let value_accessors = &agg_with_accessor.aggs.values[self.accessor_idx].value_accessors;
-        let features: Vec<ComparableDocFeature> = self
-            .inner_collector
+        let sorts: Vec<DocValueAndOrder> = self
             .req
             .sort
             .iter()
@@ -505,18 +529,12 @@ impl SegmentAggregationCollector for SegmentTopHitsCollector {
                     .0
                     .values_for_doc(doc_id)
                     .next();
-                ComparableDocFeature { value, order }
+                DocValueAndOrder { value, order }
             })
             .collect();
 
-        let retrieval_result = self
-            .inner_collector
-            .req
-            .retrieval
-            .get_document_field_data(value_accessors, doc_id);
-
-        self.inner_collector.collect(
-            ComparableDocFeatures(features, retrieval_result),
+        self.top_n.push(
+            sorts,
             DocAddress {
                 segment_ord: self.segment_ordinal,
                 doc_id,
@@ -530,11 +548,7 @@ impl SegmentAggregationCollector for SegmentTopHitsCollector {
         docs: &[crate::DocId],
         agg_with_accessor: &mut crate::aggregation::agg_req_with_accessor::AggregationsWithAccessor,
     ) -> crate::Result<()> {
-        // TODO: Consider getting fields with the column block accessor and refactor this.
-        // ---
-        // Would the additional complexity of getting fields with the column_block_accessor
-        // make sense here? Probably yes, but I want to get a first-pass review first
-        // before proceeding.
+        // TODO: Consider getting fields with the column block accessor.
         for doc in docs {
             self.collect(*doc, agg_with_accessor)?;
         }
@@ -549,7 +563,7 @@ mod tests {
     use serde_json::Value;
     use time::macros::datetime;
 
-    use super::{ComparableDocFeature, ComparableDocFeatures, Order};
+    use super::{DocSortValuesAndFields, DocValueAndOrder, Order};
     use crate::aggregation::agg_req::Aggregations;
     use crate::aggregation::agg_result::AggregationResults;
     use crate::aggregation::bucket::tests::get_test_index_from_docs;
@@ -557,44 +571,44 @@ mod tests {
     use crate::aggregation::AggregationCollector;
     use crate::collector::ComparableDoc;
     use crate::query::AllQuery;
-    use crate::schema::OwnedValue as SchemaValue;
+    use crate::schema::OwnedValue;
 
-    fn invert_order(cmp_feature: ComparableDocFeature) -> ComparableDocFeature {
-        let ComparableDocFeature { value, order } = cmp_feature;
+    fn invert_order(cmp_feature: DocValueAndOrder) -> DocValueAndOrder {
+        let DocValueAndOrder { value, order } = cmp_feature;
         let order = match order {
             Order::Asc => Order::Desc,
             Order::Desc => Order::Asc,
         };
-        ComparableDocFeature { value, order }
+        DocValueAndOrder { value, order }
     }
 
-    fn collector_with_capacity(capacity: usize) -> super::TopHitsCollector {
-        super::TopHitsCollector {
+    fn collector_with_capacity(capacity: usize) -> super::TopHitsTopNComputer {
+        super::TopHitsTopNComputer {
             top_n: super::TopNComputer::new(capacity),
-            ..Default::default()
+            req: Default::default(),
         }
     }
 
-    fn invert_order_features(cmp_features: ComparableDocFeatures) -> ComparableDocFeatures {
-        let ComparableDocFeatures(cmp_features, search_results) = cmp_features;
-        let cmp_features = cmp_features
+    fn invert_order_features(mut cmp_features: DocSortValuesAndFields) -> DocSortValuesAndFields {
+        cmp_features.sorts = cmp_features
+            .sorts
             .into_iter()
             .map(invert_order)
             .collect::<Vec<_>>();
-        ComparableDocFeatures(cmp_features, search_results)
+        cmp_features
     }
 
     #[test]
     fn test_comparable_doc_feature() -> crate::Result<()> {
-        let small = ComparableDocFeature {
+        let small = DocValueAndOrder {
             value: Some(1),
             order: Order::Asc,
         };
-        let big = ComparableDocFeature {
+        let big = DocValueAndOrder {
             value: Some(2),
             order: Order::Asc,
         };
-        let none = ComparableDocFeature {
+        let none = DocValueAndOrder {
             value: None,
             order: Order::Asc,
         };
@@ -616,21 +630,21 @@ mod tests {
 
     #[test]
     fn test_comparable_doc_features() -> crate::Result<()> {
-        let features_1 = ComparableDocFeatures(
-            vec![ComparableDocFeature {
+        let features_1 = DocSortValuesAndFields {
+            sorts: vec![DocValueAndOrder {
                 value: Some(1),
                 order: Order::Asc,
             }],
-            Default::default(),
-        );
+            doc_value_fields: Default::default(),
+        };
 
-        let features_2 = ComparableDocFeatures(
-            vec![ComparableDocFeature {
+        let features_2 = DocSortValuesAndFields {
+            sorts: vec![DocValueAndOrder {
                 value: Some(2),
                 order: Order::Asc,
             }],
-            Default::default(),
-        );
+            doc_value_fields: Default::default(),
+        };
 
         assert!(features_1 < features_2);
 
@@ -689,39 +703,39 @@ mod tests {
                     segment_ord: 0,
                     doc_id: 0,
                 },
-                feature: ComparableDocFeatures(
-                    vec![ComparableDocFeature {
+                feature: DocSortValuesAndFields {
+                    sorts: vec![DocValueAndOrder {
                         value: Some(1),
                         order: Order::Asc,
                     }],
-                    Default::default(),
-                ),
+                    doc_value_fields: Default::default(),
+                },
             },
             ComparableDoc {
                 doc: crate::DocAddress {
                     segment_ord: 0,
                     doc_id: 2,
                 },
-                feature: ComparableDocFeatures(
-                    vec![ComparableDocFeature {
+                feature: DocSortValuesAndFields {
+                    sorts: vec![DocValueAndOrder {
                         value: Some(3),
                         order: Order::Asc,
                     }],
-                    Default::default(),
-                ),
+                    doc_value_fields: Default::default(),
+                },
             },
             ComparableDoc {
                 doc: crate::DocAddress {
                     segment_ord: 0,
                     doc_id: 1,
                 },
-                feature: ComparableDocFeatures(
-                    vec![ComparableDocFeature {
+                feature: DocSortValuesAndFields {
+                    sorts: vec![DocValueAndOrder {
                         value: Some(5),
                         order: Order::Asc,
                     }],
-                    Default::default(),
-                ),
+                    doc_value_fields: Default::default(),
+                },
             },
         ];
 
@@ -730,23 +744,23 @@ mod tests {
             collector.collect(doc.feature, doc.doc);
         }
 
-        let res = collector.finalize();
+        let res = collector.into_final_result();
 
         assert_eq!(
             res,
             super::TopHitsMetricResult {
                 hits: vec![
                     super::TopHitsVecEntry {
-                        sort: vec![docs[0].feature.0[0].value],
-                        search_results: Default::default(),
+                        sort: vec![docs[0].feature.sorts[0].value],
+                        doc_value_fields: Default::default(),
                     },
                     super::TopHitsVecEntry {
-                        sort: vec![docs[1].feature.0[0].value],
-                        search_results: Default::default(),
+                        sort: vec![docs[1].feature.sorts[0].value],
+                        doc_value_fields: Default::default(),
                     },
                     super::TopHitsVecEntry {
-                        sort: vec![docs[2].feature.0[0].value],
-                        search_results: Default::default(),
+                        sort: vec![docs[2].feature.sorts[0].value],
+                        doc_value_fields: Default::default(),
                     },
                 ]
             }
@@ -803,7 +817,7 @@ mod tests {
                     {
                         "sort": [common::i64_to_u64(date_2017.unix_timestamp_nanos() as i64)],
                         "docvalue_fields": {
-                            "date": [ SchemaValue::Date(DateTime::from_utc(date_2017)) ],
+                            "date": [ OwnedValue::Date(DateTime::from_utc(date_2017)) ],
                             "text": [ "ccc" ],
                             "text2": [ "ddd" ],
                             "mixed.dyn_arr": [ 3, "4" ],
@@ -812,7 +826,7 @@ mod tests {
                     {
                         "sort": [common::i64_to_u64(date_2016.unix_timestamp_nanos() as i64)],
                         "docvalue_fields": {
-                            "date": [ SchemaValue::Date(DateTime::from_utc(date_2016)) ],
+                            "date": [ OwnedValue::Date(DateTime::from_utc(date_2016)) ],
                             "text": [ "aaa" ],
                             "text2": [ "bbb" ],
                             "mixed.dyn_arr": [ 6, "7" ],

src/aggregation/mod.rs

@@ -417,7 +417,6 @@ mod tests {
     use time::OffsetDateTime;
 
     use super::agg_req::Aggregations;
-    use super::segment_agg_result::AggregationLimits;
     use super::*;
     use crate::indexer::NoMergePolicy;
     use crate::query::{AllQuery, TermQuery};

src/aggregation/segment_agg_result.rs

@@ -16,7 +16,7 @@ use super::metric::{
     SumAggregation,
 };
 use crate::aggregation::bucket::TermMissingAgg;
-use crate::aggregation::metric::SegmentTopHitsCollector;
+use crate::aggregation::metric::TopHitsSegmentCollector;
 
 pub(crate) trait SegmentAggregationCollector: CollectorClone + Debug {
     fn add_intermediate_aggregation_result(
@@ -161,7 +161,7 @@ pub(crate) fn build_single_agg_segment_collector(
                 accessor_idx,
             )?,
         )),
-        TopHits(top_hits_req) => Ok(Box::new(SegmentTopHitsCollector::from_req(
+        TopHits(top_hits_req) => Ok(Box::new(TopHitsSegmentCollector::from_req(
             top_hits_req,
             accessor_idx,
             req.segment_ordinal,

src/collector/histogram_collector.rs

@@ -160,7 +160,7 @@ mod tests {
     use super::{add_vecs, HistogramCollector, HistogramComputer};
     use crate::schema::{Schema, FAST};
     use crate::time::{Date, Month};
-    use crate::{doc, query, DateTime, Index};
+    use crate::{query, DateTime, Index};
 
     #[test]
     fn test_add_histograms_simple() {

src/collector/tests.rs

@@ -1,15 +1,11 @@
 use columnar::{BytesColumn, Column};
 
 use super::*;
-use crate::collector::{Count, FilterCollector, TopDocs};
-use crate::index::SegmentReader;
 use crate::query::{AllQuery, QueryParser};
 use crate::schema::{Schema, FAST, TEXT};
 use crate::time::format_description::well_known::Rfc3339;
 use crate::time::OffsetDateTime;
-use crate::{
-    doc, DateTime, DocAddress, DocId, Index, Score, Searcher, SegmentOrdinal, TantivyDocument,
-};
+use crate::{DateTime, DocAddress, Index, Searcher, TantivyDocument};
 
 pub const TEST_COLLECTOR_WITH_SCORE: TestCollector = TestCollector {
     compute_score: true,

src/collector/top_score_collector.rs

@@ -732,6 +732,19 @@ pub struct TopNComputer<Score, D, const REVERSE_ORDER: bool = true> {
     top_n: usize,
     pub(crate) threshold: Option<Score>,
 }
+
+impl<Score: std::fmt::Debug, D, const REVERSE_ORDER: bool> std::fmt::Debug
+    for TopNComputer<Score, D, REVERSE_ORDER>
+{
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("TopNComputer")
+            .field("buffer_len", &self.buffer.len())
+            .field("top_n", &self.top_n)
+            .field("current_threshold", &self.threshold)
+            .finish()
+    }
+}
+
 // Intermediate struct for TopNComputer for deserialization, to keep vec capacity
 #[derive(Deserialize)]
 struct TopNComputerDeser<Score, D, const REVERSE_ORDER: bool> {

src/directory/composite_file.rs

@@ -1,6 +1,5 @@
 use std::collections::HashMap;
 use std::io::{self, Read, Write};
-use std::iter::ExactSizeIterator;
 use std::ops::Range;
 
 use common::{BinarySerializable, CountingWriter, HasLen, VInt};

src/directory/directory.rs

@@ -1,5 +1,4 @@
 use std::io::Write;
-use std::marker::{Send, Sync};
 use std::path::{Path, PathBuf};
 use std::sync::Arc;
 use std::time::Duration;
@@ -40,6 +39,7 @@ impl RetryPolicy {
 /// The `DirectoryLock` is an object that represents a file lock.
 ///
 /// It is associated with a lock file, that gets deleted on `Drop.`
+#[allow(dead_code)]
 pub struct DirectoryLock(Box<dyn Send + Sync + 'static>);
 
 struct DirectoryLockGuard {

src/directory/tests.rs

@@ -1,6 +1,6 @@
 use std::io::Write;
 use std::mem;
-use std::path::{Path, PathBuf};
+use std::path::Path;
 use std::sync::atomic::Ordering::SeqCst;
 use std::sync::atomic::{AtomicBool, AtomicUsize};
 use std::sync::Arc;

src/directory/watch_event_router.rs

@@ -32,6 +32,7 @@ pub struct WatchCallbackList {
 /// file change is detected.
 #[must_use = "This `WatchHandle` controls the lifetime of the watch and should therefore be used."]
 #[derive(Clone)]
+#[allow(dead_code)]
 pub struct WatchHandle(Arc<WatchCallback>);
 
 impl WatchHandle {

src/fastfield/mod.rs

@@ -79,7 +79,7 @@ mod tests {
     use std::ops::{Range, RangeInclusive};
     use std::path::Path;
 
-    use columnar::{Column, MonotonicallyMappableToU64, StrColumn};
+    use columnar::StrColumn;
     use common::{ByteCount, HasLen, TerminatingWrite};
     use once_cell::sync::Lazy;
     use rand::prelude::SliceRandom;

src/index/segment_id.rs

@@ -1,4 +1,4 @@
-use std::cmp::{Ord, Ordering};
+use std::cmp::Ordering;
 use std::error::Error;
 use std::fmt;
 use std::str::FromStr;

src/index/segment_reader.rs

@@ -516,8 +516,8 @@ impl fmt::Debug for SegmentReader {
 mod test {
     use super::*;
     use crate::index::Index;
-    use crate::schema::{Schema, SchemaBuilder, Term, STORED, TEXT};
-    use crate::{DocId, IndexWriter};
+    use crate::schema::{SchemaBuilder, Term, STORED, TEXT};
+    use crate::IndexWriter;
 
     #[test]
     fn test_merge_field_meta_data_same() {

src/indexer/flat_map_with_buffer.rs

@@ -22,6 +22,7 @@ where
     }
 }
 
+#[allow(dead_code)]
 pub trait FlatMapWithBufferIter: Iterator {
     /// Function similar to `flat_map`, but allows reusing a shared `Vec`.
     fn flat_map_with_buffer<F, T>(self, fill_buffer: F) -> FlatMapWithBuffer<T, F, Self>

src/indexer/log_merge_policy.rs

@@ -145,7 +145,6 @@ mod tests {
 
     use super::*;
     use crate::index::SegmentMetaInventory;
-    use crate::indexer::merge_policy::MergePolicy;
     use crate::schema::INDEXED;
     use crate::{schema, SegmentId};
 

src/indexer/merge_policy.rs

@@ -39,7 +39,6 @@ impl MergePolicy for NoMergePolicy {
 pub mod tests {
 
     use super::*;
-    use crate::index::{SegmentId, SegmentMeta};
 
     /// `MergePolicy` useful for test purposes.
     ///

src/indexer/merger.rs

@@ -576,7 +576,7 @@ impl IndexMerger {
                     //
                     // Overall the reliable way to know if we have actual frequencies loaded or not
                     // is to check whether the actual decoded array is empty or not.
-                    if has_term_freq != !postings.block_cursor.freqs().is_empty() {
+                    if has_term_freq == postings.block_cursor.freqs().is_empty() {
                         return Err(DataCorruption::comment_only(
                             "Term freqs are inconsistent across segments",
                         )

src/indexer/mod.rs

@@ -144,6 +144,115 @@ mod tests_mmap {
             assert_eq!(num_docs, 256);
         }
     }
+    #[test]
+    fn test_json_field_null_byte() {
+        // Test when field name contains a zero byte, which has special meaning in tantivy.
+        // As a workaround, we convert the zero byte to the ASCII character '0'.
+        // https://github.com/quickwit-oss/tantivy/issues/2340
+        // https://github.com/quickwit-oss/tantivy/issues/2193
+        let field_name_in = "\u{0000}";
+        let field_name_out = "0";
+        test_json_field_name(field_name_in, field_name_out);
+    }
+    #[test]
+    fn test_json_field_1byte() {
+        // Test when field name contains a 1 byte, which has special meaning in tantivy.
+        let field_name_in = "\u{0001}";
+        let field_name_out = "\u{0001}";
+        test_json_field_name(field_name_in, field_name_out);
+
+        // Test when field name contains a 1 byte, which has special meaning in tantivy.
+        let field_name_in = "\u{0001}";
+        let field_name_out = ".";
+        test_json_field_name(field_name_in, field_name_out);
+    }
+    fn test_json_field_name(field_name_in: &str, field_name_out: &str) {
+        let mut schema_builder = Schema::builder();
+
+        let options = JsonObjectOptions::from(TEXT | FAST).set_expand_dots_enabled();
+        let field = schema_builder.add_json_field("json", options);
+        let index = Index::create_in_ram(schema_builder.build());
+        let mut index_writer = index.writer_for_tests().unwrap();
+        index_writer
+            .add_document(doc!(field=>json!({format!("{field_name_in}"): "test1"})))
+            .unwrap();
+        index_writer
+            .add_document(doc!(field=>json!({format!("a{field_name_in}"): "test2"})))
+            .unwrap();
+        index_writer
+            .add_document(doc!(field=>json!({format!("a{field_name_in}a"): "test3"})))
+            .unwrap();
+        index_writer
+            .add_document(
+                doc!(field=>json!({format!("a{field_name_in}a{field_name_in}"): "test4"})),
+            )
+            .unwrap();
+        index_writer
+            .add_document(
+                doc!(field=>json!({format!("a{field_name_in}.ab{field_name_in}"): "test5"})),
+            )
+            .unwrap();
+        index_writer
+            .add_document(
+                doc!(field=>json!({format!("a{field_name_in}"): json!({format!("a{field_name_in}"): "test6"}) })),
+            )
+            .unwrap();
+        index_writer
+            .add_document(doc!(field=>json!({format!("{field_name_in}a" ): "test7"})))
+            .unwrap();
+
+        index_writer.commit().unwrap();
+        let reader = index.reader().unwrap();
+        let searcher = reader.searcher();
+        let parse_query = QueryParser::for_index(&index, Vec::new());
+        let test_query = |field_name: &str| {
+            let query = parse_query.parse_query(field_name).unwrap();
+            let num_docs = searcher.search(&query, &Count).unwrap();
+            assert_eq!(num_docs, 1);
+        };
+        test_query(format!("json.{field_name_out}:test1").as_str());
+        test_query(format!("json.a{field_name_out}:test2").as_str());
+        test_query(format!("json.a{field_name_out}a:test3").as_str());
+        test_query(format!("json.a{field_name_out}a{field_name_out}:test4").as_str());
+        test_query(format!("json.a{field_name_out}.ab{field_name_out}:test5").as_str());
+        test_query(format!("json.a{field_name_out}.a{field_name_out}:test6").as_str());
+        test_query(format!("json.{field_name_out}a:test7").as_str());
+
+        let test_agg = |field_name: &str, expected: &str| {
+            let agg_req_str = json!(
+            {
+              "termagg": {
+                "terms": {
+                  "field": field_name,
+                }
+              }
+            });
+
+            let agg_req: Aggregations = serde_json::from_value(agg_req_str).unwrap();
+            let collector = AggregationCollector::from_aggs(agg_req, Default::default());
+            let agg_res: AggregationResults = searcher.search(&AllQuery, &collector).unwrap();
+            let res = serde_json::to_value(agg_res).unwrap();
+            assert_eq!(res["termagg"]["buckets"][0]["doc_count"], 1);
+            assert_eq!(res["termagg"]["buckets"][0]["key"], expected);
+        };
+
+        test_agg(format!("json.{field_name_out}").as_str(), "test1");
+        test_agg(format!("json.a{field_name_out}").as_str(), "test2");
+        test_agg(format!("json.a{field_name_out}a").as_str(), "test3");
+        test_agg(
+            format!("json.a{field_name_out}a{field_name_out}").as_str(),
+            "test4",
+        );
+        test_agg(
+            format!("json.a{field_name_out}.ab{field_name_out}").as_str(),
+            "test5",
+        );
+        test_agg(
+            format!("json.a{field_name_out}.a{field_name_out}").as_str(),
+            "test6",
+        );
+        test_agg(format!("json.{field_name_out}a").as_str(), "test7");
+    }
 
     #[test]
     fn test_json_field_expand_dots_enabled_dot_escape_not_required() {

src/postings/compression/mod.rs

@@ -14,7 +14,6 @@ pub fn compressed_block_size(num_bits: u8) -> usize {
 pub struct BlockEncoder {
     bitpacker: BitPacker4x,
     pub output: [u8; COMPRESSED_BLOCK_MAX_SIZE],
-    pub output_len: usize,
 }
 
 impl Default for BlockEncoder {
@@ -28,7 +27,6 @@ impl BlockEncoder {
         BlockEncoder {
             bitpacker: BitPacker4x::new(),
             output: [0u8; COMPRESSED_BLOCK_MAX_SIZE],
-            output_len: 0,
         }
     }
 

src/postings/json_postings_writer.rs

@@ -67,10 +67,18 @@ impl<Rec: Recorder> PostingsWriter for JsonPostingsWriter<Rec> {
     ) -> io::Result<()> {
         let mut term_buffer = Term::with_capacity(48);
         let mut buffer_lender = BufferLender::default();
+        term_buffer.clear_with_field_and_type(Type::Json, Field::from_field_id(0));
+        let mut prev_term_id = u32::MAX;
+        let mut term_path_len = 0; // this will be set in the first iteration
         for (_field, path_id, term, addr) in term_addrs {
-            term_buffer.clear_with_field_and_type(Type::Json, Field::from_field_id(0));
-            term_buffer.append_bytes(ordered_id_to_path[path_id.path_id() as usize].as_bytes());
-            term_buffer.append_bytes(&[JSON_END_OF_PATH]);
+            if prev_term_id != path_id.path_id() {
+                term_buffer.truncate_value_bytes(0);
+                term_buffer.append_path(ordered_id_to_path[path_id.path_id() as usize].as_bytes());
+                term_buffer.append_bytes(&[JSON_END_OF_PATH]);
+                term_path_len = term_buffer.len_bytes();
+                prev_term_id = path_id.path_id();
+            }
+            term_buffer.truncate_value_bytes(term_path_len);
             term_buffer.append_bytes(term);
             if let Some(json_value) = term_buffer.value().as_json_value_bytes() {
                 let typ = json_value.typ();

src/postings/skip.rs

@@ -1,5 +1,3 @@
-use std::convert::TryInto;
-
 use crate::directory::OwnedBytes;
 use crate::postings::compression::{compressed_block_size, COMPRESSION_BLOCK_SIZE};
 use crate::query::Bm25Weight;

src/postings/term_info.rs

@@ -1,5 +1,4 @@
 use std::io;
-use std::iter::ExactSizeIterator;
 use std::ops::Range;
 
 use common::{BinarySerializable, FixedSize};

src/query/exist_query.rs

@@ -149,7 +149,7 @@ mod tests {
     use crate::query::exist_query::ExistsQuery;
     use crate::query::{BooleanQuery, RangeQuery};
     use crate::schema::{Facet, FacetOptions, Schema, FAST, INDEXED, STRING, TEXT};
-    use crate::{doc, Index, Searcher};
+    use crate::{Index, Searcher};
 
     #[test]
     fn test_exists_query_simple() -> crate::Result<()> {

src/query/fuzzy_query.rs

@@ -84,7 +84,7 @@ pub struct FuzzyTermQuery {
     distance: u8,
     /// Should a transposition cost 1 or 2?
     transposition_cost_one: bool,
-    ///
+    /// is a starts with query
     prefix: bool,
 }
 

src/query/range_query/range_query.rs

@@ -477,7 +477,7 @@ mod tests {
     use crate::schema::{
         Field, IntoIpv6Addr, Schema, TantivyDocument, FAST, INDEXED, STORED, TEXT,
     };
-    use crate::{doc, Index, IndexWriter};
+    use crate::{Index, IndexWriter};
 
     #[test]
     fn test_range_query_simple() -> crate::Result<()> {

src/query/term_query/term_query.rs

@@ -139,7 +139,7 @@ mod tests {
     use crate::collector::{Count, TopDocs};
     use crate::query::{Query, QueryParser, TermQuery};
     use crate::schema::{IndexRecordOption, IntoIpv6Addr, Schema, INDEXED, STORED};
-    use crate::{doc, Index, IndexWriter, Term};
+    use crate::{Index, IndexWriter, Term};
 
     #[test]
     fn search_ip_test() {

src/query/vec_docset.rs

@@ -53,8 +53,7 @@ impl HasLen for VecDocSet {
 pub mod tests {
 
     use super::*;
-    use crate::docset::{DocSet, COLLECT_BLOCK_BUFFER_LEN};
-    use crate::DocId;
+    use crate::docset::COLLECT_BLOCK_BUFFER_LEN;
 
     #[test]
     pub fn test_vec_postings() {

src/schema/document/de.rs

@@ -819,7 +819,6 @@ mod tests {
     use crate::schema::document::existing_type_impls::JsonObjectIter;
     use crate::schema::document::se::BinaryValueSerializer;
     use crate::schema::document::{ReferenceValue, ReferenceValueLeaf};
-    use crate::schema::OwnedValue;
 
     fn serialize_value<'a>(value: ReferenceValue<'a, &'a serde_json::Value>) -> Vec<u8> {
         let mut writer = Vec::new();

src/schema/document/default_document.rs

@@ -1,15 +1,16 @@
 use std::collections::{BTreeMap, HashMap, HashSet};
 use std::net::Ipv6Addr;
 
-use common::DateTime;
+use common::{BinarySerializable, DateTime, VInt};
 use serde_json::Map;
 
+use crate::schema::document::se::BinaryValueSerializer;
 use crate::schema::document::{
-    DeserializeError, Document, DocumentDeserialize, DocumentDeserializer,
+    BinaryDocumentDeserializer, BinaryDocumentSerializer, DeserializeError, Document, DocumentDeserialize, DocumentDeserializer, ReferenceValue, ReferenceValueLeaf
 };
 use crate::schema::field_type::ValueParsingError;
 use crate::schema::field_value::FieldValueIter;
-use crate::schema::{Facet, Field, FieldValue, NamedFieldDocument, OwnedValue, Schema};
+use crate::schema::{Facet, Field, FieldValue, NamedFieldDocument, OwnedValue, Schema, Value};
 use crate::tokenizer::PreTokenizedString;
 
 /// TantivyDocument provides a default implementation of the `Document` trait.
@@ -85,6 +86,36 @@ impl IntoIterator for TantivyDocument {
 }
 
 impl TantivyDocument {
+
+    pub fn to_bytes(&self, buffer: &mut Vec<u8>) -> std::io::Result<()> {
+        buffer.clear();
+        let num_field_values = self.field_values.len();
+
+        VInt(num_field_values as u64).serialize(buffer)?;
+        for (field, value_access) in self.iter_fields_and_values() {
+            field.serialize(buffer)?;
+            let mut serializer = BinaryValueSerializer::new(buffer);
+            match value_access.as_value() {
+                ReferenceValue::Leaf(ReferenceValueLeaf::PreTokStr(pre_tokenized_text)) => {
+                    serializer.serialize_value(ReferenceValue::Leaf::<&'_ OwnedValue>(
+                        ReferenceValueLeaf::Str(&pre_tokenized_text.text),
+                    ))?;
+                }
+                _ => {
+                    serializer.serialize_value(value_access.as_value())?;
+                }
+            }
+        }
+        Ok(())
+    }
+
+
+    pub fn from_bytes(mut payload: &[u8]) -> Self {
+        let deserializer = BinaryDocumentDeserializer::from_reader(&mut payload).unwrap();
+        Self::deserialize(deserializer).unwrap()
+
+    }
+
     /// Creates a new, empty document object
     pub fn new() -> TantivyDocument {
         TantivyDocument::default()
@@ -256,7 +287,6 @@ impl DocParsingError {
 
 #[cfg(test)]
 mod tests {
-    use crate::schema::document::default_document::TantivyDocument;
     use crate::schema::*;
 
     #[test]

src/schema/document/owned_value.rs

@@ -1,4 +1,4 @@
-use std::collections::{btree_map, BTreeMap};
+use std::collections::BTreeMap;
 use std::fmt;
 use std::net::Ipv6Addr;
 
@@ -45,7 +45,7 @@ pub enum OwnedValue {
     /// A set of values.
     Array(Vec<Self>),
     /// Dynamic object value.
-    Object(BTreeMap<String, Self>),
+    Object(Vec<(String, Self)>),
     /// IpV6 Address. Internally there is no IpV4, it needs to be converted to `Ipv6Addr`.
     IpAddr(Ipv6Addr),
 }
@@ -148,10 +148,10 @@ impl ValueDeserialize for OwnedValue {
 
             fn visit_object<'de, A>(&self, mut access: A) -> Result<Self::Value, DeserializeError>
             where A: ObjectAccess<'de> {
-                let mut elements = BTreeMap::new();
+                let mut elements = Vec::new();
 
                 while let Some((key, value)) = access.next_entry()? {
-                    elements.insert(key, value);
+                    elements.push((key, value));
                 }
 
                 Ok(OwnedValue::Object(elements))
@@ -248,12 +248,13 @@ impl<'de> serde::Deserialize<'de> for OwnedValue {
 
             fn visit_map<A>(self, mut map: A) -> Result<Self::Value, A::Error>
             where A: MapAccess<'de> {
-                let mut object = BTreeMap::new();
-
+                let mut object =
+                    map.size_hint()
+                       .map(Vec::with_capacity)
+                       .unwrap_or_default();
                 while let Some((key, value)) = map.next_entry()? {
-                    object.insert(key, value);
+                    object.push((key, value));
                 }
-
                 Ok(OwnedValue::Object(object))
             }
         }
@@ -363,7 +364,8 @@ impl From<PreTokenizedString> for OwnedValue {
 
 impl From<BTreeMap<String, OwnedValue>> for OwnedValue {
     fn from(object: BTreeMap<String, OwnedValue>) -> OwnedValue {
-        OwnedValue::Object(object)
+        let key_values = object.into_iter().collect();
+        OwnedValue::Object(key_values)
     }
 }
 
@@ -417,18 +419,15 @@ impl From<serde_json::Value> for OwnedValue {
 
 impl From<serde_json::Map<String, serde_json::Value>> for OwnedValue {
     fn from(map: serde_json::Map<String, serde_json::Value>) -> Self {
-        let mut object = BTreeMap::new();
-
-        for (key, value) in map {
-            object.insert(key, OwnedValue::from(value));
-        }
-
+        let object: Vec<(String, OwnedValue)> = map.into_iter()
+            .map(|(key, value)| (key, OwnedValue::from(value)))
+            .collect();
         OwnedValue::Object(object)
     }
 }
 
 /// A wrapper type for iterating over a serde_json object producing reference values.
-pub struct ObjectMapIter<'a>(btree_map::Iter<'a, String, OwnedValue>);
+pub struct ObjectMapIter<'a>(std::slice::Iter<'a, (String, OwnedValue)>);
 
 impl<'a> Iterator for ObjectMapIter<'a> {
     type Item = (&'a str, &'a OwnedValue);
@@ -443,9 +442,7 @@ impl<'a> Iterator for ObjectMapIter<'a> {
 mod tests {
     use super::*;
     use crate::schema::{BytesOptions, Schema};
-    use crate::time::format_description::well_known::Rfc3339;
-    use crate::time::OffsetDateTime;
-    use crate::{DateTime, Document, TantivyDocument};
+    use crate::{Document, TantivyDocument};
 
     #[test]
     fn test_parse_bytes_doc() {

src/schema/field_entry.rs

@@ -136,7 +136,6 @@ impl FieldEntry {
 
 #[cfg(test)]
 mod tests {
-    use serde_json;
 
     use super::*;
     use crate::schema::{Schema, TextFieldIndexing, TEXT};

src/schema/schema.rs

@@ -6,10 +6,8 @@ use serde::de::{SeqAccess, Visitor};
 use serde::ser::SerializeSeq;
 use serde::{Deserialize, Deserializer, Serialize, Serializer};
 
-use super::ip_options::IpAddrOptions;
 use super::*;
 use crate::json_utils::split_json_path;
-use crate::schema::bytes_options::BytesOptions;
 use crate::TantivyError;
 
 /// Tantivy has a very strict schema.
@@ -421,9 +419,7 @@ mod tests {
 
     use matches::{assert_matches, matches};
     use pretty_assertions::assert_eq;
-    use serde_json;
 
-    use crate::schema::document::Value;
     use crate::schema::field_type::ValueParsingError;
     use crate::schema::schema::DocParsingError::InvalidJson;
     use crate::schema::*;

src/schema/term.rs

@@ -1,4 +1,3 @@
-use std::convert::TryInto;
 use std::hash::{Hash, Hasher};
 use std::net::Ipv6Addr;
 use std::{fmt, str};
@@ -218,6 +217,23 @@ impl Term {
         &mut self.0[len_before..]
     }
 
+    /// Appends json path bytes to the Term.
+    /// If the path contains 0 bytes, they are replaced by a "0" string.
+    /// The 0 byte is used to mark the end of the path.
+    ///
+    /// This function returns the segment that has just been added.
+    #[inline]
+    pub fn append_path(&mut self, bytes: &[u8]) -> &mut [u8] {
+        let len_before = self.0.len();
+        if bytes.contains(&0u8) {
+            self.0
+                .extend(bytes.iter().map(|&b| if b == 0 { b'0' } else { b }));
+        } else {
+            self.0.extend_from_slice(bytes);
+        }
+        &mut self.0[len_before..]
+    }
+
     /// Appends a JSON_PATH_SEGMENT_SEP to the term.
     /// Only used for JSON type.
     #[inline]

src/store/compression_lz4_block.rs

@@ -1,4 +1,3 @@
-use core::convert::TryInto;
 use std::io::{self};
 use std::mem;
 

src/store/compressors.rs

@@ -2,12 +2,6 @@ use std::io;
 
 use serde::{Deserialize, Deserializer, Serialize};
 
-pub trait StoreCompressor {
-    fn compress(&self, uncompressed: &[u8], compressed: &mut Vec<u8>) -> io::Result<()>;
-    fn decompress(&self, compressed: &[u8], decompressed: &mut Vec<u8>) -> io::Result<()>;
-    fn get_compressor_id() -> u8;
-}
-
 /// Compressor can be used on `IndexSettings` to choose
 /// the compressor used to compress the doc store.
 ///

src/store/decompressors.rs

@@ -4,12 +4,6 @@ use serde::{Deserialize, Serialize};
 
 use super::Compressor;
 
-pub trait StoreCompressor {
-    fn compress(&self, uncompressed: &[u8], compressed: &mut Vec<u8>) -> io::Result<()>;
-    fn decompress(&self, compressed: &[u8], decompressed: &mut Vec<u8>) -> io::Result<()>;
-    fn get_compressor_id() -> u8;
-}
-
 /// Decompressor is deserialized from the doc store footer, when opening an index.
 #[derive(Clone, Debug, Copy, PartialEq, Eq, Serialize, Deserialize)]
 pub enum Decompressor {
@@ -86,7 +80,6 @@ impl Decompressor {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::store::Compressor;
 
     #[test]
     fn compressor_decompressor_id_test() {

src/store/index/mod.rs

@@ -41,7 +41,7 @@ mod tests {
 
     use std::io;
 
-    use proptest::strategy::{BoxedStrategy, Strategy};
+    use proptest::prelude::*;
 
     use super::{SkipIndex, SkipIndexBuilder};
     use crate::directory::OwnedBytes;
@@ -227,8 +227,6 @@ mod tests {
         }
     }
 
-    use proptest::prelude::*;
-
     proptest! {
         #![proptest_config(ProptestConfig::with_cases(20))]
         #[test]

sstable/benches/ord_to_term.rs

@@ -3,7 +3,7 @@ use std::sync::Arc;
 use common::file_slice::FileSlice;
 use common::OwnedBytes;
 use criterion::{criterion_group, criterion_main, Criterion};
-use tantivy_sstable::{self, Dictionary, MonotonicU64SSTable};
+use tantivy_sstable::{Dictionary, MonotonicU64SSTable};
 
 fn make_test_sstable(suffix: &str) -> FileSlice {
     let mut builder = Dictionary::<MonotonicU64SSTable>::builder(Vec::new()).unwrap();

sstable/benches/stream_bench.rs

@@ -5,7 +5,7 @@ use common::file_slice::FileSlice;
 use criterion::{criterion_group, criterion_main, Criterion};
 use rand::rngs::StdRng;
 use rand::{Rng, SeedableRng};
-use tantivy_sstable::{self, Dictionary, MonotonicU64SSTable};
+use tantivy_sstable::{Dictionary, MonotonicU64SSTable};
 
 const CHARSET: &'static [u8] = b"abcdefghij";
 

stacker/Cargo.toml

@@ -10,7 +10,7 @@ description = "term hashmap used for indexing"
 [dependencies]
 murmurhash32 = "0.3"
 common = { version = "0.6", path = "../common/", package = "tantivy-common" }
-ahash = { version = "0.8.3", default-features = false, optional = true }
+ahash = { version = "0.8.11", default-features = false, optional = true }
 rand_distr = "0.4.3"
 
 [[bench]]

stacker/src/expull.rs

@@ -151,7 +151,6 @@ impl ExpUnrolledLinkedList {
 mod tests {
     use common::{read_u32_vint, write_u32_vint};
 
-    use super::super::MemoryArena;
     use super::*;
 
     #[test]