tantivy document memory test

* Owned value relying on Vec instead of BTreeMap

* fmt

* fix build

* fix serialization

---------

Co-authored-by: Pascal Seitz <pascal.seitz@gmail.com>

.github/workflows/coverage.yml

@@ -15,11 +15,11 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - name: Install Rust
-        run: rustup toolchain install nightly-2023-09-10 --profile minimal --component llvm-tools-preview
+        run: rustup toolchain install nightly-2024-04-10 --profile minimal --component llvm-tools-preview
       - uses: Swatinem/rust-cache@v2
       - uses: taiki-e/install-action@cargo-llvm-cov
       - name: Generate code coverage
-        run: cargo +nightly-2023-09-10 llvm-cov --all-features --workspace --doctests --lcov --output-path lcov.info
+        run: cargo +nightly-2024-04-10 llvm-cov --all-features --workspace --doctests --lcov --output-path lcov.info
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v3
         continue-on-error: true

Cargo.toml

@@ -62,6 +62,7 @@ tokenizer-api = { version= "0.3", path="./tokenizer-api", package="tantivy-token
 sketches-ddsketch = { version = "0.2.1", features = ["use_serde"] }
 futures-util = { version = "0.3.28", optional = true }
 fnv = "1.0.7"
+mediumvec = "1.3.0"
 
 [target.'cfg(windows)'.dependencies]
 winapi = "0.3.9"
@@ -81,6 +82,7 @@ time = { version = "0.3.10", features = ["serde-well-known", "macros"] }
 postcard = { version = "1.0.4", features = [
   "use-std",
 ], default-features = false }
+peakmem-alloc = "0.3.0"
 
 [target.'cfg(not(windows))'.dev-dependencies]
 criterion = { version = "0.5", default-features = false }

examples/doc_mem.rs

@@ -0,0 +1,335 @@
+#![allow(unused_imports)]
+#![allow(dead_code)]
+use std::alloc::System;
+use std::env::args;
+use std::net::Ipv6Addr;
+
+use columnar::{MonotonicallyMappableToU128, MonotonicallyMappableToU64};
+use common::{BinarySerializable, CountingWriter, DateTime, FixedSize};
+use peakmem_alloc::*;
+use tantivy::schema::{Field, FieldValue, OwnedValue, FAST, INDEXED, STRING, TEXT};
+use tantivy::tokenizer::PreTokenizedString;
+use tantivy::{doc, TantivyDocument};
+
+const GH_LOGS: &str = include_str!("../benches/gh.json");
+const HDFS_LOGS: &str = include_str!("../benches/hdfs.json");
+
+#[global_allocator]
+static GLOBAL: &PeakMemAlloc<System> = &INSTRUMENTED_SYSTEM;
+
+fn main() {
+    dbg!(std::mem::size_of::<TantivyDocument>());
+    dbg!(std::mem::size_of::<DocContainerRef>());
+    dbg!(std::mem::size_of::<OwnedValue>());
+    dbg!(std::mem::size_of::<OwnedValueMedVec>());
+    dbg!(std::mem::size_of::<ValueContainerRef>());
+    dbg!(std::mem::size_of::<mediumvec::vec32::Vec32::<u8>>());
+
+    let filter = args().nth(1);
+    measure_fn(
+        test_hdfs::<TantivyDocument>,
+        "hdfs TantivyDocument",
+        &filter,
+    );
+    measure_fn(
+        test_hdfs::<TantivyDocumentMedVec>,
+        "hdfs TantivyDocumentMedVec",
+        &filter,
+    );
+    measure_fn(
+        test_hdfs::<DocContainerRef>,
+        "hdfs DocContainerRef",
+        &filter,
+    );
+    measure_fn(test_gh::<TantivyDocument>, "gh TantivyDocument", &filter);
+    measure_fn(
+        test_gh::<TantivyDocumentMedVec>,
+        "gh TantivyDocumentMedVec",
+        &filter,
+    );
+    measure_fn(test_gh::<DocContainerRef>, "gh DocContainerRef", &filter);
+}
+fn measure_fn<F: FnOnce()>(f: F, name: &str, filter: &Option<std::string::String>) {
+    if let Some(filter) = filter {
+        if !name.contains(filter) {
+            return;
+        }
+    }
+    GLOBAL.reset_peak_memory();
+    f();
+    println!("Peak Memory {} : {:#?}", GLOBAL.get_peak_memory(), name);
+}
+fn test_hdfs<T: From<TantivyDocument>>() {
+    let schema = {
+        let mut schema_builder = tantivy::schema::SchemaBuilder::new();
+        schema_builder.add_u64_field("timestamp", INDEXED);
+        schema_builder.add_text_field("body", TEXT);
+        schema_builder.add_text_field("severity", STRING);
+        schema_builder.build()
+    };
+    let mut docs: Vec<T> = Vec::with_capacity(HDFS_LOGS.lines().count());
+    for doc_json in HDFS_LOGS.lines() {
+        let doc = TantivyDocument::parse_json(&schema, doc_json)
+            .unwrap()
+            .into();
+        docs.push(doc);
+    }
+}
+
+fn test_gh<T: From<TantivyDocument>>() {
+    let schema = {
+        let mut schema_builder = tantivy::schema::SchemaBuilder::new();
+        schema_builder.add_json_field("json", FAST);
+        schema_builder.build()
+    };
+    let mut docs: Vec<T> = Vec::with_capacity(GH_LOGS.lines().count());
+    for doc_json in GH_LOGS.lines() {
+        let json_field = schema.get_field("json").unwrap();
+
+        let json_val: serde_json::Map<String, serde_json::Value> =
+            serde_json::from_str(doc_json).unwrap();
+        let doc = tantivy::doc!(json_field=>json_val).into();
+        docs.push(doc);
+    }
+}
+
+#[derive(Clone, Debug, Default)]
+#[allow(dead_code)]
+pub struct TantivyDocumentMedVec {
+    field_values: mediumvec::Vec32<FieldValueMedVec>,
+}
+
+#[derive(Debug, Clone, PartialEq)]
+pub struct FieldValueMedVec {
+    pub field: Field,
+    pub value: OwnedValueMedVec,
+}
+
+/// This is a owned variant of `Value`, that can be passed around without lifetimes.
+/// Represents the value of a any field.
+/// It is an enum over all over all of the possible field type.
+#[derive(Debug, Clone, PartialEq)]
+pub enum OwnedValueMedVec {
+    /// A null value.
+    Null,
+    /// The str type is used for any text information.
+    Str(mediumvec::vec32::Vec32<u8>),
+    /// Unsigned 64-bits Integer `u64`
+    U64(u64),
+    /// Signed 64-bits Integer `i64`
+    I64(i64),
+    /// 64-bits Float `f64`
+    F64(f64),
+    /// Bool value
+    Bool(bool),
+    /// Date/time with nanoseconds precision
+    Date(DateTime),
+    Array(mediumvec::vec32::Vec32<Self>),
+    /// Dynamic object value.
+    Object(mediumvec::vec32::Vec32<(String, Self)>),
+    /// IpV6 Address. Internally there is no IpV4, it needs to be converted to `Ipv6Addr`.
+    IpAddr(Ipv6Addr),
+    /// Pre-tokenized str type,
+    PreTokStr(Box<PreTokenizedString>),
+    /// Arbitrarily sized byte array
+    Bytes(mediumvec::vec32::Vec32<u8>),
+}
+
+impl From<TantivyDocument> for TantivyDocumentMedVec {
+    fn from(doc: TantivyDocument) -> Self {
+        let field_values = doc
+            .into_iter()
+            .map(|fv| FieldValueMedVec {
+                field: fv.field,
+                value: fv.value.into(),
+            })
+            .collect();
+        TantivyDocumentMedVec { field_values }
+    }
+}
+impl From<OwnedValue> for OwnedValueMedVec {
+    fn from(value: OwnedValue) -> Self {
+        match value {
+            OwnedValue::Null => OwnedValueMedVec::Null,
+            OwnedValue::Str(s) => {
+                let bytes = s.into_bytes();
+                let vec = mediumvec::vec32::Vec32::from_vec(bytes);
+                OwnedValueMedVec::Str(vec)
+            }
+            OwnedValue::U64(u) => OwnedValueMedVec::U64(u),
+            OwnedValue::I64(i) => OwnedValueMedVec::I64(i),
+            OwnedValue::F64(f) => OwnedValueMedVec::F64(f),
+            OwnedValue::Bool(b) => OwnedValueMedVec::Bool(b),
+            OwnedValue::Date(d) => OwnedValueMedVec::Date(d),
+            OwnedValue::Array(arr) => {
+                let arr = arr.into_iter().map(|v| v.into()).collect();
+                OwnedValueMedVec::Array(arr)
+            }
+            OwnedValue::Object(obj) => {
+                let obj = obj.into_iter().map(|(k, v)| (k, v.into())).collect();
+                OwnedValueMedVec::Object(obj)
+            }
+            OwnedValue::IpAddr(ip) => OwnedValueMedVec::IpAddr(ip),
+            _ => panic!("Unsupported value type {:?}", value),
+        }
+    }
+}
+
+#[repr(packed)]
+pub struct FieldValueContainerRef {
+    pub field: u16,
+    pub value: ValueContainerRef,
+}
+
+#[repr(packed)]
+struct DocContainerRef {
+    container: OwnedValueRefContainer,
+    field_values: mediumvec::Vec32<FieldValueContainerRef>,
+}
+
+#[derive(Default)]
+struct OwnedValueRefContainer {
+    nodes: mediumvec::Vec32<ValueContainerRef>,
+    node_data: mediumvec::Vec32<u8>,
+}
+impl OwnedValueRefContainer {
+    fn shrink_to_fit(&mut self) {
+        self.nodes.shrink_to_fit();
+        self.node_data.shrink_to_fit();
+    }
+}
+
+impl From<TantivyDocument> for DocContainerRef {
+    fn from(doc: TantivyDocument) -> Self {
+        let mut container = OwnedValueRefContainer::default();
+        let field_values = doc
+            .into_iter()
+            .map(|fv| FieldValueContainerRef {
+                field: fv.field.field_id().try_into().unwrap(),
+                value: container.add_value(fv.value),
+            })
+            .collect();
+        container.shrink_to_fit();
+        Self {
+            field_values,
+            container,
+        }
+    }
+}
+
+// References to positions in two array, one for the OwnedValueRef and the other for the encoded
+// bytes
+#[derive(Debug, Clone, PartialEq)]
+pub enum ValueContainerRef {
+    /// A null value.
+    Null,
+    /// The str type is used for any text information.
+    Str(u32),
+    /// Unsigned 64-bits Integer `u64`
+    U64(u32), // position of the serialized 8 bytes in the data array
+    /// Signed 64-bits Integer `i64`
+    I64(u32), // position of the serialized 8 bytes in the data array
+    /// 64-bits Float `f64`
+    F64(u32), // position of the serialized 8 bytes in the data array
+    /// Bool value
+    Bool(bool), // inlined bool
+    /// Date/time with nanoseconds precision
+    Date(u32), // position of the serialized 8 byte in the data array
+    Array(NodeAddress),
+    /// Dynamic object value.
+    Object(NodeAddress),
+    /// IpV6 Address. Internally there is no IpV4, it needs to be converted to `Ipv6Addr`.
+    IpAddr(u32), // position of the serialized 16 bytes in the data array
+    /// Arbitrarily sized byte array
+    Bytes(u32),
+}
+
+#[derive(Debug, Clone, PartialEq)]
+pub struct NodeAddress {
+    pos: u32,
+    num_nodes: u32,
+}
+
+impl OwnedValueRefContainer {
+    pub fn add_value(&mut self, value: OwnedValue) -> ValueContainerRef {
+        match value {
+            OwnedValue::Null => ValueContainerRef::Null,
+            OwnedValue::U64(num) => ValueContainerRef::U64(write_into(&mut self.node_data, num)),
+            OwnedValue::I64(num) => ValueContainerRef::I64(write_into(&mut self.node_data, num)),
+            OwnedValue::F64(num) => ValueContainerRef::F64(write_into(&mut self.node_data, num)),
+            OwnedValue::Bool(b) => ValueContainerRef::Bool(b),
+            OwnedValue::Date(date) => ValueContainerRef::Date(write_into(
+                &mut self.node_data,
+                date.into_timestamp_nanos(),
+            )),
+            OwnedValue::Str(bytes) => {
+                ValueContainerRef::Str(write_into(&mut self.node_data, bytes))
+            }
+            OwnedValue::Bytes(bytes) => {
+                ValueContainerRef::Bytes(write_into(&mut self.node_data, bytes))
+            }
+            OwnedValue::Array(elements) => {
+                let pos = self.nodes.len() as u32;
+                let len = elements.len() as u32;
+                for elem in elements {
+                    let ref_elem = self.add_value(elem);
+                    self.nodes.push(ref_elem);
+                }
+                ValueContainerRef::Array(NodeAddress {
+                    pos,
+                    num_nodes: len,
+                })
+            }
+            OwnedValue::Object(entries) => {
+                let pos = self.nodes.len() as u32;
+                let len = entries.len() as u32;
+                for (key, value) in entries {
+                    let ref_key = self.add_value(OwnedValue::Str(key));
+                    let ref_value = self.add_value(value);
+                    self.nodes.push(ref_key);
+                    self.nodes.push(ref_value);
+                }
+                ValueContainerRef::Object(NodeAddress {
+                    pos,
+                    num_nodes: len,
+                })
+            }
+            OwnedValue::IpAddr(num) => {
+                ValueContainerRef::IpAddr(write_into(&mut self.node_data, num.to_u128()))
+            }
+            OwnedValue::PreTokStr(_) => todo!(),
+            OwnedValue::Facet(_) => todo!(),
+        }
+    }
+}
+
+fn write_into<T: BinarySerializable>(data: &mut mediumvec::Vec32<u8>, value: T) -> u32 {
+    let pos = data.len() as u32;
+    data.as_vec(|vec| value.serialize(vec).unwrap());
+    pos
+}
+
+fn write_into_2<T: BinarySerializable>(data: &mut mediumvec::Vec32<u8>, value: T) -> NodeAddress {
+    let pos = data.len() as u32;
+    let mut len = 0;
+    data.as_vec(|vec| {
+        let mut wrt = CountingWriter::wrap(vec);
+        value.serialize(&mut wrt).unwrap();
+        len = wrt.written_bytes() as u32;
+    });
+    NodeAddress {
+        pos,
+        num_nodes: len,
+    }
+}
+
+// impl From<ContainerDocRef> for TantivyDocument {
+// fn from(doc: ContainerDocRef) -> Self {
+// let mut doc2 = TantivyDocument::new();
+// for fv in doc.field_values {
+// let field = Field::from_field_id(fv.field as u32);
+// let value = doc.container.get_value(fv.value);
+// doc2.add(FieldValue::new(field, value));
+//}
+// doc2
+//}

src/core/json_utils.rs

@@ -4,7 +4,7 @@ use rustc_hash::FxHashMap;
 
 use crate::postings::{IndexingContext, IndexingPosition, PostingsWriter};
 use crate::schema::document::{ReferenceValue, ReferenceValueLeaf, Value};
-use crate::schema::{Field, Type};
+use crate::schema::Type;
 use crate::time::format_description::well_known::Rfc3339;
 use crate::time::{OffsetDateTime, UtcOffset};
 use crate::tokenizer::TextAnalyzer;
@@ -349,44 +349,24 @@ pub(crate) fn encode_column_name(
     path.into()
 }
 
-pub fn term_from_json_paths<'a>(
-    json_field: Field,
-    paths: impl Iterator<Item = &'a str>,
-    expand_dots_enabled: bool,
-) -> Term {
-    let mut json_path = JsonPathWriter::with_expand_dots(expand_dots_enabled);
-    for path in paths {
-        json_path.push(path);
-    }
-    json_path.set_end();
-    let mut term = Term::with_type_and_field(Type::Json, json_field);
-
-    term.append_bytes(json_path.as_str().as_bytes());
-    term
-}
-
 #[cfg(test)]
 mod tests {
     use super::split_json_path;
-    use crate::json_utils::term_from_json_paths;
     use crate::schema::Field;
+    use crate::Term;
 
     #[test]
     fn test_json_writer() {
         let field = Field::from_field_id(1);
 
-        let mut term = term_from_json_paths(field, ["attributes", "color"].into_iter(), false);
+        let mut term = Term::from_field_json_path(field, "attributes.color", false);
         term.append_type_and_str("red");
         assert_eq!(
             format!("{:?}", term),
             "Term(field=1, type=Json, path=attributes.color, type=Str, \"red\")"
         );
 
-        let mut term = term_from_json_paths(
-            field,
-            ["attributes", "dimensions", "width"].into_iter(),
-            false,
-        );
+        let mut term = Term::from_field_json_path(field, "attributes.dimensions.width", false);
         term.append_type_and_fast_value(400i64);
         assert_eq!(
             format!("{:?}", term),
@@ -397,7 +377,7 @@ mod tests {
     #[test]
     fn test_string_term() {
         let field = Field::from_field_id(1);
-        let mut term = term_from_json_paths(field, ["color"].into_iter(), false);
+        let mut term = Term::from_field_json_path(field, "color", false);
         term.append_type_and_str("red");
 
         assert_eq!(term.serialized_term(), b"\x00\x00\x00\x01jcolor\x00sred")
@@ -406,7 +386,7 @@ mod tests {
     #[test]
     fn test_i64_term() {
         let field = Field::from_field_id(1);
-        let mut term = term_from_json_paths(field, ["color"].into_iter(), false);
+        let mut term = Term::from_field_json_path(field, "color", false);
         term.append_type_and_fast_value(-4i64);
 
         assert_eq!(
@@ -418,7 +398,7 @@ mod tests {
     #[test]
     fn test_u64_term() {
         let field = Field::from_field_id(1);
-        let mut term = term_from_json_paths(field, ["color"].into_iter(), false);
+        let mut term = Term::from_field_json_path(field, "color", false);
         term.append_type_and_fast_value(4u64);
 
         assert_eq!(
@@ -430,7 +410,7 @@ mod tests {
     #[test]
     fn test_f64_term() {
         let field = Field::from_field_id(1);
-        let mut term = term_from_json_paths(field, ["color"].into_iter(), false);
+        let mut term = Term::from_field_json_path(field, "color", false);
         term.append_type_and_fast_value(4.0f64);
         assert_eq!(
             term.serialized_term(),
@@ -441,7 +421,7 @@ mod tests {
     #[test]
     fn test_bool_term() {
         let field = Field::from_field_id(1);
-        let mut term = term_from_json_paths(field, ["color"].into_iter(), false);
+        let mut term = Term::from_field_json_path(field, "color", false);
         term.append_type_and_fast_value(true);
         assert_eq!(
             term.serialized_term(),

src/core/tests.rs

@@ -1,7 +1,6 @@
 use crate::collector::Count;
 use crate::directory::{RamDirectory, WatchCallback};
 use crate::indexer::{LogMergePolicy, NoMergePolicy};
-use crate::json_utils::term_from_json_paths;
 use crate::query::TermQuery;
 use crate::schema::{Field, IndexRecordOption, Schema, INDEXED, STRING, TEXT};
 use crate::tokenizer::TokenizerManager;
@@ -417,7 +416,7 @@ fn test_non_text_json_term_freq() {
     let segment_reader = searcher.segment_reader(0u32);
     let inv_idx = segment_reader.inverted_index(field).unwrap();
 
-    let mut term = term_from_json_paths(field, ["tenant_id"].iter().cloned(), false);
+    let mut term = Term::from_field_json_path(field, "tenant_id", false);
     term.append_type_and_fast_value(75u64);
 
     let postings = inv_idx
@@ -451,7 +450,7 @@ fn test_non_text_json_term_freq_bitpacked() {
     let segment_reader = searcher.segment_reader(0u32);
     let inv_idx = segment_reader.inverted_index(field).unwrap();
 
-    let mut term = term_from_json_paths(field, ["tenant_id"].iter().cloned(), false);
+    let mut term = Term::from_field_json_path(field, "tenant_id", false);
     term.append_type_and_fast_value(75u64);
 
     let mut postings = inv_idx

src/functional_test.rs

@@ -6,6 +6,7 @@ use rand::{thread_rng, Rng};
 
 use crate::indexer::index_writer::MEMORY_BUDGET_NUM_BYTES_MIN;
 use crate::schema::*;
+#[allow(deprecated)]
 use crate::{doc, schema, Index, IndexSettings, IndexSortByField, IndexWriter, Order, Searcher};
 
 fn check_index_content(searcher: &Searcher, vals: &[u64]) -> crate::Result<()> {

src/indexer/segment_writer.rs

@@ -498,7 +498,6 @@ mod tests {
     use crate::collector::{Count, TopDocs};
     use crate::directory::RamDirectory;
     use crate::fastfield::FastValue;
-    use crate::json_utils::term_from_json_paths;
     use crate::postings::TermInfo;
     use crate::query::{PhraseQuery, QueryParser};
     use crate::schema::document::Value;
@@ -647,9 +646,8 @@ mod tests {
 
         let mut term_stream = term_dict.stream().unwrap();
 
-        let term_from_path = |paths: &[&str]| -> Term {
-            term_from_json_paths(json_field, paths.iter().cloned(), false)
-        };
+        let term_from_path =
+            |path: &str| -> Term { Term::from_field_json_path(json_field, path, false) };
 
         fn set_fast_val<T: FastValue>(val: T, mut term: Term) -> Term {
             term.append_type_and_fast_value(val);
@@ -660,15 +658,14 @@ mod tests {
             term
         }
 
-        let term = term_from_path(&["bool"]);
+        let term = term_from_path("bool");
         assert!(term_stream.advance());
         assert_eq!(
             term_stream.key(),
             set_fast_val(true, term).serialized_value_bytes()
         );
 
-        let term = term_from_path(&["complexobject", "field.with.dot"]);
-
+        let term = term_from_path("complexobject.field\\.with\\.dot");
         assert!(term_stream.advance());
         assert_eq!(
             term_stream.key(),
@@ -676,7 +673,7 @@ mod tests {
         );
 
         // Date
-        let term = term_from_path(&["date"]);
+        let term = term_from_path("date");
 
         assert!(term_stream.advance());
         assert_eq!(
@@ -691,7 +688,7 @@ mod tests {
         );
 
         // Float
-        let term = term_from_path(&["float"]);
+        let term = term_from_path("float");
         assert!(term_stream.advance());
         assert_eq!(
             term_stream.key(),
@@ -699,21 +696,21 @@ mod tests {
         );
 
         // Number In Array
-        let term = term_from_path(&["my_arr"]);
+        let term = term_from_path("my_arr");
         assert!(term_stream.advance());
         assert_eq!(
             term_stream.key(),
             set_fast_val(2i64, term).serialized_value_bytes()
         );
 
-        let term = term_from_path(&["my_arr"]);
+        let term = term_from_path("my_arr");
         assert!(term_stream.advance());
         assert_eq!(
             term_stream.key(),
             set_fast_val(3i64, term).serialized_value_bytes()
         );
 
-        let term = term_from_path(&["my_arr"]);
+        let term = term_from_path("my_arr");
         assert!(term_stream.advance());
         assert_eq!(
             term_stream.key(),
@@ -721,13 +718,13 @@ mod tests {
         );
 
         // El in Array
-        let term = term_from_path(&["my_arr", "my_key"]);
+        let term = term_from_path("my_arr.my_key");
         assert!(term_stream.advance());
         assert_eq!(
             term_stream.key(),
             set_str("tokens", term).serialized_value_bytes()
         );
-        let term = term_from_path(&["my_arr", "my_key"]);
+        let term = term_from_path("my_arr.my_key");
         assert!(term_stream.advance());
         assert_eq!(
             term_stream.key(),
@@ -735,21 +732,21 @@ mod tests {
         );
 
         // Signed
-        let term = term_from_path(&["signed"]);
+        let term = term_from_path("signed");
         assert!(term_stream.advance());
         assert_eq!(
             term_stream.key(),
             set_fast_val(-2i64, term).serialized_value_bytes()
         );
 
-        let term = term_from_path(&["toto"]);
+        let term = term_from_path("toto");
         assert!(term_stream.advance());
         assert_eq!(
             term_stream.key(),
             set_str("titi", term).serialized_value_bytes()
         );
         // Unsigned
-        let term = term_from_path(&["unsigned"]);
+        let term = term_from_path("unsigned");
         assert!(term_stream.advance());
         assert_eq!(
             term_stream.key(),
@@ -776,7 +773,7 @@ mod tests {
         let searcher = reader.searcher();
         let segment_reader = searcher.segment_reader(0u32);
         let inv_index = segment_reader.inverted_index(json_field).unwrap();
-        let mut term = term_from_json_paths(json_field, ["mykey"].into_iter(), false);
+        let mut term = Term::from_field_json_path(json_field, "mykey", false);
         term.append_type_and_str("token");
         let term_info = inv_index.get_term_info(&term).unwrap().unwrap();
         assert_eq!(
@@ -815,7 +812,7 @@ mod tests {
         let searcher = reader.searcher();
         let segment_reader = searcher.segment_reader(0u32);
         let inv_index = segment_reader.inverted_index(json_field).unwrap();
-        let mut term = term_from_json_paths(json_field, ["mykey"].into_iter(), false);
+        let mut term = Term::from_field_json_path(json_field, "mykey", false);
         term.append_type_and_str("two tokens");
         let term_info = inv_index.get_term_info(&term).unwrap().unwrap();
         assert_eq!(
@@ -856,7 +853,7 @@ mod tests {
         let reader = index.reader().unwrap();
         let searcher = reader.searcher();
 
-        let term = term_from_json_paths(json_field, ["mykey", "field"].into_iter(), false);
+        let term = Term::from_field_json_path(json_field, "mykey.field", false);
 
         let mut hello_term = term.clone();
         hello_term.append_type_and_str("hello");

src/query/query_parser/query_parser.rs

@@ -11,9 +11,7 @@ use rustc_hash::FxHashMap;
 
 use super::logical_ast::*;
 use crate::index::Index;
-use crate::json_utils::{
-    convert_to_fast_value_and_append_to_json_term, split_json_path, term_from_json_paths,
-};
+use crate::json_utils::convert_to_fast_value_and_append_to_json_term;
 use crate::query::range_query::{is_type_valid_for_fastfield_range_query, RangeQuery};
 use crate::query::{
     AllQuery, BooleanQuery, BoostQuery, EmptyQuery, FuzzyTermQuery, Occur, PhrasePrefixQuery,
@@ -966,14 +964,8 @@ fn generate_literals_for_json_object(
     let index_record_option = text_options.index_option();
     let mut logical_literals = Vec::new();
 
-    let paths = split_json_path(json_path);
-    let get_term_with_path = || {
-        term_from_json_paths(
-            field,
-            paths.iter().map(|el| el.as_str()),
-            json_options.is_expand_dots_enabled(),
-        )
-    };
+    let get_term_with_path =
+        || Term::from_field_json_path(field, json_path, json_options.is_expand_dots_enabled());
 
     // Try to convert the phrase to a fast value
     if let Some(term) = convert_to_fast_value_and_append_to_json_term(get_term_with_path(), phrase)

src/schema/document/de.rs

@@ -960,13 +960,19 @@ mod tests {
             "my-third-key".to_string(),
             crate::schema::OwnedValue::F64(123.0),
         );
-        assert_eq!(value, crate::schema::OwnedValue::Object(expected_object));
+        assert_eq!(
+            value,
+            crate::schema::OwnedValue::Object(expected_object.into_iter().collect())
+        );
 
         let object = serde_json::Map::new();
         let result = serialize_value(ReferenceValue::Object(JsonObjectIter(object.iter())));
         let value = deserialize_value(result);
         let expected_object = BTreeMap::new();
-        assert_eq!(value, crate::schema::OwnedValue::Object(expected_object));
+        assert_eq!(
+            value,
+            crate::schema::OwnedValue::Object(expected_object.into_iter().collect())
+        );
 
         let mut object = serde_json::Map::new();
         object.insert("my-first-key".into(), serde_json::Value::Null);
@@ -978,7 +984,10 @@ mod tests {
         expected_object.insert("my-first-key".to_string(), crate::schema::OwnedValue::Null);
         expected_object.insert("my-second-key".to_string(), crate::schema::OwnedValue::Null);
         expected_object.insert("my-third-key".to_string(), crate::schema::OwnedValue::Null);
-        assert_eq!(value, crate::schema::OwnedValue::Object(expected_object));
+        assert_eq!(
+            value,
+            crate::schema::OwnedValue::Object(expected_object.into_iter().collect())
+        );
     }
 
     #[test]
@@ -1055,7 +1064,10 @@ mod tests {
                 .collect(),
             ),
         );
-        assert_eq!(value, crate::schema::OwnedValue::Object(expected_object));
+        assert_eq!(
+            value,
+            crate::schema::OwnedValue::Object(expected_object.into_iter().collect())
+        );
 
         // Some more extreme nesting that might behave weirdly
         let mut object = serde_json::Map::new();
@@ -1077,6 +1089,9 @@ mod tests {
                 OwnedValue::Array(vec![OwnedValue::Null]),
             ])]),
         );
-        assert_eq!(value, OwnedValue::Object(expected_object));
+        assert_eq!(
+            value,
+            OwnedValue::Object(expected_object.into_iter().collect())
+        );
     }
 }

src/schema/document/mod.rs

@@ -5,22 +5,24 @@
 //! - [Value] which provides tantivy with a way to access the document's values in a common way
 //!   without performing any additional allocations.
 //! - [DocumentDeserialize] which implements the necessary code to deserialize the document from the
-//!   doc store.
+//!   doc store. If you are fine with fetching [TantivyDocument] from the doc store, you can skip
+//!   implementing this trait for your type.
 //!
 //! Tantivy provides a few out-of-box implementations of these core traits to provide
 //! some simple usage if you don't want to implement these traits on a custom type yourself.
 //!
 //! # Out-of-box document implementations
-//! - [Document] the old document type used by Tantivy before the trait based approach was
+//! - [TantivyDocument] the old document type used by Tantivy before the trait based approach was
 //!   implemented. This type is still valid and provides all of the original behaviour you might
 //!   expect.
-//! - `BTreeMap<Field, Value>` a mapping of field_ids to their relevant schema value using a
+//! - `BTreeMap<Field, OwnedValue>` a mapping of field_ids to their relevant schema value using a
 //!   BTreeMap.
-//! - `HashMap<Field, Value>` a mapping of field_ids to their relevant schema value using a HashMap.
+//! - `HashMap<Field, OwnedValue>` a mapping of field_ids to their relevant schema value using a
+//!   HashMap.
 //!
 //! # Implementing your custom documents
 //! Often in larger projects or higher performance applications you want to avoid the extra overhead
-//! of converting your own types to the Tantivy [Document] type, this can often save you a
+//! of converting your own types to the [TantivyDocument] type, this can often save you a
 //! significant amount of time when indexing by avoiding the additional allocations.
 //!
 //! ### Important Note
@@ -46,6 +48,7 @@
 //!
 //! impl Document for MyCustomDocument {
 //!     // The value type produced by the `iter_fields_and_values` iterator.
+//!     // tantivy already implements the Value trait for serde_json::Value.
 //!     type Value<'a> = &'a serde_json::Value;
 //!     // The iterator which is produced by `iter_fields_and_values`.
 //!     // Often this is a simple new-type wrapper unless you like super long generics.
@@ -94,10 +97,11 @@
 //! implementation for.
 //!
 //! ## Implementing custom values
-//! Internally, Tantivy only works with `ReferenceValue` which is an enum that tries to borrow
-//! as much data as it can, in order to allow documents to return custom types, they must implement
-//! the `Value` trait which provides a way for Tantivy to get a `ReferenceValue` that it can then
+//! In order to allow documents to return custom types, they must implement
+//! the [Value] trait which provides a way for Tantivy to get a `ReferenceValue` that it can then
 //! index and store.
+//! Internally, Tantivy only works with `ReferenceValue` which is an enum that tries to borrow
+//! as much data as it can
 //!
 //! Values can just as easily be customised as documents by implementing the `Value` trait.
 //!
@@ -105,9 +109,9 @@
 //! hold references of the data held by the parent [Document] which can then be passed
 //! on to the [ReferenceValue].
 //!
-//! This is why `Value` is implemented for `&'a serde_json::Value` and `&'a
-//! tantivy::schema::Value` but not for their owned counterparts, as we cannot satisfy the lifetime
-//! bounds necessary when indexing the documents.
+//! This is why [Value] is implemented for `&'a serde_json::Value` and
+//! [&'a tantivy::schema::document::OwnedValue](OwnedValue) but not for their owned counterparts, as
+//! we cannot satisfy the lifetime bounds necessary when indexing the documents.
 //!
 //! ### A note about returning values
 //! The custom value type does not have to be the type stored by the document, instead the

src/schema/document/owned_value.rs

@@ -1,4 +1,4 @@
-use std::collections::{btree_map, BTreeMap};
+use std::collections::BTreeMap;
 use std::fmt;
 use std::net::Ipv6Addr;
 
@@ -45,7 +45,7 @@ pub enum OwnedValue {
     /// A set of values.
     Array(Vec<Self>),
     /// Dynamic object value.
-    Object(BTreeMap<String, Self>),
+    Object(Vec<(String, Self)>),
     /// IpV6 Address. Internally there is no IpV4, it needs to be converted to `Ipv6Addr`.
     IpAddr(Ipv6Addr),
 }
@@ -148,10 +148,10 @@ impl ValueDeserialize for OwnedValue {
 
             fn visit_object<'de, A>(&self, mut access: A) -> Result<Self::Value, DeserializeError>
             where A: ObjectAccess<'de> {
-                let mut elements = BTreeMap::new();
+                let mut elements = Vec::with_capacity(access.size_hint());
 
                 while let Some((key, value)) = access.next_entry()? {
-                    elements.insert(key, value);
+                    elements.push((key, value));
                 }
 
                 Ok(OwnedValue::Object(elements))
@@ -167,6 +167,7 @@ impl Eq for OwnedValue {}
 impl serde::Serialize for OwnedValue {
     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
     where S: serde::Serializer {
+        use serde::ser::SerializeMap;
         match *self {
             OwnedValue::Null => serializer.serialize_unit(),
             OwnedValue::Str(ref v) => serializer.serialize_str(v),
@@ -180,7 +181,13 @@ impl serde::Serialize for OwnedValue {
             }
             OwnedValue::Facet(ref facet) => facet.serialize(serializer),
             OwnedValue::Bytes(ref bytes) => serializer.serialize_str(&BASE64.encode(bytes)),
-            OwnedValue::Object(ref obj) => obj.serialize(serializer),
+            OwnedValue::Object(ref obj) => {
+                let mut map = serializer.serialize_map(Some(obj.len()))?;
+                for &(ref k, ref v) in obj {
+                    map.serialize_entry(k, v)?;
+                }
+                map.end()
+            }
             OwnedValue::IpAddr(ref ip_v6) => {
                 // Ensure IpV4 addresses get serialized as IpV4, but excluding IpV6 loopback.
                 if let Some(ip_v4) = ip_v6.to_ipv4_mapped() {
@@ -248,12 +255,10 @@ impl<'de> serde::Deserialize<'de> for OwnedValue {
 
             fn visit_map<A>(self, mut map: A) -> Result<Self::Value, A::Error>
             where A: MapAccess<'de> {
-                let mut object = BTreeMap::new();
-
+                let mut object = map.size_hint().map(Vec::with_capacity).unwrap_or_default();
                 while let Some((key, value)) = map.next_entry()? {
-                    object.insert(key, value);
+                    object.push((key, value));
                 }
-
                 Ok(OwnedValue::Object(object))
             }
         }
@@ -363,7 +368,8 @@ impl From<PreTokenizedString> for OwnedValue {
 
 impl From<BTreeMap<String, OwnedValue>> for OwnedValue {
     fn from(object: BTreeMap<String, OwnedValue>) -> OwnedValue {
-        OwnedValue::Object(object)
+        let key_values = object.into_iter().collect();
+        OwnedValue::Object(key_values)
     }
 }
 
@@ -417,18 +423,16 @@ impl From<serde_json::Value> for OwnedValue {
 
 impl From<serde_json::Map<String, serde_json::Value>> for OwnedValue {
     fn from(map: serde_json::Map<String, serde_json::Value>) -> Self {
-        let mut object = BTreeMap::new();
-
-        for (key, value) in map {
-            object.insert(key, OwnedValue::from(value));
-        }
-
+        let object: Vec<(String, OwnedValue)> = map
+            .into_iter()
+            .map(|(key, value)| (key, OwnedValue::from(value)))
+            .collect();
         OwnedValue::Object(object)
     }
 }
 
 /// A wrapper type for iterating over a serde_json object producing reference values.
-pub struct ObjectMapIter<'a>(btree_map::Iter<'a, String, OwnedValue>);
+pub struct ObjectMapIter<'a>(std::slice::Iter<'a, (String, OwnedValue)>);
 
 impl<'a> Iterator for ObjectMapIter<'a> {
     type Item = (&'a str, &'a OwnedValue);

src/schema/term.rs

@@ -4,10 +4,12 @@ use std::{fmt, str};
 
 use columnar::{MonotonicallyMappableToU128, MonotonicallyMappableToU64};
 use common::json_path_writer::{JSON_END_OF_PATH, JSON_PATH_SEGMENT_SEP_STR};
+use common::JsonPathWriter;
 
 use super::date_time_options::DATE_TIME_PRECISION_INDEXED;
 use super::Field;
 use crate::fastfield::FastValue;
+use crate::json_utils::split_json_path;
 use crate::schema::{Facet, Type};
 use crate::DateTime;
 
@@ -33,6 +35,28 @@ impl Term {
         Term(data)
     }
 
+    /// Creates a term from a json path.
+    ///
+    /// The json path can address a nested value in a JSON object.
+    /// e.g. `{"k8s": {"node": {"id": 5}}}` can be addressed via `k8s.node.id`.
+    ///
+    /// In case there are dots in the field name, and the `expand_dots_enabled` parameter is not
+    /// set they need to be escaped with a backslash.
+    /// e.g. `{"k8s.node": {"id": 5}}` can be addressed via `k8s\.node.id`.
+    pub fn from_field_json_path(field: Field, json_path: &str, expand_dots_enabled: bool) -> Term {
+        let paths = split_json_path(json_path);
+        let mut json_path = JsonPathWriter::with_expand_dots(expand_dots_enabled);
+        for path in paths {
+            json_path.push(&path);
+        }
+        json_path.set_end();
+        let mut term = Term::with_type_and_field(Type::Json, field);
+
+        term.append_bytes(json_path.as_str().as_bytes());
+
+        term
+    }
+
     pub(crate) fn with_type_and_field(typ: Type, field: Field) -> Term {
         let mut term = Self::with_capacity(8);
         term.set_field_and_type(field, typ);
@@ -165,7 +189,7 @@ impl Term {
     /// This is used in JSON type to append a fast value after the path.
     ///
     /// It will not clear existing bytes.
-    pub(crate) fn append_type_and_fast_value<T: FastValue>(&mut self, val: T) {
+    pub fn append_type_and_fast_value<T: FastValue>(&mut self, val: T) {
         self.0.push(T::to_type().to_code());
         let value = if T::to_type() == Type::Date {
             DateTime::from_u64(val.to_u64())
@@ -181,7 +205,7 @@ impl Term {
     /// This is used in JSON type to append a str after the path.
     ///
     /// It will not clear existing bytes.
-    pub(crate) fn append_type_and_str(&mut self, val: &str) {
+    pub fn append_type_and_str(&mut self, val: &str) {
         self.0.push(Type::Str.to_code());
         self.0.extend(val.as_bytes().as_ref());
     }