bump to 0.22.1

.github/workflows/coverage.yml

@@ -15,11 +15,11 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - name: Install Rust
-        run: rustup toolchain install nightly-2024-04-10 --profile minimal --component llvm-tools-preview
+        run: rustup toolchain install nightly-2023-09-10 --profile minimal --component llvm-tools-preview
       - uses: Swatinem/rust-cache@v2
       - uses: taiki-e/install-action@cargo-llvm-cov
       - name: Generate code coverage
-        run: cargo +nightly-2024-04-10 llvm-cov --all-features --workspace --doctests --lcov --output-path lcov.info
+        run: cargo +nightly-2023-09-10 llvm-cov --all-features --workspace --doctests --lcov --output-path lcov.info
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v3
         continue-on-error: true

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "tantivy"
-version = "0.23.0"
+version = "0.22.1"
 authors = ["Paul Masurel <paul.masurel@gmail.com>"]
 license = "MIT"
 categories = ["database-implementations", "data-structures"]
@@ -15,15 +15,12 @@ rust-version = "1.63"
 exclude = ["benches/*.json", "benches/*.txt"]
 
 [dependencies]
-oneshot = "0.1.7"
+oneshot = "0.1.5"
 base64 = "0.22.0"
 byteorder = "1.4.3"
 crc32fast = "1.3.2"
 once_cell = "1.10.0"
-regex = { version = "1.5.5", default-features = false, features = [
-    "std",
-    "unicode",
-] }
+regex = { version = "1.5.5", default-features = false, features = ["std", "unicode"] }
 aho-corasick = "1.0"
 tantivy-fst = "0.5"
 memmap2 = { version = "0.9.0", optional = true }
@@ -33,15 +30,14 @@ tempfile = { version = "3.3.0", optional = true }
 log = "0.4.16"
 serde = { version = "1.0.136", features = ["derive"] }
 serde_json = "1.0.79"
+num_cpus = "1.13.1"
 fs4 = { version = "0.8.0", optional = true }
 levenshtein_automata = "0.2.1"
 uuid = { version = "1.0.0", features = ["v4", "serde"] }
 crossbeam-channel = "0.5.4"
 rust-stemmers = "1.2.0"
 downcast-rs = "1.2.0"
-bitpacking = { version = "0.9.2", default-features = false, features = [
-    "bitpacker4x",
-] }
+bitpacking = { version = "0.9.2", default-features = false, features = ["bitpacker4x"] }
 census = "0.4.2"
 rustc-hash = "1.1.0"
 thiserror = "1.0.30"
@@ -52,18 +48,18 @@ smallvec = "1.8.0"
 rayon = "1.5.2"
 lru = "0.12.0"
 fastdivide = "0.4.0"
-itertools = "0.13.0"
+itertools = "0.12.0"
 measure_time = "0.8.2"
 arc-swap = "1.5.0"
 
-columnar = { version = "0.3", path = "./columnar", package = "tantivy-columnar" }
-sstable = { version = "0.3", path = "./sstable", package = "tantivy-sstable", optional = true }
-stacker = { version = "0.3", path = "./stacker", package = "tantivy-stacker" }
-query-grammar = { version = "0.22.0", path = "./query-grammar", package = "tantivy-query-grammar" }
-tantivy-bitpacker = { version = "0.6", path = "./bitpacker" }
-common = { version = "0.7", path = "./common/", package = "tantivy-common" }
-tokenizer-api = { version = "0.3", path = "./tokenizer-api", package = "tantivy-tokenizer-api" }
-sketches-ddsketch = { version = "0.3.0", features = ["use_serde"] }
+columnar = { version= "0.3", path="./columnar", package ="tantivy-columnar" }
+sstable = { version= "0.3", path="./sstable", package ="tantivy-sstable", optional = true }
+stacker = { version= "0.3", path="./stacker", package ="tantivy-stacker" }
+query-grammar = { version= "0.22.0", path="./query-grammar", package = "tantivy-query-grammar" }
+tantivy-bitpacker = { version= "0.6", path="./bitpacker" }
+common = { version= "0.7", path = "./common/", package = "tantivy-common" }
+tokenizer-api = { version= "0.3", path="./tokenizer-api", package="tantivy-tokenizer-api" }
+sketches-ddsketch = { version = "0.2.1", features = ["use_serde"] }
 futures-util = { version = "0.3.28", optional = true }
 fnv = "1.0.7"
 
@@ -71,7 +67,6 @@ fnv = "1.0.7"
 winapi = "0.3.9"
 
 [dev-dependencies]
-binggan = "0.8.0"
 rand = "0.8.5"
 maplit = "1.0.2"
 matches = "0.1.9"
@@ -117,26 +112,17 @@ lz4-compression = ["lz4_flex"]
 zstd-compression = ["zstd"]
 
 failpoints = ["fail", "fail/failpoints"]
-unstable = []                            # useful for benches.
+unstable = [] # useful for benches.
 
 quickwit = ["sstable", "futures-util"]
 
-# Compares only the hash of a string when indexing data.
+# Compares only the hash of a string when indexing data. 
 # Increases indexing speed, but may lead to extremely rare missing terms, when there's a hash collision.
 # Uses 64bit ahash.
 compare_hash_only = ["stacker/compare_hash_only"]
 
 [workspace]
-members = [
-    "query-grammar",
-    "bitpacker",
-    "common",
-    "ownedbytes",
-    "stacker",
-    "sstable",
-    "tokenizer-api",
-    "columnar",
-]
+members = ["query-grammar", "bitpacker", "common", "ownedbytes", "stacker", "sstable", "tokenizer-api", "columnar"]
 
 # Following the "fail" crate best practises, we isolate
 # tests that define specific behavior in fail check points
@@ -157,7 +143,3 @@ harness = false
 [[bench]]
 name = "index-bench"
 harness = false
-
-[[bench]]
-name = "agg_bench"
-harness = false

benches/agg_bench.rs

@@ -1,419 +0,0 @@
-use binggan::{black_box, InputGroup, PeakMemAlloc, INSTRUMENTED_SYSTEM};
-use rand::prelude::SliceRandom;
-use rand::rngs::StdRng;
-use rand::{Rng, SeedableRng};
-use rand_distr::Distribution;
-use serde_json::json;
-use tantivy::aggregation::agg_req::Aggregations;
-use tantivy::aggregation::AggregationCollector;
-use tantivy::query::{AllQuery, TermQuery};
-use tantivy::schema::{IndexRecordOption, Schema, TextFieldIndexing, FAST, STRING};
-use tantivy::{doc, Index, Term};
-
-#[global_allocator]
-pub static GLOBAL: &PeakMemAlloc<std::alloc::System> = &INSTRUMENTED_SYSTEM;
-
-/// Mini macro to register a function via its name
-/// runner.register("average_u64", move |index| average_u64(index));
-macro_rules! register {
-    ($runner:expr, $func:ident) => {
-        $runner.register(stringify!($func), move |index| $func(index))
-    };
-}
-
-fn main() {
-    let inputs = vec![
-        ("full", get_test_index_bench(Cardinality::Full).unwrap()),
-        (
-            "dense",
-            get_test_index_bench(Cardinality::OptionalDense).unwrap(),
-        ),
-        (
-            "sparse",
-            get_test_index_bench(Cardinality::OptionalSparse).unwrap(),
-        ),
-        (
-            "multivalue",
-            get_test_index_bench(Cardinality::Multivalued).unwrap(),
-        ),
-    ];
-
-    bench_agg(InputGroup::new_with_inputs(inputs));
-}
-
-fn bench_agg(mut group: InputGroup<Index>) {
-    group.set_alloc(GLOBAL); // Set the peak mem allocator. This will enable peak memory reporting.
-    register!(group, average_u64);
-    register!(group, average_f64);
-    register!(group, average_f64_u64);
-    register!(group, stats_f64);
-    register!(group, extendedstats_f64);
-    register!(group, percentiles_f64);
-    register!(group, terms_few);
-    register!(group, terms_many);
-    register!(group, terms_many_order_by_term);
-    register!(group, terms_many_with_top_hits);
-    register!(group, terms_many_with_avg_sub_agg);
-    register!(group, terms_many_json_mixed_type_with_sub_agg_card);
-    register!(group, range_agg);
-    register!(group, range_agg_with_avg_sub_agg);
-    register!(group, range_agg_with_term_agg_few);
-    register!(group, range_agg_with_term_agg_many);
-    register!(group, histogram);
-    register!(group, histogram_hard_bounds);
-    register!(group, histogram_with_avg_sub_agg);
-    register!(group, avg_and_range_with_avg_sub_agg);
-
-    group.run();
-}
-
-fn exec_term_with_agg(index: &Index, agg_req: serde_json::Value) {
-    let agg_req: Aggregations = serde_json::from_value(agg_req).unwrap();
-
-    let reader = index.reader().unwrap();
-    let text_field = reader.searcher().schema().get_field("text").unwrap();
-    let term_query = TermQuery::new(
-        Term::from_field_text(text_field, "cool"),
-        IndexRecordOption::Basic,
-    );
-    let collector = get_collector(agg_req);
-    let searcher = reader.searcher();
-    black_box(searcher.search(&term_query, &collector).unwrap());
-}
-
-fn average_u64(index: &Index) {
-    let agg_req = json!({
-        "average": { "avg": { "field": "score", } }
-    });
-    exec_term_with_agg(index, agg_req)
-}
-fn average_f64(index: &Index) {
-    let agg_req = json!({
-        "average": { "avg": { "field": "score_f64", } }
-    });
-    exec_term_with_agg(index, agg_req)
-}
-fn average_f64_u64(index: &Index) {
-    let agg_req = json!({
-        "average_f64": { "avg": { "field": "score_f64" } },
-        "average": { "avg": { "field": "score" } },
-    });
-    exec_term_with_agg(index, agg_req)
-}
-fn stats_f64(index: &Index) {
-    let agg_req = json!({
-        "average_f64": { "stats": { "field": "score_f64", } }
-    });
-    exec_term_with_agg(index, agg_req)
-}
-fn extendedstats_f64(index: &Index) {
-    let agg_req = json!({
-        "extendedstats_f64": { "extended_stats": { "field": "score_f64", } }
-    });
-    exec_term_with_agg(index, agg_req)
-}
-fn percentiles_f64(index: &Index) {
-    let agg_req = json!({
-      "mypercentiles": {
-        "percentiles": {
-          "field": "score_f64",
-          "percents": [ 95, 99, 99.9 ]
-        }
-      }
-    });
-    execute_agg(index, agg_req);
-}
-fn terms_few(index: &Index) {
-    let agg_req = json!({
-        "my_texts": { "terms": { "field": "text_few_terms" } },
-    });
-    execute_agg(index, agg_req);
-}
-fn terms_many(index: &Index) {
-    let agg_req = json!({
-        "my_texts": { "terms": { "field": "text_many_terms" } },
-    });
-    execute_agg(index, agg_req);
-}
-fn terms_many_order_by_term(index: &Index) {
-    let agg_req = json!({
-        "my_texts": { "terms": { "field": "text_many_terms", "order": { "_key": "desc" } } },
-    });
-    execute_agg(index, agg_req);
-}
-fn terms_many_with_top_hits(index: &Index) {
-    let agg_req = json!({
-        "my_texts": {
-            "terms": { "field": "text_many_terms" },
-            "aggs": {
-                "top_hits": { "top_hits":
-                    {
-                        "sort": [
-                            { "score": "desc" }
-                        ],
-                        "size": 2,
-                        "doc_value_fields": ["score_f64"]
-                    }
-                }
-            }
-        },
-    });
-    execute_agg(index, agg_req);
-}
-fn terms_many_with_avg_sub_agg(index: &Index) {
-    let agg_req = json!({
-        "my_texts": {
-            "terms": { "field": "text_many_terms" },
-            "aggs": {
-                "average_f64": { "avg": { "field": "score_f64" } }
-            }
-        },
-    });
-    execute_agg(index, agg_req);
-}
-fn terms_many_json_mixed_type_with_sub_agg_card(index: &Index) {
-    let agg_req = json!({
-        "my_texts": {
-            "terms": { "field": "json.mixed_type" },
-            "aggs": {
-                "average_f64": { "avg": { "field": "score_f64" } }
-            }
-        },
-    });
-    execute_agg(index, agg_req);
-}
-
-fn execute_agg(index: &Index, agg_req: serde_json::Value) {
-    let agg_req: Aggregations = serde_json::from_value(agg_req).unwrap();
-    let collector = get_collector(agg_req);
-
-    let reader = index.reader().unwrap();
-    let searcher = reader.searcher();
-    black_box(searcher.search(&AllQuery, &collector).unwrap());
-}
-fn range_agg(index: &Index) {
-    let agg_req = json!({
-        "range_f64": { "range": { "field": "score_f64", "ranges": [
-            { "from": 3, "to": 7000 },
-            { "from": 7000, "to": 20000 },
-            { "from": 20000, "to": 30000 },
-            { "from": 30000, "to": 40000 },
-            { "from": 40000, "to": 50000 },
-            { "from": 50000, "to": 60000 }
-        ] } },
-    });
-    execute_agg(index, agg_req);
-}
-fn range_agg_with_avg_sub_agg(index: &Index) {
-    let agg_req = json!({
-        "rangef64": {
-            "range": {
-                "field": "score_f64",
-                "ranges": [
-                    { "from": 3, "to": 7000 },
-                    { "from": 7000, "to": 20000 },
-                    { "from": 20000, "to": 30000 },
-                    { "from": 30000, "to": 40000 },
-                    { "from": 40000, "to": 50000 },
-                    { "from": 50000, "to": 60000 }
-                ]
-            },
-            "aggs": {
-                "average_f64": { "avg": { "field": "score_f64" } }
-            }
-        },
-    });
-    execute_agg(index, agg_req);
-}
-
-fn range_agg_with_term_agg_few(index: &Index) {
-    let agg_req = json!({
-        "rangef64": {
-            "range": {
-                "field": "score_f64",
-                "ranges": [
-                    { "from": 3, "to": 7000 },
-                    { "from": 7000, "to": 20000 },
-                    { "from": 20000, "to": 30000 },
-                    { "from": 30000, "to": 40000 },
-                    { "from": 40000, "to": 50000 },
-                    { "from": 50000, "to": 60000 }
-                ]
-            },
-            "aggs": {
-                "my_texts": { "terms": { "field": "text_few_terms" } },
-            }
-        },
-    });
-    execute_agg(index, agg_req);
-}
-fn range_agg_with_term_agg_many(index: &Index) {
-    let agg_req = json!({
-        "rangef64": {
-            "range": {
-                "field": "score_f64",
-                "ranges": [
-                    { "from": 3, "to": 7000 },
-                    { "from": 7000, "to": 20000 },
-                    { "from": 20000, "to": 30000 },
-                    { "from": 30000, "to": 40000 },
-                    { "from": 40000, "to": 50000 },
-                    { "from": 50000, "to": 60000 }
-                ]
-            },
-            "aggs": {
-                "my_texts": { "terms": { "field": "text_many_terms" } },
-            }
-        },
-    });
-    execute_agg(index, agg_req);
-}
-fn histogram(index: &Index) {
-    let agg_req = json!({
-        "rangef64": {
-            "histogram": {
-                "field": "score_f64",
-                "interval": 100 // 1000 buckets
-            },
-        }
-    });
-    execute_agg(index, agg_req);
-}
-fn histogram_hard_bounds(index: &Index) {
-    let agg_req = json!({
-        "rangef64": { "histogram": { "field": "score_f64", "interval": 100, "hard_bounds": { "min": 1000, "max": 300000 } } },
-    });
-    execute_agg(index, agg_req);
-}
-fn histogram_with_avg_sub_agg(index: &Index) {
-    let agg_req = json!({
-        "rangef64": {
-            "histogram": { "field": "score_f64", "interval": 100 },
-            "aggs": {
-                "average_f64": { "avg": { "field": "score_f64" } }
-            }
-        }
-    });
-    execute_agg(index, agg_req);
-}
-fn avg_and_range_with_avg_sub_agg(index: &Index) {
-    let agg_req = json!({
-        "rangef64": {
-            "range": {
-                "field": "score_f64",
-                "ranges": [
-                    { "from": 3, "to": 7000 },
-                    { "from": 7000, "to": 20000 },
-                    { "from": 20000, "to": 60000 }
-                ]
-            },
-            "aggs": {
-                "average_in_range": { "avg": { "field": "score" } }
-            }
-        },
-        "average": { "avg": { "field": "score" } }
-    });
-    execute_agg(index, agg_req);
-}
-
-#[derive(Clone, Copy, Hash, Default, Debug, PartialEq, Eq, PartialOrd, Ord)]
-enum Cardinality {
-    /// All documents contain exactly one value.
-    /// `Full` is the default for auto-detecting the Cardinality, since it is the most strict.
-    #[default]
-    Full = 0,
-    /// All documents contain at most one value.
-    OptionalDense = 1,
-    /// All documents may contain any number of values.
-    Multivalued = 2,
-    /// 1 / 20 documents has a value
-    OptionalSparse = 3,
-}
-
-fn get_collector(agg_req: Aggregations) -> AggregationCollector {
-    AggregationCollector::from_aggs(agg_req, Default::default())
-}
-
-fn get_test_index_bench(cardinality: Cardinality) -> tantivy::Result<Index> {
-    let mut schema_builder = Schema::builder();
-    let text_fieldtype = tantivy::schema::TextOptions::default()
-        .set_indexing_options(
-            TextFieldIndexing::default().set_index_option(IndexRecordOption::WithFreqs),
-        )
-        .set_stored();
-    let text_field = schema_builder.add_text_field("text", text_fieldtype);
-    let json_field = schema_builder.add_json_field("json", FAST);
-    let text_field_many_terms = schema_builder.add_text_field("text_many_terms", STRING | FAST);
-    let text_field_few_terms = schema_builder.add_text_field("text_few_terms", STRING | FAST);
-    let score_fieldtype = tantivy::schema::NumericOptions::default().set_fast();
-    let score_field = schema_builder.add_u64_field("score", score_fieldtype.clone());
-    let score_field_f64 = schema_builder.add_f64_field("score_f64", score_fieldtype.clone());
-    let score_field_i64 = schema_builder.add_i64_field("score_i64", score_fieldtype);
-    let index = Index::create_from_tempdir(schema_builder.build())?;
-    let few_terms_data = ["INFO", "ERROR", "WARN", "DEBUG"];
-
-    let lg_norm = rand_distr::LogNormal::new(2.996f64, 0.979f64).unwrap();
-
-    let many_terms_data = (0..150_000)
-        .map(|num| format!("author{num}"))
-        .collect::<Vec<_>>();
-    {
-        let mut rng = StdRng::from_seed([1u8; 32]);
-        let mut index_writer = index.writer_with_num_threads(1, 200_000_000)?;
-        // To make the different test cases comparable we just change one doc to force the
-        // cardinality
-        if cardinality == Cardinality::OptionalDense {
-            index_writer.add_document(doc!())?;
-        }
-        if cardinality == Cardinality::Multivalued {
-            index_writer.add_document(doc!(
-                json_field => json!({"mixed_type": 10.0}),
-                json_field => json!({"mixed_type": 10.0}),
-                text_field => "cool",
-                text_field => "cool",
-                text_field_many_terms => "cool",
-                text_field_many_terms => "cool",
-                text_field_few_terms => "cool",
-                text_field_few_terms => "cool",
-                score_field => 1u64,
-                score_field => 1u64,
-                score_field_f64 => lg_norm.sample(&mut rng),
-                score_field_f64 => lg_norm.sample(&mut rng),
-                score_field_i64 => 1i64,
-                score_field_i64 => 1i64,
-            ))?;
-        }
-        let mut doc_with_value = 1_000_000;
-        if cardinality == Cardinality::OptionalSparse {
-            doc_with_value /= 20;
-        }
-        let _val_max = 1_000_000.0;
-        for _ in 0..doc_with_value {
-            let val: f64 = rng.gen_range(0.0..1_000_000.0);
-            let json = if rng.gen_bool(0.1) {
-                // 10% are numeric values
-                json!({ "mixed_type": val })
-            } else {
-                json!({"mixed_type": many_terms_data.choose(&mut rng).unwrap().to_string()})
-            };
-            index_writer.add_document(doc!(
-                text_field => "cool",
-                json_field => json,
-                text_field_many_terms => many_terms_data.choose(&mut rng).unwrap().to_string(),
-                text_field_few_terms => few_terms_data.choose(&mut rng).unwrap().to_string(),
-                score_field => val as u64,
-                score_field_f64 => lg_norm.sample(&mut rng),
-                score_field_i64 => val as i64,
-            ))?;
-            if cardinality == Cardinality::OptionalSparse {
-                for _ in 0..20 {
-                    index_writer.add_document(doc!(text_field => "cool"))?;
-                }
-            }
-        }
-        // writing the segment
-        index_writer.commit()?;
-    }
-
-    Ok(index)
-}

benches/index-bench.rs

@@ -18,7 +18,7 @@ fn benchmark(
         benchmark_dynamic_json(b, input, schema, commit, parse_json)
     } else {
         _benchmark(b, input, schema, commit, parse_json, |schema, doc_json| {
-            TantivyDocument::parse_json(schema, doc_json).unwrap()
+            TantivyDocument::parse_json(&schema, doc_json).unwrap()
         })
     }
 }
@@ -90,7 +90,8 @@ fn benchmark_dynamic_json(
 ) {
     let json_field = schema.get_field("json").unwrap();
     _benchmark(b, input, schema, commit, parse_json, |_schema, doc_json| {
-        let json_val: serde_json::Value = serde_json::from_str(doc_json).unwrap();
+        let json_val: serde_json::Map<String, serde_json::Value> =
+            serde_json::from_str(doc_json).unwrap();
         tantivy::doc!(json_field=>json_val)
     })
 }
@@ -137,16 +138,15 @@ pub fn hdfs_index_benchmark(c: &mut Criterion) {
     for (prefix, schema, is_dynamic) in benches {
         for commit in [false, true] {
             let suffix = if commit { "with-commit" } else { "no-commit" };
-            {
-                let parse_json = false;
+            for parse_json in [false] {
                 // for parse_json in [false, true] {
                 let suffix = if parse_json {
-                    format!("{suffix}-with-json-parsing")
+                    format!("{}-with-json-parsing", suffix)
                 } else {
-                    suffix.to_string()
+                    format!("{}", suffix)
                 };
 
-                let bench_name = format!("{prefix}{suffix}");
+                let bench_name = format!("{}{}", prefix, suffix);
                 group.bench_function(bench_name, |b| {
                     benchmark(b, HDFS_LOGS, schema.clone(), commit, parse_json, is_dynamic)
                 });

columnar/Cargo.toml

@@ -9,7 +9,7 @@ description = "column oriented storage for tantivy"
 categories = ["database-implementations", "data-structures", "compression"]
 
 [dependencies]
-itertools = "0.13.0"
+itertools = "0.12.0"
 fastdivide = "0.4.0"
 
 stacker = { version= "0.3", path = "../stacker", package="tantivy-stacker"}
@@ -23,12 +23,6 @@ downcast-rs = "1.2.0"
 proptest = "1"
 more-asserts = "0.3.1"
 rand = "0.8"
-binggan = "0.8.1"
-
-[[bench]]
-name = "bench_merge"
-harness = false
-
 
 [features]
 unstable = []

columnar/benches/bench_merge.rs

@@ -1,101 +0,0 @@
-#![feature(test)]
-extern crate test;
-
-use core::fmt;
-use std::fmt::{Display, Formatter};
-
-use binggan::{black_box, BenchRunner};
-use tantivy_columnar::*;
-
-enum Card {
-    Multi,
-    Sparse,
-    Dense,
-}
-impl Display for Card {
-    fn fmt(&self, f: &mut Formatter) -> fmt::Result {
-        match self {
-            Card::Multi => write!(f, "multi"),
-            Card::Sparse => write!(f, "sparse"),
-            Card::Dense => write!(f, "dense"),
-        }
-    }
-}
-
-const NUM_DOCS: u32 = 100_000;
-
-fn generate_columnar(card: Card, num_docs: u32) -> ColumnarReader {
-    use tantivy_columnar::ColumnarWriter;
-
-    let mut columnar_writer = ColumnarWriter::default();
-
-    match card {
-        Card::Multi => {
-            columnar_writer.record_numerical(0, "price", 10u64);
-            columnar_writer.record_numerical(0, "price", 10u64);
-        }
-        _ => {}
-    }
-
-    for i in 0..num_docs {
-        match card {
-            Card::Multi | Card::Sparse => {
-                if i % 8 == 0 {
-                    columnar_writer.record_numerical(i, "price", i as u64);
-                }
-            }
-            Card::Dense => {
-                if i % 6 == 0 {
-                    columnar_writer.record_numerical(i, "price", i as u64);
-                }
-            }
-        }
-    }
-
-    let mut wrt: Vec<u8> = Vec::new();
-    columnar_writer.serialize(num_docs, None, &mut wrt).unwrap();
-
-    ColumnarReader::open(wrt).unwrap()
-}
-fn main() {
-    let mut inputs = Vec::new();
-
-    let mut add_combo = |card1: Card, card2: Card| {
-        inputs.push((
-            format!("merge_{card1}_and_{card2}"),
-            vec![
-                generate_columnar(card1, NUM_DOCS),
-                generate_columnar(card2, NUM_DOCS),
-            ],
-        ));
-    };
-
-    add_combo(Card::Multi, Card::Multi);
-    add_combo(Card::Dense, Card::Dense);
-    add_combo(Card::Sparse, Card::Sparse);
-    add_combo(Card::Sparse, Card::Dense);
-    add_combo(Card::Multi, Card::Dense);
-    add_combo(Card::Multi, Card::Sparse);
-
-    let runner: BenchRunner = BenchRunner::new();
-    let mut group = runner.new_group();
-    for (input_name, columnar_readers) in inputs.iter() {
-        group.register_with_input(
-            input_name,
-            columnar_readers,
-            move |columnar_readers: &Vec<ColumnarReader>| {
-                let mut out = vec![];
-                let columnar_readers = columnar_readers.iter().collect::<Vec<_>>();
-                let merge_row_order = StackMergeOrder::stack(&columnar_readers[..]);
-
-                let _ = black_box(merge_columnar(
-                    &columnar_readers,
-                    &[],
-                    merge_row_order.into(),
-                    &mut out,
-                ));
-            },
-        );
-    }
-    group.run();
-}

columnar/src/column_index/merge/mod.rs

@@ -150,62 +150,61 @@ mod tests {
         );
     }
 
-    // #[test]
-    // fn test_merge_index_multivalued_sorted() {
-    //     let column_indexes: Vec<ColumnIndex> = vec![MultiValueIndex::for_test(&[0, 2,
-    // 5]).into()];     let merge_row_order: MergeRowOrder = ShuffleMergeOrder::for_test(
-    //         &[2],
-    //         vec![
-    //             RowAddr {
-    //                 segment_ord: 0u32,
-    //                 row_id: 1u32,
-    //             },
-    //             RowAddr {
-    //                 segment_ord: 0u32,
-    //                 row_id: 0u32,
-    //             },
-    //         ],
-    //     )
-    //     .into();
-    //     let merged_column_index = merge_column_index(&column_indexes[..], &merge_row_order);
-    //     let SerializableColumnIndex::Multivalued(serializable_multivalue_index) =
-    // merged_column_index else {         panic!("Excpected a multivalued index")
-    //     };
-    //     serializable_multivalue_index.doc_ids_with_values_opt.
-    //     let start_indexes: Vec<RowId> = start_index_iterable.boxed_iter().collect();
-    //     assert_eq!(&start_indexes, &[0, 3, 5]);
-    // }
+    #[test]
+    fn test_merge_index_multivalued_sorted() {
+        let column_indexes: Vec<ColumnIndex> = vec![MultiValueIndex::for_test(&[0, 2, 5]).into()];
+        let merge_row_order: MergeRowOrder = ShuffleMergeOrder::for_test(
+            &[2],
+            vec![
+                RowAddr {
+                    segment_ord: 0u32,
+                    row_id: 1u32,
+                },
+                RowAddr {
+                    segment_ord: 0u32,
+                    row_id: 0u32,
+                },
+            ],
+        )
+        .into();
+        let merged_column_index = merge_column_index(&column_indexes[..], &merge_row_order);
+        let SerializableColumnIndex::Multivalued(start_index_iterable) = merged_column_index else {
+            panic!("Excpected a multivalued index")
+        };
+        let start_indexes: Vec<RowId> = start_index_iterable.boxed_iter().collect();
+        assert_eq!(&start_indexes, &[0, 3, 5]);
+    }
 
-    // #[test]
-    // fn test_merge_index_multivalued_sorted_several_segment() {
-    //     let column_indexes: Vec<ColumnIndex> = vec![
-    //         MultiValueIndex::for_test(&[0, 2, 5]).into(),
-    //         ColumnIndex::Empty { num_docs: 0 },
-    //         MultiValueIndex::for_test(&[0, 1, 4]).into(),
-    //     ];
-    //     let merge_row_order: MergeRowOrder = ShuffleMergeOrder::for_test(
-    //         &[2, 0, 2],
-    //         vec![
-    //             RowAddr {
-    //                 segment_ord: 2u32,
-    //                 row_id: 1u32,
-    //             },
-    //             RowAddr {
-    //                 segment_ord: 0u32,
-    //                 row_id: 0u32,
-    //             },
-    //             RowAddr {
-    //                 segment_ord: 2u32,
-    //                 row_id: 0u32,
-    //             },
-    //         ],
-    //     )
-    //     .into();
-    //     let merged_column_index = merge_column_index(&column_indexes[..], &merge_row_order);
-    //     let SerializableColumnIndex::Multivalued(serializable_multivalue_index) =
-    // merged_column_index else {         panic!("Excpected a multivalued index")
-    //     };
-    //     let start_indexes: Vec<RowId> = start_index_iterable.boxed_iter().collect();
-    //     assert_eq!(&start_indexes, &[0, 3, 5, 6]);
-    // }
+    #[test]
+    fn test_merge_index_multivalued_sorted_several_segment() {
+        let column_indexes: Vec<ColumnIndex> = vec![
+            MultiValueIndex::for_test(&[0, 2, 5]).into(),
+            ColumnIndex::Empty { num_docs: 0 },
+            MultiValueIndex::for_test(&[0, 1, 4]).into(),
+        ];
+        let merge_row_order: MergeRowOrder = ShuffleMergeOrder::for_test(
+            &[2, 0, 2],
+            vec![
+                RowAddr {
+                    segment_ord: 2u32,
+                    row_id: 1u32,
+                },
+                RowAddr {
+                    segment_ord: 0u32,
+                    row_id: 0u32,
+                },
+                RowAddr {
+                    segment_ord: 2u32,
+                    row_id: 0u32,
+                },
+            ],
+        )
+        .into();
+        let merged_column_index = merge_column_index(&column_indexes[..], &merge_row_order);
+        let SerializableColumnIndex::Multivalued(start_index_iterable) = merged_column_index else {
+            panic!("Excpected a multivalued index")
+        };
+        let start_indexes: Vec<RowId> = start_index_iterable.boxed_iter().collect();
+        assert_eq!(&start_indexes, &[0, 3, 5, 6]);
+    }
 }

columnar/src/column_index/merge/shuffled.rs

@@ -9,23 +9,22 @@ pub fn merge_column_index_shuffled<'a>(
     cardinality_after_merge: Cardinality,
     shuffle_merge_order: &'a ShuffleMergeOrder,
 ) -> SerializableColumnIndex<'a> {
-    todo!();
-    // match cardinality_after_merge {
-    //     Cardinality::Full => SerializableColumnIndex::Full,
-    //     Cardinality::Optional => {
-    //         let non_null_row_ids =
-    //             merge_column_index_shuffled_optional(column_indexes, shuffle_merge_order);
-    //         SerializableColumnIndex::Optional {
-    //             non_null_row_ids,
-    //             num_rows: shuffle_merge_order.num_rows(),
-    //         }
-    //     }
-    //     Cardinality::Multivalued => {
-    //         let multivalue_start_index =
-    //             merge_column_index_shuffled_multivalued(column_indexes, shuffle_merge_order);
-    //         SerializableColumnIndex::Multivalued(multivalue_start_index)
-    //     }
-    // }
+    match cardinality_after_merge {
+        Cardinality::Full => SerializableColumnIndex::Full,
+        Cardinality::Optional => {
+            let non_null_row_ids =
+                merge_column_index_shuffled_optional(column_indexes, shuffle_merge_order);
+            SerializableColumnIndex::Optional {
+                non_null_row_ids,
+                num_rows: shuffle_merge_order.num_rows(),
+            }
+        }
+        Cardinality::Multivalued => {
+            let multivalue_start_index =
+                merge_column_index_shuffled_multivalued(column_indexes, shuffle_merge_order);
+            SerializableColumnIndex::Multivalued(multivalue_start_index)
+        }
+    }
 }
 
 /// Merge several column indexes into one, ordering rows according to the merge_order passed as
@@ -138,35 +137,35 @@ mod tests {
         assert!(integrate_num_vals([3, 0, 10, 20].into_iter()).eq([0, 3, 3, 13, 33].into_iter()));
     }
 
-    // #[test]
-    // fn test_merge_column_index_optional_shuffle() {
-    //     let optional_index: ColumnIndex = OptionalIndex::for_test(2, &[0]).into();
-    //     let column_indexes = [optional_index, ColumnIndex::Full];
-    //     let row_addrs = vec![
-    //         RowAddr {
-    //             segment_ord: 0u32,
-    //             row_id: 1u32,
-    //         },
-    //         RowAddr {
-    //             segment_ord: 1u32,
-    //             row_id: 0u32,
-    //         },
-    //     ];
-    //     let shuffle_merge_order = ShuffleMergeOrder::for_test(&[2, 1], row_addrs);
-    //     let serializable_index = merge_column_index_shuffled(
-    //         &column_indexes[..],
-    //         Cardinality::Optional,
-    //         &shuffle_merge_order,
-    //     );
-    //     let SerializableColumnIndex::Optional {
-    //         non_null_row_ids,
-    //         num_rows,
-    //     } = serializable_index
-    //     else {
-    //         panic!()
-    //     };
-    //     assert_eq!(num_rows, 2);
-    //     let non_null_rows: Vec<RowId> = non_null_row_ids.boxed_iter().collect();
-    //     assert_eq!(&non_null_rows, &[1]);
-    // }
+    #[test]
+    fn test_merge_column_index_optional_shuffle() {
+        let optional_index: ColumnIndex = OptionalIndex::for_test(2, &[0]).into();
+        let column_indexes = [optional_index, ColumnIndex::Full];
+        let row_addrs = vec![
+            RowAddr {
+                segment_ord: 0u32,
+                row_id: 1u32,
+            },
+            RowAddr {
+                segment_ord: 1u32,
+                row_id: 0u32,
+            },
+        ];
+        let shuffle_merge_order = ShuffleMergeOrder::for_test(&[2, 1], row_addrs);
+        let serializable_index = merge_column_index_shuffled(
+            &column_indexes[..],
+            Cardinality::Optional,
+            &shuffle_merge_order,
+        );
+        let SerializableColumnIndex::Optional {
+            non_null_row_ids,
+            num_rows,
+        } = serializable_index
+        else {
+            panic!()
+        };
+        assert_eq!(num_rows, 2);
+        let non_null_rows: Vec<RowId> = non_null_row_ids.boxed_iter().collect();
+        assert_eq!(&non_null_rows, &[1]);
+    }
 }

columnar/src/column_index/merge/stacked.rs

@@ -1,8 +1,6 @@
-use std::ops::Range;
+use std::iter;
 
-use crate::column_index::multivalued_index::SerializableMultivalueIndex;
-use crate::column_index::serialize::SerializableOptionalIndex;
-use crate::column_index::SerializableColumnIndex;
+use crate::column_index::{SerializableColumnIndex, Set};
 use crate::iterable::Iterable;
 use crate::{Cardinality, ColumnIndex, RowId, StackMergeOrder};
 
@@ -17,140 +15,23 @@ pub fn merge_column_index_stacked<'a>(
 ) -> SerializableColumnIndex<'a> {
     match cardinality_after_merge {
         Cardinality::Full => SerializableColumnIndex::Full,
-        Cardinality::Optional => SerializableColumnIndex::Optional(SerializableOptionalIndex {
+        Cardinality::Optional => SerializableColumnIndex::Optional {
             non_null_row_ids: Box::new(StackedOptionalIndex {
                 columns,
                 stack_merge_order,
             }),
             num_rows: stack_merge_order.num_rows(),
-        }),
+        },
         Cardinality::Multivalued => {
-            let serializable_multivalue_index =
-                make_serializable_multivalued_index(columns, stack_merge_order);
-            SerializableColumnIndex::Multivalued(serializable_multivalue_index)
+            let stacked_multivalued_index = StackedMultivaluedIndex {
+                columns,
+                stack_merge_order,
+            };
+            SerializableColumnIndex::Multivalued(Box::new(stacked_multivalued_index))
         }
     }
 }
 
-struct StackedDocIdsWithValues<'a> {
-    column_indexes: &'a [ColumnIndex],
-    stack_merge_order: &'a StackMergeOrder,
-}
-
-impl Iterable<u32> for StackedDocIdsWithValues<'_> {
-    fn boxed_iter(&self) -> Box<dyn Iterator<Item = u32> + '_> {
-        Box::new((0..self.column_indexes.len()).flat_map(|i| {
-            let column_index = &self.column_indexes[i];
-            let doc_range = self.stack_merge_order.columnar_range(i);
-            get_doc_ids_with_values(column_index, doc_range)
-        }))
-    }
-}
-
-fn get_doc_ids_with_values<'a>(
-    column_index: &'a ColumnIndex,
-    doc_range: Range<u32>,
-) -> Box<dyn Iterator<Item = u32> + 'a> {
-    match column_index {
-        ColumnIndex::Empty { .. } => Box::new(0..0),
-        ColumnIndex::Full => Box::new(doc_range),
-        ColumnIndex::Optional(optional_index) => Box::new(
-            optional_index
-                .iter_rows()
-                .map(move |row| row + doc_range.start),
-        ),
-        ColumnIndex::Multivalued(multivalued_index) => Box::new(
-            multivalued_index
-                .optional_index
-                .iter_rows()
-                .map(move |row| row + doc_range.start),
-        ),
-    }
-}
-
-fn stack_doc_ids_with_values<'a>(
-    column_indexes: &'a [ColumnIndex],
-    stack_merge_order: &'a StackMergeOrder,
-) -> SerializableOptionalIndex<'a> {
-    let num_rows = stack_merge_order.num_rows();
-    SerializableOptionalIndex {
-        non_null_row_ids: Box::new(StackedDocIdsWithValues {
-            column_indexes,
-            stack_merge_order,
-        }),
-        num_rows,
-    }
-}
-
-struct StackedStartOffsets<'a> {
-    column_indexes: &'a [ColumnIndex],
-    stack_merge_order: &'a StackMergeOrder,
-}
-
-fn get_num_values_iterator<'a>(
-    column_index: &'a ColumnIndex,
-    num_docs: u32,
-) -> Box<dyn Iterator<Item = u32> + 'a> {
-    match column_index {
-        ColumnIndex::Empty { .. } => Box::new(std::iter::empty()),
-        ColumnIndex::Full => Box::new(std::iter::repeat(1u32).take(num_docs as usize)),
-        ColumnIndex::Optional(optional_index) => {
-            Box::new(std::iter::repeat(1u32).take(optional_index.num_non_nulls() as usize))
-        }
-        ColumnIndex::Multivalued(multivalued_index) => {
-            let vals: Vec<u32> = multivalued_index.start_index_column.iter().collect();
-            Box::new(
-                multivalued_index
-                    .start_index_column
-                    .iter()
-                    .scan(0u32, |previous_start_offset, current_start_offset| {
-                        let num_vals = current_start_offset - *previous_start_offset;
-                        *previous_start_offset = current_start_offset;
-                        Some(num_vals)
-                    })
-                    .skip(1),
-            )
-        }
-    }
-}
-
-impl<'a> Iterable for StackedStartOffsets<'a> {
-    fn boxed_iter(&self) -> Box<dyn Iterator<Item = u64> + '_> {
-        let num_values_it = (0..self.column_indexes.len()).flat_map(|columnar_id| {
-            let num_docs = self.stack_merge_order.columnar_range(columnar_id).len() as u32;
-            let column_index = &self.column_indexes[columnar_id];
-            get_num_values_iterator(column_index, num_docs)
-        });
-        Box::new(std::iter::once(0u64).chain(num_values_it.into_iter().scan(
-            0u64,
-            |cumulated, el| {
-                *cumulated += el as u64;
-                Some(*cumulated)
-            },
-        )))
-    }
-}
-
-fn stack_start_offsets<'a>(
-    column_indexes: &'a [ColumnIndex],
-    stack_merge_order: &'a StackMergeOrder,
-) -> Box<dyn Iterable + 'a> {
-    Box::new(StackedStartOffsets {
-        column_indexes,
-        stack_merge_order,
-    })
-}
-
-fn make_serializable_multivalued_index<'a>(
-    columns: &'a [ColumnIndex],
-    stack_merge_order: &'a StackMergeOrder,
-) -> SerializableMultivalueIndex<'a> {
-    SerializableMultivalueIndex {
-        doc_ids_with_values: stack_doc_ids_with_values(columns, stack_merge_order),
-        start_offsets: stack_start_offsets(columns, stack_merge_order),
-    }
-}
-
 struct StackedOptionalIndex<'a> {
     columns: &'a [ColumnIndex],
     stack_merge_order: &'a StackMergeOrder,
@@ -181,3 +62,87 @@ impl<'a> Iterable<RowId> for StackedOptionalIndex<'a> {
         )
     }
 }
+
+#[derive(Clone, Copy)]
+struct StackedMultivaluedIndex<'a> {
+    columns: &'a [ColumnIndex],
+    stack_merge_order: &'a StackMergeOrder,
+}
+
+fn convert_column_opt_to_multivalued_index<'a>(
+    column_index_opt: &'a ColumnIndex,
+    num_rows: RowId,
+) -> Box<dyn Iterator<Item = RowId> + 'a> {
+    match column_index_opt {
+        ColumnIndex::Empty { .. } => Box::new(iter::repeat(0u32).take(num_rows as usize + 1)),
+        ColumnIndex::Full => Box::new(0..num_rows + 1),
+        ColumnIndex::Optional(optional_index) => {
+            Box::new(
+                (0..num_rows)
+                    // TODO optimize
+                    .map(|row_id| optional_index.rank(row_id))
+                    .chain(std::iter::once(optional_index.num_non_nulls())),
+            )
+        }
+        ColumnIndex::Multivalued(multivalued_index) => multivalued_index.start_index_column.iter(),
+    }
+}
+
+impl<'a> Iterable<RowId> for StackedMultivaluedIndex<'a> {
+    fn boxed_iter(&self) -> Box<dyn Iterator<Item = RowId> + '_> {
+        let multivalued_indexes =
+            self.columns
+                .iter()
+                .enumerate()
+                .map(|(columnar_id, column_opt)| {
+                    let num_rows =
+                        self.stack_merge_order.columnar_range(columnar_id).len() as RowId;
+                    convert_column_opt_to_multivalued_index(column_opt, num_rows)
+                });
+        stack_multivalued_indexes(multivalued_indexes)
+    }
+}
+
+// Refactor me
+fn stack_multivalued_indexes<'a>(
+    mut multivalued_indexes: impl Iterator<Item = Box<dyn Iterator<Item = RowId> + 'a>> + 'a,
+) -> Box<dyn Iterator<Item = RowId> + 'a> {
+    let mut offset = 0;
+    let mut last_row_id = 0;
+    let mut current_it = multivalued_indexes.next();
+    Box::new(std::iter::from_fn(move || loop {
+        if let Some(row_id) = current_it.as_mut()?.next() {
+            last_row_id = offset + row_id;
+            return Some(last_row_id);
+        }
+        offset = last_row_id;
+        loop {
+            current_it = multivalued_indexes.next();
+            if current_it.as_mut()?.next().is_some() {
+                break;
+            }
+        }
+    }))
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::RowId;
+
+    fn it<'a>(row_ids: &'a [RowId]) -> Box<dyn Iterator<Item = RowId> + 'a> {
+        Box::new(row_ids.iter().copied())
+    }
+
+    #[test]
+    fn test_stack() {
+        let columns = [
+            it(&[0u32, 0u32]),
+            it(&[0u32, 1u32, 1u32, 4u32]),
+            it(&[0u32, 3u32, 5u32]),
+            it(&[0u32, 4u32]),
+        ]
+        .into_iter();
+        let start_offsets: Vec<RowId> = super::stack_multivalued_indexes(columns).collect();
+        assert_eq!(start_offsets, &[0, 0, 1, 1, 4, 7, 9, 13]);
+    }
+}

columnar/src/column_index/mod.rs

@@ -11,11 +11,8 @@ mod serialize;
 use std::ops::Range;
 
 pub use merge::merge_column_index;
-pub(crate) use multivalued_index::SerializableMultivalueIndex;
 pub use optional_index::{OptionalIndex, Set};
-pub use serialize::{
-    open_column_index, serialize_column_index, SerializableColumnIndex, SerializableOptionalIndex,
-};
+pub use serialize::{open_column_index, serialize_column_index, SerializableColumnIndex};
 
 use crate::column_index::multivalued_index::MultiValueIndex;
 use crate::{Cardinality, DocId, RowId};

columnar/src/column_index/multivalued_index.rs

@@ -3,64 +3,35 @@ use std::io::Write;
 use std::ops::Range;
 use std::sync::Arc;
 
-use common::{CountingWriter, OwnedBytes};
+use common::OwnedBytes;
 
-use super::optional_index::{open_optional_index, serialize_optional_index};
-use super::{OptionalIndex, SerializableOptionalIndex, Set};
 use crate::column_values::{
     load_u64_based_column_values, serialize_u64_based_column_values, CodecType, ColumnValues,
 };
 use crate::iterable::Iterable;
 use crate::{DocId, RowId};
 
-pub struct SerializableMultivalueIndex<'a> {
-    pub doc_ids_with_values: SerializableOptionalIndex<'a>,
-    pub start_offsets: Box<dyn Iterable<u64> + 'a>,
-}
-
-pub fn serialize_multivalued_index<'a>(
-    multivalued_index: &SerializableMultivalueIndex<'a>,
+pub fn serialize_multivalued_index(
+    multivalued_index: &dyn Iterable<RowId>,
     output: &mut impl Write,
 ) -> io::Result<()> {
-    let SerializableMultivalueIndex {
-        doc_ids_with_values,
-        start_offsets,
-    } = multivalued_index;
-    let mut count_writer = CountingWriter::wrap(output);
-    let SerializableOptionalIndex {
-        non_null_row_ids,
-        num_rows,
-    } = doc_ids_with_values;
-    serialize_optional_index(&**non_null_row_ids, *num_rows, &mut count_writer)?;
-    let optional_len = count_writer.written_bytes() as u32;
-    let output = count_writer.finish();
     serialize_u64_based_column_values(
-        &**start_offsets,
+        multivalued_index,
         &[CodecType::Bitpacked, CodecType::Linear],
         output,
     )?;
-    output.write_all(&optional_len.to_le_bytes())?;
     Ok(())
 }
 
 pub fn open_multivalued_index(bytes: OwnedBytes) -> io::Result<MultiValueIndex> {
-    let (body_bytes, optional_index_len) = bytes.rsplit(4);
-    let optional_index_len = u32::from_le_bytes(optional_index_len.as_slice().try_into().unwrap());
-    let (optional_index_bytes, start_index_bytes) = body_bytes.split(optional_index_len as usize);
-    let optional_index = open_optional_index(optional_index_bytes)?;
-    let start_index_column: Arc<dyn ColumnValues<RowId>> =
-        load_u64_based_column_values(start_index_bytes)?;
-    Ok(MultiValueIndex {
-        optional_index,
-        start_index_column,
-    })
+    let start_index_column: Arc<dyn ColumnValues<RowId>> = load_u64_based_column_values(bytes)?;
+    Ok(MultiValueIndex { start_index_column })
 }
 
 #[derive(Clone)]
 /// Index to resolve value range for given doc_id.
 /// Starts at 0.
 pub struct MultiValueIndex {
-    pub optional_index: OptionalIndex,
     pub start_index_column: Arc<dyn crate::ColumnValues<RowId>>,
 }
 
@@ -72,27 +43,16 @@ impl std::fmt::Debug for MultiValueIndex {
     }
 }
 
+impl From<Arc<dyn ColumnValues<RowId>>> for MultiValueIndex {
+    fn from(start_index_column: Arc<dyn ColumnValues<RowId>>) -> Self {
+        MultiValueIndex { start_index_column }
+    }
+}
+
 impl MultiValueIndex {
     pub fn for_test(start_offsets: &[RowId]) -> MultiValueIndex {
-        assert!(start_offsets.len() > 0);
-        assert_eq!(start_offsets[0], 0);
-        let mut doc_with_values = Vec::new();
-        let mut compact_start_offsets: Vec<u64> = vec![0];
-        for doc in 0..start_offsets.len() - 1 {
-            if start_offsets[doc] < start_offsets[doc + 1] {
-                doc_with_values.push(doc as RowId);
-                compact_start_offsets.push(start_offsets[doc + 1] as u64);
-            }
-        }
-        let serializable_multivalued_index = SerializableMultivalueIndex {
-            doc_ids_with_values: SerializableOptionalIndex {
-                non_null_row_ids: Box::new(&doc_with_values[..]),
-                num_rows: start_offsets.len() as u32 - 1,
-            },
-            start_offsets: Box::new(&compact_start_offsets[..]),
-        };
         let mut buffer = Vec::new();
-        serialize_multivalued_index(&serializable_multivalued_index, &mut buffer).unwrap();
+        serialize_multivalued_index(&start_offsets, &mut buffer).unwrap();
         let bytes = OwnedBytes::new(buffer);
         open_multivalued_index(bytes).unwrap()
     }
@@ -101,19 +61,15 @@ impl MultiValueIndex {
     /// the given document are `start..end`.
     #[inline]
     pub(crate) fn range(&self, doc_id: DocId) -> Range<RowId> {
-        let Some(rank) = self.optional_index.rank_if_exists(doc_id) else {
-            return 0..0;
-        };
-        let start = self.start_index_column.get_val(rank);
-        let end = self.start_index_column.get_val(rank + 1);
+        let start = self.start_index_column.get_val(doc_id);
+        let end = self.start_index_column.get_val(doc_id + 1);
         start..end
     }
 
     /// Returns the number of documents in the index.
     #[inline]
     pub fn num_docs(&self) -> u32 {
-        self.optional_index.num_docs()
-        // self.start_index_column.num_vals() - 1
+        self.start_index_column.num_vals() - 1
     }
 
     /// Converts a list of ranks (row ids of values) in a 1:n index to the corresponding list of
@@ -152,10 +108,6 @@ impl MultiValueIndex {
             }
         }
         ranks.truncate(write_doc_pos);
-
-        for rank in ranks.iter_mut() {
-            *rank = self.optional_index.select(*rank);
-        }
     }
 }
 
@@ -182,7 +134,6 @@ mod tests {
         let positions = &[10u32, 11, 15, 20, 21, 22];
         assert_eq!(index_to_pos_helper(&index, 0..5, positions), vec![1, 3, 4]);
         assert_eq!(index_to_pos_helper(&index, 1..5, positions), vec![1, 3, 4]);
-
         assert_eq!(index_to_pos_helper(&index, 0..5, &[9]), vec![0]);
         assert_eq!(index_to_pos_helper(&index, 1..5, &[10]), vec![1]);
         assert_eq!(index_to_pos_helper(&index, 1..5, &[11]), vec![1]);

columnar/src/column_index/optional_index/mod.rs

@@ -86,14 +86,8 @@ pub struct OptionalIndex {
     block_metas: Arc<[BlockMeta]>,
 }
 
-impl<'a> Iterable<u32> for &'a OptionalIndex {
-    fn boxed_iter(&self) -> Box<dyn Iterator<Item = u32> + '_> {
-        Box::new(self.iter_rows())
-    }
-}
-
 impl std::fmt::Debug for OptionalIndex {
-    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         f.debug_struct("OptionalIndex")
             .field("num_rows", &self.num_rows)
             .field("num_non_null_rows", &self.num_non_null_rows)
@@ -202,7 +196,6 @@ impl Set<RowId> for OptionalIndex {
         } = row_addr_from_row_id(doc_id);
         let block_meta = self.block_metas[block_id as usize];
         let block = self.block(block_meta);
-
         let block_offset_row_id = match block {
             Block::Dense(dense_block) => dense_block.rank(in_block_row_id),
             Block::Sparse(sparse_block) => sparse_block.rank(in_block_row_id),
@@ -256,10 +249,6 @@ impl Set<RowId> for OptionalIndex {
 }
 
 impl OptionalIndex {
-    pub fn new_empty(num_rows: RowId) -> OptionalIndex {
-        Self::for_test(num_rows, &[])
-    }
-
     pub fn for_test(num_rows: RowId, row_ids: &[RowId]) -> OptionalIndex {
         assert!(row_ids
             .last()

columnar/src/column_index/serialize.rs

@@ -3,41 +3,28 @@ use std::io::Write;
 
 use common::{CountingWriter, OwnedBytes};
 
-use super::multivalued_index::SerializableMultivalueIndex;
-use super::OptionalIndex;
 use crate::column_index::multivalued_index::serialize_multivalued_index;
 use crate::column_index::optional_index::serialize_optional_index;
 use crate::column_index::ColumnIndex;
 use crate::iterable::Iterable;
 use crate::{Cardinality, RowId};
 
-pub struct SerializableOptionalIndex<'a> {
-    pub non_null_row_ids: Box<dyn Iterable<RowId> + 'a>,
-    pub num_rows: RowId,
-}
-
-impl<'a> From<&'a OptionalIndex> for SerializableOptionalIndex<'a> {
-    fn from(optional_index: &'a OptionalIndex) -> Self {
-        SerializableOptionalIndex {
-            non_null_row_ids: Box::new(optional_index),
-            num_rows: optional_index.num_docs(),
-        }
-    }
-}
-
 pub enum SerializableColumnIndex<'a> {
     Full,
-    Optional(SerializableOptionalIndex<'a>),
+    Optional {
+        non_null_row_ids: Box<dyn Iterable<RowId> + 'a>,
+        num_rows: RowId,
+    },
     // TODO remove the Arc<dyn> apart from serialization this is not
     // dynamic at all.
-    Multivalued(SerializableMultivalueIndex<'a>),
+    Multivalued(Box<dyn Iterable<RowId> + 'a>),
 }
 
 impl<'a> SerializableColumnIndex<'a> {
     pub fn get_cardinality(&self) -> Cardinality {
         match self {
             SerializableColumnIndex::Full => Cardinality::Full,
-            SerializableColumnIndex::Optional(_) => Cardinality::Optional,
+            SerializableColumnIndex::Optional { .. } => Cardinality::Optional,
             SerializableColumnIndex::Multivalued(_) => Cardinality::Multivalued,
         }
     }
@@ -53,12 +40,12 @@ pub fn serialize_column_index(
     output.write_all(&[cardinality])?;
     match column_index {
         SerializableColumnIndex::Full => {}
-        SerializableColumnIndex::Optional(SerializableOptionalIndex {
+        SerializableColumnIndex::Optional {
             non_null_row_ids,
             num_rows,
-        }) => serialize_optional_index(non_null_row_ids.as_ref(), num_rows, &mut output)?,
+        } => serialize_optional_index(non_null_row_ids.as_ref(), num_rows, &mut output)?,
         SerializableColumnIndex::Multivalued(multivalued_index) => {
-            serialize_multivalued_index(&multivalued_index, &mut output)?
+            serialize_multivalued_index(&*multivalued_index, &mut output)?
         }
     }
     let column_index_num_bytes = output.written_bytes() as u32;

columnar/src/columnar/format_version.rs

@@ -8,7 +8,7 @@ const MAGIC_BYTES: [u8; 4] = [2, 113, 119, 66];
 
 pub fn footer() -> [u8; VERSION_FOOTER_NUM_BYTES] {
     let mut footer_bytes = [0u8; VERSION_FOOTER_NUM_BYTES];
-    footer_bytes[0..4].copy_from_slice(&Version::V2.to_bytes());
+    footer_bytes[0..4].copy_from_slice(&Version::V1.to_bytes());
     footer_bytes[4..8].copy_from_slice(&MAGIC_BYTES[..]);
     footer_bytes
 }
@@ -24,7 +24,6 @@ pub fn parse_footer(footer_bytes: [u8; VERSION_FOOTER_NUM_BYTES]) -> Result<Vers
 #[repr(u32)]
 pub enum Version {
     V1 = 1u32,
-    V2 = 2u32,
 }
 
 impl Version {
@@ -35,7 +34,7 @@ impl Version {
     fn try_from_bytes(bytes: [u8; 4]) -> Result<Version, InvalidData> {
         let code = u32::from_le_bytes(bytes);
         match code {
-            2u32 => Ok(Version::V2),
+            1u32 => Ok(Version::V1),
             _ => Err(InvalidData),
         }
     }
@@ -50,7 +49,7 @@ mod tests {
     #[test]
     fn test_footer_dserialization() {
         let parsed_version: Version = parse_footer(footer()).unwrap();
-        assert_eq!(Version::V2, parsed_version);
+        assert_eq!(Version::V1, parsed_version);
     }
 
     #[test]
@@ -64,7 +63,7 @@ mod tests {
         for &i in &version_to_tests {
             let version_res = Version::try_from_bytes(i.to_le_bytes());
             if let Ok(version) = version_res {
-                assert_eq!(version, Version::V2);
+                assert_eq!(version, Version::V1);
                 assert_eq!(version.to_bytes(), i.to_le_bytes());
                 valid_versions.insert(i);
             }

columnar/src/columnar/writer/mod.rs

@@ -12,7 +12,7 @@ use common::CountingWriter;
 pub(crate) use serializer::ColumnarSerializer;
 use stacker::{Addr, ArenaHashMap, MemoryArena};
 
-use crate::column_index::{SerializableColumnIndex, SerializableOptionalIndex};
+use crate::column_index::SerializableColumnIndex;
 use crate::column_values::{MonotonicallyMappableToU128, MonotonicallyMappableToU64};
 use crate::columnar::column_type::ColumnType;
 use crate::columnar::writer::column_writers::{
@@ -20,7 +20,6 @@ use crate::columnar::writer::column_writers::{
 };
 use crate::columnar::writer::value_index::{IndexBuilder, PreallocatedIndexBuilders};
 use crate::dictionary::{DictionaryBuilder, TermIdMapping, UnorderedId};
-use crate::iterable::Iterable;
 use crate::value::{Coerce, NumericalType, NumericalValue};
 use crate::{Cardinality, RowId};
 
@@ -60,6 +59,22 @@ pub struct ColumnarWriter {
     buffers: SpareBuffers,
 }
 
+#[inline]
+fn mutate_or_create_column<V, TMutator>(
+    arena_hash_map: &mut ArenaHashMap,
+    column_name: &str,
+    updater: TMutator,
+) where
+    V: Copy + 'static,
+    TMutator: FnMut(Option<V>) -> V,
+{
+    assert!(
+        !column_name.as_bytes().contains(&0u8),
+        "key may not contain the 0 byte"
+    );
+    arena_hash_map.mutate_or_create(column_name.as_bytes(), updater);
+}
+
 impl ColumnarWriter {
     pub fn mem_usage(&self) -> usize {
         self.arena.mem_usage()
@@ -160,8 +175,9 @@ impl ColumnarWriter {
                     },
                     &mut self.dictionaries,
                 );
-                hash_map.mutate_or_create(
-                    column_name.as_bytes(),
+                mutate_or_create_column(
+                    hash_map,
+                    column_name,
                     |column_opt: Option<StrOrBytesColumnWriter>| {
                         let mut column_writer = if let Some(column_writer) = column_opt {
                             column_writer
@@ -176,21 +192,24 @@ impl ColumnarWriter {
                 );
             }
             ColumnType::Bool => {
-                self.bool_field_hash_map.mutate_or_create(
-                    column_name.as_bytes(),
+                mutate_or_create_column(
+                    &mut self.bool_field_hash_map,
+                    column_name,
                     |column_opt: Option<ColumnWriter>| column_opt.unwrap_or_default(),
                 );
             }
             ColumnType::DateTime => {
-                self.datetime_field_hash_map.mutate_or_create(
-                    column_name.as_bytes(),
+                mutate_or_create_column(
+                    &mut self.datetime_field_hash_map,
+                    column_name,
                     |column_opt: Option<ColumnWriter>| column_opt.unwrap_or_default(),
                 );
             }
             ColumnType::I64 | ColumnType::F64 | ColumnType::U64 => {
                 let numerical_type = column_type.numerical_type().unwrap();
-                self.numerical_field_hash_map.mutate_or_create(
-                    column_name.as_bytes(),
+                mutate_or_create_column(
+                    &mut self.numerical_field_hash_map,
+                    column_name,
                     |column_opt: Option<NumericalColumnWriter>| {
                         let mut column: NumericalColumnWriter = column_opt.unwrap_or_default();
                         column.force_numerical_type(numerical_type);
@@ -198,8 +217,9 @@ impl ColumnarWriter {
                     },
                 );
             }
-            ColumnType::IpAddr => self.ip_addr_field_hash_map.mutate_or_create(
-                column_name.as_bytes(),
+            ColumnType::IpAddr => mutate_or_create_column(
+                &mut self.ip_addr_field_hash_map,
+                column_name,
                 |column_opt: Option<ColumnWriter>| column_opt.unwrap_or_default(),
             ),
         }
@@ -212,8 +232,9 @@ impl ColumnarWriter {
         numerical_value: T,
     ) {
         let (hash_map, arena) = (&mut self.numerical_field_hash_map, &mut self.arena);
-        hash_map.mutate_or_create(
-            column_name.as_bytes(),
+        mutate_or_create_column(
+            hash_map,
+            column_name,
             |column_opt: Option<NumericalColumnWriter>| {
                 let mut column: NumericalColumnWriter = column_opt.unwrap_or_default();
                 column.record_numerical_value(doc, numerical_value.into(), arena);
@@ -223,6 +244,10 @@ impl ColumnarWriter {
     }
 
     pub fn record_ip_addr(&mut self, doc: RowId, column_name: &str, ip_addr: Ipv6Addr) {
+        assert!(
+            !column_name.as_bytes().contains(&0u8),
+            "key may not contain the 0 byte"
+        );
         let (hash_map, arena) = (&mut self.ip_addr_field_hash_map, &mut self.arena);
         hash_map.mutate_or_create(
             column_name.as_bytes(),
@@ -236,30 +261,24 @@ impl ColumnarWriter {
 
     pub fn record_bool(&mut self, doc: RowId, column_name: &str, val: bool) {
         let (hash_map, arena) = (&mut self.bool_field_hash_map, &mut self.arena);
-        hash_map.mutate_or_create(
-            column_name.as_bytes(),
-            |column_opt: Option<ColumnWriter>| {
-                let mut column: ColumnWriter = column_opt.unwrap_or_default();
-                column.record(doc, val, arena);
-                column
-            },
-        );
+        mutate_or_create_column(hash_map, column_name, |column_opt: Option<ColumnWriter>| {
+            let mut column: ColumnWriter = column_opt.unwrap_or_default();
+            column.record(doc, val, arena);
+            column
+        });
     }
 
     pub fn record_datetime(&mut self, doc: RowId, column_name: &str, datetime: common::DateTime) {
         let (hash_map, arena) = (&mut self.datetime_field_hash_map, &mut self.arena);
-        hash_map.mutate_or_create(
-            column_name.as_bytes(),
-            |column_opt: Option<ColumnWriter>| {
-                let mut column: ColumnWriter = column_opt.unwrap_or_default();
-                column.record(
-                    doc,
-                    NumericalValue::I64(datetime.into_timestamp_nanos()),
-                    arena,
-                );
-                column
-            },
-        );
+        mutate_or_create_column(hash_map, column_name, |column_opt: Option<ColumnWriter>| {
+            let mut column: ColumnWriter = column_opt.unwrap_or_default();
+            column.record(
+                doc,
+                NumericalValue::I64(datetime.into_timestamp_nanos()),
+                arena,
+            );
+            column
+        });
     }
 
     pub fn record_str(&mut self, doc: RowId, column_name: &str, value: &str) {
@@ -284,6 +303,10 @@ impl ColumnarWriter {
     }
 
     pub fn record_bytes(&mut self, doc: RowId, column_name: &str, value: &[u8]) {
+        assert!(
+            !column_name.as_bytes().contains(&0u8),
+            "key may not contain the 0 byte"
+        );
         let (hash_map, arena, dictionaries) = (
             &mut self.bytes_field_hash_map,
             &mut self.arena,
@@ -636,16 +659,16 @@ fn send_to_serialize_column_mappable_to_u128<
             let optional_index_builder = value_index_builders.borrow_optional_index_builder();
             consume_operation_iterator(op_iterator, optional_index_builder, values);
             let optional_index = optional_index_builder.finish(num_rows);
-            SerializableColumnIndex::Optional(SerializableOptionalIndex {
+            SerializableColumnIndex::Optional {
                 num_rows,
                 non_null_row_ids: Box::new(optional_index),
-            })
+            }
         }
         Cardinality::Multivalued => {
             let multivalued_index_builder = value_index_builders.borrow_multivalued_index_builder();
             consume_operation_iterator(op_iterator, multivalued_index_builder, values);
-            let serializable_multivalued_index = multivalued_index_builder.finish(num_rows);
-            SerializableColumnIndex::Multivalued(serializable_multivalued_index)
+            let multivalued_index = multivalued_index_builder.finish(num_rows);
+            SerializableColumnIndex::Multivalued(Box::new(multivalued_index))
         }
     };
     crate::column::serialize_column_mappable_to_u128(
@@ -688,21 +711,19 @@ fn send_to_serialize_column_mappable_to_u64(
             let optional_index_builder = value_index_builders.borrow_optional_index_builder();
             consume_operation_iterator(op_iterator, optional_index_builder, values);
             let optional_index = optional_index_builder.finish(num_rows);
-            SerializableColumnIndex::Optional(SerializableOptionalIndex {
+            SerializableColumnIndex::Optional {
                 non_null_row_ids: Box::new(optional_index),
                 num_rows,
-            })
+            }
         }
         Cardinality::Multivalued => {
             let multivalued_index_builder = value_index_builders.borrow_multivalued_index_builder();
             consume_operation_iterator(op_iterator, multivalued_index_builder, values);
+            let multivalued_index = multivalued_index_builder.finish(num_rows);
             if sort_values_within_row {
-                // not supported in this hack
-                todo!()
-                // sort_values_within_row_in_place(multivalued_index, values);
+                sort_values_within_row_in_place(multivalued_index, values);
             }
-            let serializable_multivalued_index = multivalued_index_builder.finish(num_rows);
-            SerializableColumnIndex::Multivalued(serializable_multivalued_index)
+            SerializableColumnIndex::Multivalued(Box::new(multivalued_index))
         }
     };
     crate::column::serialize_column_mappable_to_u64(

columnar/src/columnar/writer/value_index.rs

@@ -1,4 +1,3 @@
-use crate::column_index::{SerializableMultivalueIndex, SerializableOptionalIndex};
 use crate::iterable::Iterable;
 use crate::RowId;
 
@@ -60,50 +59,32 @@ impl IndexBuilder for OptionalIndexBuilder {
 
 #[derive(Default)]
 pub struct MultivaluedIndexBuilder {
-    doc_with_values: Vec<RowId>,
-    start_offsets: Vec<u64>,
-    total_num_vals_seen: u64,
-    current_row: RowId,
-    current_row_has_value: bool,
+    start_offsets: Vec<RowId>,
+    total_num_vals_seen: u32,
 }
 
 impl MultivaluedIndexBuilder {
-    pub fn finish(&mut self, num_docs: RowId) -> SerializableMultivalueIndex<'_> {
-        self.start_offsets.push(self.total_num_vals_seen as u64);
-        let non_null_row_ids: Box<dyn Iterable<RowId>> = Box::new(&self.doc_with_values[..]);
-        SerializableMultivalueIndex {
-            doc_ids_with_values: SerializableOptionalIndex {
-                non_null_row_ids,
-                num_rows: num_docs,
-            },
-            start_offsets: Box::new(&self.start_offsets[..]),
-        }
+    pub fn finish(&mut self, num_docs: RowId) -> &[u32] {
+        self.start_offsets
+            .resize(num_docs as usize + 1, self.total_num_vals_seen);
+        &self.start_offsets[..]
     }
 
     fn reset(&mut self) {
-        self.doc_with_values.clear();
         self.start_offsets.clear();
+        self.start_offsets.push(0u32);
         self.total_num_vals_seen = 0;
-        self.current_row = 0;
-        self.current_row_has_value = false;
     }
 }
 
 impl IndexBuilder for MultivaluedIndexBuilder {
     fn record_row(&mut self, row_id: RowId) {
-        self.current_row = row_id;
-        self.current_row_has_value = false;
-        // self.start_offsets
-        //     .resize(row_id as usize + 1, self.total_num_vals_seen);
+        self.start_offsets
+            .resize(row_id as usize + 1, self.total_num_vals_seen);
     }
 
     fn record_value(&mut self) {
-        if !self.current_row_has_value {
-            self.current_row_has_value = true;
-            self.doc_with_values.push(self.current_row);
-            self.start_offsets.push(self.total_num_vals_seen as u64);
-        }
-        self.total_num_vals_seen += 1u64;
+        self.total_num_vals_seen += 1;
     }
 }
 
@@ -160,32 +141,6 @@ mod tests {
         );
     }
 
-    #[test]
-    fn test_multivalued_value_index_builder_simple() {
-        let mut multivalued_value_index_builder = MultivaluedIndexBuilder::default();
-        {
-            multivalued_value_index_builder.record_row(0u32);
-            multivalued_value_index_builder.record_value();
-            multivalued_value_index_builder.record_value();
-            let serialized_multivalue_index = multivalued_value_index_builder.finish(1u32);
-            let start_offsets: Vec<u64> = serialized_multivalue_index
-                .start_offsets
-                .boxed_iter()
-                .collect();
-            assert_eq!(&start_offsets, &[0, 2]);
-        }
-        multivalued_value_index_builder.reset();
-        multivalued_value_index_builder.record_row(0u32);
-        multivalued_value_index_builder.record_value();
-        multivalued_value_index_builder.record_value();
-        let serialized_multivalue_index = multivalued_value_index_builder.finish(1u32);
-        let start_offsets: Vec<u64> = serialized_multivalue_index
-            .start_offsets
-            .boxed_iter()
-            .collect();
-        assert_eq!(&start_offsets, &[0, 2]);
-    }
-
     #[test]
     fn test_multivalued_value_index_builder() {
         let mut multivalued_value_index_builder = MultivaluedIndexBuilder::default();
@@ -194,30 +149,17 @@ mod tests {
         multivalued_value_index_builder.record_value();
         multivalued_value_index_builder.record_row(2u32);
         multivalued_value_index_builder.record_value();
-        let SerializableMultivalueIndex {
-            doc_ids_with_values,
-            start_offsets,
-        } = multivalued_value_index_builder.finish(4u32);
-        assert_eq!(doc_ids_with_values.num_rows, 4u32);
-        let doc_ids_with_values: Vec<u32> =
-            doc_ids_with_values.non_null_row_ids.boxed_iter().collect();
-        assert_eq!(&doc_ids_with_values, &[1u32, 2u32]);
-        let start_offsets: Vec<u64> = start_offsets.boxed_iter().collect::<Vec<u64>>();
-        assert_eq!(&start_offsets[..], &[0, 2, 3]);
-        // assert!(doc_ids_with_values_opt.is_some());
-        // assert!(doc_ids_with_values_opt.is_some());
-
-        // assert_eq!(
-        //     multivalued_value_index_builder.finish(4u32).to_vec(),
-        //     vec![0, 0, 2, 3, 3]
-        // );
-        // multivalued_value_index_builder.reset();
-        // multivalued_value_index_builder.record_row(2u32);
-        // multivalued_value_index_builder.record_value();
-        // multivalued_value_index_builder.record_value();
-        // assert_eq!(
-        //     multivalued_value_index_builder.finish(4u32).to_vec(),
-        //     vec![0, 0, 0, 2, 2]
-        // );
+        assert_eq!(
+            multivalued_value_index_builder.finish(4u32).to_vec(),
+            vec![0, 0, 2, 3, 3]
+        );
+        multivalued_value_index_builder.reset();
+        multivalued_value_index_builder.record_row(2u32);
+        multivalued_value_index_builder.record_value();
+        multivalued_value_index_builder.record_value();
+        assert_eq!(
+            multivalued_value_index_builder.finish(4u32).to_vec(),
+            vec![0, 0, 0, 2, 2]
+        );
     }
 }

columnar/src/iterable.rs

@@ -1,7 +1,4 @@
 use std::ops::Range;
-use std::sync::Arc;
-
-use crate::{ColumnValues, RowId};
 
 pub trait Iterable<T = u64> {
     fn boxed_iter(&self) -> Box<dyn Iterator<Item = T> + '_>;
@@ -20,9 +17,3 @@ where Range<T>: Iterator<Item = T>
         Box::new(self.clone())
     }
 }
-
-impl Iterable for Arc<dyn crate::ColumnValues<RowId>> {
-    fn boxed_iter(&self) -> Box<dyn Iterator<Item = u64> + '_> {
-        Box::new(self.iter().map(|row_id| row_id as u64))
-    }
-}

columnar/src/tests.rs

@@ -11,7 +11,7 @@ use crate::columnar::{ColumnType, ColumnTypeCategory};
 use crate::dynamic_column::{DynamicColumn, DynamicColumnHandle};
 use crate::value::{Coerce, NumericalValue};
 use crate::{
-    BytesColumn, Cardinality, Column, ColumnIndex, ColumnarReader, ColumnarWriter, RowAddr, RowId,
+    BytesColumn, Cardinality, Column, ColumnarReader, ColumnarWriter, RowAddr, RowId,
     ShuffleMergeOrder, StackMergeOrder,
 };
 
@@ -79,7 +79,7 @@ fn test_dataframe_writer_u64_multivalued() {
     assert_eq!(columnar.num_columns(), 1);
     let cols: Vec<DynamicColumnHandle> = columnar.read_columns("divisor").unwrap();
     assert_eq!(cols.len(), 1);
-    assert_eq!(cols[0].num_bytes(), 50);
+    assert_eq!(cols[0].num_bytes(), 29);
     let dyn_i64_col = cols[0].open().unwrap();
     let DynamicColumn::I64(divisor_col) = dyn_i64_col else {
         panic!();
@@ -448,7 +448,6 @@ fn assert_columnar_eq(
     }
 }
 
-#[track_caller]
 fn assert_column_eq<T: Copy + PartialOrd + Debug + Send + Sync + 'static>(
     left: &Column<T>,
     right: &Column<T>,
@@ -842,27 +841,26 @@ fn columnar_docs_and_remap(
     )
 }
 
-// proptest! {
-//     #![proptest_config(ProptestConfig::with_cases(1000))]
-//     #[test]
-//     fn test_columnar_merge_and_remap_proptest((columnar_docs, shuffle_merge_order) in
-// columnar_docs_and_remap()) {         let shuffled_rows: Vec<Vec<(&'static str, ColumnValue)>> =
-// shuffle_merge_order.iter()             .map(|row_addr| columnar_docs[row_addr.segment_ord as
-// usize][row_addr.row_id as usize].clone())             .collect();
-//         let expected_merged_columnar = build_columnar(&shuffled_rows[..]);
-//         let columnar_readers: Vec<ColumnarReader> = columnar_docs.iter()
-//             .map(|docs| build_columnar(&docs[..]))
-//             .collect::<Vec<_>>();
-//         let columnar_readers_arr: Vec<&ColumnarReader> = columnar_readers.iter().collect();
-//         let mut output: Vec<u8> = Vec::new();
-//         let segment_num_rows: Vec<RowId> = columnar_docs.iter().map(|docs| docs.len() as
-// RowId).collect();         let shuffle_merge_order =
-// ShuffleMergeOrder::for_test(&segment_num_rows, shuffle_merge_order);
-//         crate::merge_columnar(&columnar_readers_arr[..], &[], shuffle_merge_order.into(), &mut
-// output).unwrap();         let merged_columnar = ColumnarReader::open(output).unwrap();
-//         assert_columnar_eq(&merged_columnar, &expected_merged_columnar, true);
-//     }
-// }
+proptest! {
+    #![proptest_config(ProptestConfig::with_cases(1000))]
+    #[test]
+    fn test_columnar_merge_and_remap_proptest((columnar_docs, shuffle_merge_order) in columnar_docs_and_remap()) {
+        let shuffled_rows: Vec<Vec<(&'static str, ColumnValue)>> = shuffle_merge_order.iter()
+            .map(|row_addr| columnar_docs[row_addr.segment_ord as usize][row_addr.row_id as usize].clone())
+            .collect();
+        let expected_merged_columnar = build_columnar(&shuffled_rows[..]);
+        let columnar_readers: Vec<ColumnarReader> = columnar_docs.iter()
+            .map(|docs| build_columnar(&docs[..]))
+            .collect::<Vec<_>>();
+        let columnar_readers_arr: Vec<&ColumnarReader> = columnar_readers.iter().collect();
+        let mut output: Vec<u8> = Vec::new();
+        let segment_num_rows: Vec<RowId> = columnar_docs.iter().map(|docs| docs.len() as RowId).collect();
+        let shuffle_merge_order = ShuffleMergeOrder::for_test(&segment_num_rows, shuffle_merge_order);
+        crate::merge_columnar(&columnar_readers_arr[..], &[], shuffle_merge_order.into(), &mut output).unwrap();
+        let merged_columnar = ColumnarReader::open(output).unwrap();
+        assert_columnar_eq(&merged_columnar, &expected_merged_columnar, true);
+    }
+}
 
 #[test]
 fn test_columnar_merge_empty() {
@@ -884,64 +882,64 @@ fn test_columnar_merge_empty() {
     assert_eq!(merged_columnar.num_columns(), 0);
 }
 
-// #[test]
-// fn test_columnar_merge_single_str_column() {
-//     let columnar_reader_1 = build_columnar(&[]);
-//     let rows: &[Vec<_>] = &[vec![("c1", ColumnValue::Str("a"))]][..];
-//     let columnar_reader_2 = build_columnar(rows);
-//     let mut output: Vec<u8> = Vec::new();
-//     let segment_num_rows: Vec<RowId> = vec![0, 1];
-//     let shuffle_merge_order = ShuffleMergeOrder::for_test(
-//         &segment_num_rows,
-//         vec![RowAddr {
-//             segment_ord: 1u32,
-//             row_id: 0u32,
-//         }],
-//     );
-//     crate::merge_columnar(
-//         &[&columnar_reader_1, &columnar_reader_2],
-//         &[],
-//         shuffle_merge_order.into(),
-//         &mut output,
-//     )
-//     .unwrap();
-//     let merged_columnar = ColumnarReader::open(output).unwrap();
-//     assert_eq!(merged_columnar.num_rows(), 1);
-//     assert_eq!(merged_columnar.num_columns(), 1);
-// }
+#[test]
+fn test_columnar_merge_single_str_column() {
+    let columnar_reader_1 = build_columnar(&[]);
+    let rows: &[Vec<_>] = &[vec![("c1", ColumnValue::Str("a"))]][..];
+    let columnar_reader_2 = build_columnar(rows);
+    let mut output: Vec<u8> = Vec::new();
+    let segment_num_rows: Vec<RowId> = vec![0, 1];
+    let shuffle_merge_order = ShuffleMergeOrder::for_test(
+        &segment_num_rows,
+        vec![RowAddr {
+            segment_ord: 1u32,
+            row_id: 0u32,
+        }],
+    );
+    crate::merge_columnar(
+        &[&columnar_reader_1, &columnar_reader_2],
+        &[],
+        shuffle_merge_order.into(),
+        &mut output,
+    )
+    .unwrap();
+    let merged_columnar = ColumnarReader::open(output).unwrap();
+    assert_eq!(merged_columnar.num_rows(), 1);
+    assert_eq!(merged_columnar.num_columns(), 1);
+}
 
-// #[test]
-// fn test_delete_decrease_cardinality() {
-//     let columnar_reader_1 = build_columnar(&[]);
-//     let rows: &[Vec<_>] = &[
-//         vec![
-//             ("c", ColumnValue::from(0i64)),
-//             ("c", ColumnValue::from(0i64)),
-//         ],
-//         vec![("c", ColumnValue::from(0i64))],
-//     ][..];
-//     // c is multivalued here
-//     let columnar_reader_2 = build_columnar(rows);
-//     let mut output: Vec<u8> = Vec::new();
-//     let shuffle_merge_order = ShuffleMergeOrder::for_test(
-//         &[0, 2],
-//         vec![RowAddr {
-//             segment_ord: 1u32,
-//             row_id: 1u32,
-//         }],
-//     );
-//     crate::merge_columnar(
-//         &[&columnar_reader_1, &columnar_reader_2],
-//         &[],
-//         shuffle_merge_order.into(),
-//         &mut output,
-//     )
-//     .unwrap();
-//     let merged_columnar = ColumnarReader::open(output).unwrap();
-//     assert_eq!(merged_columnar.num_rows(), 1);
-//     assert_eq!(merged_columnar.num_columns(), 1);
-//     let cols = merged_columnar.read_columns("c").unwrap();
-//     assert_eq!(cols.len(), 1);
-//     assert_eq!(cols[0].column_type(), ColumnType::I64);
-//     assert_eq!(cols[0].open().unwrap().get_cardinality(), Cardinality::Full);
-// }
+#[test]
+fn test_delete_decrease_cardinality() {
+    let columnar_reader_1 = build_columnar(&[]);
+    let rows: &[Vec<_>] = &[
+        vec![
+            ("c", ColumnValue::from(0i64)),
+            ("c", ColumnValue::from(0i64)),
+        ],
+        vec![("c", ColumnValue::from(0i64))],
+    ][..];
+    // c is multivalued here
+    let columnar_reader_2 = build_columnar(rows);
+    let mut output: Vec<u8> = Vec::new();
+    let shuffle_merge_order = ShuffleMergeOrder::for_test(
+        &[0, 2],
+        vec![RowAddr {
+            segment_ord: 1u32,
+            row_id: 1u32,
+        }],
+    );
+    crate::merge_columnar(
+        &[&columnar_reader_1, &columnar_reader_2],
+        &[],
+        shuffle_merge_order.into(),
+        &mut output,
+    )
+    .unwrap();
+    let merged_columnar = ColumnarReader::open(output).unwrap();
+    assert_eq!(merged_columnar.num_rows(), 1);
+    assert_eq!(merged_columnar.num_columns(), 1);
+    let cols = merged_columnar.read_columns("c").unwrap();
+    assert_eq!(cols.len(), 1);
+    assert_eq!(cols[0].column_type(), ColumnType::I64);
+    assert_eq!(cols[0].open().unwrap().get_cardinality(), Cardinality::Full);
+}

common/src/bitset.rs

@@ -1,5 +1,5 @@
 use std::io::Write;
-use std::{fmt, io};
+use std::{fmt, io, u64};
 
 use ownedbytes::OwnedBytes;
 

common/src/json_path_writer.rs

@@ -5,12 +5,6 @@ pub const JSON_PATH_SEGMENT_SEP: u8 = 1u8;
 pub const JSON_PATH_SEGMENT_SEP_STR: &str =
     unsafe { std::str::from_utf8_unchecked(&[JSON_PATH_SEGMENT_SEP]) };
 
-/// Separates the json path and the value in
-/// a JSON term binary representation.
-pub const JSON_END_OF_PATH: u8 = 0u8;
-pub const JSON_END_OF_PATH_STR: &str =
-    unsafe { std::str::from_utf8_unchecked(&[JSON_END_OF_PATH]) };
-
 /// Create a new JsonPathWriter, that creates flattened json paths for tantivy.
 #[derive(Clone, Debug, Default)]
 pub struct JsonPathWriter {
@@ -20,14 +14,6 @@ pub struct JsonPathWriter {
 }
 
 impl JsonPathWriter {
-    pub fn with_expand_dots(expand_dots: bool) -> Self {
-        JsonPathWriter {
-            path: String::new(),
-            indices: Vec::new(),
-            expand_dots,
-        }
-    }
-
     pub fn new() -> Self {
         JsonPathWriter {
             path: String::new(),
@@ -53,8 +39,8 @@ impl JsonPathWriter {
     pub fn push(&mut self, segment: &str) {
         let len_path = self.path.len();
         self.indices.push(len_path);
-        if self.indices.len() > 1 {
-            self.path.push(JSON_PATH_SEGMENT_SEP as char);
+        if !self.path.is_empty() {
+            self.path.push_str(JSON_PATH_SEGMENT_SEP_STR);
         }
         self.path.push_str(segment);
         if self.expand_dots {
@@ -69,12 +55,6 @@ impl JsonPathWriter {
         }
     }
 
-    /// Set the end of JSON path marker.
-    #[inline]
-    pub fn set_end(&mut self) {
-        self.path.push_str(JSON_END_OF_PATH_STR);
-    }
-
     /// Remove the last segment. Does nothing if the path is empty.
     #[inline]
     pub fn pop(&mut self) {
@@ -111,7 +91,6 @@ mod tests {
     #[test]
     fn json_path_writer_test() {
         let mut writer = JsonPathWriter::new();
-        writer.set_expand_dots(false);
 
         writer.push("root");
         assert_eq!(writer.as_str(), "root");
@@ -130,15 +109,4 @@ mod tests {
         writer.push("k8s.node.id");
         assert_eq!(writer.as_str(), "root\u{1}k8s\u{1}node\u{1}id");
     }
-
-    #[test]
-    fn test_json_path_expand_dots_enabled_pop_segment() {
-        let mut json_writer = JsonPathWriter::with_expand_dots(true);
-        json_writer.push("hello");
-        assert_eq!(json_writer.as_str(), "hello");
-        json_writer.push("color.hue");
-        assert_eq!(json_writer.as_str(), "hello\x01color\x01hue");
-        json_writer.pop();
-        assert_eq!(json_writer.as_str(), "hello");
-    }
 }

common/src/lib.rs

@@ -9,7 +9,7 @@ mod byte_count;
 mod datetime;
 pub mod file_slice;
 mod group_by;
-pub mod json_path_writer;
+mod json_path_writer;
 mod serialize;
 mod vint;
 mod writer;

common/src/vint.rs

@@ -151,7 +151,7 @@ pub fn read_u32_vint_no_advance(data: &[u8]) -> (u32, usize) {
     (result, vlen)
 }
 /// Write a `u32` as a vint payload.
-pub fn write_u32_vint<W: io::Write + ?Sized>(val: u32, writer: &mut W) -> io::Result<()> {
+pub fn write_u32_vint<W: io::Write>(val: u32, writer: &mut W) -> io::Result<()> {
     let mut buf = [0u8; 8];
     let data = serialize_vint_u32(val, &mut buf);
     writer.write_all(data)

examples/custom_collector.rs

@@ -11,10 +11,9 @@ use columnar::Column;
 // ---
 // Importing tantivy...
 use tantivy::collector::{Collector, SegmentCollector};
-use tantivy::index::SegmentReader;
 use tantivy::query::QueryParser;
 use tantivy::schema::{Schema, FAST, INDEXED, TEXT};
-use tantivy::{doc, Index, IndexWriter, Score};
+use tantivy::{doc, Index, IndexWriter, Score, SegmentReader};
 
 #[derive(Default)]
 struct Stats {

examples/date_time_field.rs

@@ -4,7 +4,7 @@
 
 use tantivy::collector::TopDocs;
 use tantivy::query::QueryParser;
-use tantivy::schema::{DateOptions, Document, Schema, Value, INDEXED, STORED, STRING};
+use tantivy::schema::{DateOptions, Document, OwnedValue, Schema, INDEXED, STORED, STRING};
 use tantivy::{Index, IndexWriter, TantivyDocument};
 
 fn main() -> tantivy::Result<()> {
@@ -13,7 +13,7 @@ fn main() -> tantivy::Result<()> {
     let opts = DateOptions::from(INDEXED)
         .set_stored()
         .set_fast()
-        .set_precision(tantivy::schema::DateTimePrecision::Seconds);
+        .set_precision(tantivy::DateTimePrecision::Seconds);
     // Add `occurred_at` date field type
     let occurred_at = schema_builder.add_date_field("occurred_at", opts);
     let event_type = schema_builder.add_text_field("event", STRING | STORED);
@@ -61,12 +61,10 @@ fn main() -> tantivy::Result<()> {
         assert_eq!(count_docs.len(), 1);
         for (_score, doc_address) in count_docs {
             let retrieved_doc = searcher.doc::<TantivyDocument>(doc_address)?;
-            assert!(retrieved_doc
-                .get_first(occurred_at)
-                .unwrap()
-                .as_value()
-                .as_datetime()
-                .is_some(),);
+            assert!(matches!(
+                retrieved_doc.get_first(occurred_at),
+                Some(OwnedValue::Date(_))
+            ));
             assert_eq!(
                 retrieved_doc.to_json(&schema),
                 r#"{"event":["comment"],"occurred_at":["2022-06-22T13:00:00.22Z"]}"#

examples/faceted_search_with_tweaked_score.rs

@@ -51,7 +51,7 @@ fn main() -> tantivy::Result<()> {
     let reader = index.reader()?;
     let searcher = reader.searcher();
     {
-        let facets = [
+        let facets = vec![
             Facet::from("/ingredient/egg"),
             Facet::from("/ingredient/oil"),
             Facet::from("/ingredient/garlic"),
@@ -94,8 +94,9 @@ fn main() -> tantivy::Result<()> {
                     .doc::<TantivyDocument>(*doc_id)
                     .unwrap()
                     .get_first(title)
-                    .and_then(|v| v.as_str().map(|el| el.to_string()))
+                    .and_then(|v| v.as_str())
                     .unwrap()
+                    .to_owned()
             })
             .collect();
         assert_eq!(titles, vec!["Fried egg", "Egg rolls"]);

examples/index_from_multiple_threads.rs

@@ -61,7 +61,7 @@ fn main() -> tantivy::Result<()> {
                         debris of the winter’s flooding; and sycamores with mottled, white, recumbent \
                         limbs and branches that arch over the pool"
                     ))?;
-            println!("add doc {i} from thread 1 - opstamp {opstamp}");
+            println!("add doc {} from thread 1 - opstamp {}", i, opstamp);
             thread::sleep(Duration::from_millis(20));
         }
         Result::<(), TantivyError>::Ok(())
@@ -82,7 +82,7 @@ fn main() -> tantivy::Result<()> {
                     body => "Some great book description..."
                 ))?
             };
-            println!("add doc {i} from thread 2 - opstamp {opstamp}");
+            println!("add doc {} from thread 2 - opstamp {}", i, opstamp);
             thread::sleep(Duration::from_millis(10));
         }
         Result::<(), TantivyError>::Ok(())

examples/iterating_docs_and_positions.rs

@@ -7,11 +7,10 @@
 // the list of documents containing a term, getting
 // its term frequency, and accessing its positions.
 
-use tantivy::postings::Postings;
 // ---
 // Importing tantivy...
 use tantivy::schema::*;
-use tantivy::{doc, DocSet, Index, IndexWriter, TERMINATED};
+use tantivy::{doc, DocSet, Index, IndexWriter, Postings, TERMINATED};
 
 fn main() -> tantivy::Result<()> {
     // We first create a schema for the sake of the

examples/warmer.rs

@@ -3,11 +3,10 @@ use std::collections::{HashMap, HashSet};
 use std::sync::{Arc, RwLock, Weak};
 
 use tantivy::collector::TopDocs;
-use tantivy::index::SegmentId;
 use tantivy::query::QueryParser;
 use tantivy::schema::{Schema, FAST, TEXT};
 use tantivy::{
-    doc, DocAddress, DocId, Index, IndexWriter, Opstamp, Searcher, SearcherGeneration,
+    doc, DocAddress, DocId, Index, IndexWriter, Opstamp, Searcher, SearcherGeneration, SegmentId,
     SegmentReader, Warmer,
 };
 

query-grammar/src/query_grammar.rs

@@ -1,4 +1,3 @@
-use std::borrow::Cow;
 use std::iter::once;
 
 use nom::branch::alt;
@@ -20,7 +19,7 @@ use crate::Occur;
 // Note: '-' char is only forbidden at the beginning of a field name, would be clearer to add it to
 // special characters.
 const SPECIAL_CHARS: &[char] = &[
-    '+', '^', '`', ':', '{', '}', '"', '\'', '[', ']', '(', ')', '!', '\\', '*', ' ',
+    '+', '^', '`', ':', '{', '}', '"', '[', ']', '(', ')', '!', '\\', '*', ' ',
 ];
 
 /// consume a field name followed by colon. Return the field name with escape sequence
@@ -42,92 +41,36 @@ fn field_name(inp: &str) -> IResult<&str, String> {
     )(inp)
 }
 
-const ESCAPE_IN_WORD: &[char] = &['^', '`', ':', '{', '}', '"', '\'', '[', ']', '(', ')', '\\'];
-
-fn interpret_escape(source: &str) -> String {
-    let mut res = String::with_capacity(source.len());
-    let mut in_escape = false;
-    let require_escape = |c: char| c.is_whitespace() || ESCAPE_IN_WORD.contains(&c) || c == '-';
-
-    for c in source.chars() {
-        if in_escape {
-            if !require_escape(c) {
-                // we re-add the escape sequence
-                res.push('\\');
-            }
-            res.push(c);
-            in_escape = false;
-        } else if c == '\\' {
-            in_escape = true;
-        } else {
-            res.push(c);
-        }
-    }
-    res
-}
-
 /// Consume a word outside of any context.
 // TODO should support escape sequences
-fn word(inp: &str) -> IResult<&str, Cow<str>> {
+fn word(inp: &str) -> IResult<&str, &str> {
     map_res(
         recognize(tuple((
-            alt((
-                preceded(char('\\'), anychar),
-                satisfy(|c| !c.is_whitespace() && !ESCAPE_IN_WORD.contains(&c) && c != '-'),
-            )),
-            many0(alt((
-                preceded(char('\\'), anychar),
-                satisfy(|c: char| !c.is_whitespace() && !ESCAPE_IN_WORD.contains(&c)),
-            ))),
+            satisfy(|c| {
+                !c.is_whitespace()
+                    && !['-', '^', '`', ':', '{', '}', '"', '[', ']', '(', ')'].contains(&c)
+            }),
+            many0(satisfy(|c: char| {
+                !c.is_whitespace() && ![':', '^', '{', '}', '"', '[', ']', '(', ')'].contains(&c)
+            })),
         ))),
         |s| match s {
             "OR" | "AND" | "NOT" | "IN" => Err(Error::new(inp, ErrorKind::Tag)),
-            s if s.contains('\\') => Ok(Cow::Owned(interpret_escape(s))),
-            s => Ok(Cow::Borrowed(s)),
+            _ => Ok(s),
         },
     )(inp)
 }
 
-fn word_infallible(
-    delimiter: &str,
-    emit_error: bool,
-) -> impl Fn(&str) -> JResult<&str, Option<Cow<str>>> + '_ {
-    // emit error is set when receiving an unescaped `:` should emit an error
-
-    move |inp| {
-        map(
-            opt_i_err(
-                preceded(
-                    multispace0,
-                    recognize(many1(alt((
-                        preceded(char::<&str, _>('\\'), anychar),
-                        satisfy(|c| !c.is_whitespace() && !delimiter.contains(c)),
-                    )))),
-                ),
-                "expected word",
+fn word_infallible(delimiter: &str) -> impl Fn(&str) -> JResult<&str, Option<&str>> + '_ {
+    |inp| {
+        opt_i_err(
+            preceded(
+                multispace0,
+                recognize(many1(satisfy(|c| {
+                    !c.is_whitespace() && !delimiter.contains(c)
+                }))),
             ),
-            |(opt_s, mut errors)| match opt_s {
-                Some(s) => {
-                    if emit_error
-                        && (s
-                            .as_bytes()
-                            .windows(2)
-                            .any(|window| window[0] != b'\\' && window[1] == b':')
-                            || s.starts_with(':'))
-                    {
-                        errors.push(LenientErrorInternal {
-                            pos: inp.len(),
-                            message: "parsed possible invalid field as term".to_string(),
-                        });
-                    }
-                    if s.contains('\\') {
-                        (Some(Cow::Owned(interpret_escape(s))), errors)
-                    } else {
-                        (Some(Cow::Borrowed(s)), errors)
-                    }
-                }
-                None => (None, errors),
-            },
+            "expected word",
         )(inp)
     }
 }
@@ -216,7 +159,7 @@ fn simple_term_infallible(
                 (value((), char('\'')), simple_quotes),
             ),
             // numbers are parsed with words in this case, as we allow string starting with a -
-            map(word_infallible(delimiter, true), |(text, errors)| {
+            map(word_infallible(delimiter), |(text, errors)| {
                 (text.map(|text| (Delimiter::None, text.to_string())), errors)
             }),
         )(inp)
@@ -275,14 +218,27 @@ fn term_or_phrase_infallible(inp: &str) -> JResult<&str, Option<UserInputLeaf>>
 }
 
 fn term_group(inp: &str) -> IResult<&str, UserInputAst> {
+    let occur_symbol = alt((
+        value(Occur::MustNot, char('-')),
+        value(Occur::Must, char('+')),
+    ));
+
     map(
         tuple((
             terminated(field_name, multispace0),
-            delimited(tuple((char('('), multispace0)), ast, char(')')),
+            delimited(
+                tuple((char('('), multispace0)),
+                separated_list0(multispace1, tuple((opt(occur_symbol), term_or_phrase))),
+                char(')'),
+            ),
         )),
-        |(field_name, mut ast)| {
-            ast.set_default_field(field_name);
-            ast
+        |(field_name, terms)| {
+            UserInputAst::Clause(
+                terms
+                    .into_iter()
+                    .map(|(occur, leaf)| (occur, leaf.set_field(Some(field_name.clone())).into()))
+                    .collect(),
+            )
         },
     )(inp)
 }
@@ -302,18 +258,46 @@ fn term_group_precond(inp: &str) -> IResult<&str, (), ()> {
 }
 
 fn term_group_infallible(inp: &str) -> JResult<&str, UserInputAst> {
-    let (inp, (field_name, _, _, _)) =
+    let (mut inp, (field_name, _, _, _)) =
         tuple((field_name, multispace0, char('('), multispace0))(inp).expect("precondition failed");
 
-    let res = delimited_infallible(
-        nothing,
-        map(ast_infallible, |(mut ast, errors)| {
-            ast.set_default_field(field_name.to_string());
-            (ast, errors)
-        }),
-        opt_i_err(char(')'), "expected ')'"),
-    )(inp);
-    res
+    let mut terms = Vec::new();
+    let mut errs = Vec::new();
+
+    let mut first_round = true;
+    loop {
+        let mut space_error = if first_round {
+            first_round = false;
+            Vec::new()
+        } else {
+            let (rest, (_, err)) = space1_infallible(inp)?;
+            inp = rest;
+            err
+        };
+        if inp.is_empty() {
+            errs.push(LenientErrorInternal {
+                pos: inp.len(),
+                message: "missing )".to_string(),
+            });
+            break Ok((inp, (UserInputAst::Clause(terms), errs)));
+        }
+        if let Some(inp) = inp.strip_prefix(')') {
+            break Ok((inp, (UserInputAst::Clause(terms), errs)));
+        }
+        // only append missing space error if we did not reach the end of group
+        errs.append(&mut space_error);
+
+        // here we do the assumption term_or_phrase_infallible always consume something if the
+        // first byte is not `)` or ' '. If it did not, we would end up looping.
+
+        let (rest, ((occur, leaf), mut err)) =
+            tuple_infallible((occur_symbol, term_or_phrase_infallible))(inp)?;
+        errs.append(&mut err);
+        if let Some(leaf) = leaf {
+            terms.push((occur, leaf.set_field(Some(field_name.clone())).into()));
+        }
+        inp = rest;
+    }
 }
 
 fn exists(inp: &str) -> IResult<&str, UserInputLeaf> {
@@ -379,6 +363,15 @@ fn literal_no_group_infallible(inp: &str) -> JResult<&str, Option<UserInputAst>>
         |((field_name, _, leaf), mut errors)| {
             (
                 leaf.map(|leaf| {
+                    if matches!(&leaf, UserInputLeaf::Literal(literal)
+                            if literal.phrase.contains(':') && literal.delimiter == Delimiter::None)
+                        && field_name.is_none()
+                    {
+                        errors.push(LenientErrorInternal {
+                            pos: inp.len(),
+                            message: "parsed possible invalid field as term".to_string(),
+                        });
+                    }
                     if matches!(&leaf, UserInputLeaf::Literal(literal)
                             if literal.phrase == "NOT" && literal.delimiter == Delimiter::None)
                         && field_name.is_none()
@@ -497,20 +490,20 @@ fn range_infallible(inp: &str) -> JResult<&str, UserInputLeaf> {
         tuple_infallible((
             opt_i(anychar),
             space0_infallible,
-            word_infallible("]}", false),
+            word_infallible("]}"),
             space1_infallible,
             opt_i_err(
                 terminated(tag("TO"), alt((value((), multispace1), value((), eof)))),
                 "missing keyword TO",
             ),
-            word_infallible("]}", false),
+            word_infallible("]}"),
             opt_i_err(one_of("]}"), "missing range delimiter"),
         )),
         |(
             (lower_bound_kind, _multispace0, lower, _multispace1, to, upper, upper_bound_kind),
             errs,
         )| {
-            let lower_bound = match (lower_bound_kind, lower.as_deref()) {
+            let lower_bound = match (lower_bound_kind, lower) {
                 (_, Some("*")) => UserInputBound::Unbounded,
                 (_, None) => UserInputBound::Unbounded,
                 // if it is some, TO was actually the bound (i.e. [TO TO something])
@@ -519,7 +512,7 @@ fn range_infallible(inp: &str) -> JResult<&str, UserInputLeaf> {
                 (Some('{'), Some(bound)) => UserInputBound::Exclusive(bound.to_string()),
                 _ => unreachable!("precondition failed, range did not start with [ or {{"),
             };
-            let upper_bound = match (upper_bound_kind, upper.as_deref()) {
+            let upper_bound = match (upper_bound_kind, upper) {
                 (_, Some("*")) => UserInputBound::Unbounded,
                 (_, None) => UserInputBound::Unbounded,
                 (Some(']'), Some(bound)) => UserInputBound::Inclusive(bound.to_string()),
@@ -536,7 +529,7 @@ fn range_infallible(inp: &str) -> JResult<&str, UserInputLeaf> {
             (
                 (
                     value((), tag(">=")),
-                    map(word_infallible("", false), |(bound, err)| {
+                    map(word_infallible(""), |(bound, err)| {
                         (
                             (
                                 bound
@@ -550,7 +543,7 @@ fn range_infallible(inp: &str) -> JResult<&str, UserInputLeaf> {
                 ),
                 (
                     value((), tag("<=")),
-                    map(word_infallible("", false), |(bound, err)| {
+                    map(word_infallible(""), |(bound, err)| {
                         (
                             (
                                 UserInputBound::Unbounded,
@@ -564,7 +557,7 @@ fn range_infallible(inp: &str) -> JResult<&str, UserInputLeaf> {
                 ),
                 (
                     value((), tag(">")),
-                    map(word_infallible("", false), |(bound, err)| {
+                    map(word_infallible(""), |(bound, err)| {
                         (
                             (
                                 bound
@@ -578,7 +571,7 @@ fn range_infallible(inp: &str) -> JResult<&str, UserInputLeaf> {
                 ),
                 (
                     value((), tag("<")),
-                    map(word_infallible("", false), |(bound, err)| {
+                    map(word_infallible(""), |(bound, err)| {
                         (
                             (
                                 UserInputBound::Unbounded,
@@ -1205,12 +1198,6 @@ mod test {
         test_parse_query_to_ast_helper("weight: <= 70", "\"weight\":{\"*\" TO \"70\"]");
 
         test_parse_query_to_ast_helper("weight: <= 70.5", "\"weight\":{\"*\" TO \"70.5\"]");
-
-        test_parse_query_to_ast_helper(">a", "{\"a\" TO \"*\"}");
-        test_parse_query_to_ast_helper(">=a", "[\"a\" TO \"*\"}");
-        test_parse_query_to_ast_helper("<a", "{\"*\" TO \"a\"}");
-        test_parse_query_to_ast_helper("<=a", "{\"*\" TO \"a\"]");
-        test_parse_query_to_ast_helper("<=bsd", "{\"*\" TO \"bsd\"]");
     }
 
     #[test]
@@ -1481,18 +1468,8 @@ mod test {
 
     #[test]
     fn test_parse_query_term_group() {
-        test_parse_query_to_ast_helper(r#"field:(abc)"#, r#""field":abc"#);
+        test_parse_query_to_ast_helper(r#"field:(abc)"#, r#"(*"field":abc)"#);
         test_parse_query_to_ast_helper(r#"field:(+a -"b c")"#, r#"(+"field":a -"field":"b c")"#);
-        test_parse_query_to_ast_helper(r#"field:(a AND "b c")"#, r#"(+"field":a +"field":"b c")"#);
-        test_parse_query_to_ast_helper(r#"field:(a OR "b c")"#, r#"(?"field":a ?"field":"b c")"#);
-        test_parse_query_to_ast_helper(
-            r#"field:(a OR (b AND c))"#,
-            r#"(?"field":a ?(+"field":b +"field":c))"#,
-        );
-        test_parse_query_to_ast_helper(
-            r#"field:(a [b TO c])"#,
-            r#"(*"field":a *"field":["b" TO "c"])"#,
-        );
 
         test_is_parse_err(r#"field:(+a -"b c""#, r#"(+"field":a -"field":"b c")"#);
     }
@@ -1644,21 +1621,5 @@ mod test {
             r#"myfield:'hello\"happy\'tax'"#,
             r#""myfield":'hello"happy'tax'"#,
         );
-        // we don't process escape sequence for chars which don't require it
-        test_parse_query_to_ast_helper(r#"abc\*"#, r#"abc\*"#);
-    }
-
-    #[test]
-    fn test_queries_with_colons() {
-        test_parse_query_to_ast_helper(r#""abc:def""#, r#""abc:def""#);
-        test_parse_query_to_ast_helper(r#"'abc:def'"#, r#"'abc:def'"#);
-        test_parse_query_to_ast_helper(r#"abc\:def"#, r#"abc:def"#);
-        test_parse_query_to_ast_helper(r#""abc\:def""#, r#""abc:def""#);
-        test_parse_query_to_ast_helper(r#"'abc\:def'"#, r#"'abc:def'"#);
-    }
-
-    #[test]
-    fn test_invalid_field() {
-        test_is_parse_err(r#"!bc:def"#, "!bc:def");
     }
 }

query-grammar/src/user_input_ast.rs

@@ -44,26 +44,6 @@ impl UserInputLeaf {
             },
         }
     }
-
-    pub(crate) fn set_default_field(&mut self, default_field: String) {
-        match self {
-            UserInputLeaf::Literal(ref mut literal) if literal.field_name.is_none() => {
-                literal.field_name = Some(default_field)
-            }
-            UserInputLeaf::All => {
-                *self = UserInputLeaf::Exists {
-                    field: default_field,
-                }
-            }
-            UserInputLeaf::Range { ref mut field, .. } if field.is_none() => {
-                *field = Some(default_field)
-            }
-            UserInputLeaf::Set { ref mut field, .. } if field.is_none() => {
-                *field = Some(default_field)
-            }
-            _ => (), // field was already set, do nothing
-        }
-    }
 }
 
 impl Debug for UserInputLeaf {
@@ -225,16 +205,6 @@ impl UserInputAst {
     pub fn or(asts: Vec<UserInputAst>) -> UserInputAst {
         UserInputAst::compose(Occur::Should, asts)
     }
-
-    pub(crate) fn set_default_field(&mut self, field: String) {
-        match self {
-            UserInputAst::Clause(clauses) => clauses
-                .iter_mut()
-                .for_each(|(_, ast)| ast.set_default_field(field.clone())),
-            UserInputAst::Leaf(leaf) => leaf.set_default_field(field),
-            UserInputAst::Boost(ref mut ast, _) => ast.set_default_field(field),
-        }
-    }
 }
 
 impl From<UserInputLiteral> for UserInputLeaf {

src/aggregation/agg_bench.rs

@@ -0,0 +1,585 @@
+#[cfg(all(test, feature = "unstable"))]
+mod bench {
+
+    use rand::prelude::SliceRandom;
+    use rand::rngs::StdRng;
+    use rand::{Rng, SeedableRng};
+    use rand_distr::Distribution;
+    use serde_json::json;
+    use test::{self, Bencher};
+
+    use crate::aggregation::agg_req::Aggregations;
+    use crate::aggregation::AggregationCollector;
+    use crate::query::{AllQuery, TermQuery};
+    use crate::schema::{IndexRecordOption, Schema, TextFieldIndexing, FAST, STRING};
+    use crate::{Index, Term};
+
+    #[derive(Clone, Copy, Hash, Default, Debug, PartialEq, Eq, PartialOrd, Ord)]
+    enum Cardinality {
+        /// All documents contain exactly one value.
+        /// `Full` is the default for auto-detecting the Cardinality, since it is the most strict.
+        #[default]
+        Full = 0,
+        /// All documents contain at most one value.
+        Optional = 1,
+        /// All documents may contain any number of values.
+        Multivalued = 2,
+        /// 1 / 20 documents has a value
+        Sparse = 3,
+    }
+
+    fn get_collector(agg_req: Aggregations) -> AggregationCollector {
+        AggregationCollector::from_aggs(agg_req, Default::default())
+    }
+
+    fn get_test_index_bench(cardinality: Cardinality) -> crate::Result<Index> {
+        let mut schema_builder = Schema::builder();
+        let text_fieldtype = crate::schema::TextOptions::default()
+            .set_indexing_options(
+                TextFieldIndexing::default().set_index_option(IndexRecordOption::WithFreqs),
+            )
+            .set_stored();
+        let text_field = schema_builder.add_text_field("text", text_fieldtype);
+        let json_field = schema_builder.add_json_field("json", FAST);
+        let text_field_many_terms = schema_builder.add_text_field("text_many_terms", STRING | FAST);
+        let text_field_few_terms = schema_builder.add_text_field("text_few_terms", STRING | FAST);
+        let score_fieldtype = crate::schema::NumericOptions::default().set_fast();
+        let score_field = schema_builder.add_u64_field("score", score_fieldtype.clone());
+        let score_field_f64 = schema_builder.add_f64_field("score_f64", score_fieldtype.clone());
+        let score_field_i64 = schema_builder.add_i64_field("score_i64", score_fieldtype);
+        let index = Index::create_from_tempdir(schema_builder.build())?;
+        let few_terms_data = ["INFO", "ERROR", "WARN", "DEBUG"];
+
+        let lg_norm = rand_distr::LogNormal::new(2.996f64, 0.979f64).unwrap();
+
+        let many_terms_data = (0..150_000)
+            .map(|num| format!("author{}", num))
+            .collect::<Vec<_>>();
+        {
+            let mut rng = StdRng::from_seed([1u8; 32]);
+            let mut index_writer = index.writer_with_num_threads(1, 200_000_000)?;
+            // To make the different test cases comparable we just change one doc to force the
+            // cardinality
+            if cardinality == Cardinality::Optional {
+                index_writer.add_document(doc!())?;
+            }
+            if cardinality == Cardinality::Multivalued {
+                index_writer.add_document(doc!(
+                    json_field => json!({"mixed_type": 10.0}),
+                    json_field => json!({"mixed_type": 10.0}),
+                    text_field => "cool",
+                    text_field => "cool",
+                    text_field_many_terms => "cool",
+                    text_field_many_terms => "cool",
+                    text_field_few_terms => "cool",
+                    text_field_few_terms => "cool",
+                    score_field => 1u64,
+                    score_field => 1u64,
+                    score_field_f64 => lg_norm.sample(&mut rng),
+                    score_field_f64 => lg_norm.sample(&mut rng),
+                    score_field_i64 => 1i64,
+                    score_field_i64 => 1i64,
+                ))?;
+            }
+            let mut doc_with_value = 1_000_000;
+            if cardinality == Cardinality::Sparse {
+                doc_with_value /= 20;
+            }
+            let _val_max = 1_000_000.0;
+            for _ in 0..doc_with_value {
+                let val: f64 = rng.gen_range(0.0..1_000_000.0);
+                let json = if rng.gen_bool(0.1) {
+                    // 10% are numeric values
+                    json!({ "mixed_type": val })
+                } else {
+                    json!({"mixed_type": many_terms_data.choose(&mut rng).unwrap().to_string()})
+                };
+                index_writer.add_document(doc!(
+                    text_field => "cool",
+                    json_field => json,
+                    text_field_many_terms => many_terms_data.choose(&mut rng).unwrap().to_string(),
+                    text_field_few_terms => few_terms_data.choose(&mut rng).unwrap().to_string(),
+                    score_field => val as u64,
+                    score_field_f64 => lg_norm.sample(&mut rng),
+                    score_field_i64 => val as i64,
+                ))?;
+                if cardinality == Cardinality::Sparse {
+                    for _ in 0..20 {
+                        index_writer.add_document(doc!(text_field => "cool"))?;
+                    }
+                }
+            }
+            // writing the segment
+            index_writer.commit()?;
+        }
+
+        Ok(index)
+    }
+
+    use paste::paste;
+    #[macro_export]
+    macro_rules! bench_all_cardinalities {
+        (  $x:ident ) => {
+            paste! {
+                #[bench]
+                fn $x(b: &mut Bencher) {
+                    [<$x _card>](b, Cardinality::Full)
+                }
+
+                #[bench]
+                fn [<$x _opt>](b: &mut Bencher) {
+                    [<$x _card>](b, Cardinality::Optional)
+                }
+
+                #[bench]
+                fn [<$x _multi>](b: &mut Bencher) {
+                    [<$x _card>](b, Cardinality::Multivalued)
+                }
+
+                #[bench]
+                fn [<$x _sparse>](b: &mut Bencher) {
+                    [<$x _card>](b, Cardinality::Sparse)
+                }
+
+            }
+        };
+    }
+
+    bench_all_cardinalities!(bench_aggregation_average_u64);
+
+    fn bench_aggregation_average_u64_card(b: &mut Bencher, cardinality: Cardinality) {
+        let index = get_test_index_bench(cardinality).unwrap();
+        let reader = index.reader().unwrap();
+        let text_field = reader.searcher().schema().get_field("text").unwrap();
+
+        b.iter(|| {
+            let term_query = TermQuery::new(
+                Term::from_field_text(text_field, "cool"),
+                IndexRecordOption::Basic,
+            );
+
+            let agg_req_1: Aggregations = serde_json::from_value(json!({
+                "average": { "avg": { "field": "score", } }
+            }))
+            .unwrap();
+
+            let collector = get_collector(agg_req_1);
+
+            let searcher = reader.searcher();
+            searcher.search(&term_query, &collector).unwrap()
+        });
+    }
+
+    bench_all_cardinalities!(bench_aggregation_stats_f64);
+
+    fn bench_aggregation_stats_f64_card(b: &mut Bencher, cardinality: Cardinality) {
+        let index = get_test_index_bench(cardinality).unwrap();
+        let reader = index.reader().unwrap();
+        let text_field = reader.searcher().schema().get_field("text").unwrap();
+
+        b.iter(|| {
+            let term_query = TermQuery::new(
+                Term::from_field_text(text_field, "cool"),
+                IndexRecordOption::Basic,
+            );
+
+            let agg_req_1: Aggregations = serde_json::from_value(json!({
+                "average_f64": { "stats": { "field": "score_f64", } }
+            }))
+            .unwrap();
+
+            let collector = get_collector(agg_req_1);
+
+            let searcher = reader.searcher();
+            searcher.search(&term_query, &collector).unwrap()
+        });
+    }
+
+    bench_all_cardinalities!(bench_aggregation_average_f64);
+
+    fn bench_aggregation_average_f64_card(b: &mut Bencher, cardinality: Cardinality) {
+        let index = get_test_index_bench(cardinality).unwrap();
+        let reader = index.reader().unwrap();
+        let text_field = reader.searcher().schema().get_field("text").unwrap();
+
+        b.iter(|| {
+            let term_query = TermQuery::new(
+                Term::from_field_text(text_field, "cool"),
+                IndexRecordOption::Basic,
+            );
+
+            let agg_req_1: Aggregations = serde_json::from_value(json!({
+                "average_f64": { "avg": { "field": "score_f64", } }
+            }))
+            .unwrap();
+
+            let collector = get_collector(agg_req_1);
+
+            let searcher = reader.searcher();
+            searcher.search(&term_query, &collector).unwrap()
+        });
+    }
+
+    bench_all_cardinalities!(bench_aggregation_percentiles_f64);
+
+    fn bench_aggregation_percentiles_f64_card(b: &mut Bencher, cardinality: Cardinality) {
+        let index = get_test_index_bench(cardinality).unwrap();
+        let reader = index.reader().unwrap();
+
+        b.iter(|| {
+            let agg_req_str = r#"
+            {
+              "mypercentiles": {
+                "percentiles": {
+                  "field": "score_f64",
+                  "percents": [ 95, 99, 99.9 ]
+                }
+              }
+            } "#;
+            let agg_req_1: Aggregations = serde_json::from_str(agg_req_str).unwrap();
+
+            let collector = get_collector(agg_req_1);
+
+            let searcher = reader.searcher();
+            searcher.search(&AllQuery, &collector).unwrap()
+        });
+    }
+
+    bench_all_cardinalities!(bench_aggregation_average_u64_and_f64);
+
+    fn bench_aggregation_average_u64_and_f64_card(b: &mut Bencher, cardinality: Cardinality) {
+        let index = get_test_index_bench(cardinality).unwrap();
+        let reader = index.reader().unwrap();
+        let text_field = reader.searcher().schema().get_field("text").unwrap();
+
+        b.iter(|| {
+            let term_query = TermQuery::new(
+                Term::from_field_text(text_field, "cool"),
+                IndexRecordOption::Basic,
+            );
+
+            let agg_req_1: Aggregations = serde_json::from_value(json!({
+                "average_f64": { "avg": { "field": "score_f64" } },
+                "average": { "avg": { "field": "score" } },
+            }))
+            .unwrap();
+
+            let collector = get_collector(agg_req_1);
+
+            let searcher = reader.searcher();
+            searcher.search(&term_query, &collector).unwrap()
+        });
+    }
+
+    bench_all_cardinalities!(bench_aggregation_terms_few);
+
+    fn bench_aggregation_terms_few_card(b: &mut Bencher, cardinality: Cardinality) {
+        let index = get_test_index_bench(cardinality).unwrap();
+        let reader = index.reader().unwrap();
+
+        b.iter(|| {
+            let agg_req: Aggregations = serde_json::from_value(json!({
+                "my_texts": { "terms": { "field": "text_few_terms" } },
+            }))
+            .unwrap();
+
+            let collector = get_collector(agg_req);
+
+            let searcher = reader.searcher();
+            searcher.search(&AllQuery, &collector).unwrap()
+        });
+    }
+
+    bench_all_cardinalities!(bench_aggregation_terms_many_with_top_hits_agg);
+
+    fn bench_aggregation_terms_many_with_top_hits_agg_card(
+        b: &mut Bencher,
+        cardinality: Cardinality,
+    ) {
+        let index = get_test_index_bench(cardinality).unwrap();
+        let reader = index.reader().unwrap();
+
+        b.iter(|| {
+            let agg_req: Aggregations = serde_json::from_value(json!({
+                "my_texts": {
+                    "terms": { "field": "text_many_terms" },
+                    "aggs": {
+                        "top_hits": { "top_hits":
+                            {
+                                "sort": [
+                                    { "score": "desc" }
+                                ],
+                                "size": 2,
+                                "doc_value_fields": ["score_f64"]
+                            }
+                        }
+                    }
+                },
+            }))
+            .unwrap();
+
+            let collector = get_collector(agg_req);
+
+            let searcher = reader.searcher();
+            searcher.search(&AllQuery, &collector).unwrap()
+        });
+    }
+
+    bench_all_cardinalities!(bench_aggregation_terms_many_with_sub_agg);
+
+    fn bench_aggregation_terms_many_with_sub_agg_card(b: &mut Bencher, cardinality: Cardinality) {
+        let index = get_test_index_bench(cardinality).unwrap();
+        let reader = index.reader().unwrap();
+
+        b.iter(|| {
+            let agg_req: Aggregations = serde_json::from_value(json!({
+                "my_texts": {
+                    "terms": { "field": "text_many_terms" },
+                    "aggs": {
+                        "average_f64": { "avg": { "field": "score_f64" } }
+                    }
+                },
+            }))
+            .unwrap();
+
+            let collector = get_collector(agg_req);
+
+            let searcher = reader.searcher();
+            searcher.search(&AllQuery, &collector).unwrap()
+        });
+    }
+
+    bench_all_cardinalities!(bench_aggregation_terms_many_json_mixed_type_with_sub_agg);
+
+    fn bench_aggregation_terms_many_json_mixed_type_with_sub_agg_card(
+        b: &mut Bencher,
+        cardinality: Cardinality,
+    ) {
+        let index = get_test_index_bench(cardinality).unwrap();
+        let reader = index.reader().unwrap();
+
+        b.iter(|| {
+            let agg_req: Aggregations = serde_json::from_value(json!({
+                "my_texts": {
+                    "terms": { "field": "json.mixed_type" },
+                    "aggs": {
+                        "average_f64": { "avg": { "field": "score_f64" } }
+                    }
+                },
+            }))
+            .unwrap();
+
+            let collector = get_collector(agg_req);
+
+            let searcher = reader.searcher();
+            searcher.search(&AllQuery, &collector).unwrap()
+        });
+    }
+
+    bench_all_cardinalities!(bench_aggregation_terms_many2);
+
+    fn bench_aggregation_terms_many2_card(b: &mut Bencher, cardinality: Cardinality) {
+        let index = get_test_index_bench(cardinality).unwrap();
+        let reader = index.reader().unwrap();
+
+        b.iter(|| {
+            let agg_req: Aggregations = serde_json::from_value(json!({
+                "my_texts": { "terms": { "field": "text_many_terms" } },
+            }))
+            .unwrap();
+
+            let collector = get_collector(agg_req);
+
+            let searcher = reader.searcher();
+            searcher.search(&AllQuery, &collector).unwrap()
+        });
+    }
+
+    bench_all_cardinalities!(bench_aggregation_terms_many_order_by_term);
+
+    fn bench_aggregation_terms_many_order_by_term_card(b: &mut Bencher, cardinality: Cardinality) {
+        let index = get_test_index_bench(cardinality).unwrap();
+        let reader = index.reader().unwrap();
+
+        b.iter(|| {
+            let agg_req: Aggregations = serde_json::from_value(json!({
+                "my_texts": { "terms": { "field": "text_many_terms", "order": { "_key": "desc" } } },
+            }))
+            .unwrap();
+
+            let collector = get_collector(agg_req);
+
+            let searcher = reader.searcher();
+            searcher.search(&AllQuery, &collector).unwrap()
+        });
+    }
+
+    bench_all_cardinalities!(bench_aggregation_range_only);
+
+    fn bench_aggregation_range_only_card(b: &mut Bencher, cardinality: Cardinality) {
+        let index = get_test_index_bench(cardinality).unwrap();
+        let reader = index.reader().unwrap();
+
+        b.iter(|| {
+            let agg_req_1: Aggregations = serde_json::from_value(json!({
+                "range_f64": { "range": { "field": "score_f64", "ranges": [
+                    { "from": 3, "to": 7000 },
+                    { "from": 7000, "to": 20000 },
+                    { "from": 20000, "to": 30000 },
+                    { "from": 30000, "to": 40000 },
+                    { "from": 40000, "to": 50000 },
+                    { "from": 50000, "to": 60000 }
+                ] } },
+            }))
+            .unwrap();
+
+            let collector = get_collector(agg_req_1);
+
+            let searcher = reader.searcher();
+            searcher.search(&AllQuery, &collector).unwrap()
+        });
+    }
+
+    bench_all_cardinalities!(bench_aggregation_range_with_avg);
+
+    fn bench_aggregation_range_with_avg_card(b: &mut Bencher, cardinality: Cardinality) {
+        let index = get_test_index_bench(cardinality).unwrap();
+        let reader = index.reader().unwrap();
+
+        b.iter(|| {
+            let agg_req_1: Aggregations = serde_json::from_value(json!({
+                "rangef64": {
+                    "range": {
+                        "field": "score_f64",
+                        "ranges": [
+                            { "from": 3, "to": 7000 },
+                            { "from": 7000, "to": 20000 },
+                            { "from": 20000, "to": 30000 },
+                            { "from": 30000, "to": 40000 },
+                            { "from": 40000, "to": 50000 },
+                            { "from": 50000, "to": 60000 }
+                        ]
+                    },
+                    "aggs": {
+                        "average_f64": { "avg": { "field": "score_f64" } }
+                    }
+                },
+            }))
+            .unwrap();
+
+            let collector = get_collector(agg_req_1);
+
+            let searcher = reader.searcher();
+            searcher.search(&AllQuery, &collector).unwrap()
+        });
+    }
+
+    // hard bounds has a different algorithm, because it actually limits collection range
+    //
+    bench_all_cardinalities!(bench_aggregation_histogram_only_hard_bounds);
+
+    fn bench_aggregation_histogram_only_hard_bounds_card(
+        b: &mut Bencher,
+        cardinality: Cardinality,
+    ) {
+        let index = get_test_index_bench(cardinality).unwrap();
+        let reader = index.reader().unwrap();
+
+        b.iter(|| {
+            let agg_req_1: Aggregations = serde_json::from_value(json!({
+                "rangef64": { "histogram": { "field": "score_f64", "interval": 100, "hard_bounds": { "min": 1000, "max": 300000 } } },
+            }))
+            .unwrap();
+
+            let collector = get_collector(agg_req_1);
+            let searcher = reader.searcher();
+            searcher.search(&AllQuery, &collector).unwrap()
+        });
+    }
+
+    bench_all_cardinalities!(bench_aggregation_histogram_with_avg);
+
+    fn bench_aggregation_histogram_with_avg_card(b: &mut Bencher, cardinality: Cardinality) {
+        let index = get_test_index_bench(cardinality).unwrap();
+        let reader = index.reader().unwrap();
+
+        b.iter(|| {
+            let agg_req_1: Aggregations = serde_json::from_value(json!({
+                "rangef64": {
+                    "histogram": { "field": "score_f64", "interval": 100 },
+                    "aggs": {
+                        "average_f64": { "avg": { "field": "score_f64" } }
+                    }
+                }
+            }))
+            .unwrap();
+
+            let collector = get_collector(agg_req_1);
+
+            let searcher = reader.searcher();
+            searcher.search(&AllQuery, &collector).unwrap()
+        });
+    }
+
+    bench_all_cardinalities!(bench_aggregation_histogram_only);
+
+    fn bench_aggregation_histogram_only_card(b: &mut Bencher, cardinality: Cardinality) {
+        let index = get_test_index_bench(cardinality).unwrap();
+        let reader = index.reader().unwrap();
+
+        b.iter(|| {
+            let agg_req_1: Aggregations = serde_json::from_value(json!({
+                "rangef64": {
+                    "histogram": {
+                        "field": "score_f64",
+                        "interval": 100 // 1000 buckets
+                    },
+                }
+            }))
+            .unwrap();
+
+            let collector = get_collector(agg_req_1);
+
+            let searcher = reader.searcher();
+            searcher.search(&AllQuery, &collector).unwrap()
+        });
+    }
+
+    bench_all_cardinalities!(bench_aggregation_avg_and_range_with_avg);
+
+    fn bench_aggregation_avg_and_range_with_avg_card(b: &mut Bencher, cardinality: Cardinality) {
+        let index = get_test_index_bench(cardinality).unwrap();
+        let reader = index.reader().unwrap();
+        let text_field = reader.searcher().schema().get_field("text").unwrap();
+
+        b.iter(|| {
+            let term_query = TermQuery::new(
+                Term::from_field_text(text_field, "cool"),
+                IndexRecordOption::Basic,
+            );
+
+            let agg_req_1: Aggregations = serde_json::from_value(json!({
+                "rangef64": {
+                    "range": {
+                        "field": "score_f64",
+                        "ranges": [
+                            { "from": 3, "to": 7000 },
+                            { "from": 7000, "to": 20000 },
+                            { "from": 20000, "to": 60000 }
+                        ]
+                    },
+                    "aggs": {
+                        "average_in_range": { "avg": { "field": "score" } }
+                    }
+                },
+                "average": { "avg": { "field": "score" } }
+            }))
+            .unwrap();
+
+            let collector = get_collector(agg_req_1);
+
+            let searcher = reader.searcher();
+            searcher.search(&term_query, &collector).unwrap()
+        });
+    }
+}

src/aggregation/agg_limits.rs

@@ -81,11 +81,10 @@ impl AggregationLimits {
         }
     }
 
-    pub(crate) fn add_memory_consumed(&self, add_num_bytes: u64) -> crate::Result<()> {
-        let prev_value = self
-            .memory_consumption
-            .fetch_add(add_num_bytes, Ordering::Relaxed);
-        validate_memory_consumption(prev_value + add_num_bytes, self.memory_limit)?;
+    pub(crate) fn add_memory_consumed(&self, num_bytes: u64) -> crate::Result<()> {
+        self.memory_consumption
+            .fetch_add(num_bytes, Ordering::Relaxed);
+        validate_memory_consumption(&self.memory_consumption, self.memory_limit)?;
         Ok(())
     }
 
@@ -95,11 +94,11 @@ impl AggregationLimits {
 }
 
 fn validate_memory_consumption(
-    memory_consumption: u64,
+    memory_consumption: &AtomicU64,
     memory_limit: ByteCount,
 ) -> Result<(), AggregationError> {
     // Load the estimated memory consumed by the aggregations
-    let memory_consumed: ByteCount = memory_consumption.into();
+    let memory_consumed: ByteCount = memory_consumption.load(Ordering::Relaxed).into();
     if memory_consumed > memory_limit {
         return Err(AggregationError::MemoryExceeded {
             limit: memory_limit,
@@ -119,11 +118,10 @@ pub struct ResourceLimitGuard {
 }
 
 impl ResourceLimitGuard {
-    pub(crate) fn add_memory_consumed(&self, add_num_bytes: u64) -> crate::Result<()> {
-        let prev_value = self
-            .memory_consumption
-            .fetch_add(add_num_bytes, Ordering::Relaxed);
-        validate_memory_consumption(prev_value + add_num_bytes, self.memory_limit)?;
+    pub(crate) fn add_memory_consumed(&self, num_bytes: u64) -> crate::Result<()> {
+        self.memory_consumption
+            .fetch_add(num_bytes, Ordering::Relaxed);
+        validate_memory_consumption(&self.memory_consumption, self.memory_limit)?;
         Ok(())
     }
 }

src/aggregation/agg_req.rs

@@ -34,7 +34,7 @@ use super::bucket::{
     DateHistogramAggregationReq, HistogramAggregation, RangeAggregation, TermsAggregation,
 };
 use super::metric::{
-    AverageAggregation, CountAggregation, ExtendedStatsAggregation, MaxAggregation, MinAggregation,
+    AverageAggregation, CountAggregation, MaxAggregation, MinAggregation,
     PercentilesAggregationReq, StatsAggregation, SumAggregation, TopHitsAggregation,
 };
 
@@ -146,11 +146,6 @@ pub enum AggregationVariants {
     /// extracted values.
     #[serde(rename = "stats")]
     Stats(StatsAggregation),
-    /// Computes a collection of estended statistics (`min`, `max`, `sum`, `count`, `avg`,
-    /// `sum_of_squares`, `variance`, `variance_sampling`, `std_deviation`,
-    /// `std_deviation_sampling`) over the  extracted values.
-    #[serde(rename = "extended_stats")]
-    ExtendedStats(ExtendedStatsAggregation),
     /// Computes the sum of the extracted values.
     #[serde(rename = "sum")]
     Sum(SumAggregation),
@@ -175,7 +170,6 @@ impl AggregationVariants {
             AggregationVariants::Max(max) => vec![max.field_name()],
             AggregationVariants::Min(min) => vec![min.field_name()],
             AggregationVariants::Stats(stats) => vec![stats.field_name()],
-            AggregationVariants::ExtendedStats(extended_stats) => vec![extended_stats.field_name()],
             AggregationVariants::Sum(sum) => vec![sum.field_name()],
             AggregationVariants::Percentiles(per) => vec![per.field_name()],
             AggregationVariants::TopHits(top_hits) => top_hits.field_names(),
@@ -203,12 +197,6 @@ impl AggregationVariants {
             _ => None,
         }
     }
-    pub(crate) fn as_top_hits(&self) -> Option<&TopHitsAggregation> {
-        match &self {
-            AggregationVariants::TopHits(top_hits) => Some(top_hits),
-            _ => None,
-        }
-    }
 
     pub(crate) fn as_percentile(&self) -> Option<&PercentilesAggregationReq> {
         match &self {

src/aggregation/agg_req_with_accessor.rs

@@ -11,14 +11,13 @@ use super::bucket::{
     DateHistogramAggregationReq, HistogramAggregation, RangeAggregation, TermsAggregation,
 };
 use super::metric::{
-    AverageAggregation, CountAggregation, ExtendedStatsAggregation, MaxAggregation, MinAggregation,
-    StatsAggregation, SumAggregation,
+    AverageAggregation, CountAggregation, MaxAggregation, MinAggregation, StatsAggregation,
+    SumAggregation,
 };
 use super::segment_agg_result::AggregationLimits;
 use super::VecWithNames;
 use crate::aggregation::{f64_to_fastfield_u64, Key};
-use crate::index::SegmentReader;
-use crate::SegmentOrdinal;
+use crate::{SegmentOrdinal, SegmentReader};
 
 #[derive(Default)]
 pub(crate) struct AggregationsWithAccessor {
@@ -276,10 +275,6 @@ impl AggregationWithAccessor {
                 field: ref field_name,
                 ..
             })
-            | ExtendedStats(ExtendedStatsAggregation {
-                field: ref field_name,
-                ..
-            })
             | Sum(SumAggregation {
                 field: ref field_name,
                 ..
@@ -339,8 +334,8 @@ fn get_missing_val(
         }
         _ => {
             return Err(crate::TantivyError::InvalidArgument(format!(
-                "Missing value {missing:?} for field {field_name} is not supported for column \
-                 type {column_type:?}"
+                "Missing value {:?} for field {} is not supported for column type {:?}",
+                missing, field_name, column_type
             )));
         }
     };
@@ -407,7 +402,7 @@ fn get_dynamic_columns(
         .iter()
         .map(|h| h.open())
         .collect::<io::Result<_>>()?;
-    assert!(!ff_fields.is_empty(), "field {field_name} not found");
+    assert!(!ff_fields.is_empty(), "field {} not found", field_name);
     Ok(cols)
 }
 

src/aggregation/agg_result.rs

@@ -8,9 +8,7 @@ use rustc_hash::FxHashMap;
 use serde::{Deserialize, Serialize};
 
 use super::bucket::GetDocCount;
-use super::metric::{
-    ExtendedStats, PercentilesMetricResult, SingleMetricResult, Stats, TopHitsMetricResult,
-};
+use super::metric::{PercentilesMetricResult, SingleMetricResult, Stats, TopHitsMetricResult};
 use super::{AggregationError, Key};
 use crate::TantivyError;
 
@@ -90,8 +88,6 @@ pub enum MetricResult {
     Min(SingleMetricResult),
     /// Stats metric result.
     Stats(Stats),
-    /// ExtendedStats metric result.
-    ExtendedStats(Box<ExtendedStats>),
     /// Sum metric result.
     Sum(SingleMetricResult),
     /// Percentiles metric result.
@@ -108,7 +104,6 @@ impl MetricResult {
             MetricResult::Max(max) => Ok(max.value),
             MetricResult::Min(min) => Ok(min.value),
             MetricResult::Stats(stats) => stats.get_value(agg_property),
-            MetricResult::ExtendedStats(extended_stats) => extended_stats.get_value(agg_property),
             MetricResult::Sum(sum) => Ok(sum.value),
             MetricResult::Percentiles(_) => Err(TantivyError::AggregationError(
                 AggregationError::InvalidRequest("percentiles can't be used to order".to_string()),

src/aggregation/bucket/histogram/histogram.rs

@@ -331,11 +331,9 @@ impl SegmentAggregationCollector for SegmentHistogramCollector {
         }
 
         let mem_delta = self.get_memory_consumption() - mem_pre;
-        if mem_delta > 0 {
-            bucket_agg_accessor
-                .limits
-                .add_memory_consumed(mem_delta as u64)?;
-        }
+        bucket_agg_accessor
+            .limits
+            .add_memory_consumed(mem_delta as u64)?;
 
         Ok(())
     }

src/aggregation/bucket/mod.rs

@@ -28,7 +28,6 @@ mod term_agg;
 mod term_missing_agg;
 
 use std::collections::HashMap;
-use std::fmt;
 
 pub use histogram::*;
 pub use range::*;
@@ -73,12 +72,12 @@ impl From<&str> for OrderTarget {
     }
 }
 
-impl fmt::Display for OrderTarget {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+impl ToString for OrderTarget {
+    fn to_string(&self) -> String {
         match self {
-            OrderTarget::Key => f.write_str("_key"),
-            OrderTarget::Count => f.write_str("_count"),
-            OrderTarget::SubAggregation(agg) => agg.fmt(f),
+            OrderTarget::Key => "_key".to_string(),
+            OrderTarget::Count => "_count".to_string(),
+            OrderTarget::SubAggregation(agg) => agg.to_string(),
         }
     }
 }

src/aggregation/bucket/term_agg.rs

@@ -324,11 +324,9 @@ impl SegmentAggregationCollector for SegmentTermCollector {
         }
 
         let mem_delta = self.get_memory_consumption() - mem_pre;
-        if mem_delta > 0 {
-            bucket_agg_accessor
-                .limits
-                .add_memory_consumed(mem_delta as u64)?;
-        }
+        bucket_agg_accessor
+            .limits
+            .add_memory_consumed(mem_delta as u64)?;
 
         Ok(())
     }
@@ -357,7 +355,8 @@ impl SegmentTermCollector {
     ) -> crate::Result<Self> {
         if field_type == ColumnType::Bytes {
             return Err(TantivyError::InvalidArgument(format!(
-                "terms aggregation is not supported for column type {field_type:?}"
+                "terms aggregation is not supported for column type {:?}",
+                field_type
             )));
         }
         let term_buckets = TermBuckets::default();

src/aggregation/collector.rs

@@ -8,8 +8,7 @@ use super::segment_agg_result::{
 };
 use crate::aggregation::agg_req_with_accessor::get_aggs_with_segment_accessor_and_validate;
 use crate::collector::{Collector, SegmentCollector};
-use crate::index::SegmentReader;
-use crate::{DocId, SegmentOrdinal, TantivyError};
+use crate::{DocId, SegmentOrdinal, SegmentReader, TantivyError};
 
 /// The default max bucket count, before the aggregation fails.
 pub const DEFAULT_BUCKET_LIMIT: u32 = 65000;

src/aggregation/intermediate_agg_result.rs

@@ -19,8 +19,8 @@ use super::bucket::{
     GetDocCount, Order, OrderTarget, RangeAggregation, TermsAggregation,
 };
 use super::metric::{
-    IntermediateAverage, IntermediateCount, IntermediateExtendedStats, IntermediateMax,
-    IntermediateMin, IntermediateStats, IntermediateSum, PercentilesCollector, TopHitsTopNComputer,
+    IntermediateAverage, IntermediateCount, IntermediateMax, IntermediateMin, IntermediateStats,
+    IntermediateSum, PercentilesCollector, TopHitsTopNComputer,
 };
 use super::segment_agg_result::AggregationLimits;
 use super::{format_date, AggregationError, Key, SerializedKey};
@@ -215,9 +215,6 @@ pub(crate) fn empty_from_req(req: &Aggregation) -> IntermediateAggregationResult
         Stats(_) => IntermediateAggregationResult::Metric(IntermediateMetricResult::Stats(
             IntermediateStats::default(),
         )),
-        ExtendedStats(_) => IntermediateAggregationResult::Metric(
-            IntermediateMetricResult::ExtendedStats(IntermediateExtendedStats::default()),
-        ),
         Sum(_) => IntermediateAggregationResult::Metric(IntermediateMetricResult::Sum(
             IntermediateSum::default(),
         )),
@@ -225,7 +222,7 @@ pub(crate) fn empty_from_req(req: &Aggregation) -> IntermediateAggregationResult
             IntermediateMetricResult::Percentiles(PercentilesCollector::default()),
         ),
         TopHits(ref req) => IntermediateAggregationResult::Metric(
-            IntermediateMetricResult::TopHits(TopHitsTopNComputer::new(req)),
+            IntermediateMetricResult::TopHits(TopHitsTopNComputer::new(req.clone())),
         ),
     }
 }
@@ -285,8 +282,6 @@ pub enum IntermediateMetricResult {
     Min(IntermediateMin),
     /// Intermediate stats result.
     Stats(IntermediateStats),
-    /// Intermediate stats result.
-    ExtendedStats(IntermediateExtendedStats),
     /// Intermediate sum result.
     Sum(IntermediateSum),
     /// Intermediate top_hits result
@@ -311,9 +306,6 @@ impl IntermediateMetricResult {
             IntermediateMetricResult::Stats(intermediate_stats) => {
                 MetricResult::Stats(intermediate_stats.finalize())
             }
-            IntermediateMetricResult::ExtendedStats(intermediate_stats) => {
-                MetricResult::ExtendedStats(intermediate_stats.finalize())
-            }
             IntermediateMetricResult::Sum(intermediate_sum) => {
                 MetricResult::Sum(intermediate_sum.finalize().into())
             }
@@ -354,12 +346,6 @@ impl IntermediateMetricResult {
             ) => {
                 stats_left.merge_fruits(stats_right);
             }
-            (
-                IntermediateMetricResult::ExtendedStats(extended_stats_left),
-                IntermediateMetricResult::ExtendedStats(extended_stats_right),
-            ) => {
-                extended_stats_left.merge_fruits(extended_stats_right);
-            }
             (IntermediateMetricResult::Sum(sum_left), IntermediateMetricResult::Sum(sum_right)) => {
                 sum_left.merge_fruits(sum_right);
             }

src/aggregation/metric/mod.rs

@@ -18,7 +18,6 @@
 
 mod average;
 mod count;
-mod extended_stats;
 mod max;
 mod min;
 mod percentiles;
@@ -30,7 +29,6 @@ use std::collections::HashMap;
 
 pub use average::*;
 pub use count::*;
-pub use extended_stats::*;
 pub use max::*;
 pub use min::*;
 pub use percentiles::*;

src/aggregation/metric/stats.rs

@@ -1,5 +1,3 @@
-use std::fmt::Debug;
-
 use serde::{Deserialize, Serialize};
 
 use super::*;
@@ -87,15 +85,13 @@ impl Stats {
 #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
 pub struct IntermediateStats {
     /// The number of extracted values.
-    pub(crate) count: u64,
+    count: u64,
     /// The sum of the extracted values.
-    pub(crate) sum: f64,
-    /// delta for sum needed for [Kahan algorithm for summation](https://en.wikipedia.org/wiki/Kahan_summation_algorithm)
-    pub(crate) delta: f64,
+    sum: f64,
     /// The min value.
-    pub(crate) min: f64,
+    min: f64,
     /// The max value.
-    pub(crate) max: f64,
+    max: f64,
 }
 
 impl Default for IntermediateStats {
@@ -103,7 +99,6 @@ impl Default for IntermediateStats {
         Self {
             count: 0,
             sum: 0.0,
-            delta: 0.0,
             min: f64::MAX,
             max: f64::MIN,
         }
@@ -114,13 +109,7 @@ impl IntermediateStats {
     /// Merges the other stats intermediate result into self.
     pub fn merge_fruits(&mut self, other: IntermediateStats) {
         self.count += other.count;
-
-        // kahan algorithm for sum
-        let y = other.sum - (self.delta + other.delta);
-        let t = self.sum + y;
-        self.delta = (t - self.sum) - y;
-        self.sum = t;
-
+        self.sum += other.sum;
         self.min = self.min.min(other.min);
         self.max = self.max.max(other.max);
     }
@@ -152,15 +141,9 @@ impl IntermediateStats {
     }
 
     #[inline]
-    pub(in crate::aggregation::metric) fn collect(&mut self, value: f64) {
+    fn collect(&mut self, value: f64) {
         self.count += 1;
-
-        // kahan algorithm for sum
-        let y = value - self.delta;
-        let t = self.sum + y;
-        self.delta = (t - self.sum) - y;
-        self.sum = t;
-
+        self.sum += value;
         self.min = self.min.min(value);
         self.max = self.max.max(value);
     }
@@ -305,6 +288,7 @@ impl SegmentAggregationCollector for SegmentStatsCollector {
 
 #[cfg(test)]
 mod tests {
+
     use serde_json::Value;
 
     use crate::aggregation::agg_req::{Aggregation, Aggregations};

src/aggregation/metric/top_hits.rs

@@ -1,8 +1,7 @@
 use std::collections::HashMap;
 use std::net::Ipv6Addr;
 
-use columnar::{Column, ColumnType, ColumnarReader, DynamicColumn};
-use common::json_path_writer::JSON_PATH_SEGMENT_SEP_STR;
+use columnar::{ColumnarReader, DynamicColumn};
 use common::DateTime;
 use regex::Regex;
 use serde::ser::SerializeMap;
@@ -16,6 +15,7 @@ use crate::aggregation::intermediate_agg_result::{
 use crate::aggregation::segment_agg_result::SegmentAggregationCollector;
 use crate::aggregation::AggregationError;
 use crate::collector::TopNComputer;
+use crate::schema::term::JSON_PATH_SEGMENT_SEP_STR;
 use crate::schema::OwnedValue;
 use crate::{DocAddress, DocId, SegmentOrdinal};
 
@@ -131,8 +131,8 @@ impl<'de> Deserialize<'de> for KeyOrder {
         ))?;
         if key_order.next().is_some() {
             return Err(serde::de::Error::custom(format!(
-                "Expected exactly one key-value pair in sort parameter of top_hits, found \
-                 {key_order:?}"
+                "Expected exactly one key-value pair in sort parameter of top_hits, found {:?}",
+                key_order
             )));
         }
         Ok(Self { field, order })
@@ -144,22 +144,27 @@ fn globbed_string_to_regex(glob: &str) -> Result<Regex, crate::TantivyError> {
     // Replace `*` glob with `.*` regex
     let sanitized = format!("^{}$", regex::escape(glob).replace(r"\*", ".*"));
     Regex::new(&sanitized.replace('*', ".*")).map_err(|e| {
-        crate::TantivyError::SchemaError(format!("Invalid regex '{glob}' in docvalue_fields: {e}"))
+        crate::TantivyError::SchemaError(format!(
+            "Invalid regex '{}' in docvalue_fields: {}",
+            glob, e
+        ))
     })
 }
 
 fn use_doc_value_fields_err(parameter: &str) -> crate::Result<()> {
     Err(crate::TantivyError::AggregationError(
         AggregationError::InvalidRequest(format!(
-            "The `{parameter}` parameter is not supported, only `docvalue_fields` is supported in \
-             `top_hits` aggregation"
+            "The `{}` parameter is not supported, only `docvalue_fields` is supported in \
+             `top_hits` aggregation",
+            parameter
         )),
     ))
 }
 fn unsupported_err(parameter: &str) -> crate::Result<()> {
     Err(crate::TantivyError::AggregationError(
         AggregationError::InvalidRequest(format!(
-            "The `{parameter}` parameter is not supported in the `top_hits` aggregation"
+            "The `{}` parameter is not supported in the `top_hits` aggregation",
+            parameter
         )),
     ))
 }
@@ -212,7 +217,8 @@ impl TopHitsAggregation {
                     .collect::<Vec<_>>();
                 assert!(
                     !fields.is_empty(),
-                    "No fields matched the glob '{field}' in docvalue_fields"
+                    "No fields matched the glob '{}' in docvalue_fields",
+                    field
                 );
                 Ok(fields)
             })
@@ -248,7 +254,7 @@ impl TopHitsAggregation {
             .map(|field| {
                 let accessors = accessors
                     .get(field)
-                    .unwrap_or_else(|| panic!("field '{field}' not found in accessors"));
+                    .unwrap_or_else(|| panic!("field '{}' not found in accessors", field));
 
                 let values: Vec<FastFieldValue> = accessors
                     .iter()
@@ -443,10 +449,10 @@ impl std::cmp::PartialEq for TopHitsTopNComputer {
 
 impl TopHitsTopNComputer {
     /// Create a new TopHitsCollector
-    pub fn new(req: &TopHitsAggregation) -> Self {
+    pub fn new(req: TopHitsAggregation) -> Self {
         Self {
             top_n: TopNComputer::new(req.size + req.from.unwrap_or(0)),
-            req: req.clone(),
+            req,
         }
     }
 
@@ -491,6 +497,7 @@ impl TopHitsTopNComputer {
 pub(crate) struct TopHitsSegmentCollector {
     segment_ordinal: SegmentOrdinal,
     accessor_idx: usize,
+    req: TopHitsAggregation,
     top_n: TopNComputer<Vec<DocValueAndOrder>, DocAddress, false>,
 }
 
@@ -501,6 +508,7 @@ impl TopHitsSegmentCollector {
         segment_ordinal: SegmentOrdinal,
     ) -> Self {
         Self {
+            req: req.clone(),
             top_n: TopNComputer::new(req.size + req.from.unwrap_or(0)),
             segment_ordinal,
             accessor_idx,
@@ -509,13 +517,14 @@ impl TopHitsSegmentCollector {
     fn into_top_hits_collector(
         self,
         value_accessors: &HashMap<String, Vec<DynamicColumn>>,
-        req: &TopHitsAggregation,
     ) -> TopHitsTopNComputer {
-        let mut top_hits_computer = TopHitsTopNComputer::new(req);
+        let mut top_hits_computer = TopHitsTopNComputer::new(self.req.clone());
         let top_results = self.top_n.into_vec();
 
         for res in top_results {
-            let doc_value_fields = req.get_document_field_data(value_accessors, res.doc.doc_id);
+            let doc_value_fields = self
+                .req
+                .get_document_field_data(value_accessors, res.doc.doc_id);
             top_hits_computer.collect(
                 DocSortValuesAndFields {
                     sorts: res.feature,
@@ -527,15 +536,34 @@ impl TopHitsSegmentCollector {
 
         top_hits_computer
     }
+}
 
-    /// TODO add a specialized variant for a single sort field
-    fn collect_with(
+impl SegmentAggregationCollector for TopHitsSegmentCollector {
+    fn add_intermediate_aggregation_result(
+        self: Box<Self>,
+        agg_with_accessor: &crate::aggregation::agg_req_with_accessor::AggregationsWithAccessor,
+        results: &mut crate::aggregation::intermediate_agg_result::IntermediateAggregationResults,
+    ) -> crate::Result<()> {
+        let name = agg_with_accessor.aggs.keys[self.accessor_idx].to_string();
+
+        let value_accessors = &agg_with_accessor.aggs.values[self.accessor_idx].value_accessors;
+
+        let intermediate_result =
+            IntermediateMetricResult::TopHits(self.into_top_hits_collector(value_accessors));
+        results.push(
+            name,
+            IntermediateAggregationResult::Metric(intermediate_result),
+        )
+    }
+
+    fn collect(
         &mut self,
         doc_id: crate::DocId,
-        req: &TopHitsAggregation,
-        accessors: &[(Column<u64>, ColumnType)],
+        agg_with_accessor: &mut crate::aggregation::agg_req_with_accessor::AggregationsWithAccessor,
     ) -> crate::Result<()> {
-        let sorts: Vec<DocValueAndOrder> = req
+        let accessors = &agg_with_accessor.aggs.values[self.accessor_idx].accessors;
+        let sorts: Vec<DocValueAndOrder> = self
+            .req
             .sort
             .iter()
             .enumerate()
@@ -560,62 +588,15 @@ impl TopHitsSegmentCollector {
         );
         Ok(())
     }
-}
-
-impl SegmentAggregationCollector for TopHitsSegmentCollector {
-    fn add_intermediate_aggregation_result(
-        self: Box<Self>,
-        agg_with_accessor: &crate::aggregation::agg_req_with_accessor::AggregationsWithAccessor,
-        results: &mut crate::aggregation::intermediate_agg_result::IntermediateAggregationResults,
-    ) -> crate::Result<()> {
-        let name = agg_with_accessor.aggs.keys[self.accessor_idx].to_string();
-
-        let value_accessors = &agg_with_accessor.aggs.values[self.accessor_idx].value_accessors;
-        let tophits_req = &agg_with_accessor.aggs.values[self.accessor_idx]
-            .agg
-            .agg
-            .as_top_hits()
-            .expect("aggregation request must be of type top hits");
-
-        let intermediate_result = IntermediateMetricResult::TopHits(
-            self.into_top_hits_collector(value_accessors, tophits_req),
-        );
-        results.push(
-            name,
-            IntermediateAggregationResult::Metric(intermediate_result),
-        )
-    }
-
-    /// TODO: Consider a caching layer to reduce the call overhead
-    fn collect(
-        &mut self,
-        doc_id: crate::DocId,
-        agg_with_accessor: &mut crate::aggregation::agg_req_with_accessor::AggregationsWithAccessor,
-    ) -> crate::Result<()> {
-        let tophits_req = &agg_with_accessor.aggs.values[self.accessor_idx]
-            .agg
-            .agg
-            .as_top_hits()
-            .expect("aggregation request must be of type top hits");
-        let accessors = &agg_with_accessor.aggs.values[self.accessor_idx].accessors;
-        self.collect_with(doc_id, tophits_req, accessors)?;
-        Ok(())
-    }
 
     fn collect_block(
         &mut self,
         docs: &[crate::DocId],
         agg_with_accessor: &mut crate::aggregation::agg_req_with_accessor::AggregationsWithAccessor,
     ) -> crate::Result<()> {
-        let tophits_req = &agg_with_accessor.aggs.values[self.accessor_idx]
-            .agg
-            .agg
-            .as_top_hits()
-            .expect("aggregation request must be of type top hits");
-        let accessors = &agg_with_accessor.aggs.values[self.accessor_idx].accessors;
         // TODO: Consider getting fields with the column block accessor.
         for doc in docs {
-            self.collect_with(*doc, tophits_req, accessors)?;
+            self.collect(*doc, agg_with_accessor)?;
         }
         Ok(())
     }

src/aggregation/mod.rs

@@ -143,6 +143,8 @@ use std::fmt::Display;
 #[cfg(test)]
 mod agg_tests;
 
+mod agg_bench;
+
 use core::fmt;
 
 pub use agg_limits::AggregationLimits;
@@ -157,15 +159,20 @@ use itertools::Itertools;
 use serde::de::{self, Visitor};
 use serde::{Deserialize, Deserializer, Serialize};
 
+pub(crate) fn invalid_agg_request(message: String) -> crate::TantivyError {
+    crate::TantivyError::AggregationError(AggregationError::InvalidRequest(message))
+}
+
 fn parse_str_into_f64<E: de::Error>(value: &str) -> Result<f64, E> {
-    let parsed = value
-        .parse::<f64>()
-        .map_err(|_err| de::Error::custom(format!("Failed to parse f64 from string: {value:?}")))?;
+    let parsed = value.parse::<f64>().map_err(|_err| {
+        de::Error::custom(format!("Failed to parse f64 from string: {:?}", value))
+    })?;
 
     // Check if the parsed value is NaN or infinity
     if parsed.is_nan() || parsed.is_infinite() {
         Err(de::Error::custom(format!(
-            "Value is not a valid f64 (NaN or Infinity): {value:?}"
+            "Value is not a valid f64 (NaN or Infinity): {:?}",
+            value
         )))
     } else {
         Ok(parsed)

src/aggregation/segment_agg_result.rs

@@ -11,12 +11,12 @@ use super::agg_req_with_accessor::{AggregationWithAccessor, AggregationsWithAcce
 use super::bucket::{SegmentHistogramCollector, SegmentRangeCollector, SegmentTermCollector};
 use super::intermediate_agg_result::IntermediateAggregationResults;
 use super::metric::{
-    AverageAggregation, CountAggregation, ExtendedStatsAggregation, MaxAggregation, MinAggregation,
+    AverageAggregation, CountAggregation, MaxAggregation, MinAggregation,
     SegmentPercentilesCollector, SegmentStatsCollector, SegmentStatsType, StatsAggregation,
     SumAggregation,
 };
 use crate::aggregation::bucket::TermMissingAgg;
-use crate::aggregation::metric::{SegmentExtendedStatsCollector, TopHitsSegmentCollector};
+use crate::aggregation::metric::TopHitsSegmentCollector;
 
 pub(crate) trait SegmentAggregationCollector: CollectorClone + Debug {
     fn add_intermediate_aggregation_result(
@@ -148,9 +148,6 @@ pub(crate) fn build_single_agg_segment_collector(
             accessor_idx,
             *missing,
         ))),
-        ExtendedStats(ExtendedStatsAggregation { missing, sigma, .. }) => Ok(Box::new(
-            SegmentExtendedStatsCollector::from_req(req.field_type, *sigma, accessor_idx, *missing),
-        )),
         Sum(SumAggregation { missing, .. }) => Ok(Box::new(SegmentStatsCollector::from_req(
             req.field_type,
             SegmentStatsType::Sum,

src/collector/facet_collector.rs

@@ -1,7 +1,7 @@
 use std::cmp::Ordering;
 use std::collections::{btree_map, BTreeMap, BTreeSet, BinaryHeap};
-use std::io;
 use std::ops::Bound;
+use std::{io, u64, usize};
 
 use crate::collector::{Collector, SegmentCollector};
 use crate::fastfield::FacetReader;
@@ -598,7 +598,7 @@ mod tests {
                 let mid = n % 4;
                 n /= 4;
                 let leaf = n % 5;
-                Facet::from(&format!("/top{top}/mid{mid}/leaf{leaf}"))
+                Facet::from(&format!("/top{}/mid{}/leaf{}", top, mid, leaf))
             })
             .collect();
         for i in 0..num_facets * 10 {
@@ -643,30 +643,30 @@ mod tests {
         facet_collector.add_facet(Facet::from("/country/europe"));
     }
 
-    // #[test]
-    // fn test_doc_unsorted_multifacet() -> crate::Result<()> {
-    //     let mut schema_builder = Schema::builder();
-    //     let facet_field = schema_builder.add_facet_field("facets", FacetOptions::default());
-    //     let schema = schema_builder.build();
-    //     let index = Index::create_in_ram(schema);
-    //     let mut index_writer = index.writer_for_tests()?;
-    //     index_writer.add_document(doc!(
-    //         facet_field => Facet::from_text(&"/subjects/A/a").unwrap(),
-    //         facet_field => Facet::from_text(&"/subjects/B/a").unwrap(),
-    //         facet_field => Facet::from_text(&"/subjects/A/b").unwrap(),
-    //         facet_field => Facet::from_text(&"/subjects/B/b").unwrap(),
-    //     ))?;
-    //     index_writer.commit()?;
-    //     let reader = index.reader()?;
-    //     let searcher = reader.searcher();
-    //     assert_eq!(searcher.num_docs(), 1);
-    //     let mut facet_collector = FacetCollector::for_field("facets");
-    //     facet_collector.add_facet("/subjects");
-    //     let counts = searcher.search(&AllQuery, &facet_collector)?;
-    //     let facets: Vec<(&Facet, u64)> = counts.get("/subjects").collect();
-    //     assert_eq!(facets[0].1, 1);
-    //     Ok(())
-    // }
+    #[test]
+    fn test_doc_unsorted_multifacet() -> crate::Result<()> {
+        let mut schema_builder = Schema::builder();
+        let facet_field = schema_builder.add_facet_field("facets", FacetOptions::default());
+        let schema = schema_builder.build();
+        let index = Index::create_in_ram(schema);
+        let mut index_writer = index.writer_for_tests()?;
+        index_writer.add_document(doc!(
+            facet_field => Facet::from_text(&"/subjects/A/a").unwrap(),
+            facet_field => Facet::from_text(&"/subjects/B/a").unwrap(),
+            facet_field => Facet::from_text(&"/subjects/A/b").unwrap(),
+            facet_field => Facet::from_text(&"/subjects/B/b").unwrap(),
+        ))?;
+        index_writer.commit()?;
+        let reader = index.reader()?;
+        let searcher = reader.searcher();
+        assert_eq!(searcher.num_docs(), 1);
+        let mut facet_collector = FacetCollector::for_field("facets");
+        facet_collector.add_facet("/subjects");
+        let counts = searcher.search(&AllQuery, &facet_collector)?;
+        let facets: Vec<(&Facet, u64)> = counts.get("/subjects").collect();
+        assert_eq!(facets[0].1, 1);
+        Ok(())
+    }
 
     #[test]
     fn test_doc_search_by_facet() -> crate::Result<()> {
@@ -725,99 +725,99 @@ mod tests {
         facet_collector.add_facet(Facet::from("/countryeurope"));
     }
 
-    // #[test]
-    // fn test_facet_collector_topk() {
-    //     let mut schema_builder = Schema::builder();
-    //     let facet_field = schema_builder.add_facet_field("facet", FacetOptions::default());
-    //     let schema = schema_builder.build();
-    //     let index = Index::create_in_ram(schema);
+    #[test]
+    fn test_facet_collector_topk() {
+        let mut schema_builder = Schema::builder();
+        let facet_field = schema_builder.add_facet_field("facet", FacetOptions::default());
+        let schema = schema_builder.build();
+        let index = Index::create_in_ram(schema);
 
-    //     let uniform = Uniform::new_inclusive(1, 100_000);
-    //     let mut docs: Vec<TantivyDocument> =
-    //         vec![("a", 10), ("b", 100), ("c", 7), ("d", 12), ("e", 21)]
-    //             .into_iter()
-    //             .flat_map(|(c, count)| {
-    //                 let facet = Facet::from(&format!("/facet/{c}"));
-    //                 let doc = doc!(facet_field => facet);
-    //                 iter::repeat(doc).take(count)
-    //             })
-    //             .map(|mut doc| {
-    //                 doc.add_facet(
-    //                     facet_field,
-    //                     &format!("/facet/{}", thread_rng().sample(uniform)),
-    //                 );
-    //                 doc
-    //             })
-    //             .collect();
-    //     docs[..].shuffle(&mut thread_rng());
+        let uniform = Uniform::new_inclusive(1, 100_000);
+        let mut docs: Vec<TantivyDocument> =
+            vec![("a", 10), ("b", 100), ("c", 7), ("d", 12), ("e", 21)]
+                .into_iter()
+                .flat_map(|(c, count)| {
+                    let facet = Facet::from(&format!("/facet/{}", c));
+                    let doc = doc!(facet_field => facet);
+                    iter::repeat(doc).take(count)
+                })
+                .map(|mut doc| {
+                    doc.add_facet(
+                        facet_field,
+                        &format!("/facet/{}", thread_rng().sample(uniform)),
+                    );
+                    doc
+                })
+                .collect();
+        docs[..].shuffle(&mut thread_rng());
 
-    //     let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
-    //     for doc in docs {
-    //         index_writer.add_document(doc).unwrap();
-    //     }
-    //     index_writer.commit().unwrap();
-    //     let searcher = index.reader().unwrap().searcher();
+        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
+        for doc in docs {
+            index_writer.add_document(doc).unwrap();
+        }
+        index_writer.commit().unwrap();
+        let searcher = index.reader().unwrap().searcher();
 
-    //     let mut facet_collector = FacetCollector::for_field("facet");
-    //     facet_collector.add_facet("/facet");
-    //     let counts: FacetCounts = searcher.search(&AllQuery, &facet_collector).unwrap();
+        let mut facet_collector = FacetCollector::for_field("facet");
+        facet_collector.add_facet("/facet");
+        let counts: FacetCounts = searcher.search(&AllQuery, &facet_collector).unwrap();
 
-    //     {
-    //         let facets: Vec<(&Facet, u64)> = counts.top_k("/facet", 3);
-    //         assert_eq!(
-    //             facets,
-    //             vec![
-    //                 (&Facet::from("/facet/b"), 100),
-    //                 (&Facet::from("/facet/e"), 21),
-    //                 (&Facet::from("/facet/d"), 12),
-    //             ]
-    //         );
-    //     }
-    // }
+        {
+            let facets: Vec<(&Facet, u64)> = counts.top_k("/facet", 3);
+            assert_eq!(
+                facets,
+                vec![
+                    (&Facet::from("/facet/b"), 100),
+                    (&Facet::from("/facet/e"), 21),
+                    (&Facet::from("/facet/d"), 12),
+                ]
+            );
+        }
+    }
 
-    // #[test]
-    // fn test_facet_collector_topk_tie_break() -> crate::Result<()> {
-    //     let mut schema_builder = Schema::builder();
-    //     let facet_field = schema_builder.add_facet_field("facet", FacetOptions::default());
-    //     let schema = schema_builder.build();
-    //     let index = Index::create_in_ram(schema);
+    #[test]
+    fn test_facet_collector_topk_tie_break() -> crate::Result<()> {
+        let mut schema_builder = Schema::builder();
+        let facet_field = schema_builder.add_facet_field("facet", FacetOptions::default());
+        let schema = schema_builder.build();
+        let index = Index::create_in_ram(schema);
 
-    //     let docs: Vec<TantivyDocument> = vec![("b", 2), ("a", 2), ("c", 4)]
-    //         .into_iter()
-    //         .flat_map(|(c, count)| {
-    //             let facet = Facet::from(&format!("/facet/{c}"));
-    //             let doc = doc!(facet_field => facet);
-    //             iter::repeat(doc).take(count)
-    //         })
-    //         .collect();
+        let docs: Vec<TantivyDocument> = vec![("b", 2), ("a", 2), ("c", 4)]
+            .into_iter()
+            .flat_map(|(c, count)| {
+                let facet = Facet::from(&format!("/facet/{}", c));
+                let doc = doc!(facet_field => facet);
+                iter::repeat(doc).take(count)
+            })
+            .collect();
 
-    //     let mut index_writer = index.writer_for_tests()?;
-    //     for doc in docs {
-    //         index_writer.add_document(doc)?;
-    //     }
-    //     index_writer.commit()?;
+        let mut index_writer = index.writer_for_tests()?;
+        for doc in docs {
+            index_writer.add_document(doc)?;
+        }
+        index_writer.commit()?;
 
-    //     let searcher = index.reader()?.searcher();
-    //     let mut facet_collector = FacetCollector::for_field("facet");
-    //     facet_collector.add_facet("/facet");
-    //     let counts: FacetCounts = searcher.search(&AllQuery, &facet_collector)?;
+        let searcher = index.reader()?.searcher();
+        let mut facet_collector = FacetCollector::for_field("facet");
+        facet_collector.add_facet("/facet");
+        let counts: FacetCounts = searcher.search(&AllQuery, &facet_collector)?;
 
-    //     let facets: Vec<(&Facet, u64)> = counts.top_k("/facet", 2);
-    //     assert_eq!(
-    //         facets,
-    //         vec![(&Facet::from("/facet/c"), 4), (&Facet::from("/facet/a"), 2)]
-    //     );
-    //     Ok(())
-    // }
+        let facets: Vec<(&Facet, u64)> = counts.top_k("/facet", 2);
+        assert_eq!(
+            facets,
+            vec![(&Facet::from("/facet/c"), 4), (&Facet::from("/facet/a"), 2)]
+        );
+        Ok(())
+    }
 
-    // #[test]
-    // fn is_child_facet() {
-    //     assert!(super::is_child_facet(&b"foo"[..], &b"foo\0bar"[..]));
-    //     assert!(super::is_child_facet(&b""[..], &b"foo\0bar"[..]));
-    //     assert!(super::is_child_facet(&b""[..], &b"foo"[..]));
-    //     assert!(!super::is_child_facet(&b"foo\0bar"[..], &b"foo"[..]));
-    //     assert!(!super::is_child_facet(&b"foo"[..], &b"foobar\0baz"[..]));
-    // }
+    #[test]
+    fn is_child_facet() {
+        assert!(super::is_child_facet(&b"foo"[..], &b"foo\0bar"[..]));
+        assert!(super::is_child_facet(&b""[..], &b"foo\0bar"[..]));
+        assert!(super::is_child_facet(&b""[..], &b"foo"[..]));
+        assert!(!super::is_child_facet(&b"foo\0bar"[..], &b"foo"[..]));
+        assert!(!super::is_child_facet(&b"foo"[..], &b"foobar\0baz"[..]));
+    }
 }
 
 #[cfg(all(test, feature = "unstable"))]

src/collector/top_collector.rs

@@ -4,8 +4,7 @@ use std::marker::PhantomData;
 use serde::{Deserialize, Serialize};
 
 use super::top_score_collector::TopNComputer;
-use crate::index::SegmentReader;
-use crate::{DocAddress, DocId, SegmentOrdinal};
+use crate::{DocAddress, DocId, SegmentOrdinal, SegmentReader};
 
 /// Contains a feature (field, score, etc.) of a document along with the document address.
 ///

src/collector/top_score_collector.rs

@@ -787,7 +787,7 @@ impl<Score, D, const R: bool> From<TopNComputerDeser<Score, D, R>> for TopNCompu
     }
 }
 
-impl<Score, D, const R: bool> TopNComputer<Score, D, R>
+impl<Score, D, const REVERSE_ORDER: bool> TopNComputer<Score, D, REVERSE_ORDER>
 where
     Score: PartialOrd + Clone,
     D: Serialize + DeserializeOwned + Ord + Clone,
@@ -808,7 +808,10 @@ where
     #[inline]
     pub fn push(&mut self, feature: Score, doc: D) {
         if let Some(last_median) = self.threshold.clone() {
-            if feature < last_median {
+            if !REVERSE_ORDER && feature > last_median {
+                return;
+            }
+            if REVERSE_ORDER && feature < last_median {
                 return;
             }
         }
@@ -843,7 +846,7 @@ where
     }
 
     /// Returns the top n elements in sorted order.
-    pub fn into_sorted_vec(mut self) -> Vec<ComparableDoc<Score, D, R>> {
+    pub fn into_sorted_vec(mut self) -> Vec<ComparableDoc<Score, D, REVERSE_ORDER>> {
         if self.buffer.len() > self.top_n {
             self.truncate_top_n();
         }
@@ -854,7 +857,7 @@ where
     /// Returns the top n elements in stored order.
     /// Useful if you do not need the elements in sorted order,
     /// for example when merging the results of multiple segments.
-    pub fn into_vec(mut self) -> Vec<ComparableDoc<Score, D, R>> {
+    pub fn into_vec(mut self) -> Vec<ComparableDoc<Score, D, REVERSE_ORDER>> {
         if self.buffer.len() > self.top_n {
             self.truncate_top_n();
         }
@@ -864,17 +867,16 @@ where
 
 #[cfg(test)]
 mod tests {
+    use proptest::prelude::*;
+
     use super::{TopDocs, TopNComputer};
     use crate::collector::top_collector::ComparableDoc;
-    use crate::collector::Collector;
+    use crate::collector::{Collector, DocSetCollector};
     use crate::query::{AllQuery, Query, QueryParser};
     use crate::schema::{Field, Schema, FAST, STORED, TEXT};
     use crate::time::format_description::well_known::Rfc3339;
     use crate::time::OffsetDateTime;
-    use crate::{
-        assert_nearly_equals, DateTime, DocAddress, DocId, Index, IndexWriter, Order, Score,
-        SegmentReader,
-    };
+    use crate::{DateTime, DocAddress, DocId, Index, IndexWriter, Order, Score, SegmentReader};
 
     fn make_index() -> crate::Result<Index> {
         let mut schema_builder = Schema::builder();
@@ -961,6 +963,44 @@ mod tests {
         }
     }
 
+    proptest! {
+        #[test]
+        fn test_topn_computer_asc_prop(
+          limit in 0..10_usize,
+          docs in proptest::collection::vec((0..100_u64, 0..100_u64), 0..100_usize),
+        ) {
+            let mut computer: TopNComputer<_, _, false> = TopNComputer::new(limit);
+            for (feature, doc) in &docs {
+                computer.push(*feature, *doc);
+            }
+            let mut comparable_docs = docs.into_iter().map(|(feature, doc)| ComparableDoc { feature, doc }).collect::<Vec<_>>();
+            comparable_docs.sort();
+            comparable_docs.truncate(limit);
+            prop_assert_eq!(
+                computer.into_sorted_vec(),
+                comparable_docs,
+            );
+        }
+
+        #[test]
+        fn test_topn_computer_desc_prop(
+          limit in 0..10_usize,
+          docs in proptest::collection::vec((0..100_u64, 0..100_u64), 0..100_usize),
+        ) {
+            let mut computer: TopNComputer<_, _, true> = TopNComputer::new(limit);
+            for (feature, doc) in &docs {
+                computer.push(*feature, *doc);
+            }
+            let mut comparable_docs = docs.into_iter().map(|(feature, doc)| ComparableDoc { feature, doc }).collect::<Vec<_>>();
+            comparable_docs.sort();
+            comparable_docs.truncate(limit);
+            prop_assert_eq!(
+                computer.into_sorted_vec(),
+                comparable_docs,
+            );
+        }
+    }
+
     #[test]
     fn test_top_collector_not_at_capacity_without_offset() -> crate::Result<()> {
         let index = make_index()?;
@@ -1374,4 +1414,29 @@ mod tests {
         );
         Ok(())
     }
+
+    #[test]
+    fn test_topn_computer_asc() {
+        let mut computer: TopNComputer<u32, u32, false> = TopNComputer::new(2);
+
+        computer.push(1u32, 1u32);
+        computer.push(2u32, 2u32);
+        computer.push(3u32, 3u32);
+        computer.push(2u32, 4u32);
+        computer.push(4u32, 5u32);
+        computer.push(1u32, 6u32);
+        assert_eq!(
+            computer.into_sorted_vec(),
+            &[
+                ComparableDoc {
+                    feature: 1u32,
+                    doc: 1u32,
+                },
+                ComparableDoc {
+                    feature: 1u32,
+                    doc: 6u32,
+                }
+            ]
+        );
+    }
 }

src/core/executor.rs

@@ -1,25 +1,19 @@
-use std::sync::Arc;
-
-#[cfg(feature = "quickwit")]
-use futures_util::{future::Either, FutureExt};
+use rayon::{ThreadPool, ThreadPoolBuilder};
 
 use crate::TantivyError;
 
-/// Executor makes it possible to run tasks in single thread or
-/// in a thread pool.
-#[derive(Clone)]
+/// Search executor whether search request are single thread or multithread.
+///
+/// We don't expose Rayon thread pool directly here for several reasons.
+///
+/// First dependency hell. It is not a good idea to expose the
+/// API of a dependency, knowing it might conflict with a different version
+/// used by the client. Second, we may stop using rayon in the future.
 pub enum Executor {
     /// Single thread variant of an Executor
     SingleThread,
     /// Thread pool variant of an Executor
-    ThreadPool(Arc<rayon::ThreadPool>),
-}
-
-#[cfg(feature = "quickwit")]
-impl From<Arc<rayon::ThreadPool>> for Executor {
-    fn from(thread_pool: Arc<rayon::ThreadPool>) -> Self {
-        Executor::ThreadPool(thread_pool)
-    }
+    ThreadPool(ThreadPool),
 }
 
 impl Executor {
@@ -30,11 +24,11 @@ impl Executor {
 
     /// Creates an Executor that dispatches the tasks in a thread pool.
     pub fn multi_thread(num_threads: usize, prefix: &'static str) -> crate::Result<Executor> {
-        let pool = rayon::ThreadPoolBuilder::new()
+        let pool = ThreadPoolBuilder::new()
             .num_threads(num_threads)
             .thread_name(move |num| format!("{prefix}{num}"))
             .build()?;
-        Ok(Executor::ThreadPool(Arc::new(pool)))
+        Ok(Executor::ThreadPool(pool))
     }
 
     /// Perform a map in the thread pool.
@@ -97,36 +91,11 @@ impl Executor {
             }
         }
     }
-
-    /// Spawn a task on the pool, returning a future completing on task success.
-    ///
-    /// If the task panic, returns `Err(())`.
-    #[cfg(feature = "quickwit")]
-    pub fn spawn_blocking<T: Send + 'static>(
-        &self,
-        cpu_intensive_task: impl FnOnce() -> T + Send + 'static,
-    ) -> impl std::future::Future<Output = Result<T, ()>> {
-        match self {
-            Executor::SingleThread => Either::Left(std::future::ready(Ok(cpu_intensive_task()))),
-            Executor::ThreadPool(pool) => {
-                let (sender, receiver) = oneshot::channel();
-                pool.spawn(|| {
-                    if sender.is_closed() {
-                        return;
-                    }
-                    let task_result = cpu_intensive_task();
-                    let _ = sender.send(task_result);
-                });
-
-                let res = receiver.map(|res| res.map_err(|_| ()));
-                Either::Right(res)
-            }
-        }
-    }
 }
 
 #[cfg(test)]
 mod tests {
+
     use super::Executor;
 
     #[test]
@@ -178,62 +147,4 @@ mod tests {
             assert_eq!(result[i], i * 2);
         }
     }
-
-    #[cfg(feature = "quickwit")]
-    #[test]
-    fn test_cancel_cpu_intensive_tasks() {
-        use std::sync::atomic::{AtomicU64, Ordering};
-        use std::sync::Arc;
-
-        let counter: Arc<AtomicU64> = Default::default();
-
-        let other_counter: Arc<AtomicU64> = Default::default();
-
-        let mut futures = Vec::new();
-        let mut other_futures = Vec::new();
-
-        let (tx, rx) = crossbeam_channel::bounded::<()>(0);
-        let rx = Arc::new(rx);
-        let executor = Executor::multi_thread(3, "search-test").unwrap();
-        for _ in 0..1000 {
-            let counter_clone: Arc<AtomicU64> = counter.clone();
-            let other_counter_clone: Arc<AtomicU64> = other_counter.clone();
-
-            let rx_clone = rx.clone();
-            let rx_clone2 = rx.clone();
-            let fut = executor.spawn_blocking(move || {
-                counter_clone.fetch_add(1, Ordering::SeqCst);
-                let _ = rx_clone.recv();
-            });
-            futures.push(fut);
-            let other_fut = executor.spawn_blocking(move || {
-                other_counter_clone.fetch_add(1, Ordering::SeqCst);
-                let _ = rx_clone2.recv();
-            });
-            other_futures.push(other_fut);
-        }
-
-        // We execute 100 futures.
-        for _ in 0..100 {
-            tx.send(()).unwrap();
-        }
-
-        let counter_val = counter.load(Ordering::SeqCst);
-        let other_counter_val = other_counter.load(Ordering::SeqCst);
-        assert!(counter_val >= 30);
-        assert!(other_counter_val >= 30);
-
-        drop(other_futures);
-
-        // We execute 100 futures.
-        for _ in 0..100 {
-            tx.send(()).unwrap();
-        }
-
-        let counter_val2 = counter.load(Ordering::SeqCst);
-        assert!(counter_val2 >= counter_val + 100 - 6);
-
-        let other_counter_val2 = other_counter.load(Ordering::SeqCst);
-        assert!(other_counter_val2 <= other_counter_val + 6);
-    }
 }

src/core/json_utils.rs

@@ -1,10 +1,12 @@
-use common::json_path_writer::JSON_PATH_SEGMENT_SEP;
+use columnar::MonotonicallyMappableToU64;
 use common::{replace_in_place, JsonPathWriter};
 use rustc_hash::FxHashMap;
 
+use crate::fastfield::FastValue;
 use crate::postings::{IndexingContext, IndexingPosition, PostingsWriter};
 use crate::schema::document::{ReferenceValue, ReferenceValueLeaf, Value};
-use crate::schema::Type;
+use crate::schema::term::JSON_PATH_SEGMENT_SEP;
+use crate::schema::{Field, Type, DATE_TIME_PRECISION_INDEXED};
 use crate::time::format_description::well_known::Rfc3339;
 use crate::time::{OffsetDateTime, UtcOffset};
 use crate::tokenizer::TextAnalyzer;
@@ -31,7 +33,7 @@ use crate::{DateTime, DocId, Term};
 /// position 1.
 /// As a result, with lemmatization, "The Smiths" will match our object.
 ///
-/// Worse, if a same term appears in the second object, a non increasing value would be pushed
+/// Worse, if a same term is appears in the second object, a non increasing value would be pushed
 /// to the position recorder probably provoking a panic.
 ///
 /// This problem is solved for regular multivalued object by offsetting the position
@@ -50,7 +52,7 @@ use crate::{DateTime, DocId, Term};
 /// We can therefore afford working with a map that is not imperfect. It is fine if several
 /// path map to the same index position as long as the probability is relatively low.
 #[derive(Default)]
-pub(crate) struct IndexingPositionsPerPath {
+struct IndexingPositionsPerPath {
     positions_per_path: FxHashMap<u32, IndexingPosition>,
 }
 
@@ -58,9 +60,6 @@ impl IndexingPositionsPerPath {
     fn get_position_from_id(&mut self, id: u32) -> &mut IndexingPosition {
         self.positions_per_path.entry(id).or_default()
     }
-    pub fn clear(&mut self) {
-        self.positions_per_path.clear();
-    }
 }
 
 /// Convert JSON_PATH_SEGMENT_SEP to a dot.
@@ -71,6 +70,36 @@ pub fn json_path_sep_to_dot(path: &mut str) {
     }
 }
 
+#[allow(clippy::too_many_arguments)]
+pub(crate) fn index_json_values<'a, V: Value<'a>>(
+    doc: DocId,
+    json_visitors: impl Iterator<Item = crate::Result<V::ObjectIter>>,
+    text_analyzer: &mut TextAnalyzer,
+    expand_dots_enabled: bool,
+    term_buffer: &mut Term,
+    postings_writer: &mut dyn PostingsWriter,
+    json_path_writer: &mut JsonPathWriter,
+    ctx: &mut IndexingContext,
+) -> crate::Result<()> {
+    json_path_writer.clear();
+    json_path_writer.set_expand_dots(expand_dots_enabled);
+    let mut positions_per_path: IndexingPositionsPerPath = Default::default();
+    for json_visitor_res in json_visitors {
+        let json_visitor = json_visitor_res?;
+        index_json_object::<V>(
+            doc,
+            json_visitor,
+            text_analyzer,
+            term_buffer,
+            json_path_writer,
+            postings_writer,
+            ctx,
+            &mut positions_per_path,
+        );
+    }
+    Ok(())
+}
+
 #[allow(clippy::too_many_arguments)]
 fn index_json_object<'a, V: Value<'a>>(
     doc: DocId,
@@ -99,7 +128,7 @@ fn index_json_object<'a, V: Value<'a>>(
 }
 
 #[allow(clippy::too_many_arguments)]
-pub(crate) fn index_json_value<'a, V: Value<'a>>(
+fn index_json_value<'a, V: Value<'a>>(
     doc: DocId,
     json_value: V,
     text_analyzer: &mut TextAnalyzer,
@@ -139,18 +168,12 @@ pub(crate) fn index_json_value<'a, V: Value<'a>>(
                 );
             }
             ReferenceValueLeaf::U64(val) => {
-                // try to parse to i64, since when querying we will apply the same logic and prefer
-                // i64 values
                 set_path_id(
                     term_buffer,
                     ctx.path_to_unordered_id
                         .get_or_allocate_unordered_id(json_path_writer.as_str()),
                 );
-                if let Ok(i64_val) = val.try_into() {
-                    term_buffer.append_type_and_fast_value::<i64>(i64_val);
-                } else {
-                    term_buffer.append_type_and_fast_value(val);
-                }
+                term_buffer.append_type_and_fast_value(val);
                 postings_writer.subscribe(doc, 0u32, term_buffer, ctx);
             }
             ReferenceValueLeaf::I64(val) => {
@@ -233,42 +256,71 @@ pub(crate) fn index_json_value<'a, V: Value<'a>>(
     }
 }
 
-/// Tries to infer a JSON type from a string and append it to the term.
-///
-/// The term must be json + JSON path.
-pub fn convert_to_fast_value_and_append_to_json_term(mut term: Term, phrase: &str) -> Option<Term> {
-    assert_eq!(
-        term.value()
-            .as_json_value_bytes()
-            .expect("expecting a Term with a json type and json path")
-            .as_serialized()
-            .len(),
-        0,
-        "JSON value bytes should be empty"
-    );
+// Tries to infer a JSON type from a string.
+pub fn convert_to_fast_value_and_get_term(
+    json_term_writer: &mut JsonTermWriter,
+    phrase: &str,
+) -> Option<Term> {
     if let Ok(dt) = OffsetDateTime::parse(phrase, &Rfc3339) {
         let dt_utc = dt.to_offset(UtcOffset::UTC);
-        term.append_type_and_fast_value(DateTime::from_utc(dt_utc));
-        return Some(term);
+        return Some(set_fastvalue_and_get_term(
+            json_term_writer,
+            DateTime::from_utc(dt_utc),
+        ));
     }
     if let Ok(i64_val) = str::parse::<i64>(phrase) {
-        term.append_type_and_fast_value(i64_val);
-        return Some(term);
+        return Some(set_fastvalue_and_get_term(json_term_writer, i64_val));
     }
     if let Ok(u64_val) = str::parse::<u64>(phrase) {
-        term.append_type_and_fast_value(u64_val);
-        return Some(term);
+        return Some(set_fastvalue_and_get_term(json_term_writer, u64_val));
     }
     if let Ok(f64_val) = str::parse::<f64>(phrase) {
-        term.append_type_and_fast_value(f64_val);
-        return Some(term);
+        return Some(set_fastvalue_and_get_term(json_term_writer, f64_val));
     }
     if let Ok(bool_val) = str::parse::<bool>(phrase) {
-        term.append_type_and_fast_value(bool_val);
-        return Some(term);
+        return Some(set_fastvalue_and_get_term(json_term_writer, bool_val));
     }
     None
 }
+// helper function to generate a Term from a json fastvalue
+pub(crate) fn set_fastvalue_and_get_term<T: FastValue>(
+    json_term_writer: &mut JsonTermWriter,
+    value: T,
+) -> Term {
+    json_term_writer.set_fast_value(value);
+    json_term_writer.term().clone()
+}
+
+// helper function to generate a list of terms with their positions from a textual json value
+pub(crate) fn set_string_and_get_terms(
+    json_term_writer: &mut JsonTermWriter,
+    value: &str,
+    text_analyzer: &mut TextAnalyzer,
+) -> Vec<(usize, Term)> {
+    let mut positions_and_terms = Vec::<(usize, Term)>::new();
+    json_term_writer.close_path_and_set_type(Type::Str);
+    let term_num_bytes = json_term_writer.term_buffer.len_bytes();
+    let mut token_stream = text_analyzer.token_stream(value);
+    token_stream.process(&mut |token| {
+        json_term_writer
+            .term_buffer
+            .truncate_value_bytes(term_num_bytes);
+        json_term_writer
+            .term_buffer
+            .append_bytes(token.text.as_bytes());
+        positions_and_terms.push((token.position, json_term_writer.term().clone()));
+    });
+    positions_and_terms
+}
+
+/// Writes a value of a JSON field to a `Term`.
+/// The Term format is as follows:
+/// `[JSON_TYPE][JSON_PATH][JSON_END_OF_PATH][VALUE_BYTES]`
+pub struct JsonTermWriter<'a> {
+    term_buffer: &'a mut Term,
+    path_stack: Vec<usize>,
+    expand_dots_enabled: bool,
+}
 
 /// Splits a json path supplied to the query parser in such a way that
 /// `.` can be escaped.
@@ -325,48 +377,158 @@ pub(crate) fn encode_column_name(
     path.into()
 }
 
+impl<'a> JsonTermWriter<'a> {
+    pub fn from_field_and_json_path(
+        field: Field,
+        json_path: &str,
+        expand_dots_enabled: bool,
+        term_buffer: &'a mut Term,
+    ) -> Self {
+        term_buffer.set_field_and_type(field, Type::Json);
+        let mut json_term_writer = Self::wrap(term_buffer, expand_dots_enabled);
+        for segment in split_json_path(json_path) {
+            json_term_writer.push_path_segment(&segment);
+        }
+        json_term_writer
+    }
+
+    pub fn wrap(term_buffer: &'a mut Term, expand_dots_enabled: bool) -> Self {
+        term_buffer.clear_with_type(Type::Json);
+        let mut path_stack = Vec::with_capacity(10);
+        path_stack.push(0);
+        Self {
+            term_buffer,
+            path_stack,
+            expand_dots_enabled,
+        }
+    }
+
+    fn trim_to_end_of_path(&mut self) {
+        let end_of_path = *self.path_stack.last().unwrap();
+        self.term_buffer.truncate_value_bytes(end_of_path);
+    }
+
+    pub fn close_path_and_set_type(&mut self, typ: Type) {
+        self.trim_to_end_of_path();
+        self.term_buffer.set_json_path_end();
+        self.term_buffer.append_bytes(&[typ.to_code()]);
+    }
+
+    // TODO: Remove this function and use JsonPathWriter instead.
+    pub fn push_path_segment(&mut self, segment: &str) {
+        // the path stack should never be empty.
+        self.trim_to_end_of_path();
+
+        if self.path_stack.len() > 1 {
+            self.term_buffer.set_json_path_separator();
+        }
+        let appended_segment = self.term_buffer.append_bytes(segment.as_bytes());
+        if self.expand_dots_enabled {
+            // We need to replace `.` by JSON_PATH_SEGMENT_SEP.
+            replace_in_place(b'.', JSON_PATH_SEGMENT_SEP, appended_segment);
+        }
+        self.term_buffer.add_json_path_separator();
+        self.path_stack.push(self.term_buffer.len_bytes());
+    }
+
+    pub fn pop_path_segment(&mut self) {
+        self.path_stack.pop();
+        assert!(!self.path_stack.is_empty());
+        self.trim_to_end_of_path();
+    }
+
+    /// Returns the json path of the term being currently built.
+    #[cfg(test)]
+    pub(crate) fn path(&self) -> &[u8] {
+        let end_of_path = self.path_stack.last().cloned().unwrap_or(1);
+        &self.term().serialized_value_bytes()[..end_of_path - 1]
+    }
+
+    pub(crate) fn set_fast_value<T: FastValue>(&mut self, val: T) {
+        self.close_path_and_set_type(T::to_type());
+        let value = if T::to_type() == Type::Date {
+            DateTime::from_u64(val.to_u64())
+                .truncate(DATE_TIME_PRECISION_INDEXED)
+                .to_u64()
+        } else {
+            val.to_u64()
+        };
+        self.term_buffer
+            .append_bytes(value.to_be_bytes().as_slice());
+    }
+
+    pub fn set_str(&mut self, text: &str) {
+        self.close_path_and_set_type(Type::Str);
+        self.term_buffer.append_bytes(text.as_bytes());
+    }
+
+    pub fn term(&self) -> &Term {
+        self.term_buffer
+    }
+}
+
 #[cfg(test)]
 mod tests {
-    use super::split_json_path;
-    use crate::schema::Field;
+    use super::{split_json_path, JsonTermWriter};
+    use crate::schema::{Field, Type};
     use crate::Term;
 
     #[test]
     fn test_json_writer() {
         let field = Field::from_field_id(1);
-
-        let mut term = Term::from_field_json_path(field, "attributes.color", false);
-        term.append_type_and_str("red");
+        let mut term = Term::with_type_and_field(Type::Json, field);
+        let mut json_writer = JsonTermWriter::wrap(&mut term, false);
+        json_writer.push_path_segment("attributes");
+        json_writer.push_path_segment("color");
+        json_writer.set_str("red");
         assert_eq!(
-            format!("{term:?}"),
+            format!("{:?}", json_writer.term()),
             "Term(field=1, type=Json, path=attributes.color, type=Str, \"red\")"
         );
-
-        let mut term = Term::from_field_json_path(field, "attributes.dimensions.width", false);
-        term.append_type_and_fast_value(400i64);
+        json_writer.set_str("blue");
         assert_eq!(
-            format!("{term:?}"),
+            format!("{:?}", json_writer.term()),
+            "Term(field=1, type=Json, path=attributes.color, type=Str, \"blue\")"
+        );
+        json_writer.pop_path_segment();
+        json_writer.push_path_segment("dimensions");
+        json_writer.push_path_segment("width");
+        json_writer.set_fast_value(400i64);
+        assert_eq!(
+            format!("{:?}", json_writer.term()),
             "Term(field=1, type=Json, path=attributes.dimensions.width, type=I64, 400)"
         );
+        json_writer.pop_path_segment();
+        json_writer.push_path_segment("height");
+        json_writer.set_fast_value(300i64);
+        assert_eq!(
+            format!("{:?}", json_writer.term()),
+            "Term(field=1, type=Json, path=attributes.dimensions.height, type=I64, 300)"
+        );
     }
 
     #[test]
     fn test_string_term() {
         let field = Field::from_field_id(1);
-        let mut term = Term::from_field_json_path(field, "color", false);
-        term.append_type_and_str("red");
-
-        assert_eq!(term.serialized_term(), b"\x00\x00\x00\x01jcolor\x00sred")
+        let mut term = Term::with_type_and_field(Type::Json, field);
+        let mut json_writer = JsonTermWriter::wrap(&mut term, false);
+        json_writer.push_path_segment("color");
+        json_writer.set_str("red");
+        assert_eq!(
+            json_writer.term().serialized_term(),
+            b"\x00\x00\x00\x01jcolor\x00sred"
+        )
     }
 
     #[test]
     fn test_i64_term() {
         let field = Field::from_field_id(1);
-        let mut term = Term::from_field_json_path(field, "color", false);
-        term.append_type_and_fast_value(-4i64);
-
+        let mut term = Term::with_type_and_field(Type::Json, field);
+        let mut json_writer = JsonTermWriter::wrap(&mut term, false);
+        json_writer.push_path_segment("color");
+        json_writer.set_fast_value(-4i64);
         assert_eq!(
-            term.serialized_term(),
+            json_writer.term().serialized_term(),
             b"\x00\x00\x00\x01jcolor\x00i\x7f\xff\xff\xff\xff\xff\xff\xfc"
         )
     }
@@ -374,11 +536,12 @@ mod tests {
     #[test]
     fn test_u64_term() {
         let field = Field::from_field_id(1);
-        let mut term = Term::from_field_json_path(field, "color", false);
-        term.append_type_and_fast_value(4u64);
-
+        let mut term = Term::with_type_and_field(Type::Json, field);
+        let mut json_writer = JsonTermWriter::wrap(&mut term, false);
+        json_writer.push_path_segment("color");
+        json_writer.set_fast_value(4u64);
         assert_eq!(
-            term.serialized_term(),
+            json_writer.term().serialized_term(),
             b"\x00\x00\x00\x01jcolor\x00u\x00\x00\x00\x00\x00\x00\x00\x04"
         )
     }
@@ -386,10 +549,12 @@ mod tests {
     #[test]
     fn test_f64_term() {
         let field = Field::from_field_id(1);
-        let mut term = Term::from_field_json_path(field, "color", false);
-        term.append_type_and_fast_value(4.0f64);
+        let mut term = Term::with_type_and_field(Type::Json, field);
+        let mut json_writer = JsonTermWriter::wrap(&mut term, false);
+        json_writer.push_path_segment("color");
+        json_writer.set_fast_value(4.0f64);
         assert_eq!(
-            term.serialized_term(),
+            json_writer.term().serialized_term(),
             b"\x00\x00\x00\x01jcolor\x00f\xc0\x10\x00\x00\x00\x00\x00\x00"
         )
     }
@@ -397,14 +562,90 @@ mod tests {
     #[test]
     fn test_bool_term() {
         let field = Field::from_field_id(1);
-        let mut term = Term::from_field_json_path(field, "color", false);
-        term.append_type_and_fast_value(true);
+        let mut term = Term::with_type_and_field(Type::Json, field);
+        let mut json_writer = JsonTermWriter::wrap(&mut term, false);
+        json_writer.push_path_segment("color");
+        json_writer.set_fast_value(true);
         assert_eq!(
-            term.serialized_term(),
+            json_writer.term().serialized_term(),
             b"\x00\x00\x00\x01jcolor\x00o\x00\x00\x00\x00\x00\x00\x00\x01"
         )
     }
 
+    #[test]
+    fn test_push_after_set_path_segment() {
+        let field = Field::from_field_id(1);
+        let mut term = Term::with_type_and_field(Type::Json, field);
+        let mut json_writer = JsonTermWriter::wrap(&mut term, false);
+        json_writer.push_path_segment("attribute");
+        json_writer.set_str("something");
+        json_writer.push_path_segment("color");
+        json_writer.set_str("red");
+        assert_eq!(
+            json_writer.term().serialized_term(),
+            b"\x00\x00\x00\x01jattribute\x01color\x00sred"
+        )
+    }
+
+    #[test]
+    fn test_pop_segment() {
+        let field = Field::from_field_id(1);
+        let mut term = Term::with_type_and_field(Type::Json, field);
+        let mut json_writer = JsonTermWriter::wrap(&mut term, false);
+        json_writer.push_path_segment("color");
+        json_writer.push_path_segment("hue");
+        json_writer.pop_path_segment();
+        json_writer.set_str("red");
+        assert_eq!(
+            json_writer.term().serialized_term(),
+            b"\x00\x00\x00\x01jcolor\x00sred"
+        )
+    }
+
+    #[test]
+    fn test_json_writer_path() {
+        let field = Field::from_field_id(1);
+        let mut term = Term::with_type_and_field(Type::Json, field);
+        let mut json_writer = JsonTermWriter::wrap(&mut term, false);
+        json_writer.push_path_segment("color");
+        assert_eq!(json_writer.path(), b"color");
+        json_writer.push_path_segment("hue");
+        assert_eq!(json_writer.path(), b"color\x01hue");
+        json_writer.set_str("pink");
+        assert_eq!(json_writer.path(), b"color\x01hue");
+    }
+
+    #[test]
+    fn test_json_path_expand_dots_disabled() {
+        let field = Field::from_field_id(1);
+        let mut term = Term::with_type_and_field(Type::Json, field);
+        let mut json_writer = JsonTermWriter::wrap(&mut term, false);
+        json_writer.push_path_segment("color.hue");
+        assert_eq!(json_writer.path(), b"color.hue");
+    }
+
+    #[test]
+    fn test_json_path_expand_dots_enabled() {
+        let field = Field::from_field_id(1);
+        let mut term = Term::with_type_and_field(Type::Json, field);
+        let mut json_writer = JsonTermWriter::wrap(&mut term, true);
+        json_writer.push_path_segment("color.hue");
+        assert_eq!(json_writer.path(), b"color\x01hue");
+    }
+
+    #[test]
+    fn test_json_path_expand_dots_enabled_pop_segment() {
+        let field = Field::from_field_id(1);
+        let mut term = Term::with_type_and_field(Type::Json, field);
+        let mut json_writer = JsonTermWriter::wrap(&mut term, true);
+        json_writer.push_path_segment("hello");
+        assert_eq!(json_writer.path(), b"hello");
+        json_writer.push_path_segment("color.hue");
+        assert_eq!(json_writer.path(), b"hello\x01color\x01hue");
+        json_writer.pop_path_segment();
+        assert_eq!(json_writer.path(), b"hello");
+    }
+
     #[test]
     fn test_split_json_path_simple() {
         let json_path = split_json_path("titi.toto");

src/core/searcher.rs

@@ -4,13 +4,13 @@ use std::{fmt, io};
 
 use crate::collector::Collector;
 use crate::core::Executor;
-use crate::index::{SegmentId, SegmentReader};
+use crate::index::SegmentReader;
 use crate::query::{Bm25StatisticsProvider, EnableScoring, Query};
 use crate::schema::document::DocumentDeserialize;
 use crate::schema::{Schema, Term};
 use crate::space_usage::SearcherSpaceUsage;
 use crate::store::{CacheStats, StoreReader};
-use crate::{DocAddress, Index, Opstamp, TrackedObject};
+use crate::{DocAddress, Index, Opstamp, SegmentId, TrackedObject};
 
 /// Identifies the searcher generation accessed by a [`Searcher`].
 ///
@@ -109,9 +109,8 @@ impl Searcher {
         &self,
         doc_address: DocAddress,
     ) -> crate::Result<D> {
-        let executor = self.inner.index.search_executor();
         let store_reader = &self.inner.store_readers[doc_address.segment_ord as usize];
-        store_reader.get_async(doc_address.doc_id, executor).await
+        store_reader.get_async(doc_address.doc_id).await
     }
 
     /// Access the schema associated with the index of this searcher.

src/core/tests.rs

@@ -1,14 +1,13 @@
 use crate::collector::Count;
 use crate::directory::{RamDirectory, WatchCallback};
-use crate::index::SegmentId;
 use crate::indexer::{LogMergePolicy, NoMergePolicy};
-use crate::postings::Postings;
+use crate::json_utils::JsonTermWriter;
 use crate::query::TermQuery;
-use crate::schema::{Field, IndexRecordOption, Schema, INDEXED, STRING, TEXT};
+use crate::schema::{Field, IndexRecordOption, Schema, Type, INDEXED, STRING, TEXT};
 use crate::tokenizer::TokenizerManager;
 use crate::{
-    Directory, DocSet, Index, IndexBuilder, IndexReader, IndexSettings, IndexWriter, ReloadPolicy,
-    TantivyDocument, Term,
+    Directory, DocSet, Index, IndexBuilder, IndexReader, IndexSettings, IndexWriter, Postings,
+    ReloadPolicy, SegmentId, TantivyDocument, Term,
 };
 
 #[test]
@@ -417,12 +416,16 @@ fn test_non_text_json_term_freq() {
     let searcher = reader.searcher();
     let segment_reader = searcher.segment_reader(0u32);
     let inv_idx = segment_reader.inverted_index(field).unwrap();
-
-    let mut term = Term::from_field_json_path(field, "tenant_id", false);
-    term.append_type_and_fast_value(75i64);
-
+    let mut term = Term::with_type_and_field(Type::Json, field);
+    let mut json_term_writer = JsonTermWriter::wrap(&mut term, false);
+    json_term_writer.push_path_segment("tenant_id");
+    json_term_writer.close_path_and_set_type(Type::U64);
+    json_term_writer.set_fast_value(75u64);
     let postings = inv_idx
-        .read_postings(&term, IndexRecordOption::WithFreqsAndPositions)
+        .read_postings(
+            json_term_writer.term(),
+            IndexRecordOption::WithFreqsAndPositions,
+        )
         .unwrap()
         .unwrap();
     assert_eq!(postings.doc(), 0);
@@ -451,12 +454,16 @@ fn test_non_text_json_term_freq_bitpacked() {
     let searcher = reader.searcher();
     let segment_reader = searcher.segment_reader(0u32);
     let inv_idx = segment_reader.inverted_index(field).unwrap();
-
-    let mut term = Term::from_field_json_path(field, "tenant_id", false);
-    term.append_type_and_fast_value(75i64);
-
+    let mut term = Term::with_type_and_field(Type::Json, field);
+    let mut json_term_writer = JsonTermWriter::wrap(&mut term, false);
+    json_term_writer.push_path_segment("tenant_id");
+    json_term_writer.close_path_and_set_type(Type::U64);
+    json_term_writer.set_fast_value(75u64);
     let mut postings = inv_idx
-        .read_postings(&term, IndexRecordOption::WithFreqsAndPositions)
+        .read_postings(
+            json_term_writer.term(),
+            IndexRecordOption::WithFreqsAndPositions,
+        )
         .unwrap()
         .unwrap();
     assert_eq!(postings.doc(), 0);

src/directory/mmap_directory.rs

@@ -566,7 +566,7 @@ mod tests {
         let mmap_directory = MmapDirectory::create_from_tempdir().unwrap();
         let num_paths = 10;
         let paths: Vec<PathBuf> = (0..num_paths)
-            .map(|i| PathBuf::from(&*format!("file_{i}")))
+            .map(|i| PathBuf::from(&*format!("file_{}", i)))
             .collect();
         {
             for path in &paths {

src/fastfield/facet_reader.rs

@@ -62,7 +62,8 @@ impl FacetReader {
 
 #[cfg(test)]
 mod tests {
-    use crate::schema::{Facet, FacetOptions, SchemaBuilder, Value, STORED};
+    use crate::schema::document::Value;
+    use crate::schema::{Facet, FacetOptions, SchemaBuilder, STORED};
     use crate::{DocAddress, Index, IndexWriter, TantivyDocument};
 
     #[test]
@@ -88,108 +89,103 @@ mod tests {
         let doc = searcher
             .doc::<TantivyDocument>(DocAddress::new(0u32, 0u32))
             .unwrap();
-        let value = doc
-            .get_first(facet_field)
-            .and_then(|v| v.as_value().as_facet());
+        let value = doc.get_first(facet_field).and_then(|v| v.as_facet());
         assert_eq!(value, None);
     }
 
-    // #[test]
-    // fn test_facet_several_facets_sorted() {
-    //     let mut schema_builder = SchemaBuilder::default();
-    //     let facet_field = schema_builder.add_facet_field("facet", FacetOptions::default());
-    //     let schema = schema_builder.build();
-    //     let index = Index::create_in_ram(schema);
-    //     let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
-    //     index_writer
-    //         .add_document(doc!(facet_field=>Facet::from_text("/parent/child1").unwrap()))
-    //         .unwrap();
-    //     index_writer
-    //         .add_document(doc!(
-    //             facet_field=>Facet::from_text("/parent/child2").unwrap(),
-    //             facet_field=>Facet::from_text("/parent/child1/blop").unwrap(),
-    //         ))
-    //         .unwrap();
-    //     index_writer.commit().unwrap();
-    //     let searcher = index.reader().unwrap().searcher();
-    //     let facet_reader = searcher.segment_reader(0u32).facet_reader("facet").unwrap();
-    //     let mut facet_ords = Vec::new();
+    #[test]
+    fn test_facet_several_facets_sorted() {
+        let mut schema_builder = SchemaBuilder::default();
+        let facet_field = schema_builder.add_facet_field("facet", FacetOptions::default());
+        let schema = schema_builder.build();
+        let index = Index::create_in_ram(schema);
+        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
+        index_writer
+            .add_document(doc!(facet_field=>Facet::from_text("/parent/child1").unwrap()))
+            .unwrap();
+        index_writer
+            .add_document(doc!(
+                facet_field=>Facet::from_text("/parent/child2").unwrap(),
+                facet_field=>Facet::from_text("/parent/child1/blop").unwrap(),
+            ))
+            .unwrap();
+        index_writer.commit().unwrap();
+        let searcher = index.reader().unwrap().searcher();
+        let facet_reader = searcher.segment_reader(0u32).facet_reader("facet").unwrap();
+        let mut facet_ords = Vec::new();
 
-    //     facet_ords.extend(facet_reader.facet_ords(0u32));
-    //     assert_eq!(&facet_ords, &[0u64]);
+        facet_ords.extend(facet_reader.facet_ords(0u32));
+        assert_eq!(&facet_ords, &[0u64]);
 
-    //     facet_ords.clear();
-    //     facet_ords.extend(facet_reader.facet_ords(1u32));
-    //     assert_eq!(&facet_ords, &[1u64, 2u64]);
+        facet_ords.clear();
+        facet_ords.extend(facet_reader.facet_ords(1u32));
+        assert_eq!(&facet_ords, &[1u64, 2u64]);
 
-    //     assert_eq!(facet_reader.num_facets(), 3);
-    //     let mut facet = Facet::default();
-    //     facet_reader.facet_from_ord(0, &mut facet).unwrap();
-    //     assert_eq!(facet.to_path_string(), "/parent/child1");
-    //     facet_reader.facet_from_ord(1, &mut facet).unwrap();
-    //     assert_eq!(facet.to_path_string(), "/parent/child1/blop");
-    //     facet_reader.facet_from_ord(2, &mut facet).unwrap();
-    //     assert_eq!(facet.to_path_string(), "/parent/child2");
-    // }
+        assert_eq!(facet_reader.num_facets(), 3);
+        let mut facet = Facet::default();
+        facet_reader.facet_from_ord(0, &mut facet).unwrap();
+        assert_eq!(facet.to_path_string(), "/parent/child1");
+        facet_reader.facet_from_ord(1, &mut facet).unwrap();
+        assert_eq!(facet.to_path_string(), "/parent/child1/blop");
+        facet_reader.facet_from_ord(2, &mut facet).unwrap();
+        assert_eq!(facet.to_path_string(), "/parent/child2");
+    }
 
-    // #[test]
-    // fn test_facet_stored_and_indexed() -> crate::Result<()> {
-    //     let mut schema_builder = SchemaBuilder::default();
-    //     let facet_field = schema_builder.add_facet_field("facet", STORED);
-    //     let schema = schema_builder.build();
-    //     let index = Index::create_in_ram(schema);
-    //     let mut index_writer = index.writer_for_tests()?;
-    //     index_writer.add_document(doc!(facet_field=>Facet::from_text("/a/b").unwrap()))?;
-    //     index_writer.commit()?;
-    //     let searcher = index.reader()?.searcher();
-    //     let facet_reader = searcher.segment_reader(0u32).facet_reader("facet").unwrap();
-    //     let mut facet_ords = Vec::new();
-    //     facet_ords.extend(facet_reader.facet_ords(0u32));
-    //     assert_eq!(&facet_ords, &[0u64]);
-    //     let doc = searcher.doc::<TantivyDocument>(DocAddress::new(0u32, 0u32))?;
-    //     let value: Option<Facet> = doc
-    //         .get_first(facet_field)
-    //         .and_then(|v| v.as_facet())
-    //         .map(|facet| Facet::from_encoded_string(facet.to_string()));
-    //     assert_eq!(value, Facet::from_text("/a/b").ok());
-    //     Ok(())
-    // }
+    #[test]
+    fn test_facet_stored_and_indexed() -> crate::Result<()> {
+        let mut schema_builder = SchemaBuilder::default();
+        let facet_field = schema_builder.add_facet_field("facet", STORED);
+        let schema = schema_builder.build();
+        let index = Index::create_in_ram(schema);
+        let mut index_writer = index.writer_for_tests()?;
+        index_writer.add_document(doc!(facet_field=>Facet::from_text("/a/b").unwrap()))?;
+        index_writer.commit()?;
+        let searcher = index.reader()?.searcher();
+        let facet_reader = searcher.segment_reader(0u32).facet_reader("facet").unwrap();
+        let mut facet_ords = Vec::new();
+        facet_ords.extend(facet_reader.facet_ords(0u32));
+        assert_eq!(&facet_ords, &[0u64]);
+        let doc = searcher.doc::<TantivyDocument>(DocAddress::new(0u32, 0u32))?;
+        let value: Option<&Facet> = doc.get_first(facet_field).and_then(|v| v.as_facet());
+        assert_eq!(value, Facet::from_text("/a/b").ok().as_ref());
+        Ok(())
+    }
 
-    // #[test]
-    // fn test_facet_not_populated_for_all_docs() -> crate::Result<()> {
-    //     let mut schema_builder = SchemaBuilder::default();
-    //     let facet_field = schema_builder.add_facet_field("facet", FacetOptions::default());
-    //     let schema = schema_builder.build();
-    //     let index = Index::create_in_ram(schema);
-    //     let mut index_writer = index.writer_for_tests()?;
-    //     index_writer.add_document(doc!(facet_field=>Facet::from_text("/a/b").unwrap()))?;
-    //     index_writer.add_document(TantivyDocument::default())?;
-    //     index_writer.commit()?;
-    //     let searcher = index.reader()?.searcher();
-    //     let facet_reader = searcher.segment_reader(0u32).facet_reader("facet").unwrap();
-    //     let mut facet_ords = Vec::new();
-    //     facet_ords.extend(facet_reader.facet_ords(0u32));
-    //     assert_eq!(&facet_ords, &[0u64]);
-    //     facet_ords.clear();
-    //     facet_ords.extend(facet_reader.facet_ords(1u32));
-    //     assert!(facet_ords.is_empty());
-    //     Ok(())
-    // }
+    #[test]
+    fn test_facet_not_populated_for_all_docs() -> crate::Result<()> {
+        let mut schema_builder = SchemaBuilder::default();
+        let facet_field = schema_builder.add_facet_field("facet", FacetOptions::default());
+        let schema = schema_builder.build();
+        let index = Index::create_in_ram(schema);
+        let mut index_writer = index.writer_for_tests()?;
+        index_writer.add_document(doc!(facet_field=>Facet::from_text("/a/b").unwrap()))?;
+        index_writer.add_document(TantivyDocument::default())?;
+        index_writer.commit()?;
+        let searcher = index.reader()?.searcher();
+        let facet_reader = searcher.segment_reader(0u32).facet_reader("facet").unwrap();
+        let mut facet_ords = Vec::new();
+        facet_ords.extend(facet_reader.facet_ords(0u32));
+        assert_eq!(&facet_ords, &[0u64]);
+        facet_ords.clear();
+        facet_ords.extend(facet_reader.facet_ords(1u32));
+        assert!(facet_ords.is_empty());
+        Ok(())
+    }
 
-    // #[test]
-    // fn test_facet_not_populated_for_any_docs() -> crate::Result<()> {
-    //     let mut schema_builder = SchemaBuilder::default();
-    //     schema_builder.add_facet_field("facet", FacetOptions::default());
-    //     let schema = schema_builder.build();
-    //     let index = Index::create_in_ram(schema);
-    //     let mut index_writer = index.writer_for_tests()?;
-    //     index_writer.add_document(TantivyDocument::default())?;
-    //     index_writer.add_document(TantivyDocument::default())?;
-    //     index_writer.commit()?;
-    //     let searcher = index.reader()?.searcher();
-    //     let facet_reader = searcher.segment_reader(0u32).facet_reader("facet").unwrap();
-    //     assert!(facet_reader.facet_ords(0u32).next().is_none());
-    //     assert!(facet_reader.facet_ords(1u32).next().is_none());
-    //     Ok(())
-    // }
+    #[test]
+    fn test_facet_not_populated_for_any_docs() -> crate::Result<()> {
+        let mut schema_builder = SchemaBuilder::default();
+        schema_builder.add_facet_field("facet", FacetOptions::default());
+        let schema = schema_builder.build();
+        let index = Index::create_in_ram(schema);
+        let mut index_writer = index.writer_for_tests()?;
+        index_writer.add_document(TantivyDocument::default())?;
+        index_writer.add_document(TantivyDocument::default())?;
+        index_writer.commit()?;
+        let searcher = index.reader()?.searcher();
+        let facet_reader = searcher.segment_reader(0u32).facet_reader("facet").unwrap();
+        assert!(facet_reader.facet_ords(0u32).next().is_none());
+        assert!(facet_reader.facet_ords(1u32).next().is_none());
+        Ok(())
+    }
 }

src/fastfield/mod.rs

@@ -80,7 +80,7 @@ mod tests {
     use std::path::Path;
 
     use columnar::StrColumn;
-    use common::{ByteCount, DateTimePrecision, HasLen, TerminatingWrite};
+    use common::{ByteCount, HasLen, TerminatingWrite};
     use once_cell::sync::Lazy;
     use rand::prelude::SliceRandom;
     use rand::rngs::StdRng;
@@ -88,15 +88,14 @@ mod tests {
 
     use super::*;
     use crate::directory::{Directory, RamDirectory, WritePtr};
-    use crate::index::SegmentId;
     use crate::merge_policy::NoMergePolicy;
     use crate::schema::{
-        DateOptions, Facet, FacetOptions, Field, JsonObjectOptions, Schema, SchemaBuilder,
-        TantivyDocument, TextOptions, FAST, INDEXED, STORED, STRING, TEXT,
+        Facet, FacetOptions, Field, JsonObjectOptions, Schema, SchemaBuilder, TantivyDocument,
+        TextOptions, FAST, INDEXED, STORED, STRING, TEXT,
     };
     use crate::time::OffsetDateTime;
     use crate::tokenizer::{LowerCaser, RawTokenizer, TextAnalyzer, TokenizerManager};
-    use crate::{Index, IndexWriter, SegmentReader};
+    use crate::{DateOptions, DateTimePrecision, Index, IndexWriter, SegmentId, SegmentReader};
 
     pub static SCHEMA: Lazy<Schema> = Lazy::new(|| {
         let mut schema_builder = Schema::builder();

src/fastfield/writer.rs

@@ -1,14 +1,14 @@
 use std::io;
 
 use columnar::{ColumnarWriter, NumericalValue};
-use common::{DateTimePrecision, JsonPathWriter};
+use common::JsonPathWriter;
 use tokenizer_api::Token;
 
 use crate::indexer::doc_id_mapping::DocIdMapping;
 use crate::schema::document::{Document, ReferenceValue, ReferenceValueLeaf, Value};
 use crate::schema::{value_type_to_column_type, Field, FieldType, Schema, Type};
 use crate::tokenizer::{TextAnalyzer, TokenizerManager};
-use crate::{DocId, TantivyError};
+use crate::{DateTimePrecision, DocId, TantivyError};
 
 /// Only index JSON down to a depth of 20.
 /// This is mostly to guard us from a stack overflow triggered by malicious input.
@@ -183,7 +183,8 @@ impl FastFieldsWriter {
                         .record_datetime(doc_id, field_name, truncated_datetime);
                 }
                 ReferenceValueLeaf::Facet(val) => {
-                    self.columnar_writer.record_str(doc_id, field_name, val);
+                    self.columnar_writer
+                        .record_str(doc_id, field_name, val.encoded_str());
                 }
                 ReferenceValueLeaf::Bytes(val) => {
                     self.columnar_writer.record_bytes(doc_id, field_name, val);

src/functional_test.rs

@@ -1,12 +1,9 @@
-#![allow(deprecated)] // Remove with index sorting
-
 use std::collections::HashSet;
 
 use rand::{thread_rng, Rng};
 
 use crate::indexer::index_writer::MEMORY_BUDGET_NUM_BYTES_MIN;
 use crate::schema::*;
-#[allow(deprecated)]
 use crate::{doc, schema, Index, IndexSettings, IndexSortByField, IndexWriter, Order, Searcher};
 
 fn check_index_content(searcher: &Searcher, vals: &[u64]) -> crate::Result<()> {

src/index/index.rs

@@ -3,7 +3,7 @@ use std::fmt;
 #[cfg(feature = "mmap")]
 use std::path::Path;
 use std::path::PathBuf;
-use std::thread::available_parallelism;
+use std::sync::Arc;
 
 use super::segment::Segment;
 use super::segment_reader::merge_field_meta_data;
@@ -20,7 +20,7 @@ use crate::indexer::segment_updater::save_metas;
 use crate::indexer::{IndexWriter, SingleSegmentIndexWriter};
 use crate::reader::{IndexReader, IndexReaderBuilder};
 use crate::schema::document::Document;
-use crate::schema::{Field, FieldType, Schema, Type};
+use crate::schema::{Field, FieldType, Schema};
 use crate::tokenizer::{TextAnalyzer, TokenizerManager};
 use crate::SegmentReader;
 
@@ -248,14 +248,6 @@ impl IndexBuilder {
                         sort_by_field.field
                     )));
                 }
-                let supported_field_types = [Type::I64, Type::U64, Type::F64, Type::Date];
-                let field_type = entry.field_type().value_type();
-                if !supported_field_types.contains(&field_type) {
-                    return Err(TantivyError::InvalidArgument(format!(
-                        "Unsupported field type in sort_by_field: {field_type:?}. Supported field \
-                         types: {supported_field_types:?} ",
-                    )));
-                }
             }
             Ok(())
         } else {
@@ -292,7 +284,7 @@ pub struct Index {
     directory: ManagedDirectory,
     schema: Schema,
     settings: IndexSettings,
-    executor: Executor,
+    executor: Arc<Executor>,
     tokenizers: TokenizerManager,
     fast_field_tokenizers: TokenizerManager,
     inventory: SegmentMetaInventory,
@@ -317,25 +309,29 @@ impl Index {
     ///
     /// By default the executor is single thread, and simply runs in the calling thread.
     pub fn search_executor(&self) -> &Executor {
-        &self.executor
+        self.executor.as_ref()
     }
 
     /// Replace the default single thread search executor pool
     /// by a thread pool with a given number of threads.
     pub fn set_multithread_executor(&mut self, num_threads: usize) -> crate::Result<()> {
-        self.executor = Executor::multi_thread(num_threads, "tantivy-search-")?;
+        self.executor = Arc::new(Executor::multi_thread(num_threads, "tantivy-search-")?);
         Ok(())
     }
 
     /// Custom thread pool by a outer thread pool.
-    pub fn set_executor(&mut self, executor: Executor) {
-        self.executor = executor;
+    pub fn set_shared_multithread_executor(
+        &mut self,
+        shared_thread_pool: Arc<Executor>,
+    ) -> crate::Result<()> {
+        self.executor = shared_thread_pool.clone();
+        Ok(())
     }
 
     /// Replace the default single thread search executor pool
     /// by a thread pool with as many threads as there are CPUs on the system.
     pub fn set_default_multithread_executor(&mut self) -> crate::Result<()> {
-        let default_num_threads = available_parallelism()?.get();
+        let default_num_threads = num_cpus::get();
         self.set_multithread_executor(default_num_threads)
     }
 
@@ -413,7 +409,7 @@ impl Index {
             schema,
             tokenizers: TokenizerManager::default(),
             fast_field_tokenizers: TokenizerManager::default(),
-            executor: Executor::single_thread(),
+            executor: Arc::new(Executor::single_thread()),
             inventory,
         }
     }
@@ -616,7 +612,7 @@ impl Index {
         &self,
         memory_budget_in_bytes: usize,
     ) -> crate::Result<IndexWriter<D>> {
-        let mut num_threads = std::cmp::min(available_parallelism()?.get(), MAX_NUM_THREAD);
+        let mut num_threads = std::cmp::min(num_cpus::get(), MAX_NUM_THREAD);
         let memory_budget_num_bytes_per_thread = memory_budget_in_bytes / num_threads;
         if memory_budget_num_bytes_per_thread < MEMORY_BUDGET_NUM_BYTES_MIN {
             num_threads = (memory_budget_in_bytes / MEMORY_BUDGET_NUM_BYTES_MIN).max(1);

src/index/inverted_index_reader.rs

@@ -1,13 +1,12 @@
 use std::io;
 
-use common::json_path_writer::JSON_END_OF_PATH;
 use common::BinarySerializable;
 use fnv::FnvHashSet;
 
 use crate::directory::FileSlice;
 use crate::positions::PositionReader;
 use crate::postings::{BlockSegmentPostings, SegmentPostings, TermInfo};
-use crate::schema::{IndexRecordOption, Term, Type};
+use crate::schema::{IndexRecordOption, Term, Type, JSON_END_OF_PATH};
 use crate::termdict::TermDictionary;
 
 /// The inverted index reader is in charge of accessing

src/index/mod.rs

@@ -1,3 +1,5 @@
+//! # Index Module
+//!
 //! The `index` module in Tantivy contains core components to read and write indexes.
 //!
 //! It contains `Index` and `Segment`, where a `Index` consists of one or more `Segment`s.

src/index/segment_reader.rs

@@ -318,14 +318,14 @@ impl SegmentReader {
                         if create_canonical {
                             // Without expand dots enabled dots need to be escaped.
                             let escaped_json_path = json_path.replace('.', "\\.");
-                            let full_path = format!("{field_name}.{escaped_json_path}");
+                            let full_path = format!("{}.{}", field_name, escaped_json_path);
                             let full_path_unescaped = format!("{}.{}", field_name, &json_path);
                             map_to_canonical.insert(full_path_unescaped, full_path.to_string());
                             full_path
                         } else {
                             // With expand dots enabled, we can use '.' instead of '\u{1}'.
                             json_path_sep_to_dot(&mut json_path);
-                            format!("{field_name}.{json_path}")
+                            format!("{}.{}", field_name, json_path)
                         }
                     };
                     indexed_fields.extend(
@@ -406,7 +406,7 @@ impl SegmentReader {
     }
 
     /// Returns an iterator that will iterate over the alive document ids
-    pub fn doc_ids_alive(&self) -> Box<dyn Iterator<Item = DocId> + Send + '_> {
+    pub fn doc_ids_alive(&self) -> Box<dyn Iterator<Item = DocId> + '_> {
         if let Some(alive_bitset) = &self.alive_bitset_opt {
             Box::new(alive_bitset.iter_alive())
         } else {

src/indexer/delete_queue.rs

@@ -246,9 +246,8 @@ impl DeleteCursor {
 mod tests {
 
     use super::{DeleteOperation, DeleteQueue};
-    use crate::index::SegmentReader;
     use crate::query::{Explanation, Scorer, Weight};
-    use crate::{DocId, Score};
+    use crate::{DocId, Score, SegmentReader};
 
     struct DummyWeight;
     impl Weight for DummyWeight {

src/indexer/doc_id_mapping.rs

@@ -159,7 +159,7 @@ mod tests_indexsorting {
     use crate::indexer::NoMergePolicy;
     use crate::query::QueryParser;
     use crate::schema::*;
-    use crate::{DocAddress, Index, IndexBuilder, IndexSettings, IndexSortByField, Order};
+    use crate::{DocAddress, Index, IndexSettings, IndexSortByField, Order};
 
     fn create_test_index(
         index_settings: Option<IndexSettings>,
@@ -306,10 +306,12 @@ mod tests_indexsorting {
         let my_string_field = index.schema().get_field("string_field").unwrap();
         let searcher = index.reader()?.searcher();
         {
-            assert!(searcher
-                .doc::<TantivyDocument>(DocAddress::new(0, 0))?
-                .get_first(my_string_field)
-                .is_none());
+            assert_eq!(
+                searcher
+                    .doc::<TantivyDocument>(DocAddress::new(0, 0))?
+                    .get_first(my_string_field),
+                None
+            );
             assert_eq!(
                 searcher
                     .doc::<TantivyDocument>(DocAddress::new(0, 3))?
@@ -342,7 +344,7 @@ mod tests_indexsorting {
                 Some("blublub")
             );
             let doc = searcher.doc::<TantivyDocument>(DocAddress::new(0, 4))?;
-            assert!(doc.get_first(my_string_field).is_none());
+            assert_eq!(doc.get_first(my_string_field), None);
         }
         // sort by field desc
         let index = create_test_index(
@@ -555,28 +557,4 @@ mod tests_indexsorting {
             &[2000, 8000, 3000]
         );
     }
-
-    #[test]
-    fn test_text_sort() -> crate::Result<()> {
-        let mut schema_builder = SchemaBuilder::new();
-        schema_builder.add_text_field("id", STRING | FAST | STORED);
-        schema_builder.add_text_field("name", TEXT | STORED);
-
-        let resp = IndexBuilder::new()
-            .schema(schema_builder.build())
-            .settings(IndexSettings {
-                sort_by_field: Some(IndexSortByField {
-                    field: "id".to_string(),
-                    order: Order::Asc,
-                }),
-                ..Default::default()
-            })
-            .create_in_ram();
-        assert!(resp
-            .unwrap_err()
-            .to_string()
-            .contains("Unsupported field type"));
-
-        Ok(())
-    }
 }

src/indexer/index_writer.rs

@@ -814,9 +814,10 @@ mod tests {
     use crate::indexer::index_writer::MEMORY_BUDGET_NUM_BYTES_MIN;
     use crate::indexer::NoMergePolicy;
     use crate::query::{BooleanQuery, Occur, Query, QueryParser, TermQuery};
+    use crate::schema::document::Value;
     use crate::schema::{
         self, Facet, FacetOptions, IndexRecordOption, IpAddrOptions, NumericOptions, Schema,
-        TextFieldIndexing, TextOptions, Value, FAST, INDEXED, STORED, STRING, TEXT,
+        TextFieldIndexing, TextOptions, FAST, INDEXED, STORED, STRING, TEXT,
     };
     use crate::store::DOCSTORE_CACHE_CAPACITY;
     use crate::{
@@ -1979,13 +1980,7 @@ mod tests {
                 .unwrap();
             // test store iterator
             for doc in store_reader.iter::<TantivyDocument>(segment_reader.alive_bitset()) {
-                let id = doc
-                    .unwrap()
-                    .get_first(id_field)
-                    .unwrap()
-                    .as_value()
-                    .as_u64()
-                    .unwrap();
+                let id = doc.unwrap().get_first(id_field).unwrap().as_u64().unwrap();
                 assert!(expected_ids_and_num_occurrences.contains_key(&id));
             }
             // test store random access
@@ -2018,7 +2013,7 @@ mod tests {
                     let mut bool2 = doc.get_all(multi_bools);
                     assert_eq!(bool, bool2.next().unwrap().as_bool().unwrap());
                     assert_ne!(bool, bool2.next().unwrap().as_bool().unwrap());
-                    assert!(bool2.next().is_none())
+                    assert_eq!(None, bool2.next())
                 }
             }
         }

src/indexer/log_merge_policy.rs

@@ -144,9 +144,9 @@ mod tests {
     use once_cell::sync::Lazy;
 
     use super::*;
-    use crate::index::{SegmentId, SegmentMetaInventory};
-    use crate::schema;
+    use crate::index::SegmentMetaInventory;
     use crate::schema::INDEXED;
+    use crate::{schema, SegmentId};
 
     static INVENTORY: Lazy<SegmentMetaInventory> = Lazy::new(SegmentMetaInventory::default);
 

src/indexer/merge_operation.rs

@@ -1,8 +1,7 @@
 use std::collections::HashSet;
 use std::ops::Deref;
 
-use crate::index::SegmentId;
-use crate::{Inventory, Opstamp, TrackedObject};
+use crate::{Inventory, Opstamp, SegmentId, TrackedObject};
 
 #[derive(Default)]
 pub(crate) struct MergeOperationInventory(Inventory<InnerMergeOperation>);

src/indexer/merger.rs

@@ -13,7 +13,7 @@ use crate::docset::{DocSet, TERMINATED};
 use crate::error::DataCorruption;
 use crate::fastfield::{AliveBitSet, FastFieldNotAvailableError};
 use crate::fieldnorm::{FieldNormReader, FieldNormReaders, FieldNormsSerializer, FieldNormsWriter};
-use crate::index::{Segment, SegmentComponent, SegmentReader};
+use crate::index::{Segment, SegmentReader};
 use crate::indexer::doc_id_mapping::{MappingType, SegmentDocIdMapping};
 use crate::indexer::SegmentSerializer;
 use crate::postings::{InvertedIndexSerializer, Postings, SegmentPostings};
@@ -21,7 +21,8 @@ use crate::schema::{value_type_to_column_type, Field, FieldType, Schema};
 use crate::store::StoreWriter;
 use crate::termdict::{TermMerger, TermOrdinal};
 use crate::{
-    DocAddress, DocId, IndexSettings, IndexSortByField, InvertedIndexReader, Order, SegmentOrdinal,
+    DocAddress, DocId, IndexSettings, IndexSortByField, InvertedIndexReader, Order,
+    SegmentComponent, SegmentOrdinal,
 };
 
 /// Segment's max doc must be `< MAX_DOC_LIMIT`.
@@ -787,25 +788,23 @@ impl IndexMerger {
 mod tests {
 
     use columnar::Column;
-    use proptest::prop_oneof;
-    use proptest::strategy::Strategy;
     use schema::FAST;
 
     use crate::collector::tests::{
         BytesFastFieldTestCollector, FastFieldTestCollector, TEST_COLLECTOR_WITH_SCORE,
     };
     use crate::collector::{Count, FacetCollector};
-    use crate::index::{Index, SegmentId};
-    use crate::indexer::NoMergePolicy;
+    use crate::index::Index;
     use crate::query::{AllQuery, BooleanQuery, EnableScoring, Scorer, TermQuery};
+    use crate::schema::document::Value;
     use crate::schema::{
         Facet, FacetOptions, IndexRecordOption, NumericOptions, TantivyDocument, Term,
-        TextFieldIndexing, Value, INDEXED, TEXT,
+        TextFieldIndexing, INDEXED, TEXT,
     };
     use crate::time::OffsetDateTime;
     use crate::{
         assert_nearly_equals, schema, DateTime, DocAddress, DocId, DocSet, IndexSettings,
-        IndexSortByField, IndexWriter, Order, Searcher,
+        IndexSortByField, IndexWriter, Order, Searcher, SegmentId,
     };
 
     #[test]
@@ -912,24 +911,15 @@ mod tests {
             }
             {
                 let doc = searcher.doc::<TantivyDocument>(DocAddress::new(0, 0))?;
-                assert_eq!(
-                    doc.get_first(text_field).unwrap().as_value().as_str(),
-                    Some("af b")
-                );
+                assert_eq!(doc.get_first(text_field).unwrap().as_str(), Some("af b"));
             }
             {
                 let doc = searcher.doc::<TantivyDocument>(DocAddress::new(0, 1))?;
-                assert_eq!(
-                    doc.get_first(text_field).unwrap().as_value().as_str(),
-                    Some("a b c")
-                );
+                assert_eq!(doc.get_first(text_field).unwrap().as_str(), Some("a b c"));
             }
             {
                 let doc = searcher.doc::<TantivyDocument>(DocAddress::new(0, 2))?;
-                assert_eq!(
-                    doc.get_first(text_field).unwrap().as_value().as_str(),
-                    Some("a b c d")
-                );
+                assert_eq!(doc.get_first(text_field).unwrap().as_str(), Some("a b c d"));
             }
             {
                 let doc = searcher.doc::<TantivyDocument>(DocAddress::new(0, 3))?;
@@ -1534,112 +1524,6 @@ mod tests {
         Ok(())
     }
 
-    #[derive(Debug, Clone, Copy, Eq, PartialEq)]
-    enum IndexingOp {
-        ZeroVal,
-        OneVal { val: u64 },
-        TwoVal { val: u64 },
-        Commit,
-    }
-
-    fn balanced_operation_strategy() -> impl Strategy<Value = IndexingOp> {
-        prop_oneof![
-            (0u64..1u64).prop_map(|_| IndexingOp::ZeroVal),
-            (0u64..1u64).prop_map(|val| IndexingOp::OneVal { val }),
-            (0u64..1u64).prop_map(|val| IndexingOp::TwoVal { val }),
-            (0u64..1u64).prop_map(|_| IndexingOp::Commit),
-        ]
-    }
-
-    use proptest::prelude::*;
-    proptest! {
-        #[test]
-        fn test_merge_columnar_int_proptest(ops in proptest::collection::vec(balanced_operation_strategy(), 1..20)) {
-            assert!(test_merge_int_fields(&ops[..]).is_ok());
-        }
-    }
-    fn test_merge_int_fields(ops: &[IndexingOp]) -> crate::Result<()> {
-        if ops.iter().all(|op| *op == IndexingOp::Commit) {
-            return Ok(());
-        }
-        let expected_doc_and_vals: Vec<(u32, Vec<u64>)> = ops
-            .iter()
-            .filter(|op| *op != &IndexingOp::Commit)
-            .map(|op| match op {
-                IndexingOp::ZeroVal => vec![],
-                IndexingOp::OneVal { val } => vec![*val],
-                IndexingOp::TwoVal { val } => vec![*val, *val],
-                IndexingOp::Commit => unreachable!(),
-            })
-            .enumerate()
-            .map(|(id, val)| (id as u32, val))
-            .collect();
-
-        let mut schema_builder = schema::Schema::builder();
-        let int_options = NumericOptions::default().set_fast().set_indexed();
-        let int_field = schema_builder.add_u64_field("intvals", int_options);
-        let index = Index::create_in_ram(schema_builder.build());
-        {
-            let mut index_writer = index.writer_for_tests()?;
-            index_writer.set_merge_policy(Box::new(NoMergePolicy));
-            let index_doc = |index_writer: &mut IndexWriter, int_vals: &[u64]| {
-                let mut doc = TantivyDocument::default();
-                for &val in int_vals {
-                    doc.add_u64(int_field, val);
-                }
-                index_writer.add_document(doc).unwrap();
-            };
-
-            for op in ops {
-                match op {
-                    IndexingOp::ZeroVal => index_doc(&mut index_writer, &[]),
-                    IndexingOp::OneVal { val } => index_doc(&mut index_writer, &[*val]),
-                    IndexingOp::TwoVal { val } => index_doc(&mut index_writer, &[*val, *val]),
-                    IndexingOp::Commit => {
-                        index_writer.commit().expect("commit failed");
-                    }
-                }
-            }
-            index_writer.commit().expect("commit failed");
-        }
-        {
-            let mut segment_ids = index.searchable_segment_ids()?;
-            segment_ids.sort();
-            let mut index_writer: IndexWriter = index.writer_for_tests()?;
-            index_writer.merge(&segment_ids).wait()?;
-            index_writer.wait_merging_threads()?;
-        }
-        let reader = index.reader()?;
-        reader.reload()?;
-
-        let mut vals: Vec<u64> = Vec::new();
-        let mut test_vals = move |col: &Column<u64>, doc: DocId, expected: &[u64]| {
-            vals.clear();
-            vals.extend(col.values_for_doc(doc));
-            assert_eq!(&vals[..], expected);
-        };
-
-        let mut test_col = move |col: &Column<u64>, column_expected: &[(u32, Vec<u64>)]| {
-            for (doc_id, vals) in column_expected.iter() {
-                test_vals(col, *doc_id, vals);
-            }
-        };
-
-        {
-            let searcher = reader.searcher();
-            let segment = searcher.segment_reader(0u32);
-            let col = segment
-                .fast_fields()
-                .column_opt::<u64>("intvals")
-                .unwrap()
-                .unwrap();
-
-            test_col(&col, &expected_doc_and_vals);
-        }
-
-        Ok(())
-    }
-
     #[test]
     fn test_merge_multivalued_int_fields_simple() -> crate::Result<()> {
         let mut schema_builder = schema::Schema::builder();

src/indexer/merger_sorted_index_test.rs

@@ -3,15 +3,15 @@ mod tests {
     use crate::collector::TopDocs;
     use crate::fastfield::AliveBitSet;
     use crate::index::Index;
-    use crate::postings::Postings;
     use crate::query::QueryParser;
+    use crate::schema::document::Value;
     use crate::schema::{
         self, BytesOptions, Facet, FacetOptions, IndexRecordOption, NumericOptions,
-        TextFieldIndexing, TextOptions, Value,
+        TextFieldIndexing, TextOptions,
     };
     use crate::{
-        DocAddress, DocSet, IndexSettings, IndexSortByField, IndexWriter, Order, TantivyDocument,
-        Term,
+        DocAddress, DocSet, IndexSettings, IndexSortByField, IndexWriter, Order, Postings,
+        TantivyDocument, Term,
     };
 
     fn create_test_index_posting_list_issue(index_settings: Option<IndexSettings>) -> Index {
@@ -280,16 +280,13 @@ mod tests {
                 .doc::<TantivyDocument>(DocAddress::new(0, blubber_pos))
                 .unwrap();
             assert_eq!(
-                doc.get_first(my_text_field).unwrap().as_value().as_str(),
+                doc.get_first(my_text_field).unwrap().as_str(),
                 Some("blubber")
             );
             let doc = searcher
                 .doc::<TantivyDocument>(DocAddress::new(0, 0))
                 .unwrap();
-            assert_eq!(
-                doc.get_first(int_field).unwrap().as_value().as_u64(),
-                Some(1000)
-            );
+            assert_eq!(doc.get_first(int_field).unwrap().as_u64(), Some(1000));
         }
     }
 

src/indexer/mod.rs

@@ -156,24 +156,16 @@ mod tests_mmap {
     }
     #[test]
     fn test_json_field_1byte() {
-        // Test when field name contains a '1' byte, which has special meaning in tantivy.
-        // The 1 byte can be addressed as '1' byte or '.'.
+        // Test when field name contains a 1 byte, which has special meaning in tantivy.
         let field_name_in = "\u{0001}";
         let field_name_out = "\u{0001}";
         test_json_field_name(field_name_in, field_name_out);
 
-        // Test when field name contains a '1' byte, which has special meaning in tantivy.
+        // Test when field name contains a 1 byte, which has special meaning in tantivy.
         let field_name_in = "\u{0001}";
         let field_name_out = ".";
         test_json_field_name(field_name_in, field_name_out);
     }
-    #[test]
-    fn test_json_field_dot() {
-        // Test when field name contains a '.'
-        let field_name_in = ".";
-        let field_name_out = ".";
-        test_json_field_name(field_name_in, field_name_out);
-    }
     fn test_json_field_name(field_name_in: &str, field_name_out: &str) {
         let mut schema_builder = Schema::builder();
 
@@ -182,7 +174,7 @@ mod tests_mmap {
         let index = Index::create_in_ram(schema_builder.build());
         let mut index_writer = index.writer_for_tests().unwrap();
         index_writer
-            .add_document(doc!(field=>json!({format!("{field_name_in}"): "test1", format!("num{field_name_in}"): 10})))
+            .add_document(doc!(field=>json!({format!("{field_name_in}"): "test1"})))
             .unwrap();
         index_writer
             .add_document(doc!(field=>json!({format!("a{field_name_in}"): "test2"})))
@@ -213,10 +205,10 @@ mod tests_mmap {
         let reader = index.reader().unwrap();
         let searcher = reader.searcher();
         let parse_query = QueryParser::for_index(&index, Vec::new());
-        let test_query = |query_str: &str| {
-            let query = parse_query.parse_query(query_str).unwrap();
+        let test_query = |field_name: &str| {
+            let query = parse_query.parse_query(field_name).unwrap();
             let num_docs = searcher.search(&query, &Count).unwrap();
-            assert_eq!(num_docs, 1, "{query_str}");
+            assert_eq!(num_docs, 1);
         };
         test_query(format!("json.{field_name_out}:test1").as_str());
         test_query(format!("json.a{field_name_out}:test2").as_str());
@@ -260,64 +252,6 @@ mod tests_mmap {
             "test6",
         );
         test_agg(format!("json.{field_name_out}a").as_str(), "test7");
-
-        // `.` is stored as `\u{0001}` internally in tantivy
-        let field_name_out_internal = if field_name_out == "." {
-            "\u{0001}"
-        } else {
-            field_name_out
-        };
-
-        let mut fields = reader.searcher().segment_readers()[0]
-            .inverted_index(field)
-            .unwrap()
-            .list_encoded_fields()
-            .unwrap();
-        assert_eq!(fields.len(), 8);
-        fields.sort();
-        let mut expected_fields = vec![
-            (format!("a{field_name_out_internal}"), Type::Str),
-            (format!("a{field_name_out_internal}a"), Type::Str),
-            (
-                format!("a{field_name_out_internal}a{field_name_out_internal}"),
-                Type::Str,
-            ),
-            (
-                format!("a{field_name_out_internal}\u{1}ab{field_name_out_internal}"),
-                Type::Str,
-            ),
-            (
-                format!("a{field_name_out_internal}\u{1}a{field_name_out_internal}"),
-                Type::Str,
-            ),
-            (format!("{field_name_out_internal}a"), Type::Str),
-            (format!("{field_name_out_internal}"), Type::Str),
-            (format!("num{field_name_out_internal}"), Type::I64),
-        ];
-        expected_fields.sort();
-        assert_eq!(fields, expected_fields);
-        // Check columnar reader
-        let mut columns = reader.searcher().segment_readers()[0]
-            .fast_fields()
-            .columnar()
-            .list_columns()
-            .unwrap()
-            .into_iter()
-            .map(|(name, _)| name)
-            .collect::<Vec<_>>();
-        let mut expected_columns = vec![
-            format!("json\u{1}{field_name_out_internal}"),
-            format!("json\u{1}{field_name_out_internal}a"),
-            format!("json\u{1}a{field_name_out_internal}"),
-            format!("json\u{1}a{field_name_out_internal}a"),
-            format!("json\u{1}a{field_name_out_internal}a{field_name_out_internal}"),
-            format!("json\u{1}a{field_name_out_internal}\u{1}ab{field_name_out_internal}"),
-            format!("json\u{1}a{field_name_out_internal}\u{1}a{field_name_out_internal}"),
-            format!("json\u{1}num{field_name_out_internal}"),
-        ];
-        columns.sort();
-        expected_columns.sort();
-        assert_eq!(columns, expected_columns);
     }
 
     #[test]
@@ -590,10 +524,10 @@ mod tests_mmap {
         let query_parser = QueryParser::for_index(&index, vec![]);
         // Test if field name can be queried
         for (indexed_field, val) in fields_and_vals.iter() {
-            let query_str = &format!("{indexed_field}:{val}");
+            let query_str = &format!("{}:{}", indexed_field, val);
             let query = query_parser.parse_query(query_str).unwrap();
             let count_docs = searcher.search(&*query, &TopDocs::with_limit(2)).unwrap();
-            assert!(!count_docs.is_empty(), "{indexed_field}:{val}");
+            assert!(!count_docs.is_empty(), "{}:{}", indexed_field, val);
         }
         // Test if field name can be used for aggregation
         for (field_name, val) in fields_and_vals.iter() {

src/indexer/segment_writer.rs

@@ -5,20 +5,20 @@ use tokenizer_api::BoxTokenStream;
 
 use super::doc_id_mapping::{get_doc_id_mapping_from_field, DocIdMapping};
 use super::operation::AddOperation;
+use crate::core::json_utils::index_json_values;
 use crate::fastfield::FastFieldsWriter;
 use crate::fieldnorm::{FieldNormReaders, FieldNormsWriter};
-use crate::index::{Segment, SegmentComponent};
+use crate::index::Segment;
 use crate::indexer::segment_serializer::SegmentSerializer;
-use crate::json_utils::{index_json_value, IndexingPositionsPerPath};
 use crate::postings::{
     compute_table_memory_size, serialize_postings, IndexingContext, IndexingPosition,
     PerFieldPostingsWriter, PostingsWriter,
 };
-use crate::schema::document::{Document, Value};
+use crate::schema::document::{Document, ReferenceValue, Value};
 use crate::schema::{FieldEntry, FieldType, Schema, Term, DATE_TIME_PRECISION_INDEXED};
 use crate::store::{StoreReader, StoreWriter};
 use crate::tokenizer::{FacetTokenizer, PreTokenizedStream, TextAnalyzer, Tokenizer};
-use crate::{DocId, Opstamp, TantivyError};
+use crate::{DocId, Opstamp, SegmentComponent, TantivyError};
 
 /// Computes the initial size of the hash table.
 ///
@@ -68,7 +68,6 @@ pub struct SegmentWriter {
     pub(crate) fast_field_writers: FastFieldsWriter,
     pub(crate) fieldnorms_writer: FieldNormsWriter,
     pub(crate) json_path_writer: JsonPathWriter,
-    pub(crate) json_positions_per_path: IndexingPositionsPerPath,
     pub(crate) doc_opstamps: Vec<Opstamp>,
     per_field_text_analyzers: Vec<TextAnalyzer>,
     term_buffer: Term,
@@ -120,7 +119,6 @@ impl SegmentWriter {
             per_field_postings_writers,
             fieldnorms_writer: FieldNormsWriter::for_schema(&schema),
             json_path_writer: JsonPathWriter::default(),
-            json_positions_per_path: IndexingPositionsPerPath::default(),
             segment_serializer,
             fast_field_writers: FastFieldsWriter::from_schema_and_tokenizer_manager(
                 &schema,
@@ -206,7 +204,8 @@ impl SegmentWriter {
                         // Used to help with linting and type checking.
                         let value = value_access as D::Value<'_>;
 
-                        let facet_str = value.as_facet().ok_or_else(make_schema_error)?;
+                        let facet = value.as_facet().ok_or_else(make_schema_error)?;
+                        let facet_str = facet.encoded_str();
                         let mut facet_tokenizer = facet_tokenizer.token_stream(facet_str);
                         let mut indexing_position = IndexingPosition::default();
                         postings_writer.index_text(
@@ -229,7 +228,7 @@ impl SegmentWriter {
                                 &mut self.per_field_text_analyzers[field.field_id() as usize];
                             text_analyzer.token_stream(text)
                         } else if let Some(tok_str) = value.as_pre_tokenized_text() {
-                            BoxTokenStream::new(PreTokenizedStream::from(*tok_str.clone()))
+                            BoxTokenStream::new(PreTokenizedStream::from(tok_str.clone()))
                         } else {
                             continue;
                         };
@@ -343,24 +342,26 @@ impl SegmentWriter {
                 FieldType::JsonObject(json_options) => {
                     let text_analyzer =
                         &mut self.per_field_text_analyzers[field.field_id() as usize];
+                    let json_values_it = values.map(|value_access| {
+                        // Used to help with linting and type checking.
+                        let value_access = value_access as D::Value<'_>;
+                        let value = value_access.as_value();
 
-                    self.json_positions_per_path.clear();
-                    self.json_path_writer
-                        .set_expand_dots(json_options.is_expand_dots_enabled());
-                    for json_value in values {
-                        self.json_path_writer.clear();
-
-                        index_json_value(
-                            doc_id,
-                            json_value,
-                            text_analyzer,
-                            term_buffer,
-                            &mut self.json_path_writer,
-                            postings_writer,
-                            ctx,
-                            &mut self.json_positions_per_path,
-                        );
-                    }
+                        match value {
+                            ReferenceValue::Object(object_iter) => Ok(object_iter),
+                            _ => Err(make_schema_error()),
+                        }
+                    });
+                    index_json_values::<D::Value<'_>>(
+                        doc_id,
+                        json_values_it,
+                        text_analyzer,
+                        json_options.is_expand_dots_enabled(),
+                        term_buffer,
+                        postings_writer,
+                        &mut self.json_path_writer,
+                        ctx,
+                    )?;
                 }
                 FieldType::IpAddr(_) => {
                     let mut num_vals = 0;
@@ -495,21 +496,22 @@ mod tests {
     use tempfile::TempDir;
 
     use crate::collector::{Count, TopDocs};
+    use crate::core::json_utils::JsonTermWriter;
     use crate::directory::RamDirectory;
-    use crate::fastfield::FastValue;
-    use crate::postings::{Postings, TermInfo};
+    use crate::postings::TermInfo;
     use crate::query::{PhraseQuery, QueryParser};
+    use crate::schema::document::Value;
     use crate::schema::{
-        Document, IndexRecordOption, OwnedValue, Schema, TextFieldIndexing, TextOptions, Value,
-        STORED, STRING, TEXT,
+        Document, IndexRecordOption, Schema, TextFieldIndexing, TextOptions, Type, STORED, STRING,
+        TEXT,
     };
     use crate::store::{Compressor, StoreReader, StoreWriter};
     use crate::time::format_description::well_known::Rfc3339;
     use crate::time::OffsetDateTime;
     use crate::tokenizer::{PreTokenizedString, Token};
     use crate::{
-        DateTime, Directory, DocAddress, DocSet, Index, IndexWriter, TantivyDocument, Term,
-        TERMINATED,
+        DateTime, Directory, DocAddress, DocSet, Index, IndexWriter, Postings, TantivyDocument,
+        Term, TERMINATED,
     };
 
     #[test]
@@ -554,15 +556,9 @@ mod tests {
         let reader = StoreReader::open(directory.open_read(path).unwrap(), 0).unwrap();
         let doc = reader.get::<TantivyDocument>(0).unwrap();
 
-        assert_eq!(doc.field_values().count(), 2);
-        assert_eq!(
-            doc.get_all(text_field).next().unwrap().as_value().as_str(),
-            Some("A")
-        );
-        assert_eq!(
-            doc.get_all(text_field).nth(1).unwrap().as_value().as_str(),
-            Some("title")
-        );
+        assert_eq!(doc.field_values().len(), 2);
+        assert_eq!(doc.field_values()[0].value().as_str(), Some("A"));
+        assert_eq!(doc.field_values()[1].value().as_str(), Some("title"));
     }
     #[test]
     fn test_simple_json_indexing() {
@@ -602,51 +598,12 @@ mod tests {
         assert_eq!(score_docs.len(), 2);
     }
 
-    #[test]
-    fn test_flat_json_indexing() {
-        // A JSON Object that contains mixed values on the first level
-        let mut schema_builder = Schema::builder();
-        let json_field = schema_builder.add_json_field("json", STORED | STRING);
-        let schema = schema_builder.build();
-        let index = Index::create_in_ram(schema.clone());
-        let mut writer = index.writer_for_tests().unwrap();
-        // Text, i64, u64
-        writer.add_document(doc!(json_field=>"b")).unwrap();
-        writer
-            .add_document(doc!(json_field=>OwnedValue::I64(10i64)))
-            .unwrap();
-        writer
-            .add_document(doc!(json_field=>OwnedValue::U64(55u64)))
-            .unwrap();
-        writer
-            .add_document(doc!(json_field=>json!({"my_field": "a"})))
-            .unwrap();
-        writer.commit().unwrap();
-
-        let search_and_expect = |query| {
-            let query_parser = QueryParser::for_index(&index, vec![json_field]);
-            let text_query = query_parser.parse_query(query).unwrap();
-            let score_docs: Vec<(_, DocAddress)> = index
-                .reader()
-                .unwrap()
-                .searcher()
-                .search(&text_query, &TopDocs::with_limit(4))
-                .unwrap();
-            assert_eq!(score_docs.len(), 1);
-        };
-
-        search_and_expect("my_field:a");
-        search_and_expect("b");
-        search_and_expect("10");
-        search_and_expect("55");
-    }
-
     #[test]
     fn test_json_indexing() {
         let mut schema_builder = Schema::builder();
         let json_field = schema_builder.add_json_field("json", STORED | TEXT);
         let schema = schema_builder.build();
-        let json_val: serde_json::Value = serde_json::from_str(
+        let json_val: serde_json::Map<String, serde_json::Value> = serde_json::from_str(
             r#"{
             "toto": "titi",
             "float": -0.2,
@@ -674,125 +631,129 @@ mod tests {
                 doc_id: 0u32,
             })
             .unwrap();
-        let serdeser_json_val = serde_json::from_str::<serde_json::Value>(&doc.to_json(&schema))
+        let serdeser_json_val = serde_json::from_str::<serde_json::Map<String, serde_json::Value>>(
+            &doc.to_json(&schema),
+        )
+        .unwrap()
+        .get("json")
+        .unwrap()[0]
+            .as_object()
             .unwrap()
-            .get("json")
-            .unwrap()[0]
             .clone();
         assert_eq!(json_val, serdeser_json_val);
         let segment_reader = searcher.segment_reader(0u32);
         let inv_idx = segment_reader.inverted_index(json_field).unwrap();
         let term_dict = inv_idx.terms();
 
+        let mut term = Term::with_type_and_field(Type::Json, json_field);
         let mut term_stream = term_dict.stream().unwrap();
 
-        let term_from_path =
-            |path: &str| -> Term { Term::from_field_json_path(json_field, path, false) };
+        let mut json_term_writer = JsonTermWriter::wrap(&mut term, false);
 
-        fn set_fast_val<T: FastValue>(val: T, mut term: Term) -> Term {
-            term.append_type_and_fast_value(val);
-            term
-        }
-        fn set_str(val: &str, mut term: Term) -> Term {
-            term.append_type_and_str(val);
-            term
-        }
-
-        let term = term_from_path("bool");
+        json_term_writer.push_path_segment("bool");
+        json_term_writer.set_fast_value(true);
         assert!(term_stream.advance());
         assert_eq!(
             term_stream.key(),
-            set_fast_val(true, term).serialized_value_bytes()
+            json_term_writer.term().serialized_value_bytes()
         );
 
-        let term = term_from_path("complexobject.field\\.with\\.dot");
+        json_term_writer.pop_path_segment();
+        json_term_writer.push_path_segment("complexobject");
+        json_term_writer.push_path_segment("field.with.dot");
+        json_term_writer.set_fast_value(1i64);
         assert!(term_stream.advance());
         assert_eq!(
             term_stream.key(),
-            set_fast_val(1i64, term).serialized_value_bytes()
+            json_term_writer.term().serialized_value_bytes()
         );
 
-        // Date
-        let term = term_from_path("date");
-
+        json_term_writer.pop_path_segment();
+        json_term_writer.pop_path_segment();
+        json_term_writer.push_path_segment("date");
+        json_term_writer.set_fast_value(DateTime::from_utc(
+            OffsetDateTime::parse("1985-04-12T23:20:50.52Z", &Rfc3339).unwrap(),
+        ));
         assert!(term_stream.advance());
         assert_eq!(
             term_stream.key(),
-            set_fast_val(
-                DateTime::from_utc(
-                    OffsetDateTime::parse("1985-04-12T23:20:50.52Z", &Rfc3339).unwrap(),
-                ),
-                term
-            )
-            .serialized_value_bytes()
+            json_term_writer.term().serialized_value_bytes()
         );
 
-        // Float
-        let term = term_from_path("float");
+        json_term_writer.pop_path_segment();
+        json_term_writer.push_path_segment("float");
+        json_term_writer.set_fast_value(-0.2f64);
         assert!(term_stream.advance());
         assert_eq!(
             term_stream.key(),
-            set_fast_val(-0.2f64, term).serialized_value_bytes()
+            json_term_writer.term().serialized_value_bytes()
         );
 
-        // Number In Array
-        let term = term_from_path("my_arr");
+        json_term_writer.pop_path_segment();
+        json_term_writer.push_path_segment("my_arr");
+        json_term_writer.set_fast_value(2i64);
         assert!(term_stream.advance());
         assert_eq!(
             term_stream.key(),
-            set_fast_val(2i64, term).serialized_value_bytes()
+            json_term_writer.term().serialized_value_bytes()
         );
 
-        let term = term_from_path("my_arr");
+        json_term_writer.set_fast_value(3i64);
         assert!(term_stream.advance());
         assert_eq!(
             term_stream.key(),
-            set_fast_val(3i64, term).serialized_value_bytes()
+            json_term_writer.term().serialized_value_bytes()
         );
 
-        let term = term_from_path("my_arr");
+        json_term_writer.set_fast_value(4i64);
         assert!(term_stream.advance());
         assert_eq!(
             term_stream.key(),
-            set_fast_val(4i64, term).serialized_value_bytes()
+            json_term_writer.term().serialized_value_bytes()
         );
 
-        // El in Array
-        let term = term_from_path("my_arr.my_key");
+        json_term_writer.push_path_segment("my_key");
+        json_term_writer.set_str("tokens");
         assert!(term_stream.advance());
         assert_eq!(
             term_stream.key(),
-            set_str("tokens", term).serialized_value_bytes()
-        );
-        let term = term_from_path("my_arr.my_key");
-        assert!(term_stream.advance());
-        assert_eq!(
-            term_stream.key(),
-            set_str("two", term).serialized_value_bytes()
+            json_term_writer.term().serialized_value_bytes()
         );
 
-        // Signed
-        let term = term_from_path("signed");
+        json_term_writer.set_str("two");
         assert!(term_stream.advance());
         assert_eq!(
             term_stream.key(),
-            set_fast_val(-2i64, term).serialized_value_bytes()
+            json_term_writer.term().serialized_value_bytes()
         );
 
-        let term = term_from_path("toto");
+        json_term_writer.pop_path_segment();
+        json_term_writer.pop_path_segment();
+        json_term_writer.push_path_segment("signed");
+        json_term_writer.set_fast_value(-2i64);
         assert!(term_stream.advance());
         assert_eq!(
             term_stream.key(),
-            set_str("titi", term).serialized_value_bytes()
-        );
-        // Unsigned
-        let term = term_from_path("unsigned");
-        assert!(term_stream.advance());
-        assert_eq!(
-            term_stream.key(),
-            set_fast_val(1i64, term).serialized_value_bytes()
+            json_term_writer.term().serialized_value_bytes()
         );
 
+        json_term_writer.pop_path_segment();
+        json_term_writer.push_path_segment("toto");
+        json_term_writer.set_str("titi");
+        assert!(term_stream.advance());
+        assert_eq!(
+            term_stream.key(),
+            json_term_writer.term().serialized_value_bytes()
+        );
+
+        json_term_writer.pop_path_segment();
+        json_term_writer.push_path_segment("unsigned");
+        json_term_writer.set_fast_value(1i64);
+        assert!(term_stream.advance());
+        assert_eq!(
+            term_stream.key(),
+            json_term_writer.term().serialized_value_bytes()
+        );
         assert!(!term_stream.advance());
     }
 
@@ -813,9 +774,14 @@ mod tests {
         let searcher = reader.searcher();
         let segment_reader = searcher.segment_reader(0u32);
         let inv_index = segment_reader.inverted_index(json_field).unwrap();
-        let mut term = Term::from_field_json_path(json_field, "mykey", false);
-        term.append_type_and_str("token");
-        let term_info = inv_index.get_term_info(&term).unwrap().unwrap();
+        let mut term = Term::with_type_and_field(Type::Json, json_field);
+        let mut json_term_writer = JsonTermWriter::wrap(&mut term, false);
+        json_term_writer.push_path_segment("mykey");
+        json_term_writer.set_str("token");
+        let term_info = inv_index
+            .get_term_info(json_term_writer.term())
+            .unwrap()
+            .unwrap();
         assert_eq!(
             term_info,
             TermInfo {
@@ -841,7 +807,7 @@ mod tests {
         let mut schema_builder = Schema::builder();
         let json_field = schema_builder.add_json_field("json", STRING);
         let schema = schema_builder.build();
-        let json_val: serde_json::Value =
+        let json_val: serde_json::Map<String, serde_json::Value> =
             serde_json::from_str(r#"{"mykey": "two tokens"}"#).unwrap();
         let doc = doc!(json_field=>json_val);
         let index = Index::create_in_ram(schema);
@@ -852,9 +818,14 @@ mod tests {
         let searcher = reader.searcher();
         let segment_reader = searcher.segment_reader(0u32);
         let inv_index = segment_reader.inverted_index(json_field).unwrap();
-        let mut term = Term::from_field_json_path(json_field, "mykey", false);
-        term.append_type_and_str("two tokens");
-        let term_info = inv_index.get_term_info(&term).unwrap().unwrap();
+        let mut term = Term::with_type_and_field(Type::Json, json_field);
+        let mut json_term_writer = JsonTermWriter::wrap(&mut term, false);
+        json_term_writer.push_path_segment("mykey");
+        json_term_writer.set_str("two tokens");
+        let term_info = inv_index
+            .get_term_info(json_term_writer.term())
+            .unwrap()
+            .unwrap();
         assert_eq!(
             term_info,
             TermInfo {
@@ -881,7 +852,7 @@ mod tests {
         let mut schema_builder = Schema::builder();
         let json_field = schema_builder.add_json_field("json", TEXT);
         let schema = schema_builder.build();
-        let json_val: serde_json::Value = serde_json::from_str(
+        let json_val: serde_json::Map<String, serde_json::Value> = serde_json::from_str(
             r#"{"mykey": [{"field": "hello happy tax payer"}, {"field": "nothello"}]}"#,
         )
         .unwrap();
@@ -892,18 +863,16 @@ mod tests {
         writer.commit().unwrap();
         let reader = index.reader().unwrap();
         let searcher = reader.searcher();
-
-        let term = Term::from_field_json_path(json_field, "mykey.field", false);
-
-        let mut hello_term = term.clone();
-        hello_term.append_type_and_str("hello");
-
-        let mut nothello_term = term.clone();
-        nothello_term.append_type_and_str("nothello");
-
-        let mut happy_term = term.clone();
-        happy_term.append_type_and_str("happy");
-
+        let mut term = Term::with_type_and_field(Type::Json, json_field);
+        let mut json_term_writer = JsonTermWriter::wrap(&mut term, false);
+        json_term_writer.push_path_segment("mykey");
+        json_term_writer.push_path_segment("field");
+        json_term_writer.set_str("hello");
+        let hello_term = json_term_writer.term().clone();
+        json_term_writer.set_str("nothello");
+        let nothello_term = json_term_writer.term().clone();
+        json_term_writer.set_str("happy");
+        let happy_term = json_term_writer.term().clone();
         let phrase_query = PhraseQuery::new(vec![hello_term, happy_term.clone()]);
         assert_eq!(searcher.search(&phrase_query, &Count).unwrap(), 1);
         let phrase_query = PhraseQuery::new(vec![nothello_term, happy_term]);

src/lib.rs

@@ -216,6 +216,11 @@ use once_cell::sync::Lazy;
 use serde::{Deserialize, Serialize};
 
 pub use self::docset::{DocSet, COLLECT_BLOCK_BUFFER_LEN, TERMINATED};
+#[deprecated(
+    since = "0.22.0",
+    note = "Will be removed in tantivy 0.23. Use export from snippet module instead"
+)]
+pub use self::snippet::{Snippet, SnippetGenerator};
 #[doc(hidden)]
 pub use crate::core::json_utils;
 pub use crate::core::{Executor, Searcher, SearcherGeneration};
@@ -223,10 +228,16 @@ pub use crate::directory::Directory;
 #[allow(deprecated)] // Remove with index sorting
 pub use crate::index::{
     Index, IndexBuilder, IndexMeta, IndexSettings, IndexSortByField, InvertedIndexReader, Order,
-    Segment, SegmentMeta, SegmentReader,
+    Segment, SegmentComponent, SegmentId, SegmentMeta, SegmentReader,
 };
+#[deprecated(
+    since = "0.22.0",
+    note = "Will be removed in tantivy 0.23. Use export from indexer module instead"
+)]
+pub use crate::indexer::PreparedCommit;
 pub use crate::indexer::{IndexWriter, SingleSegmentIndexWriter};
-pub use crate::schema::{Document, TantivyDocument, Term};
+pub use crate::postings::Postings;
+pub use crate::schema::{DateOptions, DateTimePrecision, Document, TantivyDocument, Term};
 
 /// Index format version.
 const INDEX_FORMAT_VERSION: u32 = 6;
@@ -244,7 +255,7 @@ pub struct Version {
 
 impl fmt::Debug for Version {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        fmt::Display::fmt(self, f)
+        write!(f, "{}", self.to_string())
     }
 }
 
@@ -255,10 +266,9 @@ static VERSION: Lazy<Version> = Lazy::new(|| Version {
     index_format_version: INDEX_FORMAT_VERSION,
 });
 
-impl fmt::Display for Version {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(
-            f,
+impl ToString for Version {
+    fn to_string(&self) -> String {
+        format!(
             "tantivy v{}.{}.{}, index_format v{}",
             self.major, self.minor, self.patch, self.index_format_version
         )
@@ -381,10 +391,9 @@ pub mod tests {
     use crate::docset::{DocSet, TERMINATED};
     use crate::index::SegmentReader;
     use crate::merge_policy::NoMergePolicy;
-    use crate::postings::Postings;
     use crate::query::BooleanQuery;
     use crate::schema::*;
-    use crate::{DateTime, DocAddress, Index, IndexWriter, ReloadPolicy};
+    use crate::{DateTime, DocAddress, Index, IndexWriter, Postings, ReloadPolicy};
 
     pub fn fixed_size_test<O: BinarySerializable + FixedSize + Default>() {
         let mut buffer = Vec::new();
@@ -397,20 +406,16 @@ pub mod tests {
     #[macro_export]
     macro_rules! assert_nearly_equals {
         ($left:expr, $right:expr) => {{
-            assert_nearly_equals!($left, $right, 0.0005);
-        }};
-        ($left:expr, $right:expr, $epsilon:expr) => {{
-            match (&$left, &$right, &$epsilon) {
-                (left_val, right_val, epsilon_val) => {
+            match (&$left, &$right) {
+                (left_val, right_val) => {
                     let diff = (left_val - right_val).abs();
-
-                    if diff > *epsilon_val {
+                    let add = left_val.abs() + right_val.abs();
+                    if diff > 0.0005 * add {
                         panic!(
-                            r#"assertion failed: `abs(left-right)>epsilon`
-    left: `{:?}`,
-    right: `{:?}`,
-    epsilon: `{:?}`"#,
-                            &*left_val, &*right_val, &*epsilon_val
+                            r#"assertion failed: `(left ~= right)`
+  left: `{:?}`,
+ right: `{:?}`"#,
+                            &*left_val, &*right_val
                         )
                     }
                 }
@@ -440,6 +445,7 @@ pub mod tests {
     }
 
     #[test]
+    #[cfg(not(feature = "lz4"))]
     fn test_version_string() {
         use regex::Regex;
         let regex_ptn = Regex::new(
@@ -939,7 +945,7 @@ pub mod tests {
         let mut schema_builder = Schema::builder();
         let json_field = schema_builder.add_json_field("json", STORED | TEXT);
         let schema = schema_builder.build();
-        let json_val: serde_json::Value = serde_json::from_str(
+        let json_val: serde_json::Map<String, serde_json::Value> = serde_json::from_str(
             r#"{
             "signed": 2,
             "float": 2.0,
@@ -1029,16 +1035,13 @@ pub mod tests {
                             text_field => "some other value",
                             other_text_field => "short");
         assert_eq!(document.len(), 3);
-        let values: Vec<OwnedValue> = document.get_all(text_field).map(OwnedValue::from).collect();
+        let values: Vec<&OwnedValue> = document.get_all(text_field).collect();
         assert_eq!(values.len(), 2);
-        assert_eq!(values[0].as_ref().as_str(), Some("tantivy"));
-        assert_eq!(values[1].as_ref().as_str(), Some("some other value"));
-        let values: Vec<OwnedValue> = document
-            .get_all(other_text_field)
-            .map(OwnedValue::from)
-            .collect();
+        assert_eq!(values[0].as_str(), Some("tantivy"));
+        assert_eq!(values[1].as_str(), Some("some other value"));
+        let values: Vec<&OwnedValue> = document.get_all(other_text_field).collect();
         assert_eq!(values.len(), 1);
-        assert_eq!(values[0].as_ref().as_str(), Some("short"));
+        assert_eq!(values[0].as_str(), Some("short"));
     }
 
     #[test]
@@ -1105,9 +1108,9 @@ pub mod tests {
     #[test]
     fn test_update_via_delete_insert() -> crate::Result<()> {
         use crate::collector::Count;
-        use crate::index::SegmentId;
         use crate::indexer::NoMergePolicy;
         use crate::query::AllQuery;
+        use crate::SegmentId;
 
         const DOC_COUNT: u64 = 2u64;
 

src/macros.rs

@@ -41,7 +41,6 @@
 /// );
 /// # }
 /// ```
-
 #[macro_export]
 macro_rules! doc(
     () => {
@@ -53,7 +52,7 @@ macro_rules! doc(
         {
             let mut document = $crate::TantivyDocument::default();
             $(
-                document.add_field_value($field, &$value);
+                document.add_field_value($field, $value);
             )*
             document
         }

src/postings/json_postings_writer.rs

@@ -1,6 +1,5 @@
 use std::io;
 
-use common::json_path_writer::JSON_END_OF_PATH;
 use stacker::Addr;
 
 use crate::indexer::doc_id_mapping::DocIdMapping;
@@ -8,7 +7,7 @@ use crate::indexer::path_to_unordered_id::OrderedPathId;
 use crate::postings::postings_writer::SpecializedPostingsWriter;
 use crate::postings::recorder::{BufferLender, DocIdRecorder, Recorder};
 use crate::postings::{FieldSerializer, IndexingContext, IndexingPosition, PostingsWriter};
-use crate::schema::{Field, Type};
+use crate::schema::{Field, Type, JSON_END_OF_PATH};
 use crate::tokenizer::TokenStream;
 use crate::{DocId, Term};
 

src/postings/serializer.rs

@@ -56,7 +56,7 @@ pub struct InvertedIndexSerializer {
 impl InvertedIndexSerializer {
     /// Open a new `InvertedIndexSerializer` for the given segment
     pub fn open(segment: &mut Segment) -> crate::Result<InvertedIndexSerializer> {
-        use crate::index::SegmentComponent::{Positions, Postings, Terms};
+        use crate::SegmentComponent::{Positions, Postings, Terms};
         let inv_index_serializer = InvertedIndexSerializer {
             terms_write: CompositeWrite::wrap(segment.open_write(Terms)?),
             postings_write: CompositeWrite::wrap(segment.open_write(Postings)?),

src/query/empty_query.rs

@@ -1,9 +1,8 @@
 use super::Scorer;
 use crate::docset::TERMINATED;
-use crate::index::SegmentReader;
 use crate::query::explanation::does_not_match;
 use crate::query::{EnableScoring, Explanation, Query, Weight};
-use crate::{DocId, DocSet, Score, Searcher};
+use crate::{DocId, DocSet, Score, Searcher, SegmentReader};
 
 /// `EmptyQuery` is a dummy `Query` in which no document matches.
 ///

src/query/fuzzy_query.rs

@@ -138,7 +138,8 @@ impl FuzzyTermQuery {
                 if json_path_type != Type::Str {
                     return Err(InvalidArgument(format!(
                         "The fuzzy term query requires a string path type for a json term. Found \
-                         {json_path_type:?}"
+                         {:?}",
+                        json_path_type
                     )));
                 }
             }

src/query/more_like_this/more_like_this.rs

@@ -180,7 +180,7 @@ impl MoreLikeThis {
                 let facets: Vec<&str> = values
                     .iter()
                     .map(|value| {
-                        value.as_facet().ok_or_else(|| {
+                        value.as_facet().map(|f| f.encoded_str()).ok_or_else(|| {
                             TantivyError::InvalidArgument("invalid field value".to_string())
                         })
                     })
@@ -220,7 +220,7 @@ impl MoreLikeThis {
                         let mut token_stream = tokenizer.token_stream(text);
                         token_stream.process(sink);
                     } else if let Some(tok_str) = value.as_pre_tokenized_text() {
-                        let mut token_stream = PreTokenizedStream::from(*tok_str.clone());
+                        let mut token_stream = PreTokenizedStream::from(tok_str.clone());
                         token_stream.process(sink);
                     }
                 }

src/query/phrase_prefix_query/phrase_prefix_scorer.rs

@@ -2,7 +2,7 @@ use crate::docset::{DocSet, TERMINATED};
 use crate::fieldnorm::FieldNormReader;
 use crate::postings::Postings;
 use crate::query::bm25::Bm25Weight;
-use crate::query::phrase_query::{intersection_count, intersection_exists, PhraseScorer};
+use crate::query::phrase_query::{intersection_count, PhraseScorer};
 use crate::query::Scorer;
 use crate::{DocId, Score};
 
@@ -92,17 +92,14 @@ impl<TPostings: Postings> Scorer for PhraseKind<TPostings> {
     }
 }
 
-pub struct PhrasePrefixScorer<TPostings: Postings, const SCORING_ENABLED: bool> {
+pub struct PhrasePrefixScorer<TPostings: Postings> {
     phrase_scorer: PhraseKind<TPostings>,
     suffixes: Vec<TPostings>,
     suffix_offset: u32,
     phrase_count: u32,
-    suffix_position_buffer: Vec<u32>,
 }
 
-impl<TPostings: Postings, const SCORING_ENABLED: bool>
-    PhrasePrefixScorer<TPostings, SCORING_ENABLED>
-{
+impl<TPostings: Postings> PhrasePrefixScorer<TPostings> {
     // If similarity_weight is None, then scoring is disabled.
     pub fn new(
         mut term_postings: Vec<(usize, TPostings)>,
@@ -110,7 +107,7 @@ impl<TPostings: Postings, const SCORING_ENABLED: bool>
         fieldnorm_reader: FieldNormReader,
         suffixes: Vec<TPostings>,
         suffix_pos: usize,
-    ) -> PhrasePrefixScorer<TPostings, SCORING_ENABLED> {
+    ) -> PhrasePrefixScorer<TPostings> {
         // correct indices so we can merge with our suffix term the PhraseScorer doesn't know about
         let max_offset = term_postings
             .iter()
@@ -143,7 +140,6 @@ impl<TPostings: Postings, const SCORING_ENABLED: bool>
             suffixes,
             suffix_offset: (max_offset - suffix_pos) as u32,
             phrase_count: 0,
-            suffix_position_buffer: Vec::with_capacity(100),
         };
         if phrase_prefix_scorer.doc() != TERMINATED && !phrase_prefix_scorer.matches_prefix() {
             phrase_prefix_scorer.advance();
@@ -157,6 +153,7 @@ impl<TPostings: Postings, const SCORING_ENABLED: bool>
 
     fn matches_prefix(&mut self) -> bool {
         let mut count = 0;
+        let mut positions = Vec::new();
         let current_doc = self.doc();
         let pos_matching = self.phrase_scorer.get_intersection();
         for suffix in &mut self.suffixes {
@@ -165,27 +162,16 @@ impl<TPostings: Postings, const SCORING_ENABLED: bool>
             }
             let doc = suffix.seek(current_doc);
             if doc == current_doc {
-                suffix.positions_with_offset(self.suffix_offset, &mut self.suffix_position_buffer);
-                if SCORING_ENABLED {
-                    count += intersection_count(pos_matching, &self.suffix_position_buffer);
-                } else {
-                    if intersection_exists(pos_matching, &self.suffix_position_buffer) {
-                        return true;
-                    }
-                }
+                suffix.positions_with_offset(self.suffix_offset, &mut positions);
+                count += intersection_count(pos_matching, &positions);
             }
         }
-        if !SCORING_ENABLED {
-            return false;
-        }
         self.phrase_count = count as u32;
         count != 0
     }
 }
 
-impl<TPostings: Postings, const SCORING_ENABLED: bool> DocSet
-    for PhrasePrefixScorer<TPostings, SCORING_ENABLED>
-{
+impl<TPostings: Postings> DocSet for PhrasePrefixScorer<TPostings> {
     fn advance(&mut self) -> DocId {
         loop {
             let doc = self.phrase_scorer.advance();
@@ -212,15 +198,9 @@ impl<TPostings: Postings, const SCORING_ENABLED: bool> DocSet
     }
 }
 
-impl<TPostings: Postings, const SCORING_ENABLED: bool> Scorer
-    for PhrasePrefixScorer<TPostings, SCORING_ENABLED>
-{
+impl<TPostings: Postings> Scorer for PhrasePrefixScorer<TPostings> {
     fn score(&mut self) -> Score {
-        if SCORING_ENABLED {
-            self.phrase_scorer.score()
-        } else {
-            1.0f32
-        }
         // TODO modify score??
+        self.phrase_scorer.score()
     }
 }

src/query/phrase_prefix_query/phrase_prefix_weight.rs

@@ -42,11 +42,11 @@ impl PhrasePrefixWeight {
         Ok(FieldNormReader::constant(reader.max_doc(), 1))
     }
 
-    pub(crate) fn phrase_prefix_scorer<const SCORING_ENABLED: bool>(
+    pub(crate) fn phrase_scorer(
         &self,
         reader: &SegmentReader,
         boost: Score,
-    ) -> crate::Result<Option<PhrasePrefixScorer<SegmentPostings, SCORING_ENABLED>>> {
+    ) -> crate::Result<Option<PhrasePrefixScorer<SegmentPostings>>> {
         let similarity_weight_opt = self
             .similarity_weight_opt
             .as_ref()
@@ -128,20 +128,15 @@ impl PhrasePrefixWeight {
 
 impl Weight for PhrasePrefixWeight {
     fn scorer(&self, reader: &SegmentReader, boost: Score) -> crate::Result<Box<dyn Scorer>> {
-        if self.similarity_weight_opt.is_some() {
-            if let Some(scorer) = self.phrase_prefix_scorer::<true>(reader, boost)? {
-                return Ok(Box::new(scorer));
-            }
+        if let Some(scorer) = self.phrase_scorer(reader, boost)? {
+            Ok(Box::new(scorer))
         } else {
-            if let Some(scorer) = self.phrase_prefix_scorer::<false>(reader, boost)? {
-                return Ok(Box::new(scorer));
-            }
+            Ok(Box::new(EmptyScorer))
         }
-        Ok(Box::new(EmptyScorer))
     }
 
     fn explain(&self, reader: &SegmentReader, doc: DocId) -> crate::Result<Explanation> {
-        let scorer_opt = self.phrase_prefix_scorer::<true>(reader, 1.0)?;
+        let scorer_opt = self.phrase_scorer(reader, 1.0)?;
         if scorer_opt.is_none() {
             return Err(does_not_match(doc));
         }
@@ -205,7 +200,7 @@ mod tests {
             .unwrap()
             .unwrap();
         let mut phrase_scorer = phrase_weight
-            .phrase_prefix_scorer::<true>(searcher.segment_reader(0u32), 1.0)?
+            .phrase_scorer(searcher.segment_reader(0u32), 1.0)?
             .unwrap();
         assert_eq!(phrase_scorer.doc(), 1);
         assert_eq!(phrase_scorer.phrase_count(), 2);
@@ -216,38 +211,6 @@ mod tests {
         Ok(())
     }
 
-    #[test]
-    pub fn test_phrase_no_count() -> crate::Result<()> {
-        let index = create_index(&[
-            "aa bb dd cc",
-            "aa aa bb c dd aa bb cc aa bb dc",
-            " aa bb cd",
-        ])?;
-        let schema = index.schema();
-        let text_field = schema.get_field("text").unwrap();
-        let searcher = index.reader()?.searcher();
-        let phrase_query = PhrasePrefixQuery::new(vec![
-            Term::from_field_text(text_field, "aa"),
-            Term::from_field_text(text_field, "bb"),
-            Term::from_field_text(text_field, "c"),
-        ]);
-        let enable_scoring = EnableScoring::enabled_from_searcher(&searcher);
-        let phrase_weight = phrase_query
-            .phrase_prefix_query_weight(enable_scoring)
-            .unwrap()
-            .unwrap();
-        let mut phrase_scorer = phrase_weight
-            .phrase_prefix_scorer::<false>(searcher.segment_reader(0u32), 1.0)?
-            .unwrap();
-        assert_eq!(phrase_scorer.doc(), 1);
-        assert_eq!(phrase_scorer.phrase_count(), 0);
-        assert_eq!(phrase_scorer.advance(), 2);
-        assert_eq!(phrase_scorer.doc(), 2);
-        assert_eq!(phrase_scorer.phrase_count(), 0);
-        assert_eq!(phrase_scorer.advance(), TERMINATED);
-        Ok(())
-    }
-
     #[test]
     pub fn test_phrase_count_mid() -> crate::Result<()> {
         let index = create_index(&["aa dd cc", "aa aa bb c dd aa bb cc aa dc", " aa bb cd"])?;
@@ -264,7 +227,7 @@ mod tests {
             .unwrap()
             .unwrap();
         let mut phrase_scorer = phrase_weight
-            .phrase_prefix_scorer::<true>(searcher.segment_reader(0u32), 1.0)?
+            .phrase_scorer(searcher.segment_reader(0u32), 1.0)?
             .unwrap();
         assert_eq!(phrase_scorer.doc(), 1);
         assert_eq!(phrase_scorer.phrase_count(), 2);

src/query/phrase_query/mod.rs

@@ -3,8 +3,8 @@ mod phrase_scorer;
 mod phrase_weight;
 
 pub use self::phrase_query::PhraseQuery;
+pub(crate) use self::phrase_scorer::intersection_count;
 pub use self::phrase_scorer::PhraseScorer;
-pub(crate) use self::phrase_scorer::{intersection_count, intersection_exists};
 pub use self::phrase_weight::PhraseWeight;
 
 #[cfg(test)]

src/query/phrase_query/phrase_scorer.rs

@@ -58,7 +58,7 @@ pub struct PhraseScorer<TPostings: Postings> {
 }
 
 /// Returns true if and only if the two sorted arrays contain a common element
-pub(crate) fn intersection_exists(left: &[u32], right: &[u32]) -> bool {
+fn intersection_exists(left: &[u32], right: &[u32]) -> bool {
     let mut left_index = 0;
     let mut right_index = 0;
     while left_index < left.len() && right_index < right.len() {

src/query/query_parser/query_parser.rs

@@ -10,8 +10,10 @@ use query_grammar::{UserInputAst, UserInputBound, UserInputLeaf, UserInputLitera
 use rustc_hash::FxHashMap;
 
 use super::logical_ast::*;
+use crate::core::json_utils::{
+    convert_to_fast_value_and_get_term, set_string_and_get_terms, JsonTermWriter,
+};
 use crate::index::Index;
-use crate::json_utils::convert_to_fast_value_and_append_to_json_term;
 use crate::query::range_query::{is_type_valid_for_fastfield_range_query, RangeQuery};
 use crate::query::{
     AllQuery, BooleanQuery, BoostQuery, EmptyQuery, FuzzyTermQuery, Occur, PhrasePrefixQuery,
@@ -963,27 +965,20 @@ fn generate_literals_for_json_object(
         })?;
     let index_record_option = text_options.index_option();
     let mut logical_literals = Vec::new();
-
-    let get_term_with_path =
-        || Term::from_field_json_path(field, json_path, json_options.is_expand_dots_enabled());
-
-    // Try to convert the phrase to a fast value
-    if let Some(term) = convert_to_fast_value_and_append_to_json_term(get_term_with_path(), phrase)
-    {
+    let mut term = Term::with_capacity(100);
+    let mut json_term_writer = JsonTermWriter::from_field_and_json_path(
+        field,
+        json_path,
+        json_options.is_expand_dots_enabled(),
+        &mut term,
+    );
+    if let Some(term) = convert_to_fast_value_and_get_term(&mut json_term_writer, phrase) {
         logical_literals.push(LogicalLiteral::Term(term));
     }
-
-    // Try to tokenize the phrase and create Terms.
-    let mut positions_and_terms = Vec::<(usize, Term)>::new();
-    let mut token_stream = text_analyzer.token_stream(phrase);
-    token_stream.process(&mut |token| {
-        let mut term = get_term_with_path();
-        term.append_type_and_str(&token.text);
-        positions_and_terms.push((token.position, term.clone()));
-    });
-
-    if positions_and_terms.len() <= 1 {
-        for (_, term) in positions_and_terms {
+    let terms = set_string_and_get_terms(&mut json_term_writer, phrase, &mut text_analyzer);
+    drop(json_term_writer);
+    if terms.len() <= 1 {
+        for (_, term) in terms {
             logical_literals.push(LogicalLiteral::Term(term));
         }
         return Ok(logical_literals);
@@ -994,7 +989,7 @@ fn generate_literals_for_json_object(
         ));
     }
     logical_literals.push(LogicalLiteral::Phrase {
-        terms: positions_and_terms,
+        terms,
         slop: 0,
         prefix: false,
     });

src/query/range_query/fast_field_range_query.rs

@@ -174,7 +174,7 @@ impl<T: Send + Sync + PartialOrd + Copy + Debug + 'static> DocSet for RangeDocSe
     }
 
     fn size_hint(&self) -> u32 {
-        self.column.num_docs()
+        0 // heuristic possible by checking number of hits when fetching a block
     }
 }
 

src/query/regex_query.rs

@@ -185,7 +185,7 @@ mod test {
             Err(crate::TantivyError::InvalidArgument(msg)) => {
                 assert!(msg.contains("error: unclosed group"))
             }
-            res => panic!("unexpected result: {res:?}"),
+            res => panic!("unexpected result: {:?}", res),
         }
     }
 }

src/query/term_query/term_scorer.rs

@@ -127,7 +127,6 @@ impl Scorer for TermScorer {
 mod tests {
     use proptest::prelude::*;
 
-    use crate::index::SegmentId;
     use crate::indexer::index_writer::MEMORY_BUDGET_NUM_BYTES_MIN;
     use crate::merge_policy::NoMergePolicy;
     use crate::postings::compression::COMPRESSION_BLOCK_SIZE;
@@ -135,7 +134,8 @@ mod tests {
     use crate::query::{Bm25Weight, EnableScoring, Scorer, TermQuery};
     use crate::schema::{IndexRecordOption, Schema, TEXT};
     use crate::{
-        assert_nearly_equals, DocId, DocSet, Index, IndexWriter, Score, Searcher, Term, TERMINATED,
+        assert_nearly_equals, DocId, DocSet, Index, IndexWriter, Score, Searcher, SegmentId, Term,
+        TERMINATED,
     };
 
     #[test]

src/reader/warming.rs

@@ -179,10 +179,9 @@ mod tests {
     use super::Warmer;
     use crate::core::searcher::SearcherGeneration;
     use crate::directory::RamDirectory;
-    use crate::index::SegmentId;
     use crate::indexer::index_writer::MEMORY_BUDGET_NUM_BYTES_MIN;
     use crate::schema::{Schema, INDEXED};
-    use crate::{Index, IndexSettings, ReloadPolicy, Searcher};
+    use crate::{Index, IndexSettings, ReloadPolicy, Searcher, SegmentId};
 
     #[derive(Default)]
     struct TestWarmer {

src/schema/document/de.rs

@@ -873,7 +873,7 @@ mod tests {
         );
 
         let facet = Facet::from_text("/hello/world").unwrap();
-        let result = serialize_value(ReferenceValueLeaf::Facet(facet.encoded_str()).into());
+        let result = serialize_value(ReferenceValueLeaf::Facet(&facet).into());
         let value = deserialize_value(result);
         assert_eq!(value, crate::schema::OwnedValue::Facet(facet));
 
@@ -881,8 +881,7 @@ mod tests {
             text: "hello, world".to_string(),
             tokens: vec![Token::default(), Token::default()],
         };
-        let result =
-            serialize_value(ReferenceValueLeaf::PreTokStr(pre_tok_str.clone().into()).into());
+        let result = serialize_value(ReferenceValueLeaf::PreTokStr(&pre_tok_str).into());
         let value = deserialize_value(result);
         assert_eq!(value, crate::schema::OwnedValue::PreTokStr(pre_tok_str));
     }
@@ -961,19 +960,13 @@ mod tests {
             "my-third-key".to_string(),
             crate::schema::OwnedValue::F64(123.0),
         );
-        assert_eq!(
-            value,
-            crate::schema::OwnedValue::Object(expected_object.into_iter().collect())
-        );
+        assert_eq!(value, crate::schema::OwnedValue::Object(expected_object));
 
         let object = serde_json::Map::new();
         let result = serialize_value(ReferenceValue::Object(JsonObjectIter(object.iter())));
         let value = deserialize_value(result);
         let expected_object = BTreeMap::new();
-        assert_eq!(
-            value,
-            crate::schema::OwnedValue::Object(expected_object.into_iter().collect())
-        );
+        assert_eq!(value, crate::schema::OwnedValue::Object(expected_object));
 
         let mut object = serde_json::Map::new();
         object.insert("my-first-key".into(), serde_json::Value::Null);
@@ -985,10 +978,7 @@ mod tests {
         expected_object.insert("my-first-key".to_string(), crate::schema::OwnedValue::Null);
         expected_object.insert("my-second-key".to_string(), crate::schema::OwnedValue::Null);
         expected_object.insert("my-third-key".to_string(), crate::schema::OwnedValue::Null);
-        assert_eq!(
-            value,
-            crate::schema::OwnedValue::Object(expected_object.into_iter().collect())
-        );
+        assert_eq!(value, crate::schema::OwnedValue::Object(expected_object));
     }
 
     #[test]
@@ -1065,10 +1055,7 @@ mod tests {
                 .collect(),
             ),
         );
-        assert_eq!(
-            value,
-            crate::schema::OwnedValue::Object(expected_object.into_iter().collect())
-        );
+        assert_eq!(value, crate::schema::OwnedValue::Object(expected_object));
 
         // Some more extreme nesting that might behave weirdly
         let mut object = serde_json::Map::new();
@@ -1090,9 +1077,6 @@ mod tests {
                 OwnedValue::Array(vec![OwnedValue::Null]),
             ])]),
         );
-        assert_eq!(
-            value,
-            OwnedValue::Object(expected_object.into_iter().collect())
-        );
+        assert_eq!(value, OwnedValue::Object(expected_object));
     }
 }

src/schema/document/default_document.rs

@@ -1,64 +1,93 @@
 use std::collections::{BTreeMap, HashMap, HashSet};
-use std::io::{self, Read, Write};
 use std::net::Ipv6Addr;
 
-use columnar::MonotonicallyMappableToU128;
-use common::{read_u32_vint_no_advance, serialize_vint_u32, BinarySerializable, DateTime, VInt};
+use common::DateTime;
 use serde_json::Map;
-pub use CompactDoc as TantivyDocument;
 
-use super::{ReferenceValue, ReferenceValueLeaf, Value};
 use crate::schema::document::{
     DeserializeError, Document, DocumentDeserialize, DocumentDeserializer,
 };
 use crate::schema::field_type::ValueParsingError;
-use crate::schema::{Facet, Field, NamedFieldDocument, OwnedValue, Schema};
+use crate::schema::field_value::FieldValueIter;
+use crate::schema::{Facet, Field, FieldValue, NamedFieldDocument, OwnedValue, Schema};
 use crate::tokenizer::PreTokenizedString;
 
-#[repr(packed)]
-#[derive(Debug, Clone)]
-/// A field value pair in the compact tantivy document
-struct FieldValueAddr {
-    pub field: u16,
-    pub value_addr: ValueAddr,
+/// TantivyDocument provides a default implementation of the `Document` trait.
+/// It is the object that can be indexed and then searched for.
+///
+/// Documents are fundamentally a collection of unordered couples `(field, value)`.
+/// In this list, one field may appear more than once.
+#[derive(Clone, Debug, serde::Serialize, serde::Deserialize, Default)]
+pub struct TantivyDocument {
+    field_values: Vec<FieldValue>,
 }
 
-#[derive(Debug, Clone)]
-/// The default document in tantivy. It encodes data in a compact form.
-pub struct CompactDoc {
-    /// `node_data` is a vec of bytes, where each value is serialized into bytes and stored. It
-    /// includes all the data of the document and also metadata like where the nodes are located
-    /// in an object or array.
-    pub node_data: Vec<u8>,
-    /// The root (Field, Value) pairs
-    field_values: Vec<FieldValueAddr>,
-}
+impl Document for TantivyDocument {
+    type Value<'a> = &'a OwnedValue;
+    type FieldsValuesIter<'a> = FieldValueIter<'a>;
 
-impl Default for CompactDoc {
-    fn default() -> Self {
-        Self::new()
+    fn iter_fields_and_values(&self) -> Self::FieldsValuesIter<'_> {
+        FieldValueIter(self.field_values.iter())
     }
 }
 
-impl CompactDoc {
-    /// Creates a new, empty document object
-    /// The reserved capacity is for the total serialized data
-    pub fn with_capacity(bytes: usize) -> CompactDoc {
-        CompactDoc {
-            node_data: Vec::with_capacity(bytes),
-            field_values: Vec::with_capacity(4),
+impl DocumentDeserialize for TantivyDocument {
+    fn deserialize<'de, D>(mut deserializer: D) -> Result<Self, DeserializeError>
+    where D: DocumentDeserializer<'de> {
+        let mut field_values = Vec::with_capacity(deserializer.size_hint());
+
+        while let Some((field, value)) = deserializer.next_field()? {
+            field_values.push(FieldValue::new(field, value));
         }
-    }
 
+        Ok(Self { field_values })
+    }
+}
+
+impl From<Vec<FieldValue>> for TantivyDocument {
+    fn from(field_values: Vec<FieldValue>) -> Self {
+        Self { field_values }
+    }
+}
+
+impl PartialEq for TantivyDocument {
+    fn eq(&self, other: &Self) -> bool {
+        // super slow, but only here for tests
+        let convert_to_comparable_map = |field_values: &[FieldValue]| {
+            let mut field_value_set: HashMap<Field, HashSet<String>> = Default::default();
+            for field_value in field_values.iter() {
+                let value = serde_json::to_string(field_value.value()).unwrap();
+                field_value_set
+                    .entry(field_value.field())
+                    .or_default()
+                    .insert(value);
+            }
+            field_value_set
+        };
+        let self_field_values: HashMap<Field, HashSet<String>> =
+            convert_to_comparable_map(&self.field_values);
+        let other_field_values: HashMap<Field, HashSet<String>> =
+            convert_to_comparable_map(&other.field_values);
+        self_field_values.eq(&other_field_values)
+    }
+}
+
+impl Eq for TantivyDocument {}
+
+impl IntoIterator for TantivyDocument {
+    type Item = FieldValue;
+
+    type IntoIter = std::vec::IntoIter<FieldValue>;
+
+    fn into_iter(self) -> Self::IntoIter {
+        self.field_values.into_iter()
+    }
+}
+
+impl TantivyDocument {
     /// Creates a new, empty document object
-    pub fn new() -> CompactDoc {
-        CompactDoc::with_capacity(1024)
-    }
-
-    /// Skrinks the capacity of the document to fit the data
-    pub fn shrink_to_fit(&mut self) {
-        self.node_data.shrink_to_fit();
-        self.field_values.shrink_to_fit();
+    pub fn new() -> TantivyDocument {
+        TantivyDocument::default()
     }
 
     /// Returns the length of the document.
@@ -70,111 +99,83 @@ impl CompactDoc {
     pub fn add_facet<F>(&mut self, field: Field, path: F)
     where Facet: From<F> {
         let facet = Facet::from(path);
-        self.add_leaf_field_value(field, ReferenceValueLeaf::Facet(facet.encoded_str()));
+        let value = OwnedValue::Facet(facet);
+        self.add_field_value(field, value);
     }
 
     /// Add a text field.
-    pub fn add_text<S: AsRef<str>>(&mut self, field: Field, text: S) {
-        self.add_leaf_field_value(field, ReferenceValueLeaf::Str(text.as_ref()));
+    pub fn add_text<S: ToString>(&mut self, field: Field, text: S) {
+        let value = OwnedValue::Str(text.to_string());
+        self.add_field_value(field, value);
     }
 
     /// Add a pre-tokenized text field.
     pub fn add_pre_tokenized_text(&mut self, field: Field, pre_tokenized_text: PreTokenizedString) {
-        self.add_leaf_field_value(field, pre_tokenized_text);
+        self.add_field_value(field, pre_tokenized_text);
     }
 
     /// Add a u64 field
     pub fn add_u64(&mut self, field: Field, value: u64) {
-        self.add_leaf_field_value(field, value);
+        self.add_field_value(field, value);
     }
 
     /// Add a IP address field. Internally only Ipv6Addr is used.
     pub fn add_ip_addr(&mut self, field: Field, value: Ipv6Addr) {
-        self.add_leaf_field_value(field, value);
+        self.add_field_value(field, value);
     }
 
     /// Add a i64 field
     pub fn add_i64(&mut self, field: Field, value: i64) {
-        self.add_leaf_field_value(field, value);
+        self.add_field_value(field, value);
     }
 
     /// Add a f64 field
     pub fn add_f64(&mut self, field: Field, value: f64) {
-        self.add_leaf_field_value(field, value);
+        self.add_field_value(field, value);
     }
 
     /// Add a bool field
     pub fn add_bool(&mut self, field: Field, value: bool) {
-        self.add_leaf_field_value(field, value);
+        self.add_field_value(field, value);
     }
 
     /// Add a date field with unspecified time zone offset
     pub fn add_date(&mut self, field: Field, value: DateTime) {
-        self.add_leaf_field_value(field, value);
+        self.add_field_value(field, value);
     }
 
     /// Add a bytes field
-    pub fn add_bytes(&mut self, field: Field, value: &[u8]) {
-        self.add_leaf_field_value(field, value);
+    pub fn add_bytes<T: Into<Vec<u8>>>(&mut self, field: Field, value: T) {
+        self.add_field_value(field, value.into());
     }
 
     /// Add a dynamic object field
     pub fn add_object(&mut self, field: Field, object: BTreeMap<String, OwnedValue>) {
-        self.add_field_value(field, &OwnedValue::from(object));
+        self.add_field_value(field, object);
     }
 
     /// Add a (field, value) to the document.
-    ///
-    /// `OwnedValue` implements Value, which should be easiest to use, but is not the most
-    /// performant.
-    pub fn add_field_value<'a, V: Value<'a>>(&mut self, field: Field, value: V) {
-        let field_value = FieldValueAddr {
-            field: field
-                .field_id()
-                .try_into()
-                .expect("support only up to u16::MAX field ids"),
-            value_addr: self.add_value(value),
-        };
-        self.field_values.push(field_value);
-    }
-
-    /// Add a (field, leaf value) to the document.
-    /// Leaf values don't have nested values.
-    pub fn add_leaf_field_value<'a, T: Into<ReferenceValueLeaf<'a>>>(
-        &mut self,
-        field: Field,
-        typed_val: T,
-    ) {
+    pub fn add_field_value<T: Into<OwnedValue>>(&mut self, field: Field, typed_val: T) {
         let value = typed_val.into();
-        let field_value = FieldValueAddr {
-            field: field
-                .field_id()
-                .try_into()
-                .expect("support only up to u16::MAX field ids"),
-            value_addr: self.add_value_leaf(value),
-        };
+        let field_value = FieldValue { field, value };
         self.field_values.push(field_value);
     }
 
     /// field_values accessor
-    pub fn field_values(&self) -> impl Iterator<Item = (Field, CompactDocValue<'_>)> {
-        self.field_values.iter().map(|field_val| {
-            let field = Field::from_field_id(field_val.field as u32);
-            let val = self.get_compact_doc_value(field_val.value_addr);
-            (field, val)
-        })
+    pub fn field_values(&self) -> &[FieldValue] {
+        &self.field_values
     }
 
-    /// Returns all of the `ReferenceValue`s associated the given field
-    pub fn get_all(&self, field: Field) -> impl Iterator<Item = CompactDocValue<'_>> + '_ {
+    /// Returns all of the `FieldValue`s associated the given field
+    pub fn get_all(&self, field: Field) -> impl Iterator<Item = &OwnedValue> {
         self.field_values
             .iter()
-            .filter(move |field_value| Field::from_field_id(field_value.field as u32) == field)
-            .map(|val| self.get_compact_doc_value(val.value_addr))
+            .filter(move |field_value| field_value.field() == field)
+            .map(FieldValue::value)
     }
 
-    /// Returns the first `ReferenceValue` associated the given field
-    pub fn get_first(&self, field: Field) -> Option<CompactDocValue<'_>> {
+    /// Returns the first `FieldValue` associated the given field
+    pub fn get_first(&self, field: Field) -> Option<&OwnedValue> {
         self.get_all(field).next()
     }
 
@@ -182,12 +183,12 @@ impl CompactDoc {
     pub fn convert_named_doc(
         schema: &Schema,
         named_doc: NamedFieldDocument,
-    ) -> Result<Self, DocParsingError> {
-        let mut document = Self::new();
+    ) -> Result<TantivyDocument, DocParsingError> {
+        let mut document = TantivyDocument::new();
         for (field_name, values) in named_doc.0 {
             if let Ok(field) = schema.get_field(&field_name) {
                 for value in values {
-                    document.add_field_value(field, &value);
+                    document.add_field_value(field, value);
                 }
             }
         }
@@ -195,7 +196,7 @@ impl CompactDoc {
     }
 
     /// Build a document object from a json-object.
-    pub fn parse_json(schema: &Schema, doc_json: &str) -> Result<Self, DocParsingError> {
+    pub fn parse_json(schema: &Schema, doc_json: &str) -> Result<TantivyDocument, DocParsingError> {
         let json_obj: Map<String, serde_json::Value> =
             serde_json::from_str(doc_json).map_err(|_| DocParsingError::invalid_json(doc_json))?;
         Self::from_json_object(schema, json_obj)
@@ -205,8 +206,8 @@ impl CompactDoc {
     pub fn from_json_object(
         schema: &Schema,
         json_obj: Map<String, serde_json::Value>,
-    ) -> Result<Self, DocParsingError> {
-        let mut doc = Self::default();
+    ) -> Result<TantivyDocument, DocParsingError> {
+        let mut doc = TantivyDocument::default();
         for (field_name, json_value) in json_obj {
             if let Ok(field) = schema.get_field(&field_name) {
                 let field_entry = schema.get_field_entry(field);
@@ -217,482 +218,20 @@ impl CompactDoc {
                             let value = field_type
                                 .value_from_json(json_item)
                                 .map_err(|e| DocParsingError::ValueError(field_name.clone(), e))?;
-                            doc.add_field_value(field, &value);
+                            doc.add_field_value(field, value);
                         }
                     }
                     _ => {
                         let value = field_type
                             .value_from_json(json_value)
                             .map_err(|e| DocParsingError::ValueError(field_name.clone(), e))?;
-                        doc.add_field_value(field, &value);
+                        doc.add_field_value(field, value);
                     }
                 }
             }
         }
         Ok(doc)
     }
-
-    fn add_value_leaf(&mut self, leaf: ReferenceValueLeaf) -> ValueAddr {
-        let type_id = ValueType::from(&leaf);
-        // Write into `node_data` and return u32 position as its address
-        // Null and bool are inlined into the address
-        let val_addr = match leaf {
-            ReferenceValueLeaf::Null => 0,
-            ReferenceValueLeaf::Str(bytes) => {
-                write_bytes_into(&mut self.node_data, bytes.as_bytes())
-            }
-            ReferenceValueLeaf::Facet(bytes) => {
-                write_bytes_into(&mut self.node_data, bytes.as_bytes())
-            }
-            ReferenceValueLeaf::Bytes(bytes) => write_bytes_into(&mut self.node_data, bytes),
-            ReferenceValueLeaf::U64(num) => write_into(&mut self.node_data, num),
-            ReferenceValueLeaf::I64(num) => write_into(&mut self.node_data, num),
-            ReferenceValueLeaf::F64(num) => write_into(&mut self.node_data, num),
-            ReferenceValueLeaf::Bool(b) => b as u32,
-            ReferenceValueLeaf::Date(date) => {
-                write_into(&mut self.node_data, date.into_timestamp_nanos())
-            }
-            ReferenceValueLeaf::IpAddr(num) => write_into(&mut self.node_data, num.to_u128()),
-            ReferenceValueLeaf::PreTokStr(pre_tok) => write_into(&mut self.node_data, *pre_tok),
-        };
-        ValueAddr { type_id, val_addr }
-    }
-    /// Adds a value and returns in address into the
-    fn add_value<'a, V: Value<'a>>(&mut self, value: V) -> ValueAddr {
-        let value = value.as_value();
-        let type_id = ValueType::from(&value);
-        match value {
-            ReferenceValue::Leaf(leaf) => self.add_value_leaf(leaf),
-            ReferenceValue::Array(elements) => {
-                // addresses of the elements in node_data
-                // Reusing a vec would be nicer, but it's not easy because of the recursion
-                // A global vec would work if every writer get it's discriminator
-                let mut addresses = Vec::new();
-                for elem in elements {
-                    let value_addr = self.add_value(elem);
-                    write_into(&mut addresses, value_addr);
-                }
-                ValueAddr {
-                    type_id,
-                    val_addr: write_bytes_into(&mut self.node_data, &addresses),
-                }
-            }
-            ReferenceValue::Object(entries) => {
-                // addresses of the elements in node_data
-                let mut addresses = Vec::new();
-                for (key, value) in entries {
-                    let key_addr = self.add_value_leaf(ReferenceValueLeaf::Str(key));
-                    let value_addr = self.add_value(value);
-                    write_into(&mut addresses, key_addr);
-                    write_into(&mut addresses, value_addr);
-                }
-                ValueAddr {
-                    type_id,
-                    val_addr: write_bytes_into(&mut self.node_data, &addresses),
-                }
-            }
-        }
-    }
-
-    /// Get CompactDocValue for address
-    fn get_compact_doc_value(&self, value_addr: ValueAddr) -> CompactDocValue<'_> {
-        CompactDocValue {
-            container: self,
-            value_addr,
-        }
-    }
-
-    /// get &[u8] reference from node_data
-    fn extract_bytes(&self, addr: Addr) -> &[u8] {
-        binary_deserialize_bytes(self.get_slice(addr))
-    }
-
-    /// get &str reference from node_data
-    fn extract_str(&self, addr: Addr) -> &str {
-        let data = self.extract_bytes(addr);
-        // Utf-8 checks would have a noticeable performance overhead here
-        unsafe { std::str::from_utf8_unchecked(data) }
-    }
-
-    /// deserialized owned value from node_data
-    fn read_from<T: BinarySerializable>(&self, addr: Addr) -> io::Result<T> {
-        let data_slice = &self.node_data[addr as usize..];
-        let mut cursor = std::io::Cursor::new(data_slice);
-        T::deserialize(&mut cursor)
-    }
-
-    /// get slice from address. The returned slice is open ended
-    fn get_slice(&self, addr: Addr) -> &[u8] {
-        &self.node_data[addr as usize..]
-    }
-}
-
-/// BinarySerializable alternative to read references
-fn binary_deserialize_bytes(data: &[u8]) -> &[u8] {
-    let (len, bytes_read) = read_u32_vint_no_advance(data);
-    &data[bytes_read..bytes_read + len as usize]
-}
-
-/// Write bytes and return the position of the written data.
-///
-/// BinarySerializable alternative to write references
-fn write_bytes_into(vec: &mut Vec<u8>, data: &[u8]) -> u32 {
-    let pos = vec.len() as u32;
-    let mut buf = [0u8; 8];
-    let len_vint_bytes = serialize_vint_u32(data.len() as u32, &mut buf);
-    vec.extend_from_slice(len_vint_bytes);
-    vec.extend_from_slice(data);
-    pos
-}
-
-/// Serialize and return the position
-fn write_into<T: BinarySerializable>(vec: &mut Vec<u8>, value: T) -> u32 {
-    let pos = vec.len() as u32;
-    value.serialize(vec).unwrap();
-    pos
-}
-
-impl PartialEq for CompactDoc {
-    fn eq(&self, other: &Self) -> bool {
-        // super slow, but only here for tests
-        let convert_to_comparable_map = |doc: &CompactDoc| {
-            let mut field_value_set: HashMap<Field, HashSet<String>> = Default::default();
-            for field_value in doc.field_values.iter() {
-                let value: OwnedValue = doc.get_compact_doc_value(field_value.value_addr).into();
-                let value = serde_json::to_string(&value).unwrap();
-                field_value_set
-                    .entry(Field::from_field_id(field_value.field as u32))
-                    .or_default()
-                    .insert(value);
-            }
-            field_value_set
-        };
-        let self_field_values: HashMap<Field, HashSet<String>> = convert_to_comparable_map(self);
-        let other_field_values: HashMap<Field, HashSet<String>> = convert_to_comparable_map(other);
-        self_field_values.eq(&other_field_values)
-    }
-}
-
-impl Eq for CompactDoc {}
-
-impl DocumentDeserialize for CompactDoc {
-    fn deserialize<'de, D>(mut deserializer: D) -> Result<Self, DeserializeError>
-    where D: DocumentDeserializer<'de> {
-        let mut doc = CompactDoc::default();
-        // TODO: Deserializing into OwnedValue is wasteful. The deserializer should be able to work
-        // on slices and referenced data.
-        while let Some((field, value)) = deserializer.next_field::<OwnedValue>()? {
-            doc.add_field_value(field, &value);
-        }
-        Ok(doc)
-    }
-}
-
-/// A value of Compact Doc needs a reference to the container to extract its payload
-#[derive(Debug, Clone, Copy)]
-pub struct CompactDocValue<'a> {
-    container: &'a CompactDoc,
-    value_addr: ValueAddr,
-}
-impl PartialEq for CompactDocValue<'_> {
-    fn eq(&self, other: &Self) -> bool {
-        let value1: OwnedValue = (*self).into();
-        let value2: OwnedValue = (*other).into();
-        value1 == value2
-    }
-}
-impl<'a> From<CompactDocValue<'a>> for OwnedValue {
-    fn from(value: CompactDocValue) -> Self {
-        value.as_value().into()
-    }
-}
-impl<'a> Value<'a> for CompactDocValue<'a> {
-    type ArrayIter = CompactDocArrayIter<'a>;
-
-    type ObjectIter = CompactDocObjectIter<'a>;
-
-    fn as_value(&self) -> ReferenceValue<'a, Self> {
-        self.get_ref_value().unwrap()
-    }
-}
-impl<'a> CompactDocValue<'a> {
-    fn get_ref_value(&self) -> io::Result<ReferenceValue<'a, CompactDocValue<'a>>> {
-        let addr = self.value_addr.val_addr;
-        match self.value_addr.type_id {
-            ValueType::Null => Ok(ReferenceValueLeaf::Null.into()),
-            ValueType::Str => {
-                let str_ref = self.container.extract_str(addr);
-                Ok(ReferenceValueLeaf::Str(str_ref).into())
-            }
-            ValueType::Facet => {
-                let str_ref = self.container.extract_str(addr);
-                Ok(ReferenceValueLeaf::Facet(str_ref).into())
-            }
-            ValueType::Bytes => {
-                let data = self.container.extract_bytes(addr);
-                Ok(ReferenceValueLeaf::Bytes(data).into())
-            }
-            ValueType::U64 => self
-                .container
-                .read_from::<u64>(addr)
-                .map(ReferenceValueLeaf::U64)
-                .map(Into::into),
-            ValueType::I64 => self
-                .container
-                .read_from::<i64>(addr)
-                .map(ReferenceValueLeaf::I64)
-                .map(Into::into),
-            ValueType::F64 => self
-                .container
-                .read_from::<f64>(addr)
-                .map(ReferenceValueLeaf::F64)
-                .map(Into::into),
-            ValueType::Bool => Ok(ReferenceValueLeaf::Bool(addr != 0).into()),
-            ValueType::Date => self
-                .container
-                .read_from::<i64>(addr)
-                .map(|ts| ReferenceValueLeaf::Date(DateTime::from_timestamp_nanos(ts)))
-                .map(Into::into),
-            ValueType::IpAddr => self
-                .container
-                .read_from::<u128>(addr)
-                .map(|num| ReferenceValueLeaf::IpAddr(Ipv6Addr::from_u128(num)))
-                .map(Into::into),
-            ValueType::PreTokStr => self
-                .container
-                .read_from::<PreTokenizedString>(addr)
-                .map(Into::into)
-                .map(ReferenceValueLeaf::PreTokStr)
-                .map(Into::into),
-            ValueType::Object => Ok(ReferenceValue::Object(CompactDocObjectIter::new(
-                self.container,
-                addr,
-            )?)),
-            ValueType::Array => Ok(ReferenceValue::Array(CompactDocArrayIter::new(
-                self.container,
-                addr,
-            )?)),
-        }
-    }
-}
-
-/// The address in the vec
-type Addr = u32;
-
-#[derive(Clone, Copy, Default)]
-#[repr(packed)]
-/// The value type and the address to its payload in the container.
-struct ValueAddr {
-    type_id: ValueType,
-    /// This is the address to the value in the vec, except for bool and null, which are inlined
-    val_addr: Addr,
-}
-impl BinarySerializable for ValueAddr {
-    fn serialize<W: Write + ?Sized>(&self, writer: &mut W) -> io::Result<()> {
-        self.type_id.serialize(writer)?;
-        VInt(self.val_addr as u64).serialize(writer)
-    }
-
-    fn deserialize<R: Read>(reader: &mut R) -> io::Result<Self> {
-        let type_id = ValueType::deserialize(reader)?;
-        let val_addr = VInt::deserialize(reader)?.0 as u32;
-        Ok(ValueAddr { type_id, val_addr })
-    }
-}
-impl std::fmt::Debug for ValueAddr {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        let val_addr = self.val_addr;
-        f.write_fmt(format_args!("{:?} at {:?}", self.type_id, val_addr))
-    }
-}
-
-/// A enum representing a value for tantivy to index.
-///
-/// Any changes need to be reflected in `BinarySerializable` for `ValueType`
-///
-/// We can't use [schema::Type] or [columnar::ColumnType] here, because they are missing
-/// some items like Array and PreTokStr.
-#[derive(Default, Clone, Copy, Debug, PartialEq)]
-#[repr(u8)]
-pub enum ValueType {
-    /// A null value.
-    #[default]
-    Null = 0,
-    /// The str type is used for any text information.
-    Str = 1,
-    /// Unsigned 64-bits Integer `u64`
-    U64 = 2,
-    /// Signed 64-bits Integer `i64`
-    I64 = 3,
-    /// 64-bits Float `f64`
-    F64 = 4,
-    /// Date/time with nanoseconds precision
-    Date = 5,
-    /// Facet
-    Facet = 6,
-    /// Arbitrarily sized byte array
-    Bytes = 7,
-    /// IpV6 Address. Internally there is no IpV4, it needs to be converted to `Ipv6Addr`.
-    IpAddr = 8,
-    /// Bool value
-    Bool = 9,
-    /// Pre-tokenized str type,
-    PreTokStr = 10,
-    /// Object
-    Object = 11,
-    /// Pre-tokenized str type,
-    Array = 12,
-}
-
-impl BinarySerializable for ValueType {
-    fn serialize<W: Write + ?Sized>(&self, writer: &mut W) -> io::Result<()> {
-        (*self as u8).serialize(writer)?;
-        Ok(())
-    }
-
-    fn deserialize<R: Read>(reader: &mut R) -> io::Result<Self> {
-        let num = u8::deserialize(reader)?;
-        let type_id = if (0..=12).contains(&num) {
-            unsafe { std::mem::transmute(num) }
-        } else {
-            return Err(io::Error::new(
-                io::ErrorKind::InvalidData,
-                format!("Invalid value type id: {num}"),
-            ));
-        };
-        Ok(type_id)
-    }
-}
-
-impl<'a, V: Value<'a>> From<&ReferenceValue<'a, V>> for ValueType {
-    fn from(value: &ReferenceValue<'a, V>) -> Self {
-        match value {
-            ReferenceValue::Leaf(leaf) => leaf.into(),
-            ReferenceValue::Array(_) => ValueType::Array,
-            ReferenceValue::Object(_) => ValueType::Object,
-        }
-    }
-}
-impl<'a> From<&ReferenceValueLeaf<'a>> for ValueType {
-    fn from(value: &ReferenceValueLeaf<'a>) -> Self {
-        match value {
-            ReferenceValueLeaf::Null => ValueType::Null,
-            ReferenceValueLeaf::Str(_) => ValueType::Str,
-            ReferenceValueLeaf::U64(_) => ValueType::U64,
-            ReferenceValueLeaf::I64(_) => ValueType::I64,
-            ReferenceValueLeaf::F64(_) => ValueType::F64,
-            ReferenceValueLeaf::Bool(_) => ValueType::Bool,
-            ReferenceValueLeaf::Date(_) => ValueType::Date,
-            ReferenceValueLeaf::IpAddr(_) => ValueType::IpAddr,
-            ReferenceValueLeaf::PreTokStr(_) => ValueType::PreTokStr,
-            ReferenceValueLeaf::Facet(_) => ValueType::Facet,
-            ReferenceValueLeaf::Bytes(_) => ValueType::Bytes,
-        }
-    }
-}
-
-#[derive(Debug, Clone)]
-/// The Iterator for the object values in the compact document
-pub struct CompactDocObjectIter<'a> {
-    container: &'a CompactDoc,
-    node_addresses_slice: &'a [u8],
-}
-
-impl<'a> CompactDocObjectIter<'a> {
-    fn new(container: &'a CompactDoc, addr: Addr) -> io::Result<Self> {
-        // Objects are `&[ValueAddr]` serialized into bytes
-        let node_addresses_slice = container.extract_bytes(addr);
-        Ok(Self {
-            container,
-            node_addresses_slice,
-        })
-    }
-}
-
-impl<'a> Iterator for CompactDocObjectIter<'a> {
-    type Item = (&'a str, CompactDocValue<'a>);
-
-    fn next(&mut self) -> Option<Self::Item> {
-        if self.node_addresses_slice.is_empty() {
-            return None;
-        }
-        let key_addr = ValueAddr::deserialize(&mut self.node_addresses_slice).ok()?;
-        let key = self.container.extract_str(key_addr.val_addr);
-        let value = ValueAddr::deserialize(&mut self.node_addresses_slice).ok()?;
-        let value = CompactDocValue {
-            container: self.container,
-            value_addr: value,
-        };
-        Some((key, value))
-    }
-}
-
-#[derive(Debug, Clone)]
-/// The Iterator for the array values in the compact document
-pub struct CompactDocArrayIter<'a> {
-    container: &'a CompactDoc,
-    node_addresses_slice: &'a [u8],
-}
-
-impl<'a> CompactDocArrayIter<'a> {
-    fn new(container: &'a CompactDoc, addr: Addr) -> io::Result<Self> {
-        // Arrays are &[ValueAddr] serialized into bytes
-        let node_addresses_slice = container.extract_bytes(addr);
-        Ok(Self {
-            container,
-            node_addresses_slice,
-        })
-    }
-}
-
-impl<'a> Iterator for CompactDocArrayIter<'a> {
-    type Item = CompactDocValue<'a>;
-
-    fn next(&mut self) -> Option<Self::Item> {
-        if self.node_addresses_slice.is_empty() {
-            return None;
-        }
-        let value = ValueAddr::deserialize(&mut self.node_addresses_slice).ok()?;
-        let value = CompactDocValue {
-            container: self.container,
-            value_addr: value,
-        };
-        Some(value)
-    }
-}
-
-impl Document for CompactDoc {
-    type Value<'a> = CompactDocValue<'a>;
-    type FieldsValuesIter<'a> = FieldValueIterRef<'a>;
-
-    fn iter_fields_and_values(&self) -> Self::FieldsValuesIter<'_> {
-        FieldValueIterRef {
-            slice: self.field_values.iter(),
-            container: self,
-        }
-    }
-}
-
-/// A helper wrapper for creating an iterator over the field values
-pub struct FieldValueIterRef<'a> {
-    slice: std::slice::Iter<'a, FieldValueAddr>,
-    container: &'a CompactDoc,
-}
-
-impl<'a> Iterator for FieldValueIterRef<'a> {
-    type Item = (Field, CompactDocValue<'a>);
-
-    fn next(&mut self) -> Option<Self::Item> {
-        self.slice.next().map(|field_value| {
-            (
-                Field::from_field_id(field_value.field as u32),
-                CompactDocValue::<'a> {
-                    container: self.container,
-                    value_addr: field_value.value_addr,
-                },
-            )
-        })
-    }
 }
 
 /// Error that may happen when deserializing
@@ -725,40 +264,7 @@ mod tests {
         let text_field = schema_builder.add_text_field("title", TEXT);
         let mut doc = TantivyDocument::default();
         doc.add_text(text_field, "My title");
-        assert_eq!(doc.field_values().count(), 1);
-
-        let schema = schema_builder.build();
-        let _val = doc.get_first(text_field).unwrap();
-        let _json = doc.to_named_doc(&schema);
-    }
-
-    #[test]
-    fn test_json_value() {
-        let json_str = r#"{ 
-            "toto": "titi",
-            "float": -0.2,
-            "bool": true,
-            "unsigned": 1,
-            "signed": -2,
-            "complexobject": {
-                "field.with.dot": 1
-            },
-            "date": "1985-04-12T23:20:50.52Z",
-            "my_arr": [2, 3, {"my_key": "two tokens"}, 4, {"nested_array": [2, 5, 6, [7, 8, {"a": [{"d": {"e":[99]}}, 9000]}, 9, 10], [5, 5]]}]
-        }"#;
-        let json_val: std::collections::BTreeMap<String, OwnedValue> =
-            serde_json::from_str(json_str).unwrap();
-
-        let mut schema_builder = Schema::builder();
-        let json_field = schema_builder.add_json_field("json", TEXT);
-        let mut doc = TantivyDocument::default();
-        doc.add_object(json_field, json_val);
-
-        let schema = schema_builder.build();
-        let json = doc.to_json(&schema);
-        let actual_json: serde_json::Value = serde_json::from_str(&json).unwrap();
-        let expected_json: serde_json::Value = serde_json::from_str(json_str).unwrap();
-        assert_eq!(actual_json["json"][0], expected_json);
+        assert_eq!(doc.field_values().len(), 1);
     }
 
     // TODO: Should this be re-added with the serialize method

src/schema/document/existing_type_impls.rs

@@ -5,39 +5,21 @@
 //! and don't care about some of the more specialised types or only want to customise
 //! part of the document structure.
 use std::collections::{btree_map, hash_map, BTreeMap, HashMap};
-use std::iter::Empty;
-use std::net::Ipv6Addr;
 
-use common::DateTime;
 use serde_json::Number;
-use time::format_description::well_known::Rfc3339;
-use time::OffsetDateTime;
 
-use super::facet::Facet;
 use super::ReferenceValueLeaf;
 use crate::schema::document::{
     ArrayAccess, DeserializeError, Document, DocumentDeserialize, DocumentDeserializer,
     ObjectAccess, ReferenceValue, Value, ValueDeserialize, ValueDeserializer, ValueVisitor,
 };
 use crate::schema::Field;
-use crate::tokenizer::PreTokenizedString;
 
 // Serde compatibility support.
-pub fn can_be_rfc3339_date_time(text: &str) -> bool {
-    if let Some(&first_byte) = text.as_bytes().first() {
-        if first_byte.is_ascii_digit() {
-            return true;
-        }
-    }
-
-    false
-}
-
 impl<'a> Value<'a> for &'a serde_json::Value {
     type ArrayIter = std::slice::Iter<'a, serde_json::Value>;
     type ObjectIter = JsonObjectIter<'a>;
 
-    #[inline]
     fn as_value(&self) -> ReferenceValue<'a, Self> {
         match self {
             serde_json::Value::Null => ReferenceValueLeaf::Null.into(),
@@ -53,19 +35,7 @@ impl<'a> Value<'a> for &'a serde_json::Value {
                     panic!("Unsupported serde_json number {number}");
                 }
             }
-            serde_json::Value::String(text) => {
-                if can_be_rfc3339_date_time(text) {
-                    match OffsetDateTime::parse(text, &Rfc3339) {
-                        Ok(dt) => {
-                            let dt_utc = dt.to_offset(time::UtcOffset::UTC);
-                            ReferenceValueLeaf::Date(DateTime::from_utc(dt_utc)).into()
-                        }
-                        Err(_) => ReferenceValueLeaf::Str(text).into(),
-                    }
-                } else {
-                    ReferenceValueLeaf::Str(text).into()
-                }
-            }
+            serde_json::Value::String(val) => ReferenceValueLeaf::Str(val).into(),
             serde_json::Value::Array(elements) => ReferenceValue::Array(elements.iter()),
             serde_json::Value::Object(object) => {
                 ReferenceValue::Object(JsonObjectIter(object.iter()))
@@ -74,126 +44,6 @@ impl<'a> Value<'a> for &'a serde_json::Value {
     }
 }
 
-impl<'a> Value<'a> for &'a String {
-    type ArrayIter = Empty<&'a String>;
-    type ObjectIter = Empty<(&'a str, &'a String)>;
-    #[inline]
-    fn as_value(&self) -> ReferenceValue<'a, Self> {
-        ReferenceValue::Leaf(ReferenceValueLeaf::Str(self))
-    }
-}
-
-impl<'a> Value<'a> for &'a Facet {
-    type ArrayIter = Empty<&'a Facet>;
-    type ObjectIter = Empty<(&'a str, &'a Facet)>;
-    #[inline]
-    fn as_value(&self) -> ReferenceValue<'a, Self> {
-        ReferenceValue::Leaf(ReferenceValueLeaf::Facet(self.encoded_str()))
-    }
-}
-
-impl<'a> Value<'a> for &'a u64 {
-    type ArrayIter = Empty<&'a u64>;
-    type ObjectIter = Empty<(&'a str, &'a u64)>;
-    #[inline]
-    fn as_value(&self) -> ReferenceValue<'a, Self> {
-        ReferenceValue::Leaf(ReferenceValueLeaf::U64(**self))
-    }
-}
-
-impl<'a> Value<'a> for &'a i64 {
-    type ArrayIter = Empty<&'a i64>;
-    type ObjectIter = Empty<(&'a str, &'a i64)>;
-    #[inline]
-    fn as_value(&self) -> ReferenceValue<'a, Self> {
-        ReferenceValue::Leaf(ReferenceValueLeaf::I64(**self))
-    }
-}
-impl<'a> Value<'a> for &'a f64 {
-    type ArrayIter = Empty<&'a f64>;
-    type ObjectIter = Empty<(&'a str, &'a f64)>;
-    #[inline]
-    fn as_value(&self) -> ReferenceValue<'a, Self> {
-        ReferenceValue::Leaf(ReferenceValueLeaf::F64(**self))
-    }
-}
-impl<'a> Value<'a> for &'a bool {
-    type ArrayIter = Empty<&'a bool>;
-    type ObjectIter = Empty<(&'a str, &'a bool)>;
-    #[inline]
-    fn as_value(&self) -> ReferenceValue<'a, Self> {
-        ReferenceValue::Leaf(ReferenceValueLeaf::Bool(**self))
-    }
-}
-impl<'a> Value<'a> for &'a str {
-    type ArrayIter = Empty<&'a str>;
-    type ObjectIter = Empty<(&'a str, &'a str)>;
-    #[inline]
-    fn as_value(&self) -> ReferenceValue<'a, Self> {
-        ReferenceValue::Leaf(ReferenceValueLeaf::Str(self))
-    }
-}
-impl<'a> Value<'a> for &'a &'a str {
-    type ArrayIter = Empty<&'a &'a str>;
-    type ObjectIter = Empty<(&'a str, &'a &'a str)>;
-    #[inline]
-    fn as_value(&self) -> ReferenceValue<'a, Self> {
-        ReferenceValue::Leaf(ReferenceValueLeaf::Str(self))
-    }
-}
-
-impl<'a> Value<'a> for &'a [u8] {
-    type ArrayIter = Empty<&'a [u8]>;
-    type ObjectIter = Empty<(&'a str, &'a [u8])>;
-    #[inline]
-    fn as_value(&self) -> ReferenceValue<'a, Self> {
-        ReferenceValue::Leaf(ReferenceValueLeaf::Bytes(self))
-    }
-}
-
-impl<'a> Value<'a> for &'a &'a [u8] {
-    type ArrayIter = Empty<&'a &'a [u8]>;
-    type ObjectIter = Empty<(&'a str, &'a &'a [u8])>;
-    #[inline]
-    fn as_value(&self) -> ReferenceValue<'a, Self> {
-        ReferenceValue::Leaf(ReferenceValueLeaf::Bytes(self))
-    }
-}
-
-impl<'a> Value<'a> for &'a Vec<u8> {
-    type ArrayIter = Empty<&'a Vec<u8>>;
-    type ObjectIter = Empty<(&'a str, &'a Vec<u8>)>;
-    #[inline]
-    fn as_value(&self) -> ReferenceValue<'a, Self> {
-        ReferenceValue::Leaf(ReferenceValueLeaf::Bytes(self))
-    }
-}
-
-impl<'a> Value<'a> for &'a DateTime {
-    type ArrayIter = Empty<&'a DateTime>;
-    type ObjectIter = Empty<(&'a str, &'a DateTime)>;
-    #[inline]
-    fn as_value(&self) -> ReferenceValue<'a, Self> {
-        ReferenceValue::Leaf(ReferenceValueLeaf::Date(**self))
-    }
-}
-impl<'a> Value<'a> for &'a Ipv6Addr {
-    type ArrayIter = Empty<&'a Ipv6Addr>;
-    type ObjectIter = Empty<(&'a str, &'a Ipv6Addr)>;
-    #[inline]
-    fn as_value(&self) -> ReferenceValue<'a, Self> {
-        ReferenceValue::Leaf(ReferenceValueLeaf::IpAddr(**self))
-    }
-}
-impl<'a> Value<'a> for &'a PreTokenizedString {
-    type ArrayIter = Empty<&'a PreTokenizedString>;
-    type ObjectIter = Empty<(&'a str, &'a PreTokenizedString)>;
-    #[inline]
-    fn as_value(&self) -> ReferenceValue<'a, Self> {
-        ReferenceValue::Leaf(ReferenceValueLeaf::PreTokStr(Box::new((*self).clone())))
-    }
-}
-
 impl ValueDeserialize for serde_json::Value {
     fn deserialize<'de, D>(deserializer: D) -> Result<Self, DeserializeError>
     where D: ValueDeserializer<'de> {

src/schema/document/mod.rs

@@ -5,24 +5,22 @@
 //! - [Value] which provides tantivy with a way to access the document's values in a common way
 //!   without performing any additional allocations.
 //! - [DocumentDeserialize] which implements the necessary code to deserialize the document from the
-//!   doc store. If you are fine with fetching [TantivyDocument] from the doc store, you can skip
-//!   implementing this trait for your type.
+//!   doc store.
 //!
 //! Tantivy provides a few out-of-box implementations of these core traits to provide
 //! some simple usage if you don't want to implement these traits on a custom type yourself.
 //!
 //! # Out-of-box document implementations
-//! - [TantivyDocument] the old document type used by Tantivy before the trait based approach was
+//! - [Document] the old document type used by Tantivy before the trait based approach was
 //!   implemented. This type is still valid and provides all of the original behaviour you might
 //!   expect.
-//! - `BTreeMap<Field, OwnedValue>` a mapping of field_ids to their relevant schema value using a
+//! - `BTreeMap<Field, Value>` a mapping of field_ids to their relevant schema value using a
 //!   BTreeMap.
-//! - `HashMap<Field, OwnedValue>` a mapping of field_ids to their relevant schema value using a
-//!   HashMap.
+//! - `HashMap<Field, Value>` a mapping of field_ids to their relevant schema value using a HashMap.
 //!
 //! # Implementing your custom documents
 //! Often in larger projects or higher performance applications you want to avoid the extra overhead
-//! of converting your own types to the [TantivyDocument] type, this can often save you a
+//! of converting your own types to the Tantivy [Document] type, this can often save you a
 //! significant amount of time when indexing by avoiding the additional allocations.
 //!
 //! ### Important Note
@@ -48,7 +46,6 @@
 //!
 //! impl Document for MyCustomDocument {
 //!     // The value type produced by the `iter_fields_and_values` iterator.
-//!     // tantivy already implements the Value trait for serde_json::Value.
 //!     type Value<'a> = &'a serde_json::Value;
 //!     // The iterator which is produced by `iter_fields_and_values`.
 //!     // Often this is a simple new-type wrapper unless you like super long generics.
@@ -97,11 +94,10 @@
 //! implementation for.
 //!
 //! ## Implementing custom values
-//! In order to allow documents to return custom types, they must implement
-//! the [Value] trait which provides a way for Tantivy to get a `ReferenceValue` that it can then
-//! index and store.
 //! Internally, Tantivy only works with `ReferenceValue` which is an enum that tries to borrow
-//! as much data as it can
+//! as much data as it can, in order to allow documents to return custom types, they must implement
+//! the `Value` trait which provides a way for Tantivy to get a `ReferenceValue` that it can then
+//! index and store.
 //!
 //! Values can just as easily be customised as documents by implementing the `Value` trait.
 //!
@@ -109,9 +105,9 @@
 //! hold references of the data held by the parent [Document] which can then be passed
 //! on to the [ReferenceValue].
 //!
-//! This is why [Value] is implemented for `&'a serde_json::Value` and
-//! [&'a tantivy::schema::document::OwnedValue](OwnedValue) but not for their owned counterparts, as
-//! we cannot satisfy the lifetime bounds necessary when indexing the documents.
+//! This is why `Value` is implemented for `&'a serde_json::Value` and `&'a
+//! tantivy::schema::Value` but not for their owned counterparts, as we cannot satisfy the lifetime
+//! bounds necessary when indexing the documents.
 //!
 //! ### A note about returning values
 //! The custom value type does not have to be the type stored by the document, instead the
@@ -172,9 +168,7 @@ pub use self::de::{
     ArrayAccess, DeserializeError, DocumentDeserialize, DocumentDeserializer, ObjectAccess,
     ValueDeserialize, ValueDeserializer, ValueType, ValueVisitor,
 };
-pub use self::default_document::{
-    CompactDocArrayIter, CompactDocObjectIter, CompactDocValue, DocParsingError, TantivyDocument,
-};
+pub use self::default_document::{DocParsingError, TantivyDocument};
 pub use self::owned_value::OwnedValue;
 pub(crate) use self::se::BinaryDocumentSerializer;
 pub use self::value::{ReferenceValue, ReferenceValueLeaf, Value};
@@ -235,7 +229,7 @@ pub trait Document: Send + Sync + 'static {
             let field_name = schema.get_field_name(field);
             let values: Vec<OwnedValue> = field_values
                 .into_iter()
-                .map(|val| OwnedValue::from(val.as_value()))
+                .map(|val| val.as_value().into())
                 .collect();
             field_map.insert(field_name.to_string(), values);
         }

src/schema/document/owned_value.rs

@@ -1,4 +1,4 @@
-use std::collections::BTreeMap;
+use std::collections::{btree_map, BTreeMap};
 use std::fmt;
 use std::net::Ipv6Addr;
 
@@ -8,7 +8,6 @@ use serde::de::{MapAccess, SeqAccess};
 use time::format_description::well_known::Rfc3339;
 use time::OffsetDateTime;
 
-use super::existing_type_impls::can_be_rfc3339_date_time;
 use super::ReferenceValueLeaf;
 use crate::schema::document::{
     ArrayAccess, DeserializeError, ObjectAccess, ReferenceValue, Value, ValueDeserialize,
@@ -46,7 +45,7 @@ pub enum OwnedValue {
     /// A set of values.
     Array(Vec<Self>),
     /// Dynamic object value.
-    Object(Vec<(String, Self)>),
+    Object(BTreeMap<String, Self>),
     /// IpV6 Address. Internally there is no IpV4, it needs to be converted to `Ipv6Addr`.
     IpAddr(Ipv6Addr),
 }
@@ -66,13 +65,13 @@ impl<'a> Value<'a> for &'a OwnedValue {
         match self {
             OwnedValue::Null => ReferenceValueLeaf::Null.into(),
             OwnedValue::Str(val) => ReferenceValueLeaf::Str(val).into(),
-            OwnedValue::PreTokStr(val) => ReferenceValueLeaf::PreTokStr(val.clone().into()).into(),
+            OwnedValue::PreTokStr(val) => ReferenceValueLeaf::PreTokStr(val).into(),
             OwnedValue::U64(val) => ReferenceValueLeaf::U64(*val).into(),
             OwnedValue::I64(val) => ReferenceValueLeaf::I64(*val).into(),
             OwnedValue::F64(val) => ReferenceValueLeaf::F64(*val).into(),
             OwnedValue::Bool(val) => ReferenceValueLeaf::Bool(*val).into(),
             OwnedValue::Date(val) => ReferenceValueLeaf::Date(*val).into(),
-            OwnedValue::Facet(val) => ReferenceValueLeaf::Facet(val.encoded_str()).into(),
+            OwnedValue::Facet(val) => ReferenceValueLeaf::Facet(val).into(),
             OwnedValue::Bytes(val) => ReferenceValueLeaf::Bytes(val).into(),
             OwnedValue::IpAddr(val) => ReferenceValueLeaf::IpAddr(*val).into(),
             OwnedValue::Array(array) => ReferenceValue::Array(array.iter()),
@@ -149,10 +148,10 @@ impl ValueDeserialize for OwnedValue {
 
             fn visit_object<'de, A>(&self, mut access: A) -> Result<Self::Value, DeserializeError>
             where A: ObjectAccess<'de> {
-                let mut elements = Vec::with_capacity(access.size_hint());
+                let mut elements = BTreeMap::new();
 
                 while let Some((key, value)) = access.next_entry()? {
-                    elements.push((key, value));
+                    elements.insert(key, value);
                 }
 
                 Ok(OwnedValue::Object(elements))
@@ -168,7 +167,6 @@ impl Eq for OwnedValue {}
 impl serde::Serialize for OwnedValue {
     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
     where S: serde::Serializer {
-        use serde::ser::SerializeMap;
         match *self {
             OwnedValue::Null => serializer.serialize_unit(),
             OwnedValue::Str(ref v) => serializer.serialize_str(v),
@@ -182,13 +180,7 @@ impl serde::Serialize for OwnedValue {
             }
             OwnedValue::Facet(ref facet) => facet.serialize(serializer),
             OwnedValue::Bytes(ref bytes) => serializer.serialize_str(&BASE64.encode(bytes)),
-            OwnedValue::Object(ref obj) => {
-                let mut map = serializer.serialize_map(Some(obj.len()))?;
-                for (k, v) in obj {
-                    map.serialize_entry(k, v)?;
-                }
-                map.end()
-            }
+            OwnedValue::Object(ref obj) => obj.serialize(serializer),
             OwnedValue::IpAddr(ref ip_v6) => {
                 // Ensure IpV4 addresses get serialized as IpV4, but excluding IpV6 loopback.
                 if let Some(ip_v4) = ip_v6.to_ipv4_mapped() {
@@ -256,10 +248,12 @@ impl<'de> serde::Deserialize<'de> for OwnedValue {
 
             fn visit_map<A>(self, mut map: A) -> Result<Self::Value, A::Error>
             where A: MapAccess<'de> {
-                let mut object = map.size_hint().map(Vec::with_capacity).unwrap_or_default();
+                let mut object = BTreeMap::new();
+
                 while let Some((key, value)) = map.next_entry()? {
-                    object.push((key, value));
+                    object.insert(key, value);
                 }
+
                 Ok(OwnedValue::Object(object))
             }
         }
@@ -278,13 +272,11 @@ impl<'a, V: Value<'a>> From<ReferenceValue<'a, V>> for OwnedValue {
                 ReferenceValueLeaf::I64(val) => OwnedValue::I64(val),
                 ReferenceValueLeaf::F64(val) => OwnedValue::F64(val),
                 ReferenceValueLeaf::Date(val) => OwnedValue::Date(val),
-                ReferenceValueLeaf::Facet(val) => {
-                    OwnedValue::Facet(Facet::from_encoded_string(val.to_string()))
-                }
+                ReferenceValueLeaf::Facet(val) => OwnedValue::Facet(val.clone()),
                 ReferenceValueLeaf::Bytes(val) => OwnedValue::Bytes(val.to_vec()),
                 ReferenceValueLeaf::IpAddr(val) => OwnedValue::IpAddr(val),
                 ReferenceValueLeaf::Bool(val) => OwnedValue::Bool(val),
-                ReferenceValueLeaf::PreTokStr(val) => OwnedValue::PreTokStr(*val.clone()),
+                ReferenceValueLeaf::PreTokStr(val) => OwnedValue::PreTokStr(val.clone()),
             },
             ReferenceValue::Array(val) => {
                 OwnedValue::Array(val.map(|v| v.as_value().into()).collect())
@@ -371,11 +363,20 @@ impl From<PreTokenizedString> for OwnedValue {
 
 impl From<BTreeMap<String, OwnedValue>> for OwnedValue {
     fn from(object: BTreeMap<String, OwnedValue>) -> OwnedValue {
-        let key_values = object.into_iter().collect();
-        OwnedValue::Object(key_values)
+        OwnedValue::Object(object)
     }
 }
 
+fn can_be_rfc3339_date_time(text: &str) -> bool {
+    if let Some(&first_byte) = text.as_bytes().first() {
+        if first_byte.is_ascii_digit() {
+            return true;
+        }
+    }
+
+    false
+}
+
 impl From<serde_json::Value> for OwnedValue {
     fn from(value: serde_json::Value) -> Self {
         match value {
@@ -416,16 +417,18 @@ impl From<serde_json::Value> for OwnedValue {
 
 impl From<serde_json::Map<String, serde_json::Value>> for OwnedValue {
     fn from(map: serde_json::Map<String, serde_json::Value>) -> Self {
-        let object: Vec<(String, OwnedValue)> = map
-            .into_iter()
-            .map(|(key, value)| (key, OwnedValue::from(value)))
-            .collect();
+        let mut object = BTreeMap::new();
+
+        for (key, value) in map {
+            object.insert(key, OwnedValue::from(value));
+        }
+
         OwnedValue::Object(object)
     }
 }
 
 /// A wrapper type for iterating over a serde_json object producing reference values.
-pub struct ObjectMapIter<'a>(std::slice::Iter<'a, (String, OwnedValue)>);
+pub struct ObjectMapIter<'a>(btree_map::Iter<'a, String, OwnedValue>);
 
 impl<'a> Iterator for ObjectMapIter<'a> {
     type Item = (&'a str, &'a OwnedValue);
@@ -463,7 +466,6 @@ mod tests {
         let mut doc = TantivyDocument::default();
         doc.add_bytes(bytes_field, "".as_bytes());
         let json_string = doc.to_json(&schema);
-
         assert_eq!(json_string, r#"{"my_bytes":[""]}"#);
     }
 

src/schema/document/se.rs

@@ -25,7 +25,6 @@ where W: Write
 
     /// Attempts to serialize a given document and write the output
     /// to the writer.
-    #[inline]
     pub(crate) fn serialize_doc<D>(&mut self, doc: &D) -> io::Result<()>
     where D: Document {
         let stored_field_values = || {
@@ -58,8 +57,9 @@ where W: Write
             return Err(io::Error::new(
                 io::ErrorKind::Other,
                 format!(
-                    "Unexpected number of entries written to serializer, expected \
-                     {num_field_values} entries, got {actual_length} entries",
+                    "Unexpected number of entries written to serializer, expected {} entries, got \
+                     {} entries",
+                    num_field_values, actual_length,
                 ),
             ));
         }
@@ -121,7 +121,7 @@ where W: Write
                 ReferenceValueLeaf::Facet(val) => {
                     self.write_type_code(type_codes::HIERARCHICAL_FACET_CODE)?;
 
-                    Cow::Borrowed(val).serialize(self.writer)
+                    val.serialize(self.writer)
                 }
                 ReferenceValueLeaf::Bytes(val) => {
                     self.write_type_code(type_codes::BYTES_CODE)?;
@@ -428,7 +428,7 @@ mod tests {
         );
 
         let facet = Facet::from_text("/hello/world").unwrap();
-        let result = serialize_value(ReferenceValueLeaf::Facet(facet.encoded_str()).into());
+        let result = serialize_value(ReferenceValueLeaf::Facet(&facet).into());
         let expected = binary_repr!(
             type_codes::HIERARCHICAL_FACET_CODE => Facet::from_text("/hello/world").unwrap(),
         );
@@ -441,8 +441,7 @@ mod tests {
             text: "hello, world".to_string(),
             tokens: vec![Token::default(), Token::default()],
         };
-        let result =
-            serialize_value(ReferenceValueLeaf::PreTokStr(pre_tok_str.clone().into()).into());
+        let result = serialize_value(ReferenceValueLeaf::PreTokStr(&pre_tok_str).into());
         let expected = binary_repr!(
             type_codes::EXT_CODE, type_codes::TOK_STR_EXT_CODE => pre_tok_str,
         );
@@ -679,7 +678,6 @@ mod tests {
         );
     }
 
-    #[inline]
     fn serialize_doc<D: Document>(doc: &D, schema: &Schema) -> Vec<u8> {
         let mut writer = Vec::new();
 

src/schema/document/value.rs

@@ -3,6 +3,7 @@ use std::net::Ipv6Addr;
 
 use common::DateTime;
 
+use crate::schema::Facet;
 use crate::tokenizer::PreTokenizedString;
 
 /// A single field value.
@@ -27,7 +28,7 @@ pub trait Value<'a>: Send + Sync + Debug {
     }
 
     #[inline]
-    /// If the Value is a leaf, returns the associated leaf. Returns None otherwise.
+    /// If the Value is a String, returns the associated str. Returns None otherwise.
     fn as_leaf(&self) -> Option<ReferenceValueLeaf<'a>> {
         if let ReferenceValue::Leaf(val) = self.as_value() {
             Some(val)
@@ -81,9 +82,8 @@ pub trait Value<'a>: Send + Sync + Debug {
     #[inline]
     /// If the Value is a pre-tokenized string, returns the associated string. Returns None
     /// otherwise.
-    fn as_pre_tokenized_text(&self) -> Option<Box<PreTokenizedString>> {
-        self.as_leaf()
-            .and_then(|leaf| leaf.into_pre_tokenized_text())
+    fn as_pre_tokenized_text(&self) -> Option<&'a PreTokenizedString> {
+        self.as_leaf().and_then(|leaf| leaf.as_pre_tokenized_text())
     }
 
     #[inline]
@@ -94,7 +94,7 @@ pub trait Value<'a>: Send + Sync + Debug {
 
     #[inline]
     /// If the Value is a facet, returns the associated facet. Returns None otherwise.
-    fn as_facet(&self) -> Option<&'a str> {
+    fn as_facet(&self) -> Option<&'a Facet> {
         self.as_leaf().and_then(|leaf| leaf.as_facet())
     }
 
@@ -132,7 +132,7 @@ pub trait Value<'a>: Send + Sync + Debug {
 }
 
 /// A enum representing a leaf value for tantivy to index.
-#[derive(Clone, Debug, PartialEq)]
+#[derive(Clone, Copy, Debug, PartialEq)]
 pub enum ReferenceValueLeaf<'a> {
     /// A null value.
     Null,
@@ -146,9 +146,8 @@ pub enum ReferenceValueLeaf<'a> {
     F64(f64),
     /// Date/time with nanoseconds precision
     Date(DateTime),
-    /// Facet string needs to match the format of
-    /// [Facet::encoded_str](crate::schema::Facet::encoded_str).
-    Facet(&'a str),
+    /// Facet
+    Facet(&'a Facet),
     /// Arbitrarily sized byte array
     Bytes(&'a [u8]),
     /// IpV6 Address. Internally there is no IpV4, it needs to be converted to `Ipv6Addr`.
@@ -156,70 +155,7 @@ pub enum ReferenceValueLeaf<'a> {
     /// Bool value
     Bool(bool),
     /// Pre-tokenized str type,
-    PreTokStr(Box<PreTokenizedString>),
-}
-
-impl From<u64> for ReferenceValueLeaf<'_> {
-    #[inline]
-    fn from(value: u64) -> Self {
-        ReferenceValueLeaf::U64(value)
-    }
-}
-
-impl From<i64> for ReferenceValueLeaf<'_> {
-    #[inline]
-    fn from(value: i64) -> Self {
-        ReferenceValueLeaf::I64(value)
-    }
-}
-
-impl From<f64> for ReferenceValueLeaf<'_> {
-    #[inline]
-    fn from(value: f64) -> Self {
-        ReferenceValueLeaf::F64(value)
-    }
-}
-
-impl From<bool> for ReferenceValueLeaf<'_> {
-    #[inline]
-    fn from(value: bool) -> Self {
-        ReferenceValueLeaf::Bool(value)
-    }
-}
-
-impl<'a> From<&'a str> for ReferenceValueLeaf<'a> {
-    #[inline]
-    fn from(value: &'a str) -> Self {
-        ReferenceValueLeaf::Str(value)
-    }
-}
-
-impl<'a> From<&'a [u8]> for ReferenceValueLeaf<'a> {
-    #[inline]
-    fn from(value: &'a [u8]) -> Self {
-        ReferenceValueLeaf::Bytes(value)
-    }
-}
-
-impl From<DateTime> for ReferenceValueLeaf<'_> {
-    #[inline]
-    fn from(value: DateTime) -> Self {
-        ReferenceValueLeaf::Date(value)
-    }
-}
-
-impl From<Ipv6Addr> for ReferenceValueLeaf<'_> {
-    #[inline]
-    fn from(value: Ipv6Addr) -> Self {
-        ReferenceValueLeaf::IpAddr(value)
-    }
-}
-
-impl From<PreTokenizedString> for ReferenceValueLeaf<'_> {
-    #[inline]
-    fn from(val: PreTokenizedString) -> Self {
-        ReferenceValueLeaf::PreTokStr(Box::new(val))
-    }
+    PreTokStr(&'a PreTokenizedString),
 }
 
 impl<'a, T: Value<'a> + ?Sized> From<ReferenceValueLeaf<'a>> for ReferenceValue<'a, T> {
@@ -323,9 +259,9 @@ impl<'a> ReferenceValueLeaf<'a> {
     }
 
     #[inline]
-    /// If the Value is a pre-tokenized string, consumes it and returns the string.
-    /// Returns None otherwise.
-    pub fn into_pre_tokenized_text(self) -> Option<Box<PreTokenizedString>> {
+    /// If the Value is a pre-tokenized string, returns the associated string. Returns None
+    /// otherwise.
+    pub fn as_pre_tokenized_text(&self) -> Option<&'a PreTokenizedString> {
         if let Self::PreTokStr(val) = self {
             Some(val)
         } else {
@@ -345,7 +281,7 @@ impl<'a> ReferenceValueLeaf<'a> {
 
     #[inline]
     /// If the Value is a facet, returns the associated facet. Returns None otherwise.
-    pub fn as_facet(&self) -> Option<&'a str> {
+    pub fn as_facet(&self) -> Option<&'a Facet> {
         if let Self::Facet(val) = self {
             Some(val)
         } else {
@@ -386,16 +322,6 @@ where V: Value<'a>
         }
     }
 
-    #[inline]
-    /// If the Value is a leaf, consume it and return the leaf. Returns None otherwise.
-    pub fn into_leaf(self) -> Option<ReferenceValueLeaf<'a>> {
-        if let Self::Leaf(val) = self {
-            Some(val)
-        } else {
-            None
-        }
-    }
-
     #[inline]
     /// If the Value is a String, returns the associated str. Returns None otherwise.
     pub fn as_str(&self) -> Option<&'a str> {
@@ -439,11 +365,10 @@ where V: Value<'a>
     }
 
     #[inline]
-    /// If the Value is a pre-tokenized string, consumes it and returns the string.
-    /// Returns None otherwise.
-    pub fn into_pre_tokenized_text(self) -> Option<Box<PreTokenizedString>> {
-        self.into_leaf()
-            .and_then(|leaf| leaf.into_pre_tokenized_text())
+    /// If the Value is a pre-tokenized string, returns the associated string. Returns None
+    /// otherwise.
+    pub fn as_pre_tokenized_text(&self) -> Option<&'a PreTokenizedString> {
+        self.as_leaf().and_then(|leaf| leaf.as_pre_tokenized_text())
     }
 
     #[inline]
@@ -454,7 +379,7 @@ where V: Value<'a>
 
     #[inline]
     /// If the Value is a facet, returns the associated facet. Returns None otherwise.
-    pub fn as_facet(&self) -> Option<&'a str> {
+    pub fn as_facet(&self) -> Option<&'a Facet> {
         self.as_leaf().and_then(|leaf| leaf.as_facet())
     }
 

src/schema/field_type.rs

@@ -568,21 +568,21 @@ mod tests {
         let schema = schema_builder.build();
         let doc = TantivyDocument::parse_json(&schema, r#"{"id": 100}"#).unwrap();
         assert_eq!(
-            OwnedValue::Str("100".to_string()),
-            doc.get_first(text_field).unwrap().into()
+            &OwnedValue::Str("100".to_string()),
+            doc.get_first(text_field).unwrap()
         );
 
         let doc = TantivyDocument::parse_json(&schema, r#"{"id": true}"#).unwrap();
         assert_eq!(
-            OwnedValue::Str("true".to_string()),
-            doc.get_first(text_field).unwrap().into()
+            &OwnedValue::Str("true".to_string()),
+            doc.get_first(text_field).unwrap()
         );
 
         // Not sure if this null coercion is the best approach
         let doc = TantivyDocument::parse_json(&schema, r#"{"id": null}"#).unwrap();
         assert_eq!(
-            OwnedValue::Str("null".to_string()),
-            doc.get_first(text_field).unwrap().into()
+            &OwnedValue::Str("null".to_string()),
+            doc.get_first(text_field).unwrap()
         );
     }
 
@@ -595,18 +595,9 @@ mod tests {
         let schema = schema_builder.build();
         let doc_json = r#"{"i64": "100", "u64": "100", "f64": "100"}"#;
         let doc = TantivyDocument::parse_json(&schema, doc_json).unwrap();
-        assert_eq!(
-            OwnedValue::I64(100),
-            doc.get_first(i64_field).unwrap().into()
-        );
-        assert_eq!(
-            OwnedValue::U64(100),
-            doc.get_first(u64_field).unwrap().into()
-        );
-        assert_eq!(
-            OwnedValue::F64(100.0),
-            doc.get_first(f64_field).unwrap().into()
-        );
+        assert_eq!(&OwnedValue::I64(100), doc.get_first(i64_field).unwrap());
+        assert_eq!(&OwnedValue::U64(100), doc.get_first(u64_field).unwrap());
+        assert_eq!(&OwnedValue::F64(100.0), doc.get_first(f64_field).unwrap());
     }
 
     #[test]
@@ -616,17 +607,11 @@ mod tests {
         let schema = schema_builder.build();
         let doc_json = r#"{"bool": "true"}"#;
         let doc = TantivyDocument::parse_json(&schema, doc_json).unwrap();
-        assert_eq!(
-            OwnedValue::Bool(true),
-            doc.get_first(bool_field).unwrap().into()
-        );
+        assert_eq!(&OwnedValue::Bool(true), doc.get_first(bool_field).unwrap());
 
         let doc_json = r#"{"bool": "false"}"#;
         let doc = TantivyDocument::parse_json(&schema, doc_json).unwrap();
-        assert_eq!(
-            OwnedValue::Bool(false),
-            doc.get_first(bool_field).unwrap().into()
-        );
+        assert_eq!(&OwnedValue::Bool(false), doc.get_first(bool_field).unwrap());
     }
 
     #[test]
@@ -659,7 +644,7 @@ mod tests {
         let schema = schema_builder.build();
         let doc_json = r#"{"date": "2019-10-12T07:20:50.52+02:00"}"#;
         let doc = TantivyDocument::parse_json(&schema, doc_json).unwrap();
-        let date = OwnedValue::from(doc.get_first(date_field).unwrap());
+        let date = doc.get_first(date_field).unwrap();
         // Time zone is converted to UTC
         assert_eq!("Date(2019-10-12T05:20:50.52Z)", format!("{date:?}"));
     }

src/schema/field_value.rs

@@ -0,0 +1,46 @@
+use crate::schema::{Field, OwnedValue};
+
+/// `FieldValue` holds together a `Field` and its `Value`.
+#[allow(missing_docs)]
+#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
+pub struct FieldValue {
+    pub field: Field,
+    pub value: OwnedValue,
+}
+
+impl FieldValue {
+    /// Constructor
+    pub fn new(field: Field, value: OwnedValue) -> FieldValue {
+        FieldValue { field, value }
+    }
+
+    /// Field accessor
+    pub fn field(&self) -> Field {
+        self.field
+    }
+
+    /// Value accessor
+    pub fn value(&self) -> &OwnedValue {
+        &self.value
+    }
+}
+
+impl From<FieldValue> for OwnedValue {
+    fn from(field_value: FieldValue) -> Self {
+        field_value.value
+    }
+}
+
+/// A helper wrapper for creating standard iterators
+/// out of the fields iterator trait.
+pub struct FieldValueIter<'a>(pub(crate) std::slice::Iter<'a, FieldValue>);
+
+impl<'a> Iterator for FieldValueIter<'a> {
+    type Item = (Field, &'a OwnedValue);
+
+    fn next(&mut self) -> Option<Self::Item> {
+        self.0
+            .next()
+            .map(|field_value| (field_value.field, &field_value.value))
+    }
+}

src/schema/flags.rs

@@ -1,6 +1,7 @@
 use std::ops::BitOr;
 
-use crate::schema::{DateOptions, NumericOptions, TextOptions};
+use crate::schema::{NumericOptions, TextOptions};
+use crate::DateOptions;
 
 #[derive(Clone)]
 pub struct StoredFlag;

src/schema/mod.rs

@@ -114,6 +114,7 @@ pub(crate) mod term;
 
 mod field_entry;
 mod field_type;
+mod field_value;
 
 mod bytes_options;
 mod date_time_options;
@@ -137,6 +138,7 @@ pub use self::facet_options::FacetOptions;
 pub use self::field::Field;
 pub use self::field_entry::FieldEntry;
 pub use self::field_type::{FieldType, Type};
+pub use self::field_value::FieldValue;
 pub use self::flags::{COERCE, FAST, INDEXED, STORED};
 pub use self::index_record_option::IndexRecordOption;
 pub use self::ip_options::{IntoIpv6Addr, IpAddrOptions};
@@ -144,7 +146,7 @@ pub use self::json_object_options::JsonObjectOptions;
 pub use self::named_field_document::NamedFieldDocument;
 pub use self::numeric_options::NumericOptions;
 pub use self::schema::{Schema, SchemaBuilder};
-pub use self::term::{Term, ValueBytes};
+pub use self::term::{Term, ValueBytes, JSON_END_OF_PATH};
 pub use self::text_options::{TextFieldIndexing, TextOptions, STRING, TEXT};
 
 /// Validator for a potential `field_name`.

src/schema/schema.rs

@@ -645,15 +645,15 @@ mod tests {
         let doc =
             TantivyDocument::convert_named_doc(&schema, NamedFieldDocument(named_doc_map)).unwrap();
         assert_eq!(
-            doc.get_all(title).map(OwnedValue::from).collect::<Vec<_>>(),
+            doc.get_all(title).collect::<Vec<_>>(),
             vec![
-                OwnedValue::from("title1".to_string()),
-                OwnedValue::from("title2".to_string())
+                &OwnedValue::from("title1".to_string()),
+                &OwnedValue::from("title2".to_string())
             ]
         );
         assert_eq!(
-            doc.get_all(val).map(OwnedValue::from).collect::<Vec<_>>(),
-            vec![OwnedValue::from(14u64), OwnedValue::from(-1i64)]
+            doc.get_all(val).collect::<Vec<_>>(),
+            vec![&OwnedValue::from(14u64), &OwnedValue::from(-1i64)]
         );
     }
 
@@ -682,7 +682,7 @@ mod tests {
         let schema = schema_builder.build();
         {
             let doc = TantivyDocument::parse_json(&schema, "{}").unwrap();
-            assert!(doc.field_values().next().is_none());
+            assert!(doc.field_values().is_empty());
         }
         {
             let doc = TantivyDocument::parse_json(

src/schema/term.rs

@@ -3,16 +3,22 @@ use std::net::Ipv6Addr;
 use std::{fmt, str};
 
 use columnar::{MonotonicallyMappableToU128, MonotonicallyMappableToU64};
-use common::json_path_writer::{JSON_END_OF_PATH, JSON_PATH_SEGMENT_SEP_STR};
-use common::JsonPathWriter;
 
 use super::date_time_options::DATE_TIME_PRECISION_INDEXED;
 use super::Field;
 use crate::fastfield::FastValue;
-use crate::json_utils::split_json_path;
 use crate::schema::{Facet, Type};
 use crate::DateTime;
 
+/// Separates the different segments of a json path.
+pub const JSON_PATH_SEGMENT_SEP: u8 = 1u8;
+pub const JSON_PATH_SEGMENT_SEP_STR: &str =
+    unsafe { std::str::from_utf8_unchecked(&[JSON_PATH_SEGMENT_SEP]) };
+
+/// Separates the json path and the value in
+/// a JSON term binary representation.
+pub const JSON_END_OF_PATH: u8 = 0u8;
+
 /// Term represents the value that the token can take.
 /// It's a serialized representation over different types.
 ///
@@ -35,28 +41,6 @@ impl Term {
         Term(data)
     }
 
-    /// Creates a term from a json path.
-    ///
-    /// The json path can address a nested value in a JSON object.
-    /// e.g. `{"k8s": {"node": {"id": 5}}}` can be addressed via `k8s.node.id`.
-    ///
-    /// In case there are dots in the field name, and the `expand_dots_enabled` parameter is not
-    /// set they need to be escaped with a backslash.
-    /// e.g. `{"k8s.node": {"id": 5}}` can be addressed via `k8s\.node.id`.
-    pub fn from_field_json_path(field: Field, json_path: &str, expand_dots_enabled: bool) -> Term {
-        let paths = split_json_path(json_path);
-        let mut json_path = JsonPathWriter::with_expand_dots(expand_dots_enabled);
-        for path in paths {
-            json_path.push(&path);
-        }
-        json_path.set_end();
-        let mut term = Term::with_type_and_field(Type::Json, field);
-
-        term.append_bytes(json_path.as_str().as_bytes());
-
-        term
-    }
-
     pub(crate) fn with_type_and_field(typ: Type, field: Field) -> Term {
         let mut term = Self::with_capacity(8);
         term.set_field_and_type(field, typ);
@@ -185,11 +169,7 @@ impl Term {
         self.set_bytes(val.to_u64().to_be_bytes().as_ref());
     }
 
-    /// Append a type marker + fast value to a term.
-    /// This is used in JSON type to append a fast value after the path.
-    ///
-    /// It will not clear existing bytes.
-    pub fn append_type_and_fast_value<T: FastValue>(&mut self, val: T) {
+    pub(crate) fn append_type_and_fast_value<T: FastValue>(&mut self, val: T) {
         self.0.push(T::to_type().to_code());
         let value = if T::to_type() == Type::Date {
             DateTime::from_u64(val.to_u64())
@@ -201,15 +181,6 @@ impl Term {
         self.0.extend(value.to_be_bytes().as_ref());
     }
 
-    /// Append a string type marker + string to a term.
-    /// This is used in JSON type to append a str after the path.
-    ///
-    /// It will not clear existing bytes.
-    pub fn append_type_and_str(&mut self, val: &str) {
-        self.0.push(Type::Str.to_code());
-        self.0.extend(val.as_bytes().as_ref());
-    }
-
     /// Sets a `Ipv6Addr` value in the term.
     pub fn set_ip_addr(&mut self, val: Ipv6Addr) {
         self.set_bytes(val.to_u128().to_be_bytes().as_ref());
@@ -221,6 +192,11 @@ impl Term {
         self.0.extend(bytes);
     }
 
+    /// Set the texts only, keeping the field untouched.
+    pub fn set_text(&mut self, text: &str) {
+        self.set_bytes(text.as_bytes());
+    }
+
     /// Truncates the value bytes of the term. Value and field type stays the same.
     pub fn truncate_value_bytes(&mut self, len: usize) {
         self.0.truncate(len + TERM_METADATA_LENGTH);
@@ -257,6 +233,27 @@ impl Term {
         }
         &mut self.0[len_before..]
     }
+
+    /// Appends a JSON_PATH_SEGMENT_SEP to the term.
+    /// Only used for JSON type.
+    #[inline]
+    pub fn add_json_path_separator(&mut self) {
+        self.0.push(JSON_PATH_SEGMENT_SEP);
+    }
+    /// Sets the current end to JSON_END_OF_PATH.
+    /// Only used for JSON type.
+    #[inline]
+    pub fn set_json_path_end(&mut self) {
+        let buffer_len = self.0.len();
+        self.0[buffer_len - 1] = JSON_END_OF_PATH;
+    }
+    /// Sets the current end to JSON_PATH_SEGMENT_SEP.
+    /// Only used for JSON type.
+    #[inline]
+    pub fn set_json_path_separator(&mut self) {
+        let buffer_len = self.0.len();
+        self.0[buffer_len - 1] = JSON_PATH_SEGMENT_SEP;
+    }
 }
 
 impl<B> Term<B>