Speed up searches by removing repeated memsets coming from vec.resize()

Also, reserve exactly the size needed, which is surprisingly needed to
get the full speedup of ~5% on a good fraction of the queries.

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "tantivy"
-version = "0.21.0"
+version = "0.22.0-dev"
 authors = ["Paul Masurel <paul.masurel@gmail.com>"]
 license = "MIT"
 categories = ["database-implementations", "data-structures"]
@@ -22,7 +22,7 @@ crc32fast = "1.3.2"
 once_cell = "1.10.0"
 regex = { version = "1.5.5", default-features = false, features = ["std", "unicode"] }
 aho-corasick = "1.0"
-tantivy-fst = "0.4.0"
+tantivy-fst = "0.5"
 memmap2 = { version = "0.9.0", optional = true }
 lz4_flex = { version = "0.11", default-features = false, optional = true }
 zstd = { version = "0.13", optional = true, default-features = false }
@@ -31,27 +31,25 @@ log = "0.4.16"
 serde = { version = "1.0.136", features = ["derive"] }
 serde_json = "1.0.79"
 num_cpus = "1.13.1"
-fs4 = { version = "0.7.0", optional = true }
+fs4 = { version = "0.8.0", optional = true }
 levenshtein_automata = "0.2.1"
 uuid = { version = "1.0.0", features = ["v4", "serde"] }
 crossbeam-channel = "0.5.4"
 rust-stemmers = "1.2.0"
 downcast-rs = "1.2.0"
-bitpacking = { git = "https://github.com/quickwit-oss/bitpacking", rev = "f730b75", default-features = false, features = ["bitpacker4x"] }
-census = "0.4.0"
+bitpacking = { version = "0.9.2", default-features = false, features = ["bitpacker4x"] }
+census = "0.4.2"
 rustc-hash = "1.1.0"
 thiserror = "1.0.30"
 htmlescape = "0.3.1"
 fail = { version = "0.5.0", optional = true }
-murmurhash32 = "0.3.0"
 time = { version = "0.3.10", features = ["serde-well-known"] }
 smallvec = "1.8.0"
 rayon = "1.5.2"
 lru = "0.12.0"
 fastdivide = "0.4.0"
-itertools = "0.11.0"
+itertools = "0.12.0"
 measure_time = "0.8.2"
-async-trait = "0.1.53"
 arc-swap = "1.5.0"
 
 columnar = { version= "0.2", path="./columnar", package ="tantivy-columnar" }
@@ -75,15 +73,14 @@ matches = "0.1.9"
 pretty_assertions = "1.2.1"
 proptest = "1.0.0"
 test-log = "0.2.10"
-env_logger = "0.10.0"
 futures = "0.3.21"
 paste = "1.0.11"
 more-asserts = "0.3.1"
 rand_distr = "0.4.3"
+time = { version = "0.3.10", features = ["serde-well-known", "macros"] }
 
 [target.'cfg(not(windows))'.dev-dependencies]
-criterion = "0.5"
-pprof = { git = "https://github.com/PSeitz/pprof-rs/", rev = "53af24b", features = ["flamegraph", "criterion"] } # temp fork that works with criterion 0.5
+criterion = { version = "0.5", default-features = false }
 
 [dev-dependencies.fail]
 version = "0.5.0"

README.md

@@ -5,19 +5,18 @@
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
 [![Crates.io](https://img.shields.io/crates/v/tantivy.svg)](https://crates.io/crates/tantivy)
 
-![Tantivy](https://tantivy-search.github.io/logo/tantivy-logo.png)
+<img src="https://tantivy-search.github.io/logo/tantivy-logo.png" alt="Tantivy, the fastest full-text search engine library written in Rust" height="250">
 
-**Tantivy** is a **full-text search engine library** written in Rust.
+## Fast full-text search engine library written in Rust
 
-It is closer to [Apache Lucene](https://lucene.apache.org/) than to [Elasticsearch](https://www.elastic.co/products/elasticsearch) or [Apache Solr](https://lucene.apache.org/solr/) in the sense it is not
-an off-the-shelf search engine server, but rather a crate that can be used
-to build such a search engine.
+**If you are looking for an alternative to Elasticsearch or Apache Solr, check out [Quickwit](https://github.com/quickwit-oss/quickwit), our distributed search engine built on top of Tantivy.**
+
+Tantivy is closer to [Apache Lucene](https://lucene.apache.org/) than to [Elasticsearch](https://www.elastic.co/products/elasticsearch) or [Apache Solr](https://lucene.apache.org/solr/) in the sense it is not
+an off-the-shelf search engine server, but rather a crate that can be used to build such a search engine.
 
 Tantivy is, in fact, strongly inspired by Lucene's design.
 
-If you are looking for an alternative to Elasticsearch or Apache Solr, check out [Quickwit](https://github.com/quickwit-oss/quickwit), our search engine built on top of Tantivy.
-
-# Benchmark
+## Benchmark
 
 The following [benchmark](https://tantivy-search.github.io/bench/) breakdowns
 performance for different types of queries/collections.
@@ -28,7 +27,7 @@ Your mileage WILL vary depending on the nature of queries and their load.
 
 Details about the benchmark can be found at this [repository](https://github.com/quickwit-oss/search-benchmark-game).
 
-# Features
+## Features
 
 - Full-text search
 - Configurable tokenizer (stemming available for 17 Latin languages) with third party support for Chinese ([tantivy-jieba](https://crates.io/crates/tantivy-jieba) and [cang-jie](https://crates.io/crates/cang-jie)), Japanese ([lindera](https://github.com/lindera-morphology/lindera-tantivy), [Vaporetto](https://crates.io/crates/vaporetto_tantivy), and [tantivy-tokenizer-tiny-segmenter](https://crates.io/crates/tantivy-tokenizer-tiny-segmenter)) and Korean ([lindera](https://github.com/lindera-morphology/lindera-tantivy) + [lindera-ko-dic-builder](https://github.com/lindera-morphology/lindera-ko-dic-builder))
@@ -54,11 +53,11 @@ Details about the benchmark can be found at this [repository](https://github.com
 - Searcher Warmer API
 - Cheesy logo with a horse
 
-## Non-features
+### Non-features
 
 Distributed search is out of the scope of Tantivy, but if you are looking for this feature, check out [Quickwit](https://github.com/quickwit-oss/quickwit/).
 
-# Getting started
+## Getting started
 
 Tantivy works on stable Rust and supports Linux, macOS, and Windows.
 
@@ -68,7 +67,7 @@ index documents, and search via the CLI or a small server with a REST API.
 It walks you through getting a Wikipedia search engine up and running in a few minutes.
 - [Reference doc for the last released version](https://docs.rs/tantivy/)
 
-# How can I support this project?
+## How can I support this project?
 
 There are many ways to support this project.
 
@@ -79,16 +78,16 @@ There are many ways to support this project.
 - Contribute code (you can join [our Discord server](https://discord.gg/MT27AG5EVE))
 - Talk about Tantivy around you
 
-# Contributing code
+## Contributing code
 
 We use the GitHub Pull Request workflow: reference a GitHub ticket and/or include a comprehensive commit message when opening a PR.
 Feel free to update CHANGELOG.md with your contribution.
 
-## Tokenizer
+### Tokenizer
 
 When implementing a tokenizer for tantivy depend on the `tantivy-tokenizer-api` crate.
 
-## Clone and build locally
+### Clone and build locally
 
 Tantivy compiles on stable Rust.
 To check out and run tests, you can simply run:
@@ -99,7 +98,7 @@ cd tantivy
 cargo test
 ```
 
-# Companies Using Tantivy
+## Companies Using Tantivy
 
 <p align="left">
 <img align="center" src="doc/assets/images/etsy.png" alt="Etsy" height="25" width="auto" />&nbsp;
@@ -111,7 +110,7 @@ cargo test
 <img align="center" src="doc/assets/images/element-dark-theme.png#gh-dark-mode-only" alt="Element.io" height="25" width="auto" />
 </p>
 
-# FAQ
+## FAQ
 
 ### Can I use Tantivy in other languages?
 

benches/index-bench.rs

@@ -1,14 +1,99 @@
-use criterion::{criterion_group, criterion_main, Criterion, Throughput};
-use pprof::criterion::{Output, PProfProfiler};
+use criterion::{criterion_group, criterion_main, BatchSize, Bencher, Criterion, Throughput};
 use tantivy::schema::{TantivyDocument, FAST, INDEXED, STORED, STRING, TEXT};
-use tantivy::{Index, IndexWriter};
+use tantivy::{tokenizer, Index, IndexWriter};
 
 const HDFS_LOGS: &str = include_str!("hdfs.json");
 const GH_LOGS: &str = include_str!("gh.json");
 const WIKI: &str = include_str!("wiki.json");
 
-fn get_lines(input: &str) -> Vec<&str> {
-    input.trim().split('\n').collect()
+fn benchmark(
+    b: &mut Bencher,
+    input: &str,
+    schema: tantivy::schema::Schema,
+    commit: bool,
+    parse_json: bool,
+    is_dynamic: bool,
+) {
+    if is_dynamic {
+        benchmark_dynamic_json(b, input, schema, commit, parse_json)
+    } else {
+        _benchmark(b, input, schema, commit, parse_json, |schema, doc_json| {
+            TantivyDocument::parse_json(&schema, doc_json).unwrap()
+        })
+    }
+}
+
+fn get_index(schema: tantivy::schema::Schema) -> Index {
+    let mut index = Index::create_in_ram(schema.clone());
+    let ff_tokenizer_manager = tokenizer::TokenizerManager::default();
+    ff_tokenizer_manager.register(
+        "raw",
+        tokenizer::TextAnalyzer::builder(tokenizer::RawTokenizer::default())
+            .filter(tokenizer::RemoveLongFilter::limit(255))
+            .build(),
+    );
+    index.set_fast_field_tokenizers(ff_tokenizer_manager.clone());
+    index
+}
+
+fn _benchmark(
+    b: &mut Bencher,
+    input: &str,
+    schema: tantivy::schema::Schema,
+    commit: bool,
+    include_json_parsing: bool,
+    create_doc: impl Fn(&tantivy::schema::Schema, &str) -> TantivyDocument,
+) {
+    if include_json_parsing {
+        let lines: Vec<&str> = input.trim().split('\n').collect();
+        b.iter(|| {
+            let index = get_index(schema.clone());
+            let mut index_writer: IndexWriter =
+                index.writer_with_num_threads(1, 100_000_000).unwrap();
+            for doc_json in &lines {
+                let doc = create_doc(&schema, doc_json);
+                index_writer.add_document(doc).unwrap();
+            }
+            if commit {
+                index_writer.commit().unwrap();
+            }
+        })
+    } else {
+        let docs: Vec<_> = input
+            .trim()
+            .split('\n')
+            .map(|doc_json| create_doc(&schema, doc_json))
+            .collect();
+        b.iter_batched(
+            || docs.clone(),
+            |docs| {
+                let index = get_index(schema.clone());
+                let mut index_writer: IndexWriter =
+                    index.writer_with_num_threads(1, 100_000_000).unwrap();
+                for doc in docs {
+                    index_writer.add_document(doc).unwrap();
+                }
+                if commit {
+                    index_writer.commit().unwrap();
+                }
+            },
+            BatchSize::SmallInput,
+        )
+    }
+}
+fn benchmark_dynamic_json(
+    b: &mut Bencher,
+    input: &str,
+    schema: tantivy::schema::Schema,
+    commit: bool,
+    parse_json: bool,
+) {
+    let json_field = schema.get_field("json").unwrap();
+    _benchmark(b, input, schema, commit, parse_json, |_schema, doc_json| {
+        let json_val: serde_json::Map<String, serde_json::Value> =
+            serde_json::from_str(doc_json).unwrap();
+        tantivy::doc!(json_field=>json_val)
+    })
 }
 
 pub fn hdfs_index_benchmark(c: &mut Criterion) {
@@ -19,7 +104,14 @@ pub fn hdfs_index_benchmark(c: &mut Criterion) {
         schema_builder.add_text_field("severity", STRING);
         schema_builder.build()
     };
-    let schema_with_store = {
+    let schema_only_fast = {
+        let mut schema_builder = tantivy::schema::SchemaBuilder::new();
+        schema_builder.add_u64_field("timestamp", FAST);
+        schema_builder.add_text_field("body", FAST);
+        schema_builder.add_text_field("severity", FAST);
+        schema_builder.build()
+    };
+    let _schema_with_store = {
         let mut schema_builder = tantivy::schema::SchemaBuilder::new();
         schema_builder.add_u64_field("timestamp", INDEXED | STORED);
         schema_builder.add_text_field("body", TEXT | STORED);
@@ -28,77 +120,39 @@ pub fn hdfs_index_benchmark(c: &mut Criterion) {
     };
     let dynamic_schema = {
         let mut schema_builder = tantivy::schema::SchemaBuilder::new();
-        schema_builder.add_json_field("json", TEXT);
+        schema_builder.add_json_field("json", TEXT | FAST);
         schema_builder.build()
     };
 
     let mut group = c.benchmark_group("index-hdfs");
     group.throughput(Throughput::Bytes(HDFS_LOGS.len() as u64));
     group.sample_size(20);
-    group.bench_function("index-hdfs-no-commit", |b| {
-        let lines = get_lines(HDFS_LOGS);
-        b.iter(|| {
-            let index = Index::create_in_ram(schema.clone());
-            let index_writer: IndexWriter = index.writer_with_num_threads(1, 100_000_000).unwrap();
-            for doc_json in &lines {
-                let doc = TantivyDocument::parse_json(&schema, doc_json).unwrap();
-                index_writer.add_document(doc).unwrap();
+
+    let benches = [
+        ("only-indexed-".to_string(), schema, false),
+        //("stored-".to_string(), _schema_with_store, false),
+        ("only-fast-".to_string(), schema_only_fast, false),
+        ("dynamic-".to_string(), dynamic_schema, true),
+    ];
+
+    for (prefix, schema, is_dynamic) in benches {
+        for commit in [false, true] {
+            let suffix = if commit { "with-commit" } else { "no-commit" };
+            for parse_json in [false] {
+                // for parse_json in [false, true] {
+                let suffix = if parse_json {
+                    format!("{}-with-json-parsing", suffix)
+                } else {
+                    format!("{}", suffix)
+                };
+
+                let bench_name = format!("{}{}", prefix, suffix);
+                group.bench_function(bench_name, |b| {
+                    benchmark(b, HDFS_LOGS, schema.clone(), commit, parse_json, is_dynamic)
+                });
             }
-        })
-    });
-    group.bench_function("index-hdfs-with-commit", |b| {
-        let lines = get_lines(HDFS_LOGS);
-        b.iter(|| {
-            let index = Index::create_in_ram(schema.clone());
-            let mut index_writer: IndexWriter =
-                index.writer_with_num_threads(1, 100_000_000).unwrap();
-            for doc_json in &lines {
-                let doc = TantivyDocument::parse_json(&schema, doc_json).unwrap();
-                index_writer.add_document(doc).unwrap();
-            }
-            index_writer.commit().unwrap();
-        })
-    });
-    group.bench_function("index-hdfs-no-commit-with-docstore", |b| {
-        let lines = get_lines(HDFS_LOGS);
-        b.iter(|| {
-            let index = Index::create_in_ram(schema_with_store.clone());
-            let index_writer: IndexWriter = index.writer_with_num_threads(1, 100_000_000).unwrap();
-            for doc_json in &lines {
-                let doc = TantivyDocument::parse_json(&schema, doc_json).unwrap();
-                index_writer.add_document(doc).unwrap();
-            }
-        })
-    });
-    group.bench_function("index-hdfs-with-commit-with-docstore", |b| {
-        let lines = get_lines(HDFS_LOGS);
-        b.iter(|| {
-            let index = Index::create_in_ram(schema_with_store.clone());
-            let mut index_writer: IndexWriter =
-                index.writer_with_num_threads(1, 100_000_000).unwrap();
-            for doc_json in &lines {
-                let doc = TantivyDocument::parse_json(&schema, doc_json).unwrap();
-                index_writer.add_document(doc).unwrap();
-            }
-            index_writer.commit().unwrap();
-        })
-    });
-    group.bench_function("index-hdfs-no-commit-json-without-docstore", |b| {
-        let lines = get_lines(HDFS_LOGS);
-        b.iter(|| {
-            let index = Index::create_in_ram(dynamic_schema.clone());
-            let json_field = dynamic_schema.get_field("json").unwrap();
-            let mut index_writer: IndexWriter =
-                index.writer_with_num_threads(1, 100_000_000).unwrap();
-            for doc_json in &lines {
-                let json_val: serde_json::Map<String, serde_json::Value> =
-                    serde_json::from_str(doc_json).unwrap();
-                let doc = tantivy::doc!(json_field=>json_val);
-                index_writer.add_document(doc).unwrap();
-            }
-            index_writer.commit().unwrap();
-        })
-    });
+        }
+    }
 }
 
 pub fn gh_index_benchmark(c: &mut Criterion) {
@@ -107,39 +161,24 @@ pub fn gh_index_benchmark(c: &mut Criterion) {
         schema_builder.add_json_field("json", TEXT | FAST);
         schema_builder.build()
     };
+    let dynamic_schema_fast = {
+        let mut schema_builder = tantivy::schema::SchemaBuilder::new();
+        schema_builder.add_json_field("json", FAST);
+        schema_builder.build()
+    };
 
     let mut group = c.benchmark_group("index-gh");
     group.throughput(Throughput::Bytes(GH_LOGS.len() as u64));
 
     group.bench_function("index-gh-no-commit", |b| {
-        let lines = get_lines(GH_LOGS);
-        b.iter(|| {
-            let json_field = dynamic_schema.get_field("json").unwrap();
-            let index = Index::create_in_ram(dynamic_schema.clone());
-            let index_writer: IndexWriter = index.writer_with_num_threads(1, 100_000_000).unwrap();
-            for doc_json in &lines {
-                let json_val: serde_json::Map<String, serde_json::Value> =
-                    serde_json::from_str(doc_json).unwrap();
-                let doc = tantivy::doc!(json_field=>json_val);
-                index_writer.add_document(doc).unwrap();
-            }
-        })
+        benchmark_dynamic_json(b, GH_LOGS, dynamic_schema.clone(), false, false)
     });
-    group.bench_function("index-gh-with-commit", |b| {
-        let lines = get_lines(GH_LOGS);
-        b.iter(|| {
-            let json_field = dynamic_schema.get_field("json").unwrap();
-            let index = Index::create_in_ram(dynamic_schema.clone());
-            let mut index_writer: IndexWriter =
-                index.writer_with_num_threads(1, 100_000_000).unwrap();
-            for doc_json in &lines {
-                let json_val: serde_json::Map<String, serde_json::Value> =
-                    serde_json::from_str(doc_json).unwrap();
-                let doc = tantivy::doc!(json_field=>json_val);
-                index_writer.add_document(doc).unwrap();
-            }
-            index_writer.commit().unwrap();
-        })
+    group.bench_function("index-gh-fast", |b| {
+        benchmark_dynamic_json(b, GH_LOGS, dynamic_schema_fast.clone(), false, false)
+    });
+
+    group.bench_function("index-gh-fast-with-commit", |b| {
+        benchmark_dynamic_json(b, GH_LOGS, dynamic_schema_fast.clone(), true, false)
     });
 }
 
@@ -154,34 +193,10 @@ pub fn wiki_index_benchmark(c: &mut Criterion) {
     group.throughput(Throughput::Bytes(WIKI.len() as u64));
 
     group.bench_function("index-wiki-no-commit", |b| {
-        let lines = get_lines(WIKI);
-        b.iter(|| {
-            let json_field = dynamic_schema.get_field("json").unwrap();
-            let index = Index::create_in_ram(dynamic_schema.clone());
-            let index_writer: IndexWriter = index.writer_with_num_threads(1, 100_000_000).unwrap();
-            for doc_json in &lines {
-                let json_val: serde_json::Map<String, serde_json::Value> =
-                    serde_json::from_str(doc_json).unwrap();
-                let doc = tantivy::doc!(json_field=>json_val);
-                index_writer.add_document(doc).unwrap();
-            }
-        })
+        benchmark_dynamic_json(b, WIKI, dynamic_schema.clone(), false, false)
     });
     group.bench_function("index-wiki-with-commit", |b| {
-        let lines = get_lines(WIKI);
-        b.iter(|| {
-            let json_field = dynamic_schema.get_field("json").unwrap();
-            let index = Index::create_in_ram(dynamic_schema.clone());
-            let mut index_writer: IndexWriter =
-                index.writer_with_num_threads(1, 100_000_000).unwrap();
-            for doc_json in &lines {
-                let json_val: serde_json::Map<String, serde_json::Value> =
-                    serde_json::from_str(doc_json).unwrap();
-                let doc = tantivy::doc!(json_field=>json_val);
-                index_writer.add_document(doc).unwrap();
-            }
-            index_writer.commit().unwrap();
-        })
+        benchmark_dynamic_json(b, WIKI, dynamic_schema.clone(), true, false)
     });
 }
 
@@ -192,12 +207,12 @@ criterion_group! {
 }
 criterion_group! {
     name = gh_benches;
-    config = Criterion::default().with_profiler(PProfProfiler::new(100, Output::Flamegraph(None)));
+    config = Criterion::default();
     targets = gh_index_benchmark
 }
 criterion_group! {
     name = wiki_benches;
-    config = Criterion::default().with_profiler(PProfProfiler::new(100, Output::Flamegraph(None)));
+    config = Criterion::default();
     targets = wiki_index_benchmark
 }
 criterion_main!(benches, gh_benches, wiki_benches);

bitpacker/Cargo.toml

@@ -15,7 +15,7 @@ homepage = "https://github.com/quickwit-oss/tantivy"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-bitpacking = {version="0.8", default-features=false, features = ["bitpacker1x"]}
+bitpacking = { version = "0.9.2", default-features = false, features = ["bitpacker1x"] }
 
 [dev-dependencies]
 rand = "0.8"

bitpacker/src/bitpacker.rs

@@ -125,6 +125,8 @@ impl BitUnpacker {
 
     // Decodes the range of bitpacked `u32` values with idx
     // in [start_idx, start_idx + output.len()).
+    // It is guaranteed to completely fill `output` and not read from it, so passing a vector with
+    // un-initialized values is safe.
     //
     // #Panics
     //
@@ -237,7 +239,19 @@ impl BitUnpacker {
         data: &[u8],
         positions: &mut Vec<u32>,
     ) {
-        positions.resize(id_range.len(), 0u32);
+        // We use the code below instead of positions.resize(id_range.len(), 0u32) for performance
+        // reasons: on some queries, the CPU cost of memsetting the array and of using a bigger
+        // vector than necessary is noticeable (~5%).
+        // In particular, searches are a few percent faster when using reserve_exact() as below
+        // instead of reserve().
+        // The un-initialized values are safe as get_batch_u32s() completely fills `positions`
+        // and does not read from it.
+        positions.clear();
+        positions.reserve_exact(id_range.len());
+        #[allow(clippy::uninit_vec)]
+        unsafe {
+            positions.set_len(id_range.len());
+        }
         self.get_batch_u32s(id_range.start, data, positions);
         crate::filter_vec::filter_vec_in_place(value_range, id_range.start, positions)
     }

columnar/Cargo.toml

@@ -9,8 +9,7 @@ description = "column oriented storage for tantivy"
 categories = ["database-implementations", "data-structures", "compression"]
 
 [dependencies]
-itertools = "0.11.0"
-fnv = "1.0.7"
+itertools = "0.12.0"
 fastdivide = "0.4.0"
 
 stacker = { version= "0.2", path = "../stacker", package="tantivy-stacker"}

columnar/columnar-cli/Cargo.toml

@@ -8,7 +8,6 @@ license = "MIT"
 columnar = {path="../", package="tantivy-columnar"}
 serde_json = "1"
 serde_json_borrow = {git="https://github.com/PSeitz/serde_json_borrow/"}
-serde = "1"
 
 [workspace]
 members = []

columnar/src/column_index/merge/stacked.rs

@@ -111,10 +111,7 @@ fn stack_multivalued_indexes<'a>(
     let mut last_row_id = 0;
     let mut current_it = multivalued_indexes.next();
     Box::new(std::iter::from_fn(move || loop {
-        let Some(multivalued_index) = current_it.as_mut() else {
-            return None;
-        };
-        if let Some(row_id) = multivalued_index.next() {
+        if let Some(row_id) = current_it.as_mut()?.next() {
             last_row_id = offset + row_id;
             return Some(last_row_id);
         }

columnar/src/column_index/mod.rs

@@ -126,18 +126,18 @@ impl ColumnIndex {
         }
     }
 
-    pub fn docid_range_to_rowids(&self, doc_id: Range<DocId>) -> Range<RowId> {
+    pub fn docid_range_to_rowids(&self, doc_id_range: Range<DocId>) -> Range<RowId> {
         match self {
             ColumnIndex::Empty { .. } => 0..0,
-            ColumnIndex::Full => doc_id,
+            ColumnIndex::Full => doc_id_range,
             ColumnIndex::Optional(optional_index) => {
-                let row_start = optional_index.rank(doc_id.start);
-                let row_end = optional_index.rank(doc_id.end);
+                let row_start = optional_index.rank(doc_id_range.start);
+                let row_end = optional_index.rank(doc_id_range.end);
                 row_start..row_end
             }
             ColumnIndex::Multivalued(multivalued_index) => {
-                let end_docid = doc_id.end.min(multivalued_index.num_docs() - 1) + 1;
-                let start_docid = doc_id.start.min(end_docid);
+                let end_docid = doc_id_range.end.min(multivalued_index.num_docs() - 1) + 1;
+                let start_docid = doc_id_range.start.min(end_docid);
 
                 let row_start = multivalued_index.start_index_column.get_val(start_docid);
                 let row_end = multivalued_index.start_index_column.get_val(end_docid);

columnar/src/column_index/optional_index/mod.rs

@@ -21,8 +21,6 @@ const DENSE_BLOCK_THRESHOLD: u32 =
 
 const ELEMENTS_PER_BLOCK: u32 = u16::MAX as u32 + 1;
 
-const BLOCK_SIZE: RowId = 1 << 16;
-
 #[derive(Copy, Clone, Debug)]
 struct BlockMeta {
     non_null_rows_before_block: u32,
@@ -109,8 +107,8 @@ struct RowAddr {
 #[inline(always)]
 fn row_addr_from_row_id(row_id: RowId) -> RowAddr {
     RowAddr {
-        block_id: (row_id / BLOCK_SIZE) as u16,
-        in_block_row_id: (row_id % BLOCK_SIZE) as u16,
+        block_id: (row_id / ELEMENTS_PER_BLOCK) as u16,
+        in_block_row_id: (row_id % ELEMENTS_PER_BLOCK) as u16,
     }
 }
 
@@ -185,8 +183,13 @@ impl Set<RowId> for OptionalIndex {
         }
     }
 
+    /// Any value doc_id is allowed.
+    /// In particular, doc_id = num_rows.
     #[inline]
     fn rank(&self, doc_id: DocId) -> RowId {
+        if doc_id >= self.num_docs() {
+            return self.num_non_nulls();
+        }
         let RowAddr {
             block_id,
             in_block_row_id,
@@ -200,13 +203,15 @@ impl Set<RowId> for OptionalIndex {
         block_meta.non_null_rows_before_block + block_offset_row_id
     }
 
+    /// Any value doc_id is allowed.
+    /// In particular, doc_id = num_rows.
     #[inline]
     fn rank_if_exists(&self, doc_id: DocId) -> Option<RowId> {
         let RowAddr {
             block_id,
             in_block_row_id,
         } = row_addr_from_row_id(doc_id);
-        let block_meta = self.block_metas[block_id as usize];
+        let block_meta = *self.block_metas.get(block_id as usize)?;
         let block = self.block(block_meta);
         let block_offset_row_id = match block {
             Block::Dense(dense_block) => dense_block.rank_if_exists(in_block_row_id),
@@ -491,7 +496,7 @@ fn deserialize_optional_index_block_metadatas(
         non_null_rows_before_block += num_non_null_rows;
     }
     block_metas.resize(
-        ((num_rows + BLOCK_SIZE - 1) / BLOCK_SIZE) as usize,
+        ((num_rows + ELEMENTS_PER_BLOCK - 1) / ELEMENTS_PER_BLOCK) as usize,
         BlockMeta {
             non_null_rows_before_block,
             start_byte_offset,

columnar/src/column_index/optional_index/set.rs

@@ -39,7 +39,8 @@ pub trait Set<T> {
     ///
     /// # Panics
     ///
-    /// May panic if rank is greater than the number of elements in the Set.
+    /// May panic if rank is greater or equal to the number of
+    /// elements in the Set.
     fn select(&self, rank: T) -> T;
 
     /// Creates a brand new select cursor.

columnar/src/column_index/optional_index/tests.rs

@@ -3,6 +3,30 @@ use proptest::strategy::Strategy;
 use proptest::{prop_oneof, proptest};
 
 use super::*;
+use crate::{ColumnarReader, ColumnarWriter, DynamicColumnHandle};
+
+#[test]
+fn test_optional_index_bug_2293() {
+    // tests for panic in docid_range_to_rowids for docid == num_docs
+    test_optional_index_with_num_docs(ELEMENTS_PER_BLOCK - 1);
+    test_optional_index_with_num_docs(ELEMENTS_PER_BLOCK);
+    test_optional_index_with_num_docs(ELEMENTS_PER_BLOCK + 1);
+}
+fn test_optional_index_with_num_docs(num_docs: u32) {
+    let mut dataframe_writer = ColumnarWriter::default();
+    dataframe_writer.record_numerical(100, "score", 80i64);
+    let mut buffer: Vec<u8> = Vec::new();
+    dataframe_writer
+        .serialize(num_docs, None, &mut buffer)
+        .unwrap();
+    let columnar = ColumnarReader::open(buffer).unwrap();
+    assert_eq!(columnar.num_columns(), 1);
+    let cols: Vec<DynamicColumnHandle> = columnar.read_columns("score").unwrap();
+    assert_eq!(cols.len(), 1);
+
+    let col = cols[0].open().unwrap();
+    col.column_index().docid_range_to_rowids(0..num_docs);
+}
 
 #[test]
 fn test_dense_block_threshold() {
@@ -35,7 +59,7 @@ proptest! {
 
 #[test]
 fn test_with_random_sets_simple() {
-    let vals = 10..BLOCK_SIZE * 2;
+    let vals = 10..ELEMENTS_PER_BLOCK * 2;
     let mut out: Vec<u8> = Vec::new();
     serialize_optional_index(&vals, 100, &mut out).unwrap();
     let null_index = open_optional_index(OwnedBytes::new(out)).unwrap();
@@ -171,7 +195,7 @@ fn test_optional_index_rank() {
     test_optional_index_rank_aux(&[0u32, 1u32]);
     let mut block = Vec::new();
     block.push(3u32);
-    block.extend((0..BLOCK_SIZE).map(|i| i + BLOCK_SIZE + 1));
+    block.extend((0..ELEMENTS_PER_BLOCK).map(|i| i + ELEMENTS_PER_BLOCK + 1));
     test_optional_index_rank_aux(&block);
 }
 
@@ -185,8 +209,8 @@ fn test_optional_index_iter_empty_one() {
 fn test_optional_index_iter_dense_block() {
     let mut block = Vec::new();
     block.push(3u32);
-    block.extend((0..BLOCK_SIZE).map(|i| i + BLOCK_SIZE + 1));
-    test_optional_index_iter_aux(&block, 3 * BLOCK_SIZE);
+    block.extend((0..ELEMENTS_PER_BLOCK).map(|i| i + ELEMENTS_PER_BLOCK + 1));
+    test_optional_index_iter_aux(&block, 3 * ELEMENTS_PER_BLOCK);
 }
 
 #[test]

columnar/src/column_values/mod.rs

@@ -101,7 +101,7 @@ pub trait ColumnValues<T: PartialOrd = u64>: Send + Sync {
         row_id_hits: &mut Vec<RowId>,
     ) {
         let row_id_range = row_id_range.start..row_id_range.end.min(self.num_vals());
-        for idx in row_id_range.start..row_id_range.end {
+        for idx in row_id_range {
             let val = self.get_val(idx);
             if value_range.contains(&val) {
                 row_id_hits.push(idx);

columnar/src/columnar/writer/column_writers.rs

@@ -269,7 +269,8 @@ impl StrOrBytesColumnWriter {
         dictionaries: &mut [DictionaryBuilder],
         arena: &mut MemoryArena,
     ) {
-        let unordered_id = dictionaries[self.dictionary_id as usize].get_or_allocate_id(bytes);
+        let unordered_id =
+            dictionaries[self.dictionary_id as usize].get_or_allocate_id(bytes, arena);
         self.column_writer.record(doc, unordered_id, arena);
     }
 

columnar/src/columnar/writer/mod.rs

@@ -437,6 +437,7 @@ impl ColumnarWriter {
                             &mut symbol_byte_buffer,
                         ),
                         buffers,
+                        &self.arena,
                         &mut column_serializer,
                     )?;
                     column_serializer.finalize()?;
@@ -490,6 +491,7 @@ impl ColumnarWriter {
 
 // Serialize [Dictionary, Column, dictionary num bytes U32::LE]
 // Column: [Column Index, Column Values, column index num bytes U32::LE]
+#[allow(clippy::too_many_arguments)]
 fn serialize_bytes_or_str_column(
     cardinality: Cardinality,
     num_docs: RowId,
@@ -497,6 +499,7 @@ fn serialize_bytes_or_str_column(
     dictionary_builder: &DictionaryBuilder,
     operation_it: impl Iterator<Item = ColumnOperation<UnorderedId>>,
     buffers: &mut SpareBuffers,
+    arena: &MemoryArena,
     wrt: impl io::Write,
 ) -> io::Result<()> {
     let SpareBuffers {
@@ -505,7 +508,8 @@ fn serialize_bytes_or_str_column(
         ..
     } = buffers;
     let mut counting_writer = CountingWriter::wrap(wrt);
-    let term_id_mapping: TermIdMapping = dictionary_builder.serialize(&mut counting_writer)?;
+    let term_id_mapping: TermIdMapping =
+        dictionary_builder.serialize(arena, &mut counting_writer)?;
     let dictionary_num_bytes: u32 = counting_writer.written_bytes() as u32;
     let mut wrt = counting_writer.finish();
     let operation_iterator = operation_it.map(|symbol: ColumnOperation<UnorderedId>| {

columnar/src/dictionary.rs

@@ -1,7 +1,7 @@
 use std::io;
 
-use fnv::FnvHashMap;
 use sstable::SSTable;
+use stacker::{MemoryArena, SharedArenaHashMap};
 
 pub(crate) struct TermIdMapping {
     unordered_to_ord: Vec<OrderedId>,
@@ -31,29 +31,38 @@ pub struct OrderedId(pub u32);
 /// mapping.
 #[derive(Default)]
 pub(crate) struct DictionaryBuilder {
-    dict: FnvHashMap<Vec<u8>, UnorderedId>,
-    memory_consumption: usize,
+    dict: SharedArenaHashMap,
 }
 
 impl DictionaryBuilder {
     /// Get or allocate an unordered id.
     /// (This ID is simply an auto-incremented id.)
-    pub fn get_or_allocate_id(&mut self, term: &[u8]) -> UnorderedId {
-        if let Some(term_id) = self.dict.get(term) {
-            return *term_id;
-        }
-        let new_id = UnorderedId(self.dict.len() as u32);
-        self.dict.insert(term.to_vec(), new_id);
-        self.memory_consumption += term.len();
-        self.memory_consumption += 40; // Term Metadata + HashMap overhead
-        new_id
+    pub fn get_or_allocate_id(&mut self, term: &[u8], arena: &mut MemoryArena) -> UnorderedId {
+        let next_id = self.dict.len() as u32;
+        let unordered_id = self
+            .dict
+            .mutate_or_create(term, arena, |unordered_id: Option<u32>| {
+                if let Some(unordered_id) = unordered_id {
+                    unordered_id
+                } else {
+                    next_id
+                }
+            });
+        UnorderedId(unordered_id)
     }
 
     /// Serialize the dictionary into an fst, and returns the
     /// `UnorderedId -> TermOrdinal` map.
-    pub fn serialize<'a, W: io::Write + 'a>(&self, wrt: &mut W) -> io::Result<TermIdMapping> {
-        let mut terms: Vec<(&[u8], UnorderedId)> =
-            self.dict.iter().map(|(k, v)| (k.as_slice(), *v)).collect();
+    pub fn serialize<'a, W: io::Write + 'a>(
+        &self,
+        arena: &MemoryArena,
+        wrt: &mut W,
+    ) -> io::Result<TermIdMapping> {
+        let mut terms: Vec<(&[u8], UnorderedId)> = self
+            .dict
+            .iter(arena)
+            .map(|(k, v)| (k, arena.read(v)))
+            .collect();
         terms.sort_unstable_by_key(|(key, _)| *key);
         // TODO Remove the allocation.
         let mut unordered_to_ord: Vec<OrderedId> = vec![OrderedId(0u32); terms.len()];
@@ -68,7 +77,7 @@ impl DictionaryBuilder {
     }
 
     pub(crate) fn mem_usage(&self) -> usize {
-        self.memory_consumption
+        self.dict.mem_usage()
     }
 }
 
@@ -78,12 +87,13 @@ mod tests {
 
     #[test]
     fn test_dictionary_builder() {
+        let mut arena = MemoryArena::default();
         let mut dictionary_builder = DictionaryBuilder::default();
-        let hello_uid = dictionary_builder.get_or_allocate_id(b"hello");
-        let happy_uid = dictionary_builder.get_or_allocate_id(b"happy");
-        let tax_uid = dictionary_builder.get_or_allocate_id(b"tax");
+        let hello_uid = dictionary_builder.get_or_allocate_id(b"hello", &mut arena);
+        let happy_uid = dictionary_builder.get_or_allocate_id(b"happy", &mut arena);
+        let tax_uid = dictionary_builder.get_or_allocate_id(b"tax", &mut arena);
         let mut buffer = Vec::new();
-        let id_mapping = dictionary_builder.serialize(&mut buffer).unwrap();
+        let id_mapping = dictionary_builder.serialize(&arena, &mut buffer).unwrap();
         assert_eq!(id_mapping.to_ord(hello_uid), OrderedId(1));
         assert_eq!(id_mapping.to_ord(happy_uid), OrderedId(0));
         assert_eq!(id_mapping.to_ord(tax_uid), OrderedId(2));

columnar/src/lib.rs

@@ -1,3 +1,22 @@
+//! # Tantivy-Columnar
+//!
+//! `tantivy-columnar`provides a columnar storage for tantivy.
+//! The crate allows for efficient read operations on specific columns rather than entire records.
+//!
+//! ## Overview
+//!
+//! - **columnar**: Reading, writing, and merging multiple columns:
+//!   - **[ColumnarWriter]**: Makes it possible to create a new columnar.
+//!   - **[ColumnarReader]**: The ColumnarReader makes it possible to access a set of columns
+//!     associated to field names.
+//!   - **[merge_columnar]**: Contains the functionalities to merge multiple ColumnarReader or
+//!     segments into a single one.
+//!
+//! - **column**: A single column, which contains
+//!     - [column_index]: Resolves the rows for a document id. Manages the cardinality of the
+//!       column.
+//!     - [column_values]: Stores the values of a column in a dense format.
+
 #![cfg_attr(all(feature = "unstable", test), feature(test))]
 
 #[cfg(test)]

columnar/src/tests.rs

@@ -26,7 +26,7 @@ fn test_dataframe_writer_str() {
     assert_eq!(columnar.num_columns(), 1);
     let cols: Vec<DynamicColumnHandle> = columnar.read_columns("my_string").unwrap();
     assert_eq!(cols.len(), 1);
-    assert_eq!(cols[0].num_bytes(), 87);
+    assert_eq!(cols[0].num_bytes(), 73);
 }
 
 #[test]
@@ -40,7 +40,7 @@ fn test_dataframe_writer_bytes() {
     assert_eq!(columnar.num_columns(), 1);
     let cols: Vec<DynamicColumnHandle> = columnar.read_columns("my_string").unwrap();
     assert_eq!(cols.len(), 1);
-    assert_eq!(cols[0].num_bytes(), 87);
+    assert_eq!(cols[0].num_bytes(), 73);
 }
 
 #[test]

query-grammar/src/infallible.rs

@@ -81,8 +81,8 @@ where
     T: InputTakeAtPosition + Clone,
     <T as InputTakeAtPosition>::Item: AsChar + Clone,
 {
-    opt_i(nom::character::complete::space0)(input)
-        .map(|(left, (spaces, errors))| (left, (spaces.expect("space0 can't fail"), errors)))
+    opt_i(nom::character::complete::multispace0)(input)
+        .map(|(left, (spaces, errors))| (left, (spaces.expect("multispace0 can't fail"), errors)))
 }
 
 pub(crate) fn space1_infallible<T>(input: T) -> JResult<T, Option<T>>
@@ -90,7 +90,7 @@ where
     T: InputTakeAtPosition + Clone + InputLength,
     <T as InputTakeAtPosition>::Item: AsChar + Clone,
 {
-    opt_i(nom::character::complete::space1)(input).map(|(left, (spaces, mut errors))| {
+    opt_i(nom::character::complete::multispace1)(input).map(|(left, (spaces, mut errors))| {
         if spaces.is_none() {
             errors.push(LenientErrorInternal {
                 pos: left.input_len(),

query-grammar/src/query_grammar.rs

@@ -3,11 +3,11 @@ use std::iter::once;
 use nom::branch::alt;
 use nom::bytes::complete::tag;
 use nom::character::complete::{
-    anychar, char, digit1, none_of, one_of, satisfy, space0, space1, u32,
+    anychar, char, digit1, multispace0, multispace1, none_of, one_of, satisfy, u32,
 };
 use nom::combinator::{eof, map, map_res, opt, peek, recognize, value, verify};
 use nom::error::{Error, ErrorKind};
-use nom::multi::{many0, many1, separated_list0, separated_list1};
+use nom::multi::{many0, many1, separated_list0};
 use nom::sequence::{delimited, preceded, separated_pair, terminated, tuple};
 use nom::IResult;
 
@@ -65,7 +65,7 @@ fn word_infallible(delimiter: &str) -> impl Fn(&str) -> JResult<&str, Option<&st
     |inp| {
         opt_i_err(
             preceded(
-                space0,
+                multispace0,
                 recognize(many1(satisfy(|c| {
                     !c.is_whitespace() && !delimiter.contains(c)
                 }))),
@@ -225,10 +225,10 @@ fn term_group(inp: &str) -> IResult<&str, UserInputAst> {
 
     map(
         tuple((
-            terminated(field_name, space0),
+            terminated(field_name, multispace0),
             delimited(
-                tuple((char('('), space0)),
-                separated_list0(space1, tuple((opt(occur_symbol), term_or_phrase))),
+                tuple((char('('), multispace0)),
+                separated_list0(multispace1, tuple((opt(occur_symbol), term_or_phrase))),
                 char(')'),
             ),
         )),
@@ -250,7 +250,7 @@ fn term_group_precond(inp: &str) -> IResult<&str, (), ()> {
         (),
         peek(tuple((
             field_name,
-            space0,
+            multispace0,
             char('('), // when we are here, we know it can't be anything but a term group
         ))),
     )(inp)
@@ -259,7 +259,7 @@ fn term_group_precond(inp: &str) -> IResult<&str, (), ()> {
 
 fn term_group_infallible(inp: &str) -> JResult<&str, UserInputAst> {
     let (mut inp, (field_name, _, _, _)) =
-        tuple((field_name, space0, char('('), space0))(inp).expect("precondition failed");
+        tuple((field_name, multispace0, char('('), multispace0))(inp).expect("precondition failed");
 
     let mut terms = Vec::new();
     let mut errs = Vec::new();
@@ -305,7 +305,7 @@ fn exists(inp: &str) -> IResult<&str, UserInputLeaf> {
         UserInputLeaf::Exists {
             field: String::new(),
         },
-        tuple((space0, char('*'))),
+        tuple((multispace0, char('*'))),
     )(inp)
 }
 
@@ -314,7 +314,7 @@ fn exists_precond(inp: &str) -> IResult<&str, (), ()> {
         (),
         peek(tuple((
             field_name,
-            space0,
+            multispace0,
             char('*'), // when we are here, we know it can't be anything but a exists
         ))),
     )(inp)
@@ -323,7 +323,7 @@ fn exists_precond(inp: &str) -> IResult<&str, (), ()> {
 
 fn exists_infallible(inp: &str) -> JResult<&str, UserInputAst> {
     let (inp, (field_name, _, _)) =
-        tuple((field_name, space0, char('*')))(inp).expect("precondition failed");
+        tuple((field_name, multispace0, char('*')))(inp).expect("precondition failed");
 
     let exists = UserInputLeaf::Exists { field: field_name }.into();
     Ok((inp, (exists, Vec::new())))
@@ -349,7 +349,7 @@ fn literal_no_group_infallible(inp: &str) -> JResult<&str, Option<UserInputAst>>
             alt_infallible(
                 (
                     (
-                        value((), tuple((tag("IN"), space0, char('[')))),
+                        value((), tuple((tag("IN"), multispace0, char('[')))),
                         map(set_infallible, |(set, errs)| (Some(set), errs)),
                     ),
                     (
@@ -430,8 +430,8 @@ fn range(inp: &str) -> IResult<&str, UserInputLeaf> {
     // check for unbounded range in the form of <5, <=10, >5, >=5
     let elastic_unbounded_range = map(
         tuple((
-            preceded(space0, alt((tag(">="), tag("<="), tag("<"), tag(">")))),
-            preceded(space0, range_term_val()),
+            preceded(multispace0, alt((tag(">="), tag("<="), tag("<"), tag(">")))),
+            preceded(multispace0, range_term_val()),
         )),
         |(comparison_sign, bound)| match comparison_sign {
             ">=" => (UserInputBound::Inclusive(bound), UserInputBound::Unbounded),
@@ -444,7 +444,7 @@ fn range(inp: &str) -> IResult<&str, UserInputLeaf> {
     );
 
     let lower_bound = map(
-        separated_pair(one_of("{["), space0, range_term_val()),
+        separated_pair(one_of("{["), multispace0, range_term_val()),
         |(boundary_char, lower_bound)| {
             if lower_bound == "*" {
                 UserInputBound::Unbounded
@@ -457,7 +457,7 @@ fn range(inp: &str) -> IResult<&str, UserInputLeaf> {
     );
 
     let upper_bound = map(
-        separated_pair(range_term_val(), space0, one_of("}]")),
+        separated_pair(range_term_val(), multispace0, one_of("}]")),
         |(upper_bound, boundary_char)| {
             if upper_bound == "*" {
                 UserInputBound::Unbounded
@@ -469,8 +469,11 @@ fn range(inp: &str) -> IResult<&str, UserInputLeaf> {
         },
     );
 
-    let lower_to_upper =
-        separated_pair(lower_bound, tuple((space1, tag("TO"), space1)), upper_bound);
+    let lower_to_upper = separated_pair(
+        lower_bound,
+        tuple((multispace1, tag("TO"), multispace1)),
+        upper_bound,
+    );
 
     map(
         alt((elastic_unbounded_range, lower_to_upper)),
@@ -490,13 +493,16 @@ fn range_infallible(inp: &str) -> JResult<&str, UserInputLeaf> {
             word_infallible("]}"),
             space1_infallible,
             opt_i_err(
-                terminated(tag("TO"), alt((value((), space1), value((), eof)))),
+                terminated(tag("TO"), alt((value((), multispace1), value((), eof)))),
                 "missing keyword TO",
             ),
             word_infallible("]}"),
             opt_i_err(one_of("]}"), "missing range delimiter"),
         )),
-        |((lower_bound_kind, _space0, lower, _space1, to, upper, upper_bound_kind), errs)| {
+        |(
+            (lower_bound_kind, _multispace0, lower, _multispace1, to, upper, upper_bound_kind),
+            errs,
+        )| {
             let lower_bound = match (lower_bound_kind, lower) {
                 (_, Some("*")) => UserInputBound::Unbounded,
                 (_, None) => UserInputBound::Unbounded,
@@ -596,10 +602,10 @@ fn range_infallible(inp: &str) -> JResult<&str, UserInputLeaf> {
 fn set(inp: &str) -> IResult<&str, UserInputLeaf> {
     map(
         preceded(
-            tuple((space0, tag("IN"), space1)),
+            tuple((multispace0, tag("IN"), multispace1)),
             delimited(
-                tuple((char('['), space0)),
-                separated_list0(space1, map(simple_term, |(_, term)| term)),
+                tuple((char('['), multispace0)),
+                separated_list0(multispace1, map(simple_term, |(_, term)| term)),
                 char(']'),
             ),
         ),
@@ -667,7 +673,7 @@ fn leaf(inp: &str) -> IResult<&str, UserInputAst> {
     alt((
         delimited(char('('), ast, char(')')),
         map(char('*'), |_| UserInputAst::from(UserInputLeaf::All)),
-        map(preceded(tuple((tag("NOT"), space1)), leaf), negate),
+        map(preceded(tuple((tag("NOT"), multispace1)), leaf), negate),
         literal,
     ))(inp)
 }
@@ -780,27 +786,23 @@ fn binary_operand(inp: &str) -> IResult<&str, BinaryOperand> {
 }
 
 fn aggregate_binary_expressions(
-    left: UserInputAst,
-    others: Vec<(BinaryOperand, UserInputAst)>,
-) -> UserInputAst {
-    let mut dnf: Vec<Vec<UserInputAst>> = vec![vec![left]];
-    for (operator, operand_ast) in others {
-        match operator {
-            BinaryOperand::And => {
-                if let Some(last) = dnf.last_mut() {
-                    last.push(operand_ast);
-                }
-            }
-            BinaryOperand::Or => {
-                dnf.push(vec![operand_ast]);
-            }
-        }
-    }
-    if dnf.len() == 1 {
-        UserInputAst::and(dnf.into_iter().next().unwrap()) //< safe
+    left: (Option<Occur>, UserInputAst),
+    others: Vec<(Option<BinaryOperand>, Option<Occur>, UserInputAst)>,
+) -> Result<UserInputAst, LenientErrorInternal> {
+    let mut leafs = Vec::with_capacity(others.len() + 1);
+    leafs.push((None, left.0, Some(left.1)));
+    leafs.extend(
+        others
+            .into_iter()
+            .map(|(operand, occur, ast)| (operand, occur, Some(ast))),
+    );
+    // the parameters we pass should statically guarantee we can't get errors
+    // (no prefix BinaryOperand is provided)
+    let (res, mut errors) = aggregate_infallible_expressions(leafs);
+    if errors.is_empty() {
+        Ok(res)
     } else {
-        let conjunctions = dnf.into_iter().map(UserInputAst::and).collect();
-        UserInputAst::or(conjunctions)
+        Err(errors.swap_remove(0))
     }
 }
 
@@ -816,30 +818,10 @@ fn aggregate_infallible_expressions(
         return (UserInputAst::empty_query(), err);
     }
 
-    let use_operand = leafs.iter().any(|(operand, _, _)| operand.is_some());
-    let all_operand = leafs
-        .iter()
-        .skip(1)
-        .all(|(operand, _, _)| operand.is_some());
     let early_operand = leafs
         .iter()
         .take(1)
         .all(|(operand, _, _)| operand.is_some());
-    let use_occur = leafs.iter().any(|(_, occur, _)| occur.is_some());
-
-    if use_operand && use_occur {
-        err.push(LenientErrorInternal {
-            pos: 0,
-            message: "Use of mixed occur and boolean operator".to_string(),
-        });
-    }
-
-    if use_operand && !all_operand {
-        err.push(LenientErrorInternal {
-            pos: 0,
-            message: "Missing boolean operator".to_string(),
-        });
-    }
 
     if early_operand {
         err.push(LenientErrorInternal {
@@ -866,7 +848,15 @@ fn aggregate_infallible_expressions(
                     Some(BinaryOperand::And) => Some(Occur::Must),
                     _ => Some(Occur::Should),
                 };
-                clauses.push(vec![(occur.or(default_op), ast.clone())]);
+                if occur == &Some(Occur::MustNot) && default_op == Some(Occur::Should) {
+                    // if occur is MustNot *and* operation is OR, we synthetize a ShouldNot
+                    clauses.push(vec![(
+                        Some(Occur::Should),
+                        ast.clone().unary(Occur::MustNot),
+                    )])
+                } else {
+                    clauses.push(vec![(occur.or(default_op), ast.clone())]);
+                }
             }
             None => {
                 let default_op = match next_operator {
@@ -874,7 +864,15 @@ fn aggregate_infallible_expressions(
                     Some(BinaryOperand::Or) => Some(Occur::Should),
                     None => None,
                 };
-                clauses.push(vec![(occur.or(default_op), ast.clone())])
+                if occur == &Some(Occur::MustNot) && default_op == Some(Occur::Should) {
+                    // if occur is MustNot *and* operation is OR, we synthetize a ShouldNot
+                    clauses.push(vec![(
+                        Some(Occur::Should),
+                        ast.clone().unary(Occur::MustNot),
+                    )])
+                } else {
+                    clauses.push(vec![(occur.or(default_op), ast.clone())])
+                }
             }
         }
     }
@@ -891,7 +889,12 @@ fn aggregate_infallible_expressions(
             }
         }
         Some(BinaryOperand::Or) => {
-            clauses.push(vec![(last_occur.or(Some(Occur::Should)), last_ast)]);
+            if last_occur == Some(Occur::MustNot) {
+                // if occur is MustNot *and* operation is OR, we synthetize a ShouldNot
+                clauses.push(vec![(Some(Occur::Should), last_ast.unary(Occur::MustNot))]);
+            } else {
+                clauses.push(vec![(last_occur.or(Some(Occur::Should)), last_ast)]);
+            }
         }
         None => clauses.push(vec![(last_occur, last_ast)]),
     }
@@ -917,35 +920,29 @@ fn aggregate_infallible_expressions(
     }
 }
 
-fn operand_leaf(inp: &str) -> IResult<&str, (BinaryOperand, UserInputAst)> {
-    tuple((
-        terminated(binary_operand, space0),
-        terminated(boosted_leaf, space0),
-    ))(inp)
+fn operand_leaf(inp: &str) -> IResult<&str, (Option<BinaryOperand>, Option<Occur>, UserInputAst)> {
+    map(
+        tuple((
+            terminated(opt(binary_operand), multispace0),
+            terminated(occur_leaf, multispace0),
+        )),
+        |(operand, (occur, ast))| (operand, occur, ast),
+    )(inp)
 }
 
 fn ast(inp: &str) -> IResult<&str, UserInputAst> {
-    let boolean_expr = map(
-        separated_pair(boosted_leaf, space1, many1(operand_leaf)),
+    let boolean_expr = map_res(
+        separated_pair(occur_leaf, multispace1, many1(operand_leaf)),
         |(left, right)| aggregate_binary_expressions(left, right),
     );
-    let whitespace_separated_leaves = map(separated_list1(space1, occur_leaf), |subqueries| {
-        if subqueries.len() == 1 {
-            let (occur_opt, ast) = subqueries.into_iter().next().unwrap();
-            match occur_opt.unwrap_or(Occur::Should) {
-                Occur::Must | Occur::Should => ast,
-                Occur::MustNot => UserInputAst::Clause(vec![(Some(Occur::MustNot), ast)]),
-            }
+    let single_leaf = map(occur_leaf, |(occur, ast)| {
+        if occur == Some(Occur::MustNot) {
+            ast.unary(Occur::MustNot)
         } else {
-            UserInputAst::Clause(subqueries.into_iter().collect())
+            ast
         }
     });
-
-    delimited(
-        space0,
-        alt((boolean_expr, whitespace_separated_leaves)),
-        space0,
-    )(inp)
+    delimited(multispace0, alt((boolean_expr, single_leaf)), multispace0)(inp)
 }
 
 fn ast_infallible(inp: &str) -> JResult<&str, UserInputAst> {
@@ -969,7 +966,7 @@ fn ast_infallible(inp: &str) -> JResult<&str, UserInputAst> {
 }
 
 pub fn parse_to_ast(inp: &str) -> IResult<&str, UserInputAst> {
-    map(delimited(space0, opt(ast), eof), |opt_ast| {
+    map(delimited(multispace0, opt(ast), eof), |opt_ast| {
         rewrite_ast(opt_ast.unwrap_or_else(UserInputAst::empty_query))
     })(inp)
 }
@@ -1145,24 +1142,43 @@ mod test {
     #[test]
     fn test_parse_query_to_ast_binary_op() {
         test_parse_query_to_ast_helper("a AND b", "(+a +b)");
+        test_parse_query_to_ast_helper("a\nAND b", "(+a +b)");
         test_parse_query_to_ast_helper("a OR b", "(?a ?b)");
         test_parse_query_to_ast_helper("a OR b AND c", "(?a ?(+b +c))");
         test_parse_query_to_ast_helper("a AND b         AND c", "(+a +b +c)");
-        test_is_parse_err("a OR b aaa", "(?a ?b *aaa)");
-        test_is_parse_err("a AND b aaa", "(?(+a +b) *aaa)");
-        test_is_parse_err("aaa a OR b ", "(*aaa ?a ?b)");
-        test_is_parse_err("aaa ccc a OR b ", "(*aaa *ccc ?a ?b)");
-        test_is_parse_err("aaa a AND b ", "(*aaa ?(+a +b))");
-        test_is_parse_err("aaa ccc a AND b ", "(*aaa *ccc ?(+a +b))");
+        test_parse_query_to_ast_helper("a OR b aaa", "(?a ?b *aaa)");
+        test_parse_query_to_ast_helper("a AND b aaa", "(?(+a +b) *aaa)");
+        test_parse_query_to_ast_helper("aaa a OR b ", "(*aaa ?a ?b)");
+        test_parse_query_to_ast_helper("aaa ccc a OR b ", "(*aaa *ccc ?a ?b)");
+        test_parse_query_to_ast_helper("aaa a AND b ", "(*aaa ?(+a +b))");
+        test_parse_query_to_ast_helper("aaa ccc a AND b ", "(*aaa *ccc ?(+a +b))");
     }
 
     #[test]
     fn test_parse_mixed_bool_occur() {
-        test_is_parse_err("a OR b +aaa", "(?a ?b +aaa)");
-        test_is_parse_err("a AND b -aaa", "(?(+a +b) -aaa)");
-        test_is_parse_err("+a OR +b aaa", "(+a +b *aaa)");
-        test_is_parse_err("-a AND -b aaa", "(?(-a -b) *aaa)");
-        test_is_parse_err("-aaa +ccc -a OR b ", "(-aaa +ccc -a ?b)");
+        test_parse_query_to_ast_helper("+a OR +b", "(+a +b)");
+
+        test_parse_query_to_ast_helper("a AND -b", "(+a -b)");
+        test_parse_query_to_ast_helper("-a AND b", "(-a +b)");
+        test_parse_query_to_ast_helper("a AND NOT b", "(+a +(-b))");
+        test_parse_query_to_ast_helper("NOT a AND b", "(+(-a) +b)");
+
+        test_parse_query_to_ast_helper("a AND NOT b AND c", "(+a +(-b) +c)");
+        test_parse_query_to_ast_helper("a AND -b AND c", "(+a -b +c)");
+
+        test_parse_query_to_ast_helper("a OR -b", "(?a ?(-b))");
+        test_parse_query_to_ast_helper("-a OR b", "(?(-a) ?b)");
+        test_parse_query_to_ast_helper("a OR NOT b", "(?a ?(-b))");
+        test_parse_query_to_ast_helper("NOT a OR b", "(?(-a) ?b)");
+
+        test_parse_query_to_ast_helper("a OR NOT b OR c", "(?a ?(-b) ?c)");
+        test_parse_query_to_ast_helper("a OR -b OR c", "(?a ?(-b) ?c)");
+
+        test_parse_query_to_ast_helper("a OR b +aaa", "(?a ?b +aaa)");
+        test_parse_query_to_ast_helper("a AND b -aaa", "(?(+a +b) -aaa)");
+        test_parse_query_to_ast_helper("+a OR +b aaa", "(+a +b *aaa)");
+        test_parse_query_to_ast_helper("-a AND -b aaa", "(?(-a -b) *aaa)");
+        test_parse_query_to_ast_helper("-aaa +ccc -a OR b ", "(-aaa +ccc ?(-a) ?b)");
     }
 
     #[test]

src/aggregation/agg_bench.rs

@@ -290,6 +290,41 @@ mod bench {
         });
     }
 
+    bench_all_cardinalities!(bench_aggregation_terms_many_with_top_hits_agg);
+
+    fn bench_aggregation_terms_many_with_top_hits_agg_card(
+        b: &mut Bencher,
+        cardinality: Cardinality,
+    ) {
+        let index = get_test_index_bench(cardinality).unwrap();
+        let reader = index.reader().unwrap();
+
+        b.iter(|| {
+            let agg_req: Aggregations = serde_json::from_value(json!({
+                "my_texts": {
+                    "terms": { "field": "text_many_terms" },
+                    "aggs": {
+                        "top_hits": { "top_hits":
+                            {
+                                "sort": [
+                                    { "score": "desc" }
+                                ],
+                                "size": 2,
+                                "doc_value_fields": ["score_f64"]
+                            }
+                        }
+                    }
+                },
+            }))
+            .unwrap();
+
+            let collector = get_collector(agg_req);
+
+            let searcher = reader.searcher();
+            searcher.search(&AllQuery, &collector).unwrap()
+        });
+    }
+
     bench_all_cardinalities!(bench_aggregation_terms_many_with_sub_agg);
 
     fn bench_aggregation_terms_many_with_sub_agg_card(b: &mut Bencher, cardinality: Cardinality) {

src/aggregation/agg_req.rs

@@ -35,7 +35,7 @@ use super::bucket::{
 };
 use super::metric::{
     AverageAggregation, CountAggregation, MaxAggregation, MinAggregation,
-    PercentilesAggregationReq, StatsAggregation, SumAggregation,
+    PercentilesAggregationReq, StatsAggregation, SumAggregation, TopHitsAggregation,
 };
 
 /// The top-level aggregation request structure, which contains [`Aggregation`] and their user
@@ -93,7 +93,12 @@ impl Aggregation {
     }
 
     fn get_fast_field_names(&self, fast_field_names: &mut HashSet<String>) {
-        fast_field_names.insert(self.agg.get_fast_field_name().to_string());
+        fast_field_names.extend(
+            self.agg
+                .get_fast_field_names()
+                .iter()
+                .map(|s| s.to_string()),
+        );
         fast_field_names.extend(get_fast_field_names(&self.sub_aggregation));
     }
 }
@@ -147,23 +152,27 @@ pub enum AggregationVariants {
     /// Computes the sum of the extracted values.
     #[serde(rename = "percentiles")]
     Percentiles(PercentilesAggregationReq),
+    /// Finds the top k values matching some order
+    #[serde(rename = "top_hits")]
+    TopHits(TopHitsAggregation),
 }
 
 impl AggregationVariants {
-    /// Returns the name of the field used by the aggregation.
-    pub fn get_fast_field_name(&self) -> &str {
+    /// Returns the name of the fields used by the aggregation.
+    pub fn get_fast_field_names(&self) -> Vec<&str> {
         match self {
-            AggregationVariants::Terms(terms) => terms.field.as_str(),
-            AggregationVariants::Range(range) => range.field.as_str(),
-            AggregationVariants::Histogram(histogram) => histogram.field.as_str(),
-            AggregationVariants::DateHistogram(histogram) => histogram.field.as_str(),
-            AggregationVariants::Average(avg) => avg.field_name(),
-            AggregationVariants::Count(count) => count.field_name(),
-            AggregationVariants::Max(max) => max.field_name(),
-            AggregationVariants::Min(min) => min.field_name(),
-            AggregationVariants::Stats(stats) => stats.field_name(),
-            AggregationVariants::Sum(sum) => sum.field_name(),
-            AggregationVariants::Percentiles(per) => per.field_name(),
+            AggregationVariants::Terms(terms) => vec![terms.field.as_str()],
+            AggregationVariants::Range(range) => vec![range.field.as_str()],
+            AggregationVariants::Histogram(histogram) => vec![histogram.field.as_str()],
+            AggregationVariants::DateHistogram(histogram) => vec![histogram.field.as_str()],
+            AggregationVariants::Average(avg) => vec![avg.field_name()],
+            AggregationVariants::Count(count) => vec![count.field_name()],
+            AggregationVariants::Max(max) => vec![max.field_name()],
+            AggregationVariants::Min(min) => vec![min.field_name()],
+            AggregationVariants::Stats(stats) => vec![stats.field_name()],
+            AggregationVariants::Sum(sum) => vec![sum.field_name()],
+            AggregationVariants::Percentiles(per) => vec![per.field_name()],
+            AggregationVariants::TopHits(top_hits) => top_hits.field_names(),
         }
     }
 

src/aggregation/agg_req_with_accessor.rs

@@ -1,6 +1,9 @@
 //! This will enhance the request tree with access to the fastfield and metadata.
 
-use columnar::{Column, ColumnBlockAccessor, ColumnType, StrColumn};
+use std::collections::HashMap;
+use std::io;
+
+use columnar::{Column, ColumnBlockAccessor, ColumnType, DynamicColumn, StrColumn};
 
 use super::agg_limits::ResourceLimitGuard;
 use super::agg_req::{Aggregation, AggregationVariants, Aggregations};
@@ -14,7 +17,7 @@ use super::metric::{
 use super::segment_agg_result::AggregationLimits;
 use super::VecWithNames;
 use crate::aggregation::{f64_to_fastfield_u64, Key};
-use crate::SegmentReader;
+use crate::{SegmentOrdinal, SegmentReader};
 
 #[derive(Default)]
 pub(crate) struct AggregationsWithAccessor {
@@ -32,6 +35,7 @@ impl AggregationsWithAccessor {
 }
 
 pub struct AggregationWithAccessor {
+    pub(crate) segment_ordinal: SegmentOrdinal,
     /// In general there can be buckets without fast field access, e.g. buckets that are created
     /// based on search terms. That is not that case currently, but eventually this needs to be
     /// Option or moved.
@@ -44,10 +48,16 @@ pub struct AggregationWithAccessor {
     pub(crate) limits: ResourceLimitGuard,
     pub(crate) column_block_accessor: ColumnBlockAccessor<u64>,
     /// Used for missing term aggregation, which checks all columns for existence.
+    /// And also for `top_hits` aggregation, which may sort on multiple fields.
     /// By convention the missing aggregation is chosen, when this property is set
     /// (instead bein set in `agg`).
     /// If this needs to used by other aggregations, we need to refactor this.
-    pub(crate) accessors: Vec<Column<u64>>,
+    // NOTE: we can make all other aggregations use this instead of the `accessor` and `field_type`
+    // (making them obsolete) But will it have a performance impact?
+    pub(crate) accessors: Vec<(Column<u64>, ColumnType)>,
+    /// Map field names to all associated column accessors.
+    /// This field is used for `docvalue_fields`, which is currently only supported for `top_hits`.
+    pub(crate) value_accessors: HashMap<String, Vec<DynamicColumn>>,
     pub(crate) agg: Aggregation,
 }
 
@@ -57,19 +67,55 @@ impl AggregationWithAccessor {
         agg: &Aggregation,
         sub_aggregation: &Aggregations,
         reader: &SegmentReader,
+        segment_ordinal: SegmentOrdinal,
         limits: AggregationLimits,
     ) -> crate::Result<Vec<AggregationWithAccessor>> {
-        let add_agg_with_accessor = |accessor: Column<u64>,
+        let mut agg = agg.clone();
+
+        let add_agg_with_accessor = |agg: &Aggregation,
+                                     accessor: Column<u64>,
                                      column_type: ColumnType,
                                      aggs: &mut Vec<AggregationWithAccessor>|
          -> crate::Result<()> {
             let res = AggregationWithAccessor {
+                segment_ordinal,
                 accessor,
-                accessors: Vec::new(),
+                accessors: Default::default(),
+                value_accessors: Default::default(),
                 field_type: column_type,
                 sub_aggregation: get_aggs_with_segment_accessor_and_validate(
                     sub_aggregation,
                     reader,
+                    segment_ordinal,
+                    &limits,
+                )?,
+                agg: agg.clone(),
+                limits: limits.new_guard(),
+                missing_value_for_accessor: None,
+                str_dict_column: None,
+                column_block_accessor: Default::default(),
+            };
+            aggs.push(res);
+            Ok(())
+        };
+
+        let add_agg_with_accessors = |agg: &Aggregation,
+                                      accessors: Vec<(Column<u64>, ColumnType)>,
+                                      aggs: &mut Vec<AggregationWithAccessor>,
+                                      value_accessors: HashMap<String, Vec<DynamicColumn>>|
+         -> crate::Result<()> {
+            let (accessor, field_type) = accessors.first().expect("at least one accessor");
+            let res = AggregationWithAccessor {
+                segment_ordinal,
+                // TODO: We should do away with the `accessor` field altogether
+                accessor: accessor.clone(),
+                value_accessors,
+                field_type: *field_type,
+                accessors,
+                sub_aggregation: get_aggs_with_segment_accessor_and_validate(
+                    sub_aggregation,
+                    reader,
+                    segment_ordinal,
                     &limits,
                 )?,
                 agg: agg.clone(),
@@ -84,32 +130,36 @@ impl AggregationWithAccessor {
 
         let mut res: Vec<AggregationWithAccessor> = Vec::new();
         use AggregationVariants::*;
-        match &agg.agg {
+
+        match agg.agg {
             Range(RangeAggregation {
-                field: field_name, ..
+                field: ref field_name,
+                ..
             }) => {
                 let (accessor, column_type) =
                     get_ff_reader(reader, field_name, Some(get_numeric_or_date_column_types()))?;
-                add_agg_with_accessor(accessor, column_type, &mut res)?;
+                add_agg_with_accessor(&agg, accessor, column_type, &mut res)?;
             }
             Histogram(HistogramAggregation {
-                field: field_name, ..
+                field: ref field_name,
+                ..
             }) => {
                 let (accessor, column_type) =
                     get_ff_reader(reader, field_name, Some(get_numeric_or_date_column_types()))?;
-                add_agg_with_accessor(accessor, column_type, &mut res)?;
+                add_agg_with_accessor(&agg, accessor, column_type, &mut res)?;
             }
             DateHistogram(DateHistogramAggregationReq {
-                field: field_name, ..
+                field: ref field_name,
+                ..
             }) => {
                 let (accessor, column_type) =
                     // Only DateTime is supported for DateHistogram
                     get_ff_reader(reader, field_name, Some(&[ColumnType::DateTime]))?;
-                add_agg_with_accessor(accessor, column_type, &mut res)?;
+                add_agg_with_accessor(&agg, accessor, column_type, &mut res)?;
             }
             Terms(TermsAggregation {
-                field: field_name,
-                missing,
+                field: ref field_name,
+                ref missing,
                 ..
             }) => {
                 let str_dict_column = reader.fast_fields().str(field_name)?;
@@ -119,8 +169,8 @@ impl AggregationWithAccessor {
                     ColumnType::F64,
                     ColumnType::Str,
                     ColumnType::DateTime,
+                    ColumnType::Bool,
                     // ColumnType::Bytes Unsupported
-                    // ColumnType::Bool Unsupported
                     // ColumnType::IpAddr Unsupported
                 ];
 
@@ -162,24 +212,11 @@ impl AggregationWithAccessor {
                     let column_and_types =
                         get_all_ff_reader_or_empty(reader, field_name, None, fallback_type)?;
 
-                    let accessors: Vec<Column> =
-                        column_and_types.iter().map(|(a, _)| a.clone()).collect();
-                    let agg_wit_acc = AggregationWithAccessor {
-                        missing_value_for_accessor: None,
-                        accessor: accessors[0].clone(),
-                        accessors,
-                        field_type: ColumnType::U64,
-                        sub_aggregation: get_aggs_with_segment_accessor_and_validate(
-                            sub_aggregation,
-                            reader,
-                            &limits,
-                        )?,
-                        agg: agg.clone(),
-                        str_dict_column: str_dict_column.clone(),
-                        limits: limits.new_guard(),
-                        column_block_accessor: Default::default(),
-                    };
-                    res.push(agg_wit_acc);
+                    let accessors = column_and_types
+                        .iter()
+                        .map(|c_t| (c_t.0.clone(), c_t.1))
+                        .collect();
+                    add_agg_with_accessors(&agg, accessors, &mut res, Default::default())?;
                 }
 
                 for (accessor, column_type) in column_and_types {
@@ -189,21 +226,25 @@ impl AggregationWithAccessor {
                         missing.clone()
                     };
 
-                    let missing_value_for_accessor =
-                        if let Some(missing) = missing_value_term_agg.as_ref() {
-                            get_missing_val(column_type, missing, agg.agg.get_fast_field_name())?
-                        } else {
-                            None
-                        };
+                    let missing_value_for_accessor = if let Some(missing) =
+                        missing_value_term_agg.as_ref()
+                    {
+                        get_missing_val(column_type, missing, agg.agg.get_fast_field_names()[0])?
+                    } else {
+                        None
+                    };
 
                     let agg = AggregationWithAccessor {
+                        segment_ordinal,
                         missing_value_for_accessor,
                         accessor,
-                        accessors: Vec::new(),
+                        accessors: Default::default(),
+                        value_accessors: Default::default(),
                         field_type: column_type,
                         sub_aggregation: get_aggs_with_segment_accessor_and_validate(
                             sub_aggregation,
                             reader,
+                            segment_ordinal,
                             &limits,
                         )?,
                         agg: agg.clone(),
@@ -215,34 +256,63 @@ impl AggregationWithAccessor {
                 }
             }
             Average(AverageAggregation {
-                field: field_name, ..
+                field: ref field_name,
+                ..
             })
             | Count(CountAggregation {
-                field: field_name, ..
+                field: ref field_name,
+                ..
             })
             | Max(MaxAggregation {
-                field: field_name, ..
+                field: ref field_name,
+                ..
             })
             | Min(MinAggregation {
-                field: field_name, ..
+                field: ref field_name,
+                ..
             })
             | Stats(StatsAggregation {
-                field: field_name, ..
+                field: ref field_name,
+                ..
             })
             | Sum(SumAggregation {
-                field: field_name, ..
+                field: ref field_name,
+                ..
             }) => {
                 let (accessor, column_type) =
                     get_ff_reader(reader, field_name, Some(get_numeric_or_date_column_types()))?;
-                add_agg_with_accessor(accessor, column_type, &mut res)?;
+                add_agg_with_accessor(&agg, accessor, column_type, &mut res)?;
             }
-            Percentiles(percentiles) => {
+            Percentiles(ref percentiles) => {
                 let (accessor, column_type) = get_ff_reader(
                     reader,
                     percentiles.field_name(),
                     Some(get_numeric_or_date_column_types()),
                 )?;
-                add_agg_with_accessor(accessor, column_type, &mut res)?;
+                add_agg_with_accessor(&agg, accessor, column_type, &mut res)?;
+            }
+            TopHits(ref mut top_hits) => {
+                top_hits.validate_and_resolve(reader.fast_fields().columnar())?;
+                let accessors: Vec<(Column<u64>, ColumnType)> = top_hits
+                    .field_names()
+                    .iter()
+                    .map(|field| {
+                        get_ff_reader(reader, field, Some(get_numeric_or_date_column_types()))
+                    })
+                    .collect::<crate::Result<_>>()?;
+
+                let value_accessors = top_hits
+                    .value_field_names()
+                    .iter()
+                    .map(|field_name| {
+                        Ok((
+                            field_name.to_string(),
+                            get_dynamic_columns(reader, field_name)?,
+                        ))
+                    })
+                    .collect::<crate::Result<_>>()?;
+
+                add_agg_with_accessors(&agg, accessors, &mut res, value_accessors)?;
             }
         };
 
@@ -284,6 +354,7 @@ fn get_numeric_or_date_column_types() -> &'static [ColumnType] {
 pub(crate) fn get_aggs_with_segment_accessor_and_validate(
     aggs: &Aggregations,
     reader: &SegmentReader,
+    segment_ordinal: SegmentOrdinal,
     limits: &AggregationLimits,
 ) -> crate::Result<AggregationsWithAccessor> {
     let mut aggss = Vec::new();
@@ -292,6 +363,7 @@ pub(crate) fn get_aggs_with_segment_accessor_and_validate(
             agg,
             agg.sub_aggregation(),
             reader,
+            segment_ordinal,
             limits.clone(),
         )?;
         for agg in aggs {
@@ -321,6 +393,19 @@ fn get_ff_reader(
     Ok(ff_field_with_type)
 }
 
+fn get_dynamic_columns(
+    reader: &SegmentReader,
+    field_name: &str,
+) -> crate::Result<Vec<columnar::DynamicColumn>> {
+    let ff_fields = reader.fast_fields().dynamic_column_handles(field_name)?;
+    let cols = ff_fields
+        .iter()
+        .map(|h| h.open())
+        .collect::<io::Result<_>>()?;
+    assert!(!ff_fields.is_empty(), "field {} not found", field_name);
+    Ok(cols)
+}
+
 /// Get all fast field reader or empty as default.
 ///
 /// Is guaranteed to return at least one column.

src/aggregation/agg_result.rs

@@ -8,7 +8,7 @@ use rustc_hash::FxHashMap;
 use serde::{Deserialize, Serialize};
 
 use super::bucket::GetDocCount;
-use super::metric::{PercentilesMetricResult, SingleMetricResult, Stats};
+use super::metric::{PercentilesMetricResult, SingleMetricResult, Stats, TopHitsMetricResult};
 use super::{AggregationError, Key};
 use crate::TantivyError;
 
@@ -90,8 +90,10 @@ pub enum MetricResult {
     Stats(Stats),
     /// Sum metric result.
     Sum(SingleMetricResult),
-    /// Sum metric result.
+    /// Percentiles metric result.
     Percentiles(PercentilesMetricResult),
+    /// Top hits metric result
+    TopHits(TopHitsMetricResult),
 }
 
 impl MetricResult {
@@ -106,6 +108,9 @@ impl MetricResult {
             MetricResult::Percentiles(_) => Err(TantivyError::AggregationError(
                 AggregationError::InvalidRequest("percentiles can't be used to order".to_string()),
             )),
+            MetricResult::TopHits(_) => Err(TantivyError::AggregationError(
+                AggregationError::InvalidRequest("top_hits can't be used to order".to_string()),
+            )),
         }
     }
 }

src/aggregation/agg_tests.rs

@@ -587,6 +587,9 @@ fn test_aggregation_on_json_object() {
     let schema = schema_builder.build();
     let index = Index::create_in_ram(schema);
     let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
+    index_writer
+        .add_document(doc!(json => json!({"color": "red"})))
+        .unwrap();
     index_writer
         .add_document(doc!(json => json!({"color": "red"})))
         .unwrap();
@@ -614,12 +617,74 @@ fn test_aggregation_on_json_object() {
         &serde_json::json!({
             "jsonagg": {
                 "buckets": [
+                    {"doc_count": 2, "key": "red"},
                     {"doc_count": 1, "key": "blue"},
+                ],
+                "doc_count_error_upper_bound": 0,
+                "sum_other_doc_count": 0
+            }
+        })
+    );
+}
+
+#[test]
+fn test_aggregation_on_nested_json_object() {
+    let mut schema_builder = Schema::builder();
+    let json = schema_builder.add_json_field("json.blub", FAST);
+    let schema = schema_builder.build();
+    let index = Index::create_in_ram(schema);
+    let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
+    index_writer
+        .add_document(doc!(json => json!({"color.dot": "red", "color": {"nested":"red"} })))
+        .unwrap();
+    index_writer
+        .add_document(doc!(json => json!({"color.dot": "blue", "color": {"nested":"blue"} })))
+        .unwrap();
+    index_writer
+        .add_document(doc!(json => json!({"color.dot": "blue", "color": {"nested":"blue"} })))
+        .unwrap();
+    index_writer.commit().unwrap();
+    let reader = index.reader().unwrap();
+    let searcher = reader.searcher();
+
+    let agg: Aggregations = serde_json::from_value(json!({
+        "jsonagg1": {
+            "terms": {
+                "field": "json\\.blub.color\\.dot",
+            }
+        },
+        "jsonagg2": {
+            "terms": {
+                "field": "json\\.blub.color.nested",
+            }
+        }
+
+    }))
+    .unwrap();
+
+    let aggregation_collector = get_collector(agg);
+    let aggregation_results = searcher.search(&AllQuery, &aggregation_collector).unwrap();
+    let aggregation_res_json = serde_json::to_value(aggregation_results).unwrap();
+    assert_eq!(
+        &aggregation_res_json,
+        &serde_json::json!({
+            "jsonagg1": {
+                "buckets": [
+                    {"doc_count": 2, "key": "blue"},
+                    {"doc_count": 1, "key": "red"}
+                ],
+                "doc_count_error_upper_bound": 0,
+                "sum_other_doc_count": 0
+            },
+            "jsonagg2": {
+                "buckets": [
+                    {"doc_count": 2, "key": "blue"},
                     {"doc_count": 1, "key": "red"}
                 ],
                 "doc_count_error_upper_bound": 0,
                 "sum_other_doc_count": 0
             }
+
         })
     );
 }
@@ -755,6 +820,12 @@ fn test_aggregation_on_json_object_mixed_types() {
         .unwrap();
     index_writer.commit().unwrap();
     // => Segment with all values text
+    index_writer
+        .add_document(doc!(json => json!({"mixed_type": "blue"})))
+        .unwrap();
+    index_writer
+        .add_document(doc!(json => json!({"mixed_type": "blue"})))
+        .unwrap();
     index_writer
         .add_document(doc!(json => json!({"mixed_type": "blue"})))
         .unwrap();
@@ -766,6 +837,9 @@ fn test_aggregation_on_json_object_mixed_types() {
     index_writer.commit().unwrap();
 
     // => Segment with mixed values
+    index_writer
+        .add_document(doc!(json => json!({"mixed_type": "red"})))
+        .unwrap();
     index_writer
         .add_document(doc!(json => json!({"mixed_type": "red"})))
         .unwrap();
@@ -811,6 +885,8 @@ fn test_aggregation_on_json_object_mixed_types() {
 
     let aggregation_results = searcher.search(&AllQuery, &aggregation_collector).unwrap();
     let aggregation_res_json = serde_json::to_value(aggregation_results).unwrap();
+    // pretty print as json
+    use pretty_assertions::assert_eq;
     assert_eq!(
         &aggregation_res_json,
         &serde_json::json!({
@@ -826,9 +902,9 @@ fn test_aggregation_on_json_object_mixed_types() {
             "buckets": [
               { "doc_count": 1, "key": 10.0, "min_price": { "value": 10.0 } },
               { "doc_count": 1, "key": -20.5, "min_price": { "value": -20.5 } },
-              // TODO bool is also not yet handled in aggregation
-              { "doc_count": 1, "key": "blue", "min_price": { "value": null } },
-              { "doc_count": 1, "key": "red", "min_price": { "value": null } },
+              { "doc_count": 2, "key": "red", "min_price": { "value": null } },
+              { "doc_count": 2, "key": 1.0, "key_as_string": "true", "min_price": { "value": null } },
+              { "doc_count": 3, "key": "blue", "min_price": { "value": null } },
             ],
             "sum_other_doc_count": 0
           }

src/aggregation/bucket/histogram/date_histogram.rs

@@ -1,7 +1,7 @@
 use serde::{Deserialize, Serialize};
 
 use super::{HistogramAggregation, HistogramBounds};
-use crate::aggregation::AggregationError;
+use crate::aggregation::*;
 
 /// DateHistogramAggregation is similar to `HistogramAggregation`, but it can only be used with date
 /// type.
@@ -307,6 +307,7 @@ pub mod tests {
     ) -> crate::Result<Index> {
         let mut schema_builder = Schema::builder();
         schema_builder.add_date_field("date", FAST);
+        schema_builder.add_json_field("mixed", FAST);
         schema_builder.add_text_field("text", FAST | STRING);
         schema_builder.add_text_field("text2", FAST | STRING);
         let schema = schema_builder.build();
@@ -351,8 +352,10 @@ pub mod tests {
         let docs = vec![
             vec![r#"{ "date": "2015-01-01T12:10:30Z", "text": "aaa" }"#],
             vec![r#"{ "date": "2015-01-01T11:11:30Z", "text": "bbb" }"#],
+            vec![r#"{ "date": "2015-01-01T11:11:30Z", "text": "bbb" }"#],
             vec![r#"{ "date": "2015-01-02T00:00:00Z", "text": "bbb" }"#],
             vec![r#"{ "date": "2015-01-06T00:00:00Z", "text": "ccc" }"#],
+            vec![r#"{ "date": "2015-01-06T00:00:00Z", "text": "ccc" }"#],
         ];
         let index = get_test_index_from_docs(merge_segments, &docs).unwrap();
 
@@ -381,7 +384,7 @@ pub mod tests {
                         {
                             "key_as_string" : "2015-01-01T00:00:00Z",
                             "key" : 1420070400000.0,
-                            "doc_count" : 4
+                            "doc_count" : 6
                         }
                     ]
                 }
@@ -419,15 +422,15 @@ pub mod tests {
                     {
                         "key_as_string" : "2015-01-01T00:00:00Z",
                         "key" : 1420070400000.0,
-                        "doc_count" : 4,
+                        "doc_count" : 6,
                         "texts": {
                             "buckets": [
                                 {
-                                "doc_count": 2,
+                                "doc_count": 3,
                                 "key": "bbb"
                                 },
                                 {
-                                "doc_count": 1,
+                                "doc_count": 2,
                                 "key": "ccc"
                                 },
                                 {
@@ -466,7 +469,7 @@ pub mod tests {
                 "sales_over_time": {
                     "buckets": [
                         {
-                            "doc_count": 2,
+                            "doc_count": 3,
                             "key": 1420070400000.0,
                             "key_as_string": "2015-01-01T00:00:00Z"
                         },
@@ -491,7 +494,7 @@ pub mod tests {
                             "key_as_string": "2015-01-05T00:00:00Z"
                         },
                         {
-                            "doc_count": 1,
+                            "doc_count": 2,
                             "key": 1420502400000.0,
                             "key_as_string": "2015-01-06T00:00:00Z"
                         }
@@ -532,7 +535,7 @@ pub mod tests {
                             "key_as_string": "2014-12-31T00:00:00Z"
                         },
                         {
-                            "doc_count": 2,
+                            "doc_count": 3,
                             "key": 1420070400000.0,
                             "key_as_string": "2015-01-01T00:00:00Z"
                         },
@@ -557,7 +560,7 @@ pub mod tests {
                             "key_as_string": "2015-01-05T00:00:00Z"
                         },
                         {
-                            "doc_count": 1,
+                            "doc_count": 2,
                             "key": 1420502400000.0,
                             "key_as_string": "2015-01-06T00:00:00Z"
                         },

src/aggregation/bucket/histogram/histogram.rs

@@ -20,7 +20,7 @@ use crate::aggregation::intermediate_agg_result::{
 use crate::aggregation::segment_agg_result::{
     build_segment_agg_collector, AggregationLimits, SegmentAggregationCollector,
 };
-use crate::aggregation::{f64_from_fastfield_u64, format_date};
+use crate::aggregation::*;
 use crate::TantivyError;
 
 /// Histogram is a bucket aggregation, where buckets are created dynamically for given `interval`.
@@ -73,6 +73,7 @@ pub struct HistogramAggregation {
     pub field: String,
     /// The interval to chunk your data range. Each bucket spans a value range of [0..interval).
     /// Must be a positive value.
+    #[serde(deserialize_with = "deserialize_f64")]
     pub interval: f64,
     /// Intervals implicitly defines an absolute grid of buckets `[interval * k, interval * (k +
     /// 1))`.
@@ -85,6 +86,7 @@ pub struct HistogramAggregation {
     /// fall into the buckets with the key 0 and 10.
     /// With offset 5 and interval 10, they would both fall into the bucket with they key 5 and the
     /// range [5..15)
+    #[serde(default, deserialize_with = "deserialize_option_f64")]
     pub offset: Option<f64>,
     /// The minimum number of documents in a bucket to be returned. Defaults to 0.
     pub min_doc_count: Option<u64>,
@@ -596,10 +598,13 @@ mod tests {
 
     use super::*;
     use crate::aggregation::agg_req::Aggregations;
+    use crate::aggregation::agg_result::AggregationResults;
     use crate::aggregation::tests::{
         exec_request, exec_request_with_query, exec_request_with_query_and_memory_limit,
         get_test_index_2_segments, get_test_index_from_values, get_test_index_with_num_docs,
     };
+    use crate::aggregation::AggregationCollector;
+    use crate::query::AllQuery;
 
     #[test]
     fn histogram_test_crooked_values() -> crate::Result<()> {
@@ -1351,6 +1356,35 @@ mod tests {
             })
         );
 
+        Ok(())
+    }
+    #[test]
+    fn test_aggregation_histogram_empty_index() -> crate::Result<()> {
+        // test index without segments
+        let values = vec![];
+
+        let index = get_test_index_from_values(false, &values)?;
+
+        let agg_req_1: Aggregations = serde_json::from_value(json!({
+            "myhisto": {
+                "histogram": {
+                    "field": "score",
+                    "interval": 10.0
+                },
+            }
+        }))
+        .unwrap();
+
+        let collector = AggregationCollector::from_aggs(agg_req_1, Default::default());
+
+        let reader = index.reader()?;
+        let searcher = reader.searcher();
+        let agg_res: AggregationResults = searcher.search(&AllQuery, &collector).unwrap();
+
+        let res: Value = serde_json::from_str(&serde_json::to_string(&agg_res)?)?;
+        // Make sure the result structure is correct
+        assert_eq!(res["myhisto"]["buckets"].as_array().unwrap().len(), 0);
+
         Ok(())
     }
 }

src/aggregation/bucket/range.rs

@@ -14,9 +14,7 @@ use crate::aggregation::intermediate_agg_result::{
 use crate::aggregation::segment_agg_result::{
     build_segment_agg_collector, SegmentAggregationCollector,
 };
-use crate::aggregation::{
-    f64_from_fastfield_u64, f64_to_fastfield_u64, format_date, Key, SerializedKey,
-};
+use crate::aggregation::*;
 use crate::TantivyError;
 
 /// Provide user-defined buckets to aggregate on.
@@ -72,11 +70,19 @@ pub struct RangeAggregationRange {
     pub key: Option<String>,
     /// The from range value, which is inclusive in the range.
     /// `None` equals to an open ended interval.
-    #[serde(skip_serializing_if = "Option::is_none", default)]
+    #[serde(
+        skip_serializing_if = "Option::is_none",
+        default,
+        deserialize_with = "deserialize_option_f64"
+    )]
     pub from: Option<f64>,
     /// The to range value, which is not inclusive in the range.
     /// `None` equals to an open ended interval.
-    #[serde(skip_serializing_if = "Option::is_none", default)]
+    #[serde(
+        skip_serializing_if = "Option::is_none",
+        default,
+        deserialize_with = "deserialize_option_f64"
+    )]
     pub to: Option<f64>,
 }
 

src/aggregation/bucket/term_agg.rs

@@ -99,24 +99,15 @@ pub struct TermsAggregation {
     #[serde(skip_serializing_if = "Option::is_none", default)]
     pub size: Option<u32>,
 
-    /// Unused by tantivy.
-    ///
-    /// Since tantivy doesn't know shards, this parameter is merely there to be used by consumers
-    /// of tantivy. shard_size is the number of terms returned by each shard.
-    /// The default value in elasticsearch is size * 1.5 + 10.
-    ///
-    /// Should never be smaller than size.
-    #[serde(skip_serializing_if = "Option::is_none", default)]
-    #[serde(alias = "shard_size")]
-    pub split_size: Option<u32>,
-
-    /// The get more accurate results, we fetch more than `size` from each segment.
+    /// To get more accurate results, we fetch more than `size` from each segment.
     ///
     /// Increasing this value is will increase the cost for more accuracy.
     ///
     /// Defaults to 10 * size.
     #[serde(skip_serializing_if = "Option::is_none", default)]
-    pub segment_size: Option<u32>,
+    #[serde(alias = "segment_size")]
+    #[serde(alias = "split_size")]
+    pub shard_size: Option<u32>,
 
     /// If you set the `show_term_doc_count_error` parameter to true, the terms aggregation will
     /// include doc_count_error_upper_bound, which is an upper bound to the error on the
@@ -205,7 +196,7 @@ impl TermsAggregationInternal {
     pub(crate) fn from_req(req: &TermsAggregation) -> Self {
         let size = req.size.unwrap_or(10);
 
-        let mut segment_size = req.segment_size.unwrap_or(size * 10);
+        let mut segment_size = req.shard_size.unwrap_or(size * 10);
 
         let order = req.order.clone().unwrap_or_default();
         segment_size = segment_size.max(size);
@@ -256,7 +247,7 @@ pub struct SegmentTermCollector {
     term_buckets: TermBuckets,
     req: TermsAggregationInternal,
     blueprint: Option<Box<dyn SegmentAggregationCollector>>,
-    field_type: ColumnType,
+    column_type: ColumnType,
     accessor_idx: usize,
 }
 
@@ -355,7 +346,7 @@ impl SegmentTermCollector {
         field_type: ColumnType,
         accessor_idx: usize,
     ) -> crate::Result<Self> {
-        if field_type == ColumnType::Bytes || field_type == ColumnType::Bool {
+        if field_type == ColumnType::Bytes {
             return Err(TantivyError::InvalidArgument(format!(
                 "terms aggregation is not supported for column type {:?}",
                 field_type
@@ -389,7 +380,7 @@ impl SegmentTermCollector {
             req: TermsAggregationInternal::from_req(req),
             term_buckets,
             blueprint,
-            field_type,
+            column_type: field_type,
             accessor_idx,
         })
     }
@@ -466,7 +457,7 @@ impl SegmentTermCollector {
                 Ok(intermediate_entry)
             };
 
-        if self.field_type == ColumnType::Str {
+        if self.column_type == ColumnType::Str {
             let term_dict = agg_with_accessor
                 .str_dict_column
                 .as_ref()
@@ -531,28 +522,34 @@ impl SegmentTermCollector {
                         });
                 }
             }
-        } else if self.field_type == ColumnType::DateTime {
+        } else if self.column_type == ColumnType::DateTime {
             for (val, doc_count) in entries {
                 let intermediate_entry = into_intermediate_bucket_entry(val, doc_count)?;
                 let val = i64::from_u64(val);
                 let date = format_date(val)?;
                 dict.insert(IntermediateKey::Str(date), intermediate_entry);
             }
+        } else if self.column_type == ColumnType::Bool {
+            for (val, doc_count) in entries {
+                let intermediate_entry = into_intermediate_bucket_entry(val, doc_count)?;
+                let val = bool::from_u64(val);
+                dict.insert(IntermediateKey::Bool(val), intermediate_entry);
+            }
         } else {
             for (val, doc_count) in entries {
                 let intermediate_entry = into_intermediate_bucket_entry(val, doc_count)?;
-                let val = f64_from_fastfield_u64(val, &self.field_type);
+                let val = f64_from_fastfield_u64(val, &self.column_type);
                 dict.insert(IntermediateKey::F64(val), intermediate_entry);
             }
         };
 
-        Ok(IntermediateBucketResult::Terms(
-            IntermediateTermBucketResult {
+        Ok(IntermediateBucketResult::Terms {
+            buckets: IntermediateTermBucketResult {
                 entries: dict,
                 sum_other_doc_count,
                 doc_count_error_upper_bound: term_doc_count_before_cutoff,
             },
-        ))
+        })
     }
 }
 
@@ -1365,7 +1362,7 @@ mod tests {
 
     #[test]
     fn terms_aggregation_different_tokenizer_on_ff_test() -> crate::Result<()> {
-        let terms = vec!["Hello Hello", "Hallo Hallo"];
+        let terms = vec!["Hello Hello", "Hallo Hallo", "Hallo Hallo"];
 
         let index = get_test_index_from_terms(true, &[terms])?;
 
@@ -1383,7 +1380,7 @@ mod tests {
         println!("{}", serde_json::to_string_pretty(&res).unwrap());
 
         assert_eq!(res["my_texts"]["buckets"][0]["key"], "Hallo Hallo");
-        assert_eq!(res["my_texts"]["buckets"][0]["doc_count"], 1);
+        assert_eq!(res["my_texts"]["buckets"][0]["doc_count"], 2);
 
         assert_eq!(res["my_texts"]["buckets"][1]["key"], "Hello Hello");
         assert_eq!(res["my_texts"]["buckets"][1]["doc_count"], 1);
@@ -1894,4 +1891,40 @@ mod tests {
 
         Ok(())
     }
+
+    #[test]
+    fn terms_aggregation_bool() -> crate::Result<()> {
+        let mut schema_builder = Schema::builder();
+        let field = schema_builder.add_bool_field("bool_field", FAST);
+        let schema = schema_builder.build();
+        let index = Index::create_in_ram(schema);
+        {
+            let mut writer = index.writer_with_num_threads(1, 15_000_000)?;
+            writer.add_document(doc!(field=>true))?;
+            writer.add_document(doc!(field=>false))?;
+            writer.add_document(doc!(field=>true))?;
+            writer.commit()?;
+        }
+
+        let agg_req: Aggregations = serde_json::from_value(json!({
+            "my_bool": {
+                "terms": {
+                    "field": "bool_field"
+                },
+            }
+        }))
+        .unwrap();
+
+        let res = exec_request_with_query(agg_req, &index, None)?;
+
+        assert_eq!(res["my_bool"]["buckets"][0]["key"], 1.0);
+        assert_eq!(res["my_bool"]["buckets"][0]["key_as_string"], "true");
+        assert_eq!(res["my_bool"]["buckets"][0]["doc_count"], 2);
+        assert_eq!(res["my_bool"]["buckets"][1]["key"], 0.0);
+        assert_eq!(res["my_bool"]["buckets"][1]["key_as_string"], "false");
+        assert_eq!(res["my_bool"]["buckets"][1]["doc_count"], 1);
+        assert_eq!(res["my_bool"]["buckets"][2]["key"], serde_json::Value::Null);
+
+        Ok(())
+    }
 }

src/aggregation/bucket/term_missing_agg.rs

@@ -73,11 +73,13 @@ impl SegmentAggregationCollector for TermMissingAgg {
 
         entries.insert(missing.into(), missing_entry);
 
-        let bucket = IntermediateBucketResult::Terms(IntermediateTermBucketResult {
-            entries,
-            sum_other_doc_count: 0,
-            doc_count_error_upper_bound: 0,
-        });
+        let bucket = IntermediateBucketResult::Terms {
+            buckets: IntermediateTermBucketResult {
+                entries,
+                sum_other_doc_count: 0,
+                doc_count_error_upper_bound: 0,
+            },
+        };
 
         results.push(name, IntermediateAggregationResult::Bucket(bucket))?;
 
@@ -90,7 +92,10 @@ impl SegmentAggregationCollector for TermMissingAgg {
         agg_with_accessor: &mut AggregationsWithAccessor,
     ) -> crate::Result<()> {
         let agg = &mut agg_with_accessor.aggs.values[self.accessor_idx];
-        let has_value = agg.accessors.iter().any(|acc| acc.index.has_value(doc));
+        let has_value = agg
+            .accessors
+            .iter()
+            .any(|(acc, _)| acc.index.has_value(doc));
         if !has_value {
             self.missing_count += 1;
             if let Some(sub_agg) = self.sub_agg.as_mut() {

src/aggregation/collector.rs

@@ -8,7 +8,7 @@ use super::segment_agg_result::{
 };
 use crate::aggregation::agg_req_with_accessor::get_aggs_with_segment_accessor_and_validate;
 use crate::collector::{Collector, SegmentCollector};
-use crate::{DocId, SegmentReader, TantivyError};
+use crate::{DocId, SegmentOrdinal, SegmentReader, TantivyError};
 
 /// The default max bucket count, before the aggregation fails.
 pub const DEFAULT_BUCKET_LIMIT: u32 = 65000;
@@ -64,10 +64,15 @@ impl Collector for DistributedAggregationCollector {
 
     fn for_segment(
         &self,
-        _segment_local_id: crate::SegmentOrdinal,
+        segment_local_id: crate::SegmentOrdinal,
         reader: &crate::SegmentReader,
     ) -> crate::Result<Self::Child> {
-        AggregationSegmentCollector::from_agg_req_and_reader(&self.agg, reader, &self.limits)
+        AggregationSegmentCollector::from_agg_req_and_reader(
+            &self.agg,
+            reader,
+            segment_local_id,
+            &self.limits,
+        )
     }
 
     fn requires_scoring(&self) -> bool {
@@ -89,10 +94,15 @@ impl Collector for AggregationCollector {
 
     fn for_segment(
         &self,
-        _segment_local_id: crate::SegmentOrdinal,
+        segment_local_id: crate::SegmentOrdinal,
         reader: &crate::SegmentReader,
     ) -> crate::Result<Self::Child> {
-        AggregationSegmentCollector::from_agg_req_and_reader(&self.agg, reader, &self.limits)
+        AggregationSegmentCollector::from_agg_req_and_reader(
+            &self.agg,
+            reader,
+            segment_local_id,
+            &self.limits,
+        )
     }
 
     fn requires_scoring(&self) -> bool {
@@ -135,10 +145,11 @@ impl AggregationSegmentCollector {
     pub fn from_agg_req_and_reader(
         agg: &Aggregations,
         reader: &SegmentReader,
+        segment_ordinal: SegmentOrdinal,
         limits: &AggregationLimits,
     ) -> crate::Result<Self> {
         let mut aggs_with_accessor =
-            get_aggs_with_segment_accessor_and_validate(agg, reader, limits)?;
+            get_aggs_with_segment_accessor_and_validate(agg, reader, segment_ordinal, limits)?;
         let result =
             BufAggregationCollector::new(build_segment_agg_collector(&mut aggs_with_accessor)?);
         Ok(AggregationSegmentCollector {

src/aggregation/intermediate_agg_result.rs

@@ -19,7 +19,7 @@ use super::bucket::{
 };
 use super::metric::{
     IntermediateAverage, IntermediateCount, IntermediateMax, IntermediateMin, IntermediateStats,
-    IntermediateSum, PercentilesCollector,
+    IntermediateSum, PercentilesCollector, TopHitsCollector,
 };
 use super::segment_agg_result::AggregationLimits;
 use super::{format_date, AggregationError, Key, SerializedKey};
@@ -41,6 +41,8 @@ pub struct IntermediateAggregationResults {
 /// This might seem redundant with `Key`, but the point is to have a different
 /// Serialize implementation.
 pub enum IntermediateKey {
+    /// Bool key
+    Bool(bool),
     /// String key
     Str(String),
     /// `f64` key
@@ -59,6 +61,7 @@ impl From<IntermediateKey> for Key {
         match value {
             IntermediateKey::Str(s) => Self::Str(s),
             IntermediateKey::F64(f) => Self::F64(f),
+            IntermediateKey::Bool(f) => Self::F64(f as u64 as f64),
         }
     }
 }
@@ -71,6 +74,7 @@ impl std::hash::Hash for IntermediateKey {
         match self {
             IntermediateKey::Str(text) => text.hash(state),
             IntermediateKey::F64(val) => val.to_bits().hash(state),
+            IntermediateKey::Bool(val) => val.hash(state),
         }
     }
 }
@@ -166,9 +170,9 @@ impl IntermediateAggregationResults {
 pub(crate) fn empty_from_req(req: &Aggregation) -> IntermediateAggregationResult {
     use AggregationVariants::*;
     match req.agg {
-        Terms(_) => IntermediateAggregationResult::Bucket(IntermediateBucketResult::Terms(
-            Default::default(),
-        )),
+        Terms(_) => IntermediateAggregationResult::Bucket(IntermediateBucketResult::Terms {
+            buckets: Default::default(),
+        }),
         Range(_) => IntermediateAggregationResult::Bucket(IntermediateBucketResult::Range(
             Default::default(),
         )),
@@ -205,6 +209,9 @@ pub(crate) fn empty_from_req(req: &Aggregation) -> IntermediateAggregationResult
         Percentiles(_) => IntermediateAggregationResult::Metric(
             IntermediateMetricResult::Percentiles(PercentilesCollector::default()),
         ),
+        TopHits(_) => IntermediateAggregationResult::Metric(IntermediateMetricResult::TopHits(
+            TopHitsCollector::default(),
+        )),
     }
 }
 
@@ -265,6 +272,8 @@ pub enum IntermediateMetricResult {
     Stats(IntermediateStats),
     /// Intermediate sum result.
     Sum(IntermediateSum),
+    /// Intermediate top_hits result
+    TopHits(TopHitsCollector),
 }
 
 impl IntermediateMetricResult {
@@ -292,9 +301,13 @@ impl IntermediateMetricResult {
                 percentiles
                     .into_final_result(req.agg.as_percentile().expect("unexpected metric type")),
             ),
+            IntermediateMetricResult::TopHits(top_hits) => {
+                MetricResult::TopHits(top_hits.finalize())
+            }
         }
     }
 
+    // TODO: this is our top-of-the-chain fruit merge mech
     fn merge_fruits(&mut self, other: IntermediateMetricResult) -> crate::Result<()> {
         match (self, other) {
             (
@@ -330,6 +343,9 @@ impl IntermediateMetricResult {
             ) => {
                 left.merge_fruits(right)?;
             }
+            (IntermediateMetricResult::TopHits(left), IntermediateMetricResult::TopHits(right)) => {
+                left.merge_fruits(right)?;
+            }
             _ => {
                 panic!("incompatible fruit types in tree or missing merge_fruits handler");
             }
@@ -351,11 +367,14 @@ pub enum IntermediateBucketResult {
     Histogram {
         /// The column_type of the underlying `Column` is DateTime
         is_date_agg: bool,
-        /// The buckets
+        /// The histogram buckets
         buckets: Vec<IntermediateHistogramBucketEntry>,
     },
     /// Term aggregation
-    Terms(IntermediateTermBucketResult),
+    Terms {
+        /// The term buckets
+        buckets: IntermediateTermBucketResult,
+    },
 }
 
 impl IntermediateBucketResult {
@@ -432,7 +451,7 @@ impl IntermediateBucketResult {
                 };
                 Ok(BucketResult::Histogram { buckets })
             }
-            IntermediateBucketResult::Terms(terms) => terms.into_final_result(
+            IntermediateBucketResult::Terms { buckets: terms } => terms.into_final_result(
                 req.agg
                     .as_term()
                     .expect("unexpected aggregation, expected term aggregation"),
@@ -445,8 +464,12 @@ impl IntermediateBucketResult {
     fn merge_fruits(&mut self, other: IntermediateBucketResult) -> crate::Result<()> {
         match (self, other) {
             (
-                IntermediateBucketResult::Terms(term_res_left),
-                IntermediateBucketResult::Terms(term_res_right),
+                IntermediateBucketResult::Terms {
+                    buckets: term_res_left,
+                },
+                IntermediateBucketResult::Terms {
+                    buckets: term_res_right,
+                },
             ) => {
                 merge_maps(&mut term_res_left.entries, term_res_right.entries)?;
                 term_res_left.sum_other_doc_count += term_res_right.sum_other_doc_count;
@@ -530,8 +553,15 @@ impl IntermediateTermBucketResult {
             .into_iter()
             .filter(|bucket| bucket.1.doc_count as u64 >= req.min_doc_count)
             .map(|(key, entry)| {
+                let key_as_string = match key {
+                    IntermediateKey::Bool(key) => {
+                        let val = if key { "true" } else { "false" };
+                        Some(val.to_string())
+                    }
+                    _ => None,
+                };
                 Ok(BucketEntry {
-                    key_as_string: None,
+                    key_as_string,
                     key: key.into(),
                     doc_count: entry.doc_count as u64,
                     sub_aggregation: entry

src/aggregation/metric/average.rs

@@ -2,7 +2,8 @@ use std::fmt::Debug;
 
 use serde::{Deserialize, Serialize};
 
-use super::{IntermediateStats, SegmentStatsCollector};
+use super::*;
+use crate::aggregation::*;
 
 /// A single-value metric aggregation that computes the average of numeric values that are
 /// extracted from the aggregated documents.
@@ -24,7 +25,7 @@ pub struct AverageAggregation {
     /// By default they will be ignored but it is also possible to treat them as if they had a
     /// value. Examples in JSON format:
     /// { "field": "my_numbers", "missing": "10.0" }
-    #[serde(default)]
+    #[serde(default, deserialize_with = "deserialize_option_f64")]
     pub missing: Option<f64>,
 }
 
@@ -65,3 +66,71 @@ impl IntermediateAverage {
         self.stats.finalize().avg
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn deserialization_with_missing_test1() {
+        let json = r#"{
+            "field": "score",
+            "missing": "10.0"
+        }"#;
+        let avg: AverageAggregation = serde_json::from_str(json).unwrap();
+        assert_eq!(avg.field, "score");
+        assert_eq!(avg.missing, Some(10.0));
+        // no dot
+        let json = r#"{
+            "field": "score",
+            "missing": "10"
+        }"#;
+        let avg: AverageAggregation = serde_json::from_str(json).unwrap();
+        assert_eq!(avg.field, "score");
+        assert_eq!(avg.missing, Some(10.0));
+
+        // from value
+        let avg: AverageAggregation = serde_json::from_value(json!({
+            "field": "score_f64",
+            "missing": 10u64,
+        }))
+        .unwrap();
+        assert_eq!(avg.missing, Some(10.0));
+        // from value
+        let avg: AverageAggregation = serde_json::from_value(json!({
+            "field": "score_f64",
+            "missing": 10u32,
+        }))
+        .unwrap();
+        assert_eq!(avg.missing, Some(10.0));
+        let avg: AverageAggregation = serde_json::from_value(json!({
+            "field": "score_f64",
+            "missing": 10i8,
+        }))
+        .unwrap();
+        assert_eq!(avg.missing, Some(10.0));
+    }
+
+    #[test]
+    fn deserialization_with_missing_test_fail() {
+        let json = r#"{
+            "field": "score",
+            "missing": "a"
+        }"#;
+        let avg: Result<AverageAggregation, _> = serde_json::from_str(json);
+        assert!(avg.is_err());
+        assert!(avg
+            .unwrap_err()
+            .to_string()
+            .contains("Failed to parse f64 from string: \"a\""));
+
+        // Disallow NaN
+        let json = r#"{
+            "field": "score",
+            "missing": "NaN"
+        }"#;
+        let avg: Result<AverageAggregation, _> = serde_json::from_str(json);
+        assert!(avg.is_err());
+        assert!(avg.unwrap_err().to_string().contains("NaN"));
+    }
+}

src/aggregation/metric/count.rs

@@ -2,7 +2,8 @@ use std::fmt::Debug;
 
 use serde::{Deserialize, Serialize};
 
-use super::{IntermediateStats, SegmentStatsCollector};
+use super::*;
+use crate::aggregation::*;
 
 /// A single-value metric aggregation that counts the number of values that are
 /// extracted from the aggregated documents.
@@ -24,7 +25,7 @@ pub struct CountAggregation {
     /// By default they will be ignored but it is also possible to treat them as if they had a
     /// value. Examples in JSON format:
     /// { "field": "my_numbers", "missing": "10.0" }
-    #[serde(default)]
+    #[serde(default, deserialize_with = "deserialize_option_f64")]
     pub missing: Option<f64>,
 }
 

src/aggregation/metric/max.rs

@@ -2,7 +2,8 @@ use std::fmt::Debug;
 
 use serde::{Deserialize, Serialize};
 
-use super::{IntermediateStats, SegmentStatsCollector};
+use super::*;
+use crate::aggregation::*;
 
 /// A single-value metric aggregation that computes the maximum of numeric values that are
 /// extracted from the aggregated documents.
@@ -24,7 +25,7 @@ pub struct MaxAggregation {
     /// By default they will be ignored but it is also possible to treat them as if they had a
     /// value. Examples in JSON format:
     /// { "field": "my_numbers", "missing": "10.0" }
-    #[serde(default)]
+    #[serde(default, deserialize_with = "deserialize_option_f64")]
     pub missing: Option<f64>,
 }
 

src/aggregation/metric/min.rs

@@ -2,7 +2,8 @@ use std::fmt::Debug;
 
 use serde::{Deserialize, Serialize};
 
-use super::{IntermediateStats, SegmentStatsCollector};
+use super::*;
+use crate::aggregation::*;
 
 /// A single-value metric aggregation that computes the minimum of numeric values that are
 /// extracted from the aggregated documents.
@@ -24,7 +25,7 @@ pub struct MinAggregation {
     /// By default they will be ignored but it is also possible to treat them as if they had a
     /// value. Examples in JSON format:
     /// { "field": "my_numbers", "missing": "10.0" }
-    #[serde(default)]
+    #[serde(default, deserialize_with = "deserialize_option_f64")]
     pub missing: Option<f64>,
 }
 

src/aggregation/metric/mod.rs

@@ -23,6 +23,8 @@ mod min;
 mod percentiles;
 mod stats;
 mod sum;
+mod top_hits;
+
 pub use average::*;
 pub use count::*;
 pub use max::*;
@@ -32,6 +34,7 @@ use rustc_hash::FxHashMap;
 use serde::{Deserialize, Serialize};
 pub use stats::*;
 pub use sum::*;
+pub use top_hits::*;
 
 /// Single-metric aggregations use this common result structure.
 ///
@@ -81,6 +84,27 @@ pub struct PercentilesMetricResult {
     pub values: PercentileValues,
 }
 
+/// The top_hits metric results entry
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+pub struct TopHitsVecEntry {
+    /// The sort values of the document, depending on the sort criteria in the request.
+    pub sort: Vec<Option<u64>>,
+
+    /// Search results, for queries that include field retrieval requests
+    /// (`docvalue_fields`).
+    #[serde(flatten)]
+    pub search_results: FieldRetrivalResult,
+}
+
+/// The top_hits metric aggregation results a list of top hits by sort criteria.
+///
+/// The main reason for wrapping it in `hits` is to match elasticsearch output structure.
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+pub struct TopHitsMetricResult {
+    /// The result of the top_hits metric.
+    pub hits: Vec<TopHitsVecEntry>,
+}
+
 #[cfg(test)]
 mod tests {
     use crate::aggregation::agg_req::Aggregations;

src/aggregation/metric/percentiles.rs

@@ -11,7 +11,7 @@ use crate::aggregation::intermediate_agg_result::{
     IntermediateAggregationResult, IntermediateAggregationResults, IntermediateMetricResult,
 };
 use crate::aggregation::segment_agg_result::SegmentAggregationCollector;
-use crate::aggregation::{f64_from_fastfield_u64, f64_to_fastfield_u64, AggregationError};
+use crate::aggregation::*;
 use crate::{DocId, TantivyError};
 
 /// # Percentiles
@@ -84,7 +84,11 @@ pub struct PercentilesAggregationReq {
     /// By default they will be ignored but it is also possible to treat them as if they had a
     /// value. Examples in JSON format:
     /// { "field": "my_numbers", "missing": "10.0" }
-    #[serde(skip_serializing_if = "Option::is_none", default)]
+    #[serde(
+        skip_serializing_if = "Option::is_none",
+        default,
+        deserialize_with = "deserialize_option_f64"
+    )]
     pub missing: Option<f64>,
 }
 fn default_percentiles() -> &'static [f64] {
@@ -133,7 +137,6 @@ pub(crate) struct SegmentPercentilesCollector {
     field_type: ColumnType,
     pub(crate) percentiles: PercentilesCollector,
     pub(crate) accessor_idx: usize,
-    val_cache: Vec<u64>,
     missing: Option<u64>,
 }
 
@@ -243,7 +246,6 @@ impl SegmentPercentilesCollector {
             field_type,
             percentiles: PercentilesCollector::new(),
             accessor_idx,
-            val_cache: Default::default(),
             missing,
         })
     }

src/aggregation/metric/stats.rs

@@ -9,7 +9,7 @@ use crate::aggregation::intermediate_agg_result::{
     IntermediateAggregationResult, IntermediateAggregationResults, IntermediateMetricResult,
 };
 use crate::aggregation::segment_agg_result::SegmentAggregationCollector;
-use crate::aggregation::{f64_from_fastfield_u64, f64_to_fastfield_u64};
+use crate::aggregation::*;
 use crate::{DocId, TantivyError};
 
 /// A multi-value metric aggregation that computes a collection of statistics on numeric values that
@@ -33,7 +33,7 @@ pub struct StatsAggregation {
     /// By default they will be ignored but it is also possible to treat them as if they had a
     /// value. Examples in JSON format:
     /// { "field": "my_numbers", "missing": "10.0" }
-    #[serde(default)]
+    #[serde(default, deserialize_with = "deserialize_option_f64")]
     pub missing: Option<f64>,
 }
 
@@ -580,6 +580,30 @@ mod tests {
             })
         );
 
+        // From string
+        let agg_req: Aggregations = serde_json::from_value(json!({
+            "my_stats": {
+                "stats": {
+                    "field": "json.partially_empty",
+                    "missing": "0.0"
+                },
+            }
+        }))
+        .unwrap();
+
+        let res = exec_request_with_query(agg_req, &index, None)?;
+
+        assert_eq!(
+            res["my_stats"],
+            json!({
+                "avg":  2.5,
+                "count": 4,
+                "max": 10.0,
+                "min": 0.0,
+                "sum": 10.0
+            })
+        );
+
         Ok(())
     }
 

src/aggregation/metric/sum.rs

@@ -2,7 +2,8 @@ use std::fmt::Debug;
 
 use serde::{Deserialize, Serialize};
 
-use super::{IntermediateStats, SegmentStatsCollector};
+use super::*;
+use crate::aggregation::*;
 
 /// A single-value metric aggregation that sums up numeric values that are
 /// extracted from the aggregated documents.
@@ -24,7 +25,7 @@ pub struct SumAggregation {
     /// By default they will be ignored but it is also possible to treat them as if they had a
     /// value. Examples in JSON format:
     /// { "field": "my_numbers", "missing": "10.0" }
-    #[serde(default)]
+    #[serde(default, deserialize_with = "deserialize_option_f64")]
     pub missing: Option<f64>,
 }
 

src/aggregation/metric/top_hits.rs

@@ -0,0 +1,837 @@
+use std::collections::HashMap;
+use std::fmt::Formatter;
+
+use columnar::{ColumnarReader, DynamicColumn};
+use regex::Regex;
+use serde::ser::SerializeMap;
+use serde::{Deserialize, Deserializer, Serialize, Serializer};
+
+use super::{TopHitsMetricResult, TopHitsVecEntry};
+use crate::aggregation::bucket::Order;
+use crate::aggregation::intermediate_agg_result::{
+    IntermediateAggregationResult, IntermediateMetricResult,
+};
+use crate::aggregation::segment_agg_result::SegmentAggregationCollector;
+use crate::collector::TopNComputer;
+use crate::schema::term::JSON_PATH_SEGMENT_SEP_STR;
+use crate::schema::OwnedValue;
+use crate::{DocAddress, DocId, SegmentOrdinal};
+
+/// # Top Hits
+///
+/// The top hits aggregation is a useful tool to answer questions like:
+/// - "What are the most recent posts by each author?"
+/// - "What are the most popular items in each category?"
+///
+/// It does so by keeping track of the most relevant document being aggregated,
+/// in terms of a sort criterion that can consist of multiple fields and their
+/// sort-orders (ascending or descending).
+///
+/// `top_hits` should not be used as a top-level aggregation. It is intended to be
+/// used as a sub-aggregation, inside a `terms` aggregation or a `filters` aggregation,
+/// for example.
+///
+/// Note that this aggregator does not return the actual document addresses, but
+/// rather a list of the values of the fields that were requested to be retrieved.
+/// These values can be specified in the `docvalue_fields` parameter, which can include
+/// a list of fast fields to be retrieved. At the moment, only fast fields are supported
+/// but it is possible that we support the `fields` parameter to retrieve any stored
+/// field in the future.
+///
+/// The following example demonstrates a request for the top_hits aggregation:
+/// ```JSON
+/// {
+///     "aggs": {
+///         "top_authors": {
+///             "terms": {
+///                 "field": "author",
+///                 "size": 5
+///             }
+///         },
+///         "aggs": {
+///             "top_hits": {
+///                 "size": 2,
+///                 "from": 0
+///                 "sort": [
+///                     { "date": "desc" }
+///                 ]
+///                 "docvalue_fields": ["date", "title", "iden"]
+///             }
+///         }
+/// }
+/// ```
+///
+/// This request will return an object containing the top two documents, sorted
+/// by the `date` field in descending order. You can also sort by multiple fields, which
+/// helps to resolve ties. The aggregation object for each bucket will look like:
+/// ```JSON
+/// {
+///     "hits": [
+///         {
+///           "score": [<time_u64>],
+///           "docvalue_fields": {
+///             "date": "<date_RFC3339>",
+///             "title": "<title>",
+///             "iden": "<iden>"
+///           }
+///         },
+///         {
+///           "score": [<time_u64>]
+///           "docvalue_fields": {
+///             "date": "<date_RFC3339>",
+///             "title": "<title>",
+///             "iden": "<iden>"
+///           }
+///         }
+///     ]
+/// }
+/// ```
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Default)]
+pub struct TopHitsAggregation {
+    sort: Vec<KeyOrder>,
+    size: usize,
+    from: Option<usize>,
+
+    #[serde(flatten)]
+    retrieval: RetrievalFields,
+}
+
+const fn default_doc_value_fields() -> Vec<String> {
+    Vec::new()
+}
+
+/// Search query spec for each matched document
+/// TODO: move this to a common module
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Default)]
+pub struct RetrievalFields {
+    /// The fast fields to return for each hit.
+    /// This is the only variant supported for now.
+    /// TODO: support the {field, format} variant for custom formatting.
+    #[serde(rename = "docvalue_fields")]
+    #[serde(default = "default_doc_value_fields")]
+    pub doc_value_fields: Vec<String>,
+}
+
+/// Search query result for each matched document
+/// TODO: move this to a common module
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Default)]
+pub struct FieldRetrivalResult {
+    /// The fast fields returned for each hit.
+    #[serde(rename = "docvalue_fields")]
+    #[serde(skip_serializing_if = "HashMap::is_empty")]
+    pub doc_value_fields: HashMap<String, OwnedValue>,
+}
+
+impl RetrievalFields {
+    fn get_field_names(&self) -> Vec<&str> {
+        self.doc_value_fields.iter().map(|s| s.as_str()).collect()
+    }
+
+    fn resolve_field_names(&mut self, reader: &ColumnarReader) -> crate::Result<()> {
+        // Tranform a glob (`pattern*`, for example) into a regex::Regex (`^pattern.*$`)
+        let globbed_string_to_regex = |glob: &str| {
+            // Replace `*` glob with `.*` regex
+            let sanitized = format!("^{}$", regex::escape(glob).replace(r"\*", ".*"));
+            Regex::new(&sanitized.replace('*', ".*")).map_err(|e| {
+                crate::TantivyError::SchemaError(format!(
+                    "Invalid regex '{}' in docvalue_fields: {}",
+                    glob, e
+                ))
+            })
+        };
+        self.doc_value_fields = self
+            .doc_value_fields
+            .iter()
+            .map(|field| {
+                if !field.contains('*')
+                    && reader
+                        .iter_columns()?
+                        .any(|(name, _)| name.as_str() == field)
+                {
+                    return Ok(vec![field.to_owned()]);
+                }
+
+                let pattern = globbed_string_to_regex(field)?;
+                let fields = reader
+                    .iter_columns()?
+                    .map(|(name, _)| {
+                        // normalize path from internal fast field repr
+                        name.replace(JSON_PATH_SEGMENT_SEP_STR, ".")
+                    })
+                    .filter(|name| pattern.is_match(name))
+                    .collect::<Vec<_>>();
+                assert!(
+                    !fields.is_empty(),
+                    "No fields matched the glob '{}' in docvalue_fields",
+                    field
+                );
+                Ok(fields)
+            })
+            .collect::<crate::Result<Vec<_>>>()?
+            .into_iter()
+            .flatten()
+            .collect();
+
+        Ok(())
+    }
+
+    fn get_document_field_data(
+        &self,
+        accessors: &HashMap<String, Vec<DynamicColumn>>,
+        doc_id: DocId,
+    ) -> FieldRetrivalResult {
+        let dvf = self
+            .doc_value_fields
+            .iter()
+            .map(|field| {
+                let accessors = accessors
+                    .get(field)
+                    .unwrap_or_else(|| panic!("field '{}' not found in accessors", field));
+
+                let values: Vec<OwnedValue> = accessors
+                    .iter()
+                    .flat_map(|accessor| match accessor {
+                        DynamicColumn::U64(accessor) => accessor
+                            .values_for_doc(doc_id)
+                            .map(OwnedValue::U64)
+                            .collect::<Vec<_>>(),
+                        DynamicColumn::I64(accessor) => accessor
+                            .values_for_doc(doc_id)
+                            .map(OwnedValue::I64)
+                            .collect::<Vec<_>>(),
+                        DynamicColumn::F64(accessor) => accessor
+                            .values_for_doc(doc_id)
+                            .map(OwnedValue::F64)
+                            .collect::<Vec<_>>(),
+                        DynamicColumn::Bytes(accessor) => accessor
+                            .term_ords(doc_id)
+                            .map(|term_ord| {
+                                let mut buffer = vec![];
+                                assert!(
+                                    accessor
+                                        .ord_to_bytes(term_ord, &mut buffer)
+                                        .expect("could not read term dictionary"),
+                                    "term corresponding to term_ord does not exist"
+                                );
+                                OwnedValue::Bytes(buffer)
+                            })
+                            .collect::<Vec<_>>(),
+                        DynamicColumn::Str(accessor) => accessor
+                            .term_ords(doc_id)
+                            .map(|term_ord| {
+                                let mut buffer = vec![];
+                                assert!(
+                                    accessor
+                                        .ord_to_bytes(term_ord, &mut buffer)
+                                        .expect("could not read term dictionary"),
+                                    "term corresponding to term_ord does not exist"
+                                );
+                                OwnedValue::Str(String::from_utf8(buffer).unwrap())
+                            })
+                            .collect::<Vec<_>>(),
+                        DynamicColumn::Bool(accessor) => accessor
+                            .values_for_doc(doc_id)
+                            .map(OwnedValue::Bool)
+                            .collect::<Vec<_>>(),
+                        DynamicColumn::IpAddr(accessor) => accessor
+                            .values_for_doc(doc_id)
+                            .map(OwnedValue::IpAddr)
+                            .collect::<Vec<_>>(),
+                        DynamicColumn::DateTime(accessor) => accessor
+                            .values_for_doc(doc_id)
+                            .map(OwnedValue::Date)
+                            .collect::<Vec<_>>(),
+                    })
+                    .collect();
+
+                (field.to_owned(), OwnedValue::Array(values))
+            })
+            .collect();
+        FieldRetrivalResult {
+            doc_value_fields: dvf,
+        }
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Default)]
+struct KeyOrder {
+    field: String,
+    order: Order,
+}
+
+impl Serialize for KeyOrder {
+    fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
+        let KeyOrder { field, order } = self;
+        let mut map = serializer.serialize_map(Some(1))?;
+        map.serialize_entry(field, order)?;
+        map.end()
+    }
+}
+
+impl<'de> Deserialize<'de> for KeyOrder {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where D: Deserializer<'de> {
+        let mut k_o = <HashMap<String, Order>>::deserialize(deserializer)?.into_iter();
+        let (k, v) = k_o.next().ok_or(serde::de::Error::custom(
+            "Expected exactly one key-value pair in KeyOrder, found none",
+        ))?;
+        if k_o.next().is_some() {
+            return Err(serde::de::Error::custom(
+                "Expected exactly one key-value pair in KeyOrder, found more",
+            ));
+        }
+        Ok(Self { field: k, order: v })
+    }
+}
+
+impl TopHitsAggregation {
+    /// Validate and resolve field retrieval parameters
+    pub fn validate_and_resolve(&mut self, reader: &ColumnarReader) -> crate::Result<()> {
+        self.retrieval.resolve_field_names(reader)
+    }
+
+    /// Return fields accessed by the aggregator, in order.
+    pub fn field_names(&self) -> Vec<&str> {
+        self.sort
+            .iter()
+            .map(|KeyOrder { field, .. }| field.as_str())
+            .collect()
+    }
+
+    /// Return fields accessed by the aggregator's value retrieval.
+    pub fn value_field_names(&self) -> Vec<&str> {
+        self.retrieval.get_field_names()
+    }
+}
+
+/// Holds a single comparable doc feature, and the order in which it should be sorted.
+#[derive(Clone, Serialize, Deserialize, Debug)]
+struct ComparableDocFeature {
+    /// Stores any u64-mappable feature.
+    value: Option<u64>,
+    /// Sort order for the doc feature
+    order: Order,
+}
+
+impl Ord for ComparableDocFeature {
+    fn cmp(&self, other: &Self) -> std::cmp::Ordering {
+        let invert = |cmp: std::cmp::Ordering| match self.order {
+            Order::Asc => cmp,
+            Order::Desc => cmp.reverse(),
+        };
+
+        match (self.value, other.value) {
+            (Some(self_value), Some(other_value)) => invert(self_value.cmp(&other_value)),
+            (Some(_), None) => std::cmp::Ordering::Greater,
+            (None, Some(_)) => std::cmp::Ordering::Less,
+            (None, None) => std::cmp::Ordering::Equal,
+        }
+    }
+}
+
+impl PartialOrd for ComparableDocFeature {
+    fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
+        Some(self.cmp(other))
+    }
+}
+
+impl PartialEq for ComparableDocFeature {
+    fn eq(&self, other: &Self) -> bool {
+        self.value.cmp(&other.value) == std::cmp::Ordering::Equal
+    }
+}
+
+impl Eq for ComparableDocFeature {}
+
+#[derive(Clone, Serialize, Deserialize, Debug)]
+struct ComparableDocFeatures(Vec<ComparableDocFeature>, FieldRetrivalResult);
+
+impl Ord for ComparableDocFeatures {
+    fn cmp(&self, other: &Self) -> std::cmp::Ordering {
+        for (self_feature, other_feature) in self.0.iter().zip(other.0.iter()) {
+            let cmp = self_feature.cmp(other_feature);
+            if cmp != std::cmp::Ordering::Equal {
+                return cmp;
+            }
+        }
+        std::cmp::Ordering::Equal
+    }
+}
+
+impl PartialOrd for ComparableDocFeatures {
+    fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
+        Some(self.cmp(other))
+    }
+}
+
+impl PartialEq for ComparableDocFeatures {
+    fn eq(&self, other: &Self) -> bool {
+        self.cmp(other) == std::cmp::Ordering::Equal
+    }
+}
+
+impl Eq for ComparableDocFeatures {}
+
+/// The TopHitsCollector used for collecting over segments and merging results.
+#[derive(Clone, Serialize, Deserialize)]
+pub struct TopHitsCollector {
+    req: TopHitsAggregation,
+    top_n: TopNComputer<ComparableDocFeatures, DocAddress, false>,
+}
+
+impl Default for TopHitsCollector {
+    fn default() -> Self {
+        Self {
+            req: TopHitsAggregation::default(),
+            top_n: TopNComputer::new(1),
+        }
+    }
+}
+
+impl std::fmt::Debug for TopHitsCollector {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("TopHitsCollector")
+            .field("req", &self.req)
+            .field("top_n_threshold", &self.top_n.threshold)
+            .finish()
+    }
+}
+
+impl std::cmp::PartialEq for TopHitsCollector {
+    fn eq(&self, _other: &Self) -> bool {
+        false
+    }
+}
+
+impl TopHitsCollector {
+    fn collect(&mut self, features: ComparableDocFeatures, doc: DocAddress) {
+        self.top_n.push(features, doc);
+    }
+
+    pub(crate) fn merge_fruits(&mut self, other_fruit: Self) -> crate::Result<()> {
+        for doc in other_fruit.top_n.into_vec() {
+            self.collect(doc.feature, doc.doc);
+        }
+        Ok(())
+    }
+
+    /// Finalize by converting self into the final result form
+    pub fn finalize(self) -> TopHitsMetricResult {
+        let mut hits: Vec<TopHitsVecEntry> = self
+            .top_n
+            .into_sorted_vec()
+            .into_iter()
+            .map(|doc| TopHitsVecEntry {
+                sort: doc.feature.0.iter().map(|f| f.value).collect(),
+                search_results: doc.feature.1,
+            })
+            .collect();
+
+        // Remove the first `from` elements
+        // Truncating from end would be more efficient, but we need to truncate from the front
+        // because `into_sorted_vec` gives us a descending order because of the inverted
+        // `Ord` semantics of the heap elements.
+        hits.drain(..self.req.from.unwrap_or(0));
+        TopHitsMetricResult { hits }
+    }
+}
+
+#[derive(Clone)]
+pub(crate) struct SegmentTopHitsCollector {
+    segment_ordinal: SegmentOrdinal,
+    accessor_idx: usize,
+    inner_collector: TopHitsCollector,
+}
+
+impl SegmentTopHitsCollector {
+    pub fn from_req(
+        req: &TopHitsAggregation,
+        accessor_idx: usize,
+        segment_ordinal: SegmentOrdinal,
+    ) -> Self {
+        Self {
+            inner_collector: TopHitsCollector {
+                req: req.clone(),
+                top_n: TopNComputer::new(req.size + req.from.unwrap_or(0)),
+            },
+            segment_ordinal,
+            accessor_idx,
+        }
+    }
+}
+
+impl std::fmt::Debug for SegmentTopHitsCollector {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("SegmentTopHitsCollector")
+            .field("segment_id", &self.segment_ordinal)
+            .field("accessor_idx", &self.accessor_idx)
+            .field("inner_collector", &self.inner_collector)
+            .finish()
+    }
+}
+
+impl SegmentAggregationCollector for SegmentTopHitsCollector {
+    fn add_intermediate_aggregation_result(
+        self: Box<Self>,
+        agg_with_accessor: &crate::aggregation::agg_req_with_accessor::AggregationsWithAccessor,
+        results: &mut crate::aggregation::intermediate_agg_result::IntermediateAggregationResults,
+    ) -> crate::Result<()> {
+        let name = agg_with_accessor.aggs.keys[self.accessor_idx].to_string();
+        let intermediate_result = IntermediateMetricResult::TopHits(self.inner_collector);
+        results.push(
+            name,
+            IntermediateAggregationResult::Metric(intermediate_result),
+        )
+    }
+
+    fn collect(
+        &mut self,
+        doc_id: crate::DocId,
+        agg_with_accessor: &mut crate::aggregation::agg_req_with_accessor::AggregationsWithAccessor,
+    ) -> crate::Result<()> {
+        let accessors = &agg_with_accessor.aggs.values[self.accessor_idx].accessors;
+        let value_accessors = &agg_with_accessor.aggs.values[self.accessor_idx].value_accessors;
+        let features: Vec<ComparableDocFeature> = self
+            .inner_collector
+            .req
+            .sort
+            .iter()
+            .enumerate()
+            .map(|(idx, KeyOrder { order, .. })| {
+                let order = *order;
+                let value = accessors
+                    .get(idx)
+                    .expect("could not find field in accessors")
+                    .0
+                    .values_for_doc(doc_id)
+                    .next();
+                ComparableDocFeature { value, order }
+            })
+            .collect();
+
+        let retrieval_result = self
+            .inner_collector
+            .req
+            .retrieval
+            .get_document_field_data(value_accessors, doc_id);
+
+        self.inner_collector.collect(
+            ComparableDocFeatures(features, retrieval_result),
+            DocAddress {
+                segment_ord: self.segment_ordinal,
+                doc_id,
+            },
+        );
+        Ok(())
+    }
+
+    fn collect_block(
+        &mut self,
+        docs: &[crate::DocId],
+        agg_with_accessor: &mut crate::aggregation::agg_req_with_accessor::AggregationsWithAccessor,
+    ) -> crate::Result<()> {
+        // TODO: Consider getting fields with the column block accessor and refactor this.
+        // ---
+        // Would the additional complexity of getting fields with the column_block_accessor
+        // make sense here? Probably yes, but I want to get a first-pass review first
+        // before proceeding.
+        for doc in docs {
+            self.collect(*doc, agg_with_accessor)?;
+        }
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use common::DateTime;
+    use pretty_assertions::assert_eq;
+    use serde_json::Value;
+    use time::macros::datetime;
+
+    use super::{ComparableDocFeature, ComparableDocFeatures, Order};
+    use crate::aggregation::agg_req::Aggregations;
+    use crate::aggregation::agg_result::AggregationResults;
+    use crate::aggregation::bucket::tests::get_test_index_from_docs;
+    use crate::aggregation::tests::get_test_index_from_values;
+    use crate::aggregation::AggregationCollector;
+    use crate::collector::ComparableDoc;
+    use crate::query::AllQuery;
+    use crate::schema::OwnedValue as SchemaValue;
+
+    fn invert_order(cmp_feature: ComparableDocFeature) -> ComparableDocFeature {
+        let ComparableDocFeature { value, order } = cmp_feature;
+        let order = match order {
+            Order::Asc => Order::Desc,
+            Order::Desc => Order::Asc,
+        };
+        ComparableDocFeature { value, order }
+    }
+
+    fn collector_with_capacity(capacity: usize) -> super::TopHitsCollector {
+        super::TopHitsCollector {
+            top_n: super::TopNComputer::new(capacity),
+            ..Default::default()
+        }
+    }
+
+    fn invert_order_features(cmp_features: ComparableDocFeatures) -> ComparableDocFeatures {
+        let ComparableDocFeatures(cmp_features, search_results) = cmp_features;
+        let cmp_features = cmp_features
+            .into_iter()
+            .map(invert_order)
+            .collect::<Vec<_>>();
+        ComparableDocFeatures(cmp_features, search_results)
+    }
+
+    #[test]
+    fn test_comparable_doc_feature() -> crate::Result<()> {
+        let small = ComparableDocFeature {
+            value: Some(1),
+            order: Order::Asc,
+        };
+        let big = ComparableDocFeature {
+            value: Some(2),
+            order: Order::Asc,
+        };
+        let none = ComparableDocFeature {
+            value: None,
+            order: Order::Asc,
+        };
+
+        assert!(small < big);
+        assert!(none < small);
+        assert!(none < big);
+
+        let small = invert_order(small);
+        let big = invert_order(big);
+        let none = invert_order(none);
+
+        assert!(small > big);
+        assert!(none < small);
+        assert!(none < big);
+
+        Ok(())
+    }
+
+    #[test]
+    fn test_comparable_doc_features() -> crate::Result<()> {
+        let features_1 = ComparableDocFeatures(
+            vec![ComparableDocFeature {
+                value: Some(1),
+                order: Order::Asc,
+            }],
+            Default::default(),
+        );
+
+        let features_2 = ComparableDocFeatures(
+            vec![ComparableDocFeature {
+                value: Some(2),
+                order: Order::Asc,
+            }],
+            Default::default(),
+        );
+
+        assert!(features_1 < features_2);
+
+        assert!(invert_order_features(features_1.clone()) > invert_order_features(features_2));
+
+        Ok(())
+    }
+
+    #[test]
+    fn test_aggregation_top_hits_empty_index() -> crate::Result<()> {
+        let values = vec![];
+
+        let index = get_test_index_from_values(false, &values)?;
+
+        let d: Aggregations = serde_json::from_value(json!({
+            "top_hits_req": {
+                "top_hits": {
+                    "size": 2,
+                    "sort": [
+                        { "date": "desc" }
+                    ],
+                    "from": 0,
+                }
+        }
+        }))
+        .unwrap();
+
+        let collector = AggregationCollector::from_aggs(d, Default::default());
+
+        let reader = index.reader()?;
+        let searcher = reader.searcher();
+        let agg_res: AggregationResults = searcher.search(&AllQuery, &collector).unwrap();
+
+        let res: Value = serde_json::from_str(
+            &serde_json::to_string(&agg_res).expect("JSON serialization failed"),
+        )
+        .expect("JSON parsing failed");
+
+        assert_eq!(
+            res,
+            json!({
+                "top_hits_req": {
+                    "hits": []
+                }
+            })
+        );
+
+        Ok(())
+    }
+
+    #[test]
+    fn test_top_hits_collector_single_feature() -> crate::Result<()> {
+        let docs = vec![
+            ComparableDoc::<_, _, false> {
+                doc: crate::DocAddress {
+                    segment_ord: 0,
+                    doc_id: 0,
+                },
+                feature: ComparableDocFeatures(
+                    vec![ComparableDocFeature {
+                        value: Some(1),
+                        order: Order::Asc,
+                    }],
+                    Default::default(),
+                ),
+            },
+            ComparableDoc {
+                doc: crate::DocAddress {
+                    segment_ord: 0,
+                    doc_id: 2,
+                },
+                feature: ComparableDocFeatures(
+                    vec![ComparableDocFeature {
+                        value: Some(3),
+                        order: Order::Asc,
+                    }],
+                    Default::default(),
+                ),
+            },
+            ComparableDoc {
+                doc: crate::DocAddress {
+                    segment_ord: 0,
+                    doc_id: 1,
+                },
+                feature: ComparableDocFeatures(
+                    vec![ComparableDocFeature {
+                        value: Some(5),
+                        order: Order::Asc,
+                    }],
+                    Default::default(),
+                ),
+            },
+        ];
+
+        let mut collector = collector_with_capacity(3);
+        for doc in docs.clone() {
+            collector.collect(doc.feature, doc.doc);
+        }
+
+        let res = collector.finalize();
+
+        assert_eq!(
+            res,
+            super::TopHitsMetricResult {
+                hits: vec![
+                    super::TopHitsVecEntry {
+                        sort: vec![docs[0].feature.0[0].value],
+                        search_results: Default::default(),
+                    },
+                    super::TopHitsVecEntry {
+                        sort: vec![docs[1].feature.0[0].value],
+                        search_results: Default::default(),
+                    },
+                    super::TopHitsVecEntry {
+                        sort: vec![docs[2].feature.0[0].value],
+                        search_results: Default::default(),
+                    },
+                ]
+            }
+        );
+
+        Ok(())
+    }
+
+    fn test_aggregation_top_hits(merge_segments: bool) -> crate::Result<()> {
+        let docs = vec![
+            vec![
+                r#"{ "date": "2015-01-02T00:00:00Z", "text": "bbb", "text2": "bbb", "mixed": { "dyn_arr": [1, "2"] } }"#,
+                r#"{ "date": "2017-06-15T00:00:00Z", "text": "ccc", "text2": "ddd", "mixed": { "dyn_arr": [3, "4"] } }"#,
+            ],
+            vec![
+                r#"{ "text": "aaa", "text2": "bbb", "date": "2018-01-02T00:00:00Z", "mixed": { "dyn_arr": ["9", 8] } }"#,
+                r#"{ "text": "aaa", "text2": "bbb", "date": "2016-01-02T00:00:00Z", "mixed": { "dyn_arr": ["7", 6] } }"#,
+            ],
+        ];
+
+        let index = get_test_index_from_docs(merge_segments, &docs)?;
+
+        let d: Aggregations = serde_json::from_value(json!({
+            "top_hits_req": {
+                "top_hits": {
+                    "size": 2,
+                    "sort": [
+                        { "date": "desc" }
+                    ],
+                    "from": 1,
+                    "docvalue_fields": [
+                        "date",
+                        "tex*",
+                        "mixed.*",
+                    ],
+                }
+        }
+        }))?;
+
+        let collector = AggregationCollector::from_aggs(d, Default::default());
+        let reader = index.reader()?;
+        let searcher = reader.searcher();
+
+        let agg_res =
+            serde_json::to_value(searcher.search(&AllQuery, &collector).unwrap()).unwrap();
+
+        let date_2017 = datetime!(2017-06-15 00:00:00 UTC);
+        let date_2016 = datetime!(2016-01-02 00:00:00 UTC);
+
+        assert_eq!(
+            agg_res["top_hits_req"],
+            json!({
+                "hits": [
+                    {
+                        "sort": [common::i64_to_u64(date_2017.unix_timestamp_nanos() as i64)],
+                        "docvalue_fields": {
+                            "date": [ SchemaValue::Date(DateTime::from_utc(date_2017)) ],
+                            "text": [ "ccc" ],
+                            "text2": [ "ddd" ],
+                            "mixed.dyn_arr": [ 3, "4" ],
+                        }
+                    },
+                    {
+                        "sort": [common::i64_to_u64(date_2016.unix_timestamp_nanos() as i64)],
+                        "docvalue_fields": {
+                            "date": [ SchemaValue::Date(DateTime::from_utc(date_2016)) ],
+                            "text": [ "aaa" ],
+                            "text2": [ "bbb" ],
+                            "mixed.dyn_arr": [ 6, "7" ],
+                        }
+                    }
+                ]
+            }),
+        );
+
+        Ok(())
+    }
+
+    #[test]
+    fn test_aggregation_top_hits_single_segment() -> crate::Result<()> {
+        test_aggregation_top_hits(true)
+    }
+
+    #[test]
+    fn test_aggregation_top_hits_multi_segment() -> crate::Result<()> {
+        test_aggregation_top_hits(false)
+    }
+}

src/aggregation/mod.rs

@@ -145,6 +145,8 @@ mod agg_tests;
 
 mod agg_bench;
 
+use core::fmt;
+
 pub use agg_limits::AggregationLimits;
 pub use collector::{
     AggregationCollector, AggregationSegmentCollector, DistributedAggregationCollector,
@@ -154,7 +156,106 @@ use columnar::{ColumnType, MonotonicallyMappableToU64};
 pub(crate) use date::format_date;
 pub use error::AggregationError;
 use itertools::Itertools;
-use serde::{Deserialize, Serialize};
+use serde::de::{self, Visitor};
+use serde::{Deserialize, Deserializer, Serialize};
+
+fn parse_str_into_f64<E: de::Error>(value: &str) -> Result<f64, E> {
+    let parsed = value.parse::<f64>().map_err(|_err| {
+        de::Error::custom(format!("Failed to parse f64 from string: {:?}", value))
+    })?;
+
+    // Check if the parsed value is NaN or infinity
+    if parsed.is_nan() || parsed.is_infinite() {
+        Err(de::Error::custom(format!(
+            "Value is not a valid f64 (NaN or Infinity): {:?}",
+            value
+        )))
+    } else {
+        Ok(parsed)
+    }
+}
+
+/// deserialize Option<f64> from string or float
+pub(crate) fn deserialize_option_f64<'de, D>(deserializer: D) -> Result<Option<f64>, D::Error>
+where D: Deserializer<'de> {
+    struct StringOrFloatVisitor;
+
+    impl<'de> Visitor<'de> for StringOrFloatVisitor {
+        type Value = Option<f64>;
+
+        fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
+            formatter.write_str("a string or a float")
+        }
+
+        fn visit_str<E>(self, value: &str) -> Result<Self::Value, E>
+        where E: de::Error {
+            parse_str_into_f64(value).map(Some)
+        }
+
+        fn visit_f64<E>(self, value: f64) -> Result<Self::Value, E>
+        where E: de::Error {
+            Ok(Some(value))
+        }
+
+        fn visit_i64<E>(self, value: i64) -> Result<Self::Value, E>
+        where E: de::Error {
+            Ok(Some(value as f64))
+        }
+
+        fn visit_u64<E>(self, value: u64) -> Result<Self::Value, E>
+        where E: de::Error {
+            Ok(Some(value as f64))
+        }
+
+        fn visit_none<E>(self) -> Result<Self::Value, E>
+        where E: de::Error {
+            Ok(None)
+        }
+
+        fn visit_unit<E>(self) -> Result<Self::Value, E>
+        where E: de::Error {
+            Ok(None)
+        }
+    }
+
+    deserializer.deserialize_any(StringOrFloatVisitor)
+}
+
+/// deserialize f64 from string or float
+pub(crate) fn deserialize_f64<'de, D>(deserializer: D) -> Result<f64, D::Error>
+where D: Deserializer<'de> {
+    struct StringOrFloatVisitor;
+
+    impl<'de> Visitor<'de> for StringOrFloatVisitor {
+        type Value = f64;
+
+        fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
+            formatter.write_str("a string or a float")
+        }
+
+        fn visit_str<E>(self, value: &str) -> Result<Self::Value, E>
+        where E: de::Error {
+            parse_str_into_f64(value)
+        }
+
+        fn visit_f64<E>(self, value: f64) -> Result<Self::Value, E>
+        where E: de::Error {
+            Ok(value)
+        }
+
+        fn visit_i64<E>(self, value: i64) -> Result<Self::Value, E>
+        where E: de::Error {
+            Ok(value as f64)
+        }
+
+        fn visit_u64<E>(self, value: u64) -> Result<Self::Value, E>
+        where E: de::Error {
+            Ok(value as f64)
+        }
+    }
+
+    deserializer.deserialize_any(StringOrFloatVisitor)
+}
 
 /// Represents an associative array `(key => values)` in a very efficient manner.
 #[derive(PartialEq, Serialize, Deserialize)]
@@ -281,6 +382,7 @@ pub(crate) fn f64_from_fastfield_u64(val: u64, field_type: &ColumnType) -> f64 {
         ColumnType::U64 => val as f64,
         ColumnType::I64 | ColumnType::DateTime => i64::from_u64(val) as f64,
         ColumnType::F64 => f64::from_u64(val),
+        ColumnType::Bool => val as f64,
         _ => {
             panic!("unexpected type {field_type:?}. This should not happen")
         }
@@ -301,6 +403,7 @@ pub(crate) fn f64_to_fastfield_u64(val: f64, field_type: &ColumnType) -> Option<
         ColumnType::U64 => Some(val as u64),
         ColumnType::I64 | ColumnType::DateTime => Some((val as i64).to_u64()),
         ColumnType::F64 => Some(val.to_u64()),
+        ColumnType::Bool => Some(val as u64),
         _ => None,
     }
 }

src/aggregation/segment_agg_result.rs

@@ -16,6 +16,7 @@ use super::metric::{
     SumAggregation,
 };
 use crate::aggregation::bucket::TermMissingAgg;
+use crate::aggregation::metric::SegmentTopHitsCollector;
 
 pub(crate) trait SegmentAggregationCollector: CollectorClone + Debug {
     fn add_intermediate_aggregation_result(
@@ -160,6 +161,11 @@ pub(crate) fn build_single_agg_segment_collector(
                 accessor_idx,
             )?,
         )),
+        TopHits(top_hits_req) => Ok(Box::new(SegmentTopHitsCollector::from_req(
+            top_hits_req,
+            accessor_idx,
+            req.segment_ordinal,
+        ))),
     }
 }
 

src/collector/facet_collector.rs

@@ -410,6 +410,7 @@ impl SegmentCollector for FacetSegmentCollector {
 
 /// Intermediary result of the `FacetCollector` that stores
 /// the facet counts for all the segments.
+#[derive(Default, Clone)]
 pub struct FacetCounts {
     facet_counts: BTreeMap<Facet, u64>,
 }
@@ -493,7 +494,7 @@ mod tests {
     use super::{FacetCollector, FacetCounts};
     use crate::collector::facet_collector::compress_mapping;
     use crate::collector::Count;
-    use crate::core::Index;
+    use crate::index::Index;
     use crate::query::{AllQuery, QueryParser, TermQuery};
     use crate::schema::{Facet, FacetOptions, IndexRecordOption, Schema, TantivyDocument};
     use crate::{IndexWriter, Term};

src/collector/mod.rs

@@ -97,6 +97,7 @@ pub use self::multi_collector::{FruitHandle, MultiCollector, MultiFruit};
 mod top_collector;
 
 mod top_score_collector;
+pub use self::top_collector::ComparableDoc;
 pub use self::top_score_collector::{TopDocs, TopNComputer};
 
 mod custom_score_top_collector;

src/collector/tests.rs

@@ -2,7 +2,7 @@ use columnar::{BytesColumn, Column};
 
 use super::*;
 use crate::collector::{Count, FilterCollector, TopDocs};
-use crate::core::SegmentReader;
+use crate::index::SegmentReader;
 use crate::query::{AllQuery, QueryParser};
 use crate::schema::{Schema, FAST, TEXT};
 use crate::time::format_description::well_known::Rfc3339;

src/collector/top_collector.rs

@@ -1,47 +1,58 @@
 use std::cmp::Ordering;
 use std::marker::PhantomData;
 
+use serde::{Deserialize, Serialize};
+
 use super::top_score_collector::TopNComputer;
 use crate::{DocAddress, DocId, SegmentOrdinal, SegmentReader};
 
 /// Contains a feature (field, score, etc.) of a document along with the document address.
 ///
-/// It has a custom implementation of `PartialOrd` that reverses the order. This is because the
-/// default Rust heap is a max heap, whereas a min heap is needed.
-///
-/// Additionally, it guarantees stable sorting: in case of a tie on the feature, the document
+/// It guarantees stable sorting: in case of a tie on the feature, the document
 /// address is used.
 ///
+/// The REVERSE_ORDER generic parameter controls whether the by-feature order
+/// should be reversed, which is useful for achieving for example largest-first
+/// semantics without having to wrap the feature in a `Reverse`.
+///
 /// WARNING: equality is not what you would expect here.
 /// Two elements are equal if their feature is equal, and regardless of whether `doc`
 /// is equal. This should be perfectly fine for this usage, but let's make sure this
 /// struct is never public.
-pub(crate) struct ComparableDoc<T, D> {
+#[derive(Clone, Default, Serialize, Deserialize)]
+pub struct ComparableDoc<T, D, const REVERSE_ORDER: bool = false> {
+    /// The feature of the document. In practice, this is
+    /// is any type that implements `PartialOrd`.
     pub feature: T,
+    /// The document address. In practice, this is any
+    /// type that implements `PartialOrd`, and is guaranteed
+    /// to be unique for each document.
     pub doc: D,
 }
-impl<T: std::fmt::Debug, D: std::fmt::Debug> std::fmt::Debug for ComparableDoc<T, D> {
+impl<T: std::fmt::Debug, D: std::fmt::Debug, const R: bool> std::fmt::Debug
+    for ComparableDoc<T, D, R>
+{
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.debug_struct("ComparableDoc")
+        f.debug_struct(format!("ComparableDoc<_, _ {R}").as_str())
             .field("feature", &self.feature)
             .field("doc", &self.doc)
             .finish()
     }
 }
 
-impl<T: PartialOrd, D: PartialOrd> PartialOrd for ComparableDoc<T, D> {
+impl<T: PartialOrd, D: PartialOrd, const R: bool> PartialOrd for ComparableDoc<T, D, R> {
     fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
         Some(self.cmp(other))
     }
 }
 
-impl<T: PartialOrd, D: PartialOrd> Ord for ComparableDoc<T, D> {
+impl<T: PartialOrd, D: PartialOrd, const R: bool> Ord for ComparableDoc<T, D, R> {
     #[inline]
     fn cmp(&self, other: &Self) -> Ordering {
-        // Reversed to make BinaryHeap work as a min-heap
-        let by_feature = other
+        let by_feature = self
             .feature
-            .partial_cmp(&self.feature)
+            .partial_cmp(&other.feature)
+            .map(|ord| if R { ord.reverse() } else { ord })
             .unwrap_or(Ordering::Equal);
 
         let lazy_by_doc_address = || self.doc.partial_cmp(&other.doc).unwrap_or(Ordering::Equal);
@@ -53,13 +64,13 @@ impl<T: PartialOrd, D: PartialOrd> Ord for ComparableDoc<T, D> {
     }
 }
 
-impl<T: PartialOrd, D: PartialOrd> PartialEq for ComparableDoc<T, D> {
+impl<T: PartialOrd, D: PartialOrd, const R: bool> PartialEq for ComparableDoc<T, D, R> {
     fn eq(&self, other: &Self) -> bool {
         self.cmp(other) == Ordering::Equal
     }
 }
 
-impl<T: PartialOrd, D: PartialOrd> Eq for ComparableDoc<T, D> {}
+impl<T: PartialOrd, D: PartialOrd, const R: bool> Eq for ComparableDoc<T, D, R> {}
 
 pub(crate) struct TopCollector<T> {
     pub limit: usize,
@@ -99,10 +110,10 @@ where T: PartialOrd + Clone
         if self.limit == 0 {
             return Ok(Vec::new());
         }
-        let mut top_collector = TopNComputer::new(self.limit + self.offset);
+        let mut top_collector: TopNComputer<_, _> = TopNComputer::new(self.limit + self.offset);
         for child_fruit in children {
             for (feature, doc) in child_fruit {
-                top_collector.push(ComparableDoc { feature, doc });
+                top_collector.push(feature, doc);
             }
         }
 
@@ -143,6 +154,8 @@ where T: PartialOrd + Clone
 /// The theoretical complexity for collecting the top `K` out of `n` documents
 /// is `O(n + K)`.
 pub(crate) struct TopSegmentCollector<T> {
+    /// We reverse the order of the feature in order to
+    /// have top-semantics instead of bottom semantics.
     topn_computer: TopNComputer<T, DocId>,
     segment_ord: u32,
 }
@@ -180,7 +193,7 @@ impl<T: PartialOrd + Clone> TopSegmentCollector<T> {
     /// will compare the lowest scoring item with the given one and keep whichever is greater.
     #[inline]
     pub fn collect(&mut self, doc: DocId, feature: T) {
-        self.topn_computer.push(ComparableDoc { feature, doc });
+        self.topn_computer.push(feature, doc);
     }
 }
 

src/collector/top_score_collector.rs

@@ -3,6 +3,8 @@ use std::marker::PhantomData;
 use std::sync::Arc;
 
 use columnar::ColumnValues;
+use serde::de::DeserializeOwned;
+use serde::{Deserialize, Serialize};
 
 use super::Collector;
 use crate::collector::custom_score_top_collector::CustomScoreTopCollector;
@@ -309,7 +311,7 @@ impl TopDocs {
     ///
     /// To comfortably work with `u64`s, `i64`s, `f64`s, or `date`s, please refer to
     /// the [.order_by_fast_field(...)](TopDocs::order_by_fast_field) method.
-    fn order_by_u64_field(
+    pub fn order_by_u64_field(
         self,
         field: impl ToString,
         order: Order,
@@ -663,7 +665,7 @@ impl Collector for TopDocs {
         reader: &SegmentReader,
     ) -> crate::Result<<Self::Child as SegmentCollector>::Fruit> {
         let heap_len = self.0.limit + self.0.offset;
-        let mut top_n = TopNComputer::new(heap_len);
+        let mut top_n: TopNComputer<_, _> = TopNComputer::new(heap_len);
 
         if let Some(alive_bitset) = reader.alive_bitset() {
             let mut threshold = Score::MIN;
@@ -672,21 +674,13 @@ impl Collector for TopDocs {
                 if alive_bitset.is_deleted(doc) {
                     return threshold;
                 }
-                let doc = ComparableDoc {
-                    feature: score,
-                    doc,
-                };
-                top_n.push(doc);
+                top_n.push(score, doc);
                 threshold = top_n.threshold.unwrap_or(Score::MIN);
                 threshold
             })?;
         } else {
             weight.for_each_pruning(Score::MIN, reader, &mut |doc, score| {
-                let doc = ComparableDoc {
-                    feature: score,
-                    doc,
-                };
-                top_n.push(doc);
+                top_n.push(score, doc);
                 top_n.threshold.unwrap_or(Score::MIN)
             })?;
         }
@@ -725,17 +719,65 @@ impl SegmentCollector for TopScoreSegmentCollector {
 
 /// Fast TopN Computation
 ///
+/// Capacity of the vec is 2 * top_n.
+/// The buffer is truncated to the top_n elements when it reaches the capacity of the Vec.
+/// That means capacity has special meaning and should be carried over when cloning or serializing.
+///
 /// For TopN == 0, it will be relative expensive.
-pub struct TopNComputer<Score, DocId> {
-    buffer: Vec<ComparableDoc<Score, DocId>>,
+#[derive(Serialize, Deserialize)]
+#[serde(from = "TopNComputerDeser<Score, D, REVERSE_ORDER>")]
+pub struct TopNComputer<Score, D, const REVERSE_ORDER: bool = true> {
+    /// The buffer reverses sort order to get top-semantics instead of bottom-semantics
+    buffer: Vec<ComparableDoc<Score, D, REVERSE_ORDER>>,
     top_n: usize,
     pub(crate) threshold: Option<Score>,
 }
+// Intermediate struct for TopNComputer for deserialization, to keep vec capacity
+#[derive(Deserialize)]
+struct TopNComputerDeser<Score, D, const REVERSE_ORDER: bool> {
+    buffer: Vec<ComparableDoc<Score, D, REVERSE_ORDER>>,
+    top_n: usize,
+    threshold: Option<Score>,
+}
 
-impl<Score, DocId> TopNComputer<Score, DocId>
+// Custom clone to keep capacity
+impl<Score: Clone, D: Clone, const REVERSE_ORDER: bool> Clone
+    for TopNComputer<Score, D, REVERSE_ORDER>
+{
+    fn clone(&self) -> Self {
+        let mut buffer_clone = Vec::with_capacity(self.buffer.capacity());
+        buffer_clone.extend(self.buffer.iter().cloned());
+
+        TopNComputer {
+            buffer: buffer_clone,
+            top_n: self.top_n,
+            threshold: self.threshold.clone(),
+        }
+    }
+}
+
+impl<Score, D, const R: bool> From<TopNComputerDeser<Score, D, R>> for TopNComputer<Score, D, R> {
+    fn from(mut value: TopNComputerDeser<Score, D, R>) -> Self {
+        let expected_cap = value.top_n.max(1) * 2;
+        let current_cap = value.buffer.capacity();
+        if current_cap < expected_cap {
+            value.buffer.reserve_exact(expected_cap - current_cap);
+        } else {
+            value.buffer.shrink_to(expected_cap);
+        }
+
+        TopNComputer {
+            buffer: value.buffer,
+            top_n: value.top_n,
+            threshold: value.threshold,
+        }
+    }
+}
+
+impl<Score, D, const R: bool> TopNComputer<Score, D, R>
 where
     Score: PartialOrd + Clone,
-    DocId: Ord + Clone,
+    D: Serialize + DeserializeOwned + Ord + Clone,
 {
     /// Create a new `TopNComputer`.
     /// Internally it will allocate a buffer of size `2 * top_n`.
@@ -748,10 +790,12 @@ where
         }
     }
 
+    /// Push a new document to the top n.
+    /// If the document is below the current threshold, it will be ignored.
     #[inline]
-    pub(crate) fn push(&mut self, doc: ComparableDoc<Score, DocId>) {
+    pub fn push(&mut self, feature: Score, doc: D) {
         if let Some(last_median) = self.threshold.clone() {
-            if doc.feature < last_median {
+            if feature < last_median {
                 return;
             }
         }
@@ -766,7 +810,7 @@ where
         let uninit = self.buffer.spare_capacity_mut();
         // This cannot panic, because we truncate_median will at least remove one element, since
         // the min capacity is 2.
-        uninit[0].write(doc);
+        uninit[0].write(ComparableDoc { doc, feature });
         // This is safe because it would panic in the line above
         unsafe {
             self.buffer.set_len(self.buffer.len() + 1);
@@ -785,13 +829,24 @@ where
         median_score
     }
 
-    pub(crate) fn into_sorted_vec(mut self) -> Vec<ComparableDoc<Score, DocId>> {
+    /// Returns the top n elements in sorted order.
+    pub fn into_sorted_vec(mut self) -> Vec<ComparableDoc<Score, D, R>> {
         if self.buffer.len() > self.top_n {
             self.truncate_top_n();
         }
         self.buffer.sort_unstable();
         self.buffer
     }
+
+    /// Returns the top n elements in stored order.
+    /// Useful if you do not need the elements in sorted order,
+    /// for example when merging the results of multiple segments.
+    pub fn into_vec(mut self) -> Vec<ComparableDoc<Score, D, R>> {
+        if self.buffer.len() > self.top_n {
+            self.truncate_top_n();
+        }
+        self.buffer
+    }
 }
 
 #[cfg(test)]
@@ -825,49 +880,44 @@ mod tests {
             crate::assert_nearly_equals!(result.0, expected.0);
         }
     }
+    #[test]
+    fn test_topn_computer_serde() {
+        let computer: TopNComputer<u32, u32> = TopNComputer::new(1);
+
+        let computer_ser = serde_json::to_string(&computer).unwrap();
+        let mut computer: TopNComputer<u32, u32> = serde_json::from_str(&computer_ser).unwrap();
+
+        computer.push(1u32, 5u32);
+        computer.push(1u32, 0u32);
+        computer.push(1u32, 7u32);
+
+        assert_eq!(
+            computer.into_sorted_vec(),
+            &[ComparableDoc {
+                feature: 1u32,
+                doc: 0u32,
+            },]
+        );
+    }
 
     #[test]
     fn test_empty_topn_computer() {
         let mut computer: TopNComputer<u32, u32> = TopNComputer::new(0);
 
-        computer.push(ComparableDoc {
-            feature: 1u32,
-            doc: 1u32,
-        });
-        computer.push(ComparableDoc {
-            feature: 1u32,
-            doc: 2u32,
-        });
-        computer.push(ComparableDoc {
-            feature: 1u32,
-            doc: 3u32,
-        });
+        computer.push(1u32, 1u32);
+        computer.push(1u32, 2u32);
+        computer.push(1u32, 3u32);
         assert!(computer.into_sorted_vec().is_empty());
     }
     #[test]
     fn test_topn_computer() {
         let mut computer: TopNComputer<u32, u32> = TopNComputer::new(2);
 
-        computer.push(ComparableDoc {
-            feature: 1u32,
-            doc: 1u32,
-        });
-        computer.push(ComparableDoc {
-            feature: 2u32,
-            doc: 2u32,
-        });
-        computer.push(ComparableDoc {
-            feature: 3u32,
-            doc: 3u32,
-        });
-        computer.push(ComparableDoc {
-            feature: 2u32,
-            doc: 4u32,
-        });
-        computer.push(ComparableDoc {
-            feature: 1u32,
-            doc: 5u32,
-        });
+        computer.push(1u32, 1u32);
+        computer.push(2u32, 2u32);
+        computer.push(3u32, 3u32);
+        computer.push(2u32, 4u32);
+        computer.push(1u32, 5u32);
         assert_eq!(
             computer.into_sorted_vec(),
             &[
@@ -889,10 +939,7 @@ mod tests {
             let mut computer: TopNComputer<u32, u32> = TopNComputer::new(top_n);
 
             for _ in 0..1 + top_n * 2 {
-                computer.push(ComparableDoc {
-                    feature: 1u32,
-                    doc: 1u32,
-                });
+                computer.push(1u32, 1u32);
             }
             let _vals = computer.into_sorted_vec();
         }

src/core/json_utils.rs

@@ -62,6 +62,14 @@ impl IndexingPositionsPerPath {
     }
 }
 
+/// Convert JSON_PATH_SEGMENT_SEP to a dot.
+pub fn json_path_sep_to_dot(path: &mut str) {
+    // This is safe since we are replacing a ASCII character by another ASCII character.
+    unsafe {
+        replace_in_place(JSON_PATH_SEGMENT_SEP, b'.', path.as_bytes_mut());
+    }
+}
+
 #[allow(clippy::too_many_arguments)]
 pub(crate) fn index_json_values<'a, V: Value<'a>>(
     doc: DocId,
@@ -320,7 +328,7 @@ pub struct JsonTermWriter<'a> {
 /// In other words,
 /// - `k8s.node` ends up as `["k8s", "node"]`.
 /// - `k8s\.node` ends up as `["k8s.node"]`.
-fn split_json_path(json_path: &str) -> Vec<String> {
+pub fn split_json_path(json_path: &str) -> Vec<String> {
     let mut escaped_state: bool = false;
     let mut json_path_segments = Vec::new();
     let mut buffer = String::new();

src/core/mod.rs

@@ -1,32 +1,14 @@
 mod executor;
-pub mod index;
-mod index_meta;
-mod inverted_index_reader;
 #[doc(hidden)]
 pub mod json_utils;
 pub mod searcher;
-mod segment;
-mod segment_component;
-mod segment_id;
-mod segment_reader;
-mod single_segment_index_writer;
 
 use std::path::Path;
 
 use once_cell::sync::Lazy;
 
 pub use self::executor::Executor;
-pub use self::index::{Index, IndexBuilder};
-pub use self::index_meta::{
-    IndexMeta, IndexSettings, IndexSortByField, Order, SegmentMeta, SegmentMetaInventory,
-};
-pub use self::inverted_index_reader::InvertedIndexReader;
 pub use self::searcher::{Searcher, SearcherGeneration};
-pub use self::segment::Segment;
-pub use self::segment_component::SegmentComponent;
-pub use self::segment_id::SegmentId;
-pub use self::segment_reader::SegmentReader;
-pub use self::single_segment_index_writer::SingleSegmentIndexWriter;
 
 /// The meta file contains all the information about the list of segments and the schema
 /// of the index.

src/core/searcher.rs

@@ -3,7 +3,8 @@ use std::sync::Arc;
 use std::{fmt, io};
 
 use crate::collector::Collector;
-use crate::core::{Executor, SegmentReader};
+use crate::core::Executor;
+use crate::index::SegmentReader;
 use crate::query::{Bm25StatisticsProvider, EnableScoring, Query};
 use crate::schema::document::DocumentDeserialize;
 use crate::schema::{Schema, Term};

src/core/tests.rs

@@ -1,12 +1,13 @@
 use crate::collector::Count;
 use crate::directory::{RamDirectory, WatchCallback};
-use crate::indexer::NoMergePolicy;
+use crate::indexer::{LogMergePolicy, NoMergePolicy};
+use crate::json_utils::JsonTermWriter;
 use crate::query::TermQuery;
-use crate::schema::{Field, IndexRecordOption, Schema, INDEXED, STRING, TEXT};
+use crate::schema::{Field, IndexRecordOption, Schema, Type, INDEXED, STRING, TEXT};
 use crate::tokenizer::TokenizerManager;
 use crate::{
-    Directory, Index, IndexBuilder, IndexReader, IndexSettings, IndexWriter, ReloadPolicy,
-    SegmentId, TantivyDocument, Term,
+    Directory, DocSet, Index, IndexBuilder, IndexReader, IndexSettings, IndexWriter, Postings,
+    ReloadPolicy, SegmentId, TantivyDocument, Term,
 };
 
 #[test]
@@ -344,3 +345,132 @@ fn test_merging_segment_update_docfreq() {
     let term_info = inv_index.get_term_info(&term).unwrap().unwrap();
     assert_eq!(term_info.doc_freq, 12);
 }
+
+// motivated by https://github.com/quickwit-oss/quickwit/issues/4130
+#[test]
+fn test_positions_merge_bug_non_text_json_vint() {
+    let mut schema_builder = Schema::builder();
+    let field = schema_builder.add_json_field("dynamic", TEXT);
+    let schema = schema_builder.build();
+    let index = Index::create_in_ram(schema.clone());
+    let mut writer: IndexWriter = index.writer_for_tests().unwrap();
+    let mut merge_policy = LogMergePolicy::default();
+    merge_policy.set_min_num_segments(2);
+    writer.set_merge_policy(Box::new(merge_policy));
+    // Here a string would work.
+    let doc_json = r#"{"tenant_id":75}"#;
+    let vals = serde_json::from_str(doc_json).unwrap();
+    let mut doc = TantivyDocument::default();
+    doc.add_object(field, vals);
+    writer.add_document(doc.clone()).unwrap();
+    writer.commit().unwrap();
+    writer.add_document(doc.clone()).unwrap();
+    writer.commit().unwrap();
+    writer.wait_merging_threads().unwrap();
+    let reader = index.reader().unwrap();
+    assert_eq!(reader.searcher().segment_readers().len(), 1);
+}
+
+// Same as above but with bitpacked blocks
+#[test]
+fn test_positions_merge_bug_non_text_json_bitpacked_block() {
+    let mut schema_builder = Schema::builder();
+    let field = schema_builder.add_json_field("dynamic", TEXT);
+    let schema = schema_builder.build();
+    let index = Index::create_in_ram(schema.clone());
+    let mut writer: IndexWriter = index.writer_for_tests().unwrap();
+    let mut merge_policy = LogMergePolicy::default();
+    merge_policy.set_min_num_segments(2);
+    writer.set_merge_policy(Box::new(merge_policy));
+    // Here a string would work.
+    let doc_json = r#"{"tenant_id":75}"#;
+    let vals = serde_json::from_str(doc_json).unwrap();
+    let mut doc = TantivyDocument::default();
+    doc.add_object(field, vals);
+    for _ in 0..128 {
+        writer.add_document(doc.clone()).unwrap();
+    }
+    writer.commit().unwrap();
+    writer.add_document(doc.clone()).unwrap();
+    writer.commit().unwrap();
+    writer.wait_merging_threads().unwrap();
+    let reader = index.reader().unwrap();
+    assert_eq!(reader.searcher().segment_readers().len(), 1);
+}
+
+#[test]
+fn test_non_text_json_term_freq() {
+    let mut schema_builder = Schema::builder();
+    let field = schema_builder.add_json_field("dynamic", TEXT);
+    let schema = schema_builder.build();
+    let index = Index::create_in_ram(schema.clone());
+    let mut writer: IndexWriter = index.writer_for_tests().unwrap();
+    // Here a string would work.
+    let doc_json = r#"{"tenant_id":75}"#;
+    let vals = serde_json::from_str(doc_json).unwrap();
+    let mut doc = TantivyDocument::default();
+    doc.add_object(field, vals);
+    writer.add_document(doc.clone()).unwrap();
+    writer.commit().unwrap();
+    let reader = index.reader().unwrap();
+    assert_eq!(reader.searcher().segment_readers().len(), 1);
+    let searcher = reader.searcher();
+    let segment_reader = searcher.segment_reader(0u32);
+    let inv_idx = segment_reader.inverted_index(field).unwrap();
+    let mut term = Term::with_type_and_field(Type::Json, field);
+    let mut json_term_writer = JsonTermWriter::wrap(&mut term, false);
+    json_term_writer.push_path_segment("tenant_id");
+    json_term_writer.close_path_and_set_type(Type::U64);
+    json_term_writer.set_fast_value(75u64);
+    let postings = inv_idx
+        .read_postings(
+            json_term_writer.term(),
+            IndexRecordOption::WithFreqsAndPositions,
+        )
+        .unwrap()
+        .unwrap();
+    assert_eq!(postings.doc(), 0);
+    assert_eq!(postings.term_freq(), 1u32);
+}
+
+#[test]
+fn test_non_text_json_term_freq_bitpacked() {
+    let mut schema_builder = Schema::builder();
+    let field = schema_builder.add_json_field("dynamic", TEXT);
+    let schema = schema_builder.build();
+    let index = Index::create_in_ram(schema.clone());
+    let mut writer: IndexWriter = index.writer_for_tests().unwrap();
+    // Here a string would work.
+    let doc_json = r#"{"tenant_id":75}"#;
+    let vals = serde_json::from_str(doc_json).unwrap();
+    let mut doc = TantivyDocument::default();
+    doc.add_object(field, vals);
+    let num_docs = 132;
+    for _ in 0..num_docs {
+        writer.add_document(doc.clone()).unwrap();
+    }
+    writer.commit().unwrap();
+    let reader = index.reader().unwrap();
+    assert_eq!(reader.searcher().segment_readers().len(), 1);
+    let searcher = reader.searcher();
+    let segment_reader = searcher.segment_reader(0u32);
+    let inv_idx = segment_reader.inverted_index(field).unwrap();
+    let mut term = Term::with_type_and_field(Type::Json, field);
+    let mut json_term_writer = JsonTermWriter::wrap(&mut term, false);
+    json_term_writer.push_path_segment("tenant_id");
+    json_term_writer.close_path_and_set_type(Type::U64);
+    json_term_writer.set_fast_value(75u64);
+    let mut postings = inv_idx
+        .read_postings(
+            json_term_writer.term(),
+            IndexRecordOption::WithFreqsAndPositions,
+        )
+        .unwrap()
+        .unwrap();
+    assert_eq!(postings.doc(), 0);
+    assert_eq!(postings.term_freq(), 1u32);
+    for i in 1..num_docs {
+        assert_eq!(postings.advance(), i);
+        assert_eq!(postings.term_freq(), 1u32);
+    }
+}

src/directory/mmap_directory.rs

@@ -479,6 +479,7 @@ impl Directory for MmapDirectory {
         let file: File = OpenOptions::new()
             .write(true)
             .create(true) //< if the file does not exist yet, create it.
+            .truncate(false)
             .open(full_path)
             .map_err(LockError::wrap_io_error)?;
         if lock.is_blocking {
@@ -673,7 +674,7 @@ mod tests {
             let num_segments = reader.searcher().segment_readers().len();
             assert!(num_segments <= 4);
             let num_components_except_deletes_and_tempstore =
-                crate::core::SegmentComponent::iterator().len() - 2;
+                crate::index::SegmentComponent::iterator().len() - 2;
             let max_num_mmapped = num_components_except_deletes_and_tempstore * num_segments;
             assert_eventually(|| {
                 let num_mmapped = mmap_directory.get_cache_info().mmapped.len();

src/directory/ram_directory.rs

@@ -85,7 +85,7 @@ impl InnerDirectory {
         self.fs
             .get(path)
             .ok_or_else(|| OpenReadError::FileDoesNotExist(PathBuf::from(path)))
-            .map(Clone::clone)
+            .cloned()
     }
 
     fn delete(&mut self, path: &Path) -> result::Result<(), DeleteError> {

src/fastfield/mod.rs

@@ -131,7 +131,7 @@ mod tests {
         }
         let file = directory.open_read(path).unwrap();
 
-        assert_eq!(file.len(), 93);
+        assert_eq!(file.len(), 80);
         let fast_field_readers = FastFieldReaders::open(file, SCHEMA.clone()).unwrap();
         let column = fast_field_readers
             .u64("field")
@@ -181,7 +181,7 @@ mod tests {
             write.terminate().unwrap();
         }
         let file = directory.open_read(path).unwrap();
-        assert_eq!(file.len(), 121);
+        assert_eq!(file.len(), 108);
         let fast_field_readers = FastFieldReaders::open(file, SCHEMA.clone()).unwrap();
         let col = fast_field_readers
             .u64("field")
@@ -214,7 +214,7 @@ mod tests {
             write.terminate().unwrap();
         }
         let file = directory.open_read(path).unwrap();
-        assert_eq!(file.len(), 94);
+        assert_eq!(file.len(), 81);
         let fast_field_readers = FastFieldReaders::open(file, SCHEMA.clone()).unwrap();
         let fast_field_reader = fast_field_readers
             .u64("field")
@@ -246,7 +246,7 @@ mod tests {
             write.terminate().unwrap();
         }
         let file = directory.open_read(path).unwrap();
-        assert_eq!(file.len(), 4489);
+        assert_eq!(file.len(), 4476);
         {
             let fast_field_readers = FastFieldReaders::open(file, SCHEMA.clone()).unwrap();
             let col = fast_field_readers
@@ -279,7 +279,7 @@ mod tests {
             write.terminate().unwrap();
         }
         let file = directory.open_read(path).unwrap();
-        assert_eq!(file.len(), 265);
+        assert_eq!(file.len(), 252);
 
         {
             let fast_field_readers = FastFieldReaders::open(file, schema).unwrap();
@@ -773,7 +773,7 @@ mod tests {
             write.terminate().unwrap();
         }
         let file = directory.open_read(path).unwrap();
-        assert_eq!(file.len(), 102);
+        assert_eq!(file.len(), 84);
         let fast_field_readers = FastFieldReaders::open(file, schema).unwrap();
         let bool_col = fast_field_readers.bool("field_bool").unwrap();
         assert_eq!(bool_col.first(0), Some(true));
@@ -805,7 +805,7 @@ mod tests {
             write.terminate().unwrap();
         }
         let file = directory.open_read(path).unwrap();
-        assert_eq!(file.len(), 114);
+        assert_eq!(file.len(), 96);
         let readers = FastFieldReaders::open(file, schema).unwrap();
         let bool_col = readers.bool("field_bool").unwrap();
         for i in 0..25 {
@@ -830,7 +830,7 @@ mod tests {
             write.terminate().unwrap();
         }
         let file = directory.open_read(path).unwrap();
-        assert_eq!(file.len(), 104);
+        assert_eq!(file.len(), 86);
         let fastfield_readers = FastFieldReaders::open(file, schema).unwrap();
         let col = fastfield_readers.bool("field_bool").unwrap();
         assert_eq!(col.first(0), None);
@@ -1288,11 +1288,18 @@ mod tests {
         index_writer.commit().unwrap();
         let searcher = index.reader().unwrap().searcher();
         let fast_field_reader = searcher.segment_reader(0u32).fast_fields();
+        // Supported for now, maybe dropped in the future.
         let column = fast_field_reader
             .column_opt::<i64>("jsonfield.attr.age")
             .unwrap()
             .unwrap();
         let vals: Vec<i64> = column.values_for_doc(0u32).collect();
         assert_eq!(&vals, &[33]);
+        let column = fast_field_reader
+            .column_opt::<i64>("jsonfield\\.attr.age")
+            .unwrap()
+            .unwrap();
+        let vals: Vec<i64> = column.values_for_doc(0u32).collect();
+        assert_eq!(&vals, &[33]);
     }
 }

src/index/index.rs

@@ -6,23 +6,23 @@ use std::path::PathBuf;
 use std::sync::Arc;
 
 use super::segment::Segment;
-use super::IndexSettings;
-use crate::core::single_segment_index_writer::SingleSegmentIndexWriter;
-use crate::core::{
-    Executor, IndexMeta, SegmentId, SegmentMeta, SegmentMetaInventory, META_FILEPATH,
-};
+use super::segment_reader::merge_field_meta_data;
+use super::{FieldMetadata, IndexSettings};
+use crate::core::{Executor, META_FILEPATH};
 use crate::directory::error::OpenReadError;
 #[cfg(feature = "mmap")]
 use crate::directory::MmapDirectory;
 use crate::directory::{Directory, ManagedDirectory, RamDirectory, INDEX_WRITER_LOCK};
 use crate::error::{DataCorruption, TantivyError};
+use crate::index::{IndexMeta, SegmentId, SegmentMeta, SegmentMetaInventory};
 use crate::indexer::index_writer::{MAX_NUM_THREAD, MEMORY_BUDGET_NUM_BYTES_MIN};
 use crate::indexer::segment_updater::save_metas;
-use crate::indexer::IndexWriter;
+use crate::indexer::{IndexWriter, SingleSegmentIndexWriter};
 use crate::reader::{IndexReader, IndexReaderBuilder};
 use crate::schema::document::Document;
 use crate::schema::{Field, FieldType, Schema};
 use crate::tokenizer::{TextAnalyzer, TokenizerManager};
+use crate::SegmentReader;
 
 fn load_metas(
     directory: &dyn Directory,
@@ -322,6 +322,15 @@ impl Index {
         Ok(())
     }
 
+    /// Custom thread pool by a outer thread pool.
+    pub fn set_shared_multithread_executor(
+        &mut self,
+        shared_thread_pool: Arc<Executor>,
+    ) -> crate::Result<()> {
+        self.executor = shared_thread_pool.clone();
+        Ok(())
+    }
+
     /// Replace the default single thread search executor pool
     /// by a thread pool with as many threads as there are CPUs on the system.
     pub fn set_default_multithread_executor(&mut self) -> crate::Result<()> {
@@ -489,6 +498,28 @@ impl Index {
         self.inventory.all()
     }
 
+    /// Returns the list of fields that have been indexed in the Index.
+    /// The field list includes the field defined in the schema as well as the fields
+    /// that have been indexed as a part of a JSON field.
+    /// The returned field name is the full field name, including the name of the JSON field.
+    ///
+    /// The returned field names can be used in queries.
+    ///
+    /// Notice: If your data contains JSON fields this is **very expensive**, as it requires
+    /// browsing through the inverted index term dictionary and the columnar field dictionary.
+    ///
+    /// Disclaimer: Some fields may not be listed here. For instance, if the schema contains a json
+    /// field that is not indexed nor a fast field but is stored, it is possible for the field
+    /// to not be listed.
+    pub fn fields_metadata(&self) -> crate::Result<Vec<FieldMetadata>> {
+        let segments = self.searchable_segments()?;
+        let fields_metadata: Vec<Vec<FieldMetadata>> = segments
+            .into_iter()
+            .map(|segment| SegmentReader::open(&segment)?.fields_metadata())
+            .collect::<Result<_, _>>()?;
+        Ok(merge_field_meta_data(fields_metadata, &self.schema()))
+    }
+
     /// Creates a new segment_meta (Advanced user only).
     ///
     /// As long as the `SegmentMeta` lives, the files associated with the

src/index/index_meta.rs

@@ -7,7 +7,7 @@ use std::sync::Arc;
 use serde::{Deserialize, Serialize};
 
 use super::SegmentComponent;
-use crate::core::SegmentId;
+use crate::index::SegmentId;
 use crate::schema::Schema;
 use crate::store::Compressor;
 use crate::{Inventory, Opstamp, TrackedObject};
@@ -19,7 +19,7 @@ struct DeleteMeta {
 }
 
 #[derive(Clone, Default)]
-pub struct SegmentMetaInventory {
+pub(crate) struct SegmentMetaInventory {
     inventory: Inventory<InnerSegmentMeta>,
 }
 
@@ -408,7 +408,7 @@ impl fmt::Debug for IndexMeta {
 mod tests {
 
     use super::IndexMeta;
-    use crate::core::index_meta::UntrackedIndexMeta;
+    use crate::index::index_meta::UntrackedIndexMeta;
     use crate::schema::{Schema, TEXT};
     use crate::store::Compressor;
     #[cfg(feature = "zstd-compression")]

src/index/inverted_index_reader.rs

@@ -75,7 +75,7 @@ impl InvertedIndexReader {
     ///
     /// Notice: This requires a full scan and therefore **very expensive**.
     /// TODO: Move to sstable to use the index.
-    pub fn list_fields(&self) -> io::Result<Vec<(String, Type)>> {
+    pub fn list_encoded_fields(&self) -> io::Result<Vec<(String, Type)>> {
         let mut stream = self.termdict.stream()?;
         let mut fields = Vec::new();
         let mut fields_set = FnvHashSet::default();
@@ -266,7 +266,9 @@ impl InvertedIndexReader {
 
     /// Warmup a block postings given a `Term`.
     /// This method is for an advanced usage only.
-    pub async fn warm_postings(&self, term: &Term, with_positions: bool) -> io::Result<()> {
+    ///
+    /// returns a boolean, whether the term was found in the dictionary
+    pub async fn warm_postings(&self, term: &Term, with_positions: bool) -> io::Result<bool> {
         let term_info_opt: Option<TermInfo> = self.get_term_info_async(term).await?;
         if let Some(term_info) = term_info_opt {
             let postings = self
@@ -280,23 +282,27 @@ impl InvertedIndexReader {
             } else {
                 postings.await?;
             }
+            Ok(true)
+        } else {
+            Ok(false)
         }
-        Ok(())
     }
 
     /// Warmup a block postings given a range of `Term`s.
     /// This method is for an advanced usage only.
+    ///
+    /// returns a boolean, whether a term matching the range was found in the dictionary
     pub async fn warm_postings_range(
         &self,
         terms: impl std::ops::RangeBounds<Term>,
         limit: Option<u64>,
         with_positions: bool,
-    ) -> io::Result<()> {
+    ) -> io::Result<bool> {
         let mut term_info = self.get_term_range_async(terms, limit).await?;
 
         let Some(first_terminfo) = term_info.next() else {
             // no key matches, nothing more to load
-            return Ok(());
+            return Ok(false);
         };
 
         let last_terminfo = term_info.last().unwrap_or_else(|| first_terminfo.clone());
@@ -316,7 +322,7 @@ impl InvertedIndexReader {
         } else {
             postings.await?;
         }
-        Ok(())
+        Ok(true)
     }
 
     /// Warmup the block postings for all terms.

src/index/mod.rs

@@ -0,0 +1,22 @@
+//! # Index Module
+//!
+//! The `index` module in Tantivy contains core components to read and write indexes.
+//!
+//! It contains `Index` and `Segment`, where a `Index` consists of one or more `Segment`s.
+
+mod index;
+mod index_meta;
+mod inverted_index_reader;
+mod segment;
+mod segment_component;
+mod segment_id;
+mod segment_reader;
+
+pub use self::index::{Index, IndexBuilder};
+pub(crate) use self::index_meta::SegmentMetaInventory;
+pub use self::index_meta::{IndexMeta, IndexSettings, IndexSortByField, Order, SegmentMeta};
+pub use self::inverted_index_reader::InvertedIndexReader;
+pub use self::segment::Segment;
+pub use self::segment_component::SegmentComponent;
+pub use self::segment_id::SegmentId;
+pub use self::segment_reader::{FieldMetadata, SegmentReader};

src/index/segment.rs

@@ -2,9 +2,9 @@ use std::fmt;
 use std::path::PathBuf;
 
 use super::SegmentComponent;
-use crate::core::{Index, SegmentId, SegmentMeta};
 use crate::directory::error::{OpenReadError, OpenWriteError};
 use crate::directory::{Directory, FileSlice, WritePtr};
+use crate::index::{Index, SegmentId, SegmentMeta};
 use crate::schema::Schema;
 use crate::Opstamp;
 

src/index/segment_reader.rs

@@ -1,12 +1,17 @@
 use std::collections::HashMap;
+use std::ops::BitOrAssign;
 use std::sync::{Arc, RwLock};
 use std::{fmt, io};
 
-use crate::core::{InvertedIndexReader, Segment, SegmentComponent, SegmentId};
+use fnv::FnvHashMap;
+use itertools::Itertools;
+
 use crate::directory::{CompositeFile, FileSlice};
 use crate::error::DataCorruption;
 use crate::fastfield::{intersect_alive_bitsets, AliveBitSet, FacetReader, FastFieldReaders};
 use crate::fieldnorm::{FieldNormReader, FieldNormReaders};
+use crate::index::{InvertedIndexReader, Segment, SegmentComponent, SegmentId};
+use crate::json_utils::json_path_sep_to_dot;
 use crate::schema::{Field, IndexRecordOption, Schema, Type};
 use crate::space_usage::SegmentSpaceUsage;
 use crate::store::StoreReader;
@@ -280,6 +285,103 @@ impl SegmentReader {
         Ok(inv_idx_reader)
     }
 
+    /// Returns the list of fields that have been indexed in the segment.
+    /// The field list includes the field defined in the schema as well as the fields
+    /// that have been indexed as a part of a JSON field.
+    /// The returned field name is the full field name, including the name of the JSON field.
+    ///
+    /// The returned field names can be used in queries.
+    ///
+    /// Notice: If your data contains JSON fields this is **very expensive**, as it requires
+    /// browsing through the inverted index term dictionary and the columnar field dictionary.
+    ///
+    /// Disclaimer: Some fields may not be listed here. For instance, if the schema contains a json
+    /// field that is not indexed nor a fast field but is stored, it is possible for the field
+    /// to not be listed.
+    pub fn fields_metadata(&self) -> crate::Result<Vec<FieldMetadata>> {
+        let mut indexed_fields: Vec<FieldMetadata> = Vec::new();
+        let mut map_to_canonical = FnvHashMap::default();
+        for (field, field_entry) in self.schema().fields() {
+            let field_name = field_entry.name().to_string();
+            let is_indexed = field_entry.is_indexed();
+
+            if is_indexed {
+                let is_json = field_entry.field_type().value_type() == Type::Json;
+                if is_json {
+                    let inv_index = self.inverted_index(field)?;
+                    let encoded_fields_in_index = inv_index.list_encoded_fields()?;
+                    let mut build_path = |field_name: &str, mut json_path: String| {
+                        // In this case we need to map the potential fast field to the field name
+                        // accepted by the query parser.
+                        let create_canonical =
+                            !field_entry.is_expand_dots_enabled() && json_path.contains('.');
+                        if create_canonical {
+                            // Without expand dots enabled dots need to be escaped.
+                            let escaped_json_path = json_path.replace('.', "\\.");
+                            let full_path = format!("{}.{}", field_name, escaped_json_path);
+                            let full_path_unescaped = format!("{}.{}", field_name, &json_path);
+                            map_to_canonical.insert(full_path_unescaped, full_path.to_string());
+                            full_path
+                        } else {
+                            // With expand dots enabled, we can use '.' instead of '\u{1}'.
+                            json_path_sep_to_dot(&mut json_path);
+                            format!("{}.{}", field_name, json_path)
+                        }
+                    };
+                    indexed_fields.extend(
+                        encoded_fields_in_index
+                            .into_iter()
+                            .map(|(name, typ)| (build_path(&field_name, name), typ))
+                            .map(|(field_name, typ)| FieldMetadata {
+                                indexed: true,
+                                stored: false,
+                                field_name,
+                                fast: false,
+                                typ,
+                            }),
+                    );
+                } else {
+                    indexed_fields.push(FieldMetadata {
+                        indexed: true,
+                        stored: false,
+                        field_name: field_name.to_string(),
+                        fast: false,
+                        typ: field_entry.field_type().value_type(),
+                    });
+                }
+            }
+        }
+        let mut fast_fields: Vec<FieldMetadata> = self
+            .fast_fields()
+            .columnar()
+            .iter_columns()?
+            .map(|(mut field_name, handle)| {
+                json_path_sep_to_dot(&mut field_name);
+                // map to canonical path, to avoid similar but different entries.
+                // Eventually we should just accept '.' seperated for all cases.
+                let field_name = map_to_canonical
+                    .get(&field_name)
+                    .unwrap_or(&field_name)
+                    .to_string();
+                FieldMetadata {
+                    indexed: false,
+                    stored: false,
+                    field_name,
+                    fast: true,
+                    typ: Type::from(handle.column_type()),
+                }
+            })
+            .collect();
+        // Since the type is encoded differently in the fast field and in the inverted index,
+        // the order of the fields is not guaranteed to be the same. Therefore, we sort the fields.
+        // If we are sure that the order is the same, we can remove this sort.
+        indexed_fields.sort_unstable();
+        fast_fields.sort_unstable();
+        let merged = merge_field_meta_data(vec![indexed_fields, fast_fields], &self.schema);
+
+        Ok(merged)
+    }
+
     /// Returns the segment id
     pub fn segment_id(&self) -> SegmentId {
         self.segment_id
@@ -330,6 +432,65 @@ impl SegmentReader {
     }
 }
 
+#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
+/// FieldMetadata
+pub struct FieldMetadata {
+    /// The field name
+    // Notice: Don't reorder the declaration of 1.field_name 2.typ, as it is used for ordering by
+    // field_name then typ.
+    pub field_name: String,
+    /// The field type
+    // Notice: Don't reorder the declaration of 1.field_name 2.typ, as it is used for ordering by
+    // field_name then typ.
+    pub typ: Type,
+    /// Is the field indexed for search
+    pub indexed: bool,
+    /// Is the field stored in the doc store
+    pub stored: bool,
+    /// Is the field stored in the columnar storage
+    pub fast: bool,
+}
+impl BitOrAssign for FieldMetadata {
+    fn bitor_assign(&mut self, rhs: Self) {
+        assert!(self.field_name == rhs.field_name);
+        assert!(self.typ == rhs.typ);
+        self.indexed |= rhs.indexed;
+        self.stored |= rhs.stored;
+        self.fast |= rhs.fast;
+    }
+}
+
+// Maybe too slow for the high cardinality case
+fn is_field_stored(field_name: &str, schema: &Schema) -> bool {
+    schema
+        .find_field(field_name)
+        .map(|(field, _path)| schema.get_field_entry(field).is_stored())
+        .unwrap_or(false)
+}
+
+/// Helper to merge the field metadata from multiple segments.
+pub fn merge_field_meta_data(
+    field_metadatas: Vec<Vec<FieldMetadata>>,
+    schema: &Schema,
+) -> Vec<FieldMetadata> {
+    let mut merged_field_metadata = Vec::new();
+    for (_key, mut group) in &field_metadatas
+        .into_iter()
+        .kmerge_by(|left, right| left < right)
+        // TODO: Remove allocation
+        .group_by(|el| (el.field_name.to_string(), el.typ))
+    {
+        let mut merged: FieldMetadata = group.next().unwrap();
+        for el in group {
+            merged |= el;
+        }
+        // Currently is_field_stored is maybe too slow for the high cardinality case
+        merged.stored = is_field_stored(&merged.field_name, schema);
+        merged_field_metadata.push(merged);
+    }
+    merged_field_metadata
+}
+
 fn intersect_alive_bitset(
     left_opt: Option<AliveBitSet>,
     right_opt: Option<AliveBitSet>,
@@ -353,10 +514,128 @@ impl fmt::Debug for SegmentReader {
 
 #[cfg(test)]
 mod test {
-    use crate::core::Index;
-    use crate::schema::{Schema, Term, STORED, TEXT};
+    use super::*;
+    use crate::index::Index;
+    use crate::schema::{Schema, SchemaBuilder, Term, STORED, TEXT};
     use crate::{DocId, IndexWriter};
 
+    #[test]
+    fn test_merge_field_meta_data_same() {
+        let schema = SchemaBuilder::new().build();
+        let field_metadata1 = FieldMetadata {
+            field_name: "a".to_string(),
+            typ: crate::schema::Type::Str,
+            indexed: true,
+            stored: false,
+            fast: true,
+        };
+        let field_metadata2 = FieldMetadata {
+            field_name: "a".to_string(),
+            typ: crate::schema::Type::Str,
+            indexed: true,
+            stored: false,
+            fast: true,
+        };
+        let res = merge_field_meta_data(
+            vec![vec![field_metadata1.clone()], vec![field_metadata2]],
+            &schema,
+        );
+        assert_eq!(res, vec![field_metadata1]);
+    }
+    #[test]
+    fn test_merge_field_meta_data_different() {
+        let schema = SchemaBuilder::new().build();
+        let field_metadata1 = FieldMetadata {
+            field_name: "a".to_string(),
+            typ: crate::schema::Type::Str,
+            indexed: false,
+            stored: false,
+            fast: true,
+        };
+        let field_metadata2 = FieldMetadata {
+            field_name: "b".to_string(),
+            typ: crate::schema::Type::Str,
+            indexed: false,
+            stored: false,
+            fast: true,
+        };
+        let field_metadata3 = FieldMetadata {
+            field_name: "a".to_string(),
+            typ: crate::schema::Type::Str,
+            indexed: true,
+            stored: false,
+            fast: false,
+        };
+        let res = merge_field_meta_data(
+            vec![
+                vec![field_metadata1.clone(), field_metadata2.clone()],
+                vec![field_metadata3],
+            ],
+            &schema,
+        );
+        let field_metadata_expected1 = FieldMetadata {
+            field_name: "a".to_string(),
+            typ: crate::schema::Type::Str,
+            indexed: true,
+            stored: false,
+            fast: true,
+        };
+        assert_eq!(res, vec![field_metadata_expected1, field_metadata2.clone()]);
+    }
+    #[test]
+    fn test_merge_field_meta_data_merge() {
+        use pretty_assertions::assert_eq;
+        let get_meta_data = |name: &str, typ: Type| FieldMetadata {
+            field_name: name.to_string(),
+            typ,
+            indexed: false,
+            stored: false,
+            fast: true,
+        };
+        let schema = SchemaBuilder::new().build();
+        let mut metas = vec![get_meta_data("d", Type::Str), get_meta_data("e", Type::U64)];
+        metas.sort();
+        let res = merge_field_meta_data(vec![vec![get_meta_data("e", Type::Str)], metas], &schema);
+        assert_eq!(
+            res,
+            vec![
+                get_meta_data("d", Type::Str),
+                get_meta_data("e", Type::Str),
+                get_meta_data("e", Type::U64),
+            ]
+        );
+    }
+    #[test]
+    fn test_merge_field_meta_data_bitxor() {
+        let field_metadata1 = FieldMetadata {
+            field_name: "a".to_string(),
+            typ: crate::schema::Type::Str,
+            indexed: false,
+            stored: false,
+            fast: true,
+        };
+        let field_metadata2 = FieldMetadata {
+            field_name: "a".to_string(),
+            typ: crate::schema::Type::Str,
+            indexed: true,
+            stored: false,
+            fast: false,
+        };
+        let field_metadata_expected = FieldMetadata {
+            field_name: "a".to_string(),
+            typ: crate::schema::Type::Str,
+            indexed: true,
+            stored: false,
+            fast: true,
+        };
+        let mut res1 = field_metadata1.clone();
+        res1 |= field_metadata2.clone();
+        let mut res2 = field_metadata2.clone();
+        res2 |= field_metadata1;
+        assert_eq!(res1, field_metadata_expected);
+        assert_eq!(res2, field_metadata_expected);
+    }
+
     #[test]
     fn test_num_alive() -> crate::Result<()> {
         let mut schema_builder = Schema::builder();

src/indexer/index_writer.rs

@@ -9,10 +9,10 @@ use smallvec::smallvec;
 use super::operation::{AddOperation, UserOperation};
 use super::segment_updater::SegmentUpdater;
 use super::{AddBatch, AddBatchReceiver, AddBatchSender, PreparedCommit};
-use crate::core::{Index, Segment, SegmentComponent, SegmentId, SegmentMeta, SegmentReader};
 use crate::directory::{DirectoryLock, GarbageCollectionResult, TerminatingWrite};
 use crate::error::TantivyError;
 use crate::fastfield::write_alive_bitset;
+use crate::index::{Index, Segment, SegmentComponent, SegmentId, SegmentMeta, SegmentReader};
 use crate::indexer::delete_queue::{DeleteCursor, DeleteQueue};
 use crate::indexer::doc_opstamp_mapping::DocToOpstampMapping;
 use crate::indexer::index_writer_status::IndexWriterStatus;
@@ -1651,6 +1651,7 @@ mod tests {
         force_end_merge: bool,
     ) -> crate::Result<Index> {
         let mut schema_builder = schema::Schema::builder();
+        let json_field = schema_builder.add_json_field("json", FAST | TEXT | STORED);
         let ip_field = schema_builder.add_ip_addr_field("ip", FAST | INDEXED | STORED);
         let ips_field = schema_builder
             .add_ip_addr_field("ips", IpAddrOptions::default().set_fast().set_indexed());
@@ -1729,7 +1730,9 @@ mod tests {
                             id_field=>id,
                         ))?;
                     } else {
+                        let json = json!({"date1": format!("2022-{id}-01T00:00:01Z"), "date2": format!("{id}-05-01T00:00:01Z"), "id": id, "ip": ip.to_string()});
                         index_writer.add_document(doc!(id_field=>id,
+                                json_field=>json,
                                 bytes_field => id.to_le_bytes().as_slice(),
                                 id_opt_field => id,
                                 ip_field => ip,

src/indexer/log_merge_policy.rs

@@ -3,7 +3,7 @@ use std::cmp;
 use itertools::Itertools;
 
 use super::merge_policy::{MergeCandidate, MergePolicy};
-use crate::core::SegmentMeta;
+use crate::index::SegmentMeta;
 
 const DEFAULT_LEVEL_LOG_SIZE: f64 = 0.75;
 const DEFAULT_MIN_LAYER_SIZE: u32 = 10_000;
@@ -144,7 +144,7 @@ mod tests {
     use once_cell::sync::Lazy;
 
     use super::*;
-    use crate::core::{SegmentId, SegmentMeta, SegmentMetaInventory};
+    use crate::index::{SegmentId, SegmentMeta, SegmentMetaInventory};
     use crate::indexer::merge_policy::MergePolicy;
     use crate::schema;
     use crate::schema::INDEXED;

src/indexer/merge_policy.rs

@@ -1,7 +1,7 @@
 use std::fmt::Debug;
 use std::marker;
 
-use crate::core::{SegmentId, SegmentMeta};
+use crate::index::{SegmentId, SegmentMeta};
 
 /// Set of segment suggested for a merge.
 #[derive(Debug, Clone)]
@@ -39,7 +39,7 @@ impl MergePolicy for NoMergePolicy {
 pub mod tests {
 
     use super::*;
-    use crate::core::{SegmentId, SegmentMeta};
+    use crate::index::{SegmentId, SegmentMeta};
 
     /// `MergePolicy` useful for test purposes.
     ///

src/indexer/merger.rs

@@ -8,12 +8,12 @@ use common::ReadOnlyBitSet;
 use itertools::Itertools;
 use measure_time::debug_time;
 
-use crate::core::{Segment, SegmentReader};
 use crate::directory::WritePtr;
 use crate::docset::{DocSet, TERMINATED};
 use crate::error::DataCorruption;
 use crate::fastfield::{AliveBitSet, FastFieldNotAvailableError};
 use crate::fieldnorm::{FieldNormReader, FieldNormReaders, FieldNormsSerializer, FieldNormsWriter};
+use crate::index::{Segment, SegmentReader};
 use crate::indexer::doc_id_mapping::{MappingType, SegmentDocIdMapping};
 use crate::indexer::SegmentSerializer;
 use crate::postings::{InvertedIndexSerializer, Postings, SegmentPostings};
@@ -552,7 +552,41 @@ impl IndexMerger {
                 continue;
             }
 
-            field_serializer.new_term(term_bytes, total_doc_freq)?;
+            // This should never happen as we early exited for total_doc_freq == 0.
+            assert!(!segment_postings_containing_the_term.is_empty());
+
+            let has_term_freq = {
+                let has_term_freq = !segment_postings_containing_the_term[0]
+                    .1
+                    .block_cursor
+                    .freqs()
+                    .is_empty();
+                for (_, postings) in &segment_postings_containing_the_term[1..] {
+                    // This may look at a strange way to test whether we have term freq or not.
+                    // With JSON object, the schema is not sufficient to know whether a term
+                    // has its term frequency encoded or not:
+                    // strings may have term frequencies, while number terms never have one.
+                    //
+                    // Ideally, we should have burnt one bit of two in the `TermInfo`.
+                    // However, we preferred not changing the codec too much and detect this
+                    // instead by
+                    // - looking at the size of the skip data for bitpacked blocks
+                    // - observing the absence of remaining data after reading the docs for vint
+                    // blocks.
+                    //
+                    // Overall the reliable way to know if we have actual frequencies loaded or not
+                    // is to check whether the actual decoded array is empty or not.
+                    if has_term_freq != !postings.block_cursor.freqs().is_empty() {
+                        return Err(DataCorruption::comment_only(
+                            "Term freqs are inconsistent across segments",
+                        )
+                        .into());
+                    }
+                }
+                has_term_freq
+            };
+
+            field_serializer.new_term(term_bytes, total_doc_freq, has_term_freq)?;
 
             // We can now serialize this postings, by pushing each document to the
             // postings serializer.
@@ -567,8 +601,17 @@ impl IndexMerger {
                     if let Some(remapped_doc_id) = old_to_new_doc_id[doc as usize] {
                         // we make sure to only write the term if
                         // there is at least one document.
-                        let term_freq = segment_postings.term_freq();
-                        segment_postings.positions(&mut positions_buffer);
+                        let term_freq = if has_term_freq {
+                            segment_postings.positions(&mut positions_buffer);
+                            segment_postings.term_freq()
+                        } else {
+                            // The positions_buffer may contain positions from the previous term
+                            // Existence of positions depend on the value type in JSON fields.
+                            // https://github.com/quickwit-oss/tantivy/issues/2283
+                            positions_buffer.clear();
+                            0u32
+                        };
+
                         // if doc_id_mapping exists, the doc_ids are reordered, they are
                         // not just stacked. The field serializer expects monotonically increasing
                         // doc_ids, so we collect and sort them first, before writing.
@@ -751,7 +794,7 @@ mod tests {
         BytesFastFieldTestCollector, FastFieldTestCollector, TEST_COLLECTOR_WITH_SCORE,
     };
     use crate::collector::{Count, FacetCollector};
-    use crate::core::Index;
+    use crate::index::Index;
     use crate::query::{AllQuery, BooleanQuery, EnableScoring, Scorer, TermQuery};
     use crate::schema::document::Value;
     use crate::schema::{

src/indexer/merger_sorted_index_test.rs

@@ -1,8 +1,8 @@
 #[cfg(test)]
 mod tests {
     use crate::collector::TopDocs;
-    use crate::core::Index;
     use crate::fastfield::AliveBitSet;
+    use crate::index::Index;
     use crate::query::QueryParser;
     use crate::schema::document::Value;
     use crate::schema::{
@@ -485,7 +485,7 @@ mod bench_sorted_index_merge {
 
     use test::{self, Bencher};
 
-    use crate::core::Index;
+    use crate::index::Index;
     use crate::indexer::merger::IndexMerger;
     use crate::schema::{NumericOptions, Schema};
     use crate::{IndexSettings, IndexSortByField, IndexWriter, Order};

src/indexer/mod.rs

@@ -25,6 +25,7 @@ mod segment_register;
 pub(crate) mod segment_serializer;
 pub(crate) mod segment_updater;
 pub(crate) mod segment_writer;
+pub(crate) mod single_segment_index_writer;
 mod stamper;
 
 use crossbeam_channel as channel;
@@ -34,13 +35,14 @@ pub use self::index_writer::IndexWriter;
 pub use self::log_merge_policy::LogMergePolicy;
 pub use self::merge_operation::MergeOperation;
 pub use self::merge_policy::{MergeCandidate, MergePolicy, NoMergePolicy};
+use self::operation::AddOperation;
 pub use self::operation::UserOperation;
 pub use self::prepared_commit::PreparedCommit;
 pub use self::segment_entry::SegmentEntry;
 pub(crate) use self::segment_serializer::SegmentSerializer;
 pub use self::segment_updater::{merge_filtered_segments, merge_indices};
 pub use self::segment_writer::SegmentWriter;
-use crate::indexer::operation::AddOperation;
+pub use self::single_segment_index_writer::SingleSegmentIndexWriter;
 
 /// Alias for the default merge policy, which is the `LogMergePolicy`.
 pub type DefaultMergePolicy = LogMergePolicy;
@@ -59,9 +61,13 @@ type AddBatchReceiver<D> = channel::Receiver<AddBatch<D>>;
 #[cfg(test)]
 mod tests_mmap {
 
-    use crate::collector::Count;
-    use crate::query::QueryParser;
-    use crate::schema::{JsonObjectOptions, Schema, Type, TEXT};
+    use crate::aggregation::agg_req::Aggregations;
+    use crate::aggregation::agg_result::AggregationResults;
+    use crate::aggregation::AggregationCollector;
+    use crate::collector::{Count, TopDocs};
+    use crate::index::FieldMetadata;
+    use crate::query::{AllQuery, QueryParser};
+    use crate::schema::{JsonObjectOptions, Schema, Type, FAST, INDEXED, STORED, TEXT};
     use crate::{Index, IndexWriter, Term};
 
     #[test]
@@ -173,8 +179,7 @@ mod tests_mmap {
     #[test]
     fn test_json_field_list_fields() {
         let mut schema_builder = Schema::builder();
-        let json_options: JsonObjectOptions =
-            JsonObjectOptions::from(TEXT).set_expand_dots_enabled();
+        let json_options: JsonObjectOptions = JsonObjectOptions::from(TEXT);
         let json_field = schema_builder.add_json_field("json", json_options);
         let index = Index::create_in_ram(schema_builder.build());
         let mut index_writer = index.writer_for_tests().unwrap();
@@ -193,9 +198,9 @@ mod tests_mmap {
         let reader = &searcher.segment_readers()[0];
         let inverted_index = reader.inverted_index(json_field).unwrap();
         assert_eq!(
-            inverted_index.list_fields().unwrap(),
+            inverted_index.list_encoded_fields().unwrap(),
             [
-                ("k8s\u{1}container\u{1}name".to_string(), Type::Str),
+                ("k8s.container.name".to_string(), Type::Str),
                 ("sub\u{1}a".to_string(), Type::I64),
                 ("sub\u{1}b".to_string(), Type::I64),
                 ("suber\u{1}a".to_string(), Type::I64),
@@ -205,4 +210,239 @@ mod tests_mmap {
             ]
         );
     }
+
+    #[test]
+    fn test_json_fields_metadata_expanded_dots_one_segment() {
+        test_json_fields_metadata(true, true);
+    }
+    #[test]
+    fn test_json_fields_metadata_expanded_dots_multi_segment() {
+        test_json_fields_metadata(true, false);
+    }
+    #[test]
+    fn test_json_fields_metadata_no_expanded_dots_one_segment() {
+        test_json_fields_metadata(false, true);
+    }
+    #[test]
+    fn test_json_fields_metadata_no_expanded_dots_multi_segment() {
+        test_json_fields_metadata(false, false);
+    }
+
+    fn test_json_fields_metadata(expanded_dots: bool, one_segment: bool) {
+        use pretty_assertions::assert_eq;
+        let mut schema_builder = Schema::builder();
+        let json_options: JsonObjectOptions =
+            JsonObjectOptions::from(TEXT).set_fast(None).set_stored();
+        let json_options = if expanded_dots {
+            json_options.set_expand_dots_enabled()
+        } else {
+            json_options
+        };
+        schema_builder.add_json_field("json.confusing", json_options.clone());
+        let json_field = schema_builder.add_json_field("json.shadow", json_options.clone());
+        let json_field2 = schema_builder.add_json_field("json", json_options.clone());
+        schema_builder.add_json_field("empty_json", json_options);
+        let number_field = schema_builder.add_u64_field("numbers", FAST);
+        schema_builder.add_u64_field("empty", FAST | INDEXED | STORED);
+        let index = Index::create_in_ram(schema_builder.build());
+        let mut index_writer = index.writer_for_tests().unwrap();
+        let json =
+            serde_json::json!({"k8s.container.name": "a", "val": "a", "sub": {"a": 1, "b": 1}});
+        index_writer.add_document(doc!(json_field=>json)).unwrap();
+        let json =
+            serde_json::json!({"k8s.container.name": "a", "val": "a", "suber": {"a": 1, "b": 1}});
+        if !one_segment {
+            index_writer.commit().unwrap();
+        }
+        index_writer.add_document(doc!(json_field=>json)).unwrap();
+        let json = serde_json::json!({"k8s.container.name": "a", "k8s.container.name": "a", "val": "a", "suber": {"a": "a", "b": 1}});
+        index_writer
+            .add_document(doc!(number_field => 50u64, json_field=>json, json_field2=>json!({"shadow": {"val": "a"}})))
+            .unwrap();
+        index_writer.commit().unwrap();
+        let reader = index.reader().unwrap();
+
+        let searcher = reader.searcher();
+        assert_eq!(searcher.num_docs(), 3);
+
+        let fields_metadata = index.fields_metadata().unwrap();
+        assert_eq!(
+            fields_metadata,
+            [
+                FieldMetadata {
+                    field_name: "empty".to_string(),
+                    indexed: true,
+                    stored: true,
+                    fast: true,
+                    typ: Type::U64
+                },
+                FieldMetadata {
+                    field_name: if expanded_dots {
+                        "json.shadow.k8s.container.name".to_string()
+                    } else {
+                        "json.shadow.k8s\\.container\\.name".to_string()
+                    },
+                    indexed: true,
+                    stored: true,
+                    fast: true,
+                    typ: Type::Str
+                },
+                FieldMetadata {
+                    field_name: "json.shadow.sub.a".to_string(),
+                    indexed: true,
+                    stored: true,
+                    fast: true,
+                    typ: Type::I64
+                },
+                FieldMetadata {
+                    field_name: "json.shadow.sub.b".to_string(),
+                    indexed: true,
+                    stored: true,
+                    fast: true,
+                    typ: Type::I64
+                },
+                FieldMetadata {
+                    field_name: "json.shadow.suber.a".to_string(),
+                    indexed: true,
+                    stored: true,
+                    fast: true,
+                    typ: Type::I64
+                },
+                FieldMetadata {
+                    field_name: "json.shadow.suber.a".to_string(),
+                    indexed: true,
+                    stored: true,
+                    fast: true,
+                    typ: Type::Str
+                },
+                FieldMetadata {
+                    field_name: "json.shadow.suber.b".to_string(),
+                    indexed: true,
+                    stored: true,
+                    fast: true,
+                    typ: Type::I64
+                },
+                FieldMetadata {
+                    field_name: "json.shadow.val".to_string(),
+                    indexed: true,
+                    stored: true,
+                    fast: true,
+                    typ: Type::Str
+                },
+                FieldMetadata {
+                    field_name: "numbers".to_string(),
+                    indexed: false,
+                    stored: false,
+                    fast: true,
+                    typ: Type::U64
+                }
+            ]
+        );
+        let query_parser = QueryParser::for_index(&index, vec![]);
+        // Test if returned field name can be queried
+        for indexed_field in fields_metadata.iter().filter(|meta| meta.indexed) {
+            let val = if indexed_field.typ == Type::Str {
+                "a"
+            } else {
+                "1"
+            };
+            let query_str = &format!("{}:{}", indexed_field.field_name, val);
+            let query = query_parser.parse_query(query_str).unwrap();
+            let count_docs = searcher.search(&*query, &TopDocs::with_limit(2)).unwrap();
+            if indexed_field.field_name.contains("empty") || indexed_field.typ == Type::Json {
+                assert_eq!(count_docs.len(), 0);
+            } else {
+                assert!(!count_docs.is_empty(), "{}", indexed_field.field_name);
+            }
+        }
+        // Test if returned field name can be used for aggregation
+        for fast_field in fields_metadata.iter().filter(|meta| meta.fast) {
+            let agg_req_str = json!(
+            {
+              "termagg": {
+                "terms": {
+                  "field": fast_field.field_name,
+                }
+              }
+            });
+
+            let agg_req: Aggregations = serde_json::from_value(agg_req_str).unwrap();
+            let collector = AggregationCollector::from_aggs(agg_req, Default::default());
+            let agg_res: AggregationResults = searcher.search(&AllQuery, &collector).unwrap();
+            let res = serde_json::to_value(agg_res).unwrap();
+            if !fast_field.field_name.contains("empty") && fast_field.typ != Type::Json {
+                assert!(
+                    !res["termagg"]["buckets"].as_array().unwrap().is_empty(),
+                    "{}",
+                    fast_field.field_name
+                );
+            }
+        }
+    }
+
+    #[test]
+    fn test_json_field_shadowing_field_name_bug() {
+        /// This test is only there to display a bug on addressing a field if it gets shadowed
+        /// The issues only occurs if the field name that shadows contains a dot.
+        ///
+        /// Happens independently of the `expand_dots` option. Since that option does not
+        /// affect the field name itself.
+        use pretty_assertions::assert_eq;
+        let mut schema_builder = Schema::builder();
+        let json_options: JsonObjectOptions =
+            JsonObjectOptions::from(TEXT).set_fast(None).set_stored();
+        // let json_options = json_options.set_expand_dots_enabled();
+        let json_field_shadow = schema_builder.add_json_field("json.shadow", json_options.clone());
+        let json_field = schema_builder.add_json_field("json", json_options.clone());
+        let index = Index::create_in_ram(schema_builder.build());
+        let mut index_writer = index.writer_for_tests().unwrap();
+        index_writer
+            .add_document(
+                doc!(json_field_shadow=>json!({"val": "b"}), json_field=>json!({"shadow": {"val": "a"}})),
+            )
+            .unwrap();
+        index_writer.commit().unwrap();
+        let reader = index.reader().unwrap();
+
+        let searcher = reader.searcher();
+
+        let fields_and_vals = [
+            ("json.shadow\u{1}val".to_string(), "a"), // Succeeds
+            //("json.shadow.val".to_string(), "a"),   // Fails
+            ("json.shadow.val".to_string(), "b"),
+        ];
+
+        let query_parser = QueryParser::for_index(&index, vec![]);
+        // Test if field name can be queried
+        for (indexed_field, val) in fields_and_vals.iter() {
+            let query_str = &format!("{}:{}", indexed_field, val);
+            let query = query_parser.parse_query(query_str).unwrap();
+            let count_docs = searcher.search(&*query, &TopDocs::with_limit(2)).unwrap();
+            assert!(!count_docs.is_empty(), "{}:{}", indexed_field, val);
+        }
+        // Test if field name can be used for aggregation
+        for (field_name, val) in fields_and_vals.iter() {
+            let agg_req_str = json!(
+            {
+              "termagg": {
+                "terms": {
+                  "field": field_name,
+                }
+              }
+            });
+
+            let agg_req: Aggregations = serde_json::from_value(agg_req_str).unwrap();
+            let collector = AggregationCollector::from_aggs(agg_req, Default::default());
+            let agg_res: AggregationResults = searcher.search(&AllQuery, &collector).unwrap();
+            let res = serde_json::to_value(agg_res).unwrap();
+            assert_eq!(
+                res["termagg"]["buckets"].as_array().unwrap()[0]["key"]
+                    .as_str()
+                    .unwrap(),
+                *val,
+                "{}",
+                field_name
+            );
+        }
+    }
 }

src/indexer/segment_entry.rs

@@ -2,7 +2,7 @@ use std::fmt;
 
 use common::BitSet;
 
-use crate::core::{SegmentId, SegmentMeta};
+use crate::index::{SegmentId, SegmentMeta};
 use crate::indexer::delete_queue::DeleteCursor;
 
 /// A segment entry describes the state of

src/indexer/segment_manager.rs

@@ -3,8 +3,8 @@ use std::fmt::{self, Debug, Formatter};
 use std::sync::{RwLock, RwLockReadGuard, RwLockWriteGuard};
 
 use super::segment_register::SegmentRegister;
-use crate::core::{SegmentId, SegmentMeta};
 use crate::error::TantivyError;
+use crate::index::{SegmentId, SegmentMeta};
 use crate::indexer::delete_queue::DeleteCursor;
 use crate::indexer::SegmentEntry;
 

src/indexer/segment_register.rs

@@ -1,7 +1,7 @@
 use std::collections::{HashMap, HashSet};
 use std::fmt::{self, Debug, Display, Formatter};
 
-use crate::core::{SegmentId, SegmentMeta};
+use crate::index::{SegmentId, SegmentMeta};
 use crate::indexer::delete_queue::DeleteCursor;
 use crate::indexer::segment_entry::SegmentEntry;
 
@@ -103,7 +103,7 @@ impl SegmentRegister {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::core::{SegmentId, SegmentMetaInventory};
+    use crate::index::{SegmentId, SegmentMetaInventory};
     use crate::indexer::delete_queue::*;
 
     fn segment_ids(segment_register: &SegmentRegister) -> Vec<SegmentId> {

src/indexer/segment_serializer.rs

@@ -1,8 +1,8 @@
 use common::TerminatingWrite;
 
-use crate::core::{Segment, SegmentComponent};
 use crate::directory::WritePtr;
 use crate::fieldnorm::FieldNormsSerializer;
+use crate::index::{Segment, SegmentComponent};
 use crate::postings::InvertedIndexSerializer;
 use crate::store::StoreWriter;
 

src/indexer/segment_updater.rs

@@ -9,11 +9,10 @@ use std::sync::{Arc, RwLock};
 use rayon::{ThreadPool, ThreadPoolBuilder};
 
 use super::segment_manager::SegmentManager;
-use crate::core::{
-    Index, IndexMeta, IndexSettings, Segment, SegmentId, SegmentMeta, META_FILEPATH,
-};
+use crate::core::META_FILEPATH;
 use crate::directory::{Directory, DirectoryClone, GarbageCollectionResult};
 use crate::fastfield::AliveBitSet;
+use crate::index::{Index, IndexMeta, IndexSettings, Segment, SegmentId, SegmentMeta};
 use crate::indexer::delete_queue::DeleteCursor;
 use crate::indexer::index_writer::advance_deletes;
 use crate::indexer::merge_operation::MergeOperationInventory;

src/indexer/segment_writer.rs

@@ -6,9 +6,9 @@ use tokenizer_api::BoxTokenStream;
 use super::doc_id_mapping::{get_doc_id_mapping_from_field, DocIdMapping};
 use super::operation::AddOperation;
 use crate::core::json_utils::index_json_values;
-use crate::core::Segment;
 use crate::fastfield::FastFieldsWriter;
 use crate::fieldnorm::{FieldNormReaders, FieldNormsWriter};
+use crate::index::Segment;
 use crate::indexer::segment_serializer::SegmentSerializer;
 use crate::postings::{
     compute_table_memory_size, serialize_postings, IndexingContext, IndexingPosition,
@@ -879,6 +879,31 @@ mod tests {
         assert_eq!(searcher.search(&phrase_query, &Count).unwrap(), 0);
     }
 
+    #[test]
+    fn test_json_term_with_numeric_merge_panic_regression_bug_2283() {
+        // https://github.com/quickwit-oss/tantivy/issues/2283
+        let mut schema_builder = Schema::builder();
+        let json = schema_builder.add_json_field("json", TEXT);
+        let schema = schema_builder.build();
+        let index = Index::create_in_ram(schema);
+        let mut writer = index.writer_for_tests().unwrap();
+        let doc = json!({"field": "a"});
+        writer.add_document(doc!(json=>doc)).unwrap();
+        writer.commit().unwrap();
+        let doc = json!({"field": "a", "id": 1});
+        writer.add_document(doc!(json=>doc.clone())).unwrap();
+        writer.commit().unwrap();
+
+        // Force Merge
+        writer.wait_merging_threads().unwrap();
+        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
+        let segment_ids = index
+            .searchable_segment_ids()
+            .expect("Searchable segments failed.");
+        index_writer.merge(&segment_ids).wait().unwrap();
+        assert!(index_writer.wait_merging_threads().is_ok());
+    }
+
     #[test]
     fn test_bug_regression_1629_position_when_array_with_a_field_value_that_does_not_contain_any_token(
     ) {

src/lib.rs

@@ -189,6 +189,7 @@ pub mod collector;
 pub mod directory;
 pub mod fastfield;
 pub mod fieldnorm;
+pub mod index;
 pub mod positions;
 pub mod postings;
 
@@ -220,18 +221,18 @@ pub use self::docset::{DocSet, TERMINATED};
 pub use self::snippet::{Snippet, SnippetGenerator};
 #[doc(hidden)]
 pub use crate::core::json_utils;
-pub use crate::core::{
-    Executor, Index, IndexBuilder, IndexMeta, IndexSettings, IndexSortByField, InvertedIndexReader,
-    Order, Searcher, SearcherGeneration, Segment, SegmentComponent, SegmentId, SegmentMeta,
-    SegmentReader, SingleSegmentIndexWriter,
-};
+pub use crate::core::{Executor, Searcher, SearcherGeneration};
 pub use crate::directory::Directory;
-pub use crate::indexer::IndexWriter;
+pub use crate::index::{
+    Index, IndexBuilder, IndexMeta, IndexSettings, IndexSortByField, InvertedIndexReader, Order,
+    Segment, SegmentComponent, SegmentId, SegmentMeta, SegmentReader,
+};
 #[deprecated(
     since = "0.22.0",
     note = "Will be removed in tantivy 0.23. Use export from indexer module instead"
 )]
-pub use crate::indexer::{merge_filtered_segments, merge_indices, PreparedCommit};
+pub use crate::indexer::PreparedCommit;
+pub use crate::indexer::{IndexWriter, SingleSegmentIndexWriter};
 pub use crate::postings::Postings;
 #[allow(deprecated)]
 pub use crate::schema::DatePrecision;
@@ -338,7 +339,7 @@ impl DocAddress {
 ///
 /// The id used for the segment is actually an ordinal
 /// in the list of `Segment`s held by a `Searcher`.
-#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
 pub struct DocAddress {
     /// The segment ordinal id that identifies the segment
     /// hosting the document in the `Searcher` it is called from.
@@ -386,8 +387,8 @@ pub mod tests {
     use time::OffsetDateTime;
 
     use crate::collector::tests::TEST_COLLECTOR_WITH_SCORE;
-    use crate::core::SegmentReader;
     use crate::docset::{DocSet, TERMINATED};
+    use crate::index::SegmentReader;
     use crate::merge_policy::NoMergePolicy;
     use crate::query::BooleanQuery;
     use crate::schema::document::Value;

src/postings/block_segment_postings.rs

@@ -383,8 +383,8 @@ mod tests {
     use common::HasLen;
 
     use super::BlockSegmentPostings;
-    use crate::core::Index;
     use crate::docset::{DocSet, TERMINATED};
+    use crate::index::Index;
     use crate::postings::compression::COMPRESSION_BLOCK_SIZE;
     use crate::postings::postings::Postings;
     use crate::postings::SegmentPostings;

src/postings/json_postings_writer.rs

@@ -11,6 +11,10 @@ use crate::schema::{Field, Type, JSON_END_OF_PATH};
 use crate::tokenizer::TokenStream;
 use crate::{DocId, Term};
 
+/// The `JsonPostingsWriter` is odd in that it relies on a hidden contract:
+///
+/// `subscribe` is called directly to index non-text tokens, while
+/// `index_text` is used to index text.
 #[derive(Default)]
 pub(crate) struct JsonPostingsWriter<Rec: Recorder> {
     str_posting_writer: SpecializedPostingsWriter<Rec>,

src/postings/mod.rs

@@ -42,9 +42,9 @@ pub mod tests {
     use std::mem;
 
     use super::{InvertedIndexSerializer, Postings};
-    use crate::core::{Index, SegmentComponent, SegmentReader};
     use crate::docset::{DocSet, TERMINATED};
     use crate::fieldnorm::FieldNormReader;
+    use crate::index::{Index, SegmentComponent, SegmentReader};
     use crate::indexer::operation::AddOperation;
     use crate::indexer::SegmentWriter;
     use crate::query::Scorer;
@@ -63,7 +63,7 @@ pub mod tests {
         let mut segment = index.new_segment();
         let mut posting_serializer = InvertedIndexSerializer::open(&mut segment)?;
         let mut field_serializer = posting_serializer.new_field(text_field, 120 * 4, None)?;
-        field_serializer.new_term("abc".as_bytes(), 12u32)?;
+        field_serializer.new_term("abc".as_bytes(), 12u32, true)?;
         for doc_id in 0u32..120u32 {
             let delta_positions = vec![1, 2, 3, 2];
             field_serializer.write_doc(doc_id, 4, &delta_positions);

src/postings/postings_writer.rs

@@ -194,7 +194,7 @@ impl<Rec: Recorder> SpecializedPostingsWriter<Rec> {
     ) -> io::Result<()> {
         let recorder: Rec = ctx.term_index.read(addr);
         let term_doc_freq = recorder.term_doc_freq().unwrap_or(0u32);
-        serializer.new_term(term, term_doc_freq)?;
+        serializer.new_term(term, term_doc_freq, recorder.has_term_freq())?;
         recorder.serialize(&ctx.arena, doc_id_map, serializer, buffer_lender);
         serializer.close_term()?;
         Ok(())

src/postings/recorder.rs

@@ -79,24 +79,20 @@ pub(crate) trait Recorder: Copy + Default + Send + Sync + 'static {
     ///
     /// Returns `None` if not available.
     fn term_doc_freq(&self) -> Option<u32>;
+
+    #[inline]
+    fn has_term_freq(&self) -> bool {
+        true
+    }
 }
 
 /// Only records the doc ids
-#[derive(Clone, Copy)]
+#[derive(Clone, Copy, Default)]
 pub struct DocIdRecorder {
     stack: ExpUnrolledLinkedList,
     current_doc: DocId,
 }
 
-impl Default for DocIdRecorder {
-    fn default() -> Self {
-        DocIdRecorder {
-            stack: ExpUnrolledLinkedList::default(),
-            current_doc: u32::MAX,
-        }
-    }
-}
-
 impl Recorder for DocIdRecorder {
     #[inline]
     fn current_doc(&self) -> DocId {
@@ -105,8 +101,9 @@ impl Recorder for DocIdRecorder {
 
     #[inline]
     fn new_doc(&mut self, doc: DocId, arena: &mut MemoryArena) {
+        let delta = doc - self.current_doc;
         self.current_doc = doc;
-        self.stack.writer(arena).write_u32_vint(doc);
+        self.stack.writer(arena).write_u32_vint(delta);
     }
 
     #[inline]
@@ -123,21 +120,20 @@ impl Recorder for DocIdRecorder {
         buffer_lender: &mut BufferLender,
     ) {
         let (buffer, doc_ids) = buffer_lender.lend_all();
-        self.stack.read_to_end(arena, buffer);
         // TODO avoid reading twice.
+        self.stack.read_to_end(arena, buffer);
         if let Some(doc_id_map) = doc_id_map {
-            doc_ids.extend(
-                VInt32Reader::new(&buffer[..])
-                    .map(|old_doc_id| doc_id_map.get_new_doc_id(old_doc_id)),
-            );
+            let iter = get_sum_reader(VInt32Reader::new(&buffer[..]));
+            doc_ids.extend(iter.map(|old_doc_id| doc_id_map.get_new_doc_id(old_doc_id)));
             doc_ids.sort_unstable();
 
             for doc in doc_ids {
                 serializer.write_doc(*doc, 0u32, &[][..]);
             }
         } else {
-            for doc in VInt32Reader::new(&buffer[..]) {
-                serializer.write_doc(doc, 0u32, &[][..]);
+            let iter = get_sum_reader(VInt32Reader::new(&buffer[..]));
+            for doc_id in iter {
+                serializer.write_doc(doc_id, 0u32, &[][..]);
             }
         }
     }
@@ -145,6 +141,19 @@ impl Recorder for DocIdRecorder {
     fn term_doc_freq(&self) -> Option<u32> {
         None
     }
+
+    fn has_term_freq(&self) -> bool {
+        false
+    }
+}
+
+/// Takes an Iterator of delta encoded elements and returns an iterator
+/// that yields the sum of the elements.
+fn get_sum_reader(iter: impl Iterator<Item = u32>) -> impl Iterator<Item = u32> {
+    iter.scan(0, |state, delta| {
+        *state += delta;
+        Some(*state)
+    })
 }
 
 /// Recorder encoding document ids, and term frequencies
@@ -164,9 +173,10 @@ impl Recorder for TermFrequencyRecorder {
 
     #[inline]
     fn new_doc(&mut self, doc: DocId, arena: &mut MemoryArena) {
+        let delta = doc - self.current_doc;
         self.term_doc_freq += 1;
         self.current_doc = doc;
-        self.stack.writer(arena).write_u32_vint(doc);
+        self.stack.writer(arena).write_u32_vint(delta);
     }
 
     #[inline]
@@ -193,9 +203,12 @@ impl Recorder for TermFrequencyRecorder {
         let mut u32_it = VInt32Reader::new(&buffer[..]);
         if let Some(doc_id_map) = doc_id_map {
             let mut doc_id_and_tf = vec![];
-            while let Some(old_doc_id) = u32_it.next() {
+            let mut prev_doc = 0;
+            while let Some(delta_doc_id) = u32_it.next() {
+                let doc_id = prev_doc + delta_doc_id;
+                prev_doc = doc_id;
                 let term_freq = u32_it.next().unwrap_or(self.current_tf);
-                doc_id_and_tf.push((doc_id_map.get_new_doc_id(old_doc_id), term_freq));
+                doc_id_and_tf.push((doc_id_map.get_new_doc_id(doc_id), term_freq));
             }
             doc_id_and_tf.sort_unstable_by_key(|&(doc_id, _)| doc_id);
 
@@ -203,9 +216,12 @@ impl Recorder for TermFrequencyRecorder {
                 serializer.write_doc(doc_id, tf, &[][..]);
             }
         } else {
-            while let Some(doc) = u32_it.next() {
+            let mut prev_doc = 0;
+            while let Some(delta_doc_id) = u32_it.next() {
+                let doc_id = prev_doc + delta_doc_id;
+                prev_doc = doc_id;
                 let term_freq = u32_it.next().unwrap_or(self.current_tf);
-                serializer.write_doc(doc, term_freq, &[][..]);
+                serializer.write_doc(doc_id, term_freq, &[][..]);
             }
         }
     }
@@ -216,23 +232,13 @@ impl Recorder for TermFrequencyRecorder {
 }
 
 /// Recorder encoding term frequencies as well as positions.
-#[derive(Clone, Copy)]
+#[derive(Clone, Copy, Default)]
 pub struct TfAndPositionRecorder {
     stack: ExpUnrolledLinkedList,
     current_doc: DocId,
     term_doc_freq: u32,
 }
 
-impl Default for TfAndPositionRecorder {
-    fn default() -> Self {
-        TfAndPositionRecorder {
-            stack: ExpUnrolledLinkedList::default(),
-            current_doc: u32::MAX,
-            term_doc_freq: 0u32,
-        }
-    }
-}
-
 impl Recorder for TfAndPositionRecorder {
     #[inline]
     fn current_doc(&self) -> DocId {
@@ -241,9 +247,10 @@ impl Recorder for TfAndPositionRecorder {
 
     #[inline]
     fn new_doc(&mut self, doc: DocId, arena: &mut MemoryArena) {
+        let delta = doc - self.current_doc;
         self.current_doc = doc;
         self.term_doc_freq += 1u32;
-        self.stack.writer(arena).write_u32_vint(doc);
+        self.stack.writer(arena).write_u32_vint(delta);
     }
 
     #[inline]
@@ -269,7 +276,10 @@ impl Recorder for TfAndPositionRecorder {
         self.stack.read_to_end(arena, buffer_u8);
         let mut u32_it = VInt32Reader::new(&buffer_u8[..]);
         let mut doc_id_and_positions = vec![];
-        while let Some(doc) = u32_it.next() {
+        let mut prev_doc = 0;
+        while let Some(delta_doc_id) = u32_it.next() {
+            let doc_id = prev_doc + delta_doc_id;
+            prev_doc = doc_id;
             let mut prev_position_plus_one = 1u32;
             buffer_positions.clear();
             loop {
@@ -287,9 +297,9 @@ impl Recorder for TfAndPositionRecorder {
             if let Some(doc_id_map) = doc_id_map {
                 // this simple variant to remap may consume to much memory
                 doc_id_and_positions
-                    .push((doc_id_map.get_new_doc_id(doc), buffer_positions.to_vec()));
+                    .push((doc_id_map.get_new_doc_id(doc_id), buffer_positions.to_vec()));
             } else {
-                serializer.write_doc(doc, buffer_positions.len() as u32, buffer_positions);
+                serializer.write_doc(doc_id, buffer_positions.len() as u32, buffer_positions);
             }
         }
         if doc_id_map.is_some() {

src/postings/segment_postings.rs

@@ -71,7 +71,7 @@ impl SegmentPostings {
         {
             let mut postings_serializer =
                 PostingsSerializer::new(&mut buffer, 0.0, IndexRecordOption::Basic, None);
-            postings_serializer.new_term(docs.len() as u32);
+            postings_serializer.new_term(docs.len() as u32, false);
             for &doc in docs {
                 postings_serializer.write_doc(doc, 1u32);
             }
@@ -120,7 +120,7 @@ impl SegmentPostings {
             IndexRecordOption::WithFreqs,
             fieldnorm_reader,
         );
-        postings_serializer.new_term(doc_and_tfs.len() as u32);
+        postings_serializer.new_term(doc_and_tfs.len() as u32, true);
         for &(doc, tf) in doc_and_tfs {
             postings_serializer.write_doc(doc, tf);
         }
@@ -238,14 +238,18 @@ impl Postings for SegmentPostings {
     }
 
     fn positions_with_offset(&mut self, offset: u32, output: &mut Vec<u32>) {
-        let term_freq = self.term_freq() as usize;
+        let term_freq = self.term_freq();
         if let Some(position_reader) = self.position_reader.as_mut() {
+            debug_assert!(
+                !self.block_cursor.freqs().is_empty(),
+                "No positions available"
+            );
             let read_offset = self.block_cursor.position_offset()
                 + (self.block_cursor.freqs()[..self.cur]
                     .iter()
                     .cloned()
                     .sum::<u32>() as u64);
-            output.resize(term_freq, 0u32);
+            output.resize(term_freq as usize, 0u32);
             position_reader.read(read_offset, &mut output[..]);
             let mut cum = offset;
             for output_mut in output.iter_mut() {

src/postings/serializer.rs

@@ -4,9 +4,9 @@ use std::io::{self, Write};
 use common::{BinarySerializable, CountingWriter, VInt};
 
 use super::TermInfo;
-use crate::core::Segment;
 use crate::directory::{CompositeWrite, WritePtr};
 use crate::fieldnorm::FieldNormReader;
+use crate::index::Segment;
 use crate::positions::PositionSerializer;
 use crate::postings::compression::{BlockEncoder, VIntEncoder, COMPRESSION_BLOCK_SIZE};
 use crate::postings::skip::SkipSerializer;
@@ -168,7 +168,12 @@ impl<'a> FieldSerializer<'a> {
     /// * term - the term. It needs to come after the previous term according to the lexicographical
     ///   order.
     /// * term_doc_freq - return the number of document containing the term.
-    pub fn new_term(&mut self, term: &[u8], term_doc_freq: u32) -> io::Result<()> {
+    pub fn new_term(
+        &mut self,
+        term: &[u8],
+        term_doc_freq: u32,
+        record_term_freq: bool,
+    ) -> io::Result<()> {
         assert!(
             !self.term_open,
             "Called new_term, while the previous term was not closed."
@@ -177,7 +182,8 @@ impl<'a> FieldSerializer<'a> {
         self.postings_serializer.clear();
         self.current_term_info = self.current_term_info();
         self.term_dictionary_builder.insert_key(term)?;
-        self.postings_serializer.new_term(term_doc_freq);
+        self.postings_serializer
+            .new_term(term_doc_freq, record_term_freq);
         Ok(())
     }
 
@@ -330,10 +336,10 @@ impl<W: Write> PostingsSerializer<W> {
         }
     }
 
-    pub fn new_term(&mut self, term_doc_freq: u32) {
+    pub fn new_term(&mut self, term_doc_freq: u32, record_term_freq: bool) {
         self.bm25_weight = None;
 
-        self.term_has_freq = self.mode.has_freq() && term_doc_freq != 0;
+        self.term_has_freq = self.mode.has_freq() && record_term_freq;
         if !self.term_has_freq {
             return;
         }
@@ -349,7 +355,7 @@ impl<W: Write> PostingsSerializer<W> {
             return;
         }
 
-        self.bm25_weight = Some(Bm25Weight::for_one_term(
+        self.bm25_weight = Some(Bm25Weight::for_one_term_without_explain(
             term_doc_freq as u64,
             num_docs_in_segment,
             self.avg_fieldnorm,

src/query/all_query.rs

@@ -1,5 +1,5 @@
-use crate::core::SegmentReader;
 use crate::docset::{DocSet, BUFFER_LEN, TERMINATED};
+use crate::index::SegmentReader;
 use crate::query::boost_query::BoostScorer;
 use crate::query::explanation::does_not_match;
 use crate::query::{EnableScoring, Explanation, Query, Scorer, Weight};

src/query/automaton_weight.rs

@@ -5,7 +5,7 @@ use common::BitSet;
 use tantivy_fst::Automaton;
 
 use super::phrase_prefix_query::prefix_end;
-use crate::core::SegmentReader;
+use crate::index::SegmentReader;
 use crate::query::{BitSetDocSet, ConstScorer, Explanation, Scorer, Weight};
 use crate::schema::{Field, IndexRecordOption};
 use crate::termdict::{TermDictionary, TermStreamer};

src/query/bm25.rs

@@ -77,7 +77,7 @@ pub struct Bm25Params {
 /// A struct used for computing BM25 scores.
 #[derive(Clone)]
 pub struct Bm25Weight {
-    idf_explain: Explanation,
+    idf_explain: Option<Explanation>,
     weight: Score,
     cache: [Score; 256],
     average_fieldnorm: Score,
@@ -147,11 +147,30 @@ impl Bm25Weight {
         idf_explain.add_const("N, total number of docs", total_num_docs as Score);
         Bm25Weight::new(idf_explain, avg_fieldnorm)
     }
+    /// Construct a [Bm25Weight] for a single term.
+    /// This method does not carry the [Explanation] for the idf.
+    pub fn for_one_term_without_explain(
+        term_doc_freq: u64,
+        total_num_docs: u64,
+        avg_fieldnorm: Score,
+    ) -> Bm25Weight {
+        let idf = idf(term_doc_freq, total_num_docs);
+        Bm25Weight::new_without_explain(idf, avg_fieldnorm)
+    }
 
     pub(crate) fn new(idf_explain: Explanation, average_fieldnorm: Score) -> Bm25Weight {
         let weight = idf_explain.value() * (1.0 + K1);
         Bm25Weight {
-            idf_explain,
+            idf_explain: Some(idf_explain),
+            weight,
+            cache: compute_tf_cache(average_fieldnorm),
+            average_fieldnorm,
+        }
+    }
+    pub(crate) fn new_without_explain(idf: f32, average_fieldnorm: Score) -> Bm25Weight {
+        let weight = idf * (1.0 + K1);
+        Bm25Weight {
+            idf_explain: None,
             weight,
             cache: compute_tf_cache(average_fieldnorm),
             average_fieldnorm,
@@ -202,7 +221,9 @@ impl Bm25Weight {
 
         let mut explanation = Explanation::new("TermQuery, product of...", score);
         explanation.add_detail(Explanation::new("(K1+1)", K1 + 1.0));
-        explanation.add_detail(self.idf_explain.clone());
+        if let Some(idf_explain) = &self.idf_explain {
+            explanation.add_detail(idf_explain.clone());
+        }
         explanation.add_detail(tf_explanation);
         explanation
     }

src/query/boolean_query/boolean_weight.rs

@@ -1,7 +1,7 @@
 use std::collections::HashMap;
 
-use crate::core::SegmentReader;
 use crate::docset::BUFFER_LEN;
+use crate::index::SegmentReader;
 use crate::postings::FreqReadingOption;
 use crate::query::explanation::does_not_match;
 use crate::query::score_combiner::{DoNothingCombiner, ScoreCombiner};

src/query/boost_query.rs

@@ -74,7 +74,8 @@ impl Weight for BoostWeight {
     fn explain(&self, reader: &SegmentReader, doc: u32) -> crate::Result<Explanation> {
         let underlying_explanation = self.weight.explain(reader, doc)?;
         let score = underlying_explanation.value() * self.boost;
-        let mut explanation = Explanation::new(format!("Boost x{} of ...", self.boost), score);
+        let mut explanation =
+            Explanation::new_with_string(format!("Boost x{} of ...", self.boost), score);
         explanation.add_detail(underlying_explanation);
         Ok(explanation)
     }
@@ -151,7 +152,7 @@ mod tests {
         let explanation = query.explain(&searcher, DocAddress::new(0, 0u32)).unwrap();
         assert_eq!(
             explanation.to_pretty_json(),
-            "{\n  \"value\": 0.2,\n  \"description\": \"Boost x0.2 of ...\",\n  \"details\": [\n    {\n      \"value\": 1.0,\n      \"description\": \"AllQuery\",\n      \"context\": []\n    }\n  ],\n  \"context\": []\n}"
+            "{\n  \"value\": 0.2,\n  \"description\": \"Boost x0.2 of ...\",\n  \"details\": [\n    {\n      \"value\": 1.0,\n      \"description\": \"AllQuery\"\n    }\n  ]\n}"
         );
         Ok(())
     }

src/query/const_score_query.rs

@@ -164,11 +164,9 @@ mod tests {
   "details": [
     {
       "value": 1.0,
-      "description": "AllQuery",
-      "context": []
+      "description": "AllQuery"
     }
-  ],
-  "context": []
+  ]
 }"#
         );
         Ok(())

src/query/exist_query.rs

@@ -3,8 +3,8 @@ use core::fmt::Debug;
 use columnar::{ColumnIndex, DynamicColumn};
 
 use super::{ConstScorer, EmptyScorer};
-use crate::core::SegmentReader;
 use crate::docset::{DocSet, TERMINATED};
+use crate::index::SegmentReader;
 use crate::query::explanation::does_not_match;
 use crate::query::{EnableScoring, Explanation, Query, Scorer, Weight};
 use crate::{DocId, Score, TantivyError};

src/query/explanation.rs

@@ -1,3 +1,4 @@
+use std::borrow::Cow;
 use std::fmt;
 
 use serde::Serialize;
@@ -16,12 +17,12 @@ pub(crate) fn does_not_match(doc: DocId) -> TantivyError {
 #[derive(Clone, Serialize)]
 pub struct Explanation {
     value: Score,
-    description: String,
-    #[serde(skip_serializing_if = "Vec::is_empty")]
-    details: Vec<Explanation>,
-    context: Vec<String>,
+    description: Cow<'static, str>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    details: Option<Vec<Explanation>>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    context: Option<Vec<String>>,
 }
-
 impl fmt::Debug for Explanation {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         write!(f, "Explanation({})", self.to_pretty_json())
@@ -30,12 +31,21 @@ impl fmt::Debug for Explanation {
 
 impl Explanation {
     /// Creates a new explanation object.
-    pub fn new<T: ToString>(description: T, value: Score) -> Explanation {
+    pub fn new_with_string(description: String, value: Score) -> Explanation {
         Explanation {
             value,
-            description: description.to_string(),
-            details: vec![],
-            context: vec![],
+            description: Cow::Owned(description),
+            details: None,
+            context: None,
+        }
+    }
+    /// Creates a new explanation object.
+    pub fn new(description: &'static str, value: Score) -> Explanation {
+        Explanation {
+            value,
+            description: Cow::Borrowed(description),
+            details: None,
+            context: None,
         }
     }
 
@@ -48,17 +58,21 @@ impl Explanation {
     ///
     /// Details are treated as child of the current node.
     pub fn add_detail(&mut self, child_explanation: Explanation) {
-        self.details.push(child_explanation);
+        self.details
+            .get_or_insert_with(Vec::new)
+            .push(child_explanation);
     }
 
     /// Adds some extra context to the explanation.
     pub fn add_context(&mut self, context: String) {
-        self.context.push(context);
+        self.context.get_or_insert_with(Vec::new).push(context);
     }
 
     /// Shortcut for `self.details.push(Explanation::new(name, value));`
-    pub fn add_const<T: ToString>(&mut self, name: T, value: Score) {
-        self.details.push(Explanation::new(name, value));
+    pub fn add_const(&mut self, name: &'static str, value: Score) {
+        self.details
+            .get_or_insert_with(Vec::new)
+            .push(Explanation::new(name, value));
     }
 
     /// Returns an indented json representation of the explanation tree for debug usage.

src/query/phrase_prefix_query/phrase_prefix_weight.rs

@@ -1,6 +1,6 @@
 use super::{prefix_end, PhrasePrefixScorer};
-use crate::core::SegmentReader;
 use crate::fieldnorm::FieldNormReader;
+use crate::index::SegmentReader;
 use crate::postings::SegmentPostings;
 use crate::query::bm25::Bm25Weight;
 use crate::query::explanation::does_not_match;
@@ -157,8 +157,8 @@ impl Weight for PhrasePrefixWeight {
 
 #[cfg(test)]
 mod tests {
-    use crate::core::Index;
     use crate::docset::TERMINATED;
+    use crate::index::Index;
     use crate::query::{EnableScoring, PhrasePrefixQuery, Query};
     use crate::schema::{Schema, TEXT};
     use crate::{DocSet, IndexWriter, Term};

src/query/phrase_query/mod.rs

@@ -14,7 +14,7 @@ pub mod tests {
 
     use super::*;
     use crate::collector::tests::{TEST_COLLECTOR_WITHOUT_SCORE, TEST_COLLECTOR_WITH_SCORE};
-    use crate::core::Index;
+    use crate::index::Index;
     use crate::query::{EnableScoring, QueryParser, Weight};
     use crate::schema::{Schema, Term, TEXT};
     use crate::{assert_nearly_equals, DocAddress, DocId, IndexWriter, TERMINATED};

src/query/phrase_query/phrase_weight.rs

@@ -1,6 +1,6 @@
 use super::PhraseScorer;
-use crate::core::SegmentReader;
 use crate::fieldnorm::FieldNormReader;
+use crate::index::SegmentReader;
 use crate::postings::SegmentPostings;
 use crate::query::bm25::Bm25Weight;
 use crate::query::explanation::does_not_match;

src/query/query_parser/query_parser.rs

@@ -13,7 +13,7 @@ use super::logical_ast::*;
 use crate::core::json_utils::{
     convert_to_fast_value_and_get_term, set_string_and_get_terms, JsonTermWriter,
 };
-use crate::core::Index;
+use crate::index::Index;
 use crate::query::range_query::{is_type_valid_for_fastfield_range_query, RangeQuery};
 use crate::query::{
     AllQuery, BooleanQuery, BoostQuery, EmptyQuery, FuzzyTermQuery, Occur, PhrasePrefixQuery,

src/query/range_query/range_query.rs

@@ -7,8 +7,8 @@ use common::{BinarySerializable, BitSet};
 
 use super::map_bound;
 use super::range_query_u64_fastfield::FastFieldRangeWeight;
-use crate::core::SegmentReader;
 use crate::error::TantivyError;
+use crate::index::SegmentReader;
 use crate::query::explanation::does_not_match;
 use crate::query::range_query::range_query_ip_fastfield::IPFastFieldRangeWeight;
 use crate::query::range_query::{is_type_valid_for_fastfield_range_query, map_bound_res};

src/query/regex_query.rs

@@ -63,7 +63,7 @@ impl RegexQuery {
     /// Creates a new RegexQuery from a given pattern
     pub fn from_pattern(regex_pattern: &str, field: Field) -> crate::Result<Self> {
         let regex = Regex::new(regex_pattern)
-            .map_err(|_| TantivyError::InvalidArgument(regex_pattern.to_string()))?;
+            .map_err(|err| TantivyError::InvalidArgument(format!("RegexQueryError: {err}")))?;
         Ok(RegexQuery::from_regex(regex, field))
     }
 
@@ -176,4 +176,16 @@ mod test {
         verify_regex_query(matching_one, matching_zero, reader);
         Ok(())
     }
+
+    #[test]
+    pub fn test_pattern_error() {
+        let (_reader, field) = build_test_index().unwrap();
+
+        match RegexQuery::from_pattern(r"(foo", field) {
+            Err(crate::TantivyError::InvalidArgument(msg)) => {
+                assert!(msg.contains("error: unclosed group"))
+            }
+            res => panic!("unexpected result: {:?}", res),
+        }
+    }
 }

src/query/term_query/term_query.rs

@@ -101,7 +101,7 @@ impl TermQuery {
                 ..
             } => Bm25Weight::for_terms(statistics_provider, &[self.term.clone()])?,
             EnableScoring::Disabled { .. } => {
-                Bm25Weight::new(Explanation::new("<no score>".to_string(), 1.0f32), 1.0f32)
+                Bm25Weight::new(Explanation::new("<no score>", 1.0f32), 1.0f32)
             }
         };
         let scoring_enabled = enable_scoring.is_scoring_enabled();

src/query/term_query/term_weight.rs

@@ -1,7 +1,7 @@
 use super::term_scorer::TermScorer;
-use crate::core::SegmentReader;
 use crate::docset::{DocSet, BUFFER_LEN};
 use crate::fieldnorm::FieldNormReader;
+use crate::index::SegmentReader;
 use crate::postings::SegmentPostings;
 use crate::query::bm25::Bm25Weight;
 use crate::query::explanation::does_not_match;