tracing

* remove Document: DocumentDeserialize dependency

The dependency requires users to implement an API they may not use.

* remove unnecessary Document bounds

.github/workflows/coverage.yml

@@ -15,13 +15,13 @@ jobs:
   coverage:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Install Rust
-        run: rustup toolchain install nightly --profile minimal --component llvm-tools-preview
+        run: rustup toolchain install nightly-2023-09-10 --profile minimal --component llvm-tools-preview
       - uses: Swatinem/rust-cache@v2
       - uses: taiki-e/install-action@cargo-llvm-cov
       - name: Generate code coverage
-        run: cargo +nightly llvm-cov --all-features --workspace --doctests --lcov --output-path lcov.info
+        run: cargo +nightly-2023-09-10 llvm-cov --all-features --workspace --doctests --lcov --output-path lcov.info
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v3
         continue-on-error: true

.github/workflows/long_running.yml

@@ -19,7 +19,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Install stable
       uses: actions-rs/toolchain@v1
       with:

.github/workflows/test.yml

@@ -20,7 +20,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
     - name: Install nightly
       uses: actions-rs/toolchain@v1
@@ -60,7 +60,7 @@ jobs:
     name: test-${{ matrix.features.label}}
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
     - name: Install stable
       uses: actions-rs/toolchain@v1

Cargo.toml

@@ -19,19 +19,20 @@ oneshot = "0.1.5"
 base64 = "0.21.0"
 byteorder = "1.4.3"
 crc32fast = "1.3.2"
+tracing = "0.1"
 once_cell = "1.10.0"
 regex = { version = "1.5.5", default-features = false, features = ["std", "unicode"] }
 aho-corasick = "1.0"
 tantivy-fst = "0.4.0"
-memmap2 = { version = "0.7.1", optional = true }
+memmap2 = { version = "0.9.0", optional = true }
 lz4_flex = { version = "0.11", default-features = false, optional = true }
-zstd = { version = "0.12", optional = true, default-features = false }
+zstd = { version = "0.13", optional = true, default-features = false }
 tempfile = { version = "3.3.0", optional = true }
 log = "0.4.16"
 serde = { version = "1.0.136", features = ["derive"] }
 serde_json = "1.0.79"
 num_cpus = "1.13.1"
-fs4 = { version = "0.6.3", optional = true }
+fs4 = { version = "0.7.0", optional = true }
 levenshtein_automata = "0.2.1"
 uuid = { version = "1.0.0", features = ["v4", "serde"] }
 crossbeam-channel = "0.5.4"
@@ -47,7 +48,7 @@ murmurhash32 = "0.3.0"
 time = { version = "0.3.10", features = ["serde-well-known"] }
 smallvec = "1.8.0"
 rayon = "1.5.2"
-lru = "0.11.0"
+lru = "0.12.0"
 fastdivide = "0.4.0"
 itertools = "0.11.0"
 measure_time = "0.8.2"
@@ -63,6 +64,7 @@ common = { version= "0.6", path = "./common/", package = "tantivy-common" }
 tokenizer-api = { version= "0.2", path="./tokenizer-api", package="tantivy-tokenizer-api" }
 sketches-ddsketch = { version = "0.2.1", features = ["use_serde"] }
 futures-util = { version = "0.3.28", optional = true }
+fnv = "1.0.7"
 
 [target.'cfg(windows)'.dependencies]
 winapi = "0.3.9"
@@ -128,7 +130,7 @@ members = ["query-grammar", "bitpacker", "common", "ownedbytes", "stacker", "sst
 [[test]]
 name = "failpoints"
 path = "tests/failpoints/mod.rs"
-required-features = ["fail/failpoints"]
+required-features = ["failpoints"]
 
 [[bench]]
 name = "analyzer"

benches/index-bench.rs

@@ -1,7 +1,7 @@
 use criterion::{criterion_group, criterion_main, Criterion, Throughput};
 use pprof::criterion::{Output, PProfProfiler};
-use tantivy::schema::{FAST, INDEXED, STORED, STRING, TEXT};
-use tantivy::Index;
+use tantivy::schema::{TantivyDocument, FAST, INDEXED, STORED, STRING, TEXT};
+use tantivy::{Index, IndexWriter};
 
 const HDFS_LOGS: &str = include_str!("hdfs.json");
 const GH_LOGS: &str = include_str!("gh.json");
@@ -39,9 +39,9 @@ pub fn hdfs_index_benchmark(c: &mut Criterion) {
         let lines = get_lines(HDFS_LOGS);
         b.iter(|| {
             let index = Index::create_in_ram(schema.clone());
-            let index_writer = index.writer_with_num_threads(1, 100_000_000).unwrap();
+            let index_writer: IndexWriter = index.writer_with_num_threads(1, 100_000_000).unwrap();
             for doc_json in &lines {
-                let doc = schema.parse_document(doc_json).unwrap();
+                let doc = TantivyDocument::parse_json(&schema, doc_json).unwrap();
                 index_writer.add_document(doc).unwrap();
             }
         })
@@ -50,9 +50,10 @@ pub fn hdfs_index_benchmark(c: &mut Criterion) {
         let lines = get_lines(HDFS_LOGS);
         b.iter(|| {
             let index = Index::create_in_ram(schema.clone());
-            let mut index_writer = index.writer_with_num_threads(1, 100_000_000).unwrap();
+            let mut index_writer: IndexWriter =
+                index.writer_with_num_threads(1, 100_000_000).unwrap();
             for doc_json in &lines {
-                let doc = schema.parse_document(doc_json).unwrap();
+                let doc = TantivyDocument::parse_json(&schema, doc_json).unwrap();
                 index_writer.add_document(doc).unwrap();
             }
             index_writer.commit().unwrap();
@@ -62,9 +63,9 @@ pub fn hdfs_index_benchmark(c: &mut Criterion) {
         let lines = get_lines(HDFS_LOGS);
         b.iter(|| {
             let index = Index::create_in_ram(schema_with_store.clone());
-            let index_writer = index.writer_with_num_threads(1, 100_000_000).unwrap();
+            let index_writer: IndexWriter = index.writer_with_num_threads(1, 100_000_000).unwrap();
             for doc_json in &lines {
-                let doc = schema.parse_document(doc_json).unwrap();
+                let doc = TantivyDocument::parse_json(&schema, doc_json).unwrap();
                 index_writer.add_document(doc).unwrap();
             }
         })
@@ -73,9 +74,10 @@ pub fn hdfs_index_benchmark(c: &mut Criterion) {
         let lines = get_lines(HDFS_LOGS);
         b.iter(|| {
             let index = Index::create_in_ram(schema_with_store.clone());
-            let mut index_writer = index.writer_with_num_threads(1, 100_000_000).unwrap();
+            let mut index_writer: IndexWriter =
+                index.writer_with_num_threads(1, 100_000_000).unwrap();
             for doc_json in &lines {
-                let doc = schema.parse_document(doc_json).unwrap();
+                let doc = TantivyDocument::parse_json(&schema, doc_json).unwrap();
                 index_writer.add_document(doc).unwrap();
             }
             index_writer.commit().unwrap();
@@ -86,7 +88,8 @@ pub fn hdfs_index_benchmark(c: &mut Criterion) {
         b.iter(|| {
             let index = Index::create_in_ram(dynamic_schema.clone());
             let json_field = dynamic_schema.get_field("json").unwrap();
-            let mut index_writer = index.writer_with_num_threads(1, 100_000_000).unwrap();
+            let mut index_writer: IndexWriter =
+                index.writer_with_num_threads(1, 100_000_000).unwrap();
             for doc_json in &lines {
                 let json_val: serde_json::Map<String, serde_json::Value> =
                     serde_json::from_str(doc_json).unwrap();
@@ -113,7 +116,7 @@ pub fn gh_index_benchmark(c: &mut Criterion) {
         b.iter(|| {
             let json_field = dynamic_schema.get_field("json").unwrap();
             let index = Index::create_in_ram(dynamic_schema.clone());
-            let index_writer = index.writer_with_num_threads(1, 100_000_000).unwrap();
+            let index_writer: IndexWriter = index.writer_with_num_threads(1, 100_000_000).unwrap();
             for doc_json in &lines {
                 let json_val: serde_json::Map<String, serde_json::Value> =
                     serde_json::from_str(doc_json).unwrap();
@@ -127,7 +130,8 @@ pub fn gh_index_benchmark(c: &mut Criterion) {
         b.iter(|| {
             let json_field = dynamic_schema.get_field("json").unwrap();
             let index = Index::create_in_ram(dynamic_schema.clone());
-            let mut index_writer = index.writer_with_num_threads(1, 100_000_000).unwrap();
+            let mut index_writer: IndexWriter =
+                index.writer_with_num_threads(1, 100_000_000).unwrap();
             for doc_json in &lines {
                 let json_val: serde_json::Map<String, serde_json::Value> =
                     serde_json::from_str(doc_json).unwrap();
@@ -154,7 +158,7 @@ pub fn wiki_index_benchmark(c: &mut Criterion) {
         b.iter(|| {
             let json_field = dynamic_schema.get_field("json").unwrap();
             let index = Index::create_in_ram(dynamic_schema.clone());
-            let index_writer = index.writer_with_num_threads(1, 100_000_000).unwrap();
+            let index_writer: IndexWriter = index.writer_with_num_threads(1, 100_000_000).unwrap();
             for doc_json in &lines {
                 let json_val: serde_json::Map<String, serde_json::Value> =
                     serde_json::from_str(doc_json).unwrap();
@@ -168,7 +172,8 @@ pub fn wiki_index_benchmark(c: &mut Criterion) {
         b.iter(|| {
             let json_field = dynamic_schema.get_field("json").unwrap();
             let index = Index::create_in_ram(dynamic_schema.clone());
-            let mut index_writer = index.writer_with_num_threads(1, 100_000_000).unwrap();
+            let mut index_writer: IndexWriter =
+                index.writer_with_num_threads(1, 100_000_000).unwrap();
             for doc_json in &lines {
                 let json_val: serde_json::Map<String, serde_json::Value> =
                     serde_json::from_str(doc_json).unwrap();

columnar/src/column_index/mod.rs

@@ -1,3 +1,8 @@
+//! # `column_index`
+//!
+//! `column_index` provides rank and select operations to associate positions when not all
+//! documents have exactly one element.
+
 mod merge;
 mod multivalued_index;
 mod optional_index;
@@ -41,10 +46,10 @@ impl ColumnIndex {
     pub fn is_multivalue(&self) -> bool {
         matches!(self, ColumnIndex::Multivalued(_))
     }
-    // Returns the cardinality of the column index.
-    //
-    // By convention, if the column contains no docs, we consider that it is
-    // full.
+    /// Returns the cardinality of the column index.
+    ///
+    /// By convention, if the column contains no docs, we consider that it is
+    /// full.
     #[inline]
     pub fn get_cardinality(&self) -> Cardinality {
         match self {

columnar/src/column_index/serialize.rs

@@ -30,6 +30,7 @@ impl<'a> SerializableColumnIndex<'a> {
     }
 }
 
+/// Serialize a column index.
 pub fn serialize_column_index(
     column_index: SerializableColumnIndex,
     output: &mut impl Write,
@@ -51,6 +52,7 @@ pub fn serialize_column_index(
     Ok(column_index_num_bytes)
 }
 
+/// Open a serialized column index.
 pub fn open_column_index(mut bytes: OwnedBytes) -> io::Result<ColumnIndex> {
     if bytes.is_empty() {
         return Err(io::Error::new(

columnar/src/lib.rs

@@ -12,7 +12,7 @@ use std::io;
 
 mod block_accessor;
 mod column;
-mod column_index;
+pub mod column_index;
 pub mod column_values;
 mod columnar;
 mod dictionary;

common/src/datetime.rs

@@ -1,11 +1,14 @@
 #![allow(deprecated)]
 
 use std::fmt;
+use std::io::{Read, Write};
 
 use serde::{Deserialize, Serialize};
 use time::format_description::well_known::Rfc3339;
 use time::{OffsetDateTime, PrimitiveDateTime, UtcOffset};
 
+use crate::BinarySerializable;
+
 /// Precision with which datetimes are truncated when stored in fast fields. This setting is only
 /// relevant for fast fields. In the docstore, datetimes are always saved with nanosecond precision.
 #[derive(
@@ -164,3 +167,15 @@ impl fmt::Debug for DateTime {
         f.write_str(&utc_rfc3339)
     }
 }
+
+impl BinarySerializable for DateTime {
+    fn serialize<W: Write + ?Sized>(&self, writer: &mut W) -> std::io::Result<()> {
+        let timestamp_micros = self.into_timestamp_micros();
+        <i64 as BinarySerializable>::serialize(&timestamp_micros, writer)
+    }
+
+    fn deserialize<R: Read>(reader: &mut R) -> std::io::Result<Self> {
+        let timestamp_micros = <i64 as BinarySerializable>::deserialize(reader)?;
+        Ok(Self::from_timestamp_micros(timestamp_micros))
+    }
+}

common/src/serialize.rs

@@ -1,3 +1,4 @@
+use std::borrow::Cow;
 use std::io::{Read, Write};
 use std::{fmt, io};
 
@@ -249,6 +250,43 @@ impl BinarySerializable for String {
     }
 }
 
+impl<'a> BinarySerializable for Cow<'a, str> {
+    fn serialize<W: Write + ?Sized>(&self, writer: &mut W) -> io::Result<()> {
+        let data: &[u8] = self.as_bytes();
+        VInt(data.len() as u64).serialize(writer)?;
+        writer.write_all(data)
+    }
+
+    fn deserialize<R: Read>(reader: &mut R) -> io::Result<Cow<'a, str>> {
+        let string_length = VInt::deserialize(reader)?.val() as usize;
+        let mut result = String::with_capacity(string_length);
+        reader
+            .take(string_length as u64)
+            .read_to_string(&mut result)?;
+        Ok(Cow::Owned(result))
+    }
+}
+
+impl<'a> BinarySerializable for Cow<'a, [u8]> {
+    fn serialize<W: Write + ?Sized>(&self, writer: &mut W) -> io::Result<()> {
+        VInt(self.len() as u64).serialize(writer)?;
+        for it in self.iter() {
+            it.serialize(writer)?;
+        }
+        Ok(())
+    }
+
+    fn deserialize<R: Read>(reader: &mut R) -> io::Result<Cow<'a, [u8]>> {
+        let num_items = VInt::deserialize(reader)?.val();
+        let mut items: Vec<u8> = Vec::with_capacity(num_items as usize);
+        for _ in 0..num_items {
+            let item = u8::deserialize(reader)?;
+            items.push(item);
+        }
+        Ok(Cow::Owned(items))
+    }
+}
+
 #[cfg(test)]
 pub mod test {
 

examples/aggregation.rs

@@ -12,7 +12,7 @@ use tantivy::aggregation::agg_result::AggregationResults;
 use tantivy::aggregation::AggregationCollector;
 use tantivy::query::AllQuery;
 use tantivy::schema::{self, IndexRecordOption, Schema, TextFieldIndexing, FAST};
-use tantivy::Index;
+use tantivy::{Index, IndexWriter, TantivyDocument};
 
 fn main() -> tantivy::Result<()> {
     // # Create Schema
@@ -132,10 +132,10 @@ fn main() -> tantivy::Result<()> {
 
     let stream = Deserializer::from_str(data).into_iter::<Value>();
 
-    let mut index_writer = index.writer(50_000_000)?;
+    let mut index_writer: IndexWriter = index.writer(50_000_000)?;
     let mut num_indexed = 0;
     for value in stream {
-        let doc = schema.parse_document(&serde_json::to_string(&value.unwrap())?)?;
+        let doc = TantivyDocument::parse_json(&schema, &serde_json::to_string(&value.unwrap())?)?;
         index_writer.add_document(doc)?;
         num_indexed += 1;
         if num_indexed > 4 {

examples/basic_search.rs

@@ -15,7 +15,7 @@
 use tantivy::collector::TopDocs;
 use tantivy::query::QueryParser;
 use tantivy::schema::*;
-use tantivy::{doc, Index, ReloadPolicy};
+use tantivy::{doc, Index, IndexWriter, ReloadPolicy};
 use tempfile::TempDir;
 
 fn main() -> tantivy::Result<()> {
@@ -75,7 +75,7 @@ fn main() -> tantivy::Result<()> {
     // Here we give tantivy a budget of `50MB`.
     // Using a bigger memory_arena for the indexer may increase
     // throughput, but 50 MB is already plenty.
-    let mut index_writer = index.writer(50_000_000)?;
+    let mut index_writer: IndexWriter = index.writer(50_000_000)?;
 
     // Let's index our documents!
     // We first need a handle on the title and the body field.
@@ -87,7 +87,7 @@ fn main() -> tantivy::Result<()> {
     let title = schema.get_field("title").unwrap();
     let body = schema.get_field("body").unwrap();
 
-    let mut old_man_doc = Document::default();
+    let mut old_man_doc = TantivyDocument::default();
     old_man_doc.add_text(title, "The Old Man and the Sea");
     old_man_doc.add_text(
         body,
@@ -217,8 +217,8 @@ fn main() -> tantivy::Result<()> {
     // the document returned will only contain
     // a title.
     for (_score, doc_address) in top_docs {
-        let retrieved_doc = searcher.doc(doc_address)?;
-        println!("{}", schema.to_json(&retrieved_doc));
+        let retrieved_doc: TantivyDocument = searcher.doc(doc_address)?;
+        println!("{}", retrieved_doc.to_json(&schema));
     }
 
     // We can also get an explanation to understand

examples/custom_collector.rs

@@ -13,7 +13,7 @@ use columnar::Column;
 use tantivy::collector::{Collector, SegmentCollector};
 use tantivy::query::QueryParser;
 use tantivy::schema::{Schema, FAST, INDEXED, TEXT};
-use tantivy::{doc, Index, Score, SegmentReader};
+use tantivy::{doc, Index, IndexWriter, Score, SegmentReader};
 
 #[derive(Default)]
 struct Stats {
@@ -142,7 +142,7 @@ fn main() -> tantivy::Result<()> {
     // this example.
     let index = Index::create_in_ram(schema);
 
-    let mut index_writer = index.writer(50_000_000)?;
+    let mut index_writer: IndexWriter = index.writer(50_000_000)?;
     index_writer.add_document(doc!(
         product_name => "Super Broom 2000",
         product_description => "While it is ok for short distance travel, this broom \

examples/custom_tokenizer.rs

@@ -6,7 +6,7 @@ use tantivy::collector::TopDocs;
 use tantivy::query::QueryParser;
 use tantivy::schema::*;
 use tantivy::tokenizer::NgramTokenizer;
-use tantivy::{doc, Index};
+use tantivy::{doc, Index, IndexWriter};
 
 fn main() -> tantivy::Result<()> {
     // # Defining the schema
@@ -62,7 +62,7 @@ fn main() -> tantivy::Result<()> {
     //
     // Here we use a buffer of 50MB per thread. Using a bigger
     // memory arena for the indexer can increase its throughput.
-    let mut index_writer = index.writer(50_000_000)?;
+    let mut index_writer: IndexWriter = index.writer(50_000_000)?;
     index_writer.add_document(doc!(
     title => "The Old Man and the Sea",
     body => "He was an old man who fished alone in a skiff in the Gulf Stream and \
@@ -103,8 +103,8 @@ fn main() -> tantivy::Result<()> {
     let top_docs = searcher.search(&query, &TopDocs::with_limit(10))?;
 
     for (_, doc_address) in top_docs {
-        let retrieved_doc = searcher.doc(doc_address)?;
-        println!("{}", schema.to_json(&retrieved_doc));
+        let retrieved_doc: TantivyDocument = searcher.doc(doc_address)?;
+        println!("{}", retrieved_doc.to_json(&schema));
     }
 
     Ok(())

examples/date_time_field.rs

@@ -4,8 +4,8 @@
 
 use tantivy::collector::TopDocs;
 use tantivy::query::QueryParser;
-use tantivy::schema::{DateOptions, Schema, Value, INDEXED, STORED, STRING};
-use tantivy::Index;
+use tantivy::schema::{DateOptions, Document, OwnedValue, Schema, INDEXED, STORED, STRING};
+use tantivy::{Index, IndexWriter, TantivyDocument};
 
 fn main() -> tantivy::Result<()> {
     // # Defining the schema
@@ -22,16 +22,18 @@ fn main() -> tantivy::Result<()> {
     // # Indexing documents
     let index = Index::create_in_ram(schema.clone());
 
-    let mut index_writer = index.writer(50_000_000)?;
+    let mut index_writer: IndexWriter = index.writer(50_000_000)?;
     // The dates are passed as string in the RFC3339 format
-    let doc = schema.parse_document(
+    let doc = TantivyDocument::parse_json(
+        &schema,
         r#"{
         "occurred_at": "2022-06-22T12:53:50.53Z",
         "event": "pull-request"
     }"#,
     )?;
     index_writer.add_document(doc)?;
-    let doc = schema.parse_document(
+    let doc = TantivyDocument::parse_json(
+        &schema,
         r#"{
         "occurred_at": "2022-06-22T13:00:00.22Z",
         "event": "comment"
@@ -58,13 +60,13 @@ fn main() -> tantivy::Result<()> {
         let count_docs = searcher.search(&*query, &TopDocs::with_limit(4))?;
         assert_eq!(count_docs.len(), 1);
         for (_score, doc_address) in count_docs {
-            let retrieved_doc = searcher.doc(doc_address)?;
+            let retrieved_doc = searcher.doc::<TantivyDocument>(doc_address)?;
             assert!(matches!(
                 retrieved_doc.get_first(occurred_at),
-                Some(Value::Date(_))
+                Some(OwnedValue::Date(_))
             ));
             assert_eq!(
-                schema.to_json(&retrieved_doc),
+                retrieved_doc.to_json(&schema),
                 r#"{"event":["comment"],"occurred_at":["2022-06-22T13:00:00.22Z"]}"#
             );
         }

examples/deleting_updating_documents.rs

@@ -11,7 +11,7 @@
 use tantivy::collector::TopDocs;
 use tantivy::query::TermQuery;
 use tantivy::schema::*;
-use tantivy::{doc, Index, IndexReader};
+use tantivy::{doc, Index, IndexReader, IndexWriter};
 
 // A simple helper function to fetch a single document
 // given its id from our index.
@@ -19,7 +19,7 @@ use tantivy::{doc, Index, IndexReader};
 fn extract_doc_given_isbn(
     reader: &IndexReader,
     isbn_term: &Term,
-) -> tantivy::Result<Option<Document>> {
+) -> tantivy::Result<Option<TantivyDocument>> {
     let searcher = reader.searcher();
 
     // This is the simplest query you can think of.
@@ -69,10 +69,10 @@ fn main() -> tantivy::Result<()> {
 
     let index = Index::create_in_ram(schema.clone());
 
-    let mut index_writer = index.writer(50_000_000)?;
+    let mut index_writer: IndexWriter = index.writer(50_000_000)?;
 
     // Let's add a couple of documents, for the sake of the example.
-    let mut old_man_doc = Document::default();
+    let mut old_man_doc = TantivyDocument::default();
     old_man_doc.add_text(title, "The Old Man and the Sea");
     index_writer.add_document(doc!(
         isbn => "978-0099908401",
@@ -94,7 +94,7 @@ fn main() -> tantivy::Result<()> {
     // Oops our frankenstein doc seems misspelled
     let frankenstein_doc_misspelled = extract_doc_given_isbn(&reader, &frankenstein_isbn)?.unwrap();
     assert_eq!(
-        schema.to_json(&frankenstein_doc_misspelled),
+        frankenstein_doc_misspelled.to_json(&schema),
         r#"{"isbn":["978-9176370711"],"title":["Frankentein"]}"#,
     );
 
@@ -136,7 +136,7 @@ fn main() -> tantivy::Result<()> {
     // No more typo!
     let frankenstein_new_doc = extract_doc_given_isbn(&reader, &frankenstein_isbn)?.unwrap();
     assert_eq!(
-        schema.to_json(&frankenstein_new_doc),
+        frankenstein_new_doc.to_json(&schema),
         r#"{"isbn":["978-9176370711"],"title":["Frankenstein"]}"#,
     );
 

examples/faceted_search.rs

@@ -17,7 +17,7 @@
 use tantivy::collector::FacetCollector;
 use tantivy::query::{AllQuery, TermQuery};
 use tantivy::schema::*;
-use tantivy::{doc, Index};
+use tantivy::{doc, Index, IndexWriter};
 
 fn main() -> tantivy::Result<()> {
     // Let's create a temporary directory for the sake of this example
@@ -30,7 +30,7 @@ fn main() -> tantivy::Result<()> {
     let schema = schema_builder.build();
     let index = Index::create_in_ram(schema);
 
-    let mut index_writer = index.writer(30_000_000)?;
+    let mut index_writer: IndexWriter = index.writer(30_000_000)?;
 
     // For convenience, tantivy also comes with a macro to
     // reduce the boilerplate above.

examples/faceted_search_with_tweaked_score.rs

@@ -12,7 +12,7 @@ use std::collections::HashSet;
 use tantivy::collector::TopDocs;
 use tantivy::query::BooleanQuery;
 use tantivy::schema::*;
-use tantivy::{doc, DocId, Index, Score, SegmentReader};
+use tantivy::{doc, DocId, Index, IndexWriter, Score, SegmentReader};
 
 fn main() -> tantivy::Result<()> {
     let mut schema_builder = Schema::builder();
@@ -23,7 +23,7 @@ fn main() -> tantivy::Result<()> {
     let schema = schema_builder.build();
     let index = Index::create_in_ram(schema);
 
-    let mut index_writer = index.writer(30_000_000)?;
+    let mut index_writer: IndexWriter = index.writer(30_000_000)?;
 
     index_writer.add_document(doc!(
         title => "Fried egg",
@@ -91,11 +91,10 @@ fn main() -> tantivy::Result<()> {
             .iter()
             .map(|(_, doc_id)| {
                 searcher
-                    .doc(*doc_id)
+                    .doc::<TantivyDocument>(*doc_id)
                     .unwrap()
                     .get_first(title)
-                    .unwrap()
-                    .as_text()
+                    .and_then(|v| v.as_str())
                     .unwrap()
                     .to_owned()
             })

examples/fuzzy_search.rs

@@ -14,7 +14,7 @@
 use tantivy::collector::{Count, TopDocs};
 use tantivy::query::FuzzyTermQuery;
 use tantivy::schema::*;
-use tantivy::{doc, Index, ReloadPolicy};
+use tantivy::{doc, Index, IndexWriter, ReloadPolicy};
 use tempfile::TempDir;
 
 fn main() -> tantivy::Result<()> {
@@ -66,7 +66,7 @@ fn main() -> tantivy::Result<()> {
     // Here we give tantivy a budget of `50MB`.
     // Using a bigger memory_arena for the indexer may increase
     // throughput, but 50 MB is already plenty.
-    let mut index_writer = index.writer(50_000_000)?;
+    let mut index_writer: IndexWriter = index.writer(50_000_000)?;
 
     // Let's index our documents!
     // We first need a handle on the title and the body field.
@@ -151,10 +151,10 @@ fn main() -> tantivy::Result<()> {
         assert_eq!(count, 3);
         assert_eq!(top_docs.len(), 3);
         for (score, doc_address) in top_docs {
-            let retrieved_doc = searcher.doc(doc_address)?;
             // Note that the score is not lower for the fuzzy hit.
             // There's an issue open for that: https://github.com/quickwit-oss/tantivy/issues/563
-            println!("score {score:?} doc {}", schema.to_json(&retrieved_doc));
+            let retrieved_doc: TantivyDocument = searcher.doc(doc_address)?;
+            println!("score {score:?} doc {}", retrieved_doc.to_json(&schema));
             // score 1.0 doc {"title":["The Diary of Muadib"]}
             //
             // score 1.0 doc {"title":["The Diary of a Young Girl"]}

examples/index_with_json.rs

@@ -21,7 +21,7 @@ fn main() -> tantivy::Result<()> {
     }"#;
 
     // We can parse our document
-    let _mice_and_men_doc = schema.parse_document(mice_and_men_doc_json)?;
+    let _mice_and_men_doc = TantivyDocument::parse_json(&schema, mice_and_men_doc_json)?;
 
     // Multi-valued field are allowed, they are
     // expressed in JSON by an array.
@@ -30,7 +30,7 @@ fn main() -> tantivy::Result<()> {
        "title": ["Frankenstein", "The Modern Prometheus"],
        "year": 1818
     }"#;
-    let _frankenstein_doc = schema.parse_document(frankenstein_json)?;
+    let _frankenstein_doc = TantivyDocument::parse_json(&schema, frankenstein_json)?;
 
     // Note that the schema is saved in your index directory.
     //

examples/integer_range_search.rs

@@ -5,7 +5,7 @@
 use tantivy::collector::Count;
 use tantivy::query::RangeQuery;
 use tantivy::schema::{Schema, INDEXED};
-use tantivy::{doc, Index, Result};
+use tantivy::{doc, Index, IndexWriter, Result};
 
 fn main() -> Result<()> {
     // For the sake of simplicity, this schema will only have 1 field
@@ -17,7 +17,7 @@ fn main() -> Result<()> {
     let index = Index::create_in_ram(schema);
     let reader = index.reader()?;
     {
-        let mut index_writer = index.writer_with_num_threads(1, 6_000_000)?;
+        let mut index_writer: IndexWriter = index.writer_with_num_threads(1, 6_000_000)?;
         for year in 1950u64..2019u64 {
             index_writer.add_document(doc!(year_field => year))?;
         }

examples/ip_field.rs

@@ -6,7 +6,7 @@
 use tantivy::collector::{Count, TopDocs};
 use tantivy::query::QueryParser;
 use tantivy::schema::{Schema, FAST, INDEXED, STORED, STRING};
-use tantivy::Index;
+use tantivy::{Index, IndexWriter, TantivyDocument};
 
 fn main() -> tantivy::Result<()> {
     // # Defining the schema
@@ -22,20 +22,22 @@ fn main() -> tantivy::Result<()> {
     // # Indexing documents
     let index = Index::create_in_ram(schema.clone());
 
-    let mut index_writer = index.writer(50_000_000)?;
+    let mut index_writer: IndexWriter = index.writer(50_000_000)?;
 
     // ### IPv4
     // Adding documents that contain an IPv4 address. Notice that the IP addresses are passed as
     // `String`. Since the field is of type ip, we parse the IP address from the string and store it
     // internally as IPv6.
-    let doc = schema.parse_document(
+    let doc = TantivyDocument::parse_json(
+        &schema,
         r#"{
             "ip": "192.168.0.33",
             "event_type": "login"
         }"#,
     )?;
     index_writer.add_document(doc)?;
-    let doc = schema.parse_document(
+    let doc = TantivyDocument::parse_json(
+        &schema,
         r#"{
             "ip": "192.168.0.80",
             "event_type": "checkout"
@@ -44,7 +46,8 @@ fn main() -> tantivy::Result<()> {
     index_writer.add_document(doc)?;
     // ### IPv6
     // Adding a document that contains an IPv6 address.
-    let doc = schema.parse_document(
+    let doc = TantivyDocument::parse_json(
+        &schema,
         r#"{
             "ip": "2001:0db8:85a3:0000:0000:8a2e:0370:7334",
             "event_type": "checkout"

examples/iterating_docs_and_positions.rs

@@ -10,7 +10,7 @@
 // ---
 // Importing tantivy...
 use tantivy::schema::*;
-use tantivy::{doc, DocSet, Index, Postings, TERMINATED};
+use tantivy::{doc, DocSet, Index, IndexWriter, Postings, TERMINATED};
 
 fn main() -> tantivy::Result<()> {
     // We first create a schema for the sake of the
@@ -24,7 +24,7 @@ fn main() -> tantivy::Result<()> {
 
     let index = Index::create_in_ram(schema);
 
-    let mut index_writer = index.writer_with_num_threads(1, 50_000_000)?;
+    let mut index_writer: IndexWriter = index.writer_with_num_threads(1, 50_000_000)?;
     index_writer.add_document(doc!(title => "The Old Man and the Sea"))?;
     index_writer.add_document(doc!(title => "Of Mice and Men"))?;
     index_writer.add_document(doc!(title => "The modern Promotheus"))?;

examples/json_field.rs

@@ -7,7 +7,7 @@
 use tantivy::collector::{Count, TopDocs};
 use tantivy::query::QueryParser;
 use tantivy::schema::{Schema, FAST, STORED, STRING, TEXT};
-use tantivy::Index;
+use tantivy::{Index, IndexWriter, TantivyDocument};
 
 fn main() -> tantivy::Result<()> {
     // # Defining the schema
@@ -20,8 +20,9 @@ fn main() -> tantivy::Result<()> {
     // # Indexing documents
     let index = Index::create_in_ram(schema.clone());
 
-    let mut index_writer = index.writer(50_000_000)?;
-    let doc = schema.parse_document(
+    let mut index_writer: IndexWriter = index.writer(50_000_000)?;
+    let doc = TantivyDocument::parse_json(
+        &schema,
         r#"{
         "timestamp": "2022-02-22T23:20:50.53Z",
         "event_type": "click",
@@ -33,7 +34,8 @@ fn main() -> tantivy::Result<()> {
     }"#,
     )?;
     index_writer.add_document(doc)?;
-    let doc = schema.parse_document(
+    let doc = TantivyDocument::parse_json(
+        &schema,
         r#"{
         "timestamp": "2022-02-22T23:20:51.53Z",
         "event_type": "click",

examples/phrase_prefix_search.rs

@@ -1,7 +1,7 @@
 use tantivy::collector::TopDocs;
 use tantivy::query::QueryParser;
 use tantivy::schema::*;
-use tantivy::{doc, Index, ReloadPolicy, Result};
+use tantivy::{doc, Index, IndexWriter, ReloadPolicy, Result};
 use tempfile::TempDir;
 
 fn main() -> Result<()> {
@@ -17,7 +17,7 @@ fn main() -> Result<()> {
 
     let index = Index::create_in_dir(&index_path, schema)?;
 
-    let mut index_writer = index.writer(50_000_000)?;
+    let mut index_writer: IndexWriter = index.writer(50_000_000)?;
 
     index_writer.add_document(doc!(
     title => "The Old Man and the Sea",
@@ -67,8 +67,12 @@ fn main() -> Result<()> {
     let mut titles = top_docs
         .into_iter()
         .map(|(_score, doc_address)| {
-            let doc = searcher.doc(doc_address)?;
-            let title = doc.get_first(title).unwrap().as_text().unwrap().to_owned();
+            let doc = searcher.doc::<TantivyDocument>(doc_address)?;
+            let title = doc
+                .get_first(title)
+                .and_then(|v| v.as_str())
+                .unwrap()
+                .to_owned();
             Ok(title)
         })
         .collect::<Result<Vec<_>>>()?;

examples/pre_tokenized_text.rs

@@ -13,7 +13,7 @@ use tantivy::collector::{Count, TopDocs};
 use tantivy::query::TermQuery;
 use tantivy::schema::*;
 use tantivy::tokenizer::{PreTokenizedString, SimpleTokenizer, Token, TokenStream, Tokenizer};
-use tantivy::{doc, Index, ReloadPolicy};
+use tantivy::{doc, Index, IndexWriter, ReloadPolicy};
 use tempfile::TempDir;
 
 fn pre_tokenize_text(text: &str) -> Vec<Token> {
@@ -38,7 +38,7 @@ fn main() -> tantivy::Result<()> {
 
     let index = Index::create_in_dir(&index_path, schema.clone())?;
 
-    let mut index_writer = index.writer(50_000_000)?;
+    let mut index_writer: IndexWriter = index.writer(50_000_000)?;
 
     // We can create a document manually, by setting the fields
     // one by one in a Document object.
@@ -83,7 +83,7 @@ fn main() -> tantivy::Result<()> {
         }]
     }"#;
 
-    let short_man_doc = schema.parse_document(short_man_json)?;
+    let short_man_doc = TantivyDocument::parse_json(&schema, short_man_json)?;
 
     index_writer.add_document(short_man_doc)?;
 
@@ -115,8 +115,8 @@ fn main() -> tantivy::Result<()> {
     // Note that the tokens are not stored along with the original text
     // in the document store
     for (_score, doc_address) in top_docs {
-        let retrieved_doc = searcher.doc(doc_address)?;
-        println!("Document: {}", schema.to_json(&retrieved_doc));
+        let retrieved_doc: TantivyDocument = searcher.doc(doc_address)?;
+        println!("{}", retrieved_doc.to_json(&schema));
     }
 
     // In contrary to the previous query, when we search for the "man" term we

examples/snippet.rs

@@ -10,7 +10,7 @@
 use tantivy::collector::TopDocs;
 use tantivy::query::QueryParser;
 use tantivy::schema::*;
-use tantivy::{doc, Index, Snippet, SnippetGenerator};
+use tantivy::{doc, Index, IndexWriter, Snippet, SnippetGenerator};
 use tempfile::TempDir;
 
 fn main() -> tantivy::Result<()> {
@@ -27,7 +27,7 @@ fn main() -> tantivy::Result<()> {
     // # Indexing documents
     let index = Index::create_in_dir(&index_path, schema)?;
 
-    let mut index_writer = index.writer(50_000_000)?;
+    let mut index_writer: IndexWriter = index.writer(50_000_000)?;
 
     // we'll only need one doc for this example.
     index_writer.add_document(doc!(
@@ -54,13 +54,10 @@ fn main() -> tantivy::Result<()> {
     let snippet_generator = SnippetGenerator::create(&searcher, &*query, body)?;
 
     for (score, doc_address) in top_docs {
-        let doc = searcher.doc(doc_address)?;
+        let doc = searcher.doc::<TantivyDocument>(doc_address)?;
         let snippet = snippet_generator.snippet_from_doc(&doc);
         println!("Document score {score}:");
-        println!(
-            "title: {}",
-            doc.get_first(title).unwrap().as_text().unwrap()
-        );
+        println!("title: {}", doc.get_first(title).unwrap().as_str().unwrap());
         println!("snippet: {}", snippet.to_html());
         println!("custom highlighting: {}", highlight(snippet));
     }

examples/stop_words.rs

@@ -15,7 +15,7 @@ use tantivy::collector::TopDocs;
 use tantivy::query::QueryParser;
 use tantivy::schema::*;
 use tantivy::tokenizer::*;
-use tantivy::{doc, Index};
+use tantivy::{doc, Index, IndexWriter};
 
 fn main() -> tantivy::Result<()> {
     // this example assumes you understand the content in `basic_search`
@@ -60,7 +60,7 @@ fn main() -> tantivy::Result<()> {
 
     index.tokenizers().register("stoppy", tokenizer);
 
-    let mut index_writer = index.writer(50_000_000)?;
+    let mut index_writer: IndexWriter = index.writer(50_000_000)?;
 
     let title = schema.get_field("title").unwrap();
     let body = schema.get_field("body").unwrap();
@@ -105,9 +105,9 @@ fn main() -> tantivy::Result<()> {
     let top_docs = searcher.search(&query, &TopDocs::with_limit(10))?;
 
     for (score, doc_address) in top_docs {
-        let retrieved_doc = searcher.doc(doc_address)?;
+        let retrieved_doc: TantivyDocument = searcher.doc(doc_address)?;
         println!("\n==\nDocument score {score}:");
-        println!("{}", schema.to_json(&retrieved_doc));
+        println!("{}", retrieved_doc.to_json(&schema));
     }
 
     Ok(())

examples/warmer.rs

@@ -6,8 +6,8 @@ use tantivy::collector::TopDocs;
 use tantivy::query::QueryParser;
 use tantivy::schema::{Schema, FAST, TEXT};
 use tantivy::{
-    doc, DocAddress, DocId, Index, Opstamp, Searcher, SearcherGeneration, SegmentId, SegmentReader,
-    Warmer,
+    doc, DocAddress, DocId, Index, IndexWriter, Opstamp, Searcher, SearcherGeneration, SegmentId,
+    SegmentReader, Warmer,
 };
 
 // This example shows how warmers can be used to
@@ -143,7 +143,7 @@ fn main() -> tantivy::Result<()> {
     const SNEAKERS: ProductId = 23222;
 
     let index = Index::create_in_ram(schema);
-    let mut writer = index.writer_with_num_threads(1, 15_000_000)?;
+    let mut writer: IndexWriter = index.writer_with_num_threads(1, 15_000_000)?;
     writer.add_document(doc!(product_id=>OLIVE_OIL, text=>"cooking olive oil from greece"))?;
     writer.add_document(doc!(product_id=>GLOVES, text=>"kitchen gloves, perfect for cooking"))?;
     writer.add_document(doc!(product_id=>SNEAKERS, text=>"uber sweet sneakers"))?;

query-grammar/src/query_grammar.rs

@@ -24,7 +24,7 @@ const SPECIAL_CHARS: &[char] = &[
 
 /// consume a field name followed by colon. Return the field name with escape sequence
 /// already interpreted
-fn field_name(i: &str) -> IResult<&str, String> {
+fn field_name(inp: &str) -> IResult<&str, String> {
     let simple_char = none_of(SPECIAL_CHARS);
     let first_char = verify(none_of(SPECIAL_CHARS), |c| *c != '-');
     let escape_sequence = || preceded(char('\\'), one_of(SPECIAL_CHARS));
@@ -38,12 +38,12 @@ fn field_name(i: &str) -> IResult<&str, String> {
             char(':'),
         ),
         |(first_char, next)| once(first_char).chain(next).collect(),
-    )(i)
+    )(inp)
 }
 
 /// Consume a word outside of any context.
 // TODO should support escape sequences
-fn word(i: &str) -> IResult<&str, &str> {
+fn word(inp: &str) -> IResult<&str, &str> {
     map_res(
         recognize(tuple((
             satisfy(|c| {
@@ -55,14 +55,14 @@ fn word(i: &str) -> IResult<&str, &str> {
             })),
         ))),
         |s| match s {
-            "OR" | "AND" | "NOT" | "IN" => Err(Error::new(i, ErrorKind::Tag)),
+            "OR" | "AND" | "NOT" | "IN" => Err(Error::new(inp, ErrorKind::Tag)),
             _ => Ok(s),
         },
-    )(i)
+    )(inp)
 }
 
 fn word_infallible(delimiter: &str) -> impl Fn(&str) -> JResult<&str, Option<&str>> + '_ {
-    |i| {
+    |inp| {
         opt_i_err(
             preceded(
                 space0,
@@ -71,29 +71,29 @@ fn word_infallible(delimiter: &str) -> impl Fn(&str) -> JResult<&str, Option<&st
                 }))),
             ),
             "expected word",
-        )(i)
+        )(inp)
     }
 }
 
 /// Consume a word inside a Range context. More values are allowed as they are
 /// not ambiguous in this context.
-fn relaxed_word(i: &str) -> IResult<&str, &str> {
+fn relaxed_word(inp: &str) -> IResult<&str, &str> {
     recognize(tuple((
         satisfy(|c| !c.is_whitespace() && !['`', '{', '}', '"', '[', ']', '(', ')'].contains(&c)),
         many0(satisfy(|c: char| {
             !c.is_whitespace() && !['{', '}', '"', '[', ']', '(', ')'].contains(&c)
         })),
-    )))(i)
+    )))(inp)
 }
 
-fn negative_number(i: &str) -> IResult<&str, &str> {
+fn negative_number(inp: &str) -> IResult<&str, &str> {
     recognize(preceded(
         char('-'),
         tuple((digit1, opt(tuple((char('.'), digit1))))),
-    ))(i)
+    ))(inp)
 }
 
-fn simple_term(i: &str) -> IResult<&str, (Delimiter, String)> {
+fn simple_term(inp: &str) -> IResult<&str, (Delimiter, String)> {
     let escaped_string = |delimiter| {
         // we need this because none_of can't accept an owned array of char.
         let not_delimiter = verify(anychar, move |parsed| *parsed != delimiter);
@@ -123,13 +123,13 @@ fn simple_term(i: &str) -> IResult<&str, (Delimiter, String)> {
         simple_quotes,
         double_quotes,
         text_no_delimiter,
-    ))(i)
+    ))(inp)
 }
 
 fn simple_term_infallible(
     delimiter: &str,
 ) -> impl Fn(&str) -> JResult<&str, Option<(Delimiter, String)>> + '_ {
-    |i| {
+    |inp| {
         let escaped_string = |delimiter| {
             // we need this because none_of can't accept an owned array of char.
             let not_delimiter = verify(anychar, move |parsed| *parsed != delimiter);
@@ -162,11 +162,11 @@ fn simple_term_infallible(
             map(word_infallible(delimiter), |(text, errors)| {
                 (text.map(|text| (Delimiter::None, text.to_string())), errors)
             }),
-        )(i)
+        )(inp)
     }
 }
 
-fn term_or_phrase(i: &str) -> IResult<&str, UserInputLeaf> {
+fn term_or_phrase(inp: &str) -> IResult<&str, UserInputLeaf> {
     map(
         tuple((simple_term, fallible(slop_or_prefix_val))),
         |((delimiter, phrase), (slop, prefix))| {
@@ -179,10 +179,10 @@ fn term_or_phrase(i: &str) -> IResult<&str, UserInputLeaf> {
             }
             .into()
         },
-    )(i)
+    )(inp)
 }
 
-fn term_or_phrase_infallible(i: &str) -> JResult<&str, Option<UserInputLeaf>> {
+fn term_or_phrase_infallible(inp: &str) -> JResult<&str, Option<UserInputLeaf>> {
     map(
         // ~* for slop/prefix, ) inside group or ast tree, ^ if boost
         tuple_infallible((simple_term_infallible("*)^"), slop_or_prefix_val)),
@@ -214,10 +214,10 @@ fn term_or_phrase_infallible(i: &str) -> JResult<&str, Option<UserInputLeaf>> {
             };
             (leaf, errors)
         },
-    )(i)
+    )(inp)
 }
 
-fn term_group(i: &str) -> IResult<&str, UserInputAst> {
+fn term_group(inp: &str) -> IResult<&str, UserInputAst> {
     let occur_symbol = alt((
         value(Occur::MustNot, char('-')),
         value(Occur::Must, char('+')),
@@ -240,12 +240,12 @@ fn term_group(i: &str) -> IResult<&str, UserInputAst> {
                     .collect(),
             )
         },
-    )(i)
+    )(inp)
 }
 
 // this is a precondition for term_group_infallible. Without it, term_group_infallible can fail
 // with a panic. It does not consume its input.
-fn term_group_precond(i: &str) -> IResult<&str, (), ()> {
+fn term_group_precond(inp: &str) -> IResult<&str, (), ()> {
     value(
         (),
         peek(tuple((
@@ -253,13 +253,13 @@ fn term_group_precond(i: &str) -> IResult<&str, (), ()> {
             space0,
             char('('), // when we are here, we know it can't be anything but a term group
         ))),
-    )(i)
+    )(inp)
     .map_err(|e| e.map(|_| ()))
 }
 
-fn term_group_infallible(i: &str) -> JResult<&str, UserInputAst> {
-    let (mut i, (field_name, _, _, _)) =
-        tuple((field_name, space0, char('('), space0))(i).expect("precondition failed");
+fn term_group_infallible(inp: &str) -> JResult<&str, UserInputAst> {
+    let (mut inp, (field_name, _, _, _)) =
+        tuple((field_name, space0, char('('), space0))(inp).expect("precondition failed");
 
     let mut terms = Vec::new();
     let mut errs = Vec::new();
@@ -270,19 +270,19 @@ fn term_group_infallible(i: &str) -> JResult<&str, UserInputAst> {
             first_round = false;
             Vec::new()
         } else {
-            let (rest, (_, err)) = space1_infallible(i)?;
-            i = rest;
+            let (rest, (_, err)) = space1_infallible(inp)?;
+            inp = rest;
             err
         };
-        if i.is_empty() {
+        if inp.is_empty() {
             errs.push(LenientErrorInternal {
-                pos: i.len(),
+                pos: inp.len(),
                 message: "missing )".to_string(),
             });
-            break Ok((i, (UserInputAst::Clause(terms), errs)));
+            break Ok((inp, (UserInputAst::Clause(terms), errs)));
         }
-        if let Some(i) = i.strip_prefix(')') {
-            break Ok((i, (UserInputAst::Clause(terms), errs)));
+        if let Some(inp) = inp.strip_prefix(')') {
+            break Ok((inp, (UserInputAst::Clause(terms), errs)));
         }
         // only append missing space error if we did not reach the end of group
         errs.append(&mut space_error);
@@ -291,26 +291,57 @@ fn term_group_infallible(i: &str) -> JResult<&str, UserInputAst> {
         // first byte is not `)` or ' '. If it did not, we would end up looping.
 
         let (rest, ((occur, leaf), mut err)) =
-            tuple_infallible((occur_symbol, term_or_phrase_infallible))(i)?;
+            tuple_infallible((occur_symbol, term_or_phrase_infallible))(inp)?;
         errs.append(&mut err);
         if let Some(leaf) = leaf {
             terms.push((occur, leaf.set_field(Some(field_name.clone())).into()));
         }
-        i = rest;
+        inp = rest;
     }
 }
 
-fn literal(i: &str) -> IResult<&str, UserInputAst> {
+fn exists(inp: &str) -> IResult<&str, UserInputLeaf> {
+    value(
+        UserInputLeaf::Exists {
+            field: String::new(),
+        },
+        tuple((space0, char('*'))),
+    )(inp)
+}
+
+fn exists_precond(inp: &str) -> IResult<&str, (), ()> {
+    value(
+        (),
+        peek(tuple((
+            field_name,
+            space0,
+            char('*'), // when we are here, we know it can't be anything but a exists
+        ))),
+    )(inp)
+    .map_err(|e| e.map(|_| ()))
+}
+
+fn exists_infallible(inp: &str) -> JResult<&str, UserInputAst> {
+    let (inp, (field_name, _, _)) =
+        tuple((field_name, space0, char('*')))(inp).expect("precondition failed");
+
+    let exists = UserInputLeaf::Exists { field: field_name }.into();
+    Ok((inp, (exists, Vec::new())))
+}
+
+fn literal(inp: &str) -> IResult<&str, UserInputAst> {
+    // * alone is already parsed by our caller, so if `exists` succeed, we can be confident
+    // something (a field name) got parsed before
     alt((
         map(
-            tuple((opt(field_name), alt((range, set, term_or_phrase)))),
+            tuple((opt(field_name), alt((range, set, exists, term_or_phrase)))),
             |(field_name, leaf): (Option<String>, UserInputLeaf)| leaf.set_field(field_name).into(),
         ),
         term_group,
-    ))(i)
+    ))(inp)
 }
 
-fn literal_no_group_infallible(i: &str) -> JResult<&str, Option<UserInputAst>> {
+fn literal_no_group_infallible(inp: &str) -> JResult<&str, Option<UserInputAst>> {
     map(
         tuple_infallible((
             opt_i(field_name),
@@ -337,7 +368,7 @@ fn literal_no_group_infallible(i: &str) -> JResult<&str, Option<UserInputAst>> {
                         && field_name.is_none()
                     {
                         errors.push(LenientErrorInternal {
-                            pos: i.len(),
+                            pos: inp.len(),
                             message: "parsed possible invalid field as term".to_string(),
                         });
                     }
@@ -346,7 +377,7 @@ fn literal_no_group_infallible(i: &str) -> JResult<&str, Option<UserInputAst>> {
                         && field_name.is_none()
                     {
                         errors.push(LenientErrorInternal {
-                            pos: i.len(),
+                            pos: inp.len(),
                             message: "parsed keyword NOT as term. It should be quoted".to_string(),
                         });
                     }
@@ -355,34 +386,40 @@ fn literal_no_group_infallible(i: &str) -> JResult<&str, Option<UserInputAst>> {
                 errors,
             )
         },
-    )(i)
+    )(inp)
 }
 
-fn literal_infallible(i: &str) -> JResult<&str, Option<UserInputAst>> {
+fn literal_infallible(inp: &str) -> JResult<&str, Option<UserInputAst>> {
     alt_infallible(
-        ((
-            term_group_precond,
-            map(term_group_infallible, |(group, errs)| (Some(group), errs)),
-        ),),
+        (
+            (
+                term_group_precond,
+                map(term_group_infallible, |(group, errs)| (Some(group), errs)),
+            ),
+            (
+                exists_precond,
+                map(exists_infallible, |(exists, errs)| (Some(exists), errs)),
+            ),
+        ),
         literal_no_group_infallible,
-    )(i)
+    )(inp)
 }
 
-fn slop_or_prefix_val(i: &str) -> JResult<&str, (u32, bool)> {
+fn slop_or_prefix_val(inp: &str) -> JResult<&str, (u32, bool)> {
     map(
         opt_i(alt((
             value((0, true), char('*')),
             map(preceded(char('~'), u32), |slop| (slop, false)),
         ))),
         |(slop_or_prefix_opt, err)| (slop_or_prefix_opt.unwrap_or_default(), err),
-    )(i)
+    )(inp)
 }
 
 /// Function that parses a range out of a Stream
 /// Supports ranges like:
 /// [5 TO 10], {5 TO 10}, [* TO 10], [10 TO *], {10 TO *], >5, <=10
 /// [a TO *], [a TO c], [abc TO bcd}
-fn range(i: &str) -> IResult<&str, UserInputLeaf> {
+fn range(inp: &str) -> IResult<&str, UserInputLeaf> {
     let range_term_val = || {
         map(
             alt((negative_number, relaxed_word, tag("*"))),
@@ -442,10 +479,10 @@ fn range(i: &str) -> IResult<&str, UserInputLeaf> {
             lower,
             upper,
         },
-    )(i)
+    )(inp)
 }
 
-fn range_infallible(i: &str) -> JResult<&str, UserInputLeaf> {
+fn range_infallible(inp: &str) -> JResult<&str, UserInputLeaf> {
     let lower_to_upper = map(
         tuple_infallible((
             opt_i(anychar),
@@ -553,10 +590,10 @@ fn range_infallible(i: &str) -> JResult<&str, UserInputLeaf> {
                 errors,
             )
         },
-    )(i)
+    )(inp)
 }
 
-fn set(i: &str) -> IResult<&str, UserInputLeaf> {
+fn set(inp: &str) -> IResult<&str, UserInputLeaf> {
     map(
         preceded(
             tuple((space0, tag("IN"), space1)),
@@ -570,10 +607,10 @@ fn set(i: &str) -> IResult<&str, UserInputLeaf> {
             field: None,
             elements,
         },
-    )(i)
+    )(inp)
 }
 
-fn set_infallible(mut i: &str) -> JResult<&str, UserInputLeaf> {
+fn set_infallible(mut inp: &str) -> JResult<&str, UserInputLeaf> {
     // `IN [` has already been parsed when we enter, we only need to parse simple terms until we
     // find a `]`
     let mut elements = Vec::new();
@@ -584,41 +621,41 @@ fn set_infallible(mut i: &str) -> JResult<&str, UserInputLeaf> {
             first_round = false;
             Vec::new()
         } else {
-            let (rest, (_, err)) = space1_infallible(i)?;
-            i = rest;
+            let (rest, (_, err)) = space1_infallible(inp)?;
+            inp = rest;
             err
         };
-        if i.is_empty() {
+        if inp.is_empty() {
             // TODO push error about missing ]
             //
             errs.push(LenientErrorInternal {
-                pos: i.len(),
+                pos: inp.len(),
                 message: "missing ]".to_string(),
             });
             let res = UserInputLeaf::Set {
                 field: None,
                 elements,
             };
-            return Ok((i, (res, errs)));
+            return Ok((inp, (res, errs)));
         }
-        if let Some(i) = i.strip_prefix(']') {
+        if let Some(inp) = inp.strip_prefix(']') {
             let res = UserInputLeaf::Set {
                 field: None,
                 elements,
             };
-            return Ok((i, (res, errs)));
+            return Ok((inp, (res, errs)));
         }
         errs.append(&mut space_error);
         // TODO
         // here we do the assumption term_or_phrase_infallible always consume something if the
         // first byte is not `)` or ' '. If it did not, we would end up looping.
 
-        let (rest, (delim_term, mut err)) = simple_term_infallible("]")(i)?;
+        let (rest, (delim_term, mut err)) = simple_term_infallible("]")(inp)?;
         errs.append(&mut err);
         if let Some((_, term)) = delim_term {
             elements.push(term);
         }
-        i = rest;
+        inp = rest;
     }
 }
 
@@ -626,16 +663,16 @@ fn negate(expr: UserInputAst) -> UserInputAst {
     expr.unary(Occur::MustNot)
 }
 
-fn leaf(i: &str) -> IResult<&str, UserInputAst> {
+fn leaf(inp: &str) -> IResult<&str, UserInputAst> {
     alt((
         delimited(char('('), ast, char(')')),
         map(char('*'), |_| UserInputAst::from(UserInputLeaf::All)),
         map(preceded(tuple((tag("NOT"), space1)), leaf), negate),
         literal,
-    ))(i)
+    ))(inp)
 }
 
-fn leaf_infallible(i: &str) -> JResult<&str, Option<UserInputAst>> {
+fn leaf_infallible(inp: &str) -> JResult<&str, Option<UserInputAst>> {
     alt_infallible(
         (
             (
@@ -665,23 +702,23 @@ fn leaf_infallible(i: &str) -> JResult<&str, Option<UserInputAst>> {
             ),
         ),
         literal_infallible,
-    )(i)
+    )(inp)
 }
 
-fn positive_float_number(i: &str) -> IResult<&str, f64> {
+fn positive_float_number(inp: &str) -> IResult<&str, f64> {
     map(
         recognize(tuple((digit1, opt(tuple((char('.'), digit1)))))),
         // TODO this is actually dangerous if the number is actually not representable as a f64
         // (too big for instance)
         |float_str: &str| float_str.parse::<f64>().unwrap(),
-    )(i)
+    )(inp)
 }
 
-fn boost(i: &str) -> JResult<&str, Option<f64>> {
-    opt_i(preceded(char('^'), positive_float_number))(i)
+fn boost(inp: &str) -> JResult<&str, Option<f64>> {
+    opt_i(preceded(char('^'), positive_float_number))(inp)
 }
 
-fn boosted_leaf(i: &str) -> IResult<&str, UserInputAst> {
+fn boosted_leaf(inp: &str) -> IResult<&str, UserInputAst> {
     map(
         tuple((leaf, fallible(boost))),
         |(leaf, boost_opt)| match boost_opt {
@@ -690,10 +727,10 @@ fn boosted_leaf(i: &str) -> IResult<&str, UserInputAst> {
             }
             _ => leaf,
         },
-    )(i)
+    )(inp)
 }
 
-fn boosted_leaf_infallible(i: &str) -> JResult<&str, Option<UserInputAst>> {
+fn boosted_leaf_infallible(inp: &str) -> JResult<&str, Option<UserInputAst>> {
     map(
         tuple_infallible((leaf_infallible, boost)),
         |((leaf, boost_opt), error)| match boost_opt {
@@ -703,30 +740,30 @@ fn boosted_leaf_infallible(i: &str) -> JResult<&str, Option<UserInputAst>> {
             ),
             _ => (leaf, error),
         },
-    )(i)
+    )(inp)
 }
 
-fn occur_symbol(i: &str) -> JResult<&str, Option<Occur>> {
+fn occur_symbol(inp: &str) -> JResult<&str, Option<Occur>> {
     opt_i(alt((
         value(Occur::MustNot, char('-')),
         value(Occur::Must, char('+')),
-    )))(i)
+    )))(inp)
 }
 
-fn occur_leaf(i: &str) -> IResult<&str, (Option<Occur>, UserInputAst)> {
-    tuple((fallible(occur_symbol), boosted_leaf))(i)
+fn occur_leaf(inp: &str) -> IResult<&str, (Option<Occur>, UserInputAst)> {
+    tuple((fallible(occur_symbol), boosted_leaf))(inp)
 }
 
 #[allow(clippy::type_complexity)]
 fn operand_occur_leaf_infallible(
-    i: &str,
+    inp: &str,
 ) -> JResult<&str, (Option<BinaryOperand>, Option<Occur>, Option<UserInputAst>)> {
     // TODO maybe this should support multiple chained AND/OR, and "fuse" them?
     tuple_infallible((
         delimited_infallible(nothing, opt_i(binary_operand), space0_infallible),
         occur_symbol,
         boosted_leaf_infallible,
-    ))(i)
+    ))(inp)
 }
 
 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
@@ -735,11 +772,11 @@ enum BinaryOperand {
     And,
 }
 
-fn binary_operand(i: &str) -> IResult<&str, BinaryOperand> {
+fn binary_operand(inp: &str) -> IResult<&str, BinaryOperand> {
     alt((
         value(BinaryOperand::And, tag("AND ")),
         value(BinaryOperand::Or, tag("OR ")),
-    ))(i)
+    ))(inp)
 }
 
 fn aggregate_binary_expressions(
@@ -880,14 +917,14 @@ fn aggregate_infallible_expressions(
     }
 }
 
-fn operand_leaf(i: &str) -> IResult<&str, (BinaryOperand, UserInputAst)> {
+fn operand_leaf(inp: &str) -> IResult<&str, (BinaryOperand, UserInputAst)> {
     tuple((
         terminated(binary_operand, space0),
         terminated(boosted_leaf, space0),
-    ))(i)
+    ))(inp)
 }
 
-fn ast(i: &str) -> IResult<&str, UserInputAst> {
+fn ast(inp: &str) -> IResult<&str, UserInputAst> {
     let boolean_expr = map(
         separated_pair(boosted_leaf, space1, many1(operand_leaf)),
         |(left, right)| aggregate_binary_expressions(left, right),
@@ -908,10 +945,10 @@ fn ast(i: &str) -> IResult<&str, UserInputAst> {
         space0,
         alt((boolean_expr, whitespace_separated_leaves)),
         space0,
-    )(i)
+    )(inp)
 }
 
-fn ast_infallible(i: &str) -> JResult<&str, UserInputAst> {
+fn ast_infallible(inp: &str) -> JResult<&str, UserInputAst> {
     // ast() parse either `term AND term OR term` or `+term term -term`
     // both are locally ambiguous, and as we allow error, it's hard to permit backtracking.
     // Instead, we allow a mix of both syntaxes, trying to make sense of what a user meant.
@@ -928,13 +965,13 @@ fn ast_infallible(i: &str) -> JResult<&str, UserInputAst> {
         },
     );
 
-    delimited_infallible(space0_infallible, expression, space0_infallible)(i)
+    delimited_infallible(space0_infallible, expression, space0_infallible)(inp)
 }
 
-pub fn parse_to_ast(i: &str) -> IResult<&str, UserInputAst> {
+pub fn parse_to_ast(inp: &str) -> IResult<&str, UserInputAst> {
     map(delimited(space0, opt(ast), eof), |opt_ast| {
         rewrite_ast(opt_ast.unwrap_or_else(UserInputAst::empty_query))
-    })(i)
+    })(inp)
 }
 
 pub fn parse_to_ast_lenient(query_str: &str) -> (UserInputAst, Vec<LenientError>) {
@@ -1538,6 +1575,17 @@ mod test {
         test_parse_query_to_ast_helper("foo:\"\"*", "\"foo\":\"\"*");
     }
 
+    #[test]
+    fn test_exist_query() {
+        test_parse_query_to_ast_helper("a:*", "\"a\":*");
+        test_parse_query_to_ast_helper("a: *", "\"a\":*");
+        // an exist followed by default term being b
+        test_is_parse_err("a:*b", "(*\"a\":* *b)");
+
+        // this is a term query (not a phrase prefix)
+        test_parse_query_to_ast_helper("a:b*", "\"a\":b*");
+    }
+
     #[test]
     fn test_not_queries_are_consistent() {
         test_parse_query_to_ast_helper("tata -toto", "(*tata -toto)");

query-grammar/src/user_input_ast.rs

@@ -16,6 +16,9 @@ pub enum UserInputLeaf {
         field: Option<String>,
         elements: Vec<String>,
     },
+    Exists {
+        field: String,
+    },
 }
 
 impl UserInputLeaf {
@@ -36,6 +39,9 @@ impl UserInputLeaf {
                 upper,
             },
             UserInputLeaf::Set { field: _, elements } => UserInputLeaf::Set { field, elements },
+            UserInputLeaf::Exists { field: _ } => UserInputLeaf::Exists {
+                field: field.expect("Exist query without a field isn't allowed"),
+            },
         }
     }
 }
@@ -74,6 +80,9 @@ impl Debug for UserInputLeaf {
                 write!(formatter, "]")
             }
             UserInputLeaf::All => write!(formatter, "*"),
+            UserInputLeaf::Exists { field } => {
+                write!(formatter, "\"{field}\":*")
+            }
         }
     }
 }

src/aggregation/agg_limits.rs

@@ -134,3 +134,142 @@ impl Drop for ResourceLimitGuard {
             .fetch_sub(self.allocated_with_the_guard, Ordering::Relaxed);
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use crate::aggregation::tests::exec_request_with_query;
+
+    // https://github.com/quickwit-oss/quickwit/issues/3837
+    #[test]
+    fn test_agg_limits_with_empty_merge() {
+        use crate::aggregation::agg_req::Aggregations;
+        use crate::aggregation::bucket::tests::get_test_index_from_docs;
+
+        let docs = vec![
+            vec![r#"{ "date": "2015-01-02T00:00:00Z", "text": "bbb", "text2": "bbb" }"#],
+            vec![r#"{ "text": "aaa", "text2": "bbb" }"#],
+        ];
+        let index = get_test_index_from_docs(false, &docs).unwrap();
+
+        {
+            let elasticsearch_compatible_json = json!(
+                {
+                    "1": {
+                        "terms": {"field": "text2", "min_doc_count": 0},
+                        "aggs": {
+                            "2":{
+                                "date_histogram": {
+                                    "field": "date",
+                                    "fixed_interval": "1d",
+                                    "extended_bounds": {
+                                        "min": "2015-01-01T00:00:00Z",
+                                        "max": "2015-01-10T00:00:00Z"
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            );
+
+            let agg_req: Aggregations = serde_json::from_str(
+                &serde_json::to_string(&elasticsearch_compatible_json).unwrap(),
+            )
+            .unwrap();
+            let res = exec_request_with_query(agg_req, &index, Some(("text", "bbb"))).unwrap();
+            let expected_res = json!({
+             "1": {
+                "buckets": [
+                  {
+                    "2": {
+                      "buckets": [
+                        { "doc_count": 0, "key": 1420070400000.0, "key_as_string": "2015-01-01T00:00:00Z" },
+                        { "doc_count": 1, "key": 1420156800000.0, "key_as_string": "2015-01-02T00:00:00Z" },
+                        { "doc_count": 0, "key": 1420243200000.0, "key_as_string": "2015-01-03T00:00:00Z" },
+                        { "doc_count": 0, "key": 1420329600000.0, "key_as_string": "2015-01-04T00:00:00Z" },
+                        { "doc_count": 0, "key": 1420416000000.0, "key_as_string": "2015-01-05T00:00:00Z" },
+                        { "doc_count": 0, "key": 1420502400000.0, "key_as_string": "2015-01-06T00:00:00Z" },
+                        { "doc_count": 0, "key": 1420588800000.0, "key_as_string": "2015-01-07T00:00:00Z" },
+                        { "doc_count": 0, "key": 1420675200000.0, "key_as_string": "2015-01-08T00:00:00Z" },
+                        { "doc_count": 0, "key": 1420761600000.0, "key_as_string": "2015-01-09T00:00:00Z" },
+                        { "doc_count": 0, "key": 1420848000000.0, "key_as_string": "2015-01-10T00:00:00Z" }
+                      ]
+                    },
+                    "doc_count": 1,
+                    "key": "bbb"
+                  }
+                ],
+                "doc_count_error_upper_bound": 0,
+                "sum_other_doc_count": 0
+              }
+            });
+            assert_eq!(res, expected_res);
+        }
+    }
+
+    // https://github.com/quickwit-oss/quickwit/issues/3837
+    #[test]
+    fn test_agg_limits_with_empty_data() {
+        use crate::aggregation::agg_req::Aggregations;
+        use crate::aggregation::bucket::tests::get_test_index_from_docs;
+
+        let docs = vec![vec![r#"{ "text": "aaa", "text2": "bbb" }"#]];
+        let index = get_test_index_from_docs(false, &docs).unwrap();
+
+        {
+            // Empty result since there is no doc with dates
+            let elasticsearch_compatible_json = json!(
+                {
+                    "1": {
+                        "terms": {"field": "text2", "min_doc_count": 0},
+                        "aggs": {
+                            "2":{
+                                "date_histogram": {
+                                    "field": "date",
+                                    "fixed_interval": "1d",
+                                    "extended_bounds": {
+                                        "min": "2015-01-01T00:00:00Z",
+                                        "max": "2015-01-10T00:00:00Z"
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            );
+
+            let agg_req: Aggregations = serde_json::from_str(
+                &serde_json::to_string(&elasticsearch_compatible_json).unwrap(),
+            )
+            .unwrap();
+            let res = exec_request_with_query(agg_req, &index, Some(("text", "bbb"))).unwrap();
+            let expected_res = json!({
+             "1": {
+                "buckets": [
+                  {
+                    "2": {
+                      "buckets": [
+                        { "doc_count": 0, "key": 1420070400000.0, "key_as_string": "2015-01-01T00:00:00Z" },
+                        { "doc_count": 0, "key": 1420156800000.0, "key_as_string": "2015-01-02T00:00:00Z" },
+                        { "doc_count": 0, "key": 1420243200000.0, "key_as_string": "2015-01-03T00:00:00Z" },
+                        { "doc_count": 0, "key": 1420329600000.0, "key_as_string": "2015-01-04T00:00:00Z" },
+                        { "doc_count": 0, "key": 1420416000000.0, "key_as_string": "2015-01-05T00:00:00Z" },
+                        { "doc_count": 0, "key": 1420502400000.0, "key_as_string": "2015-01-06T00:00:00Z" },
+                        { "doc_count": 0, "key": 1420588800000.0, "key_as_string": "2015-01-07T00:00:00Z" },
+                        { "doc_count": 0, "key": 1420675200000.0, "key_as_string": "2015-01-08T00:00:00Z" },
+                        { "doc_count": 0, "key": 1420761600000.0, "key_as_string": "2015-01-09T00:00:00Z" },
+                        { "doc_count": 0, "key": 1420848000000.0, "key_as_string": "2015-01-10T00:00:00Z" }
+                      ]
+                    },
+                    "doc_count": 0,
+                    "key": "bbb"
+                  }
+                ],
+                "doc_count_error_upper_bound": 0,
+                "sum_other_doc_count": 0
+              }
+            });
+            assert_eq!(res, expected_res);
+        }
+    }
+}

src/aggregation/agg_req_with_accessor.rs

@@ -103,7 +103,8 @@ impl AggregationWithAccessor {
                 field: field_name, ..
             }) => {
                 let (accessor, column_type) =
-                    get_ff_reader(reader, field_name, Some(get_numeric_or_date_column_types()))?;
+                    // Only DateTime is supported for DateHistogram
+                    get_ff_reader(reader, field_name, Some(&[ColumnType::DateTime]))?;
                 add_agg_with_accessor(accessor, column_type, &mut res)?;
             }
             Terms(TermsAggregation {
@@ -117,10 +118,10 @@ impl AggregationWithAccessor {
                     ColumnType::U64,
                     ColumnType::F64,
                     ColumnType::Str,
+                    ColumnType::DateTime,
                     // ColumnType::Bytes Unsupported
                     // ColumnType::Bool Unsupported
                     // ColumnType::IpAddr Unsupported
-                    // ColumnType::DateTime Unsupported
                 ];
 
                 // In case the column is empty we want the shim column to match the missing type
@@ -145,7 +146,18 @@ impl AggregationWithAccessor {
                         .map(|m| matches!(m, Key::Str(_)))
                         .unwrap_or(false);
 
-                let use_special_missing_agg = missing_and_more_than_one_col || text_on_non_text_col;
+                // Actually we could convert the text to a number and have the fast path, if it is
+                // provided in Rfc3339 format. But this use case is probably common
+                // enough to justify the effort.
+                let text_on_date_col = column_and_types.len() == 1
+                    && column_and_types[0].1 == ColumnType::DateTime
+                    && missing
+                        .as_ref()
+                        .map(|m| matches!(m, Key::Str(_)))
+                        .unwrap_or(false);
+
+                let use_special_missing_agg =
+                    missing_and_more_than_one_col || text_on_non_text_col || text_on_date_col;
                 if use_special_missing_agg {
                     let column_and_types =
                         get_all_ff_reader_or_empty(reader, field_name, None, fallback_type)?;

src/aggregation/agg_tests.rs

@@ -9,7 +9,7 @@ use crate::aggregation::tests::{get_test_index_2_segments, get_test_index_from_v
 use crate::aggregation::DistributedAggregationCollector;
 use crate::query::{AllQuery, TermQuery};
 use crate::schema::{IndexRecordOption, Schema, FAST};
-use crate::{Index, Term};
+use crate::{Index, IndexWriter, Term};
 
 fn get_avg_req(field_name: &str) -> Aggregation {
     serde_json::from_value(json!({
@@ -586,7 +586,7 @@ fn test_aggregation_on_json_object() {
     let json = schema_builder.add_json_field("json", FAST);
     let schema = schema_builder.build();
     let index = Index::create_in_ram(schema);
-    let mut index_writer = index.writer_for_tests().unwrap();
+    let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
     index_writer
         .add_document(doc!(json => json!({"color": "red"})))
         .unwrap();
@@ -630,7 +630,7 @@ fn test_aggregation_on_json_object_empty_columns() {
     let json = schema_builder.add_json_field("json", FAST);
     let schema = schema_builder.build();
     let index = Index::create_in_ram(schema);
-    let mut index_writer = index.writer_for_tests().unwrap();
+    let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
     // => Empty column when accessing color
     index_writer
         .add_document(doc!(json => json!({"price": 10.0})))
@@ -748,7 +748,7 @@ fn test_aggregation_on_json_object_mixed_types() {
     let json = schema_builder.add_json_field("json", FAST);
     let schema = schema_builder.build();
     let index = Index::create_in_ram(schema);
-    let mut index_writer = index.writer_for_tests().unwrap();
+    let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
     // => Segment with all values numeric
     index_writer
         .add_document(doc!(json => json!({"mixed_type": 10.0})))

src/aggregation/bucket/histogram/date_histogram.rs

@@ -132,6 +132,7 @@ impl DateHistogramAggregationReq {
             hard_bounds: self.hard_bounds,
             extended_bounds: self.extended_bounds,
             keyed: self.keyed,
+            is_normalized_to_ns: false,
         })
     }
 
@@ -243,15 +244,15 @@ fn parse_into_milliseconds(input: &str) -> Result<i64, AggregationError> {
 }
 
 #[cfg(test)]
-mod tests {
+pub mod tests {
     use pretty_assertions::assert_eq;
 
     use super::*;
     use crate::aggregation::agg_req::Aggregations;
     use crate::aggregation::tests::exec_request;
     use crate::indexer::NoMergePolicy;
-    use crate::schema::{Schema, FAST};
-    use crate::Index;
+    use crate::schema::{Schema, FAST, STRING};
+    use crate::{Index, IndexWriter, TantivyDocument};
 
     #[test]
     fn test_parse_into_millisecs() {
@@ -306,7 +307,8 @@ mod tests {
     ) -> crate::Result<Index> {
         let mut schema_builder = Schema::builder();
         schema_builder.add_date_field("date", FAST);
-        schema_builder.add_text_field("text", FAST);
+        schema_builder.add_text_field("text", FAST | STRING);
+        schema_builder.add_text_field("text2", FAST | STRING);
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema.clone());
         {
@@ -314,7 +316,7 @@ mod tests {
             index_writer.set_merge_policy(Box::new(NoMergePolicy));
             for values in segment_and_docs {
                 for doc_str in values {
-                    let doc = schema.parse_document(doc_str)?;
+                    let doc = TantivyDocument::parse_json(&schema, doc_str)?;
                     index_writer.add_document(doc)?;
                 }
                 // writing the segment
@@ -326,7 +328,7 @@ mod tests {
                 .searchable_segment_ids()
                 .expect("Searchable segments failed.");
             if segment_ids.len() > 1 {
-                let mut index_writer = index.writer_for_tests()?;
+                let mut index_writer: IndexWriter = index.writer_for_tests()?;
                 index_writer.merge(&segment_ids).wait()?;
                 index_writer.wait_merging_threads()?;
             }

src/aggregation/bucket/histogram/histogram.rs

@@ -122,11 +122,14 @@ pub struct HistogramAggregation {
     /// Whether to return the buckets as a hash map
     #[serde(default)]
     pub keyed: bool,
+    /// Whether the values are normalized to ns for date time values. Defaults to false.
+    #[serde(default)]
+    pub is_normalized_to_ns: bool,
 }
 
 impl HistogramAggregation {
-    pub(crate) fn normalize(&mut self, column_type: ColumnType) {
-        if column_type.is_date_time() {
+    pub(crate) fn normalize_date_time(&mut self) {
+        if !self.is_normalized_to_ns {
             // values are provided in ms, but the fastfield is in nano seconds
             self.interval *= 1_000_000.0;
             self.offset = self.offset.map(|off| off * 1_000_000.0);
@@ -138,6 +141,7 @@ impl HistogramAggregation {
                 min: bounds.min * 1_000_000.0,
                 max: bounds.max * 1_000_000.0,
             });
+            self.is_normalized_to_ns = true;
         }
     }
 
@@ -370,7 +374,7 @@ impl SegmentHistogramCollector {
 
         Ok(IntermediateBucketResult::Histogram {
             buckets,
-            column_type: Some(self.column_type),
+            is_date_agg: self.column_type == ColumnType::DateTime,
         })
     }
 
@@ -381,7 +385,9 @@ impl SegmentHistogramCollector {
         accessor_idx: usize,
     ) -> crate::Result<Self> {
         req.validate()?;
-        req.normalize(field_type);
+        if field_type == ColumnType::DateTime {
+            req.normalize_date_time();
+        }
 
         let sub_aggregation_blueprint = if sub_aggregation.is_empty() {
             None
@@ -439,6 +445,7 @@ fn intermediate_buckets_to_final_buckets_fill_gaps(
     // memory check upfront
     let (_, first_bucket_num, last_bucket_num) =
         generate_bucket_pos_with_opt_minmax(histogram_req, min_max);
+
     // It's based on user input, so we need to account for overflows
     let added_buckets = ((last_bucket_num.saturating_sub(first_bucket_num)).max(0) as u64)
         .saturating_sub(buckets.len() as u64);
@@ -482,7 +489,7 @@ fn intermediate_buckets_to_final_buckets_fill_gaps(
 // Convert to BucketEntry
 pub(crate) fn intermediate_histogram_buckets_to_final_buckets(
     buckets: Vec<IntermediateHistogramBucketEntry>,
-    column_type: Option<ColumnType>,
+    is_date_agg: bool,
     histogram_req: &HistogramAggregation,
     sub_aggregation: &Aggregations,
     limits: &AggregationLimits,
@@ -491,8 +498,8 @@ pub(crate) fn intermediate_histogram_buckets_to_final_buckets(
     // The request used in the the call to final is not yet be normalized.
     // Normalization is changing the precision from milliseconds to nanoseconds.
     let mut histogram_req = histogram_req.clone();
-    if let Some(column_type) = column_type {
-        histogram_req.normalize(column_type);
+    if is_date_agg {
+        histogram_req.normalize_date_time();
     }
     let mut buckets = if histogram_req.min_doc_count() == 0 {
         // With min_doc_count != 0, we may need to add buckets, so that there are no
@@ -516,7 +523,7 @@ pub(crate) fn intermediate_histogram_buckets_to_final_buckets(
 
     // If we have a date type on the histogram buckets, we add the `key_as_string` field as rfc339
     // and normalize from nanoseconds to milliseconds
-    if column_type == Some(ColumnType::DateTime) {
+    if is_date_agg {
         for bucket in buckets.iter_mut() {
             if let crate::aggregation::Key::F64(ref mut val) = bucket.key {
                 let key_as_string = format_date(*val as i64)?;

src/aggregation/bucket/term_agg.rs

@@ -1,6 +1,6 @@
 use std::fmt::Debug;
 
-use columnar::{BytesColumn, ColumnType, StrColumn};
+use columnar::{BytesColumn, ColumnType, MonotonicallyMappableToU64, StrColumn};
 use rustc_hash::FxHashMap;
 use serde::{Deserialize, Serialize};
 
@@ -16,7 +16,7 @@ use crate::aggregation::intermediate_agg_result::{
 use crate::aggregation::segment_agg_result::{
     build_segment_agg_collector, SegmentAggregationCollector,
 };
-use crate::aggregation::{f64_from_fastfield_u64, Key};
+use crate::aggregation::{f64_from_fastfield_u64, format_date, Key};
 use crate::error::DataCorruption;
 use crate::TantivyError;
 
@@ -531,6 +531,13 @@ impl SegmentTermCollector {
                         });
                 }
             }
+        } else if self.field_type == ColumnType::DateTime {
+            for (val, doc_count) in entries {
+                let intermediate_entry = into_intermediate_bucket_entry(val, doc_count)?;
+                let val = i64::from_u64(val);
+                let date = format_date(val)?;
+                dict.insert(IntermediateKey::Str(date), intermediate_entry);
+            }
         } else {
             for (val, doc_count) in entries {
                 let intermediate_entry = into_intermediate_bucket_entry(val, doc_count)?;
@@ -583,6 +590,9 @@ pub(crate) fn cut_off_buckets<T: GetDocCount + Debug>(
 
 #[cfg(test)]
 mod tests {
+    use common::DateTime;
+    use time::{Date, Month};
+
     use crate::aggregation::agg_req::Aggregations;
     use crate::aggregation::tests::{
         exec_request, exec_request_with_query, exec_request_with_query_and_memory_limit,
@@ -591,7 +601,7 @@ mod tests {
     use crate::aggregation::AggregationLimits;
     use crate::indexer::NoMergePolicy;
     use crate::schema::{Schema, FAST, STRING};
-    use crate::Index;
+    use crate::{Index, IndexWriter};
 
     #[test]
     fn terms_aggregation_test_single_segment() -> crate::Result<()> {
@@ -1463,7 +1473,7 @@ mod tests {
         let json = schema_builder.add_json_field("json", FAST);
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_for_tests().unwrap();
+        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
         // => Segment with empty json
         index_writer.add_document(doc!()).unwrap();
         index_writer.commit().unwrap();
@@ -1813,4 +1823,75 @@ mod tests {
 
         Ok(())
     }
+
+    #[test]
+    fn terms_aggregation_date() -> crate::Result<()> {
+        let mut schema_builder = Schema::builder();
+        let date_field = schema_builder.add_date_field("date_field", FAST);
+        let schema = schema_builder.build();
+        let index = Index::create_in_ram(schema);
+        {
+            let mut writer = index.writer_with_num_threads(1, 15_000_000)?;
+            writer.add_document(doc!(date_field=>DateTime::from_primitive(Date::from_calendar_date(1982, Month::September, 17)?.with_hms(0, 0, 0)?)))?;
+            writer.add_document(doc!(date_field=>DateTime::from_primitive(Date::from_calendar_date(1982, Month::September, 17)?.with_hms(0, 0, 0)?)))?;
+            writer.add_document(doc!(date_field=>DateTime::from_primitive(Date::from_calendar_date(1983, Month::September, 27)?.with_hms(0, 0, 0)?)))?;
+            writer.commit()?;
+        }
+
+        let agg_req: Aggregations = serde_json::from_value(json!({
+            "my_date": {
+                "terms": {
+                    "field": "date_field"
+                },
+            }
+        }))
+        .unwrap();
+
+        let res = exec_request_with_query(agg_req, &index, None)?;
+
+        // date_field field
+        assert_eq!(res["my_date"]["buckets"][0]["key"], "1982-09-17T00:00:00Z");
+        assert_eq!(res["my_date"]["buckets"][0]["doc_count"], 2);
+        assert_eq!(res["my_date"]["buckets"][1]["key"], "1983-09-27T00:00:00Z");
+        assert_eq!(res["my_date"]["buckets"][1]["doc_count"], 1);
+        assert_eq!(res["my_date"]["buckets"][2]["key"], serde_json::Value::Null);
+
+        Ok(())
+    }
+    #[test]
+    fn terms_aggregation_date_missing() -> crate::Result<()> {
+        let mut schema_builder = Schema::builder();
+        let date_field = schema_builder.add_date_field("date_field", FAST);
+        let schema = schema_builder.build();
+        let index = Index::create_in_ram(schema);
+        {
+            let mut writer = index.writer_with_num_threads(1, 15_000_000)?;
+            writer.add_document(doc!(date_field=>DateTime::from_primitive(Date::from_calendar_date(1982, Month::September, 17)?.with_hms(0, 0, 0)?)))?;
+            writer.add_document(doc!(date_field=>DateTime::from_primitive(Date::from_calendar_date(1982, Month::September, 17)?.with_hms(0, 0, 0)?)))?;
+            writer.add_document(doc!(date_field=>DateTime::from_primitive(Date::from_calendar_date(1983, Month::September, 27)?.with_hms(0, 0, 0)?)))?;
+            writer.add_document(doc!())?;
+            writer.commit()?;
+        }
+
+        let agg_req: Aggregations = serde_json::from_value(json!({
+            "my_date": {
+                "terms": {
+                    "field": "date_field",
+                    "missing": "1982-09-17T00:00:00Z"
+                },
+            }
+        }))
+        .unwrap();
+
+        let res = exec_request_with_query(agg_req, &index, None)?;
+
+        // date_field field
+        assert_eq!(res["my_date"]["buckets"][0]["key"], "1982-09-17T00:00:00Z");
+        assert_eq!(res["my_date"]["buckets"][0]["doc_count"], 3);
+        assert_eq!(res["my_date"]["buckets"][1]["key"], "1983-09-27T00:00:00Z");
+        assert_eq!(res["my_date"]["buckets"][1]["doc_count"], 1);
+        assert_eq!(res["my_date"]["buckets"][2]["key"], serde_json::Value::Null);
+
+        Ok(())
+    }
 }

src/aggregation/bucket/term_missing_agg.rs

@@ -117,7 +117,7 @@ mod tests {
     use crate::aggregation::agg_req::Aggregations;
     use crate::aggregation::tests::exec_request_with_query;
     use crate::schema::{Schema, FAST};
-    use crate::Index;
+    use crate::{Index, IndexWriter};
 
     #[test]
     fn terms_aggregation_missing_mixed_type_mult_seg_sub_agg() -> crate::Result<()> {
@@ -126,7 +126,7 @@ mod tests {
         let score = schema_builder.add_f64_field("score", FAST);
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_for_tests().unwrap();
+        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
         // => Segment with all values numeric
         index_writer
             .add_document(doc!(score => 1.0, json => json!({"mixed_type": 10.0})))
@@ -186,7 +186,7 @@ mod tests {
         let score = schema_builder.add_f64_field("score", FAST);
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_for_tests().unwrap();
+        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
         // => Segment with all values numeric
         index_writer.add_document(doc!(score => 1.0, json => json!({"mixed_type": 10.0})))?;
         index_writer.add_document(doc!(score => 5.0))?;
@@ -231,7 +231,7 @@ mod tests {
         let score = schema_builder.add_f64_field("score", FAST);
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_for_tests().unwrap();
+        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
 
         index_writer.add_document(doc!(score => 5.0))?;
         index_writer.commit().unwrap();
@@ -278,7 +278,7 @@ mod tests {
         let score = schema_builder.add_f64_field("score", FAST);
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_for_tests().unwrap();
+        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
 
         index_writer.add_document(doc!(score => 5.0))?;
         index_writer.add_document(doc!(score => 5.0))?;
@@ -323,7 +323,7 @@ mod tests {
         let json = schema_builder.add_json_field("json", FAST);
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_for_tests().unwrap();
+        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
         // => Segment with all values numeric
         index_writer
             .add_document(doc!(json => json!({"mixed_type": 10.0})))
@@ -385,7 +385,7 @@ mod tests {
         let json = schema_builder.add_json_field("json", FAST);
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_for_tests().unwrap();
+        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
         // => Segment with all values numeric
         index_writer
             .add_document(doc!(json => json!({"mixed_type": 10.0})))
@@ -427,7 +427,7 @@ mod tests {
         let json = schema_builder.add_json_field("json", FAST);
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_for_tests().unwrap();
+        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
         // => Segment with all values numeric
         index_writer
             .add_document(doc!(json => json!({"mixed_type": 10.0})))

src/aggregation/intermediate_agg_result.rs

@@ -172,10 +172,16 @@ pub(crate) fn empty_from_req(req: &Aggregation) -> IntermediateAggregationResult
         Range(_) => IntermediateAggregationResult::Bucket(IntermediateBucketResult::Range(
             Default::default(),
         )),
-        Histogram(_) | DateHistogram(_) => {
+        Histogram(_) => {
             IntermediateAggregationResult::Bucket(IntermediateBucketResult::Histogram {
                 buckets: Vec::new(),
-                column_type: None,
+                is_date_agg: false,
+            })
+        }
+        DateHistogram(_) => {
+            IntermediateAggregationResult::Bucket(IntermediateBucketResult::Histogram {
+                buckets: Vec::new(),
+                is_date_agg: true,
             })
         }
         Average(_) => IntermediateAggregationResult::Metric(IntermediateMetricResult::Average(
@@ -343,8 +349,8 @@ pub enum IntermediateBucketResult {
     /// This is the histogram entry for a bucket, which contains a key, count, and optionally
     /// sub_aggregations.
     Histogram {
-        /// The column_type of the underlying `Column`
-        column_type: Option<ColumnType>,
+        /// The column_type of the underlying `Column` is DateTime
+        is_date_agg: bool,
         /// The buckets
         buckets: Vec<IntermediateHistogramBucketEntry>,
     },
@@ -399,7 +405,7 @@ impl IntermediateBucketResult {
                 Ok(BucketResult::Range { buckets })
             }
             IntermediateBucketResult::Histogram {
-                column_type,
+                is_date_agg,
                 buckets,
             } => {
                 let histogram_req = &req
@@ -408,7 +414,7 @@ impl IntermediateBucketResult {
                     .expect("unexpected aggregation, expected histogram aggregation");
                 let buckets = intermediate_histogram_buckets_to_final_buckets(
                     buckets,
-                    column_type,
+                    is_date_agg,
                     histogram_req,
                     req.sub_aggregation(),
                     limits,
@@ -457,11 +463,11 @@ impl IntermediateBucketResult {
             (
                 IntermediateBucketResult::Histogram {
                     buckets: buckets_left,
-                    ..
+                    is_date_agg: _,
                 },
                 IntermediateBucketResult::Histogram {
                     buckets: buckets_right,
-                    ..
+                    is_date_agg: _,
                 },
             ) => {
                 let buckets: Result<Vec<IntermediateHistogramBucketEntry>, TantivyError> =

src/aggregation/metric/max.rs

@@ -71,7 +71,7 @@ mod tests {
     use crate::aggregation::agg_req::Aggregations;
     use crate::aggregation::tests::exec_request_with_query;
     use crate::schema::{Schema, FAST};
-    use crate::Index;
+    use crate::{Index, IndexWriter};
 
     #[test]
     fn test_max_agg_with_missing() -> crate::Result<()> {
@@ -79,7 +79,7 @@ mod tests {
         let json = schema_builder.add_json_field("json", FAST);
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_for_tests().unwrap();
+        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
         // => Segment with empty json
         index_writer.add_document(doc!()).unwrap();
         index_writer.commit().unwrap();

src/aggregation/metric/mod.rs

@@ -88,7 +88,7 @@ mod tests {
     use crate::aggregation::AggregationCollector;
     use crate::query::AllQuery;
     use crate::schema::{NumericOptions, Schema};
-    use crate::Index;
+    use crate::{Index, IndexWriter};
 
     #[test]
     fn test_metric_aggregations() {
@@ -96,7 +96,7 @@ mod tests {
         let field_options = NumericOptions::default().set_fast();
         let field = schema_builder.add_f64_field("price", field_options);
         let index = Index::create_in_ram(schema_builder.build());
-        let mut index_writer = index.writer_for_tests().unwrap();
+        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
 
         for i in 0..3 {
             index_writer

src/aggregation/metric/stats.rs

@@ -300,7 +300,7 @@ mod tests {
     use crate::aggregation::AggregationCollector;
     use crate::query::{AllQuery, TermQuery};
     use crate::schema::{IndexRecordOption, Schema, FAST};
-    use crate::{Index, Term};
+    use crate::{Index, IndexWriter, Term};
 
     #[test]
     fn test_aggregation_stats_empty_index() -> crate::Result<()> {
@@ -494,7 +494,7 @@ mod tests {
         let json = schema_builder.add_json_field("json", FAST);
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_for_tests().unwrap();
+        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
         // => Segment with empty json
         index_writer.add_document(doc!()).unwrap();
         index_writer.commit().unwrap();
@@ -541,7 +541,7 @@ mod tests {
         let json = schema_builder.add_json_field("json", FAST);
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_for_tests().unwrap();
+        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
         // => Segment with empty json
         index_writer.add_document(doc!()).unwrap();
         index_writer.commit().unwrap();

src/aggregation/mod.rs

@@ -319,7 +319,7 @@ mod tests {
     use crate::indexer::NoMergePolicy;
     use crate::query::{AllQuery, TermQuery};
     use crate::schema::{IndexRecordOption, Schema, TextFieldIndexing, FAST, STRING};
-    use crate::{Index, Term};
+    use crate::{Index, IndexWriter, Term};
 
     pub fn get_test_index_with_num_docs(
         merge_segments: bool,
@@ -451,7 +451,7 @@ mod tests {
                 .searchable_segment_ids()
                 .expect("Searchable segments failed.");
             if segment_ids.len() > 1 {
-                let mut index_writer = index.writer_for_tests()?;
+                let mut index_writer: IndexWriter = index.writer_for_tests()?;
                 index_writer.merge(&segment_ids).wait()?;
                 index_writer.wait_merging_threads()?;
             }
@@ -565,7 +565,7 @@ mod tests {
             let segment_ids = index
                 .searchable_segment_ids()
                 .expect("Searchable segments failed.");
-            let mut index_writer = index.writer_for_tests()?;
+            let mut index_writer: IndexWriter = index.writer_for_tests()?;
             index_writer.merge(&segment_ids).wait()?;
             index_writer.wait_merging_threads()?;
         }

src/collector/count_collector.rs

@@ -16,7 +16,7 @@ use crate::{DocId, Score, SegmentOrdinal, SegmentReader};
 /// let schema = schema_builder.build();
 /// let index = Index::create_in_ram(schema);
 ///
-/// let mut index_writer = index.writer(3_000_000).unwrap();
+/// let mut index_writer = index.writer(15_000_000).unwrap();
 /// index_writer.add_document(doc!(title => "The Name of the Wind")).unwrap();
 /// index_writer.add_document(doc!(title => "The Diary of Muadib")).unwrap();
 /// index_writer.add_document(doc!(title => "A Dairy Cow")).unwrap();

src/collector/facet_collector.rs

@@ -89,7 +89,7 @@ fn facet_depth(facet_bytes: &[u8]) -> usize {
 ///     let schema = schema_builder.build();
 ///     let index = Index::create_in_ram(schema);
 ///     {
-///         let mut index_writer = index.writer(3_000_000)?;
+///         let mut index_writer = index.writer(15_000_000)?;
 ///         // a document can be associated with any number of facets
 ///         index_writer.add_document(doc!(
 ///             title => "The Name of the Wind",
@@ -495,8 +495,8 @@ mod tests {
     use crate::collector::Count;
     use crate::core::Index;
     use crate::query::{AllQuery, QueryParser, TermQuery};
-    use crate::schema::{Document, Facet, FacetOptions, IndexRecordOption, Schema};
-    use crate::Term;
+    use crate::schema::{Facet, FacetOptions, IndexRecordOption, Schema, TantivyDocument};
+    use crate::{IndexWriter, Term};
 
     fn test_collapse_mapping_aux(
         facet_terms: &[&str],
@@ -559,7 +559,7 @@ mod tests {
         let facet_field = schema_builder.add_facet_field("facet", FacetOptions::default());
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_for_tests().unwrap();
+        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
         index_writer
             .add_document(doc!(facet_field=>Facet::from("/facet/a")))
             .unwrap();
@@ -588,7 +588,7 @@ mod tests {
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
 
-        let mut index_writer = index.writer_for_tests().unwrap();
+        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
         let num_facets: usize = 3 * 4 * 5;
         let facets: Vec<Facet> = (0..num_facets)
             .map(|mut n| {
@@ -601,7 +601,7 @@ mod tests {
             })
             .collect();
         for i in 0..num_facets * 10 {
-            let mut doc = Document::new();
+            let mut doc = TantivyDocument::new();
             doc.add_facet(facet_field, facets[i % num_facets].clone());
             index_writer.add_document(doc).unwrap();
         }
@@ -732,24 +732,25 @@ mod tests {
         let index = Index::create_in_ram(schema);
 
         let uniform = Uniform::new_inclusive(1, 100_000);
-        let mut docs: Vec<Document> = vec![("a", 10), ("b", 100), ("c", 7), ("d", 12), ("e", 21)]
-            .into_iter()
-            .flat_map(|(c, count)| {
-                let facet = Facet::from(&format!("/facet/{}", c));
-                let doc = doc!(facet_field => facet);
-                iter::repeat(doc).take(count)
-            })
-            .map(|mut doc| {
-                doc.add_facet(
-                    facet_field,
-                    &format!("/facet/{}", thread_rng().sample(uniform)),
-                );
-                doc
-            })
-            .collect();
+        let mut docs: Vec<TantivyDocument> =
+            vec![("a", 10), ("b", 100), ("c", 7), ("d", 12), ("e", 21)]
+                .into_iter()
+                .flat_map(|(c, count)| {
+                    let facet = Facet::from(&format!("/facet/{}", c));
+                    let doc = doc!(facet_field => facet);
+                    iter::repeat(doc).take(count)
+                })
+                .map(|mut doc| {
+                    doc.add_facet(
+                        facet_field,
+                        &format!("/facet/{}", thread_rng().sample(uniform)),
+                    );
+                    doc
+                })
+                .collect();
         docs[..].shuffle(&mut thread_rng());
 
-        let mut index_writer = index.writer_for_tests().unwrap();
+        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
         for doc in docs {
             index_writer.add_document(doc).unwrap();
         }
@@ -780,7 +781,7 @@ mod tests {
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
 
-        let docs: Vec<Document> = vec![("b", 2), ("a", 2), ("c", 4)]
+        let docs: Vec<TantivyDocument> = vec![("b", 2), ("a", 2), ("c", 4)]
             .into_iter()
             .flat_map(|(c, count)| {
                 let facet = Facet::from(&format!("/facet/{}", c));
@@ -828,7 +829,7 @@ mod bench {
     use crate::collector::FacetCollector;
     use crate::query::AllQuery;
     use crate::schema::{Facet, Schema, INDEXED};
-    use crate::Index;
+    use crate::{Index, IndexWriter};
 
     #[bench]
     fn bench_facet_collector(b: &mut Bencher) {
@@ -847,7 +848,7 @@ mod bench {
         // 40425 docs
         docs[..].shuffle(&mut thread_rng());
 
-        let mut index_writer = index.writer_for_tests().unwrap();
+        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
         for doc in docs {
             index_writer.add_document(doc).unwrap();
         }

src/collector/filter_collector_wrapper.rs

@@ -12,8 +12,7 @@ use std::marker::PhantomData;
 use columnar::{BytesColumn, Column, DynamicColumn, HasAssociatedColumnType};
 
 use crate::collector::{Collector, SegmentCollector};
-use crate::schema::Field;
-use crate::{DocId, Score, SegmentReader, TantivyError};
+use crate::{DocId, Score, SegmentReader};
 
 /// The `FilterCollector` filters docs using a fast field value and a predicate.
 ///
@@ -50,13 +49,13 @@ use crate::{DocId, Score, SegmentReader, TantivyError};
 ///
 /// let query_parser = QueryParser::for_index(&index, vec![title]);
 /// let query = query_parser.parse_query("diary")?;
-/// let no_filter_collector = FilterCollector::new(price, |value: u64| value > 20_120u64, TopDocs::with_limit(2));
+/// let no_filter_collector = FilterCollector::new("price".to_string(), |value: u64| value > 20_120u64, TopDocs::with_limit(2));
 /// let top_docs = searcher.search(&query, &no_filter_collector)?;
 ///
 /// assert_eq!(top_docs.len(), 1);
 /// assert_eq!(top_docs[0].1, DocAddress::new(0, 1));
 ///
-/// let filter_all_collector: FilterCollector<_, _, u64> = FilterCollector::new(price, |value| value < 5u64, TopDocs::with_limit(2));
+/// let filter_all_collector: FilterCollector<_, _, u64> = FilterCollector::new("price".to_string(), |value| value < 5u64, TopDocs::with_limit(2));
 /// let filtered_top_docs = searcher.search(&query, &filter_all_collector)?;
 ///
 /// assert_eq!(filtered_top_docs.len(), 0);
@@ -70,7 +69,7 @@ use crate::{DocId, Score, SegmentReader, TantivyError};
 pub struct FilterCollector<TCollector, TPredicate, TPredicateValue>
 where TPredicate: 'static + Clone
 {
-    field: Field,
+    field: String,
     collector: TCollector,
     predicate: TPredicate,
     t_predicate_value: PhantomData<TPredicateValue>,
@@ -83,7 +82,7 @@ where
     TPredicate: Fn(TPredicateValue) -> bool + Send + Sync + Clone,
 {
     /// Create a new `FilterCollector`.
-    pub fn new(field: Field, predicate: TPredicate, collector: TCollector) -> Self {
+    pub fn new(field: String, predicate: TPredicate, collector: TCollector) -> Self {
         Self {
             field,
             predicate,
@@ -110,18 +109,7 @@ where
         segment_local_id: u32,
         segment_reader: &SegmentReader,
     ) -> crate::Result<Self::Child> {
-        let schema = segment_reader.schema();
-        let field_entry = schema.get_field_entry(self.field);
-        if !field_entry.is_fast() {
-            return Err(TantivyError::SchemaError(format!(
-                "Field {:?} is not a fast field.",
-                field_entry.name()
-            )));
-        }
-
-        let column_opt = segment_reader
-            .fast_fields()
-            .column_opt(field_entry.name())?;
+        let column_opt = segment_reader.fast_fields().column_opt(&self.field)?;
 
         let segment_collector = self
             .collector
@@ -229,7 +217,7 @@ where
 ///
 /// let query_parser = QueryParser::for_index(&index, vec![title]);
 /// let query = query_parser.parse_query("diary")?;
-/// let filter_collector = BytesFilterCollector::new(barcode, |bytes: &[u8]| bytes.starts_with(b"01"), TopDocs::with_limit(2));
+/// let filter_collector = BytesFilterCollector::new("barcode".to_string(), |bytes: &[u8]| bytes.starts_with(b"01"), TopDocs::with_limit(2));
 /// let top_docs = searcher.search(&query, &filter_collector)?;
 ///
 /// assert_eq!(top_docs.len(), 1);
@@ -240,7 +228,7 @@ where
 pub struct BytesFilterCollector<TCollector, TPredicate>
 where TPredicate: 'static + Clone
 {
-    field: Field,
+    field: String,
     collector: TCollector,
     predicate: TPredicate,
 }
@@ -251,7 +239,7 @@ where
     TPredicate: Fn(&[u8]) -> bool + Send + Sync + Clone,
 {
     /// Create a new `BytesFilterCollector`.
-    pub fn new(field: Field, predicate: TPredicate, collector: TCollector) -> Self {
+    pub fn new(field: String, predicate: TPredicate, collector: TCollector) -> Self {
         Self {
             field,
             predicate,
@@ -274,10 +262,7 @@ where
         segment_local_id: u32,
         segment_reader: &SegmentReader,
     ) -> crate::Result<Self::Child> {
-        let schema = segment_reader.schema();
-        let field_name = schema.get_field_name(self.field);
-
-        let column_opt = segment_reader.fast_fields().bytes(field_name)?;
+        let column_opt = segment_reader.fast_fields().bytes(&self.field)?;
 
         let segment_collector = self
             .collector

src/collector/histogram_collector.rs

@@ -233,7 +233,7 @@ mod tests {
         let val_field = schema_builder.add_i64_field("val_field", FAST);
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
-        let mut writer = index.writer_with_num_threads(1, 4_000_000)?;
+        let mut writer = index.writer_for_tests()?;
         writer.add_document(doc!(val_field=>12i64))?;
         writer.add_document(doc!(val_field=>-30i64))?;
         writer.add_document(doc!(val_field=>-12i64))?;
@@ -255,7 +255,7 @@ mod tests {
         let val_field = schema_builder.add_i64_field("val_field", FAST);
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
-        let mut writer = index.writer_with_num_threads(1, 4_000_000)?;
+        let mut writer = index.writer_for_tests()?;
         writer.add_document(doc!(val_field=>12i64))?;
         writer.commit()?;
         writer.add_document(doc!(val_field=>-30i64))?;
@@ -280,7 +280,7 @@ mod tests {
         let date_field = schema_builder.add_date_field("date_field", FAST);
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
-        let mut writer = index.writer_with_num_threads(1, 4_000_000)?;
+        let mut writer = index.writer_for_tests()?;
         writer.add_document(doc!(date_field=>DateTime::from_primitive(Date::from_calendar_date(1982, Month::September, 17)?.with_hms(0, 0, 0)?)))?;
         writer.add_document(
             doc!(date_field=>DateTime::from_primitive(Date::from_calendar_date(1986, Month::March, 9)?.with_hms(0, 0, 0)?)),

src/collector/mod.rs

@@ -44,7 +44,7 @@
 //! #     let title = schema_builder.add_text_field("title", TEXT);
 //! #     let schema = schema_builder.build();
 //! #     let index = Index::create_in_ram(schema);
-//! #     let mut index_writer = index.writer(3_000_000)?;
+//! #     let mut index_writer = index.writer(15_000_000)?;
 //! #       index_writer.add_document(doc!(
 //! #       title => "The Name of the Wind",
 //! #      ))?;
@@ -97,7 +97,7 @@ pub use self::multi_collector::{FruitHandle, MultiCollector, MultiFruit};
 mod top_collector;
 
 mod top_score_collector;
-pub use self::top_score_collector::TopDocs;
+pub use self::top_score_collector::{TopDocs, TopNComputer};
 
 mod custom_score_top_collector;
 pub use self::custom_score_top_collector::{CustomScorer, CustomSegmentScorer};

src/collector/multi_collector.rs

@@ -120,7 +120,7 @@ impl<TFruit: Fruit> FruitHandle<TFruit> {
 /// let title = schema_builder.add_text_field("title", TEXT);
 /// let schema = schema_builder.build();
 /// let index = Index::create_in_ram(schema);
-/// let mut index_writer = index.writer(3_000_000)?;
+/// let mut index_writer = index.writer(15_000_000)?;
 /// index_writer.add_document(doc!(title => "The Name of the Wind"))?;
 /// index_writer.add_document(doc!(title => "The Diary of Muadib"))?;
 /// index_writer.add_document(doc!(title => "A Dairy Cow"))?;

src/collector/tests.rs

@@ -7,7 +7,9 @@ use crate::query::{AllQuery, QueryParser};
 use crate::schema::{Schema, FAST, TEXT};
 use crate::time::format_description::well_known::Rfc3339;
 use crate::time::OffsetDateTime;
-use crate::{doc, DateTime, DocAddress, DocId, Document, Index, Score, Searcher, SegmentOrdinal};
+use crate::{
+    doc, DateTime, DocAddress, DocId, Index, Score, Searcher, SegmentOrdinal, TantivyDocument,
+};
 
 pub const TEST_COLLECTOR_WITH_SCORE: TestCollector = TestCollector {
     compute_score: true,
@@ -40,7 +42,7 @@ pub fn test_filter_collector() -> crate::Result<()> {
     let query_parser = QueryParser::for_index(&index, vec![title]);
     let query = query_parser.parse_query("diary")?;
     let filter_some_collector = FilterCollector::new(
-        price,
+        "price".to_string(),
         &|value: u64| value > 20_120u64,
         TopDocs::with_limit(2),
     );
@@ -49,8 +51,11 @@ pub fn test_filter_collector() -> crate::Result<()> {
     assert_eq!(top_docs.len(), 1);
     assert_eq!(top_docs[0].1, DocAddress::new(0, 1));
 
-    let filter_all_collector: FilterCollector<_, _, u64> =
-        FilterCollector::new(price, &|value| value < 5u64, TopDocs::with_limit(2));
+    let filter_all_collector: FilterCollector<_, _, u64> = FilterCollector::new(
+        "price".to_string(),
+        &|value| value < 5u64,
+        TopDocs::with_limit(2),
+    );
     let filtered_top_docs = searcher.search(&query, &filter_all_collector).unwrap();
 
     assert_eq!(filtered_top_docs.len(), 0);
@@ -61,7 +66,8 @@ pub fn test_filter_collector() -> crate::Result<()> {
             > 0
     }
 
-    let filter_dates_collector = FilterCollector::new(date, &date_filter, TopDocs::with_limit(5));
+    let filter_dates_collector =
+        FilterCollector::new("date".to_string(), &date_filter, TopDocs::with_limit(5));
     let filtered_date_docs = searcher.search(&query, &filter_dates_collector)?;
 
     assert_eq!(filtered_date_docs.len(), 2);
@@ -280,8 +286,8 @@ fn make_test_searcher() -> crate::Result<Searcher> {
     let schema = Schema::builder().build();
     let index = Index::create_in_ram(schema);
     let mut index_writer = index.writer_for_tests()?;
-    index_writer.add_document(Document::default())?;
-    index_writer.add_document(Document::default())?;
+    index_writer.add_document(TantivyDocument::default())?;
+    index_writer.add_document(TantivyDocument::default())?;
     index_writer.commit()?;
     Ok(index.reader()?.searcher())
 }

src/collector/top_collector.rs

@@ -1,7 +1,7 @@
 use std::cmp::Ordering;
-use std::collections::BinaryHeap;
 use std::marker::PhantomData;
 
+use super::top_score_collector::TopNComputer;
 use crate::{DocAddress, DocId, SegmentOrdinal, SegmentReader};
 
 /// Contains a feature (field, score, etc.) of a document along with the document address.
@@ -20,6 +20,14 @@ pub(crate) struct ComparableDoc<T, D> {
     pub feature: T,
     pub doc: D,
 }
+impl<T: std::fmt::Debug, D: std::fmt::Debug> std::fmt::Debug for ComparableDoc<T, D> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("ComparableDoc")
+            .field("feature", &self.feature)
+            .field("doc", &self.doc)
+            .finish()
+    }
+}
 
 impl<T: PartialOrd, D: PartialOrd> PartialOrd for ComparableDoc<T, D> {
     fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
@@ -91,18 +99,13 @@ where T: PartialOrd + Clone
         if self.limit == 0 {
             return Ok(Vec::new());
         }
-        let mut top_collector = BinaryHeap::new();
+        let mut top_collector = TopNComputer::new(self.limit + self.offset);
         for child_fruit in children {
             for (feature, doc) in child_fruit {
-                if top_collector.len() < (self.limit + self.offset) {
-                    top_collector.push(ComparableDoc { feature, doc });
-                } else if let Some(mut head) = top_collector.peek_mut() {
-                    if head.feature < feature {
-                        *head = ComparableDoc { feature, doc };
-                    }
-                }
+                top_collector.push(ComparableDoc { feature, doc });
             }
         }
+
         Ok(top_collector
             .into_sorted_vec()
             .into_iter()
@@ -111,7 +114,7 @@ where T: PartialOrd + Clone
             .collect())
     }
 
-    pub(crate) fn for_segment<F: PartialOrd>(
+    pub(crate) fn for_segment<F: PartialOrd + Clone>(
         &self,
         segment_id: SegmentOrdinal,
         _: &SegmentReader,
@@ -136,20 +139,18 @@ where T: PartialOrd + Clone
 /// The Top Collector keeps track of the K documents
 /// sorted by type `T`.
 ///
-/// The implementation is based on a `BinaryHeap`.
+/// The implementation is based on a repeatedly truncating on the median after K * 2 documents
 /// The theoretical complexity for collecting the top `K` out of `n` documents
-/// is `O(n log K)`.
+/// is `O(n + K)`.
 pub(crate) struct TopSegmentCollector<T> {
-    limit: usize,
-    heap: BinaryHeap<ComparableDoc<T, DocId>>,
+    topn_computer: TopNComputer<T, DocId>,
     segment_ord: u32,
 }
 
-impl<T: PartialOrd> TopSegmentCollector<T> {
+impl<T: PartialOrd + Clone> TopSegmentCollector<T> {
     fn new(segment_ord: SegmentOrdinal, limit: usize) -> TopSegmentCollector<T> {
         TopSegmentCollector {
-            limit,
-            heap: BinaryHeap::with_capacity(limit),
+            topn_computer: TopNComputer::new(limit),
             segment_ord,
         }
     }
@@ -158,7 +159,7 @@ impl<T: PartialOrd> TopSegmentCollector<T> {
 impl<T: PartialOrd + Clone> TopSegmentCollector<T> {
     pub fn harvest(self) -> Vec<(T, DocAddress)> {
         let segment_ord = self.segment_ord;
-        self.heap
+        self.topn_computer
             .into_sorted_vec()
             .into_iter()
             .map(|comparable_doc| {
@@ -173,33 +174,13 @@ impl<T: PartialOrd + Clone> TopSegmentCollector<T> {
             .collect()
     }
 
-    /// Return true if more documents have been collected than the limit.
-    #[inline]
-    pub(crate) fn at_capacity(&self) -> bool {
-        self.heap.len() >= self.limit
-    }
-
     /// Collects a document scored by the given feature
     ///
     /// It collects documents until it has reached the max capacity. Once it reaches capacity, it
     /// will compare the lowest scoring item with the given one and keep whichever is greater.
     #[inline]
     pub fn collect(&mut self, doc: DocId, feature: T) {
-        if self.at_capacity() {
-            // It's ok to unwrap as long as a limit of 0 is forbidden.
-            if let Some(limit_feature) = self.heap.peek().map(|head| head.feature.clone()) {
-                if limit_feature < feature {
-                    if let Some(mut head) = self.heap.peek_mut() {
-                        head.feature = feature;
-                        head.doc = doc;
-                    }
-                }
-            }
-        } else {
-            // we have not reached capacity yet, so we can just push the
-            // element.
-            self.heap.push(ComparableDoc { feature, doc });
-        }
+        self.topn_computer.push(ComparableDoc { feature, doc });
     }
 }
 

src/collector/top_score_collector.rs

@@ -1,4 +1,3 @@
-use std::collections::BinaryHeap;
 use std::fmt;
 use std::marker::PhantomData;
 use std::sync::Arc;
@@ -86,12 +85,15 @@ where
 /// The `TopDocs` collector keeps track of the top `K` documents
 /// sorted by their score.
 ///
-/// The implementation is based on a `BinaryHeap`.
-/// The theoretical complexity for collecting the top `K` out of `n` documents
-/// is `O(n log K)`.
+/// The implementation is based on a repeatedly truncating on the median after K * 2 documents
+/// with pattern defeating QuickSort.
+/// The theoretical complexity for collecting the top `K` out of `N` documents
+/// is `O(N + K)`.
 ///
-/// This collector guarantees a stable sorting in case of a tie on the
-/// document score. As such, it is suitable to implement pagination.
+/// This collector does not guarantee a stable sorting in case of a tie on the
+/// document score, for stable sorting `PartialOrd` needs to resolve on other fields
+/// like docid in case of score equality.
+/// Only then, it is suitable for pagination.
 ///
 /// ```rust
 /// use tantivy::collector::TopDocs;
@@ -661,50 +663,35 @@ impl Collector for TopDocs {
         reader: &SegmentReader,
     ) -> crate::Result<<Self::Child as SegmentCollector>::Fruit> {
         let heap_len = self.0.limit + self.0.offset;
-        let mut heap: BinaryHeap<ComparableDoc<Score, DocId>> = BinaryHeap::with_capacity(heap_len);
+        let mut top_n = TopNComputer::new(heap_len);
 
         if let Some(alive_bitset) = reader.alive_bitset() {
             let mut threshold = Score::MIN;
-            weight.for_each_pruning(threshold, reader, &mut |doc, score| {
+            top_n.threshold = Some(threshold);
+            weight.for_each_pruning(Score::MIN, reader, &mut |doc, score| {
                 if alive_bitset.is_deleted(doc) {
                     return threshold;
                 }
-                let heap_item = ComparableDoc {
+                let doc = ComparableDoc {
                     feature: score,
                     doc,
                 };
-                if heap.len() < heap_len {
-                    heap.push(heap_item);
-                    if heap.len() == heap_len {
-                        threshold = heap.peek().map(|el| el.feature).unwrap_or(Score::MIN);
-                    }
-                    return threshold;
-                }
-                *heap.peek_mut().unwrap() = heap_item;
-                threshold = heap.peek().map(|el| el.feature).unwrap_or(Score::MIN);
+                top_n.push(doc);
+                threshold = top_n.threshold.unwrap_or(Score::MIN);
                 threshold
             })?;
         } else {
             weight.for_each_pruning(Score::MIN, reader, &mut |doc, score| {
-                let heap_item = ComparableDoc {
+                let doc = ComparableDoc {
                     feature: score,
                     doc,
                 };
-                if heap.len() < heap_len {
-                    heap.push(heap_item);
-                    // TODO the threshold is suboptimal for heap.len == heap_len
-                    if heap.len() == heap_len {
-                        return heap.peek().map(|el| el.feature).unwrap_or(Score::MIN);
-                    } else {
-                        return Score::MIN;
-                    }
-                }
-                *heap.peek_mut().unwrap() = heap_item;
-                heap.peek().map(|el| el.feature).unwrap_or(Score::MIN)
+                top_n.push(doc);
+                top_n.threshold.unwrap_or(Score::MIN)
             })?;
         }
 
-        let fruit = heap
+        let fruit = top_n
             .into_sorted_vec()
             .into_iter()
             .map(|cid| {
@@ -736,9 +723,81 @@ impl SegmentCollector for TopScoreSegmentCollector {
     }
 }
 
+/// Fast TopN Computation
+///
+/// For TopN == 0, it will be relative expensive.
+pub struct TopNComputer<Score, DocId> {
+    buffer: Vec<ComparableDoc<Score, DocId>>,
+    top_n: usize,
+    pub(crate) threshold: Option<Score>,
+}
+
+impl<Score, DocId> TopNComputer<Score, DocId>
+where
+    Score: PartialOrd + Clone,
+    DocId: Ord + Clone,
+{
+    /// Create a new `TopNComputer`.
+    /// Internally it will allocate a buffer of size `2 * top_n`.
+    pub fn new(top_n: usize) -> Self {
+        let vec_cap = top_n.max(1) * 2;
+        TopNComputer {
+            buffer: Vec::with_capacity(vec_cap),
+            top_n,
+            threshold: None,
+        }
+    }
+
+    #[inline]
+    pub(crate) fn push(&mut self, doc: ComparableDoc<Score, DocId>) {
+        if let Some(last_median) = self.threshold.clone() {
+            if doc.feature < last_median {
+                return;
+            }
+        }
+        if self.buffer.len() == self.buffer.capacity() {
+            let median = self.truncate_top_n();
+            self.threshold = Some(median);
+        }
+
+        // This is faster since it avoids the buffer resizing to be inlined from vec.push()
+        // (this is in the hot path)
+        // TODO: Replace with `push_within_capacity` when it's stabilized
+        let uninit = self.buffer.spare_capacity_mut();
+        // This cannot panic, because we truncate_median will at least remove one element, since
+        // the min capacity is 2.
+        uninit[0].write(doc);
+        // This is safe because it would panic in the line above
+        unsafe {
+            self.buffer.set_len(self.buffer.len() + 1);
+        }
+    }
+
+    #[inline(never)]
+    fn truncate_top_n(&mut self) -> Score {
+        // Use select_nth_unstable to find the top nth score
+        let (_, median_el, _) = self.buffer.select_nth_unstable(self.top_n);
+
+        let median_score = median_el.feature.clone();
+        // Remove all elements below the top_n
+        self.buffer.truncate(self.top_n);
+
+        median_score
+    }
+
+    pub(crate) fn into_sorted_vec(mut self) -> Vec<ComparableDoc<Score, DocId>> {
+        if self.buffer.len() > self.top_n {
+            self.truncate_top_n();
+        }
+        self.buffer.sort_unstable();
+        self.buffer
+    }
+}
+
 #[cfg(test)]
 mod tests {
-    use super::TopDocs;
+    use super::{TopDocs, TopNComputer};
+    use crate::collector::top_collector::ComparableDoc;
     use crate::collector::Collector;
     use crate::query::{AllQuery, Query, QueryParser};
     use crate::schema::{Field, Schema, FAST, STORED, TEXT};
@@ -767,6 +826,78 @@ mod tests {
         }
     }
 
+    #[test]
+    fn test_empty_topn_computer() {
+        let mut computer: TopNComputer<u32, u32> = TopNComputer::new(0);
+
+        computer.push(ComparableDoc {
+            feature: 1u32,
+            doc: 1u32,
+        });
+        computer.push(ComparableDoc {
+            feature: 1u32,
+            doc: 2u32,
+        });
+        computer.push(ComparableDoc {
+            feature: 1u32,
+            doc: 3u32,
+        });
+        assert!(computer.into_sorted_vec().is_empty());
+    }
+    #[test]
+    fn test_topn_computer() {
+        let mut computer: TopNComputer<u32, u32> = TopNComputer::new(2);
+
+        computer.push(ComparableDoc {
+            feature: 1u32,
+            doc: 1u32,
+        });
+        computer.push(ComparableDoc {
+            feature: 2u32,
+            doc: 2u32,
+        });
+        computer.push(ComparableDoc {
+            feature: 3u32,
+            doc: 3u32,
+        });
+        computer.push(ComparableDoc {
+            feature: 2u32,
+            doc: 4u32,
+        });
+        computer.push(ComparableDoc {
+            feature: 1u32,
+            doc: 5u32,
+        });
+        assert_eq!(
+            computer.into_sorted_vec(),
+            &[
+                ComparableDoc {
+                    feature: 3u32,
+                    doc: 3u32,
+                },
+                ComparableDoc {
+                    feature: 2u32,
+                    doc: 2u32,
+                }
+            ]
+        );
+    }
+
+    #[test]
+    fn test_topn_computer_no_panic() {
+        for top_n in 0..10 {
+            let mut computer: TopNComputer<u32, u32> = TopNComputer::new(top_n);
+
+            for _ in 0..1 + top_n * 2 {
+                computer.push(ComparableDoc {
+                    feature: 1u32,
+                    doc: 1u32,
+                });
+            }
+            let _vals = computer.into_sorted_vec();
+        }
+    }
+
     #[test]
     fn test_top_collector_not_at_capacity_without_offset() -> crate::Result<()> {
         let index = make_index()?;
@@ -852,20 +983,25 @@ mod tests {
         // using AllQuery to get a constant score
         let searcher = index.reader().unwrap().searcher();
 
+        let page_0 = searcher.search(&AllQuery, &TopDocs::with_limit(1)).unwrap();
+
         let page_1 = searcher.search(&AllQuery, &TopDocs::with_limit(2)).unwrap();
 
         let page_2 = searcher.search(&AllQuery, &TopDocs::with_limit(3)).unwrap();
 
         // precondition for the test to be meaningful: we did get documents
         // with the same score
+        assert!(page_0.iter().all(|result| result.0 == page_1[0].0));
         assert!(page_1.iter().all(|result| result.0 == page_1[0].0));
         assert!(page_2.iter().all(|result| result.0 == page_2[0].0));
 
         // sanity check since we're relying on make_index()
+        assert_eq!(page_0.len(), 1);
         assert_eq!(page_1.len(), 2);
         assert_eq!(page_2.len(), 3);
 
         assert_eq!(page_1, &page_2[..page_1.len()]);
+        assert_eq!(page_0, &page_2[..page_0.len()]);
     }
 
     #[test]

src/core/index.rs

@@ -16,9 +16,10 @@ use crate::directory::error::OpenReadError;
 use crate::directory::MmapDirectory;
 use crate::directory::{Directory, ManagedDirectory, RamDirectory, INDEX_WRITER_LOCK};
 use crate::error::{DataCorruption, TantivyError};
-use crate::indexer::index_writer::{MAX_NUM_THREAD, MEMORY_ARENA_NUM_BYTES_MIN};
+use crate::indexer::index_writer::{MAX_NUM_THREAD, MEMORY_BUDGET_NUM_BYTES_MIN};
 use crate::indexer::segment_updater::save_metas;
 use crate::reader::{IndexReader, IndexReaderBuilder};
+use crate::schema::document::Document;
 use crate::schema::{Field, FieldType, Schema};
 use crate::tokenizer::{TextAnalyzer, TokenizerManager};
 use crate::IndexWriter;
@@ -184,11 +185,11 @@ impl IndexBuilder {
     ///
     /// It expects an originally empty directory, and will not run any GC operation.
     #[doc(hidden)]
-    pub fn single_segment_index_writer(
+    pub fn single_segment_index_writer<D: Document>(
         self,
         dir: impl Into<Box<dyn Directory>>,
         mem_budget: usize,
-    ) -> crate::Result<SingleSegmentIndexWriter> {
+    ) -> crate::Result<SingleSegmentIndexWriter<D>> {
         let index = self.create(dir)?;
         let index_simple_writer = SingleSegmentIndexWriter::new(index, mem_budget)?;
         Ok(index_simple_writer)
@@ -523,19 +524,19 @@ impl Index {
     /// - `num_threads` defines the number of indexing workers that
     /// should work at the same time.
     ///
-    /// - `overall_memory_arena_in_bytes` sets the amount of memory
+    /// - `overall_memory_budget_in_bytes` sets the amount of memory
     /// allocated for all indexing thread.
-    /// Each thread will receive a budget of  `overall_memory_arena_in_bytes / num_threads`.
+    /// Each thread will receive a budget of  `overall_memory_budget_in_bytes / num_threads`.
     ///
     /// # Errors
     /// If the lockfile already exists, returns `Error::DirectoryLockBusy` or an `Error::IoError`.
     /// If the memory arena per thread is too small or too big, returns
     /// `TantivyError::InvalidArgument`
-    pub fn writer_with_num_threads(
+    pub fn writer_with_num_threads<D: Document>(
         &self,
         num_threads: usize,
-        overall_memory_arena_in_bytes: usize,
-    ) -> crate::Result<IndexWriter> {
+        overall_memory_budget_in_bytes: usize,
+    ) -> crate::Result<IndexWriter<D>> {
         let directory_lock = self
             .directory
             .acquire_lock(&INDEX_WRITER_LOCK)
@@ -550,7 +551,7 @@ impl Index {
                     ),
                 )
             })?;
-        let memory_arena_in_bytes_per_thread = overall_memory_arena_in_bytes / num_threads;
+        let memory_arena_in_bytes_per_thread = overall_memory_budget_in_bytes / num_threads;
         IndexWriter::new(
             self,
             num_threads,
@@ -561,10 +562,10 @@ impl Index {
 
     /// Helper to create an index writer for tests.
     ///
-    /// That index writer only simply has a single thread and a memory arena of 10 MB.
+    /// That index writer only simply has a single thread and a memory budget of 15 MB.
     /// Using a single thread gives us a deterministic allocation of DocId.
     #[cfg(test)]
-    pub fn writer_for_tests(&self) -> crate::Result<IndexWriter> {
+    pub fn writer_for_tests<D: Document>(&self) -> crate::Result<IndexWriter<D>> {
         self.writer_with_num_threads(1, 15_000_000)
     }
 
@@ -579,13 +580,16 @@ impl Index {
     /// If the lockfile already exists, returns `Error::FileAlreadyExists`.
     /// If the memory arena per thread is too small or too big, returns
     /// `TantivyError::InvalidArgument`
-    pub fn writer(&self, memory_arena_num_bytes: usize) -> crate::Result<IndexWriter> {
+    pub fn writer<D: Document>(
+        &self,
+        memory_budget_in_bytes: usize,
+    ) -> crate::Result<IndexWriter<D>> {
         let mut num_threads = std::cmp::min(num_cpus::get(), MAX_NUM_THREAD);
-        let memory_arena_num_bytes_per_thread = memory_arena_num_bytes / num_threads;
-        if memory_arena_num_bytes_per_thread < MEMORY_ARENA_NUM_BYTES_MIN {
-            num_threads = (memory_arena_num_bytes / MEMORY_ARENA_NUM_BYTES_MIN).max(1);
+        let memory_budget_num_bytes_per_thread = memory_budget_in_bytes / num_threads;
+        if memory_budget_num_bytes_per_thread < MEMORY_BUDGET_NUM_BYTES_MIN {
+            num_threads = (memory_budget_in_bytes / MEMORY_BUDGET_NUM_BYTES_MIN).max(1);
         }
-        self.writer_with_num_threads(num_threads, memory_arena_num_bytes)
+        self.writer_with_num_threads(num_threads, memory_budget_in_bytes)
     }
 
     /// Accessor to the index settings

src/core/inverted_index_reader.rs

@@ -1,11 +1,12 @@
 use std::io;
 
 use common::BinarySerializable;
+use fnv::FnvHashSet;
 
 use crate::directory::FileSlice;
 use crate::positions::PositionReader;
 use crate::postings::{BlockSegmentPostings, SegmentPostings, TermInfo};
-use crate::schema::{IndexRecordOption, Term};
+use crate::schema::{IndexRecordOption, Term, Type, JSON_END_OF_PATH};
 use crate::termdict::TermDictionary;
 
 /// The inverted index reader is in charge of accessing
@@ -69,6 +70,28 @@ impl InvertedIndexReader {
         &self.termdict
     }
 
+    /// Return the fields and types encoded in the dictionary in lexicographic oder.
+    /// Only valid on JSON fields.
+    ///
+    /// Notice: This requires a full scan and therefore **very expensive**.
+    /// TODO: Move to sstable to use the index.
+    pub fn list_fields(&self) -> io::Result<Vec<(String, Type)>> {
+        let mut stream = self.termdict.stream()?;
+        let mut fields = Vec::new();
+        let mut fields_set = FnvHashSet::default();
+        while let Some((term, _term_info)) = stream.next() {
+            if let Some(index) = term.iter().position(|&byte| byte == JSON_END_OF_PATH) {
+                if !fields_set.contains(&term[..index + 2]) {
+                    fields_set.insert(term[..index + 2].to_vec());
+                    let typ = Type::from_code(term[index + 1]).unwrap();
+                    fields.push((String::from_utf8_lossy(&term[..index]).to_string(), typ));
+                }
+            }
+        }
+
+        Ok(fields)
+    }
+
     /// Resets the block segment to another position of the postings
     /// file.
     ///

src/core/json_utils.rs

@@ -5,6 +5,7 @@ use rustc_hash::FxHashMap;
 
 use crate::fastfield::FastValue;
 use crate::postings::{IndexingContext, IndexingPosition, PostingsWriter};
+use crate::schema::document::{ReferenceValue, Value};
 use crate::schema::term::{JSON_PATH_SEGMENT_SEP, JSON_PATH_SEGMENT_SEP_STR};
 use crate::schema::{Field, Type, DATE_TIME_PRECISION_INDEXED};
 use crate::time::format_description::well_known::Rfc3339;
@@ -64,9 +65,9 @@ impl IndexingPositionsPerPath {
     }
 }
 
-pub(crate) fn index_json_values<'a>(
+pub(crate) fn index_json_values<'a, V: Value<'a>>(
     doc: DocId,
-    json_values: impl Iterator<Item = crate::Result<&'a serde_json::Map<String, serde_json::Value>>>,
+    json_visitors: impl Iterator<Item = crate::Result<V::ObjectIter>>,
     text_analyzer: &mut TextAnalyzer,
     expand_dots_enabled: bool,
     term_buffer: &mut Term,
@@ -75,11 +76,11 @@ pub(crate) fn index_json_values<'a>(
 ) -> crate::Result<()> {
     let mut json_term_writer = JsonTermWriter::wrap(term_buffer, expand_dots_enabled);
     let mut positions_per_path: IndexingPositionsPerPath = Default::default();
-    for json_value_res in json_values {
-        let json_value = json_value_res?;
-        index_json_object(
+    for json_visitor_res in json_visitors {
+        let json_visitor = json_visitor_res?;
+        index_json_object::<V>(
             doc,
-            json_value,
+            json_visitor,
             text_analyzer,
             &mut json_term_writer,
             postings_writer,
@@ -90,20 +91,20 @@ pub(crate) fn index_json_values<'a>(
     Ok(())
 }
 
-fn index_json_object(
+fn index_json_object<'a, V: Value<'a>>(
     doc: DocId,
-    json_value: &serde_json::Map<String, serde_json::Value>,
+    json_visitor: V::ObjectIter,
     text_analyzer: &mut TextAnalyzer,
     json_term_writer: &mut JsonTermWriter,
     postings_writer: &mut dyn PostingsWriter,
     ctx: &mut IndexingContext,
     positions_per_path: &mut IndexingPositionsPerPath,
 ) {
-    for (json_path_segment, json_value) in json_value {
+    for (json_path_segment, json_value_visitor) in json_visitor {
         json_term_writer.push_path_segment(json_path_segment);
         index_json_value(
             doc,
-            json_value,
+            json_value_visitor,
             text_analyzer,
             json_term_writer,
             postings_writer,
@@ -114,52 +115,65 @@ fn index_json_object(
     }
 }
 
-fn index_json_value(
+fn index_json_value<'a, V: Value<'a>>(
     doc: DocId,
-    json_value: &serde_json::Value,
+    json_value: V,
     text_analyzer: &mut TextAnalyzer,
     json_term_writer: &mut JsonTermWriter,
     postings_writer: &mut dyn PostingsWriter,
     ctx: &mut IndexingContext,
     positions_per_path: &mut IndexingPositionsPerPath,
 ) {
-    match json_value {
-        serde_json::Value::Null => {}
-        serde_json::Value::Bool(val_bool) => {
-            json_term_writer.set_fast_value(*val_bool);
+    match json_value.as_value() {
+        ReferenceValue::Null => {}
+        ReferenceValue::Str(val) => {
+            let mut token_stream = text_analyzer.token_stream(val);
+
+            // TODO: make sure the chain position works out.
+            json_term_writer.close_path_and_set_type(Type::Str);
+            let indexing_position = positions_per_path.get_position(json_term_writer.term());
+            postings_writer.index_text(
+                doc,
+                &mut *token_stream,
+                json_term_writer.term_buffer,
+                ctx,
+                indexing_position,
+            );
+        }
+        ReferenceValue::U64(val) => {
+            json_term_writer.set_fast_value(val);
             postings_writer.subscribe(doc, 0u32, json_term_writer.term(), ctx);
         }
-        serde_json::Value::Number(number) => {
-            if let Some(number_i64) = number.as_i64() {
-                json_term_writer.set_fast_value(number_i64);
-            } else if let Some(number_u64) = number.as_u64() {
-                json_term_writer.set_fast_value(number_u64);
-            } else if let Some(number_f64) = number.as_f64() {
-                json_term_writer.set_fast_value(number_f64);
-            }
+        ReferenceValue::I64(val) => {
+            json_term_writer.set_fast_value(val);
             postings_writer.subscribe(doc, 0u32, json_term_writer.term(), ctx);
         }
-        serde_json::Value::String(text) => match infer_type_from_str(text) {
-            TextOrDateTime::Text(text) => {
-                let mut token_stream = text_analyzer.token_stream(text);
-                // TODO make sure the chain position works out.
-                json_term_writer.close_path_and_set_type(Type::Str);
-                let indexing_position = positions_per_path.get_position(json_term_writer.term());
-                postings_writer.index_text(
-                    doc,
-                    &mut *token_stream,
-                    json_term_writer.term_buffer,
-                    ctx,
-                    indexing_position,
-                );
-            }
-            TextOrDateTime::DateTime(dt) => {
-                json_term_writer.set_fast_value(DateTime::from_utc(dt));
-                postings_writer.subscribe(doc, 0u32, json_term_writer.term(), ctx);
-            }
-        },
-        serde_json::Value::Array(arr) => {
-            for val in arr {
+        ReferenceValue::F64(val) => {
+            json_term_writer.set_fast_value(val);
+            postings_writer.subscribe(doc, 0u32, json_term_writer.term(), ctx);
+        }
+        ReferenceValue::Bool(val) => {
+            json_term_writer.set_fast_value(val);
+            postings_writer.subscribe(doc, 0u32, json_term_writer.term(), ctx);
+        }
+        ReferenceValue::Facet(_) => {
+            unimplemented!("Facet support in dynamic fields is not yet implemented")
+        }
+        ReferenceValue::IpAddr(_) => {
+            unimplemented!("IP address support in dynamic fields is not yet implemented")
+        }
+        ReferenceValue::Date(val) => {
+            json_term_writer.set_fast_value(val);
+            postings_writer.subscribe(doc, 0u32, json_term_writer.term(), ctx);
+        }
+        ReferenceValue::PreTokStr(_) => {
+            unimplemented!("Pre-tokenized string support in dynamic fields is not yet implemented")
+        }
+        ReferenceValue::Bytes(_) => {
+            unimplemented!("Bytes support in dynamic fields is not yet implemented")
+        }
+        ReferenceValue::Array(elements) => {
+            for val in elements {
                 index_json_value(
                     doc,
                     val,
@@ -171,10 +185,10 @@ fn index_json_value(
                 );
             }
         }
-        serde_json::Value::Object(map) => {
-            index_json_object(
+        ReferenceValue::Object(object) => {
+            index_json_object::<V>(
                 doc,
-                map,
+                object,
                 text_analyzer,
                 json_term_writer,
                 postings_writer,
@@ -185,21 +199,6 @@ fn index_json_value(
     }
 }
 
-enum TextOrDateTime<'a> {
-    Text(&'a str),
-    DateTime(OffsetDateTime),
-}
-
-fn infer_type_from_str(text: &str) -> TextOrDateTime {
-    match OffsetDateTime::parse(text, &Rfc3339) {
-        Ok(dt) => {
-            let dt_utc = dt.to_offset(UtcOffset::UTC);
-            TextOrDateTime::DateTime(dt_utc)
-        }
-        Err(_) => TextOrDateTime::Text(text),
-    }
-}
-
 // Tries to infer a JSON type from a string.
 pub fn convert_to_fast_value_and_get_term(
     json_term_writer: &mut JsonTermWriter,

src/core/searcher.rs

@@ -5,7 +5,8 @@ use std::{fmt, io};
 use crate::collector::Collector;
 use crate::core::{Executor, SegmentReader};
 use crate::query::{Bm25StatisticsProvider, EnableScoring, Query};
-use crate::schema::{Document, Schema, Term};
+use crate::schema::document::{Document, DocumentDeserialize};
+use crate::schema::{Schema, Term};
 use crate::space_usage::SearcherSpaceUsage;
 use crate::store::{CacheStats, StoreReader};
 use crate::{DocAddress, Index, Opstamp, SegmentId, TrackedObject};
@@ -83,7 +84,7 @@ impl Searcher {
     ///
     /// The searcher uses the segment ordinal to route the
     /// request to the right `Segment`.
-    pub fn doc(&self, doc_address: DocAddress) -> crate::Result<Document> {
+    pub fn doc<D: DocumentDeserialize>(&self, doc_address: DocAddress) -> crate::Result<D> {
         let store_reader = &self.inner.store_readers[doc_address.segment_ord as usize];
         store_reader.get(doc_address.doc_id)
     }
@@ -103,7 +104,10 @@ impl Searcher {
 
     /// Fetches a document in an asynchronous manner.
     #[cfg(feature = "quickwit")]
-    pub async fn doc_async(&self, doc_address: DocAddress) -> crate::Result<Document> {
+    pub async fn doc_async<D: DocumentDeserialize>(
+        &self,
+        doc_address: DocAddress,
+    ) -> crate::Result<D> {
         let store_reader = &self.inner.store_readers[doc_address.segment_ord as usize];
         store_reader.get_async(doc_address.doc_id).await
     }

src/core/segment_reader.rs

@@ -355,7 +355,7 @@ impl fmt::Debug for SegmentReader {
 mod test {
     use crate::core::Index;
     use crate::schema::{Schema, Term, STORED, TEXT};
-    use crate::DocId;
+    use crate::{DocId, IndexWriter};
 
     #[test]
     fn test_num_alive() -> crate::Result<()> {
@@ -366,7 +366,7 @@ mod test {
         let name = schema.get_field("name").unwrap();
 
         {
-            let mut index_writer = index.writer_for_tests()?;
+            let mut index_writer: IndexWriter = index.writer_for_tests()?;
             index_writer.add_document(doc!(name => "tantivy"))?;
             index_writer.add_document(doc!(name => "horse"))?;
             index_writer.add_document(doc!(name => "jockey"))?;
@@ -392,7 +392,7 @@ mod test {
         let name = schema.get_field("name").unwrap();
 
         {
-            let mut index_writer = index.writer_for_tests()?;
+            let mut index_writer: IndexWriter = index.writer_for_tests()?;
             index_writer.add_document(doc!(name => "tantivy"))?;
             index_writer.add_document(doc!(name => "horse"))?;
             index_writer.add_document(doc!(name => "jockey"))?;
@@ -402,7 +402,7 @@ mod test {
         }
 
         {
-            let mut index_writer2 = index.writer(50_000_000)?;
+            let mut index_writer2: IndexWriter = index.writer(50_000_000)?;
             index_writer2.delete_term(Term::from_field_text(name, "horse"));
             index_writer2.delete_term(Term::from_field_text(name, "cap"));
 

src/core/single_segment_index_writer.rs

@@ -1,16 +1,20 @@
+use std::marker::PhantomData;
+
 use crate::indexer::operation::AddOperation;
 use crate::indexer::segment_updater::save_metas;
 use crate::indexer::SegmentWriter;
-use crate::{Directory, Document, Index, IndexMeta, Opstamp, Segment};
+use crate::schema::document::Document;
+use crate::{Directory, Index, IndexMeta, Opstamp, Segment, TantivyDocument};
 
 #[doc(hidden)]
-pub struct SingleSegmentIndexWriter {
+pub struct SingleSegmentIndexWriter<D: Document = TantivyDocument> {
     segment_writer: SegmentWriter,
     segment: Segment,
     opstamp: Opstamp,
+    _phantom: PhantomData<D>,
 }
 
-impl SingleSegmentIndexWriter {
+impl<D: Document> SingleSegmentIndexWriter<D> {
     pub fn new(index: Index, mem_budget: usize) -> crate::Result<Self> {
         let segment = index.new_segment();
         let segment_writer = SegmentWriter::for_segment(mem_budget, segment.clone())?;
@@ -18,6 +22,7 @@ impl SingleSegmentIndexWriter {
             segment_writer,
             segment,
             opstamp: 0,
+            _phantom: PhantomData,
         })
     }
 
@@ -25,7 +30,7 @@ impl SingleSegmentIndexWriter {
         self.segment_writer.mem_usage()
     }
 
-    pub fn add_document(&mut self, document: Document) -> crate::Result<()> {
+    pub fn add_document(&mut self, document: D) -> crate::Result<()> {
         let opstamp = self.opstamp;
         self.opstamp += 1;
         self.segment_writer

src/core/tests.rs

@@ -5,8 +5,8 @@ use crate::query::TermQuery;
 use crate::schema::{Field, IndexRecordOption, Schema, INDEXED, STRING, TEXT};
 use crate::tokenizer::TokenizerManager;
 use crate::{
-    Directory, Document, Index, IndexBuilder, IndexReader, IndexSettings, ReloadPolicy, SegmentId,
-    Term,
+    Directory, Index, IndexBuilder, IndexReader, IndexSettings, IndexWriter, ReloadPolicy,
+    SegmentId, TantivyDocument, Term,
 };
 
 #[test]
@@ -159,7 +159,7 @@ mod mmap_specific {
         let schema = throw_away_schema();
         let field = schema.get_field("num_likes").unwrap();
         let mut index = Index::create_from_tempdir(schema)?;
-        let mut writer = index.writer_for_tests()?;
+        let mut writer: IndexWriter = index.writer_for_tests()?;
         writer.commit()?;
         let reader = index
             .reader_builder()
@@ -208,7 +208,7 @@ fn test_index_on_commit_reload_policy_aux(
         .watch(WatchCallback::new(move || {
             let _ = sender.send(());
         }));
-    let mut writer = index.writer_for_tests()?;
+    let mut writer: IndexWriter = index.writer_for_tests()?;
     assert_eq!(reader.searcher().num_docs(), 0);
     writer.add_document(doc!(field=>1u64))?;
     writer.commit().unwrap();
@@ -242,7 +242,7 @@ fn garbage_collect_works_as_intended() -> crate::Result<()> {
     let field = schema.get_field("num_likes").unwrap();
     let index = Index::create(directory.clone(), schema, IndexSettings::default())?;
 
-    let mut writer = index.writer_with_num_threads(1, 32_000_000).unwrap();
+    let mut writer: IndexWriter = index.writer_with_num_threads(1, 32_000_000).unwrap();
     for _seg in 0..8 {
         for i in 0u64..1_000u64 {
             writer.add_document(doc!(field => i))?;
@@ -306,7 +306,7 @@ fn test_merging_segment_update_docfreq() {
     let id_field = schema_builder.add_text_field("id", STRING);
     let schema = schema_builder.build();
     let index = Index::create_in_ram(schema);
-    let mut writer = index.writer_for_tests().unwrap();
+    let mut writer: IndexWriter = index.writer_for_tests().unwrap();
     writer.set_merge_policy(Box::new(NoMergePolicy));
     for _ in 0..5 {
         writer.add_document(doc!(text_field=>"hello")).unwrap();
@@ -317,13 +317,13 @@ fn test_merging_segment_update_docfreq() {
     writer
         .add_document(doc!(text_field=>"hello", id_field=>"TO_BE_DELETED"))
         .unwrap();
-    writer.add_document(Document::default()).unwrap();
+    writer.add_document(TantivyDocument::default()).unwrap();
     writer.commit().unwrap();
     for _ in 0..7 {
         writer.add_document(doc!(text_field=>"hello")).unwrap();
     }
-    writer.add_document(Document::default()).unwrap();
-    writer.add_document(Document::default()).unwrap();
+    writer.add_document(TantivyDocument::default()).unwrap();
+    writer.add_document(TantivyDocument::default()).unwrap();
     writer.delete_term(Term::from_field_text(id_field, "TO_BE_DELETED"));
     writer.commit().unwrap();
 

src/directory/mmap_directory.rs

@@ -1,7 +1,7 @@
 use std::collections::HashMap;
 use std::fmt;
 use std::fs::{self, File, OpenOptions};
-use std::io::{self, BufWriter, Read, Seek, Write};
+use std::io::{self, BufWriter, Read, Write};
 use std::ops::Deref;
 use std::path::{Path, PathBuf};
 use std::sync::{Arc, RwLock, Weak};
@@ -328,12 +328,6 @@ impl Write for SafeFileWriter {
     }
 }
 
-impl Seek for SafeFileWriter {
-    fn seek(&mut self, pos: io::SeekFrom) -> io::Result<u64> {
-        self.0.seek(pos)
-    }
-}
-
 impl TerminatingWrite for SafeFileWriter {
     fn terminate_ref(&mut self, _: AntiCallToken) -> io::Result<()> {
         self.0.flush()?;
@@ -539,7 +533,7 @@ mod tests {
     use super::*;
     use crate::indexer::LogMergePolicy;
     use crate::schema::{Schema, SchemaBuilder, TEXT};
-    use crate::{Index, IndexSettings, ReloadPolicy};
+    use crate::{Index, IndexSettings, IndexWriter, ReloadPolicy};
 
     #[test]
     fn test_open_non_existent_path() {
@@ -651,7 +645,7 @@ mod tests {
             let index =
                 Index::create(mmap_directory.clone(), schema, IndexSettings::default()).unwrap();
 
-            let mut index_writer = index.writer_for_tests().unwrap();
+            let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
             let mut log_merge_policy = LogMergePolicy::default();
             log_merge_policy.set_min_num_segments(3);
             index_writer.set_merge_policy(Box::new(log_merge_policy));

src/directory/ram_directory.rs

@@ -1,5 +1,5 @@
 use std::collections::HashMap;
-use std::io::{self, BufWriter, Cursor, Seek, SeekFrom, Write};
+use std::io::{self, BufWriter, Cursor, Write};
 use std::path::{Path, PathBuf};
 use std::sync::{Arc, RwLock};
 use std::{fmt, result};
@@ -48,12 +48,6 @@ impl Drop for VecWriter {
     }
 }
 
-impl Seek for VecWriter {
-    fn seek(&mut self, pos: SeekFrom) -> io::Result<u64> {
-        self.data.seek(pos)
-    }
-}
-
 impl Write for VecWriter {
     fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
         self.is_flushed = false;

src/error.rs

@@ -11,6 +11,7 @@ use crate::directory::error::{
     Incompatibility, LockError, OpenDirectoryError, OpenReadError, OpenWriteError,
 };
 use crate::fastfield::FastFieldNotAvailableError;
+use crate::schema::document::DeserializeError;
 use crate::{query, schema};
 
 /// Represents a `DataCorruption` error.
@@ -106,6 +107,9 @@ pub enum TantivyError {
     /// e.g. a datastructure is incorrectly inititalized.
     #[error("Internal error: '{0}'")]
     InternalError(String),
+    #[error("Deserialize error: {0}")]
+    /// An error occurred while attempting to deserialize a document.
+    DeserializeError(DeserializeError),
 }
 
 impl From<io::Error> for TantivyError {
@@ -176,3 +180,9 @@ impl From<rayon::ThreadPoolBuildError> for TantivyError {
         TantivyError::SystemError(error.to_string())
     }
 }
+
+impl From<DeserializeError> for TantivyError {
+    fn from(error: DeserializeError) -> TantivyError {
+        TantivyError::DeserializeError(error)
+    }
+}

src/fastfield/facet_reader.rs

@@ -62,8 +62,9 @@ impl FacetReader {
 
 #[cfg(test)]
 mod tests {
-    use crate::schema::{Facet, FacetOptions, SchemaBuilder, Value, STORED};
-    use crate::{DocAddress, Document, Index};
+    use crate::schema::document::Value;
+    use crate::schema::{Facet, FacetOptions, SchemaBuilder, STORED};
+    use crate::{DocAddress, Index, IndexWriter, TantivyDocument};
 
     #[test]
     fn test_facet_only_indexed() {
@@ -71,7 +72,7 @@ mod tests {
         let facet_field = schema_builder.add_facet_field("facet", FacetOptions::default());
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_for_tests().unwrap();
+        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
         index_writer
             .add_document(doc!(facet_field=>Facet::from_text("/a/b").unwrap()))
             .unwrap();
@@ -85,8 +86,10 @@ mod tests {
         let mut facet = Facet::default();
         facet_reader.facet_from_ord(0, &mut facet).unwrap();
         assert_eq!(facet.to_path_string(), "/a/b");
-        let doc = searcher.doc(DocAddress::new(0u32, 0u32)).unwrap();
-        let value = doc.get_first(facet_field).and_then(Value::as_facet);
+        let doc = searcher
+            .doc::<TantivyDocument>(DocAddress::new(0u32, 0u32))
+            .unwrap();
+        let value = doc.get_first(facet_field).and_then(|v| v.as_facet());
         assert_eq!(value, None);
     }
 
@@ -96,7 +99,7 @@ mod tests {
         let facet_field = schema_builder.add_facet_field("facet", FacetOptions::default());
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_for_tests().unwrap();
+        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
         index_writer
             .add_document(doc!(facet_field=>Facet::from_text("/parent/child1").unwrap()))
             .unwrap();
@@ -142,8 +145,8 @@ mod tests {
         let mut facet_ords = Vec::new();
         facet_ords.extend(facet_reader.facet_ords(0u32));
         assert_eq!(&facet_ords, &[0u64]);
-        let doc = searcher.doc(DocAddress::new(0u32, 0u32))?;
-        let value: Option<&Facet> = doc.get_first(facet_field).and_then(Value::as_facet);
+        let doc = searcher.doc::<TantivyDocument>(DocAddress::new(0u32, 0u32))?;
+        let value: Option<&Facet> = doc.get_first(facet_field).and_then(|v| v.as_facet());
         assert_eq!(value, Facet::from_text("/a/b").ok().as_ref());
         Ok(())
     }
@@ -156,7 +159,7 @@ mod tests {
         let index = Index::create_in_ram(schema);
         let mut index_writer = index.writer_for_tests()?;
         index_writer.add_document(doc!(facet_field=>Facet::from_text("/a/b").unwrap()))?;
-        index_writer.add_document(Document::default())?;
+        index_writer.add_document(TantivyDocument::default())?;
         index_writer.commit()?;
         let searcher = index.reader()?.searcher();
         let facet_reader = searcher.segment_reader(0u32).facet_reader("facet").unwrap();
@@ -176,8 +179,8 @@ mod tests {
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
         let mut index_writer = index.writer_for_tests()?;
-        index_writer.add_document(Document::default())?;
-        index_writer.add_document(Document::default())?;
+        index_writer.add_document(TantivyDocument::default())?;
+        index_writer.add_document(TantivyDocument::default())?;
         index_writer.commit()?;
         let searcher = index.reader()?.searcher();
         let facet_reader = searcher.segment_reader(0u32).facet_reader("facet").unwrap();

src/fastfield/mod.rs

@@ -90,12 +90,12 @@ mod tests {
     use crate::directory::{Directory, RamDirectory, WritePtr};
     use crate::merge_policy::NoMergePolicy;
     use crate::schema::{
-        Document, Facet, FacetOptions, Field, JsonObjectOptions, Schema, SchemaBuilder,
+        Facet, FacetOptions, Field, JsonObjectOptions, Schema, SchemaBuilder, TantivyDocument,
         TextOptions, FAST, INDEXED, STORED, STRING, TEXT,
     };
     use crate::time::OffsetDateTime;
     use crate::tokenizer::{LowerCaser, RawTokenizer, TextAnalyzer, TokenizerManager};
-    use crate::{DateOptions, DateTimePrecision, Index, SegmentId, SegmentReader};
+    use crate::{DateOptions, DateTimePrecision, Index, IndexWriter, SegmentId, SegmentReader};
 
     pub static SCHEMA: Lazy<Schema> = Lazy::new(|| {
         let mut schema_builder = Schema::builder();
@@ -271,7 +271,7 @@ mod tests {
             let mut write: WritePtr = directory.open_write(Path::new("test")).unwrap();
             let mut fast_field_writers = FastFieldsWriter::from_schema(&schema).unwrap();
             for i in -100i64..10_000i64 {
-                let mut doc = Document::default();
+                let mut doc = TantivyDocument::default();
                 doc.add_i64(i64_field, i);
                 fast_field_writers.add_document(&doc).unwrap();
             }
@@ -312,7 +312,7 @@ mod tests {
         {
             let mut write: WritePtr = directory.open_write(Path::new("test")).unwrap();
             let mut fast_field_writers = FastFieldsWriter::from_schema(&schema).unwrap();
-            let doc = Document::default();
+            let doc = TantivyDocument::default();
             fast_field_writers.add_document(&doc).unwrap();
             fast_field_writers.serialize(&mut write, None).unwrap();
             write.terminate().unwrap();
@@ -345,7 +345,7 @@ mod tests {
         {
             let mut write: WritePtr = directory.open_write(Path::new("test")).unwrap();
             let mut fast_field_writers = FastFieldsWriter::from_schema(&schema).unwrap();
-            let doc = Document::default();
+            let doc = TantivyDocument::default();
             fast_field_writers.add_document(&doc).unwrap();
             fast_field_writers.serialize(&mut write, None).unwrap();
             write.terminate().unwrap();
@@ -416,7 +416,7 @@ mod tests {
         let date_field = schema_builder.add_date_field("date", FAST);
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_for_tests().unwrap();
+        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
         index_writer.set_merge_policy(Box::new(NoMergePolicy));
         index_writer
             .add_document(doc!(date_field => DateTime::from_utc(OffsetDateTime::now_utc())))
@@ -452,7 +452,7 @@ mod tests {
 
         {
             // first segment
-            let mut index_writer = index.writer_for_tests().unwrap();
+            let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
             index_writer.set_merge_policy(Box::new(NoMergePolicy));
             index_writer
                 .add_document(doc!(
@@ -506,7 +506,7 @@ mod tests {
 
         {
             // second segment
-            let mut index_writer = index.writer_for_tests().unwrap();
+            let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
 
             index_writer
                 .add_document(doc!(
@@ -537,7 +537,7 @@ mod tests {
         // Merging the segments
         {
             let segment_ids = index.searchable_segment_ids().unwrap();
-            let mut index_writer = index.writer_for_tests().unwrap();
+            let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
             index_writer.merge(&segment_ids).wait().unwrap();
             index_writer.wait_merging_threads().unwrap();
         }
@@ -662,7 +662,7 @@ mod tests {
         // Merging the segments
         {
             let segment_ids = index.searchable_segment_ids()?;
-            let mut index_writer = index.writer_for_tests()?;
+            let mut index_writer: IndexWriter = index.writer_for_tests()?;
             index_writer.merge(&segment_ids).wait()?;
             index_writer.wait_merging_threads()?;
         }
@@ -824,7 +824,7 @@ mod tests {
         {
             let mut write: WritePtr = directory.open_write(path).unwrap();
             let mut fast_field_writers = FastFieldsWriter::from_schema(&schema).unwrap();
-            let doc = Document::default();
+            let doc = TantivyDocument::default();
             fast_field_writers.add_document(&doc).unwrap();
             fast_field_writers.serialize(&mut write, None).unwrap();
             write.terminate().unwrap();
@@ -846,7 +846,7 @@ mod tests {
         assert_eq!(col.get_val(0), true);
     }
 
-    fn get_index(docs: &[crate::Document], schema: &Schema) -> crate::Result<RamDirectory> {
+    fn get_index(docs: &[crate::TantivyDocument], schema: &Schema) -> crate::Result<RamDirectory> {
         let directory: RamDirectory = RamDirectory::create();
         {
             let mut write: WritePtr = directory.open_write(Path::new("test")).unwrap();
@@ -888,7 +888,7 @@ mod tests {
         let field = schema_builder.add_date_field("field", date_options);
         let schema = schema_builder.build();
 
-        let docs: Vec<Document> = times.iter().map(|time| doc!(field=>*time)).collect();
+        let docs: Vec<TantivyDocument> = times.iter().map(|time| doc!(field=>*time)).collect();
 
         let directory = get_index(&docs[..], &schema).unwrap();
         let path = Path::new("test");
@@ -962,11 +962,15 @@ mod tests {
         let ip_field = schema_builder.add_u64_field("ip", FAST);
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_for_tests().unwrap();
+        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
         let ip_addr = Ipv6Addr::new(1, 2, 3, 4, 5, 1, 2, 3);
-        index_writer.add_document(Document::default()).unwrap();
+        index_writer
+            .add_document(TantivyDocument::default())
+            .unwrap();
         index_writer.add_document(doc!(ip_field=>ip_addr)).unwrap();
-        index_writer.add_document(Document::default()).unwrap();
+        index_writer
+            .add_document(TantivyDocument::default())
+            .unwrap();
         index_writer.commit().unwrap();
         let searcher = index.reader().unwrap().searcher();
         let fastfields = searcher.segment_reader(0u32).fast_fields();
@@ -1086,7 +1090,7 @@ mod tests {
         let json = schema_builder.add_json_field("json", json_option);
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_for_tests().unwrap();
+        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
         index_writer
             .add_document(doc!(json => json!({"attr.age": 32})))
             .unwrap();
@@ -1112,7 +1116,7 @@ mod tests {
         let json = schema_builder.add_json_field("json", json_option);
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_for_tests().unwrap();
+        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
         index_writer
             .add_document(doc!(json => json!({"age": 32})))
             .unwrap();
@@ -1139,7 +1143,7 @@ mod tests {
         let json = schema_builder.add_json_field("json", json_option);
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_for_tests().unwrap();
+        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
         index_writer
             .add_document(doc!(json => json!({"attr.age": 32})))
             .unwrap();
@@ -1162,7 +1166,7 @@ mod tests {
         let field_with_dot = schema_builder.add_i64_field("field.with.dot", FAST);
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_for_tests().unwrap();
+        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
         index_writer
             .add_document(doc!(field_with_dot => 32i64))
             .unwrap();
@@ -1184,7 +1188,7 @@ mod tests {
         let shadowing_json_field = schema_builder.add_json_field("jsonfield.attr", FAST);
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_for_tests().unwrap();
+        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
         index_writer
             .add_document(doc!(json_field=> json!({"attr": {"age": 32}}), shadowing_json_field=>json!({"age": 33})))
             .unwrap();
@@ -1215,7 +1219,7 @@ mod tests {
 
         let mut index = Index::create_in_ram(schema);
         index.set_fast_field_tokenizers(ff_tokenizer_manager);
-        let mut index_writer = index.writer_for_tests().unwrap();
+        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
         index_writer
             .add_document(doc!(text_field => "Test1 test2"))
             .unwrap();
@@ -1244,7 +1248,7 @@ mod tests {
         let log_field = schema_builder.add_text_field("log_level", text_fieldtype);
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_for_tests().unwrap();
+        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
         index_writer
             .add_document(doc!(log_field => "info"))
             .unwrap();
@@ -1277,7 +1281,7 @@ mod tests {
         let shadowing_json_field = schema_builder.add_json_field("jsonfield.attr", json_option);
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_for_tests().unwrap();
+        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
         index_writer
             .add_document(doc!(json_field=> json!({"attr.age": 32}), shadowing_json_field=>json!({"age": 33})))
             .unwrap();

src/fastfield/readers.rs

@@ -234,6 +234,22 @@ impl FastFieldReaders {
         Ok(dynamic_column_handle_opt)
     }
 
+    /// Returning all `dynamic_column_handle`.
+    pub fn dynamic_column_handles(
+        &self,
+        field_name: &str,
+    ) -> crate::Result<Vec<DynamicColumnHandle>> {
+        let Some(resolved_field_name) = self.resolve_field(field_name)? else {
+            return Ok(Vec::new());
+        };
+        let dynamic_column_handles = self
+            .columnar
+            .read_columns(&resolved_field_name)?
+            .into_iter()
+            .collect();
+        Ok(dynamic_column_handles)
+    }
+
     #[doc(hidden)]
     pub async fn list_dynamic_column_handles(
         &self,
@@ -338,8 +354,10 @@ impl FastFieldReaders {
 
 #[cfg(test)]
 mod tests {
+    use columnar::ColumnType;
+
     use crate::schema::{JsonObjectOptions, Schema, FAST};
-    use crate::{Document, Index};
+    use crate::{Index, IndexWriter, TantivyDocument};
 
     #[test]
     fn test_fast_field_reader_resolve_with_dynamic_internal() {
@@ -355,8 +373,10 @@ mod tests {
         let dynamic_field = schema_builder.add_json_field("_dyna", FAST);
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_for_tests().unwrap();
-        index_writer.add_document(Document::default()).unwrap();
+        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
+        index_writer
+            .add_document(TantivyDocument::default())
+            .unwrap();
         index_writer.commit().unwrap();
         let reader = index.reader().unwrap();
         let searcher = reader.searcher();
@@ -417,4 +437,45 @@ mod tests {
             Some("_dyna\u{1}notinschema\u{1}attr\u{1}color".to_string())
         );
     }
+
+    #[test]
+    fn test_fast_field_reader_dynamic_column_handles() {
+        let mut schema_builder = Schema::builder();
+        let id = schema_builder.add_u64_field("id", FAST);
+        let json = schema_builder.add_json_field("json", FAST);
+        let schema = schema_builder.build();
+        let index = Index::create_in_ram(schema);
+        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
+        index_writer
+            .add_document(doc!(id=> 1u64, json => json!({"foo": 42})))
+            .unwrap();
+        index_writer
+            .add_document(doc!(id=> 2u64, json => json!({"foo": true})))
+            .unwrap();
+        index_writer
+            .add_document(doc!(id=> 3u64, json => json!({"foo": "bar"})))
+            .unwrap();
+        index_writer.commit().unwrap();
+        let reader = index.reader().unwrap();
+        let searcher = reader.searcher();
+        let reader = searcher.segment_reader(0u32);
+        let fast_fields = reader.fast_fields();
+        let id_columns = fast_fields.dynamic_column_handles("id").unwrap();
+        assert_eq!(id_columns.len(), 1);
+        assert_eq!(id_columns.first().unwrap().column_type(), ColumnType::U64);
+
+        let foo_columns = fast_fields.dynamic_column_handles("json.foo").unwrap();
+        assert_eq!(foo_columns.len(), 3);
+        assert!(foo_columns
+            .iter()
+            .any(|column| column.column_type() == ColumnType::I64));
+        assert!(foo_columns
+            .iter()
+            .any(|column| column.column_type() == ColumnType::Bool));
+        assert!(foo_columns
+            .iter()
+            .any(|column| column.column_type() == ColumnType::Str));
+
+        println!("*** {:?}", fast_fields.columnar().list_columns());
+    }
 }

src/fastfield/writer.rs

@@ -5,8 +5,9 @@ use common::replace_in_place;
 use tokenizer_api::Token;
 
 use crate::indexer::doc_id_mapping::DocIdMapping;
+use crate::schema::document::{Document, ReferenceValue, Value};
 use crate::schema::term::{JSON_PATH_SEGMENT_SEP, JSON_PATH_SEGMENT_SEP_STR};
-use crate::schema::{value_type_to_column_type, Document, FieldType, Schema, Type, Value};
+use crate::schema::{value_type_to_column_type, Field, FieldType, Schema, Type};
 use crate::tokenizer::{TextAnalyzer, TokenizerManager};
 use crate::{DateTimePrecision, DocId, TantivyError};
 
@@ -117,114 +118,115 @@ impl FastFieldsWriter {
     }
 
     /// Indexes all of the fastfields of a new document.
-    pub fn add_document(&mut self, doc: &Document) -> crate::Result<()> {
+    pub fn add_document<D: Document>(&mut self, doc: &D) -> crate::Result<()> {
         let doc_id = self.num_docs;
-        for field_value in doc.field_values() {
-            if let Some(field_name) =
-                &self.fast_field_names[field_value.field().field_id() as usize]
-            {
-                match &field_value.value {
-                    Value::U64(u64_val) => {
-                        self.columnar_writer.record_numerical(
-                            doc_id,
-                            field_name.as_str(),
-                            NumericalValue::from(*u64_val),
-                        );
-                    }
-                    Value::I64(i64_val) => {
-                        self.columnar_writer.record_numerical(
-                            doc_id,
-                            field_name.as_str(),
-                            NumericalValue::from(*i64_val),
-                        );
-                    }
-                    Value::F64(f64_val) => {
-                        self.columnar_writer.record_numerical(
-                            doc_id,
-                            field_name.as_str(),
-                            NumericalValue::from(*f64_val),
-                        );
-                    }
-                    Value::Str(text_val) => {
-                        if let Some(tokenizer) =
-                            &mut self.per_field_tokenizer[field_value.field().field_id() as usize]
-                        {
-                            let mut token_stream = tokenizer.token_stream(text_val);
-                            token_stream.process(&mut |token: &Token| {
-                                self.columnar_writer.record_str(
-                                    doc_id,
-                                    field_name.as_str(),
-                                    &token.text,
-                                );
-                            })
-                        } else {
-                            self.columnar_writer
-                                .record_str(doc_id, field_name.as_str(), text_val);
-                        }
-                    }
-                    Value::Bytes(bytes_val) => {
-                        self.columnar_writer
-                            .record_bytes(doc_id, field_name.as_str(), bytes_val);
-                    }
-                    Value::PreTokStr(pre_tok) => {
-                        for token in &pre_tok.tokens {
-                            self.columnar_writer.record_str(
-                                doc_id,
-                                field_name.as_str(),
-                                &token.text,
-                            );
-                        }
-                    }
-                    Value::Bool(bool_val) => {
-                        self.columnar_writer
-                            .record_bool(doc_id, field_name.as_str(), *bool_val);
-                    }
-                    Value::Date(datetime) => {
-                        let date_precision =
-                            self.date_precisions[field_value.field().field_id() as usize];
-                        let truncated_datetime = datetime.truncate(date_precision);
-                        self.columnar_writer.record_datetime(
-                            doc_id,
-                            field_name.as_str(),
-                            truncated_datetime,
-                        );
-                    }
-                    Value::Facet(facet) => {
-                        self.columnar_writer.record_str(
-                            doc_id,
-                            field_name.as_str(),
-                            facet.encoded_str(),
-                        );
-                    }
-                    Value::JsonObject(json_obj) => {
-                        let expand_dots = self.expand_dots[field_value.field().field_id() as usize];
-                        self.json_path_buffer.clear();
-                        self.json_path_buffer.push_str(field_name);
+        for (field, value) in doc.iter_fields_and_values() {
+            let value_access = value as D::Value<'_>;
 
-                        let text_analyzer =
-                            &mut self.per_field_tokenizer[field_value.field().field_id() as usize];
-
-                        record_json_obj_to_columnar_writer(
-                            doc_id,
-                            json_obj,
-                            expand_dots,
-                            JSON_DEPTH_LIMIT,
-                            &mut self.json_path_buffer,
-                            &mut self.columnar_writer,
-                            text_analyzer,
-                        );
-                    }
-                    Value::IpAddr(ip_addr) => {
-                        self.columnar_writer
-                            .record_ip_addr(doc_id, field_name.as_str(), *ip_addr);
-                    }
-                }
-            }
+            self.add_doc_value(doc_id, field, value_access)?;
         }
         self.num_docs += 1;
         Ok(())
     }
 
+    fn add_doc_value<'a, V: Value<'a>>(
+        &mut self,
+        doc_id: DocId,
+        field: Field,
+        value: V,
+    ) -> crate::Result<()> {
+        let field_name = match &self.fast_field_names[field.field_id() as usize] {
+            None => return Ok(()),
+            Some(name) => name,
+        };
+
+        match value.as_value() {
+            ReferenceValue::Null => {}
+            ReferenceValue::Str(val) => {
+                if let Some(tokenizer) = &mut self.per_field_tokenizer[field.field_id() as usize] {
+                    let mut token_stream = tokenizer.token_stream(val);
+                    token_stream.process(&mut |token: &Token| {
+                        self.columnar_writer
+                            .record_str(doc_id, field_name, &token.text);
+                    })
+                } else {
+                    self.columnar_writer.record_str(doc_id, field_name, val);
+                }
+            }
+            ReferenceValue::U64(val) => {
+                self.columnar_writer.record_numerical(
+                    doc_id,
+                    field_name,
+                    NumericalValue::from(val),
+                );
+            }
+            ReferenceValue::I64(val) => {
+                self.columnar_writer.record_numerical(
+                    doc_id,
+                    field_name,
+                    NumericalValue::from(val),
+                );
+            }
+            ReferenceValue::F64(val) => {
+                self.columnar_writer.record_numerical(
+                    doc_id,
+                    field_name,
+                    NumericalValue::from(val),
+                );
+            }
+            ReferenceValue::Date(val) => {
+                let date_precision = self.date_precisions[field.field_id() as usize];
+                let truncated_datetime = val.truncate(date_precision);
+                self.columnar_writer
+                    .record_datetime(doc_id, field_name, truncated_datetime);
+            }
+            ReferenceValue::Facet(val) => {
+                self.columnar_writer
+                    .record_str(doc_id, field_name, val.encoded_str());
+            }
+            ReferenceValue::Bytes(val) => {
+                self.columnar_writer.record_bytes(doc_id, field_name, val);
+            }
+            ReferenceValue::IpAddr(val) => {
+                self.columnar_writer.record_ip_addr(doc_id, field_name, val);
+            }
+            ReferenceValue::Bool(val) => {
+                self.columnar_writer.record_bool(doc_id, field_name, val);
+            }
+            ReferenceValue::PreTokStr(val) => {
+                for token in &val.tokens {
+                    self.columnar_writer
+                        .record_str(doc_id, field_name, &token.text);
+                }
+            }
+            ReferenceValue::Array(val) => {
+                // TODO: Check this is the correct behaviour we want.
+                for value in val {
+                    self.add_doc_value(doc_id, field, value)?;
+                }
+            }
+            ReferenceValue::Object(val) => {
+                let expand_dots = self.expand_dots[field.field_id() as usize];
+                self.json_path_buffer.clear();
+                self.json_path_buffer.push_str(field_name);
+
+                let text_analyzer = &mut self.per_field_tokenizer[field.field_id() as usize];
+
+                record_json_obj_to_columnar_writer::<V>(
+                    doc_id,
+                    val,
+                    expand_dots,
+                    JSON_DEPTH_LIMIT,
+                    &mut self.json_path_buffer,
+                    &mut self.columnar_writer,
+                    text_analyzer,
+                );
+            }
+        }
+
+        Ok(())
+    }
+
     /// Serializes all of the `FastFieldWriter`s by pushing them in
     /// order to the fast field serializer.
     pub fn serialize(
@@ -241,31 +243,16 @@ impl FastFieldsWriter {
     }
 }
 
-#[inline]
-fn columnar_numerical_value(json_number: &serde_json::Number) -> Option<NumericalValue> {
-    if let Some(num_i64) = json_number.as_i64() {
-        return Some(num_i64.into());
-    }
-    if let Some(num_u64) = json_number.as_u64() {
-        return Some(num_u64.into());
-    }
-    if let Some(num_f64) = json_number.as_f64() {
-        return Some(num_f64.into());
-    }
-    // This can happen with arbitrary precision.... but we do not handle it.
-    None
-}
-
-fn record_json_obj_to_columnar_writer(
+fn record_json_obj_to_columnar_writer<'a, V: Value<'a>>(
     doc: DocId,
-    json_obj: &serde_json::Map<String, serde_json::Value>,
+    json_visitor: V::ObjectIter,
     expand_dots: bool,
     remaining_depth_limit: usize,
     json_path_buffer: &mut String,
     columnar_writer: &mut columnar::ColumnarWriter,
     tokenizer: &mut Option<TextAnalyzer>,
 ) {
-    for (key, child) in json_obj {
+    for (key, child) in json_visitor {
         let len_path = json_path_buffer.len();
         if !json_path_buffer.is_empty() {
             json_path_buffer.push_str(JSON_PATH_SEGMENT_SEP_STR);
@@ -295,9 +282,9 @@ fn record_json_obj_to_columnar_writer(
     }
 }
 
-fn record_json_value_to_columnar_writer(
+fn record_json_value_to_columnar_writer<'a, V: Value<'a>>(
     doc: DocId,
-    json_val: &serde_json::Value,
+    json_val: V,
     expand_dots: bool,
     mut remaining_depth_limit: usize,
     json_path_writer: &mut String,
@@ -308,30 +295,62 @@ fn record_json_value_to_columnar_writer(
         return;
     }
     remaining_depth_limit -= 1;
-    match json_val {
-        serde_json::Value::Null => {
-            // TODO handle null
-        }
-        serde_json::Value::Bool(bool_val) => {
-            columnar_writer.record_bool(doc, json_path_writer, *bool_val);
-        }
-        serde_json::Value::Number(json_number) => {
-            if let Some(numerical_value) = columnar_numerical_value(json_number) {
-                columnar_writer.record_numerical(doc, json_path_writer.as_str(), numerical_value);
-            }
-        }
-        serde_json::Value::String(text) => {
+
+    match json_val.as_value() {
+        ReferenceValue::Null => {} // TODO: Handle null
+        ReferenceValue::Str(val) => {
             if let Some(text_analyzer) = tokenizer.as_mut() {
-                let mut token_stream = text_analyzer.token_stream(text);
+                let mut token_stream = text_analyzer.token_stream(val);
                 token_stream.process(&mut |token| {
                     columnar_writer.record_str(doc, json_path_writer.as_str(), &token.text);
                 })
             } else {
-                columnar_writer.record_str(doc, json_path_writer.as_str(), text);
+                columnar_writer.record_str(doc, json_path_writer.as_str(), val);
             }
         }
-        serde_json::Value::Array(arr) => {
-            for el in arr {
+        ReferenceValue::U64(val) => {
+            columnar_writer.record_numerical(
+                doc,
+                json_path_writer.as_str(),
+                NumericalValue::from(val),
+            );
+        }
+        ReferenceValue::I64(val) => {
+            columnar_writer.record_numerical(
+                doc,
+                json_path_writer.as_str(),
+                NumericalValue::from(val),
+            );
+        }
+        ReferenceValue::F64(val) => {
+            columnar_writer.record_numerical(
+                doc,
+                json_path_writer.as_str(),
+                NumericalValue::from(val),
+            );
+        }
+        ReferenceValue::Bool(val) => {
+            columnar_writer.record_bool(doc, json_path_writer, val);
+        }
+        ReferenceValue::Date(val) => {
+            columnar_writer.record_datetime(doc, json_path_writer.as_str(), val);
+        }
+        ReferenceValue::Facet(_) => {
+            unimplemented!("Facet support in dynamic fields is not yet implemented")
+        }
+        ReferenceValue::Bytes(_) => {
+            // TODO: This can be re added once it is added to the JSON Utils section as well.
+            // columnar_writer.record_bytes(doc, json_path_writer.as_str(), val);
+            unimplemented!("Bytes support in dynamic fields is not yet implemented")
+        }
+        ReferenceValue::IpAddr(_) => {
+            unimplemented!("IP address support in dynamic fields is not yet implemented")
+        }
+        ReferenceValue::PreTokStr(_) => {
+            unimplemented!("Pre-tokenized string support in dynamic fields is not yet implemented")
+        }
+        ReferenceValue::Array(elements) => {
+            for el in elements {
                 record_json_value_to_columnar_writer(
                     doc,
                     el,
@@ -343,10 +362,10 @@ fn record_json_value_to_columnar_writer(
                 );
             }
         }
-        serde_json::Value::Object(json_obj) => {
-            record_json_obj_to_columnar_writer(
+        ReferenceValue::Object(object) => {
+            record_json_obj_to_columnar_writer::<V>(
                 doc,
-                json_obj,
+                object,
                 expand_dots,
                 remaining_depth_limit,
                 json_path_writer,

src/functional_test.rs

@@ -2,8 +2,9 @@ use std::collections::HashSet;
 
 use rand::{thread_rng, Rng};
 
+use crate::indexer::index_writer::MEMORY_BUDGET_NUM_BYTES_MIN;
 use crate::schema::*;
-use crate::{doc, schema, Index, IndexSettings, IndexSortByField, Order, Searcher};
+use crate::{doc, schema, Index, IndexSettings, IndexSortByField, IndexWriter, Order, Searcher};
 
 fn check_index_content(searcher: &Searcher, vals: &[u64]) -> crate::Result<()> {
     assert!(searcher.segment_readers().len() < 20);
@@ -11,7 +12,7 @@ fn check_index_content(searcher: &Searcher, vals: &[u64]) -> crate::Result<()> {
     for segment_reader in searcher.segment_readers() {
         let store_reader = segment_reader.get_store_reader(1)?;
         for doc_id in 0..segment_reader.max_doc() {
-            let _doc = store_reader.get(doc_id)?;
+            let _doc: TantivyDocument = store_reader.get(doc_id)?;
         }
     }
     Ok(())
@@ -30,7 +31,8 @@ fn test_functional_store() -> crate::Result<()> {
 
     let mut rng = thread_rng();
 
-    let mut index_writer = index.writer_with_num_threads(3, 12_000_000)?;
+    let mut index_writer: IndexWriter =
+        index.writer_with_num_threads(3, MEMORY_BUDGET_NUM_BYTES_MIN)?;
 
     let mut doc_set: Vec<u64> = Vec::new();
 
@@ -90,7 +92,7 @@ fn test_functional_indexing_sorted() -> crate::Result<()> {
 
     let mut rng = thread_rng();
 
-    let mut index_writer = index.writer_with_num_threads(3, 120_000_000)?;
+    let mut index_writer: IndexWriter = index.writer_with_num_threads(3, 120_000_000)?;
 
     let mut committed_docs: HashSet<u64> = HashSet::new();
     let mut uncommitted_docs: HashSet<u64> = HashSet::new();
@@ -113,7 +115,7 @@ fn test_functional_indexing_sorted() -> crate::Result<()> {
             index_writer.delete_term(doc_id_term);
         } else {
             uncommitted_docs.insert(random_val);
-            let mut doc = Document::new();
+            let mut doc = TantivyDocument::new();
             doc.add_u64(id_field, random_val);
             for i in 1u64..10u64 {
                 doc.add_u64(multiples_field, random_val * i);
@@ -165,7 +167,7 @@ fn test_functional_indexing_unsorted() -> crate::Result<()> {
 
     let mut rng = thread_rng();
 
-    let mut index_writer = index.writer_with_num_threads(3, 120_000_000)?;
+    let mut index_writer: IndexWriter = index.writer_with_num_threads(3, 120_000_000)?;
 
     let mut committed_docs: HashSet<u64> = HashSet::new();
     let mut uncommitted_docs: HashSet<u64> = HashSet::new();
@@ -188,7 +190,7 @@ fn test_functional_indexing_unsorted() -> crate::Result<()> {
             index_writer.delete_term(doc_id_term);
         } else {
             uncommitted_docs.insert(random_val);
-            let mut doc = Document::new();
+            let mut doc = TantivyDocument::new();
             doc.add_u64(id_field, random_val);
             for i in 1u64..10u64 {
                 doc.add_u64(multiples_field, random_val * i);

src/indexer/doc_id_mapping.rs

@@ -158,6 +158,7 @@ mod tests_indexsorting {
     use crate::indexer::doc_id_mapping::DocIdMapping;
     use crate::indexer::NoMergePolicy;
     use crate::query::QueryParser;
+    use crate::schema::document::Value;
     use crate::schema::{Schema, *};
     use crate::{DocAddress, Index, IndexSettings, IndexSortByField, Order};
 
@@ -308,16 +309,16 @@ mod tests_indexsorting {
         {
             assert_eq!(
                 searcher
-                    .doc(DocAddress::new(0, 0))?
+                    .doc::<TantivyDocument>(DocAddress::new(0, 0))?
                     .get_first(my_string_field),
                 None
             );
             assert_eq!(
                 searcher
-                    .doc(DocAddress::new(0, 3))?
+                    .doc::<TantivyDocument>(DocAddress::new(0, 3))?
                     .get_first(my_string_field)
                     .unwrap()
-                    .as_text(),
+                    .as_str(),
                 Some("blublub")
             );
         }
@@ -337,13 +338,13 @@ mod tests_indexsorting {
         {
             assert_eq!(
                 searcher
-                    .doc(DocAddress::new(0, 0))?
+                    .doc::<TantivyDocument>(DocAddress::new(0, 0))?
                     .get_first(my_string_field)
                     .unwrap()
-                    .as_text(),
+                    .as_str(),
                 Some("blublub")
             );
-            let doc = searcher.doc(DocAddress::new(0, 4))?;
+            let doc = searcher.doc::<TantivyDocument>(DocAddress::new(0, 4))?;
             assert_eq!(doc.get_first(my_string_field), None);
         }
         // sort by field desc
@@ -360,9 +361,9 @@ mod tests_indexsorting {
         let my_string_field = index.schema().get_field("string_field").unwrap();
         let searcher = index.reader()?.searcher();
         {
-            let doc = searcher.doc(DocAddress::new(0, 4))?;
+            let doc = searcher.doc::<TantivyDocument>(DocAddress::new(0, 4))?;
             assert_eq!(
-                doc.get_first(my_string_field).unwrap().as_text(),
+                doc.get_first(my_string_field).unwrap().as_str(),
                 Some("blublub")
             );
         }

src/indexer/index_writer.rs

@@ -20,16 +20,17 @@ use crate::indexer::operation::DeleteOperation;
 use crate::indexer::stamper::Stamper;
 use crate::indexer::{MergePolicy, SegmentEntry, SegmentWriter};
 use crate::query::{EnableScoring, Query, TermQuery};
-use crate::schema::{Document, IndexRecordOption, Term};
+use crate::schema::document::Document;
+use crate::schema::{IndexRecordOption, TantivyDocument, Term};
 use crate::{FutureResult, Opstamp};
 
 // Size of the margin for the `memory_arena`. A segment is closed when the remaining memory
 // in the `memory_arena` goes below MARGIN_IN_BYTES.
 pub const MARGIN_IN_BYTES: usize = 1_000_000;
 
-// We impose the memory per thread to be at least 3 MB.
-pub const MEMORY_ARENA_NUM_BYTES_MIN: usize = ((MARGIN_IN_BYTES as u32) * 3u32) as usize;
-pub const MEMORY_ARENA_NUM_BYTES_MAX: usize = u32::MAX as usize - MARGIN_IN_BYTES;
+// We impose the memory per thread to be at least 15 MB, as the baseline consumption is 12MB.
+pub const MEMORY_BUDGET_NUM_BYTES_MIN: usize = ((MARGIN_IN_BYTES as u32) * 15u32) as usize;
+pub const MEMORY_BUDGET_NUM_BYTES_MAX: usize = u32::MAX as usize - MARGIN_IN_BYTES;
 
 // We impose the number of index writer threads to be at most this.
 pub const MAX_NUM_THREAD: usize = 8;
@@ -50,19 +51,20 @@ fn error_in_index_worker_thread(context: &str) -> TantivyError {
 /// indexing queue.
 /// Each indexing thread builds its own independent [`Segment`], via
 /// a `SegmentWriter` object.
-pub struct IndexWriter {
+pub struct IndexWriter<D: Document = TantivyDocument> {
     // the lock is just used to bind the
     // lifetime of the lock with that of the IndexWriter.
     _directory_lock: Option<DirectoryLock>,
 
     index: Index,
 
-    memory_arena_in_bytes_per_thread: usize,
+    // The memory budget per thread, after which a commit is triggered.
+    memory_budget_in_bytes_per_thread: usize,
 
     workers_join_handle: Vec<JoinHandle<crate::Result<()>>>,
 
-    index_writer_status: IndexWriterStatus,
-    operation_sender: AddBatchSender,
+    index_writer_status: IndexWriterStatus<D>,
+    operation_sender: AddBatchSender<D>,
 
     segment_updater: SegmentUpdater,
 
@@ -163,10 +165,10 @@ pub(crate) fn advance_deletes(
     Ok(())
 }
 
-fn index_documents(
+fn index_documents<D: Document>(
     memory_budget: usize,
     segment: Segment,
-    grouped_document_iterator: &mut dyn Iterator<Item = AddBatch>,
+    grouped_document_iterator: &mut dyn Iterator<Item = AddBatch<D>>,
     segment_updater: &SegmentUpdater,
     mut delete_cursor: DeleteCursor,
 ) -> crate::Result<()> {
@@ -246,7 +248,7 @@ fn apply_deletes(
     })
 }
 
-impl IndexWriter {
+impl<D: Document> IndexWriter<D> {
     /// Create a new index writer. Attempts to acquire a lockfile.
     ///
     /// The lockfile should be deleted on drop, but it is possible
@@ -264,23 +266,23 @@ impl IndexWriter {
     pub(crate) fn new(
         index: &Index,
         num_threads: usize,
-        memory_arena_in_bytes_per_thread: usize,
+        memory_budget_in_bytes_per_thread: usize,
         directory_lock: DirectoryLock,
-    ) -> crate::Result<IndexWriter> {
-        if memory_arena_in_bytes_per_thread < MEMORY_ARENA_NUM_BYTES_MIN {
+    ) -> crate::Result<Self> {
+        if memory_budget_in_bytes_per_thread < MEMORY_BUDGET_NUM_BYTES_MIN {
             let err_msg = format!(
                 "The memory arena in bytes per thread needs to be at least \
-                 {MEMORY_ARENA_NUM_BYTES_MIN}."
+                 {MEMORY_BUDGET_NUM_BYTES_MIN}."
             );
             return Err(TantivyError::InvalidArgument(err_msg));
         }
-        if memory_arena_in_bytes_per_thread >= MEMORY_ARENA_NUM_BYTES_MAX {
+        if memory_budget_in_bytes_per_thread >= MEMORY_BUDGET_NUM_BYTES_MAX {
             let err_msg = format!(
-                "The memory arena in bytes per thread cannot exceed {MEMORY_ARENA_NUM_BYTES_MAX}"
+                "The memory arena in bytes per thread cannot exceed {MEMORY_BUDGET_NUM_BYTES_MAX}"
             );
             return Err(TantivyError::InvalidArgument(err_msg));
         }
-        let (document_sender, document_receiver): (AddBatchSender, AddBatchReceiver) =
+        let (document_sender, document_receiver) =
             crossbeam_channel::bounded(PIPELINE_MAX_SIZE_IN_DOCS);
 
         let delete_queue = DeleteQueue::new();
@@ -292,10 +294,10 @@ impl IndexWriter {
         let segment_updater =
             SegmentUpdater::create(index.clone(), stamper.clone(), &delete_queue.cursor())?;
 
-        let mut index_writer = IndexWriter {
+        let mut index_writer = Self {
             _directory_lock: Some(directory_lock),
 
-            memory_arena_in_bytes_per_thread,
+            memory_budget_in_bytes_per_thread,
             index: index.clone(),
             index_writer_status: IndexWriterStatus::from(document_receiver),
             operation_sender: document_sender,
@@ -374,7 +376,7 @@ impl IndexWriter {
         self.index.new_segment()
     }
 
-    fn operation_receiver(&self) -> crate::Result<AddBatchReceiver> {
+    fn operation_receiver(&self) -> crate::Result<AddBatchReceiver<D>> {
         self.index_writer_status
             .operation_receiver()
             .ok_or_else(|| {
@@ -396,7 +398,7 @@ impl IndexWriter {
 
         let mut delete_cursor = self.delete_queue.cursor();
 
-        let mem_budget = self.memory_arena_in_bytes_per_thread;
+        let mem_budget = self.memory_budget_in_bytes_per_thread;
         let index = self.index.clone();
         let join_handle: JoinHandle<crate::Result<()>> = thread::Builder::new()
             .name(format!("thrd-tantivy-index{}", self.worker_id))
@@ -524,7 +526,7 @@ impl IndexWriter {
     ///
     /// Returns the former segment_ready channel.
     fn recreate_document_channel(&mut self) {
-        let (document_sender, document_receiver): (AddBatchSender, AddBatchReceiver) =
+        let (document_sender, document_receiver) =
             crossbeam_channel::bounded(PIPELINE_MAX_SIZE_IN_DOCS);
         self.operation_sender = document_sender;
         self.index_writer_status = IndexWriterStatus::from(document_receiver);
@@ -551,10 +553,10 @@ impl IndexWriter {
             .take()
             .expect("The IndexWriter does not have any lock. This is a bug, please report.");
 
-        let new_index_writer: IndexWriter = IndexWriter::new(
+        let new_index_writer = IndexWriter::new(
             &self.index,
             self.num_threads,
-            self.memory_arena_in_bytes_per_thread,
+            self.memory_budget_in_bytes_per_thread,
             directory_lock,
         )?;
 
@@ -597,7 +599,7 @@ impl IndexWriter {
     /// It is also possible to add a payload to the `commit`
     /// using this API.
     /// See [`PreparedCommit::set_payload()`].
-    pub fn prepare_commit(&mut self) -> crate::Result<PreparedCommit> {
+    pub fn prepare_commit(&mut self) -> crate::Result<PreparedCommit<D>> {
         // Here, because we join all of the worker threads,
         // all of the segment update for this commit have been
         // sent.
@@ -706,7 +708,7 @@ impl IndexWriter {
     /// The opstamp is an increasing `u64` that can
     /// be used by the client to align commits with its own
     /// document queue.
-    pub fn add_document(&self, document: Document) -> crate::Result<Opstamp> {
+    pub fn add_document(&self, document: D) -> crate::Result<Opstamp> {
         let opstamp = self.stamper.stamp();
         self.send_add_documents_batch(smallvec![AddOperation { opstamp, document }])?;
         Ok(opstamp)
@@ -743,7 +745,7 @@ impl IndexWriter {
     /// visible to readers only after calling `commit()`.
     pub fn run<I>(&self, user_operations: I) -> crate::Result<Opstamp>
     where
-        I: IntoIterator<Item = UserOperation>,
+        I: IntoIterator<Item = UserOperation<D>>,
         I::IntoIter: ExactSizeIterator,
     {
         let user_operations_it = user_operations.into_iter();
@@ -777,7 +779,7 @@ impl IndexWriter {
         Ok(batch_opstamp)
     }
 
-    fn send_add_documents_batch(&self, add_ops: AddBatch) -> crate::Result<()> {
+    fn send_add_documents_batch(&self, add_ops: AddBatch<D>) -> crate::Result<()> {
         if self.index_writer_status.is_alive() && self.operation_sender.send(add_ops).is_ok() {
             Ok(())
         } else {
@@ -786,7 +788,7 @@ impl IndexWriter {
     }
 }
 
-impl Drop for IndexWriter {
+impl<D: Document> Drop for IndexWriter<D> {
     fn drop(&mut self) {
         self.segment_updater.kill();
         self.drop_sender();
@@ -810,15 +812,18 @@ mod tests {
     use crate::collector::TopDocs;
     use crate::directory::error::LockError;
     use crate::error::*;
+    use crate::indexer::index_writer::MEMORY_BUDGET_NUM_BYTES_MIN;
     use crate::indexer::NoMergePolicy;
     use crate::query::{BooleanQuery, Occur, Query, QueryParser, TermQuery};
+    use crate::schema::document::Value;
     use crate::schema::{
         self, Facet, FacetOptions, IndexRecordOption, IpAddrOptions, NumericOptions, Schema,
         TextFieldIndexing, TextOptions, FAST, INDEXED, STORED, STRING, TEXT,
     };
     use crate::store::DOCSTORE_CACHE_CAPACITY;
     use crate::{
-        DateTime, DocAddress, Index, IndexSettings, IndexSortByField, Order, ReloadPolicy, Term,
+        DateTime, DocAddress, Index, IndexSettings, IndexSortByField, IndexWriter, Order,
+        ReloadPolicy, TantivyDocument, Term,
     };
 
     const LOREM: &str = "Doc Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do \
@@ -850,7 +855,7 @@ mod tests {
         let text_field = schema_builder.add_text_field("text", TEXT);
         let index = Index::create_in_ram(schema_builder.build());
 
-        let mut index_writer = index.writer_for_tests().unwrap();
+        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
         index_writer
             .add_document(doc!(text_field => "hello1"))
             .unwrap();
@@ -903,7 +908,7 @@ mod tests {
             .reload_policy(ReloadPolicy::Manual)
             .try_into()
             .unwrap();
-        let mut index_writer = index.writer_for_tests().unwrap();
+        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
         let a_term = Term::from_field_text(text_field, "a");
         let b_term = Term::from_field_text(text_field, "b");
         let operations = vec![
@@ -941,7 +946,7 @@ mod tests {
     fn test_empty_operations_group() {
         let schema_builder = schema::Schema::builder();
         let index = Index::create_in_ram(schema_builder.build());
-        let index_writer = index.writer(3_000_000).unwrap();
+        let index_writer: IndexWriter = index.writer_for_tests().unwrap();
         let operations1 = vec![];
         let batch_opstamp1 = index_writer.run(operations1).unwrap();
         assert_eq!(batch_opstamp1, 0u64);
@@ -954,8 +959,8 @@ mod tests {
     fn test_lockfile_stops_duplicates() {
         let schema_builder = schema::Schema::builder();
         let index = Index::create_in_ram(schema_builder.build());
-        let _index_writer = index.writer(3_000_000).unwrap();
-        match index.writer(3_000_000) {
+        let _index_writer: IndexWriter = index.writer_for_tests().unwrap();
+        match index.writer_for_tests::<TantivyDocument>() {
             Err(TantivyError::LockFailure(LockError::LockBusy, _)) => {}
             _ => panic!("Expected a `LockFailure` error"),
         }
@@ -965,8 +970,8 @@ mod tests {
     fn test_lockfile_already_exists_error_msg() {
         let schema_builder = schema::Schema::builder();
         let index = Index::create_in_ram(schema_builder.build());
-        let _index_writer = index.writer_for_tests().unwrap();
-        match index.writer_for_tests() {
+        let _index_writer: IndexWriter = index.writer_for_tests().unwrap();
+        match index.writer_for_tests::<TantivyDocument>() {
             Err(err) => {
                 let err_msg = err.to_string();
                 assert!(err_msg.contains("already an `IndexWriter`"));
@@ -979,7 +984,7 @@ mod tests {
     fn test_set_merge_policy() {
         let schema_builder = schema::Schema::builder();
         let index = Index::create_in_ram(schema_builder.build());
-        let index_writer = index.writer(3_000_000).unwrap();
+        let index_writer: IndexWriter = index.writer_for_tests().unwrap();
         assert_eq!(
             format!("{:?}", index_writer.get_merge_policy()),
             "LogMergePolicy { min_num_segments: 8, max_docs_before_merge: 10000000, \
@@ -998,11 +1003,11 @@ mod tests {
         let schema_builder = schema::Schema::builder();
         let index = Index::create_in_ram(schema_builder.build());
         {
-            let _index_writer = index.writer(3_000_000).unwrap();
+            let _index_writer: IndexWriter = index.writer_for_tests().unwrap();
             // the lock should be released when the
             // index_writer leaves the scope.
         }
-        let _index_writer_two = index.writer(3_000_000).unwrap();
+        let _index_writer_two: IndexWriter = index.writer_for_tests().unwrap();
     }
 
     #[test]
@@ -1022,7 +1027,7 @@ mod tests {
 
         {
             // writing the segment
-            let mut index_writer = index.writer(3_000_000)?;
+            let mut index_writer = index.writer_for_tests()?;
             index_writer.add_document(doc!(text_field=>"a"))?;
             index_writer.rollback()?;
             assert_eq!(index_writer.commit_opstamp(), 0u64);
@@ -1054,7 +1059,7 @@ mod tests {
             reader.searcher().doc_freq(&term_a).unwrap()
         };
         // writing the segment
-        let mut index_writer = index.writer(12_000_000).unwrap();
+        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
         index_writer.add_document(doc!(text_field=>"a"))?;
         index_writer.commit()?;
         //  this should create 1 segment
@@ -1094,7 +1099,7 @@ mod tests {
             reader.searcher().doc_freq(&term_a).unwrap()
         };
         // writing the segment
-        let mut index_writer = index.writer(12_000_000).unwrap();
+        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
         index_writer.add_document(doc!(text_field=>"a"))?;
         index_writer.commit()?;
         index_writer.add_document(doc!(text_field=>"a"))?;
@@ -1140,7 +1145,7 @@ mod tests {
             reader.searcher().doc_freq(&term_a).unwrap()
         };
         // writing the segment
-        let mut index_writer = index.writer(12_000_000).unwrap();
+        let mut index_writer = index.writer(MEMORY_BUDGET_NUM_BYTES_MIN).unwrap();
         // create 8 segments with 100 tiny docs
         for _doc in 0..100 {
             index_writer.add_document(doc!(text_field=>"a"))?;
@@ -1196,7 +1201,8 @@ mod tests {
 
         {
             // writing the segment
-            let mut index_writer = index.writer_with_num_threads(4, 12_000_000)?;
+            let mut index_writer =
+                index.writer_with_num_threads(4, MEMORY_BUDGET_NUM_BYTES_MIN * 4)?;
             // create 8 segments with 100 tiny docs
             for _doc in 0..100 {
                 index_writer.add_document(doc!(text_field => "a"))?;
@@ -1245,7 +1251,9 @@ mod tests {
             let term = Term::from_field_text(text_field, s);
             searcher.doc_freq(&term).unwrap()
         };
-        let mut index_writer = index.writer_with_num_threads(4, 12_000_000).unwrap();
+        let mut index_writer = index
+            .writer_with_num_threads(4, MEMORY_BUDGET_NUM_BYTES_MIN * 4)
+            .unwrap();
 
         let add_tstamp = index_writer.add_document(doc!(text_field => "a")).unwrap();
         let commit_tstamp = index_writer.commit().unwrap();
@@ -1262,7 +1270,9 @@ mod tests {
         let mut schema_builder = schema::Schema::builder();
         let text_field = schema_builder.add_text_field("text", TEXT);
         let index = Index::create_in_ram(schema_builder.build());
-        let mut index_writer = index.writer_with_num_threads(4, 12_000_000).unwrap();
+        let mut index_writer = index
+            .writer_with_num_threads(4, MEMORY_BUDGET_NUM_BYTES_MIN * 4)
+            .unwrap();
 
         let add_tstamp = index_writer.add_document(doc!(text_field => "a")).unwrap();
 
@@ -1311,7 +1321,9 @@ mod tests {
         let text_field = schema_builder.add_text_field("text", TEXT);
         let index = Index::create_in_ram(schema_builder.build());
         // writing the segment
-        let mut index_writer = index.writer_with_num_threads(4, 12_000_000).unwrap();
+        let mut index_writer = index
+            .writer_with_num_threads(4, MEMORY_BUDGET_NUM_BYTES_MIN * 4)
+            .unwrap();
         let res = index_writer.delete_all_documents();
         assert!(res.is_ok());
 
@@ -1338,7 +1350,9 @@ mod tests {
         let mut schema_builder = schema::Schema::builder();
         let text_field = schema_builder.add_text_field("text", TEXT);
         let index = Index::create_in_ram(schema_builder.build());
-        let mut index_writer = index.writer_with_num_threads(4, 12_000_000).unwrap();
+        let mut index_writer = index
+            .writer_with_num_threads(4, MEMORY_BUDGET_NUM_BYTES_MIN * 4)
+            .unwrap();
 
         // add one simple doc
         assert!(index_writer.add_document(doc!(text_field => "a")).is_ok());
@@ -1371,7 +1385,9 @@ mod tests {
     fn test_delete_all_documents_empty_index() {
         let schema_builder = schema::Schema::builder();
         let index = Index::create_in_ram(schema_builder.build());
-        let mut index_writer = index.writer_with_num_threads(4, 12_000_000).unwrap();
+        let mut index_writer: IndexWriter = index
+            .writer_with_num_threads(4, MEMORY_BUDGET_NUM_BYTES_MIN * 4)
+            .unwrap();
         let clear = index_writer.delete_all_documents();
         let commit = index_writer.commit();
         assert!(clear.is_ok());
@@ -1382,7 +1398,9 @@ mod tests {
     fn test_delete_all_documents_index_twice() {
         let schema_builder = schema::Schema::builder();
         let index = Index::create_in_ram(schema_builder.build());
-        let mut index_writer = index.writer_with_num_threads(4, 12_000_000).unwrap();
+        let mut index_writer: IndexWriter = index
+            .writer_with_num_threads(4, MEMORY_BUDGET_NUM_BYTES_MIN * 4)
+            .unwrap();
         let clear = index_writer.delete_all_documents();
         let commit = index_writer.commit();
         assert!(clear.is_ok());
@@ -1400,7 +1418,7 @@ mod tests {
         let schema = schema_builder.build();
 
         let index = Index::builder().schema(schema).create_in_ram().unwrap();
-        let mut index_writer = index.writer_for_tests().unwrap();
+        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
         index_writer
             .add_document(doc!(text_field => "one"))
             .unwrap();
@@ -1762,7 +1780,7 @@ mod tests {
         let num_segments_before_merge = searcher.segment_readers().len();
         if force_end_merge {
             index_writer.wait_merging_threads()?;
-            let mut index_writer = index.writer_for_tests()?;
+            let mut index_writer: IndexWriter = index.writer_for_tests()?;
             let segment_ids = index
                 .searchable_segment_ids()
                 .expect("Searchable segments failed.");
@@ -1958,14 +1976,14 @@ mod tests {
                 .get_store_reader(DOCSTORE_CACHE_CAPACITY)
                 .unwrap();
             // test store iterator
-            for doc in store_reader.iter(segment_reader.alive_bitset()) {
+            for doc in store_reader.iter::<TantivyDocument>(segment_reader.alive_bitset()) {
                 let id = doc.unwrap().get_first(id_field).unwrap().as_u64().unwrap();
                 assert!(expected_ids_and_num_occurrences.contains_key(&id));
             }
             // test store random access
             for doc_id in segment_reader.doc_ids_alive() {
                 let id = store_reader
-                    .get(doc_id)
+                    .get::<TantivyDocument>(doc_id)
                     .unwrap()
                     .get_first(id_field)
                     .unwrap()
@@ -1974,7 +1992,7 @@ mod tests {
                 assert!(expected_ids_and_num_occurrences.contains_key(&id));
                 if id_exists(id) {
                     let id2 = store_reader
-                        .get(doc_id)
+                        .get::<TantivyDocument>(doc_id)
                         .unwrap()
                         .get_first(multi_numbers)
                         .unwrap()
@@ -1982,13 +2000,13 @@ mod tests {
                         .unwrap();
                     assert_eq!(id, id2);
                     let bool = store_reader
-                        .get(doc_id)
+                        .get::<TantivyDocument>(doc_id)
                         .unwrap()
                         .get_first(bool_field)
                         .unwrap()
                         .as_bool()
                         .unwrap();
-                    let doc = store_reader.get(doc_id).unwrap();
+                    let doc = store_reader.get::<TantivyDocument>(doc_id).unwrap();
                     let mut bool2 = doc.get_all(multi_bools);
                     assert_eq!(bool, bool2.next().unwrap().as_bool().unwrap());
                     assert_ne!(bool, bool2.next().unwrap().as_bool().unwrap());
@@ -2528,7 +2546,7 @@ mod tests {
         // Merge
         {
             assert!(index_writer.wait_merging_threads().is_ok());
-            let mut index_writer = index.writer_for_tests()?;
+            let mut index_writer: IndexWriter = index.writer_for_tests()?;
             let segment_ids = index
                 .searchable_segment_ids()
                 .expect("Searchable segments failed.");
@@ -2570,7 +2588,7 @@ mod tests {
         // Merge
         {
             assert!(index_writer.wait_merging_threads().is_ok());
-            let mut index_writer = index.writer_for_tests()?;
+            let mut index_writer: IndexWriter = index.writer_for_tests()?;
             let segment_ids = index
                 .searchable_segment_ids()
                 .expect("Searchable segments failed.");

src/indexer/index_writer_status.rs

@@ -2,13 +2,15 @@ use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::{Arc, RwLock};
 
 use super::AddBatchReceiver;
+use crate::schema::document::Document;
+use crate::TantivyDocument;
 
 #[derive(Clone)]
-pub(crate) struct IndexWriterStatus {
-    inner: Arc<Inner>,
+pub(crate) struct IndexWriterStatus<D: Document = TantivyDocument> {
+    inner: Arc<Inner<D>>,
 }
 
-impl IndexWriterStatus {
+impl<D: Document> IndexWriterStatus<D> {
     /// Returns true iff the index writer is alive.
     pub fn is_alive(&self) -> bool {
         self.inner.as_ref().is_alive()
@@ -16,7 +18,7 @@ impl IndexWriterStatus {
 
     /// Returns a copy of the operation receiver.
     /// If the index writer was killed, returns `None`.
-    pub fn operation_receiver(&self) -> Option<AddBatchReceiver> {
+    pub fn operation_receiver(&self) -> Option<AddBatchReceiver<D>> {
         let rlock = self
             .inner
             .receive_channel
@@ -27,19 +29,19 @@ impl IndexWriterStatus {
 
     /// Create an index writer bomb.
     /// If dropped, the index writer status will be killed.
-    pub(crate) fn create_bomb(&self) -> IndexWriterBomb {
+    pub(crate) fn create_bomb(&self) -> IndexWriterBomb<D> {
         IndexWriterBomb {
             inner: Some(self.inner.clone()),
         }
     }
 }
 
-struct Inner {
+struct Inner<D: Document> {
     is_alive: AtomicBool,
-    receive_channel: RwLock<Option<AddBatchReceiver>>,
+    receive_channel: RwLock<Option<AddBatchReceiver<D>>>,
 }
 
-impl Inner {
+impl<D: Document> Inner<D> {
     fn is_alive(&self) -> bool {
         self.is_alive.load(Ordering::Relaxed)
     }
@@ -53,8 +55,8 @@ impl Inner {
     }
 }
 
-impl From<AddBatchReceiver> for IndexWriterStatus {
-    fn from(receiver: AddBatchReceiver) -> Self {
+impl<D: Document> From<AddBatchReceiver<D>> for IndexWriterStatus<D> {
+    fn from(receiver: AddBatchReceiver<D>) -> Self {
         IndexWriterStatus {
             inner: Arc::new(Inner {
                 is_alive: AtomicBool::new(true),
@@ -66,11 +68,11 @@ impl From<AddBatchReceiver> for IndexWriterStatus {
 
 /// If dropped, the index writer will be killed.
 /// To prevent this, clients can call `.defuse()`.
-pub(crate) struct IndexWriterBomb {
-    inner: Option<Arc<Inner>>,
+pub(crate) struct IndexWriterBomb<D: Document> {
+    inner: Option<Arc<Inner<D>>>,
 }
 
-impl IndexWriterBomb {
+impl<D: Document> IndexWriterBomb<D> {
     /// Defuses the bomb.
     ///
     /// This is the only way to drop the bomb without killing
@@ -80,7 +82,7 @@ impl IndexWriterBomb {
     }
 }
 
-impl Drop for IndexWriterBomb {
+impl<D: Document> Drop for IndexWriterBomb<D> {
     fn drop(&mut self) {
         if let Some(inner) = self.inner.take() {
             inner.kill();

src/indexer/merger.rs

@@ -753,9 +753,10 @@ mod tests {
     use crate::collector::{Count, FacetCollector};
     use crate::core::Index;
     use crate::query::{AllQuery, BooleanQuery, EnableScoring, Scorer, TermQuery};
+    use crate::schema::document::Value;
     use crate::schema::{
-        Document, Facet, FacetOptions, IndexRecordOption, NumericOptions, Term, TextFieldIndexing,
-        INDEXED, TEXT,
+        Facet, FacetOptions, IndexRecordOption, NumericOptions, TantivyDocument, Term,
+        TextFieldIndexing, INDEXED, TEXT,
     };
     use crate::time::OffsetDateTime;
     use crate::{
@@ -817,7 +818,7 @@ mod tests {
             let segment_ids = index
                 .searchable_segment_ids()
                 .expect("Searchable segments failed.");
-            let mut index_writer = index.writer_for_tests()?;
+            let mut index_writer: IndexWriter = index.writer_for_tests()?;
             index_writer.merge(&segment_ids).wait()?;
             index_writer.wait_merging_threads()?;
         }
@@ -866,30 +867,24 @@ mod tests {
                 );
             }
             {
-                let doc = searcher.doc(DocAddress::new(0, 0))?;
-                assert_eq!(doc.get_first(text_field).unwrap().as_text(), Some("af b"));
+                let doc = searcher.doc::<TantivyDocument>(DocAddress::new(0, 0))?;
+                assert_eq!(doc.get_first(text_field).unwrap().as_str(), Some("af b"));
             }
             {
-                let doc = searcher.doc(DocAddress::new(0, 1))?;
-                assert_eq!(doc.get_first(text_field).unwrap().as_text(), Some("a b c"));
+                let doc = searcher.doc::<TantivyDocument>(DocAddress::new(0, 1))?;
+                assert_eq!(doc.get_first(text_field).unwrap().as_str(), Some("a b c"));
             }
             {
-                let doc = searcher.doc(DocAddress::new(0, 2))?;
-                assert_eq!(
-                    doc.get_first(text_field).unwrap().as_text(),
-                    Some("a b c d")
-                );
+                let doc = searcher.doc::<TantivyDocument>(DocAddress::new(0, 2))?;
+                assert_eq!(doc.get_first(text_field).unwrap().as_str(), Some("a b c d"));
             }
             {
-                let doc = searcher.doc(DocAddress::new(0, 3))?;
-                assert_eq!(doc.get_first(text_field).unwrap().as_text(), Some("af b"));
+                let doc = searcher.doc::<TantivyDocument>(DocAddress::new(0, 3))?;
+                assert_eq!(doc.get_first(text_field).unwrap().as_str(), Some("af b"));
             }
             {
-                let doc = searcher.doc(DocAddress::new(0, 4))?;
-                assert_eq!(
-                    doc.get_first(text_field).unwrap().as_text(),
-                    Some("a b c g")
-                );
+                let doc = searcher.doc::<TantivyDocument>(DocAddress::new(0, 4))?;
+                assert_eq!(doc.get_first(text_field).unwrap().as_str(), Some("a b c g"));
             }
 
             {
@@ -1300,10 +1295,10 @@ mod tests {
         let reader = index.reader().unwrap();
         let mut int_val = 0;
         {
-            let mut index_writer = index.writer_for_tests().unwrap();
+            let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
             let index_doc =
                 |index_writer: &mut IndexWriter, doc_facets: &[&str], int_val: &mut u64| {
-                    let mut doc = Document::default();
+                    let mut doc = TantivyDocument::default();
                     for facet in doc_facets {
                         doc.add_facet(facet_field, Facet::from(facet));
                     }
@@ -1384,7 +1379,7 @@ mod tests {
             let segment_ids = index
                 .searchable_segment_ids()
                 .expect("Searchable segments failed.");
-            let mut index_writer = index.writer_for_tests().unwrap();
+            let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
             index_writer
                 .merge(&segment_ids)
                 .wait()
@@ -1406,7 +1401,7 @@ mod tests {
 
         // Deleting one term
         {
-            let mut index_writer = index.writer_for_tests().unwrap();
+            let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
             let facet = Facet::from_path(vec!["top", "a", "firstdoc"]);
             let facet_term = Term::from_facet(facet_field, &facet);
             index_writer.delete_term(facet_term);
@@ -1431,7 +1426,7 @@ mod tests {
         let mut schema_builder = schema::Schema::builder();
         let int_field = schema_builder.add_u64_field("intvals", INDEXED);
         let index = Index::create_in_ram(schema_builder.build());
-        let mut index_writer = index.writer_for_tests().unwrap();
+        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
         index_writer.add_document(doc!(int_field => 1u64))?;
         index_writer.commit().expect("commit failed");
         index_writer.add_document(doc!(int_field => 1u64))?;
@@ -1460,7 +1455,7 @@ mod tests {
         let reader = index.reader()?;
         {
             let mut index_writer = index.writer_for_tests()?;
-            let mut doc = Document::default();
+            let mut doc = TantivyDocument::default();
             doc.add_u64(int_field, 1);
             index_writer.add_document(doc.clone())?;
             index_writer.commit()?;
@@ -1503,7 +1498,7 @@ mod tests {
         {
             let mut index_writer = index.writer_for_tests()?;
             let index_doc = |index_writer: &mut IndexWriter, int_vals: &[u64]| {
-                let mut doc = Document::default();
+                let mut doc = TantivyDocument::default();
                 for &val in int_vals {
                     doc.add_u64(int_field, val);
                 }
@@ -1566,7 +1561,7 @@ mod tests {
         // Merging the segments
         {
             let segment_ids = index.searchable_segment_ids()?;
-            let mut index_writer = index.writer_for_tests()?;
+            let mut index_writer: IndexWriter = index.writer_for_tests()?;
             index_writer.merge(&segment_ids).wait()?;
             index_writer.wait_merging_threads()?;
         }
@@ -1613,7 +1608,7 @@ mod tests {
         writer.set_merge_policy(Box::new(policy));
 
         for i in 0..100 {
-            let mut doc = Document::new();
+            let mut doc = TantivyDocument::new();
             doc.add_f64(field, 42.0);
             doc.add_f64(multi_field, 0.24);
             doc.add_f64(multi_field, 0.27);

src/indexer/merger_sorted_index_test.rs

@@ -4,11 +4,15 @@ mod tests {
     use crate::core::Index;
     use crate::fastfield::AliveBitSet;
     use crate::query::QueryParser;
+    use crate::schema::document::Value;
     use crate::schema::{
         self, BytesOptions, Facet, FacetOptions, IndexRecordOption, NumericOptions,
         TextFieldIndexing, TextOptions,
     };
-    use crate::{DocAddress, DocSet, IndexSettings, IndexSortByField, Order, Postings, Term};
+    use crate::{
+        DocAddress, DocSet, IndexSettings, IndexSortByField, IndexWriter, Order, Postings,
+        TantivyDocument, Term,
+    };
 
     fn create_test_index_posting_list_issue(index_settings: Option<IndexSettings>) -> Index {
         let mut schema_builder = schema::Schema::builder();
@@ -26,7 +30,7 @@ mod tests {
         let index = index_builder.create_in_ram().unwrap();
 
         {
-            let mut index_writer = index.writer_for_tests().unwrap();
+            let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
             index_writer
                 .add_document(doc!(int_field=>3_u64, facet_field=> Facet::from("/crime")))
                 .unwrap();
@@ -45,7 +49,7 @@ mod tests {
             let segment_ids = index
                 .searchable_segment_ids()
                 .expect("Searchable segments failed.");
-            let mut index_writer = index.writer_for_tests().unwrap();
+            let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
             assert!(index_writer.merge(&segment_ids).wait().is_ok());
             assert!(index_writer.wait_merging_threads().is_ok());
         }
@@ -133,7 +137,7 @@ mod tests {
         // Merging the segments
         {
             let segment_ids = index.searchable_segment_ids()?;
-            let mut index_writer = index.writer_for_tests()?;
+            let mut index_writer: IndexWriter = index.writer_for_tests()?;
             index_writer.merge(&segment_ids).wait()?;
             index_writer.wait_merging_threads()?;
         }
@@ -272,12 +276,16 @@ mod tests {
             } else {
                 2
             };
-            let doc = searcher.doc(DocAddress::new(0, blubber_pos)).unwrap();
+            let doc = searcher
+                .doc::<TantivyDocument>(DocAddress::new(0, blubber_pos))
+                .unwrap();
             assert_eq!(
-                doc.get_first(my_text_field).unwrap().as_text(),
+                doc.get_first(my_text_field).unwrap().as_str(),
                 Some("blubber")
             );
-            let doc = searcher.doc(DocAddress::new(0, 0)).unwrap();
+            let doc = searcher
+                .doc::<TantivyDocument>(DocAddress::new(0, 0))
+                .unwrap();
             assert_eq!(doc.get_first(int_field).unwrap().as_u64(), Some(1000));
         }
     }
@@ -494,7 +502,7 @@ mod bench_sorted_index_merge {
         let index = index_builder.create_in_ram().unwrap();
 
         {
-            let mut index_writer = index.writer_for_tests().unwrap();
+            let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
             let index_doc = |index_writer: &mut IndexWriter, val: u64| {
                 index_writer.add_document(doc!(int_field=>val)).unwrap();
             };

src/indexer/mod.rs

@@ -44,9 +44,9 @@ pub type DefaultMergePolicy = LogMergePolicy;
 // - all docs in the operation will happen on the same segment and continuous doc_ids.
 // - all operations in the group are committed at the same time, making the group
 // atomic.
-type AddBatch = SmallVec<[AddOperation; 4]>;
-type AddBatchSender = channel::Sender<AddBatch>;
-type AddBatchReceiver = channel::Receiver<AddBatch>;
+type AddBatch<D> = SmallVec<[AddOperation<D>; 4]>;
+type AddBatchSender<D> = channel::Sender<AddBatch<D>>;
+type AddBatchReceiver<D> = channel::Receiver<AddBatch<D>>;
 
 #[cfg(feature = "mmap")]
 #[cfg(test)]
@@ -54,15 +54,15 @@ mod tests_mmap {
 
     use crate::collector::Count;
     use crate::query::QueryParser;
-    use crate::schema::{JsonObjectOptions, Schema, TEXT};
-    use crate::{Index, Term};
+    use crate::schema::{JsonObjectOptions, Schema, Type, TEXT};
+    use crate::{Index, IndexWriter, Term};
 
     #[test]
     fn test_advance_delete_bug() -> crate::Result<()> {
         let mut schema_builder = Schema::builder();
         let text_field = schema_builder.add_text_field("text", TEXT);
         let index = Index::create_from_tempdir(schema_builder.build())?;
-        let mut index_writer = index.writer_for_tests()?;
+        let mut index_writer: IndexWriter = index.writer_for_tests()?;
         // there must be one deleted document in the segment
         index_writer.add_document(doc!(text_field=>"b"))?;
         index_writer.delete_term(Term::from_field_text(text_field, "b"));
@@ -79,7 +79,7 @@ mod tests_mmap {
         let mut schema_builder = Schema::builder();
         let json_field = schema_builder.add_json_field("json", TEXT);
         let index = Index::create_in_ram(schema_builder.build());
-        let mut index_writer = index.writer_for_tests().unwrap();
+        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
         let json = serde_json::json!({"k8s.container.name": "prometheus", "val": "hello"});
         index_writer.add_document(doc!(json_field=>json)).unwrap();
         index_writer.commit().unwrap();
@@ -110,7 +110,7 @@ mod tests_mmap {
             JsonObjectOptions::from(TEXT).set_expand_dots_enabled();
         let json_field = schema_builder.add_json_field("json", json_options);
         let index = Index::create_in_ram(schema_builder.build());
-        let mut index_writer = index.writer_for_tests().unwrap();
+        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
         let json = serde_json::json!({"k8s.container.name": "prometheus", "val": "hello"});
         index_writer.add_document(doc!(json_field=>json)).unwrap();
         index_writer.commit().unwrap();
@@ -133,4 +133,40 @@ mod tests_mmap {
             assert_eq!(num_docs, 1);
         }
     }
+
+    #[test]
+    fn test_json_field_list_fields() {
+        let mut schema_builder = Schema::builder();
+        let json_options: JsonObjectOptions =
+            JsonObjectOptions::from(TEXT).set_expand_dots_enabled();
+        let json_field = schema_builder.add_json_field("json", json_options);
+        let index = Index::create_in_ram(schema_builder.build());
+        let mut index_writer = index.writer_for_tests().unwrap();
+        let json = serde_json::json!({"k8s.container.name": "prometheus", "val": "hello", "sub": {"a": 1, "b": 2}});
+        index_writer.add_document(doc!(json_field=>json)).unwrap();
+        let json = serde_json::json!({"k8s.container.name": "prometheus", "val": "hello", "suber": {"a": 1, "b": 2}});
+        index_writer.add_document(doc!(json_field=>json)).unwrap();
+        let json = serde_json::json!({"k8s.container.name": "prometheus", "val": "hello", "suber": {"a": "mixed", "b": 2}});
+        index_writer.add_document(doc!(json_field=>json)).unwrap();
+        index_writer.commit().unwrap();
+        let reader = index.reader().unwrap();
+
+        let searcher = reader.searcher();
+        assert_eq!(searcher.num_docs(), 3);
+
+        let reader = &searcher.segment_readers()[0];
+        let inverted_index = reader.inverted_index(json_field).unwrap();
+        assert_eq!(
+            inverted_index.list_fields().unwrap(),
+            [
+                ("k8s\u{1}container\u{1}name".to_string(), Type::Str),
+                ("sub\u{1}a".to_string(), Type::I64),
+                ("sub\u{1}b".to_string(), Type::I64),
+                ("suber\u{1}a".to_string(), Type::I64),
+                ("suber\u{1}a".to_string(), Type::Str),
+                ("suber\u{1}b".to_string(), Type::I64),
+                ("val".to_string(), Type::Str),
+            ]
+        );
+    }
 }

src/indexer/operation.rs

@@ -1,5 +1,6 @@
 use crate::query::Weight;
-use crate::schema::{Document, Term};
+use crate::schema::document::Document;
+use crate::schema::{TantivyDocument, Term};
 use crate::Opstamp;
 
 /// Timestamped Delete operation.
@@ -10,16 +11,16 @@ pub struct DeleteOperation {
 
 /// Timestamped Add operation.
 #[derive(Eq, PartialEq, Debug)]
-pub struct AddOperation {
+pub struct AddOperation<D: Document = TantivyDocument> {
     pub opstamp: Opstamp,
-    pub document: Document,
+    pub document: D,
 }
 
 /// UserOperation is an enum type that encapsulates other operation types.
 #[derive(Eq, PartialEq, Debug)]
-pub enum UserOperation {
+pub enum UserOperation<D: Document = TantivyDocument> {
     /// Add operation
-    Add(Document),
+    Add(D),
     /// Delete operation
     Delete(Term),
 }

src/indexer/prepared_commit.rs

@@ -1,16 +1,17 @@
 use super::IndexWriter;
-use crate::{FutureResult, Opstamp};
+use crate::schema::document::Document;
+use crate::{FutureResult, Opstamp, TantivyDocument};
 
 /// A prepared commit
-pub struct PreparedCommit<'a> {
-    index_writer: &'a mut IndexWriter,
+pub struct PreparedCommit<'a, D: Document = TantivyDocument> {
+    index_writer: &'a mut IndexWriter<D>,
     payload: Option<String>,
     opstamp: Opstamp,
 }
 
-impl<'a> PreparedCommit<'a> {
-    pub(crate) fn new(index_writer: &'a mut IndexWriter, opstamp: Opstamp) -> PreparedCommit<'_> {
-        PreparedCommit {
+impl<'a, D: Document> PreparedCommit<'a, D> {
+    pub(crate) fn new(index_writer: &'a mut IndexWriter<D>, opstamp: Opstamp) -> Self {
+        Self {
             index_writer,
             payload: None,
             opstamp,

src/indexer/segment_updater.rs

@@ -596,10 +596,15 @@ impl SegmentUpdater {
             );
             {
                 if let Some(after_merge_segment_entry) = after_merge_segment_entry.as_mut() {
+                    // Deletes and commits could have happened as we were merging.
+                    // We need to make sure we are up to date with deletes before accepting the
+                    // segment.
                     let mut delete_cursor = after_merge_segment_entry.delete_cursor().clone();
                     if let Some(delete_operation) = delete_cursor.get() {
                         let committed_opstamp = segment_updater.load_meta().opstamp;
                         if delete_operation.opstamp < committed_opstamp {
+                            // We are not up to date! Let's create a new tombstone file for our
+                            // freshly create split.
                             let index = &segment_updater.index;
                             let segment = index.segment(after_merge_segment_entry.meta().clone());
                             if let Err(advance_deletes_err) = advance_deletes(

src/indexer/segment_writer.rs

@@ -13,10 +13,11 @@ use crate::postings::{
     compute_table_memory_size, serialize_postings, IndexingContext, IndexingPosition,
     PerFieldPostingsWriter, PostingsWriter,
 };
-use crate::schema::{FieldEntry, FieldType, Schema, Term, Value, DATE_TIME_PRECISION_INDEXED};
+use crate::schema::document::{Document, ReferenceValue, Value};
+use crate::schema::{FieldEntry, FieldType, Schema, Term, DATE_TIME_PRECISION_INDEXED};
 use crate::store::{StoreReader, StoreWriter};
 use crate::tokenizer::{FacetTokenizer, PreTokenizedStream, TextAnalyzer, Tokenizer};
-use crate::{DocId, Document, Opstamp, SegmentComponent, TantivyError};
+use crate::{DocId, Opstamp, SegmentComponent, TantivyError};
 
 /// Computes the initial size of the hash table.
 ///
@@ -26,6 +27,8 @@ use crate::{DocId, Document, Opstamp, SegmentComponent, TantivyError};
 fn compute_initial_table_size(per_thread_memory_budget: usize) -> crate::Result<usize> {
     let table_memory_upper_bound = per_thread_memory_budget / 3;
     (10..20) // We cap it at 2^19 = 512K capacity.
+        // TODO: There are cases where this limit causes a
+        // reallocation in the hashmap. Check if this affects performance.
         .map(|power| 1 << power)
         .take_while(|capacity| compute_table_memory_size(*capacity) < table_memory_upper_bound)
         .last()
@@ -79,10 +82,7 @@ impl SegmentWriter {
     /// the flushing behavior as a memory limit.
     /// - segment: The segment being written
     /// - schema
-    pub fn for_segment(
-        memory_budget_in_bytes: usize,
-        segment: Segment,
-    ) -> crate::Result<SegmentWriter> {
+    pub fn for_segment(memory_budget_in_bytes: usize, segment: Segment) -> crate::Result<Self> {
         let schema = segment.schema();
         let tokenizer_manager = segment.index().tokenizers().clone();
         let tokenizer_manager_fast_field = segment.index().fast_field_tokenizer().clone();
@@ -111,7 +111,7 @@ impl SegmentWriter {
                 })
             })
             .collect::<Result<Vec<_>, _>>()?;
-        Ok(SegmentWriter {
+        Ok(Self {
             max_doc: 0,
             ctx: IndexingContext::new(table_size),
             per_field_postings_writers,
@@ -162,18 +162,21 @@ impl SegmentWriter {
             + self.segment_serializer.mem_usage()
     }
 
-    fn index_document(&mut self, doc: &Document) -> crate::Result<()> {
+    fn index_document<D: Document>(&mut self, doc: &D) -> crate::Result<()> {
         let doc_id = self.max_doc;
+
+        // TODO: Can this be optimised a bit?
         let vals_grouped_by_field = doc
-            .field_values()
-            .iter()
-            .sorted_by_key(|el| el.field())
-            .group_by(|el| el.field());
+            .iter_fields_and_values()
+            .sorted_by_key(|(field, _)| *field)
+            .group_by(|(field, _)| *field);
+
         for (field, field_values) in &vals_grouped_by_field {
-            let values = field_values.map(|field_value| field_value.value());
+            let values = field_values.map(|el| el.1);
+
             let field_entry = self.schema.get_field_entry(field);
             let make_schema_error = || {
-                crate::TantivyError::SchemaError(format!(
+                TantivyError::SchemaError(format!(
                     "Expected a {:?} for field {:?}",
                     field_entry.field_type().value_type(),
                     field_entry.name()
@@ -191,7 +194,10 @@ impl SegmentWriter {
             match field_entry.field_type() {
                 FieldType::Facet(_) => {
                     let mut facet_tokenizer = FacetTokenizer::default(); // this can be global
-                    for value in values {
+                    for value_access in values {
+                        // Used to help with linting and type checking.
+                        let value = value_access as D::Value<'_>;
+
                         let facet = value.as_facet().ok_or_else(make_schema_error)?;
                         let facet_str = facet.encoded_str();
                         let mut facet_tokenizer = facet_tokenizer.token_stream(facet_str);
@@ -207,19 +213,18 @@ impl SegmentWriter {
                 }
                 FieldType::Str(_) => {
                     let mut indexing_position = IndexingPosition::default();
-                    for value in values {
-                        let mut token_stream = match value {
-                            Value::PreTokStr(tok_str) => {
-                                BoxTokenStream::new(PreTokenizedStream::from(tok_str.clone()))
-                            }
-                            Value::Str(ref text) => {
-                                let text_analyzer =
-                                    &mut self.per_field_text_analyzers[field.field_id() as usize];
-                                text_analyzer.token_stream(text)
-                            }
-                            _ => {
-                                continue;
-                            }
+                    for value_access in values {
+                        // Used to help with linting and type checking.
+                        let value = value_access as D::Value<'_>;
+
+                        let mut token_stream = if let Some(text) = value.as_str() {
+                            let text_analyzer =
+                                &mut self.per_field_text_analyzers[field.field_id() as usize];
+                            text_analyzer.token_stream(text)
+                        } else if let Some(tok_str) = value.as_pre_tokenized_text() {
+                            BoxTokenStream::new(PreTokenizedStream::from(tok_str.clone()))
+                        } else {
+                            continue;
                         };
 
                         assert!(term_buffer.is_empty());
@@ -238,7 +243,10 @@ impl SegmentWriter {
                 }
                 FieldType::U64(_) => {
                     let mut num_vals = 0;
-                    for value in values {
+                    for value_access in values {
+                        // Used to help with linting and type checking.
+                        let value = value_access as D::Value<'_>;
+
                         num_vals += 1;
                         let u64_val = value.as_u64().ok_or_else(make_schema_error)?;
                         term_buffer.set_u64(u64_val);
@@ -250,9 +258,13 @@ impl SegmentWriter {
                 }
                 FieldType::Date(_) => {
                     let mut num_vals = 0;
-                    for value in values {
+                    for value_access in values {
+                        // Used to help with linting and type checking.
+                        let value_access = value_access as D::Value<'_>;
+                        let value = value_access.as_value();
+
                         num_vals += 1;
-                        let date_val = value.as_date().ok_or_else(make_schema_error)?;
+                        let date_val = value.as_datetime().ok_or_else(make_schema_error)?;
                         term_buffer
                             .set_u64(date_val.truncate(DATE_TIME_PRECISION_INDEXED).to_u64());
                         postings_writer.subscribe(doc_id, 0u32, term_buffer, ctx);
@@ -263,7 +275,10 @@ impl SegmentWriter {
                 }
                 FieldType::I64(_) => {
                     let mut num_vals = 0;
-                    for value in values {
+                    for value_access in values {
+                        // Used to help with linting and type checking.
+                        let value = value_access as D::Value<'_>;
+
                         num_vals += 1;
                         let i64_val = value.as_i64().ok_or_else(make_schema_error)?;
                         term_buffer.set_i64(i64_val);
@@ -275,7 +290,10 @@ impl SegmentWriter {
                 }
                 FieldType::F64(_) => {
                     let mut num_vals = 0;
-                    for value in values {
+                    for value_access in values {
+                        // Used to help with linting and type checking.
+                        let value = value_access as D::Value<'_>;
+
                         num_vals += 1;
                         let f64_val = value.as_f64().ok_or_else(make_schema_error)?;
                         term_buffer.set_f64(f64_val);
@@ -287,7 +305,10 @@ impl SegmentWriter {
                 }
                 FieldType::Bool(_) => {
                     let mut num_vals = 0;
-                    for value in values {
+                    for value_access in values {
+                        // Used to help with linting and type checking.
+                        let value = value_access as D::Value<'_>;
+
                         num_vals += 1;
                         let bool_val = value.as_bool().ok_or_else(make_schema_error)?;
                         term_buffer.set_bool(bool_val);
@@ -299,7 +320,10 @@ impl SegmentWriter {
                 }
                 FieldType::Bytes(_) => {
                     let mut num_vals = 0;
-                    for value in values {
+                    for value_access in values {
+                        // Used to help with linting and type checking.
+                        let value = value_access as D::Value<'_>;
+
                         num_vals += 1;
                         let bytes = value.as_bytes().ok_or_else(make_schema_error)?;
                         term_buffer.set_bytes(bytes);
@@ -312,9 +336,17 @@ impl SegmentWriter {
                 FieldType::JsonObject(json_options) => {
                     let text_analyzer =
                         &mut self.per_field_text_analyzers[field.field_id() as usize];
-                    let json_values_it =
-                        values.map(|value| value.as_json().ok_or_else(make_schema_error));
-                    index_json_values(
+                    let json_values_it = values.map(|value_access| {
+                        // Used to help with linting and type checking.
+                        let value_access = value_access as D::Value<'_>;
+                        let value = value_access.as_value();
+
+                        match value {
+                            ReferenceValue::Object(object_iter) => Ok(object_iter),
+                            _ => Err(make_schema_error()),
+                        }
+                    });
+                    index_json_values::<D::Value<'_>>(
                         doc_id,
                         json_values_it,
                         text_analyzer,
@@ -326,7 +358,10 @@ impl SegmentWriter {
                 }
                 FieldType::IpAddr(_) => {
                     let mut num_vals = 0;
-                    for value in values {
+                    for value_access in values {
+                        // Used to help with linting and type checking.
+                        let value = value_access as D::Value<'_>;
+
                         num_vals += 1;
                         let ip_addr = value.as_ip_addr().ok_or_else(make_schema_error)?;
                         term_buffer.set_ip_addr(ip_addr);
@@ -344,7 +379,10 @@ impl SegmentWriter {
     /// Indexes a new document
     ///
     /// As a user, you should rather use `IndexWriter`'s add_document.
-    pub fn add_document(&mut self, add_operation: AddOperation) -> crate::Result<()> {
+    pub fn add_document<D: Document>(
+        &mut self,
+        add_operation: AddOperation<D>,
+    ) -> crate::Result<()> {
         let AddOperation { document, opstamp } = add_operation;
         self.doc_opstamps.push(opstamp);
         self.fast_field_writers.add_document(&document)?;
@@ -443,6 +481,7 @@ fn remap_and_write(
 
 #[cfg(test)]
 mod tests {
+    use std::collections::BTreeMap;
     use std::path::{Path, PathBuf};
 
     use tempfile::TempDir;
@@ -453,15 +492,18 @@ mod tests {
     use crate::directory::RamDirectory;
     use crate::postings::TermInfo;
     use crate::query::PhraseQuery;
+    use crate::schema::document::Value;
     use crate::schema::{
-        IndexRecordOption, Schema, TextFieldIndexing, TextOptions, Type, STORED, STRING, TEXT,
+        Document, IndexRecordOption, Schema, TextFieldIndexing, TextOptions, Type, STORED, STRING,
+        TEXT,
     };
     use crate::store::{Compressor, StoreReader, StoreWriter};
     use crate::time::format_description::well_known::Rfc3339;
     use crate::time::OffsetDateTime;
     use crate::tokenizer::{PreTokenizedString, Token};
     use crate::{
-        DateTime, Directory, DocAddress, DocSet, Document, Index, Postings, Term, TERMINATED,
+        DateTime, Directory, DocAddress, DocSet, Index, IndexWriter, Postings, TantivyDocument,
+        Term, TERMINATED,
     };
 
     #[test]
@@ -478,7 +520,7 @@ mod tests {
         let mut schema_builder = Schema::builder();
         let text_field = schema_builder.add_text_field("title", TEXT | STORED);
         let schema = schema_builder.build();
-        let mut doc = Document::default();
+        let mut doc = TantivyDocument::default();
         let pre_tokenized_text = PreTokenizedString {
             text: String::from("A"),
             tokens: vec![Token {
@@ -502,11 +544,11 @@ mod tests {
         store_writer.close().unwrap();
 
         let reader = StoreReader::open(directory.open_read(path).unwrap(), 0).unwrap();
-        let doc = reader.get(0).unwrap();
+        let doc = reader.get::<TantivyDocument>(0).unwrap();
 
         assert_eq!(doc.field_values().len(), 2);
-        assert_eq!(doc.field_values()[0].value().as_text(), Some("A"));
-        assert_eq!(doc.field_values()[1].value().as_text(), Some("title"));
+        assert_eq!(doc.field_values()[0].value().as_str(), Some("A"));
+        assert_eq!(doc.field_values()[1].value().as_str(), Some("title"));
     }
 
     #[test]
@@ -537,13 +579,13 @@ mod tests {
         let reader = index.reader().unwrap();
         let searcher = reader.searcher();
         let doc = searcher
-            .doc(DocAddress {
+            .doc::<TantivyDocument>(DocAddress {
                 segment_ord: 0u32,
                 doc_id: 0u32,
             })
             .unwrap();
         let serdeser_json_val = serde_json::from_str::<serde_json::Map<String, serde_json::Value>>(
-            &schema.to_json(&doc),
+            &doc.to_json(&schema),
         )
         .unwrap()
         .get("json")
@@ -673,10 +715,10 @@ mod tests {
         let mut schema_builder = Schema::builder();
         let json_field = schema_builder.add_json_field("json", STORED | TEXT);
         let schema = schema_builder.build();
-        let mut doc = Document::default();
-        let json_val: serde_json::Map<String, serde_json::Value> =
+        let mut doc = TantivyDocument::default();
+        let json_val: BTreeMap<String, crate::schema::OwnedValue> =
             serde_json::from_str(r#"{"mykey": "repeated token token"}"#).unwrap();
-        doc.add_json_object(json_field, json_val);
+        doc.add_object(json_field, json_val);
         let index = Index::create_in_ram(schema);
         let mut writer = index.writer_for_tests().unwrap();
         writer.add_document(doc).unwrap();
@@ -800,11 +842,10 @@ mod tests {
         let mut schema_builder = Schema::builder();
         let text = schema_builder.add_text_field("text", TEXT);
         let schema = schema_builder.build();
-        let doc = schema
-            .parse_document(r#"{"text": [ "bbb", "aaa", "", "aaa"]}"#)
+        let doc = TantivyDocument::parse_json(&schema, r#"{"text": [ "bbb", "aaa", "", "aaa"]}"#)
             .unwrap();
         let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_for_tests().unwrap();
+        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
         index_writer.add_document(doc).unwrap();
         // On debug this did panic on the underflow
         index_writer.commit().unwrap();
@@ -829,7 +870,7 @@ mod tests {
         let mut schema_builder = Schema::builder();
         let text = schema_builder.add_text_field("text", TEXT);
         let schema = schema_builder.build();
-        let mut doc = Document::default();
+        let mut doc = TantivyDocument::default();
         // This is a bit of a contrived example.
         let tokens = PreTokenizedString {
             text: "roller-coaster".to_string(),
@@ -844,7 +885,7 @@ mod tests {
         doc.add_pre_tokenized_text(text, tokens.clone());
         doc.add_pre_tokenized_text(text, tokens);
         let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_for_tests().unwrap();
+        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
         index_writer.add_document(doc).unwrap();
         index_writer.commit().unwrap();
         let reader = index.reader().unwrap();
@@ -867,7 +908,7 @@ mod tests {
         let mut schema_builder = Schema::builder();
         let text = schema_builder.add_text_field("text", TEXT);
         let schema = schema_builder.build();
-        let mut doc = Document::default();
+        let mut doc = TantivyDocument::default();
         // This is a bit of a contrived example.
         let tokens = PreTokenizedString {
             text: "contrived-example".to_string(), //< I can't think of a use case where this corner case happens in real life.
@@ -892,7 +933,7 @@ mod tests {
         doc.add_pre_tokenized_text(text, tokens);
         doc.add_text(text, "hello");
         let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_for_tests().unwrap();
+        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
         index_writer.add_document(doc).unwrap();
         index_writer.commit().unwrap();
         let reader = index.reader().unwrap();
@@ -928,7 +969,7 @@ mod tests {
         let schema = index.schema();
         let mut index_writer = index.writer(50_000_000).unwrap();
         let title = schema.get_field("title").unwrap();
-        let mut document = Document::default();
+        let mut document = TantivyDocument::default();
         document.add_text(title, "The Old Man and the Sea");
         index_writer.add_document(document).unwrap();
         let error = index_writer.commit().unwrap_err();

src/lib.rs

@@ -21,7 +21,7 @@
 //! # use tantivy::collector::TopDocs;
 //! # use tantivy::query::QueryParser;
 //! # use tantivy::schema::*;
-//! # use tantivy::{doc, DocAddress, Index, Score};
+//! # use tantivy::{doc, DocAddress, Index, IndexWriter, Score};
 //! #
 //! # fn main() {
 //! #     // Let's create a temporary directory for the
@@ -53,7 +53,7 @@
 //!
 //! // Here we use a buffer of 100MB that will be split
 //! // between indexing threads.
-//! let mut index_writer = index.writer(100_000_000)?;
+//! let mut index_writer: IndexWriter = index.writer(100_000_000)?;
 //!
 //! // Let's index one documents!
 //! index_writer.add_document(doc!(
@@ -89,8 +89,8 @@
 //!
 //! for (_score, doc_address) in top_docs {
 //!     // Retrieve the actual content of documents given its `doc_address`.
-//!     let retrieved_doc = searcher.doc(doc_address)?;
-//!     println!("{}", schema.to_json(&retrieved_doc));
+//!     let retrieved_doc = searcher.doc::<TantivyDocument>(doc_address)?;
+//!     println!("{}", retrieved_doc.to_json(&schema));
 //! }
 //!
 //! # Ok(())
@@ -186,7 +186,7 @@ pub use crate::indexer::{merge_filtered_segments, merge_indices, IndexWriter, Pr
 pub use crate::postings::Postings;
 #[allow(deprecated)]
 pub use crate::schema::DatePrecision;
-pub use crate::schema::{DateOptions, DateTimePrecision, Document, Term};
+pub use crate::schema::{DateOptions, DateTimePrecision, Document, TantivyDocument, Term};
 
 /// Index format version.
 const INDEX_FORMAT_VERSION: u32 = 5;
@@ -342,8 +342,9 @@ pub mod tests {
     use crate::docset::{DocSet, TERMINATED};
     use crate::merge_policy::NoMergePolicy;
     use crate::query::BooleanQuery;
+    use crate::schema::document::Value;
     use crate::schema::*;
-    use crate::{DateTime, DocAddress, Index, Postings, ReloadPolicy};
+    use crate::{DateTime, DocAddress, Index, IndexWriter, Postings, ReloadPolicy};
 
     pub fn fixed_size_test<O: BinarySerializable + FixedSize + Default>() {
         let mut buffer = Vec::new();
@@ -414,7 +415,7 @@ pub mod tests {
         let schema = schema_builder.build();
         let index = Index::create_from_tempdir(schema)?;
         // writing the segment
-        let mut index_writer = index.writer_for_tests()?;
+        let mut index_writer: IndexWriter = index.writer_for_tests()?;
         {
             let doc = doc!(text_field=>"af b");
             index_writer.add_document(doc)?;
@@ -436,7 +437,7 @@ pub mod tests {
         let mut schema_builder = Schema::builder();
         let text_field = schema_builder.add_text_field("text", TEXT);
         let index = Index::create_in_ram(schema_builder.build());
-        let mut index_writer = index.writer_for_tests()?;
+        let mut index_writer: IndexWriter = index.writer_for_tests()?;
         index_writer.add_document(doc!(text_field=>"a b c"))?;
         index_writer.commit()?;
         index_writer.add_document(doc!(text_field=>"a"))?;
@@ -463,7 +464,7 @@ pub mod tests {
         let title_field = schema_builder.add_text_field("title", TEXT);
         let text_field = schema_builder.add_text_field("text", TEXT);
         let index = Index::create_in_ram(schema_builder.build());
-        let mut index_writer = index.writer_for_tests()?;
+        let mut index_writer: IndexWriter = index.writer_for_tests()?;
         index_writer.add_document(doc!(text_field=>"a b c"))?;
         index_writer.commit()?;
         let index_reader = index.reader()?;
@@ -485,7 +486,7 @@ pub mod tests {
         let mut schema_builder = Schema::builder();
         let text_field = schema_builder.add_text_field("text", TEXT);
         let index = Index::create_in_ram(schema_builder.build());
-        let mut index_writer = index.writer_for_tests()?;
+        let mut index_writer: IndexWriter = index.writer_for_tests()?;
         index_writer.add_document(doc!(text_field=>"a b c"))?;
         index_writer.add_document(doc!())?;
         index_writer.add_document(doc!(text_field=>"a b"))?;
@@ -528,7 +529,7 @@ pub mod tests {
             .unwrap();
         {
             // writing the segment
-            let mut index_writer = index.writer_for_tests()?;
+            let mut index_writer: IndexWriter = index.writer_for_tests()?;
             // 0
             index_writer.add_document(doc!(text_field=>"a b"))?;
             // 1
@@ -575,7 +576,7 @@ pub mod tests {
         }
         {
             // writing the segment
-            let mut index_writer = index.writer_for_tests()?;
+            let mut index_writer: IndexWriter = index.writer_for_tests()?;
             // 0
             index_writer.add_document(doc!(text_field=>"a b"))?;
             // 1
@@ -612,7 +613,7 @@ pub mod tests {
         }
         {
             // writing the segment
-            let mut index_writer = index.writer_for_tests()?;
+            let mut index_writer: IndexWriter = index.writer_for_tests()?;
             index_writer.add_document(doc!(text_field=>"a b"))?;
             index_writer.delete_term(Term::from_field_text(text_field, "c"));
             index_writer.rollback()?;
@@ -662,7 +663,7 @@ pub mod tests {
         let schema = schema_builder.build();
 
         let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_for_tests()?;
+        let mut index_writer: IndexWriter = index.writer_for_tests()?;
         index_writer.add_document(doc!(field=>1u64))?;
         index_writer.commit()?;
         let reader = index.reader()?;
@@ -685,7 +686,7 @@ pub mod tests {
         let schema = schema_builder.build();
 
         let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_for_tests()?;
+        let mut index_writer: IndexWriter = index.writer_for_tests()?;
         let negative_val = -1i64;
         index_writer.add_document(doc!(value_field => negative_val))?;
         index_writer.commit()?;
@@ -709,7 +710,7 @@ pub mod tests {
         let schema = schema_builder.build();
 
         let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_for_tests()?;
+        let mut index_writer: IndexWriter = index.writer_for_tests()?;
         let val = std::f64::consts::PI;
         index_writer.add_document(doc!(value_field => val))?;
         index_writer.commit()?;
@@ -733,7 +734,7 @@ pub mod tests {
         let absent_field = schema_builder.add_text_field("absent_text", TEXT);
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_for_tests()?;
+        let mut index_writer: IndexWriter = index.writer_for_tests()?;
         index_writer.add_document(doc!(text_field=>"a"))?;
         assert!(index_writer.commit().is_ok());
         let reader = index.reader()?;
@@ -756,7 +757,7 @@ pub mod tests {
             .try_into()?;
 
         // writing the segment
-        let mut index_writer = index.writer_for_tests()?;
+        let mut index_writer: IndexWriter = index.writer_for_tests()?;
         index_writer.add_document(doc!(text_field=>"63"))?;
         index_writer.add_document(doc!(text_field=>"70"))?;
         index_writer.add_document(doc!(text_field=>"34"))?;
@@ -781,7 +782,7 @@ pub mod tests {
         let index = Index::create_in_ram(schema);
         {
             // writing the segment
-            let mut index_writer = index.writer_for_tests()?;
+            let mut index_writer: IndexWriter = index.writer_for_tests()?;
             index_writer.add_document(doc!(text_field=>"af af af bc bc"))?;
             index_writer.commit()?;
         }
@@ -813,7 +814,7 @@ pub mod tests {
         let index = Index::create_in_ram(schema);
         let reader = index.reader()?;
         // writing the segment
-        let mut index_writer = index.writer_for_tests()?;
+        let mut index_writer: IndexWriter = index.writer_for_tests()?;
         index_writer.add_document(doc!(text_field=>"af af af b"))?;
         index_writer.add_document(doc!(text_field=>"a b c"))?;
         index_writer.add_document(doc!(text_field=>"a b c d"))?;
@@ -877,7 +878,7 @@ pub mod tests {
             .try_into()?;
         assert_eq!(reader.searcher().num_docs(), 0u64);
         // writing the segment
-        let mut index_writer = index.writer_for_tests()?;
+        let mut index_writer: IndexWriter = index.writer_for_tests()?;
         index_writer.add_document(doc!(text_field=>"af b"))?;
         index_writer.add_document(doc!(text_field=>"a b c"))?;
         index_writer.add_document(doc!(text_field=>"a b c d"))?;
@@ -985,13 +986,13 @@ pub mod tests {
                             text_field => "some other value",
                             other_text_field => "short");
         assert_eq!(document.len(), 3);
-        let values: Vec<&Value> = document.get_all(text_field).collect();
+        let values: Vec<&OwnedValue> = document.get_all(text_field).collect();
         assert_eq!(values.len(), 2);
-        assert_eq!(values[0].as_text(), Some("tantivy"));
-        assert_eq!(values[1].as_text(), Some("some other value"));
-        let values: Vec<&Value> = document.get_all(other_text_field).collect();
+        assert_eq!(values[0].as_str(), Some("tantivy"));
+        assert_eq!(values[1].as_str(), Some("some other value"));
+        let values: Vec<&OwnedValue> = document.get_all(other_text_field).collect();
         assert_eq!(values.len(), 1);
-        assert_eq!(values[0].as_text(), Some("short"));
+        assert_eq!(values[0].as_str(), Some("short"));
     }
 
     #[test]
@@ -1005,7 +1006,7 @@ pub mod tests {
         let schema = schema_builder.build();
 
         let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_for_tests()?;
+        let mut index_writer: IndexWriter = index.writer_for_tests()?;
         {
             let document =
                 doc!(fast_field_unsigned => 4u64, fast_field_signed=>4i64, fast_field_float=>4f64);
@@ -1071,7 +1072,7 @@ pub mod tests {
         let index = Index::create_in_ram(schema);
         let index_reader = index.reader()?;
 
-        let mut index_writer = index.writer_for_tests()?;
+        let mut index_writer: IndexWriter = index.writer_for_tests()?;
         index_writer.set_merge_policy(Box::new(NoMergePolicy));
 
         for doc_id in 0u64..DOC_COUNT {
@@ -1124,7 +1125,7 @@ pub mod tests {
         let body = builder.add_text_field("body", TEXT | STORED);
         let schema = builder.build();
         let index = Index::create_in_dir(&index_path, schema)?;
-        let mut writer = index.writer(50_000_000)?;
+        let mut writer: IndexWriter = index.writer(50_000_000)?;
         writer.set_merge_policy(Box::new(NoMergePolicy));
         for _ in 0..5000 {
             writer.add_document(doc!(body => "foo"))?;

src/macros.rs

@@ -45,12 +45,12 @@
 macro_rules! doc(
     () => {
         {
-            ($crate::Document::default())
+            ($crate::TantivyDocument::default())
         }
     }; // avoids a warning due to the useless `mut`.
     ($($field:expr => $value:expr),*) => {
         {
-            let mut document = $crate::Document::default();
+            let mut document = $crate::TantivyDocument::default();
             $(
                 document.add_field_value($field, $value);
             )*

src/postings/mod.rs

@@ -52,7 +52,7 @@ pub mod tests {
         Field, IndexRecordOption, Schema, Term, TextFieldIndexing, TextOptions, INDEXED, TEXT,
     };
     use crate::tokenizer::{SimpleTokenizer, MAX_TOKEN_LEN};
-    use crate::{DocId, HasLen, Score};
+    use crate::{DocId, HasLen, IndexWriter, Score};
 
     #[test]
     pub fn test_position_write() -> crate::Result<()> {
@@ -225,7 +225,7 @@ pub mod tests {
 
         {
             let mut segment_writer =
-                SegmentWriter::for_segment(3_000_000, segment.clone()).unwrap();
+                SegmentWriter::for_segment(15_000_000, segment.clone()).unwrap();
             {
                 // checking that position works if the field has two values
                 let op = AddOperation {
@@ -432,7 +432,7 @@ pub mod tests {
 
         // delete some of the documents
         {
-            let mut index_writer = index.writer_for_tests()?;
+            let mut index_writer: IndexWriter = index.writer_for_tests()?;
             index_writer.delete_term(term_0);
             assert!(index_writer.commit().is_ok());
         }
@@ -483,7 +483,7 @@ pub mod tests {
 
         // delete everything else
         {
-            let mut index_writer = index.writer_for_tests()?;
+            let mut index_writer: IndexWriter = index.writer_for_tests()?;
             index_writer.delete_term(term_1);
             assert!(index_writer.commit().is_ok());
         }
@@ -568,8 +568,8 @@ mod bench {
 
     use crate::docset::TERMINATED;
     use crate::query::Intersection;
-    use crate::schema::{Document, Field, IndexRecordOption, Schema, Term, STRING};
-    use crate::{tests, DocSet, Index};
+    use crate::schema::{Field, IndexRecordOption, Schema, TantivyDocument, Term, STRING};
+    use crate::{tests, DocSet, Index, IndexWriter};
 
     pub static TERM_A: Lazy<Term> = Lazy::new(|| {
         let field = Field::from_field_id(0);
@@ -598,9 +598,9 @@ mod bench {
         let index = Index::create_in_ram(schema);
         let posting_list_size = 1_000_000;
         {
-            let mut index_writer = index.writer_for_tests().unwrap();
+            let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
             for _ in 0..posting_list_size {
-                let mut doc = Document::default();
+                let mut doc = TantivyDocument::default();
                 if rng.gen_bool(1f64 / 15f64) {
                     doc.add_text(text_field, "a");
                 }

src/query/all_query.rs

@@ -99,14 +99,14 @@ mod tests {
     use crate::docset::{DocSet, BUFFER_LEN, TERMINATED};
     use crate::query::{AllScorer, EnableScoring, Query};
     use crate::schema::{Schema, TEXT};
-    use crate::Index;
+    use crate::{Index, IndexWriter};
 
     fn create_test_index() -> crate::Result<Index> {
         let mut schema_builder = Schema::builder();
         let field = schema_builder.add_text_field("text", TEXT);
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_for_tests()?;
+        let mut index_writer: IndexWriter = index.writer_for_tests()?;
         index_writer.add_document(doc!(field=>"aaa"))?;
         index_writer.add_document(doc!(field=>"bbb"))?;
         index_writer.commit()?;

src/query/automaton_weight.rs

@@ -4,6 +4,7 @@ use std::sync::Arc;
 use common::BitSet;
 use tantivy_fst::Automaton;
 
+use super::phrase_prefix_query::prefix_end;
 use crate::core::SegmentReader;
 use crate::query::{BitSetDocSet, ConstScorer, Explanation, Scorer, Weight};
 use crate::schema::{Field, IndexRecordOption};
@@ -14,6 +15,10 @@ use crate::{DocId, Score, TantivyError};
 pub struct AutomatonWeight<A> {
     field: Field,
     automaton: Arc<A>,
+    // For JSON fields, the term dictionary include terms from all paths.
+    // We apply additional filtering based on the given JSON path, when searching within the term
+    // dictionary. This prevents terms from unrelated paths from matching the search criteria.
+    json_path_bytes: Option<Box<[u8]>>,
 }
 
 impl<A> AutomatonWeight<A>
@@ -26,6 +31,20 @@ where
         AutomatonWeight {
             field,
             automaton: automaton.into(),
+            json_path_bytes: None,
+        }
+    }
+
+    /// Create a new AutomationWeight for a json path
+    pub fn new_for_json_path<IntoArcA: Into<Arc<A>>>(
+        field: Field,
+        automaton: IntoArcA,
+        json_path_bytes: &[u8],
+    ) -> AutomatonWeight<A> {
+        AutomatonWeight {
+            field,
+            automaton: automaton.into(),
+            json_path_bytes: Some(json_path_bytes.to_vec().into_boxed_slice()),
         }
     }
 
@@ -34,7 +53,15 @@ where
         term_dict: &'a TermDictionary,
     ) -> io::Result<TermStreamer<'a, &'a A>> {
         let automaton: &A = &self.automaton;
-        let term_stream_builder = term_dict.search(automaton);
+        let mut term_stream_builder = term_dict.search(automaton);
+
+        if let Some(json_path_bytes) = &self.json_path_bytes {
+            term_stream_builder = term_stream_builder.ge(json_path_bytes);
+            if let Some(end) = prefix_end(json_path_bytes) {
+                term_stream_builder = term_stream_builder.lt(&end);
+            }
+        }
+
         term_stream_builder.into_stream()
     }
 }
@@ -90,13 +117,13 @@ mod tests {
     use crate::docset::TERMINATED;
     use crate::query::Weight;
     use crate::schema::{Schema, STRING};
-    use crate::Index;
+    use crate::{Index, IndexWriter};
 
     fn create_index() -> crate::Result<Index> {
         let mut schema = Schema::builder();
         let title = schema.add_text_field("title", STRING);
         let index = Index::create_in_ram(schema.build());
-        let mut index_writer = index.writer_for_tests()?;
+        let mut index_writer: IndexWriter = index.writer_for_tests()?;
         index_writer.add_document(doc!(title=>"abc"))?;
         index_writer.add_document(doc!(title=>"bcd"))?;
         index_writer.add_document(doc!(title=>"abcd"))?;

src/query/boolean_query/boolean_query.rs

@@ -24,6 +24,7 @@ use crate::schema::{IndexRecordOption, Term};
 /// use tantivy::schema::{IndexRecordOption, Schema, TEXT};
 /// use tantivy::Term;
 /// use tantivy::Index;
+/// use tantivy::IndexWriter;
 ///
 /// fn main() -> tantivy::Result<()> {
 ///    let mut schema_builder = Schema::builder();
@@ -32,7 +33,7 @@ use crate::schema::{IndexRecordOption, Term};
 ///    let schema = schema_builder.build();
 ///    let index = Index::create_in_ram(schema);
 ///    {
-///        let mut index_writer = index.writer(3_000_000)?;
+///        let mut index_writer: IndexWriter = index.writer(15_000_000)?;
 ///        index_writer.add_document(doc!(
 ///            title => "The Name of the Wind",
 ///        ))?;

src/query/boolean_query/mod.rs

@@ -19,7 +19,7 @@ mod tests {
         TermQuery,
     };
     use crate::schema::*;
-    use crate::{assert_nearly_equals, DocAddress, DocId, Index, Score};
+    use crate::{assert_nearly_equals, DocAddress, DocId, Index, IndexWriter, Score};
 
     fn aux_test_helper() -> crate::Result<(Index, Field)> {
         let mut schema_builder = Schema::builder();
@@ -28,7 +28,7 @@ mod tests {
         let index = Index::create_in_ram(schema);
         {
             // writing the segment
-            let mut index_writer = index.writer_for_tests()?;
+            let mut index_writer: IndexWriter = index.writer_for_tests()?;
             index_writer.add_document(doc!(text_field => "a b c"))?;
             index_writer.add_document(doc!(text_field => "a c"))?;
             index_writer.add_document(doc!(text_field => "b c"))?;
@@ -224,7 +224,7 @@ mod tests {
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
         {
-            let mut index_writer = index.writer_for_tests()?;
+            let mut index_writer: IndexWriter = index.writer_for_tests()?;
             index_writer.add_document(doc!(text_field => "a b c"))?;
             index_writer.add_document(doc!(text_field => "a c"))?;
             index_writer.add_document(doc!(text_field => "b c"))?;
@@ -297,7 +297,7 @@ mod tests {
         let text = schema_builder.add_text_field("text", STRING);
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_with_num_threads(1, 5_000_000)?;
+        let mut index_writer: IndexWriter = index.writer_for_tests()?;
         index_writer.add_document(doc!(text=>"a"))?;
         index_writer.add_document(doc!(text=>"b"))?;
         index_writer.commit()?;

src/query/boost_query.rs

@@ -136,14 +136,14 @@ mod tests {
     use super::BoostQuery;
     use crate::query::{AllQuery, Query};
     use crate::schema::Schema;
-    use crate::{DocAddress, Document, Index};
+    use crate::{DocAddress, Index, IndexWriter, TantivyDocument};
 
     #[test]
     fn test_boost_query_explain() -> crate::Result<()> {
         let schema = Schema::builder().build();
         let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_for_tests()?;
-        index_writer.add_document(Document::new())?;
+        let mut index_writer: IndexWriter = index.writer_for_tests()?;
+        index_writer.add_document(TantivyDocument::new())?;
         index_writer.commit()?;
         let reader = index.reader()?;
         let searcher = reader.searcher();

src/query/const_score_query.rs

@@ -143,14 +143,14 @@ mod tests {
     use super::ConstScoreQuery;
     use crate::query::{AllQuery, Query};
     use crate::schema::Schema;
-    use crate::{DocAddress, Document, Index};
+    use crate::{DocAddress, Index, IndexWriter, TantivyDocument};
 
     #[test]
     fn test_const_score_query_explain() -> crate::Result<()> {
         let schema = Schema::builder().build();
         let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_for_tests()?;
-        index_writer.add_document(Document::new())?;
+        let mut index_writer: IndexWriter = index.writer_for_tests()?;
+        index_writer.add_document(TantivyDocument::new())?;
         index_writer.commit()?;
         let reader = index.reader()?;
         let searcher = reader.searcher();

src/query/disjunction_max_query.rs

@@ -15,6 +15,7 @@ use crate::{Score, Term};
 /// use tantivy::schema::{IndexRecordOption, Schema, TEXT};
 /// use tantivy::Term;
 /// use tantivy::Index;
+/// use tantivy::IndexWriter;
 ///
 /// fn main() -> tantivy::Result<()> {
 ///    let mut schema_builder = Schema::builder();
@@ -23,7 +24,7 @@ use crate::{Score, Term};
 ///    let schema = schema_builder.build();
 ///    let index = Index::create_in_ram(schema);
 ///    {
-///        let mut index_writer = index.writer(3_000_000)?;
+///        let mut index_writer: IndexWriter = index.writer(15_000_000)?;
 ///        index_writer.add_document(doc!(
 ///            title => "The Name of Girl",
 ///        ))?;

src/query/exist_query.rs

@@ -0,0 +1,345 @@
+use core::fmt::Debug;
+
+use columnar::{ColumnIndex, DynamicColumn};
+
+use super::{ConstScorer, EmptyScorer};
+use crate::core::SegmentReader;
+use crate::docset::{DocSet, TERMINATED};
+use crate::query::explanation::does_not_match;
+use crate::query::{EnableScoring, Explanation, Query, Scorer, Weight};
+use crate::{DocId, Score, TantivyError};
+
+/// Query that matches all documents with a non-null value in the specified field.
+///
+/// All of the matched documents get the score 1.0.
+#[derive(Clone, Debug)]
+pub struct ExistsQuery {
+    field_name: String,
+}
+
+impl ExistsQuery {
+    /// Creates a new `ExistQuery` from the given field.
+    ///
+    /// This query matches all documents with at least one non-null value in the specified field.
+    /// This constructor never fails, but executing the search with this query will return an
+    /// error if the specified field doesn't exists or is not a fast field.
+    pub fn new_exists_query(field: String) -> ExistsQuery {
+        ExistsQuery { field_name: field }
+    }
+}
+
+impl Query for ExistsQuery {
+    fn weight(&self, enable_scoring: EnableScoring) -> crate::Result<Box<dyn Weight>> {
+        let schema = enable_scoring.schema();
+        let Some((field, _path)) = schema.find_field(&self.field_name) else {
+            return Err(TantivyError::FieldNotFound(self.field_name.clone()));
+        };
+        let field_type = schema.get_field_entry(field).field_type();
+        if !field_type.is_fast() {
+            return Err(TantivyError::SchemaError(format!(
+                "Field {} is not a fast field.",
+                self.field_name
+            )));
+        }
+        Ok(Box::new(ExistsWeight {
+            field_name: self.field_name.clone(),
+        }))
+    }
+}
+
+/// Weight associated with the `ExistsQuery` query.
+pub struct ExistsWeight {
+    field_name: String,
+}
+
+impl Weight for ExistsWeight {
+    fn scorer(&self, reader: &SegmentReader, boost: Score) -> crate::Result<Box<dyn Scorer>> {
+        let fast_field_reader = reader.fast_fields();
+        let dynamic_columns: crate::Result<Vec<DynamicColumn>> = fast_field_reader
+            .dynamic_column_handles(&self.field_name)?
+            .into_iter()
+            .map(|handle| handle.open().map_err(|io_error| io_error.into()))
+            .collect();
+        let mut non_empty_columns = Vec::new();
+        for column in dynamic_columns? {
+            if !matches!(column.column_index(), ColumnIndex::Empty { .. }) {
+                non_empty_columns.push(column)
+            }
+        }
+        // TODO: we can optimizer more here since in most cases we will have only one index
+        if !non_empty_columns.is_empty() {
+            let docset = ExistsDocSet::new(non_empty_columns, reader.max_doc());
+            Ok(Box::new(ConstScorer::new(docset, boost)))
+        } else {
+            Ok(Box::new(EmptyScorer))
+        }
+    }
+
+    fn explain(&self, reader: &SegmentReader, doc: DocId) -> crate::Result<Explanation> {
+        let mut scorer = self.scorer(reader, 1.0)?;
+        if scorer.seek(doc) != doc {
+            return Err(does_not_match(doc));
+        }
+        Ok(Explanation::new("ExistsQuery", 1.0))
+    }
+}
+
+pub(crate) struct ExistsDocSet {
+    columns: Vec<DynamicColumn>,
+    doc: DocId,
+    max_doc: DocId,
+}
+
+impl ExistsDocSet {
+    pub(crate) fn new(columns: Vec<DynamicColumn>, max_doc: DocId) -> Self {
+        let mut set = Self {
+            columns,
+            doc: 0u32,
+            max_doc,
+        };
+        set.find_next();
+        set
+    }
+
+    fn find_next(&mut self) -> DocId {
+        while self.doc < self.max_doc {
+            if self
+                .columns
+                .iter()
+                .any(|col| col.column_index().has_value(self.doc))
+            {
+                return self.doc;
+            }
+            self.doc += 1;
+        }
+        self.doc = TERMINATED;
+        TERMINATED
+    }
+}
+
+impl DocSet for ExistsDocSet {
+    fn advance(&mut self) -> DocId {
+        self.seek(self.doc + 1)
+    }
+
+    fn size_hint(&self) -> u32 {
+        0
+    }
+
+    fn doc(&self) -> DocId {
+        self.doc
+    }
+
+    #[inline(always)]
+    fn seek(&mut self, target: DocId) -> DocId {
+        self.doc = target;
+        self.find_next()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::net::Ipv6Addr;
+    use std::ops::Bound;
+
+    use common::DateTime;
+    use time::OffsetDateTime;
+
+    use crate::collector::Count;
+    use crate::query::exist_query::ExistsQuery;
+    use crate::query::{BooleanQuery, RangeQuery};
+    use crate::schema::{Facet, FacetOptions, Schema, FAST, INDEXED, STRING, TEXT};
+    use crate::{doc, Index, Searcher};
+
+    #[test]
+    fn test_exists_query_simple() -> crate::Result<()> {
+        let mut schema_builder = Schema::builder();
+        let all_field = schema_builder.add_u64_field("all", INDEXED | FAST);
+        let even_field = schema_builder.add_u64_field("even", INDEXED | FAST);
+        let odd_field = schema_builder.add_text_field("odd", STRING | FAST);
+        let multi_field = schema_builder.add_text_field("multi", FAST);
+        let _never_field = schema_builder.add_u64_field("never", INDEXED | FAST);
+        let schema = schema_builder.build();
+
+        let index = Index::create_in_ram(schema);
+        {
+            let mut index_writer = index.writer_for_tests()?;
+            for i in 0u64..100u64 {
+                if i % 2 == 0 {
+                    if i % 10 == 0 {
+                        index_writer.add_document(doc!(all_field => i, even_field => i, multi_field => i.to_string(), multi_field => (i + 1).to_string()))?;
+                    } else {
+                        index_writer.add_document(doc!(all_field => i, even_field => i))?;
+                    }
+                } else {
+                    index_writer.add_document(doc!(all_field => i, odd_field => i.to_string()))?;
+                }
+            }
+            index_writer.commit()?;
+        }
+        let reader = index.reader()?;
+        let searcher = reader.searcher();
+
+        assert_eq!(count_existing_fields(&searcher, "all")?, 100);
+        assert_eq!(count_existing_fields(&searcher, "odd")?, 50);
+        assert_eq!(count_existing_fields(&searcher, "even")?, 50);
+        assert_eq!(count_existing_fields(&searcher, "multi")?, 10);
+        assert_eq!(count_existing_fields(&searcher, "never")?, 0);
+
+        // exercise seek
+        let query = BooleanQuery::intersection(vec![
+            Box::new(RangeQuery::new_u64_bounds(
+                "all".to_string(),
+                Bound::Included(50),
+                Bound::Unbounded,
+            )),
+            Box::new(ExistsQuery::new_exists_query("even".to_string())),
+        ]);
+        assert_eq!(searcher.search(&query, &Count)?, 25);
+
+        let query = BooleanQuery::intersection(vec![
+            Box::new(RangeQuery::new_u64_bounds(
+                "all".to_string(),
+                Bound::Included(0),
+                Bound::Excluded(50),
+            )),
+            Box::new(ExistsQuery::new_exists_query("odd".to_string())),
+        ]);
+        assert_eq!(searcher.search(&query, &Count)?, 25);
+
+        Ok(())
+    }
+
+    #[test]
+    fn test_exists_query_json() -> crate::Result<()> {
+        let mut schema_builder = Schema::builder();
+        let json = schema_builder.add_json_field("json", TEXT | FAST);
+        let schema = schema_builder.build();
+
+        let index = Index::create_in_ram(schema);
+        {
+            let mut index_writer = index.writer_for_tests()?;
+            for i in 0u64..100u64 {
+                if i % 2 == 0 {
+                    index_writer.add_document(doc!(json => json!({"all": i, "even": true})))?;
+                } else {
+                    index_writer
+                        .add_document(doc!(json => json!({"all": i.to_string(), "odd": true})))?;
+                }
+            }
+            index_writer.commit()?;
+        }
+        let reader = index.reader()?;
+        let searcher = reader.searcher();
+
+        assert_eq!(count_existing_fields(&searcher, "json.all")?, 100);
+        assert_eq!(count_existing_fields(&searcher, "json.even")?, 50);
+        assert_eq!(count_existing_fields(&searcher, "json.odd")?, 50);
+
+        // Handling of non-existing fields:
+        assert_eq!(count_existing_fields(&searcher, "json.absent")?, 0);
+        assert_eq!(
+            searcher
+                .search(
+                    &ExistsQuery::new_exists_query("does_not_exists.absent".to_string()),
+                    &Count
+                )
+                .unwrap_err()
+                .to_string(),
+            "The field does not exist: 'does_not_exists.absent'"
+        );
+
+        Ok(())
+    }
+
+    #[test]
+    fn test_exists_query_misc_supported_types() -> crate::Result<()> {
+        let mut schema_builder = Schema::builder();
+        let bool = schema_builder.add_bool_field("bool", FAST);
+        let bytes = schema_builder.add_bytes_field("bytes", FAST);
+        let date = schema_builder.add_date_field("date", FAST);
+        let f64 = schema_builder.add_f64_field("f64", FAST);
+        let ip_addr = schema_builder.add_ip_addr_field("ip_addr", FAST);
+        let facet = schema_builder.add_facet_field("facet", FacetOptions::default());
+        let schema = schema_builder.build();
+
+        let index = Index::create_in_ram(schema);
+        {
+            let mut index_writer = index.writer_for_tests()?;
+            let now = OffsetDateTime::now_utc().unix_timestamp();
+            for i in 0u8..100u8 {
+                if i % 2 == 0 {
+                    let date_val = DateTime::from_utc(OffsetDateTime::from_unix_timestamp(
+                        now + i as i64 * 100,
+                    )?);
+                    index_writer.add_document(
+                        doc!(bool => i % 3 == 0, bytes => vec![i, i + 1,  i + 2], date => date_val),
+                    )?;
+                } else {
+                    let ip_addr_v6 = Ipv6Addr::new(0, 0, 0, 0, 0, 0xffff, 0xc00a, i.into());
+                    index_writer
+                        .add_document(doc!(f64 => i as f64 * 0.5, ip_addr => ip_addr_v6, facet => Facet::from("/facet/foo"), facet => Facet::from("/facet/bar")))?;
+                }
+            }
+            index_writer.commit()?;
+        }
+        let reader = index.reader()?;
+        let searcher = reader.searcher();
+
+        assert_eq!(count_existing_fields(&searcher, "bool")?, 50);
+        assert_eq!(count_existing_fields(&searcher, "bytes")?, 50);
+        assert_eq!(count_existing_fields(&searcher, "date")?, 50);
+        assert_eq!(count_existing_fields(&searcher, "f64")?, 50);
+        assert_eq!(count_existing_fields(&searcher, "ip_addr")?, 50);
+        assert_eq!(count_existing_fields(&searcher, "facet")?, 50);
+
+        Ok(())
+    }
+
+    #[test]
+    fn test_exists_query_unsupported_types() -> crate::Result<()> {
+        let mut schema_builder = Schema::builder();
+        let not_fast = schema_builder.add_text_field("not_fast", TEXT);
+        let schema = schema_builder.build();
+
+        let index = Index::create_in_ram(schema);
+        {
+            let mut index_writer = index.writer_for_tests()?;
+            index_writer.add_document(doc!(
+                not_fast => "slow",
+            ))?;
+            index_writer.commit()?;
+        }
+        let reader = index.reader()?;
+        let searcher = reader.searcher();
+
+        assert_eq!(
+            searcher
+                .search(
+                    &ExistsQuery::new_exists_query("not_fast".to_string()),
+                    &Count
+                )
+                .unwrap_err()
+                .to_string(),
+            "Schema error: 'Field not_fast is not a fast field.'"
+        );
+
+        assert_eq!(
+            searcher
+                .search(
+                    &ExistsQuery::new_exists_query("does_not_exists".to_string()),
+                    &Count
+                )
+                .unwrap_err()
+                .to_string(),
+            "The field does not exist: 'does_not_exists'"
+        );
+
+        Ok(())
+    }
+
+    fn count_existing_fields(searcher: &Searcher, field: &str) -> crate::Result<usize> {
+        let query = ExistsQuery::new_exists_query(field.to_string());
+        searcher.search(&query, &Count)
+    }
+}

src/query/fuzzy_query.rs

@@ -3,7 +3,7 @@ use once_cell::sync::OnceCell;
 use tantivy_fst::Automaton;
 
 use crate::query::{AutomatonWeight, EnableScoring, Query, Weight};
-use crate::schema::Term;
+use crate::schema::{Term, Type};
 use crate::TantivyError::InvalidArgument;
 
 pub(crate) struct DfaWrapper(pub DFA);
@@ -38,7 +38,7 @@ impl Automaton for DfaWrapper {
 /// use tantivy::collector::{Count, TopDocs};
 /// use tantivy::query::FuzzyTermQuery;
 /// use tantivy::schema::{Schema, TEXT};
-/// use tantivy::{doc, Index, Term};
+/// use tantivy::{doc, Index, IndexWriter, Term};
 ///
 /// fn example() -> tantivy::Result<()> {
 ///     let mut schema_builder = Schema::builder();
@@ -46,7 +46,7 @@ impl Automaton for DfaWrapper {
 ///     let schema = schema_builder.build();
 ///     let index = Index::create_in_ram(schema);
 ///     {
-///         let mut index_writer = index.writer(3_000_000)?;
+///         let mut index_writer: IndexWriter = index.writer(15_000_000)?;
 ///         index_writer.add_document(doc!(
 ///             title => "The Name of the Wind",
 ///         ))?;
@@ -132,18 +132,46 @@ impl FuzzyTermQuery {
             });
 
         let term_value = self.term.value();
-        let term_text = term_value.as_str().ok_or_else(|| {
-            InvalidArgument("The fuzzy term query requires a string term.".to_string())
-        })?;
+
+        let term_text = if term_value.typ() == Type::Json {
+            if let Some(json_path_type) = term_value.json_path_type() {
+                if json_path_type != Type::Str {
+                    return Err(InvalidArgument(format!(
+                        "The fuzzy term query requires a string path type for a json term. Found \
+                         {:?}",
+                        json_path_type
+                    )));
+                }
+            }
+
+            std::str::from_utf8(self.term.serialized_value_bytes()).map_err(|_| {
+                InvalidArgument(
+                    "Failed to convert json term value bytes to utf8 string.".to_string(),
+                )
+            })?
+        } else {
+            term_value.as_str().ok_or_else(|| {
+                InvalidArgument("The fuzzy term query requires a string term.".to_string())
+            })?
+        };
         let automaton = if self.prefix {
             automaton_builder.build_prefix_dfa(term_text)
         } else {
             automaton_builder.build_dfa(term_text)
         };
-        Ok(AutomatonWeight::new(
-            self.term.field(),
-            DfaWrapper(automaton),
-        ))
+
+        if let Some((json_path_bytes, _)) = term_value.as_json() {
+            Ok(AutomatonWeight::new_for_json_path(
+                self.term.field(),
+                DfaWrapper(automaton),
+                json_path_bytes,
+            ))
+        } else {
+            Ok(AutomatonWeight::new(
+                self.term.field(),
+                DfaWrapper(automaton),
+            ))
+        }
     }
 }
 
@@ -157,8 +185,90 @@ impl Query for FuzzyTermQuery {
 mod test {
     use super::FuzzyTermQuery;
     use crate::collector::{Count, TopDocs};
-    use crate::schema::{Schema, TEXT};
-    use crate::{assert_nearly_equals, Index, Term};
+    use crate::indexer::NoMergePolicy;
+    use crate::query::QueryParser;
+    use crate::schema::{Schema, STORED, TEXT};
+    use crate::{assert_nearly_equals, Index, IndexWriter, TantivyDocument, Term};
+
+    #[test]
+    pub fn test_fuzzy_json_path() -> crate::Result<()> {
+        // # Defining the schema
+        let mut schema_builder = Schema::builder();
+        let attributes = schema_builder.add_json_field("attributes", TEXT | STORED);
+        let schema = schema_builder.build();
+
+        // # Indexing documents
+        let index = Index::create_in_ram(schema.clone());
+
+        let mut index_writer = index.writer_for_tests()?;
+        index_writer.set_merge_policy(Box::new(NoMergePolicy));
+        let doc = TantivyDocument::parse_json(
+            &schema,
+            r#"{
+            "attributes": {
+                "a": "japan"
+            }
+        }"#,
+        )?;
+        index_writer.add_document(doc)?;
+        let doc = TantivyDocument::parse_json(
+            &schema,
+            r#"{
+            "attributes": {
+                "aa": "japan"
+            }
+        }"#,
+        )?;
+        index_writer.add_document(doc)?;
+        index_writer.commit()?;
+
+        let reader = index.reader()?;
+        let searcher = reader.searcher();
+
+        // # Fuzzy search
+        let query_parser = QueryParser::for_index(&index, vec![attributes]);
+
+        let get_json_path_term = |query: &str| -> crate::Result<Term> {
+            let query = query_parser.parse_query(query)?;
+            let mut terms = Vec::new();
+            query.query_terms(&mut |term, _| {
+                terms.push(term.clone());
+            });
+
+            Ok(terms[0].clone())
+        };
+
+        // shall not match the first document due to json path mismatch
+        {
+            let term = get_json_path_term("attributes.aa:japan")?;
+            let fuzzy_query = FuzzyTermQuery::new(term, 2, true);
+            let top_docs = searcher.search(&fuzzy_query, &TopDocs::with_limit(2))?;
+            assert_eq!(top_docs.len(), 1, "Expected only 1 document");
+            assert_eq!(top_docs[0].1.doc_id, 1, "Expected the second document");
+        }
+
+        // shall match the first document because Levenshtein distance is 1 (substitute 'o' with
+        // 'a')
+        {
+            let term = get_json_path_term("attributes.a:japon")?;
+
+            let fuzzy_query = FuzzyTermQuery::new(term, 1, true);
+            let top_docs = searcher.search(&fuzzy_query, &TopDocs::with_limit(2))?;
+            assert_eq!(top_docs.len(), 1, "Expected only 1 document");
+            assert_eq!(top_docs[0].1.doc_id, 0, "Expected the first document");
+        }
+
+        // shall not match because non-prefix Levenshtein distance is more than 1 (add 'a' and 'n')
+        {
+            let term = get_json_path_term("attributes.a:jap")?;
+
+            let fuzzy_query = FuzzyTermQuery::new(term, 1, true);
+            let top_docs = searcher.search(&fuzzy_query, &TopDocs::with_limit(2))?;
+            assert_eq!(top_docs.len(), 0, "Expected no document");
+        }
+
+        Ok(())
+    }
 
     #[test]
     pub fn test_fuzzy_term() -> crate::Result<()> {
@@ -167,7 +277,7 @@ mod test {
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
         {
-            let mut index_writer = index.writer_for_tests()?;
+            let mut index_writer: IndexWriter = index.writer_for_tests()?;
             index_writer.add_document(doc!(
                 country_field => "japan",
             ))?;
@@ -216,7 +326,7 @@ mod test {
         let country_field = schema_builder.add_text_field("country", TEXT);
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_for_tests()?;
+        let mut index_writer: IndexWriter = index.writer_for_tests()?;
         index_writer.add_document(doc!(country_field => "japan"))?;
         index_writer.commit()?;
         let reader = index.reader()?;

src/query/mod.rs

@@ -8,6 +8,7 @@ mod const_score_query;
 mod disjunction_max_query;
 mod empty_query;
 mod exclude;
+mod exist_query;
 mod explanation;
 mod fuzzy_query;
 mod intersection;
@@ -41,6 +42,7 @@ pub use self::const_score_query::{ConstScoreQuery, ConstScorer};
 pub use self::disjunction_max_query::DisjunctionMaxQuery;
 pub use self::empty_query::{EmptyQuery, EmptyScorer, EmptyWeight};
 pub use self::exclude::Exclude;
+pub use self::exist_query::ExistsQuery;
 pub use self::explanation::Explanation;
 #[cfg(test)]
 pub(crate) use self::fuzzy_query::DfaWrapper;

src/query/more_like_this/more_like_this.rs

@@ -1,11 +1,14 @@
 use std::cmp::Reverse;
 use std::collections::{BinaryHeap, HashMap};
 
+use tokenizer_api::Token;
+
 use crate::query::bm25::idf;
 use crate::query::{BooleanQuery, BoostQuery, Occur, Query, TermQuery};
-use crate::schema::{Field, FieldType, IndexRecordOption, Term, Value};
+use crate::schema::document::{Document, Value};
+use crate::schema::{Field, FieldType, IndexRecordOption, Term};
 use crate::tokenizer::{FacetTokenizer, PreTokenizedStream, TokenStream, Tokenizer};
-use crate::{DocAddress, Result, Searcher, TantivyError};
+use crate::{DocAddress, Result, Searcher, TantivyDocument, TantivyError};
 
 #[derive(Debug, PartialEq)]
 struct ScoreTerm {
@@ -90,10 +93,10 @@ impl MoreLikeThis {
     }
 
     /// Creates a [`BooleanQuery`] using a set of field values.
-    pub fn query_with_document_fields(
+    pub fn query_with_document_fields<'a, V: Value<'a>>(
         &self,
         searcher: &Searcher,
-        doc_fields: &[(Field, Vec<Value>)],
+        doc_fields: &[(Field, Vec<V>)],
     ) -> Result<BooleanQuery> {
         let score_terms = self.retrieve_terms_from_doc_fields(searcher, doc_fields)?;
         let query = self.create_query(score_terms);
@@ -126,26 +129,18 @@ impl MoreLikeThis {
         searcher: &Searcher,
         doc_address: DocAddress,
     ) -> Result<Vec<ScoreTerm>> {
-        let doc = searcher.doc(doc_address)?;
-        let field_to_values = doc
-            .get_sorted_field_values()
-            .iter()
-            .map(|(field, values)| {
-                (
-                    *field,
-                    values.iter().map(|v| (**v).clone()).collect::<Vec<Value>>(),
-                )
-            })
-            .collect::<Vec<_>>();
+        let doc = searcher.doc::<TantivyDocument>(doc_address)?;
+
+        let field_to_values = doc.get_sorted_field_values();
         self.retrieve_terms_from_doc_fields(searcher, &field_to_values)
     }
 
     /// Finds terms for a more-like-this query.
     /// field_to_field_values is a mapping from field to possible values of that field.
-    fn retrieve_terms_from_doc_fields(
+    fn retrieve_terms_from_doc_fields<'a, V: Value<'a>>(
         &self,
         searcher: &Searcher,
-        field_to_values: &[(Field, Vec<Value>)],
+        field_to_values: &[(Field, Vec<V>)],
     ) -> Result<Vec<ScoreTerm>> {
         if field_to_values.is_empty() {
             return Err(TantivyError::InvalidArgument(
@@ -164,11 +159,11 @@ impl MoreLikeThis {
     /// Computes the frequency of values for a field while updating the term frequencies
     /// Note: A FieldValue can be made up of multiple terms.
     /// We are interested in extracting terms within FieldValue
-    fn add_term_frequencies(
+    fn add_term_frequencies<'a, V: Value<'a>>(
         &self,
         searcher: &Searcher,
         field: Field,
-        values: &[Value],
+        values: &[V],
         term_frequencies: &mut HashMap<Term, usize>,
     ) -> Result<()> {
         let schema = searcher.schema();
@@ -184,11 +179,10 @@ impl MoreLikeThis {
             FieldType::Facet(_) => {
                 let facets: Vec<&str> = values
                     .iter()
-                    .map(|value| match value {
-                        Value::Facet(ref facet) => Ok(facet.encoded_str()),
-                        _ => Err(TantivyError::InvalidArgument(
-                            "invalid field value".to_string(),
-                        )),
+                    .map(|value| {
+                        value.as_facet().map(|f| f.encoded_str()).ok_or_else(|| {
+                            TantivyError::InvalidArgument("invalid field value".to_string())
+                        })
                     })
                     .collect::<Result<Vec<_>>>()?;
                 for fake_str in facets {
@@ -203,35 +197,31 @@ impl MoreLikeThis {
                 }
             }
             FieldType::Str(text_options) => {
+                let mut tokenizer_opt = text_options
+                    .get_indexing_options()
+                    .map(|options| options.tokenizer())
+                    .and_then(|tokenizer_name| tokenizer_manager.get(tokenizer_name));
+
+                let sink = &mut |token: &Token| {
+                    if !self.is_noise_word(token.text.clone()) {
+                        let term = Term::from_field_text(field, &token.text);
+                        *term_frequencies.entry(term).or_insert(0) += 1;
+                    }
+                };
+
+                // TOOD: Validate these changed align with the HEAD branch.
                 for value in values {
-                    match value {
-                        Value::PreTokStr(tok_str) => {
-                            let mut token_stream = PreTokenizedStream::from(tok_str.clone());
-                            token_stream.process(&mut |token| {
-                                if !self.is_noise_word(token.text.clone()) {
-                                    let term = Term::from_field_text(field, &token.text);
-                                    *term_frequencies.entry(term).or_insert(0) += 1;
-                                }
-                            });
-                        }
-                        Value::Str(ref text) => {
-                            if let Some(mut tokenizer) = text_options
-                                .get_indexing_options()
-                                .map(|text_indexing_options| {
-                                    text_indexing_options.tokenizer().to_string()
-                                })
-                                .and_then(|tokenizer_name| tokenizer_manager.get(&tokenizer_name))
-                            {
-                                let mut token_stream = tokenizer.token_stream(text);
-                                token_stream.process(&mut |token| {
-                                    if !self.is_noise_word(token.text.clone()) {
-                                        let term = Term::from_field_text(field, &token.text);
-                                        *term_frequencies.entry(term).or_insert(0) += 1;
-                                    }
-                                });
-                            }
-                        }
-                        _ => (),
+                    if let Some(text) = value.as_str() {
+                        let tokenizer = match &mut tokenizer_opt {
+                            None => continue,
+                            Some(tokenizer) => tokenizer,
+                        };
+
+                        let mut token_stream = tokenizer.token_stream(text);
+                        token_stream.process(sink);
+                    } else if let Some(tok_str) = value.as_pre_tokenized_text() {
+                        let mut token_stream = PreTokenizedStream::from(tok_str.clone());
+                        token_stream.process(sink);
                     }
                 }
             }
@@ -248,7 +238,7 @@ impl MoreLikeThis {
             }
             FieldType::Date(_) => {
                 for value in values {
-                    let timestamp = value.as_date().ok_or_else(|| {
+                    let timestamp = value.as_datetime().ok_or_else(|| {
                         TantivyError::InvalidArgument("invalid value".to_string())
                     })?;
                     let term = Term::from_field_date(field, timestamp);

src/query/more_like_this/query.rs

@@ -1,6 +1,8 @@
+use std::fmt::Debug;
+
 use super::MoreLikeThis;
 use crate::query::{EnableScoring, Query, Weight};
-use crate::schema::{Field, Value};
+use crate::schema::{Field, OwnedValue};
 use crate::DocAddress;
 
 /// A query that matches all of the documents similar to a document
@@ -28,10 +30,10 @@ pub struct MoreLikeThisQuery {
     target: TargetDocument,
 }
 
-#[derive(Debug, PartialEq, Clone)]
+#[derive(Debug, Clone, PartialEq)]
 enum TargetDocument {
-    DocumentAdress(DocAddress),
-    DocumentFields(Vec<(Field, Vec<Value>)>),
+    DocumentAddress(DocAddress),
+    DocumentFields(Vec<(Field, Vec<OwnedValue>)>),
 }
 
 impl MoreLikeThisQuery {
@@ -51,14 +53,20 @@ impl Query for MoreLikeThisQuery {
             }
         };
         match &self.target {
-            TargetDocument::DocumentAdress(doc_address) => self
+            TargetDocument::DocumentAddress(doc_address) => self
                 .mlt
                 .query_with_document(searcher, *doc_address)?
                 .weight(enable_scoring),
-            TargetDocument::DocumentFields(doc_fields) => self
-                .mlt
-                .query_with_document_fields(searcher, doc_fields)?
-                .weight(enable_scoring),
+            TargetDocument::DocumentFields(doc_fields) => {
+                let values = doc_fields
+                    .iter()
+                    .map(|(field, values)| (*field, values.iter().collect::<Vec<&OwnedValue>>()))
+                    .collect::<Vec<_>>();
+
+                self.mlt
+                    .query_with_document_fields(searcher, &values)?
+                    .weight(enable_scoring)
+            }
         }
     }
 }
@@ -156,7 +164,7 @@ impl MoreLikeThisQueryBuilder {
     pub fn with_document(self, doc_address: DocAddress) -> MoreLikeThisQuery {
         MoreLikeThisQuery {
             mlt: self.mlt,
-            target: TargetDocument::DocumentAdress(doc_address),
+            target: TargetDocument::DocumentAddress(doc_address),
         }
     }
 
@@ -167,7 +175,10 @@ impl MoreLikeThisQueryBuilder {
     /// that will be used to compose the resulting query.
     /// This interface is meant to be used when you want to provide your own set of fields
     /// not necessarily from a specific document.
-    pub fn with_document_fields(self, doc_fields: Vec<(Field, Vec<Value>)>) -> MoreLikeThisQuery {
+    pub fn with_document_fields(
+        self,
+        doc_fields: Vec<(Field, Vec<OwnedValue>)>,
+    ) -> MoreLikeThisQuery {
         MoreLikeThisQuery {
             mlt: self.mlt,
             target: TargetDocument::DocumentFields(doc_fields),
@@ -180,7 +191,7 @@ mod tests {
     use super::{MoreLikeThisQuery, TargetDocument};
     use crate::collector::TopDocs;
     use crate::schema::{Schema, STORED, TEXT};
-    use crate::{DocAddress, Index};
+    use crate::{DocAddress, Index, IndexWriter};
 
     fn create_test_index() -> crate::Result<Index> {
         let mut schema_builder = Schema::builder();
@@ -188,7 +199,7 @@ mod tests {
         let body = schema_builder.add_text_field("body", TEXT | STORED);
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_for_tests().unwrap();
+        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
         index_writer.add_document(doc!(title => "aaa", body => "the old man and the sea"))?;
         index_writer.add_document(doc!(title => "bbb", body => "an old man sailing on the sea"))?;
         index_writer.add_document(doc!(title => "ccc", body=> "send this message to alice"))?;
@@ -236,7 +247,7 @@ mod tests {
         );
         assert_eq!(
             query.target,
-            TargetDocument::DocumentAdress(DocAddress::new(1, 2))
+            TargetDocument::DocumentAddress(DocAddress::new(1, 2))
         );
     }
 

src/query/phrase_prefix_query/mod.rs

@@ -6,7 +6,7 @@ pub use phrase_prefix_query::PhrasePrefixQuery;
 pub use phrase_prefix_scorer::PhrasePrefixScorer;
 pub use phrase_prefix_weight::PhrasePrefixWeight;
 
-fn prefix_end(prefix_start: &[u8]) -> Option<Vec<u8>> {
+pub(crate) fn prefix_end(prefix_start: &[u8]) -> Option<Vec<u8>> {
     let mut res = prefix_start.to_owned();
     while !res.is_empty() {
         let end = res.len() - 1;

src/query/phrase_prefix_query/phrase_prefix_weight.rs

@@ -161,7 +161,7 @@ mod tests {
     use crate::docset::TERMINATED;
     use crate::query::{EnableScoring, PhrasePrefixQuery, Query};
     use crate::schema::{Schema, TEXT};
-    use crate::{DocSet, Term};
+    use crate::{DocSet, IndexWriter, Term};
 
     pub fn create_index(texts: &[&'static str]) -> crate::Result<Index> {
         let mut schema_builder = Schema::builder();
@@ -169,7 +169,7 @@ mod tests {
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
         {
-            let mut index_writer = index.writer_for_tests()?;
+            let mut index_writer: IndexWriter = index.writer_for_tests()?;
             for &text in texts {
                 let doc = doc!(text_field=>text);
                 index_writer.add_document(doc)?;

src/query/phrase_query/mod.rs

@@ -17,7 +17,7 @@ pub mod tests {
     use crate::core::Index;
     use crate::query::{EnableScoring, QueryParser, Weight};
     use crate::schema::{Schema, Term, TEXT};
-    use crate::{assert_nearly_equals, DocAddress, DocId, TERMINATED};
+    use crate::{assert_nearly_equals, DocAddress, DocId, IndexWriter, TERMINATED};
 
     pub fn create_index(texts: &[&'static str]) -> crate::Result<Index> {
         let mut schema_builder = Schema::builder();
@@ -25,7 +25,7 @@ pub mod tests {
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
         {
-            let mut index_writer = index.writer_for_tests()?;
+            let mut index_writer: IndexWriter = index.writer_for_tests()?;
             for &text in texts {
                 let doc = doc!(text_field=>text);
                 index_writer.add_document(doc)?;
@@ -135,7 +135,7 @@ pub mod tests {
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
         {
-            let mut index_writer = index.writer_for_tests()?;
+            let mut index_writer: IndexWriter = index.writer_for_tests()?;
             index_writer.add_document(doc!(text_field=>"a b c"))?;
             index_writer.commit()?;
         }
@@ -278,7 +278,7 @@ pub mod tests {
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
         {
-            let mut index_writer = index.writer_for_tests()?;
+            let mut index_writer: IndexWriter = index.writer_for_tests()?;
             index_writer.add_document(doc!(text_field=>"b"))?;
             index_writer.add_document(doc!(text_field=>"a b"))?;
             index_writer.add_document(doc!(text_field=>"b a"))?;
@@ -310,7 +310,7 @@ pub mod tests {
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
         {
-            let mut index_writer = index.writer_for_tests()?;
+            let mut index_writer: IndexWriter = index.writer_for_tests()?;
             index_writer.add_document(doc!(text_field=>"a b c d e f g h"))?;
             index_writer.commit()?;
         }
@@ -348,7 +348,7 @@ pub mod tests {
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
         {
-            let mut index_writer = index.writer_for_tests()?;
+            let mut index_writer: IndexWriter = index.writer_for_tests()?;
             index_writer.add_document(doc!(json_field=>json!({
                 "text": "elliot smith the happy who"
             })))?;

src/query/query_parser/query_parser.rs

@@ -847,6 +847,12 @@ impl QueryParser {
                 }));
                 (Some(logical_ast), errors)
             }
+            UserInputLeaf::Exists { .. } => (
+                None,
+                vec![QueryParserError::UnsupportedQuery(
+                    "Range query need to target a specific field.".to_string(),
+                )],
+            ),
         }
     }
 }

src/query/range_query/range_query.rs

@@ -41,14 +41,14 @@ use crate::{DateTime, DocId, Score};
 /// use tantivy::collector::Count;
 /// use tantivy::query::RangeQuery;
 /// use tantivy::schema::{Schema, INDEXED};
-/// use tantivy::{doc, Index};
+/// use tantivy::{doc, Index, IndexWriter};
 /// # fn test() -> tantivy::Result<()> {
 /// let mut schema_builder = Schema::builder();
 /// let year_field = schema_builder.add_u64_field("year", INDEXED);
 /// let schema = schema_builder.build();
 ///
 /// let index = Index::create_in_ram(schema);
-/// let mut index_writer = index.writer_with_num_threads(1, 20_000_000)?;
+/// let mut index_writer: IndexWriter = index.writer_with_num_threads(1, 20_000_000)?;
 /// for year in 1950u64..2017u64 {
 ///     let num_docs_within_year = 10 + (year - 1950) * (year - 1950);
 ///     for _ in 0..num_docs_within_year {
@@ -474,8 +474,10 @@ mod tests {
     use crate::collector::{Count, TopDocs};
     use crate::indexer::NoMergePolicy;
     use crate::query::QueryParser;
-    use crate::schema::{Document, Field, IntoIpv6Addr, Schema, FAST, INDEXED, STORED, TEXT};
-    use crate::{doc, Index};
+    use crate::schema::{
+        Field, IntoIpv6Addr, Schema, TantivyDocument, FAST, INDEXED, STORED, TEXT,
+    };
+    use crate::{doc, Index, IndexWriter};
 
     #[test]
     fn test_range_query_simple() -> crate::Result<()> {
@@ -552,7 +554,7 @@ mod tests {
             index_writer.set_merge_policy(Box::new(NoMergePolicy));
 
             for i in 1..100 {
-                let mut doc = Document::new();
+                let mut doc = TantivyDocument::new();
                 for j in 1..100 {
                     if i % j == 0 {
                         doc.add_i64(int_field, j as i64);
@@ -617,7 +619,7 @@ mod tests {
             let mut index_writer = index.writer_with_num_threads(1, 60_000_000).unwrap();
             let mut docs = Vec::new();
             for i in 1..100 {
-                let mut doc = Document::new();
+                let mut doc = TantivyDocument::new();
                 for j in 1..100 {
                     if i % j == 0 {
                         doc.add_f64(float_field, j as f64);
@@ -722,7 +724,7 @@ mod tests {
         let ip_addr_2 = IpAddr::from_str("127.0.0.20").unwrap().into_ipv6_addr();
 
         {
-            let mut index_writer = index.writer_for_tests().unwrap();
+            let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
             for _ in 0..1_000 {
                 index_writer
                     .add_document(doc!(

src/query/range_query/range_query_ip_fastfield.rs

@@ -88,7 +88,7 @@ pub mod tests {
     use crate::collector::Count;
     use crate::query::QueryParser;
     use crate::schema::{Schema, FAST, INDEXED, STORED, STRING};
-    use crate::Index;
+    use crate::{Index, IndexWriter};
 
     #[derive(Clone, Debug)]
     pub struct Doc {
@@ -158,7 +158,7 @@ pub mod tests {
         let ips_field = schema_builder.add_ip_addr_field("ips", FAST | INDEXED);
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
-        let mut writer = index.writer_for_tests().unwrap();
+        let mut writer: IndexWriter = index.writer_for_tests().unwrap();
         let ip_addrs: Vec<Ipv6Addr> = [1000, 2000, 3000]
             .into_iter()
             .map(Ipv6Addr::from_u128)

src/query/range_query/range_query_u64_fastfield.rs

@@ -76,12 +76,14 @@ impl Weight for FastFieldRangeWeight {
         else {
             return Ok(Box::new(EmptyScorer));
         };
+        #[allow(clippy::reversed_empty_ranges)]
         let value_range = bound_to_value_range(
             &self.lower_bound,
             &self.upper_bound,
             column.min_value(),
             column.max_value(),
-        );
+        )
+        .unwrap_or(1..=0); // empty range
         if value_range.is_empty() {
             return Ok(Box::new(EmptyScorer));
         }
@@ -102,15 +104,17 @@ impl Weight for FastFieldRangeWeight {
     }
 }
 
+// Returns None, if the range cannot be converted to a inclusive range (which equals to a empty
+// range).
 fn bound_to_value_range<T: MonotonicallyMappableToU64>(
     lower_bound: &Bound<T>,
     upper_bound: &Bound<T>,
     min_value: T,
     max_value: T,
-) -> RangeInclusive<T> {
+) -> Option<RangeInclusive<T>> {
     let mut start_value = match lower_bound {
         Bound::Included(val) => *val,
-        Bound::Excluded(val) => T::from_u64(val.to_u64() + 1),
+        Bound::Excluded(val) => T::from_u64(val.to_u64().checked_add(1)?),
         Bound::Unbounded => min_value,
     };
     if start_value.partial_cmp(&min_value) == Some(std::cmp::Ordering::Less) {
@@ -118,10 +122,10 @@ fn bound_to_value_range<T: MonotonicallyMappableToU64>(
     }
     let end_value = match upper_bound {
         Bound::Included(val) => *val,
-        Bound::Excluded(val) => T::from_u64(val.to_u64() - 1),
+        Bound::Excluded(val) => T::from_u64(val.to_u64().checked_sub(1)?),
         Bound::Unbounded => max_value,
     };
-    start_value..=end_value
+    Some(start_value..=end_value)
 }
 
 #[cfg(test)]
@@ -137,7 +141,7 @@ pub mod tests {
     use crate::query::range_query::range_query_u64_fastfield::FastFieldRangeWeight;
     use crate::query::{QueryParser, Weight};
     use crate::schema::{NumericOptions, Schema, SchemaBuilder, FAST, INDEXED, STORED, STRING};
-    use crate::{Index, TERMINATED};
+    use crate::{Index, IndexWriter, TERMINATED};
 
     #[derive(Clone, Debug)]
     pub struct Doc {
@@ -205,7 +209,7 @@ pub mod tests {
         let field = schema_builder.add_u64_field("test_field", FAST);
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
-        let mut writer = index.writer_for_tests().unwrap();
+        let mut writer: IndexWriter = index.writer_for_tests().unwrap();
         writer.add_document(doc!(field=>52_000u64)).unwrap();
         writer.commit().unwrap();
         let searcher = index.reader().unwrap().searcher();
@@ -295,6 +299,9 @@ pub mod tests {
         let gen_query_inclusive = |field: &str, range: RangeInclusive<u64>| {
             format!("{}:[{} TO {}]", field, range.start(), range.end())
         };
+        let gen_query_exclusive = |field: &str, range: RangeInclusive<u64>| {
+            format!("{}:{{{} TO {}}}", field, range.start(), range.end())
+        };
 
         let test_sample = |sample_docs: Vec<Doc>| {
             let mut ids: Vec<u64> = sample_docs.iter().map(|doc| doc.id).collect();
@@ -310,6 +317,20 @@ pub mod tests {
             let query = gen_query_inclusive("ids", ids[0]..=ids[1]);
             assert_eq!(get_num_hits(query_from_text(&query)), expected_num_hits);
 
+            // Exclusive range
+            let expected_num_hits = docs
+                .iter()
+                .filter(|doc| {
+                    (ids[0].saturating_add(1)..=ids[1].saturating_sub(1)).contains(&doc.id)
+                })
+                .count();
+
+            let query = gen_query_exclusive("id", ids[0]..=ids[1]);
+            assert_eq!(get_num_hits(query_from_text(&query)), expected_num_hits);
+
+            let query = gen_query_exclusive("ids", ids[0]..=ids[1]);
+            assert_eq!(get_num_hits(query_from_text(&query)), expected_num_hits);
+
             // Intersection search
             let id_filter = sample_docs[0].id_name.to_string();
             let expected_num_hits = docs

src/query/regex_query.rs

@@ -18,7 +18,7 @@ use crate::schema::Field;
 /// use tantivy::collector::Count;
 /// use tantivy::query::RegexQuery;
 /// use tantivy::schema::{Schema, TEXT};
-/// use tantivy::{doc, Index, Term};
+/// use tantivy::{doc, Index, IndexWriter, Term};
 ///
 /// # fn test() -> tantivy::Result<()> {
 /// let mut schema_builder = Schema::builder();
@@ -26,7 +26,7 @@ use crate::schema::Field;
 /// let schema = schema_builder.build();
 /// let index = Index::create_in_ram(schema);
 /// {
-///     let mut index_writer = index.writer(3_000_000)?;
+///     let mut index_writer: IndexWriter = index.writer(15_000_000)?;
 ///     index_writer.add_document(doc!(
 ///         title => "The Name of the Wind",
 ///     ))?;
@@ -95,7 +95,7 @@ mod test {
     use super::RegexQuery;
     use crate::collector::TopDocs;
     use crate::schema::{Field, Schema, TEXT};
-    use crate::{assert_nearly_equals, Index, IndexReader};
+    use crate::{assert_nearly_equals, Index, IndexReader, IndexWriter};
 
     fn build_test_index() -> crate::Result<(IndexReader, Field)> {
         let mut schema_builder = Schema::builder();
@@ -103,7 +103,7 @@ mod test {
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
         {
-            let mut index_writer = index.writer_for_tests().unwrap();
+            let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
             index_writer.add_document(doc!(
                 country_field => "japan",
             ))?;