Store List of Fields in Segment

Fiels may be encoded in the columnar storage or in the inverted index for JSON fields. Add a new Segment file that contains the list of fields (schema + encoded)
improve indexing benchmark (#2275 )
2025-12-29 05:22:55 +00:00 · 2023-12-13 15:52:41 +08:00 · 2023-12-11 09:04:42 +01:00 · 2023-12-04 15:13:15 +01:00 · 2023-12-01 13:46:12 +01:00 · 2023-12-01 13:45:52 +01:00
179 changed files with 9215 additions and 3116 deletions
--- a/.github/workflows/coverage.yml
+++ b/.github/workflows/coverage.yml
@@ -3,8 +3,6 @@ name: Coverage
 on:
  push:
    branches: [main]
-  pull_request:
-    branches: [main]

 # Ensures that we cancel running jobs for the same PR / same workflow.
 concurrency:
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -39,6 +39,13 @@ jobs:

    - name: Check Formatting
      run: cargo +nightly fmt --all -- --check
+    
+    - name: Check Stable Compilation
+      run: cargo build --all-features
+
+    
+    - name: Check Bench Compilation
+      run: cargo +nightly bench --no-run --profile=dev --all-features

    - uses: actions-rs/clippy-check@v1
      with:
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,9 @@
+Tantivy 0.21.1
+================================
+#### Bugfixes
+- Range queries on fast fields with less values on that field than documents had an invalid end condition, leading to missing results. [#2226](https://github.com/quickwit-oss/tantivy/issues/2226)(@appaquet @PSeitz)
+- Increase the minimum memory budget from 3MB to 15MB to avoid single doc segments (API fix). [#2176](https://github.com/quickwit-oss/tantivy/issues/2176)(@PSeitz)
+
 Tantivy 0.21
 ================================
 #### Bugfixes
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "tantivy"
-version = "0.21.0"
+version = "0.22.0-dev"
 authors = ["Paul Masurel <paul.masurel@gmail.com>"]
 license = "MIT"
 categories = ["database-implementations", "data-structures"]
@@ -22,36 +22,34 @@ crc32fast = "1.3.2"
 once_cell = "1.10.0"
 regex = { version = "1.5.5", default-features = false, features = ["std", "unicode"] }
 aho-corasick = "1.0"
-tantivy-fst = "0.4.0"
-memmap2 = { version = "0.7.1", optional = true }
+tantivy-fst = "0.5"
+memmap2 = { version = "0.9.0", optional = true }
 lz4_flex = { version = "0.11", default-features = false, optional = true }
-zstd = { version = "0.12", optional = true, default-features = false }
+zstd = { version = "0.13", default-features = false }
 tempfile = { version = "3.3.0", optional = true }
 log = "0.4.16"
 serde = { version = "1.0.136", features = ["derive"] }
 serde_json = "1.0.79"
 num_cpus = "1.13.1"
-fs4 = { version = "0.6.3", optional = true }
+fs4 = { version = "0.7.0", optional = true }
 levenshtein_automata = "0.2.1"
 uuid = { version = "1.0.0", features = ["v4", "serde"] }
 crossbeam-channel = "0.5.4"
 rust-stemmers = "1.2.0"
 downcast-rs = "1.2.0"
-bitpacking = { version = "0.8.4", default-features = false, features = ["bitpacker4x"] }
+bitpacking = { version = "0.9.2", default-features = false, features = ["bitpacker4x"] }
 census = "0.4.0"
 rustc-hash = "1.1.0"
 thiserror = "1.0.30"
 htmlescape = "0.3.1"
 fail = { version = "0.5.0", optional = true }
-murmurhash32 = "0.3.0"
 time = { version = "0.3.10", features = ["serde-well-known"] }
 smallvec = "1.8.0"
 rayon = "1.5.2"
-lru = "0.11.0"
+lru = "0.12.0"
 fastdivide = "0.4.0"
-itertools = "0.11.0"
+itertools = "0.12.0"
 measure_time = "0.8.2"
-async-trait = "0.1.53"
 arc-swap = "1.5.0"

 columnar = { version= "0.2", path="./columnar", package ="tantivy-columnar" }
@@ -63,6 +61,7 @@ common = { version= "0.6", path = "./common/", package = "tantivy-common" }
 tokenizer-api = { version= "0.2", path="./tokenizer-api", package="tantivy-tokenizer-api" }
 sketches-ddsketch = { version = "0.2.1", features = ["use_serde"] }
 futures-util = { version = "0.3.28", optional = true }
+fnv = "1.0.7"

 [target.'cfg(windows)'.dependencies]
 winapi = "0.3.9"
@@ -74,15 +73,13 @@ matches = "0.1.9"
 pretty_assertions = "1.2.1"
 proptest = "1.0.0"
 test-log = "0.2.10"
-env_logger = "0.10.0"
 futures = "0.3.21"
 paste = "1.0.11"
 more-asserts = "0.3.1"
 rand_distr = "0.4.3"

 [target.'cfg(not(windows))'.dev-dependencies]
-criterion = "0.5"
-pprof = { git = "https://github.com/PSeitz/pprof-rs/", rev = "53af24b", features = ["flamegraph", "criterion"] } # temp fork that works with criterion 0.5
+criterion = { version = "0.5", default-features = false }

 [dev-dependencies.fail]
 version = "0.5.0"
@@ -108,13 +105,18 @@ mmap = ["fs4", "tempfile", "memmap2"]
 stopwords = []

 lz4-compression = ["lz4_flex"]
-zstd-compression = ["zstd"]
+zstd-compression = []

 failpoints = ["fail", "fail/failpoints"]
 unstable = [] # useful for benches.

 quickwit = ["sstable", "futures-util"]

+# Compares only the hash of a string when indexing data. 
+# Increases indexing speed, but may lead to extremely rare missing terms, when there's a hash collision.
+# Uses 64bit ahash.
+compare_hash_only = ["stacker/compare_hash_only"]
+
 [workspace]
 members = ["query-grammar", "bitpacker", "common", "ownedbytes", "stacker", "sstable", "tokenizer-api", "columnar"]

--- a/benches/index-bench.rs
+++ b/benches/index-bench.rs
@@ -1,14 +1,99 @@
-use criterion::{criterion_group, criterion_main, Criterion, Throughput};
-use pprof::criterion::{Output, PProfProfiler};
-use tantivy::schema::{FAST, INDEXED, STORED, STRING, TEXT};
-use tantivy::Index;
+use criterion::{criterion_group, criterion_main, BatchSize, Bencher, Criterion, Throughput};
+use tantivy::schema::{TantivyDocument, FAST, INDEXED, STORED, STRING, TEXT};
+use tantivy::{tokenizer, Index, IndexWriter};

 const HDFS_LOGS: &str = include_str!("hdfs.json");
 const GH_LOGS: &str = include_str!("gh.json");
 const WIKI: &str = include_str!("wiki.json");

-fn get_lines(input: &str) -> Vec<&str> {
-    input.trim().split('\n').collect()
+fn benchmark(
+    b: &mut Bencher,
+    input: &str,
+    schema: tantivy::schema::Schema,
+    commit: bool,
+    parse_json: bool,
+    is_dynamic: bool,
+) {
+    if is_dynamic {
+        benchmark_dynamic_json(b, input, schema, commit, parse_json)
+    } else {
+        _benchmark(b, input, schema, commit, parse_json, |schema, doc_json| {
+            TantivyDocument::parse_json(&schema, doc_json).unwrap()
+        })
+    }
+}
+
+fn get_index(schema: tantivy::schema::Schema) -> Index {
+    let mut index = Index::create_in_ram(schema.clone());
+    let ff_tokenizer_manager = tokenizer::TokenizerManager::default();
+    ff_tokenizer_manager.register(
+        "raw",
+        tokenizer::TextAnalyzer::builder(tokenizer::RawTokenizer::default())
+            .filter(tokenizer::RemoveLongFilter::limit(255))
+            .build(),
+    );
+    index.set_fast_field_tokenizers(ff_tokenizer_manager.clone());
+    index
+}
+
+fn _benchmark(
+    b: &mut Bencher,
+    input: &str,
+    schema: tantivy::schema::Schema,
+    commit: bool,
+    include_json_parsing: bool,
+    create_doc: impl Fn(&tantivy::schema::Schema, &str) -> TantivyDocument,
+) {
+    if include_json_parsing {
+        let lines: Vec<&str> = input.trim().split('\n').collect();
+        b.iter(|| {
+            let index = get_index(schema.clone());
+            let mut index_writer: IndexWriter =
+                index.writer_with_num_threads(1, 100_000_000).unwrap();
+            for doc_json in &lines {
+                let doc = create_doc(&schema, doc_json);
+                index_writer.add_document(doc).unwrap();
+            }
+            if commit {
+                index_writer.commit().unwrap();
+            }
+        })
+    } else {
+        let docs: Vec<_> = input
+            .trim()
+            .split('\n')
+            .map(|doc_json| create_doc(&schema, doc_json))
+            .collect();
+        b.iter_batched(
+            || docs.clone(),
+            |docs| {
+                let index = get_index(schema.clone());
+                let mut index_writer: IndexWriter =
+                    index.writer_with_num_threads(1, 100_000_000).unwrap();
+                for doc in docs {
+                    index_writer.add_document(doc).unwrap();
+                }
+                if commit {
+                    index_writer.commit().unwrap();
+                }
+            },
+            BatchSize::SmallInput,
+        )
+    }
+}
+fn benchmark_dynamic_json(
+    b: &mut Bencher,
+    input: &str,
+    schema: tantivy::schema::Schema,
+    commit: bool,
+    parse_json: bool,
+) {
+    let json_field = schema.get_field("json").unwrap();
+    _benchmark(b, input, schema, commit, parse_json, |_schema, doc_json| {
+        let json_val: serde_json::Map<String, serde_json::Value> =
+            serde_json::from_str(doc_json).unwrap();
+        tantivy::doc!(json_field=>json_val)
+    })
 }

 pub fn hdfs_index_benchmark(c: &mut Criterion) {
@@ -19,7 +104,14 @@ pub fn hdfs_index_benchmark(c: &mut Criterion) {
        schema_builder.add_text_field("severity", STRING);
        schema_builder.build()
    };
-    let schema_with_store = {
+    let schema_only_fast = {
+        let mut schema_builder = tantivy::schema::SchemaBuilder::new();
+        schema_builder.add_u64_field("timestamp", FAST);
+        schema_builder.add_text_field("body", FAST);
+        schema_builder.add_text_field("severity", FAST);
+        schema_builder.build()
+    };
+    let _schema_with_store = {
        let mut schema_builder = tantivy::schema::SchemaBuilder::new();
        schema_builder.add_u64_field("timestamp", INDEXED | STORED);
        schema_builder.add_text_field("body", TEXT | STORED);
@@ -28,74 +120,39 @@ pub fn hdfs_index_benchmark(c: &mut Criterion) {
    };
    let dynamic_schema = {
        let mut schema_builder = tantivy::schema::SchemaBuilder::new();
-        schema_builder.add_json_field("json", TEXT);
+        schema_builder.add_json_field("json", TEXT | FAST);
        schema_builder.build()
    };

    let mut group = c.benchmark_group("index-hdfs");
    group.throughput(Throughput::Bytes(HDFS_LOGS.len() as u64));
    group.sample_size(20);
-    group.bench_function("index-hdfs-no-commit", |b| {
-        let lines = get_lines(HDFS_LOGS);
-        b.iter(|| {
-            let index = Index::create_in_ram(schema.clone());
-            let index_writer = index.writer_with_num_threads(1, 100_000_000).unwrap();
-            for doc_json in &lines {
-                let doc = schema.parse_document(doc_json).unwrap();
-                index_writer.add_document(doc).unwrap();
+
+    let benches = [
+        ("only-indexed-".to_string(), schema, false),
+        //("stored-".to_string(), _schema_with_store, false),
+        ("only-fast-".to_string(), schema_only_fast, false),
+        ("dynamic-".to_string(), dynamic_schema, true),
+    ];
+
+    for (prefix, schema, is_dynamic) in benches {
+        for commit in [false, true] {
+            let suffix = if commit { "with-commit" } else { "no-commit" };
+            for parse_json in [false] {
+                // for parse_json in [false, true] {
+                let suffix = if parse_json {
+                    format!("{}-with-json-parsing", suffix)
+                } else {
+                    format!("{}", suffix)
+                };
+
+                let bench_name = format!("{}{}", prefix, suffix);
+                group.bench_function(bench_name, |b| {
+                    benchmark(b, HDFS_LOGS, schema.clone(), commit, parse_json, is_dynamic)
+                });
            }
-        })
-    });
-    group.bench_function("index-hdfs-with-commit", |b| {
-        let lines = get_lines(HDFS_LOGS);
-        b.iter(|| {
-            let index = Index::create_in_ram(schema.clone());
-            let mut index_writer = index.writer_with_num_threads(1, 100_000_000).unwrap();
-            for doc_json in &lines {
-                let doc = schema.parse_document(doc_json).unwrap();
-                index_writer.add_document(doc).unwrap();
-            }
-            index_writer.commit().unwrap();
-        })
-    });
-    group.bench_function("index-hdfs-no-commit-with-docstore", |b| {
-        let lines = get_lines(HDFS_LOGS);
-        b.iter(|| {
-            let index = Index::create_in_ram(schema_with_store.clone());
-            let index_writer = index.writer_with_num_threads(1, 100_000_000).unwrap();
-            for doc_json in &lines {
-                let doc = schema.parse_document(doc_json).unwrap();
-                index_writer.add_document(doc).unwrap();
-            }
-        })
-    });
-    group.bench_function("index-hdfs-with-commit-with-docstore", |b| {
-        let lines = get_lines(HDFS_LOGS);
-        b.iter(|| {
-            let index = Index::create_in_ram(schema_with_store.clone());
-            let mut index_writer = index.writer_with_num_threads(1, 100_000_000).unwrap();
-            for doc_json in &lines {
-                let doc = schema.parse_document(doc_json).unwrap();
-                index_writer.add_document(doc).unwrap();
-            }
-            index_writer.commit().unwrap();
-        })
-    });
-    group.bench_function("index-hdfs-no-commit-json-without-docstore", |b| {
-        let lines = get_lines(HDFS_LOGS);
-        b.iter(|| {
-            let index = Index::create_in_ram(dynamic_schema.clone());
-            let json_field = dynamic_schema.get_field("json").unwrap();
-            let mut index_writer = index.writer_with_num_threads(1, 100_000_000).unwrap();
-            for doc_json in &lines {
-                let json_val: serde_json::Map<String, serde_json::Value> =
-                    serde_json::from_str(doc_json).unwrap();
-                let doc = tantivy::doc!(json_field=>json_val);
-                index_writer.add_document(doc).unwrap();
-            }
-            index_writer.commit().unwrap();
-        })
-    });
+        }
+    }
 }

 pub fn gh_index_benchmark(c: &mut Criterion) {
@@ -104,38 +161,24 @@ pub fn gh_index_benchmark(c: &mut Criterion) {
        schema_builder.add_json_field("json", TEXT | FAST);
        schema_builder.build()
    };
+    let dynamic_schema_fast = {
+        let mut schema_builder = tantivy::schema::SchemaBuilder::new();
+        schema_builder.add_json_field("json", FAST);
+        schema_builder.build()
+    };

    let mut group = c.benchmark_group("index-gh");
    group.throughput(Throughput::Bytes(GH_LOGS.len() as u64));

    group.bench_function("index-gh-no-commit", |b| {
-        let lines = get_lines(GH_LOGS);
-        b.iter(|| {
-            let json_field = dynamic_schema.get_field("json").unwrap();
-            let index = Index::create_in_ram(dynamic_schema.clone());
-            let index_writer = index.writer_with_num_threads(1, 100_000_000).unwrap();
-            for doc_json in &lines {
-                let json_val: serde_json::Map<String, serde_json::Value> =
-                    serde_json::from_str(doc_json).unwrap();
-                let doc = tantivy::doc!(json_field=>json_val);
-                index_writer.add_document(doc).unwrap();
-            }
-        })
+        benchmark_dynamic_json(b, GH_LOGS, dynamic_schema.clone(), false, false)
    });
-    group.bench_function("index-gh-with-commit", |b| {
-        let lines = get_lines(GH_LOGS);
-        b.iter(|| {
-            let json_field = dynamic_schema.get_field("json").unwrap();
-            let index = Index::create_in_ram(dynamic_schema.clone());
-            let mut index_writer = index.writer_with_num_threads(1, 100_000_000).unwrap();
-            for doc_json in &lines {
-                let json_val: serde_json::Map<String, serde_json::Value> =
-                    serde_json::from_str(doc_json).unwrap();
-                let doc = tantivy::doc!(json_field=>json_val);
-                index_writer.add_document(doc).unwrap();
-            }
-            index_writer.commit().unwrap();
-        })
+    group.bench_function("index-gh-fast", |b| {
+        benchmark_dynamic_json(b, GH_LOGS, dynamic_schema_fast.clone(), false, false)
+    });
+
+    group.bench_function("index-gh-fast-with-commit", |b| {
+        benchmark_dynamic_json(b, GH_LOGS, dynamic_schema_fast.clone(), true, false)
    });
 }

@@ -150,33 +193,10 @@ pub fn wiki_index_benchmark(c: &mut Criterion) {
    group.throughput(Throughput::Bytes(WIKI.len() as u64));

    group.bench_function("index-wiki-no-commit", |b| {
-        let lines = get_lines(WIKI);
-        b.iter(|| {
-            let json_field = dynamic_schema.get_field("json").unwrap();
-            let index = Index::create_in_ram(dynamic_schema.clone());
-            let index_writer = index.writer_with_num_threads(1, 100_000_000).unwrap();
-            for doc_json in &lines {
-                let json_val: serde_json::Map<String, serde_json::Value> =
-                    serde_json::from_str(doc_json).unwrap();
-                let doc = tantivy::doc!(json_field=>json_val);
-                index_writer.add_document(doc).unwrap();
-            }
-        })
+        benchmark_dynamic_json(b, WIKI, dynamic_schema.clone(), false, false)
    });
    group.bench_function("index-wiki-with-commit", |b| {
-        let lines = get_lines(WIKI);
-        b.iter(|| {
-            let json_field = dynamic_schema.get_field("json").unwrap();
-            let index = Index::create_in_ram(dynamic_schema.clone());
-            let mut index_writer = index.writer_with_num_threads(1, 100_000_000).unwrap();
-            for doc_json in &lines {
-                let json_val: serde_json::Map<String, serde_json::Value> =
-                    serde_json::from_str(doc_json).unwrap();
-                let doc = tantivy::doc!(json_field=>json_val);
-                index_writer.add_document(doc).unwrap();
-            }
-            index_writer.commit().unwrap();
-        })
+        benchmark_dynamic_json(b, WIKI, dynamic_schema.clone(), true, false)
    });
 }

@@ -187,12 +207,12 @@ criterion_group! {
 }
 criterion_group! {
    name = gh_benches;
-    config = Criterion::default().with_profiler(PProfProfiler::new(100, Output::Flamegraph(None)));
+    config = Criterion::default();
    targets = gh_index_benchmark
 }
 criterion_group! {
    name = wiki_benches;
-    config = Criterion::default().with_profiler(PProfProfiler::new(100, Output::Flamegraph(None)));
+    config = Criterion::default();
    targets = wiki_index_benchmark
 }
 criterion_main!(benches, gh_benches, wiki_benches);
--- a/bitpacker/Cargo.toml
+++ b/bitpacker/Cargo.toml
@@ -15,7 +15,7 @@ homepage = "https://github.com/quickwit-oss/tantivy"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

 [dependencies]
-bitpacking = {version="0.8", default-features=false, features = ["bitpacker1x"]}
+bitpacking = { version = "0.9.2", default-features = false, features = ["bitpacker1x"] }

 [dev-dependencies]
 rand = "0.8"
--- a/bitpacker/src/bitpacker.rs
+++ b/bitpacker/src/bitpacker.rs
@@ -367,7 +367,7 @@ mod test {
            let mut output: Vec<u32> = Vec::new();
            for len in [0, 1, 2, 32, 33, 34, 64] {
                for start_idx in 0u32..32u32 {
-                    output.resize(len as usize, 0);
+                    output.resize(len, 0);
                    bitunpacker.get_batch_u32s(start_idx, &buffer, &mut output);
                    for i in 0..len {
                        let expected = (start_idx + i as u32) & mask;
--- a/columnar/Cargo.toml
+++ b/columnar/Cargo.toml
@@ -9,8 +9,7 @@ description = "column oriented storage for tantivy"
 categories = ["database-implementations", "data-structures", "compression"]

 [dependencies]
-itertools = "0.11.0"
-fnv = "1.0.7"
+itertools = "0.12.0"
 fastdivide = "0.4.0"

 stacker = { version= "0.2", path = "../stacker", package="tantivy-stacker"}
--- a/columnar/columnar-cli/Cargo.toml
+++ b/columnar/columnar-cli/Cargo.toml
@@ -8,7 +8,6 @@ license = "MIT"
 columnar = {path="../", package="tantivy-columnar"}
 serde_json = "1"
 serde_json_borrow = {git="https://github.com/PSeitz/serde_json_borrow/"}
-serde = "1"

 [workspace]
 members = []
--- a/columnar/src/column_index/mod.rs
+++ b/columnar/src/column_index/mod.rs
@@ -1,3 +1,8 @@
+//! # `column_index`
+//!
+//! `column_index` provides rank and select operations to associate positions when not all
+//! documents have exactly one element.
+
 mod merge;
 mod multivalued_index;
 mod optional_index;
@@ -41,10 +46,10 @@ impl ColumnIndex {
    pub fn is_multivalue(&self) -> bool {
        matches!(self, ColumnIndex::Multivalued(_))
    }
-    // Returns the cardinality of the column index.
-    //
-    // By convention, if the column contains no docs, we consider that it is
-    // full.
+    /// Returns the cardinality of the column index.
+    ///
+    /// By convention, if the column contains no docs, we consider that it is
+    /// full.
    #[inline]
    pub fn get_cardinality(&self) -> Cardinality {
        match self {
--- a/columnar/src/column_index/optional_index/tests.rs
+++ b/columnar/src/column_index/optional_index/tests.rs
@@ -215,12 +215,12 @@ mod bench {
        let vals: Vec<RowId> = (0..TOTAL_NUM_VALUES)
            .map(|_| rng.gen_bool(fill_ratio))
            .enumerate()
-            .filter(|(pos, val)| *val)
+            .filter(|(_pos, val)| *val)
            .map(|(pos, _)| pos as RowId)
            .collect();
        serialize_optional_index(&&vals[..], TOTAL_NUM_VALUES, &mut out).unwrap();
-        let codec = open_optional_index(OwnedBytes::new(out)).unwrap();
-        codec
+
+        open_optional_index(OwnedBytes::new(out)).unwrap()
    }

    fn random_range_iterator(
@@ -242,7 +242,7 @@ mod bench {
    }

    fn n_percent_step_iterator(percent: f32, num_values: u32) -> impl Iterator<Item = u32> {
-        let ratio = percent as f32 / 100.0;
+        let ratio = percent / 100.0;
        let step_size = (1f32 / ratio) as u32;
        let deviation = step_size - 1;
        random_range_iterator(0, num_values, step_size, deviation)
--- a/columnar/src/column_index/serialize.rs
+++ b/columnar/src/column_index/serialize.rs
@@ -30,6 +30,7 @@ impl<'a> SerializableColumnIndex<'a> {
    }
 }

+/// Serialize a column index.
 pub fn serialize_column_index(
    column_index: SerializableColumnIndex,
    output: &mut impl Write,
@@ -51,6 +52,7 @@ pub fn serialize_column_index(
    Ok(column_index_num_bytes)
 }

+/// Open a serialized column index.
 pub fn open_column_index(mut bytes: OwnedBytes) -> io::Result<ColumnIndex> {
    if bytes.is_empty() {
        return Err(io::Error::new(
--- a/columnar/src/columnar/column_type.rs
+++ b/columnar/src/columnar/column_type.rs
@@ -58,7 +58,7 @@ impl ColumnType {
        self == &ColumnType::DateTime
    }

-    pub(crate) fn try_from_code(code: u8) -> Result<ColumnType, InvalidData> {
+    pub fn try_from_code(code: u8) -> Result<ColumnType, InvalidData> {
        COLUMN_TYPES.get(code as usize).copied().ok_or(InvalidData)
    }
 }
--- a/columnar/src/columnar/writer/column_writers.rs
+++ b/columnar/src/columnar/writer/column_writers.rs
@@ -269,7 +269,8 @@ impl StrOrBytesColumnWriter {
        dictionaries: &mut [DictionaryBuilder],
        arena: &mut MemoryArena,
    ) {
-        let unordered_id = dictionaries[self.dictionary_id as usize].get_or_allocate_id(bytes);
+        let unordered_id =
+            dictionaries[self.dictionary_id as usize].get_or_allocate_id(bytes, arena);
        self.column_writer.record(doc, unordered_id, arena);
    }

--- a/columnar/src/columnar/writer/mod.rs
+++ b/columnar/src/columnar/writer/mod.rs
@@ -333,12 +333,12 @@ impl ColumnarWriter {
        num_docs: RowId,
        old_to_new_row_ids: Option<&[RowId]>,
        wrt: &mut dyn io::Write,
-    ) -> io::Result<()> {
+    ) -> io::Result<Vec<(String, ColumnType)>> {
        let mut serializer = ColumnarSerializer::new(wrt);
        let mut columns: Vec<(&[u8], ColumnType, Addr)> = self
            .numerical_field_hash_map
            .iter()
-            .map(|(column_name, addr, _)| {
+            .map(|(column_name, addr)| {
                let numerical_column_writer: NumericalColumnWriter =
                    self.numerical_field_hash_map.read(addr);
                let column_type = numerical_column_writer.numerical_type().into();
@@ -348,33 +348,35 @@ impl ColumnarWriter {
        columns.extend(
            self.bytes_field_hash_map
                .iter()
-                .map(|(term, addr, _)| (term, ColumnType::Bytes, addr)),
+                .map(|(term, addr)| (term, ColumnType::Bytes, addr)),
        );
        columns.extend(
            self.str_field_hash_map
                .iter()
-                .map(|(column_name, addr, _)| (column_name, ColumnType::Str, addr)),
+                .map(|(column_name, addr)| (column_name, ColumnType::Str, addr)),
        );
        columns.extend(
            self.bool_field_hash_map
                .iter()
-                .map(|(column_name, addr, _)| (column_name, ColumnType::Bool, addr)),
+                .map(|(column_name, addr)| (column_name, ColumnType::Bool, addr)),
        );
        columns.extend(
            self.ip_addr_field_hash_map
                .iter()
-                .map(|(column_name, addr, _)| (column_name, ColumnType::IpAddr, addr)),
+                .map(|(column_name, addr)| (column_name, ColumnType::IpAddr, addr)),
        );
        columns.extend(
            self.datetime_field_hash_map
                .iter()
-                .map(|(column_name, addr, _)| (column_name, ColumnType::DateTime, addr)),
+                .map(|(column_name, addr)| (column_name, ColumnType::DateTime, addr)),
        );
        columns.sort_unstable_by_key(|(column_name, col_type, _)| (*column_name, *col_type));

        let (arena, buffers, dictionaries) = (&self.arena, &mut self.buffers, &self.dictionaries);
        let mut symbol_byte_buffer: Vec<u8> = Vec::new();
-        for (column_name, column_type, addr) in columns {
+        for (column_name, column_type, addr) in columns.iter() {
+            let column_type = *column_type;
+            let addr = *addr;
            match column_type {
                ColumnType::Bool => {
                    let column_writer: ColumnWriter = self.bool_field_hash_map.read(addr);
@@ -437,6 +439,7 @@ impl ColumnarWriter {
                            &mut symbol_byte_buffer,
                        ),
                        buffers,
+                        &self.arena,
                        &mut column_serializer,
                    )?;
                    column_serializer.finalize()?;
@@ -484,12 +487,21 @@ impl ColumnarWriter {
            };
        }
        serializer.finalize(num_docs)?;
-        Ok(())
+        Ok(columns
+            .into_iter()
+            .map(|(column_name, column_type, _)| {
+                (
+                    String::from_utf8_lossy(column_name).to_string(),
+                    column_type,
+                )
+            })
+            .collect())
    }
 }

 // Serialize [Dictionary, Column, dictionary num bytes U32::LE]
 // Column: [Column Index, Column Values, column index num bytes U32::LE]
+#[allow(clippy::too_many_arguments)]
 fn serialize_bytes_or_str_column(
    cardinality: Cardinality,
    num_docs: RowId,
@@ -497,6 +509,7 @@ fn serialize_bytes_or_str_column(
    dictionary_builder: &DictionaryBuilder,
    operation_it: impl Iterator<Item = ColumnOperation<UnorderedId>>,
    buffers: &mut SpareBuffers,
+    arena: &MemoryArena,
    wrt: impl io::Write,
 ) -> io::Result<()> {
    let SpareBuffers {
@@ -505,7 +518,8 @@ fn serialize_bytes_or_str_column(
        ..
    } = buffers;
    let mut counting_writer = CountingWriter::wrap(wrt);
-    let term_id_mapping: TermIdMapping = dictionary_builder.serialize(&mut counting_writer)?;
+    let term_id_mapping: TermIdMapping =
+        dictionary_builder.serialize(arena, &mut counting_writer)?;
    let dictionary_num_bytes: u32 = counting_writer.written_bytes() as u32;
    let mut wrt = counting_writer.finish();
    let operation_iterator = operation_it.map(|symbol: ColumnOperation<UnorderedId>| {
--- a/columnar/src/dictionary.rs
+++ b/columnar/src/dictionary.rs
@@ -1,7 +1,7 @@
 use std::io;

-use fnv::FnvHashMap;
 use sstable::SSTable;
+use stacker::{MemoryArena, SharedArenaHashMap};

 pub(crate) struct TermIdMapping {
    unordered_to_ord: Vec<OrderedId>,
@@ -31,29 +31,38 @@ pub struct OrderedId(pub u32);
 /// mapping.
 #[derive(Default)]
 pub(crate) struct DictionaryBuilder {
-    dict: FnvHashMap<Vec<u8>, UnorderedId>,
-    memory_consumption: usize,
+    dict: SharedArenaHashMap,
 }

 impl DictionaryBuilder {
    /// Get or allocate an unordered id.
    /// (This ID is simply an auto-incremented id.)
-    pub fn get_or_allocate_id(&mut self, term: &[u8]) -> UnorderedId {
-        if let Some(term_id) = self.dict.get(term) {
-            return *term_id;
-        }
-        let new_id = UnorderedId(self.dict.len() as u32);
-        self.dict.insert(term.to_vec(), new_id);
-        self.memory_consumption += term.len();
-        self.memory_consumption += 40; // Term Metadata + HashMap overhead
-        new_id
+    pub fn get_or_allocate_id(&mut self, term: &[u8], arena: &mut MemoryArena) -> UnorderedId {
+        let next_id = self.dict.len() as u32;
+        let unordered_id = self
+            .dict
+            .mutate_or_create(term, arena, |unordered_id: Option<u32>| {
+                if let Some(unordered_id) = unordered_id {
+                    unordered_id
+                } else {
+                    next_id
+                }
+            });
+        UnorderedId(unordered_id)
    }

    /// Serialize the dictionary into an fst, and returns the
    /// `UnorderedId -> TermOrdinal` map.
-    pub fn serialize<'a, W: io::Write + 'a>(&self, wrt: &mut W) -> io::Result<TermIdMapping> {
-        let mut terms: Vec<(&[u8], UnorderedId)> =
-            self.dict.iter().map(|(k, v)| (k.as_slice(), *v)).collect();
+    pub fn serialize<'a, W: io::Write + 'a>(
+        &self,
+        arena: &MemoryArena,
+        wrt: &mut W,
+    ) -> io::Result<TermIdMapping> {
+        let mut terms: Vec<(&[u8], UnorderedId)> = self
+            .dict
+            .iter(arena)
+            .map(|(k, v)| (k, arena.read(v)))
+            .collect();
        terms.sort_unstable_by_key(|(key, _)| *key);
        // TODO Remove the allocation.
        let mut unordered_to_ord: Vec<OrderedId> = vec![OrderedId(0u32); terms.len()];
@@ -68,7 +77,7 @@ impl DictionaryBuilder {
    }

    pub(crate) fn mem_usage(&self) -> usize {
-        self.memory_consumption
+        self.dict.mem_usage()
    }
 }

@@ -78,12 +87,13 @@ mod tests {

    #[test]
    fn test_dictionary_builder() {
+        let mut arena = MemoryArena::default();
        let mut dictionary_builder = DictionaryBuilder::default();
-        let hello_uid = dictionary_builder.get_or_allocate_id(b"hello");
-        let happy_uid = dictionary_builder.get_or_allocate_id(b"happy");
-        let tax_uid = dictionary_builder.get_or_allocate_id(b"tax");
+        let hello_uid = dictionary_builder.get_or_allocate_id(b"hello", &mut arena);
+        let happy_uid = dictionary_builder.get_or_allocate_id(b"happy", &mut arena);
+        let tax_uid = dictionary_builder.get_or_allocate_id(b"tax", &mut arena);
        let mut buffer = Vec::new();
-        let id_mapping = dictionary_builder.serialize(&mut buffer).unwrap();
+        let id_mapping = dictionary_builder.serialize(&arena, &mut buffer).unwrap();
        assert_eq!(id_mapping.to_ord(hello_uid), OrderedId(1));
        assert_eq!(id_mapping.to_ord(happy_uid), OrderedId(0));
        assert_eq!(id_mapping.to_ord(tax_uid), OrderedId(2));
--- a/columnar/src/lib.rs
+++ b/columnar/src/lib.rs
@@ -1,3 +1,22 @@
+//! # Tantivy-Columnar
+//!
+//! `tantivy-columnar`provides a columnar storage for tantivy.
+//! The crate allows for efficient read operations on specific columns rather than entire records.
+//!
+//! ## Overview
+//!
+//! - **columnar**: Reading, writing, and merging multiple columns:
+//!   - **[ColumnarWriter]**: Makes it possible to create a new columnar.
+//!   - **[ColumnarReader]**: The ColumnarReader makes it possible to access a set of columns
+//!     associated to field names.
+//!   - **[merge_columnar]**: Contains the functionalities to merge multiple ColumnarReader or
+//!     segments into a single one.
+//!
+//! - **column**: A single column, which contains
+//!     - [column_index]: Resolves the rows for a document id. Manages the cardinality of the
+//!       column.
+//!     - [column_values]: Stores the values of a column in a dense format.
+
 #![cfg_attr(all(feature = "unstable", test), feature(test))]

 #[cfg(test)]
@@ -12,7 +31,7 @@ use std::io;

 mod block_accessor;
 mod column;
-mod column_index;
+pub mod column_index;
 pub mod column_values;
 mod columnar;
 mod dictionary;
--- a/columnar/src/tests.rs
+++ b/columnar/src/tests.rs
@@ -26,7 +26,7 @@ fn test_dataframe_writer_str() {
    assert_eq!(columnar.num_columns(), 1);
    let cols: Vec<DynamicColumnHandle> = columnar.read_columns("my_string").unwrap();
    assert_eq!(cols.len(), 1);
-    assert_eq!(cols[0].num_bytes(), 87);
+    assert_eq!(cols[0].num_bytes(), 73);
 }

 #[test]
@@ -40,7 +40,7 @@ fn test_dataframe_writer_bytes() {
    assert_eq!(columnar.num_columns(), 1);
    let cols: Vec<DynamicColumnHandle> = columnar.read_columns("my_string").unwrap();
    assert_eq!(cols.len(), 1);
-    assert_eq!(cols[0].num_bytes(), 87);
+    assert_eq!(cols[0].num_bytes(), 73);
 }

 #[test]
@@ -330,9 +330,9 @@ fn bytes_strategy() -> impl Strategy<Value = &'static [u8]> {
 // A random column value
 fn column_value_strategy() -> impl Strategy<Value = ColumnValue> {
    prop_oneof![
-        10 => string_strategy().prop_map(|s| ColumnValue::Str(s)),
-        1 => bytes_strategy().prop_map(|b| ColumnValue::Bytes(b)),
-        40 => num_strategy().prop_map(|n| ColumnValue::Numerical(n)),
+        10 => string_strategy().prop_map(ColumnValue::Str),
+        1 => bytes_strategy().prop_map(ColumnValue::Bytes),
+        40 => num_strategy().prop_map(ColumnValue::Numerical),
        1 => (1u16..3u16).prop_map(|ip_addr_byte| ColumnValue::IpAddr(Ipv6Addr::new(
            127,
            0,
@@ -343,7 +343,7 @@ fn column_value_strategy() -> impl Strategy<Value = ColumnValue> {
            0,
            ip_addr_byte
        ))),
-        1 => any::<bool>().prop_map(|b| ColumnValue::Bool(b)),
+        1 => any::<bool>().prop_map(ColumnValue::Bool),
        1 => (0_679_723_993i64..1_679_723_995i64)
            .prop_map(|val| { ColumnValue::DateTime(DateTime::from_timestamp_secs(val)) })
    ]
@@ -419,8 +419,8 @@ fn build_columnar_with_mapping(
    columnar_writer
        .serialize(num_docs, old_to_new_row_ids_opt, &mut buffer)
        .unwrap();
-    let columnar_reader = ColumnarReader::open(buffer).unwrap();
-    columnar_reader
+
+    ColumnarReader::open(buffer).unwrap()
 }

 fn build_columnar(docs: &[Vec<(&'static str, ColumnValue)>]) -> ColumnarReader {
@@ -746,7 +746,7 @@ proptest! {
        let stack_merge_order = StackMergeOrder::stack(&columnar_readers_arr[..]).into();
        crate::merge_columnar(&columnar_readers_arr[..], &[], stack_merge_order, &mut output).unwrap();
        let merged_columnar = ColumnarReader::open(output).unwrap();
-        let concat_rows: Vec<Vec<(&'static str, ColumnValue)>> = columnar_docs.iter().cloned().flatten().collect();
+        let concat_rows: Vec<Vec<(&'static str, ColumnValue)>> = columnar_docs.iter().flatten().cloned().collect();
        let expected_merged_columnar = build_columnar(&concat_rows[..]);
        assert_columnar_eq_strict(&merged_columnar, &expected_merged_columnar);
    }
@@ -772,7 +772,7 @@ fn test_columnar_merging_empty_columnar() {
    .unwrap();
    let merged_columnar = ColumnarReader::open(output).unwrap();
    let concat_rows: Vec<Vec<(&'static str, ColumnValue)>> =
-        columnar_docs.iter().cloned().flatten().collect();
+        columnar_docs.iter().flatten().cloned().collect();
    let expected_merged_columnar = build_columnar(&concat_rows[..]);
    assert_columnar_eq_strict(&merged_columnar, &expected_merged_columnar);
 }
@@ -809,7 +809,7 @@ fn test_columnar_merging_number_columns() {
    .unwrap();
    let merged_columnar = ColumnarReader::open(output).unwrap();
    let concat_rows: Vec<Vec<(&'static str, ColumnValue)>> =
-        columnar_docs.iter().cloned().flatten().collect();
+        columnar_docs.iter().flatten().cloned().collect();
    let expected_merged_columnar = build_columnar(&concat_rows[..]);
    assert_columnar_eq_strict(&merged_columnar, &expected_merged_columnar);
 }
--- a/common/src/bitset.rs
+++ b/common/src/bitset.rs
@@ -6,7 +6,7 @@ use ownedbytes::OwnedBytes;

 use crate::ByteCount;

-#[derive(Clone, Copy, Eq, PartialEq)]
+#[derive(Clone, Copy, Eq, PartialEq, Hash)]
 pub struct TinySet(u64);

 impl fmt::Debug for TinySet {
--- a/common/src/datetime.rs
+++ b/common/src/datetime.rs
@@ -1,11 +1,14 @@
 #![allow(deprecated)]

 use std::fmt;
+use std::io::{Read, Write};

 use serde::{Deserialize, Serialize};
 use time::format_description::well_known::Rfc3339;
 use time::{OffsetDateTime, PrimitiveDateTime, UtcOffset};

+use crate::BinarySerializable;
+
 /// Precision with which datetimes are truncated when stored in fast fields. This setting is only
 /// relevant for fast fields. In the docstore, datetimes are always saved with nanosecond precision.
 #[derive(
@@ -164,3 +167,15 @@ impl fmt::Debug for DateTime {
        f.write_str(&utc_rfc3339)
    }
 }
+
+impl BinarySerializable for DateTime {
+    fn serialize<W: Write + ?Sized>(&self, writer: &mut W) -> std::io::Result<()> {
+        let timestamp_micros = self.into_timestamp_micros();
+        <i64 as BinarySerializable>::serialize(&timestamp_micros, writer)
+    }
+
+    fn deserialize<R: Read>(reader: &mut R) -> std::io::Result<Self> {
+        let timestamp_micros = <i64 as BinarySerializable>::deserialize(reader)?;
+        Ok(Self::from_timestamp_micros(timestamp_micros))
+    }
+}
--- a/common/src/json_path_writer.rs
+++ b/common/src/json_path_writer.rs
@@ -0,0 +1,112 @@
+use crate::replace_in_place;
+
+/// Separates the different segments of a json path.
+pub const JSON_PATH_SEGMENT_SEP: u8 = 1u8;
+pub const JSON_PATH_SEGMENT_SEP_STR: &str =
+    unsafe { std::str::from_utf8_unchecked(&[JSON_PATH_SEGMENT_SEP]) };
+
+/// Create a new JsonPathWriter, that creates flattened json paths for tantivy.
+#[derive(Clone, Debug, Default)]
+pub struct JsonPathWriter {
+    path: String,
+    indices: Vec<usize>,
+    expand_dots: bool,
+}
+
+impl JsonPathWriter {
+    pub fn new() -> Self {
+        JsonPathWriter {
+            path: String::new(),
+            indices: Vec::new(),
+            expand_dots: false,
+        }
+    }
+
+    /// When expand_dots is enabled, json object like
+    /// `{"k8s.node.id": 5}` is processed as if it was
+    /// `{"k8s": {"node": {"id": 5}}}`.
+    /// This option has the merit of allowing users to
+    /// write queries  like `k8s.node.id:5`.
+    /// On the other, enabling that feature can lead to
+    /// ambiguity.
+    #[inline]
+    pub fn set_expand_dots(&mut self, expand_dots: bool) {
+        self.expand_dots = expand_dots;
+    }
+
+    /// Push a new segment to the path.
+    #[inline]
+    pub fn push(&mut self, segment: &str) {
+        let len_path = self.path.len();
+        self.indices.push(len_path);
+        if !self.path.is_empty() {
+            self.path.push_str(JSON_PATH_SEGMENT_SEP_STR);
+        }
+        self.path.push_str(segment);
+        if self.expand_dots {
+            // This might include the separation byte, which is ok because it is not a dot.
+            let appended_segment = &mut self.path[len_path..];
+            // The unsafe below is safe as long as b'.' and JSON_PATH_SEGMENT_SEP are
+            // valid single byte ut8 strings.
+            // By utf-8 design, they cannot be part of another codepoint.
+            unsafe {
+                replace_in_place(b'.', JSON_PATH_SEGMENT_SEP, appended_segment.as_bytes_mut())
+            };
+        }
+    }
+
+    /// Remove the last segment. Does nothing if the path is empty.
+    #[inline]
+    pub fn pop(&mut self) {
+        if let Some(last_idx) = self.indices.pop() {
+            self.path.truncate(last_idx);
+        }
+    }
+
+    /// Clear the path.
+    #[inline]
+    pub fn clear(&mut self) {
+        self.path.clear();
+        self.indices.clear();
+    }
+
+    /// Get the current path.
+    #[inline]
+    pub fn as_str(&self) -> &str {
+        &self.path
+    }
+}
+
+impl From<JsonPathWriter> for String {
+    #[inline]
+    fn from(value: JsonPathWriter) -> Self {
+        value.path
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn json_path_writer_test() {
+        let mut writer = JsonPathWriter::new();
+
+        writer.push("root");
+        assert_eq!(writer.as_str(), "root");
+
+        writer.push("child");
+        assert_eq!(writer.as_str(), "root\u{1}child");
+
+        writer.pop();
+        assert_eq!(writer.as_str(), "root");
+
+        writer.push("k8s.node.id");
+        assert_eq!(writer.as_str(), "root\u{1}k8s.node.id");
+
+        writer.set_expand_dots(true);
+        writer.pop();
+        writer.push("k8s.node.id");
+        assert_eq!(writer.as_str(), "root\u{1}k8s\u{1}node\u{1}id");
+    }
+}
--- a/common/src/lib.rs
+++ b/common/src/lib.rs
@@ -9,6 +9,7 @@ mod byte_count;
 mod datetime;
 pub mod file_slice;
 mod group_by;
+mod json_path_writer;
 mod serialize;
 mod vint;
 mod writer;
@@ -18,6 +19,7 @@ pub use byte_count::ByteCount;
 pub use datetime::DatePrecision;
 pub use datetime::{DateTime, DateTimePrecision};
 pub use group_by::GroupByIteratorExtended;
+pub use json_path_writer::JsonPathWriter;
 pub use ownedbytes::{OwnedBytes, StableDeref};
 pub use serialize::{BinarySerializable, DeserializeFrom, FixedSize};
 pub use vint::{
@@ -116,6 +118,7 @@ pub fn u64_to_f64(val: u64) -> f64 {
 ///
 /// This function assumes that the needle is rarely contained in the bytes string
 /// and offers a fast path if the needle is not present.
+#[inline]
 pub fn replace_in_place(needle: u8, replacement: u8, bytes: &mut [u8]) {
    if !bytes.contains(&needle) {
        return;
--- a/common/src/serialize.rs
+++ b/common/src/serialize.rs
@@ -1,3 +1,4 @@
+use std::borrow::Cow;
 use std::io::{Read, Write};
 use std::{fmt, io};

@@ -249,6 +250,43 @@ impl BinarySerializable for String {
    }
 }

+impl<'a> BinarySerializable for Cow<'a, str> {
+    fn serialize<W: Write + ?Sized>(&self, writer: &mut W) -> io::Result<()> {
+        let data: &[u8] = self.as_bytes();
+        VInt(data.len() as u64).serialize(writer)?;
+        writer.write_all(data)
+    }
+
+    fn deserialize<R: Read>(reader: &mut R) -> io::Result<Cow<'a, str>> {
+        let string_length = VInt::deserialize(reader)?.val() as usize;
+        let mut result = String::with_capacity(string_length);
+        reader
+            .take(string_length as u64)
+            .read_to_string(&mut result)?;
+        Ok(Cow::Owned(result))
+    }
+}
+
+impl<'a> BinarySerializable for Cow<'a, [u8]> {
+    fn serialize<W: Write + ?Sized>(&self, writer: &mut W) -> io::Result<()> {
+        VInt(self.len() as u64).serialize(writer)?;
+        for it in self.iter() {
+            it.serialize(writer)?;
+        }
+        Ok(())
+    }
+
+    fn deserialize<R: Read>(reader: &mut R) -> io::Result<Cow<'a, [u8]>> {
+        let num_items = VInt::deserialize(reader)?.val();
+        let mut items: Vec<u8> = Vec::with_capacity(num_items as usize);
+        for _ in 0..num_items {
+            let item = u8::deserialize(reader)?;
+            items.push(item);
+        }
+        Ok(Cow::Owned(items))
+    }
+}
+
 #[cfg(test)]
 pub mod test {

--- a/examples/aggregation.rs
+++ b/examples/aggregation.rs
@@ -12,7 +12,7 @@ use tantivy::aggregation::agg_result::AggregationResults;
 use tantivy::aggregation::AggregationCollector;
 use tantivy::query::AllQuery;
 use tantivy::schema::{self, IndexRecordOption, Schema, TextFieldIndexing, FAST};
-use tantivy::Index;
+use tantivy::{Index, IndexWriter, TantivyDocument};

 fn main() -> tantivy::Result<()> {
    // # Create Schema
@@ -132,10 +132,10 @@ fn main() -> tantivy::Result<()> {

    let stream = Deserializer::from_str(data).into_iter::<Value>();

-    let mut index_writer = index.writer(50_000_000)?;
+    let mut index_writer: IndexWriter = index.writer(50_000_000)?;
    let mut num_indexed = 0;
    for value in stream {
-        let doc = schema.parse_document(&serde_json::to_string(&value.unwrap())?)?;
+        let doc = TantivyDocument::parse_json(&schema, &serde_json::to_string(&value.unwrap())?)?;
        index_writer.add_document(doc)?;
        num_indexed += 1;
        if num_indexed > 4 {
--- a/examples/basic_search.rs
+++ b/examples/basic_search.rs
@@ -15,7 +15,7 @@
 use tantivy::collector::TopDocs;
 use tantivy::query::QueryParser;
 use tantivy::schema::*;
-use tantivy::{doc, Index, ReloadPolicy};
+use tantivy::{doc, Index, IndexWriter, ReloadPolicy};
 use tempfile::TempDir;

 fn main() -> tantivy::Result<()> {
@@ -75,7 +75,7 @@ fn main() -> tantivy::Result<()> {
    // Here we give tantivy a budget of `50MB`.
    // Using a bigger memory_arena for the indexer may increase
    // throughput, but 50 MB is already plenty.
-    let mut index_writer = index.writer(50_000_000)?;
+    let mut index_writer: IndexWriter = index.writer(50_000_000)?;

    // Let's index our documents!
    // We first need a handle on the title and the body field.
@@ -87,7 +87,7 @@ fn main() -> tantivy::Result<()> {
    let title = schema.get_field("title").unwrap();
    let body = schema.get_field("body").unwrap();

-    let mut old_man_doc = Document::default();
+    let mut old_man_doc = TantivyDocument::default();
    old_man_doc.add_text(title, "The Old Man and the Sea");
    old_man_doc.add_text(
        body,
@@ -164,7 +164,7 @@ fn main() -> tantivy::Result<()> {
    // will reload the index automatically after each commit.
    let reader = index
        .reader_builder()
-        .reload_policy(ReloadPolicy::OnCommit)
+        .reload_policy(ReloadPolicy::OnCommitWithDelay)
        .try_into()?;

    // We now need to acquire a searcher.
@@ -217,8 +217,8 @@ fn main() -> tantivy::Result<()> {
    // the document returned will only contain
    // a title.
    for (_score, doc_address) in top_docs {
-        let retrieved_doc = searcher.doc(doc_address)?;
-        println!("{}", schema.to_json(&retrieved_doc));
+        let retrieved_doc: TantivyDocument = searcher.doc(doc_address)?;
+        println!("{}", retrieved_doc.to_json(&schema));
    }

    // We can also get an explanation to understand
--- a/examples/custom_collector.rs
+++ b/examples/custom_collector.rs
@@ -13,7 +13,7 @@ use columnar::Column;
 use tantivy::collector::{Collector, SegmentCollector};
 use tantivy::query::QueryParser;
 use tantivy::schema::{Schema, FAST, INDEXED, TEXT};
-use tantivy::{doc, Index, Score, SegmentReader};
+use tantivy::{doc, Index, IndexWriter, Score, SegmentReader};

 #[derive(Default)]
 struct Stats {
@@ -142,7 +142,7 @@ fn main() -> tantivy::Result<()> {
    // this example.
    let index = Index::create_in_ram(schema);

-    let mut index_writer = index.writer(50_000_000)?;
+    let mut index_writer: IndexWriter = index.writer(50_000_000)?;
    index_writer.add_document(doc!(
        product_name => "Super Broom 2000",
        product_description => "While it is ok for short distance travel, this broom \
--- a/examples/custom_tokenizer.rs
+++ b/examples/custom_tokenizer.rs
@@ -6,7 +6,7 @@ use tantivy::collector::TopDocs;
 use tantivy::query::QueryParser;
 use tantivy::schema::*;
 use tantivy::tokenizer::NgramTokenizer;
-use tantivy::{doc, Index};
+use tantivy::{doc, Index, IndexWriter};

 fn main() -> tantivy::Result<()> {
    // # Defining the schema
@@ -62,7 +62,7 @@ fn main() -> tantivy::Result<()> {
    //
    // Here we use a buffer of 50MB per thread. Using a bigger
    // memory arena for the indexer can increase its throughput.
-    let mut index_writer = index.writer(50_000_000)?;
+    let mut index_writer: IndexWriter = index.writer(50_000_000)?;
    index_writer.add_document(doc!(
    title => "The Old Man and the Sea",
    body => "He was an old man who fished alone in a skiff in the Gulf Stream and \
@@ -103,8 +103,8 @@ fn main() -> tantivy::Result<()> {
    let top_docs = searcher.search(&query, &TopDocs::with_limit(10))?;

    for (_, doc_address) in top_docs {
-        let retrieved_doc = searcher.doc(doc_address)?;
-        println!("{}", schema.to_json(&retrieved_doc));
+        let retrieved_doc: TantivyDocument = searcher.doc(doc_address)?;
+        println!("{}", retrieved_doc.to_json(&schema));
    }

    Ok(())
--- a/examples/date_time_field.rs
+++ b/examples/date_time_field.rs
@@ -4,8 +4,8 @@

 use tantivy::collector::TopDocs;
 use tantivy::query::QueryParser;
-use tantivy::schema::{DateOptions, Schema, Value, INDEXED, STORED, STRING};
-use tantivy::Index;
+use tantivy::schema::{DateOptions, Document, OwnedValue, Schema, INDEXED, STORED, STRING};
+use tantivy::{Index, IndexWriter, TantivyDocument};

 fn main() -> tantivy::Result<()> {
    // # Defining the schema
@@ -22,16 +22,18 @@ fn main() -> tantivy::Result<()> {
    // # Indexing documents
    let index = Index::create_in_ram(schema.clone());

-    let mut index_writer = index.writer(50_000_000)?;
+    let mut index_writer: IndexWriter = index.writer(50_000_000)?;
    // The dates are passed as string in the RFC3339 format
-    let doc = schema.parse_document(
+    let doc = TantivyDocument::parse_json(
+        &schema,
        r#"{
        "occurred_at": "2022-06-22T12:53:50.53Z",
        "event": "pull-request"
    }"#,
    )?;
    index_writer.add_document(doc)?;
-    let doc = schema.parse_document(
+    let doc = TantivyDocument::parse_json(
+        &schema,
        r#"{
        "occurred_at": "2022-06-22T13:00:00.22Z",
        "event": "comment"
@@ -58,13 +60,13 @@ fn main() -> tantivy::Result<()> {
        let count_docs = searcher.search(&*query, &TopDocs::with_limit(4))?;
        assert_eq!(count_docs.len(), 1);
        for (_score, doc_address) in count_docs {
-            let retrieved_doc = searcher.doc(doc_address)?;
+            let retrieved_doc = searcher.doc::<TantivyDocument>(doc_address)?;
            assert!(matches!(
                retrieved_doc.get_first(occurred_at),
-                Some(Value::Date(_))
+                Some(OwnedValue::Date(_))
            ));
            assert_eq!(
-                schema.to_json(&retrieved_doc),
+                retrieved_doc.to_json(&schema),
                r#"{"event":["comment"],"occurred_at":["2022-06-22T13:00:00.22Z"]}"#
            );
        }
--- a/examples/deleting_updating_documents.rs
+++ b/examples/deleting_updating_documents.rs
@@ -11,7 +11,7 @@
 use tantivy::collector::TopDocs;
 use tantivy::query::TermQuery;
 use tantivy::schema::*;
-use tantivy::{doc, Index, IndexReader};
+use tantivy::{doc, Index, IndexReader, IndexWriter};

 // A simple helper function to fetch a single document
 // given its id from our index.
@@ -19,7 +19,7 @@ use tantivy::{doc, Index, IndexReader};
 fn extract_doc_given_isbn(
    reader: &IndexReader,
    isbn_term: &Term,
-) -> tantivy::Result<Option<Document>> {
+) -> tantivy::Result<Option<TantivyDocument>> {
    let searcher = reader.searcher();

    // This is the simplest query you can think of.
@@ -69,10 +69,10 @@ fn main() -> tantivy::Result<()> {

    let index = Index::create_in_ram(schema.clone());

-    let mut index_writer = index.writer(50_000_000)?;
+    let mut index_writer: IndexWriter = index.writer(50_000_000)?;

    // Let's add a couple of documents, for the sake of the example.
-    let mut old_man_doc = Document::default();
+    let mut old_man_doc = TantivyDocument::default();
    old_man_doc.add_text(title, "The Old Man and the Sea");
    index_writer.add_document(doc!(
        isbn => "978-0099908401",
@@ -94,7 +94,7 @@ fn main() -> tantivy::Result<()> {
    // Oops our frankenstein doc seems misspelled
    let frankenstein_doc_misspelled = extract_doc_given_isbn(&reader, &frankenstein_isbn)?.unwrap();
    assert_eq!(
-        schema.to_json(&frankenstein_doc_misspelled),
+        frankenstein_doc_misspelled.to_json(&schema),
        r#"{"isbn":["978-9176370711"],"title":["Frankentein"]}"#,
    );

@@ -136,7 +136,7 @@ fn main() -> tantivy::Result<()> {
    // No more typo!
    let frankenstein_new_doc = extract_doc_given_isbn(&reader, &frankenstein_isbn)?.unwrap();
    assert_eq!(
-        schema.to_json(&frankenstein_new_doc),
+        frankenstein_new_doc.to_json(&schema),
        r#"{"isbn":["978-9176370711"],"title":["Frankenstein"]}"#,
    );

--- a/examples/faceted_search.rs
+++ b/examples/faceted_search.rs
@@ -17,7 +17,7 @@
 use tantivy::collector::FacetCollector;
 use tantivy::query::{AllQuery, TermQuery};
 use tantivy::schema::*;
-use tantivy::{doc, Index};
+use tantivy::{doc, Index, IndexWriter};

 fn main() -> tantivy::Result<()> {
    // Let's create a temporary directory for the sake of this example
@@ -30,7 +30,7 @@ fn main() -> tantivy::Result<()> {
    let schema = schema_builder.build();
    let index = Index::create_in_ram(schema);

-    let mut index_writer = index.writer(30_000_000)?;
+    let mut index_writer: IndexWriter = index.writer(30_000_000)?;

    // For convenience, tantivy also comes with a macro to
    // reduce the boilerplate above.
--- a/examples/faceted_search_with_tweaked_score.rs
+++ b/examples/faceted_search_with_tweaked_score.rs
@@ -12,7 +12,7 @@ use std::collections::HashSet;
 use tantivy::collector::TopDocs;
 use tantivy::query::BooleanQuery;
 use tantivy::schema::*;
-use tantivy::{doc, DocId, Index, Score, SegmentReader};
+use tantivy::{doc, DocId, Index, IndexWriter, Score, SegmentReader};

 fn main() -> tantivy::Result<()> {
    let mut schema_builder = Schema::builder();
@@ -23,7 +23,7 @@ fn main() -> tantivy::Result<()> {
    let schema = schema_builder.build();
    let index = Index::create_in_ram(schema);

-    let mut index_writer = index.writer(30_000_000)?;
+    let mut index_writer: IndexWriter = index.writer(30_000_000)?;

    index_writer.add_document(doc!(
        title => "Fried egg",
@@ -91,11 +91,10 @@ fn main() -> tantivy::Result<()> {
            .iter()
            .map(|(_, doc_id)| {
                searcher
-                    .doc(*doc_id)
+                    .doc::<TantivyDocument>(*doc_id)
                    .unwrap()
                    .get_first(title)
-                    .unwrap()
-                    .as_text()
+                    .and_then(|v| v.as_str())
                    .unwrap()
                    .to_owned()
            })
--- a/examples/fuzzy_search.rs
+++ b/examples/fuzzy_search.rs
@@ -14,7 +14,7 @@
 use tantivy::collector::{Count, TopDocs};
 use tantivy::query::FuzzyTermQuery;
 use tantivy::schema::*;
-use tantivy::{doc, Index, ReloadPolicy};
+use tantivy::{doc, Index, IndexWriter, ReloadPolicy};
 use tempfile::TempDir;

 fn main() -> tantivy::Result<()> {
@@ -66,7 +66,7 @@ fn main() -> tantivy::Result<()> {
    // Here we give tantivy a budget of `50MB`.
    // Using a bigger memory_arena for the indexer may increase
    // throughput, but 50 MB is already plenty.
-    let mut index_writer = index.writer(50_000_000)?;
+    let mut index_writer: IndexWriter = index.writer(50_000_000)?;

    // Let's index our documents!
    // We first need a handle on the title and the body field.
@@ -123,7 +123,7 @@ fn main() -> tantivy::Result<()> {
    // will reload the index automatically after each commit.
    let reader = index
        .reader_builder()
-        .reload_policy(ReloadPolicy::OnCommit)
+        .reload_policy(ReloadPolicy::OnCommitWithDelay)
        .try_into()?;

    // We now need to acquire a searcher.
@@ -151,10 +151,10 @@ fn main() -> tantivy::Result<()> {
        assert_eq!(count, 3);
        assert_eq!(top_docs.len(), 3);
        for (score, doc_address) in top_docs {
-            let retrieved_doc = searcher.doc(doc_address)?;
            // Note that the score is not lower for the fuzzy hit.
            // There's an issue open for that: https://github.com/quickwit-oss/tantivy/issues/563
-            println!("score {score:?} doc {}", schema.to_json(&retrieved_doc));
+            let retrieved_doc: TantivyDocument = searcher.doc(doc_address)?;
+            println!("score {score:?} doc {}", retrieved_doc.to_json(&schema));
            // score 1.0 doc {"title":["The Diary of Muadib"]}
            //
            // score 1.0 doc {"title":["The Diary of a Young Girl"]}
--- a/examples/index_with_json.rs
+++ b/examples/index_with_json.rs
@@ -21,7 +21,7 @@ fn main() -> tantivy::Result<()> {
    }"#;

    // We can parse our document
-    let _mice_and_men_doc = schema.parse_document(mice_and_men_doc_json)?;
+    let _mice_and_men_doc = TantivyDocument::parse_json(&schema, mice_and_men_doc_json)?;

    // Multi-valued field are allowed, they are
    // expressed in JSON by an array.
@@ -30,7 +30,7 @@ fn main() -> tantivy::Result<()> {
       "title": ["Frankenstein", "The Modern Prometheus"],
       "year": 1818
    }"#;
-    let _frankenstein_doc = schema.parse_document(frankenstein_json)?;
+    let _frankenstein_doc = TantivyDocument::parse_json(&schema, frankenstein_json)?;

    // Note that the schema is saved in your index directory.
    //
--- a/examples/integer_range_search.rs
+++ b/examples/integer_range_search.rs
@@ -5,7 +5,7 @@
 use tantivy::collector::Count;
 use tantivy::query::RangeQuery;
 use tantivy::schema::{Schema, INDEXED};
-use tantivy::{doc, Index, Result};
+use tantivy::{doc, Index, IndexWriter, Result};

 fn main() -> Result<()> {
    // For the sake of simplicity, this schema will only have 1 field
@@ -17,7 +17,7 @@ fn main() -> Result<()> {
    let index = Index::create_in_ram(schema);
    let reader = index.reader()?;
    {
-        let mut index_writer = index.writer_with_num_threads(1, 6_000_000)?;
+        let mut index_writer: IndexWriter = index.writer_with_num_threads(1, 6_000_000)?;
        for year in 1950u64..2019u64 {
            index_writer.add_document(doc!(year_field => year))?;
        }
--- a/examples/ip_field.rs
+++ b/examples/ip_field.rs
@@ -6,7 +6,7 @@
 use tantivy::collector::{Count, TopDocs};
 use tantivy::query::QueryParser;
 use tantivy::schema::{Schema, FAST, INDEXED, STORED, STRING};
-use tantivy::Index;
+use tantivy::{Index, IndexWriter, TantivyDocument};

 fn main() -> tantivy::Result<()> {
    // # Defining the schema
@@ -22,20 +22,22 @@ fn main() -> tantivy::Result<()> {
    // # Indexing documents
    let index = Index::create_in_ram(schema.clone());

-    let mut index_writer = index.writer(50_000_000)?;
+    let mut index_writer: IndexWriter = index.writer(50_000_000)?;

    // ### IPv4
    // Adding documents that contain an IPv4 address. Notice that the IP addresses are passed as
    // `String`. Since the field is of type ip, we parse the IP address from the string and store it
    // internally as IPv6.
-    let doc = schema.parse_document(
+    let doc = TantivyDocument::parse_json(
+        &schema,
        r#"{
            "ip": "192.168.0.33",
            "event_type": "login"
        }"#,
    )?;
    index_writer.add_document(doc)?;
-    let doc = schema.parse_document(
+    let doc = TantivyDocument::parse_json(
+        &schema,
        r#"{
            "ip": "192.168.0.80",
            "event_type": "checkout"
@@ -44,7 +46,8 @@ fn main() -> tantivy::Result<()> {
    index_writer.add_document(doc)?;
    // ### IPv6
    // Adding a document that contains an IPv6 address.
-    let doc = schema.parse_document(
+    let doc = TantivyDocument::parse_json(
+        &schema,
        r#"{
            "ip": "2001:0db8:85a3:0000:0000:8a2e:0370:7334",
            "event_type": "checkout"
--- a/examples/iterating_docs_and_positions.rs
+++ b/examples/iterating_docs_and_positions.rs
@@ -10,7 +10,7 @@
 // ---
 // Importing tantivy...
 use tantivy::schema::*;
-use tantivy::{doc, DocSet, Index, Postings, TERMINATED};
+use tantivy::{doc, DocSet, Index, IndexWriter, Postings, TERMINATED};

 fn main() -> tantivy::Result<()> {
    // We first create a schema for the sake of the
@@ -24,7 +24,7 @@ fn main() -> tantivy::Result<()> {

    let index = Index::create_in_ram(schema);

-    let mut index_writer = index.writer_with_num_threads(1, 50_000_000)?;
+    let mut index_writer: IndexWriter = index.writer_with_num_threads(1, 50_000_000)?;
    index_writer.add_document(doc!(title => "The Old Man and the Sea"))?;
    index_writer.add_document(doc!(title => "Of Mice and Men"))?;
    index_writer.add_document(doc!(title => "The modern Promotheus"))?;
--- a/examples/json_field.rs
+++ b/examples/json_field.rs
@@ -7,7 +7,7 @@
 use tantivy::collector::{Count, TopDocs};
 use tantivy::query::QueryParser;
 use tantivy::schema::{Schema, FAST, STORED, STRING, TEXT};
-use tantivy::Index;
+use tantivy::{Index, IndexWriter, TantivyDocument};

 fn main() -> tantivy::Result<()> {
    // # Defining the schema
@@ -20,8 +20,9 @@ fn main() -> tantivy::Result<()> {
    // # Indexing documents
    let index = Index::create_in_ram(schema.clone());

-    let mut index_writer = index.writer(50_000_000)?;
-    let doc = schema.parse_document(
+    let mut index_writer: IndexWriter = index.writer(50_000_000)?;
+    let doc = TantivyDocument::parse_json(
+        &schema,
        r#"{
        "timestamp": "2022-02-22T23:20:50.53Z",
        "event_type": "click",
@@ -33,7 +34,8 @@ fn main() -> tantivy::Result<()> {
    }"#,
    )?;
    index_writer.add_document(doc)?;
-    let doc = schema.parse_document(
+    let doc = TantivyDocument::parse_json(
+        &schema,
        r#"{
        "timestamp": "2022-02-22T23:20:51.53Z",
        "event_type": "click",
--- a/examples/phrase_prefix_search.rs
+++ b/examples/phrase_prefix_search.rs
@@ -1,7 +1,7 @@
 use tantivy::collector::TopDocs;
 use tantivy::query::QueryParser;
 use tantivy::schema::*;
-use tantivy::{doc, Index, ReloadPolicy, Result};
+use tantivy::{doc, Index, IndexWriter, ReloadPolicy, Result};
 use tempfile::TempDir;

 fn main() -> Result<()> {
@@ -17,7 +17,7 @@ fn main() -> Result<()> {

    let index = Index::create_in_dir(&index_path, schema)?;

-    let mut index_writer = index.writer(50_000_000)?;
+    let mut index_writer: IndexWriter = index.writer(50_000_000)?;

    index_writer.add_document(doc!(
    title => "The Old Man and the Sea",
@@ -51,7 +51,7 @@ fn main() -> Result<()> {

    let reader = index
        .reader_builder()
-        .reload_policy(ReloadPolicy::OnCommit)
+        .reload_policy(ReloadPolicy::OnCommitWithDelay)
        .try_into()?;

    let searcher = reader.searcher();
@@ -67,8 +67,12 @@ fn main() -> Result<()> {
    let mut titles = top_docs
        .into_iter()
        .map(|(_score, doc_address)| {
-            let doc = searcher.doc(doc_address)?;
-            let title = doc.get_first(title).unwrap().as_text().unwrap().to_owned();
+            let doc = searcher.doc::<TantivyDocument>(doc_address)?;
+            let title = doc
+                .get_first(title)
+                .and_then(|v| v.as_str())
+                .unwrap()
+                .to_owned();
            Ok(title)
        })
        .collect::<Result<Vec<_>>>()?;
--- a/examples/pre_tokenized_text.rs
+++ b/examples/pre_tokenized_text.rs
@@ -13,7 +13,7 @@ use tantivy::collector::{Count, TopDocs};
 use tantivy::query::TermQuery;
 use tantivy::schema::*;
 use tantivy::tokenizer::{PreTokenizedString, SimpleTokenizer, Token, TokenStream, Tokenizer};
-use tantivy::{doc, Index, ReloadPolicy};
+use tantivy::{doc, Index, IndexWriter, ReloadPolicy};
 use tempfile::TempDir;

 fn pre_tokenize_text(text: &str) -> Vec<Token> {
@@ -38,7 +38,7 @@ fn main() -> tantivy::Result<()> {

    let index = Index::create_in_dir(&index_path, schema.clone())?;

-    let mut index_writer = index.writer(50_000_000)?;
+    let mut index_writer: IndexWriter = index.writer(50_000_000)?;

    // We can create a document manually, by setting the fields
    // one by one in a Document object.
@@ -83,7 +83,7 @@ fn main() -> tantivy::Result<()> {
        }]
    }"#;

-    let short_man_doc = schema.parse_document(short_man_json)?;
+    let short_man_doc = TantivyDocument::parse_json(&schema, short_man_json)?;

    index_writer.add_document(short_man_doc)?;

@@ -94,7 +94,7 @@ fn main() -> tantivy::Result<()> {

    let reader = index
        .reader_builder()
-        .reload_policy(ReloadPolicy::OnCommit)
+        .reload_policy(ReloadPolicy::OnCommitWithDelay)
        .try_into()?;

    let searcher = reader.searcher();
@@ -115,8 +115,8 @@ fn main() -> tantivy::Result<()> {
    // Note that the tokens are not stored along with the original text
    // in the document store
    for (_score, doc_address) in top_docs {
-        let retrieved_doc = searcher.doc(doc_address)?;
-        println!("Document: {}", schema.to_json(&retrieved_doc));
+        let retrieved_doc: TantivyDocument = searcher.doc(doc_address)?;
+        println!("{}", retrieved_doc.to_json(&schema));
    }

    // In contrary to the previous query, when we search for the "man" term we
--- a/examples/snippet.rs
+++ b/examples/snippet.rs
@@ -10,7 +10,8 @@
 use tantivy::collector::TopDocs;
 use tantivy::query::QueryParser;
 use tantivy::schema::*;
-use tantivy::{doc, Index, Snippet, SnippetGenerator};
+use tantivy::snippet::{Snippet, SnippetGenerator};
+use tantivy::{doc, Index, IndexWriter};
 use tempfile::TempDir;

 fn main() -> tantivy::Result<()> {
@@ -27,7 +28,7 @@ fn main() -> tantivy::Result<()> {
    // # Indexing documents
    let index = Index::create_in_dir(&index_path, schema)?;

-    let mut index_writer = index.writer(50_000_000)?;
+    let mut index_writer: IndexWriter = index.writer(50_000_000)?;

    // we'll only need one doc for this example.
    index_writer.add_document(doc!(
@@ -54,13 +55,10 @@ fn main() -> tantivy::Result<()> {
    let snippet_generator = SnippetGenerator::create(&searcher, &*query, body)?;

    for (score, doc_address) in top_docs {
-        let doc = searcher.doc(doc_address)?;
+        let doc = searcher.doc::<TantivyDocument>(doc_address)?;
        let snippet = snippet_generator.snippet_from_doc(&doc);
        println!("Document score {score}:");
-        println!(
-            "title: {}",
-            doc.get_first(title).unwrap().as_text().unwrap()
-        );
+        println!("title: {}", doc.get_first(title).unwrap().as_str().unwrap());
        println!("snippet: {}", snippet.to_html());
        println!("custom highlighting: {}", highlight(snippet));
    }
--- a/examples/stop_words.rs
+++ b/examples/stop_words.rs
@@ -15,7 +15,7 @@ use tantivy::collector::TopDocs;
 use tantivy::query::QueryParser;
 use tantivy::schema::*;
 use tantivy::tokenizer::*;
-use tantivy::{doc, Index};
+use tantivy::{doc, Index, IndexWriter};

 fn main() -> tantivy::Result<()> {
    // this example assumes you understand the content in `basic_search`
@@ -60,7 +60,7 @@ fn main() -> tantivy::Result<()> {

    index.tokenizers().register("stoppy", tokenizer);

-    let mut index_writer = index.writer(50_000_000)?;
+    let mut index_writer: IndexWriter = index.writer(50_000_000)?;

    let title = schema.get_field("title").unwrap();
    let body = schema.get_field("body").unwrap();
@@ -105,9 +105,9 @@ fn main() -> tantivy::Result<()> {
    let top_docs = searcher.search(&query, &TopDocs::with_limit(10))?;

    for (score, doc_address) in top_docs {
-        let retrieved_doc = searcher.doc(doc_address)?;
+        let retrieved_doc: TantivyDocument = searcher.doc(doc_address)?;
        println!("\n==\nDocument score {score}:");
-        println!("{}", schema.to_json(&retrieved_doc));
+        println!("{}", retrieved_doc.to_json(&schema));
    }

    Ok(())
--- a/examples/warmer.rs
+++ b/examples/warmer.rs
@@ -6,8 +6,8 @@ use tantivy::collector::TopDocs;
 use tantivy::query::QueryParser;
 use tantivy::schema::{Schema, FAST, TEXT};
 use tantivy::{
-    doc, DocAddress, DocId, Index, Opstamp, Searcher, SearcherGeneration, SegmentId, SegmentReader,
-    Warmer,
+    doc, DocAddress, DocId, Index, IndexWriter, Opstamp, Searcher, SearcherGeneration, SegmentId,
+    SegmentReader, Warmer,
 };

 // This example shows how warmers can be used to
@@ -143,7 +143,7 @@ fn main() -> tantivy::Result<()> {
    const SNEAKERS: ProductId = 23222;

    let index = Index::create_in_ram(schema);
-    let mut writer = index.writer_with_num_threads(1, 15_000_000)?;
+    let mut writer: IndexWriter = index.writer_with_num_threads(1, 15_000_000)?;
    writer.add_document(doc!(product_id=>OLIVE_OIL, text=>"cooking olive oil from greece"))?;
    writer.add_document(doc!(product_id=>GLOVES, text=>"kitchen gloves, perfect for cooking"))?;
    writer.add_document(doc!(product_id=>SNEAKERS, text=>"uber sweet sneakers"))?;
--- a/query-grammar/src/query_grammar.rs
+++ b/query-grammar/src/query_grammar.rs
@@ -185,7 +185,7 @@ fn term_or_phrase(inp: &str) -> IResult<&str, UserInputLeaf> {
 fn term_or_phrase_infallible(inp: &str) -> JResult<&str, Option<UserInputLeaf>> {
    map(
        // ~* for slop/prefix, ) inside group or ast tree, ^ if boost
-        tuple_infallible((simple_term_infallible("*)^"), slop_or_prefix_val)),
+        tuple_infallible((simple_term_infallible(")^"), slop_or_prefix_val)),
        |((delimiter_phrase, (slop, prefix)), errors)| {
            let leaf = if let Some((delimiter, phrase)) = delimiter_phrase {
                Some(
@@ -1113,6 +1113,9 @@ mod test {
        test_parse_query_to_ast_helper("'www-form-encoded'", "'www-form-encoded'");
        test_parse_query_to_ast_helper("www-form-encoded", "www-form-encoded");
        test_parse_query_to_ast_helper("www-form-encoded", "www-form-encoded");
+        test_parse_query_to_ast_helper("mr james bo?d", "(*mr *james *bo?d)");
+        test_parse_query_to_ast_helper("mr james bo*", "(*mr *james *bo*)");
+        test_parse_query_to_ast_helper("mr james b*d", "(*mr *james *b*d)");
    }

    #[test]
--- a/src/aggregation/agg_bench.rs
+++ b/src/aggregation/agg_bench.rs
@@ -48,7 +48,7 @@ mod bench {
        let score_field_f64 = schema_builder.add_f64_field("score_f64", score_fieldtype.clone());
        let score_field_i64 = schema_builder.add_i64_field("score_i64", score_fieldtype);
        let index = Index::create_from_tempdir(schema_builder.build())?;
-        let few_terms_data = vec!["INFO", "ERROR", "WARN", "DEBUG"];
+        let few_terms_data = ["INFO", "ERROR", "WARN", "DEBUG"];

        let lg_norm = rand_distr::LogNormal::new(2.996f64, 0.979f64).unwrap();

@@ -85,7 +85,7 @@ mod bench {
            if cardinality == Cardinality::Sparse {
                doc_with_value /= 20;
            }
-            let val_max = 1_000_000.0;
+            let _val_max = 1_000_000.0;
            for _ in 0..doc_with_value {
                let val: f64 = rng.gen_range(0.0..1_000_000.0);
                let json = if rng.gen_bool(0.1) {
--- a/src/aggregation/agg_limits.rs
+++ b/src/aggregation/agg_limits.rs
@@ -73,9 +73,9 @@ impl AggregationLimits {
    /// Create a new ResourceLimitGuard, that will release the memory when dropped.
    pub fn new_guard(&self) -> ResourceLimitGuard {
        ResourceLimitGuard {
-            /// The counter which is shared between the aggregations for one request.
+            // The counter which is shared between the aggregations for one request.
            memory_consumption: Arc::clone(&self.memory_consumption),
-            /// The memory_limit in bytes
+            // The memory_limit in bytes
            memory_limit: self.memory_limit,
            allocated_with_the_guard: 0,
        }
--- a/src/aggregation/agg_tests.rs
+++ b/src/aggregation/agg_tests.rs
@@ -9,7 +9,7 @@ use crate::aggregation::tests::{get_test_index_2_segments, get_test_index_from_v
 use crate::aggregation::DistributedAggregationCollector;
 use crate::query::{AllQuery, TermQuery};
 use crate::schema::{IndexRecordOption, Schema, FAST};
-use crate::{Index, Term};
+use crate::{Index, IndexWriter, Term};

 fn get_avg_req(field_name: &str) -> Aggregation {
    serde_json::from_value(json!({
@@ -586,7 +586,7 @@ fn test_aggregation_on_json_object() {
    let json = schema_builder.add_json_field("json", FAST);
    let schema = schema_builder.build();
    let index = Index::create_in_ram(schema);
-    let mut index_writer = index.writer_for_tests().unwrap();
+    let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
    index_writer
        .add_document(doc!(json => json!({"color": "red"})))
        .unwrap();
@@ -624,13 +624,72 @@ fn test_aggregation_on_json_object() {
    );
 }

+#[test]
+fn test_aggregation_on_nested_json_object() {
+    let mut schema_builder = Schema::builder();
+    let json = schema_builder.add_json_field("json.blub", FAST);
+    let schema = schema_builder.build();
+    let index = Index::create_in_ram(schema);
+    let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
+    index_writer
+        .add_document(doc!(json => json!({"color.dot": "red", "color": {"nested":"red"} })))
+        .unwrap();
+    index_writer
+        .add_document(doc!(json => json!({"color.dot": "blue", "color": {"nested":"blue"} })))
+        .unwrap();
+    index_writer.commit().unwrap();
+    let reader = index.reader().unwrap();
+    let searcher = reader.searcher();
+
+    let agg: Aggregations = serde_json::from_value(json!({
+        "jsonagg1": {
+            "terms": {
+                "field": "json\\.blub.color\\.dot",
+            }
+        },
+        "jsonagg2": {
+            "terms": {
+                "field": "json\\.blub.color.nested",
+            }
+        }
+
+    }))
+    .unwrap();
+
+    let aggregation_collector = get_collector(agg);
+    let aggregation_results = searcher.search(&AllQuery, &aggregation_collector).unwrap();
+    let aggregation_res_json = serde_json::to_value(aggregation_results).unwrap();
+    assert_eq!(
+        &aggregation_res_json,
+        &serde_json::json!({
+            "jsonagg1": {
+                "buckets": [
+                    {"doc_count": 1, "key": "blue"},
+                    {"doc_count": 1, "key": "red"}
+                ],
+                "doc_count_error_upper_bound": 0,
+                "sum_other_doc_count": 0
+            },
+            "jsonagg2": {
+                "buckets": [
+                    {"doc_count": 1, "key": "blue"},
+                    {"doc_count": 1, "key": "red"}
+                ],
+                "doc_count_error_upper_bound": 0,
+                "sum_other_doc_count": 0
+            }
+
+        })
+    );
+}
+
 #[test]
 fn test_aggregation_on_json_object_empty_columns() {
    let mut schema_builder = Schema::builder();
    let json = schema_builder.add_json_field("json", FAST);
    let schema = schema_builder.build();
    let index = Index::create_in_ram(schema);
-    let mut index_writer = index.writer_for_tests().unwrap();
+    let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
    // => Empty column when accessing color
    index_writer
        .add_document(doc!(json => json!({"price": 10.0})))
@@ -748,7 +807,7 @@ fn test_aggregation_on_json_object_mixed_types() {
    let json = schema_builder.add_json_field("json", FAST);
    let schema = schema_builder.build();
    let index = Index::create_in_ram(schema);
-    let mut index_writer = index.writer_for_tests().unwrap();
+    let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
    // => Segment with all values numeric
    index_writer
        .add_document(doc!(json => json!({"mixed_type": 10.0})))
--- a/src/aggregation/bucket/histogram/date_histogram.rs
+++ b/src/aggregation/bucket/histogram/date_histogram.rs
@@ -252,7 +252,7 @@ pub mod tests {
    use crate::aggregation::tests::exec_request;
    use crate::indexer::NoMergePolicy;
    use crate::schema::{Schema, FAST, STRING};
-    use crate::Index;
+    use crate::{Index, IndexWriter, TantivyDocument};

    #[test]
    fn test_parse_into_millisecs() {
@@ -316,7 +316,7 @@ pub mod tests {
            index_writer.set_merge_policy(Box::new(NoMergePolicy));
            for values in segment_and_docs {
                for doc_str in values {
-                    let doc = schema.parse_document(doc_str)?;
+                    let doc = TantivyDocument::parse_json(&schema, doc_str)?;
                    index_writer.add_document(doc)?;
                }
                // writing the segment
@@ -328,7 +328,7 @@ pub mod tests {
                .searchable_segment_ids()
                .expect("Searchable segments failed.");
            if segment_ids.len() > 1 {
-                let mut index_writer = index.writer_for_tests()?;
+                let mut index_writer: IndexWriter = index.writer_for_tests()?;
                index_writer.merge(&segment_ids).wait()?;
                index_writer.wait_merging_threads()?;
            }
--- a/src/aggregation/bucket/term_agg.rs
+++ b/src/aggregation/bucket/term_agg.rs
@@ -601,7 +601,7 @@ mod tests {
    use crate::aggregation::AggregationLimits;
    use crate::indexer::NoMergePolicy;
    use crate::schema::{Schema, FAST, STRING};
-    use crate::Index;
+    use crate::{Index, IndexWriter};

    #[test]
    fn terms_aggregation_test_single_segment() -> crate::Result<()> {
@@ -1473,7 +1473,7 @@ mod tests {
        let json = schema_builder.add_json_field("json", FAST);
        let schema = schema_builder.build();
        let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_for_tests().unwrap();
+        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
        // => Segment with empty json
        index_writer.add_document(doc!()).unwrap();
        index_writer.commit().unwrap();
--- a/src/aggregation/bucket/term_missing_agg.rs
+++ b/src/aggregation/bucket/term_missing_agg.rs
@@ -117,7 +117,7 @@ mod tests {
    use crate::aggregation::agg_req::Aggregations;
    use crate::aggregation::tests::exec_request_with_query;
    use crate::schema::{Schema, FAST};
-    use crate::Index;
+    use crate::{Index, IndexWriter};

    #[test]
    fn terms_aggregation_missing_mixed_type_mult_seg_sub_agg() -> crate::Result<()> {
@@ -126,7 +126,7 @@ mod tests {
        let score = schema_builder.add_f64_field("score", FAST);
        let schema = schema_builder.build();
        let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_for_tests().unwrap();
+        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
        // => Segment with all values numeric
        index_writer
            .add_document(doc!(score => 1.0, json => json!({"mixed_type": 10.0})))
@@ -186,7 +186,7 @@ mod tests {
        let score = schema_builder.add_f64_field("score", FAST);
        let schema = schema_builder.build();
        let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_for_tests().unwrap();
+        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
        // => Segment with all values numeric
        index_writer.add_document(doc!(score => 1.0, json => json!({"mixed_type": 10.0})))?;
        index_writer.add_document(doc!(score => 5.0))?;
@@ -231,7 +231,7 @@ mod tests {
        let score = schema_builder.add_f64_field("score", FAST);
        let schema = schema_builder.build();
        let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_for_tests().unwrap();
+        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();

        index_writer.add_document(doc!(score => 5.0))?;
        index_writer.commit().unwrap();
@@ -278,7 +278,7 @@ mod tests {
        let score = schema_builder.add_f64_field("score", FAST);
        let schema = schema_builder.build();
        let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_for_tests().unwrap();
+        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();

        index_writer.add_document(doc!(score => 5.0))?;
        index_writer.add_document(doc!(score => 5.0))?;
@@ -323,7 +323,7 @@ mod tests {
        let json = schema_builder.add_json_field("json", FAST);
        let schema = schema_builder.build();
        let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_for_tests().unwrap();
+        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
        // => Segment with all values numeric
        index_writer
            .add_document(doc!(json => json!({"mixed_type": 10.0})))
@@ -385,7 +385,7 @@ mod tests {
        let json = schema_builder.add_json_field("json", FAST);
        let schema = schema_builder.build();
        let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_for_tests().unwrap();
+        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
        // => Segment with all values numeric
        index_writer
            .add_document(doc!(json => json!({"mixed_type": 10.0})))
@@ -427,7 +427,7 @@ mod tests {
        let json = schema_builder.add_json_field("json", FAST);
        let schema = schema_builder.build();
        let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_for_tests().unwrap();
+        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
        // => Segment with all values numeric
        index_writer
            .add_document(doc!(json => json!({"mixed_type": 10.0})))
--- a/src/aggregation/metric/max.rs
+++ b/src/aggregation/metric/max.rs
@@ -71,7 +71,7 @@ mod tests {
    use crate::aggregation::agg_req::Aggregations;
    use crate::aggregation::tests::exec_request_with_query;
    use crate::schema::{Schema, FAST};
-    use crate::Index;
+    use crate::{Index, IndexWriter};

    #[test]
    fn test_max_agg_with_missing() -> crate::Result<()> {
@@ -79,7 +79,7 @@ mod tests {
        let json = schema_builder.add_json_field("json", FAST);
        let schema = schema_builder.build();
        let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_for_tests().unwrap();
+        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
        // => Segment with empty json
        index_writer.add_document(doc!()).unwrap();
        index_writer.commit().unwrap();
--- a/src/aggregation/metric/mod.rs
+++ b/src/aggregation/metric/mod.rs
@@ -88,7 +88,7 @@ mod tests {
    use crate::aggregation::AggregationCollector;
    use crate::query::AllQuery;
    use crate::schema::{NumericOptions, Schema};
-    use crate::Index;
+    use crate::{Index, IndexWriter};

    #[test]
    fn test_metric_aggregations() {
@@ -96,7 +96,7 @@ mod tests {
        let field_options = NumericOptions::default().set_fast();
        let field = schema_builder.add_f64_field("price", field_options);
        let index = Index::create_in_ram(schema_builder.build());
-        let mut index_writer = index.writer_for_tests().unwrap();
+        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();

        for i in 0..3 {
            index_writer
--- a/src/aggregation/metric/stats.rs
+++ b/src/aggregation/metric/stats.rs
@@ -300,7 +300,7 @@ mod tests {
    use crate::aggregation::AggregationCollector;
    use crate::query::{AllQuery, TermQuery};
    use crate::schema::{IndexRecordOption, Schema, FAST};
-    use crate::{Index, Term};
+    use crate::{Index, IndexWriter, Term};

    #[test]
    fn test_aggregation_stats_empty_index() -> crate::Result<()> {
@@ -494,7 +494,7 @@ mod tests {
        let json = schema_builder.add_json_field("json", FAST);
        let schema = schema_builder.build();
        let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_for_tests().unwrap();
+        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
        // => Segment with empty json
        index_writer.add_document(doc!()).unwrap();
        index_writer.commit().unwrap();
@@ -541,7 +541,7 @@ mod tests {
        let json = schema_builder.add_json_field("json", FAST);
        let schema = schema_builder.build();
        let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_for_tests().unwrap();
+        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
        // => Segment with empty json
        index_writer.add_document(doc!()).unwrap();
        index_writer.commit().unwrap();
--- a/src/aggregation/mod.rs
+++ b/src/aggregation/mod.rs
@@ -319,7 +319,7 @@ mod tests {
    use crate::indexer::NoMergePolicy;
    use crate::query::{AllQuery, TermQuery};
    use crate::schema::{IndexRecordOption, Schema, TextFieldIndexing, FAST, STRING};
-    use crate::{Index, Term};
+    use crate::{Index, IndexWriter, Term};

    pub fn get_test_index_with_num_docs(
        merge_segments: bool,
@@ -451,7 +451,7 @@ mod tests {
                .searchable_segment_ids()
                .expect("Searchable segments failed.");
            if segment_ids.len() > 1 {
-                let mut index_writer = index.writer_for_tests()?;
+                let mut index_writer: IndexWriter = index.writer_for_tests()?;
                index_writer.merge(&segment_ids).wait()?;
                index_writer.wait_merging_threads()?;
            }
@@ -565,7 +565,7 @@ mod tests {
            let segment_ids = index
                .searchable_segment_ids()
                .expect("Searchable segments failed.");
-            let mut index_writer = index.writer_for_tests()?;
+            let mut index_writer: IndexWriter = index.writer_for_tests()?;
            index_writer.merge(&segment_ids).wait()?;
            index_writer.wait_merging_threads()?;
        }
--- a/src/collector/facet_collector.rs
+++ b/src/collector/facet_collector.rs
@@ -495,8 +495,8 @@ mod tests {
    use crate::collector::Count;
    use crate::core::Index;
    use crate::query::{AllQuery, QueryParser, TermQuery};
-    use crate::schema::{Document, Facet, FacetOptions, IndexRecordOption, Schema};
-    use crate::Term;
+    use crate::schema::{Facet, FacetOptions, IndexRecordOption, Schema, TantivyDocument};
+    use crate::{IndexWriter, Term};

    fn test_collapse_mapping_aux(
        facet_terms: &[&str],
@@ -559,7 +559,7 @@ mod tests {
        let facet_field = schema_builder.add_facet_field("facet", FacetOptions::default());
        let schema = schema_builder.build();
        let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_for_tests().unwrap();
+        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
        index_writer
            .add_document(doc!(facet_field=>Facet::from("/facet/a")))
            .unwrap();
@@ -588,7 +588,7 @@ mod tests {
        let schema = schema_builder.build();
        let index = Index::create_in_ram(schema);

-        let mut index_writer = index.writer_for_tests().unwrap();
+        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
        let num_facets: usize = 3 * 4 * 5;
        let facets: Vec<Facet> = (0..num_facets)
            .map(|mut n| {
@@ -601,7 +601,7 @@ mod tests {
            })
            .collect();
        for i in 0..num_facets * 10 {
-            let mut doc = Document::new();
+            let mut doc = TantivyDocument::new();
            doc.add_facet(facet_field, facets[i % num_facets].clone());
            index_writer.add_document(doc).unwrap();
        }
@@ -732,24 +732,25 @@ mod tests {
        let index = Index::create_in_ram(schema);

        let uniform = Uniform::new_inclusive(1, 100_000);
-        let mut docs: Vec<Document> = vec![("a", 10), ("b", 100), ("c", 7), ("d", 12), ("e", 21)]
-            .into_iter()
-            .flat_map(|(c, count)| {
-                let facet = Facet::from(&format!("/facet/{}", c));
-                let doc = doc!(facet_field => facet);
-                iter::repeat(doc).take(count)
-            })
-            .map(|mut doc| {
-                doc.add_facet(
-                    facet_field,
-                    &format!("/facet/{}", thread_rng().sample(uniform)),
-                );
-                doc
-            })
-            .collect();
+        let mut docs: Vec<TantivyDocument> =
+            vec![("a", 10), ("b", 100), ("c", 7), ("d", 12), ("e", 21)]
+                .into_iter()
+                .flat_map(|(c, count)| {
+                    let facet = Facet::from(&format!("/facet/{}", c));
+                    let doc = doc!(facet_field => facet);
+                    iter::repeat(doc).take(count)
+                })
+                .map(|mut doc| {
+                    doc.add_facet(
+                        facet_field,
+                        &format!("/facet/{}", thread_rng().sample(uniform)),
+                    );
+                    doc
+                })
+                .collect();
        docs[..].shuffle(&mut thread_rng());

-        let mut index_writer = index.writer_for_tests().unwrap();
+        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
        for doc in docs {
            index_writer.add_document(doc).unwrap();
        }
@@ -780,7 +781,7 @@ mod tests {
        let schema = schema_builder.build();
        let index = Index::create_in_ram(schema);

-        let docs: Vec<Document> = vec![("b", 2), ("a", 2), ("c", 4)]
+        let docs: Vec<TantivyDocument> = vec![("b", 2), ("a", 2), ("c", 4)]
            .into_iter()
            .flat_map(|(c, count)| {
                let facet = Facet::from(&format!("/facet/{}", c));
@@ -828,7 +829,7 @@ mod bench {
    use crate::collector::FacetCollector;
    use crate::query::AllQuery;
    use crate::schema::{Facet, Schema, INDEXED};
-    use crate::Index;
+    use crate::{Index, IndexWriter};

    #[bench]
    fn bench_facet_collector(b: &mut Bencher) {
@@ -847,7 +848,7 @@ mod bench {
        // 40425 docs
        docs[..].shuffle(&mut thread_rng());

-        let mut index_writer = index.writer_for_tests().unwrap();
+        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
        for doc in docs {
            index_writer.add_document(doc).unwrap();
        }
--- a/src/collector/filter_collector_wrapper.rs
+++ b/src/collector/filter_collector_wrapper.rs
@@ -12,8 +12,7 @@ use std::marker::PhantomData;
 use columnar::{BytesColumn, Column, DynamicColumn, HasAssociatedColumnType};

 use crate::collector::{Collector, SegmentCollector};
-use crate::schema::Field;
-use crate::{DocId, Score, SegmentReader, TantivyError};
+use crate::{DocId, Score, SegmentReader};

 /// The `FilterCollector` filters docs using a fast field value and a predicate.
 ///
@@ -50,13 +49,13 @@ use crate::{DocId, Score, SegmentReader, TantivyError};
 ///
 /// let query_parser = QueryParser::for_index(&index, vec![title]);
 /// let query = query_parser.parse_query("diary")?;
-/// let no_filter_collector = FilterCollector::new(price, |value: u64| value > 20_120u64, TopDocs::with_limit(2));
+/// let no_filter_collector = FilterCollector::new("price".to_string(), |value: u64| value > 20_120u64, TopDocs::with_limit(2));
 /// let top_docs = searcher.search(&query, &no_filter_collector)?;
 ///
 /// assert_eq!(top_docs.len(), 1);
 /// assert_eq!(top_docs[0].1, DocAddress::new(0, 1));
 ///
-/// let filter_all_collector: FilterCollector<_, _, u64> = FilterCollector::new(price, |value| value < 5u64, TopDocs::with_limit(2));
+/// let filter_all_collector: FilterCollector<_, _, u64> = FilterCollector::new("price".to_string(), |value| value < 5u64, TopDocs::with_limit(2));
 /// let filtered_top_docs = searcher.search(&query, &filter_all_collector)?;
 ///
 /// assert_eq!(filtered_top_docs.len(), 0);
@@ -70,7 +69,7 @@ use crate::{DocId, Score, SegmentReader, TantivyError};
 pub struct FilterCollector<TCollector, TPredicate, TPredicateValue>
 where TPredicate: 'static + Clone
 {
-    field: Field,
+    field: String,
    collector: TCollector,
    predicate: TPredicate,
    t_predicate_value: PhantomData<TPredicateValue>,
@@ -83,7 +82,7 @@ where
    TPredicate: Fn(TPredicateValue) -> bool + Send + Sync + Clone,
 {
    /// Create a new `FilterCollector`.
-    pub fn new(field: Field, predicate: TPredicate, collector: TCollector) -> Self {
+    pub fn new(field: String, predicate: TPredicate, collector: TCollector) -> Self {
        Self {
            field,
            predicate,
@@ -110,18 +109,7 @@ where
        segment_local_id: u32,
        segment_reader: &SegmentReader,
    ) -> crate::Result<Self::Child> {
-        let schema = segment_reader.schema();
-        let field_entry = schema.get_field_entry(self.field);
-        if !field_entry.is_fast() {
-            return Err(TantivyError::SchemaError(format!(
-                "Field {:?} is not a fast field.",
-                field_entry.name()
-            )));
-        }
-
-        let column_opt = segment_reader
-            .fast_fields()
-            .column_opt(field_entry.name())?;
+        let column_opt = segment_reader.fast_fields().column_opt(&self.field)?;

        let segment_collector = self
            .collector
@@ -229,7 +217,7 @@ where
 ///
 /// let query_parser = QueryParser::for_index(&index, vec![title]);
 /// let query = query_parser.parse_query("diary")?;
-/// let filter_collector = BytesFilterCollector::new(barcode, |bytes: &[u8]| bytes.starts_with(b"01"), TopDocs::with_limit(2));
+/// let filter_collector = BytesFilterCollector::new("barcode".to_string(), |bytes: &[u8]| bytes.starts_with(b"01"), TopDocs::with_limit(2));
 /// let top_docs = searcher.search(&query, &filter_collector)?;
 ///
 /// assert_eq!(top_docs.len(), 1);
@@ -240,7 +228,7 @@ where
 pub struct BytesFilterCollector<TCollector, TPredicate>
 where TPredicate: 'static + Clone
 {
-    field: Field,
+    field: String,
    collector: TCollector,
    predicate: TPredicate,
 }
@@ -251,7 +239,7 @@ where
    TPredicate: Fn(&[u8]) -> bool + Send + Sync + Clone,
 {
    /// Create a new `BytesFilterCollector`.
-    pub fn new(field: Field, predicate: TPredicate, collector: TCollector) -> Self {
+    pub fn new(field: String, predicate: TPredicate, collector: TCollector) -> Self {
        Self {
            field,
            predicate,
@@ -274,10 +262,7 @@ where
        segment_local_id: u32,
        segment_reader: &SegmentReader,
    ) -> crate::Result<Self::Child> {
-        let schema = segment_reader.schema();
-        let field_name = schema.get_field_name(self.field);
-
-        let column_opt = segment_reader.fast_fields().bytes(field_name)?;
+        let column_opt = segment_reader.fast_fields().bytes(&self.field)?;

        let segment_collector = self
            .collector
--- a/src/collector/mod.rs
+++ b/src/collector/mod.rs
@@ -97,7 +97,7 @@ pub use self::multi_collector::{FruitHandle, MultiCollector, MultiFruit};
 mod top_collector;

 mod top_score_collector;
-pub use self::top_score_collector::TopDocs;
+pub use self::top_score_collector::{TopDocs, TopNComputer};

 mod custom_score_top_collector;
 pub use self::custom_score_top_collector::{CustomScorer, CustomSegmentScorer};
--- a/src/collector/tests.rs
+++ b/src/collector/tests.rs
@@ -7,7 +7,9 @@ use crate::query::{AllQuery, QueryParser};
 use crate::schema::{Schema, FAST, TEXT};
 use crate::time::format_description::well_known::Rfc3339;
 use crate::time::OffsetDateTime;
-use crate::{doc, DateTime, DocAddress, DocId, Document, Index, Score, Searcher, SegmentOrdinal};
+use crate::{
+    doc, DateTime, DocAddress, DocId, Index, Score, Searcher, SegmentOrdinal, TantivyDocument,
+};

 pub const TEST_COLLECTOR_WITH_SCORE: TestCollector = TestCollector {
    compute_score: true,
@@ -40,7 +42,7 @@ pub fn test_filter_collector() -> crate::Result<()> {
    let query_parser = QueryParser::for_index(&index, vec![title]);
    let query = query_parser.parse_query("diary")?;
    let filter_some_collector = FilterCollector::new(
-        price,
+        "price".to_string(),
        &|value: u64| value > 20_120u64,
        TopDocs::with_limit(2),
    );
@@ -49,8 +51,11 @@ pub fn test_filter_collector() -> crate::Result<()> {
    assert_eq!(top_docs.len(), 1);
    assert_eq!(top_docs[0].1, DocAddress::new(0, 1));

-    let filter_all_collector: FilterCollector<_, _, u64> =
-        FilterCollector::new(price, &|value| value < 5u64, TopDocs::with_limit(2));
+    let filter_all_collector: FilterCollector<_, _, u64> = FilterCollector::new(
+        "price".to_string(),
+        &|value| value < 5u64,
+        TopDocs::with_limit(2),
+    );
    let filtered_top_docs = searcher.search(&query, &filter_all_collector).unwrap();

    assert_eq!(filtered_top_docs.len(), 0);
@@ -61,7 +66,8 @@ pub fn test_filter_collector() -> crate::Result<()> {
            > 0
    }

-    let filter_dates_collector = FilterCollector::new(date, &date_filter, TopDocs::with_limit(5));
+    let filter_dates_collector =
+        FilterCollector::new("date".to_string(), &date_filter, TopDocs::with_limit(5));
    let filtered_date_docs = searcher.search(&query, &filter_dates_collector)?;

    assert_eq!(filtered_date_docs.len(), 2);
@@ -280,8 +286,8 @@ fn make_test_searcher() -> crate::Result<Searcher> {
    let schema = Schema::builder().build();
    let index = Index::create_in_ram(schema);
    let mut index_writer = index.writer_for_tests()?;
-    index_writer.add_document(Document::default())?;
-    index_writer.add_document(Document::default())?;
+    index_writer.add_document(TantivyDocument::default())?;
+    index_writer.add_document(TantivyDocument::default())?;
    index_writer.commit()?;
    Ok(index.reader()?.searcher())
 }
--- a/src/collector/top_collector.rs
+++ b/src/collector/top_collector.rs
@@ -1,7 +1,7 @@
 use std::cmp::Ordering;
-use std::collections::BinaryHeap;
 use std::marker::PhantomData;

+use super::top_score_collector::TopNComputer;
 use crate::{DocAddress, DocId, SegmentOrdinal, SegmentReader};

 /// Contains a feature (field, score, etc.) of a document along with the document address.
@@ -20,6 +20,14 @@ pub(crate) struct ComparableDoc<T, D> {
    pub feature: T,
    pub doc: D,
 }
+impl<T: std::fmt::Debug, D: std::fmt::Debug> std::fmt::Debug for ComparableDoc<T, D> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("ComparableDoc")
+            .field("feature", &self.feature)
+            .field("doc", &self.doc)
+            .finish()
+    }
+}

 impl<T: PartialOrd, D: PartialOrd> PartialOrd for ComparableDoc<T, D> {
    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
@@ -91,18 +99,13 @@ where T: PartialOrd + Clone
        if self.limit == 0 {
            return Ok(Vec::new());
        }
-        let mut top_collector = BinaryHeap::new();
+        let mut top_collector = TopNComputer::new(self.limit + self.offset);
        for child_fruit in children {
            for (feature, doc) in child_fruit {
-                if top_collector.len() < (self.limit + self.offset) {
-                    top_collector.push(ComparableDoc { feature, doc });
-                } else if let Some(mut head) = top_collector.peek_mut() {
-                    if head.feature < feature {
-                        *head = ComparableDoc { feature, doc };
-                    }
-                }
+                top_collector.push(ComparableDoc { feature, doc });
            }
        }
+
        Ok(top_collector
            .into_sorted_vec()
            .into_iter()
@@ -111,7 +114,7 @@ where T: PartialOrd + Clone
            .collect())
    }

-    pub(crate) fn for_segment<F: PartialOrd>(
+    pub(crate) fn for_segment<F: PartialOrd + Clone>(
        &self,
        segment_id: SegmentOrdinal,
        _: &SegmentReader,
@@ -136,20 +139,18 @@ where T: PartialOrd + Clone
 /// The Top Collector keeps track of the K documents
 /// sorted by type `T`.
 ///
-/// The implementation is based on a `BinaryHeap`.
+/// The implementation is based on a repeatedly truncating on the median after K * 2 documents
 /// The theoretical complexity for collecting the top `K` out of `n` documents
-/// is `O(n log K)`.
+/// is `O(n + K)`.
 pub(crate) struct TopSegmentCollector<T> {
-    limit: usize,
-    heap: BinaryHeap<ComparableDoc<T, DocId>>,
+    topn_computer: TopNComputer<T, DocId>,
    segment_ord: u32,
 }

-impl<T: PartialOrd> TopSegmentCollector<T> {
+impl<T: PartialOrd + Clone> TopSegmentCollector<T> {
    fn new(segment_ord: SegmentOrdinal, limit: usize) -> TopSegmentCollector<T> {
        TopSegmentCollector {
-            limit,
-            heap: BinaryHeap::with_capacity(limit),
+            topn_computer: TopNComputer::new(limit),
            segment_ord,
        }
    }
@@ -158,7 +159,7 @@ impl<T: PartialOrd> TopSegmentCollector<T> {
 impl<T: PartialOrd + Clone> TopSegmentCollector<T> {
    pub fn harvest(self) -> Vec<(T, DocAddress)> {
        let segment_ord = self.segment_ord;
-        self.heap
+        self.topn_computer
            .into_sorted_vec()
            .into_iter()
            .map(|comparable_doc| {
@@ -173,33 +174,13 @@ impl<T: PartialOrd + Clone> TopSegmentCollector<T> {
            .collect()
    }

-    /// Return true if more documents have been collected than the limit.
-    #[inline]
-    pub(crate) fn at_capacity(&self) -> bool {
-        self.heap.len() >= self.limit
-    }
-
    /// Collects a document scored by the given feature
    ///
    /// It collects documents until it has reached the max capacity. Once it reaches capacity, it
    /// will compare the lowest scoring item with the given one and keep whichever is greater.
    #[inline]
    pub fn collect(&mut self, doc: DocId, feature: T) {
-        if self.at_capacity() {
-            // It's ok to unwrap as long as a limit of 0 is forbidden.
-            if let Some(limit_feature) = self.heap.peek().map(|head| head.feature.clone()) {
-                if limit_feature < feature {
-                    if let Some(mut head) = self.heap.peek_mut() {
-                        head.feature = feature;
-                        head.doc = doc;
-                    }
-                }
-            }
-        } else {
-            // we have not reached capacity yet, so we can just push the
-            // element.
-            self.heap.push(ComparableDoc { feature, doc });
-        }
+        self.topn_computer.push(ComparableDoc { feature, doc });
    }
 }

--- a/src/collector/top_score_collector.rs
+++ b/src/collector/top_score_collector.rs
@@ -1,4 +1,3 @@
-use std::collections::BinaryHeap;
 use std::fmt;
 use std::marker::PhantomData;
 use std::sync::Arc;
@@ -86,12 +85,15 @@ where
 /// The `TopDocs` collector keeps track of the top `K` documents
 /// sorted by their score.
 ///
-/// The implementation is based on a `BinaryHeap`.
-/// The theoretical complexity for collecting the top `K` out of `n` documents
-/// is `O(n log K)`.
+/// The implementation is based on a repeatedly truncating on the median after K * 2 documents
+/// with pattern defeating QuickSort.
+/// The theoretical complexity for collecting the top `K` out of `N` documents
+/// is `O(N + K)`.
 ///
-/// This collector guarantees a stable sorting in case of a tie on the
-/// document score. As such, it is suitable to implement pagination.
+/// This collector does not guarantee a stable sorting in case of a tie on the
+/// document score, for stable sorting `PartialOrd` needs to resolve on other fields
+/// like docid in case of score equality.
+/// Only then, it is suitable for pagination.
 ///
 /// ```rust
 /// use tantivy::collector::TopDocs;
@@ -661,50 +663,35 @@ impl Collector for TopDocs {
        reader: &SegmentReader,
    ) -> crate::Result<<Self::Child as SegmentCollector>::Fruit> {
        let heap_len = self.0.limit + self.0.offset;
-        let mut heap: BinaryHeap<ComparableDoc<Score, DocId>> = BinaryHeap::with_capacity(heap_len);
+        let mut top_n = TopNComputer::new(heap_len);

        if let Some(alive_bitset) = reader.alive_bitset() {
            let mut threshold = Score::MIN;
-            weight.for_each_pruning(threshold, reader, &mut |doc, score| {
+            top_n.threshold = Some(threshold);
+            weight.for_each_pruning(Score::MIN, reader, &mut |doc, score| {
                if alive_bitset.is_deleted(doc) {
                    return threshold;
                }
-                let heap_item = ComparableDoc {
+                let doc = ComparableDoc {
                    feature: score,
                    doc,
                };
-                if heap.len() < heap_len {
-                    heap.push(heap_item);
-                    if heap.len() == heap_len {
-                        threshold = heap.peek().map(|el| el.feature).unwrap_or(Score::MIN);
-                    }
-                    return threshold;
-                }
-                *heap.peek_mut().unwrap() = heap_item;
-                threshold = heap.peek().map(|el| el.feature).unwrap_or(Score::MIN);
+                top_n.push(doc);
+                threshold = top_n.threshold.unwrap_or(Score::MIN);
                threshold
            })?;
        } else {
            weight.for_each_pruning(Score::MIN, reader, &mut |doc, score| {
-                let heap_item = ComparableDoc {
+                let doc = ComparableDoc {
                    feature: score,
                    doc,
                };
-                if heap.len() < heap_len {
-                    heap.push(heap_item);
-                    // TODO the threshold is suboptimal for heap.len == heap_len
-                    if heap.len() == heap_len {
-                        return heap.peek().map(|el| el.feature).unwrap_or(Score::MIN);
-                    } else {
-                        return Score::MIN;
-                    }
-                }
-                *heap.peek_mut().unwrap() = heap_item;
-                heap.peek().map(|el| el.feature).unwrap_or(Score::MIN)
+                top_n.push(doc);
+                top_n.threshold.unwrap_or(Score::MIN)
            })?;
        }

-        let fruit = heap
+        let fruit = top_n
            .into_sorted_vec()
            .into_iter()
            .map(|cid| {
@@ -736,9 +723,81 @@ impl SegmentCollector for TopScoreSegmentCollector {
    }
 }

+/// Fast TopN Computation
+///
+/// For TopN == 0, it will be relative expensive.
+pub struct TopNComputer<Score, DocId> {
+    buffer: Vec<ComparableDoc<Score, DocId>>,
+    top_n: usize,
+    pub(crate) threshold: Option<Score>,
+}
+
+impl<Score, DocId> TopNComputer<Score, DocId>
+where
+    Score: PartialOrd + Clone,
+    DocId: Ord + Clone,
+{
+    /// Create a new `TopNComputer`.
+    /// Internally it will allocate a buffer of size `2 * top_n`.
+    pub fn new(top_n: usize) -> Self {
+        let vec_cap = top_n.max(1) * 2;
+        TopNComputer {
+            buffer: Vec::with_capacity(vec_cap),
+            top_n,
+            threshold: None,
+        }
+    }
+
+    #[inline]
+    pub(crate) fn push(&mut self, doc: ComparableDoc<Score, DocId>) {
+        if let Some(last_median) = self.threshold.clone() {
+            if doc.feature < last_median {
+                return;
+            }
+        }
+        if self.buffer.len() == self.buffer.capacity() {
+            let median = self.truncate_top_n();
+            self.threshold = Some(median);
+        }
+
+        // This is faster since it avoids the buffer resizing to be inlined from vec.push()
+        // (this is in the hot path)
+        // TODO: Replace with `push_within_capacity` when it's stabilized
+        let uninit = self.buffer.spare_capacity_mut();
+        // This cannot panic, because we truncate_median will at least remove one element, since
+        // the min capacity is 2.
+        uninit[0].write(doc);
+        // This is safe because it would panic in the line above
+        unsafe {
+            self.buffer.set_len(self.buffer.len() + 1);
+        }
+    }
+
+    #[inline(never)]
+    fn truncate_top_n(&mut self) -> Score {
+        // Use select_nth_unstable to find the top nth score
+        let (_, median_el, _) = self.buffer.select_nth_unstable(self.top_n);
+
+        let median_score = median_el.feature.clone();
+        // Remove all elements below the top_n
+        self.buffer.truncate(self.top_n);
+
+        median_score
+    }
+
+    pub(crate) fn into_sorted_vec(mut self) -> Vec<ComparableDoc<Score, DocId>> {
+        if self.buffer.len() > self.top_n {
+            self.truncate_top_n();
+        }
+        self.buffer.sort_unstable();
+        self.buffer
+    }
+}
+
 #[cfg(test)]
 mod tests {
-    use super::TopDocs;
+    use super::{TopDocs, TopNComputer};
+    use crate::collector::top_collector::ComparableDoc;
    use crate::collector::Collector;
    use crate::query::{AllQuery, Query, QueryParser};
    use crate::schema::{Field, Schema, FAST, STORED, TEXT};
@@ -767,6 +826,78 @@ mod tests {
        }
    }

+    #[test]
+    fn test_empty_topn_computer() {
+        let mut computer: TopNComputer<u32, u32> = TopNComputer::new(0);
+
+        computer.push(ComparableDoc {
+            feature: 1u32,
+            doc: 1u32,
+        });
+        computer.push(ComparableDoc {
+            feature: 1u32,
+            doc: 2u32,
+        });
+        computer.push(ComparableDoc {
+            feature: 1u32,
+            doc: 3u32,
+        });
+        assert!(computer.into_sorted_vec().is_empty());
+    }
+    #[test]
+    fn test_topn_computer() {
+        let mut computer: TopNComputer<u32, u32> = TopNComputer::new(2);
+
+        computer.push(ComparableDoc {
+            feature: 1u32,
+            doc: 1u32,
+        });
+        computer.push(ComparableDoc {
+            feature: 2u32,
+            doc: 2u32,
+        });
+        computer.push(ComparableDoc {
+            feature: 3u32,
+            doc: 3u32,
+        });
+        computer.push(ComparableDoc {
+            feature: 2u32,
+            doc: 4u32,
+        });
+        computer.push(ComparableDoc {
+            feature: 1u32,
+            doc: 5u32,
+        });
+        assert_eq!(
+            computer.into_sorted_vec(),
+            &[
+                ComparableDoc {
+                    feature: 3u32,
+                    doc: 3u32,
+                },
+                ComparableDoc {
+                    feature: 2u32,
+                    doc: 2u32,
+                }
+            ]
+        );
+    }
+
+    #[test]
+    fn test_topn_computer_no_panic() {
+        for top_n in 0..10 {
+            let mut computer: TopNComputer<u32, u32> = TopNComputer::new(top_n);
+
+            for _ in 0..1 + top_n * 2 {
+                computer.push(ComparableDoc {
+                    feature: 1u32,
+                    doc: 1u32,
+                });
+            }
+            let _vals = computer.into_sorted_vec();
+        }
+    }
+
    #[test]
    fn test_top_collector_not_at_capacity_without_offset() -> crate::Result<()> {
        let index = make_index()?;
@@ -852,20 +983,25 @@ mod tests {
        // using AllQuery to get a constant score
        let searcher = index.reader().unwrap().searcher();

+        let page_0 = searcher.search(&AllQuery, &TopDocs::with_limit(1)).unwrap();
+
        let page_1 = searcher.search(&AllQuery, &TopDocs::with_limit(2)).unwrap();

        let page_2 = searcher.search(&AllQuery, &TopDocs::with_limit(3)).unwrap();

        // precondition for the test to be meaningful: we did get documents
        // with the same score
+        assert!(page_0.iter().all(|result| result.0 == page_1[0].0));
        assert!(page_1.iter().all(|result| result.0 == page_1[0].0));
        assert!(page_2.iter().all(|result| result.0 == page_2[0].0));

        // sanity check since we're relying on make_index()
+        assert_eq!(page_0.len(), 1);
        assert_eq!(page_1.len(), 2);
        assert_eq!(page_2.len(), 3);

        assert_eq!(page_1, &page_2[..page_1.len()]);
+        assert_eq!(page_0, &page_2[..page_0.len()]);
    }

    #[test]
--- a/src/core/index.rs
+++ b/src/core/index.rs
@@ -18,10 +18,12 @@ use crate::directory::{Directory, ManagedDirectory, RamDirectory, INDEX_WRITER_L
 use crate::error::{DataCorruption, TantivyError};
 use crate::indexer::index_writer::{MAX_NUM_THREAD, MEMORY_BUDGET_NUM_BYTES_MIN};
 use crate::indexer::segment_updater::save_metas;
+use crate::indexer::IndexWriter;
 use crate::reader::{IndexReader, IndexReaderBuilder};
+use crate::schema::document::Document;
 use crate::schema::{Field, FieldType, Schema};
 use crate::tokenizer::{TextAnalyzer, TokenizerManager};
-use crate::IndexWriter;
+use crate::{merge_field_meta_data, FieldMetadata, SegmentReader};

 fn load_metas(
    directory: &dyn Directory,
@@ -184,11 +186,11 @@ impl IndexBuilder {
    ///
    /// It expects an originally empty directory, and will not run any GC operation.
    #[doc(hidden)]
-    pub fn single_segment_index_writer(
+    pub fn single_segment_index_writer<D: Document>(
        self,
        dir: impl Into<Box<dyn Directory>>,
        mem_budget: usize,
-    ) -> crate::Result<SingleSegmentIndexWriter> {
+    ) -> crate::Result<SingleSegmentIndexWriter<D>> {
        let index = self.create(dir)?;
        let index_simple_writer = SingleSegmentIndexWriter::new(index, mem_budget)?;
        Ok(index_simple_writer)
@@ -488,6 +490,28 @@ impl Index {
        self.inventory.all()
    }

+    /// Returns the list of fields that have been indexed in the Index.
+    /// The field list includes the field defined in the schema as well as the fields
+    /// that have been indexed as a part of a JSON field.
+    /// The returned field name is the full field name, including the name of the JSON field.
+    ///
+    /// The returned field names can be used in queries.
+    ///
+    /// Notice: If your data contains JSON fields this is **very expensive**, as it requires
+    /// browsing through the inverted index term dictionary and the columnar field dictionary.
+    ///
+    /// Disclaimer: Some fields may not be listed here. For instance, if the schema contains a json
+    /// field that is not indexed nor a fast field but is stored, it is possible for the field
+    /// to not be listed.
+    pub fn fields_metadata(&self) -> crate::Result<Vec<FieldMetadata>> {
+        let segments = self.searchable_segments()?;
+        let fields_metadata: Vec<Vec<FieldMetadata>> = segments
+            .into_iter()
+            .map(|segment| SegmentReader::open(&segment)?.fields_metadata())
+            .collect::<Result<_, _>>()?;
+        Ok(merge_field_meta_data(fields_metadata, &self.schema()))
+    }
+
    /// Creates a new segment_meta (Advanced user only).
    ///
    /// As long as the `SegmentMeta` lives, the files associated with the
@@ -531,11 +555,11 @@ impl Index {
    /// If the lockfile already exists, returns `Error::DirectoryLockBusy` or an `Error::IoError`.
    /// If the memory arena per thread is too small or too big, returns
    /// `TantivyError::InvalidArgument`
-    pub fn writer_with_num_threads(
+    pub fn writer_with_num_threads<D: Document>(
        &self,
        num_threads: usize,
        overall_memory_budget_in_bytes: usize,
-    ) -> crate::Result<IndexWriter> {
+    ) -> crate::Result<IndexWriter<D>> {
        let directory_lock = self
            .directory
            .acquire_lock(&INDEX_WRITER_LOCK)
@@ -564,8 +588,8 @@ impl Index {
    /// That index writer only simply has a single thread and a memory budget of 15 MB.
    /// Using a single thread gives us a deterministic allocation of DocId.
    #[cfg(test)]
-    pub fn writer_for_tests(&self) -> crate::Result<IndexWriter> {
-        self.writer_with_num_threads(1, 15_000_000)
+    pub fn writer_for_tests<D: Document>(&self) -> crate::Result<IndexWriter<D>> {
+        self.writer_with_num_threads(1, MEMORY_BUDGET_NUM_BYTES_MIN)
    }

    /// Creates a multithreaded writer
@@ -579,7 +603,10 @@ impl Index {
    /// If the lockfile already exists, returns `Error::FileAlreadyExists`.
    /// If the memory arena per thread is too small or too big, returns
    /// `TantivyError::InvalidArgument`
-    pub fn writer(&self, memory_budget_in_bytes: usize) -> crate::Result<IndexWriter> {
+    pub fn writer<D: Document>(
+        &self,
+        memory_budget_in_bytes: usize,
+    ) -> crate::Result<IndexWriter<D>> {
        let mut num_threads = std::cmp::min(num_cpus::get(), MAX_NUM_THREAD);
        let memory_budget_num_bytes_per_thread = memory_budget_in_bytes / num_threads;
        if memory_budget_num_bytes_per_thread < MEMORY_BUDGET_NUM_BYTES_MIN {
--- a/src/core/index_meta.rs
+++ b/src/core/index_meta.rs
@@ -142,6 +142,7 @@ impl SegmentMeta {
            SegmentComponent::FastFields => ".fast".to_string(),
            SegmentComponent::FieldNorms => ".fieldnorm".to_string(),
            SegmentComponent::Delete => format!(".{}.del", self.delete_opstamp().unwrap_or(0)),
+            SegmentComponent::FieldList => ".fieldlist".to_string(),
        });
        PathBuf::from(path)
    }
--- a/src/core/inverted_index_reader.rs
+++ b/src/core/inverted_index_reader.rs
@@ -1,11 +1,12 @@
 use std::io;

 use common::BinarySerializable;
+use fnv::FnvHashSet;

 use crate::directory::FileSlice;
 use crate::positions::PositionReader;
 use crate::postings::{BlockSegmentPostings, SegmentPostings, TermInfo};
-use crate::schema::{IndexRecordOption, Term};
+use crate::schema::{IndexRecordOption, Term, Type, JSON_END_OF_PATH};
 use crate::termdict::TermDictionary;

 /// The inverted index reader is in charge of accessing
@@ -69,6 +70,28 @@ impl InvertedIndexReader {
        &self.termdict
    }

+    /// Return the fields and types encoded in the dictionary in lexicographic order.
+    /// Only valid on JSON fields.
+    ///
+    /// Notice: This requires a full scan and therefore **very expensive**.
+    /// TODO: Move to sstable to use the index.
+    pub fn list_encoded_fields(&self) -> io::Result<Vec<(String, Type)>> {
+        let mut stream = self.termdict.stream()?;
+        let mut fields = Vec::new();
+        let mut fields_set = FnvHashSet::default();
+        while let Some((term, _term_info)) = stream.next() {
+            if let Some(index) = term.iter().position(|&byte| byte == JSON_END_OF_PATH) {
+                if !fields_set.contains(&term[..index + 2]) {
+                    fields_set.insert(term[..index + 2].to_vec());
+                    let typ = Type::from_code(term[index + 1]).unwrap();
+                    fields.push((String::from_utf8_lossy(&term[..index]).to_string(), typ));
+                }
+            }
+        }
+
+        Ok(fields)
+    }
+
    /// Resets the block segment to another position of the postings
    /// file.
    ///
--- a/src/core/json_utils.rs
+++ b/src/core/json_utils.rs
@@ -1,11 +1,11 @@
-use columnar::MonotonicallyMappableToU64;
-use common::replace_in_place;
-use murmurhash32::murmurhash2;
+use columnar::{ColumnType, MonotonicallyMappableToU64};
+use common::{replace_in_place, JsonPathWriter};
 use rustc_hash::FxHashMap;

 use crate::fastfield::FastValue;
 use crate::postings::{IndexingContext, IndexingPosition, PostingsWriter};
-use crate::schema::term::{JSON_PATH_SEGMENT_SEP, JSON_PATH_SEGMENT_SEP_STR};
+use crate::schema::document::{ReferenceValue, ReferenceValueLeaf, Value};
+use crate::schema::term::JSON_PATH_SEGMENT_SEP;
 use crate::schema::{Field, Type, DATE_TIME_PRECISION_INDEXED};
 use crate::time::format_description::well_known::Rfc3339;
 use crate::time::{OffsetDateTime, UtcOffset};
@@ -57,31 +57,41 @@ struct IndexingPositionsPerPath {
 }

 impl IndexingPositionsPerPath {
-    fn get_position(&mut self, term: &Term) -> &mut IndexingPosition {
-        self.positions_per_path
-            .entry(murmurhash2(term.serialized_term()))
-            .or_default()
+    fn get_position_from_id(&mut self, id: u32) -> &mut IndexingPosition {
+        self.positions_per_path.entry(id).or_default()
    }
 }

-pub(crate) fn index_json_values<'a>(
+/// Convert JSON_PATH_SEGMENT_SEP to a dot.
+pub fn json_path_sep_to_dot(path: &mut str) {
+    // This is safe since we are replacing a ASCII character by another ASCII character.
+    unsafe {
+        replace_in_place(JSON_PATH_SEGMENT_SEP, b'.', path.as_bytes_mut());
+    }
+}
+
+#[allow(clippy::too_many_arguments)]
+pub(crate) fn index_json_values<'a, V: Value<'a>>(
    doc: DocId,
-    json_values: impl Iterator<Item = crate::Result<&'a serde_json::Map<String, serde_json::Value>>>,
+    json_visitors: impl Iterator<Item = crate::Result<V::ObjectIter>>,
    text_analyzer: &mut TextAnalyzer,
    expand_dots_enabled: bool,
    term_buffer: &mut Term,
    postings_writer: &mut dyn PostingsWriter,
+    json_path_writer: &mut JsonPathWriter,
    ctx: &mut IndexingContext,
 ) -> crate::Result<()> {
-    let mut json_term_writer = JsonTermWriter::wrap(term_buffer, expand_dots_enabled);
+    json_path_writer.clear();
+    json_path_writer.set_expand_dots(expand_dots_enabled);
    let mut positions_per_path: IndexingPositionsPerPath = Default::default();
-    for json_value_res in json_values {
-        let json_value = json_value_res?;
-        index_json_object(
+    for json_visitor_res in json_visitors {
+        let json_visitor = json_visitor_res?;
+        index_json_object::<V>(
            doc,
-            json_value,
+            json_visitor,
            text_analyzer,
-            &mut json_term_writer,
+            term_buffer,
+            json_path_writer,
            postings_writer,
            ctx,
            &mut positions_per_path,
@@ -90,93 +100,156 @@ pub(crate) fn index_json_values<'a>(
    Ok(())
 }

-fn index_json_object(
+#[allow(clippy::too_many_arguments)]
+fn index_json_object<'a, V: Value<'a>>(
    doc: DocId,
-    json_value: &serde_json::Map<String, serde_json::Value>,
+    json_visitor: V::ObjectIter,
    text_analyzer: &mut TextAnalyzer,
-    json_term_writer: &mut JsonTermWriter,
+    term_buffer: &mut Term,
+    json_path_writer: &mut JsonPathWriter,
    postings_writer: &mut dyn PostingsWriter,
    ctx: &mut IndexingContext,
    positions_per_path: &mut IndexingPositionsPerPath,
 ) {
-    for (json_path_segment, json_value) in json_value {
-        json_term_writer.push_path_segment(json_path_segment);
+    for (json_path_segment, json_value_visitor) in json_visitor {
+        json_path_writer.push(json_path_segment);
        index_json_value(
            doc,
-            json_value,
+            json_value_visitor,
            text_analyzer,
-            json_term_writer,
+            term_buffer,
+            json_path_writer,
            postings_writer,
            ctx,
            positions_per_path,
        );
-        json_term_writer.pop_path_segment();
+        json_path_writer.pop();
    }
 }

-fn index_json_value(
+#[allow(clippy::too_many_arguments)]
+fn index_json_value<'a, V: Value<'a>>(
    doc: DocId,
-    json_value: &serde_json::Value,
+    json_value: V,
    text_analyzer: &mut TextAnalyzer,
-    json_term_writer: &mut JsonTermWriter,
+    term_buffer: &mut Term,
+    json_path_writer: &mut JsonPathWriter,
    postings_writer: &mut dyn PostingsWriter,
    ctx: &mut IndexingContext,
    positions_per_path: &mut IndexingPositionsPerPath,
 ) {
-    match json_value {
-        serde_json::Value::Null => {}
-        serde_json::Value::Bool(val_bool) => {
-            json_term_writer.set_fast_value(*val_bool);
-            postings_writer.subscribe(doc, 0u32, json_term_writer.term(), ctx);
-        }
-        serde_json::Value::Number(number) => {
-            if let Some(number_i64) = number.as_i64() {
-                json_term_writer.set_fast_value(number_i64);
-            } else if let Some(number_u64) = number.as_u64() {
-                json_term_writer.set_fast_value(number_u64);
-            } else if let Some(number_f64) = number.as_f64() {
-                json_term_writer.set_fast_value(number_f64);
-            }
-            postings_writer.subscribe(doc, 0u32, json_term_writer.term(), ctx);
-        }
-        serde_json::Value::String(text) => match infer_type_from_str(text) {
-            TextOrDateTime::Text(text) => {
-                let mut token_stream = text_analyzer.token_stream(text);
-                // TODO make sure the chain position works out.
-                json_term_writer.close_path_and_set_type(Type::Str);
-                let indexing_position = positions_per_path.get_position(json_term_writer.term());
+    let set_path_id = |term_buffer: &mut Term, unordered_id: u32| {
+        term_buffer.truncate_value_bytes(0);
+        term_buffer.append_bytes(&unordered_id.to_be_bytes());
+    };
+    let set_type = |term_buffer: &mut Term, typ: Type| {
+        term_buffer.append_bytes(&[typ.to_code()]);
+    };
+
+    match json_value.as_value() {
+        ReferenceValue::Leaf(leaf) => match leaf {
+            ReferenceValueLeaf::Null => {}
+            ReferenceValueLeaf::Str(val) => {
+                let mut token_stream = text_analyzer.token_stream(val);
+                let unordered_id = ctx
+                    .path_to_unordered_id
+                    .get_or_allocate_unordered_id(json_path_writer.as_str(), ColumnType::Str);
+
+                // TODO: make sure the chain position works out.
+                set_path_id(term_buffer, unordered_id);
+                set_type(term_buffer, Type::Str);
+                let indexing_position = positions_per_path.get_position_from_id(unordered_id);
                postings_writer.index_text(
                    doc,
                    &mut *token_stream,
-                    json_term_writer.term_buffer,
+                    term_buffer,
                    ctx,
                    indexing_position,
                );
            }
-            TextOrDateTime::DateTime(dt) => {
-                json_term_writer.set_fast_value(DateTime::from_utc(dt));
-                postings_writer.subscribe(doc, 0u32, json_term_writer.term(), ctx);
+            ReferenceValueLeaf::U64(val) => {
+                set_path_id(
+                    term_buffer,
+                    ctx.path_to_unordered_id
+                        .get_or_allocate_unordered_id(json_path_writer.as_str(), ColumnType::U64),
+                );
+                term_buffer.append_type_and_fast_value(val);
+                postings_writer.subscribe(doc, 0u32, term_buffer, ctx);
+            }
+            ReferenceValueLeaf::I64(val) => {
+                set_path_id(
+                    term_buffer,
+                    ctx.path_to_unordered_id
+                        .get_or_allocate_unordered_id(json_path_writer.as_str(), ColumnType::I64),
+                );
+                term_buffer.append_type_and_fast_value(val);
+                postings_writer.subscribe(doc, 0u32, term_buffer, ctx);
+            }
+            ReferenceValueLeaf::F64(val) => {
+                set_path_id(
+                    term_buffer,
+                    ctx.path_to_unordered_id
+                        .get_or_allocate_unordered_id(json_path_writer.as_str(), ColumnType::F64),
+                );
+                term_buffer.append_type_and_fast_value(val);
+                postings_writer.subscribe(doc, 0u32, term_buffer, ctx);
+            }
+            ReferenceValueLeaf::Bool(val) => {
+                set_path_id(
+                    term_buffer,
+                    ctx.path_to_unordered_id
+                        .get_or_allocate_unordered_id(json_path_writer.as_str(), ColumnType::Bool),
+                );
+                term_buffer.append_type_and_fast_value(val);
+                postings_writer.subscribe(doc, 0u32, term_buffer, ctx);
+            }
+            ReferenceValueLeaf::Date(val) => {
+                set_path_id(
+                    term_buffer,
+                    ctx.path_to_unordered_id.get_or_allocate_unordered_id(
+                        json_path_writer.as_str(),
+                        ColumnType::DateTime,
+                    ),
+                );
+                term_buffer.append_type_and_fast_value(val);
+                postings_writer.subscribe(doc, 0u32, term_buffer, ctx);
+            }
+            ReferenceValueLeaf::PreTokStr(_) => {
+                unimplemented!(
+                    "Pre-tokenized string support in dynamic fields is not yet implemented"
+                )
+            }
+            ReferenceValueLeaf::Bytes(_) => {
+                unimplemented!("Bytes support in dynamic fields is not yet implemented")
+            }
+            ReferenceValueLeaf::Facet(_) => {
+                unimplemented!("Facet support in dynamic fields is not yet implemented")
+            }
+            ReferenceValueLeaf::IpAddr(_) => {
+                unimplemented!("IP address support in dynamic fields is not yet implemented")
            }
        },
-        serde_json::Value::Array(arr) => {
-            for val in arr {
+        ReferenceValue::Array(elements) => {
+            for val in elements {
                index_json_value(
                    doc,
                    val,
                    text_analyzer,
-                    json_term_writer,
+                    term_buffer,
+                    json_path_writer,
                    postings_writer,
                    ctx,
                    positions_per_path,
                );
            }
        }
-        serde_json::Value::Object(map) => {
-            index_json_object(
+        ReferenceValue::Object(object) => {
+            index_json_object::<V>(
                doc,
-                map,
+                object,
                text_analyzer,
-                json_term_writer,
+                term_buffer,
+                json_path_writer,
                postings_writer,
                ctx,
                positions_per_path,
@@ -185,21 +258,6 @@ fn index_json_value(
    }
 }

-enum TextOrDateTime<'a> {
-    Text(&'a str),
-    DateTime(OffsetDateTime),
-}
-
-fn infer_type_from_str(text: &str) -> TextOrDateTime {
-    match OffsetDateTime::parse(text, &Rfc3339) {
-        Ok(dt) => {
-            let dt_utc = dt.to_offset(UtcOffset::UTC);
-            TextOrDateTime::DateTime(dt_utc)
-        }
-        Err(_) => TextOrDateTime::Text(text),
-    }
-}
-
 // Tries to infer a JSON type from a string.
 pub fn convert_to_fast_value_and_get_term(
    json_term_writer: &mut JsonTermWriter,
@@ -272,7 +330,7 @@ pub struct JsonTermWriter<'a> {
 /// In other words,
 /// - `k8s.node` ends up as `["k8s", "node"]`.
 /// - `k8s\.node` ends up as `["k8s.node"]`.
-fn split_json_path(json_path: &str) -> Vec<String> {
+pub fn split_json_path(json_path: &str) -> Vec<String> {
    let mut escaped_state: bool = false;
    let mut json_path_segments = Vec::new();
    let mut buffer = String::new();
@@ -312,17 +370,13 @@ pub(crate) fn encode_column_name(
    json_path: &str,
    expand_dots_enabled: bool,
 ) -> String {
-    let mut column_key: String = String::with_capacity(field_name.len() + json_path.len() + 1);
-    column_key.push_str(field_name);
-    for mut segment in split_json_path(json_path) {
-        column_key.push_str(JSON_PATH_SEGMENT_SEP_STR);
-        if expand_dots_enabled {
-            // We need to replace `.` by JSON_PATH_SEGMENT_SEP.
-            unsafe { replace_in_place(b'.', JSON_PATH_SEGMENT_SEP, segment.as_bytes_mut()) };
-        }
-        column_key.push_str(&segment);
+    let mut path = JsonPathWriter::default();
+    path.push(field_name);
+    path.set_expand_dots(expand_dots_enabled);
+    for segment in split_json_path(json_path) {
+        path.push(&segment);
    }
-    column_key
+    path.into()
 }

 impl<'a> JsonTermWriter<'a> {
@@ -362,6 +416,7 @@ impl<'a> JsonTermWriter<'a> {
        self.term_buffer.append_bytes(&[typ.to_code()]);
    }

+    // TODO: Remove this function and use JsonPathWriter instead.
    pub fn push_path_segment(&mut self, segment: &str) {
        // the path stack should never be empty.
        self.trim_to_end_of_path();
--- a/src/core/mod.rs
+++ b/src/core/mod.rs
@@ -25,7 +25,7 @@ pub use self::searcher::{Searcher, SearcherGeneration};
 pub use self::segment::Segment;
 pub use self::segment_component::SegmentComponent;
 pub use self::segment_id::SegmentId;
-pub use self::segment_reader::SegmentReader;
+pub use self::segment_reader::{merge_field_meta_data, FieldMetadata, SegmentReader};
 pub use self::single_segment_index_writer::SingleSegmentIndexWriter;

 /// The meta file contains all the information about the list of segments and the schema
--- a/src/core/searcher.rs
+++ b/src/core/searcher.rs
@@ -5,7 +5,8 @@ use std::{fmt, io};
 use crate::collector::Collector;
 use crate::core::{Executor, SegmentReader};
 use crate::query::{Bm25StatisticsProvider, EnableScoring, Query};
-use crate::schema::{Document, Schema, Term};
+use crate::schema::document::DocumentDeserialize;
+use crate::schema::{Schema, Term};
 use crate::space_usage::SearcherSpaceUsage;
 use crate::store::{CacheStats, StoreReader};
 use crate::{DocAddress, Index, Opstamp, SegmentId, TrackedObject};
@@ -83,7 +84,7 @@ impl Searcher {
    ///
    /// The searcher uses the segment ordinal to route the
    /// request to the right `Segment`.
-    pub fn doc(&self, doc_address: DocAddress) -> crate::Result<Document> {
+    pub fn doc<D: DocumentDeserialize>(&self, doc_address: DocAddress) -> crate::Result<D> {
        let store_reader = &self.inner.store_readers[doc_address.segment_ord as usize];
        store_reader.get(doc_address.doc_id)
    }
@@ -103,7 +104,10 @@ impl Searcher {

    /// Fetches a document in an asynchronous manner.
    #[cfg(feature = "quickwit")]
-    pub async fn doc_async(&self, doc_address: DocAddress) -> crate::Result<Document> {
+    pub async fn doc_async<D: DocumentDeserialize>(
+        &self,
+        doc_address: DocAddress,
+    ) -> crate::Result<D> {
        let store_reader = &self.inner.store_readers[doc_address.segment_ord as usize];
        store_reader.get_async(doc_address.doc_id).await
    }
--- a/src/core/segment_component.rs
+++ b/src/core/segment_component.rs
@@ -27,12 +27,14 @@ pub enum SegmentComponent {
    /// Bitset describing which document of the segment is alive.
    /// (It was representing deleted docs but changed to represent alive docs from v0.17)
    Delete,
+    /// Field list describing the fields in the segment.
+    FieldList,
 }

 impl SegmentComponent {
    /// Iterates through the components.
    pub fn iterator() -> slice::Iter<'static, SegmentComponent> {
-        static SEGMENT_COMPONENTS: [SegmentComponent; 8] = [
+        static SEGMENT_COMPONENTS: [SegmentComponent; 9] = [
            SegmentComponent::Postings,
            SegmentComponent::Positions,
            SegmentComponent::FastFields,
@@ -41,6 +43,7 @@ impl SegmentComponent {
            SegmentComponent::Store,
            SegmentComponent::TempStore,
            SegmentComponent::Delete,
+            SegmentComponent::FieldList,
        ];
        SEGMENT_COMPONENTS.iter()
    }
--- a/src/core/segment_reader.rs
+++ b/src/core/segment_reader.rs
@@ -1,11 +1,15 @@
 use std::collections::HashMap;
+use std::ops::BitOrAssign;
 use std::sync::{Arc, RwLock};
 use std::{fmt, io};

+use itertools::Itertools;
+
 use crate::core::{InvertedIndexReader, Segment, SegmentComponent, SegmentId};
 use crate::directory::{CompositeFile, FileSlice};
 use crate::error::DataCorruption;
 use crate::fastfield::{intersect_alive_bitsets, AliveBitSet, FacetReader, FastFieldReaders};
+use crate::field_list::read_split_fields;
 use crate::fieldnorm::{FieldNormReader, FieldNormReaders};
 use crate::schema::{Field, IndexRecordOption, Schema, Type};
 use crate::space_usage::SegmentSpaceUsage;
@@ -39,6 +43,7 @@ pub struct SegmentReader {
    fast_fields_readers: FastFieldReaders,
    fieldnorm_readers: FieldNormReaders,

+    list_fields_file: Option<FileSlice>, // Optional field list file for backwards compatibility
    store_file: FileSlice,
    alive_bitset_opt: Option<AliveBitSet>,
    schema: Schema,
@@ -148,6 +153,7 @@ impl SegmentReader {
        let termdict_composite = CompositeFile::open(&termdict_file)?;

        let store_file = segment.open_read(SegmentComponent::Store)?;
+        let list_fields_file = segment.open_read(SegmentComponent::FieldList).ok();

        crate::fail_point!("SegmentReader::open#middle");

@@ -196,6 +202,7 @@ impl SegmentReader {
            segment_id: segment.id(),
            delete_opstamp: segment.meta().delete_opstamp(),
            store_file,
+            list_fields_file,
            alive_bitset_opt,
            positions_composite,
            schema,
@@ -280,6 +287,41 @@ impl SegmentReader {
        Ok(inv_idx_reader)
    }

+    /// Returns the list of fields that have been indexed in the segment.
+    /// The field list includes the field defined in the schema as well as the fields
+    /// that have been indexed as a part of a JSON field.
+    /// The returned field name is the full field name, including the name of the JSON field.
+    ///
+    /// The returned field names can be used in queries.
+    ///
+    /// Notice: If your data contains JSON fields this is **very expensive**, as it requires
+    /// browsing through the inverted index term dictionary and the columnar field dictionary.
+    ///
+    /// Disclaimer: Some fields may not be listed here. For instance, if the schema contains a json
+    /// field that is not indexed nor a fast field but is stored, it is possible for the field
+    /// to not be listed.
+    pub fn fields_metadata(&self) -> crate::Result<Vec<FieldMetadata>> {
+        if let Some(list_fields_file) = self.list_fields_file.as_ref() {
+            let file = list_fields_file.read_bytes()?;
+            let fields_metadata =
+                read_split_fields(file)?.collect::<io::Result<Vec<FieldMetadata>>>();
+            fields_metadata.map_err(|e| e.into())
+        } else {
+            // Schema fallback
+            Ok(self
+                .schema()
+                .fields()
+                .map(|(_field, entry)| FieldMetadata {
+                    field_name: entry.name().to_string(),
+                    typ: entry.field_type().value_type(),
+                    indexed: entry.is_indexed(),
+                    stored: entry.is_stored(),
+                    fast: entry.is_fast(),
+                })
+                .collect())
+        }
+    }
+
    /// Returns the segment id
    pub fn segment_id(&self) -> SegmentId {
        self.segment_id
@@ -330,6 +372,65 @@ impl SegmentReader {
    }
 }

+#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
+/// FieldMetadata
+pub struct FieldMetadata {
+    /// The field name
+    // Notice: Don't reorder the declaration of 1.field_name 2.typ, as it is used for ordering by
+    // field_name then typ.
+    pub field_name: String,
+    /// The field type
+    // Notice: Don't reorder the declaration of 1.field_name 2.typ, as it is used for ordering by
+    // field_name then typ.
+    pub typ: Type,
+    /// Is the field indexed for search
+    pub indexed: bool,
+    /// Is the field stored in the doc store
+    pub stored: bool,
+    /// Is the field stored in the columnar storage
+    pub fast: bool,
+}
+impl BitOrAssign for FieldMetadata {
+    fn bitor_assign(&mut self, rhs: Self) {
+        assert!(self.field_name == rhs.field_name);
+        assert!(self.typ == rhs.typ);
+        self.indexed |= rhs.indexed;
+        self.stored |= rhs.stored;
+        self.fast |= rhs.fast;
+    }
+}
+
+// Maybe too slow for the high cardinality case
+fn is_field_stored(field_name: &str, schema: &Schema) -> bool {
+    schema
+        .find_field(field_name)
+        .map(|(field, _path)| schema.get_field_entry(field).is_stored())
+        .unwrap_or(false)
+}
+
+/// Helper to merge the field metadata from multiple segments.
+pub fn merge_field_meta_data(
+    field_metadatas: Vec<Vec<FieldMetadata>>,
+    schema: &Schema,
+) -> Vec<FieldMetadata> {
+    let mut merged_field_metadata = Vec::new();
+    for (_key, mut group) in &field_metadatas
+        .into_iter()
+        .kmerge_by(|left, right| left < right)
+        // TODO: Remove allocation
+        .group_by(|el| (el.field_name.to_string(), el.typ))
+    {
+        let mut merged: FieldMetadata = group.next().unwrap();
+        for el in group {
+            merged |= el;
+        }
+        // Currently is_field_stored is maybe too slow for the high cardinality case
+        merged.stored = is_field_stored(&merged.field_name, schema);
+        merged_field_metadata.push(merged);
+    }
+    merged_field_metadata
+}
+
 fn intersect_alive_bitset(
    left_opt: Option<AliveBitSet>,
    right_opt: Option<AliveBitSet>,
@@ -353,9 +454,127 @@ impl fmt::Debug for SegmentReader {

 #[cfg(test)]
 mod test {
+    use super::*;
    use crate::core::Index;
-    use crate::schema::{Schema, Term, STORED, TEXT};
-    use crate::DocId;
+    use crate::schema::{Schema, SchemaBuilder, Term, STORED, TEXT};
+    use crate::{DocId, FieldMetadata, IndexWriter};
+
+    #[test]
+    fn test_merge_field_meta_data_same() {
+        let schema = SchemaBuilder::new().build();
+        let field_metadata1 = FieldMetadata {
+            field_name: "a".to_string(),
+            typ: crate::schema::Type::Str,
+            indexed: true,
+            stored: false,
+            fast: true,
+        };
+        let field_metadata2 = FieldMetadata {
+            field_name: "a".to_string(),
+            typ: crate::schema::Type::Str,
+            indexed: true,
+            stored: false,
+            fast: true,
+        };
+        let res = merge_field_meta_data(
+            vec![vec![field_metadata1.clone()], vec![field_metadata2]],
+            &schema,
+        );
+        assert_eq!(res, vec![field_metadata1]);
+    }
+    #[test]
+    fn test_merge_field_meta_data_different() {
+        let schema = SchemaBuilder::new().build();
+        let field_metadata1 = FieldMetadata {
+            field_name: "a".to_string(),
+            typ: crate::schema::Type::Str,
+            indexed: false,
+            stored: false,
+            fast: true,
+        };
+        let field_metadata2 = FieldMetadata {
+            field_name: "b".to_string(),
+            typ: crate::schema::Type::Str,
+            indexed: false,
+            stored: false,
+            fast: true,
+        };
+        let field_metadata3 = FieldMetadata {
+            field_name: "a".to_string(),
+            typ: crate::schema::Type::Str,
+            indexed: true,
+            stored: false,
+            fast: false,
+        };
+        let res = merge_field_meta_data(
+            vec![
+                vec![field_metadata1.clone(), field_metadata2.clone()],
+                vec![field_metadata3],
+            ],
+            &schema,
+        );
+        let field_metadata_expected1 = FieldMetadata {
+            field_name: "a".to_string(),
+            typ: crate::schema::Type::Str,
+            indexed: true,
+            stored: false,
+            fast: true,
+        };
+        assert_eq!(res, vec![field_metadata_expected1, field_metadata2.clone()]);
+    }
+    #[test]
+    fn test_merge_field_meta_data_merge() {
+        use pretty_assertions::assert_eq;
+        let get_meta_data = |name: &str, typ: Type| FieldMetadata {
+            field_name: name.to_string(),
+            typ,
+            indexed: false,
+            stored: false,
+            fast: true,
+        };
+        let schema = SchemaBuilder::new().build();
+        let mut metas = vec![get_meta_data("d", Type::Str), get_meta_data("e", Type::U64)];
+        metas.sort();
+        let res = merge_field_meta_data(vec![vec![get_meta_data("e", Type::Str)], metas], &schema);
+        assert_eq!(
+            res,
+            vec![
+                get_meta_data("d", Type::Str),
+                get_meta_data("e", Type::Str),
+                get_meta_data("e", Type::U64),
+            ]
+        );
+    }
+    #[test]
+    fn test_merge_field_meta_data_bitxor() {
+        let field_metadata1 = FieldMetadata {
+            field_name: "a".to_string(),
+            typ: crate::schema::Type::Str,
+            indexed: false,
+            stored: false,
+            fast: true,
+        };
+        let field_metadata2 = FieldMetadata {
+            field_name: "a".to_string(),
+            typ: crate::schema::Type::Str,
+            indexed: true,
+            stored: false,
+            fast: false,
+        };
+        let field_metadata_expected = FieldMetadata {
+            field_name: "a".to_string(),
+            typ: crate::schema::Type::Str,
+            indexed: true,
+            stored: false,
+            fast: true,
+        };
+        let mut res1 = field_metadata1.clone();
+        res1 |= field_metadata2.clone();
+        let mut res2 = field_metadata2.clone();
+        res2 |= field_metadata1;
+        assert_eq!(res1, field_metadata_expected);
+        assert_eq!(res2, field_metadata_expected);
+    }

    #[test]
    fn test_num_alive() -> crate::Result<()> {
@@ -366,7 +585,7 @@ mod test {
        let name = schema.get_field("name").unwrap();

        {
-            let mut index_writer = index.writer_for_tests()?;
+            let mut index_writer: IndexWriter = index.writer_for_tests()?;
            index_writer.add_document(doc!(name => "tantivy"))?;
            index_writer.add_document(doc!(name => "horse"))?;
            index_writer.add_document(doc!(name => "jockey"))?;
@@ -392,7 +611,7 @@ mod test {
        let name = schema.get_field("name").unwrap();

        {
-            let mut index_writer = index.writer_for_tests()?;
+            let mut index_writer: IndexWriter = index.writer_for_tests()?;
            index_writer.add_document(doc!(name => "tantivy"))?;
            index_writer.add_document(doc!(name => "horse"))?;
            index_writer.add_document(doc!(name => "jockey"))?;
@@ -402,7 +621,7 @@ mod test {
        }

        {
-            let mut index_writer2 = index.writer(50_000_000)?;
+            let mut index_writer2: IndexWriter = index.writer(50_000_000)?;
            index_writer2.delete_term(Term::from_field_text(name, "horse"));
            index_writer2.delete_term(Term::from_field_text(name, "cap"));

--- a/src/core/single_segment_index_writer.rs
+++ b/src/core/single_segment_index_writer.rs
@@ -1,16 +1,20 @@
+use std::marker::PhantomData;
+
 use crate::indexer::operation::AddOperation;
 use crate::indexer::segment_updater::save_metas;
 use crate::indexer::SegmentWriter;
-use crate::{Directory, Document, Index, IndexMeta, Opstamp, Segment};
+use crate::schema::document::Document;
+use crate::{Directory, Index, IndexMeta, Opstamp, Segment, TantivyDocument};

 #[doc(hidden)]
-pub struct SingleSegmentIndexWriter {
+pub struct SingleSegmentIndexWriter<D: Document = TantivyDocument> {
    segment_writer: SegmentWriter,
    segment: Segment,
    opstamp: Opstamp,
+    _phantom: PhantomData<D>,
 }

-impl SingleSegmentIndexWriter {
+impl<D: Document> SingleSegmentIndexWriter<D> {
    pub fn new(index: Index, mem_budget: usize) -> crate::Result<Self> {
        let segment = index.new_segment();
        let segment_writer = SegmentWriter::for_segment(mem_budget, segment.clone())?;
@@ -18,6 +22,7 @@ impl SingleSegmentIndexWriter {
            segment_writer,
            segment,
            opstamp: 0,
+            _phantom: PhantomData,
        })
    }

@@ -25,7 +30,7 @@ impl SingleSegmentIndexWriter {
        self.segment_writer.mem_usage()
    }

-    pub fn add_document(&mut self, document: Document) -> crate::Result<()> {
+    pub fn add_document(&mut self, document: D) -> crate::Result<()> {
        let opstamp = self.opstamp;
        self.opstamp += 1;
        self.segment_writer
--- a/src/core/tests.rs
+++ b/src/core/tests.rs
@@ -1,12 +1,13 @@
 use crate::collector::Count;
 use crate::directory::{RamDirectory, WatchCallback};
-use crate::indexer::NoMergePolicy;
+use crate::indexer::{LogMergePolicy, NoMergePolicy};
+use crate::json_utils::JsonTermWriter;
 use crate::query::TermQuery;
-use crate::schema::{Field, IndexRecordOption, Schema, INDEXED, STRING, TEXT};
+use crate::schema::{Field, IndexRecordOption, Schema, Type, INDEXED, STRING, TEXT};
 use crate::tokenizer::TokenizerManager;
 use crate::{
-    Directory, Document, Index, IndexBuilder, IndexReader, IndexSettings, ReloadPolicy, SegmentId,
-    Term,
+    Directory, DocSet, Index, IndexBuilder, IndexReader, IndexSettings, IndexWriter, Postings,
+    ReloadPolicy, SegmentId, TantivyDocument, Term,
 };

 #[test]
@@ -121,7 +122,7 @@ fn test_index_on_commit_reload_policy() -> crate::Result<()> {
    let index = Index::create_in_ram(schema);
    let reader = index
        .reader_builder()
-        .reload_policy(ReloadPolicy::OnCommit)
+        .reload_policy(ReloadPolicy::OnCommitWithDelay)
        .try_into()
        .unwrap();
    assert_eq!(reader.searcher().num_docs(), 0);
@@ -147,7 +148,7 @@ mod mmap_specific {
        let index = Index::create_in_dir(tempdir_path, schema).unwrap();
        let reader = index
            .reader_builder()
-            .reload_policy(ReloadPolicy::OnCommit)
+            .reload_policy(ReloadPolicy::OnCommitWithDelay)
            .try_into()
            .unwrap();
        assert_eq!(reader.searcher().num_docs(), 0);
@@ -159,7 +160,7 @@ mod mmap_specific {
        let schema = throw_away_schema();
        let field = schema.get_field("num_likes").unwrap();
        let mut index = Index::create_from_tempdir(schema)?;
-        let mut writer = index.writer_for_tests()?;
+        let mut writer: IndexWriter = index.writer_for_tests()?;
        writer.commit()?;
        let reader = index
            .reader_builder()
@@ -189,7 +190,7 @@ mod mmap_specific {
        let read_index = Index::open_in_dir(&tempdir_path).unwrap();
        let reader = read_index
            .reader_builder()
-            .reload_policy(ReloadPolicy::OnCommit)
+            .reload_policy(ReloadPolicy::OnCommitWithDelay)
            .try_into()
            .unwrap();
        assert_eq!(reader.searcher().num_docs(), 0);
@@ -208,7 +209,7 @@ fn test_index_on_commit_reload_policy_aux(
        .watch(WatchCallback::new(move || {
            let _ = sender.send(());
        }));
-    let mut writer = index.writer_for_tests()?;
+    let mut writer: IndexWriter = index.writer_for_tests()?;
    assert_eq!(reader.searcher().num_docs(), 0);
    writer.add_document(doc!(field=>1u64))?;
    writer.commit().unwrap();
@@ -242,7 +243,7 @@ fn garbage_collect_works_as_intended() -> crate::Result<()> {
    let field = schema.get_field("num_likes").unwrap();
    let index = Index::create(directory.clone(), schema, IndexSettings::default())?;

-    let mut writer = index.writer_with_num_threads(1, 32_000_000).unwrap();
+    let mut writer: IndexWriter = index.writer_with_num_threads(1, 32_000_000).unwrap();
    for _seg in 0..8 {
        for i in 0u64..1_000u64 {
            writer.add_document(doc!(field => i))?;
@@ -306,7 +307,7 @@ fn test_merging_segment_update_docfreq() {
    let id_field = schema_builder.add_text_field("id", STRING);
    let schema = schema_builder.build();
    let index = Index::create_in_ram(schema);
-    let mut writer = index.writer_for_tests().unwrap();
+    let mut writer: IndexWriter = index.writer_for_tests().unwrap();
    writer.set_merge_policy(Box::new(NoMergePolicy));
    for _ in 0..5 {
        writer.add_document(doc!(text_field=>"hello")).unwrap();
@@ -317,13 +318,13 @@ fn test_merging_segment_update_docfreq() {
    writer
        .add_document(doc!(text_field=>"hello", id_field=>"TO_BE_DELETED"))
        .unwrap();
-    writer.add_document(Document::default()).unwrap();
+    writer.add_document(TantivyDocument::default()).unwrap();
    writer.commit().unwrap();
    for _ in 0..7 {
        writer.add_document(doc!(text_field=>"hello")).unwrap();
    }
-    writer.add_document(Document::default()).unwrap();
-    writer.add_document(Document::default()).unwrap();
+    writer.add_document(TantivyDocument::default()).unwrap();
+    writer.add_document(TantivyDocument::default()).unwrap();
    writer.delete_term(Term::from_field_text(id_field, "TO_BE_DELETED"));
    writer.commit().unwrap();

@@ -344,3 +345,132 @@ fn test_merging_segment_update_docfreq() {
    let term_info = inv_index.get_term_info(&term).unwrap().unwrap();
    assert_eq!(term_info.doc_freq, 12);
 }
+
+// motivated by https://github.com/quickwit-oss/quickwit/issues/4130
+#[test]
+fn test_positions_merge_bug_non_text_json_vint() {
+    let mut schema_builder = Schema::builder();
+    let field = schema_builder.add_json_field("dynamic", TEXT);
+    let schema = schema_builder.build();
+    let index = Index::create_in_ram(schema.clone());
+    let mut writer: IndexWriter = index.writer_for_tests().unwrap();
+    let mut merge_policy = LogMergePolicy::default();
+    merge_policy.set_min_num_segments(2);
+    writer.set_merge_policy(Box::new(merge_policy));
+    // Here a string would work.
+    let doc_json = r#"{"tenant_id":75}"#;
+    let vals = serde_json::from_str(doc_json).unwrap();
+    let mut doc = TantivyDocument::default();
+    doc.add_object(field, vals);
+    writer.add_document(doc.clone()).unwrap();
+    writer.commit().unwrap();
+    writer.add_document(doc.clone()).unwrap();
+    writer.commit().unwrap();
+    writer.wait_merging_threads().unwrap();
+    let reader = index.reader().unwrap();
+    assert_eq!(reader.searcher().segment_readers().len(), 1);
+}
+
+// Same as above but with bitpacked blocks
+#[test]
+fn test_positions_merge_bug_non_text_json_bitpacked_block() {
+    let mut schema_builder = Schema::builder();
+    let field = schema_builder.add_json_field("dynamic", TEXT);
+    let schema = schema_builder.build();
+    let index = Index::create_in_ram(schema.clone());
+    let mut writer: IndexWriter = index.writer_for_tests().unwrap();
+    let mut merge_policy = LogMergePolicy::default();
+    merge_policy.set_min_num_segments(2);
+    writer.set_merge_policy(Box::new(merge_policy));
+    // Here a string would work.
+    let doc_json = r#"{"tenant_id":75}"#;
+    let vals = serde_json::from_str(doc_json).unwrap();
+    let mut doc = TantivyDocument::default();
+    doc.add_object(field, vals);
+    for _ in 0..128 {
+        writer.add_document(doc.clone()).unwrap();
+    }
+    writer.commit().unwrap();
+    writer.add_document(doc.clone()).unwrap();
+    writer.commit().unwrap();
+    writer.wait_merging_threads().unwrap();
+    let reader = index.reader().unwrap();
+    assert_eq!(reader.searcher().segment_readers().len(), 1);
+}
+
+#[test]
+fn test_non_text_json_term_freq() {
+    let mut schema_builder = Schema::builder();
+    let field = schema_builder.add_json_field("dynamic", TEXT);
+    let schema = schema_builder.build();
+    let index = Index::create_in_ram(schema.clone());
+    let mut writer: IndexWriter = index.writer_for_tests().unwrap();
+    // Here a string would work.
+    let doc_json = r#"{"tenant_id":75}"#;
+    let vals = serde_json::from_str(doc_json).unwrap();
+    let mut doc = TantivyDocument::default();
+    doc.add_object(field, vals);
+    writer.add_document(doc.clone()).unwrap();
+    writer.commit().unwrap();
+    let reader = index.reader().unwrap();
+    assert_eq!(reader.searcher().segment_readers().len(), 1);
+    let searcher = reader.searcher();
+    let segment_reader = searcher.segment_reader(0u32);
+    let inv_idx = segment_reader.inverted_index(field).unwrap();
+    let mut term = Term::with_type_and_field(Type::Json, field);
+    let mut json_term_writer = JsonTermWriter::wrap(&mut term, false);
+    json_term_writer.push_path_segment("tenant_id");
+    json_term_writer.close_path_and_set_type(Type::U64);
+    json_term_writer.set_fast_value(75u64);
+    let postings = inv_idx
+        .read_postings(
+            &json_term_writer.term(),
+            IndexRecordOption::WithFreqsAndPositions,
+        )
+        .unwrap()
+        .unwrap();
+    assert_eq!(postings.doc(), 0);
+    assert_eq!(postings.term_freq(), 1u32);
+}
+
+#[test]
+fn test_non_text_json_term_freq_bitpacked() {
+    let mut schema_builder = Schema::builder();
+    let field = schema_builder.add_json_field("dynamic", TEXT);
+    let schema = schema_builder.build();
+    let index = Index::create_in_ram(schema.clone());
+    let mut writer: IndexWriter = index.writer_for_tests().unwrap();
+    // Here a string would work.
+    let doc_json = r#"{"tenant_id":75}"#;
+    let vals = serde_json::from_str(doc_json).unwrap();
+    let mut doc = TantivyDocument::default();
+    doc.add_object(field, vals);
+    let num_docs = 132;
+    for _ in 0..num_docs {
+        writer.add_document(doc.clone()).unwrap();
+    }
+    writer.commit().unwrap();
+    let reader = index.reader().unwrap();
+    assert_eq!(reader.searcher().segment_readers().len(), 1);
+    let searcher = reader.searcher();
+    let segment_reader = searcher.segment_reader(0u32);
+    let inv_idx = segment_reader.inverted_index(field).unwrap();
+    let mut term = Term::with_type_and_field(Type::Json, field);
+    let mut json_term_writer = JsonTermWriter::wrap(&mut term, false);
+    json_term_writer.push_path_segment("tenant_id");
+    json_term_writer.close_path_and_set_type(Type::U64);
+    json_term_writer.set_fast_value(75u64);
+    let mut postings = inv_idx
+        .read_postings(
+            &json_term_writer.term(),
+            IndexRecordOption::WithFreqsAndPositions,
+        )
+        .unwrap()
+        .unwrap();
+    assert_eq!(postings.doc(), 0);
+    assert_eq!(postings.term_freq(), 1u32);
+    for i in 1..num_docs {
+        assert_eq!(postings.advance(), i);
+        assert_eq!(postings.term_freq(), 1u32);
+    }
+}
--- a/src/directory/directory.rs
+++ b/src/directory/directory.rs
@@ -222,8 +222,8 @@ pub trait Directory: DirectoryClone + fmt::Debug + Send + Sync + 'static {
    /// registered (and whose [`WatchHandle`] is still alive) are triggered.
    ///
    /// Internally, tantivy only uses this API to detect new commits to implement the
-    /// `OnCommit` `ReloadPolicy`. Not implementing watch in a `Directory` only prevents the
-    /// `OnCommit` `ReloadPolicy` to work properly.
+    /// `OnCommitWithDelay` `ReloadPolicy`. Not implementing watch in a `Directory` only prevents
+    /// the `OnCommitWithDelay` `ReloadPolicy` to work properly.
    fn watch(&self, watch_callback: WatchCallback) -> crate::Result<WatchHandle>;
 }

--- a/src/directory/footer.rs
+++ b/src/directory/footer.rs
@@ -7,7 +7,7 @@ use serde::{Deserialize, Serialize};

 use crate::directory::error::Incompatibility;
 use crate::directory::{AntiCallToken, FileSlice, TerminatingWrite};
-use crate::{Version, INDEX_FORMAT_VERSION};
+use crate::{Version, INDEX_FORMAT_OLDEST_SUPPORTED_VERSION, INDEX_FORMAT_VERSION};

 const FOOTER_MAX_LEN: u32 = 50_000;

@@ -102,10 +102,11 @@ impl Footer {
    /// Confirms that the index will be read correctly by this version of tantivy
    /// Has to be called after `extract_footer` to make sure it's not accessing uninitialised memory
    pub fn is_compatible(&self) -> Result<(), Incompatibility> {
+        const SUPPORTED_INDEX_FORMAT_VERSION_RANGE: std::ops::RangeInclusive<u32> =
+            INDEX_FORMAT_OLDEST_SUPPORTED_VERSION..=INDEX_FORMAT_VERSION;
+
        let library_version = crate::version();
-        if self.version.index_format_version < 4
-            || self.version.index_format_version > INDEX_FORMAT_VERSION
-        {
+        if !SUPPORTED_INDEX_FORMAT_VERSION_RANGE.contains(&self.version.index_format_version) {
            return Err(Incompatibility::IndexMismatch {
                library_version: library_version.clone(),
                index_version: self.version.clone(),
--- a/src/directory/mmap_directory.rs
+++ b/src/directory/mmap_directory.rs
@@ -1,13 +1,15 @@
 use std::collections::HashMap;
 use std::fmt;
 use std::fs::{self, File, OpenOptions};
-use std::io::{self, BufWriter, Read, Seek, Write};
+use std::io::{self, BufWriter, Read, Write};
 use std::ops::Deref;
 use std::path::{Path, PathBuf};
 use std::sync::{Arc, RwLock, Weak};

 use common::StableDeref;
 use fs4::FileExt;
+#[cfg(all(feature = "mmap", unix))]
+pub use memmap2::Advice;
 use memmap2::Mmap;
 use serde::{Deserialize, Serialize};
 use tempfile::TempDir;
@@ -21,8 +23,6 @@ use crate::directory::{
    AntiCallToken, Directory, DirectoryLock, FileHandle, Lock, OwnedBytes, TerminatingWrite,
    WatchCallback, WatchHandle, WritePtr,
 };
-#[cfg(unix)]
-use crate::Advice;

 pub type ArcBytes = Arc<dyn Deref<Target = [u8]> + Send + Sync + 'static>;
 pub type WeakArcBytes = Weak<dyn Deref<Target = [u8]> + Send + Sync + 'static>;
@@ -328,12 +328,6 @@ impl Write for SafeFileWriter {
    }
 }

-impl Seek for SafeFileWriter {
-    fn seek(&mut self, pos: io::SeekFrom) -> io::Result<u64> {
-        self.0.seek(pos)
-    }
-}
-
 impl TerminatingWrite for SafeFileWriter {
    fn terminate_ref(&mut self, _: AntiCallToken) -> io::Result<()> {
        self.0.flush()?;
@@ -539,7 +533,7 @@ mod tests {
    use super::*;
    use crate::indexer::LogMergePolicy;
    use crate::schema::{Schema, SchemaBuilder, TEXT};
-    use crate::{Index, IndexSettings, ReloadPolicy};
+    use crate::{Index, IndexSettings, IndexWriter, ReloadPolicy};

    #[test]
    fn test_open_non_existent_path() {
@@ -651,7 +645,7 @@ mod tests {
            let index =
                Index::create(mmap_directory.clone(), schema, IndexSettings::default()).unwrap();

-            let mut index_writer = index.writer_for_tests().unwrap();
+            let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
            let mut log_merge_policy = LogMergePolicy::default();
            log_merge_policy.set_min_num_segments(3);
            index_writer.set_merge_policy(Box::new(log_merge_policy));
--- a/src/directory/mod.rs
+++ b/src/directory/mod.rs
@@ -42,6 +42,9 @@ pub struct GarbageCollectionResult {
    pub failed_to_delete_files: Vec<PathBuf>,
 }

+#[cfg(all(feature = "mmap", unix))]
+pub use memmap2::Advice;
+
 pub use self::managed_directory::ManagedDirectory;
 #[cfg(feature = "mmap")]
 pub use self::mmap_directory::MmapDirectory;
--- a/src/directory/ram_directory.rs
+++ b/src/directory/ram_directory.rs
@@ -1,5 +1,5 @@
 use std::collections::HashMap;
-use std::io::{self, BufWriter, Cursor, Seek, SeekFrom, Write};
+use std::io::{self, BufWriter, Cursor, Write};
 use std::path::{Path, PathBuf};
 use std::sync::{Arc, RwLock};
 use std::{fmt, result};
@@ -48,12 +48,6 @@ impl Drop for VecWriter {
    }
 }

-impl Seek for VecWriter {
-    fn seek(&mut self, pos: SeekFrom) -> io::Result<u64> {
-        self.data.seek(pos)
-    }
-}
-
 impl Write for VecWriter {
    fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
        self.is_flushed = false;
--- a/src/docset.rs
+++ b/src/docset.rs
@@ -17,7 +17,7 @@ pub trait DocSet: Send {
    ///
    /// The DocId of the next element is returned.
    /// In other words we should always have :
-    /// ```ignore
+    /// ```compile_fail
    /// let doc = docset.advance();
    /// assert_eq!(doc, docset.doc());
    /// ```
--- a/src/error.rs
+++ b/src/error.rs
@@ -11,6 +11,7 @@ use crate::directory::error::{
    Incompatibility, LockError, OpenDirectoryError, OpenReadError, OpenWriteError,
 };
 use crate::fastfield::FastFieldNotAvailableError;
+use crate::schema::document::DeserializeError;
 use crate::{query, schema};

 /// Represents a `DataCorruption` error.
@@ -106,6 +107,9 @@ pub enum TantivyError {
    /// e.g. a datastructure is incorrectly inititalized.
    #[error("Internal error: '{0}'")]
    InternalError(String),
+    #[error("Deserialize error: {0}")]
+    /// An error occurred while attempting to deserialize a document.
+    DeserializeError(DeserializeError),
 }

 impl From<io::Error> for TantivyError {
@@ -176,3 +180,9 @@ impl From<rayon::ThreadPoolBuildError> for TantivyError {
        TantivyError::SystemError(error.to_string())
    }
 }
+
+impl From<DeserializeError> for TantivyError {
+    fn from(error: DeserializeError) -> TantivyError {
+        TantivyError::DeserializeError(error)
+    }
+}
--- a/src/fastfield/facet_reader.rs
+++ b/src/fastfield/facet_reader.rs
@@ -62,8 +62,9 @@ impl FacetReader {

 #[cfg(test)]
 mod tests {
-    use crate::schema::{Facet, FacetOptions, SchemaBuilder, Value, STORED};
-    use crate::{DocAddress, Document, Index};
+    use crate::schema::document::Value;
+    use crate::schema::{Facet, FacetOptions, SchemaBuilder, STORED};
+    use crate::{DocAddress, Index, IndexWriter, TantivyDocument};

    #[test]
    fn test_facet_only_indexed() {
@@ -71,7 +72,7 @@ mod tests {
        let facet_field = schema_builder.add_facet_field("facet", FacetOptions::default());
        let schema = schema_builder.build();
        let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_for_tests().unwrap();
+        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
        index_writer
            .add_document(doc!(facet_field=>Facet::from_text("/a/b").unwrap()))
            .unwrap();
@@ -85,8 +86,10 @@ mod tests {
        let mut facet = Facet::default();
        facet_reader.facet_from_ord(0, &mut facet).unwrap();
        assert_eq!(facet.to_path_string(), "/a/b");
-        let doc = searcher.doc(DocAddress::new(0u32, 0u32)).unwrap();
-        let value = doc.get_first(facet_field).and_then(Value::as_facet);
+        let doc = searcher
+            .doc::<TantivyDocument>(DocAddress::new(0u32, 0u32))
+            .unwrap();
+        let value = doc.get_first(facet_field).and_then(|v| v.as_facet());
        assert_eq!(value, None);
    }

@@ -96,7 +99,7 @@ mod tests {
        let facet_field = schema_builder.add_facet_field("facet", FacetOptions::default());
        let schema = schema_builder.build();
        let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_for_tests().unwrap();
+        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
        index_writer
            .add_document(doc!(facet_field=>Facet::from_text("/parent/child1").unwrap()))
            .unwrap();
@@ -142,8 +145,8 @@ mod tests {
        let mut facet_ords = Vec::new();
        facet_ords.extend(facet_reader.facet_ords(0u32));
        assert_eq!(&facet_ords, &[0u64]);
-        let doc = searcher.doc(DocAddress::new(0u32, 0u32))?;
-        let value: Option<&Facet> = doc.get_first(facet_field).and_then(Value::as_facet);
+        let doc = searcher.doc::<TantivyDocument>(DocAddress::new(0u32, 0u32))?;
+        let value: Option<&Facet> = doc.get_first(facet_field).and_then(|v| v.as_facet());
        assert_eq!(value, Facet::from_text("/a/b").ok().as_ref());
        Ok(())
    }
@@ -156,7 +159,7 @@ mod tests {
        let index = Index::create_in_ram(schema);
        let mut index_writer = index.writer_for_tests()?;
        index_writer.add_document(doc!(facet_field=>Facet::from_text("/a/b").unwrap()))?;
-        index_writer.add_document(Document::default())?;
+        index_writer.add_document(TantivyDocument::default())?;
        index_writer.commit()?;
        let searcher = index.reader()?.searcher();
        let facet_reader = searcher.segment_reader(0u32).facet_reader("facet").unwrap();
@@ -176,8 +179,8 @@ mod tests {
        let schema = schema_builder.build();
        let index = Index::create_in_ram(schema);
        let mut index_writer = index.writer_for_tests()?;
-        index_writer.add_document(Document::default())?;
-        index_writer.add_document(Document::default())?;
+        index_writer.add_document(TantivyDocument::default())?;
+        index_writer.add_document(TantivyDocument::default())?;
        index_writer.commit()?;
        let searcher = index.reader()?.searcher();
        let facet_reader = searcher.segment_reader(0u32).facet_reader("facet").unwrap();
--- a/src/fastfield/mod.rs
+++ b/src/fastfield/mod.rs
@@ -90,12 +90,12 @@ mod tests {
    use crate::directory::{Directory, RamDirectory, WritePtr};
    use crate::merge_policy::NoMergePolicy;
    use crate::schema::{
-        Document, Facet, FacetOptions, Field, JsonObjectOptions, Schema, SchemaBuilder,
+        Facet, FacetOptions, Field, JsonObjectOptions, Schema, SchemaBuilder, TantivyDocument,
        TextOptions, FAST, INDEXED, STORED, STRING, TEXT,
    };
    use crate::time::OffsetDateTime;
    use crate::tokenizer::{LowerCaser, RawTokenizer, TextAnalyzer, TokenizerManager};
-    use crate::{DateOptions, DateTimePrecision, Index, SegmentId, SegmentReader};
+    use crate::{DateOptions, DateTimePrecision, Index, IndexWriter, SegmentId, SegmentReader};

    pub static SCHEMA: Lazy<Schema> = Lazy::new(|| {
        let mut schema_builder = Schema::builder();
@@ -131,7 +131,7 @@ mod tests {
        }
        let file = directory.open_read(path).unwrap();

-        assert_eq!(file.len(), 93);
+        assert_eq!(file.len(), 80);
        let fast_field_readers = FastFieldReaders::open(file, SCHEMA.clone()).unwrap();
        let column = fast_field_readers
            .u64("field")
@@ -181,7 +181,7 @@ mod tests {
            write.terminate().unwrap();
        }
        let file = directory.open_read(path).unwrap();
-        assert_eq!(file.len(), 121);
+        assert_eq!(file.len(), 108);
        let fast_field_readers = FastFieldReaders::open(file, SCHEMA.clone()).unwrap();
        let col = fast_field_readers
            .u64("field")
@@ -214,7 +214,7 @@ mod tests {
            write.terminate().unwrap();
        }
        let file = directory.open_read(path).unwrap();
-        assert_eq!(file.len(), 94);
+        assert_eq!(file.len(), 81);
        let fast_field_readers = FastFieldReaders::open(file, SCHEMA.clone()).unwrap();
        let fast_field_reader = fast_field_readers
            .u64("field")
@@ -246,7 +246,7 @@ mod tests {
            write.terminate().unwrap();
        }
        let file = directory.open_read(path).unwrap();
-        assert_eq!(file.len(), 4489);
+        assert_eq!(file.len(), 4476);
        {
            let fast_field_readers = FastFieldReaders::open(file, SCHEMA.clone()).unwrap();
            let col = fast_field_readers
@@ -271,7 +271,7 @@ mod tests {
            let mut write: WritePtr = directory.open_write(Path::new("test")).unwrap();
            let mut fast_field_writers = FastFieldsWriter::from_schema(&schema).unwrap();
            for i in -100i64..10_000i64 {
-                let mut doc = Document::default();
+                let mut doc = TantivyDocument::default();
                doc.add_i64(i64_field, i);
                fast_field_writers.add_document(&doc).unwrap();
            }
@@ -279,7 +279,7 @@ mod tests {
            write.terminate().unwrap();
        }
        let file = directory.open_read(path).unwrap();
-        assert_eq!(file.len(), 265);
+        assert_eq!(file.len(), 252);

        {
            let fast_field_readers = FastFieldReaders::open(file, schema).unwrap();
@@ -312,7 +312,7 @@ mod tests {
        {
            let mut write: WritePtr = directory.open_write(Path::new("test")).unwrap();
            let mut fast_field_writers = FastFieldsWriter::from_schema(&schema).unwrap();
-            let doc = Document::default();
+            let doc = TantivyDocument::default();
            fast_field_writers.add_document(&doc).unwrap();
            fast_field_writers.serialize(&mut write, None).unwrap();
            write.terminate().unwrap();
@@ -345,7 +345,7 @@ mod tests {
        {
            let mut write: WritePtr = directory.open_write(Path::new("test")).unwrap();
            let mut fast_field_writers = FastFieldsWriter::from_schema(&schema).unwrap();
-            let doc = Document::default();
+            let doc = TantivyDocument::default();
            fast_field_writers.add_document(&doc).unwrap();
            fast_field_writers.serialize(&mut write, None).unwrap();
            write.terminate().unwrap();
@@ -416,7 +416,7 @@ mod tests {
        let date_field = schema_builder.add_date_field("date", FAST);
        let schema = schema_builder.build();
        let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_for_tests().unwrap();
+        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
        index_writer.set_merge_policy(Box::new(NoMergePolicy));
        index_writer
            .add_document(doc!(date_field => DateTime::from_utc(OffsetDateTime::now_utc())))
@@ -452,7 +452,7 @@ mod tests {

        {
            // first segment
-            let mut index_writer = index.writer_for_tests().unwrap();
+            let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
            index_writer.set_merge_policy(Box::new(NoMergePolicy));
            index_writer
                .add_document(doc!(
@@ -506,7 +506,7 @@ mod tests {

        {
            // second segment
-            let mut index_writer = index.writer_for_tests().unwrap();
+            let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();

            index_writer
                .add_document(doc!(
@@ -537,7 +537,7 @@ mod tests {
        // Merging the segments
        {
            let segment_ids = index.searchable_segment_ids().unwrap();
-            let mut index_writer = index.writer_for_tests().unwrap();
+            let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
            index_writer.merge(&segment_ids).wait().unwrap();
            index_writer.wait_merging_threads().unwrap();
        }
@@ -662,7 +662,7 @@ mod tests {
        // Merging the segments
        {
            let segment_ids = index.searchable_segment_ids()?;
-            let mut index_writer = index.writer_for_tests()?;
+            let mut index_writer: IndexWriter = index.writer_for_tests()?;
            index_writer.merge(&segment_ids).wait()?;
            index_writer.wait_merging_threads()?;
        }
@@ -773,7 +773,7 @@ mod tests {
            write.terminate().unwrap();
        }
        let file = directory.open_read(path).unwrap();
-        assert_eq!(file.len(), 102);
+        assert_eq!(file.len(), 84);
        let fast_field_readers = FastFieldReaders::open(file, schema).unwrap();
        let bool_col = fast_field_readers.bool("field_bool").unwrap();
        assert_eq!(bool_col.first(0), Some(true));
@@ -805,7 +805,7 @@ mod tests {
            write.terminate().unwrap();
        }
        let file = directory.open_read(path).unwrap();
-        assert_eq!(file.len(), 114);
+        assert_eq!(file.len(), 96);
        let readers = FastFieldReaders::open(file, schema).unwrap();
        let bool_col = readers.bool("field_bool").unwrap();
        for i in 0..25 {
@@ -824,13 +824,13 @@ mod tests {
        {
            let mut write: WritePtr = directory.open_write(path).unwrap();
            let mut fast_field_writers = FastFieldsWriter::from_schema(&schema).unwrap();
-            let doc = Document::default();
+            let doc = TantivyDocument::default();
            fast_field_writers.add_document(&doc).unwrap();
            fast_field_writers.serialize(&mut write, None).unwrap();
            write.terminate().unwrap();
        }
        let file = directory.open_read(path).unwrap();
-        assert_eq!(file.len(), 104);
+        assert_eq!(file.len(), 86);
        let fastfield_readers = FastFieldReaders::open(file, schema).unwrap();
        let col = fastfield_readers.bool("field_bool").unwrap();
        assert_eq!(col.first(0), None);
@@ -846,7 +846,7 @@ mod tests {
        assert_eq!(col.get_val(0), true);
    }

-    fn get_index(docs: &[crate::Document], schema: &Schema) -> crate::Result<RamDirectory> {
+    fn get_index(docs: &[crate::TantivyDocument], schema: &Schema) -> crate::Result<RamDirectory> {
        let directory: RamDirectory = RamDirectory::create();
        {
            let mut write: WritePtr = directory.open_write(Path::new("test")).unwrap();
@@ -888,7 +888,7 @@ mod tests {
        let field = schema_builder.add_date_field("field", date_options);
        let schema = schema_builder.build();

-        let docs: Vec<Document> = times.iter().map(|time| doc!(field=>*time)).collect();
+        let docs: Vec<TantivyDocument> = times.iter().map(|time| doc!(field=>*time)).collect();

        let directory = get_index(&docs[..], &schema).unwrap();
        let path = Path::new("test");
@@ -962,11 +962,15 @@ mod tests {
        let ip_field = schema_builder.add_u64_field("ip", FAST);
        let schema = schema_builder.build();
        let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_for_tests().unwrap();
+        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
        let ip_addr = Ipv6Addr::new(1, 2, 3, 4, 5, 1, 2, 3);
-        index_writer.add_document(Document::default()).unwrap();
+        index_writer
+            .add_document(TantivyDocument::default())
+            .unwrap();
        index_writer.add_document(doc!(ip_field=>ip_addr)).unwrap();
-        index_writer.add_document(Document::default()).unwrap();
+        index_writer
+            .add_document(TantivyDocument::default())
+            .unwrap();
        index_writer.commit().unwrap();
        let searcher = index.reader().unwrap().searcher();
        let fastfields = searcher.segment_reader(0u32).fast_fields();
@@ -1086,7 +1090,7 @@ mod tests {
        let json = schema_builder.add_json_field("json", json_option);
        let schema = schema_builder.build();
        let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_for_tests().unwrap();
+        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
        index_writer
            .add_document(doc!(json => json!({"attr.age": 32})))
            .unwrap();
@@ -1112,7 +1116,7 @@ mod tests {
        let json = schema_builder.add_json_field("json", json_option);
        let schema = schema_builder.build();
        let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_for_tests().unwrap();
+        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
        index_writer
            .add_document(doc!(json => json!({"age": 32})))
            .unwrap();
@@ -1139,7 +1143,7 @@ mod tests {
        let json = schema_builder.add_json_field("json", json_option);
        let schema = schema_builder.build();
        let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_for_tests().unwrap();
+        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
        index_writer
            .add_document(doc!(json => json!({"attr.age": 32})))
            .unwrap();
@@ -1162,7 +1166,7 @@ mod tests {
        let field_with_dot = schema_builder.add_i64_field("field.with.dot", FAST);
        let schema = schema_builder.build();
        let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_for_tests().unwrap();
+        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
        index_writer
            .add_document(doc!(field_with_dot => 32i64))
            .unwrap();
@@ -1184,7 +1188,7 @@ mod tests {
        let shadowing_json_field = schema_builder.add_json_field("jsonfield.attr", FAST);
        let schema = schema_builder.build();
        let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_for_tests().unwrap();
+        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
        index_writer
            .add_document(doc!(json_field=> json!({"attr": {"age": 32}}), shadowing_json_field=>json!({"age": 33})))
            .unwrap();
@@ -1215,7 +1219,7 @@ mod tests {

        let mut index = Index::create_in_ram(schema);
        index.set_fast_field_tokenizers(ff_tokenizer_manager);
-        let mut index_writer = index.writer_for_tests().unwrap();
+        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
        index_writer
            .add_document(doc!(text_field => "Test1 test2"))
            .unwrap();
@@ -1244,7 +1248,7 @@ mod tests {
        let log_field = schema_builder.add_text_field("log_level", text_fieldtype);
        let schema = schema_builder.build();
        let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_for_tests().unwrap();
+        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
        index_writer
            .add_document(doc!(log_field => "info"))
            .unwrap();
@@ -1277,18 +1281,25 @@ mod tests {
        let shadowing_json_field = schema_builder.add_json_field("jsonfield.attr", json_option);
        let schema = schema_builder.build();
        let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_for_tests().unwrap();
+        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
        index_writer
            .add_document(doc!(json_field=> json!({"attr.age": 32}), shadowing_json_field=>json!({"age": 33})))
            .unwrap();
        index_writer.commit().unwrap();
        let searcher = index.reader().unwrap().searcher();
        let fast_field_reader = searcher.segment_reader(0u32).fast_fields();
+        // Supported for now, maybe dropped in the future.
        let column = fast_field_reader
            .column_opt::<i64>("jsonfield.attr.age")
            .unwrap()
            .unwrap();
        let vals: Vec<i64> = column.values_for_doc(0u32).collect();
        assert_eq!(&vals, &[33]);
+        let column = fast_field_reader
+            .column_opt::<i64>("jsonfield\\.attr.age")
+            .unwrap()
+            .unwrap();
+        let vals: Vec<i64> = column.values_for_doc(0u32).collect();
+        assert_eq!(&vals, &[33]);
    }
 }
--- a/src/fastfield/readers.rs
+++ b/src/fastfield/readers.rs
@@ -357,7 +357,7 @@ mod tests {
    use columnar::ColumnType;

    use crate::schema::{JsonObjectOptions, Schema, FAST};
-    use crate::{Document, Index};
+    use crate::{Index, IndexWriter, TantivyDocument};

    #[test]
    fn test_fast_field_reader_resolve_with_dynamic_internal() {
@@ -373,8 +373,10 @@ mod tests {
        let dynamic_field = schema_builder.add_json_field("_dyna", FAST);
        let schema = schema_builder.build();
        let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_for_tests().unwrap();
-        index_writer.add_document(Document::default()).unwrap();
+        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
+        index_writer
+            .add_document(TantivyDocument::default())
+            .unwrap();
        index_writer.commit().unwrap();
        let reader = index.reader().unwrap();
        let searcher = reader.searcher();
@@ -443,7 +445,7 @@ mod tests {
        let json = schema_builder.add_json_field("json", FAST);
        let schema = schema_builder.build();
        let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_for_tests().unwrap();
+        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
        index_writer
            .add_document(doc!(id=> 1u64, json => json!({"foo": 42})))
            .unwrap();
--- a/src/fastfield/writer.rs
+++ b/src/fastfield/writer.rs
@@ -1,12 +1,12 @@
 use std::io;

 use columnar::{ColumnarWriter, NumericalValue};
-use common::replace_in_place;
+use common::JsonPathWriter;
 use tokenizer_api::Token;

 use crate::indexer::doc_id_mapping::DocIdMapping;
-use crate::schema::term::{JSON_PATH_SEGMENT_SEP, JSON_PATH_SEGMENT_SEP_STR};
-use crate::schema::{value_type_to_column_type, Document, FieldType, Schema, Type, Value};
+use crate::schema::document::{Document, ReferenceValue, ReferenceValueLeaf, Value};
+use crate::schema::{value_type_to_column_type, Field, FieldType, Schema, Type};
 use crate::tokenizer::{TextAnalyzer, TokenizerManager};
 use crate::{DateTimePrecision, DocId, TantivyError};

@@ -23,7 +23,7 @@ pub struct FastFieldsWriter {
    expand_dots: Vec<bool>,
    num_docs: DocId,
    // Buffer that we recycle to avoid allocation.
-    json_path_buffer: String,
+    json_path_buffer: JsonPathWriter,
 }

 impl FastFieldsWriter {
@@ -97,7 +97,7 @@ impl FastFieldsWriter {
            num_docs: 0u32,
            date_precisions,
            expand_dots,
-            json_path_buffer: String::new(),
+            json_path_buffer: JsonPathWriter::default(),
        })
    }

@@ -117,190 +117,168 @@ impl FastFieldsWriter {
    }

    /// Indexes all of the fastfields of a new document.
-    pub fn add_document(&mut self, doc: &Document) -> crate::Result<()> {
+    pub fn add_document<D: Document>(&mut self, doc: &D) -> crate::Result<()> {
        let doc_id = self.num_docs;
-        for field_value in doc.field_values() {
-            if let Some(field_name) =
-                &self.fast_field_names[field_value.field().field_id() as usize]
-            {
-                match &field_value.value {
-                    Value::U64(u64_val) => {
-                        self.columnar_writer.record_numerical(
-                            doc_id,
-                            field_name.as_str(),
-                            NumericalValue::from(*u64_val),
-                        );
-                    }
-                    Value::I64(i64_val) => {
-                        self.columnar_writer.record_numerical(
-                            doc_id,
-                            field_name.as_str(),
-                            NumericalValue::from(*i64_val),
-                        );
-                    }
-                    Value::F64(f64_val) => {
-                        self.columnar_writer.record_numerical(
-                            doc_id,
-                            field_name.as_str(),
-                            NumericalValue::from(*f64_val),
-                        );
-                    }
-                    Value::Str(text_val) => {
-                        if let Some(tokenizer) =
-                            &mut self.per_field_tokenizer[field_value.field().field_id() as usize]
-                        {
-                            let mut token_stream = tokenizer.token_stream(text_val);
-                            token_stream.process(&mut |token: &Token| {
-                                self.columnar_writer.record_str(
-                                    doc_id,
-                                    field_name.as_str(),
-                                    &token.text,
-                                );
-                            })
-                        } else {
-                            self.columnar_writer
-                                .record_str(doc_id, field_name.as_str(), text_val);
-                        }
-                    }
-                    Value::Bytes(bytes_val) => {
-                        self.columnar_writer
-                            .record_bytes(doc_id, field_name.as_str(), bytes_val);
-                    }
-                    Value::PreTokStr(pre_tok) => {
-                        for token in &pre_tok.tokens {
-                            self.columnar_writer.record_str(
-                                doc_id,
-                                field_name.as_str(),
-                                &token.text,
-                            );
-                        }
-                    }
-                    Value::Bool(bool_val) => {
-                        self.columnar_writer
-                            .record_bool(doc_id, field_name.as_str(), *bool_val);
-                    }
-                    Value::Date(datetime) => {
-                        let date_precision =
-                            self.date_precisions[field_value.field().field_id() as usize];
-                        let truncated_datetime = datetime.truncate(date_precision);
-                        self.columnar_writer.record_datetime(
-                            doc_id,
-                            field_name.as_str(),
-                            truncated_datetime,
-                        );
-                    }
-                    Value::Facet(facet) => {
-                        self.columnar_writer.record_str(
-                            doc_id,
-                            field_name.as_str(),
-                            facet.encoded_str(),
-                        );
-                    }
-                    Value::JsonObject(json_obj) => {
-                        let expand_dots = self.expand_dots[field_value.field().field_id() as usize];
-                        self.json_path_buffer.clear();
-                        self.json_path_buffer.push_str(field_name);
+        for (field, value) in doc.iter_fields_and_values() {
+            let value_access = value as D::Value<'_>;

-                        let text_analyzer =
-                            &mut self.per_field_tokenizer[field_value.field().field_id() as usize];
-
-                        record_json_obj_to_columnar_writer(
-                            doc_id,
-                            json_obj,
-                            expand_dots,
-                            JSON_DEPTH_LIMIT,
-                            &mut self.json_path_buffer,
-                            &mut self.columnar_writer,
-                            text_analyzer,
-                        );
-                    }
-                    Value::IpAddr(ip_addr) => {
-                        self.columnar_writer
-                            .record_ip_addr(doc_id, field_name.as_str(), *ip_addr);
-                    }
-                }
-            }
+            self.add_doc_value(doc_id, field, value_access)?;
        }
        self.num_docs += 1;
        Ok(())
    }

+    fn add_doc_value<'a, V: Value<'a>>(
+        &mut self,
+        doc_id: DocId,
+        field: Field,
+        value: V,
+    ) -> crate::Result<()> {
+        let field_name = match &self.fast_field_names[field.field_id() as usize] {
+            None => return Ok(()),
+            Some(name) => name,
+        };
+
+        match value.as_value() {
+            ReferenceValue::Leaf(leaf) => match leaf {
+                ReferenceValueLeaf::Null => {}
+                ReferenceValueLeaf::Str(val) => {
+                    if let Some(tokenizer) =
+                        &mut self.per_field_tokenizer[field.field_id() as usize]
+                    {
+                        let mut token_stream = tokenizer.token_stream(val);
+                        token_stream.process(&mut |token: &Token| {
+                            self.columnar_writer
+                                .record_str(doc_id, field_name, &token.text);
+                        })
+                    } else {
+                        self.columnar_writer.record_str(doc_id, field_name, val);
+                    }
+                }
+                ReferenceValueLeaf::U64(val) => {
+                    self.columnar_writer.record_numerical(
+                        doc_id,
+                        field_name,
+                        NumericalValue::from(val),
+                    );
+                }
+                ReferenceValueLeaf::I64(val) => {
+                    self.columnar_writer.record_numerical(
+                        doc_id,
+                        field_name,
+                        NumericalValue::from(val),
+                    );
+                }
+                ReferenceValueLeaf::F64(val) => {
+                    self.columnar_writer.record_numerical(
+                        doc_id,
+                        field_name,
+                        NumericalValue::from(val),
+                    );
+                }
+                ReferenceValueLeaf::Date(val) => {
+                    let date_precision = self.date_precisions[field.field_id() as usize];
+                    let truncated_datetime = val.truncate(date_precision);
+                    self.columnar_writer
+                        .record_datetime(doc_id, field_name, truncated_datetime);
+                }
+                ReferenceValueLeaf::Facet(val) => {
+                    self.columnar_writer
+                        .record_str(doc_id, field_name, val.encoded_str());
+                }
+                ReferenceValueLeaf::Bytes(val) => {
+                    self.columnar_writer.record_bytes(doc_id, field_name, val);
+                }
+                ReferenceValueLeaf::IpAddr(val) => {
+                    self.columnar_writer.record_ip_addr(doc_id, field_name, val);
+                }
+                ReferenceValueLeaf::Bool(val) => {
+                    self.columnar_writer.record_bool(doc_id, field_name, val);
+                }
+                ReferenceValueLeaf::PreTokStr(val) => {
+                    for token in &val.tokens {
+                        self.columnar_writer
+                            .record_str(doc_id, field_name, &token.text);
+                    }
+                }
+            },
+            ReferenceValue::Array(val) => {
+                // TODO: Check this is the correct behaviour we want.
+                for value in val {
+                    self.add_doc_value(doc_id, field, value)?;
+                }
+            }
+            ReferenceValue::Object(val) => {
+                let expand_dots = self.expand_dots[field.field_id() as usize];
+                self.json_path_buffer.clear();
+                // First field should not be expanded.
+                self.json_path_buffer.set_expand_dots(false);
+                self.json_path_buffer.push(field_name);
+                self.json_path_buffer.set_expand_dots(expand_dots);
+
+                let text_analyzer = &mut self.per_field_tokenizer[field.field_id() as usize];
+
+                record_json_obj_to_columnar_writer::<V>(
+                    doc_id,
+                    val,
+                    JSON_DEPTH_LIMIT,
+                    &mut self.json_path_buffer,
+                    &mut self.columnar_writer,
+                    text_analyzer,
+                );
+            }
+        }
+
+        Ok(())
+    }
+
    /// Serializes all of the `FastFieldWriter`s by pushing them in
    /// order to the fast field serializer.
    pub fn serialize(
        mut self,
        wrt: &mut dyn io::Write,
        doc_id_map_opt: Option<&DocIdMapping>,
-    ) -> io::Result<()> {
+    ) -> io::Result<Vec<(String, Type)>> {
        let num_docs = self.num_docs;
        let old_to_new_row_ids =
            doc_id_map_opt.map(|doc_id_mapping| doc_id_mapping.old_to_new_ids());
-        self.columnar_writer
+        let columns = self
+            .columnar_writer
            .serialize(num_docs, old_to_new_row_ids, wrt)?;
-        Ok(())
+        Ok(columns
+            .into_iter()
+            .map(|(field_name, column)| (field_name.to_string(), column.into()))
+            .collect())
    }
 }

-#[inline]
-fn columnar_numerical_value(json_number: &serde_json::Number) -> Option<NumericalValue> {
-    if let Some(num_i64) = json_number.as_i64() {
-        return Some(num_i64.into());
-    }
-    if let Some(num_u64) = json_number.as_u64() {
-        return Some(num_u64.into());
-    }
-    if let Some(num_f64) = json_number.as_f64() {
-        return Some(num_f64.into());
-    }
-    // This can happen with arbitrary precision.... but we do not handle it.
-    None
-}
-
-fn record_json_obj_to_columnar_writer(
+fn record_json_obj_to_columnar_writer<'a, V: Value<'a>>(
    doc: DocId,
-    json_obj: &serde_json::Map<String, serde_json::Value>,
-    expand_dots: bool,
+    json_visitor: V::ObjectIter,
    remaining_depth_limit: usize,
-    json_path_buffer: &mut String,
+    json_path_buffer: &mut JsonPathWriter,
    columnar_writer: &mut columnar::ColumnarWriter,
    tokenizer: &mut Option<TextAnalyzer>,
 ) {
-    for (key, child) in json_obj {
-        let len_path = json_path_buffer.len();
-        if !json_path_buffer.is_empty() {
-            json_path_buffer.push_str(JSON_PATH_SEGMENT_SEP_STR);
-        }
-        json_path_buffer.push_str(key);
-        if expand_dots {
-            // This might include the separation byte, which is ok because it is not a dot.
-            let appended_segment = &mut json_path_buffer[len_path..];
-            // The unsafe below is safe as long as b'.' and JSON_PATH_SEGMENT_SEP are
-            // valid single byte ut8 strings.
-            // By utf-8 design, they cannot be part of another codepoint.
-            replace_in_place(b'.', JSON_PATH_SEGMENT_SEP, unsafe {
-                appended_segment.as_bytes_mut()
-            });
-        }
+    for (key, child) in json_visitor {
+        json_path_buffer.push(key);
        record_json_value_to_columnar_writer(
            doc,
            child,
-            expand_dots,
            remaining_depth_limit,
            json_path_buffer,
            columnar_writer,
            tokenizer,
        );
-        // popping our sub path.
-        json_path_buffer.truncate(len_path);
+        json_path_buffer.pop();
    }
 }

-fn record_json_value_to_columnar_writer(
+fn record_json_value_to_columnar_writer<'a, V: Value<'a>>(
    doc: DocId,
-    json_val: &serde_json::Value,
-    expand_dots: bool,
+    json_val: V,
    mut remaining_depth_limit: usize,
-    json_path_writer: &mut String,
+    json_path_writer: &mut JsonPathWriter,
    columnar_writer: &mut columnar::ColumnarWriter,
    tokenizer: &mut Option<TextAnalyzer>,
 ) {
@@ -308,34 +286,69 @@ fn record_json_value_to_columnar_writer(
        return;
    }
    remaining_depth_limit -= 1;
-    match json_val {
-        serde_json::Value::Null => {
-            // TODO handle null
-        }
-        serde_json::Value::Bool(bool_val) => {
-            columnar_writer.record_bool(doc, json_path_writer, *bool_val);
-        }
-        serde_json::Value::Number(json_number) => {
-            if let Some(numerical_value) = columnar_numerical_value(json_number) {
-                columnar_writer.record_numerical(doc, json_path_writer.as_str(), numerical_value);
+
+    match json_val.as_value() {
+        ReferenceValue::Leaf(leaf) => match leaf {
+            ReferenceValueLeaf::Null => {} // TODO: Handle null
+            ReferenceValueLeaf::Str(val) => {
+                if let Some(text_analyzer) = tokenizer.as_mut() {
+                    let mut token_stream = text_analyzer.token_stream(val);
+                    token_stream.process(&mut |token| {
+                        columnar_writer.record_str(doc, json_path_writer.as_str(), &token.text);
+                    })
+                } else {
+                    columnar_writer.record_str(doc, json_path_writer.as_str(), val);
+                }
            }
-        }
-        serde_json::Value::String(text) => {
-            if let Some(text_analyzer) = tokenizer.as_mut() {
-                let mut token_stream = text_analyzer.token_stream(text);
-                token_stream.process(&mut |token| {
-                    columnar_writer.record_str(doc, json_path_writer.as_str(), &token.text);
-                })
-            } else {
-                columnar_writer.record_str(doc, json_path_writer.as_str(), text);
+            ReferenceValueLeaf::U64(val) => {
+                columnar_writer.record_numerical(
+                    doc,
+                    json_path_writer.as_str(),
+                    NumericalValue::from(val),
+                );
            }
-        }
-        serde_json::Value::Array(arr) => {
-            for el in arr {
+            ReferenceValueLeaf::I64(val) => {
+                columnar_writer.record_numerical(
+                    doc,
+                    json_path_writer.as_str(),
+                    NumericalValue::from(val),
+                );
+            }
+            ReferenceValueLeaf::F64(val) => {
+                columnar_writer.record_numerical(
+                    doc,
+                    json_path_writer.as_str(),
+                    NumericalValue::from(val),
+                );
+            }
+            ReferenceValueLeaf::Bool(val) => {
+                columnar_writer.record_bool(doc, json_path_writer.as_str(), val);
+            }
+            ReferenceValueLeaf::Date(val) => {
+                columnar_writer.record_datetime(doc, json_path_writer.as_str(), val);
+            }
+            ReferenceValueLeaf::Facet(_) => {
+                unimplemented!("Facet support in dynamic fields is not yet implemented")
+            }
+            ReferenceValueLeaf::Bytes(_) => {
+                // TODO: This can be re added once it is added to the JSON Utils section as well.
+                // columnar_writer.record_bytes(doc, json_path_writer.as_str(), val);
+                unimplemented!("Bytes support in dynamic fields is not yet implemented")
+            }
+            ReferenceValueLeaf::IpAddr(_) => {
+                unimplemented!("IP address support in dynamic fields is not yet implemented")
+            }
+            ReferenceValueLeaf::PreTokStr(_) => {
+                unimplemented!(
+                    "Pre-tokenized string support in dynamic fields is not yet implemented"
+                )
+            }
+        },
+        ReferenceValue::Array(elements) => {
+            for el in elements {
                record_json_value_to_columnar_writer(
                    doc,
                    el,
-                    expand_dots,
                    remaining_depth_limit,
                    json_path_writer,
                    columnar_writer,
@@ -343,11 +356,10 @@ fn record_json_value_to_columnar_writer(
                );
            }
        }
-        serde_json::Value::Object(json_obj) => {
-            record_json_obj_to_columnar_writer(
+        ReferenceValue::Object(object) => {
+            record_json_obj_to_columnar_writer::<V>(
                doc,
-                json_obj,
-                expand_dots,
+                object,
                remaining_depth_limit,
                json_path_writer,
                columnar_writer,
@@ -360,6 +372,7 @@ fn record_json_value_to_columnar_writer(
 #[cfg(test)]
 mod tests {
    use columnar::{Column, ColumnarReader, ColumnarWriter, StrColumn};
+    use common::JsonPathWriter;

    use super::record_json_value_to_columnar_writer;
    use crate::fastfield::writer::JSON_DEPTH_LIMIT;
@@ -370,12 +383,12 @@ mod tests {
        expand_dots: bool,
    ) -> ColumnarReader {
        let mut columnar_writer = ColumnarWriter::default();
-        let mut json_path = String::new();
+        let mut json_path = JsonPathWriter::default();
+        json_path.set_expand_dots(expand_dots);
        for (doc, json_doc) in json_docs.iter().enumerate() {
            record_json_value_to_columnar_writer(
                doc as u32,
                json_doc,
-                expand_dots,
                JSON_DEPTH_LIMIT,
                &mut json_path,
                &mut columnar_writer,
--- a/src/field_list/mod.rs
+++ b/src/field_list/mod.rs
@@ -0,0 +1,369 @@
+//! The list of fields that are stored in a `tantivy` `Index`.
+
+use std::collections::HashSet;
+use std::io::{self, ErrorKind, Read};
+
+use columnar::ColumnType;
+use common::TinySet;
+use fnv::FnvHashMap;
+
+use crate::indexer::path_to_unordered_id::OrderedPathId;
+use crate::json_utils::json_path_sep_to_dot;
+use crate::postings::IndexingContext;
+use crate::schema::{Field, Schema, Type};
+use crate::{merge_field_meta_data, FieldMetadata, Term};
+
+#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash)]
+pub(crate) struct FieldConfig {
+    pub typ: Type,
+    pub indexed: bool,
+    pub stored: bool,
+    pub fast: bool,
+}
+
+impl FieldConfig {
+    fn serialize(&self) -> [u8; 2] {
+        let typ = self.typ.to_code();
+        let flags = (self.indexed as u8) << 2 | (self.stored as u8) << 1 | (self.fast as u8);
+        [typ, flags]
+    }
+    fn deserialize_from(data: [u8; 2]) -> io::Result<FieldConfig> {
+        let typ = Type::from_code(data[0]).ok_or_else(|| {
+            io::Error::new(
+                ErrorKind::InvalidData,
+                format!("could not deserialize type {}", data[0]),
+            )
+        })?;
+
+        let data = data[1];
+        let indexed = (data & 0b100) != 0;
+        let stored = (data & 0b010) != 0;
+        let fast = (data & 0b001) != 0;
+
+        Ok(FieldConfig {
+            typ,
+            indexed,
+            stored,
+            fast,
+        })
+    }
+}
+/// Serializes the split fields.
+pub(crate) fn serialize_segment_fields(
+    ctx: IndexingContext,
+    wrt: &mut dyn io::Write,
+    schema: &Schema,
+    unordered_id_to_ordered_id: &[(OrderedPathId, TinySet)],
+    mut columns: Vec<(String, Type)>,
+) -> crate::Result<()> {
+    let mut field_list_set: HashSet<(Field, OrderedPathId, TinySet)> = HashSet::default();
+    let mut encoded_fields = Vec::new();
+    let mut map_to_canonical = FnvHashMap::default();
+
+    // Replace unordered ids by ordered ids to be able to sort
+    let ordered_id_to_path = ctx.path_to_unordered_id.ordered_id_to_path();
+
+    for (key, _addr) in ctx.term_index.iter() {
+        let field = Term::wrap(key).field();
+        let field_entry = schema.get_field_entry(field);
+        if field_entry.field_type().value_type() == Type::Json {
+            let byte_range_unordered_id = 5..5 + 4;
+            let unordered_id =
+                u32::from_be_bytes(key[byte_range_unordered_id.clone()].try_into().unwrap());
+            let (path_id, typ_code_bitvec) = unordered_id_to_ordered_id[unordered_id as usize];
+            if !field_list_set.contains(&(field, path_id, typ_code_bitvec)) {
+                field_list_set.insert((field, path_id, typ_code_bitvec));
+                let mut build_path = |field_name: &str, mut json_path: String| {
+                    // In this case we need to map the potential fast field to the field name
+                    // accepted by the query parser.
+                    let create_canonical =
+                        !field_entry.is_expand_dots_enabled() && json_path.contains('.');
+                    if create_canonical {
+                        // Without expand dots enabled dots need to be escaped.
+                        let escaped_json_path = json_path.replace('.', "\\.");
+                        let full_path = format!("{}.{}", field_name, escaped_json_path);
+                        let full_path_unescaped = format!("{}.{}", field_name, &json_path);
+                        map_to_canonical.insert(full_path_unescaped, full_path.to_string());
+                        full_path
+                    } else {
+                        // With expand dots enabled, we can use '.' instead of '\u{1}'.
+                        json_path_sep_to_dot(&mut json_path);
+                        format!("{}.{}", field_name, json_path)
+                    }
+                };
+
+                let path = build_path(
+                    field_entry.name(),
+                    ordered_id_to_path[path_id.path_id() as usize].to_string(), /* String::from_utf8(key[5..].to_vec()).unwrap(), */
+                );
+                encoded_fields.push((path, typ_code_bitvec));
+            }
+        }
+    }
+
+    let mut indexed_fields: Vec<FieldMetadata> = Vec::new();
+    for (_field, field_entry) in schema.fields() {
+        let field_name = field_entry.name().to_string();
+        let is_indexed = field_entry.is_indexed();
+
+        let is_json = field_entry.field_type().value_type() == Type::Json;
+        if is_indexed && !is_json {
+            indexed_fields.push(FieldMetadata {
+                indexed: true,
+                stored: false,
+                field_name: field_name.to_string(),
+                fast: false,
+                typ: field_entry.field_type().value_type(),
+            });
+        }
+    }
+    for (field_name, field_type_set) in encoded_fields {
+        for field_type in field_type_set {
+            let column_type = ColumnType::try_from_code(field_type as u8).unwrap();
+            indexed_fields.push(FieldMetadata {
+                indexed: true,
+                stored: false,
+                field_name: field_name.to_string(),
+                fast: false,
+                typ: Type::from(column_type),
+            });
+        }
+    }
+    let mut fast_fields: Vec<FieldMetadata> = columns
+        .iter_mut()
+        .map(|(field_name, typ)| {
+            json_path_sep_to_dot(field_name);
+            // map to canonical path, to avoid similar but different entries.
+            // Eventually we should just accept '.' seperated for all cases.
+            let field_name = map_to_canonical
+                .get(field_name)
+                .unwrap_or(field_name)
+                .to_string();
+            FieldMetadata {
+                indexed: false,
+                stored: false,
+                field_name,
+                fast: true,
+                typ: *typ,
+            }
+        })
+        .collect();
+    // Since the type is encoded differently in the fast field and in the inverted index,
+    // the order of the fields is not guaranteed to be the same. Therefore, we sort the fields.
+    // If we are sure that the order is the same, we can remove this sort.
+    indexed_fields.sort_unstable();
+    fast_fields.sort_unstable();
+    let merged = merge_field_meta_data(vec![indexed_fields, fast_fields], schema);
+    let out = serialize_split_fields(&merged);
+    wrt.write_all(&out)?;
+
+    Ok(())
+}
+
+/// Serializes the Split fields.
+///
+/// `fields_metadata` has to be sorted.
+pub fn serialize_split_fields(fields_metadata: &[FieldMetadata]) -> Vec<u8> {
+    // ensure that fields_metadata is strictly sorted.
+    debug_assert!(fields_metadata.windows(2).all(|w| w[0] < w[1]));
+    let mut payload = Vec::new();
+    // Write Num Fields
+    let length = fields_metadata.len() as u32;
+    payload.extend_from_slice(&length.to_le_bytes());
+
+    for field_metadata in fields_metadata {
+        write_field(field_metadata, &mut payload);
+    }
+    let compression_level = 3;
+    let payload_compressed = zstd::stream::encode_all(&mut &payload[..], compression_level)
+        .expect("zstd encoding failed");
+    let mut out = Vec::new();
+    // Write Header -- Format Version
+    let format_version = 1u8;
+    out.push(format_version);
+    // Write Payload
+    out.extend_from_slice(&payload_compressed);
+    out
+}
+
+fn write_field(field_metadata: &FieldMetadata, out: &mut Vec<u8>) {
+    let field_config = FieldConfig {
+        typ: field_metadata.typ,
+        indexed: field_metadata.indexed,
+        stored: field_metadata.stored,
+        fast: field_metadata.fast,
+    };
+
+    // Write Config 2 bytes
+    out.extend_from_slice(&field_config.serialize());
+    let str_length = field_metadata.field_name.len() as u16;
+    // Write String length 2 bytes
+    out.extend_from_slice(&str_length.to_le_bytes());
+    out.extend_from_slice(field_metadata.field_name.as_bytes());
+}
+
+/// Reads a fixed number of bytes into an array and returns the array.
+fn read_exact_array<R: Read, const N: usize>(reader: &mut R) -> io::Result<[u8; N]> {
+    let mut buffer = [0u8; N];
+    reader.read_exact(&mut buffer)?;
+    Ok(buffer)
+}
+
+/// Reads the Split fields from a zstd compressed stream of bytes
+pub fn read_split_fields<R: Read>(
+    mut reader: R,
+) -> io::Result<impl Iterator<Item = io::Result<FieldMetadata>>> {
+    let format_version = read_exact_array::<_, 1>(&mut reader)?[0];
+    assert_eq!(format_version, 1);
+    let reader = zstd::Decoder::new(reader)?;
+    read_split_fields_from_zstd(reader)
+}
+
+fn read_field<R: Read>(reader: &mut R) -> io::Result<FieldMetadata> {
+    // Read FieldConfig (2 bytes)
+    let config_bytes = read_exact_array::<_, 2>(reader)?;
+    let field_config = FieldConfig::deserialize_from(config_bytes)?; // Assuming this returns a Result
+
+    // Read field name length and the field name
+    let name_len = u16::from_le_bytes(read_exact_array::<_, 2>(reader)?) as usize;
+
+    let mut data = vec![0; name_len];
+    reader.read_exact(&mut data)?;
+
+    let field_name = String::from_utf8(data).map_err(|err| {
+        io::Error::new(
+            ErrorKind::InvalidData,
+            format!(
+                "Encountered invalid utf8 when deserializing field name: {}",
+                err
+            ),
+        )
+    })?;
+    Ok(FieldMetadata {
+        field_name,
+        typ: field_config.typ,
+        indexed: field_config.indexed,
+        stored: field_config.stored,
+        fast: field_config.fast,
+    })
+}
+
+/// Reads the Split fields from a stream of bytes
+fn read_split_fields_from_zstd<R: Read>(
+    mut reader: R,
+) -> io::Result<impl Iterator<Item = io::Result<FieldMetadata>>> {
+    let mut num_fields = u32::from_le_bytes(read_exact_array::<_, 4>(&mut reader)?);
+
+    Ok(std::iter::from_fn(move || {
+        if num_fields == 0 {
+            return None;
+        }
+        num_fields -= 1;
+
+        Some(read_field(&mut reader))
+    }))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn field_config_deser_test() {
+        let field_config = FieldConfig {
+            typ: Type::Str,
+            indexed: true,
+            stored: false,
+            fast: true,
+        };
+        let serialized = field_config.serialize();
+        let deserialized = FieldConfig::deserialize_from(serialized).unwrap();
+        assert_eq!(field_config, deserialized);
+    }
+    #[test]
+    fn write_read_field_test() {
+        for typ in Type::iter_values() {
+            let field_metadata = FieldMetadata {
+                field_name: "test".to_string(),
+                typ,
+                indexed: true,
+                stored: true,
+                fast: true,
+            };
+            let mut out = Vec::new();
+            write_field(&field_metadata, &mut out);
+            let deserialized = read_field(&mut &out[..]).unwrap();
+            assert_eq!(field_metadata, deserialized);
+        }
+        let field_metadata = FieldMetadata {
+            field_name: "test".to_string(),
+            typ: Type::Str,
+            indexed: false,
+            stored: true,
+            fast: true,
+        };
+        let mut out = Vec::new();
+        write_field(&field_metadata, &mut out);
+        let deserialized = read_field(&mut &out[..]).unwrap();
+        assert_eq!(field_metadata, deserialized);
+
+        let field_metadata = FieldMetadata {
+            field_name: "test".to_string(),
+            typ: Type::Str,
+            indexed: false,
+            stored: false,
+            fast: true,
+        };
+        let mut out = Vec::new();
+        write_field(&field_metadata, &mut out);
+        let deserialized = read_field(&mut &out[..]).unwrap();
+        assert_eq!(field_metadata, deserialized);
+
+        let field_metadata = FieldMetadata {
+            field_name: "test".to_string(),
+            typ: Type::Str,
+            indexed: true,
+            stored: false,
+            fast: false,
+        };
+        let mut out = Vec::new();
+        write_field(&field_metadata, &mut out);
+        let deserialized = read_field(&mut &out[..]).unwrap();
+        assert_eq!(field_metadata, deserialized);
+    }
+    #[test]
+    fn write_split_fields_test() {
+        let fields_metadata = vec![
+            FieldMetadata {
+                field_name: "test".to_string(),
+                typ: Type::Str,
+                indexed: true,
+                stored: true,
+                fast: true,
+            },
+            FieldMetadata {
+                field_name: "test2".to_string(),
+                typ: Type::Str,
+                indexed: true,
+                stored: false,
+                fast: false,
+            },
+            FieldMetadata {
+                field_name: "test3".to_string(),
+                typ: Type::U64,
+                indexed: true,
+                stored: false,
+                fast: true,
+            },
+        ];
+
+        let out = serialize_split_fields(&fields_metadata);
+
+        let deserialized: Vec<FieldMetadata> = read_split_fields(&mut &out[..])
+            .unwrap()
+            .map(|el| el.unwrap())
+            .collect();
+
+        assert_eq!(fields_metadata, deserialized);
+    }
+}
--- a/src/functional_test.rs
+++ b/src/functional_test.rs
@@ -4,7 +4,7 @@ use rand::{thread_rng, Rng};

 use crate::indexer::index_writer::MEMORY_BUDGET_NUM_BYTES_MIN;
 use crate::schema::*;
-use crate::{doc, schema, Index, IndexSettings, IndexSortByField, Order, Searcher};
+use crate::{doc, schema, Index, IndexSettings, IndexSortByField, IndexWriter, Order, Searcher};

 fn check_index_content(searcher: &Searcher, vals: &[u64]) -> crate::Result<()> {
    assert!(searcher.segment_readers().len() < 20);
@@ -12,7 +12,7 @@ fn check_index_content(searcher: &Searcher, vals: &[u64]) -> crate::Result<()> {
    for segment_reader in searcher.segment_readers() {
        let store_reader = segment_reader.get_store_reader(1)?;
        for doc_id in 0..segment_reader.max_doc() {
-            let _doc = store_reader.get(doc_id)?;
+            let _doc: TantivyDocument = store_reader.get(doc_id)?;
        }
    }
    Ok(())
@@ -31,7 +31,8 @@ fn test_functional_store() -> crate::Result<()> {

    let mut rng = thread_rng();

-    let mut index_writer = index.writer_with_num_threads(3, MEMORY_BUDGET_NUM_BYTES_MIN)?;
+    let mut index_writer: IndexWriter =
+        index.writer_with_num_threads(3, 3 * MEMORY_BUDGET_NUM_BYTES_MIN)?;

    let mut doc_set: Vec<u64> = Vec::new();

@@ -91,7 +92,8 @@ fn test_functional_indexing_sorted() -> crate::Result<()> {

    let mut rng = thread_rng();

-    let mut index_writer = index.writer_with_num_threads(3, 120_000_000)?;
+    let mut index_writer: IndexWriter =
+        index.writer_with_num_threads(3, 3 * MEMORY_BUDGET_NUM_BYTES_MIN)?;

    let mut committed_docs: HashSet<u64> = HashSet::new();
    let mut uncommitted_docs: HashSet<u64> = HashSet::new();
@@ -114,7 +116,7 @@ fn test_functional_indexing_sorted() -> crate::Result<()> {
            index_writer.delete_term(doc_id_term);
        } else {
            uncommitted_docs.insert(random_val);
-            let mut doc = Document::new();
+            let mut doc = TantivyDocument::new();
            doc.add_u64(id_field, random_val);
            for i in 1u64..10u64 {
                doc.add_u64(multiples_field, random_val * i);
@@ -166,7 +168,8 @@ fn test_functional_indexing_unsorted() -> crate::Result<()> {

    let mut rng = thread_rng();

-    let mut index_writer = index.writer_with_num_threads(3, 120_000_000)?;
+    let mut index_writer: IndexWriter =
+        index.writer_with_num_threads(3, 3 * MEMORY_BUDGET_NUM_BYTES_MIN)?;

    let mut committed_docs: HashSet<u64> = HashSet::new();
    let mut uncommitted_docs: HashSet<u64> = HashSet::new();
@@ -189,7 +192,7 @@ fn test_functional_indexing_unsorted() -> crate::Result<()> {
            index_writer.delete_term(doc_id_term);
        } else {
            uncommitted_docs.insert(random_val);
-            let mut doc = Document::new();
+            let mut doc = TantivyDocument::new();
            doc.add_u64(id_field, random_val);
            for i in 1u64..10u64 {
                doc.add_u64(multiples_field, random_val * i);
--- a/src/indexer/doc_id_mapping.rs
+++ b/src/indexer/doc_id_mapping.rs
@@ -158,6 +158,7 @@ mod tests_indexsorting {
    use crate::indexer::doc_id_mapping::DocIdMapping;
    use crate::indexer::NoMergePolicy;
    use crate::query::QueryParser;
+    use crate::schema::document::Value;
    use crate::schema::{Schema, *};
    use crate::{DocAddress, Index, IndexSettings, IndexSortByField, Order};

@@ -308,16 +309,16 @@ mod tests_indexsorting {
        {
            assert_eq!(
                searcher
-                    .doc(DocAddress::new(0, 0))?
+                    .doc::<TantivyDocument>(DocAddress::new(0, 0))?
                    .get_first(my_string_field),
                None
            );
            assert_eq!(
                searcher
-                    .doc(DocAddress::new(0, 3))?
+                    .doc::<TantivyDocument>(DocAddress::new(0, 3))?
                    .get_first(my_string_field)
                    .unwrap()
-                    .as_text(),
+                    .as_str(),
                Some("blublub")
            );
        }
@@ -337,13 +338,13 @@ mod tests_indexsorting {
        {
            assert_eq!(
                searcher
-                    .doc(DocAddress::new(0, 0))?
+                    .doc::<TantivyDocument>(DocAddress::new(0, 0))?
                    .get_first(my_string_field)
                    .unwrap()
-                    .as_text(),
+                    .as_str(),
                Some("blublub")
            );
-            let doc = searcher.doc(DocAddress::new(0, 4))?;
+            let doc = searcher.doc::<TantivyDocument>(DocAddress::new(0, 4))?;
            assert_eq!(doc.get_first(my_string_field), None);
        }
        // sort by field desc
@@ -360,9 +361,9 @@ mod tests_indexsorting {
        let my_string_field = index.schema().get_field("string_field").unwrap();
        let searcher = index.reader()?.searcher();
        {
-            let doc = searcher.doc(DocAddress::new(0, 4))?;
+            let doc = searcher.doc::<TantivyDocument>(DocAddress::new(0, 4))?;
            assert_eq!(
-                doc.get_first(my_string_field).unwrap().as_text(),
+                doc.get_first(my_string_field).unwrap().as_str(),
                Some("blublub")
            );
        }
--- a/src/indexer/index_writer.rs
+++ b/src/indexer/index_writer.rs
@@ -20,7 +20,8 @@ use crate::indexer::operation::DeleteOperation;
 use crate::indexer::stamper::Stamper;
 use crate::indexer::{MergePolicy, SegmentEntry, SegmentWriter};
 use crate::query::{EnableScoring, Query, TermQuery};
-use crate::schema::{Document, IndexRecordOption, Term};
+use crate::schema::document::Document;
+use crate::schema::{IndexRecordOption, TantivyDocument, Term};
 use crate::{FutureResult, Opstamp};

 // Size of the margin for the `memory_arena`. A segment is closed when the remaining memory
@@ -50,7 +51,7 @@ fn error_in_index_worker_thread(context: &str) -> TantivyError {
 /// indexing queue.
 /// Each indexing thread builds its own independent [`Segment`], via
 /// a `SegmentWriter` object.
-pub struct IndexWriter {
+pub struct IndexWriter<D: Document = TantivyDocument> {
    // the lock is just used to bind the
    // lifetime of the lock with that of the IndexWriter.
    _directory_lock: Option<DirectoryLock>,
@@ -62,8 +63,8 @@ pub struct IndexWriter {

    workers_join_handle: Vec<JoinHandle<crate::Result<()>>>,

-    index_writer_status: IndexWriterStatus,
-    operation_sender: AddBatchSender,
+    index_writer_status: IndexWriterStatus<D>,
+    operation_sender: AddBatchSender<D>,

    segment_updater: SegmentUpdater,

@@ -164,10 +165,10 @@ pub(crate) fn advance_deletes(
    Ok(())
 }

-fn index_documents(
+fn index_documents<D: Document>(
    memory_budget: usize,
    segment: Segment,
-    grouped_document_iterator: &mut dyn Iterator<Item = AddBatch>,
+    grouped_document_iterator: &mut dyn Iterator<Item = AddBatch<D>>,
    segment_updater: &SegmentUpdater,
    mut delete_cursor: DeleteCursor,
 ) -> crate::Result<()> {
@@ -247,7 +248,7 @@ fn apply_deletes(
    })
 }

-impl IndexWriter {
+impl<D: Document> IndexWriter<D> {
    /// Create a new index writer. Attempts to acquire a lockfile.
    ///
    /// The lockfile should be deleted on drop, but it is possible
@@ -267,7 +268,7 @@ impl IndexWriter {
        num_threads: usize,
        memory_budget_in_bytes_per_thread: usize,
        directory_lock: DirectoryLock,
-    ) -> crate::Result<IndexWriter> {
+    ) -> crate::Result<Self> {
        if memory_budget_in_bytes_per_thread < MEMORY_BUDGET_NUM_BYTES_MIN {
            let err_msg = format!(
                "The memory arena in bytes per thread needs to be at least \
@@ -281,7 +282,7 @@ impl IndexWriter {
            );
            return Err(TantivyError::InvalidArgument(err_msg));
        }
-        let (document_sender, document_receiver): (AddBatchSender, AddBatchReceiver) =
+        let (document_sender, document_receiver) =
            crossbeam_channel::bounded(PIPELINE_MAX_SIZE_IN_DOCS);

        let delete_queue = DeleteQueue::new();
@@ -293,7 +294,7 @@ impl IndexWriter {
        let segment_updater =
            SegmentUpdater::create(index.clone(), stamper.clone(), &delete_queue.cursor())?;

-        let mut index_writer = IndexWriter {
+        let mut index_writer = Self {
            _directory_lock: Some(directory_lock),

            memory_budget_in_bytes_per_thread,
@@ -375,7 +376,7 @@ impl IndexWriter {
        self.index.new_segment()
    }

-    fn operation_receiver(&self) -> crate::Result<AddBatchReceiver> {
+    fn operation_receiver(&self) -> crate::Result<AddBatchReceiver<D>> {
        self.index_writer_status
            .operation_receiver()
            .ok_or_else(|| {
@@ -525,7 +526,7 @@ impl IndexWriter {
    ///
    /// Returns the former segment_ready channel.
    fn recreate_document_channel(&mut self) {
-        let (document_sender, document_receiver): (AddBatchSender, AddBatchReceiver) =
+        let (document_sender, document_receiver) =
            crossbeam_channel::bounded(PIPELINE_MAX_SIZE_IN_DOCS);
        self.operation_sender = document_sender;
        self.index_writer_status = IndexWriterStatus::from(document_receiver);
@@ -552,7 +553,7 @@ impl IndexWriter {
            .take()
            .expect("The IndexWriter does not have any lock. This is a bug, please report.");

-        let new_index_writer: IndexWriter = IndexWriter::new(
+        let new_index_writer = IndexWriter::new(
            &self.index,
            self.num_threads,
            self.memory_budget_in_bytes_per_thread,
@@ -598,7 +599,7 @@ impl IndexWriter {
    /// It is also possible to add a payload to the `commit`
    /// using this API.
    /// See [`PreparedCommit::set_payload()`].
-    pub fn prepare_commit(&mut self) -> crate::Result<PreparedCommit> {
+    pub fn prepare_commit(&mut self) -> crate::Result<PreparedCommit<D>> {
        // Here, because we join all of the worker threads,
        // all of the segment update for this commit have been
        // sent.
@@ -620,7 +621,7 @@ impl IndexWriter {
        for worker_handle in former_workers_join_handle {
            let indexing_worker_result = worker_handle
                .join()
-                .map_err(|e| TantivyError::ErrorInThread(e.to_string()))?;
+                .map_err(|e| TantivyError::ErrorInThread(format!("{e:?}")))?;
            indexing_worker_result?;
            self.add_indexing_worker()?;
        }
@@ -707,7 +708,7 @@ impl IndexWriter {
    /// The opstamp is an increasing `u64` that can
    /// be used by the client to align commits with its own
    /// document queue.
-    pub fn add_document(&self, document: Document) -> crate::Result<Opstamp> {
+    pub fn add_document(&self, document: D) -> crate::Result<Opstamp> {
        let opstamp = self.stamper.stamp();
        self.send_add_documents_batch(smallvec![AddOperation { opstamp, document }])?;
        Ok(opstamp)
@@ -744,7 +745,7 @@ impl IndexWriter {
    /// visible to readers only after calling `commit()`.
    pub fn run<I>(&self, user_operations: I) -> crate::Result<Opstamp>
    where
-        I: IntoIterator<Item = UserOperation>,
+        I: IntoIterator<Item = UserOperation<D>>,
        I::IntoIter: ExactSizeIterator,
    {
        let user_operations_it = user_operations.into_iter();
@@ -778,7 +779,7 @@ impl IndexWriter {
        Ok(batch_opstamp)
    }

-    fn send_add_documents_batch(&self, add_ops: AddBatch) -> crate::Result<()> {
+    fn send_add_documents_batch(&self, add_ops: AddBatch<D>) -> crate::Result<()> {
        if self.index_writer_status.is_alive() && self.operation_sender.send(add_ops).is_ok() {
            Ok(())
        } else {
@@ -787,7 +788,7 @@ impl IndexWriter {
    }
 }

-impl Drop for IndexWriter {
+impl<D: Document> Drop for IndexWriter<D> {
    fn drop(&mut self) {
        self.segment_updater.kill();
        self.drop_sender();
@@ -814,13 +815,15 @@ mod tests {
    use crate::indexer::index_writer::MEMORY_BUDGET_NUM_BYTES_MIN;
    use crate::indexer::NoMergePolicy;
    use crate::query::{BooleanQuery, Occur, Query, QueryParser, TermQuery};
+    use crate::schema::document::Value;
    use crate::schema::{
        self, Facet, FacetOptions, IndexRecordOption, IpAddrOptions, NumericOptions, Schema,
        TextFieldIndexing, TextOptions, FAST, INDEXED, STORED, STRING, TEXT,
    };
    use crate::store::DOCSTORE_CACHE_CAPACITY;
    use crate::{
-        DateTime, DocAddress, Index, IndexSettings, IndexSortByField, Order, ReloadPolicy, Term,
+        DateTime, DocAddress, Index, IndexSettings, IndexSortByField, IndexWriter, Order,
+        ReloadPolicy, TantivyDocument, Term,
    };

    const LOREM: &str = "Doc Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do \
@@ -852,7 +855,7 @@ mod tests {
        let text_field = schema_builder.add_text_field("text", TEXT);
        let index = Index::create_in_ram(schema_builder.build());

-        let mut index_writer = index.writer_for_tests().unwrap();
+        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
        index_writer
            .add_document(doc!(text_field => "hello1"))
            .unwrap();
@@ -905,7 +908,7 @@ mod tests {
            .reload_policy(ReloadPolicy::Manual)
            .try_into()
            .unwrap();
-        let mut index_writer = index.writer_for_tests().unwrap();
+        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
        let a_term = Term::from_field_text(text_field, "a");
        let b_term = Term::from_field_text(text_field, "b");
        let operations = vec![
@@ -943,7 +946,7 @@ mod tests {
    fn test_empty_operations_group() {
        let schema_builder = schema::Schema::builder();
        let index = Index::create_in_ram(schema_builder.build());
-        let index_writer = index.writer_for_tests().unwrap();
+        let index_writer: IndexWriter = index.writer_for_tests().unwrap();
        let operations1 = vec![];
        let batch_opstamp1 = index_writer.run(operations1).unwrap();
        assert_eq!(batch_opstamp1, 0u64);
@@ -956,8 +959,8 @@ mod tests {
    fn test_lockfile_stops_duplicates() {
        let schema_builder = schema::Schema::builder();
        let index = Index::create_in_ram(schema_builder.build());
-        let _index_writer = index.writer_for_tests().unwrap();
-        match index.writer_for_tests() {
+        let _index_writer: IndexWriter = index.writer_for_tests().unwrap();
+        match index.writer_for_tests::<TantivyDocument>() {
            Err(TantivyError::LockFailure(LockError::LockBusy, _)) => {}
            _ => panic!("Expected a `LockFailure` error"),
        }
@@ -967,8 +970,8 @@ mod tests {
    fn test_lockfile_already_exists_error_msg() {
        let schema_builder = schema::Schema::builder();
        let index = Index::create_in_ram(schema_builder.build());
-        let _index_writer = index.writer_for_tests().unwrap();
-        match index.writer_for_tests() {
+        let _index_writer: IndexWriter = index.writer_for_tests().unwrap();
+        match index.writer_for_tests::<TantivyDocument>() {
            Err(err) => {
                let err_msg = err.to_string();
                assert!(err_msg.contains("already an `IndexWriter`"));
@@ -981,7 +984,7 @@ mod tests {
    fn test_set_merge_policy() {
        let schema_builder = schema::Schema::builder();
        let index = Index::create_in_ram(schema_builder.build());
-        let index_writer = index.writer_for_tests().unwrap();
+        let index_writer: IndexWriter = index.writer_for_tests().unwrap();
        assert_eq!(
            format!("{:?}", index_writer.get_merge_policy()),
            "LogMergePolicy { min_num_segments: 8, max_docs_before_merge: 10000000, \
@@ -1000,11 +1003,11 @@ mod tests {
        let schema_builder = schema::Schema::builder();
        let index = Index::create_in_ram(schema_builder.build());
        {
-            let _index_writer = index.writer_for_tests().unwrap();
+            let _index_writer: IndexWriter = index.writer_for_tests().unwrap();
            // the lock should be released when the
            // index_writer leaves the scope.
        }
-        let _index_writer_two = index.writer_for_tests().unwrap();
+        let _index_writer_two: IndexWriter = index.writer_for_tests().unwrap();
    }

    #[test]
@@ -1056,7 +1059,7 @@ mod tests {
            reader.searcher().doc_freq(&term_a).unwrap()
        };
        // writing the segment
-        let mut index_writer = index.writer_for_tests().unwrap();
+        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
        index_writer.add_document(doc!(text_field=>"a"))?;
        index_writer.commit()?;
        //  this should create 1 segment
@@ -1096,7 +1099,7 @@ mod tests {
            reader.searcher().doc_freq(&term_a).unwrap()
        };
        // writing the segment
-        let mut index_writer = index.writer_for_tests().unwrap();
+        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
        index_writer.add_document(doc!(text_field=>"a"))?;
        index_writer.commit()?;
        index_writer.add_document(doc!(text_field=>"a"))?;
@@ -1382,7 +1385,7 @@ mod tests {
    fn test_delete_all_documents_empty_index() {
        let schema_builder = schema::Schema::builder();
        let index = Index::create_in_ram(schema_builder.build());
-        let mut index_writer = index
+        let mut index_writer: IndexWriter = index
            .writer_with_num_threads(4, MEMORY_BUDGET_NUM_BYTES_MIN * 4)
            .unwrap();
        let clear = index_writer.delete_all_documents();
@@ -1395,7 +1398,7 @@ mod tests {
    fn test_delete_all_documents_index_twice() {
        let schema_builder = schema::Schema::builder();
        let index = Index::create_in_ram(schema_builder.build());
-        let mut index_writer = index
+        let mut index_writer: IndexWriter = index
            .writer_with_num_threads(4, MEMORY_BUDGET_NUM_BYTES_MIN * 4)
            .unwrap();
        let clear = index_writer.delete_all_documents();
@@ -1415,7 +1418,7 @@ mod tests {
        let schema = schema_builder.build();

        let index = Index::builder().schema(schema).create_in_ram().unwrap();
-        let mut index_writer = index.writer_for_tests().unwrap();
+        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
        index_writer
            .add_document(doc!(text_field => "one"))
            .unwrap();
@@ -1703,7 +1706,8 @@ mod tests {

        let old_reader = index.reader()?;

-        let id_exists = |id| id % 3 != 0; // 0 does not exist
+        // Every 3rd doc has only id field
+        let id_is_full_doc = |id| id % 3 != 0;

        let multi_text_field_text1 = "test1 test2 test3 test1 test2 test3";
        // rotate left
@@ -1719,7 +1723,7 @@ mod tests {
                    let facet = Facet::from(&("/cola/".to_string() + &id.to_string()));
                    let ip = ip_from_id(id);

-                    if !id_exists(id) {
+                    if !id_is_full_doc(id) {
                        // every 3rd doc has no ip field
                        index_writer.add_document(doc!(
                            id_field=>id,
@@ -1777,7 +1781,7 @@ mod tests {
        let num_segments_before_merge = searcher.segment_readers().len();
        if force_end_merge {
            index_writer.wait_merging_threads()?;
-            let mut index_writer = index.writer_for_tests()?;
+            let mut index_writer: IndexWriter = index.writer_for_tests()?;
            let segment_ids = index
                .searchable_segment_ids()
                .expect("Searchable segments failed.");
@@ -1839,7 +1843,7 @@ mod tests {

        let num_docs_with_values = expected_ids_and_num_occurrences
            .iter()
-            .filter(|(id, _id_occurrences)| id_exists(**id))
+            .filter(|(id, _id_occurrences)| id_is_full_doc(**id))
            .map(|(_, id_occurrences)| *id_occurrences as usize)
            .sum::<usize>();

@@ -1863,7 +1867,7 @@ mod tests {
        if force_end_merge && num_segments_before_merge > 1 && num_segments_after_merge == 1 {
            let mut expected_multi_ips: Vec<_> = id_list
                .iter()
-                .filter(|id| id_exists(**id))
+                .filter(|id| id_is_full_doc(**id))
                .flat_map(|id| vec![ip_from_id(*id), ip_from_id(*id)])
                .collect();
            assert_eq!(num_ips, expected_multi_ips.len() as u32);
@@ -1901,7 +1905,7 @@ mod tests {
        let expected_ips = expected_ids_and_num_occurrences
            .keys()
            .flat_map(|id| {
-                if !id_exists(*id) {
+                if !id_is_full_doc(*id) {
                    None
                } else {
                    Some(Ipv6Addr::from_u128(*id as u128))
@@ -1913,7 +1917,7 @@ mod tests {
        let expected_ips = expected_ids_and_num_occurrences
            .keys()
            .filter_map(|id| {
-                if !id_exists(*id) {
+                if !id_is_full_doc(*id) {
                    None
                } else {
                    Some(Ipv6Addr::from_u128(*id as u128))
@@ -1948,7 +1952,7 @@ mod tests {
                let id = id_reader.first(doc).unwrap();

                let vals: Vec<u64> = ff_reader.values_for_doc(doc).collect();
-                if id_exists(id) {
+                if id_is_full_doc(id) {
                    assert_eq!(vals.len(), 2);
                    assert_eq!(vals[0], vals[1]);
                    assert!(expected_ids_and_num_occurrences.contains_key(&vals[0]));
@@ -1958,7 +1962,7 @@ mod tests {
                }

                let bool_vals: Vec<bool> = bool_ff_reader.values_for_doc(doc).collect();
-                if id_exists(id) {
+                if id_is_full_doc(id) {
                    assert_eq!(bool_vals.len(), 2);
                    assert_ne!(bool_vals[0], bool_vals[1]);
                } else {
@@ -1973,23 +1977,23 @@ mod tests {
                .get_store_reader(DOCSTORE_CACHE_CAPACITY)
                .unwrap();
            // test store iterator
-            for doc in store_reader.iter(segment_reader.alive_bitset()) {
+            for doc in store_reader.iter::<TantivyDocument>(segment_reader.alive_bitset()) {
                let id = doc.unwrap().get_first(id_field).unwrap().as_u64().unwrap();
                assert!(expected_ids_and_num_occurrences.contains_key(&id));
            }
            // test store random access
            for doc_id in segment_reader.doc_ids_alive() {
                let id = store_reader
-                    .get(doc_id)
+                    .get::<TantivyDocument>(doc_id)
                    .unwrap()
                    .get_first(id_field)
                    .unwrap()
                    .as_u64()
                    .unwrap();
                assert!(expected_ids_and_num_occurrences.contains_key(&id));
-                if id_exists(id) {
+                if id_is_full_doc(id) {
                    let id2 = store_reader
-                        .get(doc_id)
+                        .get::<TantivyDocument>(doc_id)
                        .unwrap()
                        .get_first(multi_numbers)
                        .unwrap()
@@ -1997,13 +2001,13 @@ mod tests {
                        .unwrap();
                    assert_eq!(id, id2);
                    let bool = store_reader
-                        .get(doc_id)
+                        .get::<TantivyDocument>(doc_id)
                        .unwrap()
                        .get_first(bool_field)
                        .unwrap()
                        .as_bool()
                        .unwrap();
-                    let doc = store_reader.get(doc_id).unwrap();
+                    let doc = store_reader.get::<TantivyDocument>(doc_id).unwrap();
                    let mut bool2 = doc.get_all(multi_bools);
                    assert_eq!(bool, bool2.next().unwrap().as_bool().unwrap());
                    assert_ne!(bool, bool2.next().unwrap().as_bool().unwrap());
@@ -2034,7 +2038,7 @@ mod tests {
            let (existing_id, count) = (*id, *count);
            let get_num_hits = |field| do_search(&existing_id.to_string(), field).len() as u64;
            assert_eq!(get_num_hits(id_field), count);
-            if !id_exists(existing_id) {
+            if !id_is_full_doc(existing_id) {
                continue;
            }
            assert_eq!(get_num_hits(text_field), count);
@@ -2084,7 +2088,7 @@ mod tests {
        //
        for (existing_id, count) in &expected_ids_and_num_occurrences {
            let (existing_id, count) = (*existing_id, *count);
-            if !id_exists(existing_id) {
+            if !id_is_full_doc(existing_id) {
                continue;
            }
            let do_search_ip_field = |term: &str| do_search(term, ip_field).len() as u64;
@@ -2101,34 +2105,84 @@ mod tests {
            }
        }

-        // assert data is like expected
+        // Range query
        //
-        for (existing_id, count) in expected_ids_and_num_occurrences.iter().take(10) {
-            let (existing_id, count) = (*existing_id, *count);
-            if !id_exists(existing_id) {
-                continue;
-            }
-            let gen_query_inclusive = |field: &str, from: Ipv6Addr, to: Ipv6Addr| {
-                format!("{}:[{} TO {}]", field, &from.to_string(), &to.to_string())
+        // Take half as sample
+        let mut sample: Vec<_> = expected_ids_and_num_occurrences.iter().collect();
+        sample.sort_by_key(|(k, _num_occurences)| *k);
+        // sample.truncate(sample.len() / 2);
+        if !sample.is_empty() {
+            let (left_sample, right_sample) = sample.split_at(sample.len() / 2);
+
+            let expected_count = |sample: &[(&u64, &u64)]| {
+                sample
+                    .iter()
+                    .filter(|(id, _)| id_is_full_doc(**id))
+                    .map(|(_id, num_occurences)| **num_occurences)
+                    .sum::<u64>()
            };
-            let ip = ip_from_id(existing_id);
+            fn gen_query_inclusive<T1: ToString, T2: ToString>(
+                field: &str,
+                from: T1,
+                to: T2,
+            ) -> String {
+                format!("{}:[{} TO {}]", field, &from.to_string(), &to.to_string())
+            }

-            let do_search_ip_field = |term: &str| do_search(term, ip_field).len() as u64;
-            // Range query on single value field
-            let query = gen_query_inclusive("ip", ip, ip);
-            assert_eq!(do_search_ip_field(&query), count);
+            // Query first half
+            if !left_sample.is_empty() {
+                let expected_count = expected_count(left_sample);

-            // Range query on multi value field
-            let query = gen_query_inclusive("ips", ip, ip);
+                let start_range = *left_sample[0].0;
+                let end_range = *left_sample.last().unwrap().0;
+                let query = gen_query_inclusive("id_opt", start_range, end_range);
+                assert_eq!(do_search(&query, id_opt_field).len() as u64, expected_count);

-            assert_eq!(do_search_ip_field(&query), count);
+                // Range query on ip field
+                let ip1 = ip_from_id(start_range);
+                let ip2 = ip_from_id(end_range);
+                let do_search_ip_field = |term: &str| do_search(term, ip_field).len() as u64;
+                let query = gen_query_inclusive("ip", ip1, ip2);
+                assert_eq!(do_search_ip_field(&query), expected_count);
+                let query = gen_query_inclusive("ip", "*", ip2);
+                assert_eq!(do_search_ip_field(&query), expected_count);
+                // Range query on multi value field
+                let query = gen_query_inclusive("ips", ip1, ip2);
+                assert_eq!(do_search_ip_field(&query), expected_count);
+                let query = gen_query_inclusive("ips", "*", ip2);
+                assert_eq!(do_search_ip_field(&query), expected_count);
+            }
+            // Query second half
+            if !right_sample.is_empty() {
+                let expected_count = expected_count(right_sample);
+                let start_range = *right_sample[0].0;
+                let end_range = *right_sample.last().unwrap().0;
+                // Range query on id opt field
+                let query =
+                    gen_query_inclusive("id_opt", start_range.to_string(), end_range.to_string());
+                assert_eq!(do_search(&query, id_opt_field).len() as u64, expected_count);
+
+                // Range query on ip field
+                let ip1 = ip_from_id(start_range);
+                let ip2 = ip_from_id(end_range);
+                let do_search_ip_field = |term: &str| do_search(term, ip_field).len() as u64;
+                let query = gen_query_inclusive("ip", ip1, ip2);
+                assert_eq!(do_search_ip_field(&query), expected_count);
+                let query = gen_query_inclusive("ip", ip1, "*");
+                assert_eq!(do_search_ip_field(&query), expected_count);
+                // Range query on multi value field
+                let query = gen_query_inclusive("ips", ip1, ip2);
+                assert_eq!(do_search_ip_field(&query), expected_count);
+                let query = gen_query_inclusive("ips", ip1, "*");
+                assert_eq!(do_search_ip_field(&query), expected_count);
+            }
        }

        // ip range query on fast field
        //
        for (existing_id, count) in expected_ids_and_num_occurrences.iter().take(10) {
            let (existing_id, count) = (*existing_id, *count);
-            if !id_exists(existing_id) {
+            if !id_is_full_doc(existing_id) {
                continue;
            }
            let gen_query_inclusive = |field: &str, from: Ipv6Addr, to: Ipv6Addr| {
@@ -2156,7 +2210,7 @@ mod tests {
                .first_or_default_col(9999);
            for doc_id in segment_reader.doc_ids_alive() {
                let id = ff_reader.get_val(doc_id);
-                if !id_exists(id) {
+                if !id_is_full_doc(id) {
                    continue;
                }
                let facet_ords: Vec<u64> = facet_reader.facet_ords(doc_id).collect();
@@ -2194,6 +2248,12 @@ mod tests {
        Ok(index)
    }

+    #[test]
+    fn test_fast_field_range() {
+        let ops: Vec<_> = (0..1000).map(|id| IndexingOp::AddDoc { id }).collect();
+        assert!(test_operation_strategy(&ops, false, true).is_ok());
+    }
+
    #[test]
    fn test_sort_index_on_opt_field_regression() {
        assert!(test_operation_strategy(
@@ -2543,7 +2603,7 @@ mod tests {
        // Merge
        {
            assert!(index_writer.wait_merging_threads().is_ok());
-            let mut index_writer = index.writer_for_tests()?;
+            let mut index_writer: IndexWriter = index.writer_for_tests()?;
            let segment_ids = index
                .searchable_segment_ids()
                .expect("Searchable segments failed.");
@@ -2585,7 +2645,7 @@ mod tests {
        // Merge
        {
            assert!(index_writer.wait_merging_threads().is_ok());
-            let mut index_writer = index.writer_for_tests()?;
+            let mut index_writer: IndexWriter = index.writer_for_tests()?;
            let segment_ids = index
                .searchable_segment_ids()
                .expect("Searchable segments failed.");
--- a/src/indexer/index_writer_status.rs
+++ b/src/indexer/index_writer_status.rs
@@ -2,13 +2,15 @@ use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::{Arc, RwLock};

 use super::AddBatchReceiver;
+use crate::schema::document::Document;
+use crate::TantivyDocument;

 #[derive(Clone)]
-pub(crate) struct IndexWriterStatus {
-    inner: Arc<Inner>,
+pub(crate) struct IndexWriterStatus<D: Document = TantivyDocument> {
+    inner: Arc<Inner<D>>,
 }

-impl IndexWriterStatus {
+impl<D: Document> IndexWriterStatus<D> {
    /// Returns true iff the index writer is alive.
    pub fn is_alive(&self) -> bool {
        self.inner.as_ref().is_alive()
@@ -16,7 +18,7 @@ impl IndexWriterStatus {

    /// Returns a copy of the operation receiver.
    /// If the index writer was killed, returns `None`.
-    pub fn operation_receiver(&self) -> Option<AddBatchReceiver> {
+    pub fn operation_receiver(&self) -> Option<AddBatchReceiver<D>> {
        let rlock = self
            .inner
            .receive_channel
@@ -27,19 +29,19 @@ impl IndexWriterStatus {

    /// Create an index writer bomb.
    /// If dropped, the index writer status will be killed.
-    pub(crate) fn create_bomb(&self) -> IndexWriterBomb {
+    pub(crate) fn create_bomb(&self) -> IndexWriterBomb<D> {
        IndexWriterBomb {
            inner: Some(self.inner.clone()),
        }
    }
 }

-struct Inner {
+struct Inner<D: Document> {
    is_alive: AtomicBool,
-    receive_channel: RwLock<Option<AddBatchReceiver>>,
+    receive_channel: RwLock<Option<AddBatchReceiver<D>>>,
 }

-impl Inner {
+impl<D: Document> Inner<D> {
    fn is_alive(&self) -> bool {
        self.is_alive.load(Ordering::Relaxed)
    }
@@ -53,8 +55,8 @@ impl Inner {
    }
 }

-impl From<AddBatchReceiver> for IndexWriterStatus {
-    fn from(receiver: AddBatchReceiver) -> Self {
+impl<D: Document> From<AddBatchReceiver<D>> for IndexWriterStatus<D> {
+    fn from(receiver: AddBatchReceiver<D>) -> Self {
        IndexWriterStatus {
            inner: Arc::new(Inner {
                is_alive: AtomicBool::new(true),
@@ -66,11 +68,11 @@ impl From<AddBatchReceiver> for IndexWriterStatus {

 /// If dropped, the index writer will be killed.
 /// To prevent this, clients can call `.defuse()`.
-pub(crate) struct IndexWriterBomb {
-    inner: Option<Arc<Inner>>,
+pub(crate) struct IndexWriterBomb<D: Document> {
+    inner: Option<Arc<Inner<D>>>,
 }

-impl IndexWriterBomb {
+impl<D: Document> IndexWriterBomb<D> {
    /// Defuses the bomb.
    ///
    /// This is the only way to drop the bomb without killing
@@ -80,7 +82,7 @@ impl IndexWriterBomb {
    }
 }

-impl Drop for IndexWriterBomb {
+impl<D: Document> Drop for IndexWriterBomb<D> {
    fn drop(&mut self) {
        if let Some(inner) = self.inner.take() {
            inner.kill();
--- a/src/indexer/merge_operation.rs
+++ b/src/indexer/merge_operation.rs
@@ -63,10 +63,13 @@ impl MergeOperation {
        }
    }

+    /// Returns the opstamp up to which we want to consume the delete queue and reflect their
+    /// deletes.
    pub fn target_opstamp(&self) -> Opstamp {
        self.inner.target_opstamp
    }

+    /// Returns the list of segment to be merged.
    pub fn segment_ids(&self) -> &[SegmentId] {
        &self.inner.segment_ids[..]
    }
--- a/src/indexer/merger.rs
+++ b/src/indexer/merger.rs
@@ -1,3 +1,4 @@
+use std::io::Write;
 use std::sync::Arc;

 use columnar::{
@@ -13,6 +14,7 @@ use crate::directory::WritePtr;
 use crate::docset::{DocSet, TERMINATED};
 use crate::error::DataCorruption;
 use crate::fastfield::{AliveBitSet, FastFieldNotAvailableError};
+use crate::field_list::serialize_split_fields;
 use crate::fieldnorm::{FieldNormReader, FieldNormReaders, FieldNormsSerializer, FieldNormsWriter};
 use crate::indexer::doc_id_mapping::{MappingType, SegmentDocIdMapping};
 use crate::indexer::SegmentSerializer;
@@ -21,8 +23,8 @@ use crate::schema::{value_type_to_column_type, Field, FieldType, Schema};
 use crate::store::StoreWriter;
 use crate::termdict::{TermMerger, TermOrdinal};
 use crate::{
-    DocAddress, DocId, IndexSettings, IndexSortByField, InvertedIndexReader, Order,
-    SegmentComponent, SegmentOrdinal,
+    merge_field_meta_data, DocAddress, DocId, FieldMetadata, IndexSettings, IndexSortByField,
+    InvertedIndexReader, Order, SegmentComponent, SegmentOrdinal,
 };

 /// Segment's max doc must be `< MAX_DOC_LIMIT`.
@@ -255,6 +257,19 @@ impl IndexMerger {
        Ok(())
    }

+    fn write_field_list(&self, list_field_wrt: &mut WritePtr) -> crate::Result<()> {
+        let field_metadatas: Vec<Vec<FieldMetadata>> = self
+            .readers
+            .iter()
+            .map(|reader| reader.fields_metadata())
+            .collect::<crate::Result<Vec<_>>>()?;
+        let merged = merge_field_meta_data(field_metadatas, &self.schema);
+        let out = serialize_split_fields(&merged);
+        list_field_wrt.write_all(&out)?;
+
+        Ok(())
+    }
+
    fn write_fast_fields(
        &self,
        fast_field_wrt: &mut WritePtr,
@@ -552,7 +567,41 @@ impl IndexMerger {
                continue;
            }

-            field_serializer.new_term(term_bytes, total_doc_freq)?;
+            // This should never happen as we early exited for total_doc_freq == 0.
+            assert!(!segment_postings_containing_the_term.is_empty());
+
+            let has_term_freq = {
+                let has_term_freq = !segment_postings_containing_the_term[0]
+                    .1
+                    .block_cursor
+                    .freqs()
+                    .is_empty();
+                for (_, postings) in &segment_postings_containing_the_term[1..] {
+                    // This may look at a strange way to test whether we have term freq or not.
+                    // With JSON object, the schema is not sufficient to know whether a term
+                    // has its term frequency encoded or not:
+                    // strings may have term frequencies, while number terms never have one.
+                    //
+                    // Ideally, we should have burnt one bit of two in the `TermInfo`.
+                    // However, we preferred not changing the codec too much and detect this
+                    // instead by
+                    // - looking at the size of the skip data for bitpacked blocks
+                    // - observing the absence of remaining data after reading the docs for vint
+                    // blocks.
+                    //
+                    // Overall the reliable way to know if we have actual frequencies loaded or not
+                    // is to check whether the actual decoded array is empty or not.
+                    if has_term_freq != !postings.block_cursor.freqs().is_empty() {
+                        return Err(DataCorruption::comment_only(
+                            "Term freqs are inconsistent across segments",
+                        )
+                        .into());
+                    }
+                }
+                has_term_freq
+            };
+
+            field_serializer.new_term(term_bytes, total_doc_freq, has_term_freq)?;

            // We can now serialize this postings, by pushing each document to the
            // postings serializer.
@@ -567,8 +616,13 @@ impl IndexMerger {
                    if let Some(remapped_doc_id) = old_to_new_doc_id[doc as usize] {
                        // we make sure to only write the term if
                        // there is at least one document.
-                        let term_freq = segment_postings.term_freq();
-                        segment_postings.positions(&mut positions_buffer);
+                        let term_freq = if has_term_freq {
+                            segment_postings.positions(&mut positions_buffer);
+                            segment_postings.term_freq()
+                        } else {
+                            0u32
+                        };
+
                        // if doc_id_mapping exists, the doc_ids are reordered, they are
                        // not just stacked. The field serializer expects monotonically increasing
                        // doc_ids, so we collect and sort them first, before writing.
@@ -734,6 +788,7 @@ impl IndexMerger {
        self.write_storable_fields(serializer.get_store_writer(), &doc_id_mapping)?;
        debug!("write-fastfields");
        self.write_fast_fields(serializer.get_fast_field_write(), doc_id_mapping)?;
+        self.write_field_list(serializer.get_field_list_write())?;

        debug!("close-serializer");
        serializer.close()?;
@@ -753,9 +808,10 @@ mod tests {
    use crate::collector::{Count, FacetCollector};
    use crate::core::Index;
    use crate::query::{AllQuery, BooleanQuery, EnableScoring, Scorer, TermQuery};
+    use crate::schema::document::Value;
    use crate::schema::{
-        Document, Facet, FacetOptions, IndexRecordOption, NumericOptions, Term, TextFieldIndexing,
-        INDEXED, TEXT,
+        Facet, FacetOptions, IndexRecordOption, NumericOptions, TantivyDocument, Term,
+        TextFieldIndexing, INDEXED, TEXT,
    };
    use crate::time::OffsetDateTime;
    use crate::{
@@ -817,7 +873,7 @@ mod tests {
            let segment_ids = index
                .searchable_segment_ids()
                .expect("Searchable segments failed.");
-            let mut index_writer = index.writer_for_tests()?;
+            let mut index_writer: IndexWriter = index.writer_for_tests()?;
            index_writer.merge(&segment_ids).wait()?;
            index_writer.wait_merging_threads()?;
        }
@@ -866,30 +922,24 @@ mod tests {
                );
            }
            {
-                let doc = searcher.doc(DocAddress::new(0, 0))?;
-                assert_eq!(doc.get_first(text_field).unwrap().as_text(), Some("af b"));
+                let doc = searcher.doc::<TantivyDocument>(DocAddress::new(0, 0))?;
+                assert_eq!(doc.get_first(text_field).unwrap().as_str(), Some("af b"));
            }
            {
-                let doc = searcher.doc(DocAddress::new(0, 1))?;
-                assert_eq!(doc.get_first(text_field).unwrap().as_text(), Some("a b c"));
+                let doc = searcher.doc::<TantivyDocument>(DocAddress::new(0, 1))?;
+                assert_eq!(doc.get_first(text_field).unwrap().as_str(), Some("a b c"));
            }
            {
-                let doc = searcher.doc(DocAddress::new(0, 2))?;
-                assert_eq!(
-                    doc.get_first(text_field).unwrap().as_text(),
-                    Some("a b c d")
-                );
+                let doc = searcher.doc::<TantivyDocument>(DocAddress::new(0, 2))?;
+                assert_eq!(doc.get_first(text_field).unwrap().as_str(), Some("a b c d"));
            }
            {
-                let doc = searcher.doc(DocAddress::new(0, 3))?;
-                assert_eq!(doc.get_first(text_field).unwrap().as_text(), Some("af b"));
+                let doc = searcher.doc::<TantivyDocument>(DocAddress::new(0, 3))?;
+                assert_eq!(doc.get_first(text_field).unwrap().as_str(), Some("af b"));
            }
            {
-                let doc = searcher.doc(DocAddress::new(0, 4))?;
-                assert_eq!(
-                    doc.get_first(text_field).unwrap().as_text(),
-                    Some("a b c g")
-                );
+                let doc = searcher.doc::<TantivyDocument>(DocAddress::new(0, 4))?;
+                assert_eq!(doc.get_first(text_field).unwrap().as_str(), Some("a b c g"));
            }

            {
@@ -1300,10 +1350,10 @@ mod tests {
        let reader = index.reader().unwrap();
        let mut int_val = 0;
        {
-            let mut index_writer = index.writer_for_tests().unwrap();
+            let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
            let index_doc =
                |index_writer: &mut IndexWriter, doc_facets: &[&str], int_val: &mut u64| {
-                    let mut doc = Document::default();
+                    let mut doc = TantivyDocument::default();
                    for facet in doc_facets {
                        doc.add_facet(facet_field, Facet::from(facet));
                    }
@@ -1384,7 +1434,7 @@ mod tests {
            let segment_ids = index
                .searchable_segment_ids()
                .expect("Searchable segments failed.");
-            let mut index_writer = index.writer_for_tests().unwrap();
+            let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
            index_writer
                .merge(&segment_ids)
                .wait()
@@ -1406,7 +1456,7 @@ mod tests {

        // Deleting one term
        {
-            let mut index_writer = index.writer_for_tests().unwrap();
+            let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
            let facet = Facet::from_path(vec!["top", "a", "firstdoc"]);
            let facet_term = Term::from_facet(facet_field, &facet);
            index_writer.delete_term(facet_term);
@@ -1431,7 +1481,7 @@ mod tests {
        let mut schema_builder = schema::Schema::builder();
        let int_field = schema_builder.add_u64_field("intvals", INDEXED);
        let index = Index::create_in_ram(schema_builder.build());
-        let mut index_writer = index.writer_for_tests().unwrap();
+        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
        index_writer.add_document(doc!(int_field => 1u64))?;
        index_writer.commit().expect("commit failed");
        index_writer.add_document(doc!(int_field => 1u64))?;
@@ -1460,7 +1510,7 @@ mod tests {
        let reader = index.reader()?;
        {
            let mut index_writer = index.writer_for_tests()?;
-            let mut doc = Document::default();
+            let mut doc = TantivyDocument::default();
            doc.add_u64(int_field, 1);
            index_writer.add_document(doc.clone())?;
            index_writer.commit()?;
@@ -1503,7 +1553,7 @@ mod tests {
        {
            let mut index_writer = index.writer_for_tests()?;
            let index_doc = |index_writer: &mut IndexWriter, int_vals: &[u64]| {
-                let mut doc = Document::default();
+                let mut doc = TantivyDocument::default();
                for &val in int_vals {
                    doc.add_u64(int_field, val);
                }
@@ -1566,7 +1616,7 @@ mod tests {
        // Merging the segments
        {
            let segment_ids = index.searchable_segment_ids()?;
-            let mut index_writer = index.writer_for_tests()?;
+            let mut index_writer: IndexWriter = index.writer_for_tests()?;
            index_writer.merge(&segment_ids).wait()?;
            index_writer.wait_merging_threads()?;
        }
@@ -1613,7 +1663,7 @@ mod tests {
        writer.set_merge_policy(Box::new(policy));

        for i in 0..100 {
-            let mut doc = Document::new();
+            let mut doc = TantivyDocument::new();
            doc.add_f64(field, 42.0);
            doc.add_f64(multi_field, 0.24);
            doc.add_f64(multi_field, 0.27);
--- a/src/indexer/merger_sorted_index_test.rs
+++ b/src/indexer/merger_sorted_index_test.rs
@@ -4,11 +4,15 @@ mod tests {
    use crate::core::Index;
    use crate::fastfield::AliveBitSet;
    use crate::query::QueryParser;
+    use crate::schema::document::Value;
    use crate::schema::{
        self, BytesOptions, Facet, FacetOptions, IndexRecordOption, NumericOptions,
        TextFieldIndexing, TextOptions,
    };
-    use crate::{DocAddress, DocSet, IndexSettings, IndexSortByField, Order, Postings, Term};
+    use crate::{
+        DocAddress, DocSet, IndexSettings, IndexSortByField, IndexWriter, Order, Postings,
+        TantivyDocument, Term,
+    };

    fn create_test_index_posting_list_issue(index_settings: Option<IndexSettings>) -> Index {
        let mut schema_builder = schema::Schema::builder();
@@ -26,7 +30,7 @@ mod tests {
        let index = index_builder.create_in_ram().unwrap();

        {
-            let mut index_writer = index.writer_for_tests().unwrap();
+            let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
            index_writer
                .add_document(doc!(int_field=>3_u64, facet_field=> Facet::from("/crime")))
                .unwrap();
@@ -45,7 +49,7 @@ mod tests {
            let segment_ids = index
                .searchable_segment_ids()
                .expect("Searchable segments failed.");
-            let mut index_writer = index.writer_for_tests().unwrap();
+            let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
            assert!(index_writer.merge(&segment_ids).wait().is_ok());
            assert!(index_writer.wait_merging_threads().is_ok());
        }
@@ -133,7 +137,7 @@ mod tests {
        // Merging the segments
        {
            let segment_ids = index.searchable_segment_ids()?;
-            let mut index_writer = index.writer_for_tests()?;
+            let mut index_writer: IndexWriter = index.writer_for_tests()?;
            index_writer.merge(&segment_ids).wait()?;
            index_writer.wait_merging_threads()?;
        }
@@ -272,12 +276,16 @@ mod tests {
            } else {
                2
            };
-            let doc = searcher.doc(DocAddress::new(0, blubber_pos)).unwrap();
+            let doc = searcher
+                .doc::<TantivyDocument>(DocAddress::new(0, blubber_pos))
+                .unwrap();
            assert_eq!(
-                doc.get_first(my_text_field).unwrap().as_text(),
+                doc.get_first(my_text_field).unwrap().as_str(),
                Some("blubber")
            );
-            let doc = searcher.doc(DocAddress::new(0, 0)).unwrap();
+            let doc = searcher
+                .doc::<TantivyDocument>(DocAddress::new(0, 0))
+                .unwrap();
            assert_eq!(doc.get_first(int_field).unwrap().as_u64(), Some(1000));
        }
    }
@@ -494,7 +502,7 @@ mod bench_sorted_index_merge {
        let index = index_builder.create_in_ram().unwrap();

        {
-            let mut index_writer = index.writer_for_tests().unwrap();
+            let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
            let index_doc = |index_writer: &mut IndexWriter, val: u64| {
                index_writer.add_document(doc!(int_field=>val)).unwrap();
            };
@@ -535,7 +543,7 @@ mod bench_sorted_index_merge {
    //);
    //(doc_addr.doc_id, reader, u64_reader)
    //});
-    //// add values in order of the new doc_ids
+    /// add values in order of the new doc_ids
    // let mut val = 0;
    // for (doc_id, _reader, field_reader) in sorted_doc_ids {
    // val = field_reader.get_val(doc_id);
--- a/src/indexer/mod.rs
+++ b/src/indexer/mod.rs
@@ -1,23 +1,30 @@
-pub mod delete_queue;
+//! Indexing and merging data.
+//!
+//! Contains code to create and merge segments.
+//! `IndexWriter` is the main entry point for that, which created from
+//! [`Index::writer`](crate::Index::writer).

-pub mod doc_id_mapping;
+pub(crate) mod delete_queue;
+pub(crate) mod path_to_unordered_id;
+
+pub(crate) mod doc_id_mapping;
 mod doc_opstamp_mapping;
 mod flat_map_with_buffer;
-pub mod index_writer;
-mod index_writer_status;
+pub(crate) mod index_writer;
+pub(crate) mod index_writer_status;
 mod log_merge_policy;
 mod merge_operation;
-pub mod merge_policy;
-pub mod merger;
+pub(crate) mod merge_policy;
+pub(crate) mod merger;
 mod merger_sorted_index_test;
-pub mod operation;
-pub mod prepared_commit;
+pub(crate) mod operation;
+pub(crate) mod prepared_commit;
 mod segment_entry;
 mod segment_manager;
 mod segment_register;
-pub mod segment_serializer;
-pub mod segment_updater;
-mod segment_writer;
+pub(crate) mod segment_serializer;
+pub(crate) mod segment_updater;
+pub(crate) mod segment_writer;
 mod stamper;

 use crossbeam_channel as channel;
@@ -27,10 +34,10 @@ pub use self::index_writer::IndexWriter;
 pub use self::log_merge_policy::LogMergePolicy;
 pub use self::merge_operation::MergeOperation;
 pub use self::merge_policy::{MergeCandidate, MergePolicy, NoMergePolicy};
+pub use self::operation::UserOperation;
 pub use self::prepared_commit::PreparedCommit;
 pub use self::segment_entry::SegmentEntry;
-pub use self::segment_manager::SegmentManager;
-pub use self::segment_serializer::SegmentSerializer;
+pub(crate) use self::segment_serializer::SegmentSerializer;
 pub use self::segment_updater::{merge_filtered_segments, merge_indices};
 pub use self::segment_writer::SegmentWriter;
 use crate::indexer::operation::AddOperation;
@@ -44,25 +51,28 @@ pub type DefaultMergePolicy = LogMergePolicy;
 // - all docs in the operation will happen on the same segment and continuous doc_ids.
 // - all operations in the group are committed at the same time, making the group
 // atomic.
-type AddBatch = SmallVec<[AddOperation; 4]>;
-type AddBatchSender = channel::Sender<AddBatch>;
-type AddBatchReceiver = channel::Receiver<AddBatch>;
+type AddBatch<D> = SmallVec<[AddOperation<D>; 4]>;
+type AddBatchSender<D> = channel::Sender<AddBatch<D>>;
+type AddBatchReceiver<D> = channel::Receiver<AddBatch<D>>;

 #[cfg(feature = "mmap")]
 #[cfg(test)]
 mod tests_mmap {

-    use crate::collector::Count;
-    use crate::query::QueryParser;
-    use crate::schema::{JsonObjectOptions, Schema, TEXT};
-    use crate::{Index, Term};
+    use crate::aggregation::agg_req::Aggregations;
+    use crate::aggregation::agg_result::AggregationResults;
+    use crate::aggregation::AggregationCollector;
+    use crate::collector::{Count, TopDocs};
+    use crate::query::{AllQuery, QueryParser};
+    use crate::schema::{JsonObjectOptions, Schema, Type, FAST, INDEXED, STORED, TEXT};
+    use crate::{FieldMetadata, Index, IndexWriter, Term};

    #[test]
    fn test_advance_delete_bug() -> crate::Result<()> {
        let mut schema_builder = Schema::builder();
        let text_field = schema_builder.add_text_field("text", TEXT);
        let index = Index::create_from_tempdir(schema_builder.build())?;
-        let mut index_writer = index.writer_for_tests()?;
+        let mut index_writer: IndexWriter = index.writer_for_tests()?;
        // there must be one deleted document in the segment
        index_writer.add_document(doc!(text_field=>"b"))?;
        index_writer.delete_term(Term::from_field_text(text_field, "b"));
@@ -79,7 +89,7 @@ mod tests_mmap {
        let mut schema_builder = Schema::builder();
        let json_field = schema_builder.add_json_field("json", TEXT);
        let index = Index::create_in_ram(schema_builder.build());
-        let mut index_writer = index.writer_for_tests().unwrap();
+        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
        let json = serde_json::json!({"k8s.container.name": "prometheus", "val": "hello"});
        index_writer.add_document(doc!(json_field=>json)).unwrap();
        index_writer.commit().unwrap();
@@ -103,6 +113,35 @@ mod tests_mmap {
        }
    }

+    #[test]
+    fn test_json_field_number() {
+        // this test was added specifically to reach some cases related to using json fields, with
+        // frequency enabled, to store integers, with enough documents containing a single integer
+        // that the posting list can be bitpacked.
+        let mut schema_builder = Schema::builder();
+
+        let json_field = schema_builder.add_json_field("json", TEXT);
+        let index = Index::create_in_ram(schema_builder.build());
+        let mut index_writer = index.writer_for_tests().unwrap();
+        for _ in 0..256 {
+            let json = serde_json::json!({"somekey": 1u64, "otherkey": -2i64});
+            index_writer.add_document(doc!(json_field=>json)).unwrap();
+
+            let json = serde_json::json!({"somekey": "1str", "otherkey": "2str"});
+            index_writer.add_document(doc!(json_field=>json)).unwrap();
+        }
+        index_writer.commit().unwrap();
+        let reader = index.reader().unwrap();
+        let searcher = reader.searcher();
+        assert_eq!(searcher.num_docs(), 512);
+        let parse_query = QueryParser::for_index(&index, Vec::new());
+        {
+            let query = parse_query.parse_query(r"json.somekey:1").unwrap();
+            let num_docs = searcher.search(&query, &Count).unwrap();
+            assert_eq!(num_docs, 256);
+        }
+    }
+
    #[test]
    fn test_json_field_expand_dots_enabled_dot_escape_not_required() {
        let mut schema_builder = Schema::builder();
@@ -110,7 +149,7 @@ mod tests_mmap {
            JsonObjectOptions::from(TEXT).set_expand_dots_enabled();
        let json_field = schema_builder.add_json_field("json", json_options);
        let index = Index::create_in_ram(schema_builder.build());
-        let mut index_writer = index.writer_for_tests().unwrap();
+        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
        let json = serde_json::json!({"k8s.container.name": "prometheus", "val": "hello"});
        index_writer.add_document(doc!(json_field=>json)).unwrap();
        index_writer.commit().unwrap();
@@ -133,4 +172,275 @@ mod tests_mmap {
            assert_eq!(num_docs, 1);
        }
    }
+
+    #[test]
+    fn test_json_field_list_fields() {
+        let mut schema_builder = Schema::builder();
+        let json_options: JsonObjectOptions = JsonObjectOptions::from(TEXT);
+        let json_field = schema_builder.add_json_field("json", json_options);
+        let index = Index::create_in_ram(schema_builder.build());
+        let mut index_writer = index.writer_for_tests().unwrap();
+        let json = serde_json::json!({"k8s.container.name": "prometheus", "val": "hello", "sub": {"a": 1, "b": 2}});
+        index_writer.add_document(doc!(json_field=>json)).unwrap();
+        let json = serde_json::json!({"k8s.container.name": "prometheus", "val": "hello", "suber": {"a": 1, "b": 2}});
+        index_writer.add_document(doc!(json_field=>json)).unwrap();
+        let json = serde_json::json!({"k8s.container.name": "prometheus", "val": "hello", "suber": {"a": "mixed", "b": 2}});
+        index_writer.add_document(doc!(json_field=>json)).unwrap();
+        index_writer.commit().unwrap();
+        let reader = index.reader().unwrap();
+
+        let searcher = reader.searcher();
+        assert_eq!(searcher.num_docs(), 3);
+
+        let reader = &searcher.segment_readers()[0];
+        let inverted_index = reader.inverted_index(json_field).unwrap();
+        assert_eq!(
+            inverted_index.list_encoded_fields().unwrap(),
+            [
+                ("k8s.container.name".to_string(), Type::Str),
+                ("sub\u{1}a".to_string(), Type::I64),
+                ("sub\u{1}b".to_string(), Type::I64),
+                ("suber\u{1}a".to_string(), Type::I64),
+                ("suber\u{1}a".to_string(), Type::Str),
+                ("suber\u{1}b".to_string(), Type::I64),
+                ("val".to_string(), Type::Str),
+            ]
+        );
+    }
+
+    #[test]
+    fn test_json_fields_metadata_expanded_dots_one_segment() {
+        test_json_fields_metadata(true, true);
+    }
+    #[test]
+    fn test_json_fields_metadata_expanded_dots_multi_segment() {
+        test_json_fields_metadata(true, false);
+    }
+    #[test]
+    fn test_json_fields_metadata_no_expanded_dots_one_segment() {
+        test_json_fields_metadata(false, true);
+    }
+    #[test]
+    fn test_json_fields_metadata_no_expanded_dots_multi_segment() {
+        test_json_fields_metadata(false, false);
+    }
+
+    fn test_json_fields_metadata(expanded_dots: bool, one_segment: bool) {
+        use pretty_assertions::assert_eq;
+        let mut schema_builder = Schema::builder();
+        let json_options: JsonObjectOptions =
+            JsonObjectOptions::from(TEXT).set_fast(None).set_stored();
+        let json_options = if expanded_dots {
+            json_options.set_expand_dots_enabled()
+        } else {
+            json_options
+        };
+        schema_builder.add_json_field("json.confusing", json_options.clone());
+        let json_field = schema_builder.add_json_field("json.shadow", json_options.clone());
+        let json_field2 = schema_builder.add_json_field("json", json_options.clone());
+        schema_builder.add_json_field("empty_json", json_options);
+        let number_field = schema_builder.add_u64_field("numbers", FAST);
+        schema_builder.add_u64_field("empty", FAST | INDEXED | STORED);
+        let index = Index::create_in_ram(schema_builder.build());
+        let mut index_writer = index.writer_for_tests().unwrap();
+        let json =
+            serde_json::json!({"k8s.container.name": "a", "val": "a", "sub": {"a": 1, "b": 1}});
+        index_writer.add_document(doc!(json_field=>json)).unwrap();
+        let json =
+            serde_json::json!({"k8s.container.name": "a", "val": "a", "suber": {"a": 1, "b": 1}});
+        if !one_segment {
+            index_writer.commit().unwrap();
+        }
+        index_writer.add_document(doc!(json_field=>json)).unwrap();
+        let json = serde_json::json!({"k8s.container.name": "a", "k8s.container.name": "a", "val": "a", "suber": {"a": "a", "b": 1}});
+        index_writer
+            .add_document(doc!(number_field => 50u64, json_field=>json, json_field2=>json!({"shadow": {"val": "a"}})))
+            .unwrap();
+        index_writer.commit().unwrap();
+        let reader = index.reader().unwrap();
+
+        let searcher = reader.searcher();
+        assert_eq!(searcher.num_docs(), 3);
+
+        let fields_metadata = index.fields_metadata().unwrap();
+        assert_eq!(
+            fields_metadata,
+            [
+                FieldMetadata {
+                    field_name: "empty".to_string(),
+                    indexed: true,
+                    stored: true,
+                    fast: true,
+                    typ: Type::U64
+                },
+                FieldMetadata {
+                    field_name: if expanded_dots {
+                        "json.shadow.k8s.container.name".to_string()
+                    } else {
+                        "json.shadow.k8s\\.container\\.name".to_string()
+                    },
+                    indexed: true,
+                    stored: true,
+                    fast: true,
+                    typ: Type::Str
+                },
+                FieldMetadata {
+                    field_name: "json.shadow.sub.a".to_string(),
+                    indexed: true,
+                    stored: true,
+                    fast: true,
+                    typ: Type::I64
+                },
+                FieldMetadata {
+                    field_name: "json.shadow.sub.b".to_string(),
+                    indexed: true,
+                    stored: true,
+                    fast: true,
+                    typ: Type::I64
+                },
+                FieldMetadata {
+                    field_name: "json.shadow.suber.a".to_string(),
+                    indexed: true,
+                    stored: true,
+                    fast: true,
+                    typ: Type::I64
+                },
+                FieldMetadata {
+                    field_name: "json.shadow.suber.a".to_string(),
+                    indexed: true,
+                    stored: true,
+                    fast: true,
+                    typ: Type::Str
+                },
+                FieldMetadata {
+                    field_name: "json.shadow.suber.b".to_string(),
+                    indexed: true,
+                    stored: true,
+                    fast: true,
+                    typ: Type::I64
+                },
+                FieldMetadata {
+                    field_name: "json.shadow.val".to_string(),
+                    indexed: true,
+                    stored: true,
+                    fast: true,
+                    typ: Type::Str
+                },
+                FieldMetadata {
+                    field_name: "numbers".to_string(),
+                    indexed: false,
+                    stored: false,
+                    fast: true,
+                    typ: Type::U64
+                }
+            ]
+        );
+        let query_parser = QueryParser::for_index(&index, vec![]);
+        // Test if returned field name can be queried
+        for indexed_field in fields_metadata.iter().filter(|meta| meta.indexed) {
+            let val = if indexed_field.typ == Type::Str {
+                "a"
+            } else {
+                "1"
+            };
+            let query_str = &format!("{}:{}", indexed_field.field_name, val);
+            let query = query_parser.parse_query(query_str).unwrap();
+            let count_docs = searcher.search(&*query, &TopDocs::with_limit(2)).unwrap();
+            if indexed_field.field_name.contains("empty") || indexed_field.typ == Type::Json {
+                assert_eq!(count_docs.len(), 0);
+            } else {
+                assert!(!count_docs.is_empty(), "{}", indexed_field.field_name);
+            }
+        }
+        // Test if returned field name can be used for aggregation
+        for fast_field in fields_metadata.iter().filter(|meta| meta.fast) {
+            let agg_req_str = json!(
+            {
+              "termagg": {
+                "terms": {
+                  "field": fast_field.field_name,
+                }
+              }
+            });
+
+            let agg_req: Aggregations = serde_json::from_value(agg_req_str).unwrap();
+            let collector = AggregationCollector::from_aggs(agg_req, Default::default());
+            let agg_res: AggregationResults = searcher.search(&AllQuery, &collector).unwrap();
+            let res = serde_json::to_value(agg_res).unwrap();
+            if !fast_field.field_name.contains("empty") && fast_field.typ != Type::Json {
+                assert!(
+                    !res["termagg"]["buckets"].as_array().unwrap().is_empty(),
+                    "{}",
+                    fast_field.field_name
+                );
+            }
+        }
+    }
+
+    #[test]
+    fn test_json_field_shadowing_field_name_bug() {
+        /// This test is only there to display a bug on addressing a field if it gets shadowed
+        /// The issues only occurs if the field name that shadows contains a dot.
+        ///
+        /// Happens independently of the `expand_dots` option. Since that option does not
+        /// affect the field name itself.
+        use pretty_assertions::assert_eq;
+        let mut schema_builder = Schema::builder();
+        let json_options: JsonObjectOptions =
+            JsonObjectOptions::from(TEXT).set_fast(None).set_stored();
+        // let json_options = json_options.set_expand_dots_enabled();
+        let json_field_shadow = schema_builder.add_json_field("json.shadow", json_options.clone());
+        let json_field = schema_builder.add_json_field("json", json_options.clone());
+        let index = Index::create_in_ram(schema_builder.build());
+        let mut index_writer = index.writer_for_tests().unwrap();
+        index_writer
+            .add_document(
+                doc!(json_field_shadow=>json!({"val": "b"}), json_field=>json!({"shadow": {"val": "a"}})),
+            )
+            .unwrap();
+        index_writer.commit().unwrap();
+        let reader = index.reader().unwrap();
+
+        let searcher = reader.searcher();
+
+        let fields_and_vals = vec![
+            // Only way to address or it gets shadowed by `json.shadow` field
+            ("json.shadow\u{1}val".to_string(), "a"), // Succeeds
+            //("json.shadow.val".to_string(), "a"),   // Fails
+            ("json.shadow.val".to_string(), "b"), // Succeeds
+        ];
+
+        let query_parser = QueryParser::for_index(&index, vec![]);
+        // Test if field name can be queried
+        for (indexed_field, val) in fields_and_vals.iter() {
+            let query_str = &format!("{}:{}", indexed_field, val);
+            let query = query_parser.parse_query(query_str).unwrap();
+            let count_docs = searcher.search(&*query, &TopDocs::with_limit(2)).unwrap();
+            assert!(!count_docs.is_empty(), "{}:{}", indexed_field, val);
+        }
+        // Test if field name can be used for aggregation
+        for (field_name, val) in fields_and_vals.iter() {
+            let agg_req_str = json!(
+            {
+              "termagg": {
+                "terms": {
+                  "field": field_name,
+                }
+              }
+            });
+
+            let agg_req: Aggregations = serde_json::from_value(agg_req_str).unwrap();
+            let collector = AggregationCollector::from_aggs(agg_req, Default::default());
+            let agg_res: AggregationResults = searcher.search(&AllQuery, &collector).unwrap();
+            let res = serde_json::to_value(agg_res).unwrap();
+            assert_eq!(
+                res["termagg"]["buckets"].as_array().unwrap()[0]["key"]
+                    .as_str()
+                    .unwrap(),
+                *val,
+                "{}",
+                field_name
+            );
+        }
+    }
 }
--- a/src/indexer/operation.rs
+++ b/src/indexer/operation.rs
@@ -1,5 +1,6 @@
 use crate::query::Weight;
-use crate::schema::{Document, Term};
+use crate::schema::document::Document;
+use crate::schema::{TantivyDocument, Term};
 use crate::Opstamp;

 /// Timestamped Delete operation.
@@ -10,16 +11,16 @@ pub struct DeleteOperation {

 /// Timestamped Add operation.
 #[derive(Eq, PartialEq, Debug)]
-pub struct AddOperation {
+pub struct AddOperation<D: Document = TantivyDocument> {
    pub opstamp: Opstamp,
-    pub document: Document,
+    pub document: D,
 }

 /// UserOperation is an enum type that encapsulates other operation types.
 #[derive(Eq, PartialEq, Debug)]
-pub enum UserOperation {
+pub enum UserOperation<D: Document = TantivyDocument> {
    /// Add operation
-    Add(Document),
+    Add(D),
    /// Delete operation
    Delete(Term),
 }
--- a/src/indexer/path_to_unordered_id.rs
+++ b/src/indexer/path_to_unordered_id.rs
@@ -0,0 +1,104 @@
+use columnar::ColumnType;
+use common::TinySet;
+use fnv::FnvHashMap;
+
+/// `Field` is represented by an unsigned 32-bit integer type.
+/// The schema holds the mapping between field names and `Field` objects.
+#[derive(Copy, Default, Clone, Debug, PartialEq, PartialOrd, Eq, Ord, Hash)]
+pub struct OrderedPathId(u32);
+
+impl OrderedPathId {
+    /// Create a new field object for the given PathId.
+    pub const fn from_ordered_id(field_id: u32) -> OrderedPathId {
+        OrderedPathId(field_id)
+    }
+
+    /// Returns a u32 identifying uniquely a path within a schema.
+    pub const fn path_id(self) -> u32 {
+        self.0
+    }
+}
+impl From<u32> for OrderedPathId {
+    fn from(id: u32) -> Self {
+        Self(id)
+    }
+}
+
+#[derive(Default)]
+pub(crate) struct PathToUnorderedId {
+    /// TinySet contains the type codes of the columns in the path.
+    map: FnvHashMap<String, (u32, TinySet)>,
+}
+
+impl PathToUnorderedId {
+    #[inline]
+    pub(crate) fn get_or_allocate_unordered_id(&mut self, path: &str, typ: ColumnType) -> u32 {
+        let code_bit = typ.to_code();
+        if let Some((id, all_codes)) = self.map.get_mut(path) {
+            *all_codes = all_codes.insert(code_bit as u32);
+            return *id;
+        }
+        self.insert_new_path(path, code_bit)
+    }
+    #[cold]
+    fn insert_new_path(&mut self, path: &str, typ_code: u8) -> u32 {
+        let next_id = self.map.len() as u32;
+        self.map.insert(
+            path.to_string(),
+            (next_id, TinySet::singleton(typ_code as u32)),
+        );
+        next_id
+    }
+
+    /// Retuns ids which reflect the lexical order of the paths.
+    ///
+    /// The returned vec can be indexed with the unordered id to get the ordered id.
+    pub(crate) fn unordered_id_to_ordered_id(&self) -> Vec<(OrderedPathId, TinySet)> {
+        let mut sorted_ids: Vec<(&str, (u32, TinySet))> = self
+            .map
+            .iter()
+            .map(|(k, (id, typ_code))| (k.as_str(), (*id, *typ_code)))
+            .collect();
+        sorted_ids.sort_unstable_by_key(|(path, _)| *path);
+        let mut result = vec![(OrderedPathId::default(), TinySet::empty()); sorted_ids.len()];
+        for (ordered, (unordered, typ_code)) in sorted_ids.iter().map(|(_k, v)| v).enumerate() {
+            result[*unordered as usize] =
+                (OrderedPathId::from_ordered_id(ordered as u32), *typ_code);
+        }
+        result
+    }
+
+    /// Retuns the paths so they can be queried by the ordered id (which is the index).
+    pub(crate) fn ordered_id_to_path(&self) -> Vec<&str> {
+        let mut paths = self.map.keys().map(String::as_str).collect::<Vec<_>>();
+        paths.sort_unstable();
+        paths
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn path_to_unordered_test() {
+        let mut path_to_id = PathToUnorderedId::default();
+        let terms = vec!["b", "a", "b", "c"];
+        let ids = terms
+            .iter()
+            .map(|term| path_to_id.get_or_allocate_unordered_id(term, ColumnType::Str))
+            .collect::<Vec<u32>>();
+        assert_eq!(ids, vec![0, 1, 0, 2]);
+        let ordered_ids = ids
+            .iter()
+            .map(|id| path_to_id.unordered_id_to_ordered_id()[*id as usize].0)
+            .collect::<Vec<OrderedPathId>>();
+        assert_eq!(ordered_ids, vec![1.into(), 0.into(), 1.into(), 2.into()]);
+        // Fetch terms
+        let terms_fetched = ordered_ids
+            .iter()
+            .map(|id| path_to_id.ordered_id_to_path()[id.path_id() as usize])
+            .collect::<Vec<&str>>();
+        assert_eq!(terms_fetched, terms);
+    }
+}
--- a/src/indexer/prepared_commit.rs
+++ b/src/indexer/prepared_commit.rs
@@ -1,16 +1,17 @@
 use super::IndexWriter;
-use crate::{FutureResult, Opstamp};
+use crate::schema::document::Document;
+use crate::{FutureResult, Opstamp, TantivyDocument};

 /// A prepared commit
-pub struct PreparedCommit<'a> {
-    index_writer: &'a mut IndexWriter,
+pub struct PreparedCommit<'a, D: Document = TantivyDocument> {
+    index_writer: &'a mut IndexWriter<D>,
    payload: Option<String>,
    opstamp: Opstamp,
 }

-impl<'a> PreparedCommit<'a> {
-    pub(crate) fn new(index_writer: &'a mut IndexWriter, opstamp: Opstamp) -> PreparedCommit<'_> {
-        PreparedCommit {
+impl<'a, D: Document> PreparedCommit<'a, D> {
+    pub(crate) fn new(index_writer: &'a mut IndexWriter<D>, opstamp: Opstamp) -> Self {
+        Self {
            index_writer,
            payload: None,
            opstamp,
--- a/src/indexer/segment_serializer.rs
+++ b/src/indexer/segment_serializer.rs
@@ -12,6 +12,7 @@ pub struct SegmentSerializer {
    segment: Segment,
    pub(crate) store_writer: StoreWriter,
    fast_field_write: WritePtr,
+    field_list_write: WritePtr,
    fieldnorms_serializer: Option<FieldNormsSerializer>,
    postings_serializer: InvertedIndexSerializer,
 }
@@ -49,6 +50,7 @@ impl SegmentSerializer {
        };

        let fast_field_write = segment.open_write(SegmentComponent::FastFields)?;
+        let field_list_write = segment.open_write(SegmentComponent::FieldList)?;

        let fieldnorms_write = segment.open_write(SegmentComponent::FieldNorms)?;
        let fieldnorms_serializer = FieldNormsSerializer::from_write(fieldnorms_write)?;
@@ -58,6 +60,7 @@ impl SegmentSerializer {
            segment,
            store_writer,
            fast_field_write,
+            field_list_write,
            fieldnorms_serializer: Some(fieldnorms_serializer),
            postings_serializer,
        })
@@ -81,6 +84,11 @@ impl SegmentSerializer {
        &mut self.postings_serializer
    }

+    /// Accessor to the ``.
+    pub fn get_field_list_write(&mut self) -> &mut WritePtr {
+        &mut self.field_list_write
+    }
+
    /// Accessor to the `FastFieldSerializer`.
    pub fn get_fast_field_write(&mut self) -> &mut WritePtr {
        &mut self.fast_field_write
@@ -104,6 +112,7 @@ impl SegmentSerializer {
            fieldnorms_serializer.close()?;
        }
        self.fast_field_write.terminate()?;
+        self.field_list_write.terminate()?;
        self.postings_serializer.close()?;
        self.store_writer.close()?;
        Ok(())
--- a/src/indexer/segment_writer.rs
+++ b/src/indexer/segment_writer.rs
@@ -1,4 +1,5 @@
 use columnar::MonotonicallyMappableToU64;
+use common::JsonPathWriter;
 use itertools::Itertools;
 use tokenizer_api::BoxTokenStream;

@@ -7,16 +8,18 @@ use super::operation::AddOperation;
 use crate::core::json_utils::index_json_values;
 use crate::core::Segment;
 use crate::fastfield::FastFieldsWriter;
+use crate::field_list::serialize_segment_fields;
 use crate::fieldnorm::{FieldNormReaders, FieldNormsWriter};
 use crate::indexer::segment_serializer::SegmentSerializer;
 use crate::postings::{
    compute_table_memory_size, serialize_postings, IndexingContext, IndexingPosition,
    PerFieldPostingsWriter, PostingsWriter,
 };
-use crate::schema::{FieldEntry, FieldType, Schema, Term, Value, DATE_TIME_PRECISION_INDEXED};
+use crate::schema::document::{Document, ReferenceValue, Value};
+use crate::schema::{FieldEntry, FieldType, Schema, Term, DATE_TIME_PRECISION_INDEXED};
 use crate::store::{StoreReader, StoreWriter};
 use crate::tokenizer::{FacetTokenizer, PreTokenizedStream, TextAnalyzer, Tokenizer};
-use crate::{DocId, Document, Opstamp, SegmentComponent, TantivyError};
+use crate::{DocId, Opstamp, SegmentComponent, TantivyError};

 /// Computes the initial size of the hash table.
 ///
@@ -65,6 +68,7 @@ pub struct SegmentWriter {
    pub(crate) segment_serializer: SegmentSerializer,
    pub(crate) fast_field_writers: FastFieldsWriter,
    pub(crate) fieldnorms_writer: FieldNormsWriter,
+    pub(crate) json_path_writer: JsonPathWriter,
    pub(crate) doc_opstamps: Vec<Opstamp>,
    per_field_text_analyzers: Vec<TextAnalyzer>,
    term_buffer: Term,
@@ -81,10 +85,7 @@ impl SegmentWriter {
    /// the flushing behavior as a memory limit.
    /// - segment: The segment being written
    /// - schema
-    pub fn for_segment(
-        memory_budget_in_bytes: usize,
-        segment: Segment,
-    ) -> crate::Result<SegmentWriter> {
+    pub fn for_segment(memory_budget_in_bytes: usize, segment: Segment) -> crate::Result<Self> {
        let schema = segment.schema();
        let tokenizer_manager = segment.index().tokenizers().clone();
        let tokenizer_manager_fast_field = segment.index().fast_field_tokenizer().clone();
@@ -113,11 +114,12 @@ impl SegmentWriter {
                })
            })
            .collect::<Result<Vec<_>, _>>()?;
-        Ok(SegmentWriter {
+        Ok(Self {
            max_doc: 0,
            ctx: IndexingContext::new(table_size),
            per_field_postings_writers,
            fieldnorms_writer: FieldNormsWriter::for_schema(&schema),
+            json_path_writer: JsonPathWriter::default(),
            segment_serializer,
            fast_field_writers: FastFieldsWriter::from_schema_and_tokenizer_manager(
                &schema,
@@ -146,6 +148,7 @@ impl SegmentWriter {
            .map(|sort_by_field| get_doc_id_mapping_from_field(sort_by_field, &self))
            .transpose()?;
        remap_and_write(
+            self.schema,
            &self.per_field_postings_writers,
            self.ctx,
            self.fast_field_writers,
@@ -157,6 +160,8 @@ impl SegmentWriter {
        Ok(doc_opstamps)
    }

+    /// Returns an estimation of the current memory usage of the segment writer.
+    /// If the mem usage exceeds the `memory_budget`, the segment be serialized.
    pub fn mem_usage(&self) -> usize {
        self.ctx.mem_usage()
            + self.fieldnorms_writer.mem_usage()
@@ -164,18 +169,21 @@ impl SegmentWriter {
            + self.segment_serializer.mem_usage()
    }

-    fn index_document(&mut self, doc: &Document) -> crate::Result<()> {
+    fn index_document<D: Document>(&mut self, doc: &D) -> crate::Result<()> {
        let doc_id = self.max_doc;
+
+        // TODO: Can this be optimised a bit?
        let vals_grouped_by_field = doc
-            .field_values()
-            .iter()
-            .sorted_by_key(|el| el.field())
-            .group_by(|el| el.field());
+            .iter_fields_and_values()
+            .sorted_by_key(|(field, _)| *field)
+            .group_by(|(field, _)| *field);
+
        for (field, field_values) in &vals_grouped_by_field {
-            let values = field_values.map(|field_value| field_value.value());
+            let values = field_values.map(|el| el.1);
+
            let field_entry = self.schema.get_field_entry(field);
            let make_schema_error = || {
-                crate::TantivyError::SchemaError(format!(
+                TantivyError::SchemaError(format!(
                    "Expected a {:?} for field {:?}",
                    field_entry.field_type().value_type(),
                    field_entry.name()
@@ -193,7 +201,10 @@ impl SegmentWriter {
            match field_entry.field_type() {
                FieldType::Facet(_) => {
                    let mut facet_tokenizer = FacetTokenizer::default(); // this can be global
-                    for value in values {
+                    for value_access in values {
+                        // Used to help with linting and type checking.
+                        let value = value_access as D::Value<'_>;
+
                        let facet = value.as_facet().ok_or_else(make_schema_error)?;
                        let facet_str = facet.encoded_str();
                        let mut facet_tokenizer = facet_tokenizer.token_stream(facet_str);
@@ -209,19 +220,18 @@ impl SegmentWriter {
                }
                FieldType::Str(_) => {
                    let mut indexing_position = IndexingPosition::default();
-                    for value in values {
-                        let mut token_stream = match value {
-                            Value::PreTokStr(tok_str) => {
-                                BoxTokenStream::new(PreTokenizedStream::from(tok_str.clone()))
-                            }
-                            Value::Str(ref text) => {
-                                let text_analyzer =
-                                    &mut self.per_field_text_analyzers[field.field_id() as usize];
-                                text_analyzer.token_stream(text)
-                            }
-                            _ => {
-                                continue;
-                            }
+                    for value_access in values {
+                        // Used to help with linting and type checking.
+                        let value = value_access as D::Value<'_>;
+
+                        let mut token_stream = if let Some(text) = value.as_str() {
+                            let text_analyzer =
+                                &mut self.per_field_text_analyzers[field.field_id() as usize];
+                            text_analyzer.token_stream(text)
+                        } else if let Some(tok_str) = value.as_pre_tokenized_text() {
+                            BoxTokenStream::new(PreTokenizedStream::from(tok_str.clone()))
+                        } else {
+                            continue;
                        };

                        assert!(term_buffer.is_empty());
@@ -240,7 +250,10 @@ impl SegmentWriter {
                }
                FieldType::U64(_) => {
                    let mut num_vals = 0;
-                    for value in values {
+                    for value_access in values {
+                        // Used to help with linting and type checking.
+                        let value = value_access as D::Value<'_>;
+
                        num_vals += 1;
                        let u64_val = value.as_u64().ok_or_else(make_schema_error)?;
                        term_buffer.set_u64(u64_val);
@@ -252,9 +265,13 @@ impl SegmentWriter {
                }
                FieldType::Date(_) => {
                    let mut num_vals = 0;
-                    for value in values {
+                    for value_access in values {
+                        // Used to help with linting and type checking.
+                        let value_access = value_access as D::Value<'_>;
+                        let value = value_access.as_value();
+
                        num_vals += 1;
-                        let date_val = value.as_date().ok_or_else(make_schema_error)?;
+                        let date_val = value.as_datetime().ok_or_else(make_schema_error)?;
                        term_buffer
                            .set_u64(date_val.truncate(DATE_TIME_PRECISION_INDEXED).to_u64());
                        postings_writer.subscribe(doc_id, 0u32, term_buffer, ctx);
@@ -265,7 +282,10 @@ impl SegmentWriter {
                }
                FieldType::I64(_) => {
                    let mut num_vals = 0;
-                    for value in values {
+                    for value_access in values {
+                        // Used to help with linting and type checking.
+                        let value = value_access as D::Value<'_>;
+
                        num_vals += 1;
                        let i64_val = value.as_i64().ok_or_else(make_schema_error)?;
                        term_buffer.set_i64(i64_val);
@@ -277,7 +297,10 @@ impl SegmentWriter {
                }
                FieldType::F64(_) => {
                    let mut num_vals = 0;
-                    for value in values {
+                    for value_access in values {
+                        // Used to help with linting and type checking.
+                        let value = value_access as D::Value<'_>;
+
                        num_vals += 1;
                        let f64_val = value.as_f64().ok_or_else(make_schema_error)?;
                        term_buffer.set_f64(f64_val);
@@ -289,7 +312,10 @@ impl SegmentWriter {
                }
                FieldType::Bool(_) => {
                    let mut num_vals = 0;
-                    for value in values {
+                    for value_access in values {
+                        // Used to help with linting and type checking.
+                        let value = value_access as D::Value<'_>;
+
                        num_vals += 1;
                        let bool_val = value.as_bool().ok_or_else(make_schema_error)?;
                        term_buffer.set_bool(bool_val);
@@ -301,7 +327,10 @@ impl SegmentWriter {
                }
                FieldType::Bytes(_) => {
                    let mut num_vals = 0;
-                    for value in values {
+                    for value_access in values {
+                        // Used to help with linting and type checking.
+                        let value = value_access as D::Value<'_>;
+
                        num_vals += 1;
                        let bytes = value.as_bytes().ok_or_else(make_schema_error)?;
                        term_buffer.set_bytes(bytes);
@@ -314,21 +343,33 @@ impl SegmentWriter {
                FieldType::JsonObject(json_options) => {
                    let text_analyzer =
                        &mut self.per_field_text_analyzers[field.field_id() as usize];
-                    let json_values_it =
-                        values.map(|value| value.as_json().ok_or_else(make_schema_error));
-                    index_json_values(
+                    let json_values_it = values.map(|value_access| {
+                        // Used to help with linting and type checking.
+                        let value_access = value_access as D::Value<'_>;
+                        let value = value_access.as_value();
+
+                        match value {
+                            ReferenceValue::Object(object_iter) => Ok(object_iter),
+                            _ => Err(make_schema_error()),
+                        }
+                    });
+                    index_json_values::<D::Value<'_>>(
                        doc_id,
                        json_values_it,
                        text_analyzer,
                        json_options.is_expand_dots_enabled(),
                        term_buffer,
                        postings_writer,
+                        &mut self.json_path_writer,
                        ctx,
                    )?;
                }
                FieldType::IpAddr(_) => {
                    let mut num_vals = 0;
-                    for value in values {
+                    for value_access in values {
+                        // Used to help with linting and type checking.
+                        let value = value_access as D::Value<'_>;
+
                        num_vals += 1;
                        let ip_addr = value.as_ip_addr().ok_or_else(make_schema_error)?;
                        term_buffer.set_ip_addr(ip_addr);
@@ -346,7 +387,10 @@ impl SegmentWriter {
    /// Indexes a new document
    ///
    /// As a user, you should rather use `IndexWriter`'s add_document.
-    pub fn add_document(&mut self, add_operation: AddOperation) -> crate::Result<()> {
+    pub fn add_document<D: Document>(
+        &mut self,
+        add_operation: AddOperation<D>,
+    ) -> crate::Result<()> {
        let AddOperation { document, opstamp } = add_operation;
        self.doc_opstamps.push(opstamp);
        self.fast_field_writers.add_document(&document)?;
@@ -384,6 +428,7 @@ impl SegmentWriter {
 ///
 /// `doc_id_map` is used to map to the new doc_id order.
 fn remap_and_write(
+    schema: Schema,
    per_field_postings_writers: &PerFieldPostingsWriter,
    ctx: IndexingContext,
    fast_field_writers: FastFieldsWriter,
@@ -399,15 +444,29 @@ fn remap_and_write(
        .segment()
        .open_read(SegmentComponent::FieldNorms)?;
    let fieldnorm_readers = FieldNormReaders::open(fieldnorm_data)?;
+
+    let unordered_id_to_ordered_id = ctx.path_to_unordered_id.unordered_id_to_ordered_id();
+
    serialize_postings(
-        ctx,
+        &ctx,
+        schema.clone(),
        per_field_postings_writers,
        fieldnorm_readers,
        doc_id_map,
+        &unordered_id_to_ordered_id,
        serializer.get_postings_serializer(),
    )?;
    debug!("fastfield-serialize");
-    fast_field_writers.serialize(serializer.get_fast_field_write(), doc_id_map)?;
+    let columns = fast_field_writers.serialize(serializer.get_fast_field_write(), doc_id_map)?;
+
+    let field_list_serializer = serializer.get_field_list_write();
+    serialize_segment_fields(
+        ctx,
+        field_list_serializer,
+        &schema,
+        &unordered_id_to_ordered_id,
+        columns,
+    )?;

    // finalize temp docstore and create version, which reflects the doc_id_map
    if let Some(doc_id_map) = doc_id_map {
@@ -445,32 +504,37 @@ fn remap_and_write(

 #[cfg(test)]
 mod tests {
+    use std::collections::BTreeMap;
    use std::path::{Path, PathBuf};

    use tempfile::TempDir;

-    use super::compute_initial_table_size;
-    use crate::collector::Count;
+    use crate::collector::{Count, TopDocs};
    use crate::core::json_utils::JsonTermWriter;
    use crate::directory::RamDirectory;
    use crate::postings::TermInfo;
-    use crate::query::PhraseQuery;
+    use crate::query::{PhraseQuery, QueryParser};
+    use crate::schema::document::Value;
    use crate::schema::{
-        IndexRecordOption, Schema, TextFieldIndexing, TextOptions, Type, STORED, STRING, TEXT,
+        Document, IndexRecordOption, Schema, TextFieldIndexing, TextOptions, Type, STORED, STRING,
+        TEXT,
    };
    use crate::store::{Compressor, StoreReader, StoreWriter};
    use crate::time::format_description::well_known::Rfc3339;
    use crate::time::OffsetDateTime;
    use crate::tokenizer::{PreTokenizedString, Token};
    use crate::{
-        DateTime, Directory, DocAddress, DocSet, Document, Index, Postings, Term, TERMINATED,
+        DateTime, Directory, DocAddress, DocSet, Index, IndexWriter, Postings, TantivyDocument,
+        Term, TERMINATED,
    };

    #[test]
+    #[cfg(not(feature = "compare_hash_only"))]
    fn test_hashmap_size() {
-        assert_eq!(compute_initial_table_size(100_000).unwrap(), 1 << 11);
-        assert_eq!(compute_initial_table_size(1_000_000).unwrap(), 1 << 14);
-        assert_eq!(compute_initial_table_size(15_000_000).unwrap(), 1 << 18);
+        use super::compute_initial_table_size;
+        assert_eq!(compute_initial_table_size(100_000).unwrap(), 1 << 12);
+        assert_eq!(compute_initial_table_size(1_000_000).unwrap(), 1 << 15);
+        assert_eq!(compute_initial_table_size(15_000_000).unwrap(), 1 << 19);
        assert_eq!(compute_initial_table_size(1_000_000_000).unwrap(), 1 << 19);
        assert_eq!(compute_initial_table_size(4_000_000_000).unwrap(), 1 << 19);
    }
@@ -480,7 +544,7 @@ mod tests {
        let mut schema_builder = Schema::builder();
        let text_field = schema_builder.add_text_field("title", TEXT | STORED);
        let schema = schema_builder.build();
-        let mut doc = Document::default();
+        let mut doc = TantivyDocument::default();
        let pre_tokenized_text = PreTokenizedString {
            text: String::from("A"),
            tokens: vec![Token {
@@ -504,11 +568,48 @@ mod tests {
        store_writer.close().unwrap();

        let reader = StoreReader::open(directory.open_read(path).unwrap(), 0).unwrap();
-        let doc = reader.get(0).unwrap();
+        let doc = reader.get::<TantivyDocument>(0).unwrap();

        assert_eq!(doc.field_values().len(), 2);
-        assert_eq!(doc.field_values()[0].value().as_text(), Some("A"));
-        assert_eq!(doc.field_values()[1].value().as_text(), Some("title"));
+        assert_eq!(doc.field_values()[0].value().as_str(), Some("A"));
+        assert_eq!(doc.field_values()[1].value().as_str(), Some("title"));
+    }
+    #[test]
+    fn test_simple_json_indexing() {
+        let mut schema_builder = Schema::builder();
+        let json_field = schema_builder.add_json_field("json", STORED | STRING);
+        let schema = schema_builder.build();
+        let index = Index::create_in_ram(schema.clone());
+        let mut writer = index.writer_for_tests().unwrap();
+        writer
+            .add_document(doc!(json_field=>json!({"my_field": "b"})))
+            .unwrap();
+        writer
+            .add_document(doc!(json_field=>json!({"my_field": "a"})))
+            .unwrap();
+        writer
+            .add_document(doc!(json_field=>json!({"my_field": "b"})))
+            .unwrap();
+        writer.commit().unwrap();
+
+        let query_parser = QueryParser::for_index(&index, vec![json_field]);
+        let text_query = query_parser.parse_query("my_field:a").unwrap();
+        let score_docs: Vec<(_, DocAddress)> = index
+            .reader()
+            .unwrap()
+            .searcher()
+            .search(&text_query, &TopDocs::with_limit(4))
+            .unwrap();
+        assert_eq!(score_docs.len(), 1);
+
+        let text_query = query_parser.parse_query("my_field:b").unwrap();
+        let score_docs: Vec<(_, DocAddress)> = index
+            .reader()
+            .unwrap()
+            .searcher()
+            .search(&text_query, &TopDocs::with_limit(4))
+            .unwrap();
+        assert_eq!(score_docs.len(), 2);
    }

    #[test]
@@ -539,13 +640,13 @@ mod tests {
        let reader = index.reader().unwrap();
        let searcher = reader.searcher();
        let doc = searcher
-            .doc(DocAddress {
+            .doc::<TantivyDocument>(DocAddress {
                segment_ord: 0u32,
                doc_id: 0u32,
            })
            .unwrap();
        let serdeser_json_val = serde_json::from_str::<serde_json::Map<String, serde_json::Value>>(
-            &schema.to_json(&doc),
+            &doc.to_json(&schema),
        )
        .unwrap()
        .get("json")
@@ -675,10 +776,10 @@ mod tests {
        let mut schema_builder = Schema::builder();
        let json_field = schema_builder.add_json_field("json", STORED | TEXT);
        let schema = schema_builder.build();
-        let mut doc = Document::default();
-        let json_val: serde_json::Map<String, serde_json::Value> =
+        let mut doc = TantivyDocument::default();
+        let json_val: BTreeMap<String, crate::schema::OwnedValue> =
            serde_json::from_str(r#"{"mykey": "repeated token token"}"#).unwrap();
-        doc.add_json_object(json_field, json_val);
+        doc.add_object(json_field, json_val);
        let index = Index::create_in_ram(schema);
        let mut writer = index.writer_for_tests().unwrap();
        writer.add_document(doc).unwrap();
@@ -802,11 +903,10 @@ mod tests {
        let mut schema_builder = Schema::builder();
        let text = schema_builder.add_text_field("text", TEXT);
        let schema = schema_builder.build();
-        let doc = schema
-            .parse_document(r#"{"text": [ "bbb", "aaa", "", "aaa"]}"#)
+        let doc = TantivyDocument::parse_json(&schema, r#"{"text": [ "bbb", "aaa", "", "aaa"]}"#)
            .unwrap();
        let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_for_tests().unwrap();
+        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
        index_writer.add_document(doc).unwrap();
        // On debug this did panic on the underflow
        index_writer.commit().unwrap();
@@ -831,7 +931,7 @@ mod tests {
        let mut schema_builder = Schema::builder();
        let text = schema_builder.add_text_field("text", TEXT);
        let schema = schema_builder.build();
-        let mut doc = Document::default();
+        let mut doc = TantivyDocument::default();
        // This is a bit of a contrived example.
        let tokens = PreTokenizedString {
            text: "roller-coaster".to_string(),
@@ -846,7 +946,7 @@ mod tests {
        doc.add_pre_tokenized_text(text, tokens.clone());
        doc.add_pre_tokenized_text(text, tokens);
        let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_for_tests().unwrap();
+        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
        index_writer.add_document(doc).unwrap();
        index_writer.commit().unwrap();
        let reader = index.reader().unwrap();
@@ -869,7 +969,7 @@ mod tests {
        let mut schema_builder = Schema::builder();
        let text = schema_builder.add_text_field("text", TEXT);
        let schema = schema_builder.build();
-        let mut doc = Document::default();
+        let mut doc = TantivyDocument::default();
        // This is a bit of a contrived example.
        let tokens = PreTokenizedString {
            text: "contrived-example".to_string(), //< I can't think of a use case where this corner case happens in real life.
@@ -894,7 +994,7 @@ mod tests {
        doc.add_pre_tokenized_text(text, tokens);
        doc.add_text(text, "hello");
        let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_for_tests().unwrap();
+        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
        index_writer.add_document(doc).unwrap();
        index_writer.commit().unwrap();
        let reader = index.reader().unwrap();
@@ -930,7 +1030,7 @@ mod tests {
        let schema = index.schema();
        let mut index_writer = index.writer(50_000_000).unwrap();
        let title = schema.get_field("title").unwrap();
-        let mut document = Document::default();
+        let mut document = TantivyDocument::default();
        document.add_text(title, "The Old Man and the Sea");
        index_writer.add_document(document).unwrap();
        let error = index_writer.commit().unwrap_err();
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -21,7 +21,7 @@
 //! # use tantivy::collector::TopDocs;
 //! # use tantivy::query::QueryParser;
 //! # use tantivy::schema::*;
-//! # use tantivy::{doc, DocAddress, Index, Score};
+//! # use tantivy::{doc, DocAddress, Index, IndexWriter, Score};
 //! #
 //! # fn main() {
 //! #     // Let's create a temporary directory for the
@@ -53,7 +53,7 @@
 //!
 //! // Here we use a buffer of 100MB that will be split
 //! // between indexing threads.
-//! let mut index_writer = index.writer(100_000_000)?;
+//! let mut index_writer: IndexWriter = index.writer(100_000_000)?;
 //!
 //! // Let's index one documents!
 //! index_writer.add_document(doc!(
@@ -89,8 +89,8 @@
 //!
 //! for (_score, doc_address) in top_docs {
 //!     // Retrieve the actual content of documents given its `doc_address`.
-//!     let retrieved_doc = searcher.doc(doc_address)?;
-//!     println!("{}", schema.to_json(&retrieved_doc));
+//!     let retrieved_doc = searcher.doc::<TantivyDocument>(doc_address)?;
+//!     println!("{}", retrieved_doc.to_json(&schema));
 //! }
 //!
 //! # Ok(())
@@ -103,7 +103,48 @@
 //! the example code (
 //! [literate programming](https://tantivy-search.github.io/examples/basic_search.html) /
 //! [source code](https://github.com/quickwit-oss/tantivy/blob/main/examples/basic_search.rs))
-
+//!
+//! # Tantivy Architecture Overview
+//!
+//! Tantivy is inspired by Lucene, the Architecture is very similar.
+//!
+//! ## Core Concepts
+//!
+//! - **[Index]**: A collection of segments. The top level entry point for tantivy users to search
+//!   and index data.
+//!
+//! - **[Segment]**: At the heart of Tantivy's indexing structure is the [Segment]. It contains
+//!   documents and indices and is the atomic unit of indexing and search.
+//!
+//! - **[Schema](schema)**: A schema is a set of fields in an index. Each field has a specific data
+//!   type and set of attributes.
+//!
+//! - **[IndexWriter]**: Responsible creating and merging segments. It executes the indexing
+//!   pipeline including tokenization, creating indices, and storing the index in the
+//!   [Directory](directory).
+//!
+//! - **Searching**: [Searcher] searches the segments with anything that implements
+//!   [Query](query::Query) and merges the results. The list of [supported
+//! queries](query::Query#implementors). Custom Queries are supported by implementing the
+//! [Query](query::Query) trait.
+//!
+//! - **[Directory](directory)**: Abstraction over the storage where the index data is stored.
+//!
+//! - **[Tokenizer](tokenizer)**: Breaks down text into individual tokens. Users can implement or
+//!   use provided tokenizers.
+//!
+//! ## Architecture Flow
+//!
+//! 1. **Document Addition**: Users create documents according to the defined schema. The documents
+//!    fields are tokenized, processed, and added to the current segment. See
+//!    [Document](schema::document) for the structure and usage.
+//!
+//! 2. **Segment Creation**: Once the memory limit threshold is reached or a commit is called, the
+//!    segment is written to the Directory. Documents are searchable after `commit`.
+//!
+//! 3. **Merging**: To optimize space and search speed, segments might be merged. This operation is
+//!    performed in the background. Customize the merge behaviour via
+//!    [IndexWriter::set_merge_policy].
 #[cfg_attr(test, macro_use)]
 extern crate serde_json;
 #[macro_use]
@@ -137,7 +178,7 @@ pub use crate::future_result::FutureResult;
 pub type Result<T> = std::result::Result<T, TantivyError>;

 mod core;
-mod indexer;
+pub mod indexer;

 #[allow(unused_doc_comments)]
 pub mod error;
@@ -147,6 +188,7 @@ pub mod aggregation;
 pub mod collector;
 pub mod directory;
 pub mod fastfield;
+pub mod field_list;
 pub mod fieldnorm;
 pub mod positions;
 pub mod postings;
@@ -161,8 +203,7 @@ pub mod termdict;
 mod reader;

 pub use self::reader::{IndexReader, IndexReaderBuilder, ReloadPolicy, Warmer};
-mod snippet;
-pub use self::snippet::{Snippet, SnippetGenerator};
+pub mod snippet;

 mod docset;
 use std::fmt;
@@ -173,26 +214,36 @@ use once_cell::sync::Lazy;
 use serde::{Deserialize, Serialize};

 pub use self::docset::{DocSet, TERMINATED};
+#[deprecated(
+    since = "0.22.0",
+    note = "Will be removed in tantivy 0.23. Use export from snippet module instead"
+)]
+pub use self::snippet::{Snippet, SnippetGenerator};
 #[doc(hidden)]
 pub use crate::core::json_utils;
 pub use crate::core::{
-    Executor, Index, IndexBuilder, IndexMeta, IndexSettings, IndexSortByField, InvertedIndexReader,
-    Order, Searcher, SearcherGeneration, Segment, SegmentComponent, SegmentId, SegmentMeta,
-    SegmentReader, SingleSegmentIndexWriter,
+    merge_field_meta_data, Executor, FieldMetadata, Index, IndexBuilder, IndexMeta, IndexSettings,
+    IndexSortByField, InvertedIndexReader, Order, Searcher, SearcherGeneration, Segment,
+    SegmentComponent, SegmentId, SegmentMeta, SegmentReader, SingleSegmentIndexWriter,
 };
 pub use crate::directory::Directory;
-pub use crate::indexer::operation::UserOperation;
-pub use crate::indexer::{merge_filtered_segments, merge_indices, IndexWriter, PreparedCommit};
+pub use crate::indexer::IndexWriter;
+#[deprecated(
+    since = "0.22.0",
+    note = "Will be removed in tantivy 0.23. Use export from indexer module instead"
+)]
+pub use crate::indexer::{merge_filtered_segments, merge_indices, PreparedCommit};
 pub use crate::postings::Postings;
 #[allow(deprecated)]
 pub use crate::schema::DatePrecision;
-pub use crate::schema::{DateOptions, DateTimePrecision, Document, Term};
+pub use crate::schema::{DateOptions, DateTimePrecision, Document, TantivyDocument, Term};

 /// Index format version.
-const INDEX_FORMAT_VERSION: u32 = 5;
-
-#[cfg(all(feature = "mmap", unix))]
-pub use memmap2::Advice;
+///
+/// Version 7: Add `.fieldlist` file containing the list of fields in a segment.
+const INDEX_FORMAT_VERSION: u32 = 7;
+/// Oldest index format version this tantivy version can read.
+const INDEX_FORMAT_OLDEST_SUPPORTED_VERSION: u32 = 4;

 /// Structure version for the index.
 #[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
@@ -342,8 +393,9 @@ pub mod tests {
    use crate::docset::{DocSet, TERMINATED};
    use crate::merge_policy::NoMergePolicy;
    use crate::query::BooleanQuery;
+    use crate::schema::document::Value;
    use crate::schema::*;
-    use crate::{DateTime, DocAddress, Index, Postings, ReloadPolicy};
+    use crate::{DateTime, DocAddress, Index, IndexWriter, Postings, ReloadPolicy};

    pub fn fixed_size_test<O: BinarySerializable + FixedSize + Default>() {
        let mut buffer = Vec::new();
@@ -414,7 +466,7 @@ pub mod tests {
        let schema = schema_builder.build();
        let index = Index::create_from_tempdir(schema)?;
        // writing the segment
-        let mut index_writer = index.writer_for_tests()?;
+        let mut index_writer: IndexWriter = index.writer_for_tests()?;
        {
            let doc = doc!(text_field=>"af b");
            index_writer.add_document(doc)?;
@@ -436,7 +488,7 @@ pub mod tests {
        let mut schema_builder = Schema::builder();
        let text_field = schema_builder.add_text_field("text", TEXT);
        let index = Index::create_in_ram(schema_builder.build());
-        let mut index_writer = index.writer_for_tests()?;
+        let mut index_writer: IndexWriter = index.writer_for_tests()?;
        index_writer.add_document(doc!(text_field=>"a b c"))?;
        index_writer.commit()?;
        index_writer.add_document(doc!(text_field=>"a"))?;
@@ -463,7 +515,7 @@ pub mod tests {
        let title_field = schema_builder.add_text_field("title", TEXT);
        let text_field = schema_builder.add_text_field("text", TEXT);
        let index = Index::create_in_ram(schema_builder.build());
-        let mut index_writer = index.writer_for_tests()?;
+        let mut index_writer: IndexWriter = index.writer_for_tests()?;
        index_writer.add_document(doc!(text_field=>"a b c"))?;
        index_writer.commit()?;
        let index_reader = index.reader()?;
@@ -485,7 +537,7 @@ pub mod tests {
        let mut schema_builder = Schema::builder();
        let text_field = schema_builder.add_text_field("text", TEXT);
        let index = Index::create_in_ram(schema_builder.build());
-        let mut index_writer = index.writer_for_tests()?;
+        let mut index_writer: IndexWriter = index.writer_for_tests()?;
        index_writer.add_document(doc!(text_field=>"a b c"))?;
        index_writer.add_document(doc!())?;
        index_writer.add_document(doc!(text_field=>"a b"))?;
@@ -528,7 +580,7 @@ pub mod tests {
            .unwrap();
        {
            // writing the segment
-            let mut index_writer = index.writer_for_tests()?;
+            let mut index_writer: IndexWriter = index.writer_for_tests()?;
            // 0
            index_writer.add_document(doc!(text_field=>"a b"))?;
            // 1
@@ -575,7 +627,7 @@ pub mod tests {
        }
        {
            // writing the segment
-            let mut index_writer = index.writer_for_tests()?;
+            let mut index_writer: IndexWriter = index.writer_for_tests()?;
            // 0
            index_writer.add_document(doc!(text_field=>"a b"))?;
            // 1
@@ -612,7 +664,7 @@ pub mod tests {
        }
        {
            // writing the segment
-            let mut index_writer = index.writer_for_tests()?;
+            let mut index_writer: IndexWriter = index.writer_for_tests()?;
            index_writer.add_document(doc!(text_field=>"a b"))?;
            index_writer.delete_term(Term::from_field_text(text_field, "c"));
            index_writer.rollback()?;
@@ -662,7 +714,7 @@ pub mod tests {
        let schema = schema_builder.build();

        let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_for_tests()?;
+        let mut index_writer: IndexWriter = index.writer_for_tests()?;
        index_writer.add_document(doc!(field=>1u64))?;
        index_writer.commit()?;
        let reader = index.reader()?;
@@ -685,7 +737,7 @@ pub mod tests {
        let schema = schema_builder.build();

        let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_for_tests()?;
+        let mut index_writer: IndexWriter = index.writer_for_tests()?;
        let negative_val = -1i64;
        index_writer.add_document(doc!(value_field => negative_val))?;
        index_writer.commit()?;
@@ -709,7 +761,7 @@ pub mod tests {
        let schema = schema_builder.build();

        let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_for_tests()?;
+        let mut index_writer: IndexWriter = index.writer_for_tests()?;
        let val = std::f64::consts::PI;
        index_writer.add_document(doc!(value_field => val))?;
        index_writer.commit()?;
@@ -733,7 +785,7 @@ pub mod tests {
        let absent_field = schema_builder.add_text_field("absent_text", TEXT);
        let schema = schema_builder.build();
        let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_for_tests()?;
+        let mut index_writer: IndexWriter = index.writer_for_tests()?;
        index_writer.add_document(doc!(text_field=>"a"))?;
        assert!(index_writer.commit().is_ok());
        let reader = index.reader()?;
@@ -756,7 +808,7 @@ pub mod tests {
            .try_into()?;

        // writing the segment
-        let mut index_writer = index.writer_for_tests()?;
+        let mut index_writer: IndexWriter = index.writer_for_tests()?;
        index_writer.add_document(doc!(text_field=>"63"))?;
        index_writer.add_document(doc!(text_field=>"70"))?;
        index_writer.add_document(doc!(text_field=>"34"))?;
@@ -781,7 +833,7 @@ pub mod tests {
        let index = Index::create_in_ram(schema);
        {
            // writing the segment
-            let mut index_writer = index.writer_for_tests()?;
+            let mut index_writer: IndexWriter = index.writer_for_tests()?;
            index_writer.add_document(doc!(text_field=>"af af af bc bc"))?;
            index_writer.commit()?;
        }
@@ -813,7 +865,7 @@ pub mod tests {
        let index = Index::create_in_ram(schema);
        let reader = index.reader()?;
        // writing the segment
-        let mut index_writer = index.writer_for_tests()?;
+        let mut index_writer: IndexWriter = index.writer_for_tests()?;
        index_writer.add_document(doc!(text_field=>"af af af b"))?;
        index_writer.add_document(doc!(text_field=>"a b c"))?;
        index_writer.add_document(doc!(text_field=>"a b c d"))?;
@@ -877,7 +929,7 @@ pub mod tests {
            .try_into()?;
        assert_eq!(reader.searcher().num_docs(), 0u64);
        // writing the segment
-        let mut index_writer = index.writer_for_tests()?;
+        let mut index_writer: IndexWriter = index.writer_for_tests()?;
        index_writer.add_document(doc!(text_field=>"af b"))?;
        index_writer.add_document(doc!(text_field=>"a b c"))?;
        index_writer.add_document(doc!(text_field=>"a b c d"))?;
@@ -985,13 +1037,13 @@ pub mod tests {
                            text_field => "some other value",
                            other_text_field => "short");
        assert_eq!(document.len(), 3);
-        let values: Vec<&Value> = document.get_all(text_field).collect();
+        let values: Vec<&OwnedValue> = document.get_all(text_field).collect();
        assert_eq!(values.len(), 2);
-        assert_eq!(values[0].as_text(), Some("tantivy"));
-        assert_eq!(values[1].as_text(), Some("some other value"));
-        let values: Vec<&Value> = document.get_all(other_text_field).collect();
+        assert_eq!(values[0].as_str(), Some("tantivy"));
+        assert_eq!(values[1].as_str(), Some("some other value"));
+        let values: Vec<&OwnedValue> = document.get_all(other_text_field).collect();
        assert_eq!(values.len(), 1);
-        assert_eq!(values[0].as_text(), Some("short"));
+        assert_eq!(values[0].as_str(), Some("short"));
    }

    #[test]
@@ -1005,7 +1057,7 @@ pub mod tests {
        let schema = schema_builder.build();

        let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_for_tests()?;
+        let mut index_writer: IndexWriter = index.writer_for_tests()?;
        {
            let document =
                doc!(fast_field_unsigned => 4u64, fast_field_signed=>4i64, fast_field_float=>4f64);
@@ -1071,7 +1123,7 @@ pub mod tests {
        let index = Index::create_in_ram(schema);
        let index_reader = index.reader()?;

-        let mut index_writer = index.writer_for_tests()?;
+        let mut index_writer: IndexWriter = index.writer_for_tests()?;
        index_writer.set_merge_policy(Box::new(NoMergePolicy));

        for doc_id in 0u64..DOC_COUNT {
@@ -1124,7 +1176,7 @@ pub mod tests {
        let body = builder.add_text_field("body", TEXT | STORED);
        let schema = builder.build();
        let index = Index::create_in_dir(&index_path, schema)?;
-        let mut writer = index.writer(50_000_000)?;
+        let mut writer: IndexWriter = index.writer(50_000_000)?;
        writer.set_merge_policy(Box::new(NoMergePolicy));
        for _ in 0..5000 {
            writer.add_document(doc!(body => "foo"))?;
--- a/src/macros.rs
+++ b/src/macros.rs
@@ -45,12 +45,12 @@
 macro_rules! doc(
    () => {
        {
-            ($crate::Document::default())
+            ($crate::TantivyDocument::default())
        }
    }; // avoids a warning due to the useless `mut`.
    ($($field:expr => $value:expr),*) => {
        {
-            let mut document = $crate::Document::default();
+            let mut document = $crate::TantivyDocument::default();
            $(
                document.add_field_value($field, $value);
            )*
--- a/src/positions/reader.rs
+++ b/src/positions/reader.rs
@@ -92,7 +92,7 @@ impl PositionReader {
            // that block is bitpacked.
            let bit_width = bit_widths[block_rel_id];
            self.block_decoder
-                .uncompress_block_unsorted(compressed_data, bit_width);
+                .uncompress_block_unsorted(compressed_data, bit_width, false);
        } else {
            // that block is vint encoded.
            self.block_decoder
--- a/src/positions/serializer.rs
+++ b/src/positions/serializer.rs
@@ -62,8 +62,9 @@ impl<W: io::Write> PositionSerializer<W> {
            return;
        }
        if self.block.len() == COMPRESSION_BLOCK_SIZE {
-            let (bit_width, block_encoded): (u8, &[u8]) =
-                self.block_encoder.compress_block_unsorted(&self.block[..]);
+            let (bit_width, block_encoded): (u8, &[u8]) = self
+                .block_encoder
+                .compress_block_unsorted(&self.block[..], false);
            self.bit_widths.push(bit_width);
            self.positions_buffer.extend(block_encoded);
        } else {
--- a/src/postings/block_segment_postings.rs
+++ b/src/postings/block_segment_postings.rs
@@ -24,13 +24,13 @@ fn max_score<I: Iterator<Item = Score>>(mut it: I) -> Option<Score> {
 #[derive(Clone)]
 pub struct BlockSegmentPostings {
    pub(crate) doc_decoder: BlockDecoder,
-    loaded_offset: usize,
+    block_loaded: bool,
    freq_decoder: BlockDecoder,
    freq_reading_option: FreqReadingOption,
    block_max_score_cache: Option<Score>,
    doc_freq: u32,
    data: OwnedBytes,
-    pub(crate) skip_reader: SkipReader,
+    skip_reader: SkipReader,
 }

 fn decode_bitpacked_block(
@@ -40,10 +40,16 @@ fn decode_bitpacked_block(
    doc_offset: DocId,
    doc_num_bits: u8,
    tf_num_bits: u8,
+    strict_delta: bool,
 ) {
-    let num_consumed_bytes = doc_decoder.uncompress_block_sorted(data, doc_offset, doc_num_bits);
+    let num_consumed_bytes =
+        doc_decoder.uncompress_block_sorted(data, doc_offset, doc_num_bits, strict_delta);
    if let Some(freq_decoder) = freq_decoder_opt {
-        freq_decoder.uncompress_block_unsorted(&data[num_consumed_bytes..], tf_num_bits);
+        freq_decoder.uncompress_block_unsorted(
+            &data[num_consumed_bytes..],
+            tf_num_bits,
+            strict_delta,
+        );
    }
 }

@@ -57,11 +63,15 @@ fn decode_vint_block(
    let num_consumed_bytes =
        doc_decoder.uncompress_vint_sorted(data, doc_offset, num_vint_docs, TERMINATED);
    if let Some(freq_decoder) = freq_decoder_opt {
-        freq_decoder.uncompress_vint_unsorted(
-            &data[num_consumed_bytes..],
-            num_vint_docs,
-            TERMINATED,
-        );
+        // if it's a json term with freq, containing less than 256 docs, we can reach here thinking
+        // we have a freq, despite not really having one.
+        if data.len() > num_consumed_bytes {
+            freq_decoder.uncompress_vint_unsorted(
+                &data[num_consumed_bytes..],
+                num_vint_docs,
+                TERMINATED,
+            );
+        }
    }
 }

@@ -78,28 +88,46 @@ fn split_into_skips_and_postings(
 }

 impl BlockSegmentPostings {
+    /// Opens a `BlockSegmentPostings`.
+    /// `doc_freq` is the number of documents in the posting list.
+    /// `record_option` represents the amount of data available according to the schema.
+    /// `requested_option` is the amount of data requested by the user.
+    /// If for instance, we do not request for term frequencies, this function will not decompress
+    /// term frequency blocks.
    pub(crate) fn open(
        doc_freq: u32,
        data: FileSlice,
-        record_option: IndexRecordOption,
+        mut record_option: IndexRecordOption,
        requested_option: IndexRecordOption,
    ) -> io::Result<BlockSegmentPostings> {
+        let bytes = data.read_bytes()?;
+        let (skip_data_opt, postings_data) = split_into_skips_and_postings(doc_freq, bytes)?;
+        let skip_reader = match skip_data_opt {
+            Some(skip_data) => {
+                let block_count = doc_freq as usize / COMPRESSION_BLOCK_SIZE;
+                // 8 is the minimum size of a block with frequency (can be more if pos are stored
+                // too)
+                if skip_data.len() < 8 * block_count {
+                    // the field might be encoded with frequency, but this term in particular isn't.
+                    // This can happen for JSON field with term frequencies:
+                    // - text terms are encoded with term freqs.
+                    // - numerical terms are encoded without term freqs.
+                    record_option = IndexRecordOption::Basic;
+                }
+                SkipReader::new(skip_data, doc_freq, record_option)
+            }
+            None => SkipReader::new(OwnedBytes::empty(), doc_freq, record_option),
+        };
+
        let freq_reading_option = match (record_option, requested_option) {
            (IndexRecordOption::Basic, _) => FreqReadingOption::NoFreq,
            (_, IndexRecordOption::Basic) => FreqReadingOption::SkipFreq,
            (_, _) => FreqReadingOption::ReadFreq,
        };

-        let bytes = data.read_bytes()?;
-        let (skip_data_opt, postings_data) = split_into_skips_and_postings(doc_freq, bytes)?;
-        let skip_reader = match skip_data_opt {
-            Some(skip_data) => SkipReader::new(skip_data, doc_freq, record_option),
-            None => SkipReader::new(OwnedBytes::empty(), doc_freq, record_option),
-        };
-
        let mut block_segment_postings = BlockSegmentPostings {
            doc_decoder: BlockDecoder::with_val(TERMINATED),
-            loaded_offset: usize::MAX,
+            block_loaded: false,
            freq_decoder: BlockDecoder::with_val(1),
            freq_reading_option,
            block_max_score_cache: None,
@@ -169,7 +197,7 @@ impl BlockSegmentPostings {
            split_into_skips_and_postings(doc_freq, postings_data)?;
        self.data = postings_data;
        self.block_max_score_cache = None;
-        self.loaded_offset = usize::MAX;
+        self.block_loaded = false;
        if let Some(skip_data) = skip_data_opt {
            self.skip_reader.reset(skip_data, doc_freq);
        } else {
@@ -265,22 +293,23 @@ impl BlockSegmentPostings {
    pub(crate) fn shallow_seek(&mut self, target_doc: DocId) {
        if self.skip_reader.seek(target_doc) {
            self.block_max_score_cache = None;
+            self.block_loaded = false;
        }
    }

    pub(crate) fn block_is_loaded(&self) -> bool {
-        self.loaded_offset == self.skip_reader.byte_offset()
+        self.block_loaded
    }

    pub(crate) fn load_block(&mut self) {
        let offset = self.skip_reader.byte_offset();
-        if self.loaded_offset == offset {
+        if self.block_is_loaded() {
            return;
        }
-        self.loaded_offset = offset;
        match self.skip_reader.block_info() {
            BlockInfo::BitPacked {
                doc_num_bits,
+                strict_delta_encoded,
                tf_num_bits,
                ..
            } => {
@@ -295,6 +324,7 @@ impl BlockSegmentPostings {
                    self.skip_reader.last_doc_in_previous_block,
                    doc_num_bits,
                    tf_num_bits,
+                    strict_delta_encoded,
                );
            }
            BlockInfo::VInt { num_docs } => {
@@ -318,13 +348,13 @@ impl BlockSegmentPostings {
                );
            }
        }
+        self.block_loaded = true;
    }

    /// Advance to the next block.
-    ///
-    /// Returns false if and only if there is no remaining block.
    pub fn advance(&mut self) {
        self.skip_reader.advance();
+        self.block_loaded = false;
        self.block_max_score_cache = None;
        self.load_block();
    }
@@ -333,7 +363,7 @@ impl BlockSegmentPostings {
    pub fn empty() -> BlockSegmentPostings {
        BlockSegmentPostings {
            doc_decoder: BlockDecoder::with_val(TERMINATED),
-            loaded_offset: 0,
+            block_loaded: true,
            freq_decoder: BlockDecoder::with_val(1),
            freq_reading_option: FreqReadingOption::NoFreq,
            block_max_score_cache: None,
@@ -342,6 +372,10 @@ impl BlockSegmentPostings {
            skip_reader: SkipReader::new(OwnedBytes::empty(), 0, IndexRecordOption::Basic),
        }
    }
+
+    pub(crate) fn skip_reader(&self) -> &SkipReader {
+        &self.skip_reader
+    }
 }

 #[cfg(test)]
--- a/src/postings/compression/mod.rs
+++ b/src/postings/compression/mod.rs
@@ -33,14 +33,40 @@ impl BlockEncoder {
    }

    pub fn compress_block_sorted(&mut self, block: &[u32], offset: u32) -> (u8, &[u8]) {
-        let num_bits = self.bitpacker.num_bits_sorted(offset, block);
+        // if offset is zero, convert it to None. This is correct as long as we do the same when
+        // decompressing. It's required in case the block starts with an actual zero.
+        let offset = if offset == 0u32 { None } else { Some(offset) };
+
+        let num_bits = self.bitpacker.num_bits_strictly_sorted(offset, block);
        let written_size =
            self.bitpacker
-                .compress_sorted(offset, block, &mut self.output[..], num_bits);
+                .compress_strictly_sorted(offset, block, &mut self.output[..], num_bits);
        (num_bits, &self.output[..written_size])
    }

-    pub fn compress_block_unsorted(&mut self, block: &[u32]) -> (u8, &[u8]) {
+    /// Compress a single block of unsorted numbers.
+    ///
+    /// If `minus_one_encoded` is set, each value must be >= 1, and will be encoded in a sligly
+    /// more compact format. This is useful for some values where 0 isn't a correct value, such
+    /// as term frequency, but isn't correct for some usages like position lists, where 0 can
+    /// appear.
+    pub fn compress_block_unsorted(
+        &mut self,
+        block: &[u32],
+        minus_one_encoded: bool,
+    ) -> (u8, &[u8]) {
+        debug_assert!(!minus_one_encoded || !block.contains(&0));
+
+        let mut block_minus_one = [0; COMPRESSION_BLOCK_SIZE];
+        let block = if minus_one_encoded {
+            for (elem_min_one, elem) in block_minus_one.iter_mut().zip(block) {
+                *elem_min_one = elem - 1;
+            }
+            &block_minus_one
+        } else {
+            block
+        };
+
        let num_bits = self.bitpacker.num_bits(block);
        let written_size = self
            .bitpacker
@@ -71,21 +97,55 @@ impl BlockDecoder {
        }
    }

+    /// Decompress block of sorted integers.
+    ///
+    /// `strict_delta` depends on what encoding was used. Older version of tantivy never use strict
+    /// deltas, newer versions always use them.
    pub fn uncompress_block_sorted(
        &mut self,
        compressed_data: &[u8],
        offset: u32,
        num_bits: u8,
+        strict_delta: bool,
    ) -> usize {
-        self.output_len = COMPRESSION_BLOCK_SIZE;
-        self.bitpacker
-            .decompress_sorted(offset, compressed_data, &mut self.output, num_bits)
+        if strict_delta {
+            let offset = std::num::NonZeroU32::new(offset).map(std::num::NonZeroU32::get);
+
+            self.output_len = COMPRESSION_BLOCK_SIZE;
+            self.bitpacker.decompress_strictly_sorted(
+                offset,
+                compressed_data,
+                &mut self.output,
+                num_bits,
+            )
+        } else {
+            self.output_len = COMPRESSION_BLOCK_SIZE;
+            self.bitpacker
+                .decompress_sorted(offset, compressed_data, &mut self.output, num_bits)
+        }
    }

-    pub fn uncompress_block_unsorted(&mut self, compressed_data: &[u8], num_bits: u8) -> usize {
+    /// Decompress block of unsorted integers.
+    ///
+    /// `minus_one_encoded` depends on what encoding was used. Older version of tantivy never use
+    /// that encoding. Newer version use it for some structures, but not all. See the corresponding
+    /// call to `BlockEncoder::compress_block_unsorted`.
+    pub fn uncompress_block_unsorted(
+        &mut self,
+        compressed_data: &[u8],
+        num_bits: u8,
+        minus_one_encoded: bool,
+    ) -> usize {
        self.output_len = COMPRESSION_BLOCK_SIZE;
-        self.bitpacker
-            .decompress(compressed_data, &mut self.output, num_bits)
+        let res = self
+            .bitpacker
+            .decompress(compressed_data, &mut self.output, num_bits);
+        if minus_one_encoded {
+            for val in &mut self.output {
+                *val += 1;
+            }
+        }
+        res
    }

    #[inline]
@@ -218,7 +278,8 @@ pub mod tests {
        let (num_bits, compressed_data) = encoder.compress_block_sorted(&vals, 0);
        let mut decoder = BlockDecoder::default();
        {
-            let consumed_num_bytes = decoder.uncompress_block_sorted(compressed_data, 0, num_bits);
+            let consumed_num_bytes =
+                decoder.uncompress_block_sorted(compressed_data, 0, num_bits, true);
            assert_eq!(consumed_num_bytes, compressed_data.len());
        }
        for i in 0..128 {
@@ -233,7 +294,8 @@ pub mod tests {
        let (num_bits, compressed_data) = encoder.compress_block_sorted(&vals, 10);
        let mut decoder = BlockDecoder::default();
        {
-            let consumed_num_bytes = decoder.uncompress_block_sorted(compressed_data, 10, num_bits);
+            let consumed_num_bytes =
+                decoder.uncompress_block_sorted(compressed_data, 10, num_bits, true);
            assert_eq!(consumed_num_bytes, compressed_data.len());
        }
        for i in 0..128 {
@@ -252,7 +314,8 @@ pub mod tests {
        compressed.push(173u8);
        let mut decoder = BlockDecoder::default();
        {
-            let consumed_num_bytes = decoder.uncompress_block_sorted(&compressed, 10, num_bits);
+            let consumed_num_bytes =
+                decoder.uncompress_block_sorted(&compressed, 10, num_bits, true);
            assert_eq!(consumed_num_bytes, compressed.len() - 1);
            assert_eq!(compressed[consumed_num_bytes], 173u8);
        }
@@ -263,21 +326,25 @@ pub mod tests {

    #[test]
    fn test_encode_unsorted_block_with_junk() {
-        let mut compressed: Vec<u8> = Vec::new();
-        let n = 128;
-        let vals: Vec<u32> = (0..n).map(|i| 11u32 + (i as u32) * 7u32 % 12).collect();
-        let mut encoder = BlockEncoder::default();
-        let (num_bits, compressed_data) = encoder.compress_block_unsorted(&vals);
-        compressed.extend_from_slice(compressed_data);
-        compressed.push(173u8);
-        let mut decoder = BlockDecoder::default();
-        {
-            let consumed_num_bytes = decoder.uncompress_block_unsorted(&compressed, num_bits);
-            assert_eq!(consumed_num_bytes + 1, compressed.len());
-            assert_eq!(compressed[consumed_num_bytes], 173u8);
-        }
-        for i in 0..n {
-            assert_eq!(vals[i], decoder.output(i));
+        for minus_one_encode in [false, true] {
+            let mut compressed: Vec<u8> = Vec::new();
+            let n = 128;
+            let vals: Vec<u32> = (0..n).map(|i| 11u32 + (i as u32) * 7u32 % 12).collect();
+            let mut encoder = BlockEncoder::default();
+            let (num_bits, compressed_data) =
+                encoder.compress_block_unsorted(&vals, minus_one_encode);
+            compressed.extend_from_slice(compressed_data);
+            compressed.push(173u8);
+            let mut decoder = BlockDecoder::default();
+            {
+                let consumed_num_bytes =
+                    decoder.uncompress_block_unsorted(&compressed, num_bits, minus_one_encode);
+                assert_eq!(consumed_num_bytes + 1, compressed.len());
+                assert_eq!(compressed[consumed_num_bytes], 173u8);
+            }
+            for i in 0..n {
+                assert_eq!(vals[i], decoder.output(i));
+            }
        }
    }

@@ -344,7 +411,7 @@ mod bench {
        let (num_bits, compressed) = encoder.compress_block_sorted(&data, 0u32);
        let mut decoder = BlockDecoder::default();
        b.iter(|| {
-            decoder.uncompress_block_sorted(compressed, 0u32, num_bits);
+            decoder.uncompress_block_sorted(compressed, 0u32, num_bits, true);
        });
    }

--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Pascal Seitz	bb57e63522	Store List of Fields in Segment Fiels may be encoded in the columnar storage or in the inverted index for JSON fields. Add a new Segment file that contains the list of fields (schema + encoded)	2023-12-13 15:52:41 +08:00
PSeitz	bff7c58497	improve indexing benchmark (#2275 )	2023-12-11 09:04:42 +01:00
trinity-1686a	9ebc5ed053	use fst for sstable index (#2268 ) * read path for new fst based index * implement BlockAddrStoreWriter * extract slop/derivation computation * use better linear approximator and allow negative correction to approximator * document format and reorder some fields * optimize single block sstable size * plug backward compat	2023-12-04 15:13:15 +01:00
PSeitz	0b56c88e69	Revert "Preparing for 0.21.2 release." (#2258 ) * Revert "Preparing for 0.21.2 release. (#2256)" This reverts commit `9caab45136`. * bump version to 0.21.1 * set version to 0.22.0-dev	2023-12-01 13:46:12 +01:00
PSeitz	24841f0b2a	update bitpacker dep (#2269 )	2023-12-01 13:45:52 +01:00
PSeitz	1a9fc10be9	add fields_metadata to SegmentReader, add columnar docs (#2222 ) * add fields_metadata to SegmentReader, add columnar docs * use schema to resolve field, add test * normalize paths * merge for FieldsMetadata, add fields_metadata on Index * Update src/core/segment_reader.rs Co-authored-by: Paul Masurel <paul@quickwit.io> * merge code paths * add Hash * move function oustide --------- Co-authored-by: Paul Masurel <paul@quickwit.io>	2023-11-22 12:29:53 +01:00
PSeitz	07573a7f19	update fst (#2267 ) update fst to 0.5 (deduplicates regex-syntax in the dep tree) deps cleanup	2023-11-21 16:06:57 +01:00
BlackHoleFox	daad2dc151	Take string references instead of owned values building Facet paths (#2265 )	2023-11-20 09:40:44 +01:00
PSeitz	054f49dc31	support escaped dot, add agg test (#2250 ) add agg test for nested JSON allow escaping of dot	2023-11-20 03:00:57 +01:00
PSeitz	47009ed2d3	remove unused deps (#2264 ) found with cargo machete remove pprof (doesn't work)	2023-11-20 02:59:59 +01:00
PSeitz	0aae31d7d7	reduce number of allocations (#2257 ) * reduce number of allocations Explanation makes up around 50% of all allocations (numbers not perf). It's created during serialization but not called. - Make Explanation optional in BM25 - Avoid allocations when using Explanation * use Cow	2023-11-16 13:47:36 +01:00
Paul Masurel	9caab45136	Preparing for 0.21.2 release. (#2256 )	2023-11-15 10:43:36 +09:00
Chris Tam	6d9a7b7eb0	Derive Debug for SchemaBuilder (#2254 )	2023-11-15 01:03:44 +01:00
dependabot[bot]	7a2c5804b1	Update itertools requirement from 0.11.0 to 0.12.0 (#2255 ) Updates the requirements on [itertools](https://github.com/rust-itertools/itertools) to permit the latest version. - [Changelog](https://github.com/rust-itertools/itertools/blob/master/CHANGELOG.md) - [Commits](https://github.com/rust-itertools/itertools/compare/v0.11.0...v0.12.0) --- updated-dependencies: - dependency-name: itertools dependency-type: direct:production ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>	2023-11-15 01:03:08 +01:00
François Massot	5319977171	Merge pull request #2253 from quickwit-oss/issue/2251-bug-merge-json-object-with-number Fix bug occuring when merging JSON object indexed with positions.	2023-11-14 17:28:29 +01:00
trinity-1686a	828632e8c4	rustfmt	2023-11-14 15:05:16 +01:00
Paul Masurel	6b59ec6fd5	Fix bug occuring when merging JSON object indexed with positions. In JSON Object field the presence of term frequencies depend on the field. Typically, a string with postiions indexed will have positions while numbers won't. The presence or absence of term freqs for a given term is unfortunately encoded in a very passive way. It is given by the presence of extra information in the skip info, or the lack of term freqs after decoding vint blocks. Before, after writing a segment, we would encode the segment correctly (without any term freq for number in json object field). However during merge, we would get the default term freq=1 value. (this is default in the absence of encoded term freqs) The merger would then proceed and attempt to decode 1 position when there are in fact none. This PR requires to explictly tell the posting serialize whether term frequencies should be serialized for each new term. Closes #2251	2023-11-14 22:41:48 +09:00
PSeitz	b60d862150	docid deltas while indexing (#2249 ) * docid deltas while indexing storing deltas is especially helpful for repetitive data like logs. In those cases, recording a doc on a term costed 4 bytes instead of 1 byte now. HDFS Indexing 1.1GB Total memory consumption: Before: 760 MB Now: 590 MB * use scan for delta decoding	2023-11-13 05:14:27 +01:00
PSeitz	4837c7811a	add missing inlines (#2245 )	2023-11-10 08:00:42 +01:00
PSeitz	5a2397d57e	add sstable ord_to_term benchmark (#2242 )	2023-11-10 07:27:48 +01:00
PSeitz	927b4432c9	Perf: use term hashmap in fastfield (#2243 ) * add shared arena hashmap * bench fastfield indexing * use shared arena hashmap in columnar lower minimum resize in hashtable * clippy * add comments	2023-11-09 13:44:02 +01:00
trinity-1686a	7a0064db1f	bump index version (#2237 ) * bump index version and add constant for lowest supported version * use range instead of handcoded bounds	2023-11-06 19:02:37 +01:00
PSeitz	2e7327205d	fix coverage run (#2232 ) coverage run uses the compare_hash_only feature which is not compativle with the test_hashmap_size test	2023-11-06 11:18:38 +00:00
Paul Masurel	7bc5bf78e2	Fixing functional tests. (#2239 )	2023-11-05 18:18:39 +09:00
giovannicuccu	ef603c8c7e	rename ReloadPolicy onCommit to onCommitWithDelay (#2235 ) * rename ReloadPolicy onCommit to onCommitWithDelay * fix format issues --------- Co-authored-by: Giovanni Cuccu <gcuccu@imolainformatica.it>	2023-11-03 12:22:10 +01:00
PSeitz	28dd6b6546	collect json paths in indexing (#2231 ) * collect json paths in indexing * remove unsafe iter_mut_keys	2023-11-01 11:25:17 +01:00
trinity-1686a	1dda2bb537	handle * inside term in query parser (#2228 )	2023-10-27 08:57:02 +02:00
PSeitz	bf6544cf28	fix mmap::Advice reexport (#2230 )	2023-10-27 14:09:25 +09:00
PSeitz	ccecf946f7	tantivy 0.21.1 (#2227 )	2023-10-27 05:01:44 +02:00
PSeitz	19a859d6fd	term hashmap remove copy in is_empty, unused unordered_id (#2229 )	2023-10-27 05:01:32 +02:00
PSeitz	83af14caa4	Fix range query (#2226 ) Fix range query end check in advance Rename vars to reduce ambiguity add tests Fixes #2225	2023-10-25 09:17:31 +02:00
PSeitz	4feeb2323d	fix clippy (#2223 )	2023-10-24 10:05:22 +02:00
PSeitz	07bf66a197	json path writer (#2224 ) * refactor logic to JsonPathWriter * use in encode_column_name * add inlines * move unsafe block	2023-10-24 09:45:50 +02:00
trinity-1686a	0d4589219b	encode some part of posting list as -1 instead of direct values (#2185 ) * add support for delta-1 encoding posting list * encode term frequency minus one * don't emit tf for json integer terms * make skipreader not pub(crate) mutable	2023-10-20 16:58:26 +02:00
PSeitz	c2b0469180	improve docs, rework exports (#2220 ) * rework exports move snippet and advice make indexer pub, remove indexer reexports * add deprecation warning * add architecture overview	2023-10-18 09:22:24 +02:00
PSeitz	7e1980b218	run coverage only after merge (#2212 ) * run coverage only after merge coverage is a quite slow step in CI. It can be run only after merging * Apply suggestions from code review Co-authored-by: Paul Masurel <paul@quickwit.io> --------- Co-authored-by: Paul Masurel <paul@quickwit.io>	2023-10-18 07:19:36 +02:00
PSeitz	ecb9a89a9f	add compat mode for JSON (#2219 )	2023-10-17 10:00:55 +02:00
PSeitz	5e06e504e6	split into ReferenceValueLeaf (#2217 )	2023-10-16 16:31:30 +02:00
PSeitz	182f58cea6	remove Document: DocumentDeserialize dependency (#2211 ) * remove Document: DocumentDeserialize dependency The dependency requires users to implement an API they may not use. * remove unnecessary Document bounds	2023-10-13 07:59:54 +02:00
dependabot[bot]	337ffadefd	Update lru requirement from 0.11.0 to 0.12.0 (#2208 ) Updates the requirements on [lru](https://github.com/jeromefroe/lru-rs) to permit the latest version. - [Changelog](https://github.com/jeromefroe/lru-rs/blob/master/CHANGELOG.md) - [Commits](https://github.com/jeromefroe/lru-rs/compare/0.11.0...0.12.0) --- updated-dependencies: - dependency-name: lru dependency-type: direct:production ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>	2023-10-12 12:09:56 +02:00
dependabot[bot]	22aa4daf19	Update zstd requirement from 0.12 to 0.13 (#2214 ) Updates the requirements on [zstd](https://github.com/gyscos/zstd-rs) to permit the latest version. - [Release notes](https://github.com/gyscos/zstd-rs/releases) - [Commits](https://github.com/gyscos/zstd-rs/compare/v0.12.0...v0.13.0) --- updated-dependencies: - dependency-name: zstd dependency-type: direct:production ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>	2023-10-12 04:24:44 +02:00
PSeitz	493f9b2f2a	Read list of JSON fields encoded in dictionary (#2184 ) * Read list of JSON fields encoded in dictionary add method to get list of fields on InvertedIndexReader * add field type	2023-10-09 12:06:22 +02:00
PSeitz	e246e5765d	replace ReferenceValue with Self in Value (#2210 )	2023-10-06 08:22:15 +02:00
PSeitz	6097235eff	fix numeric order, refactor Document (#2209 ) fix numeric order to prefer i64 rename and move Document stuff	2023-10-05 16:39:56 +02:00
PSeitz	b700c42246	add AsRef, expose object and array iter on Value (#2207 ) add AsRef expose object and array iter add to_json on Document	2023-10-05 03:55:35 +02:00
PSeitz	5b1bf1a993	replace Field with field name (#2196 )	2023-10-04 06:21:40 +02:00
PSeitz	041d4fced7	move to_named_doc to Document trait (#2205 )	2023-10-04 06:03:07 +02:00
dependabot[bot]	166fc15239	Update memmap2 requirement from 0.7.1 to 0.9.0 (#2204 ) Updates the requirements on [memmap2](https://github.com/RazrFalcon/memmap2-rs) to permit the latest version. - [Changelog](https://github.com/RazrFalcon/memmap2-rs/blob/master/CHANGELOG.md) - [Commits](https://github.com/RazrFalcon/memmap2-rs/compare/v0.7.1...v0.9.0) --- updated-dependencies: - dependency-name: memmap2 dependency-type: direct:production ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>	2023-10-04 05:00:46 +02:00
PSeitz	514a6e7fef	fix bench compile, fix Document reexport (#2203 )	2023-10-03 17:28:36 +02:00
dependabot[bot]	82d9127191	Update fs4 requirement from 0.6.3 to 0.7.0 (#2199 ) Updates the requirements on [fs4](https://github.com/al8n/fs4-rs) to permit the latest version. - [Release notes](https://github.com/al8n/fs4-rs/releases) - [Commits](https://github.com/al8n/fs4-rs/commits/0.7.0) --- updated-dependencies: - dependency-name: fs4 dependency-type: direct:production ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>	2023-10-03 04:43:09 +02:00
PSeitz	03a1f40767	rename DocValue to Value (#2197 ) rename DocValue to Value to avoid confusion with lucene DocValues rename Value to OwnedValue	2023-10-02 17:03:00 +02:00
Harrison Burt	1c7c6fd591	POC: Tantivy documents as a trait (#2071 ) * fix windows build (#1) * Fix windows build * Add doc traits * Add field value iter * Add value and serialization * Adjust order * Fix bug * Correct type * Fix generic bugs * Reformat code * Add generic to index writer which I forgot about * Fix missing generics on single segment writer * Add missing type export * Add default methods for convenience * Cleanup * Fix more-like-this query to use standard types * Update API and fix tests * Add doc traits * Add field value iter * Add value and serialization * Adjust order * Fix bug * Correct type * Rebase main and fix conflicts * Reformat code * Merge upstream * Fix missing generics on single segment writer * Add missing type export * Add default methods for convenience * Cleanup * Fix more-like-this query to use standard types * Update API and fix tests * Add tokenizer improvements from previous commits * Add tokenizer improvements from previous commits * Reformat * Fix unit tests * Fix unit tests * Use enum in changes * Stage changes * Add new deserializer logic * Add serializer integration * Add document deserializer * Implement new (de)serialization api for existing types * Fix bugs and type errors * Add helper implementations * Fix errors * Reformat code * Add unit tests and some code organisation for serialization * Add unit tests to deserializer * Add some small docs * Add support for deserializing serde values * Reformat * Fix typo * Fix typo * Change repr of facet * Remove unused trait methods * Add child value type * Resolve comments * Fix build * Fix more build errors * Fix more build errors * Fix the tests I missed * Fix examples * fix numerical order, serialize PreTok Str * fix coverage * rename Document to TantivyDocument, rename DocumentAccess to Document add Binary prefix to binary de/serialization * fix coverage --------- Co-authored-by: Pascal Seitz <pascal.seitz@gmail.com>	2023-10-02 10:01:16 +02:00
PSeitz	b525f653c0	replace BinaryHeap for TopN (#2186 ) * replace BinaryHeap for TopN replace BinaryHeap for TopN with variant that selects the median with QuickSort, which runs in O(n) time. add merge_fruits fast path * call truncate unconditionally, extend test * remove special early exit * add TODO, fmt * truncate top n instead median, return vec * simplify code	2023-09-27 09:25:30 +02:00
ethever.eth	90586bc1e2	chore: remove unused Seek impl for Writers (#2187 ) (#2189 ) Co-authored-by: famouscat <onismaa@gmail.com>	2023-09-26 17:03:28 +09:00
PSeitz	832f1633de	handle exclusive out of bounds ranges on fastfield range queries (#2174 ) closes https://github.com/quickwit-oss/quickwit/issues/3790	2023-09-26 08:00:40 +02:00
PSeitz	38db53c465	make column_index pub (#2181 )	2023-09-22 08:06:45 +02:00