tracing

.github/workflows/coverage.yml

@@ -3,6 +3,8 @@ name: Coverage
 on:
   push:
     branches: [main]
+  pull_request:
+    branches: [main]
 
 # Ensures that we cancel running jobs for the same PR / same workflow.
 concurrency:

.github/workflows/test.yml

@@ -39,13 +39,6 @@ jobs:
 
     - name: Check Formatting
       run: cargo +nightly fmt --all -- --check
-    
-    - name: Check Stable Compilation
-      run: cargo build --all-features
-
-    
-    - name: Check Bench Compilation
-      run: cargo +nightly bench --no-run --profile=dev --all-features
 
     - uses: actions-rs/clippy-check@v1
       with:

CHANGELOG.md

@@ -1,9 +1,3 @@
-Tantivy 0.21.1
-================================
-#### Bugfixes
-- Range queries on fast fields with less values on that field than documents had an invalid end condition, leading to missing results. [#2226](https://github.com/quickwit-oss/tantivy/issues/2226)(@appaquet @PSeitz)
-- Increase the minimum memory budget from 3MB to 15MB to avoid single doc segments (API fix). [#2176](https://github.com/quickwit-oss/tantivy/issues/2176)(@PSeitz)
-
 Tantivy 0.21
 ================================
 #### Bugfixes

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "tantivy"
-version = "0.22.0-dev"
+version = "0.21.0"
 authors = ["Paul Masurel <paul.masurel@gmail.com>"]
 license = "MIT"
 categories = ["database-implementations", "data-structures"]
@@ -19,10 +19,11 @@ oneshot = "0.1.5"
 base64 = "0.21.0"
 byteorder = "1.4.3"
 crc32fast = "1.3.2"
+tracing = "0.1"
 once_cell = "1.10.0"
 regex = { version = "1.5.5", default-features = false, features = ["std", "unicode"] }
 aho-corasick = "1.0"
-tantivy-fst = "0.5"
+tantivy-fst = "0.4.0"
 memmap2 = { version = "0.9.0", optional = true }
 lz4_flex = { version = "0.11", default-features = false, optional = true }
 zstd = { version = "0.13", optional = true, default-features = false }
@@ -37,19 +38,21 @@ uuid = { version = "1.0.0", features = ["v4", "serde"] }
 crossbeam-channel = "0.5.4"
 rust-stemmers = "1.2.0"
 downcast-rs = "1.2.0"
-bitpacking = { version = "0.9.2", default-features = false, features = ["bitpacker4x"] }
+bitpacking = { version = "0.8.4", default-features = false, features = ["bitpacker4x"] }
 census = "0.4.0"
 rustc-hash = "1.1.0"
 thiserror = "1.0.30"
 htmlescape = "0.3.1"
 fail = { version = "0.5.0", optional = true }
+murmurhash32 = "0.3.0"
 time = { version = "0.3.10", features = ["serde-well-known"] }
 smallvec = "1.8.0"
 rayon = "1.5.2"
 lru = "0.12.0"
 fastdivide = "0.4.0"
-itertools = "0.12.0"
+itertools = "0.11.0"
 measure_time = "0.8.2"
+async-trait = "0.1.53"
 arc-swap = "1.5.0"
 
 columnar = { version= "0.2", path="./columnar", package ="tantivy-columnar" }
@@ -73,13 +76,15 @@ matches = "0.1.9"
 pretty_assertions = "1.2.1"
 proptest = "1.0.0"
 test-log = "0.2.10"
+env_logger = "0.10.0"
 futures = "0.3.21"
 paste = "1.0.11"
 more-asserts = "0.3.1"
 rand_distr = "0.4.3"
 
 [target.'cfg(not(windows))'.dev-dependencies]
-criterion = { version = "0.5", default-features = false }
+criterion = "0.5"
+pprof = { git = "https://github.com/PSeitz/pprof-rs/", rev = "53af24b", features = ["flamegraph", "criterion"] } # temp fork that works with criterion 0.5
 
 [dev-dependencies.fail]
 version = "0.5.0"
@@ -112,11 +117,6 @@ unstable = [] # useful for benches.
 
 quickwit = ["sstable", "futures-util"]
 
-# Compares only the hash of a string when indexing data. 
-# Increases indexing speed, but may lead to extremely rare missing terms, when there's a hash collision.
-# Uses 64bit ahash.
-compare_hash_only = ["stacker/compare_hash_only"]
-
 [workspace]
 members = ["query-grammar", "bitpacker", "common", "ownedbytes", "stacker", "sstable", "tokenizer-api", "columnar"]
 

benches/index-bench.rs

@@ -1,99 +1,14 @@
-use criterion::{criterion_group, criterion_main, BatchSize, Bencher, Criterion, Throughput};
+use criterion::{criterion_group, criterion_main, Criterion, Throughput};
+use pprof::criterion::{Output, PProfProfiler};
 use tantivy::schema::{TantivyDocument, FAST, INDEXED, STORED, STRING, TEXT};
-use tantivy::{tokenizer, Index, IndexWriter};
+use tantivy::{Index, IndexWriter};
 
 const HDFS_LOGS: &str = include_str!("hdfs.json");
 const GH_LOGS: &str = include_str!("gh.json");
 const WIKI: &str = include_str!("wiki.json");
 
-fn benchmark(
-    b: &mut Bencher,
-    input: &str,
-    schema: tantivy::schema::Schema,
-    commit: bool,
-    parse_json: bool,
-    is_dynamic: bool,
-) {
-    if is_dynamic {
-        benchmark_dynamic_json(b, input, schema, commit, parse_json)
-    } else {
-        _benchmark(b, input, schema, commit, parse_json, |schema, doc_json| {
-            TantivyDocument::parse_json(&schema, doc_json).unwrap()
-        })
-    }
-}
-
-fn get_index(schema: tantivy::schema::Schema) -> Index {
-    let mut index = Index::create_in_ram(schema.clone());
-    let ff_tokenizer_manager = tokenizer::TokenizerManager::default();
-    ff_tokenizer_manager.register(
-        "raw",
-        tokenizer::TextAnalyzer::builder(tokenizer::RawTokenizer::default())
-            .filter(tokenizer::RemoveLongFilter::limit(255))
-            .build(),
-    );
-    index.set_fast_field_tokenizers(ff_tokenizer_manager.clone());
-    index
-}
-
-fn _benchmark(
-    b: &mut Bencher,
-    input: &str,
-    schema: tantivy::schema::Schema,
-    commit: bool,
-    include_json_parsing: bool,
-    create_doc: impl Fn(&tantivy::schema::Schema, &str) -> TantivyDocument,
-) {
-    if include_json_parsing {
-        let lines: Vec<&str> = input.trim().split('\n').collect();
-        b.iter(|| {
-            let index = get_index(schema.clone());
-            let mut index_writer: IndexWriter =
-                index.writer_with_num_threads(1, 100_000_000).unwrap();
-            for doc_json in &lines {
-                let doc = create_doc(&schema, doc_json);
-                index_writer.add_document(doc).unwrap();
-            }
-            if commit {
-                index_writer.commit().unwrap();
-            }
-        })
-    } else {
-        let docs: Vec<_> = input
-            .trim()
-            .split('\n')
-            .map(|doc_json| create_doc(&schema, doc_json))
-            .collect();
-        b.iter_batched(
-            || docs.clone(),
-            |docs| {
-                let index = get_index(schema.clone());
-                let mut index_writer: IndexWriter =
-                    index.writer_with_num_threads(1, 100_000_000).unwrap();
-                for doc in docs {
-                    index_writer.add_document(doc).unwrap();
-                }
-                if commit {
-                    index_writer.commit().unwrap();
-                }
-            },
-            BatchSize::SmallInput,
-        )
-    }
-}
-fn benchmark_dynamic_json(
-    b: &mut Bencher,
-    input: &str,
-    schema: tantivy::schema::Schema,
-    commit: bool,
-    parse_json: bool,
-) {
-    let json_field = schema.get_field("json").unwrap();
-    _benchmark(b, input, schema, commit, parse_json, |_schema, doc_json| {
-        let json_val: serde_json::Map<String, serde_json::Value> =
-            serde_json::from_str(doc_json).unwrap();
-        tantivy::doc!(json_field=>json_val)
-    })
+fn get_lines(input: &str) -> Vec<&str> {
+    input.trim().split('\n').collect()
 }
 
 pub fn hdfs_index_benchmark(c: &mut Criterion) {
@@ -104,14 +19,7 @@ pub fn hdfs_index_benchmark(c: &mut Criterion) {
         schema_builder.add_text_field("severity", STRING);
         schema_builder.build()
     };
-    let schema_only_fast = {
-        let mut schema_builder = tantivy::schema::SchemaBuilder::new();
-        schema_builder.add_u64_field("timestamp", FAST);
-        schema_builder.add_text_field("body", FAST);
-        schema_builder.add_text_field("severity", FAST);
-        schema_builder.build()
-    };
-    let _schema_with_store = {
+    let schema_with_store = {
         let mut schema_builder = tantivy::schema::SchemaBuilder::new();
         schema_builder.add_u64_field("timestamp", INDEXED | STORED);
         schema_builder.add_text_field("body", TEXT | STORED);
@@ -120,39 +28,77 @@ pub fn hdfs_index_benchmark(c: &mut Criterion) {
     };
     let dynamic_schema = {
         let mut schema_builder = tantivy::schema::SchemaBuilder::new();
-        schema_builder.add_json_field("json", TEXT | FAST);
+        schema_builder.add_json_field("json", TEXT);
         schema_builder.build()
     };
 
     let mut group = c.benchmark_group("index-hdfs");
     group.throughput(Throughput::Bytes(HDFS_LOGS.len() as u64));
     group.sample_size(20);
-
-    let benches = [
-        ("only-indexed-".to_string(), schema, false),
-        //("stored-".to_string(), _schema_with_store, false),
-        ("only-fast-".to_string(), schema_only_fast, false),
-        ("dynamic-".to_string(), dynamic_schema, true),
-    ];
-
-    for (prefix, schema, is_dynamic) in benches {
-        for commit in [false, true] {
-            let suffix = if commit { "with-commit" } else { "no-commit" };
-            for parse_json in [false] {
-                // for parse_json in [false, true] {
-                let suffix = if parse_json {
-                    format!("{}-with-json-parsing", suffix)
-                } else {
-                    format!("{}", suffix)
-                };
-
-                let bench_name = format!("{}{}", prefix, suffix);
-                group.bench_function(bench_name, |b| {
-                    benchmark(b, HDFS_LOGS, schema.clone(), commit, parse_json, is_dynamic)
-                });
+    group.bench_function("index-hdfs-no-commit", |b| {
+        let lines = get_lines(HDFS_LOGS);
+        b.iter(|| {
+            let index = Index::create_in_ram(schema.clone());
+            let index_writer: IndexWriter = index.writer_with_num_threads(1, 100_000_000).unwrap();
+            for doc_json in &lines {
+                let doc = TantivyDocument::parse_json(&schema, doc_json).unwrap();
+                index_writer.add_document(doc).unwrap();
             }
-        }
-    }
+        })
+    });
+    group.bench_function("index-hdfs-with-commit", |b| {
+        let lines = get_lines(HDFS_LOGS);
+        b.iter(|| {
+            let index = Index::create_in_ram(schema.clone());
+            let mut index_writer: IndexWriter =
+                index.writer_with_num_threads(1, 100_000_000).unwrap();
+            for doc_json in &lines {
+                let doc = TantivyDocument::parse_json(&schema, doc_json).unwrap();
+                index_writer.add_document(doc).unwrap();
+            }
+            index_writer.commit().unwrap();
+        })
+    });
+    group.bench_function("index-hdfs-no-commit-with-docstore", |b| {
+        let lines = get_lines(HDFS_LOGS);
+        b.iter(|| {
+            let index = Index::create_in_ram(schema_with_store.clone());
+            let index_writer: IndexWriter = index.writer_with_num_threads(1, 100_000_000).unwrap();
+            for doc_json in &lines {
+                let doc = TantivyDocument::parse_json(&schema, doc_json).unwrap();
+                index_writer.add_document(doc).unwrap();
+            }
+        })
+    });
+    group.bench_function("index-hdfs-with-commit-with-docstore", |b| {
+        let lines = get_lines(HDFS_LOGS);
+        b.iter(|| {
+            let index = Index::create_in_ram(schema_with_store.clone());
+            let mut index_writer: IndexWriter =
+                index.writer_with_num_threads(1, 100_000_000).unwrap();
+            for doc_json in &lines {
+                let doc = TantivyDocument::parse_json(&schema, doc_json).unwrap();
+                index_writer.add_document(doc).unwrap();
+            }
+            index_writer.commit().unwrap();
+        })
+    });
+    group.bench_function("index-hdfs-no-commit-json-without-docstore", |b| {
+        let lines = get_lines(HDFS_LOGS);
+        b.iter(|| {
+            let index = Index::create_in_ram(dynamic_schema.clone());
+            let json_field = dynamic_schema.get_field("json").unwrap();
+            let mut index_writer: IndexWriter =
+                index.writer_with_num_threads(1, 100_000_000).unwrap();
+            for doc_json in &lines {
+                let json_val: serde_json::Map<String, serde_json::Value> =
+                    serde_json::from_str(doc_json).unwrap();
+                let doc = tantivy::doc!(json_field=>json_val);
+                index_writer.add_document(doc).unwrap();
+            }
+            index_writer.commit().unwrap();
+        })
+    });
 }
 
 pub fn gh_index_benchmark(c: &mut Criterion) {
@@ -161,24 +107,39 @@ pub fn gh_index_benchmark(c: &mut Criterion) {
         schema_builder.add_json_field("json", TEXT | FAST);
         schema_builder.build()
     };
-    let dynamic_schema_fast = {
-        let mut schema_builder = tantivy::schema::SchemaBuilder::new();
-        schema_builder.add_json_field("json", FAST);
-        schema_builder.build()
-    };
 
     let mut group = c.benchmark_group("index-gh");
     group.throughput(Throughput::Bytes(GH_LOGS.len() as u64));
 
     group.bench_function("index-gh-no-commit", |b| {
-        benchmark_dynamic_json(b, GH_LOGS, dynamic_schema.clone(), false, false)
+        let lines = get_lines(GH_LOGS);
+        b.iter(|| {
+            let json_field = dynamic_schema.get_field("json").unwrap();
+            let index = Index::create_in_ram(dynamic_schema.clone());
+            let index_writer: IndexWriter = index.writer_with_num_threads(1, 100_000_000).unwrap();
+            for doc_json in &lines {
+                let json_val: serde_json::Map<String, serde_json::Value> =
+                    serde_json::from_str(doc_json).unwrap();
+                let doc = tantivy::doc!(json_field=>json_val);
+                index_writer.add_document(doc).unwrap();
+            }
+        })
     });
-    group.bench_function("index-gh-fast", |b| {
-        benchmark_dynamic_json(b, GH_LOGS, dynamic_schema_fast.clone(), false, false)
-    });
-
-    group.bench_function("index-gh-fast-with-commit", |b| {
-        benchmark_dynamic_json(b, GH_LOGS, dynamic_schema_fast.clone(), true, false)
+    group.bench_function("index-gh-with-commit", |b| {
+        let lines = get_lines(GH_LOGS);
+        b.iter(|| {
+            let json_field = dynamic_schema.get_field("json").unwrap();
+            let index = Index::create_in_ram(dynamic_schema.clone());
+            let mut index_writer: IndexWriter =
+                index.writer_with_num_threads(1, 100_000_000).unwrap();
+            for doc_json in &lines {
+                let json_val: serde_json::Map<String, serde_json::Value> =
+                    serde_json::from_str(doc_json).unwrap();
+                let doc = tantivy::doc!(json_field=>json_val);
+                index_writer.add_document(doc).unwrap();
+            }
+            index_writer.commit().unwrap();
+        })
     });
 }
 
@@ -193,10 +154,34 @@ pub fn wiki_index_benchmark(c: &mut Criterion) {
     group.throughput(Throughput::Bytes(WIKI.len() as u64));
 
     group.bench_function("index-wiki-no-commit", |b| {
-        benchmark_dynamic_json(b, WIKI, dynamic_schema.clone(), false, false)
+        let lines = get_lines(WIKI);
+        b.iter(|| {
+            let json_field = dynamic_schema.get_field("json").unwrap();
+            let index = Index::create_in_ram(dynamic_schema.clone());
+            let index_writer: IndexWriter = index.writer_with_num_threads(1, 100_000_000).unwrap();
+            for doc_json in &lines {
+                let json_val: serde_json::Map<String, serde_json::Value> =
+                    serde_json::from_str(doc_json).unwrap();
+                let doc = tantivy::doc!(json_field=>json_val);
+                index_writer.add_document(doc).unwrap();
+            }
+        })
     });
     group.bench_function("index-wiki-with-commit", |b| {
-        benchmark_dynamic_json(b, WIKI, dynamic_schema.clone(), true, false)
+        let lines = get_lines(WIKI);
+        b.iter(|| {
+            let json_field = dynamic_schema.get_field("json").unwrap();
+            let index = Index::create_in_ram(dynamic_schema.clone());
+            let mut index_writer: IndexWriter =
+                index.writer_with_num_threads(1, 100_000_000).unwrap();
+            for doc_json in &lines {
+                let json_val: serde_json::Map<String, serde_json::Value> =
+                    serde_json::from_str(doc_json).unwrap();
+                let doc = tantivy::doc!(json_field=>json_val);
+                index_writer.add_document(doc).unwrap();
+            }
+            index_writer.commit().unwrap();
+        })
     });
 }
 
@@ -207,12 +192,12 @@ criterion_group! {
 }
 criterion_group! {
     name = gh_benches;
-    config = Criterion::default();
+    config = Criterion::default().with_profiler(PProfProfiler::new(100, Output::Flamegraph(None)));
     targets = gh_index_benchmark
 }
 criterion_group! {
     name = wiki_benches;
-    config = Criterion::default();
+    config = Criterion::default().with_profiler(PProfProfiler::new(100, Output::Flamegraph(None)));
     targets = wiki_index_benchmark
 }
 criterion_main!(benches, gh_benches, wiki_benches);

bitpacker/Cargo.toml

@@ -15,7 +15,7 @@ homepage = "https://github.com/quickwit-oss/tantivy"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-bitpacking = { version = "0.9.2", default-features = false, features = ["bitpacker1x"] }
+bitpacking = {version="0.8", default-features=false, features = ["bitpacker1x"]}
 
 [dev-dependencies]
 rand = "0.8"

bitpacker/src/bitpacker.rs

@@ -367,7 +367,7 @@ mod test {
             let mut output: Vec<u32> = Vec::new();
             for len in [0, 1, 2, 32, 33, 34, 64] {
                 for start_idx in 0u32..32u32 {
-                    output.resize(len, 0);
+                    output.resize(len as usize, 0);
                     bitunpacker.get_batch_u32s(start_idx, &buffer, &mut output);
                     for i in 0..len {
                         let expected = (start_idx + i as u32) & mask;

columnar/Cargo.toml

@@ -9,7 +9,8 @@ description = "column oriented storage for tantivy"
 categories = ["database-implementations", "data-structures", "compression"]
 
 [dependencies]
-itertools = "0.12.0"
+itertools = "0.11.0"
+fnv = "1.0.7"
 fastdivide = "0.4.0"
 
 stacker = { version= "0.2", path = "../stacker", package="tantivy-stacker"}

columnar/columnar-cli/Cargo.toml

@@ -8,6 +8,7 @@ license = "MIT"
 columnar = {path="../", package="tantivy-columnar"}
 serde_json = "1"
 serde_json_borrow = {git="https://github.com/PSeitz/serde_json_borrow/"}
+serde = "1"
 
 [workspace]
 members = []

columnar/src/column/mod.rs

@@ -110,9 +110,6 @@ impl<T: PartialOrd + Copy + Debug + Send + Sync + 'static> Column<T> {
     }
 
     /// Get the docids of values which are in the provided value range.
-    ///
-    /// # Panic
-    /// Panics if a value in the selected_docid_range range is larger than the number of documents.
     #[inline]
     pub fn get_docids_for_value_range(
         &self,

columnar/src/column_index/mod.rs

@@ -126,8 +126,6 @@ impl ColumnIndex {
         }
     }
 
-    /// # Panic
-    /// Panics if a value in the doc_id range is larger than the number of documents.
     pub fn docid_range_to_rowids(&self, doc_id: Range<DocId>) -> Range<RowId> {
         match self {
             ColumnIndex::Empty { .. } => 0..0,

columnar/src/column_index/optional_index/mod.rs

@@ -21,6 +21,8 @@ const DENSE_BLOCK_THRESHOLD: u32 =
 
 const ELEMENTS_PER_BLOCK: u32 = u16::MAX as u32 + 1;
 
+const BLOCK_SIZE: RowId = 1 << 16;
+
 #[derive(Copy, Clone, Debug)]
 struct BlockMeta {
     non_null_rows_before_block: u32,
@@ -107,8 +109,8 @@ struct RowAddr {
 #[inline(always)]
 fn row_addr_from_row_id(row_id: RowId) -> RowAddr {
     RowAddr {
-        block_id: (row_id / ELEMENTS_PER_BLOCK) as u16,
-        in_block_row_id: (row_id % ELEMENTS_PER_BLOCK) as u16,
+        block_id: (row_id / BLOCK_SIZE) as u16,
+        in_block_row_id: (row_id % BLOCK_SIZE) as u16,
     }
 }
 
@@ -488,9 +490,8 @@ fn deserialize_optional_index_block_metadatas(
         start_byte_offset += block_variant.num_bytes_in_block();
         non_null_rows_before_block += num_non_null_rows;
     }
-    let last_block = row_addr_from_row_id(num_rows).block_id;
     block_metas.resize(
-        last_block as usize + 1, // +1 since last block is an index
+        ((num_rows + BLOCK_SIZE - 1) / BLOCK_SIZE) as usize,
         BlockMeta {
             non_null_rows_before_block,
             start_byte_offset,

columnar/src/column_index/optional_index/tests.rs

@@ -3,29 +3,6 @@ use proptest::strategy::Strategy;
 use proptest::{prop_oneof, proptest};
 
 use super::*;
-use crate::{ColumnarReader, ColumnarWriter, DynamicColumnHandle};
-
-#[test]
-fn test_optional_index_bug_2293() {
-    test_optional_index_with_num_docs(ELEMENTS_PER_BLOCK - 1);
-    test_optional_index_with_num_docs(ELEMENTS_PER_BLOCK);
-    test_optional_index_with_num_docs(ELEMENTS_PER_BLOCK + 1);
-}
-fn test_optional_index_with_num_docs(num_docs: u32) {
-    let mut dataframe_writer = ColumnarWriter::default();
-    dataframe_writer.record_numerical(100, "score", 80i64);
-    let mut buffer: Vec<u8> = Vec::new();
-    dataframe_writer
-        .serialize(num_docs, None, &mut buffer)
-        .unwrap();
-    let columnar = ColumnarReader::open(buffer).unwrap();
-    assert_eq!(columnar.num_columns(), 1);
-    let cols: Vec<DynamicColumnHandle> = columnar.read_columns("score").unwrap();
-    assert_eq!(cols.len(), 1);
-
-    let col = cols[0].open().unwrap();
-    col.column_index().docid_range_to_rowids(0..num_docs);
-}
 
 #[test]
 fn test_dense_block_threshold() {
@@ -58,7 +35,7 @@ proptest! {
 
 #[test]
 fn test_with_random_sets_simple() {
-    let vals = 10..ELEMENTS_PER_BLOCK * 2;
+    let vals = 10..BLOCK_SIZE * 2;
     let mut out: Vec<u8> = Vec::new();
     serialize_optional_index(&vals, 100, &mut out).unwrap();
     let null_index = open_optional_index(OwnedBytes::new(out)).unwrap();
@@ -194,7 +171,7 @@ fn test_optional_index_rank() {
     test_optional_index_rank_aux(&[0u32, 1u32]);
     let mut block = Vec::new();
     block.push(3u32);
-    block.extend((0..ELEMENTS_PER_BLOCK).map(|i| i + ELEMENTS_PER_BLOCK + 1));
+    block.extend((0..BLOCK_SIZE).map(|i| i + BLOCK_SIZE + 1));
     test_optional_index_rank_aux(&block);
 }
 
@@ -208,8 +185,8 @@ fn test_optional_index_iter_empty_one() {
 fn test_optional_index_iter_dense_block() {
     let mut block = Vec::new();
     block.push(3u32);
-    block.extend((0..ELEMENTS_PER_BLOCK).map(|i| i + ELEMENTS_PER_BLOCK + 1));
-    test_optional_index_iter_aux(&block, 3 * ELEMENTS_PER_BLOCK);
+    block.extend((0..BLOCK_SIZE).map(|i| i + BLOCK_SIZE + 1));
+    test_optional_index_iter_aux(&block, 3 * BLOCK_SIZE);
 }
 
 #[test]
@@ -238,12 +215,12 @@ mod bench {
         let vals: Vec<RowId> = (0..TOTAL_NUM_VALUES)
             .map(|_| rng.gen_bool(fill_ratio))
             .enumerate()
-            .filter(|(_pos, val)| *val)
+            .filter(|(pos, val)| *val)
             .map(|(pos, _)| pos as RowId)
             .collect();
         serialize_optional_index(&&vals[..], TOTAL_NUM_VALUES, &mut out).unwrap();
-
-        open_optional_index(OwnedBytes::new(out)).unwrap()
+        let codec = open_optional_index(OwnedBytes::new(out)).unwrap();
+        codec
     }
 
     fn random_range_iterator(
@@ -265,7 +242,7 @@ mod bench {
     }
 
     fn n_percent_step_iterator(percent: f32, num_values: u32) -> impl Iterator<Item = u32> {
-        let ratio = percent / 100.0;
+        let ratio = percent as f32 / 100.0;
         let step_size = (1f32 / ratio) as u32;
         let deviation = step_size - 1;
         random_range_iterator(0, num_values, step_size, deviation)

columnar/src/columnar/writer/column_writers.rs

@@ -269,8 +269,7 @@ impl StrOrBytesColumnWriter {
         dictionaries: &mut [DictionaryBuilder],
         arena: &mut MemoryArena,
     ) {
-        let unordered_id =
-            dictionaries[self.dictionary_id as usize].get_or_allocate_id(bytes, arena);
+        let unordered_id = dictionaries[self.dictionary_id as usize].get_or_allocate_id(bytes);
         self.column_writer.record(doc, unordered_id, arena);
     }
 

columnar/src/columnar/writer/mod.rs

@@ -338,7 +338,7 @@ impl ColumnarWriter {
         let mut columns: Vec<(&[u8], ColumnType, Addr)> = self
             .numerical_field_hash_map
             .iter()
-            .map(|(column_name, addr)| {
+            .map(|(column_name, addr, _)| {
                 let numerical_column_writer: NumericalColumnWriter =
                     self.numerical_field_hash_map.read(addr);
                 let column_type = numerical_column_writer.numerical_type().into();
@@ -348,27 +348,27 @@ impl ColumnarWriter {
         columns.extend(
             self.bytes_field_hash_map
                 .iter()
-                .map(|(term, addr)| (term, ColumnType::Bytes, addr)),
+                .map(|(term, addr, _)| (term, ColumnType::Bytes, addr)),
         );
         columns.extend(
             self.str_field_hash_map
                 .iter()
-                .map(|(column_name, addr)| (column_name, ColumnType::Str, addr)),
+                .map(|(column_name, addr, _)| (column_name, ColumnType::Str, addr)),
         );
         columns.extend(
             self.bool_field_hash_map
                 .iter()
-                .map(|(column_name, addr)| (column_name, ColumnType::Bool, addr)),
+                .map(|(column_name, addr, _)| (column_name, ColumnType::Bool, addr)),
         );
         columns.extend(
             self.ip_addr_field_hash_map
                 .iter()
-                .map(|(column_name, addr)| (column_name, ColumnType::IpAddr, addr)),
+                .map(|(column_name, addr, _)| (column_name, ColumnType::IpAddr, addr)),
         );
         columns.extend(
             self.datetime_field_hash_map
                 .iter()
-                .map(|(column_name, addr)| (column_name, ColumnType::DateTime, addr)),
+                .map(|(column_name, addr, _)| (column_name, ColumnType::DateTime, addr)),
         );
         columns.sort_unstable_by_key(|(column_name, col_type, _)| (*column_name, *col_type));
 
@@ -437,7 +437,6 @@ impl ColumnarWriter {
                             &mut symbol_byte_buffer,
                         ),
                         buffers,
-                        &self.arena,
                         &mut column_serializer,
                     )?;
                     column_serializer.finalize()?;
@@ -491,7 +490,6 @@ impl ColumnarWriter {
 
 // Serialize [Dictionary, Column, dictionary num bytes U32::LE]
 // Column: [Column Index, Column Values, column index num bytes U32::LE]
-#[allow(clippy::too_many_arguments)]
 fn serialize_bytes_or_str_column(
     cardinality: Cardinality,
     num_docs: RowId,
@@ -499,7 +497,6 @@ fn serialize_bytes_or_str_column(
     dictionary_builder: &DictionaryBuilder,
     operation_it: impl Iterator<Item = ColumnOperation<UnorderedId>>,
     buffers: &mut SpareBuffers,
-    arena: &MemoryArena,
     wrt: impl io::Write,
 ) -> io::Result<()> {
     let SpareBuffers {
@@ -508,8 +505,7 @@ fn serialize_bytes_or_str_column(
         ..
     } = buffers;
     let mut counting_writer = CountingWriter::wrap(wrt);
-    let term_id_mapping: TermIdMapping =
-        dictionary_builder.serialize(arena, &mut counting_writer)?;
+    let term_id_mapping: TermIdMapping = dictionary_builder.serialize(&mut counting_writer)?;
     let dictionary_num_bytes: u32 = counting_writer.written_bytes() as u32;
     let mut wrt = counting_writer.finish();
     let operation_iterator = operation_it.map(|symbol: ColumnOperation<UnorderedId>| {

columnar/src/dictionary.rs

@@ -1,7 +1,7 @@
 use std::io;
 
+use fnv::FnvHashMap;
 use sstable::SSTable;
-use stacker::{MemoryArena, SharedArenaHashMap};
 
 pub(crate) struct TermIdMapping {
     unordered_to_ord: Vec<OrderedId>,
@@ -31,38 +31,29 @@ pub struct OrderedId(pub u32);
 /// mapping.
 #[derive(Default)]
 pub(crate) struct DictionaryBuilder {
-    dict: SharedArenaHashMap,
+    dict: FnvHashMap<Vec<u8>, UnorderedId>,
+    memory_consumption: usize,
 }
 
 impl DictionaryBuilder {
     /// Get or allocate an unordered id.
     /// (This ID is simply an auto-incremented id.)
-    pub fn get_or_allocate_id(&mut self, term: &[u8], arena: &mut MemoryArena) -> UnorderedId {
-        let next_id = self.dict.len() as u32;
-        let unordered_id = self
-            .dict
-            .mutate_or_create(term, arena, |unordered_id: Option<u32>| {
-                if let Some(unordered_id) = unordered_id {
-                    unordered_id
-                } else {
-                    next_id
-                }
-            });
-        UnorderedId(unordered_id)
+    pub fn get_or_allocate_id(&mut self, term: &[u8]) -> UnorderedId {
+        if let Some(term_id) = self.dict.get(term) {
+            return *term_id;
+        }
+        let new_id = UnorderedId(self.dict.len() as u32);
+        self.dict.insert(term.to_vec(), new_id);
+        self.memory_consumption += term.len();
+        self.memory_consumption += 40; // Term Metadata + HashMap overhead
+        new_id
     }
 
     /// Serialize the dictionary into an fst, and returns the
     /// `UnorderedId -> TermOrdinal` map.
-    pub fn serialize<'a, W: io::Write + 'a>(
-        &self,
-        arena: &MemoryArena,
-        wrt: &mut W,
-    ) -> io::Result<TermIdMapping> {
-        let mut terms: Vec<(&[u8], UnorderedId)> = self
-            .dict
-            .iter(arena)
-            .map(|(k, v)| (k, arena.read(v)))
-            .collect();
+    pub fn serialize<'a, W: io::Write + 'a>(&self, wrt: &mut W) -> io::Result<TermIdMapping> {
+        let mut terms: Vec<(&[u8], UnorderedId)> =
+            self.dict.iter().map(|(k, v)| (k.as_slice(), *v)).collect();
         terms.sort_unstable_by_key(|(key, _)| *key);
         // TODO Remove the allocation.
         let mut unordered_to_ord: Vec<OrderedId> = vec![OrderedId(0u32); terms.len()];
@@ -77,7 +68,7 @@ impl DictionaryBuilder {
     }
 
     pub(crate) fn mem_usage(&self) -> usize {
-        self.dict.mem_usage()
+        self.memory_consumption
     }
 }
 
@@ -87,13 +78,12 @@ mod tests {
 
     #[test]
     fn test_dictionary_builder() {
-        let mut arena = MemoryArena::default();
         let mut dictionary_builder = DictionaryBuilder::default();
-        let hello_uid = dictionary_builder.get_or_allocate_id(b"hello", &mut arena);
-        let happy_uid = dictionary_builder.get_or_allocate_id(b"happy", &mut arena);
-        let tax_uid = dictionary_builder.get_or_allocate_id(b"tax", &mut arena);
+        let hello_uid = dictionary_builder.get_or_allocate_id(b"hello");
+        let happy_uid = dictionary_builder.get_or_allocate_id(b"happy");
+        let tax_uid = dictionary_builder.get_or_allocate_id(b"tax");
         let mut buffer = Vec::new();
-        let id_mapping = dictionary_builder.serialize(&arena, &mut buffer).unwrap();
+        let id_mapping = dictionary_builder.serialize(&mut buffer).unwrap();
         assert_eq!(id_mapping.to_ord(hello_uid), OrderedId(1));
         assert_eq!(id_mapping.to_ord(happy_uid), OrderedId(0));
         assert_eq!(id_mapping.to_ord(tax_uid), OrderedId(2));

columnar/src/lib.rs

@@ -1,22 +1,3 @@
-//! # Tantivy-Columnar
-//!
-//! `tantivy-columnar`provides a columnar storage for tantivy.
-//! The crate allows for efficient read operations on specific columns rather than entire records.
-//!
-//! ## Overview
-//!
-//! - **columnar**: Reading, writing, and merging multiple columns:
-//!   - **[ColumnarWriter]**: Makes it possible to create a new columnar.
-//!   - **[ColumnarReader]**: The ColumnarReader makes it possible to access a set of columns
-//!     associated to field names.
-//!   - **[merge_columnar]**: Contains the functionalities to merge multiple ColumnarReader or
-//!     segments into a single one.
-//!
-//! - **column**: A single column, which contains
-//!     - [column_index]: Resolves the rows for a document id. Manages the cardinality of the
-//!       column.
-//!     - [column_values]: Stores the values of a column in a dense format.
-
 #![cfg_attr(all(feature = "unstable", test), feature(test))]
 
 #[cfg(test)]

columnar/src/tests.rs

@@ -26,7 +26,7 @@ fn test_dataframe_writer_str() {
     assert_eq!(columnar.num_columns(), 1);
     let cols: Vec<DynamicColumnHandle> = columnar.read_columns("my_string").unwrap();
     assert_eq!(cols.len(), 1);
-    assert_eq!(cols[0].num_bytes(), 73);
+    assert_eq!(cols[0].num_bytes(), 87);
 }
 
 #[test]
@@ -40,7 +40,7 @@ fn test_dataframe_writer_bytes() {
     assert_eq!(columnar.num_columns(), 1);
     let cols: Vec<DynamicColumnHandle> = columnar.read_columns("my_string").unwrap();
     assert_eq!(cols.len(), 1);
-    assert_eq!(cols[0].num_bytes(), 73);
+    assert_eq!(cols[0].num_bytes(), 87);
 }
 
 #[test]
@@ -330,9 +330,9 @@ fn bytes_strategy() -> impl Strategy<Value = &'static [u8]> {
 // A random column value
 fn column_value_strategy() -> impl Strategy<Value = ColumnValue> {
     prop_oneof![
-        10 => string_strategy().prop_map(ColumnValue::Str),
-        1 => bytes_strategy().prop_map(ColumnValue::Bytes),
-        40 => num_strategy().prop_map(ColumnValue::Numerical),
+        10 => string_strategy().prop_map(|s| ColumnValue::Str(s)),
+        1 => bytes_strategy().prop_map(|b| ColumnValue::Bytes(b)),
+        40 => num_strategy().prop_map(|n| ColumnValue::Numerical(n)),
         1 => (1u16..3u16).prop_map(|ip_addr_byte| ColumnValue::IpAddr(Ipv6Addr::new(
             127,
             0,
@@ -343,7 +343,7 @@ fn column_value_strategy() -> impl Strategy<Value = ColumnValue> {
             0,
             ip_addr_byte
         ))),
-        1 => any::<bool>().prop_map(ColumnValue::Bool),
+        1 => any::<bool>().prop_map(|b| ColumnValue::Bool(b)),
         1 => (0_679_723_993i64..1_679_723_995i64)
             .prop_map(|val| { ColumnValue::DateTime(DateTime::from_timestamp_secs(val)) })
     ]
@@ -419,8 +419,8 @@ fn build_columnar_with_mapping(
     columnar_writer
         .serialize(num_docs, old_to_new_row_ids_opt, &mut buffer)
         .unwrap();
-
-    ColumnarReader::open(buffer).unwrap()
+    let columnar_reader = ColumnarReader::open(buffer).unwrap();
+    columnar_reader
 }
 
 fn build_columnar(docs: &[Vec<(&'static str, ColumnValue)>]) -> ColumnarReader {
@@ -746,7 +746,7 @@ proptest! {
         let stack_merge_order = StackMergeOrder::stack(&columnar_readers_arr[..]).into();
         crate::merge_columnar(&columnar_readers_arr[..], &[], stack_merge_order, &mut output).unwrap();
         let merged_columnar = ColumnarReader::open(output).unwrap();
-        let concat_rows: Vec<Vec<(&'static str, ColumnValue)>> = columnar_docs.iter().flatten().cloned().collect();
+        let concat_rows: Vec<Vec<(&'static str, ColumnValue)>> = columnar_docs.iter().cloned().flatten().collect();
         let expected_merged_columnar = build_columnar(&concat_rows[..]);
         assert_columnar_eq_strict(&merged_columnar, &expected_merged_columnar);
     }
@@ -772,7 +772,7 @@ fn test_columnar_merging_empty_columnar() {
     .unwrap();
     let merged_columnar = ColumnarReader::open(output).unwrap();
     let concat_rows: Vec<Vec<(&'static str, ColumnValue)>> =
-        columnar_docs.iter().flatten().cloned().collect();
+        columnar_docs.iter().cloned().flatten().collect();
     let expected_merged_columnar = build_columnar(&concat_rows[..]);
     assert_columnar_eq_strict(&merged_columnar, &expected_merged_columnar);
 }
@@ -809,7 +809,7 @@ fn test_columnar_merging_number_columns() {
     .unwrap();
     let merged_columnar = ColumnarReader::open(output).unwrap();
     let concat_rows: Vec<Vec<(&'static str, ColumnValue)>> =
-        columnar_docs.iter().flatten().cloned().collect();
+        columnar_docs.iter().cloned().flatten().collect();
     let expected_merged_columnar = build_columnar(&concat_rows[..]);
     assert_columnar_eq_strict(&merged_columnar, &expected_merged_columnar);
 }

common/src/json_path_writer.rs

@@ -1,112 +0,0 @@
-use crate::replace_in_place;
-
-/// Separates the different segments of a json path.
-pub const JSON_PATH_SEGMENT_SEP: u8 = 1u8;
-pub const JSON_PATH_SEGMENT_SEP_STR: &str =
-    unsafe { std::str::from_utf8_unchecked(&[JSON_PATH_SEGMENT_SEP]) };
-
-/// Create a new JsonPathWriter, that creates flattened json paths for tantivy.
-#[derive(Clone, Debug, Default)]
-pub struct JsonPathWriter {
-    path: String,
-    indices: Vec<usize>,
-    expand_dots: bool,
-}
-
-impl JsonPathWriter {
-    pub fn new() -> Self {
-        JsonPathWriter {
-            path: String::new(),
-            indices: Vec::new(),
-            expand_dots: false,
-        }
-    }
-
-    /// When expand_dots is enabled, json object like
-    /// `{"k8s.node.id": 5}` is processed as if it was
-    /// `{"k8s": {"node": {"id": 5}}}`.
-    /// This option has the merit of allowing users to
-    /// write queries  like `k8s.node.id:5`.
-    /// On the other, enabling that feature can lead to
-    /// ambiguity.
-    #[inline]
-    pub fn set_expand_dots(&mut self, expand_dots: bool) {
-        self.expand_dots = expand_dots;
-    }
-
-    /// Push a new segment to the path.
-    #[inline]
-    pub fn push(&mut self, segment: &str) {
-        let len_path = self.path.len();
-        self.indices.push(len_path);
-        if !self.path.is_empty() {
-            self.path.push_str(JSON_PATH_SEGMENT_SEP_STR);
-        }
-        self.path.push_str(segment);
-        if self.expand_dots {
-            // This might include the separation byte, which is ok because it is not a dot.
-            let appended_segment = &mut self.path[len_path..];
-            // The unsafe below is safe as long as b'.' and JSON_PATH_SEGMENT_SEP are
-            // valid single byte ut8 strings.
-            // By utf-8 design, they cannot be part of another codepoint.
-            unsafe {
-                replace_in_place(b'.', JSON_PATH_SEGMENT_SEP, appended_segment.as_bytes_mut())
-            };
-        }
-    }
-
-    /// Remove the last segment. Does nothing if the path is empty.
-    #[inline]
-    pub fn pop(&mut self) {
-        if let Some(last_idx) = self.indices.pop() {
-            self.path.truncate(last_idx);
-        }
-    }
-
-    /// Clear the path.
-    #[inline]
-    pub fn clear(&mut self) {
-        self.path.clear();
-        self.indices.clear();
-    }
-
-    /// Get the current path.
-    #[inline]
-    pub fn as_str(&self) -> &str {
-        &self.path
-    }
-}
-
-impl From<JsonPathWriter> for String {
-    #[inline]
-    fn from(value: JsonPathWriter) -> Self {
-        value.path
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn json_path_writer_test() {
-        let mut writer = JsonPathWriter::new();
-
-        writer.push("root");
-        assert_eq!(writer.as_str(), "root");
-
-        writer.push("child");
-        assert_eq!(writer.as_str(), "root\u{1}child");
-
-        writer.pop();
-        assert_eq!(writer.as_str(), "root");
-
-        writer.push("k8s.node.id");
-        assert_eq!(writer.as_str(), "root\u{1}k8s.node.id");
-
-        writer.set_expand_dots(true);
-        writer.pop();
-        writer.push("k8s.node.id");
-        assert_eq!(writer.as_str(), "root\u{1}k8s\u{1}node\u{1}id");
-    }
-}

common/src/lib.rs

@@ -9,7 +9,6 @@ mod byte_count;
 mod datetime;
 pub mod file_slice;
 mod group_by;
-mod json_path_writer;
 mod serialize;
 mod vint;
 mod writer;
@@ -19,7 +18,6 @@ pub use byte_count::ByteCount;
 pub use datetime::DatePrecision;
 pub use datetime::{DateTime, DateTimePrecision};
 pub use group_by::GroupByIteratorExtended;
-pub use json_path_writer::JsonPathWriter;
 pub use ownedbytes::{OwnedBytes, StableDeref};
 pub use serialize::{BinarySerializable, DeserializeFrom, FixedSize};
 pub use vint::{
@@ -118,7 +116,6 @@ pub fn u64_to_f64(val: u64) -> f64 {
 ///
 /// This function assumes that the needle is rarely contained in the bytes string
 /// and offers a fast path if the needle is not present.
-#[inline]
 pub fn replace_in_place(needle: u8, replacement: u8, bytes: &mut [u8]) {
     if !bytes.contains(&needle) {
         return;

examples/basic_search.rs

@@ -164,7 +164,7 @@ fn main() -> tantivy::Result<()> {
     // will reload the index automatically after each commit.
     let reader = index
         .reader_builder()
-        .reload_policy(ReloadPolicy::OnCommitWithDelay)
+        .reload_policy(ReloadPolicy::OnCommit)
         .try_into()?;
 
     // We now need to acquire a searcher.

examples/fuzzy_search.rs

@@ -123,7 +123,7 @@ fn main() -> tantivy::Result<()> {
     // will reload the index automatically after each commit.
     let reader = index
         .reader_builder()
-        .reload_policy(ReloadPolicy::OnCommitWithDelay)
+        .reload_policy(ReloadPolicy::OnCommit)
         .try_into()?;
 
     // We now need to acquire a searcher.

examples/phrase_prefix_search.rs

@@ -51,7 +51,7 @@ fn main() -> Result<()> {
 
     let reader = index
         .reader_builder()
-        .reload_policy(ReloadPolicy::OnCommitWithDelay)
+        .reload_policy(ReloadPolicy::OnCommit)
         .try_into()?;
 
     let searcher = reader.searcher();

examples/pre_tokenized_text.rs

@@ -94,7 +94,7 @@ fn main() -> tantivy::Result<()> {
 
     let reader = index
         .reader_builder()
-        .reload_policy(ReloadPolicy::OnCommitWithDelay)
+        .reload_policy(ReloadPolicy::OnCommit)
         .try_into()?;
 
     let searcher = reader.searcher();

examples/snippet.rs

@@ -10,8 +10,7 @@
 use tantivy::collector::TopDocs;
 use tantivy::query::QueryParser;
 use tantivy::schema::*;
-use tantivy::snippet::{Snippet, SnippetGenerator};
-use tantivy::{doc, Index, IndexWriter};
+use tantivy::{doc, Index, IndexWriter, Snippet, SnippetGenerator};
 use tempfile::TempDir;
 
 fn main() -> tantivy::Result<()> {

query-grammar/src/query_grammar.rs

@@ -185,7 +185,7 @@ fn term_or_phrase(inp: &str) -> IResult<&str, UserInputLeaf> {
 fn term_or_phrase_infallible(inp: &str) -> JResult<&str, Option<UserInputLeaf>> {
     map(
         // ~* for slop/prefix, ) inside group or ast tree, ^ if boost
-        tuple_infallible((simple_term_infallible(")^"), slop_or_prefix_val)),
+        tuple_infallible((simple_term_infallible("*)^"), slop_or_prefix_val)),
         |((delimiter_phrase, (slop, prefix)), errors)| {
             let leaf = if let Some((delimiter, phrase)) = delimiter_phrase {
                 Some(
@@ -1113,9 +1113,6 @@ mod test {
         test_parse_query_to_ast_helper("'www-form-encoded'", "'www-form-encoded'");
         test_parse_query_to_ast_helper("www-form-encoded", "www-form-encoded");
         test_parse_query_to_ast_helper("www-form-encoded", "www-form-encoded");
-        test_parse_query_to_ast_helper("mr james bo?d", "(*mr *james *bo?d)");
-        test_parse_query_to_ast_helper("mr james bo*", "(*mr *james *bo*)");
-        test_parse_query_to_ast_helper("mr james b*d", "(*mr *james *b*d)");
     }
 
     #[test]

src/aggregation/agg_bench.rs

@@ -48,7 +48,7 @@ mod bench {
         let score_field_f64 = schema_builder.add_f64_field("score_f64", score_fieldtype.clone());
         let score_field_i64 = schema_builder.add_i64_field("score_i64", score_fieldtype);
         let index = Index::create_from_tempdir(schema_builder.build())?;
-        let few_terms_data = ["INFO", "ERROR", "WARN", "DEBUG"];
+        let few_terms_data = vec!["INFO", "ERROR", "WARN", "DEBUG"];
 
         let lg_norm = rand_distr::LogNormal::new(2.996f64, 0.979f64).unwrap();
 
@@ -85,7 +85,7 @@ mod bench {
             if cardinality == Cardinality::Sparse {
                 doc_with_value /= 20;
             }
-            let _val_max = 1_000_000.0;
+            let val_max = 1_000_000.0;
             for _ in 0..doc_with_value {
                 let val: f64 = rng.gen_range(0.0..1_000_000.0);
                 let json = if rng.gen_bool(0.1) {

src/aggregation/agg_limits.rs

@@ -73,9 +73,9 @@ impl AggregationLimits {
     /// Create a new ResourceLimitGuard, that will release the memory when dropped.
     pub fn new_guard(&self) -> ResourceLimitGuard {
         ResourceLimitGuard {
-            // The counter which is shared between the aggregations for one request.
+            /// The counter which is shared between the aggregations for one request.
             memory_consumption: Arc::clone(&self.memory_consumption),
-            // The memory_limit in bytes
+            /// The memory_limit in bytes
             memory_limit: self.memory_limit,
             allocated_with_the_guard: 0,
         }

src/aggregation/agg_tests.rs

@@ -624,65 +624,6 @@ fn test_aggregation_on_json_object() {
     );
 }
 
-#[test]
-fn test_aggregation_on_nested_json_object() {
-    let mut schema_builder = Schema::builder();
-    let json = schema_builder.add_json_field("json.blub", FAST);
-    let schema = schema_builder.build();
-    let index = Index::create_in_ram(schema);
-    let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
-    index_writer
-        .add_document(doc!(json => json!({"color.dot": "red", "color": {"nested":"red"} })))
-        .unwrap();
-    index_writer
-        .add_document(doc!(json => json!({"color.dot": "blue", "color": {"nested":"blue"} })))
-        .unwrap();
-    index_writer.commit().unwrap();
-    let reader = index.reader().unwrap();
-    let searcher = reader.searcher();
-
-    let agg: Aggregations = serde_json::from_value(json!({
-        "jsonagg1": {
-            "terms": {
-                "field": "json\\.blub.color\\.dot",
-            }
-        },
-        "jsonagg2": {
-            "terms": {
-                "field": "json\\.blub.color.nested",
-            }
-        }
-
-    }))
-    .unwrap();
-
-    let aggregation_collector = get_collector(agg);
-    let aggregation_results = searcher.search(&AllQuery, &aggregation_collector).unwrap();
-    let aggregation_res_json = serde_json::to_value(aggregation_results).unwrap();
-    assert_eq!(
-        &aggregation_res_json,
-        &serde_json::json!({
-            "jsonagg1": {
-                "buckets": [
-                    {"doc_count": 1, "key": "blue"},
-                    {"doc_count": 1, "key": "red"}
-                ],
-                "doc_count_error_upper_bound": 0,
-                "sum_other_doc_count": 0
-            },
-            "jsonagg2": {
-                "buckets": [
-                    {"doc_count": 1, "key": "blue"},
-                    {"doc_count": 1, "key": "red"}
-                ],
-                "doc_count_error_upper_bound": 0,
-                "sum_other_doc_count": 0
-            }
-
-        })
-    );
-}
-
 #[test]
 fn test_aggregation_on_json_object_empty_columns() {
     let mut schema_builder = Schema::builder();

src/core/index.rs

@@ -18,12 +18,11 @@ use crate::directory::{Directory, ManagedDirectory, RamDirectory, INDEX_WRITER_L
 use crate::error::{DataCorruption, TantivyError};
 use crate::indexer::index_writer::{MAX_NUM_THREAD, MEMORY_BUDGET_NUM_BYTES_MIN};
 use crate::indexer::segment_updater::save_metas;
-use crate::indexer::IndexWriter;
 use crate::reader::{IndexReader, IndexReaderBuilder};
 use crate::schema::document::Document;
 use crate::schema::{Field, FieldType, Schema};
 use crate::tokenizer::{TextAnalyzer, TokenizerManager};
-use crate::{merge_field_meta_data, FieldMetadata, SegmentReader};
+use crate::IndexWriter;
 
 fn load_metas(
     directory: &dyn Directory,
@@ -490,28 +489,6 @@ impl Index {
         self.inventory.all()
     }
 
-    /// Returns the list of fields that have been indexed in the Index.
-    /// The field list includes the field defined in the schema as well as the fields
-    /// that have been indexed as a part of a JSON field.
-    /// The returned field name is the full field name, including the name of the JSON field.
-    ///
-    /// The returned field names can be used in queries.
-    ///
-    /// Notice: If your data contains JSON fields this is **very expensive**, as it requires
-    /// browsing through the inverted index term dictionary and the columnar field dictionary.
-    ///
-    /// Disclaimer: Some fields may not be listed here. For instance, if the schema contains a json
-    /// field that is not indexed nor a fast field but is stored, it is possible for the field
-    /// to not be listed.
-    pub fn fields_metadata(&self) -> crate::Result<Vec<FieldMetadata>> {
-        let segments = self.searchable_segments()?;
-        let fields_metadata: Vec<Vec<FieldMetadata>> = segments
-            .into_iter()
-            .map(|segment| SegmentReader::open(&segment)?.fields_metadata())
-            .collect::<Result<_, _>>()?;
-        Ok(merge_field_meta_data(fields_metadata, &self.schema()))
-    }
-
     /// Creates a new segment_meta (Advanced user only).
     ///
     /// As long as the `SegmentMeta` lives, the files associated with the
@@ -589,7 +566,7 @@ impl Index {
     /// Using a single thread gives us a deterministic allocation of DocId.
     #[cfg(test)]
     pub fn writer_for_tests<D: Document>(&self) -> crate::Result<IndexWriter<D>> {
-        self.writer_with_num_threads(1, MEMORY_BUDGET_NUM_BYTES_MIN)
+        self.writer_with_num_threads(1, 15_000_000)
     }
 
     /// Creates a multithreaded writer

src/core/inverted_index_reader.rs

@@ -75,7 +75,7 @@ impl InvertedIndexReader {
     ///
     /// Notice: This requires a full scan and therefore **very expensive**.
     /// TODO: Move to sstable to use the index.
-    pub fn list_encoded_fields(&self) -> io::Result<Vec<(String, Type)>> {
+    pub fn list_fields(&self) -> io::Result<Vec<(String, Type)>> {
         let mut stream = self.termdict.stream()?;
         let mut fields = Vec::new();
         let mut fields_set = FnvHashSet::default();

src/core/json_utils.rs

@@ -1,11 +1,12 @@
 use columnar::MonotonicallyMappableToU64;
-use common::{replace_in_place, JsonPathWriter};
+use common::replace_in_place;
+use murmurhash32::murmurhash2;
 use rustc_hash::FxHashMap;
 
 use crate::fastfield::FastValue;
 use crate::postings::{IndexingContext, IndexingPosition, PostingsWriter};
-use crate::schema::document::{ReferenceValue, ReferenceValueLeaf, Value};
-use crate::schema::term::JSON_PATH_SEGMENT_SEP;
+use crate::schema::document::{ReferenceValue, Value};
+use crate::schema::term::{JSON_PATH_SEGMENT_SEP, JSON_PATH_SEGMENT_SEP_STR};
 use crate::schema::{Field, Type, DATE_TIME_PRECISION_INDEXED};
 use crate::time::format_description::well_known::Rfc3339;
 use crate::time::{OffsetDateTime, UtcOffset};
@@ -57,20 +58,13 @@ struct IndexingPositionsPerPath {
 }
 
 impl IndexingPositionsPerPath {
-    fn get_position_from_id(&mut self, id: u32) -> &mut IndexingPosition {
-        self.positions_per_path.entry(id).or_default()
+    fn get_position(&mut self, term: &Term) -> &mut IndexingPosition {
+        self.positions_per_path
+            .entry(murmurhash2(term.serialized_term()))
+            .or_default()
     }
 }
 
-/// Convert JSON_PATH_SEGMENT_SEP to a dot.
-pub fn json_path_sep_to_dot(path: &mut str) {
-    // This is safe since we are replacing a ASCII character by another ASCII character.
-    unsafe {
-        replace_in_place(JSON_PATH_SEGMENT_SEP, b'.', path.as_bytes_mut());
-    }
-}
-
-#[allow(clippy::too_many_arguments)]
 pub(crate) fn index_json_values<'a, V: Value<'a>>(
     doc: DocId,
     json_visitors: impl Iterator<Item = crate::Result<V::ObjectIter>>,
@@ -78,11 +72,9 @@ pub(crate) fn index_json_values<'a, V: Value<'a>>(
     expand_dots_enabled: bool,
     term_buffer: &mut Term,
     postings_writer: &mut dyn PostingsWriter,
-    json_path_writer: &mut JsonPathWriter,
     ctx: &mut IndexingContext,
 ) -> crate::Result<()> {
-    json_path_writer.clear();
-    json_path_writer.set_expand_dots(expand_dots_enabled);
+    let mut json_term_writer = JsonTermWriter::wrap(term_buffer, expand_dots_enabled);
     let mut positions_per_path: IndexingPositionsPerPath = Default::default();
     for json_visitor_res in json_visitors {
         let json_visitor = json_visitor_res?;
@@ -90,8 +82,7 @@ pub(crate) fn index_json_values<'a, V: Value<'a>>(
             doc,
             json_visitor,
             text_analyzer,
-            term_buffer,
-            json_path_writer,
+            &mut json_term_writer,
             postings_writer,
             ctx,
             &mut positions_per_path,
@@ -100,141 +91,94 @@ pub(crate) fn index_json_values<'a, V: Value<'a>>(
     Ok(())
 }
 
-#[allow(clippy::too_many_arguments)]
 fn index_json_object<'a, V: Value<'a>>(
     doc: DocId,
     json_visitor: V::ObjectIter,
     text_analyzer: &mut TextAnalyzer,
-    term_buffer: &mut Term,
-    json_path_writer: &mut JsonPathWriter,
+    json_term_writer: &mut JsonTermWriter,
     postings_writer: &mut dyn PostingsWriter,
     ctx: &mut IndexingContext,
     positions_per_path: &mut IndexingPositionsPerPath,
 ) {
     for (json_path_segment, json_value_visitor) in json_visitor {
-        json_path_writer.push(json_path_segment);
+        json_term_writer.push_path_segment(json_path_segment);
         index_json_value(
             doc,
             json_value_visitor,
             text_analyzer,
-            term_buffer,
-            json_path_writer,
+            json_term_writer,
             postings_writer,
             ctx,
             positions_per_path,
         );
-        json_path_writer.pop();
+        json_term_writer.pop_path_segment();
     }
 }
 
-#[allow(clippy::too_many_arguments)]
 fn index_json_value<'a, V: Value<'a>>(
     doc: DocId,
     json_value: V,
     text_analyzer: &mut TextAnalyzer,
-    term_buffer: &mut Term,
-    json_path_writer: &mut JsonPathWriter,
+    json_term_writer: &mut JsonTermWriter,
     postings_writer: &mut dyn PostingsWriter,
     ctx: &mut IndexingContext,
     positions_per_path: &mut IndexingPositionsPerPath,
 ) {
-    let set_path_id = |term_buffer: &mut Term, unordered_id: u32| {
-        term_buffer.truncate_value_bytes(0);
-        term_buffer.append_bytes(&unordered_id.to_be_bytes());
-    };
-    let set_type = |term_buffer: &mut Term, typ: Type| {
-        term_buffer.append_bytes(&[typ.to_code()]);
-    };
-
     match json_value.as_value() {
-        ReferenceValue::Leaf(leaf) => match leaf {
-            ReferenceValueLeaf::Null => {}
-            ReferenceValueLeaf::Str(val) => {
-                let mut token_stream = text_analyzer.token_stream(val);
-                let unordered_id = ctx
-                    .path_to_unordered_id
-                    .get_or_allocate_unordered_id(json_path_writer.as_str());
+        ReferenceValue::Null => {}
+        ReferenceValue::Str(val) => {
+            let mut token_stream = text_analyzer.token_stream(val);
 
-                // TODO: make sure the chain position works out.
-                set_path_id(term_buffer, unordered_id);
-                set_type(term_buffer, Type::Str);
-                let indexing_position = positions_per_path.get_position_from_id(unordered_id);
-                postings_writer.index_text(
-                    doc,
-                    &mut *token_stream,
-                    term_buffer,
-                    ctx,
-                    indexing_position,
-                );
-            }
-            ReferenceValueLeaf::U64(val) => {
-                set_path_id(
-                    term_buffer,
-                    ctx.path_to_unordered_id
-                        .get_or_allocate_unordered_id(json_path_writer.as_str()),
-                );
-                term_buffer.append_type_and_fast_value(val);
-                postings_writer.subscribe(doc, 0u32, term_buffer, ctx);
-            }
-            ReferenceValueLeaf::I64(val) => {
-                set_path_id(
-                    term_buffer,
-                    ctx.path_to_unordered_id
-                        .get_or_allocate_unordered_id(json_path_writer.as_str()),
-                );
-                term_buffer.append_type_and_fast_value(val);
-                postings_writer.subscribe(doc, 0u32, term_buffer, ctx);
-            }
-            ReferenceValueLeaf::F64(val) => {
-                set_path_id(
-                    term_buffer,
-                    ctx.path_to_unordered_id
-                        .get_or_allocate_unordered_id(json_path_writer.as_str()),
-                );
-                term_buffer.append_type_and_fast_value(val);
-                postings_writer.subscribe(doc, 0u32, term_buffer, ctx);
-            }
-            ReferenceValueLeaf::Bool(val) => {
-                set_path_id(
-                    term_buffer,
-                    ctx.path_to_unordered_id
-                        .get_or_allocate_unordered_id(json_path_writer.as_str()),
-                );
-                term_buffer.append_type_and_fast_value(val);
-                postings_writer.subscribe(doc, 0u32, term_buffer, ctx);
-            }
-            ReferenceValueLeaf::Date(val) => {
-                set_path_id(
-                    term_buffer,
-                    ctx.path_to_unordered_id
-                        .get_or_allocate_unordered_id(json_path_writer.as_str()),
-                );
-                term_buffer.append_type_and_fast_value(val);
-                postings_writer.subscribe(doc, 0u32, term_buffer, ctx);
-            }
-            ReferenceValueLeaf::PreTokStr(_) => {
-                unimplemented!(
-                    "Pre-tokenized string support in dynamic fields is not yet implemented"
-                )
-            }
-            ReferenceValueLeaf::Bytes(_) => {
-                unimplemented!("Bytes support in dynamic fields is not yet implemented")
-            }
-            ReferenceValueLeaf::Facet(_) => {
-                unimplemented!("Facet support in dynamic fields is not yet implemented")
-            }
-            ReferenceValueLeaf::IpAddr(_) => {
-                unimplemented!("IP address support in dynamic fields is not yet implemented")
-            }
-        },
+            // TODO: make sure the chain position works out.
+            json_term_writer.close_path_and_set_type(Type::Str);
+            let indexing_position = positions_per_path.get_position(json_term_writer.term());
+            postings_writer.index_text(
+                doc,
+                &mut *token_stream,
+                json_term_writer.term_buffer,
+                ctx,
+                indexing_position,
+            );
+        }
+        ReferenceValue::U64(val) => {
+            json_term_writer.set_fast_value(val);
+            postings_writer.subscribe(doc, 0u32, json_term_writer.term(), ctx);
+        }
+        ReferenceValue::I64(val) => {
+            json_term_writer.set_fast_value(val);
+            postings_writer.subscribe(doc, 0u32, json_term_writer.term(), ctx);
+        }
+        ReferenceValue::F64(val) => {
+            json_term_writer.set_fast_value(val);
+            postings_writer.subscribe(doc, 0u32, json_term_writer.term(), ctx);
+        }
+        ReferenceValue::Bool(val) => {
+            json_term_writer.set_fast_value(val);
+            postings_writer.subscribe(doc, 0u32, json_term_writer.term(), ctx);
+        }
+        ReferenceValue::Facet(_) => {
+            unimplemented!("Facet support in dynamic fields is not yet implemented")
+        }
+        ReferenceValue::IpAddr(_) => {
+            unimplemented!("IP address support in dynamic fields is not yet implemented")
+        }
+        ReferenceValue::Date(val) => {
+            json_term_writer.set_fast_value(val);
+            postings_writer.subscribe(doc, 0u32, json_term_writer.term(), ctx);
+        }
+        ReferenceValue::PreTokStr(_) => {
+            unimplemented!("Pre-tokenized string support in dynamic fields is not yet implemented")
+        }
+        ReferenceValue::Bytes(_) => {
+            unimplemented!("Bytes support in dynamic fields is not yet implemented")
+        }
         ReferenceValue::Array(elements) => {
             for val in elements {
                 index_json_value(
                     doc,
                     val,
                     text_analyzer,
-                    term_buffer,
-                    json_path_writer,
+                    json_term_writer,
                     postings_writer,
                     ctx,
                     positions_per_path,
@@ -246,8 +190,7 @@ fn index_json_value<'a, V: Value<'a>>(
                 doc,
                 object,
                 text_analyzer,
-                term_buffer,
-                json_path_writer,
+                json_term_writer,
                 postings_writer,
                 ctx,
                 positions_per_path,
@@ -328,7 +271,7 @@ pub struct JsonTermWriter<'a> {
 /// In other words,
 /// - `k8s.node` ends up as `["k8s", "node"]`.
 /// - `k8s\.node` ends up as `["k8s.node"]`.
-pub fn split_json_path(json_path: &str) -> Vec<String> {
+fn split_json_path(json_path: &str) -> Vec<String> {
     let mut escaped_state: bool = false;
     let mut json_path_segments = Vec::new();
     let mut buffer = String::new();
@@ -368,13 +311,17 @@ pub(crate) fn encode_column_name(
     json_path: &str,
     expand_dots_enabled: bool,
 ) -> String {
-    let mut path = JsonPathWriter::default();
-    path.push(field_name);
-    path.set_expand_dots(expand_dots_enabled);
-    for segment in split_json_path(json_path) {
-        path.push(&segment);
+    let mut column_key: String = String::with_capacity(field_name.len() + json_path.len() + 1);
+    column_key.push_str(field_name);
+    for mut segment in split_json_path(json_path) {
+        column_key.push_str(JSON_PATH_SEGMENT_SEP_STR);
+        if expand_dots_enabled {
+            // We need to replace `.` by JSON_PATH_SEGMENT_SEP.
+            unsafe { replace_in_place(b'.', JSON_PATH_SEGMENT_SEP, segment.as_bytes_mut()) };
+        }
+        column_key.push_str(&segment);
     }
-    path.into()
+    column_key
 }
 
 impl<'a> JsonTermWriter<'a> {
@@ -414,7 +361,6 @@ impl<'a> JsonTermWriter<'a> {
         self.term_buffer.append_bytes(&[typ.to_code()]);
     }
 
-    // TODO: Remove this function and use JsonPathWriter instead.
     pub fn push_path_segment(&mut self, segment: &str) {
         // the path stack should never be empty.
         self.trim_to_end_of_path();

src/core/mod.rs

@@ -25,7 +25,7 @@ pub use self::searcher::{Searcher, SearcherGeneration};
 pub use self::segment::Segment;
 pub use self::segment_component::SegmentComponent;
 pub use self::segment_id::SegmentId;
-pub use self::segment_reader::{merge_field_meta_data, FieldMetadata, SegmentReader};
+pub use self::segment_reader::SegmentReader;
 pub use self::single_segment_index_writer::SingleSegmentIndexWriter;
 
 /// The meta file contains all the information about the list of segments and the schema

src/core/searcher.rs

@@ -5,7 +5,7 @@ use std::{fmt, io};
 use crate::collector::Collector;
 use crate::core::{Executor, SegmentReader};
 use crate::query::{Bm25StatisticsProvider, EnableScoring, Query};
-use crate::schema::document::DocumentDeserialize;
+use crate::schema::document::{Document, DocumentDeserialize};
 use crate::schema::{Schema, Term};
 use crate::space_usage::SearcherSpaceUsage;
 use crate::store::{CacheStats, StoreReader};

src/core/segment_reader.rs

@@ -1,17 +1,12 @@
 use std::collections::HashMap;
-use std::ops::BitOrAssign;
 use std::sync::{Arc, RwLock};
 use std::{fmt, io};
 
-use fnv::FnvHashMap;
-use itertools::Itertools;
-
 use crate::core::{InvertedIndexReader, Segment, SegmentComponent, SegmentId};
 use crate::directory::{CompositeFile, FileSlice};
 use crate::error::DataCorruption;
 use crate::fastfield::{intersect_alive_bitsets, AliveBitSet, FacetReader, FastFieldReaders};
 use crate::fieldnorm::{FieldNormReader, FieldNormReaders};
-use crate::json_utils::json_path_sep_to_dot;
 use crate::schema::{Field, IndexRecordOption, Schema, Type};
 use crate::space_usage::SegmentSpaceUsage;
 use crate::store::StoreReader;
@@ -285,103 +280,6 @@ impl SegmentReader {
         Ok(inv_idx_reader)
     }
 
-    /// Returns the list of fields that have been indexed in the segment.
-    /// The field list includes the field defined in the schema as well as the fields
-    /// that have been indexed as a part of a JSON field.
-    /// The returned field name is the full field name, including the name of the JSON field.
-    ///
-    /// The returned field names can be used in queries.
-    ///
-    /// Notice: If your data contains JSON fields this is **very expensive**, as it requires
-    /// browsing through the inverted index term dictionary and the columnar field dictionary.
-    ///
-    /// Disclaimer: Some fields may not be listed here. For instance, if the schema contains a json
-    /// field that is not indexed nor a fast field but is stored, it is possible for the field
-    /// to not be listed.
-    pub fn fields_metadata(&self) -> crate::Result<Vec<FieldMetadata>> {
-        let mut indexed_fields: Vec<FieldMetadata> = Vec::new();
-        let mut map_to_canonical = FnvHashMap::default();
-        for (field, field_entry) in self.schema().fields() {
-            let field_name = field_entry.name().to_string();
-            let is_indexed = field_entry.is_indexed();
-
-            if is_indexed {
-                let is_json = field_entry.field_type().value_type() == Type::Json;
-                if is_json {
-                    let inv_index = self.inverted_index(field)?;
-                    let encoded_fields_in_index = inv_index.list_encoded_fields()?;
-                    let mut build_path = |field_name: &str, mut json_path: String| {
-                        // In this case we need to map the potential fast field to the field name
-                        // accepted by the query parser.
-                        let create_canonical =
-                            !field_entry.is_expand_dots_enabled() && json_path.contains('.');
-                        if create_canonical {
-                            // Without expand dots enabled dots need to be escaped.
-                            let escaped_json_path = json_path.replace('.', "\\.");
-                            let full_path = format!("{}.{}", field_name, escaped_json_path);
-                            let full_path_unescaped = format!("{}.{}", field_name, &json_path);
-                            map_to_canonical.insert(full_path_unescaped, full_path.to_string());
-                            full_path
-                        } else {
-                            // With expand dots enabled, we can use '.' instead of '\u{1}'.
-                            json_path_sep_to_dot(&mut json_path);
-                            format!("{}.{}", field_name, json_path)
-                        }
-                    };
-                    indexed_fields.extend(
-                        encoded_fields_in_index
-                            .into_iter()
-                            .map(|(name, typ)| (build_path(&field_name, name), typ))
-                            .map(|(field_name, typ)| FieldMetadata {
-                                indexed: true,
-                                stored: false,
-                                field_name,
-                                fast: false,
-                                typ,
-                            }),
-                    );
-                } else {
-                    indexed_fields.push(FieldMetadata {
-                        indexed: true,
-                        stored: false,
-                        field_name: field_name.to_string(),
-                        fast: false,
-                        typ: field_entry.field_type().value_type(),
-                    });
-                }
-            }
-        }
-        let mut fast_fields: Vec<FieldMetadata> = self
-            .fast_fields()
-            .columnar()
-            .iter_columns()?
-            .map(|(mut field_name, handle)| {
-                json_path_sep_to_dot(&mut field_name);
-                // map to canonical path, to avoid similar but different entries.
-                // Eventually we should just accept '.' seperated for all cases.
-                let field_name = map_to_canonical
-                    .get(&field_name)
-                    .unwrap_or(&field_name)
-                    .to_string();
-                FieldMetadata {
-                    indexed: false,
-                    stored: false,
-                    field_name,
-                    fast: true,
-                    typ: Type::from(handle.column_type()),
-                }
-            })
-            .collect();
-        // Since the type is encoded differently in the fast field and in the inverted index,
-        // the order of the fields is not guaranteed to be the same. Therefore, we sort the fields.
-        // If we are sure that the order is the same, we can remove this sort.
-        indexed_fields.sort_unstable();
-        fast_fields.sort_unstable();
-        let merged = merge_field_meta_data(vec![indexed_fields, fast_fields], &self.schema);
-
-        Ok(merged)
-    }
-
     /// Returns the segment id
     pub fn segment_id(&self) -> SegmentId {
         self.segment_id
@@ -432,65 +330,6 @@ impl SegmentReader {
     }
 }
 
-#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
-/// FieldMetadata
-pub struct FieldMetadata {
-    /// The field name
-    // Notice: Don't reorder the declaration of 1.field_name 2.typ, as it is used for ordering by
-    // field_name then typ.
-    pub field_name: String,
-    /// The field type
-    // Notice: Don't reorder the declaration of 1.field_name 2.typ, as it is used for ordering by
-    // field_name then typ.
-    pub typ: Type,
-    /// Is the field indexed for search
-    pub indexed: bool,
-    /// Is the field stored in the doc store
-    pub stored: bool,
-    /// Is the field stored in the columnar storage
-    pub fast: bool,
-}
-impl BitOrAssign for FieldMetadata {
-    fn bitor_assign(&mut self, rhs: Self) {
-        assert!(self.field_name == rhs.field_name);
-        assert!(self.typ == rhs.typ);
-        self.indexed |= rhs.indexed;
-        self.stored |= rhs.stored;
-        self.fast |= rhs.fast;
-    }
-}
-
-// Maybe too slow for the high cardinality case
-fn is_field_stored(field_name: &str, schema: &Schema) -> bool {
-    schema
-        .find_field(field_name)
-        .map(|(field, _path)| schema.get_field_entry(field).is_stored())
-        .unwrap_or(false)
-}
-
-/// Helper to merge the field metadata from multiple segments.
-pub fn merge_field_meta_data(
-    field_metadatas: Vec<Vec<FieldMetadata>>,
-    schema: &Schema,
-) -> Vec<FieldMetadata> {
-    let mut merged_field_metadata = Vec::new();
-    for (_key, mut group) in &field_metadatas
-        .into_iter()
-        .kmerge_by(|left, right| left < right)
-        // TODO: Remove allocation
-        .group_by(|el| (el.field_name.to_string(), el.typ))
-    {
-        let mut merged: FieldMetadata = group.next().unwrap();
-        for el in group {
-            merged |= el;
-        }
-        // Currently is_field_stored is maybe too slow for the high cardinality case
-        merged.stored = is_field_stored(&merged.field_name, schema);
-        merged_field_metadata.push(merged);
-    }
-    merged_field_metadata
-}
-
 fn intersect_alive_bitset(
     left_opt: Option<AliveBitSet>,
     right_opt: Option<AliveBitSet>,
@@ -514,127 +353,9 @@ impl fmt::Debug for SegmentReader {
 
 #[cfg(test)]
 mod test {
-    use super::*;
     use crate::core::Index;
-    use crate::schema::{Schema, SchemaBuilder, Term, STORED, TEXT};
-    use crate::{DocId, FieldMetadata, IndexWriter};
-
-    #[test]
-    fn test_merge_field_meta_data_same() {
-        let schema = SchemaBuilder::new().build();
-        let field_metadata1 = FieldMetadata {
-            field_name: "a".to_string(),
-            typ: crate::schema::Type::Str,
-            indexed: true,
-            stored: false,
-            fast: true,
-        };
-        let field_metadata2 = FieldMetadata {
-            field_name: "a".to_string(),
-            typ: crate::schema::Type::Str,
-            indexed: true,
-            stored: false,
-            fast: true,
-        };
-        let res = merge_field_meta_data(
-            vec![vec![field_metadata1.clone()], vec![field_metadata2]],
-            &schema,
-        );
-        assert_eq!(res, vec![field_metadata1]);
-    }
-    #[test]
-    fn test_merge_field_meta_data_different() {
-        let schema = SchemaBuilder::new().build();
-        let field_metadata1 = FieldMetadata {
-            field_name: "a".to_string(),
-            typ: crate::schema::Type::Str,
-            indexed: false,
-            stored: false,
-            fast: true,
-        };
-        let field_metadata2 = FieldMetadata {
-            field_name: "b".to_string(),
-            typ: crate::schema::Type::Str,
-            indexed: false,
-            stored: false,
-            fast: true,
-        };
-        let field_metadata3 = FieldMetadata {
-            field_name: "a".to_string(),
-            typ: crate::schema::Type::Str,
-            indexed: true,
-            stored: false,
-            fast: false,
-        };
-        let res = merge_field_meta_data(
-            vec![
-                vec![field_metadata1.clone(), field_metadata2.clone()],
-                vec![field_metadata3],
-            ],
-            &schema,
-        );
-        let field_metadata_expected1 = FieldMetadata {
-            field_name: "a".to_string(),
-            typ: crate::schema::Type::Str,
-            indexed: true,
-            stored: false,
-            fast: true,
-        };
-        assert_eq!(res, vec![field_metadata_expected1, field_metadata2.clone()]);
-    }
-    #[test]
-    fn test_merge_field_meta_data_merge() {
-        use pretty_assertions::assert_eq;
-        let get_meta_data = |name: &str, typ: Type| FieldMetadata {
-            field_name: name.to_string(),
-            typ,
-            indexed: false,
-            stored: false,
-            fast: true,
-        };
-        let schema = SchemaBuilder::new().build();
-        let mut metas = vec![get_meta_data("d", Type::Str), get_meta_data("e", Type::U64)];
-        metas.sort();
-        let res = merge_field_meta_data(vec![vec![get_meta_data("e", Type::Str)], metas], &schema);
-        assert_eq!(
-            res,
-            vec![
-                get_meta_data("d", Type::Str),
-                get_meta_data("e", Type::Str),
-                get_meta_data("e", Type::U64),
-            ]
-        );
-    }
-    #[test]
-    fn test_merge_field_meta_data_bitxor() {
-        let field_metadata1 = FieldMetadata {
-            field_name: "a".to_string(),
-            typ: crate::schema::Type::Str,
-            indexed: false,
-            stored: false,
-            fast: true,
-        };
-        let field_metadata2 = FieldMetadata {
-            field_name: "a".to_string(),
-            typ: crate::schema::Type::Str,
-            indexed: true,
-            stored: false,
-            fast: false,
-        };
-        let field_metadata_expected = FieldMetadata {
-            field_name: "a".to_string(),
-            typ: crate::schema::Type::Str,
-            indexed: true,
-            stored: false,
-            fast: true,
-        };
-        let mut res1 = field_metadata1.clone();
-        res1 |= field_metadata2.clone();
-        let mut res2 = field_metadata2.clone();
-        res2 |= field_metadata1;
-        assert_eq!(res1, field_metadata_expected);
-        assert_eq!(res2, field_metadata_expected);
-    }
+    use crate::schema::{Schema, Term, STORED, TEXT};
+    use crate::{DocId, IndexWriter};
 
     #[test]
     fn test_num_alive() -> crate::Result<()> {

src/core/tests.rs

@@ -1,13 +1,12 @@
 use crate::collector::Count;
 use crate::directory::{RamDirectory, WatchCallback};
-use crate::indexer::{LogMergePolicy, NoMergePolicy};
-use crate::json_utils::JsonTermWriter;
+use crate::indexer::NoMergePolicy;
 use crate::query::TermQuery;
-use crate::schema::{Field, IndexRecordOption, Schema, Type, INDEXED, STRING, TEXT};
+use crate::schema::{Field, IndexRecordOption, Schema, INDEXED, STRING, TEXT};
 use crate::tokenizer::TokenizerManager;
 use crate::{
-    Directory, DocSet, Index, IndexBuilder, IndexReader, IndexSettings, IndexWriter, Postings,
-    ReloadPolicy, SegmentId, TantivyDocument, Term,
+    Directory, Index, IndexBuilder, IndexReader, IndexSettings, IndexWriter, ReloadPolicy,
+    SegmentId, TantivyDocument, Term,
 };
 
 #[test]
@@ -122,7 +121,7 @@ fn test_index_on_commit_reload_policy() -> crate::Result<()> {
     let index = Index::create_in_ram(schema);
     let reader = index
         .reader_builder()
-        .reload_policy(ReloadPolicy::OnCommitWithDelay)
+        .reload_policy(ReloadPolicy::OnCommit)
         .try_into()
         .unwrap();
     assert_eq!(reader.searcher().num_docs(), 0);
@@ -148,7 +147,7 @@ mod mmap_specific {
         let index = Index::create_in_dir(tempdir_path, schema).unwrap();
         let reader = index
             .reader_builder()
-            .reload_policy(ReloadPolicy::OnCommitWithDelay)
+            .reload_policy(ReloadPolicy::OnCommit)
             .try_into()
             .unwrap();
         assert_eq!(reader.searcher().num_docs(), 0);
@@ -190,7 +189,7 @@ mod mmap_specific {
         let read_index = Index::open_in_dir(&tempdir_path).unwrap();
         let reader = read_index
             .reader_builder()
-            .reload_policy(ReloadPolicy::OnCommitWithDelay)
+            .reload_policy(ReloadPolicy::OnCommit)
             .try_into()
             .unwrap();
         assert_eq!(reader.searcher().num_docs(), 0);
@@ -345,132 +344,3 @@ fn test_merging_segment_update_docfreq() {
     let term_info = inv_index.get_term_info(&term).unwrap().unwrap();
     assert_eq!(term_info.doc_freq, 12);
 }
-
-// motivated by https://github.com/quickwit-oss/quickwit/issues/4130
-#[test]
-fn test_positions_merge_bug_non_text_json_vint() {
-    let mut schema_builder = Schema::builder();
-    let field = schema_builder.add_json_field("dynamic", TEXT);
-    let schema = schema_builder.build();
-    let index = Index::create_in_ram(schema.clone());
-    let mut writer: IndexWriter = index.writer_for_tests().unwrap();
-    let mut merge_policy = LogMergePolicy::default();
-    merge_policy.set_min_num_segments(2);
-    writer.set_merge_policy(Box::new(merge_policy));
-    // Here a string would work.
-    let doc_json = r#"{"tenant_id":75}"#;
-    let vals = serde_json::from_str(doc_json).unwrap();
-    let mut doc = TantivyDocument::default();
-    doc.add_object(field, vals);
-    writer.add_document(doc.clone()).unwrap();
-    writer.commit().unwrap();
-    writer.add_document(doc.clone()).unwrap();
-    writer.commit().unwrap();
-    writer.wait_merging_threads().unwrap();
-    let reader = index.reader().unwrap();
-    assert_eq!(reader.searcher().segment_readers().len(), 1);
-}
-
-// Same as above but with bitpacked blocks
-#[test]
-fn test_positions_merge_bug_non_text_json_bitpacked_block() {
-    let mut schema_builder = Schema::builder();
-    let field = schema_builder.add_json_field("dynamic", TEXT);
-    let schema = schema_builder.build();
-    let index = Index::create_in_ram(schema.clone());
-    let mut writer: IndexWriter = index.writer_for_tests().unwrap();
-    let mut merge_policy = LogMergePolicy::default();
-    merge_policy.set_min_num_segments(2);
-    writer.set_merge_policy(Box::new(merge_policy));
-    // Here a string would work.
-    let doc_json = r#"{"tenant_id":75}"#;
-    let vals = serde_json::from_str(doc_json).unwrap();
-    let mut doc = TantivyDocument::default();
-    doc.add_object(field, vals);
-    for _ in 0..128 {
-        writer.add_document(doc.clone()).unwrap();
-    }
-    writer.commit().unwrap();
-    writer.add_document(doc.clone()).unwrap();
-    writer.commit().unwrap();
-    writer.wait_merging_threads().unwrap();
-    let reader = index.reader().unwrap();
-    assert_eq!(reader.searcher().segment_readers().len(), 1);
-}
-
-#[test]
-fn test_non_text_json_term_freq() {
-    let mut schema_builder = Schema::builder();
-    let field = schema_builder.add_json_field("dynamic", TEXT);
-    let schema = schema_builder.build();
-    let index = Index::create_in_ram(schema.clone());
-    let mut writer: IndexWriter = index.writer_for_tests().unwrap();
-    // Here a string would work.
-    let doc_json = r#"{"tenant_id":75}"#;
-    let vals = serde_json::from_str(doc_json).unwrap();
-    let mut doc = TantivyDocument::default();
-    doc.add_object(field, vals);
-    writer.add_document(doc.clone()).unwrap();
-    writer.commit().unwrap();
-    let reader = index.reader().unwrap();
-    assert_eq!(reader.searcher().segment_readers().len(), 1);
-    let searcher = reader.searcher();
-    let segment_reader = searcher.segment_reader(0u32);
-    let inv_idx = segment_reader.inverted_index(field).unwrap();
-    let mut term = Term::with_type_and_field(Type::Json, field);
-    let mut json_term_writer = JsonTermWriter::wrap(&mut term, false);
-    json_term_writer.push_path_segment("tenant_id");
-    json_term_writer.close_path_and_set_type(Type::U64);
-    json_term_writer.set_fast_value(75u64);
-    let postings = inv_idx
-        .read_postings(
-            &json_term_writer.term(),
-            IndexRecordOption::WithFreqsAndPositions,
-        )
-        .unwrap()
-        .unwrap();
-    assert_eq!(postings.doc(), 0);
-    assert_eq!(postings.term_freq(), 1u32);
-}
-
-#[test]
-fn test_non_text_json_term_freq_bitpacked() {
-    let mut schema_builder = Schema::builder();
-    let field = schema_builder.add_json_field("dynamic", TEXT);
-    let schema = schema_builder.build();
-    let index = Index::create_in_ram(schema.clone());
-    let mut writer: IndexWriter = index.writer_for_tests().unwrap();
-    // Here a string would work.
-    let doc_json = r#"{"tenant_id":75}"#;
-    let vals = serde_json::from_str(doc_json).unwrap();
-    let mut doc = TantivyDocument::default();
-    doc.add_object(field, vals);
-    let num_docs = 132;
-    for _ in 0..num_docs {
-        writer.add_document(doc.clone()).unwrap();
-    }
-    writer.commit().unwrap();
-    let reader = index.reader().unwrap();
-    assert_eq!(reader.searcher().segment_readers().len(), 1);
-    let searcher = reader.searcher();
-    let segment_reader = searcher.segment_reader(0u32);
-    let inv_idx = segment_reader.inverted_index(field).unwrap();
-    let mut term = Term::with_type_and_field(Type::Json, field);
-    let mut json_term_writer = JsonTermWriter::wrap(&mut term, false);
-    json_term_writer.push_path_segment("tenant_id");
-    json_term_writer.close_path_and_set_type(Type::U64);
-    json_term_writer.set_fast_value(75u64);
-    let mut postings = inv_idx
-        .read_postings(
-            &json_term_writer.term(),
-            IndexRecordOption::WithFreqsAndPositions,
-        )
-        .unwrap()
-        .unwrap();
-    assert_eq!(postings.doc(), 0);
-    assert_eq!(postings.term_freq(), 1u32);
-    for i in 1..num_docs {
-        assert_eq!(postings.advance(), i);
-        assert_eq!(postings.term_freq(), 1u32);
-    }
-}

src/directory/directory.rs

@@ -222,8 +222,8 @@ pub trait Directory: DirectoryClone + fmt::Debug + Send + Sync + 'static {
     /// registered (and whose [`WatchHandle`] is still alive) are triggered.
     ///
     /// Internally, tantivy only uses this API to detect new commits to implement the
-    /// `OnCommitWithDelay` `ReloadPolicy`. Not implementing watch in a `Directory` only prevents
-    /// the `OnCommitWithDelay` `ReloadPolicy` to work properly.
+    /// `OnCommit` `ReloadPolicy`. Not implementing watch in a `Directory` only prevents the
+    /// `OnCommit` `ReloadPolicy` to work properly.
     fn watch(&self, watch_callback: WatchCallback) -> crate::Result<WatchHandle>;
 }
 

src/directory/footer.rs

@@ -7,7 +7,7 @@ use serde::{Deserialize, Serialize};
 
 use crate::directory::error::Incompatibility;
 use crate::directory::{AntiCallToken, FileSlice, TerminatingWrite};
-use crate::{Version, INDEX_FORMAT_OLDEST_SUPPORTED_VERSION, INDEX_FORMAT_VERSION};
+use crate::{Version, INDEX_FORMAT_VERSION};
 
 const FOOTER_MAX_LEN: u32 = 50_000;
 
@@ -102,11 +102,10 @@ impl Footer {
     /// Confirms that the index will be read correctly by this version of tantivy
     /// Has to be called after `extract_footer` to make sure it's not accessing uninitialised memory
     pub fn is_compatible(&self) -> Result<(), Incompatibility> {
-        const SUPPORTED_INDEX_FORMAT_VERSION_RANGE: std::ops::RangeInclusive<u32> =
-            INDEX_FORMAT_OLDEST_SUPPORTED_VERSION..=INDEX_FORMAT_VERSION;
-
         let library_version = crate::version();
-        if !SUPPORTED_INDEX_FORMAT_VERSION_RANGE.contains(&self.version.index_format_version) {
+        if self.version.index_format_version < 4
+            || self.version.index_format_version > INDEX_FORMAT_VERSION
+        {
             return Err(Incompatibility::IndexMismatch {
                 library_version: library_version.clone(),
                 index_version: self.version.clone(),

src/directory/mmap_directory.rs

@@ -8,8 +8,6 @@ use std::sync::{Arc, RwLock, Weak};
 
 use common::StableDeref;
 use fs4::FileExt;
-#[cfg(all(feature = "mmap", unix))]
-pub use memmap2::Advice;
 use memmap2::Mmap;
 use serde::{Deserialize, Serialize};
 use tempfile::TempDir;
@@ -23,6 +21,8 @@ use crate::directory::{
     AntiCallToken, Directory, DirectoryLock, FileHandle, Lock, OwnedBytes, TerminatingWrite,
     WatchCallback, WatchHandle, WritePtr,
 };
+#[cfg(unix)]
+use crate::Advice;
 
 pub type ArcBytes = Arc<dyn Deref<Target = [u8]> + Send + Sync + 'static>;
 pub type WeakArcBytes = Weak<dyn Deref<Target = [u8]> + Send + Sync + 'static>;

src/directory/mod.rs

@@ -42,9 +42,6 @@ pub struct GarbageCollectionResult {
     pub failed_to_delete_files: Vec<PathBuf>,
 }
 
-#[cfg(all(feature = "mmap", unix))]
-pub use memmap2::Advice;
-
 pub use self::managed_directory::ManagedDirectory;
 #[cfg(feature = "mmap")]
 pub use self::mmap_directory::MmapDirectory;

src/docset.rs

@@ -17,7 +17,7 @@ pub trait DocSet: Send {
     ///
     /// The DocId of the next element is returned.
     /// In other words we should always have :
-    /// ```compile_fail
+    /// ```ignore
     /// let doc = docset.advance();
     /// assert_eq!(doc, docset.doc());
     /// ```

src/fastfield/mod.rs

@@ -131,7 +131,7 @@ mod tests {
         }
         let file = directory.open_read(path).unwrap();
 
-        assert_eq!(file.len(), 80);
+        assert_eq!(file.len(), 93);
         let fast_field_readers = FastFieldReaders::open(file, SCHEMA.clone()).unwrap();
         let column = fast_field_readers
             .u64("field")
@@ -181,7 +181,7 @@ mod tests {
             write.terminate().unwrap();
         }
         let file = directory.open_read(path).unwrap();
-        assert_eq!(file.len(), 108);
+        assert_eq!(file.len(), 121);
         let fast_field_readers = FastFieldReaders::open(file, SCHEMA.clone()).unwrap();
         let col = fast_field_readers
             .u64("field")
@@ -214,7 +214,7 @@ mod tests {
             write.terminate().unwrap();
         }
         let file = directory.open_read(path).unwrap();
-        assert_eq!(file.len(), 81);
+        assert_eq!(file.len(), 94);
         let fast_field_readers = FastFieldReaders::open(file, SCHEMA.clone()).unwrap();
         let fast_field_reader = fast_field_readers
             .u64("field")
@@ -246,7 +246,7 @@ mod tests {
             write.terminate().unwrap();
         }
         let file = directory.open_read(path).unwrap();
-        assert_eq!(file.len(), 4476);
+        assert_eq!(file.len(), 4489);
         {
             let fast_field_readers = FastFieldReaders::open(file, SCHEMA.clone()).unwrap();
             let col = fast_field_readers
@@ -279,7 +279,7 @@ mod tests {
             write.terminate().unwrap();
         }
         let file = directory.open_read(path).unwrap();
-        assert_eq!(file.len(), 252);
+        assert_eq!(file.len(), 265);
 
         {
             let fast_field_readers = FastFieldReaders::open(file, schema).unwrap();
@@ -773,7 +773,7 @@ mod tests {
             write.terminate().unwrap();
         }
         let file = directory.open_read(path).unwrap();
-        assert_eq!(file.len(), 84);
+        assert_eq!(file.len(), 102);
         let fast_field_readers = FastFieldReaders::open(file, schema).unwrap();
         let bool_col = fast_field_readers.bool("field_bool").unwrap();
         assert_eq!(bool_col.first(0), Some(true));
@@ -805,7 +805,7 @@ mod tests {
             write.terminate().unwrap();
         }
         let file = directory.open_read(path).unwrap();
-        assert_eq!(file.len(), 96);
+        assert_eq!(file.len(), 114);
         let readers = FastFieldReaders::open(file, schema).unwrap();
         let bool_col = readers.bool("field_bool").unwrap();
         for i in 0..25 {
@@ -830,7 +830,7 @@ mod tests {
             write.terminate().unwrap();
         }
         let file = directory.open_read(path).unwrap();
-        assert_eq!(file.len(), 86);
+        assert_eq!(file.len(), 104);
         let fastfield_readers = FastFieldReaders::open(file, schema).unwrap();
         let col = fastfield_readers.bool("field_bool").unwrap();
         assert_eq!(col.first(0), None);
@@ -1288,18 +1288,11 @@ mod tests {
         index_writer.commit().unwrap();
         let searcher = index.reader().unwrap().searcher();
         let fast_field_reader = searcher.segment_reader(0u32).fast_fields();
-        // Supported for now, maybe dropped in the future.
         let column = fast_field_reader
             .column_opt::<i64>("jsonfield.attr.age")
             .unwrap()
             .unwrap();
         let vals: Vec<i64> = column.values_for_doc(0u32).collect();
         assert_eq!(&vals, &[33]);
-        let column = fast_field_reader
-            .column_opt::<i64>("jsonfield\\.attr.age")
-            .unwrap()
-            .unwrap();
-        let vals: Vec<i64> = column.values_for_doc(0u32).collect();
-        assert_eq!(&vals, &[33]);
     }
 }

src/fastfield/writer.rs

@@ -1,11 +1,12 @@
 use std::io;
 
 use columnar::{ColumnarWriter, NumericalValue};
-use common::JsonPathWriter;
+use common::replace_in_place;
 use tokenizer_api::Token;
 
 use crate::indexer::doc_id_mapping::DocIdMapping;
-use crate::schema::document::{Document, ReferenceValue, ReferenceValueLeaf, Value};
+use crate::schema::document::{Document, ReferenceValue, Value};
+use crate::schema::term::{JSON_PATH_SEGMENT_SEP, JSON_PATH_SEGMENT_SEP_STR};
 use crate::schema::{value_type_to_column_type, Field, FieldType, Schema, Type};
 use crate::tokenizer::{TextAnalyzer, TokenizerManager};
 use crate::{DateTimePrecision, DocId, TantivyError};
@@ -23,7 +24,7 @@ pub struct FastFieldsWriter {
     expand_dots: Vec<bool>,
     num_docs: DocId,
     // Buffer that we recycle to avoid allocation.
-    json_path_buffer: JsonPathWriter,
+    json_path_buffer: String,
 }
 
 impl FastFieldsWriter {
@@ -97,7 +98,7 @@ impl FastFieldsWriter {
             num_docs: 0u32,
             date_precisions,
             expand_dots,
-            json_path_buffer: JsonPathWriter::default(),
+            json_path_buffer: String::new(),
         })
     }
 
@@ -140,68 +141,64 @@ impl FastFieldsWriter {
         };
 
         match value.as_value() {
-            ReferenceValue::Leaf(leaf) => match leaf {
-                ReferenceValueLeaf::Null => {}
-                ReferenceValueLeaf::Str(val) => {
-                    if let Some(tokenizer) =
-                        &mut self.per_field_tokenizer[field.field_id() as usize]
-                    {
-                        let mut token_stream = tokenizer.token_stream(val);
-                        token_stream.process(&mut |token: &Token| {
-                            self.columnar_writer
-                                .record_str(doc_id, field_name, &token.text);
-                        })
-                    } else {
-                        self.columnar_writer.record_str(doc_id, field_name, val);
-                    }
-                }
-                ReferenceValueLeaf::U64(val) => {
-                    self.columnar_writer.record_numerical(
-                        doc_id,
-                        field_name,
-                        NumericalValue::from(val),
-                    );
-                }
-                ReferenceValueLeaf::I64(val) => {
-                    self.columnar_writer.record_numerical(
-                        doc_id,
-                        field_name,
-                        NumericalValue::from(val),
-                    );
-                }
-                ReferenceValueLeaf::F64(val) => {
-                    self.columnar_writer.record_numerical(
-                        doc_id,
-                        field_name,
-                        NumericalValue::from(val),
-                    );
-                }
-                ReferenceValueLeaf::Date(val) => {
-                    let date_precision = self.date_precisions[field.field_id() as usize];
-                    let truncated_datetime = val.truncate(date_precision);
-                    self.columnar_writer
-                        .record_datetime(doc_id, field_name, truncated_datetime);
-                }
-                ReferenceValueLeaf::Facet(val) => {
-                    self.columnar_writer
-                        .record_str(doc_id, field_name, val.encoded_str());
-                }
-                ReferenceValueLeaf::Bytes(val) => {
-                    self.columnar_writer.record_bytes(doc_id, field_name, val);
-                }
-                ReferenceValueLeaf::IpAddr(val) => {
-                    self.columnar_writer.record_ip_addr(doc_id, field_name, val);
-                }
-                ReferenceValueLeaf::Bool(val) => {
-                    self.columnar_writer.record_bool(doc_id, field_name, val);
-                }
-                ReferenceValueLeaf::PreTokStr(val) => {
-                    for token in &val.tokens {
+            ReferenceValue::Null => {}
+            ReferenceValue::Str(val) => {
+                if let Some(tokenizer) = &mut self.per_field_tokenizer[field.field_id() as usize] {
+                    let mut token_stream = tokenizer.token_stream(val);
+                    token_stream.process(&mut |token: &Token| {
                         self.columnar_writer
                             .record_str(doc_id, field_name, &token.text);
-                    }
+                    })
+                } else {
+                    self.columnar_writer.record_str(doc_id, field_name, val);
                 }
-            },
+            }
+            ReferenceValue::U64(val) => {
+                self.columnar_writer.record_numerical(
+                    doc_id,
+                    field_name,
+                    NumericalValue::from(val),
+                );
+            }
+            ReferenceValue::I64(val) => {
+                self.columnar_writer.record_numerical(
+                    doc_id,
+                    field_name,
+                    NumericalValue::from(val),
+                );
+            }
+            ReferenceValue::F64(val) => {
+                self.columnar_writer.record_numerical(
+                    doc_id,
+                    field_name,
+                    NumericalValue::from(val),
+                );
+            }
+            ReferenceValue::Date(val) => {
+                let date_precision = self.date_precisions[field.field_id() as usize];
+                let truncated_datetime = val.truncate(date_precision);
+                self.columnar_writer
+                    .record_datetime(doc_id, field_name, truncated_datetime);
+            }
+            ReferenceValue::Facet(val) => {
+                self.columnar_writer
+                    .record_str(doc_id, field_name, val.encoded_str());
+            }
+            ReferenceValue::Bytes(val) => {
+                self.columnar_writer.record_bytes(doc_id, field_name, val);
+            }
+            ReferenceValue::IpAddr(val) => {
+                self.columnar_writer.record_ip_addr(doc_id, field_name, val);
+            }
+            ReferenceValue::Bool(val) => {
+                self.columnar_writer.record_bool(doc_id, field_name, val);
+            }
+            ReferenceValue::PreTokStr(val) => {
+                for token in &val.tokens {
+                    self.columnar_writer
+                        .record_str(doc_id, field_name, &token.text);
+                }
+            }
             ReferenceValue::Array(val) => {
                 // TODO: Check this is the correct behaviour we want.
                 for value in val {
@@ -211,16 +208,14 @@ impl FastFieldsWriter {
             ReferenceValue::Object(val) => {
                 let expand_dots = self.expand_dots[field.field_id() as usize];
                 self.json_path_buffer.clear();
-                // First field should not be expanded.
-                self.json_path_buffer.set_expand_dots(false);
-                self.json_path_buffer.push(field_name);
-                self.json_path_buffer.set_expand_dots(expand_dots);
+                self.json_path_buffer.push_str(field_name);
 
                 let text_analyzer = &mut self.per_field_tokenizer[field.field_id() as usize];
 
                 record_json_obj_to_columnar_writer::<V>(
                     doc_id,
                     val,
+                    expand_dots,
                     JSON_DEPTH_LIMIT,
                     &mut self.json_path_buffer,
                     &mut self.columnar_writer,
@@ -251,30 +246,48 @@ impl FastFieldsWriter {
 fn record_json_obj_to_columnar_writer<'a, V: Value<'a>>(
     doc: DocId,
     json_visitor: V::ObjectIter,
+    expand_dots: bool,
     remaining_depth_limit: usize,
-    json_path_buffer: &mut JsonPathWriter,
+    json_path_buffer: &mut String,
     columnar_writer: &mut columnar::ColumnarWriter,
     tokenizer: &mut Option<TextAnalyzer>,
 ) {
     for (key, child) in json_visitor {
-        json_path_buffer.push(key);
+        let len_path = json_path_buffer.len();
+        if !json_path_buffer.is_empty() {
+            json_path_buffer.push_str(JSON_PATH_SEGMENT_SEP_STR);
+        }
+        json_path_buffer.push_str(key);
+        if expand_dots {
+            // This might include the separation byte, which is ok because it is not a dot.
+            let appended_segment = &mut json_path_buffer[len_path..];
+            // The unsafe below is safe as long as b'.' and JSON_PATH_SEGMENT_SEP are
+            // valid single byte ut8 strings.
+            // By utf-8 design, they cannot be part of another codepoint.
+            replace_in_place(b'.', JSON_PATH_SEGMENT_SEP, unsafe {
+                appended_segment.as_bytes_mut()
+            });
+        }
         record_json_value_to_columnar_writer(
             doc,
             child,
+            expand_dots,
             remaining_depth_limit,
             json_path_buffer,
             columnar_writer,
             tokenizer,
         );
-        json_path_buffer.pop();
+        // popping our sub path.
+        json_path_buffer.truncate(len_path);
     }
 }
 
 fn record_json_value_to_columnar_writer<'a, V: Value<'a>>(
     doc: DocId,
     json_val: V,
+    expand_dots: bool,
     mut remaining_depth_limit: usize,
-    json_path_writer: &mut JsonPathWriter,
+    json_path_writer: &mut String,
     columnar_writer: &mut columnar::ColumnarWriter,
     tokenizer: &mut Option<TextAnalyzer>,
 ) {
@@ -284,67 +297,64 @@ fn record_json_value_to_columnar_writer<'a, V: Value<'a>>(
     remaining_depth_limit -= 1;
 
     match json_val.as_value() {
-        ReferenceValue::Leaf(leaf) => match leaf {
-            ReferenceValueLeaf::Null => {} // TODO: Handle null
-            ReferenceValueLeaf::Str(val) => {
-                if let Some(text_analyzer) = tokenizer.as_mut() {
-                    let mut token_stream = text_analyzer.token_stream(val);
-                    token_stream.process(&mut |token| {
-                        columnar_writer.record_str(doc, json_path_writer.as_str(), &token.text);
-                    })
-                } else {
-                    columnar_writer.record_str(doc, json_path_writer.as_str(), val);
-                }
+        ReferenceValue::Null => {} // TODO: Handle null
+        ReferenceValue::Str(val) => {
+            if let Some(text_analyzer) = tokenizer.as_mut() {
+                let mut token_stream = text_analyzer.token_stream(val);
+                token_stream.process(&mut |token| {
+                    columnar_writer.record_str(doc, json_path_writer.as_str(), &token.text);
+                })
+            } else {
+                columnar_writer.record_str(doc, json_path_writer.as_str(), val);
             }
-            ReferenceValueLeaf::U64(val) => {
-                columnar_writer.record_numerical(
-                    doc,
-                    json_path_writer.as_str(),
-                    NumericalValue::from(val),
-                );
-            }
-            ReferenceValueLeaf::I64(val) => {
-                columnar_writer.record_numerical(
-                    doc,
-                    json_path_writer.as_str(),
-                    NumericalValue::from(val),
-                );
-            }
-            ReferenceValueLeaf::F64(val) => {
-                columnar_writer.record_numerical(
-                    doc,
-                    json_path_writer.as_str(),
-                    NumericalValue::from(val),
-                );
-            }
-            ReferenceValueLeaf::Bool(val) => {
-                columnar_writer.record_bool(doc, json_path_writer.as_str(), val);
-            }
-            ReferenceValueLeaf::Date(val) => {
-                columnar_writer.record_datetime(doc, json_path_writer.as_str(), val);
-            }
-            ReferenceValueLeaf::Facet(_) => {
-                unimplemented!("Facet support in dynamic fields is not yet implemented")
-            }
-            ReferenceValueLeaf::Bytes(_) => {
-                // TODO: This can be re added once it is added to the JSON Utils section as well.
-                // columnar_writer.record_bytes(doc, json_path_writer.as_str(), val);
-                unimplemented!("Bytes support in dynamic fields is not yet implemented")
-            }
-            ReferenceValueLeaf::IpAddr(_) => {
-                unimplemented!("IP address support in dynamic fields is not yet implemented")
-            }
-            ReferenceValueLeaf::PreTokStr(_) => {
-                unimplemented!(
-                    "Pre-tokenized string support in dynamic fields is not yet implemented"
-                )
-            }
-        },
+        }
+        ReferenceValue::U64(val) => {
+            columnar_writer.record_numerical(
+                doc,
+                json_path_writer.as_str(),
+                NumericalValue::from(val),
+            );
+        }
+        ReferenceValue::I64(val) => {
+            columnar_writer.record_numerical(
+                doc,
+                json_path_writer.as_str(),
+                NumericalValue::from(val),
+            );
+        }
+        ReferenceValue::F64(val) => {
+            columnar_writer.record_numerical(
+                doc,
+                json_path_writer.as_str(),
+                NumericalValue::from(val),
+            );
+        }
+        ReferenceValue::Bool(val) => {
+            columnar_writer.record_bool(doc, json_path_writer, val);
+        }
+        ReferenceValue::Date(val) => {
+            columnar_writer.record_datetime(doc, json_path_writer.as_str(), val);
+        }
+        ReferenceValue::Facet(_) => {
+            unimplemented!("Facet support in dynamic fields is not yet implemented")
+        }
+        ReferenceValue::Bytes(_) => {
+            // TODO: This can be re added once it is added to the JSON Utils section as well.
+            // columnar_writer.record_bytes(doc, json_path_writer.as_str(), val);
+            unimplemented!("Bytes support in dynamic fields is not yet implemented")
+        }
+        ReferenceValue::IpAddr(_) => {
+            unimplemented!("IP address support in dynamic fields is not yet implemented")
+        }
+        ReferenceValue::PreTokStr(_) => {
+            unimplemented!("Pre-tokenized string support in dynamic fields is not yet implemented")
+        }
         ReferenceValue::Array(elements) => {
             for el in elements {
                 record_json_value_to_columnar_writer(
                     doc,
                     el,
+                    expand_dots,
                     remaining_depth_limit,
                     json_path_writer,
                     columnar_writer,
@@ -356,6 +366,7 @@ fn record_json_value_to_columnar_writer<'a, V: Value<'a>>(
             record_json_obj_to_columnar_writer::<V>(
                 doc,
                 object,
+                expand_dots,
                 remaining_depth_limit,
                 json_path_writer,
                 columnar_writer,
@@ -368,7 +379,6 @@ fn record_json_value_to_columnar_writer<'a, V: Value<'a>>(
 #[cfg(test)]
 mod tests {
     use columnar::{Column, ColumnarReader, ColumnarWriter, StrColumn};
-    use common::JsonPathWriter;
 
     use super::record_json_value_to_columnar_writer;
     use crate::fastfield::writer::JSON_DEPTH_LIMIT;
@@ -379,12 +389,12 @@ mod tests {
         expand_dots: bool,
     ) -> ColumnarReader {
         let mut columnar_writer = ColumnarWriter::default();
-        let mut json_path = JsonPathWriter::default();
-        json_path.set_expand_dots(expand_dots);
+        let mut json_path = String::new();
         for (doc, json_doc) in json_docs.iter().enumerate() {
             record_json_value_to_columnar_writer(
                 doc as u32,
                 json_doc,
+                expand_dots,
                 JSON_DEPTH_LIMIT,
                 &mut json_path,
                 &mut columnar_writer,

src/functional_test.rs

@@ -32,7 +32,7 @@ fn test_functional_store() -> crate::Result<()> {
     let mut rng = thread_rng();
 
     let mut index_writer: IndexWriter =
-        index.writer_with_num_threads(3, 3 * MEMORY_BUDGET_NUM_BYTES_MIN)?;
+        index.writer_with_num_threads(3, MEMORY_BUDGET_NUM_BYTES_MIN)?;
 
     let mut doc_set: Vec<u64> = Vec::new();
 
@@ -92,8 +92,7 @@ fn test_functional_indexing_sorted() -> crate::Result<()> {
 
     let mut rng = thread_rng();
 
-    let mut index_writer: IndexWriter =
-        index.writer_with_num_threads(3, 3 * MEMORY_BUDGET_NUM_BYTES_MIN)?;
+    let mut index_writer: IndexWriter = index.writer_with_num_threads(3, 120_000_000)?;
 
     let mut committed_docs: HashSet<u64> = HashSet::new();
     let mut uncommitted_docs: HashSet<u64> = HashSet::new();
@@ -168,8 +167,7 @@ fn test_functional_indexing_unsorted() -> crate::Result<()> {
 
     let mut rng = thread_rng();
 
-    let mut index_writer: IndexWriter =
-        index.writer_with_num_threads(3, 3 * MEMORY_BUDGET_NUM_BYTES_MIN)?;
+    let mut index_writer: IndexWriter = index.writer_with_num_threads(3, 120_000_000)?;
 
     let mut committed_docs: HashSet<u64> = HashSet::new();
     let mut uncommitted_docs: HashSet<u64> = HashSet::new();

src/indexer/index_writer.rs

@@ -1651,7 +1651,6 @@ mod tests {
         force_end_merge: bool,
     ) -> crate::Result<Index> {
         let mut schema_builder = schema::Schema::builder();
-        let json_field = schema_builder.add_json_field("json", FAST | TEXT | STORED);
         let ip_field = schema_builder.add_ip_addr_field("ip", FAST | INDEXED | STORED);
         let ips_field = schema_builder
             .add_ip_addr_field("ips", IpAddrOptions::default().set_fast().set_indexed());
@@ -1707,8 +1706,7 @@ mod tests {
 
         let old_reader = index.reader()?;
 
-        // Every 3rd doc has only id field
-        let id_is_full_doc = |id| id % 3 != 0;
+        let id_exists = |id| id % 3 != 0; // 0 does not exist
 
         let multi_text_field_text1 = "test1 test2 test3 test1 test2 test3";
         // rotate left
@@ -1724,15 +1722,13 @@ mod tests {
                     let facet = Facet::from(&("/cola/".to_string() + &id.to_string()));
                     let ip = ip_from_id(id);
 
-                    if !id_is_full_doc(id) {
+                    if !id_exists(id) {
                         // every 3rd doc has no ip field
                         index_writer.add_document(doc!(
                             id_field=>id,
                         ))?;
                     } else {
-                        let json = json!({"date1": format!("2022-{id}-01T00:00:01Z"), "date2": format!("{id}-05-01T00:00:01Z"), "id": id, "ip": ip.to_string()});
                         index_writer.add_document(doc!(id_field=>id,
-                                json_field=>json,
                                 bytes_field => id.to_le_bytes().as_slice(),
                                 id_opt_field => id,
                                 ip_field => ip,
@@ -1846,7 +1842,7 @@ mod tests {
 
         let num_docs_with_values = expected_ids_and_num_occurrences
             .iter()
-            .filter(|(id, _id_occurrences)| id_is_full_doc(**id))
+            .filter(|(id, _id_occurrences)| id_exists(**id))
             .map(|(_, id_occurrences)| *id_occurrences as usize)
             .sum::<usize>();
 
@@ -1870,7 +1866,7 @@ mod tests {
         if force_end_merge && num_segments_before_merge > 1 && num_segments_after_merge == 1 {
             let mut expected_multi_ips: Vec<_> = id_list
                 .iter()
-                .filter(|id| id_is_full_doc(**id))
+                .filter(|id| id_exists(**id))
                 .flat_map(|id| vec![ip_from_id(*id), ip_from_id(*id)])
                 .collect();
             assert_eq!(num_ips, expected_multi_ips.len() as u32);
@@ -1908,7 +1904,7 @@ mod tests {
         let expected_ips = expected_ids_and_num_occurrences
             .keys()
             .flat_map(|id| {
-                if !id_is_full_doc(*id) {
+                if !id_exists(*id) {
                     None
                 } else {
                     Some(Ipv6Addr::from_u128(*id as u128))
@@ -1920,7 +1916,7 @@ mod tests {
         let expected_ips = expected_ids_and_num_occurrences
             .keys()
             .filter_map(|id| {
-                if !id_is_full_doc(*id) {
+                if !id_exists(*id) {
                     None
                 } else {
                     Some(Ipv6Addr::from_u128(*id as u128))
@@ -1955,7 +1951,7 @@ mod tests {
                 let id = id_reader.first(doc).unwrap();
 
                 let vals: Vec<u64> = ff_reader.values_for_doc(doc).collect();
-                if id_is_full_doc(id) {
+                if id_exists(id) {
                     assert_eq!(vals.len(), 2);
                     assert_eq!(vals[0], vals[1]);
                     assert!(expected_ids_and_num_occurrences.contains_key(&vals[0]));
@@ -1965,7 +1961,7 @@ mod tests {
                 }
 
                 let bool_vals: Vec<bool> = bool_ff_reader.values_for_doc(doc).collect();
-                if id_is_full_doc(id) {
+                if id_exists(id) {
                     assert_eq!(bool_vals.len(), 2);
                     assert_ne!(bool_vals[0], bool_vals[1]);
                 } else {
@@ -1994,7 +1990,7 @@ mod tests {
                     .as_u64()
                     .unwrap();
                 assert!(expected_ids_and_num_occurrences.contains_key(&id));
-                if id_is_full_doc(id) {
+                if id_exists(id) {
                     let id2 = store_reader
                         .get::<TantivyDocument>(doc_id)
                         .unwrap()
@@ -2041,7 +2037,7 @@ mod tests {
             let (existing_id, count) = (*id, *count);
             let get_num_hits = |field| do_search(&existing_id.to_string(), field).len() as u64;
             assert_eq!(get_num_hits(id_field), count);
-            if !id_is_full_doc(existing_id) {
+            if !id_exists(existing_id) {
                 continue;
             }
             assert_eq!(get_num_hits(text_field), count);
@@ -2091,7 +2087,7 @@ mod tests {
         //
         for (existing_id, count) in &expected_ids_and_num_occurrences {
             let (existing_id, count) = (*existing_id, *count);
-            if !id_is_full_doc(existing_id) {
+            if !id_exists(existing_id) {
                 continue;
             }
             let do_search_ip_field = |term: &str| do_search(term, ip_field).len() as u64;
@@ -2108,84 +2104,34 @@ mod tests {
             }
         }
 
-        // Range query
+        // assert data is like expected
         //
-        // Take half as sample
-        let mut sample: Vec<_> = expected_ids_and_num_occurrences.iter().collect();
-        sample.sort_by_key(|(k, _num_occurences)| *k);
-        // sample.truncate(sample.len() / 2);
-        if !sample.is_empty() {
-            let (left_sample, right_sample) = sample.split_at(sample.len() / 2);
-
-            let expected_count = |sample: &[(&u64, &u64)]| {
-                sample
-                    .iter()
-                    .filter(|(id, _)| id_is_full_doc(**id))
-                    .map(|(_id, num_occurences)| **num_occurences)
-                    .sum::<u64>()
-            };
-            fn gen_query_inclusive<T1: ToString, T2: ToString>(
-                field: &str,
-                from: T1,
-                to: T2,
-            ) -> String {
+        for (existing_id, count) in expected_ids_and_num_occurrences.iter().take(10) {
+            let (existing_id, count) = (*existing_id, *count);
+            if !id_exists(existing_id) {
+                continue;
+            }
+            let gen_query_inclusive = |field: &str, from: Ipv6Addr, to: Ipv6Addr| {
                 format!("{}:[{} TO {}]", field, &from.to_string(), &to.to_string())
-            }
+            };
+            let ip = ip_from_id(existing_id);
 
-            // Query first half
-            if !left_sample.is_empty() {
-                let expected_count = expected_count(left_sample);
+            let do_search_ip_field = |term: &str| do_search(term, ip_field).len() as u64;
+            // Range query on single value field
+            let query = gen_query_inclusive("ip", ip, ip);
+            assert_eq!(do_search_ip_field(&query), count);
 
-                let start_range = *left_sample[0].0;
-                let end_range = *left_sample.last().unwrap().0;
-                let query = gen_query_inclusive("id_opt", start_range, end_range);
-                assert_eq!(do_search(&query, id_opt_field).len() as u64, expected_count);
+            // Range query on multi value field
+            let query = gen_query_inclusive("ips", ip, ip);
 
-                // Range query on ip field
-                let ip1 = ip_from_id(start_range);
-                let ip2 = ip_from_id(end_range);
-                let do_search_ip_field = |term: &str| do_search(term, ip_field).len() as u64;
-                let query = gen_query_inclusive("ip", ip1, ip2);
-                assert_eq!(do_search_ip_field(&query), expected_count);
-                let query = gen_query_inclusive("ip", "*", ip2);
-                assert_eq!(do_search_ip_field(&query), expected_count);
-                // Range query on multi value field
-                let query = gen_query_inclusive("ips", ip1, ip2);
-                assert_eq!(do_search_ip_field(&query), expected_count);
-                let query = gen_query_inclusive("ips", "*", ip2);
-                assert_eq!(do_search_ip_field(&query), expected_count);
-            }
-            // Query second half
-            if !right_sample.is_empty() {
-                let expected_count = expected_count(right_sample);
-                let start_range = *right_sample[0].0;
-                let end_range = *right_sample.last().unwrap().0;
-                // Range query on id opt field
-                let query =
-                    gen_query_inclusive("id_opt", start_range.to_string(), end_range.to_string());
-                assert_eq!(do_search(&query, id_opt_field).len() as u64, expected_count);
-
-                // Range query on ip field
-                let ip1 = ip_from_id(start_range);
-                let ip2 = ip_from_id(end_range);
-                let do_search_ip_field = |term: &str| do_search(term, ip_field).len() as u64;
-                let query = gen_query_inclusive("ip", ip1, ip2);
-                assert_eq!(do_search_ip_field(&query), expected_count);
-                let query = gen_query_inclusive("ip", ip1, "*");
-                assert_eq!(do_search_ip_field(&query), expected_count);
-                // Range query on multi value field
-                let query = gen_query_inclusive("ips", ip1, ip2);
-                assert_eq!(do_search_ip_field(&query), expected_count);
-                let query = gen_query_inclusive("ips", ip1, "*");
-                assert_eq!(do_search_ip_field(&query), expected_count);
-            }
+            assert_eq!(do_search_ip_field(&query), count);
         }
 
         // ip range query on fast field
         //
         for (existing_id, count) in expected_ids_and_num_occurrences.iter().take(10) {
             let (existing_id, count) = (*existing_id, *count);
-            if !id_is_full_doc(existing_id) {
+            if !id_exists(existing_id) {
                 continue;
             }
             let gen_query_inclusive = |field: &str, from: Ipv6Addr, to: Ipv6Addr| {
@@ -2213,7 +2159,7 @@ mod tests {
                 .first_or_default_col(9999);
             for doc_id in segment_reader.doc_ids_alive() {
                 let id = ff_reader.get_val(doc_id);
-                if !id_is_full_doc(id) {
+                if !id_exists(id) {
                     continue;
                 }
                 let facet_ords: Vec<u64> = facet_reader.facet_ords(doc_id).collect();
@@ -2251,12 +2197,6 @@ mod tests {
         Ok(index)
     }
 
-    #[test]
-    fn test_fast_field_range() {
-        let ops: Vec<_> = (0..1000).map(|id| IndexingOp::AddDoc { id }).collect();
-        assert!(test_operation_strategy(&ops, false, true).is_ok());
-    }
-
     #[test]
     fn test_sort_index_on_opt_field_regression() {
         assert!(test_operation_strategy(

src/indexer/merge_operation.rs

@@ -63,13 +63,10 @@ impl MergeOperation {
         }
     }
 
-    /// Returns the opstamp up to which we want to consume the delete queue and reflect their
-    /// deletes.
     pub fn target_opstamp(&self) -> Opstamp {
         self.inner.target_opstamp
     }
 
-    /// Returns the list of segment to be merged.
     pub fn segment_ids(&self) -> &[SegmentId] {
         &self.inner.segment_ids[..]
     }

src/indexer/merger.rs

@@ -552,41 +552,7 @@ impl IndexMerger {
                 continue;
             }
 
-            // This should never happen as we early exited for total_doc_freq == 0.
-            assert!(!segment_postings_containing_the_term.is_empty());
-
-            let has_term_freq = {
-                let has_term_freq = !segment_postings_containing_the_term[0]
-                    .1
-                    .block_cursor
-                    .freqs()
-                    .is_empty();
-                for (_, postings) in &segment_postings_containing_the_term[1..] {
-                    // This may look at a strange way to test whether we have term freq or not.
-                    // With JSON object, the schema is not sufficient to know whether a term
-                    // has its term frequency encoded or not:
-                    // strings may have term frequencies, while number terms never have one.
-                    //
-                    // Ideally, we should have burnt one bit of two in the `TermInfo`.
-                    // However, we preferred not changing the codec too much and detect this
-                    // instead by
-                    // - looking at the size of the skip data for bitpacked blocks
-                    // - observing the absence of remaining data after reading the docs for vint
-                    // blocks.
-                    //
-                    // Overall the reliable way to know if we have actual frequencies loaded or not
-                    // is to check whether the actual decoded array is empty or not.
-                    if has_term_freq != !postings.block_cursor.freqs().is_empty() {
-                        return Err(DataCorruption::comment_only(
-                            "Term freqs are inconsistent across segments",
-                        )
-                        .into());
-                    }
-                }
-                has_term_freq
-            };
-
-            field_serializer.new_term(term_bytes, total_doc_freq, has_term_freq)?;
+            field_serializer.new_term(term_bytes, total_doc_freq)?;
 
             // We can now serialize this postings, by pushing each document to the
             // postings serializer.
@@ -601,17 +567,8 @@ impl IndexMerger {
                     if let Some(remapped_doc_id) = old_to_new_doc_id[doc as usize] {
                         // we make sure to only write the term if
                         // there is at least one document.
-                        let term_freq = if has_term_freq {
-                            segment_postings.positions(&mut positions_buffer);
-                            segment_postings.term_freq()
-                        } else {
-                            // The positions_buffer may contain positions from the previous term
-                            // Existence of positions depend on the value type in JSON fields.
-                            // https://github.com/quickwit-oss/tantivy/issues/2283
-                            positions_buffer.clear();
-                            0u32
-                        };
-
+                        let term_freq = segment_postings.term_freq();
+                        segment_postings.positions(&mut positions_buffer);
                         // if doc_id_mapping exists, the doc_ids are reordered, they are
                         // not just stacked. The field serializer expects monotonically increasing
                         // doc_ids, so we collect and sort them first, before writing.

src/indexer/merger_sorted_index_test.rs

@@ -543,7 +543,7 @@ mod bench_sorted_index_merge {
     //);
     //(doc_addr.doc_id, reader, u64_reader)
     //});
-    /// add values in order of the new doc_ids
+    //// add values in order of the new doc_ids
     // let mut val = 0;
     // for (doc_id, _reader, field_reader) in sorted_doc_ids {
     // val = field_reader.get_val(doc_id);

src/indexer/mod.rs

@@ -1,30 +1,23 @@
-//! Indexing and merging data.
-//!
-//! Contains code to create and merge segments.
-//! `IndexWriter` is the main entry point for that, which created from
-//! [`Index::writer`](crate::Index::writer).
+pub mod delete_queue;
 
-pub(crate) mod delete_queue;
-pub(crate) mod path_to_unordered_id;
-
-pub(crate) mod doc_id_mapping;
+pub mod doc_id_mapping;
 mod doc_opstamp_mapping;
 mod flat_map_with_buffer;
-pub(crate) mod index_writer;
-pub(crate) mod index_writer_status;
+pub mod index_writer;
+mod index_writer_status;
 mod log_merge_policy;
 mod merge_operation;
-pub(crate) mod merge_policy;
-pub(crate) mod merger;
+pub mod merge_policy;
+pub mod merger;
 mod merger_sorted_index_test;
-pub(crate) mod operation;
-pub(crate) mod prepared_commit;
+pub mod operation;
+pub mod prepared_commit;
 mod segment_entry;
 mod segment_manager;
 mod segment_register;
-pub(crate) mod segment_serializer;
-pub(crate) mod segment_updater;
-pub(crate) mod segment_writer;
+pub mod segment_serializer;
+pub mod segment_updater;
+mod segment_writer;
 mod stamper;
 
 use crossbeam_channel as channel;
@@ -34,10 +27,10 @@ pub use self::index_writer::IndexWriter;
 pub use self::log_merge_policy::LogMergePolicy;
 pub use self::merge_operation::MergeOperation;
 pub use self::merge_policy::{MergeCandidate, MergePolicy, NoMergePolicy};
-pub use self::operation::UserOperation;
 pub use self::prepared_commit::PreparedCommit;
 pub use self::segment_entry::SegmentEntry;
-pub(crate) use self::segment_serializer::SegmentSerializer;
+pub use self::segment_manager::SegmentManager;
+pub use self::segment_serializer::SegmentSerializer;
 pub use self::segment_updater::{merge_filtered_segments, merge_indices};
 pub use self::segment_writer::SegmentWriter;
 use crate::indexer::operation::AddOperation;
@@ -59,13 +52,10 @@ type AddBatchReceiver<D> = channel::Receiver<AddBatch<D>>;
 #[cfg(test)]
 mod tests_mmap {
 
-    use crate::aggregation::agg_req::Aggregations;
-    use crate::aggregation::agg_result::AggregationResults;
-    use crate::aggregation::AggregationCollector;
-    use crate::collector::{Count, TopDocs};
-    use crate::query::{AllQuery, QueryParser};
-    use crate::schema::{JsonObjectOptions, Schema, Type, FAST, INDEXED, STORED, TEXT};
-    use crate::{FieldMetadata, Index, IndexWriter, Term};
+    use crate::collector::Count;
+    use crate::query::QueryParser;
+    use crate::schema::{JsonObjectOptions, Schema, Type, TEXT};
+    use crate::{Index, IndexWriter, Term};
 
     #[test]
     fn test_advance_delete_bug() -> crate::Result<()> {
@@ -113,35 +103,6 @@ mod tests_mmap {
         }
     }
 
-    #[test]
-    fn test_json_field_number() {
-        // this test was added specifically to reach some cases related to using json fields, with
-        // frequency enabled, to store integers, with enough documents containing a single integer
-        // that the posting list can be bitpacked.
-        let mut schema_builder = Schema::builder();
-
-        let json_field = schema_builder.add_json_field("json", TEXT);
-        let index = Index::create_in_ram(schema_builder.build());
-        let mut index_writer = index.writer_for_tests().unwrap();
-        for _ in 0..256 {
-            let json = serde_json::json!({"somekey": 1u64, "otherkey": -2i64});
-            index_writer.add_document(doc!(json_field=>json)).unwrap();
-
-            let json = serde_json::json!({"somekey": "1str", "otherkey": "2str"});
-            index_writer.add_document(doc!(json_field=>json)).unwrap();
-        }
-        index_writer.commit().unwrap();
-        let reader = index.reader().unwrap();
-        let searcher = reader.searcher();
-        assert_eq!(searcher.num_docs(), 512);
-        let parse_query = QueryParser::for_index(&index, Vec::new());
-        {
-            let query = parse_query.parse_query(r"json.somekey:1").unwrap();
-            let num_docs = searcher.search(&query, &Count).unwrap();
-            assert_eq!(num_docs, 256);
-        }
-    }
-
     #[test]
     fn test_json_field_expand_dots_enabled_dot_escape_not_required() {
         let mut schema_builder = Schema::builder();
@@ -176,7 +137,8 @@ mod tests_mmap {
     #[test]
     fn test_json_field_list_fields() {
         let mut schema_builder = Schema::builder();
-        let json_options: JsonObjectOptions = JsonObjectOptions::from(TEXT);
+        let json_options: JsonObjectOptions =
+            JsonObjectOptions::from(TEXT).set_expand_dots_enabled();
         let json_field = schema_builder.add_json_field("json", json_options);
         let index = Index::create_in_ram(schema_builder.build());
         let mut index_writer = index.writer_for_tests().unwrap();
@@ -195,9 +157,9 @@ mod tests_mmap {
         let reader = &searcher.segment_readers()[0];
         let inverted_index = reader.inverted_index(json_field).unwrap();
         assert_eq!(
-            inverted_index.list_encoded_fields().unwrap(),
+            inverted_index.list_fields().unwrap(),
             [
-                ("k8s.container.name".to_string(), Type::Str),
+                ("k8s\u{1}container\u{1}name".to_string(), Type::Str),
                 ("sub\u{1}a".to_string(), Type::I64),
                 ("sub\u{1}b".to_string(), Type::I64),
                 ("suber\u{1}a".to_string(), Type::I64),
@@ -207,240 +169,4 @@ mod tests_mmap {
             ]
         );
     }
-
-    #[test]
-    fn test_json_fields_metadata_expanded_dots_one_segment() {
-        test_json_fields_metadata(true, true);
-    }
-    #[test]
-    fn test_json_fields_metadata_expanded_dots_multi_segment() {
-        test_json_fields_metadata(true, false);
-    }
-    #[test]
-    fn test_json_fields_metadata_no_expanded_dots_one_segment() {
-        test_json_fields_metadata(false, true);
-    }
-    #[test]
-    fn test_json_fields_metadata_no_expanded_dots_multi_segment() {
-        test_json_fields_metadata(false, false);
-    }
-
-    fn test_json_fields_metadata(expanded_dots: bool, one_segment: bool) {
-        use pretty_assertions::assert_eq;
-        let mut schema_builder = Schema::builder();
-        let json_options: JsonObjectOptions =
-            JsonObjectOptions::from(TEXT).set_fast(None).set_stored();
-        let json_options = if expanded_dots {
-            json_options.set_expand_dots_enabled()
-        } else {
-            json_options
-        };
-        schema_builder.add_json_field("json.confusing", json_options.clone());
-        let json_field = schema_builder.add_json_field("json.shadow", json_options.clone());
-        let json_field2 = schema_builder.add_json_field("json", json_options.clone());
-        schema_builder.add_json_field("empty_json", json_options);
-        let number_field = schema_builder.add_u64_field("numbers", FAST);
-        schema_builder.add_u64_field("empty", FAST | INDEXED | STORED);
-        let index = Index::create_in_ram(schema_builder.build());
-        let mut index_writer = index.writer_for_tests().unwrap();
-        let json =
-            serde_json::json!({"k8s.container.name": "a", "val": "a", "sub": {"a": 1, "b": 1}});
-        index_writer.add_document(doc!(json_field=>json)).unwrap();
-        let json =
-            serde_json::json!({"k8s.container.name": "a", "val": "a", "suber": {"a": 1, "b": 1}});
-        if !one_segment {
-            index_writer.commit().unwrap();
-        }
-        index_writer.add_document(doc!(json_field=>json)).unwrap();
-        let json = serde_json::json!({"k8s.container.name": "a", "k8s.container.name": "a", "val": "a", "suber": {"a": "a", "b": 1}});
-        index_writer
-            .add_document(doc!(number_field => 50u64, json_field=>json, json_field2=>json!({"shadow": {"val": "a"}})))
-            .unwrap();
-        index_writer.commit().unwrap();
-        let reader = index.reader().unwrap();
-
-        let searcher = reader.searcher();
-        assert_eq!(searcher.num_docs(), 3);
-
-        let fields_metadata = index.fields_metadata().unwrap();
-        assert_eq!(
-            fields_metadata,
-            [
-                FieldMetadata {
-                    field_name: "empty".to_string(),
-                    indexed: true,
-                    stored: true,
-                    fast: true,
-                    typ: Type::U64
-                },
-                FieldMetadata {
-                    field_name: if expanded_dots {
-                        "json.shadow.k8s.container.name".to_string()
-                    } else {
-                        "json.shadow.k8s\\.container\\.name".to_string()
-                    },
-                    indexed: true,
-                    stored: true,
-                    fast: true,
-                    typ: Type::Str
-                },
-                FieldMetadata {
-                    field_name: "json.shadow.sub.a".to_string(),
-                    indexed: true,
-                    stored: true,
-                    fast: true,
-                    typ: Type::I64
-                },
-                FieldMetadata {
-                    field_name: "json.shadow.sub.b".to_string(),
-                    indexed: true,
-                    stored: true,
-                    fast: true,
-                    typ: Type::I64
-                },
-                FieldMetadata {
-                    field_name: "json.shadow.suber.a".to_string(),
-                    indexed: true,
-                    stored: true,
-                    fast: true,
-                    typ: Type::I64
-                },
-                FieldMetadata {
-                    field_name: "json.shadow.suber.a".to_string(),
-                    indexed: true,
-                    stored: true,
-                    fast: true,
-                    typ: Type::Str
-                },
-                FieldMetadata {
-                    field_name: "json.shadow.suber.b".to_string(),
-                    indexed: true,
-                    stored: true,
-                    fast: true,
-                    typ: Type::I64
-                },
-                FieldMetadata {
-                    field_name: "json.shadow.val".to_string(),
-                    indexed: true,
-                    stored: true,
-                    fast: true,
-                    typ: Type::Str
-                },
-                FieldMetadata {
-                    field_name: "numbers".to_string(),
-                    indexed: false,
-                    stored: false,
-                    fast: true,
-                    typ: Type::U64
-                }
-            ]
-        );
-        let query_parser = QueryParser::for_index(&index, vec![]);
-        // Test if returned field name can be queried
-        for indexed_field in fields_metadata.iter().filter(|meta| meta.indexed) {
-            let val = if indexed_field.typ == Type::Str {
-                "a"
-            } else {
-                "1"
-            };
-            let query_str = &format!("{}:{}", indexed_field.field_name, val);
-            let query = query_parser.parse_query(query_str).unwrap();
-            let count_docs = searcher.search(&*query, &TopDocs::with_limit(2)).unwrap();
-            if indexed_field.field_name.contains("empty") || indexed_field.typ == Type::Json {
-                assert_eq!(count_docs.len(), 0);
-            } else {
-                assert!(!count_docs.is_empty(), "{}", indexed_field.field_name);
-            }
-        }
-        // Test if returned field name can be used for aggregation
-        for fast_field in fields_metadata.iter().filter(|meta| meta.fast) {
-            let agg_req_str = json!(
-            {
-              "termagg": {
-                "terms": {
-                  "field": fast_field.field_name,
-                }
-              }
-            });
-
-            let agg_req: Aggregations = serde_json::from_value(agg_req_str).unwrap();
-            let collector = AggregationCollector::from_aggs(agg_req, Default::default());
-            let agg_res: AggregationResults = searcher.search(&AllQuery, &collector).unwrap();
-            let res = serde_json::to_value(agg_res).unwrap();
-            if !fast_field.field_name.contains("empty") && fast_field.typ != Type::Json {
-                assert!(
-                    !res["termagg"]["buckets"].as_array().unwrap().is_empty(),
-                    "{}",
-                    fast_field.field_name
-                );
-            }
-        }
-    }
-
-    #[test]
-    fn test_json_field_shadowing_field_name_bug() {
-        /// This test is only there to display a bug on addressing a field if it gets shadowed
-        /// The issues only occurs if the field name that shadows contains a dot.
-        ///
-        /// Happens independently of the `expand_dots` option. Since that option does not
-        /// affect the field name itself.
-        use pretty_assertions::assert_eq;
-        let mut schema_builder = Schema::builder();
-        let json_options: JsonObjectOptions =
-            JsonObjectOptions::from(TEXT).set_fast(None).set_stored();
-        // let json_options = json_options.set_expand_dots_enabled();
-        let json_field_shadow = schema_builder.add_json_field("json.shadow", json_options.clone());
-        let json_field = schema_builder.add_json_field("json", json_options.clone());
-        let index = Index::create_in_ram(schema_builder.build());
-        let mut index_writer = index.writer_for_tests().unwrap();
-        index_writer
-            .add_document(
-                doc!(json_field_shadow=>json!({"val": "b"}), json_field=>json!({"shadow": {"val": "a"}})),
-            )
-            .unwrap();
-        index_writer.commit().unwrap();
-        let reader = index.reader().unwrap();
-
-        let searcher = reader.searcher();
-
-        let fields_and_vals = vec![
-            // Only way to address or it gets shadowed by `json.shadow` field
-            ("json.shadow\u{1}val".to_string(), "a"), // Succeeds
-            //("json.shadow.val".to_string(), "a"),   // Fails
-            ("json.shadow.val".to_string(), "b"), // Succeeds
-        ];
-
-        let query_parser = QueryParser::for_index(&index, vec![]);
-        // Test if field name can be queried
-        for (indexed_field, val) in fields_and_vals.iter() {
-            let query_str = &format!("{}:{}", indexed_field, val);
-            let query = query_parser.parse_query(query_str).unwrap();
-            let count_docs = searcher.search(&*query, &TopDocs::with_limit(2)).unwrap();
-            assert!(!count_docs.is_empty(), "{}:{}", indexed_field, val);
-        }
-        // Test if field name can be used for aggregation
-        for (field_name, val) in fields_and_vals.iter() {
-            let agg_req_str = json!(
-            {
-              "termagg": {
-                "terms": {
-                  "field": field_name,
-                }
-              }
-            });
-
-            let agg_req: Aggregations = serde_json::from_value(agg_req_str).unwrap();
-            let collector = AggregationCollector::from_aggs(agg_req, Default::default());
-            let agg_res: AggregationResults = searcher.search(&AllQuery, &collector).unwrap();
-            let res = serde_json::to_value(agg_res).unwrap();
-            assert_eq!(
-                res["termagg"]["buckets"].as_array().unwrap()[0]["key"]
-                    .as_str()
-                    .unwrap(),
-                *val,
-                "{}",
-                field_name
-            );
-        }
-    }
 }

src/indexer/path_to_unordered_id.rs

@@ -1,92 +0,0 @@
-use fnv::FnvHashMap;
-
-/// `Field` is represented by an unsigned 32-bit integer type.
-/// The schema holds the mapping between field names and `Field` objects.
-#[derive(Copy, Default, Clone, Debug, PartialEq, PartialOrd, Eq, Ord, Hash)]
-pub struct OrderedPathId(u32);
-
-impl OrderedPathId {
-    /// Create a new field object for the given PathId.
-    pub const fn from_ordered_id(field_id: u32) -> OrderedPathId {
-        OrderedPathId(field_id)
-    }
-
-    /// Returns a u32 identifying uniquely a path within a schema.
-    pub const fn path_id(self) -> u32 {
-        self.0
-    }
-}
-impl From<u32> for OrderedPathId {
-    fn from(id: u32) -> Self {
-        Self(id)
-    }
-}
-
-#[derive(Default)]
-pub(crate) struct PathToUnorderedId {
-    map: FnvHashMap<String, u32>,
-}
-
-impl PathToUnorderedId {
-    #[inline]
-    pub(crate) fn get_or_allocate_unordered_id(&mut self, path: &str) -> u32 {
-        if let Some(id) = self.map.get(path) {
-            return *id;
-        }
-        self.insert_new_path(path)
-    }
-    #[cold]
-    fn insert_new_path(&mut self, path: &str) -> u32 {
-        let next_id = self.map.len() as u32;
-        self.map.insert(path.to_string(), next_id);
-        next_id
-    }
-
-    /// Retuns ids which reflect the lexical order of the paths.
-    ///
-    /// The returned vec can be indexed with the unordered id to get the ordered id.
-    pub(crate) fn unordered_id_to_ordered_id(&self) -> Vec<OrderedPathId> {
-        let mut sorted_ids: Vec<(&str, &u32)> =
-            self.map.iter().map(|(k, v)| (k.as_str(), v)).collect();
-        sorted_ids.sort_unstable_by_key(|(path, _)| *path);
-        let mut result = vec![OrderedPathId::default(); sorted_ids.len()];
-        for (ordered, unordered) in sorted_ids.iter().map(|(_k, v)| v).enumerate() {
-            result[**unordered as usize] = OrderedPathId::from_ordered_id(ordered as u32);
-        }
-        result
-    }
-
-    /// Retuns the paths so they can be queried by the ordered id (which is the index).
-    pub(crate) fn ordered_id_to_path(&self) -> Vec<&str> {
-        let mut paths = self.map.keys().map(String::as_str).collect::<Vec<_>>();
-        paths.sort_unstable();
-        paths
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn path_to_unordered_test() {
-        let mut path_to_id = PathToUnorderedId::default();
-        let terms = vec!["b", "a", "b", "c"];
-        let ids = terms
-            .iter()
-            .map(|term| path_to_id.get_or_allocate_unordered_id(term))
-            .collect::<Vec<u32>>();
-        assert_eq!(ids, vec![0, 1, 0, 2]);
-        let ordered_ids = ids
-            .iter()
-            .map(|id| path_to_id.unordered_id_to_ordered_id()[*id as usize])
-            .collect::<Vec<OrderedPathId>>();
-        assert_eq!(ordered_ids, vec![1.into(), 0.into(), 1.into(), 2.into()]);
-        // Fetch terms
-        let terms_fetched = ordered_ids
-            .iter()
-            .map(|id| path_to_id.ordered_id_to_path()[id.path_id() as usize])
-            .collect::<Vec<&str>>();
-        assert_eq!(terms_fetched, terms);
-    }
-}

src/indexer/segment_writer.rs

@@ -1,5 +1,4 @@
 use columnar::MonotonicallyMappableToU64;
-use common::JsonPathWriter;
 use itertools::Itertools;
 use tokenizer_api::BoxTokenStream;
 
@@ -67,7 +66,6 @@ pub struct SegmentWriter {
     pub(crate) segment_serializer: SegmentSerializer,
     pub(crate) fast_field_writers: FastFieldsWriter,
     pub(crate) fieldnorms_writer: FieldNormsWriter,
-    pub(crate) json_path_writer: JsonPathWriter,
     pub(crate) doc_opstamps: Vec<Opstamp>,
     per_field_text_analyzers: Vec<TextAnalyzer>,
     term_buffer: Term,
@@ -118,7 +116,6 @@ impl SegmentWriter {
             ctx: IndexingContext::new(table_size),
             per_field_postings_writers,
             fieldnorms_writer: FieldNormsWriter::for_schema(&schema),
-            json_path_writer: JsonPathWriter::default(),
             segment_serializer,
             fast_field_writers: FastFieldsWriter::from_schema_and_tokenizer_manager(
                 &schema,
@@ -147,7 +144,6 @@ impl SegmentWriter {
             .map(|sort_by_field| get_doc_id_mapping_from_field(sort_by_field, &self))
             .transpose()?;
         remap_and_write(
-            self.schema,
             &self.per_field_postings_writers,
             self.ctx,
             self.fast_field_writers,
@@ -159,8 +155,6 @@ impl SegmentWriter {
         Ok(doc_opstamps)
     }
 
-    /// Returns an estimation of the current memory usage of the segment writer.
-    /// If the mem usage exceeds the `memory_budget`, the segment be serialized.
     pub fn mem_usage(&self) -> usize {
         self.ctx.mem_usage()
             + self.fieldnorms_writer.mem_usage()
@@ -359,7 +353,6 @@ impl SegmentWriter {
                         json_options.is_expand_dots_enabled(),
                         term_buffer,
                         postings_writer,
-                        &mut self.json_path_writer,
                         ctx,
                     )?;
                 }
@@ -427,7 +420,6 @@ impl SegmentWriter {
 ///
 /// `doc_id_map` is used to map to the new doc_id order.
 fn remap_and_write(
-    schema: Schema,
     per_field_postings_writers: &PerFieldPostingsWriter,
     ctx: IndexingContext,
     fast_field_writers: FastFieldsWriter,
@@ -445,7 +437,6 @@ fn remap_and_write(
     let fieldnorm_readers = FieldNormReaders::open(fieldnorm_data)?;
     serialize_postings(
         ctx,
-        schema,
         per_field_postings_writers,
         fieldnorm_readers,
         doc_id_map,
@@ -495,11 +486,12 @@ mod tests {
 
     use tempfile::TempDir;
 
-    use crate::collector::{Count, TopDocs};
+    use super::compute_initial_table_size;
+    use crate::collector::Count;
     use crate::core::json_utils::JsonTermWriter;
     use crate::directory::RamDirectory;
     use crate::postings::TermInfo;
-    use crate::query::{PhraseQuery, QueryParser};
+    use crate::query::PhraseQuery;
     use crate::schema::document::Value;
     use crate::schema::{
         Document, IndexRecordOption, Schema, TextFieldIndexing, TextOptions, Type, STORED, STRING,
@@ -515,12 +507,10 @@ mod tests {
     };
 
     #[test]
-    #[cfg(not(feature = "compare_hash_only"))]
     fn test_hashmap_size() {
-        use super::compute_initial_table_size;
-        assert_eq!(compute_initial_table_size(100_000).unwrap(), 1 << 12);
-        assert_eq!(compute_initial_table_size(1_000_000).unwrap(), 1 << 15);
-        assert_eq!(compute_initial_table_size(15_000_000).unwrap(), 1 << 19);
+        assert_eq!(compute_initial_table_size(100_000).unwrap(), 1 << 11);
+        assert_eq!(compute_initial_table_size(1_000_000).unwrap(), 1 << 14);
+        assert_eq!(compute_initial_table_size(15_000_000).unwrap(), 1 << 18);
         assert_eq!(compute_initial_table_size(1_000_000_000).unwrap(), 1 << 19);
         assert_eq!(compute_initial_table_size(4_000_000_000).unwrap(), 1 << 19);
     }
@@ -560,43 +550,6 @@ mod tests {
         assert_eq!(doc.field_values()[0].value().as_str(), Some("A"));
         assert_eq!(doc.field_values()[1].value().as_str(), Some("title"));
     }
-    #[test]
-    fn test_simple_json_indexing() {
-        let mut schema_builder = Schema::builder();
-        let json_field = schema_builder.add_json_field("json", STORED | STRING);
-        let schema = schema_builder.build();
-        let index = Index::create_in_ram(schema.clone());
-        let mut writer = index.writer_for_tests().unwrap();
-        writer
-            .add_document(doc!(json_field=>json!({"my_field": "b"})))
-            .unwrap();
-        writer
-            .add_document(doc!(json_field=>json!({"my_field": "a"})))
-            .unwrap();
-        writer
-            .add_document(doc!(json_field=>json!({"my_field": "b"})))
-            .unwrap();
-        writer.commit().unwrap();
-
-        let query_parser = QueryParser::for_index(&index, vec![json_field]);
-        let text_query = query_parser.parse_query("my_field:a").unwrap();
-        let score_docs: Vec<(_, DocAddress)> = index
-            .reader()
-            .unwrap()
-            .searcher()
-            .search(&text_query, &TopDocs::with_limit(4))
-            .unwrap();
-        assert_eq!(score_docs.len(), 1);
-
-        let text_query = query_parser.parse_query("my_field:b").unwrap();
-        let score_docs: Vec<(_, DocAddress)> = index
-            .reader()
-            .unwrap()
-            .searcher()
-            .search(&text_query, &TopDocs::with_limit(4))
-            .unwrap();
-        assert_eq!(score_docs.len(), 2);
-    }
 
     #[test]
     fn test_json_indexing() {
@@ -879,31 +832,6 @@ mod tests {
         assert_eq!(searcher.search(&phrase_query, &Count).unwrap(), 0);
     }
 
-    #[test]
-    fn test_json_term_with_numeric_merge_panic_regression_bug_2283() {
-        // https://github.com/quickwit-oss/tantivy/issues/2283
-        let mut schema_builder = Schema::builder();
-        let json = schema_builder.add_json_field("json", TEXT);
-        let schema = schema_builder.build();
-        let index = Index::create_in_ram(schema);
-        let mut writer = index.writer_for_tests().unwrap();
-        let doc = json!({"field": "a"});
-        writer.add_document(doc!(json=>doc)).unwrap();
-        writer.commit().unwrap();
-        let doc = json!({"field": "a", "id": 1});
-        writer.add_document(doc!(json=>doc.clone())).unwrap();
-        writer.commit().unwrap();
-
-        // Force Merge
-        writer.wait_merging_threads().unwrap();
-        let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
-        let segment_ids = index
-            .searchable_segment_ids()
-            .expect("Searchable segments failed.");
-        index_writer.merge(&segment_ids).wait().unwrap();
-        assert!(index_writer.wait_merging_threads().is_ok());
-    }
-
     #[test]
     fn test_bug_regression_1629_position_when_array_with_a_field_value_that_does_not_contain_any_token(
     ) {

src/lib.rs

@@ -103,48 +103,7 @@
 //! the example code (
 //! [literate programming](https://tantivy-search.github.io/examples/basic_search.html) /
 //! [source code](https://github.com/quickwit-oss/tantivy/blob/main/examples/basic_search.rs))
-//!
-//! # Tantivy Architecture Overview
-//!
-//! Tantivy is inspired by Lucene, the Architecture is very similar.
-//!
-//! ## Core Concepts
-//!
-//! - **[Index]**: A collection of segments. The top level entry point for tantivy users to search
-//!   and index data.
-//!
-//! - **[Segment]**: At the heart of Tantivy's indexing structure is the [Segment]. It contains
-//!   documents and indices and is the atomic unit of indexing and search.
-//!
-//! - **[Schema](schema)**: A schema is a set of fields in an index. Each field has a specific data
-//!   type and set of attributes.
-//!
-//! - **[IndexWriter]**: Responsible creating and merging segments. It executes the indexing
-//!   pipeline including tokenization, creating indices, and storing the index in the
-//!   [Directory](directory).
-//!
-//! - **Searching**: [Searcher] searches the segments with anything that implements
-//!   [Query](query::Query) and merges the results. The list of [supported
-//! queries](query::Query#implementors). Custom Queries are supported by implementing the
-//! [Query](query::Query) trait.
-//!
-//! - **[Directory](directory)**: Abstraction over the storage where the index data is stored.
-//!
-//! - **[Tokenizer](tokenizer)**: Breaks down text into individual tokens. Users can implement or
-//!   use provided tokenizers.
-//!
-//! ## Architecture Flow
-//!
-//! 1. **Document Addition**: Users create documents according to the defined schema. The documents
-//!    fields are tokenized, processed, and added to the current segment. See
-//!    [Document](schema::document) for the structure and usage.
-//!
-//! 2. **Segment Creation**: Once the memory limit threshold is reached or a commit is called, the
-//!    segment is written to the Directory. Documents are searchable after `commit`.
-//!
-//! 3. **Merging**: To optimize space and search speed, segments might be merged. This operation is
-//!    performed in the background. Customize the merge behaviour via
-//!    [IndexWriter::set_merge_policy].
+
 #[cfg_attr(test, macro_use)]
 extern crate serde_json;
 #[macro_use]
@@ -178,7 +137,7 @@ pub use crate::future_result::FutureResult;
 pub type Result<T> = std::result::Result<T, TantivyError>;
 
 mod core;
-pub mod indexer;
+mod indexer;
 
 #[allow(unused_doc_comments)]
 pub mod error;
@@ -202,7 +161,8 @@ pub mod termdict;
 mod reader;
 
 pub use self::reader::{IndexReader, IndexReaderBuilder, ReloadPolicy, Warmer};
-pub mod snippet;
+mod snippet;
+pub use self::snippet::{Snippet, SnippetGenerator};
 
 mod docset;
 use std::fmt;
@@ -213,34 +173,26 @@ use once_cell::sync::Lazy;
 use serde::{Deserialize, Serialize};
 
 pub use self::docset::{DocSet, TERMINATED};
-#[deprecated(
-    since = "0.22.0",
-    note = "Will be removed in tantivy 0.23. Use export from snippet module instead"
-)]
-pub use self::snippet::{Snippet, SnippetGenerator};
 #[doc(hidden)]
 pub use crate::core::json_utils;
 pub use crate::core::{
-    merge_field_meta_data, Executor, FieldMetadata, Index, IndexBuilder, IndexMeta, IndexSettings,
-    IndexSortByField, InvertedIndexReader, Order, Searcher, SearcherGeneration, Segment,
-    SegmentComponent, SegmentId, SegmentMeta, SegmentReader, SingleSegmentIndexWriter,
+    Executor, Index, IndexBuilder, IndexMeta, IndexSettings, IndexSortByField, InvertedIndexReader,
+    Order, Searcher, SearcherGeneration, Segment, SegmentComponent, SegmentId, SegmentMeta,
+    SegmentReader, SingleSegmentIndexWriter,
 };
 pub use crate::directory::Directory;
-pub use crate::indexer::IndexWriter;
-#[deprecated(
-    since = "0.22.0",
-    note = "Will be removed in tantivy 0.23. Use export from indexer module instead"
-)]
-pub use crate::indexer::{merge_filtered_segments, merge_indices, PreparedCommit};
+pub use crate::indexer::operation::UserOperation;
+pub use crate::indexer::{merge_filtered_segments, merge_indices, IndexWriter, PreparedCommit};
 pub use crate::postings::Postings;
 #[allow(deprecated)]
 pub use crate::schema::DatePrecision;
 pub use crate::schema::{DateOptions, DateTimePrecision, Document, TantivyDocument, Term};
 
 /// Index format version.
-const INDEX_FORMAT_VERSION: u32 = 6;
-/// Oldest index format version this tantivy version can read.
-const INDEX_FORMAT_OLDEST_SUPPORTED_VERSION: u32 = 4;
+const INDEX_FORMAT_VERSION: u32 = 5;
+
+#[cfg(all(feature = "mmap", unix))]
+pub use memmap2::Advice;
 
 /// Structure version for the index.
 #[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]

src/positions/reader.rs

@@ -92,7 +92,7 @@ impl PositionReader {
             // that block is bitpacked.
             let bit_width = bit_widths[block_rel_id];
             self.block_decoder
-                .uncompress_block_unsorted(compressed_data, bit_width, false);
+                .uncompress_block_unsorted(compressed_data, bit_width);
         } else {
             // that block is vint encoded.
             self.block_decoder

src/positions/serializer.rs

@@ -62,9 +62,8 @@ impl<W: io::Write> PositionSerializer<W> {
             return;
         }
         if self.block.len() == COMPRESSION_BLOCK_SIZE {
-            let (bit_width, block_encoded): (u8, &[u8]) = self
-                .block_encoder
-                .compress_block_unsorted(&self.block[..], false);
+            let (bit_width, block_encoded): (u8, &[u8]) =
+                self.block_encoder.compress_block_unsorted(&self.block[..]);
             self.bit_widths.push(bit_width);
             self.positions_buffer.extend(block_encoded);
         } else {

src/postings/block_segment_postings.rs

@@ -24,13 +24,13 @@ fn max_score<I: Iterator<Item = Score>>(mut it: I) -> Option<Score> {
 #[derive(Clone)]
 pub struct BlockSegmentPostings {
     pub(crate) doc_decoder: BlockDecoder,
-    block_loaded: bool,
+    loaded_offset: usize,
     freq_decoder: BlockDecoder,
     freq_reading_option: FreqReadingOption,
     block_max_score_cache: Option<Score>,
     doc_freq: u32,
     data: OwnedBytes,
-    skip_reader: SkipReader,
+    pub(crate) skip_reader: SkipReader,
 }
 
 fn decode_bitpacked_block(
@@ -40,16 +40,10 @@ fn decode_bitpacked_block(
     doc_offset: DocId,
     doc_num_bits: u8,
     tf_num_bits: u8,
-    strict_delta: bool,
 ) {
-    let num_consumed_bytes =
-        doc_decoder.uncompress_block_sorted(data, doc_offset, doc_num_bits, strict_delta);
+    let num_consumed_bytes = doc_decoder.uncompress_block_sorted(data, doc_offset, doc_num_bits);
     if let Some(freq_decoder) = freq_decoder_opt {
-        freq_decoder.uncompress_block_unsorted(
-            &data[num_consumed_bytes..],
-            tf_num_bits,
-            strict_delta,
-        );
+        freq_decoder.uncompress_block_unsorted(&data[num_consumed_bytes..], tf_num_bits);
     }
 }
 
@@ -63,15 +57,11 @@ fn decode_vint_block(
     let num_consumed_bytes =
         doc_decoder.uncompress_vint_sorted(data, doc_offset, num_vint_docs, TERMINATED);
     if let Some(freq_decoder) = freq_decoder_opt {
-        // if it's a json term with freq, containing less than 256 docs, we can reach here thinking
-        // we have a freq, despite not really having one.
-        if data.len() > num_consumed_bytes {
-            freq_decoder.uncompress_vint_unsorted(
-                &data[num_consumed_bytes..],
-                num_vint_docs,
-                TERMINATED,
-            );
-        }
+        freq_decoder.uncompress_vint_unsorted(
+            &data[num_consumed_bytes..],
+            num_vint_docs,
+            TERMINATED,
+        );
     }
 }
 
@@ -88,46 +78,28 @@ fn split_into_skips_and_postings(
 }
 
 impl BlockSegmentPostings {
-    /// Opens a `BlockSegmentPostings`.
-    /// `doc_freq` is the number of documents in the posting list.
-    /// `record_option` represents the amount of data available according to the schema.
-    /// `requested_option` is the amount of data requested by the user.
-    /// If for instance, we do not request for term frequencies, this function will not decompress
-    /// term frequency blocks.
     pub(crate) fn open(
         doc_freq: u32,
         data: FileSlice,
-        mut record_option: IndexRecordOption,
+        record_option: IndexRecordOption,
         requested_option: IndexRecordOption,
     ) -> io::Result<BlockSegmentPostings> {
-        let bytes = data.read_bytes()?;
-        let (skip_data_opt, postings_data) = split_into_skips_and_postings(doc_freq, bytes)?;
-        let skip_reader = match skip_data_opt {
-            Some(skip_data) => {
-                let block_count = doc_freq as usize / COMPRESSION_BLOCK_SIZE;
-                // 8 is the minimum size of a block with frequency (can be more if pos are stored
-                // too)
-                if skip_data.len() < 8 * block_count {
-                    // the field might be encoded with frequency, but this term in particular isn't.
-                    // This can happen for JSON field with term frequencies:
-                    // - text terms are encoded with term freqs.
-                    // - numerical terms are encoded without term freqs.
-                    record_option = IndexRecordOption::Basic;
-                }
-                SkipReader::new(skip_data, doc_freq, record_option)
-            }
-            None => SkipReader::new(OwnedBytes::empty(), doc_freq, record_option),
-        };
-
         let freq_reading_option = match (record_option, requested_option) {
             (IndexRecordOption::Basic, _) => FreqReadingOption::NoFreq,
             (_, IndexRecordOption::Basic) => FreqReadingOption::SkipFreq,
             (_, _) => FreqReadingOption::ReadFreq,
         };
 
+        let bytes = data.read_bytes()?;
+        let (skip_data_opt, postings_data) = split_into_skips_and_postings(doc_freq, bytes)?;
+        let skip_reader = match skip_data_opt {
+            Some(skip_data) => SkipReader::new(skip_data, doc_freq, record_option),
+            None => SkipReader::new(OwnedBytes::empty(), doc_freq, record_option),
+        };
+
         let mut block_segment_postings = BlockSegmentPostings {
             doc_decoder: BlockDecoder::with_val(TERMINATED),
-            block_loaded: false,
+            loaded_offset: usize::MAX,
             freq_decoder: BlockDecoder::with_val(1),
             freq_reading_option,
             block_max_score_cache: None,
@@ -197,7 +169,7 @@ impl BlockSegmentPostings {
             split_into_skips_and_postings(doc_freq, postings_data)?;
         self.data = postings_data;
         self.block_max_score_cache = None;
-        self.block_loaded = false;
+        self.loaded_offset = usize::MAX;
         if let Some(skip_data) = skip_data_opt {
             self.skip_reader.reset(skip_data, doc_freq);
         } else {
@@ -293,23 +265,22 @@ impl BlockSegmentPostings {
     pub(crate) fn shallow_seek(&mut self, target_doc: DocId) {
         if self.skip_reader.seek(target_doc) {
             self.block_max_score_cache = None;
-            self.block_loaded = false;
         }
     }
 
     pub(crate) fn block_is_loaded(&self) -> bool {
-        self.block_loaded
+        self.loaded_offset == self.skip_reader.byte_offset()
     }
 
     pub(crate) fn load_block(&mut self) {
         let offset = self.skip_reader.byte_offset();
-        if self.block_is_loaded() {
+        if self.loaded_offset == offset {
             return;
         }
+        self.loaded_offset = offset;
         match self.skip_reader.block_info() {
             BlockInfo::BitPacked {
                 doc_num_bits,
-                strict_delta_encoded,
                 tf_num_bits,
                 ..
             } => {
@@ -324,7 +295,6 @@ impl BlockSegmentPostings {
                     self.skip_reader.last_doc_in_previous_block,
                     doc_num_bits,
                     tf_num_bits,
-                    strict_delta_encoded,
                 );
             }
             BlockInfo::VInt { num_docs } => {
@@ -348,13 +318,13 @@ impl BlockSegmentPostings {
                 );
             }
         }
-        self.block_loaded = true;
     }
 
     /// Advance to the next block.
+    ///
+    /// Returns false if and only if there is no remaining block.
     pub fn advance(&mut self) {
         self.skip_reader.advance();
-        self.block_loaded = false;
         self.block_max_score_cache = None;
         self.load_block();
     }
@@ -363,7 +333,7 @@ impl BlockSegmentPostings {
     pub fn empty() -> BlockSegmentPostings {
         BlockSegmentPostings {
             doc_decoder: BlockDecoder::with_val(TERMINATED),
-            block_loaded: true,
+            loaded_offset: 0,
             freq_decoder: BlockDecoder::with_val(1),
             freq_reading_option: FreqReadingOption::NoFreq,
             block_max_score_cache: None,
@@ -372,10 +342,6 @@ impl BlockSegmentPostings {
             skip_reader: SkipReader::new(OwnedBytes::empty(), 0, IndexRecordOption::Basic),
         }
     }
-
-    pub(crate) fn skip_reader(&self) -> &SkipReader {
-        &self.skip_reader
-    }
 }
 
 #[cfg(test)]

src/postings/compression/mod.rs

@@ -33,40 +33,14 @@ impl BlockEncoder {
     }
 
     pub fn compress_block_sorted(&mut self, block: &[u32], offset: u32) -> (u8, &[u8]) {
-        // if offset is zero, convert it to None. This is correct as long as we do the same when
-        // decompressing. It's required in case the block starts with an actual zero.
-        let offset = if offset == 0u32 { None } else { Some(offset) };
-
-        let num_bits = self.bitpacker.num_bits_strictly_sorted(offset, block);
+        let num_bits = self.bitpacker.num_bits_sorted(offset, block);
         let written_size =
             self.bitpacker
-                .compress_strictly_sorted(offset, block, &mut self.output[..], num_bits);
+                .compress_sorted(offset, block, &mut self.output[..], num_bits);
         (num_bits, &self.output[..written_size])
     }
 
-    /// Compress a single block of unsorted numbers.
-    ///
-    /// If `minus_one_encoded` is set, each value must be >= 1, and will be encoded in a sligly
-    /// more compact format. This is useful for some values where 0 isn't a correct value, such
-    /// as term frequency, but isn't correct for some usages like position lists, where 0 can
-    /// appear.
-    pub fn compress_block_unsorted(
-        &mut self,
-        block: &[u32],
-        minus_one_encoded: bool,
-    ) -> (u8, &[u8]) {
-        debug_assert!(!minus_one_encoded || !block.contains(&0));
-
-        let mut block_minus_one = [0; COMPRESSION_BLOCK_SIZE];
-        let block = if minus_one_encoded {
-            for (elem_min_one, elem) in block_minus_one.iter_mut().zip(block) {
-                *elem_min_one = elem - 1;
-            }
-            &block_minus_one
-        } else {
-            block
-        };
-
+    pub fn compress_block_unsorted(&mut self, block: &[u32]) -> (u8, &[u8]) {
         let num_bits = self.bitpacker.num_bits(block);
         let written_size = self
             .bitpacker
@@ -97,55 +71,21 @@ impl BlockDecoder {
         }
     }
 
-    /// Decompress block of sorted integers.
-    ///
-    /// `strict_delta` depends on what encoding was used. Older version of tantivy never use strict
-    /// deltas, newer versions always use them.
     pub fn uncompress_block_sorted(
         &mut self,
         compressed_data: &[u8],
         offset: u32,
         num_bits: u8,
-        strict_delta: bool,
-    ) -> usize {
-        if strict_delta {
-            let offset = std::num::NonZeroU32::new(offset).map(std::num::NonZeroU32::get);
-
-            self.output_len = COMPRESSION_BLOCK_SIZE;
-            self.bitpacker.decompress_strictly_sorted(
-                offset,
-                compressed_data,
-                &mut self.output,
-                num_bits,
-            )
-        } else {
-            self.output_len = COMPRESSION_BLOCK_SIZE;
-            self.bitpacker
-                .decompress_sorted(offset, compressed_data, &mut self.output, num_bits)
-        }
-    }
-
-    /// Decompress block of unsorted integers.
-    ///
-    /// `minus_one_encoded` depends on what encoding was used. Older version of tantivy never use
-    /// that encoding. Newer version use it for some structures, but not all. See the corresponding
-    /// call to `BlockEncoder::compress_block_unsorted`.
-    pub fn uncompress_block_unsorted(
-        &mut self,
-        compressed_data: &[u8],
-        num_bits: u8,
-        minus_one_encoded: bool,
     ) -> usize {
         self.output_len = COMPRESSION_BLOCK_SIZE;
-        let res = self
-            .bitpacker
-            .decompress(compressed_data, &mut self.output, num_bits);
-        if minus_one_encoded {
-            for val in &mut self.output {
-                *val += 1;
-            }
-        }
-        res
+        self.bitpacker
+            .decompress_sorted(offset, compressed_data, &mut self.output, num_bits)
+    }
+
+    pub fn uncompress_block_unsorted(&mut self, compressed_data: &[u8], num_bits: u8) -> usize {
+        self.output_len = COMPRESSION_BLOCK_SIZE;
+        self.bitpacker
+            .decompress(compressed_data, &mut self.output, num_bits)
     }
 
     #[inline]
@@ -278,8 +218,7 @@ pub mod tests {
         let (num_bits, compressed_data) = encoder.compress_block_sorted(&vals, 0);
         let mut decoder = BlockDecoder::default();
         {
-            let consumed_num_bytes =
-                decoder.uncompress_block_sorted(compressed_data, 0, num_bits, true);
+            let consumed_num_bytes = decoder.uncompress_block_sorted(compressed_data, 0, num_bits);
             assert_eq!(consumed_num_bytes, compressed_data.len());
         }
         for i in 0..128 {
@@ -294,8 +233,7 @@ pub mod tests {
         let (num_bits, compressed_data) = encoder.compress_block_sorted(&vals, 10);
         let mut decoder = BlockDecoder::default();
         {
-            let consumed_num_bytes =
-                decoder.uncompress_block_sorted(compressed_data, 10, num_bits, true);
+            let consumed_num_bytes = decoder.uncompress_block_sorted(compressed_data, 10, num_bits);
             assert_eq!(consumed_num_bytes, compressed_data.len());
         }
         for i in 0..128 {
@@ -314,8 +252,7 @@ pub mod tests {
         compressed.push(173u8);
         let mut decoder = BlockDecoder::default();
         {
-            let consumed_num_bytes =
-                decoder.uncompress_block_sorted(&compressed, 10, num_bits, true);
+            let consumed_num_bytes = decoder.uncompress_block_sorted(&compressed, 10, num_bits);
             assert_eq!(consumed_num_bytes, compressed.len() - 1);
             assert_eq!(compressed[consumed_num_bytes], 173u8);
         }
@@ -326,25 +263,21 @@ pub mod tests {
 
     #[test]
     fn test_encode_unsorted_block_with_junk() {
-        for minus_one_encode in [false, true] {
-            let mut compressed: Vec<u8> = Vec::new();
-            let n = 128;
-            let vals: Vec<u32> = (0..n).map(|i| 11u32 + (i as u32) * 7u32 % 12).collect();
-            let mut encoder = BlockEncoder::default();
-            let (num_bits, compressed_data) =
-                encoder.compress_block_unsorted(&vals, minus_one_encode);
-            compressed.extend_from_slice(compressed_data);
-            compressed.push(173u8);
-            let mut decoder = BlockDecoder::default();
-            {
-                let consumed_num_bytes =
-                    decoder.uncompress_block_unsorted(&compressed, num_bits, minus_one_encode);
-                assert_eq!(consumed_num_bytes + 1, compressed.len());
-                assert_eq!(compressed[consumed_num_bytes], 173u8);
-            }
-            for i in 0..n {
-                assert_eq!(vals[i], decoder.output(i));
-            }
+        let mut compressed: Vec<u8> = Vec::new();
+        let n = 128;
+        let vals: Vec<u32> = (0..n).map(|i| 11u32 + (i as u32) * 7u32 % 12).collect();
+        let mut encoder = BlockEncoder::default();
+        let (num_bits, compressed_data) = encoder.compress_block_unsorted(&vals);
+        compressed.extend_from_slice(compressed_data);
+        compressed.push(173u8);
+        let mut decoder = BlockDecoder::default();
+        {
+            let consumed_num_bytes = decoder.uncompress_block_unsorted(&compressed, num_bits);
+            assert_eq!(consumed_num_bytes + 1, compressed.len());
+            assert_eq!(compressed[consumed_num_bytes], 173u8);
+        }
+        for i in 0..n {
+            assert_eq!(vals[i], decoder.output(i));
         }
     }
 
@@ -411,7 +344,7 @@ mod bench {
         let (num_bits, compressed) = encoder.compress_block_sorted(&data, 0u32);
         let mut decoder = BlockDecoder::default();
         b.iter(|| {
-            decoder.uncompress_block_sorted(compressed, 0u32, num_bits, true);
+            decoder.uncompress_block_sorted(compressed, 0u32, num_bits);
         });
     }
 

src/postings/indexing_context.rs

@@ -1,7 +1,5 @@
 use stacker::{ArenaHashMap, MemoryArena};
 
-use crate::indexer::path_to_unordered_id::PathToUnorderedId;
-
 /// IndexingContext contains all of the transient memory arenas
 /// required for building the inverted index.
 pub(crate) struct IndexingContext {
@@ -10,7 +8,6 @@ pub(crate) struct IndexingContext {
     pub term_index: ArenaHashMap,
     /// Arena is a memory arena that stores posting lists / term frequencies / positions.
     pub arena: MemoryArena,
-    pub path_to_unordered_id: PathToUnorderedId,
 }
 
 impl IndexingContext {
@@ -20,7 +17,6 @@ impl IndexingContext {
         IndexingContext {
             arena: MemoryArena::default(),
             term_index,
-            path_to_unordered_id: PathToUnorderedId::default(),
         }
     }
 

src/postings/json_postings_writer.rs

@@ -3,18 +3,13 @@ use std::io;
 use stacker::Addr;
 
 use crate::indexer::doc_id_mapping::DocIdMapping;
-use crate::indexer::path_to_unordered_id::OrderedPathId;
 use crate::postings::postings_writer::SpecializedPostingsWriter;
 use crate::postings::recorder::{BufferLender, DocIdRecorder, Recorder};
 use crate::postings::{FieldSerializer, IndexingContext, IndexingPosition, PostingsWriter};
-use crate::schema::{Field, Type, JSON_END_OF_PATH};
+use crate::schema::Type;
 use crate::tokenizer::TokenStream;
 use crate::{DocId, Term};
 
-/// The `JsonPostingsWriter` is odd in that it relies on a hidden contract:
-///
-/// `subscribe` is called directly to index non-text tokens, while
-/// `index_text` is used to index text.
 #[derive(Default)]
 pub(crate) struct JsonPostingsWriter<Rec: Recorder> {
     str_posting_writer: SpecializedPostingsWriter<Rec>,
@@ -59,24 +54,18 @@ impl<Rec: Recorder> PostingsWriter for JsonPostingsWriter<Rec> {
     /// The actual serialization format is handled by the `PostingsSerializer`.
     fn serialize(
         &self,
-        term_addrs: &[(Field, OrderedPathId, &[u8], Addr)],
-        ordered_id_to_path: &[&str],
+        term_addrs: &[(Term<&[u8]>, Addr)],
         doc_id_map: Option<&DocIdMapping>,
         ctx: &IndexingContext,
         serializer: &mut FieldSerializer,
     ) -> io::Result<()> {
-        let mut term_buffer = Term::with_capacity(48);
         let mut buffer_lender = BufferLender::default();
-        for (_field, path_id, term, addr) in term_addrs {
-            term_buffer.clear_with_field_and_type(Type::Json, Field::from_field_id(0));
-            term_buffer.append_bytes(ordered_id_to_path[path_id.path_id() as usize].as_bytes());
-            term_buffer.append_bytes(&[JSON_END_OF_PATH]);
-            term_buffer.append_bytes(term);
-            if let Some(json_value) = term_buffer.value().as_json_value_bytes() {
+        for (term, addr) in term_addrs {
+            if let Some(json_value) = term.value().as_json_value_bytes() {
                 let typ = json_value.typ();
                 if typ == Type::Str {
                     SpecializedPostingsWriter::<Rec>::serialize_one_term(
-                        term_buffer.serialized_value_bytes(),
+                        term,
                         *addr,
                         doc_id_map,
                         &mut buffer_lender,
@@ -85,7 +74,7 @@ impl<Rec: Recorder> PostingsWriter for JsonPostingsWriter<Rec> {
                     )?;
                 } else {
                     SpecializedPostingsWriter::<DocIdRecorder>::serialize_one_term(
-                        term_buffer.serialized_value_bytes(),
+                        term,
                         *addr,
                         doc_id_map,
                         &mut buffer_lender,

src/postings/mod.rs

@@ -63,7 +63,7 @@ pub mod tests {
         let mut segment = index.new_segment();
         let mut posting_serializer = InvertedIndexSerializer::open(&mut segment)?;
         let mut field_serializer = posting_serializer.new_field(text_field, 120 * 4, None)?;
-        field_serializer.new_term("abc".as_bytes(), 12u32, true)?;
+        field_serializer.new_term("abc".as_bytes(), 12u32)?;
         for doc_id in 0u32..120u32 {
             let delta_positions = vec![1, 2, 3, 2];
             field_serializer.write_doc(doc_id, 4, &delta_positions);

src/postings/postings_writer.rs

@@ -6,23 +6,20 @@ use stacker::Addr;
 
 use crate::fieldnorm::FieldNormReaders;
 use crate::indexer::doc_id_mapping::DocIdMapping;
-use crate::indexer::path_to_unordered_id::OrderedPathId;
 use crate::postings::recorder::{BufferLender, Recorder};
 use crate::postings::{
     FieldSerializer, IndexingContext, InvertedIndexSerializer, PerFieldPostingsWriter,
 };
-use crate::schema::{Field, Schema, Term, Type};
+use crate::schema::{Field, Term};
 use crate::tokenizer::{Token, TokenStream, MAX_TOKEN_LEN};
 use crate::DocId;
 
 const POSITION_GAP: u32 = 1;
 
-fn make_field_partition(
-    term_offsets: &[(Field, OrderedPathId, &[u8], Addr)],
-) -> Vec<(Field, Range<usize>)> {
+fn make_field_partition(term_offsets: &[(Term<&[u8]>, Addr)]) -> Vec<(Field, Range<usize>)> {
     let term_offsets_it = term_offsets
         .iter()
-        .map(|(field, _, _, _)| *field)
+        .map(|(term, _)| term.field())
         .enumerate();
     let mut prev_field_opt = None;
     let mut fields = vec![];
@@ -47,36 +44,19 @@ fn make_field_partition(
 /// postings serializer.
 pub(crate) fn serialize_postings(
     ctx: IndexingContext,
-    schema: Schema,
     per_field_postings_writers: &PerFieldPostingsWriter,
     fieldnorm_readers: FieldNormReaders,
     doc_id_map: Option<&DocIdMapping>,
     serializer: &mut InvertedIndexSerializer,
 ) -> crate::Result<()> {
-    // Replace unordered ids by ordered ids to be able to sort
-    let unordered_id_to_ordered_id: Vec<OrderedPathId> =
-        ctx.path_to_unordered_id.unordered_id_to_ordered_id();
-
-    let mut term_offsets: Vec<(Field, OrderedPathId, &[u8], Addr)> =
-        Vec::with_capacity(ctx.term_index.len());
-    term_offsets.extend(ctx.term_index.iter().map(|(key, addr)| {
-        let field = Term::wrap(key).field();
-        if schema.get_field_entry(field).field_type().value_type() == Type::Json {
-            let byte_range_path = 5..5 + 4;
-            let unordered_id = u32::from_be_bytes(key[byte_range_path.clone()].try_into().unwrap());
-            let path_id = unordered_id_to_ordered_id[unordered_id as usize];
-            (field, path_id, &key[byte_range_path.end..], addr)
-        } else {
-            (field, 0.into(), &key[5..], addr)
-        }
-    }));
-    // Sort by field, path, and term
-    term_offsets.sort_unstable_by(
-        |(field1, path_id1, bytes1, _), (field2, path_id2, bytes2, _)| {
-            (field1, path_id1, bytes1).cmp(&(field2, path_id2, bytes2))
-        },
+    let mut term_offsets: Vec<(Term<&[u8]>, Addr)> = Vec::with_capacity(ctx.term_index.len());
+    term_offsets.extend(
+        ctx.term_index
+            .iter()
+            .map(|(bytes, addr, _unordered_id)| (Term::wrap(bytes), addr)),
     );
-    let ordered_id_to_path = ctx.path_to_unordered_id.ordered_id_to_path();
+    term_offsets.sort_unstable_by_key(|(k, _)| k.clone());
+
     let field_offsets = make_field_partition(&term_offsets);
     for (field, byte_offsets) in field_offsets {
         let postings_writer = per_field_postings_writers.get_for_field(field);
@@ -85,14 +65,12 @@ pub(crate) fn serialize_postings(
             serializer.new_field(field, postings_writer.total_num_tokens(), fieldnorm_reader)?;
         postings_writer.serialize(
             &term_offsets[byte_offsets],
-            &ordered_id_to_path,
             doc_id_map,
             &ctx,
             &mut field_serializer,
         )?;
         field_serializer.close()?;
     }
-
     Ok(())
 }
 
@@ -120,8 +98,7 @@ pub(crate) trait PostingsWriter: Send + Sync {
     /// The actual serialization format is handled by the `PostingsSerializer`.
     fn serialize(
         &self,
-        term_addrs: &[(Field, OrderedPathId, &[u8], Addr)],
-        ordered_id_to_path: &[&str],
+        term_addrs: &[(Term<&[u8]>, Addr)],
         doc_id_map: Option<&DocIdMapping>,
         ctx: &IndexingContext,
         serializer: &mut FieldSerializer,
@@ -185,7 +162,7 @@ impl<Rec: Recorder> From<SpecializedPostingsWriter<Rec>> for Box<dyn PostingsWri
 impl<Rec: Recorder> SpecializedPostingsWriter<Rec> {
     #[inline]
     pub(crate) fn serialize_one_term(
-        term: &[u8],
+        term: &Term<&[u8]>,
         addr: Addr,
         doc_id_map: Option<&DocIdMapping>,
         buffer_lender: &mut BufferLender,
@@ -194,7 +171,7 @@ impl<Rec: Recorder> SpecializedPostingsWriter<Rec> {
     ) -> io::Result<()> {
         let recorder: Rec = ctx.term_index.read(addr);
         let term_doc_freq = recorder.term_doc_freq().unwrap_or(0u32);
-        serializer.new_term(term, term_doc_freq, recorder.has_term_freq())?;
+        serializer.new_term(term.serialized_value_bytes(), term_doc_freq)?;
         recorder.serialize(&ctx.arena, doc_id_map, serializer, buffer_lender);
         serializer.close_term()?;
         Ok(())
@@ -227,14 +204,13 @@ impl<Rec: Recorder> PostingsWriter for SpecializedPostingsWriter<Rec> {
 
     fn serialize(
         &self,
-        term_addrs: &[(Field, OrderedPathId, &[u8], Addr)],
-        _ordered_id_to_path: &[&str],
+        term_addrs: &[(Term<&[u8]>, Addr)],
         doc_id_map: Option<&DocIdMapping>,
         ctx: &IndexingContext,
         serializer: &mut FieldSerializer,
     ) -> io::Result<()> {
         let mut buffer_lender = BufferLender::default();
-        for (_field, _path_id, term, addr) in term_addrs {
+        for (term, addr) in term_addrs {
             Self::serialize_one_term(term, *addr, doc_id_map, &mut buffer_lender, ctx, serializer)?;
         }
         Ok(())

src/postings/recorder.rs

@@ -79,20 +79,24 @@ pub(crate) trait Recorder: Copy + Default + Send + Sync + 'static {
     ///
     /// Returns `None` if not available.
     fn term_doc_freq(&self) -> Option<u32>;
-
-    #[inline]
-    fn has_term_freq(&self) -> bool {
-        true
-    }
 }
 
 /// Only records the doc ids
-#[derive(Clone, Copy, Default)]
+#[derive(Clone, Copy)]
 pub struct DocIdRecorder {
     stack: ExpUnrolledLinkedList,
     current_doc: DocId,
 }
 
+impl Default for DocIdRecorder {
+    fn default() -> Self {
+        DocIdRecorder {
+            stack: ExpUnrolledLinkedList::default(),
+            current_doc: u32::MAX,
+        }
+    }
+}
+
 impl Recorder for DocIdRecorder {
     #[inline]
     fn current_doc(&self) -> DocId {
@@ -101,9 +105,8 @@ impl Recorder for DocIdRecorder {
 
     #[inline]
     fn new_doc(&mut self, doc: DocId, arena: &mut MemoryArena) {
-        let delta = doc - self.current_doc;
         self.current_doc = doc;
-        self.stack.writer(arena).write_u32_vint(delta);
+        self.stack.writer(arena).write_u32_vint(doc);
     }
 
     #[inline]
@@ -120,20 +123,21 @@ impl Recorder for DocIdRecorder {
         buffer_lender: &mut BufferLender,
     ) {
         let (buffer, doc_ids) = buffer_lender.lend_all();
-        // TODO avoid reading twice.
         self.stack.read_to_end(arena, buffer);
+        // TODO avoid reading twice.
         if let Some(doc_id_map) = doc_id_map {
-            let iter = get_sum_reader(VInt32Reader::new(&buffer[..]));
-            doc_ids.extend(iter.map(|old_doc_id| doc_id_map.get_new_doc_id(old_doc_id)));
+            doc_ids.extend(
+                VInt32Reader::new(&buffer[..])
+                    .map(|old_doc_id| doc_id_map.get_new_doc_id(old_doc_id)),
+            );
             doc_ids.sort_unstable();
 
             for doc in doc_ids {
                 serializer.write_doc(*doc, 0u32, &[][..]);
             }
         } else {
-            let iter = get_sum_reader(VInt32Reader::new(&buffer[..]));
-            for doc_id in iter {
-                serializer.write_doc(doc_id, 0u32, &[][..]);
+            for doc in VInt32Reader::new(&buffer[..]) {
+                serializer.write_doc(doc, 0u32, &[][..]);
             }
         }
     }
@@ -141,19 +145,6 @@ impl Recorder for DocIdRecorder {
     fn term_doc_freq(&self) -> Option<u32> {
         None
     }
-
-    fn has_term_freq(&self) -> bool {
-        false
-    }
-}
-
-/// Takes an Iterator of delta encoded elements and returns an iterator
-/// that yields the sum of the elements.
-fn get_sum_reader(iter: impl Iterator<Item = u32>) -> impl Iterator<Item = u32> {
-    iter.scan(0, |state, delta| {
-        *state += delta;
-        Some(*state)
-    })
 }
 
 /// Recorder encoding document ids, and term frequencies
@@ -173,10 +164,9 @@ impl Recorder for TermFrequencyRecorder {
 
     #[inline]
     fn new_doc(&mut self, doc: DocId, arena: &mut MemoryArena) {
-        let delta = doc - self.current_doc;
         self.term_doc_freq += 1;
         self.current_doc = doc;
-        self.stack.writer(arena).write_u32_vint(delta);
+        self.stack.writer(arena).write_u32_vint(doc);
     }
 
     #[inline]
@@ -203,12 +193,9 @@ impl Recorder for TermFrequencyRecorder {
         let mut u32_it = VInt32Reader::new(&buffer[..]);
         if let Some(doc_id_map) = doc_id_map {
             let mut doc_id_and_tf = vec![];
-            let mut prev_doc = 0;
-            while let Some(delta_doc_id) = u32_it.next() {
-                let doc_id = prev_doc + delta_doc_id;
-                prev_doc = doc_id;
+            while let Some(old_doc_id) = u32_it.next() {
                 let term_freq = u32_it.next().unwrap_or(self.current_tf);
-                doc_id_and_tf.push((doc_id_map.get_new_doc_id(doc_id), term_freq));
+                doc_id_and_tf.push((doc_id_map.get_new_doc_id(old_doc_id), term_freq));
             }
             doc_id_and_tf.sort_unstable_by_key(|&(doc_id, _)| doc_id);
 
@@ -216,12 +203,9 @@ impl Recorder for TermFrequencyRecorder {
                 serializer.write_doc(doc_id, tf, &[][..]);
             }
         } else {
-            let mut prev_doc = 0;
-            while let Some(delta_doc_id) = u32_it.next() {
-                let doc_id = prev_doc + delta_doc_id;
-                prev_doc = doc_id;
+            while let Some(doc) = u32_it.next() {
                 let term_freq = u32_it.next().unwrap_or(self.current_tf);
-                serializer.write_doc(doc_id, term_freq, &[][..]);
+                serializer.write_doc(doc, term_freq, &[][..]);
             }
         }
     }
@@ -232,13 +216,23 @@ impl Recorder for TermFrequencyRecorder {
 }
 
 /// Recorder encoding term frequencies as well as positions.
-#[derive(Clone, Copy, Default)]
+#[derive(Clone, Copy)]
 pub struct TfAndPositionRecorder {
     stack: ExpUnrolledLinkedList,
     current_doc: DocId,
     term_doc_freq: u32,
 }
 
+impl Default for TfAndPositionRecorder {
+    fn default() -> Self {
+        TfAndPositionRecorder {
+            stack: ExpUnrolledLinkedList::default(),
+            current_doc: u32::MAX,
+            term_doc_freq: 0u32,
+        }
+    }
+}
+
 impl Recorder for TfAndPositionRecorder {
     #[inline]
     fn current_doc(&self) -> DocId {
@@ -247,10 +241,9 @@ impl Recorder for TfAndPositionRecorder {
 
     #[inline]
     fn new_doc(&mut self, doc: DocId, arena: &mut MemoryArena) {
-        let delta = doc - self.current_doc;
         self.current_doc = doc;
         self.term_doc_freq += 1u32;
-        self.stack.writer(arena).write_u32_vint(delta);
+        self.stack.writer(arena).write_u32_vint(doc);
     }
 
     #[inline]
@@ -276,10 +269,7 @@ impl Recorder for TfAndPositionRecorder {
         self.stack.read_to_end(arena, buffer_u8);
         let mut u32_it = VInt32Reader::new(&buffer_u8[..]);
         let mut doc_id_and_positions = vec![];
-        let mut prev_doc = 0;
-        while let Some(delta_doc_id) = u32_it.next() {
-            let doc_id = prev_doc + delta_doc_id;
-            prev_doc = doc_id;
+        while let Some(doc) = u32_it.next() {
             let mut prev_position_plus_one = 1u32;
             buffer_positions.clear();
             loop {
@@ -297,9 +287,9 @@ impl Recorder for TfAndPositionRecorder {
             if let Some(doc_id_map) = doc_id_map {
                 // this simple variant to remap may consume to much memory
                 doc_id_and_positions
-                    .push((doc_id_map.get_new_doc_id(doc_id), buffer_positions.to_vec()));
+                    .push((doc_id_map.get_new_doc_id(doc), buffer_positions.to_vec()));
             } else {
-                serializer.write_doc(doc_id, buffer_positions.len() as u32, buffer_positions);
+                serializer.write_doc(doc, buffer_positions.len() as u32, buffer_positions);
             }
         }
         if doc_id_map.is_some() {

src/postings/segment_postings.rs

@@ -71,7 +71,7 @@ impl SegmentPostings {
         {
             let mut postings_serializer =
                 PostingsSerializer::new(&mut buffer, 0.0, IndexRecordOption::Basic, None);
-            postings_serializer.new_term(docs.len() as u32, false);
+            postings_serializer.new_term(docs.len() as u32);
             for &doc in docs {
                 postings_serializer.write_doc(doc, 1u32);
             }
@@ -120,7 +120,7 @@ impl SegmentPostings {
             IndexRecordOption::WithFreqs,
             fieldnorm_reader,
         );
-        postings_serializer.new_term(doc_and_tfs.len() as u32, true);
+        postings_serializer.new_term(doc_and_tfs.len() as u32);
         for &(doc, tf) in doc_and_tfs {
             postings_serializer.write_doc(doc, tf);
         }
@@ -238,18 +238,14 @@ impl Postings for SegmentPostings {
     }
 
     fn positions_with_offset(&mut self, offset: u32, output: &mut Vec<u32>) {
-        let term_freq = self.term_freq();
+        let term_freq = self.term_freq() as usize;
         if let Some(position_reader) = self.position_reader.as_mut() {
-            debug_assert!(
-                !self.block_cursor.freqs().is_empty(),
-                "No positions available"
-            );
             let read_offset = self.block_cursor.position_offset()
                 + (self.block_cursor.freqs()[..self.cur]
                     .iter()
                     .cloned()
                     .sum::<u32>() as u64);
-            output.resize(term_freq as usize, 0u32);
+            output.resize(term_freq, 0u32);
             position_reader.read(read_offset, &mut output[..]);
             let mut cum = offset;
             for output_mut in output.iter_mut() {

src/postings/serializer.rs

@@ -168,12 +168,7 @@ impl<'a> FieldSerializer<'a> {
     /// * term - the term. It needs to come after the previous term according to the lexicographical
     ///   order.
     /// * term_doc_freq - return the number of document containing the term.
-    pub fn new_term(
-        &mut self,
-        term: &[u8],
-        term_doc_freq: u32,
-        record_term_freq: bool,
-    ) -> io::Result<()> {
+    pub fn new_term(&mut self, term: &[u8], term_doc_freq: u32) -> io::Result<()> {
         assert!(
             !self.term_open,
             "Called new_term, while the previous term was not closed."
@@ -182,8 +177,7 @@ impl<'a> FieldSerializer<'a> {
         self.postings_serializer.clear();
         self.current_term_info = self.current_term_info();
         self.term_dictionary_builder.insert_key(term)?;
-        self.postings_serializer
-            .new_term(term_doc_freq, record_term_freq);
+        self.postings_serializer.new_term(term_doc_freq);
         Ok(())
     }
 
@@ -307,7 +301,6 @@ pub struct PostingsSerializer<W: Write> {
     bm25_weight: Option<Bm25Weight>,
     avg_fieldnorm: Score, /* Average number of term in the field for that segment.
                            * this value is used to compute the block wand information. */
-    term_has_freq: bool,
 }
 
 impl<W: Write> PostingsSerializer<W> {
@@ -332,15 +325,13 @@ impl<W: Write> PostingsSerializer<W> {
             fieldnorm_reader,
             bm25_weight: None,
             avg_fieldnorm,
-            term_has_freq: false,
         }
     }
 
-    pub fn new_term(&mut self, term_doc_freq: u32, record_term_freq: bool) {
+    pub fn new_term(&mut self, term_doc_freq: u32) {
         self.bm25_weight = None;
 
-        self.term_has_freq = self.mode.has_freq() && record_term_freq;
-        if !self.term_has_freq {
+        if !self.mode.has_freq() {
             return;
         }
 
@@ -355,7 +346,7 @@ impl<W: Write> PostingsSerializer<W> {
             return;
         }
 
-        self.bm25_weight = Some(Bm25Weight::for_one_term_without_explain(
+        self.bm25_weight = Some(Bm25Weight::for_one_term(
             term_doc_freq as u64,
             num_docs_in_segment,
             self.avg_fieldnorm,
@@ -374,10 +365,10 @@ impl<W: Write> PostingsSerializer<W> {
             // last el block 0, offset block 1,
             self.postings_write.extend(block_encoded);
         }
-        if self.term_has_freq {
+        if self.mode.has_freq() {
             let (num_bits, block_encoded): (u8, &[u8]) = self
                 .block_encoder
-                .compress_block_unsorted(self.block.term_freqs(), true);
+                .compress_block_unsorted(self.block.term_freqs());
             self.postings_write.extend(block_encoded);
             self.skip_write.write_term_freq(num_bits);
             if self.mode.has_positions() {
@@ -441,7 +432,7 @@ impl<W: Write> PostingsSerializer<W> {
                 self.postings_write.write_all(block_encoded)?;
             }
             // ... Idem for term frequencies
-            if self.term_has_freq {
+            if self.mode.has_freq() {
                 let block_encoded = self
                     .block_encoder
                     .compress_vint_unsorted(self.block.term_freqs());

src/postings/skip.rs

@@ -6,22 +6,6 @@ use crate::query::Bm25Weight;
 use crate::schema::IndexRecordOption;
 use crate::{DocId, Score, TERMINATED};
 
-// doc num bits uses the following encoding:
-// given 0b a b cdefgh
-//         |1|2|   3  |
-// - 1: unused
-// - 2: is delta-1 encoded. 0 if not, 1, if yes
-// - 3: a 6 bit number in 0..=32, the actual bitwidth
-fn encode_bitwidth(bitwidth: u8, delta_1: bool) -> u8 {
-    bitwidth | ((delta_1 as u8) << 6)
-}
-
-fn decode_bitwidth(raw_bitwidth: u8) -> (u8, bool) {
-    let delta_1 = (raw_bitwidth >> 6 & 1) != 0;
-    let bitwidth = raw_bitwidth & 0x3f;
-    (bitwidth, delta_1)
-}
-
 #[inline]
 fn encode_block_wand_max_tf(max_tf: u32) -> u8 {
     max_tf.min(u8::MAX as u32) as u8
@@ -57,7 +41,7 @@ impl SkipSerializer {
 
     pub fn write_doc(&mut self, last_doc: DocId, doc_num_bits: u8) {
         write_u32(last_doc, &mut self.buffer);
-        self.buffer.push(encode_bitwidth(doc_num_bits, true));
+        self.buffer.push(doc_num_bits);
     }
 
     pub fn write_term_freq(&mut self, tf_num_bits: u8) {
@@ -101,7 +85,6 @@ pub(crate) struct SkipReader {
 pub(crate) enum BlockInfo {
     BitPacked {
         doc_num_bits: u8,
-        strict_delta_encoded: bool,
         tf_num_bits: u8,
         tf_sum: u32,
         block_wand_fieldnorm_id: u8,
@@ -189,13 +172,12 @@ impl SkipReader {
         let bytes = self.owned_read.as_slice();
         let advance_len: usize;
         self.last_doc_in_block = read_u32(bytes);
-        let (doc_num_bits, strict_delta_encoded) = decode_bitwidth(bytes[4]);
+        let doc_num_bits = bytes[4];
         match self.skip_info {
             IndexRecordOption::Basic => {
                 advance_len = 5;
                 self.block_info = BlockInfo::BitPacked {
                     doc_num_bits,
-                    strict_delta_encoded,
                     tf_num_bits: 0,
                     tf_sum: 0,
                     block_wand_fieldnorm_id: 0,
@@ -209,7 +191,6 @@ impl SkipReader {
                 advance_len = 8;
                 self.block_info = BlockInfo::BitPacked {
                     doc_num_bits,
-                    strict_delta_encoded,
                     tf_num_bits,
                     tf_sum: 0,
                     block_wand_fieldnorm_id,
@@ -224,7 +205,6 @@ impl SkipReader {
                 advance_len = 12;
                 self.block_info = BlockInfo::BitPacked {
                     doc_num_bits,
-                    strict_delta_encoded,
                     tf_num_bits,
                     tf_sum,
                     block_wand_fieldnorm_id,
@@ -288,9 +268,7 @@ impl SkipReader {
 #[cfg(test)]
 mod tests {
 
-    use super::{
-        decode_bitwidth, encode_bitwidth, BlockInfo, IndexRecordOption, SkipReader, SkipSerializer,
-    };
+    use super::{BlockInfo, IndexRecordOption, SkipReader, SkipSerializer};
     use crate::directory::OwnedBytes;
     use crate::postings::compression::COMPRESSION_BLOCK_SIZE;
 
@@ -332,7 +310,6 @@ mod tests {
             skip_reader.block_info,
             BlockInfo::BitPacked {
                 doc_num_bits: 2u8,
-                strict_delta_encoded: true,
                 tf_num_bits: 3u8,
                 tf_sum: 0,
                 block_wand_fieldnorm_id: 13,
@@ -345,7 +322,6 @@ mod tests {
             skip_reader.block_info(),
             BlockInfo::BitPacked {
                 doc_num_bits: 5u8,
-                strict_delta_encoded: true,
                 tf_num_bits: 2u8,
                 tf_sum: 0,
                 block_wand_fieldnorm_id: 8,
@@ -376,7 +352,6 @@ mod tests {
             skip_reader.block_info(),
             BlockInfo::BitPacked {
                 doc_num_bits: 2u8,
-                strict_delta_encoded: true,
                 tf_num_bits: 0,
                 tf_sum: 0u32,
                 block_wand_fieldnorm_id: 0,
@@ -389,7 +364,6 @@ mod tests {
             skip_reader.block_info(),
             BlockInfo::BitPacked {
                 doc_num_bits: 5u8,
-                strict_delta_encoded: true,
                 tf_num_bits: 0,
                 tf_sum: 0u32,
                 block_wand_fieldnorm_id: 0,
@@ -419,7 +393,6 @@ mod tests {
             skip_reader.block_info(),
             BlockInfo::BitPacked {
                 doc_num_bits: 2u8,
-                strict_delta_encoded: true,
                 tf_num_bits: 0,
                 tf_sum: 0u32,
                 block_wand_fieldnorm_id: 0,
@@ -429,18 +402,4 @@ mod tests {
         skip_reader.advance();
         assert_eq!(skip_reader.block_info(), BlockInfo::VInt { num_docs: 0u32 });
     }
-
-    #[test]
-    fn test_encode_decode_bitwidth() {
-        for bitwidth in 0..=32 {
-            for delta_1 in [false, true] {
-                assert_eq!(
-                    (bitwidth, delta_1),
-                    decode_bitwidth(encode_bitwidth(bitwidth, delta_1))
-                );
-            }
-        }
-        assert_eq!(0b01000010, encode_bitwidth(0b10, true));
-        assert_eq!(0b00000010, encode_bitwidth(0b10, false));
-    }
 }

src/query/bm25.rs

@@ -77,7 +77,7 @@ pub struct Bm25Params {
 /// A struct used for computing BM25 scores.
 #[derive(Clone)]
 pub struct Bm25Weight {
-    idf_explain: Option<Explanation>,
+    idf_explain: Explanation,
     weight: Score,
     cache: [Score; 256],
     average_fieldnorm: Score,
@@ -147,30 +147,11 @@ impl Bm25Weight {
         idf_explain.add_const("N, total number of docs", total_num_docs as Score);
         Bm25Weight::new(idf_explain, avg_fieldnorm)
     }
-    /// Construct a [Bm25Weight] for a single term.
-    /// This method does not carry the [Explanation] for the idf.
-    pub fn for_one_term_without_explain(
-        term_doc_freq: u64,
-        total_num_docs: u64,
-        avg_fieldnorm: Score,
-    ) -> Bm25Weight {
-        let idf = idf(term_doc_freq, total_num_docs);
-        Bm25Weight::new_without_explain(idf, avg_fieldnorm)
-    }
 
     pub(crate) fn new(idf_explain: Explanation, average_fieldnorm: Score) -> Bm25Weight {
         let weight = idf_explain.value() * (1.0 + K1);
         Bm25Weight {
-            idf_explain: Some(idf_explain),
-            weight,
-            cache: compute_tf_cache(average_fieldnorm),
-            average_fieldnorm,
-        }
-    }
-    pub(crate) fn new_without_explain(idf: f32, average_fieldnorm: Score) -> Bm25Weight {
-        let weight = idf * (1.0 + K1);
-        Bm25Weight {
-            idf_explain: None,
+            idf_explain,
             weight,
             cache: compute_tf_cache(average_fieldnorm),
             average_fieldnorm,
@@ -221,9 +202,7 @@ impl Bm25Weight {
 
         let mut explanation = Explanation::new("TermQuery, product of...", score);
         explanation.add_detail(Explanation::new("(K1+1)", K1 + 1.0));
-        if let Some(idf_explain) = &self.idf_explain {
-            explanation.add_detail(idf_explain.clone());
-        }
+        explanation.add_detail(self.idf_explain.clone());
         explanation.add_detail(tf_explanation);
         explanation
     }

src/query/boost_query.rs

@@ -74,8 +74,7 @@ impl Weight for BoostWeight {
     fn explain(&self, reader: &SegmentReader, doc: u32) -> crate::Result<Explanation> {
         let underlying_explanation = self.weight.explain(reader, doc)?;
         let score = underlying_explanation.value() * self.boost;
-        let mut explanation =
-            Explanation::new_with_string(format!("Boost x{} of ...", self.boost), score);
+        let mut explanation = Explanation::new(format!("Boost x{} of ...", self.boost), score);
         explanation.add_detail(underlying_explanation);
         Ok(explanation)
     }
@@ -152,7 +151,7 @@ mod tests {
         let explanation = query.explain(&searcher, DocAddress::new(0, 0u32)).unwrap();
         assert_eq!(
             explanation.to_pretty_json(),
-            "{\n  \"value\": 0.2,\n  \"description\": \"Boost x0.2 of ...\",\n  \"details\": [\n    {\n      \"value\": 1.0,\n      \"description\": \"AllQuery\"\n    }\n  ]\n}"
+            "{\n  \"value\": 0.2,\n  \"description\": \"Boost x0.2 of ...\",\n  \"details\": [\n    {\n      \"value\": 1.0,\n      \"description\": \"AllQuery\",\n      \"context\": []\n    }\n  ],\n  \"context\": []\n}"
         );
         Ok(())
     }

src/query/const_score_query.rs

@@ -164,9 +164,11 @@ mod tests {
   "details": [
     {
       "value": 1.0,
-      "description": "AllQuery"
+      "description": "AllQuery",
+      "context": []
     }
-  ]
+  ],
+  "context": []
 }"#
         );
         Ok(())

src/query/explanation.rs

@@ -1,4 +1,3 @@
-use std::borrow::Cow;
 use std::fmt;
 
 use serde::Serialize;
@@ -17,12 +16,12 @@ pub(crate) fn does_not_match(doc: DocId) -> TantivyError {
 #[derive(Clone, Serialize)]
 pub struct Explanation {
     value: Score,
-    description: Cow<'static, str>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    details: Option<Vec<Explanation>>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    context: Option<Vec<String>>,
+    description: String,
+    #[serde(skip_serializing_if = "Vec::is_empty")]
+    details: Vec<Explanation>,
+    context: Vec<String>,
 }
+
 impl fmt::Debug for Explanation {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         write!(f, "Explanation({})", self.to_pretty_json())
@@ -31,21 +30,12 @@ impl fmt::Debug for Explanation {
 
 impl Explanation {
     /// Creates a new explanation object.
-    pub fn new_with_string(description: String, value: Score) -> Explanation {
+    pub fn new<T: ToString>(description: T, value: Score) -> Explanation {
         Explanation {
             value,
-            description: Cow::Owned(description),
-            details: None,
-            context: None,
-        }
-    }
-    /// Creates a new explanation object.
-    pub fn new(description: &'static str, value: Score) -> Explanation {
-        Explanation {
-            value,
-            description: Cow::Borrowed(description),
-            details: None,
-            context: None,
+            description: description.to_string(),
+            details: vec![],
+            context: vec![],
         }
     }
 
@@ -58,21 +48,17 @@ impl Explanation {
     ///
     /// Details are treated as child of the current node.
     pub fn add_detail(&mut self, child_explanation: Explanation) {
-        self.details
-            .get_or_insert_with(Vec::new)
-            .push(child_explanation);
+        self.details.push(child_explanation);
     }
 
     /// Adds some extra context to the explanation.
     pub fn add_context(&mut self, context: String) {
-        self.context.get_or_insert_with(Vec::new).push(context);
+        self.context.push(context);
     }
 
     /// Shortcut for `self.details.push(Explanation::new(name, value));`
-    pub fn add_const(&mut self, name: &'static str, value: Score) {
-        self.details
-            .get_or_insert_with(Vec::new)
-            .push(Explanation::new(name, value));
+    pub fn add_const<T: ToString>(&mut self, name: T, value: Score) {
+        self.details.push(Explanation::new(name, value));
     }
 
     /// Returns an indented json representation of the explanation tree for debug usage.

src/query/range_query/fast_field_range_query.rs

@@ -31,8 +31,8 @@ impl VecCursor {
         self.current_pos = 0;
         &mut self.docs
     }
-    fn last_doc(&self) -> Option<u32> {
-        self.docs.last().cloned()
+    fn last_value(&self) -> Option<u32> {
+        self.docs.iter().last().cloned()
     }
     fn is_empty(&self) -> bool {
         self.current().is_none()
@@ -112,15 +112,15 @@ impl<T: Send + Sync + PartialOrd + Copy + Debug + 'static> RangeDocSet<T> {
             finished_to_end = true;
         }
 
-        let last_doc = self.loaded_docs.last_doc();
+        let last_value = self.loaded_docs.last_value();
         let doc_buffer: &mut Vec<DocId> = self.loaded_docs.get_cleared_data();
         self.column.get_docids_for_value_range(
             self.value_range.clone(),
             self.next_fetch_start..end,
             doc_buffer,
         );
-        if let Some(last_doc) = last_doc {
-            while self.loaded_docs.current() == Some(last_doc) {
+        if let Some(last_value) = last_value {
+            while self.loaded_docs.current() == Some(last_value) {
                 self.loaded_docs.next();
             }
         }
@@ -136,7 +136,7 @@ impl<T: Send + Sync + PartialOrd + Copy + Debug + 'static> DocSet for RangeDocSe
         if let Some(docid) = self.loaded_docs.next() {
             return docid;
         }
-        if self.next_fetch_start >= self.column.num_docs() {
+        if self.next_fetch_start >= self.column.values.num_vals() {
             return TERMINATED;
         }
         self.fetch_block();
@@ -177,54 +177,3 @@ impl<T: Send + Sync + PartialOrd + Copy + Debug + 'static> DocSet for RangeDocSe
         0 // heuristic possible by checking number of hits when fetching a block
     }
 }
-
-#[cfg(test)]
-mod tests {
-    use crate::collector::Count;
-    use crate::directory::RamDirectory;
-    use crate::query::RangeQuery;
-    use crate::{schema, IndexBuilder, TantivyDocument};
-
-    #[test]
-    fn range_query_fast_optional_field_minimum() {
-        let mut schema_builder = schema::SchemaBuilder::new();
-        let id_field = schema_builder.add_text_field("id", schema::STRING);
-        let score_field = schema_builder.add_u64_field("score", schema::FAST | schema::INDEXED);
-
-        let dir = RamDirectory::default();
-        let index = IndexBuilder::new()
-            .schema(schema_builder.build())
-            .open_or_create(dir)
-            .unwrap();
-
-        {
-            let mut writer = index.writer(15_000_000).unwrap();
-
-            let count = 1000;
-            for i in 0..count {
-                let mut doc = TantivyDocument::new();
-                doc.add_text(id_field, format!("doc{i}"));
-
-                let nb_scores = i % 2; // 0 or 1 scores
-                for _ in 0..nb_scores {
-                    doc.add_u64(score_field, 80);
-                }
-
-                writer.add_document(doc).unwrap();
-            }
-            writer.commit().unwrap();
-        }
-
-        let reader = index.reader().unwrap();
-        let searcher = reader.searcher();
-
-        let query = RangeQuery::new_u64_bounds(
-            "score".to_string(),
-            std::ops::Bound::Included(70),
-            std::ops::Bound::Unbounded,
-        );
-
-        let count = searcher.search(&query, &Count).unwrap();
-        assert_eq!(count, 500);
-    }
-}

src/query/range_query/mod.rs

@@ -7,7 +7,7 @@ mod range_query;
 mod range_query_ip_fastfield;
 mod range_query_u64_fastfield;
 
-pub use self::range_query::RangeQuery;
+pub use self::range_query::{RangeQuery, RangeWeight};
 pub use self::range_query_ip_fastfield::IPFastFieldRangeWeight;
 pub use self::range_query_u64_fastfield::FastFieldRangeWeight;
 

src/query/regex_query.rs

@@ -63,7 +63,7 @@ impl RegexQuery {
     /// Creates a new RegexQuery from a given pattern
     pub fn from_pattern(regex_pattern: &str, field: Field) -> crate::Result<Self> {
         let regex = Regex::new(regex_pattern)
-            .map_err(|err| TantivyError::InvalidArgument(format!("RegexQueryError: {err}")))?;
+            .map_err(|_| TantivyError::InvalidArgument(regex_pattern.to_string()))?;
         Ok(RegexQuery::from_regex(regex, field))
     }
 
@@ -176,16 +176,4 @@ mod test {
         verify_regex_query(matching_one, matching_zero, reader);
         Ok(())
     }
-
-    #[test]
-    pub fn test_pattern_error() {
-        let (_reader, field) = build_test_index().unwrap();
-
-        match RegexQuery::from_pattern(r"(foo", field) {
-            Err(crate::TantivyError::InvalidArgument(msg)) => {
-                assert!(msg.contains("error: unclosed group"))
-            }
-            res => panic!("unexpected result: {:?}", res),
-        }
-    }
 }

src/query/term_query/mod.rs

@@ -4,6 +4,8 @@ mod term_weight;
 
 pub use self::term_query::TermQuery;
 pub use self::term_scorer::TermScorer;
+pub use self::term_weight::TermWeight;
+
 #[cfg(test)]
 mod tests {
 

src/query/term_query/term_query.rs

@@ -101,7 +101,7 @@ impl TermQuery {
                 ..
             } => Bm25Weight::for_terms(statistics_provider, &[self.term.clone()])?,
             EnableScoring::Disabled { .. } => {
-                Bm25Weight::new(Explanation::new("<no score>", 1.0f32), 1.0f32)
+                Bm25Weight::new(Explanation::new("<no score>".to_string(), 1.0f32), 1.0f32)
             }
         };
         let scoring_enabled = enable_scoring.is_scoring_enabled();

src/query/term_query/term_scorer.rs

@@ -93,7 +93,7 @@ impl TermScorer {
     }
 
     pub fn last_doc_in_block(&self) -> DocId {
-        self.postings.block_cursor.skip_reader().last_doc_in_block()
+        self.postings.block_cursor.skip_reader.last_doc_in_block()
     }
 }
 
@@ -127,7 +127,6 @@ impl Scorer for TermScorer {
 mod tests {
     use proptest::prelude::*;
 
-    use crate::indexer::index_writer::MEMORY_BUDGET_NUM_BYTES_MIN;
     use crate::merge_policy::NoMergePolicy;
     use crate::postings::compression::COMPRESSION_BLOCK_SIZE;
     use crate::query::term_query::TermScorer;
@@ -297,8 +296,7 @@ mod tests {
         let text_field = schema_builder.add_text_field("text", TEXT);
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
-        let mut writer: IndexWriter =
-            index.writer_with_num_threads(3, 3 * MEMORY_BUDGET_NUM_BYTES_MIN)?;
+        let mut writer: IndexWriter = index.writer_with_num_threads(3, 30_000_000)?;
         use rand::Rng;
         let mut rng = rand::thread_rng();
         writer.set_merge_policy(Box::new(NoMergePolicy));

src/reader/mod.rs

@@ -28,7 +28,7 @@ pub enum ReloadPolicy {
     Manual,
     /// The index is reloaded within milliseconds after a new commit is available.
     /// This is made possible by watching changes in the `meta.json` file.
-    OnCommitWithDelay, // TODO add NEAR_REAL_TIME(target_ms)
+    OnCommit, // TODO add NEAR_REAL_TIME(target_ms)
 }
 
 /// [`IndexReader`] builder
@@ -51,7 +51,7 @@ impl IndexReaderBuilder {
     #[must_use]
     pub(crate) fn new(index: Index) -> IndexReaderBuilder {
         IndexReaderBuilder {
-            reload_policy: ReloadPolicy::OnCommitWithDelay,
+            reload_policy: ReloadPolicy::OnCommit,
             index,
             warmers: Vec::new(),
             num_warming_threads: 1,
@@ -83,7 +83,7 @@ impl IndexReaderBuilder {
                 // No need to set anything...
                 None
             }
-            ReloadPolicy::OnCommitWithDelay => {
+            ReloadPolicy::OnCommit => {
                 let inner_reader_arc_clone = inner_reader_arc.clone();
                 let callback = move || {
                     if let Err(err) = inner_reader_arc_clone.reload() {
@@ -282,7 +282,7 @@ impl IndexReader {
     /// Update searchers so that they reflect the state of the last
     /// `.commit()`.
     ///
-    /// If you set up the [`ReloadPolicy::OnCommitWithDelay`] (which is the default)
+    /// If you set up the [`ReloadPolicy::OnCommit`] (which is the default)
     /// every commit should be rapidly reflected on your `IndexReader` and you should
     /// not need to call `reload()` at all.
     ///

src/schema/bytes_options.rs

@@ -40,25 +40,21 @@ impl From<BytesOptionsDeser> for BytesOptions {
 
 impl BytesOptions {
     /// Returns true if the value is indexed.
-    #[inline]
     pub fn is_indexed(&self) -> bool {
         self.indexed
     }
 
     /// Returns true if and only if the value is normed.
-    #[inline]
     pub fn fieldnorms(&self) -> bool {
         self.fieldnorms
     }
 
     /// Returns true if the value is a fast field.
-    #[inline]
     pub fn is_fast(&self) -> bool {
         self.fast
     }
 
     /// Returns true if the value is stored.
-    #[inline]
     pub fn is_stored(&self) -> bool {
         self.stored
     }

src/schema/date_time_options.rs

@@ -27,25 +27,21 @@ pub struct DateOptions {
 
 impl DateOptions {
     /// Returns true iff the value is stored.
-    #[inline]
     pub fn is_stored(&self) -> bool {
         self.stored
     }
 
     /// Returns true iff the value is indexed and therefore searchable.
-    #[inline]
     pub fn is_indexed(&self) -> bool {
         self.indexed
     }
 
     /// Returns true iff the field has fieldnorm.
-    #[inline]
     pub fn fieldnorms(&self) -> bool {
         self.fieldnorms && self.indexed
     }
 
     /// Returns true iff the value is a fast field.
-    #[inline]
     pub fn is_fast(&self) -> bool {
         self.fast
     }

src/schema/document/de.rs

@@ -18,8 +18,6 @@ use std::sync::Arc;
 use columnar::MonotonicallyMappableToU128;
 use common::{u64_to_f64, BinarySerializable, DateTime, VInt};
 
-use super::se::BinaryObjectSerializer;
-use super::{OwnedValue, Value};
 use crate::schema::document::type_codes;
 use crate::schema::{Facet, Field};
 use crate::tokenizer::PreTokenizedString;
@@ -159,9 +157,6 @@ pub enum ValueType {
     Array,
     /// A dynamic object value.
     Object,
-    /// A JSON object value. Deprecated.
-    #[deprecated]
-    JSONObject,
 }
 
 /// A value visitor for deserializing a document value.
@@ -381,8 +376,6 @@ where R: Read
             type_codes::NULL_CODE => ValueType::Null,
             type_codes::ARRAY_CODE => ValueType::Array,
             type_codes::OBJECT_CODE => ValueType::Object,
-            #[allow(deprecated)]
-            type_codes::JSON_OBJ_CODE => ValueType::JSONObject,
             _ => {
                 return Err(DeserializeError::from(io::Error::new(
                     io::ErrorKind::InvalidData,
@@ -521,26 +514,6 @@ where R: Read
                 let access = BinaryObjectDeserializer::from_reader(self.reader)?;
                 visitor.visit_object(access)
             }
-            #[allow(deprecated)]
-            ValueType::JSONObject => {
-                // This is a compatibility layer
-                // The implementation is slow, but is temporary anyways
-                let mut de = serde_json::Deserializer::from_reader(self.reader);
-                let json_map = <serde_json::Map::<String, serde_json::Value> as serde::Deserialize>::deserialize(&mut de).map_err(|err| DeserializeError::Custom(err.to_string()))?;
-                let mut out = Vec::new();
-                let mut serializer = BinaryObjectSerializer::begin(json_map.len(), &mut out)?;
-                for (key, val) in json_map {
-                    let val: OwnedValue = val.into();
-                    serializer.serialize_entry(&key, (&val).as_value())?;
-                }
-                serializer.end()?;
-
-                let out_rc = std::rc::Rc::new(out);
-                let mut slice: &[u8] = &out_rc;
-                let access = BinaryObjectDeserializer::from_reader(&mut slice)?;
-
-                visitor.visit_object(access)
-            }
         }
     }
 }
@@ -818,8 +791,7 @@ mod tests {
     use super::*;
     use crate::schema::document::existing_type_impls::JsonObjectIter;
     use crate::schema::document::se::BinaryValueSerializer;
-    use crate::schema::document::{ReferenceValue, ReferenceValueLeaf};
-    use crate::schema::OwnedValue;
+    use crate::schema::document::ReferenceValue;
 
     fn serialize_value<'a>(value: ReferenceValue<'a, &'a serde_json::Value>) -> Vec<u8> {
         let mut writer = Vec::new();
@@ -838,35 +810,34 @@ mod tests {
 
     #[test]
     fn test_simple_value_serialize() {
-        let result = serialize_value(ReferenceValueLeaf::Null.into());
+        let result = serialize_value(ReferenceValue::Null);
         let value = deserialize_value(result);
         assert_eq!(value, crate::schema::OwnedValue::Null);
 
-        let result = serialize_value(ReferenceValueLeaf::Str("hello, world").into());
+        let result = serialize_value(ReferenceValue::Str("hello, world"));
         let value = deserialize_value(result);
         assert_eq!(
             value,
             crate::schema::OwnedValue::Str(String::from("hello, world"))
         );
 
-        let result = serialize_value(ReferenceValueLeaf::U64(123).into());
+        let result = serialize_value(ReferenceValue::U64(123));
         let value = deserialize_value(result);
         assert_eq!(value, crate::schema::OwnedValue::U64(123));
 
-        let result = serialize_value(ReferenceValueLeaf::I64(-123).into());
+        let result = serialize_value(ReferenceValue::I64(-123));
         let value = deserialize_value(result);
         assert_eq!(value, crate::schema::OwnedValue::I64(-123));
 
-        let result = serialize_value(ReferenceValueLeaf::F64(123.3845).into());
+        let result = serialize_value(ReferenceValue::F64(123.3845));
         let value = deserialize_value(result);
         assert_eq!(value, crate::schema::OwnedValue::F64(123.3845));
 
-        let result = serialize_value(ReferenceValueLeaf::Bool(false).into());
+        let result = serialize_value(ReferenceValue::Bool(false));
         let value = deserialize_value(result);
         assert_eq!(value, crate::schema::OwnedValue::Bool(false));
 
-        let result =
-            serialize_value(ReferenceValueLeaf::Date(DateTime::from_timestamp_micros(100)).into());
+        let result = serialize_value(ReferenceValue::Date(DateTime::from_timestamp_micros(100)));
         let value = deserialize_value(result);
         assert_eq!(
             value,
@@ -874,7 +845,7 @@ mod tests {
         );
 
         let facet = Facet::from_text("/hello/world").unwrap();
-        let result = serialize_value(ReferenceValueLeaf::Facet(&facet).into());
+        let result = serialize_value(ReferenceValue::Facet(&facet));
         let value = deserialize_value(result);
         assert_eq!(value, crate::schema::OwnedValue::Facet(facet));
 
@@ -882,7 +853,7 @@ mod tests {
             text: "hello, world".to_string(),
             tokens: vec![Token::default(), Token::default()],
         };
-        let result = serialize_value(ReferenceValueLeaf::PreTokStr(&pre_tok_str).into());
+        let result = serialize_value(ReferenceValue::PreTokStr(&pre_tok_str));
         let value = deserialize_value(result);
         assert_eq!(value, crate::schema::OwnedValue::PreTokStr(pre_tok_str));
     }
@@ -982,25 +953,6 @@ mod tests {
         assert_eq!(value, crate::schema::OwnedValue::Object(expected_object));
     }
 
-    #[test]
-    fn test_json_compat() {
-        let data = [
-            8, 123, 34, 107, 101, 121, 97, 58, 34, 58, 34, 98, 108, 117, 98, 34, 44, 34, 118, 97,
-            108, 115, 34, 58, 123, 34, 104, 101, 121, 34, 58, 34, 104, 111, 34, 125, 125,
-        ]
-        .to_vec();
-        let expected = json!({
-            "keya:": "blub",
-            "vals": {
-                "hey": "ho"
-            }
-        });
-        let expected_val: OwnedValue = expected.clone().into();
-
-        let value = deserialize_value(data);
-        assert_eq!(value, expected_val);
-    }
-
     #[test]
     fn test_nested_serialize() {
         let mut object = serde_json::Map::new();
@@ -1073,11 +1025,11 @@ mod tests {
         let mut expected_object = BTreeMap::new();
         expected_object.insert(
             "my-array".to_string(),
-            OwnedValue::Array(vec![OwnedValue::Array(vec![
-                OwnedValue::Array(vec![]),
-                OwnedValue::Array(vec![OwnedValue::Null]),
+            crate::schema::OwnedValue::Array(vec![crate::schema::OwnedValue::Array(vec![
+                crate::schema::OwnedValue::Array(vec![]),
+                crate::schema::OwnedValue::Array(vec![crate::schema::OwnedValue::Null]),
             ])]),
         );
-        assert_eq!(value, OwnedValue::Object(expected_object));
+        assert_eq!(value, crate::schema::OwnedValue::Object(expected_object));
     }
 }

src/schema/document/existing_type_impls.rs

@@ -8,7 +8,6 @@ use std::collections::{btree_map, hash_map, BTreeMap, HashMap};
 
 use serde_json::Number;
 
-use super::ReferenceValueLeaf;
 use crate::schema::document::{
     ArrayAccess, DeserializeError, Document, DocumentDeserialize, DocumentDeserializer,
     ObjectAccess, ReferenceValue, Value, ValueDeserialize, ValueDeserializer, ValueVisitor,
@@ -22,20 +21,20 @@ impl<'a> Value<'a> for &'a serde_json::Value {
 
     fn as_value(&self) -> ReferenceValue<'a, Self> {
         match self {
-            serde_json::Value::Null => ReferenceValueLeaf::Null.into(),
-            serde_json::Value::Bool(value) => ReferenceValueLeaf::Bool(*value).into(),
+            serde_json::Value::Null => ReferenceValue::Null,
+            serde_json::Value::Bool(value) => ReferenceValue::Bool(*value),
             serde_json::Value::Number(number) => {
                 if let Some(val) = number.as_i64() {
-                    ReferenceValueLeaf::I64(val).into()
+                    ReferenceValue::I64(val)
                 } else if let Some(val) = number.as_u64() {
-                    ReferenceValueLeaf::U64(val).into()
+                    ReferenceValue::U64(val)
                 } else if let Some(val) = number.as_f64() {
-                    ReferenceValueLeaf::F64(val).into()
+                    ReferenceValue::F64(val)
                 } else {
                     panic!("Unsupported serde_json number {number}");
                 }
             }
-            serde_json::Value::String(val) => ReferenceValueLeaf::Str(val).into(),
+            serde_json::Value::String(val) => ReferenceValue::Str(val),
             serde_json::Value::Array(elements) => ReferenceValue::Array(elements.iter()),
             serde_json::Value::Object(object) => {
                 ReferenceValue::Object(JsonObjectIter(object.iter()))
@@ -78,7 +77,7 @@ impl ValueDeserialize for serde_json::Value {
             }
 
             fn visit_bool(&self, val: bool) -> Result<Self::Value, DeserializeError> {
-                Ok(serde_json::Value::Bool(val))
+                Ok(serde_json::Value::Bool(val.into()))
             }
 
             fn visit_array<'de, A>(&self, mut access: A) -> Result<Self::Value, DeserializeError>

src/schema/document/mod.rs

@@ -116,7 +116,6 @@
 //!
 //! ```
 //! use tantivy::schema::document::ReferenceValue;
-//! use tantivy::schema::document::ReferenceValueLeaf;
 //! use tantivy::schema::{Value};
 //!
 //! #[derive(Debug)]
@@ -142,9 +141,9 @@
 //!     fn as_value(&self) -> ReferenceValue<'a, Self> {
 //!         // We can support any type that Tantivy itself supports.
 //!         match self {
-//!             MyCustomValue::String(val) => ReferenceValue::Leaf(ReferenceValueLeaf::Str(*val)),
-//!             MyCustomValue::Float(val) => ReferenceValue::Leaf(ReferenceValueLeaf::F64(*val)),
-//!             MyCustomValue::Bool(val) => ReferenceValue::Leaf(ReferenceValueLeaf::Bool(*val)),
+//!             MyCustomValue::String(val) => ReferenceValue::Str(val),
+//!             MyCustomValue::Float(val) => ReferenceValue::F64(*val),
+//!             MyCustomValue::Bool(val) => ReferenceValue::Bool(*val),
 //!         }
 //!     }
 //!
@@ -171,7 +170,7 @@ pub use self::de::{
 pub use self::default_document::{DocParsingError, TantivyDocument};
 pub use self::owned_value::OwnedValue;
 pub(crate) use self::se::BinaryDocumentSerializer;
-pub use self::value::{ReferenceValue, ReferenceValueLeaf, Value};
+pub use self::value::{ReferenceValue, Value};
 use super::*;
 
 /// The core trait representing a document within the index.
@@ -254,9 +253,8 @@ pub(crate) mod type_codes {
     pub const DATE_CODE: u8 = 5;
     pub const F64_CODE: u8 = 6;
     pub const EXT_CODE: u8 = 7;
-
-    #[deprecated]
-    pub const JSON_OBJ_CODE: u8 = 8; // Replaced by the `OBJECT_CODE`.
+    // Replaced by the `OBJECT_CODE`.
+    // -- pub const JSON_OBJ_CODE: u8 = 8;
     pub const BOOL_CODE: u8 = 9;
     pub const IP_CODE: u8 = 10;
     pub const NULL_CODE: u8 = 11;

src/schema/document/owned_value.rs

@@ -8,7 +8,6 @@ use serde::de::{MapAccess, SeqAccess};
 use time::format_description::well_known::Rfc3339;
 use time::OffsetDateTime;
 
-use super::ReferenceValueLeaf;
 use crate::schema::document::{
     ArrayAccess, DeserializeError, ObjectAccess, ReferenceValue, Value, ValueDeserialize,
     ValueDeserializer, ValueVisitor,
@@ -63,17 +62,17 @@ impl<'a> Value<'a> for &'a OwnedValue {
 
     fn as_value(&self) -> ReferenceValue<'a, Self> {
         match self {
-            OwnedValue::Null => ReferenceValueLeaf::Null.into(),
-            OwnedValue::Str(val) => ReferenceValueLeaf::Str(val).into(),
-            OwnedValue::PreTokStr(val) => ReferenceValueLeaf::PreTokStr(val).into(),
-            OwnedValue::U64(val) => ReferenceValueLeaf::U64(*val).into(),
-            OwnedValue::I64(val) => ReferenceValueLeaf::I64(*val).into(),
-            OwnedValue::F64(val) => ReferenceValueLeaf::F64(*val).into(),
-            OwnedValue::Bool(val) => ReferenceValueLeaf::Bool(*val).into(),
-            OwnedValue::Date(val) => ReferenceValueLeaf::Date(*val).into(),
-            OwnedValue::Facet(val) => ReferenceValueLeaf::Facet(val).into(),
-            OwnedValue::Bytes(val) => ReferenceValueLeaf::Bytes(val).into(),
-            OwnedValue::IpAddr(val) => ReferenceValueLeaf::IpAddr(*val).into(),
+            OwnedValue::Null => ReferenceValue::Null,
+            OwnedValue::Str(val) => ReferenceValue::Str(val),
+            OwnedValue::PreTokStr(val) => ReferenceValue::PreTokStr(val),
+            OwnedValue::U64(val) => ReferenceValue::U64(*val),
+            OwnedValue::I64(val) => ReferenceValue::I64(*val),
+            OwnedValue::F64(val) => ReferenceValue::F64(*val),
+            OwnedValue::Bool(val) => ReferenceValue::Bool(*val),
+            OwnedValue::Date(val) => ReferenceValue::Date(*val),
+            OwnedValue::Facet(val) => ReferenceValue::Facet(val),
+            OwnedValue::Bytes(val) => ReferenceValue::Bytes(val),
+            OwnedValue::IpAddr(val) => ReferenceValue::IpAddr(*val),
             OwnedValue::Array(array) => ReferenceValue::Array(array.iter()),
             OwnedValue::Object(object) => ReferenceValue::Object(ObjectMapIter(object.iter())),
         }
@@ -265,19 +264,17 @@ impl<'de> serde::Deserialize<'de> for OwnedValue {
 impl<'a, V: Value<'a>> From<ReferenceValue<'a, V>> for OwnedValue {
     fn from(val: ReferenceValue<'a, V>) -> OwnedValue {
         match val {
-            ReferenceValue::Leaf(leaf) => match leaf {
-                ReferenceValueLeaf::Null => OwnedValue::Null,
-                ReferenceValueLeaf::Str(val) => OwnedValue::Str(val.to_string()),
-                ReferenceValueLeaf::U64(val) => OwnedValue::U64(val),
-                ReferenceValueLeaf::I64(val) => OwnedValue::I64(val),
-                ReferenceValueLeaf::F64(val) => OwnedValue::F64(val),
-                ReferenceValueLeaf::Date(val) => OwnedValue::Date(val),
-                ReferenceValueLeaf::Facet(val) => OwnedValue::Facet(val.clone()),
-                ReferenceValueLeaf::Bytes(val) => OwnedValue::Bytes(val.to_vec()),
-                ReferenceValueLeaf::IpAddr(val) => OwnedValue::IpAddr(val),
-                ReferenceValueLeaf::Bool(val) => OwnedValue::Bool(val),
-                ReferenceValueLeaf::PreTokStr(val) => OwnedValue::PreTokStr(val.clone()),
-            },
+            ReferenceValue::Null => OwnedValue::Null,
+            ReferenceValue::Str(val) => OwnedValue::Str(val.to_string()),
+            ReferenceValue::U64(val) => OwnedValue::U64(val),
+            ReferenceValue::I64(val) => OwnedValue::I64(val),
+            ReferenceValue::F64(val) => OwnedValue::F64(val),
+            ReferenceValue::Date(val) => OwnedValue::Date(val),
+            ReferenceValue::Facet(val) => OwnedValue::Facet(val.clone()),
+            ReferenceValue::Bytes(val) => OwnedValue::Bytes(val.to_vec()),
+            ReferenceValue::IpAddr(val) => OwnedValue::IpAddr(val),
+            ReferenceValue::Bool(val) => OwnedValue::Bool(val),
+            ReferenceValue::PreTokStr(val) => OwnedValue::PreTokStr(val.clone()),
             ReferenceValue::Array(val) => {
                 OwnedValue::Array(val.map(|v| v.as_value().into()).collect())
             }
@@ -369,7 +366,7 @@ impl From<BTreeMap<String, OwnedValue>> for OwnedValue {
 
 fn can_be_rfc3339_date_time(text: &str) -> bool {
     if let Some(&first_byte) = text.as_bytes().first() {
-        if first_byte.is_ascii_digit() {
+        if (b'0'..=b'9').contains(&first_byte) {
             return true;
         }
     }

src/schema/document/se.rs

@@ -5,7 +5,6 @@ use std::io::Write;
 use columnar::MonotonicallyMappableToU128;
 use common::{f64_to_u64, BinarySerializable, VInt};
 
-use super::{OwnedValue, ReferenceValueLeaf};
 use crate::schema::document::{type_codes, Document, ReferenceValue, Value};
 use crate::schema::Schema;
 
@@ -40,10 +39,10 @@ where W: Write
 
             let mut serializer = BinaryValueSerializer::new(self.writer);
             match value_access.as_value() {
-                ReferenceValue::Leaf(ReferenceValueLeaf::PreTokStr(pre_tokenized_text)) => {
-                    serializer.serialize_value(ReferenceValue::Leaf::<&'_ OwnedValue>(
-                        ReferenceValueLeaf::Str(&pre_tokenized_text.text),
-                    ))?;
+                ReferenceValue::PreTokStr(pre_tokenized_text) => {
+                    serializer.serialize_value(ReferenceValue::Str::<
+                        &'_ crate::schema::OwnedValue,
+                    >(&pre_tokenized_text.text))?;
                 }
                 _ => {
                     serializer.serialize_value(value_access.as_value())?;
@@ -91,61 +90,59 @@ where W: Write
         V: Value<'a>,
     {
         match value {
-            ReferenceValue::Leaf(leaf) => match leaf {
-                ReferenceValueLeaf::Null => self.write_type_code(type_codes::NULL_CODE),
-                ReferenceValueLeaf::Str(val) => {
-                    self.write_type_code(type_codes::TEXT_CODE)?;
+            ReferenceValue::Null => self.write_type_code(type_codes::NULL_CODE),
+            ReferenceValue::Str(val) => {
+                self.write_type_code(type_codes::TEXT_CODE)?;
 
-                    let temp_val = Cow::Borrowed(val);
-                    temp_val.serialize(self.writer)
-                }
-                ReferenceValueLeaf::U64(val) => {
-                    self.write_type_code(type_codes::U64_CODE)?;
+                let temp_val = Cow::Borrowed(val);
+                temp_val.serialize(self.writer)
+            }
+            ReferenceValue::U64(val) => {
+                self.write_type_code(type_codes::U64_CODE)?;
 
-                    val.serialize(self.writer)
-                }
-                ReferenceValueLeaf::I64(val) => {
-                    self.write_type_code(type_codes::I64_CODE)?;
+                val.serialize(self.writer)
+            }
+            ReferenceValue::I64(val) => {
+                self.write_type_code(type_codes::I64_CODE)?;
 
-                    val.serialize(self.writer)
-                }
-                ReferenceValueLeaf::F64(val) => {
-                    self.write_type_code(type_codes::F64_CODE)?;
+                val.serialize(self.writer)
+            }
+            ReferenceValue::F64(val) => {
+                self.write_type_code(type_codes::F64_CODE)?;
 
-                    f64_to_u64(val).serialize(self.writer)
-                }
-                ReferenceValueLeaf::Date(val) => {
-                    self.write_type_code(type_codes::DATE_CODE)?;
-                    val.serialize(self.writer)
-                }
-                ReferenceValueLeaf::Facet(val) => {
-                    self.write_type_code(type_codes::HIERARCHICAL_FACET_CODE)?;
+                f64_to_u64(val).serialize(self.writer)
+            }
+            ReferenceValue::Date(val) => {
+                self.write_type_code(type_codes::DATE_CODE)?;
+                val.serialize(self.writer)
+            }
+            ReferenceValue::Facet(val) => {
+                self.write_type_code(type_codes::HIERARCHICAL_FACET_CODE)?;
 
-                    val.serialize(self.writer)
-                }
-                ReferenceValueLeaf::Bytes(val) => {
-                    self.write_type_code(type_codes::BYTES_CODE)?;
+                val.serialize(self.writer)
+            }
+            ReferenceValue::Bytes(val) => {
+                self.write_type_code(type_codes::BYTES_CODE)?;
 
-                    let temp_val = Cow::Borrowed(val);
-                    temp_val.serialize(self.writer)
-                }
-                ReferenceValueLeaf::IpAddr(val) => {
-                    self.write_type_code(type_codes::IP_CODE)?;
+                let temp_val = Cow::Borrowed(val);
+                temp_val.serialize(self.writer)
+            }
+            ReferenceValue::IpAddr(val) => {
+                self.write_type_code(type_codes::IP_CODE)?;
 
-                    val.to_u128().serialize(self.writer)
-                }
-                ReferenceValueLeaf::Bool(val) => {
-                    self.write_type_code(type_codes::BOOL_CODE)?;
+                val.to_u128().serialize(self.writer)
+            }
+            ReferenceValue::Bool(val) => {
+                self.write_type_code(type_codes::BOOL_CODE)?;
 
-                    val.serialize(self.writer)
-                }
-                ReferenceValueLeaf::PreTokStr(val) => {
-                    self.write_type_code(type_codes::EXT_CODE)?;
-                    self.write_type_code(type_codes::TOK_STR_EXT_CODE)?;
+                val.serialize(self.writer)
+            }
+            ReferenceValue::PreTokStr(val) => {
+                self.write_type_code(type_codes::EXT_CODE)?;
+                self.write_type_code(type_codes::TOK_STR_EXT_CODE)?;
 
-                    val.serialize(self.writer)
-                }
-            },
+                val.serialize(self.writer)
+            }
             ReferenceValue::Array(elements) => {
                 self.write_type_code(type_codes::ARRAY_CODE)?;
 
@@ -275,7 +272,7 @@ where W: Write
         // as we could avoid writing the extra byte per key. But the gain is
         // largely not worth it for the extra complexity it brings.
         self.inner
-            .serialize_value(ReferenceValue::<'a, V>::Leaf(ReferenceValueLeaf::Str(key)))?;
+            .serialize_value(ReferenceValue::<'a, V>::Str(key))?;
         self.inner.serialize_value(value)?;
 
         self.actual_length += 1;
@@ -364,7 +361,7 @@ mod tests {
 
     #[test]
     fn test_simple_value_serialize() {
-        let result = serialize_value(ReferenceValueLeaf::Null.into());
+        let result = serialize_value(ReferenceValue::Null);
         let expected = binary_repr!(
             type_codes::NULL_CODE => (),
         );
@@ -373,7 +370,7 @@ mod tests {
             "Expected serialized value to match the binary representation"
         );
 
-        let result = serialize_value(ReferenceValueLeaf::Str("hello, world").into());
+        let result = serialize_value(ReferenceValue::Str("hello, world"));
         let expected = binary_repr!(
             type_codes::TEXT_CODE => String::from("hello, world"),
         );
@@ -382,7 +379,7 @@ mod tests {
             "Expected serialized value to match the binary representation"
         );
 
-        let result = serialize_value(ReferenceValueLeaf::U64(123).into());
+        let result = serialize_value(ReferenceValue::U64(123));
         let expected = binary_repr!(
             type_codes::U64_CODE => 123u64,
         );
@@ -391,7 +388,7 @@ mod tests {
             "Expected serialized value to match the binary representation"
         );
 
-        let result = serialize_value(ReferenceValueLeaf::I64(-123).into());
+        let result = serialize_value(ReferenceValue::I64(-123));
         let expected = binary_repr!(
             type_codes::I64_CODE => -123i64,
         );
@@ -400,7 +397,7 @@ mod tests {
             "Expected serialized value to match the binary representation"
         );
 
-        let result = serialize_value(ReferenceValueLeaf::F64(123.3845f64).into());
+        let result = serialize_value(ReferenceValue::F64(123.3845));
         let expected = binary_repr!(
             type_codes::F64_CODE => f64_to_u64(123.3845f64),
         );
@@ -409,7 +406,7 @@ mod tests {
             "Expected serialized value to match the binary representation"
         );
 
-        let result = serialize_value(ReferenceValueLeaf::Bool(false).into());
+        let result = serialize_value(ReferenceValue::Bool(false));
         let expected = binary_repr!(
             type_codes::BOOL_CODE => false,
         );
@@ -418,7 +415,7 @@ mod tests {
             "Expected serialized value to match the binary representation"
         );
 
-        let result = serialize_value(ReferenceValueLeaf::Date(DateTime::MAX).into());
+        let result = serialize_value(ReferenceValue::Date(DateTime::MAX));
         let expected = binary_repr!(
             type_codes::DATE_CODE => DateTime::MAX,
         );
@@ -428,7 +425,7 @@ mod tests {
         );
 
         let facet = Facet::from_text("/hello/world").unwrap();
-        let result = serialize_value(ReferenceValueLeaf::Facet(&facet).into());
+        let result = serialize_value(ReferenceValue::Facet(&facet));
         let expected = binary_repr!(
             type_codes::HIERARCHICAL_FACET_CODE => Facet::from_text("/hello/world").unwrap(),
         );
@@ -441,7 +438,7 @@ mod tests {
             text: "hello, world".to_string(),
             tokens: vec![Token::default(), Token::default()],
         };
-        let result = serialize_value(ReferenceValueLeaf::PreTokStr(&pre_tok_str).into());
+        let result = serialize_value(ReferenceValue::PreTokStr(&pre_tok_str));
         let expected = binary_repr!(
             type_codes::EXT_CODE, type_codes::TOK_STR_EXT_CODE => pre_tok_str,
         );

src/schema/document/value.rs

@@ -21,81 +21,108 @@ pub trait Value<'a>: Send + Sync + Debug {
     #[inline]
     /// Returns if the value is `null` or not.
     fn is_null(&self) -> bool {
-        matches!(
-            self.as_value(),
-            ReferenceValue::Leaf(ReferenceValueLeaf::Null)
-        )
+        matches!(self.as_value(), ReferenceValue::Null)
     }
 
     #[inline]
     /// If the Value is a String, returns the associated str. Returns None otherwise.
-    fn as_leaf(&self) -> Option<ReferenceValueLeaf<'a>> {
-        if let ReferenceValue::Leaf(val) = self.as_value() {
+    fn as_str(&self) -> Option<&'a str> {
+        if let ReferenceValue::Str(val) = self.as_value() {
             Some(val)
         } else {
             None
         }
     }
 
-    #[inline]
-    /// If the Value is a String, returns the associated str. Returns None otherwise.
-    fn as_str(&self) -> Option<&'a str> {
-        self.as_leaf().and_then(|leaf| leaf.as_str())
-    }
-
     #[inline]
     /// If the Value is a u64, returns the associated u64. Returns None otherwise.
     fn as_u64(&self) -> Option<u64> {
-        self.as_leaf().and_then(|leaf| leaf.as_u64())
+        if let ReferenceValue::U64(val) = self.as_value() {
+            Some(val)
+        } else {
+            None
+        }
     }
 
     #[inline]
     /// If the Value is a i64, returns the associated i64. Returns None otherwise.
     fn as_i64(&self) -> Option<i64> {
-        self.as_leaf().and_then(|leaf| leaf.as_i64())
+        if let ReferenceValue::I64(val) = self.as_value() {
+            Some(val)
+        } else {
+            None
+        }
     }
 
     #[inline]
     /// If the Value is a f64, returns the associated f64. Returns None otherwise.
     fn as_f64(&self) -> Option<f64> {
-        self.as_leaf().and_then(|leaf| leaf.as_f64())
+        if let ReferenceValue::F64(val) = self.as_value() {
+            Some(val)
+        } else {
+            None
+        }
     }
 
     #[inline]
     /// If the Value is a datetime, returns the associated datetime. Returns None otherwise.
     fn as_datetime(&self) -> Option<DateTime> {
-        self.as_leaf().and_then(|leaf| leaf.as_datetime())
+        if let ReferenceValue::Date(val) = self.as_value() {
+            Some(val)
+        } else {
+            None
+        }
     }
 
     #[inline]
     /// If the Value is a IP address, returns the associated IP. Returns None otherwise.
     fn as_ip_addr(&self) -> Option<Ipv6Addr> {
-        self.as_leaf().and_then(|leaf| leaf.as_ip_addr())
+        if let ReferenceValue::IpAddr(val) = self.as_value() {
+            Some(val)
+        } else {
+            None
+        }
     }
 
     #[inline]
     /// If the Value is a bool, returns the associated bool. Returns None otherwise.
     fn as_bool(&self) -> Option<bool> {
-        self.as_leaf().and_then(|leaf| leaf.as_bool())
+        if let ReferenceValue::Bool(val) = self.as_value() {
+            Some(val)
+        } else {
+            None
+        }
     }
 
     #[inline]
     /// If the Value is a pre-tokenized string, returns the associated string. Returns None
     /// otherwise.
     fn as_pre_tokenized_text(&self) -> Option<&'a PreTokenizedString> {
-        self.as_leaf().and_then(|leaf| leaf.as_pre_tokenized_text())
+        if let ReferenceValue::PreTokStr(val) = self.as_value() {
+            Some(val)
+        } else {
+            None
+        }
     }
 
     #[inline]
     /// If the Value is a bytes value, returns the associated set of bytes. Returns None otherwise.
     fn as_bytes(&self) -> Option<&'a [u8]> {
-        self.as_leaf().and_then(|leaf| leaf.as_bytes())
+        if let ReferenceValue::Bytes(val) = self.as_value() {
+            Some(val)
+        } else {
+            None
+        }
     }
 
     #[inline]
     /// If the Value is a facet, returns the associated facet. Returns None otherwise.
     fn as_facet(&self) -> Option<&'a Facet> {
-        self.as_leaf().and_then(|leaf| leaf.as_facet())
+        if let ReferenceValue::Facet(val) = self.as_value() {
+            Some(val)
+        } else {
+            None
+        }
     }
 
     #[inline]
@@ -131,9 +158,11 @@ pub trait Value<'a>: Send + Sync + Debug {
     }
 }
 
-/// A enum representing a leaf value for tantivy to index.
-#[derive(Clone, Copy, Debug, PartialEq)]
-pub enum ReferenceValueLeaf<'a> {
+/// A enum representing a value for tantivy to index.
+#[derive(Clone, Debug, PartialEq)]
+pub enum ReferenceValue<'a, V>
+where V: Value<'a> + ?Sized
+{
     /// A null value.
     Null,
     /// The str type is used for any text information.
@@ -156,32 +185,15 @@ pub enum ReferenceValueLeaf<'a> {
     Bool(bool),
     /// Pre-tokenized str type,
     PreTokStr(&'a PreTokenizedString),
+    /// A an array containing multiple values.
+    Array(V::ArrayIter),
+    /// A nested / dynamic object.
+    Object(V::ObjectIter),
 }
 
-impl<'a, T: Value<'a> + ?Sized> From<ReferenceValueLeaf<'a>> for ReferenceValue<'a, T> {
-    #[inline]
-    fn from(value: ReferenceValueLeaf<'a>) -> Self {
-        match value {
-            ReferenceValueLeaf::Null => ReferenceValue::Leaf(ReferenceValueLeaf::Null),
-            ReferenceValueLeaf::Str(val) => ReferenceValue::Leaf(ReferenceValueLeaf::Str(val)),
-            ReferenceValueLeaf::U64(val) => ReferenceValue::Leaf(ReferenceValueLeaf::U64(val)),
-            ReferenceValueLeaf::I64(val) => ReferenceValue::Leaf(ReferenceValueLeaf::I64(val)),
-            ReferenceValueLeaf::F64(val) => ReferenceValue::Leaf(ReferenceValueLeaf::F64(val)),
-            ReferenceValueLeaf::Date(val) => ReferenceValue::Leaf(ReferenceValueLeaf::Date(val)),
-            ReferenceValueLeaf::Facet(val) => ReferenceValue::Leaf(ReferenceValueLeaf::Facet(val)),
-            ReferenceValueLeaf::Bytes(val) => ReferenceValue::Leaf(ReferenceValueLeaf::Bytes(val)),
-            ReferenceValueLeaf::IpAddr(val) => {
-                ReferenceValue::Leaf(ReferenceValueLeaf::IpAddr(val))
-            }
-            ReferenceValueLeaf::Bool(val) => ReferenceValue::Leaf(ReferenceValueLeaf::Bool(val)),
-            ReferenceValueLeaf::PreTokStr(val) => {
-                ReferenceValue::Leaf(ReferenceValueLeaf::PreTokStr(val))
-            }
-        }
-    }
-}
-
-impl<'a> ReferenceValueLeaf<'a> {
+impl<'a, V> ReferenceValue<'a, V>
+where V: Value<'a>
+{
     #[inline]
     /// Returns if the value is `null` or not.
     pub fn is_null(&self) -> bool {
@@ -288,105 +300,11 @@ impl<'a> ReferenceValueLeaf<'a> {
             None
         }
     }
-}
-
-/// A enum representing a value for tantivy to index.
-#[derive(Clone, Debug, PartialEq)]
-pub enum ReferenceValue<'a, V>
-where V: Value<'a> + ?Sized
-{
-    /// A null value.
-    Leaf(ReferenceValueLeaf<'a>),
-    /// A an array containing multiple values.
-    Array(V::ArrayIter),
-    /// A nested / dynamic object.
-    Object(V::ObjectIter),
-}
-
-impl<'a, V> ReferenceValue<'a, V>
-where V: Value<'a>
-{
-    #[inline]
-    /// Returns if the value is `null` or not.
-    pub fn is_null(&self) -> bool {
-        matches!(self, Self::Leaf(ReferenceValueLeaf::Null))
-    }
-
-    #[inline]
-    /// If the Value is a leaf, returns the associated leaf. Returns None otherwise.
-    pub fn as_leaf(&self) -> Option<&ReferenceValueLeaf<'a>> {
-        if let Self::Leaf(val) = self {
-            Some(val)
-        } else {
-            None
-        }
-    }
-
-    #[inline]
-    /// If the Value is a String, returns the associated str. Returns None otherwise.
-    pub fn as_str(&self) -> Option<&'a str> {
-        self.as_leaf().and_then(|leaf| leaf.as_str())
-    }
-
-    #[inline]
-    /// If the Value is a u64, returns the associated u64. Returns None otherwise.
-    pub fn as_u64(&self) -> Option<u64> {
-        self.as_leaf().and_then(|leaf| leaf.as_u64())
-    }
-
-    #[inline]
-    /// If the Value is a i64, returns the associated i64. Returns None otherwise.
-    pub fn as_i64(&self) -> Option<i64> {
-        self.as_leaf().and_then(|leaf| leaf.as_i64())
-    }
-
-    #[inline]
-    /// If the Value is a f64, returns the associated f64. Returns None otherwise.
-    pub fn as_f64(&self) -> Option<f64> {
-        self.as_leaf().and_then(|leaf| leaf.as_f64())
-    }
-
-    #[inline]
-    /// If the Value is a datetime, returns the associated datetime. Returns None otherwise.
-    pub fn as_datetime(&self) -> Option<DateTime> {
-        self.as_leaf().and_then(|leaf| leaf.as_datetime())
-    }
-
-    #[inline]
-    /// If the Value is a IP address, returns the associated IP. Returns None otherwise.
-    pub fn as_ip_addr(&self) -> Option<Ipv6Addr> {
-        self.as_leaf().and_then(|leaf| leaf.as_ip_addr())
-    }
-
-    #[inline]
-    /// If the Value is a bool, returns the associated bool. Returns None otherwise.
-    pub fn as_bool(&self) -> Option<bool> {
-        self.as_leaf().and_then(|leaf| leaf.as_bool())
-    }
-
-    #[inline]
-    /// If the Value is a pre-tokenized string, returns the associated string. Returns None
-    /// otherwise.
-    pub fn as_pre_tokenized_text(&self) -> Option<&'a PreTokenizedString> {
-        self.as_leaf().and_then(|leaf| leaf.as_pre_tokenized_text())
-    }
-
-    #[inline]
-    /// If the Value is a bytes value, returns the associated set of bytes. Returns None otherwise.
-    pub fn as_bytes(&self) -> Option<&'a [u8]> {
-        self.as_leaf().and_then(|leaf| leaf.as_bytes())
-    }
-
-    #[inline]
-    /// If the Value is a facet, returns the associated facet. Returns None otherwise.
-    pub fn as_facet(&self) -> Option<&'a Facet> {
-        self.as_leaf().and_then(|leaf| leaf.as_facet())
-    }
 
     #[inline]
     /// Returns true if the Value is an array.
     pub fn is_array(&self) -> bool {
-        matches!(self, Self::Array(_))
+        matches!(self, Self::Object(_))
     }
 
     #[inline]

src/schema/facet.rs

@@ -131,16 +131,16 @@ impl Facet {
     pub fn from_path<Path>(path: Path) -> Facet
     where
         Path: IntoIterator,
-        Path::Item: AsRef<str>,
+        Path::Item: ToString,
     {
         let mut facet_string: String = String::with_capacity(100);
         let mut step_it = path.into_iter();
         if let Some(step) = step_it.next() {
-            facet_string.push_str(step.as_ref());
+            facet_string.push_str(&step.to_string());
         }
         for step in step_it {
             facet_string.push(FACET_SEP_CHAR);
-            facet_string.push_str(step.as_ref());
+            facet_string.push_str(&step.to_string());
         }
         Facet(facet_string)
     }

src/schema/facet_options.rs

@@ -14,7 +14,6 @@ pub struct FacetOptions {
 
 impl FacetOptions {
     /// Returns true if the value is stored.
-    #[inline]
     pub fn is_stored(&self) -> bool {
         self.stored
     }

src/schema/field_entry.rs

@@ -108,16 +108,7 @@ impl FieldEntry {
         self.field_type.is_fast()
     }
 
-    /// Returns true if the field has the expand dots option set (for json fields)
-    pub fn is_expand_dots_enabled(&self) -> bool {
-        match self.field_type {
-            FieldType::JsonObject(ref options) => options.is_expand_dots_enabled(),
-            _ => false,
-        }
-    }
-
     /// Returns true if the field is stored
-    #[inline]
     pub fn is_stored(&self) -> bool {
         match self.field_type {
             FieldType::U64(ref options)

src/schema/field_type.rs

@@ -3,7 +3,6 @@ use std::str::FromStr;
 
 use base64::engine::general_purpose::STANDARD as BASE64;
 use base64::Engine;
-use columnar::ColumnType;
 use serde::{Deserialize, Serialize};
 use serde_json::Value as JsonValue;
 use thiserror::Error;
@@ -48,7 +47,7 @@ pub enum ValueParsingError {
 ///
 /// Contrary to FieldType, this does
 /// not include the way the field must be indexed.
-#[derive(Copy, Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash)]
+#[derive(Copy, Clone, Debug, Eq, PartialEq)]
 #[repr(u8)]
 pub enum Type {
     /// `&str`
@@ -73,21 +72,6 @@ pub enum Type {
     IpAddr = b'p',
 }
 
-impl From<ColumnType> for Type {
-    fn from(value: ColumnType) -> Self {
-        match value {
-            ColumnType::Str => Type::Str,
-            ColumnType::U64 => Type::U64,
-            ColumnType::I64 => Type::I64,
-            ColumnType::F64 => Type::F64,
-            ColumnType::Bool => Type::Bool,
-            ColumnType::DateTime => Type::Date,
-            ColumnType::Bytes => Type::Bytes,
-            ColumnType::IpAddr => Type::IpAddr,
-        }
-    }
-}
-
 const ALL_TYPES: [Type; 10] = [
     Type::Str,
     Type::U64,
@@ -109,7 +93,6 @@ impl Type {
     }
 
     /// Returns a 1 byte code used to identify the type.
-    #[inline]
     pub fn to_code(&self) -> u8 {
         *self as u8
     }
@@ -132,7 +115,6 @@ impl Type {
 
     /// Interprets a 1byte code as a type.
     /// Returns `None` if the code is invalid.
-    #[inline]
     pub fn from_code(code: u8) -> Option<Self> {
         match code {
             b's' => Some(Type::Str),

src/schema/ip_options.rs

@@ -31,25 +31,21 @@ pub struct IpAddrOptions {
 
 impl IpAddrOptions {
     /// Returns true iff the value is a fast field.
-    #[inline]
     pub fn is_fast(&self) -> bool {
         self.fast
     }
 
     /// Returns `true` if the ip address should be stored in the doc store.
-    #[inline]
     pub fn is_stored(&self) -> bool {
         self.stored
     }
 
     /// Returns true iff the value is indexed and therefore searchable.
-    #[inline]
     pub fn is_indexed(&self) -> bool {
         self.indexed
     }
 
     /// Returns true if and only if the value is normed.
-    #[inline]
     pub fn fieldnorms(&self) -> bool {
         self.fieldnorms
     }

src/schema/json_object_options.rs

@@ -46,20 +46,17 @@ pub struct JsonObjectOptions {
 
 impl JsonObjectOptions {
     /// Returns `true` if the json object should be stored.
-    #[inline]
     pub fn is_stored(&self) -> bool {
         self.stored
     }
 
     /// Returns `true` iff the json object should be indexed.
-    #[inline]
     pub fn is_indexed(&self) -> bool {
         self.indexing.is_some()
     }
 
     /// Returns true if and only if the json object fields are
     /// to be treated as fast fields.
-    #[inline]
     pub fn is_fast(&self) -> bool {
         matches!(self.fast, FastFieldTextOptions::IsEnabled(true))
             || matches!(
@@ -69,7 +66,6 @@ impl JsonObjectOptions {
     }
 
     /// Returns true if and only if the value is a fast field.
-    #[inline]
     pub fn get_fast_field_tokenizer_name(&self) -> Option<&str> {
         match &self.fast {
             FastFieldTextOptions::IsEnabled(true) | FastFieldTextOptions::IsEnabled(false) => None,
@@ -91,7 +87,6 @@ impl JsonObjectOptions {
     ///
     /// If disabled, the "." needs to be escaped:
     /// `k8s\.node\.id:5`.
-    #[inline]
     pub fn is_expand_dots_enabled(&self) -> bool {
         self.expand_dots_enabled
     }
@@ -108,7 +103,6 @@ impl JsonObjectOptions {
     /// If set to `Some` then both int and str values will be indexed.
     /// The inner `TextFieldIndexing` will however, only apply to the str values
     /// in the json object.
-    #[inline]
     pub fn get_text_indexing_options(&self) -> Option<&TextFieldIndexing> {
         self.indexing.as_ref()
     }

src/schema/numeric_options.rs

@@ -57,31 +57,26 @@ impl From<NumericOptionsDeser> for NumericOptions {
 
 impl NumericOptions {
     /// Returns true iff the value is stored in the doc store.
-    #[inline]
     pub fn is_stored(&self) -> bool {
         self.stored
     }
 
     /// Returns true iff the value is indexed and therefore searchable.
-    #[inline]
     pub fn is_indexed(&self) -> bool {
         self.indexed
     }
 
     /// Returns true iff the field has fieldnorm.
-    #[inline]
     pub fn fieldnorms(&self) -> bool {
         self.fieldnorms && self.indexed
     }
 
     /// Returns true iff the value is a fast field.
-    #[inline]
     pub fn is_fast(&self) -> bool {
         self.fast
     }
 
     /// Returns true if values should be coerced to numbers.
-    #[inline]
     pub fn should_coerce(&self) -> bool {
         self.coerce
     }

src/schema/schema.rs

@@ -8,7 +8,6 @@ use serde::{Deserialize, Deserializer, Serialize, Serializer};
 
 use super::ip_options::IpAddrOptions;
 use super::*;
-use crate::json_utils::split_json_path;
 use crate::schema::bytes_options::BytesOptions;
 use crate::TantivyError;
 
@@ -31,7 +30,7 @@ use crate::TantivyError;
 /// let body_field = schema_builder.add_text_field("body", TEXT);
 /// let schema = schema_builder.build();
 /// ```
-#[derive(Debug, Default)]
+#[derive(Default)]
 pub struct SchemaBuilder {
     fields: Vec<FieldEntry>,
     fields_map: HashMap<String, Field>,
@@ -279,7 +278,6 @@ fn locate_splitting_dots(field_path: &str) -> Vec<usize> {
 
 impl Schema {
     /// Return the `FieldEntry` associated with a `Field`.
-    #[inline]
     pub fn get_field_entry(&self, field: Field) -> &FieldEntry {
         &self.0.fields[field.field_id() as usize]
     }
@@ -329,19 +327,12 @@ impl Schema {
         if let Some(field) = self.0.fields_map.get(full_path) {
             return Some((*field, ""));
         }
-
         let mut splitting_period_pos: Vec<usize> = locate_splitting_dots(full_path);
         while let Some(pos) = splitting_period_pos.pop() {
             let (prefix, suffix) = full_path.split_at(pos);
-
             if let Some(field) = self.0.fields_map.get(prefix) {
                 return Some((*field, &suffix[1..]));
             }
-            // JSON path may contain a dot, for now we try both variants to find the field.
-            let prefix = split_json_path(prefix).join(".");
-            if let Some(field) = self.0.fields_map.get(&prefix) {
-                return Some((*field, &suffix[1..]));
-            }
         }
         None
     }
@@ -357,7 +348,6 @@ impl Schema {
     pub fn find_field_with_default<'a>(
         &self,
         full_path: &'a str,
-
         default_field_opt: Option<Field>,
     ) -> Option<(Field, &'a str)> {
         let (field, json_path) = self

src/schema/term.rs

@@ -3,7 +3,7 @@ use std::hash::{Hash, Hasher};
 use std::net::Ipv6Addr;
 use std::{fmt, str};
 
-use columnar::{MonotonicallyMappableToU128, MonotonicallyMappableToU64};
+use columnar::MonotonicallyMappableToU128;
 
 use super::date_time_options::DATE_TIME_PRECISION_INDEXED;
 use super::Field;
@@ -170,18 +170,6 @@ impl Term {
         self.set_bytes(val.to_u64().to_be_bytes().as_ref());
     }
 
-    pub(crate) fn append_type_and_fast_value<T: FastValue>(&mut self, val: T) {
-        self.0.push(T::to_type().to_code());
-        let value = if T::to_type() == Type::Date {
-            DateTime::from_u64(val.to_u64())
-                .truncate(DATE_TIME_PRECISION_INDEXED)
-                .to_u64()
-        } else {
-            val.to_u64()
-        };
-        self.0.extend(value.to_be_bytes().as_ref());
-    }
-
     /// Sets a `Ipv6Addr` value in the term.
     pub fn set_ip_addr(&mut self, val: Ipv6Addr) {
         self.set_bytes(val.to_u128().to_be_bytes().as_ref());

src/schema/text_options.rs

@@ -72,19 +72,16 @@ fn is_false(val: &bool) -> bool {
 
 impl TextOptions {
     /// Returns the indexing options.
-    #[inline]
     pub fn get_indexing_options(&self) -> Option<&TextFieldIndexing> {
         self.indexing.as_ref()
     }
 
     /// Returns true if the text is to be stored.
-    #[inline]
     pub fn is_stored(&self) -> bool {
         self.stored
     }
 
     /// Returns true if and only if the value is a fast field.
-    #[inline]
     pub fn is_fast(&self) -> bool {
         matches!(self.fast, FastFieldTextOptions::IsEnabled(true))
             || matches!(
@@ -94,7 +91,6 @@ impl TextOptions {
     }
 
     /// Returns true if and only if the value is a fast field.
-    #[inline]
     pub fn get_fast_field_tokenizer_name(&self) -> Option<&str> {
         match &self.fast {
             FastFieldTextOptions::IsEnabled(true) | FastFieldTextOptions::IsEnabled(false) => None,
@@ -105,7 +101,6 @@ impl TextOptions {
     }
 
     /// Returns true if values should be coerced to strings (numbers, null).
-    #[inline]
     pub fn should_coerce(&self) -> bool {
         self.coerce
     }

src/snippet/mod.rs

@@ -1,59 +1,3 @@
-//! [`SnippetGenerator`]
-//! Generates a text snippet for a given document, and some highlighted parts inside it.
-//! Imagine you doing a text search in a document
-//! and want to show a preview of where in the document the search terms occur,
-//! along with some surrounding text to give context, and the search terms highlighted.
-//!
-//! [`SnippetGenerator`] serves this purpose.
-//! It scans a document and constructs a snippet, which consists of sections where the search terms
-//! have been found, stitched together with "..." in between sections if necessary.
-//!
-//! ## Example
-//!
-//! ```rust
-//! # use tantivy::query::QueryParser;
-//! # use tantivy::schema::{Schema, TEXT};
-//! # use tantivy::{doc, Index};
-//! use tantivy::snippet::SnippetGenerator;
-//!
-//! # fn main() -> tantivy::Result<()> {
-//! #    let mut schema_builder = Schema::builder();
-//! #    let text_field = schema_builder.add_text_field("text", TEXT);
-//! #    let schema = schema_builder.build();
-//! #    let index = Index::create_in_ram(schema);
-//! #    let mut index_writer = index.writer_with_num_threads(1, 20_000_000)?;
-//! #    let doc = doc!(text_field => r#"Comme je descendais des Fleuves impassibles,
-//! #   Je ne me sentis plus guidé par les haleurs :
-//! #  Des Peaux-Rouges criards les avaient pris pour cibles,
-//! #  Les ayant cloués nus aux poteaux de couleurs.
-//! #
-//! #  J'étais insoucieux de tous les équipages,
-//! #  Porteur de blés flamands ou de cotons anglais.
-//! #  Quand avec mes haleurs ont fini ces tapages,
-//! #  Les Fleuves m'ont laissé descendre où je voulais.
-//! #  "#);
-//! #    index_writer.add_document(doc.clone())?;
-//! #    index_writer.commit()?;
-//! #    let query_parser = QueryParser::for_index(&index, vec![text_field]);
-//! // ...
-//! let query = query_parser.parse_query("haleurs flamands").unwrap();
-//! # let reader = index.reader()?;
-//! # let searcher = reader.searcher();
-//! let mut snippet_generator = SnippetGenerator::create(&searcher, &*query, text_field)?;
-//! snippet_generator.set_max_num_chars(100);
-//! let snippet = snippet_generator.snippet_from_doc(&doc);
-//! let snippet_html: String = snippet.to_html();
-//! assert_eq!(snippet_html, "Comme je descendais des Fleuves impassibles,\n  Je ne me sentis plus guidé par les <b>haleurs</b> :\n Des");
-//! #    Ok(())
-//! # }
-//! ```
-//!
-//! You can also specify the maximum number of characters for the snippets generated with the
-//! `set_max_num_chars` method. By default, this limit is set to 150.
-//!
-//! SnippetGenerator needs to be created from the `Searcher` and the query, and the field on which
-//! the `SnippetGenerator` should generate the snippets.
-
 use std::cmp::Ordering;
 use std::collections::{BTreeMap, BTreeSet};
 use std::ops::Range;
@@ -72,7 +16,7 @@ const DEFAULT_SNIPPET_PREFIX: &str = "<b>";
 const DEFAULT_SNIPPET_POSTFIX: &str = "</b>";
 
 #[derive(Debug)]
-pub(crate) struct FragmentCandidate {
+pub struct FragmentCandidate {
     score: Score,
     start_offset: usize,
     stop_offset: usize,
@@ -312,7 +256,7 @@ fn is_sorted(mut it: impl Iterator<Item = usize>) -> bool {
 /// # use tantivy::query::QueryParser;
 /// # use tantivy::schema::{Schema, TEXT};
 /// # use tantivy::{doc, Index};
-/// use tantivy::snippet::SnippetGenerator;
+/// use tantivy::SnippetGenerator;
 ///
 /// # fn main() -> tantivy::Result<()> {
 /// #    let mut schema_builder = Schema::builder();
@@ -402,7 +346,7 @@ impl SnippetGenerator {
         })
     }
 
-    /// Sets a maximum number of chars. Default is 150.
+    /// Sets a maximum number of chars.
     pub fn set_max_num_chars(&mut self, max_num_chars: usize) {
         self.max_num_chars = max_num_chars;
     }
@@ -454,9 +398,8 @@ mod tests {
     use super::{collapse_overlapped_ranges, search_fragments, select_best_fragment_combination};
     use crate::query::QueryParser;
     use crate::schema::{IndexRecordOption, Schema, TextFieldIndexing, TextOptions, TEXT};
-    use crate::snippet::SnippetGenerator;
     use crate::tokenizer::{NgramTokenizer, SimpleTokenizer};
-    use crate::Index;
+    use crate::{Index, SnippetGenerator};
 
     const TEST_TEXT: &str = r#"Rust is a systems programming language sponsored by
 Mozilla which describes it as a "safe, concurrent, practical language", supporting functional and

src/termdict/mod.rs

@@ -1,5 +1,5 @@
 //! The term dictionary main role is to associate the sorted [`Term`s](crate::Term) to
-//! a [`TermInfo`] struct that contains some meta-information
+//! a [`TermInfo`](crate::postings::TermInfo) struct that contains some meta-information
 //! about the term.
 //!
 //! Internally, the term dictionary relies on the `fst` crate to store
@@ -16,7 +16,8 @@
 //! `f64`-terms are transformed to `u64` using a mapping that preserve order, and are then treated
 //! as `u64`.
 //!
-//! A second datastructure makes it possible to access a [`TermInfo`].
+//! A second datastructure makes it possible to access a
+//! [`TermInfo`](crate::postings::TermInfo).
 
 #[cfg(not(feature = "quickwit"))]
 mod fst_termdict;

sstable/Cargo.toml

@@ -11,22 +11,17 @@ description = "sstables for tantivy"
 
 [dependencies]
 common = {version= "0.6", path="../common", package="tantivy-common"}
-tantivy-bitpacker = { version= "0.5", path="../bitpacker" }
-tantivy-fst = "0.5"
+tantivy-fst = "0.4"
 # experimental gives us access to Decompressor::upper_bound
 zstd = { version = "0.13", features = ["experimental"] }
+tracing = "0.1"
 
 [dev-dependencies]
 proptest = "1"
-criterion = { version = "0.5", default-features = false }
+criterion = "0.5"
 names = "0.14"
 rand = "0.8"
 
 [[bench]]
 name = "stream_bench"
 harness = false
-
-[[bench]]
-name = "ord_to_term"
-harness = false
-

sstable/README.md

@@ -89,71 +89,33 @@ Note: as the SSTable does not support redundant keys, there is no ambiguity betw
 
 ### SSTFooter
 ```
-+-----+----------------+-------------+-------------+---------+---------+
-| Fst | BlockAddrStore | StoreOffset | IndexOffset | NumTerm | Version |
-+-----+----------------+-------------+-------------+---------+---------+
++-------+-------+-----+-------------+---------+---------+
+| Block | Block | ... | IndexOffset | NumTerm | Version |
++-------+-------+-----+-------------+---------+---------+
+|----( # of blocks)---|
 ```
-- Fst(Fst): finite state transducer mapping keys to a block number
-- BlockAddrStore(BlockAddrStore): store mapping a block number to its BlockAddr
-- StoreOffset(u64): Offset to start of the BlockAddrStore. If zero, see the SingleBlockSStable section
+- Block(SSTBlock): uses IndexValue for its Values format
 - IndexOffset(u64): Offset to the start of the SSTFooter
 - NumTerm(u64): number of terms in the sstable
-- Version(u32): Currently equal to 3
+- Version(u32): Currently equal to 2
 
-### Fst
+### IndexValue
+```
++------------+----------+-------+-------+-----+
+| EntryCount | StartPos | Entry | Entry | ... |
++------------+----------+-------+-------+-----+
+                        |---( # of entries)---|
+```
 
-Fst is in the format of tantivy\_fst
+- EntryCount(VInt): number of entries
+- StartPos(VInt): the start pos of the first (data) block referenced by this (index) block
+- Entry (IndexEntry)
 
-### BlockAddrStore
-
-+---------+-----------+-----------+-----+-----------+-----------+-----+
-| MetaLen | BlockMeta | BlockMeta | ... | BlockData | BlockData | ... |
-+---------+-----------+-----------+-----+-----------+-----------+-----+
-          |---------(N blocks)----------|---------(N blocks)----------|
-
-- MetaLen(u64): length of the BlockMeta section
-- BlockMeta(BlockAddrBlockMetadata): metadata to seek through BlockData
-- BlockData(CompactedBlockAddr): bitpacked per block metadata
-
-### BlockAddrBlockMetadata
-
-+--------+------------+--------------+------------+--------------+-------------------+-----------------+----------+
-| Offset | RangeStart | FirstOrdinal | RangeSlope | OrdinalSlope | FirstOrdinalNBits | RangeStartNBits | BlockLen |
-+--------+------------+--------------+------------+--------------+-------------------+-----------------+----------+
-
-- Offset(u64): offset of the corresponding BlockData in the datastream
-- RangeStart(u64): the start position of the first block
-- FirstOrdinal(u64): the first ordinal of the first block
-- RangeSlope(u32): slope predicted for start range evolution (see computation in BlockData)
-- OrdinalSlope(u64): slope predicted for first ordinal evolution (see computation in BlockData)
-- FirstOrdinalNBits(u8): number of bits per ordinal in datastream (see computation in BlockData)
-- RangeStartNBits(u8): number of bits per range start in datastream (see computation in BlockData)
-
-### BlockData
-
-+-----------------+-------------------+---------------+
-| RangeStartDelta | FirstOrdinalDelta | FinalRangeEnd |
-+-----------------+-------------------+---------------+
-|------(BlockLen repetitions)---------|
-
-- RangeStartDelta(var): RangeStartNBits *bits* of little endian number. See below for decoding
-- FirstOrdinalDelta(var): FirstOrdinalNBits *bits* of little endian number. See below for decoding
-- FinalRangeEnd(var): RangeStartNBits *bits* of integer. See below for decoding
-
-converting a BlockData of index Index and a BlockAddrBlockMetadata to an actual block address is done as follow:
-range\_prediction := RangeStart + Index * RangeSlop;
-range\_derivation := RangeStartDelta - (1 << (RangeStartNBits-1));
-range\_start := range\_prediction + range\_derivation
-
-The same computation can be done for ordinal.
-
-Note that `range_derivation` can take negative value. `RangeStartDelta` is just its translation to a positive range.
-
-
-## SingleBlockSStable
-
-The format used for the index is meant to be compact, however it has a constant cost of around 70
-bytes, which isn't negligible for a table containing very few keys.
-To limit the impact of that constant cost, single block sstable omit the Fst and BlockAddrStore from
-their index. Instead a block with first ordinal of 0, range start of 0 and range end of IndexOffset
-is implicitly used for every operations.
+### Entry
+```
++----------+--------------+
+| BlockLen | FirstOrdinal |
++----------+--------------+
+```
+- BlockLen(VInt): length of the block
+- FirstOrdinal(VInt): ordinal of the first element in the given block

sstable/benches/ord_to_term.rs

@@ -1,110 +0,0 @@
-use std::sync::Arc;
-
-use common::file_slice::FileSlice;
-use common::OwnedBytes;
-use criterion::{criterion_group, criterion_main, Criterion};
-use tantivy_sstable::{self, Dictionary, MonotonicU64SSTable};
-
-fn make_test_sstable(suffix: &str) -> FileSlice {
-    let mut builder = Dictionary::<MonotonicU64SSTable>::builder(Vec::new()).unwrap();
-
-    // 125 mio elements
-    for elem in 0..125_000_000 {
-        let key = format!("prefix.{elem:07X}{suffix}").into_bytes();
-        builder.insert(&key, &elem).unwrap();
-    }
-
-    let table = builder.finish().unwrap();
-    let table = Arc::new(OwnedBytes::new(table));
-    let slice = common::file_slice::FileSlice::new(table.clone());
-
-    slice
-}
-
-pub fn criterion_benchmark(c: &mut Criterion) {
-    {
-        let slice = make_test_sstable(".suffix");
-        let dict = Dictionary::<MonotonicU64SSTable>::open(slice.clone()).unwrap();
-        c.bench_function("ord_to_term_suffix", |b| {
-            let mut res = Vec::new();
-            b.iter(|| {
-                assert!(dict.ord_to_term(100_000, &mut res).unwrap());
-                assert!(dict.ord_to_term(19_000_000, &mut res).unwrap());
-            })
-        });
-        c.bench_function("open_and_ord_to_term_suffix", |b| {
-            let mut res = Vec::new();
-            b.iter(|| {
-                let dict = Dictionary::<MonotonicU64SSTable>::open(slice.clone()).unwrap();
-                assert!(dict.ord_to_term(100_000, &mut res).unwrap());
-                assert!(dict.ord_to_term(19_000_000, &mut res).unwrap());
-            })
-        });
-        c.bench_function("term_ord_suffix", |b| {
-            b.iter(|| {
-                assert_eq!(
-                    dict.term_ord(b"prefix.00186A0.suffix").unwrap().unwrap(),
-                    100_000
-                );
-                assert_eq!(
-                    dict.term_ord(b"prefix.121EAC0.suffix").unwrap().unwrap(),
-                    19_000_000
-                );
-            })
-        });
-        c.bench_function("open_and_term_ord_suffix", |b| {
-            b.iter(|| {
-                let dict = Dictionary::<MonotonicU64SSTable>::open(slice.clone()).unwrap();
-                assert_eq!(
-                    dict.term_ord(b"prefix.00186A0.suffix").unwrap().unwrap(),
-                    100_000
-                );
-                assert_eq!(
-                    dict.term_ord(b"prefix.121EAC0.suffix").unwrap().unwrap(),
-                    19_000_000
-                );
-            })
-        });
-    }
-    {
-        let slice = make_test_sstable("");
-        let dict = Dictionary::<MonotonicU64SSTable>::open(slice.clone()).unwrap();
-        c.bench_function("ord_to_term", |b| {
-            let mut res = Vec::new();
-            b.iter(|| {
-                assert!(dict.ord_to_term(100_000, &mut res).unwrap());
-                assert!(dict.ord_to_term(19_000_000, &mut res).unwrap());
-            })
-        });
-        c.bench_function("open_and_ord_to_term", |b| {
-            let mut res = Vec::new();
-            b.iter(|| {
-                let dict = Dictionary::<MonotonicU64SSTable>::open(slice.clone()).unwrap();
-                assert!(dict.ord_to_term(100_000, &mut res).unwrap());
-                assert!(dict.ord_to_term(19_000_000, &mut res).unwrap());
-            })
-        });
-        c.bench_function("term_ord", |b| {
-            b.iter(|| {
-                assert_eq!(dict.term_ord(b"prefix.00186A0").unwrap().unwrap(), 100_000);
-                assert_eq!(
-                    dict.term_ord(b"prefix.121EAC0").unwrap().unwrap(),
-                    19_000_000
-                );
-            })
-        });
-        c.bench_function("open_and_term_ord", |b| {
-            b.iter(|| {
-                let dict = Dictionary::<MonotonicU64SSTable>::open(slice.clone()).unwrap();
-                assert_eq!(dict.term_ord(b"prefix.00186A0").unwrap().unwrap(), 100_000);
-                assert_eq!(
-                    dict.term_ord(b"prefix.121EAC0").unwrap().unwrap(),
-                    19_000_000
-                );
-            })
-        });
-    }
-}
-
-criterion_group!(benches, criterion_benchmark);
-criterion_main!(benches);

sstable/src/dictionary.rs

@@ -3,17 +3,15 @@ use std::io;
 use std::marker::PhantomData;
 use std::ops::{Bound, RangeBounds};
 use std::sync::Arc;
+use tracing::instrument;
 
 use common::file_slice::FileSlice;
 use common::{BinarySerializable, OwnedBytes};
 use tantivy_fst::automaton::AlwaysMatch;
 use tantivy_fst::Automaton;
 
-use crate::sstable_index_v3::SSTableIndexV3Empty;
 use crate::streamer::{Streamer, StreamerBuilder};
-use crate::{
-    BlockAddr, DeltaReader, Reader, SSTable, SSTableIndex, SSTableIndexV3, TermOrdinal, VoidSSTable,
-};
+use crate::{BlockAddr, DeltaReader, Reader, SSTable, SSTableIndex, TermOrdinal, VoidSSTable};
 
 /// An SSTable is a sorted map that associates sorted `&[u8]` keys
 /// to any kind of typed values.
@@ -97,6 +95,7 @@ impl<TSSTable: SSTable> Dictionary<TSSTable> {
         Ok(TSSTable::delta_reader(data))
     }
 
+    #[instrument(skip_all)]
     pub(crate) async fn sstable_delta_reader_block_async(
         &self,
         block_addr: BlockAddr,
@@ -183,41 +182,24 @@ impl<TSSTable: SSTable> Dictionary<TSSTable> {
     pub fn open(term_dictionary_file: FileSlice) -> io::Result<Self> {
         let (main_slice, footer_len_slice) = term_dictionary_file.split_from_end(20);
         let mut footer_len_bytes: OwnedBytes = footer_len_slice.read_bytes()?;
+
         let index_offset = u64::deserialize(&mut footer_len_bytes)?;
         let num_terms = u64::deserialize(&mut footer_len_bytes)?;
         let version = u32::deserialize(&mut footer_len_bytes)?;
+        if version != crate::SSTABLE_VERSION {
+            return Err(io::Error::new(
+                io::ErrorKind::Other,
+                format!(
+                    "Unsuported sstable version, expected {version}, found {}",
+                    crate::SSTABLE_VERSION,
+                ),
+            ));
+        }
+
         let (sstable_slice, index_slice) = main_slice.split(index_offset as usize);
         let sstable_index_bytes = index_slice.read_bytes()?;
-
-        let sstable_index = match version {
-            2 => SSTableIndex::V2(
-                crate::sstable_index_v2::SSTableIndex::load(sstable_index_bytes).map_err(|_| {
-                    io::Error::new(io::ErrorKind::InvalidData, "SSTable corruption")
-                })?,
-            ),
-            3 => {
-                let (sstable_index_bytes, mut footerv3_len_bytes) = sstable_index_bytes.rsplit(8);
-                let store_offset = u64::deserialize(&mut footerv3_len_bytes)?;
-                if store_offset != 0 {
-                    SSTableIndex::V3(
-                        SSTableIndexV3::load(sstable_index_bytes, store_offset).map_err(|_| {
-                            io::Error::new(io::ErrorKind::InvalidData, "SSTable corruption")
-                        })?,
-                    )
-                } else {
-                    // if store_offset is zero, there is no index, so we build a pseudo-index
-                    // assuming a single block of sstable covering everything.
-                    SSTableIndex::V3Empty(SSTableIndexV3Empty::load(index_offset as usize))
-                }
-            }
-            _ => {
-                return Err(io::Error::new(
-                    io::ErrorKind::Other,
-                    format!("Unsuported sstable version, expected one of [2, 3], found {version}"),
-                ))
-            }
-        };
-
+        let sstable_index = SSTableIndex::load(sstable_index_bytes)
+            .map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "SSTable corruption"))?;
         Ok(Dictionary {
             sstable_slice,
             sstable_index,
@@ -252,6 +234,7 @@ impl<TSSTable: SSTable> Dictionary<TSSTable> {
     /// If the key was not found, returns Ok(None).
     /// After calling this function, it is possible to call `DeltaReader::value` to get the
     /// associated value.
+    #[instrument(skip_all)]
     fn decode_up_to_key<K: AsRef<[u8]>>(
         &self,
         key: K,
@@ -364,6 +347,7 @@ impl<TSSTable: SSTable> Dictionary<TSSTable> {
     }
 
     /// Lookups the value corresponding to the key.
+    #[instrument(skip_all)]
     pub async fn get_async<K: AsRef<[u8]>>(&self, key: K) -> io::Result<Option<TSSTable::Value>> {
         if let Some(block_addr) = self.sstable_index.get_block_with_key(key.as_ref()) {
             let sstable_reader = self.sstable_delta_reader_block_async(block_addr).await?;

sstable/src/lib.rs

@@ -10,9 +10,8 @@ pub mod merge;
 mod streamer;
 pub mod value;
 
-mod sstable_index_v3;
-pub use sstable_index_v3::{BlockAddr, SSTableIndex, SSTableIndexBuilder, SSTableIndexV3};
-mod sstable_index_v2;
+mod sstable_index;
+pub use sstable_index::{BlockAddr, SSTableIndex, SSTableIndexBuilder};
 pub(crate) mod vint;
 pub use dictionary::Dictionary;
 pub use streamer::{Streamer, StreamerBuilder};
@@ -29,7 +28,7 @@ use crate::value::{RangeValueReader, RangeValueWriter};
 pub type TermOrdinal = u64;
 
 const DEFAULT_KEY_CAPACITY: usize = 50;
-const SSTABLE_VERSION: u32 = 3;
+const SSTABLE_VERSION: u32 = 2;
 
 /// Given two byte string returns the length of
 /// the longest common prefix.
@@ -305,8 +304,7 @@ where
 
         let offset = wrt.written_bytes();
 
-        let fst_len: u64 = self.index_builder.serialize(&mut wrt)?;
-        wrt.write_all(&fst_len.to_le_bytes())?;
+        self.index_builder.serialize(&mut wrt)?;
         wrt.write_all(&offset.to_le_bytes())?;
         wrt.write_all(&self.num_terms.to_le_bytes())?;
 
@@ -387,10 +385,13 @@ mod test {
                 16, 17, 33, 18, 19, 17, 20, // data block
                 0, 0, 0, 0, // no more block
                 // index
-                0, 0, 0, 0, 0, 0, 0, 0, // fst lenght
+                8, 0, 0, 0, // size of index block
+                0, // compression
+                1, 0, 12, 0, 32, 17, 20, // index block
+                0, 0, 0, 0, // no more index block
                 16, 0, 0, 0, 0, 0, 0, 0, // index start offset
                 3, 0, 0, 0, 0, 0, 0, 0, // num term
-                3, 0, 0, 0, // version
+                2, 0, 0, 0, // version
             ]
         );
         let buffer = OwnedBytes::new(buffer);