update basic_search example

* style: simplify strings with string interpolation

* fix: formatting

Cargo.toml

@@ -16,7 +16,7 @@ exclude = ["benches/*.json", "benches/*.txt"]
 
 [dependencies]
 # Switch back to the non-forked oneshot crate once https://github.com/faern/oneshot/pull/35 is merged
-oneshot = { git = "https://github.com/fulmicoton/oneshot.git", rev = "c10a3ba" }
+oneshot = { git = "https://github.com/fulmicoton/oneshot.git", rev = "b208f49" }
 base64 = "0.22.0"
 byteorder = "1.4.3"
 crc32fast = "1.3.2"
@@ -53,7 +53,7 @@ smallvec = "1.8.0"
 rayon = "1.5.2"
 lru = "0.12.0"
 fastdivide = "0.4.0"
-itertools = "0.12.0"
+itertools = "0.13.0"
 measure_time = "0.8.2"
 arc-swap = "1.5.0"
 
@@ -72,7 +72,7 @@ fnv = "1.0.7"
 winapi = "0.3.9"
 
 [dev-dependencies]
-binggan = "0.6.2"
+binggan = "0.8.0"
 rand = "0.8.5"
 maplit = "1.0.2"
 matches = "0.1.9"

benches/agg_bench.rs

@@ -349,7 +349,7 @@ fn get_test_index_bench(cardinality: Cardinality) -> tantivy::Result<Index> {
     let lg_norm = rand_distr::LogNormal::new(2.996f64, 0.979f64).unwrap();
 
     let many_terms_data = (0..150_000)
-        .map(|num| format!("author{}", num))
+        .map(|num| format!("author{num}"))
         .collect::<Vec<_>>();
     {
         let mut rng = StdRng::from_seed([1u8; 32]);

benches/index-bench.rs

@@ -18,7 +18,7 @@ fn benchmark(
         benchmark_dynamic_json(b, input, schema, commit, parse_json)
     } else {
         _benchmark(b, input, schema, commit, parse_json, |schema, doc_json| {
-            TantivyDocument::parse_json(&schema, doc_json).unwrap()
+            TantivyDocument::parse_json(schema, doc_json).unwrap()
         })
     }
 }
@@ -90,8 +90,7 @@ fn benchmark_dynamic_json(
 ) {
     let json_field = schema.get_field("json").unwrap();
     _benchmark(b, input, schema, commit, parse_json, |_schema, doc_json| {
-        let json_val: serde_json::Map<String, serde_json::Value> =
-            serde_json::from_str(doc_json).unwrap();
+        let json_val: serde_json::Value = serde_json::from_str(doc_json).unwrap();
         tantivy::doc!(json_field=>json_val)
     })
 }
@@ -138,15 +137,16 @@ pub fn hdfs_index_benchmark(c: &mut Criterion) {
     for (prefix, schema, is_dynamic) in benches {
         for commit in [false, true] {
             let suffix = if commit { "with-commit" } else { "no-commit" };
-            for parse_json in [false] {
+            {
+                let parse_json = false;
                 // for parse_json in [false, true] {
                 let suffix = if parse_json {
-                    format!("{}-with-json-parsing", suffix)
+                    format!("{suffix}-with-json-parsing")
                 } else {
-                    format!("{}", suffix)
+                    suffix.to_string()
                 };
 
-                let bench_name = format!("{}{}", prefix, suffix);
+                let bench_name = format!("{prefix}{suffix}");
                 group.bench_function(bench_name, |b| {
                     benchmark(b, HDFS_LOGS, schema.clone(), commit, parse_json, is_dynamic)
                 });

columnar/Cargo.toml

@@ -9,7 +9,7 @@ description = "column oriented storage for tantivy"
 categories = ["database-implementations", "data-structures", "compression"]
 
 [dependencies]
-itertools = "0.12.0"
+itertools = "0.13.0"
 fastdivide = "0.4.0"
 
 stacker = { version= "0.3", path = "../stacker", package="tantivy-stacker"}

common/src/vint.rs

@@ -151,7 +151,7 @@ pub fn read_u32_vint_no_advance(data: &[u8]) -> (u32, usize) {
     (result, vlen)
 }
 /// Write a `u32` as a vint payload.
-pub fn write_u32_vint<W: io::Write>(val: u32, writer: &mut W) -> io::Result<()> {
+pub fn write_u32_vint<W: io::Write + ?Sized>(val: u32, writer: &mut W) -> io::Result<()> {
     let mut buf = [0u8; 8];
     let data = serialize_vint_u32(val, &mut buf);
     writer.write_all(data)

examples/basic_search.rs

@@ -19,13 +19,14 @@ use tantivy::{doc, Index, IndexWriter, ReloadPolicy};
 use tempfile::TempDir;
 
 fn main() -> tantivy::Result<()> {
-    // Let's create a temporary directory for the
-    // sake of this example
+    // Normally you would use `MMapDirectory` instead to persist data on disk.
+    // https://docs.rs/tantivy/latest/tantivy/directory/struct.MmapDirectory.html
+    // But for this example, we will use a temporary directory `TempDir`.
     let index_path = TempDir::new()?;
 
     // # Defining the schema
     //
-    // The Tantivy index requires a very strict schema.
+    // The Tantivy index requires a schema.
     // The schema declares which fields are in the index,
     // and for each field, its type and "the way it should
     // be indexed".

examples/date_time_field.rs

@@ -4,7 +4,7 @@
 
 use tantivy::collector::TopDocs;
 use tantivy::query::QueryParser;
-use tantivy::schema::{DateOptions, Document, OwnedValue, Schema, INDEXED, STORED, STRING};
+use tantivy::schema::{DateOptions, Document, Schema, Value, INDEXED, STORED, STRING};
 use tantivy::{Index, IndexWriter, TantivyDocument};
 
 fn main() -> tantivy::Result<()> {
@@ -61,10 +61,12 @@ fn main() -> tantivy::Result<()> {
         assert_eq!(count_docs.len(), 1);
         for (_score, doc_address) in count_docs {
             let retrieved_doc = searcher.doc::<TantivyDocument>(doc_address)?;
-            assert!(matches!(
-                retrieved_doc.get_first(occurred_at),
-                Some(OwnedValue::Date(_))
-            ));
+            assert!(retrieved_doc
+                .get_first(occurred_at)
+                .unwrap()
+                .as_value()
+                .as_datetime()
+                .is_some(),);
             assert_eq!(
                 retrieved_doc.to_json(&schema),
                 r#"{"event":["comment"],"occurred_at":["2022-06-22T13:00:00.22Z"]}"#

examples/faceted_search_with_tweaked_score.rs

@@ -51,7 +51,7 @@ fn main() -> tantivy::Result<()> {
     let reader = index.reader()?;
     let searcher = reader.searcher();
     {
-        let facets = vec![
+        let facets = [
             Facet::from("/ingredient/egg"),
             Facet::from("/ingredient/oil"),
             Facet::from("/ingredient/garlic"),
@@ -94,9 +94,8 @@ fn main() -> tantivy::Result<()> {
                     .doc::<TantivyDocument>(*doc_id)
                     .unwrap()
                     .get_first(title)
-                    .and_then(|v| v.as_str())
+                    .and_then(|v| v.as_str().map(|el| el.to_string()))
                     .unwrap()
-                    .to_owned()
             })
             .collect();
         assert_eq!(titles, vec!["Fried egg", "Egg rolls"]);

examples/index_from_multiple_threads.rs

@@ -61,7 +61,7 @@ fn main() -> tantivy::Result<()> {
                         debris of the winter’s flooding; and sycamores with mottled, white, recumbent \
                         limbs and branches that arch over the pool"
                     ))?;
-            println!("add doc {} from thread 1 - opstamp {}", i, opstamp);
+            println!("add doc {i} from thread 1 - opstamp {opstamp}");
             thread::sleep(Duration::from_millis(20));
         }
         Result::<(), TantivyError>::Ok(())
@@ -82,7 +82,7 @@ fn main() -> tantivy::Result<()> {
                     body => "Some great book description..."
                 ))?
             };
-            println!("add doc {} from thread 2 - opstamp {}", i, opstamp);
+            println!("add doc {i} from thread 2 - opstamp {opstamp}");
             thread::sleep(Duration::from_millis(10));
         }
         Result::<(), TantivyError>::Ok(())

src/aggregation/agg_req_with_accessor.rs

@@ -335,8 +335,8 @@ fn get_missing_val(
         }
         _ => {
             return Err(crate::TantivyError::InvalidArgument(format!(
-                "Missing value {:?} for field {} is not supported for column type {:?}",
-                missing, field_name, column_type
+                "Missing value {missing:?} for field {field_name} is not supported for column \
+                 type {column_type:?}"
             )));
         }
     };
@@ -403,7 +403,7 @@ fn get_dynamic_columns(
         .iter()
         .map(|h| h.open())
         .collect::<io::Result<_>>()?;
-    assert!(!ff_fields.is_empty(), "field {} not found", field_name);
+    assert!(!ff_fields.is_empty(), "field {field_name} not found");
     Ok(cols)
 }
 

src/aggregation/bucket/term_agg.rs

@@ -357,8 +357,7 @@ impl SegmentTermCollector {
     ) -> crate::Result<Self> {
         if field_type == ColumnType::Bytes {
             return Err(TantivyError::InvalidArgument(format!(
-                "terms aggregation is not supported for column type {:?}",
-                field_type
+                "terms aggregation is not supported for column type {field_type:?}"
             )));
         }
         let term_buckets = TermBuckets::default();

src/aggregation/metric/top_hits.rs

@@ -131,8 +131,8 @@ impl<'de> Deserialize<'de> for KeyOrder {
         ))?;
         if key_order.next().is_some() {
             return Err(serde::de::Error::custom(format!(
-                "Expected exactly one key-value pair in sort parameter of top_hits, found {:?}",
-                key_order
+                "Expected exactly one key-value pair in sort parameter of top_hits, found \
+                 {key_order:?}"
             )));
         }
         Ok(Self { field, order })
@@ -144,27 +144,22 @@ fn globbed_string_to_regex(glob: &str) -> Result<Regex, crate::TantivyError> {
     // Replace `*` glob with `.*` regex
     let sanitized = format!("^{}$", regex::escape(glob).replace(r"\*", ".*"));
     Regex::new(&sanitized.replace('*', ".*")).map_err(|e| {
-        crate::TantivyError::SchemaError(format!(
-            "Invalid regex '{}' in docvalue_fields: {}",
-            glob, e
-        ))
+        crate::TantivyError::SchemaError(format!("Invalid regex '{glob}' in docvalue_fields: {e}"))
     })
 }
 
 fn use_doc_value_fields_err(parameter: &str) -> crate::Result<()> {
     Err(crate::TantivyError::AggregationError(
         AggregationError::InvalidRequest(format!(
-            "The `{}` parameter is not supported, only `docvalue_fields` is supported in \
-             `top_hits` aggregation",
-            parameter
+            "The `{parameter}` parameter is not supported, only `docvalue_fields` is supported in \
+             `top_hits` aggregation"
         )),
     ))
 }
 fn unsupported_err(parameter: &str) -> crate::Result<()> {
     Err(crate::TantivyError::AggregationError(
         AggregationError::InvalidRequest(format!(
-            "The `{}` parameter is not supported in the `top_hits` aggregation",
-            parameter
+            "The `{parameter}` parameter is not supported in the `top_hits` aggregation"
         )),
     ))
 }
@@ -217,8 +212,7 @@ impl TopHitsAggregation {
                     .collect::<Vec<_>>();
                 assert!(
                     !fields.is_empty(),
-                    "No fields matched the glob '{}' in docvalue_fields",
-                    field
+                    "No fields matched the glob '{field}' in docvalue_fields"
                 );
                 Ok(fields)
             })
@@ -254,7 +248,7 @@ impl TopHitsAggregation {
             .map(|field| {
                 let accessors = accessors
                     .get(field)
-                    .unwrap_or_else(|| panic!("field '{}' not found in accessors", field));
+                    .unwrap_or_else(|| panic!("field '{field}' not found in accessors"));
 
                 let values: Vec<FastFieldValue> = accessors
                     .iter()

src/aggregation/mod.rs

@@ -158,15 +158,14 @@ use serde::de::{self, Visitor};
 use serde::{Deserialize, Deserializer, Serialize};
 
 fn parse_str_into_f64<E: de::Error>(value: &str) -> Result<f64, E> {
-    let parsed = value.parse::<f64>().map_err(|_err| {
-        de::Error::custom(format!("Failed to parse f64 from string: {:?}", value))
-    })?;
+    let parsed = value
+        .parse::<f64>()
+        .map_err(|_err| de::Error::custom(format!("Failed to parse f64 from string: {value:?}")))?;
 
     // Check if the parsed value is NaN or infinity
     if parsed.is_nan() || parsed.is_infinite() {
         Err(de::Error::custom(format!(
-            "Value is not a valid f64 (NaN or Infinity): {:?}",
-            value
+            "Value is not a valid f64 (NaN or Infinity): {value:?}"
         )))
     } else {
         Ok(parsed)

src/collector/facet_collector.rs

@@ -598,7 +598,7 @@ mod tests {
                 let mid = n % 4;
                 n /= 4;
                 let leaf = n % 5;
-                Facet::from(&format!("/top{}/mid{}/leaf{}", top, mid, leaf))
+                Facet::from(&format!("/top{top}/mid{mid}/leaf{leaf}"))
             })
             .collect();
         for i in 0..num_facets * 10 {
@@ -737,7 +737,7 @@ mod tests {
             vec![("a", 10), ("b", 100), ("c", 7), ("d", 12), ("e", 21)]
                 .into_iter()
                 .flat_map(|(c, count)| {
-                    let facet = Facet::from(&format!("/facet/{}", c));
+                    let facet = Facet::from(&format!("/facet/{c}"));
                     let doc = doc!(facet_field => facet);
                     iter::repeat(doc).take(count)
                 })
@@ -785,7 +785,7 @@ mod tests {
         let docs: Vec<TantivyDocument> = vec![("b", 2), ("a", 2), ("c", 4)]
             .into_iter()
             .flat_map(|(c, count)| {
-                let facet = Facet::from(&format!("/facet/{}", c));
+                let facet = Facet::from(&format!("/facet/{c}"));
                 let doc = doc!(facet_field => facet);
                 iter::repeat(doc).take(count)
             })

src/core/executor.rs

@@ -184,28 +184,56 @@ mod tests {
     fn test_cancel_cpu_intensive_tasks() {
         use std::sync::atomic::{AtomicU64, Ordering};
         use std::sync::Arc;
-        use std::time::Duration;
 
         let counter: Arc<AtomicU64> = Default::default();
+
+        let other_counter: Arc<AtomicU64> = Default::default();
+
         let mut futures = Vec::new();
+        let mut other_futures = Vec::new();
+
+        let (tx, rx) = crossbeam_channel::bounded::<()>(0);
+        let rx = Arc::new(rx);
         let executor = Executor::multi_thread(3, "search-test").unwrap();
-        for _ in 0..1_000 {
-            let counter_clone = counter.clone();
+        for i in 0..1000 {
+            let counter_clone: Arc<AtomicU64> = counter.clone();
+            let other_counter_clone: Arc<AtomicU64> = other_counter.clone();
+
+            let rx_clone = rx.clone();
+            let rx_clone2 = rx.clone();
             let fut = executor.spawn_blocking(move || {
-                std::thread::sleep(Duration::from_millis(4));
-                counter_clone.fetch_add(1, Ordering::SeqCst)
+                counter_clone.fetch_add(1, Ordering::SeqCst);
+                let () = rx_clone.recv().unwrap();
             });
             futures.push(fut);
+            let other_fut = executor.spawn_blocking(move || {
+                other_counter_clone.fetch_add(1, Ordering::SeqCst);
+                let () = rx_clone2.recv().unwrap();
+            });
+            other_futures.push(other_fut);
         }
-        std::thread::sleep(Duration::from_millis(5));
-        // The first few num_cores tasks should run, but the other should get cancelled.
-        drop(futures);
-        while Arc::strong_count(&counter) > 1 {
-            std::thread::sleep(Duration::from_millis(10));
+
+        // We execute 100 futures.
+        for i in 0..100 {
+            tx.send(()).unwrap();
         }
-        // with ideal timing, we expect the result to always be 6, but as long as we run some, and
-        // cancelled most, the test is a success
-        assert!(counter.load(Ordering::SeqCst) > 0);
-        assert!(counter.load(Ordering::SeqCst) < 50);
+
+        let counter_val = counter.load(Ordering::SeqCst);
+        let other_counter_val = other_counter.load(Ordering::SeqCst);
+        assert!(counter_val >= 30);
+        assert!(other_counter_val >= 30);
+
+        drop(other_futures);
+
+        // We execute 100 futures.
+        for i in 0..100 {
+            tx.send(()).unwrap();
+        }
+
+        let counter_val2 = counter.load(Ordering::SeqCst);
+        assert!(counter_val2 >= counter_val + 100 - 6);
+
+        let other_counter_val2 = other_counter.load(Ordering::SeqCst);
+        assert!(other_counter_val2 <= other_counter_val + 6);
     }
 }

src/core/json_utils.rs

@@ -338,14 +338,14 @@ mod tests {
         let mut term = Term::from_field_json_path(field, "attributes.color", false);
         term.append_type_and_str("red");
         assert_eq!(
-            format!("{:?}", term),
+            format!("{term:?}"),
             "Term(field=1, type=Json, path=attributes.color, type=Str, \"red\")"
         );
 
         let mut term = Term::from_field_json_path(field, "attributes.dimensions.width", false);
         term.append_type_and_fast_value(400i64);
         assert_eq!(
-            format!("{:?}", term),
+            format!("{term:?}"),
             "Term(field=1, type=Json, path=attributes.dimensions.width, type=I64, 400)"
         );
     }

src/directory/mmap_directory.rs

@@ -566,7 +566,7 @@ mod tests {
         let mmap_directory = MmapDirectory::create_from_tempdir().unwrap();
         let num_paths = 10;
         let paths: Vec<PathBuf> = (0..num_paths)
-            .map(|i| PathBuf::from(&*format!("file_{}", i)))
+            .map(|i| PathBuf::from(&*format!("file_{i}")))
             .collect();
         {
             for path in &paths {

src/fastfield/facet_reader.rs

@@ -62,8 +62,7 @@ impl FacetReader {
 
 #[cfg(test)]
 mod tests {
-    use crate::schema::document::Value;
-    use crate::schema::{Facet, FacetOptions, SchemaBuilder, STORED};
+    use crate::schema::{Facet, FacetOptions, SchemaBuilder, Value, STORED};
     use crate::{DocAddress, Index, IndexWriter, TantivyDocument};
 
     #[test]
@@ -89,7 +88,9 @@ mod tests {
         let doc = searcher
             .doc::<TantivyDocument>(DocAddress::new(0u32, 0u32))
             .unwrap();
-        let value = doc.get_first(facet_field).and_then(|v| v.as_facet());
+        let value = doc
+            .get_first(facet_field)
+            .and_then(|v| v.as_value().as_facet());
         assert_eq!(value, None);
     }
 

src/index/index.rs

@@ -252,9 +252,8 @@ impl IndexBuilder {
                 let field_type = entry.field_type().value_type();
                 if !supported_field_types.contains(&field_type) {
                     return Err(TantivyError::InvalidArgument(format!(
-                        "Unsupported field type in sort_by_field: {:?}. Supported field types: \
-                         {:?} ",
-                        field_type, supported_field_types,
+                        "Unsupported field type in sort_by_field: {field_type:?}. Supported field \
+                         types: {supported_field_types:?} ",
                     )));
                 }
             }

src/index/segment_reader.rs

@@ -318,14 +318,14 @@ impl SegmentReader {
                         if create_canonical {
                             // Without expand dots enabled dots need to be escaped.
                             let escaped_json_path = json_path.replace('.', "\\.");
-                            let full_path = format!("{}.{}", field_name, escaped_json_path);
+                            let full_path = format!("{field_name}.{escaped_json_path}");
                             let full_path_unescaped = format!("{}.{}", field_name, &json_path);
                             map_to_canonical.insert(full_path_unescaped, full_path.to_string());
                             full_path
                         } else {
                             // With expand dots enabled, we can use '.' instead of '\u{1}'.
                             json_path_sep_to_dot(&mut json_path);
-                            format!("{}.{}", field_name, json_path)
+                            format!("{field_name}.{json_path}")
                         }
                     };
                     indexed_fields.extend(

src/indexer/doc_id_mapping.rs

@@ -306,12 +306,10 @@ mod tests_indexsorting {
         let my_string_field = index.schema().get_field("string_field").unwrap();
         let searcher = index.reader()?.searcher();
         {
-            assert_eq!(
-                searcher
-                    .doc::<TantivyDocument>(DocAddress::new(0, 0))?
-                    .get_first(my_string_field),
-                None
-            );
+            assert!(searcher
+                .doc::<TantivyDocument>(DocAddress::new(0, 0))?
+                .get_first(my_string_field)
+                .is_none());
             assert_eq!(
                 searcher
                     .doc::<TantivyDocument>(DocAddress::new(0, 3))?
@@ -344,7 +342,7 @@ mod tests_indexsorting {
                 Some("blublub")
             );
             let doc = searcher.doc::<TantivyDocument>(DocAddress::new(0, 4))?;
-            assert_eq!(doc.get_first(my_string_field), None);
+            assert!(doc.get_first(my_string_field).is_none());
         }
         // sort by field desc
         let index = create_test_index(

src/indexer/index_writer.rs

@@ -814,10 +814,9 @@ mod tests {
     use crate::indexer::index_writer::MEMORY_BUDGET_NUM_BYTES_MIN;
     use crate::indexer::NoMergePolicy;
     use crate::query::{BooleanQuery, Occur, Query, QueryParser, TermQuery};
-    use crate::schema::document::Value;
     use crate::schema::{
         self, Facet, FacetOptions, IndexRecordOption, IpAddrOptions, NumericOptions, Schema,
-        TextFieldIndexing, TextOptions, FAST, INDEXED, STORED, STRING, TEXT,
+        TextFieldIndexing, TextOptions, Value, FAST, INDEXED, STORED, STRING, TEXT,
     };
     use crate::store::DOCSTORE_CACHE_CAPACITY;
     use crate::{
@@ -1980,7 +1979,13 @@ mod tests {
                 .unwrap();
             // test store iterator
             for doc in store_reader.iter::<TantivyDocument>(segment_reader.alive_bitset()) {
-                let id = doc.unwrap().get_first(id_field).unwrap().as_u64().unwrap();
+                let id = doc
+                    .unwrap()
+                    .get_first(id_field)
+                    .unwrap()
+                    .as_value()
+                    .as_u64()
+                    .unwrap();
                 assert!(expected_ids_and_num_occurrences.contains_key(&id));
             }
             // test store random access
@@ -2013,7 +2018,7 @@ mod tests {
                     let mut bool2 = doc.get_all(multi_bools);
                     assert_eq!(bool, bool2.next().unwrap().as_bool().unwrap());
                     assert_ne!(bool, bool2.next().unwrap().as_bool().unwrap());
-                    assert_eq!(None, bool2.next())
+                    assert!(bool2.next().is_none())
                 }
             }
         }

src/indexer/merger.rs

@@ -795,10 +795,9 @@ mod tests {
     use crate::collector::{Count, FacetCollector};
     use crate::index::{Index, SegmentId};
     use crate::query::{AllQuery, BooleanQuery, EnableScoring, Scorer, TermQuery};
-    use crate::schema::document::Value;
     use crate::schema::{
         Facet, FacetOptions, IndexRecordOption, NumericOptions, TantivyDocument, Term,
-        TextFieldIndexing, INDEXED, TEXT,
+        TextFieldIndexing, Value, INDEXED, TEXT,
     };
     use crate::time::OffsetDateTime;
     use crate::{
@@ -910,15 +909,24 @@ mod tests {
             }
             {
                 let doc = searcher.doc::<TantivyDocument>(DocAddress::new(0, 0))?;
-                assert_eq!(doc.get_first(text_field).unwrap().as_str(), Some("af b"));
+                assert_eq!(
+                    doc.get_first(text_field).unwrap().as_value().as_str(),
+                    Some("af b")
+                );
             }
             {
                 let doc = searcher.doc::<TantivyDocument>(DocAddress::new(0, 1))?;
-                assert_eq!(doc.get_first(text_field).unwrap().as_str(), Some("a b c"));
+                assert_eq!(
+                    doc.get_first(text_field).unwrap().as_value().as_str(),
+                    Some("a b c")
+                );
             }
             {
                 let doc = searcher.doc::<TantivyDocument>(DocAddress::new(0, 2))?;
-                assert_eq!(doc.get_first(text_field).unwrap().as_str(), Some("a b c d"));
+                assert_eq!(
+                    doc.get_first(text_field).unwrap().as_value().as_str(),
+                    Some("a b c d")
+                );
             }
             {
                 let doc = searcher.doc::<TantivyDocument>(DocAddress::new(0, 3))?;

src/indexer/merger_sorted_index_test.rs

@@ -5,10 +5,9 @@ mod tests {
     use crate::index::Index;
     use crate::postings::Postings;
     use crate::query::QueryParser;
-    use crate::schema::document::Value;
     use crate::schema::{
         self, BytesOptions, Facet, FacetOptions, IndexRecordOption, NumericOptions,
-        TextFieldIndexing, TextOptions,
+        TextFieldIndexing, TextOptions, Value,
     };
     use crate::{
         DocAddress, DocSet, IndexSettings, IndexSortByField, IndexWriter, Order, TantivyDocument,
@@ -281,13 +280,16 @@ mod tests {
                 .doc::<TantivyDocument>(DocAddress::new(0, blubber_pos))
                 .unwrap();
             assert_eq!(
-                doc.get_first(my_text_field).unwrap().as_str(),
+                doc.get_first(my_text_field).unwrap().as_value().as_str(),
                 Some("blubber")
             );
             let doc = searcher
                 .doc::<TantivyDocument>(DocAddress::new(0, 0))
                 .unwrap();
-            assert_eq!(doc.get_first(int_field).unwrap().as_u64(), Some(1000));
+            assert_eq!(
+                doc.get_first(int_field).unwrap().as_value().as_u64(),
+                Some(1000)
+            );
         }
     }
 

src/indexer/mod.rs

@@ -216,7 +216,7 @@ mod tests_mmap {
         let test_query = |query_str: &str| {
             let query = parse_query.parse_query(query_str).unwrap();
             let num_docs = searcher.search(&query, &Count).unwrap();
-            assert_eq!(num_docs, 1, "{}", query_str);
+            assert_eq!(num_docs, 1, "{query_str}");
         };
         test_query(format!("json.{field_name_out}:test1").as_str());
         test_query(format!("json.a{field_name_out}:test2").as_str());
@@ -590,10 +590,10 @@ mod tests_mmap {
         let query_parser = QueryParser::for_index(&index, vec![]);
         // Test if field name can be queried
         for (indexed_field, val) in fields_and_vals.iter() {
-            let query_str = &format!("{}:{}", indexed_field, val);
+            let query_str = &format!("{indexed_field}:{val}");
             let query = query_parser.parse_query(query_str).unwrap();
             let count_docs = searcher.search(&*query, &TopDocs::with_limit(2)).unwrap();
-            assert!(!count_docs.is_empty(), "{}:{}", indexed_field, val);
+            assert!(!count_docs.is_empty(), "{indexed_field}:{val}");
         }
         // Test if field name can be used for aggregation
         for (field_name, val) in fields_and_vals.iter() {

src/indexer/segment_writer.rs

@@ -499,10 +499,9 @@ mod tests {
     use crate::fastfield::FastValue;
     use crate::postings::{Postings, TermInfo};
     use crate::query::{PhraseQuery, QueryParser};
-    use crate::schema::document::Value;
     use crate::schema::{
-        Document, IndexRecordOption, OwnedValue, Schema, TextFieldIndexing, TextOptions, STORED,
-        STRING, TEXT,
+        Document, IndexRecordOption, OwnedValue, Schema, TextFieldIndexing, TextOptions, Value,
+        STORED, STRING, TEXT,
     };
     use crate::store::{Compressor, StoreReader, StoreWriter};
     use crate::time::format_description::well_known::Rfc3339;
@@ -555,9 +554,15 @@ mod tests {
         let reader = StoreReader::open(directory.open_read(path).unwrap(), 0).unwrap();
         let doc = reader.get::<TantivyDocument>(0).unwrap();
 
-        assert_eq!(doc.field_values().len(), 2);
-        assert_eq!(doc.field_values()[0].value().as_str(), Some("A"));
-        assert_eq!(doc.field_values()[1].value().as_str(), Some("title"));
+        assert_eq!(doc.field_values().count(), 2);
+        assert_eq!(
+            doc.get_all(text_field).next().unwrap().as_value().as_str(),
+            Some("A")
+        );
+        assert_eq!(
+            doc.get_all(text_field).nth(1).unwrap().as_value().as_str(),
+            Some("title")
+        );
     }
     #[test]
     fn test_simple_json_indexing() {
@@ -641,7 +646,7 @@ mod tests {
         let mut schema_builder = Schema::builder();
         let json_field = schema_builder.add_json_field("json", STORED | TEXT);
         let schema = schema_builder.build();
-        let json_val: serde_json::Map<String, serde_json::Value> = serde_json::from_str(
+        let json_val: serde_json::Value = serde_json::from_str(
             r#"{
             "toto": "titi",
             "float": -0.2,
@@ -669,14 +674,10 @@ mod tests {
                 doc_id: 0u32,
             })
             .unwrap();
-        let serdeser_json_val = serde_json::from_str::<serde_json::Map<String, serde_json::Value>>(
-            &doc.to_json(&schema),
-        )
-        .unwrap()
-        .get("json")
-        .unwrap()[0]
-            .as_object()
+        let serdeser_json_val = serde_json::from_str::<serde_json::Value>(&doc.to_json(&schema))
             .unwrap()
+            .get("json")
+            .unwrap()[0]
             .clone();
         assert_eq!(json_val, serdeser_json_val);
         let segment_reader = searcher.segment_reader(0u32);
@@ -840,7 +841,7 @@ mod tests {
         let mut schema_builder = Schema::builder();
         let json_field = schema_builder.add_json_field("json", STRING);
         let schema = schema_builder.build();
-        let json_val: serde_json::Map<String, serde_json::Value> =
+        let json_val: serde_json::Value =
             serde_json::from_str(r#"{"mykey": "two tokens"}"#).unwrap();
         let doc = doc!(json_field=>json_val);
         let index = Index::create_in_ram(schema);
@@ -880,7 +881,7 @@ mod tests {
         let mut schema_builder = Schema::builder();
         let json_field = schema_builder.add_json_field("json", TEXT);
         let schema = schema_builder.build();
-        let json_val: serde_json::Map<String, serde_json::Value> = serde_json::from_str(
+        let json_val: serde_json::Value = serde_json::from_str(
             r#"{"mykey": [{"field": "hello happy tax payer"}, {"field": "nothello"}]}"#,
         )
         .unwrap();

src/lib.rs

@@ -935,7 +935,7 @@ pub mod tests {
         let mut schema_builder = Schema::builder();
         let json_field = schema_builder.add_json_field("json", STORED | TEXT);
         let schema = schema_builder.build();
-        let json_val: serde_json::Map<String, serde_json::Value> = serde_json::from_str(
+        let json_val: serde_json::Value = serde_json::from_str(
             r#"{
             "signed": 2,
             "float": 2.0,
@@ -1025,13 +1025,16 @@ pub mod tests {
                             text_field => "some other value",
                             other_text_field => "short");
         assert_eq!(document.len(), 3);
-        let values: Vec<&OwnedValue> = document.get_all(text_field).collect();
+        let values: Vec<OwnedValue> = document.get_all(text_field).map(OwnedValue::from).collect();
         assert_eq!(values.len(), 2);
-        assert_eq!(values[0].as_str(), Some("tantivy"));
-        assert_eq!(values[1].as_str(), Some("some other value"));
-        let values: Vec<&OwnedValue> = document.get_all(other_text_field).collect();
+        assert_eq!(values[0].as_ref().as_str(), Some("tantivy"));
+        assert_eq!(values[1].as_ref().as_str(), Some("some other value"));
+        let values: Vec<OwnedValue> = document
+            .get_all(other_text_field)
+            .map(OwnedValue::from)
+            .collect();
         assert_eq!(values.len(), 1);
-        assert_eq!(values[0].as_str(), Some("short"));
+        assert_eq!(values[0].as_ref().as_str(), Some("short"));
     }
 
     #[test]

src/macros.rs

@@ -41,6 +41,7 @@
 /// );
 /// # }
 /// ```
+
 #[macro_export]
 macro_rules! doc(
     () => {
@@ -52,7 +53,7 @@ macro_rules! doc(
         {
             let mut document = $crate::TantivyDocument::default();
             $(
-                document.add_field_value($field, $value);
+                document.add_field_value($field, &$value);
             )*
             document
         }

src/query/fuzzy_query.rs

@@ -138,8 +138,7 @@ impl FuzzyTermQuery {
                 if json_path_type != Type::Str {
                     return Err(InvalidArgument(format!(
                         "The fuzzy term query requires a string path type for a json term. Found \
-                         {:?}",
-                        json_path_type
+                         {json_path_type:?}"
                     )));
                 }
             }

src/query/regex_query.rs

@@ -185,7 +185,7 @@ mod test {
             Err(crate::TantivyError::InvalidArgument(msg)) => {
                 assert!(msg.contains("error: unclosed group"))
             }
-            res => panic!("unexpected result: {:?}", res),
+            res => panic!("unexpected result: {res:?}"),
         }
     }
 }

src/schema/document/default_document.rs

@@ -1,93 +1,64 @@
 use std::collections::{BTreeMap, HashMap, HashSet};
+use std::io::{self, Read, Write};
 use std::net::Ipv6Addr;
 
-use common::DateTime;
+use columnar::MonotonicallyMappableToU128;
+use common::{read_u32_vint_no_advance, serialize_vint_u32, BinarySerializable, DateTime, VInt};
 use serde_json::Map;
+pub use CompactDoc as TantivyDocument;
 
+use super::{ReferenceValue, ReferenceValueLeaf, Value};
 use crate::schema::document::{
     DeserializeError, Document, DocumentDeserialize, DocumentDeserializer,
 };
 use crate::schema::field_type::ValueParsingError;
-use crate::schema::field_value::FieldValueIter;
-use crate::schema::{Facet, Field, FieldValue, NamedFieldDocument, OwnedValue, Schema};
+use crate::schema::{Facet, Field, NamedFieldDocument, OwnedValue, Schema};
 use crate::tokenizer::PreTokenizedString;
 
-/// TantivyDocument provides a default implementation of the `Document` trait.
-/// It is the object that can be indexed and then searched for.
-///
-/// Documents are fundamentally a collection of unordered couples `(field, value)`.
-/// In this list, one field may appear more than once.
-#[derive(Clone, Debug, serde::Serialize, serde::Deserialize, Default)]
-pub struct TantivyDocument {
-    field_values: Vec<FieldValue>,
+#[repr(packed)]
+#[derive(Debug, Clone)]
+/// A field value pair in the compact tantivy document
+struct FieldValueAddr {
+    pub field: u16,
+    pub value_addr: ValueAddr,
 }
 
-impl Document for TantivyDocument {
-    type Value<'a> = &'a OwnedValue;
-    type FieldsValuesIter<'a> = FieldValueIter<'a>;
+#[derive(Debug, Clone)]
+/// The default document in tantivy. It encodes data in a compact form.
+pub struct CompactDoc {
+    /// `node_data` is a vec of bytes, where each value is serialized into bytes and stored. It
+    /// includes all the data of the document and also metadata like where the nodes are located
+    /// in an object or array.
+    pub node_data: Vec<u8>,
+    /// The root (Field, Value) pairs
+    field_values: Vec<FieldValueAddr>,
+}
 
-    fn iter_fields_and_values(&self) -> Self::FieldsValuesIter<'_> {
-        FieldValueIter(self.field_values.iter())
+impl Default for CompactDoc {
+    fn default() -> Self {
+        Self::new()
     }
 }
 
-impl DocumentDeserialize for TantivyDocument {
-    fn deserialize<'de, D>(mut deserializer: D) -> Result<Self, DeserializeError>
-    where D: DocumentDeserializer<'de> {
-        let mut field_values = Vec::with_capacity(deserializer.size_hint());
-
-        while let Some((field, value)) = deserializer.next_field()? {
-            field_values.push(FieldValue::new(field, value));
-        }
-
-        Ok(Self { field_values })
-    }
-}
-
-impl From<Vec<FieldValue>> for TantivyDocument {
-    fn from(field_values: Vec<FieldValue>) -> Self {
-        Self { field_values }
-    }
-}
-
-impl PartialEq for TantivyDocument {
-    fn eq(&self, other: &Self) -> bool {
-        // super slow, but only here for tests
-        let convert_to_comparable_map = |field_values: &[FieldValue]| {
-            let mut field_value_set: HashMap<Field, HashSet<String>> = Default::default();
-            for field_value in field_values.iter() {
-                let value = serde_json::to_string(field_value.value()).unwrap();
-                field_value_set
-                    .entry(field_value.field())
-                    .or_default()
-                    .insert(value);
-            }
-            field_value_set
-        };
-        let self_field_values: HashMap<Field, HashSet<String>> =
-            convert_to_comparable_map(&self.field_values);
-        let other_field_values: HashMap<Field, HashSet<String>> =
-            convert_to_comparable_map(&other.field_values);
-        self_field_values.eq(&other_field_values)
-    }
-}
-
-impl Eq for TantivyDocument {}
-
-impl IntoIterator for TantivyDocument {
-    type Item = FieldValue;
-
-    type IntoIter = std::vec::IntoIter<FieldValue>;
-
-    fn into_iter(self) -> Self::IntoIter {
-        self.field_values.into_iter()
-    }
-}
-
-impl TantivyDocument {
+impl CompactDoc {
     /// Creates a new, empty document object
-    pub fn new() -> TantivyDocument {
-        TantivyDocument::default()
+    /// The reserved capacity is for the total serialized data
+    pub fn with_capacity(bytes: usize) -> CompactDoc {
+        CompactDoc {
+            node_data: Vec::with_capacity(bytes),
+            field_values: Vec::with_capacity(4),
+        }
+    }
+
+    /// Creates a new, empty document object
+    pub fn new() -> CompactDoc {
+        CompactDoc::with_capacity(1024)
+    }
+
+    /// Skrinks the capacity of the document to fit the data
+    pub fn shrink_to_fit(&mut self) {
+        self.node_data.shrink_to_fit();
+        self.field_values.shrink_to_fit();
     }
 
     /// Returns the length of the document.
@@ -99,83 +70,111 @@ impl TantivyDocument {
     pub fn add_facet<F>(&mut self, field: Field, path: F)
     where Facet: From<F> {
         let facet = Facet::from(path);
-        let value = OwnedValue::Facet(facet);
-        self.add_field_value(field, value);
+        self.add_leaf_field_value(field, ReferenceValueLeaf::Facet(facet.encoded_str()));
     }
 
     /// Add a text field.
-    pub fn add_text<S: ToString>(&mut self, field: Field, text: S) {
-        let value = OwnedValue::Str(text.to_string());
-        self.add_field_value(field, value);
+    pub fn add_text<S: AsRef<str>>(&mut self, field: Field, text: S) {
+        self.add_leaf_field_value(field, ReferenceValueLeaf::Str(text.as_ref()));
     }
 
     /// Add a pre-tokenized text field.
     pub fn add_pre_tokenized_text(&mut self, field: Field, pre_tokenized_text: PreTokenizedString) {
-        self.add_field_value(field, pre_tokenized_text);
+        self.add_leaf_field_value(field, pre_tokenized_text);
     }
 
     /// Add a u64 field
     pub fn add_u64(&mut self, field: Field, value: u64) {
-        self.add_field_value(field, value);
+        self.add_leaf_field_value(field, value);
     }
 
     /// Add a IP address field. Internally only Ipv6Addr is used.
     pub fn add_ip_addr(&mut self, field: Field, value: Ipv6Addr) {
-        self.add_field_value(field, value);
+        self.add_leaf_field_value(field, value);
     }
 
     /// Add a i64 field
     pub fn add_i64(&mut self, field: Field, value: i64) {
-        self.add_field_value(field, value);
+        self.add_leaf_field_value(field, value);
     }
 
     /// Add a f64 field
     pub fn add_f64(&mut self, field: Field, value: f64) {
-        self.add_field_value(field, value);
+        self.add_leaf_field_value(field, value);
     }
 
     /// Add a bool field
     pub fn add_bool(&mut self, field: Field, value: bool) {
-        self.add_field_value(field, value);
+        self.add_leaf_field_value(field, value);
     }
 
     /// Add a date field with unspecified time zone offset
     pub fn add_date(&mut self, field: Field, value: DateTime) {
-        self.add_field_value(field, value);
+        self.add_leaf_field_value(field, value);
     }
 
     /// Add a bytes field
-    pub fn add_bytes<T: Into<Vec<u8>>>(&mut self, field: Field, value: T) {
-        self.add_field_value(field, value.into());
+    pub fn add_bytes(&mut self, field: Field, value: &[u8]) {
+        self.add_leaf_field_value(field, value);
     }
 
     /// Add a dynamic object field
     pub fn add_object(&mut self, field: Field, object: BTreeMap<String, OwnedValue>) {
-        self.add_field_value(field, object);
+        self.add_field_value(field, &OwnedValue::from(object));
     }
 
     /// Add a (field, value) to the document.
-    pub fn add_field_value<T: Into<OwnedValue>>(&mut self, field: Field, typed_val: T) {
+    ///
+    /// `OwnedValue` implements Value, which should be easiest to use, but is not the most
+    /// performant.
+    pub fn add_field_value<'a, V: Value<'a>>(&mut self, field: Field, value: V) {
+        let field_value = FieldValueAddr {
+            field: field
+                .field_id()
+                .try_into()
+                .expect("support only up to u16::MAX field ids"),
+            value_addr: self.add_value(value),
+        };
+        self.field_values.push(field_value);
+    }
+
+    /// Add a (field, leaf value) to the document.
+    /// Leaf values don't have nested values.
+    pub fn add_leaf_field_value<'a, T: Into<ReferenceValueLeaf<'a>>>(
+        &mut self,
+        field: Field,
+        typed_val: T,
+    ) {
         let value = typed_val.into();
-        let field_value = FieldValue { field, value };
+        let field_value = FieldValueAddr {
+            field: field
+                .field_id()
+                .try_into()
+                .expect("support only up to u16::MAX field ids"),
+            value_addr: self.add_value_leaf(value),
+        };
         self.field_values.push(field_value);
     }
 
     /// field_values accessor
-    pub fn field_values(&self) -> &[FieldValue] {
-        &self.field_values
+    pub fn field_values(&self) -> impl Iterator<Item = (Field, CompactDocValue<'_>)> {
+        self.field_values.iter().map(|field_val| {
+            let field = Field::from_field_id(field_val.field as u32);
+            let val = self.get_compact_doc_value(field_val.value_addr);
+            (field, val)
+        })
     }
 
-    /// Returns all of the `FieldValue`s associated the given field
-    pub fn get_all(&self, field: Field) -> impl Iterator<Item = &OwnedValue> {
+    /// Returns all of the `ReferenceValue`s associated the given field
+    pub fn get_all(&self, field: Field) -> impl Iterator<Item = CompactDocValue<'_>> + '_ {
         self.field_values
             .iter()
-            .filter(move |field_value| field_value.field() == field)
-            .map(FieldValue::value)
+            .filter(move |field_value| Field::from_field_id(field_value.field as u32) == field)
+            .map(|val| self.get_compact_doc_value(val.value_addr))
     }
 
-    /// Returns the first `FieldValue` associated the given field
-    pub fn get_first(&self, field: Field) -> Option<&OwnedValue> {
+    /// Returns the first `ReferenceValue` associated the given field
+    pub fn get_first(&self, field: Field) -> Option<CompactDocValue<'_>> {
         self.get_all(field).next()
     }
 
@@ -183,12 +182,12 @@ impl TantivyDocument {
     pub fn convert_named_doc(
         schema: &Schema,
         named_doc: NamedFieldDocument,
-    ) -> Result<TantivyDocument, DocParsingError> {
-        let mut document = TantivyDocument::new();
+    ) -> Result<Self, DocParsingError> {
+        let mut document = Self::new();
         for (field_name, values) in named_doc.0 {
             if let Ok(field) = schema.get_field(&field_name) {
                 for value in values {
-                    document.add_field_value(field, value);
+                    document.add_field_value(field, &value);
                 }
             }
         }
@@ -196,7 +195,7 @@ impl TantivyDocument {
     }
 
     /// Build a document object from a json-object.
-    pub fn parse_json(schema: &Schema, doc_json: &str) -> Result<TantivyDocument, DocParsingError> {
+    pub fn parse_json(schema: &Schema, doc_json: &str) -> Result<Self, DocParsingError> {
         let json_obj: Map<String, serde_json::Value> =
             serde_json::from_str(doc_json).map_err(|_| DocParsingError::invalid_json(doc_json))?;
         Self::from_json_object(schema, json_obj)
@@ -206,8 +205,8 @@ impl TantivyDocument {
     pub fn from_json_object(
         schema: &Schema,
         json_obj: Map<String, serde_json::Value>,
-    ) -> Result<TantivyDocument, DocParsingError> {
-        let mut doc = TantivyDocument::default();
+    ) -> Result<Self, DocParsingError> {
+        let mut doc = Self::default();
         for (field_name, json_value) in json_obj {
             if let Ok(field) = schema.get_field(&field_name) {
                 let field_entry = schema.get_field_entry(field);
@@ -218,20 +217,482 @@ impl TantivyDocument {
                             let value = field_type
                                 .value_from_json(json_item)
                                 .map_err(|e| DocParsingError::ValueError(field_name.clone(), e))?;
-                            doc.add_field_value(field, value);
+                            doc.add_field_value(field, &value);
                         }
                     }
                     _ => {
                         let value = field_type
                             .value_from_json(json_value)
                             .map_err(|e| DocParsingError::ValueError(field_name.clone(), e))?;
-                        doc.add_field_value(field, value);
+                        doc.add_field_value(field, &value);
                     }
                 }
             }
         }
         Ok(doc)
     }
+
+    fn add_value_leaf(&mut self, leaf: ReferenceValueLeaf) -> ValueAddr {
+        let type_id = ValueType::from(&leaf);
+        // Write into `node_data` and return u32 position as its address
+        // Null and bool are inlined into the address
+        let val_addr = match leaf {
+            ReferenceValueLeaf::Null => 0,
+            ReferenceValueLeaf::Str(bytes) => {
+                write_bytes_into(&mut self.node_data, bytes.as_bytes())
+            }
+            ReferenceValueLeaf::Facet(bytes) => {
+                write_bytes_into(&mut self.node_data, bytes.as_bytes())
+            }
+            ReferenceValueLeaf::Bytes(bytes) => write_bytes_into(&mut self.node_data, bytes),
+            ReferenceValueLeaf::U64(num) => write_into(&mut self.node_data, num),
+            ReferenceValueLeaf::I64(num) => write_into(&mut self.node_data, num),
+            ReferenceValueLeaf::F64(num) => write_into(&mut self.node_data, num),
+            ReferenceValueLeaf::Bool(b) => b as u32,
+            ReferenceValueLeaf::Date(date) => {
+                write_into(&mut self.node_data, date.into_timestamp_nanos())
+            }
+            ReferenceValueLeaf::IpAddr(num) => write_into(&mut self.node_data, num.to_u128()),
+            ReferenceValueLeaf::PreTokStr(pre_tok) => write_into(&mut self.node_data, *pre_tok),
+        };
+        ValueAddr { type_id, val_addr }
+    }
+    /// Adds a value and returns in address into the
+    fn add_value<'a, V: Value<'a>>(&mut self, value: V) -> ValueAddr {
+        let value = value.as_value();
+        let type_id = ValueType::from(&value);
+        match value {
+            ReferenceValue::Leaf(leaf) => self.add_value_leaf(leaf),
+            ReferenceValue::Array(elements) => {
+                // addresses of the elements in node_data
+                // Reusing a vec would be nicer, but it's not easy because of the recursion
+                // A global vec would work if every writer get it's discriminator
+                let mut addresses = Vec::new();
+                for elem in elements {
+                    let value_addr = self.add_value(elem);
+                    write_into(&mut addresses, value_addr);
+                }
+                ValueAddr {
+                    type_id,
+                    val_addr: write_bytes_into(&mut self.node_data, &addresses),
+                }
+            }
+            ReferenceValue::Object(entries) => {
+                // addresses of the elements in node_data
+                let mut addresses = Vec::new();
+                for (key, value) in entries {
+                    let key_addr = self.add_value_leaf(ReferenceValueLeaf::Str(key));
+                    let value_addr = self.add_value(value);
+                    write_into(&mut addresses, key_addr);
+                    write_into(&mut addresses, value_addr);
+                }
+                ValueAddr {
+                    type_id,
+                    val_addr: write_bytes_into(&mut self.node_data, &addresses),
+                }
+            }
+        }
+    }
+
+    /// Get CompactDocValue for address
+    fn get_compact_doc_value(&self, value_addr: ValueAddr) -> CompactDocValue<'_> {
+        CompactDocValue {
+            container: self,
+            value_addr,
+        }
+    }
+
+    /// get &[u8] reference from node_data
+    fn extract_bytes(&self, addr: Addr) -> &[u8] {
+        binary_deserialize_bytes(self.get_slice(addr))
+    }
+
+    /// get &str reference from node_data
+    fn extract_str(&self, addr: Addr) -> &str {
+        let data = self.extract_bytes(addr);
+        // Utf-8 checks would have a noticeable performance overhead here
+        unsafe { std::str::from_utf8_unchecked(data) }
+    }
+
+    /// deserialized owned value from node_data
+    fn read_from<T: BinarySerializable>(&self, addr: Addr) -> io::Result<T> {
+        let data_slice = &self.node_data[addr as usize..];
+        let mut cursor = std::io::Cursor::new(data_slice);
+        T::deserialize(&mut cursor)
+    }
+
+    /// get slice from address. The returned slice is open ended
+    fn get_slice(&self, addr: Addr) -> &[u8] {
+        &self.node_data[addr as usize..]
+    }
+}
+
+/// BinarySerializable alternative to read references
+fn binary_deserialize_bytes(data: &[u8]) -> &[u8] {
+    let (len, bytes_read) = read_u32_vint_no_advance(data);
+    &data[bytes_read..bytes_read + len as usize]
+}
+
+/// Write bytes and return the position of the written data.
+///
+/// BinarySerializable alternative to write references
+fn write_bytes_into(vec: &mut Vec<u8>, data: &[u8]) -> u32 {
+    let pos = vec.len() as u32;
+    let mut buf = [0u8; 8];
+    let len_vint_bytes = serialize_vint_u32(data.len() as u32, &mut buf);
+    vec.extend_from_slice(len_vint_bytes);
+    vec.extend_from_slice(data);
+    pos
+}
+
+/// Serialize and return the position
+fn write_into<T: BinarySerializable>(vec: &mut Vec<u8>, value: T) -> u32 {
+    let pos = vec.len() as u32;
+    value.serialize(vec).unwrap();
+    pos
+}
+
+impl PartialEq for CompactDoc {
+    fn eq(&self, other: &Self) -> bool {
+        // super slow, but only here for tests
+        let convert_to_comparable_map = |doc: &CompactDoc| {
+            let mut field_value_set: HashMap<Field, HashSet<String>> = Default::default();
+            for field_value in doc.field_values.iter() {
+                let value: OwnedValue = doc.get_compact_doc_value(field_value.value_addr).into();
+                let value = serde_json::to_string(&value).unwrap();
+                field_value_set
+                    .entry(Field::from_field_id(field_value.field as u32))
+                    .or_default()
+                    .insert(value);
+            }
+            field_value_set
+        };
+        let self_field_values: HashMap<Field, HashSet<String>> = convert_to_comparable_map(self);
+        let other_field_values: HashMap<Field, HashSet<String>> = convert_to_comparable_map(other);
+        self_field_values.eq(&other_field_values)
+    }
+}
+
+impl Eq for CompactDoc {}
+
+impl DocumentDeserialize for CompactDoc {
+    fn deserialize<'de, D>(mut deserializer: D) -> Result<Self, DeserializeError>
+    where D: DocumentDeserializer<'de> {
+        let mut doc = CompactDoc::default();
+        // TODO: Deserializing into OwnedValue is wasteful. The deserializer should be able to work
+        // on slices and referenced data.
+        while let Some((field, value)) = deserializer.next_field::<OwnedValue>()? {
+            doc.add_field_value(field, &value);
+        }
+        Ok(doc)
+    }
+}
+
+/// A value of Compact Doc needs a reference to the container to extract its payload
+#[derive(Debug, Clone, Copy)]
+pub struct CompactDocValue<'a> {
+    container: &'a CompactDoc,
+    value_addr: ValueAddr,
+}
+impl PartialEq for CompactDocValue<'_> {
+    fn eq(&self, other: &Self) -> bool {
+        let value1: OwnedValue = (*self).into();
+        let value2: OwnedValue = (*other).into();
+        value1 == value2
+    }
+}
+impl<'a> From<CompactDocValue<'a>> for OwnedValue {
+    fn from(value: CompactDocValue) -> Self {
+        value.as_value().into()
+    }
+}
+impl<'a> Value<'a> for CompactDocValue<'a> {
+    type ArrayIter = CompactDocArrayIter<'a>;
+
+    type ObjectIter = CompactDocObjectIter<'a>;
+
+    fn as_value(&self) -> ReferenceValue<'a, Self> {
+        self.get_ref_value().unwrap()
+    }
+}
+impl<'a> CompactDocValue<'a> {
+    fn get_ref_value(&self) -> io::Result<ReferenceValue<'a, CompactDocValue<'a>>> {
+        let addr = self.value_addr.val_addr;
+        match self.value_addr.type_id {
+            ValueType::Null => Ok(ReferenceValueLeaf::Null.into()),
+            ValueType::Str => {
+                let str_ref = self.container.extract_str(addr);
+                Ok(ReferenceValueLeaf::Str(str_ref).into())
+            }
+            ValueType::Facet => {
+                let str_ref = self.container.extract_str(addr);
+                Ok(ReferenceValueLeaf::Facet(str_ref).into())
+            }
+            ValueType::Bytes => {
+                let data = self.container.extract_bytes(addr);
+                Ok(ReferenceValueLeaf::Bytes(data).into())
+            }
+            ValueType::U64 => self
+                .container
+                .read_from::<u64>(addr)
+                .map(ReferenceValueLeaf::U64)
+                .map(Into::into),
+            ValueType::I64 => self
+                .container
+                .read_from::<i64>(addr)
+                .map(ReferenceValueLeaf::I64)
+                .map(Into::into),
+            ValueType::F64 => self
+                .container
+                .read_from::<f64>(addr)
+                .map(ReferenceValueLeaf::F64)
+                .map(Into::into),
+            ValueType::Bool => Ok(ReferenceValueLeaf::Bool(addr != 0).into()),
+            ValueType::Date => self
+                .container
+                .read_from::<i64>(addr)
+                .map(|ts| ReferenceValueLeaf::Date(DateTime::from_timestamp_nanos(ts)))
+                .map(Into::into),
+            ValueType::IpAddr => self
+                .container
+                .read_from::<u128>(addr)
+                .map(|num| ReferenceValueLeaf::IpAddr(Ipv6Addr::from_u128(num)))
+                .map(Into::into),
+            ValueType::PreTokStr => self
+                .container
+                .read_from::<PreTokenizedString>(addr)
+                .map(Into::into)
+                .map(ReferenceValueLeaf::PreTokStr)
+                .map(Into::into),
+            ValueType::Object => Ok(ReferenceValue::Object(CompactDocObjectIter::new(
+                self.container,
+                addr,
+            )?)),
+            ValueType::Array => Ok(ReferenceValue::Array(CompactDocArrayIter::new(
+                self.container,
+                addr,
+            )?)),
+        }
+    }
+}
+
+/// The address in the vec
+type Addr = u32;
+
+#[derive(Clone, Copy, Default)]
+#[repr(packed)]
+/// The value type and the address to its payload in the container.
+struct ValueAddr {
+    type_id: ValueType,
+    /// This is the address to the value in the vec, except for bool and null, which are inlined
+    val_addr: Addr,
+}
+impl BinarySerializable for ValueAddr {
+    fn serialize<W: Write + ?Sized>(&self, writer: &mut W) -> io::Result<()> {
+        self.type_id.serialize(writer)?;
+        VInt(self.val_addr as u64).serialize(writer)
+    }
+
+    fn deserialize<R: Read>(reader: &mut R) -> io::Result<Self> {
+        let type_id = ValueType::deserialize(reader)?;
+        let val_addr = VInt::deserialize(reader)?.0 as u32;
+        Ok(ValueAddr { type_id, val_addr })
+    }
+}
+impl std::fmt::Debug for ValueAddr {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let val_addr = self.val_addr;
+        f.write_fmt(format_args!("{:?} at {:?}", self.type_id, val_addr))
+    }
+}
+
+/// A enum representing a value for tantivy to index.
+///
+/// Any changes need to be reflected in `BinarySerializable` for `ValueType`
+///
+/// We can't use [schema::Type] or [columnar::ColumnType] here, because they are missing
+/// some items like Array and PreTokStr.
+#[derive(Default, Clone, Copy, Debug, PartialEq)]
+#[repr(u8)]
+pub enum ValueType {
+    /// A null value.
+    #[default]
+    Null = 0,
+    /// The str type is used for any text information.
+    Str = 1,
+    /// Unsigned 64-bits Integer `u64`
+    U64 = 2,
+    /// Signed 64-bits Integer `i64`
+    I64 = 3,
+    /// 64-bits Float `f64`
+    F64 = 4,
+    /// Date/time with nanoseconds precision
+    Date = 5,
+    /// Facet
+    Facet = 6,
+    /// Arbitrarily sized byte array
+    Bytes = 7,
+    /// IpV6 Address. Internally there is no IpV4, it needs to be converted to `Ipv6Addr`.
+    IpAddr = 8,
+    /// Bool value
+    Bool = 9,
+    /// Pre-tokenized str type,
+    PreTokStr = 10,
+    /// Object
+    Object = 11,
+    /// Pre-tokenized str type,
+    Array = 12,
+}
+
+impl BinarySerializable for ValueType {
+    fn serialize<W: Write + ?Sized>(&self, writer: &mut W) -> io::Result<()> {
+        (*self as u8).serialize(writer)?;
+        Ok(())
+    }
+
+    fn deserialize<R: Read>(reader: &mut R) -> io::Result<Self> {
+        let num = u8::deserialize(reader)?;
+        let type_id = if (0..=12).contains(&num) {
+            unsafe { std::mem::transmute(num) }
+        } else {
+            return Err(io::Error::new(
+                io::ErrorKind::InvalidData,
+                format!("Invalid value type id: {num}"),
+            ));
+        };
+        Ok(type_id)
+    }
+}
+
+impl<'a, V: Value<'a>> From<&ReferenceValue<'a, V>> for ValueType {
+    fn from(value: &ReferenceValue<'a, V>) -> Self {
+        match value {
+            ReferenceValue::Leaf(leaf) => leaf.into(),
+            ReferenceValue::Array(_) => ValueType::Array,
+            ReferenceValue::Object(_) => ValueType::Object,
+        }
+    }
+}
+impl<'a> From<&ReferenceValueLeaf<'a>> for ValueType {
+    fn from(value: &ReferenceValueLeaf<'a>) -> Self {
+        match value {
+            ReferenceValueLeaf::Null => ValueType::Null,
+            ReferenceValueLeaf::Str(_) => ValueType::Str,
+            ReferenceValueLeaf::U64(_) => ValueType::U64,
+            ReferenceValueLeaf::I64(_) => ValueType::I64,
+            ReferenceValueLeaf::F64(_) => ValueType::F64,
+            ReferenceValueLeaf::Bool(_) => ValueType::Bool,
+            ReferenceValueLeaf::Date(_) => ValueType::Date,
+            ReferenceValueLeaf::IpAddr(_) => ValueType::IpAddr,
+            ReferenceValueLeaf::PreTokStr(_) => ValueType::PreTokStr,
+            ReferenceValueLeaf::Facet(_) => ValueType::Facet,
+            ReferenceValueLeaf::Bytes(_) => ValueType::Bytes,
+        }
+    }
+}
+
+#[derive(Debug, Clone)]
+/// The Iterator for the object values in the compact document
+pub struct CompactDocObjectIter<'a> {
+    container: &'a CompactDoc,
+    node_addresses_slice: &'a [u8],
+}
+
+impl<'a> CompactDocObjectIter<'a> {
+    fn new(container: &'a CompactDoc, addr: Addr) -> io::Result<Self> {
+        // Objects are `&[ValueAddr]` serialized into bytes
+        let node_addresses_slice = container.extract_bytes(addr);
+        Ok(Self {
+            container,
+            node_addresses_slice,
+        })
+    }
+}
+
+impl<'a> Iterator for CompactDocObjectIter<'a> {
+    type Item = (&'a str, CompactDocValue<'a>);
+
+    fn next(&mut self) -> Option<Self::Item> {
+        if self.node_addresses_slice.is_empty() {
+            return None;
+        }
+        let key_addr = ValueAddr::deserialize(&mut self.node_addresses_slice).ok()?;
+        let key = self.container.extract_str(key_addr.val_addr);
+        let value = ValueAddr::deserialize(&mut self.node_addresses_slice).ok()?;
+        let value = CompactDocValue {
+            container: self.container,
+            value_addr: value,
+        };
+        Some((key, value))
+    }
+}
+
+#[derive(Debug, Clone)]
+/// The Iterator for the array values in the compact document
+pub struct CompactDocArrayIter<'a> {
+    container: &'a CompactDoc,
+    node_addresses_slice: &'a [u8],
+}
+
+impl<'a> CompactDocArrayIter<'a> {
+    fn new(container: &'a CompactDoc, addr: Addr) -> io::Result<Self> {
+        // Arrays are &[ValueAddr] serialized into bytes
+        let node_addresses_slice = container.extract_bytes(addr);
+        Ok(Self {
+            container,
+            node_addresses_slice,
+        })
+    }
+}
+
+impl<'a> Iterator for CompactDocArrayIter<'a> {
+    type Item = CompactDocValue<'a>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        if self.node_addresses_slice.is_empty() {
+            return None;
+        }
+        let value = ValueAddr::deserialize(&mut self.node_addresses_slice).ok()?;
+        let value = CompactDocValue {
+            container: self.container,
+            value_addr: value,
+        };
+        Some(value)
+    }
+}
+
+impl Document for CompactDoc {
+    type Value<'a> = CompactDocValue<'a>;
+    type FieldsValuesIter<'a> = FieldValueIterRef<'a>;
+
+    fn iter_fields_and_values(&self) -> Self::FieldsValuesIter<'_> {
+        FieldValueIterRef {
+            slice: self.field_values.iter(),
+            container: self,
+        }
+    }
+}
+
+/// A helper wrapper for creating an iterator over the field values
+pub struct FieldValueIterRef<'a> {
+    slice: std::slice::Iter<'a, FieldValueAddr>,
+    container: &'a CompactDoc,
+}
+
+impl<'a> Iterator for FieldValueIterRef<'a> {
+    type Item = (Field, CompactDocValue<'a>);
+
+    fn next(&mut self) -> Option<Self::Item> {
+        self.slice.next().map(|field_value| {
+            (
+                Field::from_field_id(field_value.field as u32),
+                CompactDocValue::<'a> {
+                    container: self.container,
+                    value_addr: field_value.value_addr,
+                },
+            )
+        })
+    }
 }
 
 /// Error that may happen when deserializing
@@ -264,7 +725,40 @@ mod tests {
         let text_field = schema_builder.add_text_field("title", TEXT);
         let mut doc = TantivyDocument::default();
         doc.add_text(text_field, "My title");
-        assert_eq!(doc.field_values().len(), 1);
+        assert_eq!(doc.field_values().count(), 1);
+
+        let schema = schema_builder.build();
+        let _val = doc.get_first(text_field).unwrap();
+        let _json = doc.to_named_doc(&schema);
+    }
+
+    #[test]
+    fn test_json_value() {
+        let json_str = r#"{ 
+            "toto": "titi",
+            "float": -0.2,
+            "bool": true,
+            "unsigned": 1,
+            "signed": -2,
+            "complexobject": {
+                "field.with.dot": 1
+            },
+            "date": "1985-04-12T23:20:50.52Z",
+            "my_arr": [2, 3, {"my_key": "two tokens"}, 4, {"nested_array": [2, 5, 6, [7, 8, {"a": [{"d": {"e":[99]}}, 9000]}, 9, 10], [5, 5]]}]
+        }"#;
+        let json_val: std::collections::BTreeMap<String, OwnedValue> =
+            serde_json::from_str(json_str).unwrap();
+
+        let mut schema_builder = Schema::builder();
+        let json_field = schema_builder.add_json_field("json", TEXT);
+        let mut doc = TantivyDocument::default();
+        doc.add_object(json_field, json_val);
+
+        let schema = schema_builder.build();
+        let json = doc.to_json(&schema);
+        let actual_json: serde_json::Value = serde_json::from_str(&json).unwrap();
+        let expected_json: serde_json::Value = serde_json::from_str(json_str).unwrap();
+        assert_eq!(actual_json["json"][0], expected_json);
     }
 
     // TODO: Should this be re-added with the serialize method

src/schema/document/existing_type_impls.rs

@@ -5,21 +5,39 @@
 //! and don't care about some of the more specialised types or only want to customise
 //! part of the document structure.
 use std::collections::{btree_map, hash_map, BTreeMap, HashMap};
+use std::iter::Empty;
+use std::net::Ipv6Addr;
 
+use common::DateTime;
 use serde_json::Number;
+use time::format_description::well_known::Rfc3339;
+use time::OffsetDateTime;
 
+use super::facet::Facet;
 use super::ReferenceValueLeaf;
 use crate::schema::document::{
     ArrayAccess, DeserializeError, Document, DocumentDeserialize, DocumentDeserializer,
     ObjectAccess, ReferenceValue, Value, ValueDeserialize, ValueDeserializer, ValueVisitor,
 };
 use crate::schema::Field;
+use crate::tokenizer::PreTokenizedString;
 
 // Serde compatibility support.
+pub fn can_be_rfc3339_date_time(text: &str) -> bool {
+    if let Some(&first_byte) = text.as_bytes().first() {
+        if first_byte.is_ascii_digit() {
+            return true;
+        }
+    }
+
+    false
+}
+
 impl<'a> Value<'a> for &'a serde_json::Value {
     type ArrayIter = std::slice::Iter<'a, serde_json::Value>;
     type ObjectIter = JsonObjectIter<'a>;
 
+    #[inline]
     fn as_value(&self) -> ReferenceValue<'a, Self> {
         match self {
             serde_json::Value::Null => ReferenceValueLeaf::Null.into(),
@@ -35,7 +53,19 @@ impl<'a> Value<'a> for &'a serde_json::Value {
                     panic!("Unsupported serde_json number {number}");
                 }
             }
-            serde_json::Value::String(val) => ReferenceValueLeaf::Str(val).into(),
+            serde_json::Value::String(text) => {
+                if can_be_rfc3339_date_time(text) {
+                    match OffsetDateTime::parse(text, &Rfc3339) {
+                        Ok(dt) => {
+                            let dt_utc = dt.to_offset(time::UtcOffset::UTC);
+                            ReferenceValueLeaf::Date(DateTime::from_utc(dt_utc)).into()
+                        }
+                        Err(_) => ReferenceValueLeaf::Str(text).into(),
+                    }
+                } else {
+                    ReferenceValueLeaf::Str(text).into()
+                }
+            }
             serde_json::Value::Array(elements) => ReferenceValue::Array(elements.iter()),
             serde_json::Value::Object(object) => {
                 ReferenceValue::Object(JsonObjectIter(object.iter()))
@@ -44,6 +74,126 @@ impl<'a> Value<'a> for &'a serde_json::Value {
     }
 }
 
+impl<'a> Value<'a> for &'a String {
+    type ArrayIter = Empty<&'a String>;
+    type ObjectIter = Empty<(&'a str, &'a String)>;
+    #[inline]
+    fn as_value(&self) -> ReferenceValue<'a, Self> {
+        ReferenceValue::Leaf(ReferenceValueLeaf::Str(self))
+    }
+}
+
+impl<'a> Value<'a> for &'a Facet {
+    type ArrayIter = Empty<&'a Facet>;
+    type ObjectIter = Empty<(&'a str, &'a Facet)>;
+    #[inline]
+    fn as_value(&self) -> ReferenceValue<'a, Self> {
+        ReferenceValue::Leaf(ReferenceValueLeaf::Facet(self.encoded_str()))
+    }
+}
+
+impl<'a> Value<'a> for &'a u64 {
+    type ArrayIter = Empty<&'a u64>;
+    type ObjectIter = Empty<(&'a str, &'a u64)>;
+    #[inline]
+    fn as_value(&self) -> ReferenceValue<'a, Self> {
+        ReferenceValue::Leaf(ReferenceValueLeaf::U64(**self))
+    }
+}
+
+impl<'a> Value<'a> for &'a i64 {
+    type ArrayIter = Empty<&'a i64>;
+    type ObjectIter = Empty<(&'a str, &'a i64)>;
+    #[inline]
+    fn as_value(&self) -> ReferenceValue<'a, Self> {
+        ReferenceValue::Leaf(ReferenceValueLeaf::I64(**self))
+    }
+}
+impl<'a> Value<'a> for &'a f64 {
+    type ArrayIter = Empty<&'a f64>;
+    type ObjectIter = Empty<(&'a str, &'a f64)>;
+    #[inline]
+    fn as_value(&self) -> ReferenceValue<'a, Self> {
+        ReferenceValue::Leaf(ReferenceValueLeaf::F64(**self))
+    }
+}
+impl<'a> Value<'a> for &'a bool {
+    type ArrayIter = Empty<&'a bool>;
+    type ObjectIter = Empty<(&'a str, &'a bool)>;
+    #[inline]
+    fn as_value(&self) -> ReferenceValue<'a, Self> {
+        ReferenceValue::Leaf(ReferenceValueLeaf::Bool(**self))
+    }
+}
+impl<'a> Value<'a> for &'a str {
+    type ArrayIter = Empty<&'a str>;
+    type ObjectIter = Empty<(&'a str, &'a str)>;
+    #[inline]
+    fn as_value(&self) -> ReferenceValue<'a, Self> {
+        ReferenceValue::Leaf(ReferenceValueLeaf::Str(self))
+    }
+}
+impl<'a> Value<'a> for &'a &'a str {
+    type ArrayIter = Empty<&'a &'a str>;
+    type ObjectIter = Empty<(&'a str, &'a &'a str)>;
+    #[inline]
+    fn as_value(&self) -> ReferenceValue<'a, Self> {
+        ReferenceValue::Leaf(ReferenceValueLeaf::Str(self))
+    }
+}
+
+impl<'a> Value<'a> for &'a [u8] {
+    type ArrayIter = Empty<&'a [u8]>;
+    type ObjectIter = Empty<(&'a str, &'a [u8])>;
+    #[inline]
+    fn as_value(&self) -> ReferenceValue<'a, Self> {
+        ReferenceValue::Leaf(ReferenceValueLeaf::Bytes(self))
+    }
+}
+
+impl<'a> Value<'a> for &'a &'a [u8] {
+    type ArrayIter = Empty<&'a &'a [u8]>;
+    type ObjectIter = Empty<(&'a str, &'a &'a [u8])>;
+    #[inline]
+    fn as_value(&self) -> ReferenceValue<'a, Self> {
+        ReferenceValue::Leaf(ReferenceValueLeaf::Bytes(self))
+    }
+}
+
+impl<'a> Value<'a> for &'a Vec<u8> {
+    type ArrayIter = Empty<&'a Vec<u8>>;
+    type ObjectIter = Empty<(&'a str, &'a Vec<u8>)>;
+    #[inline]
+    fn as_value(&self) -> ReferenceValue<'a, Self> {
+        ReferenceValue::Leaf(ReferenceValueLeaf::Bytes(self))
+    }
+}
+
+impl<'a> Value<'a> for &'a DateTime {
+    type ArrayIter = Empty<&'a DateTime>;
+    type ObjectIter = Empty<(&'a str, &'a DateTime)>;
+    #[inline]
+    fn as_value(&self) -> ReferenceValue<'a, Self> {
+        ReferenceValue::Leaf(ReferenceValueLeaf::Date(**self))
+    }
+}
+impl<'a> Value<'a> for &'a Ipv6Addr {
+    type ArrayIter = Empty<&'a Ipv6Addr>;
+    type ObjectIter = Empty<(&'a str, &'a Ipv6Addr)>;
+    #[inline]
+    fn as_value(&self) -> ReferenceValue<'a, Self> {
+        ReferenceValue::Leaf(ReferenceValueLeaf::IpAddr(**self))
+    }
+}
+impl<'a> Value<'a> for &'a PreTokenizedString {
+    type ArrayIter = Empty<&'a PreTokenizedString>;
+    type ObjectIter = Empty<(&'a str, &'a PreTokenizedString)>;
+    #[inline]
+    fn as_value(&self) -> ReferenceValue<'a, Self> {
+        ReferenceValue::Leaf(ReferenceValueLeaf::PreTokStr(Box::new((*self).clone())))
+    }
+}
+
 impl ValueDeserialize for serde_json::Value {
     fn deserialize<'de, D>(deserializer: D) -> Result<Self, DeserializeError>
     where D: ValueDeserializer<'de> {

src/schema/document/mod.rs

@@ -172,7 +172,9 @@ pub use self::de::{
     ArrayAccess, DeserializeError, DocumentDeserialize, DocumentDeserializer, ObjectAccess,
     ValueDeserialize, ValueDeserializer, ValueType, ValueVisitor,
 };
-pub use self::default_document::{DocParsingError, TantivyDocument};
+pub use self::default_document::{
+    CompactDocArrayIter, CompactDocObjectIter, CompactDocValue, DocParsingError, TantivyDocument,
+};
 pub use self::owned_value::OwnedValue;
 pub(crate) use self::se::BinaryDocumentSerializer;
 pub use self::value::{ReferenceValue, ReferenceValueLeaf, Value};
@@ -233,7 +235,7 @@ pub trait Document: Send + Sync + 'static {
             let field_name = schema.get_field_name(field);
             let values: Vec<OwnedValue> = field_values
                 .into_iter()
-                .map(|val| val.as_value().into())
+                .map(|val| OwnedValue::from(val.as_value()))
                 .collect();
             field_map.insert(field_name.to_string(), values);
         }

src/schema/document/owned_value.rs

@@ -8,6 +8,7 @@ use serde::de::{MapAccess, SeqAccess};
 use time::format_description::well_known::Rfc3339;
 use time::OffsetDateTime;
 
+use super::existing_type_impls::can_be_rfc3339_date_time;
 use super::ReferenceValueLeaf;
 use crate::schema::document::{
     ArrayAccess, DeserializeError, ObjectAccess, ReferenceValue, Value, ValueDeserialize,
@@ -375,16 +376,6 @@ impl From<BTreeMap<String, OwnedValue>> for OwnedValue {
     }
 }
 
-fn can_be_rfc3339_date_time(text: &str) -> bool {
-    if let Some(&first_byte) = text.as_bytes().first() {
-        if first_byte.is_ascii_digit() {
-            return true;
-        }
-    }
-
-    false
-}
-
 impl From<serde_json::Value> for OwnedValue {
     fn from(value: serde_json::Value) -> Self {
         match value {
@@ -472,6 +463,7 @@ mod tests {
         let mut doc = TantivyDocument::default();
         doc.add_bytes(bytes_field, "".as_bytes());
         let json_string = doc.to_json(&schema);
+
         assert_eq!(json_string, r#"{"my_bytes":[""]}"#);
     }
 

src/schema/document/se.rs

@@ -25,6 +25,7 @@ where W: Write
 
     /// Attempts to serialize a given document and write the output
     /// to the writer.
+    #[inline]
     pub(crate) fn serialize_doc<D>(&mut self, doc: &D) -> io::Result<()>
     where D: Document {
         let stored_field_values = || {
@@ -57,9 +58,8 @@ where W: Write
             return Err(io::Error::new(
                 io::ErrorKind::Other,
                 format!(
-                    "Unexpected number of entries written to serializer, expected {} entries, got \
-                     {} entries",
-                    num_field_values, actual_length,
+                    "Unexpected number of entries written to serializer, expected \
+                     {num_field_values} entries, got {actual_length} entries",
                 ),
             ));
         }
@@ -679,6 +679,7 @@ mod tests {
         );
     }
 
+    #[inline]
     fn serialize_doc<D: Document>(doc: &D, schema: &Schema) -> Vec<u8> {
         let mut writer = Vec::new();
 

src/schema/document/value.rs

@@ -159,6 +159,69 @@ pub enum ReferenceValueLeaf<'a> {
     PreTokStr(Box<PreTokenizedString>),
 }
 
+impl From<u64> for ReferenceValueLeaf<'_> {
+    #[inline]
+    fn from(value: u64) -> Self {
+        ReferenceValueLeaf::U64(value)
+    }
+}
+
+impl From<i64> for ReferenceValueLeaf<'_> {
+    #[inline]
+    fn from(value: i64) -> Self {
+        ReferenceValueLeaf::I64(value)
+    }
+}
+
+impl From<f64> for ReferenceValueLeaf<'_> {
+    #[inline]
+    fn from(value: f64) -> Self {
+        ReferenceValueLeaf::F64(value)
+    }
+}
+
+impl From<bool> for ReferenceValueLeaf<'_> {
+    #[inline]
+    fn from(value: bool) -> Self {
+        ReferenceValueLeaf::Bool(value)
+    }
+}
+
+impl<'a> From<&'a str> for ReferenceValueLeaf<'a> {
+    #[inline]
+    fn from(value: &'a str) -> Self {
+        ReferenceValueLeaf::Str(value)
+    }
+}
+
+impl<'a> From<&'a [u8]> for ReferenceValueLeaf<'a> {
+    #[inline]
+    fn from(value: &'a [u8]) -> Self {
+        ReferenceValueLeaf::Bytes(value)
+    }
+}
+
+impl From<DateTime> for ReferenceValueLeaf<'_> {
+    #[inline]
+    fn from(value: DateTime) -> Self {
+        ReferenceValueLeaf::Date(value)
+    }
+}
+
+impl From<Ipv6Addr> for ReferenceValueLeaf<'_> {
+    #[inline]
+    fn from(value: Ipv6Addr) -> Self {
+        ReferenceValueLeaf::IpAddr(value)
+    }
+}
+
+impl From<PreTokenizedString> for ReferenceValueLeaf<'_> {
+    #[inline]
+    fn from(val: PreTokenizedString) -> Self {
+        ReferenceValueLeaf::PreTokStr(Box::new(val))
+    }
+}
+
 impl<'a, T: Value<'a> + ?Sized> From<ReferenceValueLeaf<'a>> for ReferenceValue<'a, T> {
     #[inline]
     fn from(value: ReferenceValueLeaf<'a>) -> Self {

src/schema/field_type.rs

@@ -568,21 +568,21 @@ mod tests {
         let schema = schema_builder.build();
         let doc = TantivyDocument::parse_json(&schema, r#"{"id": 100}"#).unwrap();
         assert_eq!(
-            &OwnedValue::Str("100".to_string()),
-            doc.get_first(text_field).unwrap()
+            OwnedValue::Str("100".to_string()),
+            doc.get_first(text_field).unwrap().into()
         );
 
         let doc = TantivyDocument::parse_json(&schema, r#"{"id": true}"#).unwrap();
         assert_eq!(
-            &OwnedValue::Str("true".to_string()),
-            doc.get_first(text_field).unwrap()
+            OwnedValue::Str("true".to_string()),
+            doc.get_first(text_field).unwrap().into()
         );
 
         // Not sure if this null coercion is the best approach
         let doc = TantivyDocument::parse_json(&schema, r#"{"id": null}"#).unwrap();
         assert_eq!(
-            &OwnedValue::Str("null".to_string()),
-            doc.get_first(text_field).unwrap()
+            OwnedValue::Str("null".to_string()),
+            doc.get_first(text_field).unwrap().into()
         );
     }
 
@@ -595,9 +595,18 @@ mod tests {
         let schema = schema_builder.build();
         let doc_json = r#"{"i64": "100", "u64": "100", "f64": "100"}"#;
         let doc = TantivyDocument::parse_json(&schema, doc_json).unwrap();
-        assert_eq!(&OwnedValue::I64(100), doc.get_first(i64_field).unwrap());
-        assert_eq!(&OwnedValue::U64(100), doc.get_first(u64_field).unwrap());
-        assert_eq!(&OwnedValue::F64(100.0), doc.get_first(f64_field).unwrap());
+        assert_eq!(
+            OwnedValue::I64(100),
+            doc.get_first(i64_field).unwrap().into()
+        );
+        assert_eq!(
+            OwnedValue::U64(100),
+            doc.get_first(u64_field).unwrap().into()
+        );
+        assert_eq!(
+            OwnedValue::F64(100.0),
+            doc.get_first(f64_field).unwrap().into()
+        );
     }
 
     #[test]
@@ -607,11 +616,17 @@ mod tests {
         let schema = schema_builder.build();
         let doc_json = r#"{"bool": "true"}"#;
         let doc = TantivyDocument::parse_json(&schema, doc_json).unwrap();
-        assert_eq!(&OwnedValue::Bool(true), doc.get_first(bool_field).unwrap());
+        assert_eq!(
+            OwnedValue::Bool(true),
+            doc.get_first(bool_field).unwrap().into()
+        );
 
         let doc_json = r#"{"bool": "false"}"#;
         let doc = TantivyDocument::parse_json(&schema, doc_json).unwrap();
-        assert_eq!(&OwnedValue::Bool(false), doc.get_first(bool_field).unwrap());
+        assert_eq!(
+            OwnedValue::Bool(false),
+            doc.get_first(bool_field).unwrap().into()
+        );
     }
 
     #[test]
@@ -644,7 +659,7 @@ mod tests {
         let schema = schema_builder.build();
         let doc_json = r#"{"date": "2019-10-12T07:20:50.52+02:00"}"#;
         let doc = TantivyDocument::parse_json(&schema, doc_json).unwrap();
-        let date = doc.get_first(date_field).unwrap();
+        let date = OwnedValue::from(doc.get_first(date_field).unwrap());
         // Time zone is converted to UTC
         assert_eq!("Date(2019-10-12T05:20:50.52Z)", format!("{date:?}"));
     }

src/schema/field_value.rs

@@ -1,46 +0,0 @@
-use crate::schema::{Field, OwnedValue};
-
-/// `FieldValue` holds together a `Field` and its `Value`.
-#[allow(missing_docs)]
-#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
-pub struct FieldValue {
-    pub field: Field,
-    pub value: OwnedValue,
-}
-
-impl FieldValue {
-    /// Constructor
-    pub fn new(field: Field, value: OwnedValue) -> FieldValue {
-        FieldValue { field, value }
-    }
-
-    /// Field accessor
-    pub fn field(&self) -> Field {
-        self.field
-    }
-
-    /// Value accessor
-    pub fn value(&self) -> &OwnedValue {
-        &self.value
-    }
-}
-
-impl From<FieldValue> for OwnedValue {
-    fn from(field_value: FieldValue) -> Self {
-        field_value.value
-    }
-}
-
-/// A helper wrapper for creating standard iterators
-/// out of the fields iterator trait.
-pub struct FieldValueIter<'a>(pub(crate) std::slice::Iter<'a, FieldValue>);
-
-impl<'a> Iterator for FieldValueIter<'a> {
-    type Item = (Field, &'a OwnedValue);
-
-    fn next(&mut self) -> Option<Self::Item> {
-        self.0
-            .next()
-            .map(|field_value| (field_value.field, &field_value.value))
-    }
-}

src/schema/mod.rs

@@ -114,7 +114,6 @@ pub(crate) mod term;
 
 mod field_entry;
 mod field_type;
-mod field_value;
 
 mod bytes_options;
 mod date_time_options;
@@ -138,7 +137,6 @@ pub use self::facet_options::FacetOptions;
 pub use self::field::Field;
 pub use self::field_entry::FieldEntry;
 pub use self::field_type::{FieldType, Type};
-pub use self::field_value::FieldValue;
 pub use self::flags::{COERCE, FAST, INDEXED, STORED};
 pub use self::index_record_option::IndexRecordOption;
 pub use self::ip_options::{IntoIpv6Addr, IpAddrOptions};

src/schema/schema.rs

@@ -645,15 +645,15 @@ mod tests {
         let doc =
             TantivyDocument::convert_named_doc(&schema, NamedFieldDocument(named_doc_map)).unwrap();
         assert_eq!(
-            doc.get_all(title).collect::<Vec<_>>(),
+            doc.get_all(title).map(OwnedValue::from).collect::<Vec<_>>(),
             vec![
-                &OwnedValue::from("title1".to_string()),
-                &OwnedValue::from("title2".to_string())
+                OwnedValue::from("title1".to_string()),
+                OwnedValue::from("title2".to_string())
             ]
         );
         assert_eq!(
-            doc.get_all(val).collect::<Vec<_>>(),
-            vec![&OwnedValue::from(14u64), &OwnedValue::from(-1i64)]
+            doc.get_all(val).map(OwnedValue::from).collect::<Vec<_>>(),
+            vec![OwnedValue::from(14u64), OwnedValue::from(-1i64)]
         );
     }
 
@@ -682,7 +682,7 @@ mod tests {
         let schema = schema_builder.build();
         {
             let doc = TantivyDocument::parse_json(&schema, "{}").unwrap();
-            assert!(doc.field_values().is_empty());
+            assert!(doc.field_values().next().is_none());
         }
         {
             let doc = TantivyDocument::parse_json(

src/store/mod.rs

@@ -59,9 +59,8 @@ pub mod tests {
     use super::*;
     use crate::directory::{Directory, RamDirectory, WritePtr};
     use crate::fastfield::AliveBitSet;
-    use crate::schema::document::Value;
     use crate::schema::{
-        self, Schema, TantivyDocument, TextFieldIndexing, TextOptions, STORED, TEXT,
+        self, Schema, TantivyDocument, TextFieldIndexing, TextOptions, Value, STORED, TEXT,
     };
     use crate::{Index, IndexWriter, Term};
 
@@ -92,8 +91,8 @@ pub mod tests {
                 StoreWriter::new(writer, compressor, blocksize, separate_thread).unwrap();
             for i in 0..num_docs {
                 let mut doc = TantivyDocument::default();
-                doc.add_field_value(field_body, LOREM.to_string());
-                doc.add_field_value(field_title, format!("Doc {i}"));
+                doc.add_text(field_body, LOREM);
+                doc.add_text(field_title, format!("Doc {i}"));
                 store_writer.store(&doc, &schema).unwrap();
             }
             store_writer.close().unwrap();
@@ -119,10 +118,11 @@ pub mod tests {
         let store = StoreReader::open(store_file, 10)?;
         for i in 0..NUM_DOCS as u32 {
             assert_eq!(
-                *store
+                store
                     .get::<TantivyDocument>(i)?
                     .get_first(field_title)
                     .unwrap()
+                    .as_value()
                     .as_str()
                     .unwrap(),
                 format!("Doc {i}")
@@ -131,7 +131,13 @@ pub mod tests {
 
         for doc in store.iter::<TantivyDocument>(Some(&alive_bitset)) {
             let doc = doc?;
-            let title_content = doc.get_first(field_title).unwrap().as_str().unwrap();
+            let title_content = doc
+                .get_first(field_title)
+                .unwrap()
+                .as_value()
+                .as_str()
+                .unwrap()
+                .to_string();
             if !title_content.starts_with("Doc ") {
                 panic!("unexpected title_content {title_content}");
             }

src/store/reader.rs

@@ -403,8 +403,7 @@ mod tests {
 
     use super::*;
     use crate::directory::RamDirectory;
-    use crate::schema::document::Value;
-    use crate::schema::{Field, TantivyDocument};
+    use crate::schema::{Field, TantivyDocument, Value};
     use crate::store::tests::write_lorem_ipsum_store;
     use crate::store::Compressor;
     use crate::Directory;
@@ -412,7 +411,7 @@ mod tests {
     const BLOCK_SIZE: usize = 16_384;
 
     fn get_text_field<'a>(doc: &'a TantivyDocument, field: &'a Field) -> Option<&'a str> {
-        doc.get_first(*field).and_then(|f| f.as_str())
+        doc.get_first(*field).and_then(|f| f.as_value().as_str())
     }
 
     #[test]

src/termdict/fst_termdict/termdict.rs

@@ -93,7 +93,7 @@ fn open_fst_index(fst_file: FileSlice) -> io::Result<tantivy_fst::Map<OwnedBytes
     let fst = Fst::new(bytes).map_err(|err| {
         io::Error::new(
             io::ErrorKind::InvalidData,
-            format!("Fst data is corrupted: {:?}", err),
+            format!("Fst data is corrupted: {err:?}"),
         )
     })?;
     Ok(tantivy_fst::Map::from(fst))

src/termdict/tests.rs

@@ -95,7 +95,7 @@ fn test_term_dictionary_simple() -> crate::Result<()> {
 #[test]
 fn test_term_dictionary_stream() -> crate::Result<()> {
     let ids: Vec<_> = (0u32..10_000u32)
-        .map(|i| (format!("doc{:0>6}", i), i))
+        .map(|i| (format!("doc{i:0>6}"), i))
         .collect();
     let buffer: Vec<u8> = {
         let mut term_dictionary_builder = TermDictionaryBuilder::create(vec![]).unwrap();
@@ -156,7 +156,7 @@ fn test_stream_high_range_prefix_suffix() -> crate::Result<()> {
 #[test]
 fn test_stream_range() -> crate::Result<()> {
     let ids: Vec<_> = (0u32..10_000u32)
-        .map(|i| (format!("doc{:0>6}", i), i))
+        .map(|i| (format!("doc{i:0>6}"), i))
         .collect();
     let buffer: Vec<u8> = {
         let mut term_dictionary_builder = TermDictionaryBuilder::create(vec![]).unwrap();

src/tokenizer/facet_tokenizer.rs

@@ -96,7 +96,7 @@ mod tests {
         {
             let mut add_token = |token: &Token| {
                 let facet = Facet::from_encoded(token.text.as_bytes().to_owned()).unwrap();
-                tokens.push(format!("{}", facet));
+                tokens.push(format!("{facet}"));
             };
             FacetTokenizer::default()
                 .token_stream(facet.encoded_str())
@@ -116,7 +116,7 @@ mod tests {
         {
             let mut add_token = |token: &Token| {
                 let facet = Facet::from_encoded(token.text.as_bytes().to_owned()).unwrap(); // ok test
-                tokens.push(format!("{}", facet));
+                tokens.push(format!("{facet}"));
             };
             FacetTokenizer::default()
                 .token_stream(facet.encoded_str()) // ok test