persistor

Updates the requirements on [base64](https://github.com/marshallpierce/rust-base64) to permit the latest version.
- [Release notes](https://github.com/marshallpierce/rust-base64/releases)
- [Changelog](https://github.com/marshallpierce/rust-base64/blob/master/RELEASE-NOTES.md)
- [Commits](https://github.com/marshallpierce/rust-base64/compare/v0.11.0...v0.12.0)

Signed-off-by: dependabot-preview[bot] <support@dependabot.com>

Co-authored-by: dependabot-preview[bot] <27856297+dependabot-preview[bot]@users.noreply.github.com>

CHANGELOG.md

@@ -1,3 +1,18 @@
+Tantivy 0.12.0
+======================
+- Removing static dispatch in tokenizers for simplicity. (#762)
+- Added backward iteration for `TermDictionary` stream. (@halvorboe)
+- Fixed a performance issue when searching for the posting lists of a missing term (@audunhalland)
+- Added a configurable maximum number of docs (10M by default) for a segment to be considered for merge (@hntd187, landed by @halvorboe #713) 
+- Important Bugfix #777, causing tantivy to retain memory mapping. (diagnosed by @poljar)
+- Added support for field boosting. (#547, @fulmicoton)
+
+## How to update?
+
+Crates relying on custom tokenizer, or registering tokenizer in the manager will require some 
+minor changes. Check https://github.com/tantivy-search/tantivy/blob/master/examples/custom_tokenizer.rs
+to check for some code sample.
+
 Tantivy 0.11.3
 =======================
 - Fixed DateTime as a fast field (#735)

Cargo.toml

@@ -1,11 +1,11 @@
 [package]
 name = "tantivy"
-version = "0.11.3"
+version = "0.12.0"
 authors = ["Paul Masurel <paul.masurel@gmail.com>"]
 license = "MIT"
 categories = ["database-implementations", "data-structures"]
 description = """Search engine library"""
-documentation = "https://tantivy-search.github.io/tantivy/tantivy/index.html"
+documentation = "https://docs.rs/tantivy/"
 homepage = "https://github.com/tantivy-search/tantivy"
 repository = "https://github.com/tantivy-search/tantivy"
 readme = "README.md"
@@ -13,25 +13,23 @@ keywords = ["search", "information", "retrieval"]
 edition = "2018"
 
 [dependencies]
-base64 = "0.11.0"
+base64 = "0.12.0"
 byteorder = "1.0"
 crc32fast = "1.2.0"
 once_cell = "1.0"
 regex ={version = "1.3.0", default-features = false, features = ["std"]}
-tantivy-fst = "0.1"
+tantivy-fst = "0.2.1"
 memmap = {version = "0.7", optional=true}
 lz4 = {version="1.20", optional=true}
-snap = {version="0.2"}
+snap = "1"
 atomicwrites = {version="0.2.2", optional=true}
 tempfile = "3.0"
 log = "0.4"
-serde = "1.0"
-serde_derive = "1.0"
+serde = {version="1.0", features=["derive"]}
 serde_json = "1.0"
 num_cpus = "1.2"
 fs2={version="0.4", optional=true}
-itertools = "0.8"
-levenshtein_automata = {version="0.1", features=["fst_automaton"]}
+levenshtein_automata = "0.1"
 notify = {version="4", optional=true}
 uuid = { version = "0.8", features = ["v4", "serde"] }
 crossbeam = "0.7"
@@ -40,9 +38,9 @@ owning_ref = "0.4"
 stable_deref_trait = "1.0.0"
 rust-stemmers = "1.2"
 downcast-rs = { version="1.0" }
-tantivy-query-grammar = { version="0.11", path="./query-grammar" }
+tantivy-query-grammar = { version="0.12", path="./query-grammar" }
 bitpacking = {version="0.8", default-features = false, features=["bitpacker4x"]}
-census = "0.4"
+census =  {path="../census"}
 fnv = "1.0.6"
 owned-read = "0.4"
 failure = "0.1"
@@ -60,7 +58,6 @@ winapi = "0.3"
 rand = "0.7"
 maplit = "1"
 matches = "0.1.8"
-time = "0.1.42"
 
 [dev-dependencies.fail]
 version = "0.3"

README.md

@@ -40,7 +40,7 @@ performance for different type of queries / collection.
 # Features
 
 - Full-text search
-- Configurable tokenizer (stemming available for 17 Latin languages with third party support for Chinese ([tantivy-jieba](https://crates.io/crates/tantivy-jieba) and [cang-jie](https://crates.io/crates/cang-jie)) and [Japanese](https://crates.io/crates/tantivy-tokenizer-tiny-segmenter))
+- Configurable tokenizer (stemming available for 17 Latin languages with third party support for Chinese ([tantivy-jieba](https://crates.io/crates/tantivy-jieba) and [cang-jie](https://crates.io/crates/cang-jie)), Japanese ([lindera](https://github.com/lindera-morphology/lindera-tantivy) and [tantivy-tokenizer-tiny-segmente](https://crates.io/crates/tantivy-tokenizer-tiny-segmenter)) and Korean ([lindera](https://github.com/lindera-morphology/lindera-tantivy) + [lindera-ko-dic-builder](https://github.com/lindera-morphology/lindera-ko-dic-builder))
 - Fast (check out the :racehorse: :sparkles: [benchmark](https://tantivy-search.github.io/bench/) :sparkles: :racehorse:)
 - Tiny startup time (<10ms), perfect for command line tools
 - BM25 scoring (the same as Lucene)
@@ -59,18 +59,17 @@ performance for different type of queries / collection.
 - Configurable indexing (optional term frequency and position indexing)
 - Cheesy logo with a horse
 
-# Non-features
+## Non-features
 
 - Distributed search is out of the scope of Tantivy. That being said, Tantivy is a
 library upon which one could build a distributed search. Serializable/mergeable collector state for instance, 
 are within the scope of Tantivy.
 
-# Supported OS and compiler
-
-Tantivy works on stable Rust (>= 1.27) and supports Linux, MacOS, and Windows.
 
 # Getting started
 
+Tantivy works on stable Rust (>= 1.27) and supports Linux, MacOS, and Windows.
+
 - [Tantivy's simple search example](https://tantivy-search.github.io/examples/basic_search.html)
 - [tantivy-cli and its tutorial](https://github.com/tantivy-search/tantivy-cli) - `tantivy-cli` is an actual command line interface that makes it easy for you to create a search engine,
 index documents, and search via the CLI or a small server with a REST API.

TODO.md

@@ -0,0 +1,9 @@
+- segment writer serializes if already flush
+- flush if exceed max doc. 
+- unit tests
+- reader from `IndexWriter`
+- configurable limit to flush 
+- bundle directory
+- add index worker when exceeds some doc limit
+- flush before prepare commit.
+- segment_writer should not receive a segment 

examples/faceted_search.rs

@@ -13,63 +13,100 @@
 // ---
 // Importing tantivy...
 use tantivy::collector::FacetCollector;
-use tantivy::query::AllQuery;
+use tantivy::query::{AllQuery, TermQuery};
 use tantivy::schema::*;
 use tantivy::{doc, Index};
-use tempfile::TempDir;
 
 fn main() -> tantivy::Result<()> {
-    // Let's create a temporary directory for the
-    // sake of this example
-    let index_path = TempDir::new()?;
+    // Let's create a temporary directory for the sake of this example
     let mut schema_builder = Schema::builder();
 
-    schema_builder.add_text_field("name", TEXT | STORED);
-
-    // this is our faceted field
-    schema_builder.add_facet_field("tags");
+    let name = schema_builder.add_text_field("felin_name", TEXT | STORED);
+    // this is our faceted field: its scientific classification
+    let classification = schema_builder.add_facet_field("classification");
 
     let schema = schema_builder.build();
+    let index = Index::create_in_ram(schema);
 
-    let index = Index::create_in_dir(&index_path, schema.clone())?;
-
-    let mut index_writer = index.writer(50_000_000)?;
-
-    let name = schema.get_field("name").unwrap();
-    let tags = schema.get_field("tags").unwrap();
+    let mut index_writer = index.writer(30_000_000)?;
 
     // For convenience, tantivy also comes with a macro to
     // reduce the boilerplate above.
     index_writer.add_document(doc!(
-        name => "the ditch",
-        tags => Facet::from("/pools/north")
+        name => "Cat",
+        classification => Facet::from("/Felidae/Felinae/Felis")
     ));
-
     index_writer.add_document(doc!(
-        name => "little stacey",
-        tags => Facet::from("/pools/south")
+        name => "Canada lynx",
+        classification => Facet::from("/Felidae/Felinae/Lynx")
+    ));
+    index_writer.add_document(doc!(
+        name => "Cheetah",
+        classification => Facet::from("/Felidae/Felinae/Acinonyx")
+    ));
+    index_writer.add_document(doc!(
+        name => "Tiger",
+        classification => Facet::from("/Felidae/Pantherinae/Panthera")
+    ));
+    index_writer.add_document(doc!(
+        name => "Lion",
+        classification => Facet::from("/Felidae/Pantherinae/Panthera")
+    ));
+    index_writer.add_document(doc!(
+        name => "Jaguar",
+        classification => Facet::from("/Felidae/Pantherinae/Panthera")
+    ));
+    index_writer.add_document(doc!(
+        name => "Sunda clouded leopard",
+        classification => Facet::from("/Felidae/Pantherinae/Neofelis")
+    ));
+    index_writer.add_document(doc!(
+        name => "Fossa",
+        classification => Facet::from("/Eupleridae/Cryptoprocta")
     ));
-
     index_writer.commit()?;
 
     let reader = index.reader()?;
-
     let searcher = reader.searcher();
+    {
+        let mut facet_collector = FacetCollector::for_field(classification);
+        facet_collector.add_facet("/Felidae");
+        let facet_counts = searcher.search(&AllQuery, &facet_collector)?;
+        // This lists all of the facet counts, right below "/Felidae".
+        let facets: Vec<(&Facet, u64)> = facet_counts.get("/Felidae").collect();
+        assert_eq!(
+            facets,
+            vec![
+                (&Facet::from("/Felidae/Felinae"), 3),
+                (&Facet::from("/Felidae/Pantherinae"), 4),
+            ]
+        );
+    }
 
-    let mut facet_collector = FacetCollector::for_field(tags);
-    facet_collector.add_facet("/pools");
+    // Facets are also searchable.
+    //
+    // For instance a common UI pattern is to allow the user someone to click on a facet link
+    // (e.g: `Pantherinae`) to drill down and filter the current result set with this subfacet.
+    //
+    // The search would then look as follows.
 
-    let facet_counts = searcher.search(&AllQuery, &facet_collector).unwrap();
-
-    // This lists all of the facet counts
-    let facets: Vec<(&Facet, u64)> = facet_counts.get("/pools").collect();
-    assert_eq!(
-        facets,
-        vec![
-            (&Facet::from("/pools/north"), 1),
-            (&Facet::from("/pools/south"), 1),
-        ]
-    );
+    // Check the reference doc for different ways to create a `Facet` object.
+    {
+        let facet = Facet::from_text("/Felidae/Pantherinae");
+        let facet_term = Term::from_facet(classification, &facet);
+        let facet_term_query = TermQuery::new(facet_term, IndexRecordOption::Basic);
+        let mut facet_collector = FacetCollector::for_field(classification);
+        facet_collector.add_facet("/Felidae/Pantherinae");
+        let facet_counts = searcher.search(&facet_term_query, &facet_collector)?;
+        let facets: Vec<(&Facet, u64)> = facet_counts.get("/Felidae/Pantherinae").collect();
+        assert_eq!(
+            facets,
+            vec![
+                (&Facet::from("/Felidae/Pantherinae/Neofelis"), 1),
+                (&Facet::from("/Felidae/Pantherinae/Panthera"), 3),
+            ]
+        );
+    }
 
     Ok(())
 }

examples/pre_tokenized_text.rs

@@ -9,11 +9,10 @@
 // - import tokenized text straight from json,
 // - perform a search on documents with pre-tokenized text
 
-use tantivy::tokenizer::{PreTokenizedString, SimpleTokenizer, Token, TokenStream, Tokenizer};
-
 use tantivy::collector::{Count, TopDocs};
 use tantivy::query::TermQuery;
 use tantivy::schema::*;
+use tantivy::tokenizer::{PreTokenizedString, SimpleTokenizer, Token, Tokenizer};
 use tantivy::{doc, Index, ReloadPolicy};
 use tempfile::TempDir;
 

examples/stop_words.rs

@@ -50,7 +50,7 @@ fn main() -> tantivy::Result<()> {
 
     // This tokenizer lowers all of the text (to help with stop word matching)
     // then removes all instances of `the` and `and` from the corpus
-    let tokenizer = SimpleTokenizer
+    let tokenizer = TextAnalyzer::from(SimpleTokenizer)
         .filter(LowerCaser)
         .filter(StopWordFilter::remove(vec![
             "the".to_string(),

query-grammar/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "tantivy-query-grammar"
-version = "0.11.0"
+version = "0.12.0"
 authors = ["Paul Masurel <paul.masurel@gmail.com>"]
 license = "MIT"
 categories = ["database-implementations", "data-structures"]
@@ -13,4 +13,4 @@ keywords = ["search", "information", "retrieval"]
 edition = "2018"
 
 [dependencies]
-combine = ">=3.6.0,<4.0.0"
+combine = {version="4", default-features=false, features=[] }

query-grammar/src/lib.rs

@@ -1,5 +1,3 @@
-#![recursion_limit = "100"]
-
 mod occur;
 mod query_grammar;
 mod user_input_ast;

query-grammar/src/query_grammar.rs

@@ -1,150 +1,153 @@
-use super::user_input_ast::*;
+use super::user_input_ast::{UserInputAST, UserInputBound, UserInputLeaf, UserInputLiteral};
 use crate::Occur;
-use combine::char::*;
-use combine::error::StreamError;
-use combine::stream::StreamErrorFor;
-use combine::*;
+use combine::error::StringStreamError;
+use combine::parser::char::{char, digit, letter, space, spaces, string};
+use combine::parser::Parser;
+use combine::{
+    attempt, choice, eof, many, many1, one_of, optional, parser, satisfy, skip_many1, value,
+};
 
-parser! {
-    fn field[I]()(I) -> String
-    where [I: Stream<Item = char>] {
-        (
-            letter(),
-            many(satisfy(|c: char| c.is_alphanumeric() || c == '_')),
-        ).skip(char(':')).map(|(s1, s2): (char, String)| format!("{}{}", s1, s2))
-    }
-}
-
-parser! {
-    fn word[I]()(I) -> String
-    where [I: Stream<Item = char>] {
-        (
-            satisfy(|c: char| !c.is_whitespace() && !['-', '`', ':', '{', '}', '"', '[', ']', '(',')'].contains(&c) ),
-            many(satisfy(|c: char| !c.is_whitespace() && ![':', '{', '}', '"', '[', ']', '(',')'].contains(&c)))
-        )
+fn field<'a>() -> impl Parser<&'a str, Output = String> {
+    (
+        letter(),
+        many(satisfy(|c: char| c.is_alphanumeric() || c == '_')),
+    )
+        .skip(char(':'))
         .map(|(s1, s2): (char, String)| format!("{}{}", s1, s2))
-        .and_then(|s: String|
-           match s.as_str() {
-             "OR" => Err(StreamErrorFor::<I>::unexpected_static_message("OR")),
-             "AND" => Err(StreamErrorFor::<I>::unexpected_static_message("AND")),
-             "NOT" => Err(StreamErrorFor::<I>::unexpected_static_message("NOT")),
-             _ => Ok(s)
-           })
-    }
 }
 
-parser! {
-    fn literal[I]()(I) -> UserInputLeaf
-    where [I: Stream<Item = char>]
-    {
-        let term_val = || {
-            let phrase = char('"').with(many1(satisfy(|c| c != '"'))).skip(char('"'));
-            phrase.or(word())
-        };
-        let term_val_with_field = negative_number().or(term_val());
-        let term_query =
-            (field(), term_val_with_field)
-            .map(|(field_name, phrase)| UserInputLiteral {
-                field_name: Some(field_name),
-                phrase,
-            });
-        let term_default_field = term_val().map(|phrase| UserInputLiteral {
-            field_name: None,
-            phrase,
-        });
-        attempt(term_query)
-            .or(term_default_field)
-            .map(UserInputLeaf::from)
-    }
-}
-
-parser! {
-    fn negative_number[I]()(I) -> String
-    where [I: Stream<Item = char>]
-    {
-        (char('-'), many1(satisfy(char::is_numeric)),
-         optional((char('.'), many1(satisfy(char::is_numeric)))))
-            .map(|(s1, s2, s3): (char, String, Option<(char, String)>)| {
-                if let Some(('.', s3)) = s3 {
-                    format!("{}{}.{}", s1, s2, s3)
-                } else {
-                    format!("{}{}", s1, s2)
-                }
-            })
-    }
-}
-
-parser! {
-    fn spaces1[I]()(I) -> ()
-    where [I: Stream<Item = char>] {
-        skip_many1(space())
-    }
-}
-
-parser! {
-    /// Function that parses a range out of a Stream
-    /// Supports ranges like:
-    /// [5 TO 10], {5 TO 10}, [* TO 10], [10 TO *], {10 TO *], >5, <=10
-    /// [a TO *], [a TO c], [abc TO bcd}
-    fn range[I]()(I) -> UserInputLeaf
-    where [I: Stream<Item = char>] {
-        let range_term_val = || {
-            word().or(negative_number()).or(char('*').with(value("*".to_string())))
-        };
-
-        // check for unbounded range in the form of <5, <=10, >5, >=5
-        let elastic_unbounded_range = (choice([attempt(string(">=")),
-                                               attempt(string("<=")),
-                                               attempt(string("<")),
-                                               attempt(string(">"))])
-                                       .skip(spaces()),
-                                       range_term_val()).
-            map(|(comparison_sign, bound): (&str, String)|
-                match comparison_sign {
-                    ">=" => (UserInputBound::Inclusive(bound), UserInputBound::Unbounded),
-                    "<=" => (UserInputBound::Unbounded, UserInputBound::Inclusive(bound)),
-                    "<" => (UserInputBound::Unbounded, UserInputBound::Exclusive(bound)),
-                    ">" => (UserInputBound::Exclusive(bound), UserInputBound::Unbounded),
-                    // default case
-                    _ => (UserInputBound::Unbounded, UserInputBound::Unbounded)
-                });
-        let lower_bound = (one_of("{[".chars()), range_term_val())
-            .map(|(boundary_char, lower_bound): (char, String)|
-                 if lower_bound == "*" {
-                     UserInputBound::Unbounded
-                 } else if boundary_char == '{' {
-                         UserInputBound::Exclusive(lower_bound)
-                 } else {
-                     UserInputBound::Inclusive(lower_bound)
-                 });
-        let upper_bound = (range_term_val(), one_of("}]".chars()))
-            .map(|(higher_bound, boundary_char): (String, char)|
-                 if higher_bound == "*" {
-                     UserInputBound::Unbounded
-                 } else if boundary_char == '}' {
-                     UserInputBound::Exclusive(higher_bound)
-                 } else {
-                     UserInputBound::Inclusive(higher_bound)
-                 });
-         // return only lower and upper
-        let lower_to_upper = (lower_bound.
-                                    skip((spaces(),
-                                          string("TO"),
-                                          spaces())),
-                                    upper_bound);
-
-        (optional(field()).skip(spaces()),
-         // try elastic first, if it matches, the range is unbounded
-         attempt(elastic_unbounded_range).or(lower_to_upper))
-            .map(|(field, (lower, upper))|
-                 // Construct the leaf from extracted field (optional)
-                 // and bounds
-                 UserInputLeaf::Range {
-                     field,
-                     lower,
-                     upper
+fn word<'a>() -> impl Parser<&'a str, Output = String> {
+    (
+        satisfy(|c: char| {
+            !c.is_whitespace()
+                && !['-', '^', '`', ':', '{', '}', '"', '[', ']', '(', ')'].contains(&c)
+        }),
+        many(satisfy(|c: char| {
+            !c.is_whitespace() && ![':', '^', '{', '}', '"', '[', ']', '(', ')'].contains(&c)
+        })),
+    )
+        .map(|(s1, s2): (char, String)| format!("{}{}", s1, s2))
+        .and_then(|s: String| match s.as_str() {
+            "OR" | "AND " | "NOT" => Err(StringStreamError::UnexpectedParse),
+            _ => Ok(s),
         })
-    }
+}
+
+fn term_val<'a>() -> impl Parser<&'a str, Output = String> {
+    let phrase = char('"').with(many1(satisfy(|c| c != '"'))).skip(char('"'));
+    phrase.or(word())
+}
+
+fn term_query<'a>() -> impl Parser<&'a str, Output = UserInputLiteral> {
+    let term_val_with_field = negative_number().or(term_val());
+    (field(), term_val_with_field).map(|(field_name, phrase)| UserInputLiteral {
+        field_name: Some(field_name),
+        phrase,
+    })
+}
+
+fn literal<'a>() -> impl Parser<&'a str, Output = UserInputLeaf> {
+    let term_default_field = term_val().map(|phrase| UserInputLiteral {
+        field_name: None,
+        phrase,
+    });
+    attempt(term_query())
+        .or(term_default_field)
+        .map(UserInputLeaf::from)
+}
+
+fn negative_number<'a>() -> impl Parser<&'a str, Output = String> {
+    (
+        char('-'),
+        many1(digit()),
+        optional((char('.'), many1(digit()))),
+    )
+        .map(|(s1, s2, s3): (char, String, Option<(char, String)>)| {
+            if let Some(('.', s3)) = s3 {
+                format!("{}{}.{}", s1, s2, s3)
+            } else {
+                format!("{}{}", s1, s2)
+            }
+        })
+}
+
+fn spaces1<'a>() -> impl Parser<&'a str, Output = ()> {
+    skip_many1(space())
+}
+
+/// Function that parses a range out of a Stream
+/// Supports ranges like:
+/// [5 TO 10], {5 TO 10}, [* TO 10], [10 TO *], {10 TO *], >5, <=10
+/// [a TO *], [a TO c], [abc TO bcd}
+fn range<'a>() -> impl Parser<&'a str, Output = UserInputLeaf> {
+    let range_term_val = || {
+        word()
+            .or(negative_number())
+            .or(char('*').with(value("*".to_string())))
+    };
+
+    // check for unbounded range in the form of <5, <=10, >5, >=5
+    let elastic_unbounded_range = (
+        choice([
+            attempt(string(">=")),
+            attempt(string("<=")),
+            attempt(string("<")),
+            attempt(string(">")),
+        ])
+        .skip(spaces()),
+        range_term_val(),
+    )
+        .map(
+            |(comparison_sign, bound): (&str, String)| match comparison_sign {
+                ">=" => (UserInputBound::Inclusive(bound), UserInputBound::Unbounded),
+                "<=" => (UserInputBound::Unbounded, UserInputBound::Inclusive(bound)),
+                "<" => (UserInputBound::Unbounded, UserInputBound::Exclusive(bound)),
+                ">" => (UserInputBound::Exclusive(bound), UserInputBound::Unbounded),
+                // default case
+                _ => (UserInputBound::Unbounded, UserInputBound::Unbounded),
+            },
+        );
+    let lower_bound = (one_of("{[".chars()), range_term_val()).map(
+        |(boundary_char, lower_bound): (char, String)| {
+            if lower_bound == "*" {
+                UserInputBound::Unbounded
+            } else if boundary_char == '{' {
+                UserInputBound::Exclusive(lower_bound)
+            } else {
+                UserInputBound::Inclusive(lower_bound)
+            }
+        },
+    );
+    let upper_bound = (range_term_val(), one_of("}]".chars())).map(
+        |(higher_bound, boundary_char): (String, char)| {
+            if higher_bound == "*" {
+                UserInputBound::Unbounded
+            } else if boundary_char == '}' {
+                UserInputBound::Exclusive(higher_bound)
+            } else {
+                UserInputBound::Inclusive(higher_bound)
+            }
+        },
+    );
+    // return only lower and upper
+    let lower_to_upper = (
+        lower_bound.skip((spaces(), string("TO"), spaces())),
+        upper_bound,
+    );
+
+    (
+        optional(field()).skip(spaces()),
+        // try elastic first, if it matches, the range is unbounded
+        attempt(elastic_unbounded_range).or(lower_to_upper),
+    )
+        .map(|(field, (lower, upper))|
+             // Construct the leaf from extracted field (optional)
+             // and bounds
+             UserInputLeaf::Range {
+                 field,
+                 lower,
+                 upper
+    })
 }
 
 fn negate(expr: UserInputAST) -> UserInputAST {
@@ -155,17 +158,48 @@ fn must(expr: UserInputAST) -> UserInputAST {
     expr.unary(Occur::Must)
 }
 
-parser! {
-    fn leaf[I]()(I) -> UserInputAST
-    where [I: Stream<Item = char>] {
-            char('-').with(leaf()).map(negate)
-        .or(char('+').with(leaf()).map(must))
-        .or(char('(').with(ast()).skip(char(')')))
-        .or(char('*').map(|_| UserInputAST::from(UserInputLeaf::All)))
-        .or(attempt(string("NOT").skip(spaces1()).with(leaf()).map(negate)))
-        .or(attempt(range().map(UserInputAST::from)))
-        .or(literal().map(UserInputAST::from))
-    }
+fn leaf<'a>() -> impl Parser<&'a str, Output = UserInputAST> {
+    parser(|input| {
+        char('-')
+            .with(leaf())
+            .map(negate)
+            .or(char('+').with(leaf()).map(must))
+            .or(char('(').with(ast()).skip(char(')')))
+            .or(char('*').map(|_| UserInputAST::from(UserInputLeaf::All)))
+            .or(attempt(
+                string("NOT").skip(spaces1()).with(leaf()).map(negate),
+            ))
+            .or(attempt(range().map(UserInputAST::from)))
+            .or(literal().map(UserInputAST::from))
+            .parse_stream(input)
+            .into_result()
+    })
+}
+
+fn positive_float_number<'a>() -> impl Parser<&'a str, Output = f32> {
+    (many1(digit()), optional((char('.'), many1(digit())))).map(
+        |(int_part, decimal_part_opt): (String, Option<(char, String)>)| {
+            let mut float_str = int_part;
+            if let Some((chr, decimal_str)) = decimal_part_opt {
+                float_str.push(chr);
+                float_str.push_str(&decimal_str);
+            }
+            float_str.parse::<f32>().unwrap()
+        },
+    )
+}
+
+fn boost<'a>() -> impl Parser<&'a str, Output = f32> {
+    (char('^'), positive_float_number()).map(|(_, boost)| boost)
+}
+
+fn boosted_leaf<'a>() -> impl Parser<&'a str, Output = UserInputAST> {
+    (leaf(), optional(boost())).map(|(leaf, boost_opt)| match boost_opt {
+        Some(boost) if (boost - 1.0).abs() > std::f32::EPSILON => {
+            UserInputAST::Boost(Box::new(leaf), boost)
+        }
+        _ => leaf,
+    })
 }
 
 #[derive(Clone, Copy)]
@@ -174,13 +208,10 @@ enum BinaryOperand {
     And,
 }
 
-parser! {
-    fn binary_operand[I]()(I) -> BinaryOperand
-    where [I: Stream<Item = char>]
-    {
-       string("AND").with(value(BinaryOperand::And))
-       .or(string("OR").with(value(BinaryOperand::Or)))
-    }
+fn binary_operand<'a>() -> impl Parser<&'a str, Output = BinaryOperand> {
+    string("AND")
+        .with(value(BinaryOperand::And))
+        .or(string("OR").with(value(BinaryOperand::Or)))
 }
 
 fn aggregate_binary_expressions(
@@ -208,37 +239,67 @@ fn aggregate_binary_expressions(
     }
 }
 
-parser! {
-    pub fn ast[I]()(I) -> UserInputAST
-    where [I: Stream<Item = char>]
-    {
-        let operand_leaf = (binary_operand().skip(spaces()), leaf().skip(spaces()));
-        let boolean_expr = (leaf().skip(spaces().silent()), many1(operand_leaf)).map(
-            |(left, right)| aggregate_binary_expressions(left,right));
-        let whitespace_separated_leaves = many1(leaf().skip(spaces().silent()))
-        .map(|subqueries: Vec<UserInputAST>|
+pub fn ast<'a>() -> impl Parser<&'a str, Output = UserInputAST> {
+    let operand_leaf = (
+        binary_operand().skip(spaces()),
+        boosted_leaf().skip(spaces()),
+    );
+    let boolean_expr = (boosted_leaf().skip(spaces().silent()), many1(operand_leaf))
+        .map(|(left, right)| aggregate_binary_expressions(left, right));
+    let whitespace_separated_leaves =
+        many1(boosted_leaf().skip(spaces().silent())).map(|subqueries: Vec<UserInputAST>| {
             if subqueries.len() == 1 {
                 subqueries.into_iter().next().unwrap()
             } else {
                 UserInputAST::Clause(subqueries.into_iter().collect())
-            });
-        let expr = attempt(boolean_expr).or(whitespace_separated_leaves);
-        spaces().with(expr).skip(spaces())
-    }
+            }
+        });
+    let expr = attempt(boolean_expr).or(whitespace_separated_leaves);
+    spaces().with(expr).skip(spaces())
 }
 
-parser! {
-    pub fn parse_to_ast[I]()(I) -> UserInputAST
-    where [I: Stream<Item = char>]
-    {
-        spaces().with(optional(ast()).skip(eof())).map(|opt_ast| opt_ast.unwrap_or_else(UserInputAST::empty_query))
-    }
+pub fn parse_to_ast<'a>() -> impl Parser<&'a str, Output = UserInputAST> {
+    spaces()
+        .with(optional(ast()).skip(eof()))
+        .map(|opt_ast| opt_ast.unwrap_or_else(UserInputAST::empty_query))
 }
 
 #[cfg(test)]
 mod test {
 
     use super::*;
+    use combine::parser::Parser;
+
+    pub fn nearly_equals(a: f32, b: f32) -> bool {
+        (a - b).abs() < 0.0005 * (a + b).abs()
+    }
+
+    fn assert_nearly_equals(expected: f32, val: f32) {
+        assert!(
+            nearly_equals(val, expected),
+            "Got {}, expected {}.",
+            val,
+            expected
+        );
+    }
+
+    #[test]
+    fn test_positive_float_number() {
+        fn valid_parse(float_str: &str, expected_val: f32, expected_remaining: &str) {
+            let (val, remaining) = positive_float_number().parse(float_str).unwrap();
+            assert_eq!(remaining, expected_remaining);
+            assert_nearly_equals(val, expected_val);
+        }
+        fn error_parse(float_str: &str) {
+            assert!(positive_float_number().parse(float_str).is_err());
+        }
+        valid_parse("1.0", 1.0f32, "");
+        valid_parse("1", 1.0f32, "");
+        valid_parse("0.234234 aaa", 0.234234f32, " aaa");
+        error_parse(".3332");
+        error_parse("1.");
+        error_parse("-1.");
+    }
 
     fn test_parse_query_to_ast_helper(query: &str, expected: &str) {
         let query = parse_to_ast().parse(query).unwrap().0;
@@ -272,6 +333,15 @@ mod test {
         test_parse_query_to_ast_helper("NOT a", "-(\"a\")");
     }
 
+    #[test]
+    fn test_boosting() {
+        assert!(parse_to_ast().parse("a^2^3").is_err());
+        assert!(parse_to_ast().parse("a^2^").is_err());
+        test_parse_query_to_ast_helper("a^3", "(\"a\")^3");
+        test_parse_query_to_ast_helper("a^3 b^2", "((\"a\")^3 (\"b\")^2)");
+        test_parse_query_to_ast_helper("a^1", "\"a\"");
+    }
+
     #[test]
     fn test_parse_query_to_ast_binary_op() {
         test_parse_query_to_ast_helper("a AND b", "(+(\"a\") +(\"b\"))");

query-grammar/src/user_input_ast.rs

@@ -88,6 +88,7 @@ pub enum UserInputAST {
     Clause(Vec<UserInputAST>),
     Unary(Occur, Box<UserInputAST>),
     Leaf(Box<UserInputLeaf>),
+    Boost(Box<UserInputAST>, f32),
 }
 
 impl UserInputAST {
@@ -154,6 +155,7 @@ impl fmt::Debug for UserInputAST {
                 write!(formatter, "{}({:?})", occur, subquery)
             }
             UserInputAST::Leaf(ref subquery) => write!(formatter, "{:?}", subquery),
+            UserInputAST::Boost(ref leaf, boost) => write!(formatter, "({:?})^{}", leaf, boost),
         }
     }
 }

src/collector/count_collector.rs

@@ -1,7 +1,6 @@
 use super::Collector;
 use crate::collector::SegmentCollector;
 use crate::DocId;
-use crate::Result;
 use crate::Score;
 use crate::SegmentLocalId;
 use crate::SegmentReader;
@@ -44,7 +43,11 @@ impl Collector for Count {
 
     type Child = SegmentCountCollector;
 
-    fn for_segment(&self, _: SegmentLocalId, _: &SegmentReader) -> Result<SegmentCountCollector> {
+    fn for_segment(
+        &self,
+        _: SegmentLocalId,
+        _: &SegmentReader,
+    ) -> crate::Result<SegmentCountCollector> {
         Ok(SegmentCountCollector::default())
     }
 
@@ -52,7 +55,7 @@ impl Collector for Count {
         false
     }
 
-    fn merge_fruits(&self, segment_counts: Vec<usize>) -> Result<usize> {
+    fn merge_fruits(&self, segment_counts: Vec<usize>) -> crate::Result<usize> {
         Ok(segment_counts.into_iter().sum())
     }
 }

src/collector/custom_score_top_collector.rs

@@ -1,6 +1,5 @@
 use crate::collector::top_collector::{TopCollector, TopSegmentCollector};
 use crate::collector::{Collector, SegmentCollector};
-use crate::Result;
 use crate::{DocAddress, DocId, Score, SegmentReader};
 
 pub(crate) struct CustomScoreTopCollector<TCustomScorer, TScore = Score> {
@@ -42,7 +41,7 @@ pub trait CustomScorer<TScore>: Sync {
     type Child: CustomSegmentScorer<TScore>;
     /// Builds a child scorer for a specific segment. The child scorer is associated to
     /// a specific segment.
-    fn segment_scorer(&self, segment_reader: &SegmentReader) -> Result<Self::Child>;
+    fn segment_scorer(&self, segment_reader: &SegmentReader) -> crate::Result<Self::Child>;
 }
 
 impl<TCustomScorer, TScore> Collector for CustomScoreTopCollector<TCustomScorer, TScore>
@@ -58,7 +57,7 @@ where
         &self,
         segment_local_id: u32,
         segment_reader: &SegmentReader,
-    ) -> Result<Self::Child> {
+    ) -> crate::Result<Self::Child> {
         let segment_scorer = self.custom_scorer.segment_scorer(segment_reader)?;
         let segment_collector = self
             .collector
@@ -73,7 +72,7 @@ where
         false
     }
 
-    fn merge_fruits(&self, segment_fruits: Vec<Self::Fruit>) -> Result<Self::Fruit> {
+    fn merge_fruits(&self, segment_fruits: Vec<Self::Fruit>) -> crate::Result<Self::Fruit> {
         self.collector.merge_fruits(segment_fruits)
     }
 }
@@ -111,7 +110,7 @@ where
 {
     type Child = T;
 
-    fn segment_scorer(&self, segment_reader: &SegmentReader) -> Result<Self::Child> {
+    fn segment_scorer(&self, segment_reader: &SegmentReader) -> crate::Result<Self::Child> {
         Ok((self)(segment_reader))
     }
 }

src/collector/facet_collector.rs

@@ -5,7 +5,6 @@ use crate::fastfield::FacetReader;
 use crate::schema::Facet;
 use crate::schema::Field;
 use crate::DocId;
-use crate::Result;
 use crate::Score;
 use crate::SegmentLocalId;
 use crate::SegmentReader;
@@ -84,9 +83,9 @@ fn facet_depth(facet_bytes: &[u8]) -> usize {
 /// use tantivy::collector::FacetCollector;
 /// use tantivy::query::AllQuery;
 /// use tantivy::schema::{Facet, Schema, TEXT};
-/// use tantivy::{doc, Index, Result};
+/// use tantivy::{doc, Index};
 ///
-/// fn example() -> Result<()> {
+/// fn example() -> tantivy::Result<()> {
 ///     let mut schema_builder = Schema::builder();
 ///
 ///     // Facet have their own specific type.
@@ -262,7 +261,7 @@ impl Collector for FacetCollector {
         &self,
         _: SegmentLocalId,
         reader: &SegmentReader,
-    ) -> Result<FacetSegmentCollector> {
+    ) -> crate::Result<FacetSegmentCollector> {
         let field_name = reader.schema().get_field_name(self.field);
         let facet_reader = reader.facet_reader(self.field).ok_or_else(|| {
             TantivyError::SchemaError(format!("Field {:?} is not a facet field.", field_name))
@@ -328,7 +327,7 @@ impl Collector for FacetCollector {
         false
     }
 
-    fn merge_fruits(&self, segments_facet_counts: Vec<FacetCounts>) -> Result<FacetCounts> {
+    fn merge_fruits(&self, segments_facet_counts: Vec<FacetCounts>) -> crate::Result<FacetCounts> {
         let mut facet_counts: BTreeMap<Facet, u64> = BTreeMap::new();
         for segment_facet_counts in segments_facet_counts {
             for (facet, count) in segment_facet_counts.facet_counts {
@@ -452,9 +451,11 @@ impl FacetCounts {
 #[cfg(test)]
 mod tests {
     use super::{FacetCollector, FacetCounts};
+    use crate::collector::Count;
     use crate::core::Index;
-    use crate::query::AllQuery;
-    use crate::schema::{Document, Facet, Field, Schema};
+    use crate::query::{AllQuery, QueryParser, TermQuery};
+    use crate::schema::{Document, Facet, Field, IndexRecordOption, Schema};
+    use crate::Term;
     use rand::distributions::Uniform;
     use rand::prelude::SliceRandom;
     use rand::{thread_rng, Rng};
@@ -544,6 +545,56 @@ mod tests {
         assert_eq!(facets[0].1, 1);
     }
 
+    #[test]
+    fn test_doc_search_by_facet() {
+        let mut schema_builder = Schema::builder();
+        let facet_field = schema_builder.add_facet_field("facet");
+        let schema = schema_builder.build();
+        let index = Index::create_in_ram(schema);
+        let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
+        index_writer.add_document(doc!(
+            facet_field => Facet::from_text(&"/A/A"),
+        ));
+        index_writer.add_document(doc!(
+            facet_field => Facet::from_text(&"/A/B"),
+        ));
+        index_writer.add_document(doc!(
+            facet_field => Facet::from_text(&"/A/C/A"),
+        ));
+        index_writer.add_document(doc!(
+            facet_field => Facet::from_text(&"/D/C/A"),
+        ));
+        index_writer.commit().unwrap();
+        let reader = index.reader().unwrap();
+        let searcher = reader.searcher();
+        assert_eq!(searcher.num_docs(), 4);
+
+        let count_facet = |facet_str: &str| {
+            let term = Term::from_facet(facet_field, &Facet::from_text(facet_str));
+            searcher
+                .search(&TermQuery::new(term, IndexRecordOption::Basic), &Count)
+                .unwrap()
+        };
+
+        assert_eq!(count_facet("/"), 4);
+        assert_eq!(count_facet("/A"), 3);
+        assert_eq!(count_facet("/A/B"), 1);
+        assert_eq!(count_facet("/A/C"), 1);
+        assert_eq!(count_facet("/A/C/A"), 1);
+        assert_eq!(count_facet("/C/A"), 0);
+        {
+            let query_parser = QueryParser::for_index(&index, vec![]);
+            {
+                let query = query_parser.parse_query("facet:/A/B").unwrap();
+                assert_eq!(1, searcher.search(&query, &Count).unwrap());
+            }
+            {
+                let query = query_parser.parse_query("facet:/A").unwrap();
+                assert_eq!(3, searcher.search(&query, &Count).unwrap());
+            }
+        }
+    }
+
     #[test]
     fn test_non_used_facet_collector() {
         let mut facet_collector = FacetCollector::for_field(Field::from_field_id(0));

src/collector/mod.rs

@@ -85,7 +85,6 @@ See the `custom_collector` example.
 */
 
 use crate::DocId;
-use crate::Result;
 use crate::Score;
 use crate::SegmentLocalId;
 use crate::SegmentReader;
@@ -147,14 +146,14 @@ pub trait Collector: Sync {
         &self,
         segment_local_id: SegmentLocalId,
         segment: &SegmentReader,
-    ) -> Result<Self::Child>;
+    ) -> crate::Result<Self::Child>;
 
     /// Returns true iff the collector requires to compute scores for documents.
     fn requires_scoring(&self) -> bool;
 
     /// Combines the fruit associated to the collection of each segments
     /// into one fruit.
-    fn merge_fruits(&self, segment_fruits: Vec<Self::Fruit>) -> Result<Self::Fruit>;
+    fn merge_fruits(&self, segment_fruits: Vec<Self::Fruit>) -> crate::Result<Self::Fruit>;
 }
 
 /// The `SegmentCollector` is the trait in charge of defining the
@@ -185,7 +184,11 @@ where
     type Fruit = (Left::Fruit, Right::Fruit);
     type Child = (Left::Child, Right::Child);
 
-    fn for_segment(&self, segment_local_id: u32, segment: &SegmentReader) -> Result<Self::Child> {
+    fn for_segment(
+        &self,
+        segment_local_id: u32,
+        segment: &SegmentReader,
+    ) -> crate::Result<Self::Child> {
         let left = self.0.for_segment(segment_local_id, segment)?;
         let right = self.1.for_segment(segment_local_id, segment)?;
         Ok((left, right))
@@ -198,7 +201,7 @@ where
     fn merge_fruits(
         &self,
         children: Vec<(Left::Fruit, Right::Fruit)>,
-    ) -> Result<(Left::Fruit, Right::Fruit)> {
+    ) -> crate::Result<(Left::Fruit, Right::Fruit)> {
         let mut left_fruits = vec![];
         let mut right_fruits = vec![];
         for (left_fruit, right_fruit) in children {
@@ -240,7 +243,11 @@ where
     type Fruit = (One::Fruit, Two::Fruit, Three::Fruit);
     type Child = (One::Child, Two::Child, Three::Child);
 
-    fn for_segment(&self, segment_local_id: u32, segment: &SegmentReader) -> Result<Self::Child> {
+    fn for_segment(
+        &self,
+        segment_local_id: u32,
+        segment: &SegmentReader,
+    ) -> crate::Result<Self::Child> {
         let one = self.0.for_segment(segment_local_id, segment)?;
         let two = self.1.for_segment(segment_local_id, segment)?;
         let three = self.2.for_segment(segment_local_id, segment)?;
@@ -251,7 +258,7 @@ where
         self.0.requires_scoring() || self.1.requires_scoring() || self.2.requires_scoring()
     }
 
-    fn merge_fruits(&self, children: Vec<Self::Fruit>) -> Result<Self::Fruit> {
+    fn merge_fruits(&self, children: Vec<Self::Fruit>) -> crate::Result<Self::Fruit> {
         let mut one_fruits = vec![];
         let mut two_fruits = vec![];
         let mut three_fruits = vec![];
@@ -299,7 +306,11 @@ where
     type Fruit = (One::Fruit, Two::Fruit, Three::Fruit, Four::Fruit);
     type Child = (One::Child, Two::Child, Three::Child, Four::Child);
 
-    fn for_segment(&self, segment_local_id: u32, segment: &SegmentReader) -> Result<Self::Child> {
+    fn for_segment(
+        &self,
+        segment_local_id: u32,
+        segment: &SegmentReader,
+    ) -> crate::Result<Self::Child> {
         let one = self.0.for_segment(segment_local_id, segment)?;
         let two = self.1.for_segment(segment_local_id, segment)?;
         let three = self.2.for_segment(segment_local_id, segment)?;
@@ -314,7 +325,7 @@ where
             || self.3.requires_scoring()
     }
 
-    fn merge_fruits(&self, children: Vec<Self::Fruit>) -> Result<Self::Fruit> {
+    fn merge_fruits(&self, children: Vec<Self::Fruit>) -> crate::Result<Self::Fruit> {
         let mut one_fruits = vec![];
         let mut two_fruits = vec![];
         let mut three_fruits = vec![];

src/collector/multi_collector.rs

@@ -2,7 +2,6 @@ use super::Collector;
 use super::SegmentCollector;
 use crate::collector::Fruit;
 use crate::DocId;
-use crate::Result;
 use crate::Score;
 use crate::SegmentLocalId;
 use crate::SegmentReader;
@@ -24,7 +23,7 @@ impl<TCollector: Collector> Collector for CollectorWrapper<TCollector> {
         &self,
         segment_local_id: u32,
         reader: &SegmentReader,
-    ) -> Result<Box<dyn BoxableSegmentCollector>> {
+    ) -> crate::Result<Box<dyn BoxableSegmentCollector>> {
         let child = self.0.for_segment(segment_local_id, reader)?;
         Ok(Box::new(SegmentCollectorWrapper(child)))
     }
@@ -33,7 +32,10 @@ impl<TCollector: Collector> Collector for CollectorWrapper<TCollector> {
         self.0.requires_scoring()
     }
 
-    fn merge_fruits(&self, children: Vec<<Self as Collector>::Fruit>) -> Result<Box<dyn Fruit>> {
+    fn merge_fruits(
+        &self,
+        children: Vec<<Self as Collector>::Fruit>,
+    ) -> crate::Result<Box<dyn Fruit>> {
         let typed_fruit: Vec<TCollector::Fruit> = children
             .into_iter()
             .map(|untyped_fruit| {
@@ -44,7 +46,7 @@ impl<TCollector: Collector> Collector for CollectorWrapper<TCollector> {
                         TantivyError::InvalidArgument("Failed to cast child fruit.".to_string())
                     })
             })
-            .collect::<Result<_>>()?;
+            .collect::<crate::Result<_>>()?;
         let merged_fruit = self.0.merge_fruits(typed_fruit)?;
         Ok(Box::new(merged_fruit))
     }
@@ -175,12 +177,12 @@ impl<'a> Collector for MultiCollector<'a> {
         &self,
         segment_local_id: SegmentLocalId,
         segment: &SegmentReader,
-    ) -> Result<MultiCollectorChild> {
+    ) -> crate::Result<MultiCollectorChild> {
         let children = self
             .collector_wrappers
             .iter()
             .map(|collector_wrapper| collector_wrapper.for_segment(segment_local_id, segment))
-            .collect::<Result<Vec<_>>>()?;
+            .collect::<crate::Result<Vec<_>>>()?;
         Ok(MultiCollectorChild { children })
     }
 
@@ -191,7 +193,7 @@ impl<'a> Collector for MultiCollector<'a> {
             .any(Collector::requires_scoring)
     }
 
-    fn merge_fruits(&self, segments_multifruits: Vec<MultiFruit>) -> Result<MultiFruit> {
+    fn merge_fruits(&self, segments_multifruits: Vec<MultiFruit>) -> crate::Result<MultiFruit> {
         let mut segment_fruits_list: Vec<Vec<Box<dyn Fruit>>> = (0..self.collector_wrappers.len())
             .map(|_| Vec::with_capacity(segments_multifruits.len()))
             .collect::<Vec<_>>();
@@ -209,7 +211,7 @@ impl<'a> Collector for MultiCollector<'a> {
             .map(|(child_collector, segment_fruits)| {
                 Ok(Some(child_collector.merge_fruits(segment_fruits)?))
             })
-            .collect::<Result<_>>()?;
+            .collect::<crate::Result<_>>()?;
         Ok(MultiFruit { sub_fruits })
     }
 }

src/collector/tests.rs

@@ -55,7 +55,7 @@ impl Collector for TestCollector {
         &self,
         segment_id: SegmentLocalId,
         _reader: &SegmentReader,
-    ) -> Result<TestSegmentCollector> {
+    ) -> crate::Result<TestSegmentCollector> {
         Ok(TestSegmentCollector {
             segment_id,
             fruit: TestFruit::default(),
@@ -66,7 +66,7 @@ impl Collector for TestCollector {
         self.compute_score
     }
 
-    fn merge_fruits(&self, mut children: Vec<TestFruit>) -> Result<TestFruit> {
+    fn merge_fruits(&self, mut children: Vec<TestFruit>) -> crate::Result<TestFruit> {
         children.sort_by_key(|fruit| {
             if fruit.docs().is_empty() {
                 0
@@ -124,7 +124,7 @@ impl Collector for FastFieldTestCollector {
         &self,
         _: SegmentLocalId,
         segment_reader: &SegmentReader,
-    ) -> Result<FastFieldSegmentCollector> {
+    ) -> crate::Result<FastFieldSegmentCollector> {
         let reader = segment_reader
             .fast_fields()
             .u64(self.field)
@@ -139,7 +139,7 @@ impl Collector for FastFieldTestCollector {
         false
     }
 
-    fn merge_fruits(&self, children: Vec<Vec<u64>>) -> Result<Vec<u64>> {
+    fn merge_fruits(&self, children: Vec<Vec<u64>>) -> crate::Result<Vec<u64>> {
         Ok(children.into_iter().flat_map(|v| v.into_iter()).collect())
     }
 }
@@ -184,7 +184,7 @@ impl Collector for BytesFastFieldTestCollector {
         &self,
         _segment_local_id: u32,
         segment_reader: &SegmentReader,
-    ) -> Result<BytesFastFieldSegmentCollector> {
+    ) -> crate::Result<BytesFastFieldSegmentCollector> {
         Ok(BytesFastFieldSegmentCollector {
             vals: Vec::new(),
             reader: segment_reader
@@ -198,7 +198,7 @@ impl Collector for BytesFastFieldTestCollector {
         false
     }
 
-    fn merge_fruits(&self, children: Vec<Vec<u8>>) -> Result<Vec<u8>> {
+    fn merge_fruits(&self, children: Vec<Vec<u8>>) -> crate::Result<Vec<u8>> {
         Ok(children.into_iter().flat_map(|c| c.into_iter()).collect())
     }
 }

src/collector/top_collector.rs

@@ -1,6 +1,5 @@
 use crate::DocAddress;
 use crate::DocId;
-use crate::Result;
 use crate::SegmentLocalId;
 use crate::SegmentReader;
 use serde::export::PhantomData;
@@ -86,7 +85,7 @@ where
     pub fn merge_fruits(
         &self,
         children: Vec<Vec<(T, DocAddress)>>,
-    ) -> Result<Vec<(T, DocAddress)>> {
+    ) -> crate::Result<Vec<(T, DocAddress)>> {
         if self.limit == 0 {
             return Ok(Vec::new());
         }
@@ -113,7 +112,7 @@ where
         &self,
         segment_id: SegmentLocalId,
         _: &SegmentReader,
-    ) -> Result<TopSegmentCollector<F>> {
+    ) -> crate::Result<TopSegmentCollector<F>> {
         Ok(TopSegmentCollector::new(segment_id, self.limit))
     }
 }

src/collector/top_score_collector.rs

@@ -6,10 +6,10 @@ use crate::collector::tweak_score_top_collector::TweakedScoreTopCollector;
 use crate::collector::{
     CustomScorer, CustomSegmentScorer, ScoreSegmentTweaker, ScoreTweaker, SegmentCollector,
 };
+use crate::fastfield::FastFieldReader;
 use crate::schema::Field;
 use crate::DocAddress;
 use crate::DocId;
-use crate::Result;
 use crate::Score;
 use crate::SegmentLocalId;
 use crate::SegmentReader;
@@ -61,6 +61,37 @@ impl fmt::Debug for TopDocs {
     }
 }
 
+struct ScorerByFastFieldReader {
+    ff_reader: FastFieldReader<u64>,
+}
+
+impl CustomSegmentScorer<u64> for ScorerByFastFieldReader {
+    fn score(&self, doc: DocId) -> u64 {
+        self.ff_reader.get_u64(u64::from(doc))
+    }
+}
+
+struct ScorerByField {
+    field: Field,
+}
+
+impl CustomScorer<u64> for ScorerByField {
+    type Child = ScorerByFastFieldReader;
+
+    fn segment_scorer(&self, segment_reader: &SegmentReader) -> crate::Result<Self::Child> {
+        let ff_reader = segment_reader
+            .fast_fields()
+            .u64(self.field)
+            .ok_or_else(|| {
+                crate::TantivyError::SchemaError(format!(
+                    "Field requested ({:?}) is not a i64/u64 fast field.",
+                    self.field
+                ))
+            })?;
+        Ok(ScorerByFastFieldReader { ff_reader })
+    }
+}
+
 impl TopDocs {
     /// Creates a top score collector, with a number of documents equal to "limit".
     ///
@@ -74,7 +105,7 @@ impl TopDocs {
     ///
     /// ```rust
     /// # use tantivy::schema::{Schema, FAST, TEXT};
-    /// # use tantivy::{doc, Index, Result, DocAddress};
+    /// # use tantivy::{doc, Index, DocAddress};
     /// # use tantivy::query::{Query, QueryParser};
     /// use tantivy::Searcher;
     /// use tantivy::collector::TopDocs;
@@ -111,7 +142,7 @@ impl TopDocs {
     /// fn docs_sorted_by_rating(searcher: &Searcher,
     ///                          query: &dyn Query,
     ///                          sort_by_field: Field)
-    ///     -> Result<Vec<(u64, DocAddress)>> {
+    ///     -> tantivy::Result<Vec<(u64, DocAddress)>> {
     ///
     ///     // This is where we build our topdocs collector
     ///     //
@@ -143,14 +174,7 @@ impl TopDocs {
         self,
         field: Field,
     ) -> impl Collector<Fruit = Vec<(u64, DocAddress)>> {
-        self.custom_score(move |segment_reader: &SegmentReader| {
-            let ff_reader = segment_reader
-                .fast_fields()
-                .u64(field)
-                .expect("Field requested is not a i64/u64 fast field.");
-            //TODO error message missmatch actual behavior for i64
-            move |doc: DocId| ff_reader.get(doc)
-        })
+        self.custom_score(ScorerByField { field })
     }
 
     /// Ranks the documents using a custom score.
@@ -384,7 +408,7 @@ impl Collector for TopDocs {
         &self,
         segment_local_id: SegmentLocalId,
         reader: &SegmentReader,
-    ) -> Result<Self::Child> {
+    ) -> crate::Result<Self::Child> {
         let collector = self.0.for_segment(segment_local_id, reader)?;
         Ok(TopScoreSegmentCollector(collector))
     }
@@ -393,7 +417,10 @@ impl Collector for TopDocs {
         true
     }
 
-    fn merge_fruits(&self, child_fruits: Vec<Vec<(Score, DocAddress)>>) -> Result<Self::Fruit> {
+    fn merge_fruits(
+        &self,
+        child_fruits: Vec<Vec<(Score, DocAddress)>>,
+    ) -> crate::Result<Self::Fruit> {
         self.0.merge_fruits(child_fruits)
     }
 }
@@ -423,7 +450,6 @@ mod tests {
     use crate::Index;
     use crate::IndexWriter;
     use crate::Score;
-    use itertools::Itertools;
 
     fn make_index() -> Index {
         let mut schema_builder = Schema::builder();
@@ -497,8 +523,8 @@ mod tests {
 
         // precondition for the test to be meaningful: we did get documents
         // with the same score
-        assert!(page_1.iter().map(|result| result.0).all_equal());
-        assert!(page_2.iter().map(|result| result.0).all_equal());
+        assert!(page_1.iter().all(|result| result.0 == page_1[0].0));
+        assert!(page_2.iter().all(|result| result.0 == page_2[0].0));
 
         // sanity check since we're relying on make_index()
         assert_eq!(page_1.len(), 2);
@@ -572,7 +598,6 @@ mod tests {
     }
 
     #[test]
-    #[should_panic(expected = "Field requested is not a i64/u64 fast field")]
     fn test_field_not_fast_field() {
         let mut schema_builder = Schema::builder();
         let title = schema_builder.add_text_field(TITLE, TEXT);
@@ -587,7 +612,15 @@ mod tests {
         let searcher = index.reader().unwrap().searcher();
         let segment = searcher.segment_reader(0);
         let top_collector = TopDocs::with_limit(4).order_by_u64_field(size);
-        assert!(top_collector.for_segment(0, segment).is_ok());
+        let err = top_collector.for_segment(0, segment);
+        if let Err(crate::TantivyError::SchemaError(msg)) = err {
+            assert_eq!(
+                msg,
+                "Field requested (Field(1)) is not a i64/u64 fast field."
+            );
+        } else {
+            assert!(false);
+        }
     }
 
     fn index(

src/common/composite_file.rs

@@ -190,11 +190,12 @@ mod test {
     use crate::schema::Field;
     use std::io::Write;
     use std::path::Path;
+    
 
     #[test]
     fn test_composite_file() {
         let path = Path::new("test_path");
-        let mut directory = RAMDirectory::create();
+        let mut directory = RAMDirectory::default();
         {
             let w = directory.open_write(path).unwrap();
             let mut composite_write = CompositeWrite::wrap(w);

src/common/mod.rs

@@ -2,6 +2,7 @@ pub mod bitpacker;
 mod bitset;
 mod composite_file;
 mod counting_writer;
+mod mutable_enum;
 mod serialize;
 mod vint;
 
@@ -9,6 +10,7 @@ pub use self::bitset::BitSet;
 pub(crate) use self::bitset::TinySet;
 pub(crate) use self::composite_file::{CompositeFile, CompositeWrite};
 pub use self::counting_writer::CountingWriter;
+pub(crate) use self::mutable_enum::MutableEnum;
 pub use self::serialize::{BinarySerializable, FixedSize};
 pub use self::vint::{read_u32_vint, serialize_vint_u32, write_u32_vint, VInt};
 pub use byteorder::LittleEndian as Endianness;
@@ -18,6 +20,19 @@ pub use byteorder::LittleEndian as Endianness;
 /// We do not allow segments with more than
 pub const MAX_DOC_LIMIT: u32 = 1 << 31;
 
+pub fn minmax<I, T>(mut vals: I) -> Option<(T, T)>
+where
+    I: Iterator<Item = T>,
+    T: Copy + Ord,
+{
+    if let Some(first_el) = vals.next() {
+        return Some(vals.fold((first_el, first_el), |(min_val, max_val), el| {
+            (min_val.min(el), max_val.max(el))
+        }));
+    }
+    None
+}
+
 /// Computes the number of bits that will be used for bitpacking.
 ///
 /// In general the target is the minimum number of bits
@@ -134,6 +149,7 @@ pub fn u64_to_f64(val: u64) -> f64 {
 #[cfg(test)]
 pub(crate) mod test {
 
+    pub use super::minmax;
     pub use super::serialize::test::fixed_size_test;
     use super::{compute_num_bits, f64_to_u64, i64_to_u64, u64_to_f64, u64_to_i64};
     use std::f64;
@@ -199,4 +215,21 @@ pub(crate) mod test {
         assert!(((super::MAX_DOC_LIMIT - 1) as i32) >= 0);
         assert!((super::MAX_DOC_LIMIT as i32) < 0);
     }
+
+    #[test]
+    fn test_minmax_empty() {
+        let vals: Vec<u32> = vec![];
+        assert_eq!(minmax(vals.into_iter()), None);
+    }
+
+    #[test]
+    fn test_minmax_one() {
+        assert_eq!(minmax(vec![1].into_iter()), Some((1, 1)));
+    }
+
+    #[test]
+    fn test_minmax_two() {
+        assert_eq!(minmax(vec![1, 2].into_iter()), Some((1, 2)));
+        assert_eq!(minmax(vec![2, 1].into_iter()), Some((1, 2)));
+    }
 }

src/common/mutable_enum.rs

@@ -0,0 +1,37 @@
+use std::ops::{Deref, DerefMut};
+
+pub(crate) struct MutableEnum<T>(Option<T>);
+
+impl<T> MutableEnum<T> {
+    pub fn wrap(val: T) -> Self {
+        MutableEnum(Some(val))
+    }
+
+    pub fn into(self) -> T {
+        self.0.unwrap()
+    }
+}
+
+impl<T> MutableEnum<T> {
+    pub fn map_mutate<E, F>(&mut self, transformation: F) -> Result<(), E>
+    where
+        F: FnOnce(T) -> Result<T, E>,
+    {
+        self.0 = self.0.take().map(transformation).transpose()?;
+        Ok(())
+    }
+}
+
+impl<T> Deref for MutableEnum<T> {
+    type Target = T;
+
+    fn deref(&self) -> &Self::Target {
+        self.0.as_ref().unwrap()
+    }
+}
+
+impl<T> DerefMut for MutableEnum<T> {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        self.0.as_mut().unwrap()
+    }
+}

src/core/executor.rs

@@ -1,4 +1,3 @@
-use crate::Result;
 use crossbeam::channel;
 use rayon::{ThreadPool, ThreadPoolBuilder};
 
@@ -10,7 +9,9 @@ use rayon::{ThreadPool, ThreadPoolBuilder};
 /// API of a dependency, knowing it might conflict with a different version
 /// used by the client. Second, we may stop using rayon in the future.
 pub enum Executor {
+    /// Single thread variant of an Executor
     SingleThread,
+    /// Thread pool variant of an Executor
     ThreadPool(ThreadPool),
 }
 
@@ -20,8 +21,8 @@ impl Executor {
         Executor::SingleThread
     }
 
-    // Creates an Executor that dispatches the tasks in a thread pool.
-    pub fn multi_thread(num_threads: usize, prefix: &'static str) -> Result<Executor> {
+    /// Creates an Executor that dispatches the tasks in a thread pool.
+    pub fn multi_thread(num_threads: usize, prefix: &'static str) -> crate::Result<Executor> {
         let pool = ThreadPoolBuilder::new()
             .num_threads(num_threads)
             .thread_name(move |num| format!("{}{}", prefix, num))
@@ -29,22 +30,22 @@ impl Executor {
         Ok(Executor::ThreadPool(pool))
     }
 
-    // Perform a map in the thread pool.
-    //
-    // Regardless of the executor (`SingleThread` or `ThreadPool`), panics in the task
-    // will propagate to the caller.
+    /// Perform a map in the thread pool.
+    ///
+    /// Regardless of the executor (`SingleThread` or `ThreadPool`), panics in the task
+    /// will propagate to the caller.
     pub fn map<
         A: Send,
         R: Send,
         AIterator: Iterator<Item = A>,
-        F: Sized + Sync + Fn(A) -> Result<R>,
+        F: Sized + Sync + Fn(A) -> crate::Result<R>,
     >(
         &self,
         f: F,
         args: AIterator,
-    ) -> Result<Vec<R>> {
+    ) -> crate::Result<Vec<R>> {
         match self {
-            Executor::SingleThread => args.map(f).collect::<Result<_>>(),
+            Executor::SingleThread => args.map(f).collect::<crate::Result<_>>(),
             Executor::ThreadPool(pool) => {
                 let args_with_indices: Vec<(usize, A)> = args.enumerate().collect();
                 let num_fruits = args_with_indices.len();

src/core/index.rs

@@ -1,4 +1,3 @@
-use super::segment::create_segment;
 use super::segment::Segment;
 use crate::core::Executor;
 use crate::core::IndexMeta;
@@ -13,17 +12,15 @@ use crate::directory::INDEX_WRITER_LOCK;
 use crate::directory::{Directory, RAMDirectory};
 use crate::error::DataCorruption;
 use crate::error::TantivyError;
-use crate::indexer::index_writer::HEAP_SIZE_MIN;
 use crate::indexer::segment_updater::save_new_metas;
+use crate::indexer::IndexWriterConfig;
 use crate::reader::IndexReader;
 use crate::reader::IndexReaderBuilder;
 use crate::schema::Field;
 use crate::schema::FieldType;
 use crate::schema::Schema;
-use crate::tokenizer::BoxedTokenizer;
-use crate::tokenizer::TokenizerManager;
+use crate::tokenizer::{TextAnalyzer, TokenizerManager};
 use crate::IndexWriter;
-use crate::Result;
 use num_cpus;
 use std::borrow::BorrowMut;
 use std::collections::HashSet;
@@ -32,7 +29,10 @@ use std::fmt;
 use std::path::{Path, PathBuf};
 use std::sync::Arc;
 
-fn load_metas(directory: &dyn Directory, inventory: &SegmentMetaInventory) -> Result<IndexMeta> {
+fn load_metas(
+    directory: &dyn Directory,
+    inventory: &SegmentMetaInventory,
+) -> crate::Result<IndexMeta> {
     let meta_data = directory.atomic_read(&META_FILEPATH)?;
     let meta_string = String::from_utf8_lossy(&meta_data);
     IndexMeta::deserialize(&meta_string, &inventory)
@@ -73,14 +73,14 @@ impl Index {
 
     /// Replace the default single thread search executor pool
     /// by a thread pool with a given number of threads.
-    pub fn set_multithread_executor(&mut self, num_threads: usize) -> Result<()> {
+    pub fn set_multithread_executor(&mut self, num_threads: usize) -> crate::Result<()> {
         self.executor = Arc::new(Executor::multi_thread(num_threads, "thrd-tantivy-search-")?);
         Ok(())
     }
 
     /// Replace the default single thread search executor pool
     /// by a thread pool with a given number of threads.
-    pub fn set_default_multithread_executor(&mut self) -> Result<()> {
+    pub fn set_default_multithread_executor(&mut self) -> crate::Result<()> {
         let default_num_threads = num_cpus::get();
         self.set_multithread_executor(default_num_threads)
     }
@@ -99,7 +99,10 @@ impl Index {
     ///
     /// If a previous index was in this directory, then its meta file will be destroyed.
     #[cfg(feature = "mmap")]
-    pub fn create_in_dir<P: AsRef<Path>>(directory_path: P, schema: Schema) -> Result<Index> {
+    pub fn create_in_dir<P: AsRef<Path>>(
+        directory_path: P,
+        schema: Schema,
+    ) -> crate::Result<Index> {
         let mmap_directory = MmapDirectory::open(directory_path)?;
         if Index::exists(&mmap_directory) {
             return Err(TantivyError::IndexAlreadyExists);
@@ -108,7 +111,7 @@ impl Index {
     }
 
     /// Opens or creates a new index in the provided directory
-    pub fn open_or_create<Dir: Directory>(dir: Dir, schema: Schema) -> Result<Index> {
+    pub fn open_or_create<Dir: Directory>(dir: Dir, schema: Schema) -> crate::Result<Index> {
         if !Index::exists(&dir) {
             return Index::create(dir, schema);
         }
@@ -131,13 +134,13 @@ impl Index {
     /// The temp directory is only used for testing the `MmapDirectory`.
     /// For other unit tests, prefer the `RAMDirectory`, see: `create_in_ram`.
     #[cfg(feature = "mmap")]
-    pub fn create_from_tempdir(schema: Schema) -> Result<Index> {
+    pub fn create_from_tempdir(schema: Schema) -> crate::Result<Index> {
         let mmap_directory = MmapDirectory::create_from_tempdir()?;
         Index::create(mmap_directory, schema)
     }
 
     /// Creates a new index given an implementation of the trait `Directory`
-    pub fn create<Dir: Directory>(dir: Dir, schema: Schema) -> Result<Index> {
+    pub fn create<Dir: Directory>(dir: Dir, schema: Schema) -> crate::Result<Index> {
         let directory = ManagedDirectory::wrap(dir)?;
         Index::from_directory(directory, schema)
     }
@@ -145,7 +148,7 @@ impl Index {
     /// Create a new index from a directory.
     ///
     /// This will overwrite existing meta.json
-    fn from_directory(mut directory: ManagedDirectory, schema: Schema) -> Result<Index> {
+    fn from_directory(mut directory: ManagedDirectory, schema: Schema) -> crate::Result<Index> {
         save_new_metas(schema.clone(), directory.borrow_mut())?;
         let metas = IndexMeta::with_schema(schema);
         Index::create_from_metas(directory, &metas, SegmentMetaInventory::default())
@@ -156,7 +159,7 @@ impl Index {
         directory: ManagedDirectory,
         metas: &IndexMeta,
         inventory: SegmentMetaInventory,
-    ) -> Result<Index> {
+    ) -> crate::Result<Index> {
         let schema = metas.schema.clone();
         Ok(Index {
             directory,
@@ -173,11 +176,11 @@ impl Index {
     }
 
     /// Helper to access the tokenizer associated to a specific field.
-    pub fn tokenizer_for_field(&self, field: Field) -> Result<BoxedTokenizer> {
+    pub fn tokenizer_for_field(&self, field: Field) -> crate::Result<TextAnalyzer> {
         let field_entry = self.schema.get_field_entry(field);
         let field_type = field_entry.field_type();
         let tokenizer_manager: &TokenizerManager = self.tokenizers();
-        let tokenizer_name_opt: Option<BoxedTokenizer> = match field_type {
+        let tokenizer_name_opt: Option<TextAnalyzer> = match field_type {
             FieldType::Str(text_options) => text_options
                 .get_indexing_options()
                 .map(|text_indexing_options| text_indexing_options.tokenizer().to_string())
@@ -196,7 +199,7 @@ impl Index {
     /// Create a default `IndexReader` for the given index.
     ///
     /// See [`Index.reader_builder()`](#method.reader_builder).
-    pub fn reader(&self) -> Result<IndexReader> {
+    pub fn reader(&self) -> crate::Result<IndexReader> {
         self.reader_builder().try_into()
     }
 
@@ -211,7 +214,7 @@ impl Index {
 
     /// Opens a new directory from an index path.
     #[cfg(feature = "mmap")]
-    pub fn open_in_dir<P: AsRef<Path>>(directory_path: P) -> Result<Index> {
+    pub fn open_in_dir<P: AsRef<Path>>(directory_path: P) -> crate::Result<Index> {
         let mmap_directory = MmapDirectory::open(directory_path)?;
         Index::open(mmap_directory)
     }
@@ -235,7 +238,7 @@ impl Index {
     }
 
     /// Open the index using the provided directory
-    pub fn open<D: Directory>(directory: D) -> Result<Index> {
+    pub fn open<D: Directory>(directory: D) -> crate::Result<Index> {
         let directory = ManagedDirectory::wrap(directory)?;
         let inventory = SegmentMetaInventory::default();
         let metas = load_metas(&directory, &inventory)?;
@@ -243,7 +246,7 @@ impl Index {
     }
 
     /// Reads the index meta file from the directory.
-    pub fn load_metas(&self) -> Result<IndexMeta> {
+    pub fn load_metas(&self) -> crate::Result<IndexMeta> {
         load_metas(self.directory(), &self.inventory)
     }
 
@@ -270,30 +273,14 @@ impl Index {
     pub fn writer_with_num_threads(
         &self,
         num_threads: usize,
-        overall_heap_size_in_bytes: usize,
-    ) -> Result<IndexWriter> {
-        let directory_lock = self
-            .directory
-            .acquire_lock(&INDEX_WRITER_LOCK)
-            .map_err(|err| {
-                TantivyError::LockFailure(
-                    err,
-                    Some(
-                        "Failed to acquire index lock. If you are using\
-                         a regular directory, this means there is already an \
-                         `IndexWriter` working on this `Directory`, in this process \
-                         or in a different process."
-                            .to_string(),
-                    ),
-                )
-            })?;
-        let heap_size_in_bytes_per_thread = overall_heap_size_in_bytes / num_threads;
-        IndexWriter::new(
-            self,
-            num_threads,
-            heap_size_in_bytes_per_thread,
-            directory_lock,
-        )
+        overall_heap_size_in_bytes: u64,
+    ) -> crate::Result<IndexWriter> {
+        let config = IndexWriterConfig {
+            max_indexing_threads: num_threads,
+            memory_budget: overall_heap_size_in_bytes,
+            ..Default::default()
+        };
+        self.writer_from_config(config)
     }
 
     /// Creates a multithreaded writer
@@ -306,13 +293,36 @@ impl Index {
     /// If the lockfile already exists, returns `Error::FileAlreadyExists`.
     /// # Panics
     /// If the heap size per thread is too small, panics.
-    pub fn writer(&self, overall_heap_size_in_bytes: usize) -> Result<IndexWriter> {
-        let mut num_threads = num_cpus::get();
-        let heap_size_in_bytes_per_thread = overall_heap_size_in_bytes / num_threads;
-        if heap_size_in_bytes_per_thread < HEAP_SIZE_MIN {
-            num_threads = (overall_heap_size_in_bytes / HEAP_SIZE_MIN).max(1);
-        }
-        self.writer_with_num_threads(num_threads, overall_heap_size_in_bytes)
+    pub fn writer(&self, overall_heap_size_in_bytes: u64) -> crate::Result<IndexWriter> {
+        let config = IndexWriterConfig {
+            max_indexing_threads: num_cpus::get(),
+            memory_budget: overall_heap_size_in_bytes,
+            ..Default::default()
+        };
+        self.writer_from_config(config)
+    }
+
+    /// Creates a new writer with a given configuration.
+    ///
+    /// See [`IndexWriterConfig`](./struct.IndexWriterConfig.html) for more information.
+    pub fn writer_from_config(&self, mut config: IndexWriterConfig) -> crate::Result<IndexWriter> {
+        config.validate()?;
+        let directory_lock = self
+            .directory
+            .acquire_lock(&INDEX_WRITER_LOCK)
+            .map_err(|err| {
+                TantivyError::LockFailure(
+                    err,
+                    Some(
+                        "Failed to acquire index lock. If you are using \
+                         a regular directory, this means there is already an \
+                         `IndexWriter` working on this `Directory`, in this process \
+                         or in a different process."
+                            .to_string(),
+                    ),
+                )
+            })?;
+        IndexWriter::new(self, config, directory_lock)
     }
 
     /// Accessor to the index schema
@@ -323,7 +333,7 @@ impl Index {
     }
 
     /// Returns the list of segments that are searchable
-    pub fn searchable_segments(&self) -> Result<Vec<Segment>> {
+    pub fn searchable_segments(&self) -> crate::Result<Vec<Segment>> {
         Ok(self
             .searchable_segment_metas()?
             .into_iter()
@@ -333,7 +343,7 @@ impl Index {
 
     #[doc(hidden)]
     pub fn segment(&self, segment_meta: SegmentMeta) -> Segment {
-        create_segment(self.clone(), segment_meta)
+        Segment::for_index(self.clone(), segment_meta)
     }
 
     /// Creates a new segment.
@@ -356,12 +366,12 @@ impl Index {
 
     /// Reads the meta.json and returns the list of
     /// `SegmentMeta` from the last commit.
-    pub fn searchable_segment_metas(&self) -> Result<Vec<SegmentMeta>> {
+    pub fn searchable_segment_metas(&self) -> crate::Result<Vec<SegmentMeta>> {
         Ok(self.load_metas()?.segments)
     }
 
     /// Returns the list of segment ids that are searchable.
-    pub fn searchable_segment_ids(&self) -> Result<Vec<SegmentId>> {
+    pub fn searchable_segment_ids(&self) -> crate::Result<Vec<SegmentId>> {
         Ok(self
             .searchable_segment_metas()?
             .iter()
@@ -370,7 +380,7 @@ impl Index {
     }
 
     /// Returns the set of corrupted files
-    pub fn validate_checksum(&self) -> Result<HashSet<PathBuf>> {
+    pub fn validate_checksum(&self) -> crate::Result<HashSet<PathBuf>> {
         self.directory.list_damaged().map_err(Into::into)
     }
 }
@@ -466,7 +476,7 @@ mod tests {
             .try_into()
             .unwrap();
         assert_eq!(reader.searcher().num_docs(), 0);
-        test_index_on_commit_reload_policy_aux(field, &index, &reader);
+        test_index_on_commit_reload_policy_aux(field, index.clone(), &index, &reader);
     }
 
     #[cfg(feature = "mmap")]
@@ -490,7 +500,7 @@ mod tests {
                 .try_into()
                 .unwrap();
             assert_eq!(reader.searcher().num_docs(), 0);
-            test_index_on_commit_reload_policy_aux(field, &index, &reader);
+            test_index_on_commit_reload_policy_aux(field, index.clone(), &index, &reader);
         }
 
         #[test]
@@ -532,12 +542,16 @@ mod tests {
                 .try_into()
                 .unwrap();
             assert_eq!(reader.searcher().num_docs(), 0);
-            test_index_on_commit_reload_policy_aux(field, &write_index, &reader);
+            test_index_on_commit_reload_policy_aux(field, read_index, &write_index, &reader);
         }
     }
 
-    fn test_index_on_commit_reload_policy_aux(field: Field, index: &Index, reader: &IndexReader) {
-        let mut reader_index = reader.index();
+    fn test_index_on_commit_reload_policy_aux(
+        field: Field,
+        mut reader_index: Index,
+        index: &Index,
+        reader: &IndexReader,
+    ) {
         let (sender, receiver) = crossbeam::channel::unbounded();
         let _watch_handle = reader_index.directory_mut().watch(Box::new(move || {
             let _ = sender.send(());

src/core/index_meta.rs

@@ -4,6 +4,7 @@ use crate::schema::Schema;
 use crate::Opstamp;
 use census::{Inventory, TrackedObject};
 use serde;
+use serde::{Deserialize, Serialize};
 use serde_json;
 use std::collections::HashSet;
 use std::fmt;

src/core/inverted_index_reader.rs

@@ -60,7 +60,7 @@ impl InvertedIndexReader {
             .get_index_record_option()
             .unwrap_or(IndexRecordOption::Basic);
         InvertedIndexReader {
-            termdict: TermDictionary::empty(&field_type),
+            termdict: TermDictionary::empty(),
             postings_source: ReadOnlySource::empty(),
             positions_source: ReadOnlySource::empty(),
             positions_idx_source: ReadOnlySource::empty(),

src/core/searcher.rs

@@ -14,7 +14,6 @@ use crate::store::StoreReader;
 use crate::termdict::TermMerger;
 use crate::DocAddress;
 use crate::Index;
-use crate::Result;
 use std::fmt;
 use std::sync::Arc;
 
@@ -23,8 +22,8 @@ fn collect_segment<C: Collector>(
     weight: &dyn Weight,
     segment_ord: u32,
     segment_reader: &SegmentReader,
-) -> Result<C::Fruit> {
-    let mut scorer = weight.scorer(segment_reader)?;
+) -> crate::Result<C::Fruit> {
+    let mut scorer = weight.scorer(segment_reader, 1.0f32)?;
     let mut segment_collector = collector.for_segment(segment_ord as u32, segment_reader)?;
     if let Some(delete_bitset) = segment_reader.delete_bitset() {
         scorer.for_each(&mut |doc, score| {
@@ -78,7 +77,7 @@ impl Searcher {
     ///
     /// The searcher uses the segment ordinal to route the
     /// the request to the right `Segment`.
-    pub fn doc(&self, doc_address: DocAddress) -> Result<Document> {
+    pub fn doc(&self, doc_address: DocAddress) -> crate::Result<Document> {
         let DocAddress(segment_local_id, doc_id) = doc_address;
         let store_reader = &self.store_readers[segment_local_id as usize];
         store_reader.get(doc_id)
@@ -132,7 +131,11 @@ impl Searcher {
     ///
     ///  Finally, the Collector merges each of the child collectors into itself for result usability
     ///  by the caller.
-    pub fn search<C: Collector>(&self, query: &dyn Query, collector: &C) -> Result<C::Fruit> {
+    pub fn search<C: Collector>(
+        &self,
+        query: &dyn Query,
+        collector: &C,
+    ) -> crate::Result<C::Fruit> {
         let executor = self.index.search_executor();
         self.search_with_executor(query, collector, executor)
     }
@@ -154,7 +157,7 @@ impl Searcher {
         query: &dyn Query,
         collector: &C,
         executor: &Executor,
-    ) -> Result<C::Fruit> {
+    ) -> crate::Result<C::Fruit> {
         let scoring_enabled = collector.requires_scoring();
         let weight = query.weight(self, scoring_enabled)?;
         let segment_readers = self.segment_readers();

src/core/segment.rs

@@ -3,46 +3,140 @@ use crate::core::Index;
 use crate::core::SegmentId;
 use crate::core::SegmentMeta;
 use crate::directory::error::{OpenReadError, OpenWriteError};
-use crate::directory::Directory;
+use crate::directory::{Directory, ManagedDirectory, RAMDirectory};
 use crate::directory::{ReadOnlySource, WritePtr};
 use crate::indexer::segment_serializer::SegmentSerializer;
 use crate::schema::Schema;
 use crate::Opstamp;
-use crate::Result;
 use std::fmt;
+use std::ops::{Deref, DerefMut};
 use std::path::PathBuf;
-use std::result;
+use crate::indexer::{ResourceManager};
+
+#[derive(Clone)]
+pub(crate) enum SegmentDirectory {
+    Persisted(ManagedDirectory),
+    Volatile {
+        directory: RAMDirectory,
+    },
+}
+
+impl SegmentDirectory {
+    fn new_volatile(memory_manager: ResourceManager) -> SegmentDirectory {
+        SegmentDirectory::Volatile {
+            directory: RAMDirectory::create_with_memory_manager(memory_manager),
+        }
+    }
+}
+
+impl fmt::Debug for SegmentDirectory {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match self {
+            SegmentDirectory::Volatile { .. } => write!(f, "volatile")?,
+            SegmentDirectory::Persisted(dir) => write!(f, "Persisted({:?})", dir)?,
+        }
+        Ok(())
+    }
+}
+
+impl From<ManagedDirectory> for SegmentDirectory {
+    fn from(directory: ManagedDirectory) -> Self {
+        SegmentDirectory::Persisted(directory)
+    }
+}
+
+impl Deref for SegmentDirectory {
+    type Target = dyn Directory;
+
+    fn deref(&self) -> &Self::Target {
+        match self {
+            SegmentDirectory::Volatile {
+                directory, ..
+            } => directory,
+            SegmentDirectory::Persisted(dir) => dir,
+        }
+    }
+}
+
+impl DerefMut for SegmentDirectory {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        match self {
+            SegmentDirectory::Volatile { directory, .. } => directory,
+            SegmentDirectory::Persisted(dir) => dir,
+        }
+    }
+}
 
 /// A segment is a piece of the index.
 #[derive(Clone)]
 pub struct Segment {
-    index: Index,
+    schema: Schema,
     meta: SegmentMeta,
+    directory: SegmentDirectory,
 }
 
 impl fmt::Debug for Segment {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(f, "Segment({:?})", self.id().uuid_string())
+        write!(
+            f,
+            "Segment(id={:?}, directory={:?})",
+            self.id().uuid_string(),
+            self.directory
+        )
     }
 }
 
-/// Creates a new segment given an `Index` and a `SegmentId`
-///
-/// The function is here to make it private outside `tantivy`.
-/// #[doc(hidden)]
-pub fn create_segment(index: Index, meta: SegmentMeta) -> Segment {
-    Segment { index, meta }
-}
-
 impl Segment {
-    /// Returns the index the segment belongs to.
-    pub fn index(&self) -> &Index {
-        &self.index
+    pub(crate) fn new_persisted(
+        meta: SegmentMeta,
+        directory: ManagedDirectory,
+        schema: Schema,
+    ) -> Segment {
+        Segment {
+            meta,
+            schema,
+            directory: SegmentDirectory::from(directory),
+        }
+    }
+
+    /// Creates a new segment that embeds its own `RAMDirectory`.
+    ///
+    /// That segment is entirely dissociated from the index directory.
+    /// It will be persisted by a background thread in charge of IO.
+    pub fn new_volatile(meta: SegmentMeta, schema: Schema, resource_manager: ResourceManager) -> Segment {
+        Segment {
+            schema,
+            meta,
+            directory: SegmentDirectory::new_volatile(resource_manager),
+        }
+    }
+
+    /// Creates a new segment given an `Index` and a `SegmentId`
+    pub(crate) fn for_index(index: Index, meta: SegmentMeta) -> Segment {
+        let segment_directory = index.directory().clone();
+        Segment::new_persisted(meta, segment_directory, index.schema())
+    }
+
+    /// Persists a given `Segment` to a directory.
+    pub fn persist(&mut self, mut dest_directory: ManagedDirectory) -> crate::Result<()> {
+        if let SegmentDirectory::Persisted(_) = self.directory {
+            // this segment is already persisted.
+            return Ok(());
+        }
+        if let SegmentDirectory::Volatile { directory, ..} = &self.directory {
+            directory.persist(&mut dest_directory)?;
+        }
+        self.directory = SegmentDirectory::Persisted(dest_directory);
+        Ok(())
+    }
+
+    pub fn into_volatile(&self, memory_manager: ResourceManager) -> Segment {
+        Segment::new_volatile(self.meta.clone(), self.schema.clone(), memory_manager)
     }
 
     /// Returns our index's schema.
     pub fn schema(&self) -> Schema {
-        self.index.schema()
+        self.schema.clone()
     }
 
     /// Returns the segment meta-information
@@ -56,16 +150,18 @@ impl Segment {
     /// as we finalize a fresh new segment.
     pub(crate) fn with_max_doc(self, max_doc: u32) -> Segment {
         Segment {
-            index: self.index,
+            schema: self.schema,
             meta: self.meta.with_max_doc(max_doc),
+            directory: self.directory,
         }
     }
 
     #[doc(hidden)]
     pub fn with_delete_meta(self, num_deleted_docs: u32, opstamp: Opstamp) -> Segment {
         Segment {
-            index: self.index,
+            schema: self.schema,
             meta: self.meta.with_delete_meta(num_deleted_docs, opstamp),
+            directory: self.directory,
         }
     }
 
@@ -83,23 +179,17 @@ impl Segment {
     }
 
     /// Open one of the component file for a *regular* read.
-    pub fn open_read(
-        &self,
-        component: SegmentComponent,
-    ) -> result::Result<ReadOnlySource, OpenReadError> {
+    pub fn open_read(&self, component: SegmentComponent) -> Result<ReadOnlySource, OpenReadError> {
         let path = self.relative_path(component);
-        let source = self.index.directory().open_read(&path)?;
+        let source = self.directory.open_read(&path)?;
         Ok(source)
     }
 
     /// Open one of the component file for *regular* write.
-    pub fn open_write(
-        &mut self,
-        component: SegmentComponent,
-    ) -> result::Result<WritePtr, OpenWriteError> {
+    pub fn open_write(&mut self, component: SegmentComponent) -> Result<WritePtr, OpenWriteError> {
         let path = self.relative_path(component);
-        let write = self.index.directory_mut().open_write(&path)?;
-        Ok(write)
+        let wrt = self.directory.open_write(&path)?;
+        Ok(wrt)
     }
 }
 
@@ -109,5 +199,5 @@ pub trait SerializableSegment {
     ///
     /// # Returns
     /// The number of documents in the segment.
-    fn write(&self, serializer: SegmentSerializer) -> Result<u32>;
+    fn write(&self, serializer: SegmentSerializer) -> crate::Result<u32>;
 }

src/core/segment_id.rs

@@ -4,6 +4,7 @@ use uuid::Uuid;
 
 #[cfg(test)]
 use once_cell::sync::Lazy;
+use serde::{Deserialize, Serialize};
 use std::error::Error;
 use std::str::FromStr;
 #[cfg(test)]

src/core/segment_reader.rs

@@ -16,7 +16,6 @@ use crate::space_usage::SegmentSpaceUsage;
 use crate::store::StoreReader;
 use crate::termdict::TermDictionary;
 use crate::DocId;
-use crate::Result;
 use fail::fail_point;
 use std::collections::HashMap;
 use std::fmt;
@@ -145,7 +144,7 @@ impl SegmentReader {
     }
 
     /// Open a new segment for reading.
-    pub fn open(segment: &Segment) -> Result<SegmentReader> {
+    pub fn open(segment: &Segment) -> crate::Result<SegmentReader> {
         let termdict_source = segment.open_read(SegmentComponent::TERMS)?;
         let termdict_composite = CompositeFile::open(&termdict_source)?;
 

src/directory/footer.rs

@@ -8,6 +8,8 @@ use crc32fast::Hasher;
 use std::io;
 use std::io::Write;
 
+const FOOTER_MAX_LEN: usize = 10_000;
+
 type CrcHashU32 = u32;
 
 #[derive(Debug, Clone, PartialEq)]
@@ -143,12 +145,23 @@ impl BinarySerializable for VersionedFooter {
             }
         }
         BinarySerializable::serialize(&VInt(buf.len() as u64), writer)?;
+        assert!(buf.len() <= FOOTER_MAX_LEN);
         writer.write_all(&buf[..])?;
         Ok(())
     }
 
     fn deserialize<R: io::Read>(reader: &mut R) -> io::Result<Self> {
         let len = VInt::deserialize(reader)?.0 as usize;
+        if len > FOOTER_MAX_LEN {
+            return Err(io::Error::new(
+                io::ErrorKind::InvalidData,
+                format!(
+                    "Footer seems invalid as it suggests a footer len of {}. File is corrupted, \
+            or the index was created with a different & old version of tantivy.",
+                    len
+                ),
+            ));
+        }
         let mut buf = vec![0u8; len];
         reader.read_exact(&mut buf[..])?;
         let mut cursor = &buf[..];
@@ -221,11 +234,12 @@ mod tests {
 
     use super::CrcHashU32;
     use super::FooterProxy;
-    use crate::common::BinarySerializable;
+    use crate::common::{BinarySerializable, VInt};
     use crate::directory::footer::{Footer, VersionedFooter};
     use crate::directory::TerminatingWrite;
     use byteorder::{ByteOrder, LittleEndian};
     use regex::Regex;
+    use std::io;
 
     #[test]
     fn test_versioned_footer() {
@@ -336,4 +350,20 @@ mod tests {
         let res = footer.is_compatible();
         assert!(res.is_err());
     }
+
+    #[test]
+    fn test_deserialize_too_large_footer() {
+        let mut buf = vec![];
+        assert!(FooterProxy::new(&mut buf).terminate().is_ok());
+        let mut long_len_buf = [0u8; 10];
+        let num_bytes = VInt(super::FOOTER_MAX_LEN as u64 + 1u64).serialize_into(&mut long_len_buf);
+        buf[0..num_bytes].copy_from_slice(&long_len_buf[..num_bytes]);
+        let err = Footer::deserialize(&mut &buf[..]).unwrap_err();
+        assert_eq!(err.kind(), io::ErrorKind::InvalidData);
+        assert_eq!(
+            err.to_string(),
+            "Footer seems invalid as it suggests a footer len of 10001. File is corrupted, \
+            or the index was created with a different & old version of tantivy."
+        );
+    }
 }

src/directory/managed_directory.rs

@@ -150,7 +150,7 @@ impl ManagedDirectory {
                 }
                 Err(err) => {
                     error!("Failed to acquire lock for GC");
-                    return Err(crate::Error::from(err));
+                    return Err(crate::TantivyError::from(err));
                 }
             }
         }

src/directory/mmap_directory.rs

@@ -22,6 +22,7 @@ use crate::directory::WatchHandle;
 use crate::directory::{TerminatingWrite, WritePtr};
 use atomicwrites;
 use memmap::Mmap;
+use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
 use std::convert::From;
 use std::fmt;
@@ -131,14 +132,13 @@ impl MmapCache {
         }
         self.cache.remove(full_path);
         self.counters.miss += 1;
-        Ok(if let Some(mmap) = open_mmap(full_path)? {
+        let mmap_opt = open_mmap(full_path)?;
+        Ok(mmap_opt.map(|mmap| {
             let mmap_arc: Arc<BoxedData> = Arc::new(Box::new(mmap));
             let mmap_weak = Arc::downgrade(&mmap_arc);
             self.cache.insert(full_path.to_owned(), mmap_weak);
-            Some(mmap_arc)
-        } else {
-            None
-        })
+            mmap_arc
+        }))
     }
 }
 

src/directory/mod.rs

@@ -13,7 +13,10 @@ mod footer;
 mod managed_directory;
 mod ram_directory;
 mod read_only_source;
+mod spilling_writer;
 mod watch_event_router;
+mod persistor;
+
 
 /// Errors specific to the directory module.
 pub mod error;
@@ -23,6 +26,7 @@ pub use self::directory::{Directory, DirectoryClone};
 pub use self::directory_lock::{Lock, INDEX_WRITER_LOCK, META_LOCK};
 pub use self::ram_directory::RAMDirectory;
 pub use self::read_only_source::ReadOnlySource;
+pub(crate) use self::spilling_writer::{SpillingResult, SpillingWriter};
 pub use self::watch_event_router::{WatchCallback, WatchCallbackList, WatchHandle};
 use std::io::{self, BufWriter, Write};
 use std::path::PathBuf;
@@ -79,10 +83,16 @@ impl<W: TerminatingWrite> TerminatingWrite for BufWriter<W> {
     }
 }
 
+impl TerminatingWrite for Vec<u8> {
+    fn terminate_ref(&mut self, _a: AntiCallToken) -> io::Result<()> {
+        Ok(())
+    }
+}
+
 #[cfg(test)]
 impl<'a> TerminatingWrite for &'a mut Vec<u8> {
     fn terminate_ref(&mut self, _a: AntiCallToken) -> io::Result<()> {
-        self.flush()
+        Ok(())
     }
 }
 

src/directory/persistor.rs

@@ -0,0 +1,38 @@
+use crate::indexer::{SegmentManager, ResourceManager, MergeOperationInventory};
+use std::thread::JoinHandle;
+use crate::{IndexWriterConfig, SegmentId};
+use std::collections::HashSet;
+
+pub(crate) struct Persistor {
+    memory_manager: ResourceManager,
+    thread_handle: JoinHandle<()>,
+}
+
+impl Persistor {
+    pub(crate) fn create_and_start(segment_manager: SegmentManager,
+                            memory_manager: ResourceManager,
+                            config: IndexWriterConfig) -> crate::Result<Persistor> {
+        let memory_manager_clone = memory_manager.clone();
+        let thread_handle = std::thread::Builder::new()
+            .name("persistor-thread".to_string())
+            .spawn(move || {
+                while let Ok(_) = memory_manager_clone.wait_until_in_range(config.persist_low..) {
+                    segment_manager.largest_segment_not_in_merge();
+                }
+            }).map_err(|_err| crate::TantivyError::ErrorInThread("Failed to start persistor thread.".to_string()))?;
+        Ok(Persistor {
+            memory_manager,
+            thread_handle
+        })
+    }
+
+    /// Stop the persisting thread.
+    ///
+    /// The memory manager will be terminated, which will unlock the thread from any waiting
+    /// position.
+    /// This method blocks for a short amount of tim until the persistor thread has terminated.
+    pub fn stop(self) {
+        self.memory_manager.terminate();
+        let _ = self.thread_handle.join();
+    }
+}

src/directory/ram_directory.rs

@@ -11,6 +11,7 @@ use std::io::{self, BufWriter, Cursor, Seek, SeekFrom, Write};
 use std::path::{Path, PathBuf};
 use std::result;
 use std::sync::{Arc, RwLock};
+use crate::indexer::ResourceManager;
 
 /// Writer associated with the `RAMDirectory`
 ///
@@ -82,11 +83,12 @@ impl TerminatingWrite for VecWriter {
 struct InnerDirectory {
     fs: HashMap<PathBuf, ReadOnlySource>,
     watch_router: WatchCallbackList,
+    memory_manager: ResourceManager,
 }
 
 impl InnerDirectory {
     fn write(&mut self, path: PathBuf, data: &[u8]) -> bool {
-        let data = ReadOnlySource::new(Vec::from(data));
+        let data = ReadOnlySource::new_with_allocation(Vec::from(data), &self.memory_manager);
         self.fs.insert(path, data).is_some()
     }
 
@@ -112,8 +114,8 @@ impl InnerDirectory {
         self.watch_router.subscribe(watch_handle)
     }
 
-    fn total_mem_usage(&self) -> usize {
-        self.fs.values().map(|f| f.len()).sum()
+    fn total_mem_usage(&self) -> u64 {
+        self.fs.values().map(|source| source.len() as u64).sum()
     }
 }
 
@@ -134,16 +136,48 @@ pub struct RAMDirectory {
 }
 
 impl RAMDirectory {
-    /// Constructor
+
+    /// Creates a new RAMDirectory.
+    ///
+    /// Check `.create_with_memory_manager(..)` if you want to associate an external memory
+    /// manager to your RAMDirectory.
     pub fn create() -> RAMDirectory {
-        Self::default()
+        RAMDirectory::default()
+    }
+
+    /// Constructor
+    pub fn create_with_memory_manager(memory_manager: ResourceManager) -> RAMDirectory {
+        let inner_directory = InnerDirectory {
+            fs: Default::default(),
+            watch_router: Default::default(),
+            memory_manager
+        };
+        RAMDirectory {
+            fs: Arc::new(RwLock::new(inner_directory))
+        }
     }
 
     /// Returns the sum of the size of the different files
     /// in the RAMDirectory.
-    pub fn total_mem_usage(&self) -> usize {
+    pub fn total_mem_usage(&self) -> u64 {
         self.fs.read().unwrap().total_mem_usage()
     }
+
+    /// Write a copy of all of the files saved in the RAMDirectory in the target `Directory`.
+    ///
+    /// Files are all written using the `Directory::write` meaning, even if they were
+    /// written using the `atomic_write` api.
+    ///
+    /// If an error is encounterred, files may be persisted partially.
+    pub fn persist(&self, dest: &mut dyn Directory) -> crate::Result<()> {
+        let wlock = self.fs.write().unwrap();
+        for (path, source) in wlock.fs.iter() {
+            let mut dest_wrt = dest.open_write(path)?;
+            dest_wrt.write_all(source.as_slice())?;
+            dest_wrt.terminate()?;
+        }
+        Ok(())
+    }
 }
 
 impl Directory for RAMDirectory {
@@ -204,3 +238,86 @@ impl Directory for RAMDirectory {
         Ok(self.fs.write().unwrap().watch(watch_callback))
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::RAMDirectory;
+    use crate::Directory;
+    use std::io::Write;
+    use std::path::Path;
+    use crate::indexer::ResourceManager;
+    use crate::directory::TerminatingWrite;
+    use std::mem;
+
+    #[test]
+    fn test_persist() {
+        let msg_atomic: &'static [u8] = b"atomic is the way";
+        let msg_seq: &'static [u8] = b"sequential is the way";
+        let path_atomic: &'static Path = Path::new("atomic");
+        let path_seq: &'static Path = Path::new("seq");
+        let mut directory = RAMDirectory::create();
+        assert!(directory.atomic_write(path_atomic, msg_atomic).is_ok());
+        let mut wrt = directory.open_write(path_seq).unwrap();
+        assert!(wrt.write_all(msg_seq).is_ok());
+        assert!(wrt.flush().is_ok());
+        let mut directory_copy = RAMDirectory::create();
+        assert!(directory.persist(&mut directory_copy).is_ok());
+        assert_eq!(directory_copy.atomic_read(path_atomic).unwrap(), msg_atomic);
+        assert_eq!(directory_copy.atomic_read(path_seq).unwrap(), msg_seq);
+    }
+
+    #[test]
+    fn test_memory_manager_several_path() {
+        let memory_manager = ResourceManager::default();
+        let mut ram_directory = RAMDirectory::create_with_memory_manager(memory_manager.clone());
+        assert!(ram_directory.atomic_write(Path::new("/titi"), b"abcd").is_ok());
+        assert_eq!(memory_manager.total_amount(), 4u64);
+        assert!(ram_directory.atomic_write(Path::new("/toto"), b"abcde").is_ok());
+        assert_eq!(memory_manager.total_amount(), 9u64);
+    }
+
+    #[test]
+    fn test_memory_manager_override() {
+        let memory_manager = ResourceManager::default();
+        let mut ram_directory = RAMDirectory::create_with_memory_manager(memory_manager.clone());
+        assert!(ram_directory.atomic_write(Path::new("/titi"), b"abcde").is_ok());
+        assert_eq!(memory_manager.total_amount(), 5u64);
+        assert!(ram_directory.atomic_write(Path::new("/titi"), b"abcdef").is_ok());
+        assert_eq!(memory_manager.total_amount(), 6u64);
+    }
+
+    #[test]
+    fn test_memory_manager_seq_wrt() {
+        let memory_manager = ResourceManager::default();
+        let mut ram_directory = RAMDirectory::create_with_memory_manager(memory_manager.clone());
+        let mut wrt = ram_directory.open_write(Path::new("/titi")).unwrap();
+        assert!(wrt.write_all(b"abcde").is_ok());
+        assert!(wrt.terminate().is_ok());
+        assert_eq!(memory_manager.total_amount(), 5u64);
+        assert!(ram_directory.atomic_write(Path::new("/titi"), b"abcdef").is_ok());
+        assert_eq!(memory_manager.total_amount(), 6u64);
+    }
+
+    #[test]
+    fn test_release_on_drop() {
+        let memory_manager = ResourceManager::default();
+        let mut ram_directory = RAMDirectory::create_with_memory_manager(memory_manager.clone());
+        let mut wrt = ram_directory.open_write(Path::new("/titi")).unwrap();
+        assert!(wrt.write_all(b"abcde").is_ok());
+        assert!(wrt.terminate().is_ok());
+        assert_eq!(memory_manager.total_amount(), 5u64);
+        let mut wrt2 = ram_directory.open_write(Path::new("/toto")).unwrap();
+        assert!(wrt2.write_all(b"abcdefghijkl").is_ok());
+        assert!(wrt2.terminate().is_ok());
+        assert_eq!(memory_manager.total_amount(), 17u64);
+        let source = ram_directory.open_read(Path::new("/titi")).unwrap();
+        let source_clone = source.clone();
+        assert_eq!(memory_manager.total_amount(), 17u64);
+        mem::drop(ram_directory);
+        assert_eq!(memory_manager.total_amount(), 5u64);
+        mem::drop(source);
+        assert_eq!(memory_manager.total_amount(), 5u64);
+        mem::drop(source_clone);
+        assert_eq!(memory_manager.total_amount(), 0u64);
+    }
+}

src/directory/read_only_source.rs

@@ -2,6 +2,7 @@ use crate::common::HasLen;
 use stable_deref_trait::{CloneStableDeref, StableDeref};
 use std::ops::Deref;
 use std::sync::Arc;
+use crate::indexer::{Allocation, ResourceManager};
 
 pub type BoxedData = Box<dyn Deref<Target = [u8]> + Send + Sync + 'static>;
 
@@ -15,6 +16,7 @@ pub struct ReadOnlySource {
     data: Arc<BoxedData>,
     start: usize,
     stop: usize,
+    allocation: Option<Arc<Allocation>>
 }
 
 unsafe impl StableDeref for ReadOnlySource {}
@@ -35,11 +37,13 @@ impl From<Arc<BoxedData>> for ReadOnlySource {
             data,
             start: 0,
             stop: len,
+            allocation: None
         }
     }
 }
 
 impl ReadOnlySource {
+
     pub(crate) fn new<D>(data: D) -> ReadOnlySource
     where
         D: Deref<Target = [u8]> + Send + Sync + 'static,
@@ -49,9 +53,24 @@ impl ReadOnlySource {
             data: Arc::new(Box::new(data)),
             start: 0,
             stop: len,
+            allocation: None
         }
     }
 
+    pub(crate) fn new_with_allocation<D>(data: D, memory_manager: &ResourceManager) -> ReadOnlySource
+        where
+            D: Deref<Target = [u8]> + Send + Sync + 'static,
+    {
+        let len = data.len();
+        ReadOnlySource {
+            data: Arc::new(Box::new(data)),
+            start: 0,
+            stop: len,
+            allocation: Some(Arc::new(memory_manager.allocate(len as u64)))
+        }
+    }
+
+
     /// Creates an empty ReadOnlySource
     pub fn empty() -> ReadOnlySource {
         ReadOnlySource::new(&[][..])
@@ -98,6 +117,7 @@ impl ReadOnlySource {
             data: self.data.clone(),
             start: self.start + start,
             stop: self.start + stop,
+            allocation: self.allocation.clone()
         }
     }
 

src/directory/spilling_writer.rs

@@ -0,0 +1,186 @@
+use crate::common::MutableEnum;
+use crate::directory::{TerminatingWrite, WritePtr};
+use std::io::{self, Write};
+
+/// Represents the state of the `SpillingWriter`.
+enum SpillingState {
+    Buffer {
+        buffer: Vec<u8>,
+        capacity: usize,
+        write_factory: Box<dyn FnOnce() -> io::Result<WritePtr>>,
+    },
+    Spilled(WritePtr),
+}
+
+impl SpillingState {
+    fn new(
+        limit: usize,
+        write_factory: Box<dyn FnOnce() -> io::Result<WritePtr>>,
+    ) -> SpillingState {
+        SpillingState::Buffer {
+            buffer: Vec::with_capacity(limit),
+            capacity: limit,
+            write_factory,
+        }
+    }
+
+    // Change the state in such a way that it is ready to accept
+    // `extra_capacity` bytes.
+    //
+    fn reserve(self, extra_capacity: usize) -> io::Result<SpillingState> {
+        match self {
+            SpillingState::Buffer {
+                buffer,
+                capacity,
+                write_factory,
+            } => {
+                if capacity >= extra_capacity {
+                    Ok(SpillingState::Buffer {
+                        buffer,
+                        capacity: capacity - extra_capacity,
+                        write_factory,
+                    })
+                } else {
+                    let mut wrt = write_factory()?;
+                    wrt.write_all(&buffer[..])?;
+                    Ok(SpillingState::Spilled(wrt))
+                }
+            }
+            SpillingState::Spilled(wrt) => Ok(SpillingState::Spilled(wrt)),
+        }
+    }
+}
+
+/// The `SpillingWriter` is a writer that start by writing in a
+/// buffer.
+///
+/// Once a memory limit is reached, the spilling writer will
+/// call a given `WritePtr` factory and start spilling into it.
+///
+/// Spilling here includes:
+/// - writing all of the data that were written in the in-memory buffer so far
+/// - writing subsequent data as well.
+///
+/// Once entering "spilling" mode, the `SpillingWriter` stays in this mode.
+pub struct SpillingWriter {
+    state: MutableEnum<SpillingState>,
+}
+
+impl SpillingWriter {
+    //// Creates a new `Spilling Writer`.
+    pub fn new(
+        limit: usize,
+        write_factory: Box<dyn FnOnce() -> io::Result<WritePtr>>,
+    ) -> SpillingWriter {
+        let state = SpillingState::new(limit, write_factory);
+        SpillingWriter {
+            state: MutableEnum::wrap(state),
+        }
+    }
+
+    /// Finalizes the `SpillingWriter`.
+    ///
+    /// The `SpillingResult` object is an enum specific
+    /// to whether the `SpillingWriter` reached the spilling limit
+    /// (In that case, the buffer is returned).
+    ///
+    /// If the writer reached the spilling mode, the underlying `WritePtr`
+    /// is terminated and SpillingResult::Spilled is returned.
+    pub fn finalize(self) -> io::Result<SpillingResult> {
+        match self.state.into() {
+            SpillingState::Spilled(wrt) => {
+                wrt.terminate()?;
+                Ok(SpillingResult::Spilled)
+            }
+            SpillingState::Buffer { buffer, .. } => Ok(SpillingResult::Buffer(buffer)),
+        }
+    }
+}
+
+/// enum used as the result of `.finalize()`.
+pub enum SpillingResult {
+    Spilled,
+    Buffer(Vec<u8>),
+}
+
+impl io::Write for SpillingWriter {
+    fn write(&mut self, payload: &[u8]) -> io::Result<usize> {
+        self.write_all(payload)?;
+        Ok(payload.len())
+    }
+
+    fn flush(&mut self) -> io::Result<()> {
+        if let SpillingState::Spilled(wrt) = &mut *self.state {
+            wrt.flush()?;
+        }
+        Ok(())
+    }
+
+    fn write_all(&mut self, payload: &[u8]) -> io::Result<()> {
+        self.state.map_mutate(|mut state| {
+            state = state.reserve(payload.len())?;
+            match &mut state {
+                SpillingState::Buffer { buffer, .. } => {
+                    buffer.extend_from_slice(payload);
+                }
+                SpillingState::Spilled(wrt) => {
+                    wrt.write_all(payload)?;
+                }
+            }
+            Ok(state)
+        })
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::SpillingWriter;
+    use crate::directory::spilling_writer::SpillingResult;
+    use crate::directory::RAMDirectory;
+    use crate::Directory;
+    use std::io::{self, Write};
+    use std::path::Path;
+
+    #[test]
+    fn test_no_spilling() {
+        let ram_directory = RAMDirectory::create();
+        let mut ram_directory_clone = ram_directory.clone();
+        let path = Path::new("test");
+        let write_factory = Box::new(move || {
+            ram_directory_clone
+                .open_write(path)
+                .map_err(|err| io::Error::new(io::ErrorKind::Other, err))
+        });
+        let mut spilling_wrt = SpillingWriter::new(10, write_factory);
+        assert!(spilling_wrt.write_all(b"abcd").is_ok());
+        if let SpillingResult::Buffer(buf) = spilling_wrt.finalize().unwrap() {
+            assert_eq!(buf, b"abcd")
+        } else {
+            panic!("spill writer should not have spilled");
+        }
+        assert!(!ram_directory.exists(path));
+    }
+
+    #[test]
+    fn test_spilling() {
+        let ram_directory = RAMDirectory::create();
+        let mut ram_directory_clone = ram_directory.clone();
+        let path = Path::new("test");
+        let write_factory = Box::new(move || {
+            ram_directory_clone
+                .open_write(path)
+                .map_err(|err| io::Error::new(io::ErrorKind::Other, err))
+        });
+        let mut spilling_wrt = SpillingWriter::new(10, write_factory);
+        assert!(spilling_wrt.write_all(b"abcd").is_ok());
+        assert!(spilling_wrt.write_all(b"efghijklmnop").is_ok());
+        if let SpillingResult::Spilled = spilling_wrt.finalize().unwrap() {
+        } else {
+            panic!("spill writer should have spilled");
+        }
+        assert_eq!(
+            ram_directory.atomic_read(path).unwrap(),
+            b"abcdefghijklmnop"
+        );
+    }
+}

src/fastfield/delete.rs

@@ -89,7 +89,7 @@ mod tests {
 
     fn test_delete_bitset_helper(bitset: &BitSet, max_doc: u32) {
         let test_path = PathBuf::from("test");
-        let mut directory = RAMDirectory::create();
+        let mut directory = RAMDirectory::default();
         {
             let mut writer = directory.open_write(&*test_path).unwrap();
             write_delete_bitset(bitset, max_doc, &mut writer).unwrap();

src/fastfield/multivalued/mod.rs

@@ -7,9 +7,6 @@ pub use self::writer::MultiValueIntFastFieldWriter;
 #[cfg(test)]
 mod tests {
 
-    use time;
-
-    use self::time::Duration;
     use crate::collector::TopDocs;
     use crate::query::QueryParser;
     use crate::schema::Cardinality;
@@ -17,6 +14,7 @@ mod tests {
     use crate::schema::IntOptions;
     use crate::schema::Schema;
     use crate::Index;
+    use chrono::Duration;
 
     #[test]
     fn test_multivalued_u64() {

src/fastfield/multivalued/writer.rs

@@ -6,7 +6,6 @@ use crate::schema::{Document, Field};
 use crate::termdict::TermOrdinal;
 use crate::DocId;
 use fnv::FnvHashMap;
-use itertools::Itertools;
 use std::io;
 
 /// Writer for multi-valued (as in, more than one value per document)
@@ -151,8 +150,8 @@ impl MultiValueIntFastFieldWriter {
                     }
                 }
                 None => {
-                    let val_min_max = self.vals.iter().cloned().minmax();
-                    let (val_min, val_max) = val_min_max.into_option().unwrap_or((0u64, 0u64));
+                    let val_min_max = crate::common::minmax(self.vals.iter().cloned());
+                    let (val_min, val_max) = val_min_max.unwrap_or((0u64, 0u64));
                     value_serializer =
                         serializer.new_u64_fast_field_with_idx(self.field, val_min, val_max, 1)?;
                     for &val in &self.vals {

src/fastfield/readers.rs

@@ -4,7 +4,6 @@ use crate::fastfield::MultiValueIntFastFieldReader;
 use crate::fastfield::{FastFieldNotAvailableError, FastFieldReader};
 use crate::schema::{Cardinality, Field, FieldType, Schema};
 use crate::space_usage::PerFieldSpaceUsage;
-use crate::Result;
 use std::collections::HashMap;
 
 /// Provides access to all of the FastFieldReader.
@@ -54,7 +53,7 @@ impl FastFieldReaders {
     pub(crate) fn load_all(
         schema: &Schema,
         fast_fields_composite: &CompositeFile,
-    ) -> Result<FastFieldReaders> {
+    ) -> crate::Result<FastFieldReaders> {
         let mut fast_field_readers = FastFieldReaders {
             fast_field_i64: Default::default(),
             fast_field_u64: Default::default(),

src/indexer/index_writer.rs

@@ -8,22 +8,26 @@ use crate::core::SegmentComponent;
 use crate::core::SegmentId;
 use crate::core::SegmentMeta;
 use crate::core::SegmentReader;
-use crate::directory::TerminatingWrite;
 use crate::directory::{DirectoryLock, GarbageCollectionResult};
+use crate::directory::{TerminatingWrite, WatchCallbackList};
 use crate::docset::DocSet;
 use crate::error::TantivyError;
 use crate::fastfield::write_delete_bitset;
 use crate::indexer::delete_queue::{DeleteCursor, DeleteQueue};
 use crate::indexer::doc_opstamp_mapping::DocToOpstampMapping;
 use crate::indexer::operation::DeleteOperation;
+use crate::indexer::segment_manager::SegmentRegisters;
+use crate::indexer::segment_register::SegmentRegister;
 use crate::indexer::stamper::Stamper;
-use crate::indexer::MergePolicy;
-use crate::indexer::SegmentEntry;
+use crate::indexer::{SegmentEntry, ResourceManager};
 use crate::indexer::SegmentWriter;
+use crate::indexer::{IndexWriterConfig, MergePolicy};
+use crate::reader::NRTReader;
 use crate::schema::Document;
 use crate::schema::IndexRecordOption;
 use crate::schema::Term;
-use crate::Opstamp;
+use crate::tokenizer::TokenizerManager;
+use crate::{IndexReader, Opstamp};
 use crossbeam::channel;
 use futures::executor::block_on;
 use futures::future::Future;
@@ -31,18 +35,10 @@ use smallvec::smallvec;
 use smallvec::SmallVec;
 use std::mem;
 use std::ops::Range;
-use std::sync::Arc;
+use std::sync::{Arc, RwLock};
 use std::thread;
 use std::thread::JoinHandle;
 
-// Size of the margin for the heap. A segment is closed when the remaining memory
-// in the heap goes below MARGIN_IN_BYTES.
-pub const MARGIN_IN_BYTES: usize = 1_000_000;
-
-// We impose the memory per thread to be at least 3 MB.
-pub const HEAP_SIZE_MIN: usize = ((MARGIN_IN_BYTES as u32) * 3u32) as usize;
-pub const HEAP_SIZE_MAX: usize = u32::max_value() as usize - MARGIN_IN_BYTES;
-
 // Add document will block if the number of docs waiting in the queue to be indexed
 // reaches `PIPELINE_MAX_SIZE_IN_DOCS`
 const PIPELINE_MAX_SIZE_IN_DOCS: usize = 10_000;
@@ -69,8 +65,9 @@ pub struct IndexWriter {
     _directory_lock: Option<DirectoryLock>,
 
     index: Index,
+    config: IndexWriterConfig,
 
-    heap_size_in_bytes_per_thread: usize,
+    segment_registers: Arc<RwLock<SegmentRegisters>>,
 
     workers_join_handle: Vec<JoinHandle<crate::Result<()>>>,
 
@@ -80,13 +77,14 @@ pub struct IndexWriter {
     segment_updater: SegmentUpdater,
 
     worker_id: usize,
-
-    num_threads: usize,
-
     delete_queue: DeleteQueue,
 
     stamper: Stamper,
     committed_opstamp: Opstamp,
+
+    on_commit: WatchCallbackList,
+
+    memory_manager: ResourceManager,
 }
 
 fn compute_deleted_bitset(
@@ -133,7 +131,6 @@ fn compute_deleted_bitset(
 /// For instance, there was no delete operation between the state of the `segment_entry` and
 /// the `target_opstamp`, `segment_entry` is not updated.
 pub(crate) fn advance_deletes(
-    mut segment: Segment,
     segment_entry: &mut SegmentEntry,
     target_opstamp: Opstamp,
 ) -> crate::Result<()> {
@@ -142,28 +139,40 @@ pub(crate) fn advance_deletes(
         return Ok(());
     }
 
-    if segment_entry.delete_bitset().is_none() && segment_entry.delete_cursor().get().is_none() {
+    let delete_bitset_opt = segment_entry.take_delete_bitset();
+
+    // We avoid directly advancing the `SegmentEntry` delete cursor, because
+    // we do not want to end up in an invalid state if the delete bitset
+    // serialization fails.
+    let mut delete_cursor = segment_entry.delete_cursor();
+
+    if delete_bitset_opt.is_none() && segment_entry.delete_cursor().get().is_none() {
         // There has been no `DeleteOperation` between the segment status and `target_opstamp`.
         return Ok(());
     }
 
+    // We open our current serialized segment to compute the new deleted bitset.
+    let segment = segment_entry.segment().clone();
     let segment_reader = SegmentReader::open(&segment)?;
 
     let max_doc = segment_reader.max_doc();
-    let mut delete_bitset: BitSet = match segment_entry.delete_bitset() {
-        Some(previous_delete_bitset) => (*previous_delete_bitset).clone(),
-        None => BitSet::with_max_value(max_doc),
-    };
+
+    let mut delete_bitset: BitSet =
+        delete_bitset_opt.unwrap_or_else(|| BitSet::with_max_value(max_doc));
+
+    let num_deleted_docs_before = segment.meta().num_deleted_docs();
 
     compute_deleted_bitset(
         &mut delete_bitset,
         &segment_reader,
-        segment_entry.delete_cursor(),
+        &mut delete_cursor,
         &DocToOpstampMapping::None,
         target_opstamp,
     )?;
 
     // TODO optimize
+    // It should be possible to do something smarter by manipulation bitsets directly
+    // to compute this union.
     if let Some(seg_delete_bitset) = segment_reader.delete_bitset() {
         for doc in 0u32..max_doc {
             if seg_delete_bitset.is_deleted(doc) {
@@ -172,34 +181,43 @@ pub(crate) fn advance_deletes(
         }
     }
 
-    let num_deleted_docs = delete_bitset.len();
-    if num_deleted_docs > 0 {
-        segment = segment.with_delete_meta(num_deleted_docs as u32, target_opstamp);
-        let mut delete_file = segment.open_write(SegmentComponent::DELETE)?;
+    let num_deleted_docs: u32 = delete_bitset.len() as u32;
+    if num_deleted_docs > num_deleted_docs_before {
+        // There are new deletes. We need to write a new delete file.
+        let mut delete_file = segment
+            .with_delete_meta(num_deleted_docs as u32, target_opstamp)
+            .open_write(SegmentComponent::DELETE)?;
         write_delete_bitset(&delete_bitset, max_doc, &mut delete_file)?;
         delete_file.terminate()?;
+        segment_entry.reset_delete_meta(num_deleted_docs as u32, target_opstamp);
     }
 
-    segment_entry.set_meta(segment.meta().clone());
+    // Regardless of whether we did end up having to write a new file or not
+    // we advance the `delete_cursor`. This is an optimisation. We want to ensure we do not
+    // check that a given deleted term does not match any of our docs more than once.
+    segment_entry.set_delete_cursor(delete_cursor);
     Ok(())
 }
 
 fn index_documents(
-    memory_budget: usize,
+    config: IndexWriterConfig,
     segment: Segment,
     grouped_document_iterator: &mut dyn Iterator<Item = OperationGroup>,
     segment_updater: &mut SegmentUpdater,
+    tokenizers: &TokenizerManager,
     mut delete_cursor: DeleteCursor,
+    memory_manager: ResourceManager
 ) -> crate::Result<bool> {
     let schema = segment.schema();
 
-    let mut segment_writer = SegmentWriter::for_segment(memory_budget, segment.clone(), &schema)?;
+    let mut segment_writer = SegmentWriter::for_segment(
+        &config, segment, &schema, tokenizers, memory_manager)?;
     for document_group in grouped_document_iterator {
         for doc in document_group {
             segment_writer.add_document(doc, &schema)?;
         }
         let mem_usage = segment_writer.mem_usage();
-        if mem_usage >= memory_budget - MARGIN_IN_BYTES {
+        if mem_usage >= config.heap_size_before_flushing() {
             info!(
                 "Buffer limit reached, flushing segment with maxdoc={}.",
                 segment_writer.max_doc()
@@ -218,24 +236,14 @@ fn index_documents(
     // the worker thread.
     assert!(max_doc > 0);
 
-    let doc_opstamps: Vec<Opstamp> = segment_writer.finalize()?;
-
-    let segment_with_max_doc = segment.with_max_doc(max_doc);
+    let (segment, doc_opstamps): (Segment, Vec<Opstamp>) = segment_writer.finalize()?;
 
     let last_docstamp: Opstamp = *(doc_opstamps.last().unwrap());
 
-    let delete_bitset_opt = apply_deletes(
-        &segment_with_max_doc,
-        &mut delete_cursor,
-        &doc_opstamps,
-        last_docstamp,
-    )?;
+    let delete_bitset_opt =
+        apply_deletes(&segment, &mut delete_cursor, &doc_opstamps, last_docstamp)?;
 
-    let segment_entry = SegmentEntry::new(
-        segment_with_max_doc.meta().clone(),
-        delete_cursor,
-        delete_bitset_opt,
-    );
+    let segment_entry = SegmentEntry::new(segment, delete_cursor, delete_bitset_opt);
     block_on(segment_updater.schedule_add_segment(segment_entry))?;
     Ok(true)
 }
@@ -287,21 +295,10 @@ impl IndexWriter {
     /// If the heap size per thread is too small, panics.
     pub(crate) fn new(
         index: &Index,
-        num_threads: usize,
-        heap_size_in_bytes_per_thread: usize,
+        mut config: IndexWriterConfig,
         directory_lock: DirectoryLock,
     ) -> crate::Result<IndexWriter> {
-        if heap_size_in_bytes_per_thread < HEAP_SIZE_MIN {
-            let err_msg = format!(
-                "The heap size per thread needs to be at least {}.",
-                HEAP_SIZE_MIN
-            );
-            return Err(TantivyError::InvalidArgument(err_msg));
-        }
-        if heap_size_in_bytes_per_thread >= HEAP_SIZE_MAX {
-            let err_msg = format!("The heap size per thread cannot exceed {}", HEAP_SIZE_MAX);
-            return Err(TantivyError::InvalidArgument(err_msg));
-        }
+        config.validate()?;
         let (document_sender, document_receiver): (OperationSender, OperationReceiver) =
             channel::bounded(PIPELINE_MAX_SIZE_IN_DOCS);
 
@@ -309,16 +306,26 @@ impl IndexWriter {
 
         let current_opstamp = index.load_metas()?.opstamp;
 
+        let meta = index.load_metas()?;
+
         let stamper = Stamper::new(current_opstamp);
 
+        let commited_segments = SegmentRegister::new(
+            index.directory(),
+            &index.schema(),
+            meta.segments,
+            &delete_queue.cursor(),
+        );
+        let segment_registers = Arc::new(RwLock::new(SegmentRegisters::new(commited_segments)));
+
         let segment_updater =
-            SegmentUpdater::create(index.clone(), stamper.clone(), &delete_queue.cursor())?;
+            SegmentUpdater::create(segment_registers.clone(), index.clone(), stamper.clone())?;
 
         let mut index_writer = IndexWriter {
             _directory_lock: Some(directory_lock),
 
-            heap_size_in_bytes_per_thread,
             index: index.clone(),
+            config,
 
             operation_receiver: document_receiver,
             operation_sender: document_sender,
@@ -326,7 +333,6 @@ impl IndexWriter {
             segment_updater,
 
             workers_join_handle: vec![],
-            num_threads,
 
             delete_queue,
 
@@ -334,6 +340,10 @@ impl IndexWriter {
             stamper,
 
             worker_id: 0,
+            segment_registers,
+            on_commit: Default::default(),
+
+            memory_manager: Default::default()
         };
         index_writer.start_workers()?;
         Ok(index_writer)
@@ -361,23 +371,11 @@ impl IndexWriter {
                 })?;
         }
 
-        let result = self
+        self
             .segment_updater
-            .wait_merging_thread()
-            .map_err(|_| TantivyError::ErrorInThread("Failed to join merging thread.".into()));
+            .wait_merging_thread();
 
-        if let Err(ref e) = result {
-            error!("Some merging thread failed {:?}", e);
-        }
-
-        result
-    }
-
-    #[doc(hidden)]
-    pub fn add_segment(&self, segment_meta: SegmentMeta) -> crate::Result<()> {
-        let delete_cursor = self.delete_queue.cursor();
-        let segment_entry = SegmentEntry::new(segment_meta, delete_cursor, None);
-        block_on(self.segment_updater.schedule_add_segment(segment_entry))
+        Ok(())
     }
 
     /// Creates a new segment.
@@ -400,8 +398,9 @@ impl IndexWriter {
 
         let mut delete_cursor = self.delete_queue.cursor();
 
-        let mem_budget = self.heap_size_in_bytes_per_thread;
         let index = self.index.clone();
+        let config = self.config.clone();
+        let memory_manager = self.memory_manager.clone();
         let join_handle: JoinHandle<crate::Result<()>> = thread::Builder::new()
             .name(format!("thrd-tantivy-index{}", self.worker_id))
             .spawn(move || {
@@ -430,11 +429,13 @@ impl IndexWriter {
                     }
                     let segment = index.new_segment();
                     index_documents(
-                        mem_budget,
+                        config.clone(),
                         segment,
                         &mut document_iterator,
                         &mut segment_updater,
+                        index.tokenizers(),
                         delete_cursor.clone(),
+                        memory_manager.clone()
                     )?;
                 }
             })?;
@@ -454,7 +455,7 @@ impl IndexWriter {
     }
 
     fn start_workers(&mut self) -> crate::Result<()> {
-        for _ in 0..self.num_threads {
+        for _ in 0..self.config.max_indexing_threads {
             self.add_indexing_worker()?;
         }
         Ok(())
@@ -559,12 +560,8 @@ impl IndexWriter {
             .take()
             .expect("The IndexWriter does not have any lock. This is a bug, please report.");
 
-        let new_index_writer: IndexWriter = IndexWriter::new(
-            &self.index,
-            self.num_threads,
-            self.heap_size_in_bytes_per_thread,
-            directory_lock,
-        )?;
+        let new_index_writer: IndexWriter =
+            IndexWriter::new(&self.index, self.config.clone(), directory_lock)?;
 
         // the current `self` is dropped right away because of this call.
         //
@@ -603,7 +600,7 @@ impl IndexWriter {
     /// It is also possible to add a payload to the `commit`
     /// using this API.
     /// See [`PreparedCommit::set_payload()`](PreparedCommit.html)
-    pub fn prepare_commit(&mut self) -> crate::Result<PreparedCommit> {
+    pub fn prepare_commit(&mut self, soft_commit: bool) -> crate::Result<PreparedCommit> {
         // Here, because we join all of the worker threads,
         // all of the segment update for this commit have been
         // sent.
@@ -631,7 +628,7 @@ impl IndexWriter {
         }
 
         let commit_opstamp = self.stamper.stamp();
-        let prepared_commit = PreparedCommit::new(self, commit_opstamp);
+        let prepared_commit = PreparedCommit::new(self, commit_opstamp, soft_commit);
         info!("Prepared commit {}", commit_opstamp);
         Ok(prepared_commit)
     }
@@ -651,7 +648,25 @@ impl IndexWriter {
     /// that made it in the commit.
     ///
     pub fn commit(&mut self) -> crate::Result<Opstamp> {
-        self.prepare_commit()?.commit()
+        self.prepare_commit(false)?.commit()
+    }
+
+    pub fn soft_commit(&mut self) -> crate::Result<Opstamp> {
+        self.prepare_commit(true)?.commit()
+    }
+
+    pub(crate) fn trigger_commit(&self) -> impl Future<Output = ()> {
+        self.on_commit.broadcast()
+    }
+
+    pub fn reader(&self, num_searchers: usize) -> crate::Result<IndexReader> {
+        let nrt_reader = NRTReader::create(
+            num_searchers,
+            self.index.clone(),
+            self.segment_registers.clone(),
+            &self.on_commit,
+        )?;
+        Ok(IndexReader::NRT(nrt_reader))
     }
 
     pub(crate) fn segment_updater(&self) -> &SegmentUpdater {
@@ -803,6 +818,46 @@ mod tests {
         assert_eq!(batch_opstamp1, 2u64);
     }
 
+    #[test]
+    fn test_no_need_to_rewrite_delete_file_if_no_new_deletes() {
+        let mut schema_builder = schema::Schema::builder();
+        let text_field = schema_builder.add_text_field("text", schema::TEXT);
+        let index = Index::create_in_ram(schema_builder.build());
+
+        let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
+        index_writer.add_document(doc!(text_field => "hello1"));
+        index_writer.add_document(doc!(text_field => "hello2"));
+        assert!(index_writer.commit().is_ok());
+
+        let reader = index.reader().unwrap();
+        let searcher = reader.searcher();
+        assert_eq!(searcher.segment_readers().len(), 1);
+        assert_eq!(searcher.segment_reader(0u32).num_deleted_docs(), 0);
+
+        index_writer.delete_term(Term::from_field_text(text_field, "hello1"));
+        assert!(index_writer.commit().is_ok());
+
+        assert!(reader.reload().is_ok());
+        let searcher = reader.searcher();
+        assert_eq!(searcher.segment_readers().len(), 1);
+        assert_eq!(searcher.segment_reader(0u32).num_deleted_docs(), 1);
+
+        let previous_delete_opstamp = index.load_metas().unwrap().segments[0].delete_opstamp();
+
+        // All docs containing hello1 have been already removed.
+        // We should not update the delete meta.
+        index_writer.delete_term(Term::from_field_text(text_field, "hello1"));
+        assert!(index_writer.commit().is_ok());
+
+        assert!(reader.reload().is_ok());
+        let searcher = reader.searcher();
+        assert_eq!(searcher.segment_readers().len(), 1);
+        assert_eq!(searcher.segment_reader(0u32).num_deleted_docs(), 1);
+
+        let after_delete_opstamp = index.load_metas().unwrap().segments[0].delete_opstamp();
+        assert_eq!(after_delete_opstamp, previous_delete_opstamp);
+    }
+
     #[test]
     fn test_ordered_batched_operations() {
         // * one delete for `doc!(field=>"a")`
@@ -897,7 +952,7 @@ mod tests {
         let index_writer = index.writer(3_000_000).unwrap();
         assert_eq!(
             format!("{:?}", index_writer.get_merge_policy()),
-            "LogMergePolicy { min_merge_size: 8, min_layer_size: 10000, \
+            "LogMergePolicy { min_merge_size: 8, max_merge_size: 10000000, min_layer_size: 10000, \
              level_log_size: 0.75 }"
         );
         let merge_policy = Box::new(NoMergePolicy::default());
@@ -1009,7 +1064,8 @@ mod tests {
                 index_writer.add_document(doc!(text_field => "a"));
             }
             {
-                let mut prepared_commit = index_writer.prepare_commit().expect("commit failed");
+                let mut prepared_commit =
+                    index_writer.prepare_commit(false).expect("commit failed");
                 prepared_commit.set_payload("first commit");
                 prepared_commit.commit().expect("commit failed");
             }
@@ -1042,7 +1098,8 @@ mod tests {
                 index_writer.add_document(doc!(text_field => "a"));
             }
             {
-                let mut prepared_commit = index_writer.prepare_commit().expect("commit failed");
+                let mut prepared_commit =
+                    index_writer.prepare_commit(false).expect("commit failed");
                 prepared_commit.set_payload("first commit");
                 prepared_commit.abort().expect("commit failed");
             }
@@ -1220,4 +1277,41 @@ mod tests {
         let commit = index_writer.commit();
         assert!(commit.is_ok());
     }
+
+    #[test]
+    fn test_index_writer_reader() {
+        let mut schema_builder = schema::Schema::builder();
+        let idfield = schema_builder.add_text_field("id", STRING);
+        schema_builder.add_text_field("optfield", STRING);
+        let index = Index::create_in_ram(schema_builder.build());
+        let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
+        index_writer.add_document(doc!(idfield=>"myid"));
+        assert!(index_writer.commit().is_ok());
+        let reader = index_writer.reader(2).unwrap();
+        let searcher = reader.searcher();
+        assert_eq!(searcher.num_docs(), 1u64);
+        index_writer.add_document(doc!(idfield=>"myid"));
+        assert!(index_writer.commit().is_ok());
+        assert_eq!(reader.searcher().num_docs(), 2u64);
+        assert_eq!(searcher.num_docs(), 1u64);
+    }
+
+    #[test]
+    fn test_index_writer_reader_soft_commit() {
+        let mut schema_builder = schema::Schema::builder();
+        let idfield = schema_builder.add_text_field("id", STRING);
+        schema_builder.add_text_field("optfield", STRING);
+        let index = Index::create_in_ram(schema_builder.build());
+        let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
+        index_writer.add_document(doc!(idfield=>"myid"));
+        assert!(index_writer.soft_commit().is_ok());
+        let nrt_reader = index_writer.reader(2).unwrap();
+        let normal_reader = index.reader_builder().try_into().unwrap();
+        assert_eq!(nrt_reader.searcher().num_docs(), 1u64);
+        assert_eq!(normal_reader.searcher().num_docs(), 0u64);
+        assert!(index_writer.commit().is_ok());
+        assert!(normal_reader.reload().is_ok());
+        assert_eq!(nrt_reader.searcher().num_docs(), 1u64);
+        assert_eq!(normal_reader.searcher().num_docs(), 1u64);
+    }
 }

src/indexer/index_writer_config.rs

@@ -0,0 +1,133 @@
+use serde::{Deserialize, Serialize};
+
+// Size of the margin for the heap. A segment is closed when the remaining memory
+// in the heap goes below MARGIN_IN_BYTES.
+const MARGIN_IN_BYTES: u64 = 1_000_000;
+
+// We impose the memory per thread to be at least 3 MB.
+const HEAP_SIZE_MIN: u64 = MARGIN_IN_BYTES * 3u64;
+const HEAP_SIZE_MAX: u64 = u32::max_value() as u64 - MARGIN_IN_BYTES;
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct IndexWriterConfig {
+    pub max_indexing_threads: usize,
+    pub max_merging_threads: usize,
+    pub memory_budget: u64,
+    pub store_flush_num_bytes: u64,
+    pub persist_low: u64,
+    pub persist_high: u64,
+}
+
+impl Default for IndexWriterConfig {
+    fn default() -> Self {
+        IndexWriterConfig {
+            max_indexing_threads: 1,
+            max_merging_threads: 3,
+            memory_budget: 50_000_000u64,
+            store_flush_num_bytes: 10_000_000u64,
+            persist_low: 10_000_000u64,
+            persist_high: 50_000_000u64
+        }
+    }
+}
+
+impl IndexWriterConfig {
+    #[cfg(test)]
+    pub fn for_test() -> IndexWriterConfig {
+        IndexWriterConfig {
+            max_indexing_threads: 1,
+            max_merging_threads: 5,
+            memory_budget: 4_000_000u64,
+            store_flush_num_bytes: 500_000u64,
+            persist_low: 2_000_000u64,
+            persist_high: 3_000_000u64,
+        }
+    }
+
+    // Ensures the `IndexWriterConfig` is correct.
+    //
+    // This method checks that the values in the `IndexWriterConfig`
+    // are valid. If it is not, it may mutate some of the values (like `max_num_threads`) to
+    // fit the contracts or return an error with an explicit error message.
+    //
+    // If called twice, the config is guaranteed to not be updated the second time.
+    pub fn validate(&mut self) -> crate::Result<()> {
+        if self.memory_budget < HEAP_SIZE_MIN {
+            let err_msg = format!(
+                "The heap size per thread needs to be at least {}.",
+                HEAP_SIZE_MIN
+            );
+            return Err(crate::TantivyError::InvalidArgument(err_msg));
+        }
+        let heap_size_in_bytes_per_thread = self.heap_size_in_byte_per_thread();
+        if heap_size_in_bytes_per_thread >= HEAP_SIZE_MAX {
+            let err_msg = format!("The heap size per thread cannot exceed {}", HEAP_SIZE_MAX);
+            return Err(crate::TantivyError::InvalidArgument(err_msg));
+        }
+        if heap_size_in_bytes_per_thread < HEAP_SIZE_MIN {
+            self.max_indexing_threads = (self.memory_budget / HEAP_SIZE_MIN) as usize;
+        }
+        Ok(())
+    }
+
+    pub fn heap_size_in_byte_per_thread(&self) -> u64 {
+        self.memory_budget / self.max_indexing_threads as u64
+    }
+
+    pub fn heap_size_before_flushing(&self) -> u64 {
+        self.heap_size_in_byte_per_thread() - MARGIN_IN_BYTES
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::IndexWriterConfig;
+
+    #[test]
+    fn test_index_writer_config_simple() {
+        let mut index = IndexWriterConfig {
+            max_indexing_threads: 3,
+            memory_budget: super::HEAP_SIZE_MIN * 3,
+            ..Default::default()
+        };
+        assert!(index.validate().is_ok());
+        assert_eq!(index.max_indexing_threads, 3);
+        assert_eq!(index.heap_size_in_byte_per_thread(), super::HEAP_SIZE_MIN);
+    }
+
+    #[test]
+    fn test_index_writer_config_reduce_num_threads() {
+        let mut index = IndexWriterConfig {
+            max_indexing_threads: 3,
+            memory_budget: super::HEAP_SIZE_MIN,
+            ..Default::default()
+        };
+        assert!(index.validate().is_ok());
+        assert_eq!(index.max_indexing_threads, 1);
+        assert_eq!(index.heap_size_in_byte_per_thread(), super::HEAP_SIZE_MIN);
+    }
+
+    #[test]
+    fn test_index_writer_config_not_enough_memory() {
+        let mut index = IndexWriterConfig {
+            max_indexing_threads: 1,
+            memory_budget: super::HEAP_SIZE_MIN - 1,
+            ..Default::default()
+        };
+        assert!(
+            matches!(index.validate(), Err(crate::TantivyError::InvalidArgument(msg) ) if msg.contains("The heap size per thread needs to be at least"))
+        );
+    }
+
+    #[test]
+    fn test_index_writer_config_too_much_memory() {
+        let mut index = IndexWriterConfig {
+            max_indexing_threads: 1,
+            memory_budget: (u32::max_value() as u64) + 1,
+            ..Default::default()
+        };
+        assert!(
+            matches!(index.validate(), Err(crate::TantivyError::InvalidArgument(msg) ) if msg.contains("The heap size per thread cannot exceed"))
+        );
+    }
+}

src/indexer/log_merge_policy.rs

@@ -6,12 +6,14 @@ use std::f64;
 const DEFAULT_LEVEL_LOG_SIZE: f64 = 0.75;
 const DEFAULT_MIN_LAYER_SIZE: u32 = 10_000;
 const DEFAULT_MIN_MERGE_SIZE: usize = 8;
+const DEFAULT_MAX_MERGE_SIZE: usize = 10_000_000;
 
 /// `LogMergePolicy` tries tries to merge segments that have a similar number of
 /// documents.
 #[derive(Debug, Clone)]
 pub struct LogMergePolicy {
     min_merge_size: usize,
+    max_merge_size: usize,
     min_layer_size: u32,
     level_log_size: f64,
 }
@@ -26,6 +28,12 @@ impl LogMergePolicy {
         self.min_merge_size = min_merge_size;
     }
 
+    /// Set the maximum number docs in a segment for it to be considered for
+    /// merging.
+    pub fn set_max_merge_size(&mut self, max_merge_size: usize) {
+        self.max_merge_size = max_merge_size;
+    }
+
     /// Set the minimum segment size under which all segment belong
     /// to the same level.
     pub fn set_min_layer_size(&mut self, min_layer_size: u32) {
@@ -53,6 +61,7 @@ impl MergePolicy for LogMergePolicy {
         let mut size_sorted_tuples = segments
             .iter()
             .map(SegmentMeta::num_docs)
+            .filter(|s| s <= &(self.max_merge_size as u32))
             .enumerate()
             .collect::<Vec<(usize, u32)>>();
 
@@ -86,6 +95,7 @@ impl Default for LogMergePolicy {
     fn default() -> LogMergePolicy {
         LogMergePolicy {
             min_merge_size: DEFAULT_MIN_MERGE_SIZE,
+            max_merge_size: DEFAULT_MAX_MERGE_SIZE,
             min_layer_size: DEFAULT_MIN_LAYER_SIZE,
             level_log_size: DEFAULT_LEVEL_LOG_SIZE,
         }
@@ -104,6 +114,7 @@ mod tests {
     fn test_merge_policy() -> LogMergePolicy {
         let mut log_merge_policy = LogMergePolicy::default();
         log_merge_policy.set_min_merge_size(3);
+        log_merge_policy.set_max_merge_size(100_000);
         log_merge_policy.set_min_layer_size(2);
         log_merge_policy
     }
@@ -141,11 +152,11 @@ mod tests {
             create_random_segment_meta(10),
             create_random_segment_meta(10),
             create_random_segment_meta(10),
-            create_random_segment_meta(1000),
-            create_random_segment_meta(1000),
-            create_random_segment_meta(1000),
-            create_random_segment_meta(10000),
-            create_random_segment_meta(10000),
+            create_random_segment_meta(1_000),
+            create_random_segment_meta(1_000),
+            create_random_segment_meta(1_000),
+            create_random_segment_meta(10_000),
+            create_random_segment_meta(10_000),
             create_random_segment_meta(10),
             create_random_segment_meta(10),
             create_random_segment_meta(10),
@@ -182,4 +193,19 @@ mod tests {
         let result_list = test_merge_policy().compute_merge_candidates(&test_input);
         assert_eq!(result_list.len(), 1);
     }
+
+    #[test]
+    fn test_large_merge_segments() {
+        let test_input = vec![
+            create_random_segment_meta(1_000_000),
+            create_random_segment_meta(100_001),
+            create_random_segment_meta(100_000),
+            create_random_segment_meta(100_000),
+            create_random_segment_meta(100_000),
+        ];
+        let result_list = test_merge_policy().compute_merge_candidates(&test_input);
+        // Do not include large segments
+        assert_eq!(result_list.len(), 1);
+        assert_eq!(result_list[0].0.len(), 3)
+    }
 }

src/indexer/merge_operation.rs

@@ -1,17 +1,22 @@
+use crate::indexer::resource_manager::{Allocation, ResourceManager};
 use crate::Opstamp;
 use crate::SegmentId;
 use census::{Inventory, TrackedObject};
 use std::collections::HashSet;
+use std::fmt;
 use std::ops::Deref;
 
-#[derive(Default)]
-pub(crate) struct MergeOperationInventory(Inventory<InnerMergeOperation>);
+#[derive(Default, Clone)]
+pub(crate) struct MergeOperationInventory {
+    inventory: Inventory<InnerMergeOperation>,
+    num_merge_watcher: ResourceManager,
+}
 
 impl Deref for MergeOperationInventory {
     type Target = Inventory<InnerMergeOperation>;
 
     fn deref(&self) -> &Self::Target {
-        &self.0
+        &self.inventory
     }
 }
 
@@ -25,6 +30,10 @@ impl MergeOperationInventory {
         }
         segment_in_merge
     }
+
+    pub fn wait_until_empty(&self) {
+        let _ = self.num_merge_watcher.wait_until_in_range(0..1);
+    }
 }
 
 /// A `MergeOperation` has two roles.
@@ -47,6 +56,17 @@ pub struct MergeOperation {
 pub(crate) struct InnerMergeOperation {
     target_opstamp: Opstamp,
     segment_ids: Vec<SegmentId>,
+    _allocation: Allocation,
+}
+
+impl fmt::Debug for InnerMergeOperation {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(
+            f,
+            "MergeOp(target_opstamp={:?}, segment_ids={:?})",
+            self.target_opstamp, self.segment_ids
+        )
+    }
 }
 
 impl MergeOperation {
@@ -55,9 +75,11 @@ impl MergeOperation {
         target_opstamp: Opstamp,
         segment_ids: Vec<SegmentId>,
     ) -> MergeOperation {
+        let allocation = inventory.num_merge_watcher.allocate(1);
         let inner_merge_operation = InnerMergeOperation {
             target_opstamp,
             segment_ids,
+            _allocation: allocation,
         };
         MergeOperation {
             inner: inventory.track(inner_merge_operation),

src/indexer/merger.rs

@@ -2,6 +2,8 @@ use crate::common::MAX_DOC_LIMIT;
 use crate::core::Segment;
 use crate::core::SegmentReader;
 use crate::core::SerializableSegment;
+use crate::directory::TerminatingWrite;
+use crate::directory::WritePtr;
 use crate::docset::DocSet;
 use crate::fastfield::BytesFastFieldReader;
 use crate::fastfield::DeleteBitSet;
@@ -21,9 +23,6 @@ use crate::store::StoreWriter;
 use crate::termdict::TermMerger;
 use crate::termdict::TermOrdinal;
 use crate::DocId;
-use crate::Result;
-use crate::TantivyError;
-use itertools::Itertools;
 use std::cmp;
 use std::collections::HashMap;
 
@@ -72,11 +71,11 @@ fn compute_min_max_val(
             Some(delete_bitset) => {
                 // some deleted documents,
                 // we need to recompute the max / min
-                (0..max_doc)
-                    .filter(|doc_id| delete_bitset.is_alive(*doc_id))
-                    .map(|doc_id| u64_reader.get(doc_id))
-                    .minmax()
-                    .into_option()
+                crate::common::minmax(
+                    (0..max_doc)
+                        .filter(|doc_id| delete_bitset.is_alive(*doc_id))
+                        .map(|doc_id| u64_reader.get(doc_id)),
+                )
             }
             None => {
                 // no deleted documents,
@@ -143,7 +142,7 @@ impl DeltaComputer {
 }
 
 impl IndexMerger {
-    pub fn open(schema: Schema, segments: &[Segment]) -> Result<IndexMerger> {
+    pub fn open(schema: Schema, segments: &[Segment]) -> crate::Result<IndexMerger> {
         let mut readers = vec![];
         let mut max_doc: u32 = 0u32;
         for segment in segments {
@@ -159,7 +158,7 @@ impl IndexMerger {
                  which exceeds the limit {}.",
                 max_doc, MAX_DOC_LIMIT
             );
-            return Err(TantivyError::InvalidArgument(err_msg));
+            return Err(crate::TantivyError::InvalidArgument(err_msg));
         }
         Ok(IndexMerger {
             schema,
@@ -168,7 +167,10 @@ impl IndexMerger {
         })
     }
 
-    fn write_fieldnorms(&self, fieldnorms_serializer: &mut FieldNormsSerializer) -> Result<()> {
+    fn write_fieldnorms(
+        &self,
+        fieldnorms_serializer: &mut FieldNormsSerializer,
+    ) -> crate::Result<()> {
         let fields = FieldNormsWriter::fields_with_fieldnorm(&self.schema);
         let mut fieldnorms_data = Vec::with_capacity(self.max_doc as usize);
         for field in fields {
@@ -189,7 +191,7 @@ impl IndexMerger {
         &self,
         fast_field_serializer: &mut FastFieldSerializer,
         mut term_ord_mappings: HashMap<Field, TermOrdinalMapping>,
-    ) -> Result<()> {
+    ) -> crate::Result<()> {
         for (field, field_entry) in self.schema.fields() {
             let field_type = field_entry.field_type();
             match *field_type {
@@ -234,7 +236,7 @@ impl IndexMerger {
         &self,
         field: Field,
         fast_field_serializer: &mut FastFieldSerializer,
-    ) -> Result<()> {
+    ) -> crate::Result<()> {
         let mut u64_readers = vec![];
         let mut min_value = u64::max_value();
         let mut max_value = u64::min_value();
@@ -284,7 +286,7 @@ impl IndexMerger {
         &self,
         field: Field,
         fast_field_serializer: &mut FastFieldSerializer,
-    ) -> Result<()> {
+    ) -> crate::Result<()> {
         let mut total_num_vals = 0u64;
         let mut u64s_readers: Vec<MultiValueIntFastFieldReader<u64>> = Vec::new();
 
@@ -331,7 +333,7 @@ impl IndexMerger {
         field: Field,
         term_ordinal_mappings: &TermOrdinalMapping,
         fast_field_serializer: &mut FastFieldSerializer,
-    ) -> Result<()> {
+    ) -> crate::Result<()> {
         // Multifastfield consists in 2 fastfields.
         // The first serves as an index into the second one and is stricly increasing.
         // The second contains the actual values.
@@ -371,7 +373,7 @@ impl IndexMerger {
         &self,
         field: Field,
         fast_field_serializer: &mut FastFieldSerializer,
-    ) -> Result<()> {
+    ) -> crate::Result<()> {
         // Multifastfield consists in 2 fastfields.
         // The first serves as an index into the second one and is stricly increasing.
         // The second contains the actual values.
@@ -436,7 +438,7 @@ impl IndexMerger {
         &self,
         field: Field,
         fast_field_serializer: &mut FastFieldSerializer,
-    ) -> Result<()> {
+    ) -> crate::Result<()> {
         let mut total_num_vals = 0u64;
         let mut bytes_readers: Vec<BytesFastFieldReader> = Vec::new();
 
@@ -492,7 +494,7 @@ impl IndexMerger {
         indexed_field: Field,
         field_type: &FieldType,
         serializer: &mut InvertedIndexSerializer,
-    ) -> Result<Option<TermOrdinalMapping>> {
+    ) -> crate::Result<Option<TermOrdinalMapping>> {
         let mut positions_buffer: Vec<u32> = Vec::with_capacity(1_000);
         let mut delta_computer = DeltaComputer::new();
         let field_readers = self
@@ -646,7 +648,7 @@ impl IndexMerger {
     fn write_postings(
         &self,
         serializer: &mut InvertedIndexSerializer,
-    ) -> Result<HashMap<Field, TermOrdinalMapping>> {
+    ) -> crate::Result<HashMap<Field, TermOrdinalMapping>> {
         let mut term_ordinal_mappings = HashMap::new();
         for (field, field_entry) in self.schema.fields() {
             if field_entry.is_indexed() {
@@ -660,7 +662,8 @@ impl IndexMerger {
         Ok(term_ordinal_mappings)
     }
 
-    fn write_storable_fields(&self, store_writer: &mut StoreWriter) -> Result<()> {
+    pub fn write_storable_fields(&self, store_wrt: WritePtr) -> crate::Result<()> {
+        let mut store_writer = StoreWriter::new(store_wrt);
         for reader in &self.readers {
             let store_reader = reader.get_store_reader();
             if reader.num_deleted_docs() > 0 {
@@ -672,16 +675,17 @@ impl IndexMerger {
                 store_writer.stack(&store_reader)?;
             }
         }
+        let store_wrt = store_writer.close()?;
+        store_wrt.terminate()?;
         Ok(())
     }
 }
 
 impl SerializableSegment for IndexMerger {
-    fn write(&self, mut serializer: SegmentSerializer) -> Result<u32> {
+    fn write(&self, mut serializer: SegmentSerializer) -> crate::Result<u32> {
         let term_ord_mappings = self.write_postings(serializer.get_postings_serializer())?;
         self.write_fieldnorms(serializer.get_fieldnorms_serializer())?;
         self.write_fast_fields(serializer.get_fast_field_serializer(), term_ord_mappings)?;
-        self.write_storable_fields(serializer.get_store_writer())?;
         serializer.close()?;
         Ok(self.max_doc)
     }

src/indexer/mod.rs

@@ -2,12 +2,14 @@ pub mod delete_queue;
 
 mod doc_opstamp_mapping;
 pub mod index_writer;
+mod index_writer_config;
 mod log_merge_policy;
 mod merge_operation;
 pub mod merge_policy;
 pub mod merger;
 pub mod operation;
 mod prepared_commit;
+mod resource_manager;
 mod segment_entry;
 mod segment_manager;
 mod segment_register;
@@ -16,13 +18,17 @@ pub mod segment_updater;
 mod segment_writer;
 mod stamper;
 
+pub(crate) use self::resource_manager::{Allocation, ResourceManager};
+pub(crate) use self::merge_operation::MergeOperationInventory;
 pub use self::index_writer::IndexWriter;
+pub use self::index_writer_config::IndexWriterConfig;
 pub use self::log_merge_policy::LogMergePolicy;
 pub use self::merge_operation::MergeOperation;
 pub use self::merge_policy::{MergeCandidate, MergePolicy, NoMergePolicy};
 pub use self::prepared_commit::PreparedCommit;
 pub use self::segment_entry::SegmentEntry;
 pub use self::segment_manager::SegmentManager;
+pub(crate) use self::segment_manager::SegmentRegisters;
 pub use self::segment_serializer::SegmentSerializer;
 pub use self::segment_writer::SegmentWriter;
 

src/indexer/operation.rs

@@ -19,6 +19,8 @@ pub struct AddOperation {
 /// UserOperation is an enum type that encapsulates other operation types.
 #[derive(Eq, PartialEq, Debug)]
 pub enum UserOperation {
+    /// Add operation
     Add(Document),
+    /// Delete operation
     Delete(Term),
 }

src/indexer/prepared_commit.rs

@@ -1,6 +1,5 @@
 use super::IndexWriter;
 use crate::Opstamp;
-use crate::Result;
 use futures::executor::block_on;
 
 /// A prepared commit
@@ -8,14 +7,20 @@ pub struct PreparedCommit<'a> {
     index_writer: &'a mut IndexWriter,
     payload: Option<String>,
     opstamp: Opstamp,
+    soft_commit: bool,
 }
 
 impl<'a> PreparedCommit<'a> {
-    pub(crate) fn new(index_writer: &'a mut IndexWriter, opstamp: Opstamp) -> PreparedCommit<'_> {
+    pub(crate) fn new(
+        index_writer: &'a mut IndexWriter,
+        opstamp: Opstamp,
+        soft_commit: bool,
+    ) -> PreparedCommit<'_> {
         PreparedCommit {
             index_writer,
             payload: None,
             opstamp,
+            soft_commit,
         }
     }
 
@@ -27,17 +32,18 @@ impl<'a> PreparedCommit<'a> {
         self.payload = Some(payload.to_string())
     }
 
-    pub fn abort(self) -> Result<Opstamp> {
+    pub fn abort(self) -> crate::Result<Opstamp> {
         self.index_writer.rollback()
     }
 
-    pub fn commit(self) -> Result<Opstamp> {
+    pub fn commit(self) -> crate::Result<Opstamp> {
         info!("committing {}", self.opstamp);
-        let _ = block_on(
-            self.index_writer
-                .segment_updater()
-                .schedule_commit(self.opstamp, self.payload),
-        );
+        block_on(self.index_writer.segment_updater().schedule_commit(
+            self.opstamp,
+            self.payload,
+            self.soft_commit,
+        ))?;
+        block_on(self.index_writer.trigger_commit());
         Ok(self.opstamp)
     }
 }

src/indexer/resource_manager.rs

@@ -0,0 +1,213 @@
+use std::ops::RangeBounds;
+use std::sync::{Arc, Condvar, Mutex, MutexGuard, RwLock};
+
+struct LockedData {
+    count: u64,
+    enabled: bool
+}
+
+impl Default for LockedData {
+    fn default() -> Self {
+        LockedData {
+            count: 0u64,
+            enabled: true
+        }
+    }
+}
+
+#[derive(Default)]
+struct Inner {
+    resource_level: Mutex<LockedData>,
+    convdvar: Condvar,
+}
+
+
+/// The resource manager makes it possible to track the amount of level of a given resource.
+/// There is no magic here : it is to the description of the user to declare how much
+/// of the resource is being held.
+///
+/// Allocation of a resource is bound to the lifetime of a `Allocation` instance.
+///
+/// ```rust
+/// let resource_manager = ResourceManager::default();
+///
+/// ```
+///
+/// In tantivy, this is used to check the number of merging thread and the number of memory
+/// used by the volatile segments.
+///
+#[derive(Clone, Default)]
+pub struct ResourceManager {
+    inner: Arc<Inner>,
+}
+
+impl ResourceManager {
+    /// Return the total amount of reousrce allocated
+    pub fn total_amount(&self) -> u64 {
+        self.lock().count
+    }
+
+    fn lock(&self) -> MutexGuard<LockedData> {
+        self.inner
+            .resource_level
+            .lock()
+            .expect("Failed to obtain lock for ReservedMemory. This should never happen.")
+    }
+
+    fn record_delta(&self, delta: i64) {
+        if delta == 0i64 {
+            return;
+        }
+        let mut lock = self.lock();
+        let new_val = lock.count as i64 + delta;
+        lock.count = new_val as u64;
+        self.inner.convdvar.notify_all();
+    }
+
+    /// Records a new allocation.
+    ///
+    /// The returned allocate object is used to automatically release the allocated resource
+    /// on drop.
+    pub fn allocate(&self, amount: u64) -> Allocation {
+        self.record_delta(amount as i64);
+        Allocation {
+            resource_manager: self.clone(),
+            amount: RwLock::new(amount),
+        }
+    }
+
+    /// Stops the resource manager.
+    ///
+    /// If any thread is waiting via `.wait_until_in_range(...)`, the method will stop
+    /// being blocking and will return an error.
+    pub fn terminate(&self) {
+        self.lock().enabled = false;
+        self.inner.convdvar.notify_all();
+    }
+
+    /// Blocks the current thread until the resource level reaches the given range,
+    /// in a cpu-efficient way.
+    ///
+    /// This method does not necessarily wakes up the current thread at every transition
+    /// into the targetted range, but any durable entry in the range will be detected.
+    pub fn wait_until_in_range<R: RangeBounds<u64>>(&self, range: R) -> Result<u64, u64> {
+        let mut levels = self.lock();
+        if !levels.enabled {
+            return Err(levels.count)
+        }
+        while !range.contains(&levels.count) {
+           levels = self.inner.convdvar.wait(levels).unwrap();
+            if !levels.enabled {
+                return Err(levels.count)
+            }
+        }
+        Ok(levels.count)
+    }
+}
+
+pub struct Allocation {
+    resource_manager: ResourceManager,
+    amount: RwLock<u64>,
+}
+
+impl Allocation {
+    pub fn amount(&self) -> u64 {
+        *self.amount.read().unwrap()
+    }
+
+    pub fn modify(&self, new_amount: u64) {
+        let mut wlock = self.amount.write().unwrap();
+        let delta = new_amount as i64 - *wlock as i64;
+        *wlock = new_amount;
+        self.resource_manager.record_delta(delta);
+    }
+}
+
+impl Drop for Allocation {
+    fn drop(&mut self) {
+        let amount = self.amount();
+        self.resource_manager.record_delta(-(amount as i64))
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::ResourceManager;
+    use futures::channel::oneshot;
+    use futures::executor::block_on;
+    use std::{mem, thread};
+
+    #[test]
+    fn test_simple_allocation() {
+        let memory = ResourceManager::default();
+        assert_eq!(memory.total_amount(), 0u64);
+        let _allocation = memory.allocate(10u64);
+        assert_eq!(memory.total_amount(), 10u64);
+    }
+
+    #[test]
+    fn test_multiple_allocation() {
+        let memory = ResourceManager::default();
+        assert_eq!(memory.total_amount(), 0u64);
+        let _allocation = memory.allocate(10u64);
+        let _allocation_2 = memory.allocate(11u64);
+        assert_eq!(memory.total_amount(), 21u64);
+    }
+
+    #[test]
+    fn test_release_on_drop() {
+        let memory = ResourceManager::default();
+        assert_eq!(memory.total_amount(), 0u64);
+        let allocation = memory.allocate(10u64);
+        let allocation_2 = memory.allocate(11u64);
+        assert_eq!(memory.total_amount(), 21u64);
+        mem::drop(allocation);
+        assert_eq!(memory.total_amount(), 11u64);
+        mem::drop(allocation_2);
+        assert_eq!(memory.total_amount(), 0u64);
+    }
+
+    #[test]
+    fn test_wait_until() {
+        let memory = ResourceManager::default();
+        let (send, recv) = oneshot::channel::<()>();
+        let memory_clone = memory.clone();
+        thread::spawn(move || {
+            let _allocation1 = memory_clone.allocate(2u64);
+            let _allocation2 = memory_clone.allocate(3u64);
+            let _allocation3 = memory_clone.allocate(4u64);
+            std::mem::drop(_allocation3);
+            assert!(block_on(recv).is_ok());
+        });
+        assert_eq!(memory.wait_until_in_range(5u64..8u64), Ok(5u64));
+        assert!(send.send(()).is_ok());
+    }
+
+    #[test]
+    fn test_modify_amount() {
+        let memory = ResourceManager::default();
+        let alloc = memory.allocate(2u64);
+        assert_eq!(memory.total_amount(), 2u64);
+        assert_eq!(alloc.amount(), 2u64);
+        let alloc2 = memory.allocate(3u64);
+        assert_eq!(memory.total_amount(), 2u64 + 3u64);
+        assert_eq!(alloc2.amount(), 3u64);
+        alloc.modify(14u64);
+        assert_eq!(alloc.amount(), 14u64);
+        assert_eq!(memory.total_amount(), 14u64 + 3u64)
+    }
+    
+    #[test]
+    fn test_stop_resource_manager() {
+        let resource_manager = ResourceManager::default();
+        let resource_manager_clone = resource_manager.clone();
+        let (sender, recv) = oneshot::channel();
+        let join_handle = thread::spawn(move || {
+            assert!(sender.send(()).is_ok());
+            resource_manager_clone.wait_until_in_range(10..20)
+        });
+        let _ = block_on(recv);
+        resource_manager.terminate();
+        assert_eq!(join_handle.join().unwrap(), Err(0u64));
+    }
+}

src/indexer/segment_entry.rs

@@ -1,7 +1,9 @@
 use crate::common::BitSet;
 use crate::core::SegmentId;
 use crate::core::SegmentMeta;
+use crate::directory::ManagedDirectory;
 use crate::indexer::delete_queue::DeleteCursor;
+use crate::{Opstamp, Segment};
 use std::fmt;
 
 /// A segment entry describes the state of
@@ -19,7 +21,7 @@ use std::fmt;
 /// in the .del file or in the `delete_bitset`.
 #[derive(Clone)]
 pub struct SegmentEntry {
-    meta: SegmentMeta,
+    segment: Segment,
     delete_bitset: Option<BitSet>,
     delete_cursor: DeleteCursor,
 }
@@ -27,47 +29,67 @@ pub struct SegmentEntry {
 impl SegmentEntry {
     /// Create a new `SegmentEntry`
     pub fn new(
-        segment_meta: SegmentMeta,
+        segment: Segment,
         delete_cursor: DeleteCursor,
         delete_bitset: Option<BitSet>,
     ) -> SegmentEntry {
         SegmentEntry {
-            meta: segment_meta,
+            segment,
             delete_bitset,
             delete_cursor,
         }
     }
 
-    /// Return a reference to the segment entry deleted bitset.
-    ///
-    /// `DocId` in this bitset are flagged as deleted.
-    pub fn delete_bitset(&self) -> Option<&BitSet> {
-        self.delete_bitset.as_ref()
+    pub fn persist(&mut self, dest_directory: ManagedDirectory) -> crate::Result<()> {
+        // TODO take in account delete bitset?
+        self.segment.persist(dest_directory)?;
+        Ok(())
     }
 
-    /// Set the `SegmentMeta` for this segment.
-    pub fn set_meta(&mut self, segment_meta: SegmentMeta) {
-        self.meta = segment_meta;
+    pub fn set_delete_cursor(&mut self, delete_cursor: DeleteCursor) {
+        self.delete_cursor = delete_cursor;
+    }
+
+    /// `Takes` (as in Option::take) the delete bitset of a segment entry.
+    /// `DocId` in this bitset are flagged as deleted.
+    pub fn take_delete_bitset(&mut self) -> Option<BitSet> {
+        self.delete_bitset.take()
+    }
+
+    /// Reset the delete information in this segment.
+    ///
+    /// The `SegmentEntry` segment's `SegmentMeta` gets updated, and
+    /// any delete bitset is drop and set to None.
+    pub fn reset_delete_meta(&mut self, num_deleted_docs: u32, target_opstamp: Opstamp) {
+        self.segment = self
+            .segment
+            .clone()
+            .with_delete_meta(num_deleted_docs, target_opstamp);
+        self.delete_bitset = None;
     }
 
     /// Return a reference to the segment_entry's delete cursor
-    pub fn delete_cursor(&mut self) -> &mut DeleteCursor {
-        &mut self.delete_cursor
+    pub fn delete_cursor(&mut self) -> DeleteCursor {
+        self.delete_cursor.clone()
     }
 
     /// Returns the segment id.
     pub fn segment_id(&self) -> SegmentId {
-        self.meta.id()
+        self.meta().id()
     }
 
+    /// Returns the `segment` associated to the `SegmentEntry`.
+    pub fn segment(&self) -> &Segment {
+        &self.segment
+    }
     /// Accessor to the `SegmentMeta`
     pub fn meta(&self) -> &SegmentMeta {
-        &self.meta
+        self.segment.meta()
     }
 }
 
 impl fmt::Debug for SegmentEntry {
     fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(formatter, "SegmentEntry({:?})", self.meta)
+        write!(formatter, "SegmentEntry({:?})", self.meta())
     }
 }

src/indexer/segment_manager.rs

@@ -2,16 +2,15 @@ use super::segment_register::SegmentRegister;
 use crate::core::SegmentId;
 use crate::core::SegmentMeta;
 use crate::error::TantivyError;
-use crate::indexer::delete_queue::DeleteCursor;
-use crate::indexer::SegmentEntry;
-use crate::Result as TantivyResult;
+use crate::indexer::{SegmentEntry, MergeOperationInventory, MergeCandidate, MergeOperation};
+use crate::{Segment, Opstamp};
 use std::collections::hash_set::HashSet;
 use std::fmt::{self, Debug, Formatter};
-use std::sync::RwLock;
+use std::sync::{Arc, RwLock};
 use std::sync::{RwLockReadGuard, RwLockWriteGuard};
 
 #[derive(Default)]
-struct SegmentRegisters {
+pub(crate) struct SegmentRegisters {
     uncommitted: SegmentRegister,
     committed: SegmentRegister,
 }
@@ -23,6 +22,17 @@ pub(crate) enum SegmentsStatus {
 }
 
 impl SegmentRegisters {
+    pub fn new(committed: SegmentRegister) -> SegmentRegisters {
+        SegmentRegisters {
+            uncommitted: Default::default(),
+            committed,
+        }
+    }
+
+    pub fn committed_segment(&self) -> Vec<Segment> {
+        self.committed.segments()
+    }
+
     /// Check if all the segments are committed or uncommited.
     ///
     /// If some segment is missing or segments are in a different state (this should not happen
@@ -45,11 +55,12 @@ impl SegmentRegisters {
 /// changes (merges especially)
 #[derive(Default)]
 pub struct SegmentManager {
-    registers: RwLock<SegmentRegisters>,
+    registers: Arc<RwLock<SegmentRegisters>>,
+    merge_operations: MergeOperationInventory,
 }
 
 impl Debug for SegmentManager {
-    fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), fmt::Error> {
+    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
         let lock = self.read();
         write!(
             f,
@@ -59,34 +70,28 @@ impl Debug for SegmentManager {
     }
 }
 
-pub fn get_mergeable_segments(
-    in_merge_segment_ids: &HashSet<SegmentId>,
-    segment_manager: &SegmentManager,
-) -> (Vec<SegmentMeta>, Vec<SegmentMeta>) {
-    let registers_lock = segment_manager.read();
-    (
-        registers_lock
-            .committed
-            .get_mergeable_segments(in_merge_segment_ids),
-        registers_lock
-            .uncommitted
-            .get_mergeable_segments(in_merge_segment_ids),
-    )
-}
 
 impl SegmentManager {
-    pub fn from_segments(
-        segment_metas: Vec<SegmentMeta>,
-        delete_cursor: &DeleteCursor,
-    ) -> SegmentManager {
+    pub(crate) fn new(registers: Arc<RwLock<SegmentRegisters>>) -> SegmentManager {
         SegmentManager {
-            registers: RwLock::new(SegmentRegisters {
-                uncommitted: SegmentRegister::default(),
-                committed: SegmentRegister::new(segment_metas, delete_cursor),
-            }),
+            registers,
+            merge_operations: Default::default()
         }
     }
 
+    pub fn new_merge_operation(&self, opstamp: Opstamp, merge_candidate: MergeCandidate) -> MergeOperation {
+        MergeOperation::new(
+            &self.merge_operations,
+            opstamp,
+            merge_candidate.0
+
+        )
+    }
+
+    pub fn wait_merging_thread(&self) {
+        self.merge_operations.wait_until_empty()
+    }
+
     /// Returns all of the segment entries (committed or uncommitted)
     pub fn segment_entries(&self) -> Vec<SegmentEntry> {
         let registers_lock = self.read();
@@ -95,6 +100,34 @@ impl SegmentManager {
         segment_entries
     }
 
+    /// Returns the segments that are currently not in merge.
+    ///
+    /// They are split over two `Vec`. The committed segments on one hand
+    /// and the uncommitted ones on the other hand.
+    ///
+    /// This method is useful when searching for merge candidate or segments
+    /// to persists.
+    pub fn segments_not_in_merge(&self) -> (Vec<SegmentMeta>, Vec<SegmentMeta>) {
+        let in_merge_segment_ids: HashSet<SegmentId> = self.merge_operations.segment_in_merge();
+        let registers_lock = self.read();
+        (
+            registers_lock
+                .committed
+                .get_mergeable_segments(&in_merge_segment_ids),
+            registers_lock
+                .uncommitted
+                .get_mergeable_segments(&in_merge_segment_ids),
+        )
+    }
+
+    pub fn largest_segment_not_in_merge(&self) -> Option<SegmentMeta> {
+        let (committed, uncommitted) = self.segments_not_in_merge();
+        let mut segments = vec![];
+        segments.extend(committed);
+        segments.extend(uncommitted);
+        None
+    }
+
     // Lock poisoning should never happen :
     // The lock is acquired and released within this class,
     // and the operations cannot panic.
@@ -145,7 +178,7 @@ impl SegmentManager {
     /// Returns an error if some segments are missing, or if
     /// the `segment_ids` are not either all committed or all
     /// uncommitted.
-    pub fn start_merge(&self, segment_ids: &[SegmentId]) -> TantivyResult<Vec<SegmentEntry>> {
+    pub fn start_merge(&self, segment_ids: &[SegmentId]) -> crate::Result<Vec<SegmentEntry>> {
         let registers_lock = self.read();
         let mut segment_entries = vec![];
         if registers_lock.uncommitted.contains_all(segment_ids) {
@@ -188,7 +221,7 @@ impl SegmentManager {
             .segments_status(before_merge_segment_ids)
             .ok_or_else(|| {
                 warn!("couldn't find segment in SegmentManager");
-                crate::Error::InvalidArgument(
+                crate::TantivyError::InvalidArgument(
                     "The segments that were merged could not be found in the SegmentManager. \
                      This is not necessarily a bug, and can happen after a rollback for instance."
                         .to_string(),

src/indexer/segment_register.rs

@@ -1,7 +1,10 @@
 use crate::core::SegmentId;
 use crate::core::SegmentMeta;
+use crate::directory::ManagedDirectory;
 use crate::indexer::delete_queue::DeleteCursor;
 use crate::indexer::segment_entry::SegmentEntry;
+use crate::schema::Schema;
+use crate::Segment;
 use std::collections::HashMap;
 use std::collections::HashSet;
 use std::fmt::{self, Debug, Formatter};
@@ -46,6 +49,13 @@ impl SegmentRegister {
             .collect()
     }
 
+    pub fn segments(&self) -> Vec<Segment> {
+        self.segment_states
+            .values()
+            .map(|segment_entry| segment_entry.segment().clone())
+            .collect()
+    }
+
     pub fn segment_entries(&self) -> Vec<SegmentEntry> {
         self.segment_states.values().cloned().collect()
     }
@@ -79,11 +89,17 @@ impl SegmentRegister {
         self.segment_states.get(segment_id).cloned()
     }
 
-    pub fn new(segment_metas: Vec<SegmentMeta>, delete_cursor: &DeleteCursor) -> SegmentRegister {
+    pub fn new(
+        directory: &ManagedDirectory,
+        schema: &Schema,
+        segment_metas: Vec<SegmentMeta>,
+        delete_cursor: &DeleteCursor,
+    ) -> SegmentRegister {
         let mut segment_states = HashMap::new();
         for segment_meta in segment_metas {
             let segment_id = segment_meta.id();
-            let segment_entry = SegmentEntry::new(segment_meta, delete_cursor.clone(), None);
+            let segment = Segment::new_persisted(segment_meta, directory.clone(), schema.clone());
+            let segment_entry = SegmentEntry::new(segment, delete_cursor.clone(), None);
             segment_states.insert(segment_id, segment_entry);
         }
         SegmentRegister { segment_states }
@@ -95,6 +111,7 @@ mod tests {
     use super::*;
     use crate::core::{SegmentId, SegmentMetaInventory};
     use crate::indexer::delete_queue::*;
+    use crate::indexer::ResourceManager;
 
     fn segment_ids(segment_register: &SegmentRegister) -> Vec<SegmentId> {
         segment_register
@@ -108,28 +125,34 @@ mod tests {
     fn test_segment_register() {
         let inventory = SegmentMetaInventory::default();
         let delete_queue = DeleteQueue::new();
+        let schema = Schema::builder().build();
 
         let mut segment_register = SegmentRegister::default();
         let segment_id_a = SegmentId::generate_random();
         let segment_id_b = SegmentId::generate_random();
         let segment_id_merged = SegmentId::generate_random();
 
+        let memory_manager = ResourceManager::default();
+
         {
-            let segment_meta = inventory.new_segment_meta(segment_id_a, 0u32);
-            let segment_entry = SegmentEntry::new(segment_meta, delete_queue.cursor(), None);
+            let meta = inventory.new_segment_meta(segment_id_a, 0u32);
+            let segment = Segment::new_volatile(meta, schema.clone(), memory_manager.clone());
+            let segment_entry = SegmentEntry::new(segment, delete_queue.cursor(), None);
             segment_register.add_segment_entry(segment_entry);
         }
         assert_eq!(segment_ids(&segment_register), vec![segment_id_a]);
         {
-            let segment_meta = inventory.new_segment_meta(segment_id_b, 0u32);
-            let segment_entry = SegmentEntry::new(segment_meta, delete_queue.cursor(), None);
+            let meta = inventory.new_segment_meta(segment_id_b, 0u32);
+            let segment = Segment::new_volatile(meta, schema.clone(), memory_manager.clone());
+            let segment_entry = SegmentEntry::new(segment, delete_queue.cursor(), None);
             segment_register.add_segment_entry(segment_entry);
         }
         segment_register.remove_segment(&segment_id_a);
         segment_register.remove_segment(&segment_id_b);
         {
             let segment_meta_merged = inventory.new_segment_meta(segment_id_merged, 0u32);
-            let segment_entry = SegmentEntry::new(segment_meta_merged, delete_queue.cursor(), None);
+            let segment_merged = Segment::new_volatile(segment_meta_merged, schema.clone(), memory_manager.clone());
+            let segment_entry = SegmentEntry::new(segment_merged, delete_queue.cursor(), None);
             segment_register.add_segment_entry(segment_entry);
         }
         assert_eq!(segment_ids(&segment_register), vec![segment_id_merged]);

src/indexer/segment_serializer.rs

@@ -1,16 +1,12 @@
-use crate::Result;
-
 use crate::core::Segment;
 use crate::core::SegmentComponent;
 use crate::fastfield::FastFieldSerializer;
 use crate::fieldnorm::FieldNormsSerializer;
 use crate::postings::InvertedIndexSerializer;
-use crate::store::StoreWriter;
 
 /// Segment serializer is in charge of laying out on disk
 /// the data accumulated and sorted by the `SegmentWriter`.
 pub struct SegmentSerializer {
-    store_writer: StoreWriter,
     fast_field_serializer: FastFieldSerializer,
     fieldnorms_serializer: FieldNormsSerializer,
     postings_serializer: InvertedIndexSerializer,
@@ -18,9 +14,7 @@ pub struct SegmentSerializer {
 
 impl SegmentSerializer {
     /// Creates a new `SegmentSerializer`.
-    pub fn for_segment(segment: &mut Segment) -> Result<SegmentSerializer> {
-        let store_write = segment.open_write(SegmentComponent::STORE)?;
-
+    pub fn for_segment(segment: &mut Segment) -> crate::Result<SegmentSerializer> {
         let fast_field_write = segment.open_write(SegmentComponent::FASTFIELDS)?;
         let fast_field_serializer = FastFieldSerializer::from_write(fast_field_write)?;
 
@@ -29,7 +23,6 @@ impl SegmentSerializer {
 
         let postings_serializer = InvertedIndexSerializer::open(segment)?;
         Ok(SegmentSerializer {
-            store_writer: StoreWriter::new(store_write),
             fast_field_serializer,
             fieldnorms_serializer,
             postings_serializer,
@@ -51,16 +44,10 @@ impl SegmentSerializer {
         &mut self.fieldnorms_serializer
     }
 
-    /// Accessor to the `StoreWriter`.
-    pub fn get_store_writer(&mut self) -> &mut StoreWriter {
-        &mut self.store_writer
-    }
-
     /// Finalize the segment serialization.
-    pub fn close(self) -> Result<()> {
+    pub fn close(self) -> crate::Result<()> {
         self.fast_field_serializer.close()?;
         self.postings_serializer.close()?;
-        self.store_writer.close()?;
         self.fieldnorms_serializer.close()?;
         Ok(())
     }

src/indexer/segment_updater.rs

@@ -1,4 +1,4 @@
-use super::segment_manager::{get_mergeable_segments, SegmentManager};
+use super::segment_manager::SegmentManager;
 use crate::core::Index;
 use crate::core::IndexMeta;
 use crate::core::Segment;
@@ -7,18 +7,16 @@ use crate::core::SegmentMeta;
 use crate::core::SerializableSegment;
 use crate::core::META_FILEPATH;
 use crate::directory::{Directory, DirectoryClone, GarbageCollectionResult};
-use crate::indexer::delete_queue::DeleteCursor;
 use crate::indexer::index_writer::advance_deletes;
-use crate::indexer::merge_operation::MergeOperationInventory;
 use crate::indexer::merger::IndexMerger;
-use crate::indexer::segment_manager::SegmentsStatus;
+use crate::indexer::segment_manager::{SegmentRegisters, SegmentsStatus};
 use crate::indexer::stamper::Stamper;
 use crate::indexer::SegmentEntry;
 use crate::indexer::SegmentSerializer;
 use crate::indexer::{DefaultMergePolicy, MergePolicy};
 use crate::indexer::{MergeCandidate, MergeOperation};
 use crate::schema::Schema;
-use crate::Opstamp;
+use crate::{Opstamp, SegmentComponent};
 use futures::channel::oneshot;
 use futures::executor::{ThreadPool, ThreadPoolBuilder};
 use futures::future::Future;
@@ -117,15 +115,14 @@ fn merge(
 
     // First we apply all of the delet to the merged segment, up to the target opstamp.
     for segment_entry in &mut segment_entries {
-        let segment = index.segment(segment_entry.meta().clone());
-        advance_deletes(segment, segment_entry, target_opstamp)?;
+        advance_deletes(segment_entry, target_opstamp)?;
     }
 
-    let delete_cursor = segment_entries[0].delete_cursor().clone();
+    let delete_cursor = segment_entries[0].delete_cursor();
 
     let segments: Vec<Segment> = segment_entries
         .iter()
-        .map(|segment_entry| index.segment(segment_entry.meta().clone()))
+        .map(|segment_entry| segment_entry.segment().clone())
         .collect();
 
     // An IndexMerger is like a "view" of our merged segments.
@@ -134,11 +131,16 @@ fn merge(
     // ... we just serialize this index merger in our new segment to merge the two segments.
     let segment_serializer = SegmentSerializer::for_segment(&mut merged_segment)?;
 
-    let num_docs = merger.write(segment_serializer)?;
+    let store_wrt = merged_segment.open_write(SegmentComponent::STORE)?;
+    merger.write_storable_fields(store_wrt)?;
 
-    let segment_meta = index.new_segment_meta(merged_segment.id(), num_docs);
+    let max_doc = merger.write(segment_serializer)?;
 
-    Ok(SegmentEntry::new(segment_meta, delete_cursor, None))
+    Ok(SegmentEntry::new(
+        merged_segment.with_max_doc(max_doc),
+        delete_cursor,
+        None,
+    ))
 }
 
 pub(crate) struct InnerSegmentUpdater {
@@ -157,30 +159,32 @@ pub(crate) struct InnerSegmentUpdater {
     merge_policy: RwLock<Arc<Box<dyn MergePolicy>>>,
     killed: AtomicBool,
     stamper: Stamper,
-    merge_operations: MergeOperationInventory,
 }
 
 impl SegmentUpdater {
     pub fn create(
+        segment_registers: Arc<RwLock<SegmentRegisters>>,
         index: Index,
         stamper: Stamper,
-        delete_cursor: &DeleteCursor,
     ) -> crate::Result<SegmentUpdater> {
-        let segments = index.searchable_segment_metas()?;
-        let segment_manager = SegmentManager::from_segments(segments, delete_cursor);
+        let segment_manager = SegmentManager::new(segment_registers);
         let pool = ThreadPoolBuilder::new()
             .name_prefix("segment_updater")
             .pool_size(1)
             .create()
             .map_err(|_| {
-                crate::Error::SystemError("Failed to spawn segment updater thread".to_string())
+                crate::TantivyError::SystemError(
+                    "Failed to spawn segment updater thread".to_string(),
+                )
             })?;
         let merge_thread_pool = ThreadPoolBuilder::new()
             .name_prefix("merge_thread")
             .pool_size(NUM_MERGE_THREADS)
             .create()
             .map_err(|_| {
-                crate::Error::SystemError("Failed to spawn segment merging thread".to_string())
+                crate::TantivyError::SystemError(
+                    "Failed to spawn segment merging thread".to_string(),
+                )
             })?;
         let index_meta = index.load_metas()?;
         Ok(SegmentUpdater(Arc::new(InnerSegmentUpdater {
@@ -192,7 +196,6 @@ impl SegmentUpdater {
             merge_policy: RwLock::new(Arc::new(Box::new(DefaultMergePolicy::default()))),
             killed: AtomicBool::new(false),
             stamper,
-            merge_operations: Default::default(),
         })))
     }
 
@@ -222,7 +225,7 @@ impl SegmentUpdater {
         receiver.unwrap_or_else(|_| {
             let err_msg =
                 "A segment_updater future did not success. This should never happen.".to_string();
-            Err(crate::Error::SystemError(err_msg))
+            Err(crate::TantivyError::SystemError(err_msg))
         })
     }
 
@@ -258,8 +261,7 @@ impl SegmentUpdater {
     fn purge_deletes(&self, target_opstamp: Opstamp) -> crate::Result<Vec<SegmentEntry>> {
         let mut segment_entries = self.segment_manager.segment_entries();
         for segment_entry in &mut segment_entries {
-            let segment = self.index.segment(segment_entry.meta().clone());
-            advance_deletes(segment, segment_entry, target_opstamp)?;
+            advance_deletes(segment_entry, target_opstamp)?;
         }
         Ok(segment_entries)
     }
@@ -327,12 +329,21 @@ impl SegmentUpdater {
         &self,
         opstamp: Opstamp,
         payload: Option<String>,
+        soft_commit: bool,
     ) -> impl Future<Output = crate::Result<()>> {
         let segment_updater: SegmentUpdater = self.clone();
+        let directory = self.index.directory().clone();
         self.schedule_future(async move {
-            let segment_entries = segment_updater.purge_deletes(opstamp)?;
+            let mut segment_entries = segment_updater.purge_deletes(opstamp)?;
+            if !soft_commit {
+                for segment_entry in &mut segment_entries {
+                    segment_entry.persist(directory.clone())?;
+                }
+            }
             segment_updater.segment_manager.commit(segment_entries);
-            segment_updater.save_metas(opstamp, payload)?;
+            if !soft_commit {
+                segment_updater.save_metas(opstamp, payload)?;
+            }
             let _ = garbage_collect_files(segment_updater.clone()).await;
             segment_updater.consider_merge_options().await;
             Ok(())
@@ -349,7 +360,7 @@ impl SegmentUpdater {
 
     pub(crate) fn make_merge_operation(&self, segment_ids: &[SegmentId]) -> MergeOperation {
         let commit_opstamp = self.load_metas().opstamp;
-        MergeOperation::new(&self.merge_operations, commit_opstamp, segment_ids.to_vec())
+        self.segment_manager.new_merge_operation(commit_opstamp, MergeCandidate(segment_ids.to_vec()))
     }
 
     // Starts a merge operation. This function will block until the merge operation is effectively
@@ -419,13 +430,12 @@ impl SegmentUpdater {
         });
 
         Ok(merging_future_recv
-            .unwrap_or_else(|_| Err(crate::Error::SystemError("Merge failed".to_string()))))
+            .unwrap_or_else(|_| Err(crate::TantivyError::SystemError("Merge failed".to_string()))))
     }
 
     async fn consider_merge_options(&self) {
-        let merge_segment_ids: HashSet<SegmentId> = self.merge_operations.segment_in_merge();
         let (committed_segments, uncommitted_segments) =
-            get_mergeable_segments(&merge_segment_ids, &self.segment_manager);
+            self.segment_manager.segments_not_in_merge();
 
         // Committed segments cannot be merged with uncommitted_segments.
         // We therefore consider merges using these two sets of segments independently.
@@ -436,7 +446,7 @@ impl SegmentUpdater {
             .compute_merge_candidates(&uncommitted_segments)
             .into_iter()
             .map(|merge_candidate| {
-                MergeOperation::new(&self.merge_operations, current_opstamp, merge_candidate.0)
+                self.segment_manager.new_merge_operation(current_opstamp, merge_candidate)
             })
             .collect();
 
@@ -445,7 +455,7 @@ impl SegmentUpdater {
             .compute_merge_candidates(&committed_segments)
             .into_iter()
             .map(|merge_candidate: MergeCandidate| {
-                MergeOperation::new(&self.merge_operations, commit_opstamp, merge_candidate.0)
+                self.segment_manager.new_merge_operation(commit_opstamp, merge_candidate)
             })
             .collect::<Vec<_>>();
         merge_candidates.extend(committed_merge_candidates.into_iter());
@@ -470,17 +480,13 @@ impl SegmentUpdater {
         let end_merge_future = self.schedule_future(async move {
             info!("End merge {:?}", after_merge_segment_entry.meta());
             {
-                let mut delete_cursor = after_merge_segment_entry.delete_cursor().clone();
+                let mut delete_cursor = after_merge_segment_entry.delete_cursor();
                 if let Some(delete_operation) = delete_cursor.get() {
                     let committed_opstamp = segment_updater.load_metas().opstamp;
                     if delete_operation.opstamp < committed_opstamp {
-                        let index = &segment_updater.index;
-                        let segment = index.segment(after_merge_segment_entry.meta().clone());
-                        if let Err(e) = advance_deletes(
-                            segment,
-                            &mut after_merge_segment_entry,
-                            committed_opstamp,
-                        ) {
+                        if let Err(e) =
+                            advance_deletes(&mut after_merge_segment_entry, committed_opstamp)
+                        {
                             error!(
                                 "Merge of {:?} was cancelled (advancing deletes failed): {:?}",
                                 merge_operation.segment_ids(),
@@ -529,9 +535,8 @@ impl SegmentUpdater {
     ///
     /// Obsolete files will eventually be cleaned up
     /// by the directory garbage collector.
-    pub fn wait_merging_thread(&self) -> crate::Result<()> {
-        self.merge_operations.wait_until_empty();
-        Ok(())
+    pub fn wait_merging_thread(&self) {
+        self.segment_manager.wait_merging_thread()
     }
 }
 
@@ -540,7 +545,8 @@ mod tests {
 
     use crate::indexer::merge_policy::tests::MergeWheneverPossible;
     use crate::schema::*;
-    use crate::Index;
+    use crate::{Index, SegmentId};
+    use futures::executor::block_on;
 
     #[test]
     fn test_delete_during_merge() {
@@ -691,4 +697,27 @@ mod tests {
             .segment_entries();
         assert!(seg_vec.is_empty());
     }
+
+    #[test]
+    fn test_merge_over_soft_commit() {
+        let mut schema_builder = Schema::builder();
+        let text_field = schema_builder.add_text_field("text", TEXT);
+        let schema = schema_builder.build();
+        let index = Index::create_in_ram(schema);
+        // writing the segment
+        let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
+        index_writer.add_document(doc!(text_field=>"a"));
+        assert!(index_writer.soft_commit().is_ok());
+        index_writer.add_document(doc!(text_field=>"a"));
+        assert!(index_writer.soft_commit().is_ok());
+
+        let reader = index_writer.reader(1).unwrap();
+        let segment_ids: Vec<SegmentId> = reader
+            .searcher()
+            .segment_readers()
+            .iter()
+            .map(|reader| reader.segment_id())
+            .collect();
+        assert!(block_on(index_writer.merge(&segment_ids)).is_ok());
+    }
 }

src/indexer/segment_writer.rs

@@ -1,9 +1,11 @@
 use super::operation::AddOperation;
 use crate::core::Segment;
 use crate::core::SerializableSegment;
+use crate::directory::{SpillingResult, SpillingWriter, TerminatingWrite};
 use crate::fastfield::FastFieldsWriter;
 use crate::fieldnorm::FieldNormsWriter;
 use crate::indexer::segment_serializer::SegmentSerializer;
+use crate::indexer::IndexWriterConfig;
 use crate::postings::compute_table_size;
 use crate::postings::MultiFieldPostingsWriter;
 use crate::schema::FieldType;
@@ -11,29 +13,29 @@ use crate::schema::Schema;
 use crate::schema::Term;
 use crate::schema::Value;
 use crate::schema::{Field, FieldEntry};
-use crate::tokenizer::BoxedTokenizer;
-use crate::tokenizer::FacetTokenizer;
-use crate::tokenizer::PreTokenizedStream;
-use crate::tokenizer::{TokenStream, TokenStreamChain, Tokenizer};
-use crate::DocId;
+use crate::store::StoreWriter;
+use crate::tokenizer::{BoxTokenStream, PreTokenizedStream};
+use crate::tokenizer::{FacetTokenizer, TextAnalyzer, TokenizerManager};
+use crate::tokenizer::{TokenStreamChain, Tokenizer};
 use crate::Opstamp;
-use crate::Result;
-use crate::TantivyError;
+use crate::{DocId, SegmentComponent};
 use std::io;
+use std::io::Write;
 use std::str;
+use crate::indexer::resource_manager::ResourceManager;
 
 /// Computes the initial size of the hash table.
 ///
 /// Returns a number of bit `b`, such that the recommended initial table size is 2^b.
-fn initial_table_size(per_thread_memory_budget: usize) -> Result<usize> {
-    let table_memory_upper_bound = per_thread_memory_budget / 3;
+fn initial_table_size(per_thread_memory_budget: u64) -> crate::Result<usize> {
+    let table_memory_upper_bound = per_thread_memory_budget / 3u64;
     if let Some(limit) = (10..)
-        .take_while(|num_bits: &usize| compute_table_size(*num_bits) < table_memory_upper_bound)
+        .take_while(|num_bits| compute_table_size(*num_bits) < table_memory_upper_bound)
         .last()
     {
         Ok(limit.min(19)) // we cap it at 2^19 = 512K.
     } else {
-        Err(TantivyError::InvalidArgument(
+        Err(crate::TantivyError::InvalidArgument(
             format!("per thread memory budget (={}) is too small. Raise the memory budget or lower the number of threads.", per_thread_memory_budget)))
     }
 }
@@ -46,11 +48,13 @@ fn initial_table_size(per_thread_memory_budget: usize) -> Result<usize> {
 pub struct SegmentWriter {
     max_doc: DocId,
     multifield_postings: MultiFieldPostingsWriter,
-    segment_serializer: SegmentSerializer,
+    segment: Segment,
     fast_field_writers: FastFieldsWriter,
     fieldnorms_writer: FieldNormsWriter,
     doc_opstamps: Vec<Opstamp>,
-    tokenizers: Vec<Option<BoxedTokenizer>>,
+    tokenizers: Vec<Option<TextAnalyzer>>,
+    store_writer: StoreWriter<SpillingWriter>,
+    memory_manager: ResourceManager,
 }
 
 impl SegmentWriter {
@@ -64,12 +68,13 @@ impl SegmentWriter {
     /// - segment: The segment being written
     /// - schema
     pub fn for_segment(
-        memory_budget: usize,
-        mut segment: Segment,
+        config: &IndexWriterConfig,
+        segment: Segment,
         schema: &Schema,
-    ) -> Result<SegmentWriter> {
-        let table_num_bits = initial_table_size(memory_budget)?;
-        let segment_serializer = SegmentSerializer::for_segment(&mut segment)?;
+        tokenizer_manager: &TokenizerManager,
+        memory_manager: ResourceManager
+    ) -> crate::Result<SegmentWriter> {
+        let table_num_bits = initial_table_size(config.heap_size_in_byte_per_thread())?;
         let multifield_postings = MultiFieldPostingsWriter::new(schema, table_num_bits);
         let tokenizers = schema
             .fields()
@@ -79,20 +84,32 @@ impl SegmentWriter {
                         .get_indexing_options()
                         .and_then(|text_index_option| {
                             let tokenizer_name = &text_index_option.tokenizer();
-                            segment.index().tokenizers().get(tokenizer_name)
+                            tokenizer_manager.get(tokenizer_name)
                         }),
                     _ => None,
                 },
             )
             .collect();
+        let mut segment_clone = segment.clone();
+        let spilling_wrt = SpillingWriter::new(
+            50_000_000,
+            Box::new(move || {
+                segment_clone
+                    .open_write(SegmentComponent::STORE)
+                    .map_err(|err| io::Error::new(io::ErrorKind::Other, err))
+            }),
+        );
+        let store_writer = StoreWriter::new(spilling_wrt);
         Ok(SegmentWriter {
             max_doc: 0,
             multifield_postings,
             fieldnorms_writer: FieldNormsWriter::for_schema(schema),
-            segment_serializer,
+            segment,
             fast_field_writers: FastFieldsWriter::from_schema(schema),
             doc_opstamps: Vec::with_capacity(1_000),
             tokenizers,
+            store_writer,
+            memory_manager
         })
     }
 
@@ -100,18 +117,33 @@ impl SegmentWriter {
     ///
     /// Finalize consumes the `SegmentWriter`, so that it cannot
     /// be used afterwards.
-    pub fn finalize(mut self) -> Result<Vec<u64>> {
+    pub fn finalize(mut self) -> crate::Result<(Segment, Vec<u64>)> {
         self.fieldnorms_writer.fill_up_to_max_doc(self.max_doc);
+        let spilling_wrt = self.store_writer.close()?;
+        let mut segment: Segment;
+        match spilling_wrt.finalize()? {
+            SpillingResult::Spilled => {
+                segment = self.segment.clone();
+            }
+            SpillingResult::Buffer(buf) => {
+                segment = self.segment.into_volatile(self.memory_manager.clone());
+                let mut store_wrt = segment.open_write(SegmentComponent::STORE)?;
+                store_wrt.write_all(&buf[..])?;
+                store_wrt.terminate()?;
+            }
+        }
+        let segment_serializer = SegmentSerializer::for_segment(&mut segment)?;
+        segment = segment.with_max_doc(self.max_doc);
         write(
             &self.multifield_postings,
             &self.fast_field_writers,
             &self.fieldnorms_writer,
-            self.segment_serializer,
+            segment_serializer,
         )?;
-        Ok(self.doc_opstamps)
+        Ok((segment, self.doc_opstamps))
     }
 
-    pub fn mem_usage(&self) -> usize {
+    pub fn mem_usage(&self) -> u64 {
         self.multifield_postings.mem_usage()
     }
 
@@ -159,7 +191,7 @@ impl SegmentWriter {
                     }
                 }
                 FieldType::Str(_) => {
-                    let mut token_streams: Vec<Box<dyn TokenStream>> = vec![];
+                    let mut token_streams: Vec<BoxTokenStream> = vec![];
                     let mut offsets = vec![];
                     let mut total_offset = 0;
 
@@ -172,7 +204,7 @@ impl SegmentWriter {
                                 }
 
                                 token_streams
-                                    .push(Box::new(PreTokenizedStream::from(tok_str.clone())));
+                                    .push(PreTokenizedStream::from(tok_str.clone()).into());
                             }
                             Value::Str(ref text) => {
                                 if let Some(ref mut tokenizer) =
@@ -191,8 +223,7 @@ impl SegmentWriter {
                     let num_tokens = if token_streams.is_empty() {
                         0
                     } else {
-                        let mut token_stream: Box<dyn TokenStream> =
-                            Box::new(TokenStreamChain::new(offsets, token_streams));
+                        let mut token_stream = TokenStreamChain::new(offsets, token_streams);
                         self.multifield_postings
                             .index_text(doc_id, field, &mut token_stream)
                     };
@@ -250,8 +281,7 @@ impl SegmentWriter {
         }
         doc.filter_fields(|field| schema.get_field_entry(field).is_stored());
         doc.prepare_for_store();
-        let doc_writer = self.segment_serializer.get_store_writer();
-        doc_writer.store(&doc)?;
+        self.store_writer.store(&doc)?;
         self.max_doc += 1;
         Ok(())
     }
@@ -283,7 +313,7 @@ fn write(
     fast_field_writers: &FastFieldsWriter,
     fieldnorms_writer: &FieldNormsWriter,
     mut serializer: SegmentSerializer,
-) -> Result<()> {
+) -> crate::Result<()> {
     let term_ord_map = multifield_postings.serialize(serializer.get_postings_serializer())?;
     fast_field_writers.serialize(serializer.get_fast_field_serializer(), &term_ord_map)?;
     fieldnorms_writer.serialize(serializer.get_fieldnorms_serializer())?;
@@ -292,7 +322,7 @@ fn write(
 }
 
 impl SerializableSegment for SegmentWriter {
-    fn write(&self, serializer: SegmentSerializer) -> Result<u32> {
+    fn write(&self, serializer: SegmentSerializer) -> crate::Result<u32> {
         let max_doc = self.max_doc;
         write(
             &self.multifield_postings,

src/indexer/stamper.rs

@@ -1,18 +1,76 @@
 use crate::Opstamp;
 use std::ops::Range;
-use std::sync::atomic::{AtomicU64, Ordering};
+use std::sync::atomic::Ordering;
 use std::sync::Arc;
 
+#[cfg(not(target_arch = "arm"))]
+mod atomic_impl {
+
+    use crate::Opstamp;
+    use std::sync::atomic::{AtomicU64, Ordering};
+
+    #[derive(Default)]
+    pub struct AtomicU64Wrapper(AtomicU64);
+
+    impl AtomicU64Wrapper {
+        pub fn new(first_opstamp: Opstamp) -> AtomicU64Wrapper {
+            AtomicU64Wrapper(AtomicU64::new(first_opstamp as u64))
+        }
+
+        pub fn fetch_add(&self, val: u64, order: Ordering) -> u64 {
+            self.0.fetch_add(val as u64, order) as u64
+        }
+
+        pub fn revert(&self, val: u64, order: Ordering) -> u64 {
+            self.0.store(val, order);
+            val
+        }
+    }
+}
+
+#[cfg(target_arch = "arm")]
+mod atomic_impl {
+
+    use crate::Opstamp;
+    /// Under other architecture, we rely on a mutex.
+    use std::sync::atomic::Ordering;
+    use std::sync::RwLock;
+
+    #[derive(Default)]
+    pub struct AtomicU64Wrapper(RwLock<u64>);
+
+    impl AtomicU64Wrapper {
+        pub fn new(first_opstamp: Opstamp) -> AtomicU64Wrapper {
+            AtomicU64Wrapper(RwLock::new(first_opstamp))
+        }
+
+        pub fn fetch_add(&self, incr: u64, _order: Ordering) -> u64 {
+            let mut lock = self.0.write().unwrap();
+            let previous_val = *lock;
+            *lock = previous_val + incr;
+            previous_val
+        }
+
+        pub fn revert(&self, val: u64, _order: Ordering) -> u64 {
+            let mut lock = self.0.write().unwrap();
+            *lock = val;
+            val
+        }
+    }
+}
+
+use self::atomic_impl::AtomicU64Wrapper;
+
 /// Stamper provides Opstamps, which is just an auto-increment id to label
 /// an operation.
 ///
 /// Cloning does not "fork" the stamp generation. The stamper actually wraps an `Arc`.
 #[derive(Clone, Default)]
-pub struct Stamper(Arc<AtomicU64>);
+pub struct Stamper(Arc<AtomicU64Wrapper>);
 
 impl Stamper {
     pub fn new(first_opstamp: Opstamp) -> Stamper {
-        Stamper(Arc::new(AtomicU64::new(first_opstamp)))
+        Stamper(Arc::new(AtomicU64Wrapper::new(first_opstamp)))
     }
 
     pub fn stamp(&self) -> Opstamp {
@@ -31,8 +89,7 @@ impl Stamper {
 
     /// Reverts the stamper to a given `Opstamp` value and returns it
     pub fn revert(&self, to_opstamp: Opstamp) -> Opstamp {
-        self.0.store(to_opstamp, Ordering::SeqCst);
-        to_opstamp
+        self.0.revert(to_opstamp, Ordering::SeqCst)
     }
 }
 

src/lib.rs

@@ -98,9 +98,6 @@
 //! [literate programming](https://tantivy-search.github.io/examples/basic_search.html) /
 //! [source code](https://github.com/tantivy-search/tantivy/blob/master/examples/basic_search.rs))
 
-#[macro_use]
-extern crate serde_derive;
-
 #[cfg_attr(test, macro_use)]
 extern crate serde_json;
 
@@ -121,13 +118,13 @@ mod functional_test;
 mod macros;
 
 pub use crate::error::TantivyError;
-
-#[deprecated(since = "0.7.0", note = "please use `tantivy::TantivyError` instead")]
-pub use crate::error::TantivyError as Error;
 pub use chrono;
 
 /// Tantivy result.
-pub type Result<T> = std::result::Result<T, error::TantivyError>;
+///
+/// Within tantivy, please avoid importing `Result` using `use crate::Result`
+/// and instead, refer to this as `crate::Result<T>`.
+pub type Result<T> = std::result::Result<T, TantivyError>;
 
 /// Tantivy DateTime
 pub type DateTime = chrono::DateTime<chrono::Utc>;
@@ -161,17 +158,19 @@ pub use self::snippet::{Snippet, SnippetGenerator};
 mod docset;
 pub use self::docset::{DocSet, SkipResult};
 pub use crate::common::{f64_to_u64, i64_to_u64, u64_to_f64, u64_to_i64};
-pub use crate::core::SegmentComponent;
+pub use crate::core::{Executor, SegmentComponent};
 pub use crate::core::{Index, IndexMeta, Searcher, Segment, SegmentId, SegmentMeta};
 pub use crate::core::{InvertedIndexReader, SegmentReader};
 pub use crate::directory::Directory;
-pub use crate::indexer::IndexWriter;
+pub use crate::indexer::operation::UserOperation;
+pub use crate::indexer::{IndexWriter, IndexWriterConfig};
 pub use crate::postings::Postings;
 pub use crate::reader::LeasedItem;
 pub use crate::schema::{Document, Term};
 use std::fmt;
 
 use once_cell::sync::Lazy;
+use serde::{Deserialize, Serialize};
 
 /// Index format version.
 const INDEX_FORMAT_VERSION: u32 = 1;

src/postings/block_search.rs

@@ -106,7 +106,7 @@ impl BlockSearcher {
     /// the target.
     ///
     /// The results should be equivalent to
-    /// ```ignore
+    /// ```compile_fail
     /// block[..]
     //       .iter()
     //       .take_while(|&&val| val < target)

src/postings/mod.rs

@@ -54,7 +54,7 @@ pub mod tests {
     use crate::docset::{DocSet, SkipResult};
     use crate::fieldnorm::FieldNormReader;
     use crate::indexer::operation::AddOperation;
-    use crate::indexer::SegmentWriter;
+    use crate::indexer::{IndexWriterConfig, SegmentWriter, ResourceManager};
     use crate::merge_policy::NoMergePolicy;
     use crate::query::Scorer;
     use crate::schema::{Document, Schema, Term, INDEXED, STRING, TEXT};
@@ -216,11 +216,17 @@ pub mod tests {
         let text_field = schema_builder.add_text_field("text", TEXT);
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema.clone());
-        let segment = index.new_segment();
 
-        {
-            let mut segment_writer =
-                SegmentWriter::for_segment(3_000_000, segment.clone(), &schema).unwrap();
+        let resource_manager = ResourceManager::default();
+        let segment = {
+            let mut segment_writer = SegmentWriter::for_segment(
+                &IndexWriterConfig::for_test(),
+                index.new_segment(),
+                &schema,
+                index.tokenizers(),
+                resource_manager
+            )
+            .unwrap();
             {
                 let mut doc = Document::default();
                 // checking that position works if the field has two values
@@ -252,8 +258,9 @@ pub mod tests {
                 };
                 segment_writer.add_document(op, &schema).unwrap();
             }
-            segment_writer.finalize().unwrap();
-        }
+            let (segment, _) = segment_writer.finalize().unwrap();
+            segment
+        };
         {
             let segment_reader = SegmentReader::open(&segment).unwrap();
             {

src/postings/postings_writer.rs

@@ -11,7 +11,6 @@ use crate::termdict::TermOrdinal;
 use crate::tokenizer::TokenStream;
 use crate::tokenizer::{Token, MAX_TOKEN_LEN};
 use crate::DocId;
-use crate::Result;
 use fnv::FnvHashMap;
 use std::collections::HashMap;
 use std::io;
@@ -96,7 +95,7 @@ impl MultiFieldPostingsWriter {
         }
     }
 
-    pub fn mem_usage(&self) -> usize {
+    pub fn mem_usage(&self) -> u64 {
         self.term_index.mem_usage() + self.heap.mem_usage()
     }
 
@@ -129,7 +128,7 @@ impl MultiFieldPostingsWriter {
     pub fn serialize(
         &self,
         serializer: &mut InvertedIndexSerializer,
-    ) -> Result<HashMap<Field, FnvHashMap<UnorderedTermId, TermOrdinal>>> {
+    ) -> crate::Result<HashMap<Field, FnvHashMap<UnorderedTermId, TermOrdinal>>> {
         let mut term_offsets: Vec<(&[u8], Addr, UnorderedTermId)> =
             self.term_index.iter().collect();
         term_offsets.sort_unstable_by_key(|&(k, _, _)| k);

src/postings/serializer.rs

@@ -11,7 +11,6 @@ use crate::schema::Schema;
 use crate::schema::{Field, FieldEntry, FieldType};
 use crate::termdict::{TermDictionaryBuilder, TermOrdinal};
 use crate::DocId;
-use crate::Result;
 use std::io::{self, Write};
 
 /// `InvertedIndexSerializer` is in charge of serializing
@@ -61,7 +60,7 @@ impl InvertedIndexSerializer {
         positions_write: CompositeWrite<WritePtr>,
         positionsidx_write: CompositeWrite<WritePtr>,
         schema: Schema,
-    ) -> Result<InvertedIndexSerializer> {
+    ) -> crate::Result<InvertedIndexSerializer> {
         Ok(InvertedIndexSerializer {
             terms_write,
             postings_write,
@@ -72,7 +71,7 @@ impl InvertedIndexSerializer {
     }
 
     /// Open a new `PostingsSerializer` for the given segment
-    pub fn open(segment: &mut Segment) -> Result<InvertedIndexSerializer> {
+    pub fn open(segment: &mut Segment) -> crate::Result<InvertedIndexSerializer> {
         use crate::SegmentComponent::{POSITIONS, POSITIONSSKIP, POSTINGS, TERMS};
         InvertedIndexSerializer::create(
             CompositeWrite::wrap(segment.open_write(TERMS)?),
@@ -148,8 +147,7 @@ impl<'a> FieldSerializer<'a> {
             }
             _ => (false, false),
         };
-        let term_dictionary_builder =
-            TermDictionaryBuilder::create(term_dictionary_write, &field_type)?;
+        let term_dictionary_builder = TermDictionaryBuilder::create(term_dictionary_write)?;
         let postings_serializer =
             PostingsSerializer::new(postings_write, term_freq_enabled, position_enabled);
         let positions_serializer_opt = if position_enabled {

src/postings/stacker/memory_arena.rs

@@ -106,8 +106,8 @@ impl MemoryArena {
     ///
     /// Internally, it counts a number of `1MB` pages
     /// and therefore delivers an upperbound.
-    pub fn mem_usage(&self) -> usize {
-        self.pages.len() * PAGE_SIZE
+    pub fn mem_usage(&self) -> u64 {
+        (self.pages.len() as u64) * (PAGE_SIZE as u64)
     }
 
     pub fn write_at<Item: Copy + 'static>(&mut self, addr: Addr, val: Item) {

src/postings/stacker/term_hashmap.rs

@@ -12,8 +12,8 @@ use std::slice;
 
 /// Returns the actual memory size in bytes
 /// required to create a table of size $2^num_bits$.
-pub fn compute_table_size(num_bits: usize) -> usize {
-    (1 << num_bits) * mem::size_of::<KeyValue>()
+pub fn compute_table_size(num_bits: usize) -> u64 {
+    (1u64 << num_bits as u64) * mem::size_of::<KeyValue>() as u64
 }
 
 /// `KeyValue` is the item stored in the hash table.
@@ -116,8 +116,8 @@ impl TermHashMap {
         QuadraticProbing::compute(hash as usize, self.mask)
     }
 
-    pub fn mem_usage(&self) -> usize {
-        self.table.len() * mem::size_of::<KeyValue>()
+    pub fn mem_usage(&self) -> u64 {
+        self.table.len() as u64 * mem::size_of::<KeyValue>() as u64
     }
 
     fn is_saturated(&self) -> bool {

src/query/all_query.rs

@@ -1,10 +1,10 @@
 use crate::core::Searcher;
 use crate::core::SegmentReader;
 use crate::docset::DocSet;
+use crate::query::boost_query::BoostScorer;
 use crate::query::explanation::does_not_match;
 use crate::query::{Explanation, Query, Scorer, Weight};
 use crate::DocId;
-use crate::Result;
 use crate::Score;
 
 /// Query that matches all of the documents.
@@ -14,7 +14,7 @@ use crate::Score;
 pub struct AllQuery;
 
 impl Query for AllQuery {
-    fn weight(&self, _: &Searcher, _: bool) -> Result<Box<dyn Weight>> {
+    fn weight(&self, _: &Searcher, _: bool) -> crate::Result<Box<dyn Weight>> {
         Ok(Box::new(AllWeight))
     }
 }
@@ -23,15 +23,16 @@ impl Query for AllQuery {
 pub struct AllWeight;
 
 impl Weight for AllWeight {
-    fn scorer(&self, reader: &SegmentReader) -> Result<Box<dyn Scorer>> {
-        Ok(Box::new(AllScorer {
+    fn scorer(&self, reader: &SegmentReader, boost: f32) -> crate::Result<Box<dyn Scorer>> {
+        let all_scorer = AllScorer {
             state: State::NotStarted,
             doc: 0u32,
             max_doc: reader.max_doc(),
-        }))
+        };
+        Ok(Box::new(BoostScorer::new(all_scorer, boost)))
     }
 
-    fn explain(&self, reader: &SegmentReader, doc: DocId) -> Result<Explanation> {
+    fn explain(&self, reader: &SegmentReader, doc: DocId) -> crate::Result<Explanation> {
         if doc >= reader.max_doc() {
             return Err(does_not_match(doc));
         }
@@ -91,14 +92,12 @@ impl Scorer for AllScorer {
 
 #[cfg(test)]
 mod tests {
-
     use super::AllQuery;
     use crate::query::Query;
     use crate::schema::{Schema, TEXT};
     use crate::Index;
 
-    #[test]
-    fn test_all_query() {
+    fn create_test_index() -> Index {
         let mut schema_builder = Schema::builder();
         let field = schema_builder.add_text_field("text", TEXT);
         let schema = schema_builder.build();
@@ -109,13 +108,18 @@ mod tests {
         index_writer.commit().unwrap();
         index_writer.add_document(doc!(field=>"ccc"));
         index_writer.commit().unwrap();
+        index
+    }
+
+    #[test]
+    fn test_all_query() {
+        let index = create_test_index();
         let reader = index.reader().unwrap();
-        reader.reload().unwrap();
         let searcher = reader.searcher();
         let weight = AllQuery.weight(&searcher, false).unwrap();
         {
             let reader = searcher.segment_reader(0);
-            let mut scorer = weight.scorer(reader).unwrap();
+            let mut scorer = weight.scorer(reader, 1.0f32).unwrap();
             assert!(scorer.advance());
             assert_eq!(scorer.doc(), 0u32);
             assert!(scorer.advance());
@@ -124,10 +128,31 @@ mod tests {
         }
         {
             let reader = searcher.segment_reader(1);
-            let mut scorer = weight.scorer(reader).unwrap();
+            let mut scorer = weight.scorer(reader, 1.0f32).unwrap();
             assert!(scorer.advance());
             assert_eq!(scorer.doc(), 0u32);
             assert!(!scorer.advance());
         }
     }
+
+    #[test]
+    fn test_all_query_with_boost() {
+        let index = create_test_index();
+        let reader = index.reader().unwrap();
+        let searcher = reader.searcher();
+        let weight = AllQuery.weight(&searcher, false).unwrap();
+        let reader = searcher.segment_reader(0);
+        {
+            let mut scorer = weight.scorer(reader, 2.0f32).unwrap();
+            assert!(scorer.advance());
+            assert_eq!(scorer.doc(), 0u32);
+            assert_eq!(scorer.score(), 2.0f32);
+        }
+        {
+            let mut scorer = weight.scorer(reader, 1.5f32).unwrap();
+            assert!(scorer.advance());
+            assert_eq!(scorer.doc(), 0u32);
+            assert_eq!(scorer.score(), 1.5f32);
+        }
+    }
 }

src/query/automaton_weight.rs

@@ -40,7 +40,7 @@ impl<A> Weight for AutomatonWeight<A>
 where
     A: Automaton + Send + Sync + 'static,
 {
-    fn scorer(&self, reader: &SegmentReader) -> Result<Box<dyn Scorer>> {
+    fn scorer(&self, reader: &SegmentReader, boost: f32) -> Result<Box<dyn Scorer>> {
         let max_doc = reader.max_doc();
         let mut doc_bitset = BitSet::with_max_value(max_doc);
 
@@ -58,11 +58,12 @@ where
             }
         }
         let doc_bitset = BitSetDocSet::from(doc_bitset);
-        Ok(Box::new(ConstScorer::new(doc_bitset)))
+        let const_scorer = ConstScorer::new(doc_bitset, boost);
+        Ok(Box::new(const_scorer))
     }
 
     fn explain(&self, reader: &SegmentReader, doc: DocId) -> Result<Explanation> {
-        let mut scorer = self.scorer(reader)?;
+        let mut scorer = self.scorer(reader, 1.0f32)?;
         if scorer.skip_next(doc) == SkipResult::Reached {
             Ok(Explanation::new("AutomatonScorer", 1.0f32))
         } else {
@@ -72,3 +73,95 @@ where
         }
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::AutomatonWeight;
+    use crate::query::Weight;
+    use crate::schema::{Schema, STRING};
+    use crate::Index;
+    use tantivy_fst::Automaton;
+
+    fn create_index() -> Index {
+        let mut schema = Schema::builder();
+        let title = schema.add_text_field("title", STRING);
+        let index = Index::create_in_ram(schema.build());
+        let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
+        index_writer.add_document(doc!(title=>"abc"));
+        index_writer.add_document(doc!(title=>"bcd"));
+        index_writer.add_document(doc!(title=>"abcd"));
+        assert!(index_writer.commit().is_ok());
+        index
+    }
+
+    enum State {
+        Start,
+        NotMatching,
+        AfterA,
+    }
+
+    struct PrefixedByA;
+
+    impl Automaton for PrefixedByA {
+        type State = State;
+
+        fn start(&self) -> Self::State {
+            State::Start
+        }
+
+        fn is_match(&self, state: &Self::State) -> bool {
+            match *state {
+                State::AfterA => true,
+                _ => false,
+            }
+        }
+
+        fn accept(&self, state: &Self::State, byte: u8) -> Self::State {
+            match *state {
+                State::Start => {
+                    if byte == b'a' {
+                        State::AfterA
+                    } else {
+                        State::NotMatching
+                    }
+                }
+                State::AfterA => State::AfterA,
+                State::NotMatching => State::NotMatching,
+            }
+        }
+    }
+
+    #[test]
+    fn test_automaton_weight() {
+        let index = create_index();
+        let field = index.schema().get_field("title").unwrap();
+        let automaton_weight = AutomatonWeight::new(field, PrefixedByA);
+        let reader = index.reader().unwrap();
+        let searcher = reader.searcher();
+        let mut scorer = automaton_weight
+            .scorer(searcher.segment_reader(0u32), 1.0f32)
+            .unwrap();
+        assert!(scorer.advance());
+        assert_eq!(scorer.doc(), 0u32);
+        assert_eq!(scorer.score(), 1.0f32);
+        assert!(scorer.advance());
+        assert_eq!(scorer.doc(), 2u32);
+        assert_eq!(scorer.score(), 1.0f32);
+        assert!(!scorer.advance());
+    }
+
+    #[test]
+    fn test_automaton_weight_boost() {
+        let index = create_index();
+        let field = index.schema().get_field("title").unwrap();
+        let automaton_weight = AutomatonWeight::new(field, PrefixedByA);
+        let reader = index.reader().unwrap();
+        let searcher = reader.searcher();
+        let mut scorer = automaton_weight
+            .scorer(searcher.segment_reader(0u32), 1.32f32)
+            .unwrap();
+        assert!(scorer.advance());
+        assert_eq!(scorer.doc(), 0u32);
+        assert_eq!(scorer.score(), 1.32f32);
+    }
+}

src/query/bm25.rs

@@ -25,7 +25,6 @@ fn compute_tf_cache(average_fieldnorm: f32) -> [f32; 256] {
     cache
 }
 
-#[derive(Clone)]
 pub struct BM25Weight {
     idf_explain: Explanation,
     weight: f32,
@@ -34,6 +33,15 @@ pub struct BM25Weight {
 }
 
 impl BM25Weight {
+    pub fn boost_by(&self, boost: f32) -> BM25Weight {
+        BM25Weight {
+            idf_explain: self.idf_explain.clone(),
+            weight: self.weight * boost,
+            cache: self.cache,
+            average_fieldnorm: self.average_fieldnorm,
+        }
+    }
+
     pub fn for_terms(searcher: &Searcher, terms: &[Term]) -> BM25Weight {
         assert!(!terms.is_empty(), "BM25 requires at least one term");
         let field = terms[0].field();

src/query/boolean_query/boolean_query.rs

@@ -5,7 +5,6 @@ use crate::query::TermQuery;
 use crate::query::Weight;
 use crate::schema::IndexRecordOption;
 use crate::schema::Term;
-use crate::Result;
 use crate::Searcher;
 use std::collections::BTreeSet;
 
@@ -30,9 +29,9 @@ use std::collections::BTreeSet;
 ///use tantivy::query::{BooleanQuery, Occur, PhraseQuery, Query, TermQuery};
 ///use tantivy::schema::{IndexRecordOption, Schema, TEXT};
 ///use tantivy::Term;
-///use tantivy::{Index, Result};
+///use tantivy::Index;
 ///
-///fn main() -> Result<()> {
+///fn main() -> tantivy::Result<()> {
 ///    let mut schema_builder = Schema::builder();
 ///    let title = schema_builder.add_text_field("title", TEXT);
 ///    let body = schema_builder.add_text_field("body", TEXT);
@@ -149,14 +148,14 @@ impl From<Vec<(Occur, Box<dyn Query>)>> for BooleanQuery {
 }
 
 impl Query for BooleanQuery {
-    fn weight(&self, searcher: &Searcher, scoring_enabled: bool) -> Result<Box<dyn Weight>> {
+    fn weight(&self, searcher: &Searcher, scoring_enabled: bool) -> crate::Result<Box<dyn Weight>> {
         let sub_weights = self
             .subqueries
             .iter()
             .map(|&(ref occur, ref subquery)| {
                 Ok((*occur, subquery.weight(searcher, scoring_enabled)?))
             })
-            .collect::<Result<_>>()?;
+            .collect::<crate::Result<_>>()?;
         Ok(Box::new(BooleanWeight::new(sub_weights, scoring_enabled)))
     }
 

src/query/boolean_query/boolean_weight.rs

@@ -10,7 +10,6 @@ use crate::query::Scorer;
 use crate::query::Union;
 use crate::query::Weight;
 use crate::query::{intersect_scorers, Explanation};
-use crate::Result;
 use crate::{DocId, SkipResult};
 use std::collections::HashMap;
 
@@ -56,10 +55,11 @@ impl BooleanWeight {
     fn per_occur_scorers(
         &self,
         reader: &SegmentReader,
-    ) -> Result<HashMap<Occur, Vec<Box<dyn Scorer>>>> {
+        boost: f32,
+    ) -> crate::Result<HashMap<Occur, Vec<Box<dyn Scorer>>>> {
         let mut per_occur_scorers: HashMap<Occur, Vec<Box<dyn Scorer>>> = HashMap::new();
         for &(ref occur, ref subweight) in &self.weights {
-            let sub_scorer: Box<dyn Scorer> = subweight.scorer(reader)?;
+            let sub_scorer: Box<dyn Scorer> = subweight.scorer(reader, boost)?;
             per_occur_scorers
                 .entry(*occur)
                 .or_insert_with(Vec::new)
@@ -71,8 +71,9 @@ impl BooleanWeight {
     fn complex_scorer<TScoreCombiner: ScoreCombiner>(
         &self,
         reader: &SegmentReader,
-    ) -> Result<Box<dyn Scorer>> {
-        let mut per_occur_scorers = self.per_occur_scorers(reader)?;
+        boost: f32,
+    ) -> crate::Result<Box<dyn Scorer>> {
+        let mut per_occur_scorers = self.per_occur_scorers(reader, boost)?;
 
         let should_scorer_opt: Option<Box<dyn Scorer>> = per_occur_scorers
             .remove(&Occur::Should)
@@ -113,7 +114,7 @@ impl BooleanWeight {
 }
 
 impl Weight for BooleanWeight {
-    fn scorer(&self, reader: &SegmentReader) -> Result<Box<dyn Scorer>> {
+    fn scorer(&self, reader: &SegmentReader, boost: f32) -> crate::Result<Box<dyn Scorer>> {
         if self.weights.is_empty() {
             Ok(Box::new(EmptyScorer))
         } else if self.weights.len() == 1 {
@@ -121,17 +122,17 @@ impl Weight for BooleanWeight {
             if occur == Occur::MustNot {
                 Ok(Box::new(EmptyScorer))
             } else {
-                weight.scorer(reader)
+                weight.scorer(reader, boost)
             }
         } else if self.scoring_enabled {
-            self.complex_scorer::<SumWithCoordsCombiner>(reader)
+            self.complex_scorer::<SumWithCoordsCombiner>(reader, boost)
         } else {
-            self.complex_scorer::<DoNothingCombiner>(reader)
+            self.complex_scorer::<DoNothingCombiner>(reader, boost)
         }
     }
 
-    fn explain(&self, reader: &SegmentReader, doc: DocId) -> Result<Explanation> {
-        let mut scorer = self.scorer(reader)?;
+    fn explain(&self, reader: &SegmentReader, doc: DocId) -> crate::Result<Explanation> {
+        let mut scorer = self.scorer(reader, 1.0f32)?;
         if scorer.skip_next(doc) != SkipResult::Reached {
             return Err(does_not_match(doc));
         }

src/query/boolean_query/mod.rs

@@ -18,6 +18,7 @@ mod tests {
     use crate::query::Scorer;
     use crate::query::TermQuery;
     use crate::schema::*;
+    use crate::tests::assert_nearly_equals;
     use crate::Index;
     use crate::{DocAddress, DocId};
 
@@ -70,7 +71,9 @@ mod tests {
         let query = query_parser.parse_query("+a").unwrap();
         let searcher = index.reader().unwrap().searcher();
         let weight = query.weight(&searcher, true).unwrap();
-        let scorer = weight.scorer(searcher.segment_reader(0u32)).unwrap();
+        let scorer = weight
+            .scorer(searcher.segment_reader(0u32), 1.0f32)
+            .unwrap();
         assert!(scorer.is::<TermScorer>());
     }
 
@@ -82,13 +85,17 @@ mod tests {
         {
             let query = query_parser.parse_query("+a +b +c").unwrap();
             let weight = query.weight(&searcher, true).unwrap();
-            let scorer = weight.scorer(searcher.segment_reader(0u32)).unwrap();
+            let scorer = weight
+                .scorer(searcher.segment_reader(0u32), 1.0f32)
+                .unwrap();
             assert!(scorer.is::<Intersection<TermScorer>>());
         }
         {
             let query = query_parser.parse_query("+a +(b c)").unwrap();
             let weight = query.weight(&searcher, true).unwrap();
-            let scorer = weight.scorer(searcher.segment_reader(0u32)).unwrap();
+            let scorer = weight
+                .scorer(searcher.segment_reader(0u32), 1.0f32)
+                .unwrap();
             assert!(scorer.is::<Intersection<Box<dyn Scorer>>>());
         }
     }
@@ -101,7 +108,9 @@ mod tests {
         {
             let query = query_parser.parse_query("+a b").unwrap();
             let weight = query.weight(&searcher, true).unwrap();
-            let scorer = weight.scorer(searcher.segment_reader(0u32)).unwrap();
+            let scorer = weight
+                .scorer(searcher.segment_reader(0u32), 1.0f32)
+                .unwrap();
             assert!(scorer.is::<RequiredOptionalScorer<
                 Box<dyn Scorer>,
                 Box<dyn Scorer>,
@@ -111,7 +120,9 @@ mod tests {
         {
             let query = query_parser.parse_query("+a b").unwrap();
             let weight = query.weight(&searcher, false).unwrap();
-            let scorer = weight.scorer(searcher.segment_reader(0u32)).unwrap();
+            let scorer = weight
+                .scorer(searcher.segment_reader(0u32), 1.0f32)
+                .unwrap();
             assert!(scorer.is::<TermScorer>());
         }
     }
@@ -179,6 +190,50 @@ mod tests {
         }
     }
 
+    #[test]
+    pub fn test_boolean_query_with_weight() {
+        let mut schema_builder = Schema::builder();
+        let text_field = schema_builder.add_text_field("text", TEXT);
+        let schema = schema_builder.build();
+        let index = Index::create_in_ram(schema);
+        {
+            let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
+            index_writer.add_document(doc!(text_field => "a b c"));
+            index_writer.add_document(doc!(text_field => "a c"));
+            index_writer.add_document(doc!(text_field => "b c"));
+            assert!(index_writer.commit().is_ok());
+        }
+        let term_a: Box<dyn Query> = Box::new(TermQuery::new(
+            Term::from_field_text(text_field, "a"),
+            IndexRecordOption::WithFreqs,
+        ));
+        let term_b: Box<dyn Query> = Box::new(TermQuery::new(
+            Term::from_field_text(text_field, "b"),
+            IndexRecordOption::WithFreqs,
+        ));
+        let reader = index.reader().unwrap();
+        let searcher = reader.searcher();
+        let boolean_query =
+            BooleanQuery::from(vec![(Occur::Should, term_a), (Occur::Should, term_b)]);
+        let boolean_weight = boolean_query.weight(&searcher, true).unwrap();
+        {
+            let mut boolean_scorer = boolean_weight
+                .scorer(searcher.segment_reader(0u32), 1.0f32)
+                .unwrap();
+            assert!(boolean_scorer.advance());
+            assert_eq!(boolean_scorer.doc(), 0u32);
+            assert_nearly_equals(boolean_scorer.score(), 0.84163445f32);
+        }
+        {
+            let mut boolean_scorer = boolean_weight
+                .scorer(searcher.segment_reader(0u32), 2.0f32)
+                .unwrap();
+            assert!(boolean_scorer.advance());
+            assert_eq!(boolean_scorer.doc(), 0u32);
+            assert_nearly_equals(boolean_scorer.score(), 1.6832689f32);
+        }
+    }
+
     #[test]
     pub fn test_intersection_score() {
         let (index, text_field) = aux_test_helper();
@@ -249,7 +304,9 @@ mod tests {
         let query_parser = QueryParser::for_index(&index, vec![title, text]);
         let query = query_parser.parse_query("Оксана Лифенко").unwrap();
         let weight = query.weight(&searcher, true).unwrap();
-        let mut scorer = weight.scorer(searcher.segment_reader(0u32)).unwrap();
+        let mut scorer = weight
+            .scorer(searcher.segment_reader(0u32), 1.0f32)
+            .unwrap();
         scorer.advance();
 
         let explanation = query.explain(&searcher, DocAddress(0u32, 0u32)).unwrap();

src/query/boost_query.rs

@@ -0,0 +1,164 @@
+use crate::common::BitSet;
+use crate::fastfield::DeleteBitSet;
+use crate::query::explanation::does_not_match;
+use crate::query::{Explanation, Query, Scorer, Weight};
+use crate::{DocId, DocSet, Searcher, SegmentReader, SkipResult, Term};
+use std::collections::BTreeSet;
+use std::fmt;
+
+/// `BoostQuery` is a wrapper over a query used to boost its score.
+///
+/// The document set matched by the `BoostQuery` is strictly the same as the underlying query.
+/// The score of each document, is the score of the underlying query multiplied by the `boost`
+/// factor.
+pub struct BoostQuery {
+    query: Box<dyn Query>,
+    boost: f32,
+}
+
+impl BoostQuery {
+    /// Builds a boost query.
+    pub fn new(query: Box<dyn Query>, boost: f32) -> BoostQuery {
+        BoostQuery { query, boost }
+    }
+}
+
+impl Clone for BoostQuery {
+    fn clone(&self) -> Self {
+        BoostQuery {
+            query: self.query.box_clone(),
+            boost: self.boost,
+        }
+    }
+}
+
+impl fmt::Debug for BoostQuery {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "Boost(query={:?}, boost={})", self.query, self.boost)
+    }
+}
+
+impl Query for BoostQuery {
+    fn weight(&self, searcher: &Searcher, scoring_enabled: bool) -> crate::Result<Box<dyn Weight>> {
+        let weight_without_boost = self.query.weight(searcher, scoring_enabled)?;
+        let boosted_weight = if scoring_enabled {
+            Box::new(BoostWeight::new(weight_without_boost, self.boost))
+        } else {
+            weight_without_boost
+        };
+        Ok(boosted_weight)
+    }
+
+    fn query_terms(&self, term_set: &mut BTreeSet<Term>) {
+        self.query.query_terms(term_set)
+    }
+}
+
+pub(crate) struct BoostWeight {
+    weight: Box<dyn Weight>,
+    boost: f32,
+}
+
+impl BoostWeight {
+    pub fn new(weight: Box<dyn Weight>, boost: f32) -> Self {
+        BoostWeight { weight, boost }
+    }
+}
+
+impl Weight for BoostWeight {
+    fn scorer(&self, reader: &SegmentReader, boost: f32) -> crate::Result<Box<dyn Scorer>> {
+        self.weight.scorer(reader, boost * self.boost)
+    }
+
+    fn explain(&self, reader: &SegmentReader, doc: u32) -> crate::Result<Explanation> {
+        let mut scorer = self.scorer(reader, 1.0f32)?;
+        if scorer.skip_next(doc) != SkipResult::Reached {
+            return Err(does_not_match(doc));
+        }
+        let mut explanation =
+            Explanation::new(format!("Boost x{} of ...", self.boost), scorer.score());
+        let underlying_explanation = self.weight.explain(reader, doc)?;
+        explanation.add_detail(underlying_explanation);
+        Ok(explanation)
+    }
+
+    fn count(&self, reader: &SegmentReader) -> crate::Result<u32> {
+        self.weight.count(reader)
+    }
+}
+
+pub(crate) struct BoostScorer<S: Scorer> {
+    underlying: S,
+    boost: f32,
+}
+
+impl<S: Scorer> BoostScorer<S> {
+    pub fn new(underlying: S, boost: f32) -> BoostScorer<S> {
+        BoostScorer { underlying, boost }
+    }
+}
+
+impl<S: Scorer> DocSet for BoostScorer<S> {
+    fn advance(&mut self) -> bool {
+        self.underlying.advance()
+    }
+
+    fn skip_next(&mut self, target: DocId) -> SkipResult {
+        self.underlying.skip_next(target)
+    }
+
+    fn fill_buffer(&mut self, buffer: &mut [DocId]) -> usize {
+        self.underlying.fill_buffer(buffer)
+    }
+
+    fn doc(&self) -> u32 {
+        self.underlying.doc()
+    }
+
+    fn size_hint(&self) -> u32 {
+        self.underlying.size_hint()
+    }
+
+    fn append_to_bitset(&mut self, bitset: &mut BitSet) {
+        self.underlying.append_to_bitset(bitset)
+    }
+
+    fn count(&mut self, delete_bitset: &DeleteBitSet) -> u32 {
+        self.underlying.count(delete_bitset)
+    }
+
+    fn count_including_deleted(&mut self) -> u32 {
+        self.underlying.count_including_deleted()
+    }
+}
+
+impl<S: Scorer> Scorer for BoostScorer<S> {
+    fn score(&mut self) -> f32 {
+        self.underlying.score() * self.boost
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::BoostQuery;
+    use crate::query::{AllQuery, Query};
+    use crate::schema::Schema;
+    use crate::{DocAddress, Document, Index};
+
+    #[test]
+    fn test_boost_query_explain() {
+        let schema = Schema::builder().build();
+        let index = Index::create_in_ram(schema);
+        let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
+        index_writer.add_document(Document::new());
+        assert!(index_writer.commit().is_ok());
+        let reader = index.reader().unwrap();
+        let searcher = reader.searcher();
+        let query = BoostQuery::new(Box::new(AllQuery), 0.2);
+        let explanation = query.explain(&searcher, DocAddress(0, 0u32)).unwrap();
+        assert_eq!(
+            explanation.to_pretty_json(),
+            "{\n  \"value\": 0.2,\n  \"description\": \"Boost x0.2 of ...\",\n  \"details\": [\n    {\n      \"value\": 1.0,\n      \"description\": \"AllQuery\"\n    }\n  ]\n}"
+        )
+    }
+}

src/query/empty_query.rs

@@ -4,7 +4,6 @@ use crate::query::Weight;
 use crate::query::{Explanation, Query};
 use crate::DocId;
 use crate::DocSet;
-use crate::Result;
 use crate::Score;
 use crate::Searcher;
 use crate::SegmentReader;
@@ -16,11 +15,15 @@ use crate::SegmentReader;
 pub struct EmptyQuery;
 
 impl Query for EmptyQuery {
-    fn weight(&self, _searcher: &Searcher, _scoring_enabled: bool) -> Result<Box<dyn Weight>> {
+    fn weight(
+        &self,
+        _searcher: &Searcher,
+        _scoring_enabled: bool,
+    ) -> crate::Result<Box<dyn Weight>> {
         Ok(Box::new(EmptyWeight))
     }
 
-    fn count(&self, _searcher: &Searcher) -> Result<usize> {
+    fn count(&self, _searcher: &Searcher) -> crate::Result<usize> {
         Ok(0)
     }
 }
@@ -30,11 +33,11 @@ impl Query for EmptyQuery {
 /// It is useful for tests and handling edge cases.
 pub struct EmptyWeight;
 impl Weight for EmptyWeight {
-    fn scorer(&self, _reader: &SegmentReader) -> Result<Box<dyn Scorer>> {
+    fn scorer(&self, _reader: &SegmentReader, _boost: f32) -> crate::Result<Box<dyn Scorer>> {
         Ok(Box::new(EmptyScorer))
     }
 
-    fn explain(&self, _reader: &SegmentReader, doc: DocId) -> Result<Explanation> {
+    fn explain(&self, _reader: &SegmentReader, doc: DocId) -> crate::Result<Explanation> {
         Err(does_not_match(doc))
     }
 }

src/query/explanation.rs

@@ -1,4 +1,5 @@
 use crate::{DocId, TantivyError};
+use serde::Serialize;
 
 pub(crate) fn does_not_match(doc: DocId) -> TantivyError {
     TantivyError::InvalidArgument(format!("Document #({}) does not match", doc))

src/query/fuzzy_query.rs

@@ -1,8 +1,7 @@
-use crate::error::TantivyError::InvalidArgument;
 use crate::query::{AutomatonWeight, Query, Weight};
 use crate::schema::Term;
-use crate::Result;
 use crate::Searcher;
+use crate::TantivyError::InvalidArgument;
 use levenshtein_automata::{LevenshteinAutomatonBuilder, DFA};
 use once_cell::sync::Lazy;
 use std::collections::HashMap;
@@ -10,7 +9,7 @@ use std::ops::Range;
 
 /// A range of Levenshtein distances that we will build DFAs for our terms
 /// The computation is exponential, so best keep it to low single digits
-const VALID_LEVENSHTEIN_DISTANCE_RANGE: Range<u8> = (0..3);
+const VALID_LEVENSHTEIN_DISTANCE_RANGE: Range<u8> = 0..3;
 
 static LEV_BUILDER: Lazy<HashMap<(u8, bool), LevenshteinAutomatonBuilder>> = Lazy::new(|| {
     let mut lev_builder_cache = HashMap::new();
@@ -31,9 +30,9 @@ static LEV_BUILDER: Lazy<HashMap<(u8, bool), LevenshteinAutomatonBuilder>> = Laz
 /// use tantivy::collector::{Count, TopDocs};
 /// use tantivy::query::FuzzyTermQuery;
 /// use tantivy::schema::{Schema, TEXT};
-/// use tantivy::{doc, Index, Result, Term};
+/// use tantivy::{doc, Index, Term};
 ///
-/// fn example() -> Result<()> {
+/// fn example() -> tantivy::Result<()> {
 ///     let mut schema_builder = Schema::builder();
 ///     let title = schema_builder.add_text_field("title", TEXT);
 ///     let schema = schema_builder.build();
@@ -102,7 +101,7 @@ impl FuzzyTermQuery {
         }
     }
 
-    fn specialized_weight(&self) -> Result<AutomatonWeight<DFA>> {
+    fn specialized_weight(&self) -> crate::Result<AutomatonWeight<DFA>> {
         // LEV_BUILDER is a HashMap, whose `get` method returns an Option
         match LEV_BUILDER.get(&(self.distance, false)) {
             // Unwrap the option and build the Ok(AutomatonWeight)
@@ -119,7 +118,11 @@ impl FuzzyTermQuery {
 }
 
 impl Query for FuzzyTermQuery {
-    fn weight(&self, _searcher: &Searcher, _scoring_enabled: bool) -> Result<Box<dyn Weight>> {
+    fn weight(
+        &self,
+        _searcher: &Searcher,
+        _scoring_enabled: bool,
+    ) -> crate::Result<Box<dyn Weight>> {
         Ok(Box::new(self.specialized_weight()?))
     }
 }

src/query/mod.rs

@@ -7,6 +7,7 @@ mod automaton_weight;
 mod bitset;
 mod bm25;
 mod boolean_query;
+mod boost_query;
 mod empty_query;
 mod exclude;
 mod explanation;
@@ -37,6 +38,7 @@ pub use self::all_query::{AllQuery, AllScorer, AllWeight};
 pub use self::automaton_weight::AutomatonWeight;
 pub use self::bitset::BitSetDocSet;
 pub use self::boolean_query::BooleanQuery;
+pub use self::boost_query::BoostQuery;
 pub use self::empty_query::{EmptyQuery, EmptyScorer, EmptyWeight};
 pub use self::exclude::Exclude;
 pub use self::explanation::Explanation;

src/query/phrase_query/mod.rs

@@ -7,7 +7,7 @@ pub use self::phrase_scorer::PhraseScorer;
 pub use self::phrase_weight::PhraseWeight;
 
 #[cfg(test)]
-mod tests {
+pub mod tests {
 
     use super::*;
     use crate::collector::tests::{TEST_COLLECTOR_WITHOUT_SCORE, TEST_COLLECTOR_WITH_SCORE};
@@ -15,10 +15,10 @@ mod tests {
     use crate::error::TantivyError;
     use crate::schema::{Schema, Term, TEXT};
     use crate::tests::assert_nearly_equals;
+    use crate::DocAddress;
     use crate::DocId;
-    use crate::{DocAddress, DocSet};
 
-    fn create_index(texts: &[&'static str]) -> Index {
+    pub fn create_index(texts: &[&'static str]) -> Index {
         let mut schema_builder = Schema::builder();
         let text_field = schema_builder.add_text_field("text", TEXT);
         let schema = schema_builder.build();
@@ -102,30 +102,6 @@ mod tests {
         assert!(test_query(vec!["g", "a"]).is_empty());
     }
 
-    #[test]
-    pub fn test_phrase_count() {
-        let index = create_index(&["a c", "a a b d a b c", " a b"]);
-        let schema = index.schema();
-        let text_field = schema.get_field("text").unwrap();
-        let searcher = index.reader().unwrap().searcher();
-        let phrase_query = PhraseQuery::new(vec![
-            Term::from_field_text(text_field, "a"),
-            Term::from_field_text(text_field, "b"),
-        ]);
-        let phrase_weight = phrase_query.phrase_weight(&searcher, true).unwrap();
-        let mut phrase_scorer = phrase_weight
-            .phrase_scorer(searcher.segment_reader(0u32))
-            .unwrap()
-            .unwrap();
-        assert!(phrase_scorer.advance());
-        assert_eq!(phrase_scorer.doc(), 1);
-        assert_eq!(phrase_scorer.phrase_count(), 2);
-        assert!(phrase_scorer.advance());
-        assert_eq!(phrase_scorer.doc(), 2);
-        assert_eq!(phrase_scorer.phrase_count(), 1);
-        assert!(!phrase_scorer.advance());
-    }
-
     #[test]
     pub fn test_phrase_query_no_positions() {
         let mut schema_builder = Schema::builder();

src/query/phrase_query/phrase_query.rs

@@ -1,12 +1,10 @@
 use super::PhraseWeight;
 use crate::core::searcher::Searcher;
-use crate::error::TantivyError;
 use crate::query::bm25::BM25Weight;
 use crate::query::Query;
 use crate::query::Weight;
 use crate::schema::IndexRecordOption;
 use crate::schema::{Field, Term};
-use crate::Result;
 use std::collections::BTreeSet;
 
 /// `PhraseQuery` matches a specific sequence of words.
@@ -81,7 +79,7 @@ impl PhraseQuery {
         &self,
         searcher: &Searcher,
         scoring_enabled: bool,
-    ) -> Result<PhraseWeight> {
+    ) -> crate::Result<PhraseWeight> {
         let schema = searcher.schema();
         let field_entry = schema.get_field_entry(self.field);
         let has_positions = field_entry
@@ -91,7 +89,7 @@ impl PhraseQuery {
             .unwrap_or(false);
         if !has_positions {
             let field_name = field_entry.name();
-            return Err(TantivyError::SchemaError(format!(
+            return Err(crate::TantivyError::SchemaError(format!(
                 "Applied phrase query on field {:?}, which does not have positions indexed",
                 field_name
             )));
@@ -110,7 +108,7 @@ impl Query for PhraseQuery {
     /// Create the weight associated to a query.
     ///
     /// See [`Weight`](./trait.Weight.html).
-    fn weight(&self, searcher: &Searcher, scoring_enabled: bool) -> Result<Box<dyn Weight>> {
+    fn weight(&self, searcher: &Searcher, scoring_enabled: bool) -> crate::Result<Box<dyn Weight>> {
         let phrase_weight = self.phrase_weight(searcher, scoring_enabled)?;
         Ok(Box::new(phrase_weight))
     }

src/query/phrase_query/phrase_weight.rs

@@ -37,11 +37,12 @@ impl PhraseWeight {
         reader.get_fieldnorms_reader(field)
     }
 
-    pub fn phrase_scorer(
+    fn phrase_scorer(
         &self,
         reader: &SegmentReader,
+        boost: f32,
     ) -> Result<Option<PhraseScorer<SegmentPostings>>> {
-        let similarity_weight = self.similarity_weight.clone();
+        let similarity_weight = self.similarity_weight.boost_by(boost);
         let fieldnorm_reader = self.fieldnorm_reader(reader);
         if reader.has_deletes() {
             let mut term_postings_list = Vec::new();
@@ -84,8 +85,8 @@ impl PhraseWeight {
 }
 
 impl Weight for PhraseWeight {
-    fn scorer(&self, reader: &SegmentReader) -> Result<Box<dyn Scorer>> {
-        if let Some(scorer) = self.phrase_scorer(reader)? {
+    fn scorer(&self, reader: &SegmentReader, boost: f32) -> Result<Box<dyn Scorer>> {
+        if let Some(scorer) = self.phrase_scorer(reader, boost)? {
             Ok(Box::new(scorer))
         } else {
             Ok(Box::new(EmptyScorer))
@@ -93,7 +94,7 @@ impl Weight for PhraseWeight {
     }
 
     fn explain(&self, reader: &SegmentReader, doc: DocId) -> Result<Explanation> {
-        let scorer_opt = self.phrase_scorer(reader)?;
+        let scorer_opt = self.phrase_scorer(reader, 1.0f32)?;
         if scorer_opt.is_none() {
             return Err(does_not_match(doc));
         }
@@ -109,3 +110,34 @@ impl Weight for PhraseWeight {
         Ok(explanation)
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::super::tests::create_index;
+    use crate::query::PhraseQuery;
+    use crate::{DocSet, Term};
+
+    #[test]
+    pub fn test_phrase_count() {
+        let index = create_index(&["a c", "a a b d a b c", " a b"]);
+        let schema = index.schema();
+        let text_field = schema.get_field("text").unwrap();
+        let searcher = index.reader().unwrap().searcher();
+        let phrase_query = PhraseQuery::new(vec![
+            Term::from_field_text(text_field, "a"),
+            Term::from_field_text(text_field, "b"),
+        ]);
+        let phrase_weight = phrase_query.phrase_weight(&searcher, true).unwrap();
+        let mut phrase_scorer = phrase_weight
+            .phrase_scorer(searcher.segment_reader(0u32), 1.0f32)
+            .unwrap()
+            .unwrap();
+        assert!(phrase_scorer.advance());
+        assert_eq!(phrase_scorer.doc(), 1);
+        assert_eq!(phrase_scorer.phrase_count(), 2);
+        assert!(phrase_scorer.advance());
+        assert_eq!(phrase_scorer.doc(), 2);
+        assert_eq!(phrase_scorer.phrase_count(), 1);
+        assert!(!phrase_scorer.advance());
+    }
+}

src/query/query.rs

@@ -2,7 +2,6 @@ use super::Weight;
 use crate::core::searcher::Searcher;
 use crate::query::Explanation;
 use crate::DocAddress;
-use crate::Result;
 use crate::Term;
 use downcast_rs::impl_downcast;
 use std::collections::BTreeSet;
@@ -48,17 +47,17 @@ pub trait Query: QueryClone + downcast_rs::Downcast + fmt::Debug {
     /// can increase performances.
     ///
     /// See [`Weight`](./trait.Weight.html).
-    fn weight(&self, searcher: &Searcher, scoring_enabled: bool) -> Result<Box<dyn Weight>>;
+    fn weight(&self, searcher: &Searcher, scoring_enabled: bool) -> crate::Result<Box<dyn Weight>>;
 
     /// Returns an `Explanation` for the score of the document.
-    fn explain(&self, searcher: &Searcher, doc_address: DocAddress) -> Result<Explanation> {
+    fn explain(&self, searcher: &Searcher, doc_address: DocAddress) -> crate::Result<Explanation> {
         let reader = searcher.segment_reader(doc_address.segment_ord());
         let weight = self.weight(searcher, true)?;
         weight.explain(reader, doc_address.doc())
     }
 
     /// Returns the number of documents matching the query.
-    fn count(&self, searcher: &Searcher) -> Result<usize> {
+    fn count(&self, searcher: &Searcher) -> crate::Result<usize> {
         let weight = self.weight(searcher, false)?;
         let mut result = 0;
         for reader in searcher.segment_readers() {
@@ -86,11 +85,11 @@ where
 }
 
 impl Query for Box<dyn Query> {
-    fn weight(&self, searcher: &Searcher, scoring_enabled: bool) -> Result<Box<dyn Weight>> {
+    fn weight(&self, searcher: &Searcher, scoring_enabled: bool) -> crate::Result<Box<dyn Weight>> {
         self.as_ref().weight(searcher, scoring_enabled)
     }
 
-    fn count(&self, searcher: &Searcher) -> Result<usize> {
+    fn count(&self, searcher: &Searcher) -> crate::Result<usize> {
         self.as_ref().count(searcher)
     }
 

src/query/query_parser/logical_ast.rs

@@ -21,6 +21,17 @@ pub enum LogicalLiteral {
 pub enum LogicalAST {
     Clause(Vec<(Occur, LogicalAST)>),
     Leaf(Box<LogicalLiteral>),
+    Boost(Box<LogicalAST>, f32),
+}
+
+impl LogicalAST {
+    pub fn boost(self, boost: f32) -> LogicalAST {
+        if (boost - 1.0f32).abs() < std::f32::EPSILON {
+            self
+        } else {
+            LogicalAST::Boost(Box::new(self), boost)
+        }
+    }
 }
 
 fn occur_letter(occur: Occur) -> &'static str {
@@ -47,6 +58,7 @@ impl fmt::Debug for LogicalAST {
                 }
                 Ok(())
             }
+            LogicalAST::Boost(ref ast, boost) => write!(formatter, "{:?}^{}", ast, boost),
             LogicalAST::Leaf(ref literal) => write!(formatter, "{:?}", literal),
         }
     }

src/query/query_parser/query_parser.rs

@@ -1,6 +1,5 @@
 use super::logical_ast::*;
 use crate::core::Index;
-use crate::query::AllQuery;
 use crate::query::BooleanQuery;
 use crate::query::EmptyQuery;
 use crate::query::Occur;
@@ -8,11 +7,13 @@ use crate::query::PhraseQuery;
 use crate::query::Query;
 use crate::query::RangeQuery;
 use crate::query::TermQuery;
-use crate::schema::IndexRecordOption;
+use crate::query::{AllQuery, BoostQuery};
+use crate::schema::{Facet, IndexRecordOption};
 use crate::schema::{Field, Schema};
 use crate::schema::{FieldType, Term};
 use crate::tokenizer::TokenizerManager;
 use std::borrow::Cow;
+use std::collections::HashMap;
 use std::num::{ParseFloatError, ParseIntError};
 use std::ops::Bound;
 use std::str::FromStr;
@@ -144,7 +145,6 @@ fn trim_ast(logical_ast: LogicalAST) -> Option<LogicalAST> {
 ///
 /// * must terms: By prepending a term by a `+`, a term can be made required for the search.
 ///
-///
 /// * phrase terms: Quoted terms become phrase searches on fields that have positions indexed.
 ///   e.g., `title:"Barack Obama"` will only find documents that have "barack" immediately followed
 ///   by "obama".
@@ -158,12 +158,20 @@ fn trim_ast(logical_ast: LogicalAST) -> Option<LogicalAST> {
 ///
 /// *  all docs query: A plain `*` will match all documents in the index.
 ///
+/// Parts of the queries can be boosted by appending `^boostfactor`.
+/// For instance, `"SRE"^2.0 OR devops^0.4` will boost documents containing `SRE` instead of
+/// devops. Negative boosts are not allowed.
+///
+/// It is also possible to define a boost for a some specific field, at the query parser level.
+/// (See [`set_boost(...)`](#method.set_field_boost) ). Typically you may want to boost a title
+/// field.
 #[derive(Clone)]
 pub struct QueryParser {
     schema: Schema,
     default_fields: Vec<Field>,
     conjunction_by_default: bool,
     tokenizer_manager: TokenizerManager,
+    boost: HashMap<Field, f32>,
 }
 
 impl QueryParser {
@@ -181,6 +189,7 @@ impl QueryParser {
             default_fields,
             tokenizer_manager,
             conjunction_by_default: false,
+            boost: Default::default(),
         }
     }
 
@@ -201,6 +210,17 @@ impl QueryParser {
         self.conjunction_by_default = true;
     }
 
+    /// Sets a boost for a specific field.
+    ///
+    /// The parse query will automatically boost this field.
+    ///
+    /// If the query defines a query boost through the query language (e.g: `country:France^3.0`),
+    /// the two boosts (the one defined in the query, and the one defined in the `QueryParser`)
+    /// are multiplied together.
+    pub fn set_field_boost(&mut self, field: Field, boost: f32) {
+        self.boost.insert(field, boost);
+    }
+
     /// Parse a query
     ///
     /// Note that `parse_query` returns an error if the input
@@ -319,7 +339,10 @@ impl QueryParser {
                     ))
                 }
             }
-            FieldType::HierarchicalFacet => Ok(vec![(0, Term::from_field_text(field, phrase))]),
+            FieldType::HierarchicalFacet => {
+                let facet = Facet::from_text(phrase);
+                Ok(vec![(0, Term::from_field_text(field, facet.encoded_str()))])
+            }
             FieldType::Bytes => {
                 let field_name = self.schema.get_field_name(field).to_string();
                 Err(QueryParserError::FieldNotIndexed(field_name))
@@ -404,6 +427,10 @@ impl QueryParser {
                     self.compute_logical_ast_with_occur(*subquery)?;
                 Ok((Occur::compose(left_occur, right_occur), logical_sub_queries))
             }
+            UserInputAST::Boost(ast, boost) => {
+                let (occur, ast_without_occur) = self.compute_logical_ast_with_occur(*ast)?;
+                Ok((occur, ast_without_occur.boost(boost)))
+            }
             UserInputAST::Leaf(leaf) => {
                 let result_ast = self.compute_logical_ast_from_leaf(*leaf)?;
                 Ok((Occur::Should, result_ast))
@@ -411,6 +438,10 @@ impl QueryParser {
         }
     }
 
+    fn field_boost(&self, field: Field) -> f32 {
+        self.boost.get(&field).cloned().unwrap_or(1.0f32)
+    }
+
     fn compute_logical_ast_from_leaf(
         &self,
         leaf: UserInputLeaf,
@@ -436,7 +467,9 @@ impl QueryParser {
                 let mut asts: Vec<LogicalAST> = Vec::new();
                 for (field, phrase) in term_phrases {
                     if let Some(ast) = self.compute_logical_ast_for_leaf(field, &phrase)? {
-                        asts.push(LogicalAST::Leaf(Box::new(ast)));
+                        // Apply some field specific boost defined at the query parser level.
+                        let boost = self.field_boost(field);
+                        asts.push(LogicalAST::Leaf(Box::new(ast)).boost(boost));
                     }
                 }
                 let result_ast: LogicalAST = if asts.len() == 1 {
@@ -456,14 +489,16 @@ impl QueryParser {
                 let mut clauses = fields
                     .iter()
                     .map(|&field| {
+                        let boost = self.field_boost(field);
                         let field_entry = self.schema.get_field_entry(field);
                         let value_type = field_entry.field_type().value_type();
-                        Ok(LogicalAST::Leaf(Box::new(LogicalLiteral::Range {
+                        let logical_ast = LogicalAST::Leaf(Box::new(LogicalLiteral::Range {
                             field,
                             value_type,
                             lower: self.resolve_bound(field, &lower)?,
                             upper: self.resolve_bound(field, &upper)?,
-                        })))
+                        }));
+                        Ok(logical_ast.boost(boost))
                     })
                     .collect::<Result<Vec<_>, QueryParserError>>()?;
                 let result_ast = if clauses.len() == 1 {
@@ -516,6 +551,11 @@ fn convert_to_query(logical_ast: LogicalAST) -> Box<dyn Query> {
         Some(LogicalAST::Leaf(trimmed_logical_literal)) => {
             convert_literal_to_query(*trimmed_logical_literal)
         }
+        Some(LogicalAST::Boost(ast, boost)) => {
+            let query = convert_to_query(*ast);
+            let boosted_query = BoostQuery::new(query, boost);
+            Box::new(boosted_query)
+        }
         None => Box::new(EmptyQuery),
     }
 }
@@ -530,12 +570,12 @@ mod test {
     use crate::schema::{IndexRecordOption, TextFieldIndexing, TextOptions};
     use crate::schema::{Schema, Term, INDEXED, STORED, STRING, TEXT};
     use crate::tokenizer::{
-        LowerCaser, SimpleTokenizer, StopWordFilter, Tokenizer, TokenizerManager,
+        LowerCaser, SimpleTokenizer, StopWordFilter, TextAnalyzer, TokenizerManager,
     };
     use crate::Index;
     use matches::assert_matches;
 
-    fn make_query_parser() -> QueryParser {
+    fn make_schema() -> Schema {
         let mut schema_builder = Schema::builder();
         let text_field_indexing = TextFieldIndexing::default()
             .set_tokenizer("en_with_stop_words")
@@ -543,8 +583,8 @@ mod test {
         let text_options = TextOptions::default()
             .set_indexing_options(text_field_indexing)
             .set_stored();
-        let title = schema_builder.add_text_field("title", TEXT);
-        let text = schema_builder.add_text_field("text", TEXT);
+        schema_builder.add_text_field("title", TEXT);
+        schema_builder.add_text_field("text", TEXT);
         schema_builder.add_i64_field("signed", INDEXED);
         schema_builder.add_u64_field("unsigned", INDEXED);
         schema_builder.add_text_field("notindexed_text", STORED);
@@ -554,12 +594,20 @@ mod test {
         schema_builder.add_text_field("with_stop_words", text_options);
         schema_builder.add_date_field("date", INDEXED);
         schema_builder.add_f64_field("float", INDEXED);
-        let schema = schema_builder.build();
-        let default_fields = vec![title, text];
+        schema_builder.add_facet_field("facet");
+        schema_builder.build()
+    }
+
+    fn make_query_parser() -> QueryParser {
+        let schema = make_schema();
+        let default_fields: Vec<Field> = vec!["title", "text"]
+            .into_iter()
+            .flat_map(|field_name| schema.get_field(field_name))
+            .collect();
         let tokenizer_manager = TokenizerManager::default();
         tokenizer_manager.register(
             "en_with_stop_words",
-            SimpleTokenizer
+            TextAnalyzer::from(SimpleTokenizer)
                 .filter(LowerCaser)
                 .filter(StopWordFilter::remove(vec!["the".to_string()])),
         );
@@ -588,9 +636,52 @@ mod test {
     }
 
     #[test]
-    pub fn test_parse_query_simple() {
+    pub fn test_parse_query_facet() {
         let query_parser = make_query_parser();
-        assert!(query_parser.parse_query("toto").is_ok());
+        let query = query_parser.parse_query("facet:/root/branch/leaf").unwrap();
+        assert_eq!(
+            format!("{:?}", query),
+            "TermQuery(Term(field=11,bytes=[114, 111, 111, 116, 0, 98, 114, 97, 110, 99, 104, 0, 108, 101, 97, 102]))"
+        );
+    }
+
+    #[test]
+    pub fn test_parse_query_with_boost() {
+        let mut query_parser = make_query_parser();
+        let schema = make_schema();
+        let text_field = schema.get_field("text").unwrap();
+        query_parser.set_field_boost(text_field, 2.0f32);
+        let query = query_parser.parse_query("text:hello").unwrap();
+        assert_eq!(
+            format!("{:?}", query),
+            "Boost(query=TermQuery(Term(field=1,bytes=[104, 101, 108, 108, 111])), boost=2)"
+        );
+    }
+
+    #[test]
+    pub fn test_parse_query_range_with_boost() {
+        let mut query_parser = make_query_parser();
+        let schema = make_schema();
+        let title_field = schema.get_field("title").unwrap();
+        query_parser.set_field_boost(title_field, 2.0f32);
+        let query = query_parser.parse_query("title:[A TO B]").unwrap();
+        assert_eq!(
+            format!("{:?}", query),
+            "Boost(query=RangeQuery { field: Field(0), value_type: Str, left_bound: Included([97]), right_bound: Included([98]) }, boost=2)"
+        );
+    }
+
+    #[test]
+    pub fn test_parse_query_with_default_boost_and_custom_boost() {
+        let mut query_parser = make_query_parser();
+        let schema = make_schema();
+        let text_field = schema.get_field("text").unwrap();
+        query_parser.set_field_boost(text_field, 2.0f32);
+        let query = query_parser.parse_query("text:hello^2").unwrap();
+        assert_eq!(
+            format!("{:?}", query),
+            "Boost(query=Boost(query=TermQuery(Term(field=1,bytes=[104, 101, 108, 108, 111])), boost=2), boost=2)"
+        );
     }
 
     #[test]

src/query/range_query.rs

@@ -289,7 +289,7 @@ impl RangeWeight {
 }
 
 impl Weight for RangeWeight {
-    fn scorer(&self, reader: &SegmentReader) -> Result<Box<dyn Scorer>> {
+    fn scorer(&self, reader: &SegmentReader, boost: f32) -> Result<Box<dyn Scorer>> {
         let max_doc = reader.max_doc();
         let mut doc_bitset = BitSet::with_max_value(max_doc);
 
@@ -307,11 +307,11 @@ impl Weight for RangeWeight {
             }
         }
         let doc_bitset = BitSetDocSet::from(doc_bitset);
-        Ok(Box::new(ConstScorer::new(doc_bitset)))
+        Ok(Box::new(ConstScorer::new(doc_bitset, boost)))
     }
 
     fn explain(&self, reader: &SegmentReader, doc: DocId) -> Result<Explanation> {
-        let mut scorer = self.scorer(reader)?;
+        let mut scorer = self.scorer(reader, 1.0f32)?;
         if scorer.skip_next(doc) != SkipResult::Reached {
             return Err(does_not_match(doc));
         }

src/query/regex_query.rs

@@ -1,7 +1,6 @@
 use crate::error::TantivyError;
 use crate::query::{AutomatonWeight, Query, Weight};
 use crate::schema::Field;
-use crate::Result;
 use crate::Searcher;
 use std::clone::Clone;
 use std::sync::Arc;
@@ -58,7 +57,7 @@ pub struct RegexQuery {
 
 impl RegexQuery {
     /// Creates a new RegexQuery from a given pattern
-    pub fn from_pattern(regex_pattern: &str, field: Field) -> Result<Self> {
+    pub fn from_pattern(regex_pattern: &str, field: Field) -> crate::Result<Self> {
         let regex = Regex::new(&regex_pattern)
             .map_err(|_| TantivyError::InvalidArgument(regex_pattern.to_string()))?;
         Ok(RegexQuery::from_regex(regex, field))
@@ -78,7 +77,11 @@ impl RegexQuery {
 }
 
 impl Query for RegexQuery {
-    fn weight(&self, _searcher: &Searcher, _scoring_enabled: bool) -> Result<Box<dyn Weight>> {
+    fn weight(
+        &self,
+        _searcher: &Searcher,
+        _scoring_enabled: bool,
+    ) -> crate::Result<Box<dyn Weight>> {
         Ok(Box::new(self.specialized_weight()))
     }
 }

src/query/reqopt_scorer.rs

@@ -115,8 +115,8 @@ mod tests {
         let req = vec![1, 3, 7];
         let mut reqoptscorer: RequiredOptionalScorer<_, _, SumCombiner> =
             RequiredOptionalScorer::new(
-                ConstScorer::new(VecDocSet::from(req.clone())),
-                ConstScorer::new(VecDocSet::from(vec![])),
+                ConstScorer::from(VecDocSet::from(req.clone())),
+                ConstScorer::from(VecDocSet::from(vec![])),
             );
         let mut docs = vec![];
         while reqoptscorer.advance() {
@@ -129,8 +129,8 @@ mod tests {
     fn test_reqopt_scorer() {
         let mut reqoptscorer: RequiredOptionalScorer<_, _, SumCombiner> =
             RequiredOptionalScorer::new(
-                ConstScorer::new(VecDocSet::from(vec![1, 3, 7, 8, 9, 10, 13, 15])),
-                ConstScorer::new(VecDocSet::from(vec![1, 2, 7, 11, 12, 15])),
+                ConstScorer::new(VecDocSet::from(vec![1, 3, 7, 8, 9, 10, 13, 15]), 1.0f32),
+                ConstScorer::new(VecDocSet::from(vec![1, 2, 7, 11, 12, 15]), 1.0f32),
             );
         {
             assert!(reqoptscorer.advance());
@@ -183,8 +183,8 @@ mod tests {
         test_skip_against_unoptimized(
             || {
                 Box::new(RequiredOptionalScorer::<_, _, DoNothingCombiner>::new(
-                    ConstScorer::new(VecDocSet::from(req_docs.clone())),
-                    ConstScorer::new(VecDocSet::from(opt_docs.clone())),
+                    ConstScorer::from(VecDocSet::from(req_docs.clone())),
+                    ConstScorer::from(VecDocSet::from(opt_docs.clone())),
                 ))
             },
             skip_docs,

src/query/scorer.rs

@@ -49,16 +49,14 @@ pub struct ConstScorer<TDocSet: DocSet> {
 
 impl<TDocSet: DocSet> ConstScorer<TDocSet> {
     /// Creates a new `ConstScorer`.
-    pub fn new(docset: TDocSet) -> ConstScorer<TDocSet> {
-        ConstScorer {
-            docset,
-            score: 1f32,
-        }
+    pub fn new(docset: TDocSet, score: f32) -> ConstScorer<TDocSet> {
+        ConstScorer { docset, score }
     }
+}
 
-    /// Sets the constant score to a different value.
-    pub fn set_score(&mut self, score: Score) {
-        self.score = score;
+impl<TDocSet: DocSet> From<TDocSet> for ConstScorer<TDocSet> {
+    fn from(docset: TDocSet) -> Self {
+        ConstScorer::new(docset, 1.0f32)
     }
 }
 
@@ -90,6 +88,6 @@ impl<TDocSet: DocSet> DocSet for ConstScorer<TDocSet> {
 
 impl<TDocSet: DocSet + 'static> Scorer for ConstScorer<TDocSet> {
     fn score(&mut self) -> Score {
-        1f32
+        self.score
     }
 }

src/query/term_query/mod.rs

@@ -39,7 +39,7 @@ mod tests {
         );
         let term_weight = term_query.weight(&searcher, true).unwrap();
         let segment_reader = searcher.segment_reader(0);
-        let mut term_scorer = term_weight.scorer(segment_reader).unwrap();
+        let mut term_scorer = term_weight.scorer(segment_reader, 1.0f32).unwrap();
         assert!(term_scorer.advance());
         assert_eq!(term_scorer.doc(), 0);
         assert_eq!(term_scorer.score(), 0.28768212);

src/query/term_query/term_query.rs

@@ -3,7 +3,6 @@ use crate::query::bm25::BM25Weight;
 use crate::query::Query;
 use crate::query::Weight;
 use crate::schema::IndexRecordOption;
-use crate::Result;
 use crate::Searcher;
 use crate::Term;
 use std::collections::BTreeSet;
@@ -101,7 +100,7 @@ impl TermQuery {
 }
 
 impl Query for TermQuery {
-    fn weight(&self, searcher: &Searcher, scoring_enabled: bool) -> Result<Box<dyn Weight>> {
+    fn weight(&self, searcher: &Searcher, scoring_enabled: bool) -> crate::Result<Box<dyn Weight>> {
         Ok(Box::new(self.specialized_weight(searcher, scoring_enabled)))
     }
     fn query_terms(&self, term_set: &mut BTreeSet<Term>) {

src/query/term_query/term_weight.rs

@@ -18,13 +18,13 @@ pub struct TermWeight {
 }
 
 impl Weight for TermWeight {
-    fn scorer(&self, reader: &SegmentReader) -> Result<Box<dyn Scorer>> {
-        let term_scorer = self.scorer_specialized(reader)?;
+    fn scorer(&self, reader: &SegmentReader, boost: f32) -> Result<Box<dyn Scorer>> {
+        let term_scorer = self.scorer_specialized(reader, boost)?;
         Ok(Box::new(term_scorer))
     }
 
     fn explain(&self, reader: &SegmentReader, doc: DocId) -> Result<Explanation> {
-        let mut scorer = self.scorer_specialized(reader)?;
+        let mut scorer = self.scorer_specialized(reader, 1.0f32)?;
         if scorer.skip_next(doc) != SkipResult::Reached {
             return Err(does_not_match(doc));
         }
@@ -33,7 +33,7 @@ impl Weight for TermWeight {
 
     fn count(&self, reader: &SegmentReader) -> Result<u32> {
         if let Some(delete_bitset) = reader.delete_bitset() {
-            Ok(self.scorer(reader)?.count(delete_bitset))
+            Ok(self.scorer(reader, 1.0f32)?.count(delete_bitset))
         } else {
             let field = self.term.field();
             Ok(reader
@@ -58,11 +58,11 @@ impl TermWeight {
         }
     }
 
-    fn scorer_specialized(&self, reader: &SegmentReader) -> Result<TermScorer> {
+    fn scorer_specialized(&self, reader: &SegmentReader, boost: f32) -> Result<TermScorer> {
         let field = self.term.field();
         let inverted_index = reader.inverted_index(field);
         let fieldnorm_reader = reader.get_fieldnorms_reader(field);
-        let similarity_weight = self.similarity_weight.clone();
+        let similarity_weight = self.similarity_weight.boost_by(boost);
         let postings_opt: Option<SegmentPostings> =
             inverted_index.read_postings(&self.term, self.index_record_option);
         if let Some(segment_postings) = postings_opt {

src/query/union.rs

@@ -145,26 +145,6 @@ where
         }
     }
 
-    fn count_including_deleted(&mut self) -> u32 {
-        let mut count = self.bitsets[self.cursor..HORIZON_NUM_TINYBITSETS]
-            .iter()
-            .map(|bitset| bitset.len())
-            .sum::<u32>();
-        for bitset in self.bitsets.iter_mut() {
-            bitset.clear();
-        }
-        while self.refill() {
-            count += self.bitsets.iter().map(|bitset| bitset.len()).sum::<u32>();
-            for bitset in self.bitsets.iter_mut() {
-                bitset.clear();
-            }
-        }
-        self.cursor = HORIZON_NUM_TINYBITSETS;
-        count
-    }
-
-    // TODO implement `count` efficiently.
-
     fn skip_next(&mut self, target: DocId) -> SkipResult {
         if !self.advance() {
             return SkipResult::End;
@@ -243,6 +223,8 @@ where
         }
     }
 
+    // TODO implement `count` efficiently.
+
     fn doc(&self) -> DocId {
         self.doc
     }
@@ -250,6 +232,24 @@ where
     fn size_hint(&self) -> u32 {
         0u32
     }
+
+    fn count_including_deleted(&mut self) -> u32 {
+        let mut count = self.bitsets[self.cursor..HORIZON_NUM_TINYBITSETS]
+            .iter()
+            .map(|bitset| bitset.len())
+            .sum::<u32>();
+        for bitset in self.bitsets.iter_mut() {
+            bitset.clear();
+        }
+        while self.refill() {
+            count += self.bitsets.iter().map(|bitset| bitset.len()).sum::<u32>();
+            for bitset in self.bitsets.iter_mut() {
+                bitset.clear();
+            }
+        }
+        self.cursor = HORIZON_NUM_TINYBITSETS;
+        count
+    }
 }
 
 impl<TScorer, TScoreCombiner> Scorer for Union<TScorer, TScoreCombiner>
@@ -290,7 +290,7 @@ mod tests {
                 vals.iter()
                     .cloned()
                     .map(VecDocSet::from)
-                    .map(ConstScorer::new)
+                    .map(|docset| ConstScorer::new(docset, 1.0f32))
                     .collect::<Vec<ConstScorer<VecDocSet>>>(),
             )
         };
@@ -339,7 +339,7 @@ mod tests {
                     .iter()
                     .map(|docs| docs.clone())
                     .map(VecDocSet::from)
-                    .map(ConstScorer::new)
+                    .map(|docset| ConstScorer::new(docset, 1.0f32))
                     .collect::<Vec<_>>(),
             ));
             res
@@ -369,8 +369,8 @@ mod tests {
     #[test]
     fn test_union_skip_corner_case3() {
         let mut docset = Union::<_, DoNothingCombiner>::from(vec![
-            ConstScorer::new(VecDocSet::from(vec![0u32, 5u32])),
-            ConstScorer::new(VecDocSet::from(vec![1u32, 4u32])),
+            ConstScorer::from(VecDocSet::from(vec![0u32, 5u32])),
+            ConstScorer::from(VecDocSet::from(vec![1u32, 4u32])),
         ]);
         assert!(docset.advance());
         assert_eq!(docset.doc(), 0u32);

src/query/weight.rs

@@ -1,7 +1,7 @@
 use super::Scorer;
 use crate::core::SegmentReader;
 use crate::query::Explanation;
-use crate::{DocId, Result};
+use crate::DocId;
 
 /// A Weight is the specialization of a Query
 /// for a given set of segments.
@@ -9,15 +9,18 @@ use crate::{DocId, Result};
 /// See [`Query`](./trait.Query.html).
 pub trait Weight: Send + Sync + 'static {
     /// Returns the scorer for the given segment.
+    ///
+    /// `boost` is a multiplier to apply to the score.
+    ///
     /// See [`Query`](./trait.Query.html).
-    fn scorer(&self, reader: &SegmentReader) -> Result<Box<dyn Scorer>>;
+    fn scorer(&self, reader: &SegmentReader, boost: f32) -> crate::Result<Box<dyn Scorer>>;
 
     /// Returns an `Explanation` for the given document.
-    fn explain(&self, reader: &SegmentReader, doc: DocId) -> Result<Explanation>;
+    fn explain(&self, reader: &SegmentReader, doc: DocId) -> crate::Result<Explanation>;
 
     /// Returns the number documents within the given `SegmentReader`.
-    fn count(&self, reader: &SegmentReader) -> Result<u32> {
-        let mut scorer = self.scorer(reader)?;
+    fn count(&self, reader: &SegmentReader) -> crate::Result<u32> {
+        let mut scorer = self.scorer(reader, 1.0f32)?;
         if let Some(delete_bitset) = reader.delete_bitset() {
             Ok(scorer.count(delete_bitset))
         } else {

src/reader/index_writer_reader.rs

@@ -0,0 +1,84 @@
+use crate::directory::{WatchCallbackList, WatchHandle};
+use crate::indexer::SegmentRegisters;
+use crate::reader::pool::Pool;
+use crate::{Index, LeasedItem, Searcher, Segment, SegmentReader};
+use std::iter::repeat_with;
+use std::sync::{Arc, RwLock, Weak};
+
+struct InnerNRTReader {
+    num_searchers: usize,
+    index: Index,
+    searcher_pool: Pool<Searcher>,
+    segment_registers: Arc<RwLock<SegmentRegisters>>,
+}
+
+impl InnerNRTReader {
+    fn load_segment_readers(&self) -> crate::Result<Vec<SegmentReader>> {
+        let segments: Vec<Segment> = self
+            .segment_registers
+            .read()
+            .expect("lock should never be polluted. Please report.")
+            .committed_segment();
+        segments
+            .iter()
+            .map(SegmentReader::open)
+            .collect::<crate::Result<Vec<SegmentReader>>>()
+    }
+
+    pub fn reload(&self) -> crate::Result<()> {
+        let segment_readers: Vec<SegmentReader> = self.load_segment_readers()?;
+        let schema = self.index.schema();
+        let searchers = repeat_with(|| {
+            Searcher::new(schema.clone(), self.index.clone(), segment_readers.clone())
+        })
+        .take(self.num_searchers)
+        .collect();
+        self.searcher_pool.publish_new_generation(searchers);
+        Ok(())
+    }
+
+    pub fn searcher(&self) -> LeasedItem<Searcher> {
+        self.searcher_pool.acquire()
+    }
+}
+
+#[derive(Clone)]
+pub struct NRTReader {
+    inner: Arc<InnerNRTReader>,
+    watch_handle: WatchHandle,
+}
+
+impl NRTReader {
+    pub fn reload(&self) -> crate::Result<()> {
+        self.inner.reload()
+    }
+
+    pub fn searcher(&self) -> LeasedItem<Searcher> {
+        self.inner.searcher()
+    }
+
+    pub(crate) fn create(
+        num_searchers: usize,
+        index: Index,
+        segment_registers: Arc<RwLock<SegmentRegisters>>,
+        watch_callback_list: &WatchCallbackList,
+    ) -> crate::Result<Self> {
+        let inner_reader: Arc<InnerNRTReader> = Arc::new(InnerNRTReader {
+            num_searchers,
+            index,
+            searcher_pool: Pool::new(),
+            segment_registers,
+        });
+        let inner_reader_weak: Weak<InnerNRTReader> = Arc::downgrade(&inner_reader);
+        let watch_handle = watch_callback_list.subscribe(Box::new(move || {
+            if let Some(nrt_reader_arc) = inner_reader_weak.upgrade() {
+                let _ = nrt_reader_arc.reload();
+            }
+        }));
+        inner_reader.reload()?;
+        Ok(NRTReader {
+            inner: inner_reader,
+            watch_handle,
+        })
+    }
+}

src/reader/meta_file_reader.rs

@@ -0,0 +1,177 @@
+use super::pool::Pool;
+use crate::core::Segment;
+use crate::directory::Directory;
+use crate::directory::WatchHandle;
+use crate::directory::META_LOCK;
+use crate::Searcher;
+use crate::SegmentReader;
+use crate::{Index, LeasedItem};
+use crate::{IndexReader, Result};
+use std::iter::repeat_with;
+use std::sync::Arc;
+
+/// Defines when a new version of the index should be reloaded.
+///
+/// Regardless of whether you search and index in the same process, tantivy does not necessarily
+/// reflects the change that are commited to your index. `ReloadPolicy` precisely helps you define
+/// when you want your index to be reloaded.
+#[derive(Clone, Copy)]
+pub enum ReloadPolicy {
+    /// The index is entirely reloaded manually.
+    /// All updates of the index should be manual.
+    ///
+    /// No change is reflected automatically. You are required to call `.load_seacher()` manually.
+    Manual,
+    /// The index is reloaded within milliseconds after a new commit is available.
+    /// This is made possible by watching changes in the `meta.json` file.
+    OnCommit, // TODO add NEAR_REAL_TIME(target_ms)
+}
+
+/// `IndexReader` builder
+///
+/// It makes it possible to set the following values.
+///
+/// - `num_searchers` (by default, the number of detected CPU threads):
+///
+///   When `num_searchers` queries are requested at the same time, the `num_searchers` will block
+///   until the one of the searcher in-use gets released.
+/// - `reload_policy` (by default `ReloadPolicy::OnCommit`):
+///
+///   See [`ReloadPolicy`](./enum.ReloadPolicy.html) for more details.
+#[derive(Clone)]
+pub struct IndexReaderBuilder {
+    num_searchers: usize,
+    reload_policy: ReloadPolicy,
+    index: Index,
+}
+
+impl IndexReaderBuilder {
+    pub(crate) fn new(index: Index) -> IndexReaderBuilder {
+        IndexReaderBuilder {
+            num_searchers: num_cpus::get(),
+            reload_policy: ReloadPolicy::Manual,
+            index,
+        }
+    }
+
+    /// Sets the reload_policy.
+    ///
+    /// See [`ReloadPolicy`](./enum.ReloadPolicy.html) for more details.
+    pub fn reload_policy(mut self, reload_policy: ReloadPolicy) -> IndexReaderBuilder {
+        self.reload_policy = reload_policy;
+        self
+    }
+
+    /// Sets the number of `Searcher` in the searcher pool.
+    pub fn num_searchers(mut self, num_searchers: usize) -> IndexReaderBuilder {
+        self.num_searchers = num_searchers;
+        self
+    }
+
+    /// Building the reader is a non-trivial operation that requires
+    /// to open different segment readers. It may take hundreds of milliseconds
+    /// of time and it may return an error.
+    pub fn try_into(self) -> crate::Result<IndexReader> {
+        let inner_reader = MetaFileIndexReaderInner {
+            index: self.index,
+            num_searchers: self.num_searchers,
+            searcher_pool: Pool::new(),
+        };
+        inner_reader.reload()?;
+        let inner_reader_arc = Arc::new(inner_reader);
+        let watch_handle_opt: Option<WatchHandle>;
+        match self.reload_policy {
+            ReloadPolicy::Manual => {
+                // No need to set anything...
+                watch_handle_opt = None;
+            }
+            ReloadPolicy::OnCommit => {
+                let inner_reader_arc_clone = inner_reader_arc.clone();
+                let callback = move || {
+                    if let Err(err) = inner_reader_arc_clone.reload() {
+                        error!(
+                            "Error while loading searcher after commit was detected. {:?}",
+                            err
+                        );
+                    }
+                };
+                let watch_handle = inner_reader_arc
+                    .index
+                    .directory()
+                    .watch(Box::new(callback))?;
+                watch_handle_opt = Some(watch_handle);
+            }
+        }
+        Ok(IndexReader::from(MetaFileIndexReader {
+            inner: inner_reader_arc,
+            watch_handle_opt,
+        }))
+    }
+}
+
+struct MetaFileIndexReaderInner {
+    num_searchers: usize,
+    searcher_pool: Pool<Searcher>,
+    index: Index,
+}
+
+impl MetaFileIndexReaderInner {
+    fn load_segment_readers(&self) -> crate::Result<Vec<SegmentReader>> {
+        // We keep the lock until we have effectively finished opening the
+        // the `SegmentReader` because it prevents a diffferent process
+        // to garbage collect these file while we open them.
+        //
+        // Once opened, on linux & mac, the mmap will remain valid after
+        // the file has been deleted
+        // On windows, the file deletion will fail.
+        let _meta_lock = self.index.directory().acquire_lock(&META_LOCK)?;
+        let searchable_segments = self.searchable_segments()?;
+        searchable_segments
+            .iter()
+            .map(SegmentReader::open)
+            .collect::<Result<_>>()
+    }
+
+    fn reload(&self) -> crate::Result<()> {
+        let segment_readers: Vec<SegmentReader> = self.load_segment_readers()?;
+        let schema = self.index.schema();
+        let searchers = repeat_with(|| {
+            Searcher::new(schema.clone(), self.index.clone(), segment_readers.clone())
+        })
+        .take(self.num_searchers)
+        .collect();
+        self.searcher_pool.publish_new_generation(searchers);
+        Ok(())
+    }
+
+    /// Returns the list of segments that are searchable
+    fn searchable_segments(&self) -> crate::Result<Vec<Segment>> {
+        self.index.searchable_segments()
+    }
+
+    fn searcher(&self) -> LeasedItem<Searcher> {
+        self.searcher_pool.acquire()
+    }
+}
+
+/// `IndexReader` is your entry point to read and search the index.
+///
+/// It controls when a new version of the index should be loaded and lends
+/// you instances of `Searcher` for the last loaded version.
+///
+/// `Clone` does not clone the different pool of searcher. `IndexReader`
+/// just wraps and `Arc`.
+#[derive(Clone)]
+pub struct MetaFileIndexReader {
+    inner: Arc<MetaFileIndexReaderInner>,
+    watch_handle_opt: Option<WatchHandle>,
+}
+
+impl MetaFileIndexReader {
+    pub fn reload(&self) -> crate::Result<()> {
+        self.inner.reload()
+    }
+    pub fn searcher(&self) -> LeasedItem<Searcher> {
+        self.inner.searcher()
+    }
+}

src/reader/mod.rs

@@ -1,152 +1,14 @@
+mod index_writer_reader;
+mod meta_file_reader;
 mod pool;
 
+use self::meta_file_reader::MetaFileIndexReader;
+pub use self::meta_file_reader::{IndexReaderBuilder, ReloadPolicy};
 pub use self::pool::LeasedItem;
-use self::pool::Pool;
-use crate::core::Segment;
-use crate::directory::Directory;
-use crate::directory::WatchHandle;
-use crate::directory::META_LOCK;
-use crate::Index;
-use crate::Result;
+
+pub(crate) use crate::reader::index_writer_reader::NRTReader;
+
 use crate::Searcher;
-use crate::SegmentReader;
-use std::sync::Arc;
-
-/// Defines when a new version of the index should be reloaded.
-///
-/// Regardless of whether you search and index in the same process, tantivy does not necessarily
-/// reflects the change that are commited to your index. `ReloadPolicy` precisely helps you define
-/// when you want your index to be reloaded.
-#[derive(Clone, Copy)]
-pub enum ReloadPolicy {
-    /// The index is entirely reloaded manually.
-    /// All updates of the index should be manual.
-    ///
-    /// No change is reflected automatically. You are required to call `.load_seacher()` manually.
-    Manual,
-    /// The index is reloaded within milliseconds after a new commit is available.
-    /// This is made possible by watching changes in the `meta.json` file.
-    OnCommit, // TODO add NEAR_REAL_TIME(target_ms)
-}
-
-/// `IndexReader` builder
-///
-/// It makes it possible to set the following values.
-///
-/// - `num_searchers` (by default, the number of detected CPU threads):
-///
-///   When `num_searchers` queries are requested at the same time, the `num_searchers` will block
-///   until the one of the searcher in-use gets released.
-/// - `reload_policy` (by default `ReloadPolicy::OnCommit`):
-///
-///   See [`ReloadPolicy`](./enum.ReloadPolicy.html) for more details.
-#[derive(Clone)]
-pub struct IndexReaderBuilder {
-    num_searchers: usize,
-    reload_policy: ReloadPolicy,
-    index: Index,
-}
-
-impl IndexReaderBuilder {
-    pub(crate) fn new(index: Index) -> IndexReaderBuilder {
-        IndexReaderBuilder {
-            num_searchers: num_cpus::get(),
-            reload_policy: ReloadPolicy::OnCommit,
-            index,
-        }
-    }
-
-    /// Builds the reader.
-    ///
-    /// Building the reader is a non-trivial operation that requires
-    /// to open different segment readers. It may take hundreds of milliseconds
-    /// of time and it may return an error.
-    /// TODO(pmasurel) Use the `TryInto` trait once it is available in stable.
-    pub fn try_into(self) -> Result<IndexReader> {
-        let inner_reader = InnerIndexReader {
-            index: self.index,
-            num_searchers: self.num_searchers,
-            searcher_pool: Pool::new(),
-        };
-        inner_reader.reload()?;
-        let inner_reader_arc = Arc::new(inner_reader);
-        let watch_handle_opt: Option<WatchHandle>;
-        match self.reload_policy {
-            ReloadPolicy::Manual => {
-                // No need to set anything...
-                watch_handle_opt = None;
-            }
-            ReloadPolicy::OnCommit => {
-                let inner_reader_arc_clone = inner_reader_arc.clone();
-                let callback = move || {
-                    if let Err(err) = inner_reader_arc_clone.reload() {
-                        error!(
-                            "Error while loading searcher after commit was detected. {:?}",
-                            err
-                        );
-                    }
-                };
-                let watch_handle = inner_reader_arc
-                    .index
-                    .directory()
-                    .watch(Box::new(callback))?;
-                watch_handle_opt = Some(watch_handle);
-            }
-        }
-        Ok(IndexReader {
-            inner: inner_reader_arc,
-            watch_handle_opt,
-        })
-    }
-
-    /// Sets the reload_policy.
-    ///
-    /// See [`ReloadPolicy`](./enum.ReloadPolicy.html) for more details.
-    pub fn reload_policy(mut self, reload_policy: ReloadPolicy) -> IndexReaderBuilder {
-        self.reload_policy = reload_policy;
-        self
-    }
-
-    /// Sets the number of `Searcher` in the searcher pool.
-    pub fn num_searchers(mut self, num_searchers: usize) -> IndexReaderBuilder {
-        self.num_searchers = num_searchers;
-        self
-    }
-}
-
-struct InnerIndexReader {
-    num_searchers: usize,
-    searcher_pool: Pool<Searcher>,
-    index: Index,
-}
-
-impl InnerIndexReader {
-    fn reload(&self) -> Result<()> {
-        let segment_readers: Vec<SegmentReader> = {
-            let _meta_lock = self.index.directory().acquire_lock(&META_LOCK)?;
-            let searchable_segments = self.searchable_segments()?;
-            searchable_segments
-                .iter()
-                .map(SegmentReader::open)
-                .collect::<Result<_>>()?
-        };
-        let schema = self.index.schema();
-        let searchers = (0..self.num_searchers)
-            .map(|_| Searcher::new(schema.clone(), self.index.clone(), segment_readers.clone()))
-            .collect();
-        self.searcher_pool.publish_new_generation(searchers);
-        Ok(())
-    }
-
-    /// Returns the list of segments that are searchable
-    fn searchable_segments(&self) -> Result<Vec<Segment>> {
-        self.index.searchable_segments()
-    }
-
-    fn searcher(&self) -> LeasedItem<Searcher> {
-        self.searcher_pool.acquire()
-    }
-}
 
 /// `IndexReader` is your entry point to read and search the index.
 ///
@@ -156,17 +18,12 @@ impl InnerIndexReader {
 /// `Clone` does not clone the different pool of searcher. `IndexReader`
 /// just wraps and `Arc`.
 #[derive(Clone)]
-pub struct IndexReader {
-    inner: Arc<InnerIndexReader>,
-    watch_handle_opt: Option<WatchHandle>,
+pub enum IndexReader {
+    FromMetaFile(MetaFileIndexReader),
+    NRT(NRTReader),
 }
 
 impl IndexReader {
-    #[cfg(test)]
-    pub(crate) fn index(&self) -> Index {
-        self.inner.index.clone()
-    }
-
     /// Update searchers so that they reflect the state of the last
     /// `.commit()`.
     ///
@@ -176,8 +33,11 @@ impl IndexReader {
     ///
     /// This automatic reload can take 10s of milliseconds to kick in however, and in unit tests
     /// it can be nice to deterministically force the reload of searchers.
-    pub fn reload(&self) -> Result<()> {
-        self.inner.reload()
+    pub fn reload(&self) -> crate::Result<()> {
+        match self {
+            IndexReader::FromMetaFile(meta_file_reader) => meta_file_reader.reload(),
+            IndexReader::NRT(nrt_reader) => nrt_reader.reload(),
+        }
     }
 
     /// Returns a searcher
@@ -191,6 +51,21 @@ impl IndexReader {
     /// The same searcher must be used for a given query, as it ensures
     /// the use of a consistent segment set.
     pub fn searcher(&self) -> LeasedItem<Searcher> {
-        self.inner.searcher()
+        match self {
+            IndexReader::FromMetaFile(meta_file_reader) => meta_file_reader.searcher(),
+            IndexReader::NRT(nrt_reader) => nrt_reader.searcher(),
+        }
+    }
+}
+
+impl From<MetaFileIndexReader> for IndexReader {
+    fn from(meta_file_reader: MetaFileIndexReader) -> Self {
+        IndexReader::FromMetaFile(meta_file_reader)
+    }
+}
+
+impl From<NRTReader> for IndexReader {
+    fn from(nrt_reader: NRTReader) -> Self {
+        IndexReader::NRT(nrt_reader)
     }
 }

src/reader/pool.rs

@@ -68,7 +68,9 @@ impl<T> Pool<T> {
     /// After publish, all new `Searcher` acquired will be
     /// of the new generation.
     pub fn publish_new_generation(&self, items: Vec<T>) {
+        assert!(!items.is_empty());
         let next_generation = self.next_generation.fetch_add(1, Ordering::SeqCst) + 1;
+        let num_items = items.len();
         for item in items {
             let gen_item = GenerationItem {
                 item,
@@ -77,6 +79,23 @@ impl<T> Pool<T> {
             self.queue.push(gen_item);
         }
         self.advertise_generation(next_generation);
+        // Purge possible previous searchers.
+        //
+        // Assuming at this point no searcher is held more than duration T by the user,
+        // this guarantees that an obsolete searcher will not be uselessly held (and its associated
+        // mmap) for more than duration T.
+        //
+        // Proof: At this point, obsolete searcher that are held by the user will be held for less
+        // than T. When released, they will be dropped as their generation is detected obsolete.
+        //
+        // We still need to ensure that the searcher that are obsolete and in the pool get removed.
+        // The queue currently contains up to 2n searchers, in any random order.
+        //
+        // Half of them are obsoletes. By requesting `(n+1)` fresh searchers, we ensure that all
+        // searcher will be inspected.
+        for _ in 0..=num_items {
+            let _ = self.acquire();
+        }
     }
 
     /// At the exit of this method,

src/schema/document.rs

@@ -3,8 +3,9 @@ use crate::common::BinarySerializable;
 use crate::common::VInt;
 use crate::tokenizer::PreTokenizedString;
 use crate::DateTime;
-use itertools::Itertools;
+use serde;
 use std::io::{self, Read, Write};
+use std::mem;
 
 /// Tantivy's Document is the object that can
 /// be indexed and then searched for.
@@ -16,7 +17,7 @@ use std::io::{self, Read, Write};
 
 /// Documents are really just a list of couple `(field, value)`.
 /// In this list, one field may appear more than once.
-#[derive(Clone, Debug, Serialize, Deserialize, Default)]
+#[derive(Clone, Debug, serde::Serialize, serde::Deserialize, Default)]
 pub struct Document {
     field_values: Vec<FieldValue>,
 }
@@ -131,12 +132,34 @@ impl Document {
     pub fn get_sorted_field_values(&self) -> Vec<(Field, Vec<&FieldValue>)> {
         let mut field_values: Vec<&FieldValue> = self.field_values().iter().collect();
         field_values.sort_by_key(|field_value| field_value.field());
-        field_values
-            .into_iter()
-            .group_by(|field_value| field_value.field())
-            .into_iter()
-            .map(|(key, group)| (key, group.collect()))
-            .collect::<Vec<(Field, Vec<&FieldValue>)>>()
+
+        let mut grouped_field_values = vec![];
+
+        let mut current_field;
+        let mut current_group;
+
+        let mut field_values_it = field_values.into_iter();
+        if let Some(field_value) = field_values_it.next() {
+            current_field = field_value.field();
+            current_group = vec![field_value]
+        } else {
+            return grouped_field_values;
+        }
+
+        for field_value in field_values_it {
+            if field_value.field() == current_field {
+                current_group.push(field_value);
+            } else {
+                grouped_field_values.push((
+                    current_field,
+                    mem::replace(&mut current_group, vec![field_value]),
+                ));
+                current_field = field_value.field();
+            }
+        }
+
+        grouped_field_values.push((current_field, current_group));
+        grouped_field_values
     }
 
     /// Returns all of the `FieldValue`s associated the given field

src/schema/facet.rs

@@ -122,6 +122,11 @@ impl Facet {
     pub fn to_path(&self) -> Vec<&str> {
         self.encoded_str().split(|c| c == FACET_SEP_CHAR).collect()
     }
+
+    /// This function is the inverse of Facet::from(&str).
+    pub fn to_path_string(&self) -> String {
+        format!("{}", self)
+    }
 }
 
 impl Borrow<str> for Facet {
@@ -265,4 +270,21 @@ mod tests {
         let facet = Facet::from_path(v.iter());
         assert_eq!(facet.to_path(), v);
     }
+
+    #[test]
+    fn test_to_path_string() {
+        let v = ["first", "second", "third/not_fourth"];
+        let facet = Facet::from_path(v.iter());
+        assert_eq!(
+            facet.to_path_string(),
+            String::from("/first/second/third\\/not_fourth")
+        );
+    }
+
+    #[test]
+    fn test_to_path_string_empty() {
+        let v: Vec<&str> = vec![];
+        let facet = Facet::from_path(v.iter());
+        assert_eq!(facet.to_path_string(), "/");
+    }
 }

src/schema/field.rs

@@ -1,11 +1,14 @@
 use crate::common::BinarySerializable;
+use serde;
 use std::io;
 use std::io::Read;
 use std::io::Write;
 
 /// `Field` is represented by an unsigned 32-bit integer type
 /// The schema holds the mapping between field names and `Field` objects.
-#[derive(Copy, Clone, Debug, PartialEq, PartialOrd, Eq, Ord, Hash, Serialize, Deserialize)]
+#[derive(
+    Copy, Clone, Debug, PartialEq, PartialOrd, Eq, Ord, Hash, serde::Serialize, serde::Deserialize,
+)]
 pub struct Field(u32);
 
 impl Field {

src/schema/field_value.rs

@@ -1,12 +1,13 @@
 use crate::common::BinarySerializable;
 use crate::schema::Field;
 use crate::schema::Value;
+use serde;
 use std::io;
 use std::io::Read;
 use std::io::Write;
 
 /// `FieldValue` holds together a `Field` and its `Value`.
-#[derive(Debug, Clone, Ord, PartialEq, Eq, PartialOrd, Serialize, Deserialize)]
+#[derive(Debug, Clone, Ord, PartialEq, Eq, PartialOrd, serde::Serialize, serde::Deserialize)]
 pub struct FieldValue {
     field: Field,
     value: Value,

src/schema/index_record_option.rs

@@ -1,3 +1,5 @@
+use serde::{Deserialize, Serialize};
+
 /// `IndexRecordOption` describes an amount information associated
 /// to a given indexed field.
 ///