macos ci

u64-based fields are actually 4+8=12 bytes long

.travis.yml

@@ -41,8 +41,8 @@ matrix:
     - env: TARGET=x86_64-unknown-linux-gnu CODECOV=1 #UPLOAD_DOCS=1
     # - env: TARGET=x86_64-unknown-linux-musl CODECOV=1
     # OSX
-    #- env: TARGET=x86_64-apple-darwin
+    - env: TARGET=x86_64-apple-darwin
-    #  os: osx
+      os: osx
 
 before_install:
   - set -e

CHANGELOG.md

@@ -7,10 +7,13 @@ Tantivy 0.11.0
     - Better handling of whitespaces.
 - Closes #498 - add support for Elastic-style unbounded range queries for alphanumeric types eg. "title:>hello", "weight:>=70.5", "height:<200" (@petr-tik)
 - API change around `Box<BoxableTokenizer>`. See detail in #629
-
+- Avoid rebuilding Regex automaton whenever a regex query is reused. #630 (@brainlock)
+ 
 ## How to update?
 
-`Box<dyn BoxableTokenizer>` has been replaced by a `BoxedTokenizer` struct.
+- `Box<dyn BoxableTokenizer>` has been replaced by a `BoxedTokenizer` struct.
+- Regex are now compiled when the `RegexQuery` instance is built. As a result, it can now return
+an error and handling the `Result` is required.
 
 Tantivy 0.10.1
 =====================

Cargo.toml

@@ -15,8 +15,8 @@ edition = "2018"
 [dependencies]
 base64 = "0.10.0"
 byteorder = "1.0"
-once_cell = "0.2"
+once_cell = "1.0"
-regex = "1.0"
+regex ={version = "1.3.0", default-features = false, features = ["std"]}
 tantivy-fst = "0.1"
 memmap = {version = "0.7", optional=true}
 lz4 = {version="1.20", optional=true}
@@ -24,7 +24,6 @@ snap = {version="0.2"}
 atomicwrites = {version="0.2.2", optional=true}
 tempfile = "3.0"
 log = "0.4"
-combine = ">=3.6.0,<4.0.0"
 serde = "1.0"
 serde_derive = "1.0"
 serde_json = "1.0"
@@ -42,6 +41,7 @@ owning_ref = "0.4"
 stable_deref_trait = "1.0.0"
 rust-stemmers = "1.1"
 downcast-rs = { version="1.0" }
+tantivy-query-grammar = { path="./query-grammar" }
 bitpacking = {version="0.8", default-features = false, features=["bitpacker4x"]}
 census = "0.2"
 fnv = "1.0.6"
@@ -80,10 +80,14 @@ failpoints = ["fail/failpoints"]
 unstable = [] # useful for benches.
 wasm-bindgen = ["uuid/wasm-bindgen"]
 
+[workspace]
+members = ["query-grammar"]
+
 [badges]
 travis-ci = { repository = "tantivy-search/tantivy" }
 
 [dev-dependencies.fail]
+version = "0.3"
 features = ["failpoints"]
 
 # Following the "fail" crate best practises, we isolate

Makefile

@@ -0,0 +1,3 @@
+test:
+	echo "Run test only... No examples."
+	cargo test --tests --lib

ci/script.sh

@@ -7,7 +7,7 @@ set -ex
 main() {
     if [ ! -z $CODECOV ]; then
         echo "Codecov"
-        cargo build --verbose && cargo coverage --verbose && bash <(curl -s https://codecov.io/bash) -s target/kcov
+        cargo build --verbose && cargo coverage --verbose --all && bash <(curl -s https://codecov.io/bash) -s target/kcov
     else
         echo "Build"
         cross build --target $TARGET
@@ -15,7 +15,8 @@ main() {
             return
         fi
         echo "Test"
-        cross test --target $TARGET --no-default-features --features mmap -- --test-threads 1
+        cross test --target $TARGET --no-default-features --features mmap
+        cross test --target $TARGET --no-default-features --features mmap query-grammar
     fi
     for example in $(ls examples/*.rs)
     do

examples/basic_search.rs

@@ -5,20 +5,17 @@
 //
 // We will :
 // - define our schema
-// = create an index in a directory
+// - create an index in a directory
-// - index few documents in our index
+// - index a few documents into our index
-// - search for the best document matchings "sea whale"
+// - search for the best document matching a basic query
-// - retrieve the best document original content.
+// - retrieve the best document's original content.
 
 // ---
 // Importing tantivy...
-#[macro_use]
-extern crate tantivy;
 use tantivy::collector::TopDocs;
 use tantivy::query::QueryParser;
 use tantivy::schema::*;
-use tantivy::Index;
+use tantivy::{doc, Index, ReloadPolicy};
-use tantivy::ReloadPolicy;
 use tempfile::TempDir;
 
 fn main() -> tantivy::Result<()> {
@@ -33,7 +30,7 @@ fn main() -> tantivy::Result<()> {
     // and for each field, its type and "the way it should
     // be indexed".
 
-    // first we need to define a schema ...
+    // First we need to define a schema ...
     let mut schema_builder = Schema::builder();
 
     // Our first field is title.
@@ -48,7 +45,7 @@ fn main() -> tantivy::Result<()> {
     //
     // `STORED` means that the field will also be saved
     // in a compressed, row-oriented key-value store.
-    // This store is useful to reconstruct the
+    // This store is useful for reconstructing the
     // documents that were selected during the search phase.
     schema_builder.add_text_field("title", TEXT | STORED);
 
@@ -57,8 +54,7 @@ fn main() -> tantivy::Result<()> {
     // need to be able to be able to retrieve it
     // for our application.
     //
-    // We can make our index lighter and
+    // We can make our index lighter by omitting the `STORED` flag.
-    // by omitting `STORED` flag.
     schema_builder.add_text_field("body", TEXT);
 
     let schema = schema_builder.build();
@@ -71,7 +67,7 @@ fn main() -> tantivy::Result<()> {
     // with our schema in the directory.
     let index = Index::create_in_dir(&index_path, schema.clone())?;
 
-    // To insert document we need an index writer.
+    // To insert a document we will need an index writer.
     // There must be only one writer at a time.
     // This single `IndexWriter` is already
     // multithreaded.
@@ -149,8 +145,8 @@ fn main() -> tantivy::Result<()> {
     // At this point our documents are not searchable.
     //
     //
-    // We need to call .commit() explicitly to force the
+    // We need to call `.commit()` explicitly to force the
-    // index_writer to finish processing the documents in the queue,
+    // `index_writer` to finish processing the documents in the queue,
     // flush the current index to the disk, and advertise
     // the existence of new documents.
     //
@@ -162,14 +158,14 @@ fn main() -> tantivy::Result<()> {
     // persistently indexed.
     //
     // In the scenario of a crash or a power failure,
-    // tantivy behaves as if has rolled back to its last
+    // tantivy behaves as if it has rolled back to its last
     // commit.
 
     // # Searching
     //
     // ### Searcher
     //
-    // A reader is required to get search the index.
+    // A reader is required first in order to search an index.
     // It acts as a `Searcher` pool that reloads itself,
     // depending on a `ReloadPolicy`.
     //
@@ -185,7 +181,7 @@ fn main() -> tantivy::Result<()> {
 
     // We now need to acquire a searcher.
     //
-    // A searcher points to snapshotted, immutable version of the index.
+    // A searcher points to a snapshotted, immutable version of the index.
     //
     // Some search experience might require more than
     // one query. Using the same searcher ensures that all of these queries will run on the
@@ -205,7 +201,7 @@ fn main() -> tantivy::Result<()> {
     // in both title and body.
     let query_parser = QueryParser::for_index(&index, vec![title, body]);
 
-    // QueryParser may fail if the query is not in the right
+    // `QueryParser` may fail if the query is not in the right
     // format. For user facing applications, this can be a problem.
     // A ticket has been opened regarding this problem.
     let query = query_parser.parse_query("sea whale")?;
@@ -221,7 +217,7 @@ fn main() -> tantivy::Result<()> {
     //
     // We are not interested in all of the documents but
     // only in the top 10. Keeping track of our top 10 best documents
-    // is the role of the TopDocs.
+    // is the role of the `TopDocs` collector.
 
     // We can now perform our query.
     let top_docs = searcher.search(&query, &TopDocs::with_limit(10))?;

examples/custom_collector.rs

@@ -9,15 +9,12 @@
 
 // ---
 // Importing tantivy...
-#[macro_use]
-extern crate tantivy;
 use tantivy::collector::{Collector, SegmentCollector};
 use tantivy::fastfield::FastFieldReader;
 use tantivy::query::QueryParser;
 use tantivy::schema::Field;
 use tantivy::schema::{Schema, FAST, INDEXED, TEXT};
-use tantivy::SegmentReader;
+use tantivy::{doc, Index, SegmentReader, TantivyError};
-use tantivy::{Index, TantivyError};
 
 #[derive(Default)]
 struct Stats {

examples/custom_tokenizer.rs

@@ -2,14 +2,11 @@
 //
 // In this example, we'll see how to define a tokenizer pipeline
 // by aligning a bunch of `TokenFilter`.
-
-#[macro_use]
-extern crate tantivy;
 use tantivy::collector::TopDocs;
 use tantivy::query::QueryParser;
 use tantivy::schema::*;
 use tantivy::tokenizer::NgramTokenizer;
-use tantivy::Index;
+use tantivy::{doc, Index};
 
 fn main() -> tantivy::Result<()> {
     // # Defining the schema

examples/deleting_updating_documents.rs

@@ -8,13 +8,10 @@
 //
 // ---
 // Importing tantivy...
-#[macro_use]
-extern crate tantivy;
 use tantivy::collector::TopDocs;
 use tantivy::query::TermQuery;
 use tantivy::schema::*;
-use tantivy::Index;
+use tantivy::{doc, Index, IndexReader};
-use tantivy::IndexReader;
 
 // A simple helper function to fetch a single document
 // given its id from our index.

examples/faceted_search.rs

@@ -12,12 +12,10 @@
 
 // ---
 // Importing tantivy...
-#[macro_use]
-extern crate tantivy;
 use tantivy::collector::FacetCollector;
 use tantivy::query::AllQuery;
 use tantivy::schema::*;
-use tantivy::Index;
+use tantivy::{doc, Index};
 use tempfile::TempDir;
 
 fn main() -> tantivy::Result<()> {

examples/integer_range_search.rs

@@ -2,14 +2,10 @@
 //
 // Below is an example of creating an indexed integer field in your schema
 // You can use RangeQuery to get a Count of all occurrences in a given range.
-
-#[macro_use]
-extern crate tantivy;
 use tantivy::collector::Count;
 use tantivy::query::RangeQuery;
 use tantivy::schema::{Schema, INDEXED};
-use tantivy::Index;
+use tantivy::{doc, Index, Result};
-use tantivy::Result;
 
 fn run() -> Result<()> {
     // For the sake of simplicity, this schema will only have 1 field

examples/iterating_docs_and_positions.rs

@@ -9,11 +9,8 @@
 
 // ---
 // Importing tantivy...
-#[macro_use]
-extern crate tantivy;
 use tantivy::schema::*;
-use tantivy::Index;
+use tantivy::{doc, DocId, DocSet, Index, Postings};
-use tantivy::{DocId, DocSet, Postings};
 
 fn main() -> tantivy::Result<()> {
     // We first create a schema for the sake of the

examples/multiple_producer.rs

@@ -25,14 +25,11 @@
 
 // ---
 // Importing tantivy...
-#[macro_use]
-extern crate tantivy;
 use std::sync::{Arc, RwLock};
 use std::thread;
 use std::time::Duration;
 use tantivy::schema::{Schema, STORED, TEXT};
-use tantivy::Opstamp;
+use tantivy::{doc, Index, IndexWriter, Opstamp};
-use tantivy::{Index, IndexWriter};
 
 fn main() -> tantivy::Result<()> {
     // # Defining the schema
@@ -49,10 +46,9 @@ fn main() -> tantivy::Result<()> {
     thread::spawn(move || {
         // we index 100 times the document... for the sake of the example.
         for i in 0..100 {
-            let opstamp = {
+            let opstamp = index_writer_clone_1
-                // A read lock is sufficient here.
+                .read().unwrap() //< A read lock is sufficient here.
-                let index_writer_rlock = index_writer_clone_1.read().unwrap();
+                .add_document(
-                index_writer_rlock.add_document(
                     doc!(
                         title => "Of Mice and Men",
                         body => "A few miles south of Soledad, the Salinas River drops in close to the hillside \
@@ -63,8 +59,7 @@ fn main() -> tantivy::Result<()> {
                         fresh and green with every spring, carrying in their lower leaf junctures the \
                         debris of the winter’s flooding; and sycamores with mottled, white, recumbent \
                         limbs and branches that arch over the pool"
-                    ))
+                    ));
-            };
             println!("add doc {} from thread 1 - opstamp {}", i, opstamp);
             thread::sleep(Duration::from_millis(20));
         }

examples/snippet.rs

@@ -7,13 +7,10 @@
 
 // ---
 // Importing tantivy...
-#[macro_use]
-extern crate tantivy;
 use tantivy::collector::TopDocs;
 use tantivy::query::QueryParser;
 use tantivy::schema::*;
-use tantivy::Index;
+use tantivy::{doc, Index, Snippet, SnippetGenerator};
-use tantivy::{Snippet, SnippetGenerator};
 use tempfile::TempDir;
 
 fn main() -> tantivy::Result<()> {

examples/stop_words.rs

@@ -11,13 +11,11 @@
 
 // ---
 // Importing tantivy...
-#[macro_use]
-extern crate tantivy;
 use tantivy::collector::TopDocs;
 use tantivy::query::QueryParser;
 use tantivy::schema::*;
 use tantivy::tokenizer::*;
-use tantivy::Index;
+use tantivy::{doc, Index};
 
 fn main() -> tantivy::Result<()> {
     // this example assumes you understand the content in `basic_search`

query-grammar/Cargo.toml

@@ -0,0 +1,16 @@
+[package]
+name = "tantivy-query-grammar"
+version = "0.11.0"
+authors = ["Paul Masurel <paul.masurel@gmail.com>"]
+license = "MIT"
+categories = ["database-implementations", "data-structures"]
+description = """Search engine library"""
+documentation = "https://tantivy-search.github.io/tantivy/tantivy/index.html"
+homepage = "https://github.com/tantivy-search/tantivy"
+repository = "https://github.com/tantivy-search/tantivy"
+readme = "README.md"
+keywords = ["search", "information", "retrieval"]
+edition = "2018"
+
+[dependencies]
+combine = ">=3.6.0,<4.0.0"

query-grammar/src/lib.rs

@@ -0,0 +1,17 @@
+#![recursion_limit = "100"]
+
+mod occur;
+mod query_grammar;
+mod user_input_ast;
+use combine::parser::Parser;
+
+pub use crate::occur::Occur;
+use crate::query_grammar::parse_to_ast;
+pub use crate::user_input_ast::{UserInputAST, UserInputBound, UserInputLeaf, UserInputLiteral};
+
+pub struct Error;
+
+pub fn parse_query(query: &str) -> Result<UserInputAST, Error> {
+    let (user_input_ast, _remaining) = parse_to_ast().parse(query).map_err(|_| Error)?;
+    Ok(user_input_ast)
+}

query-grammar/src/occur.rs

@@ -1,3 +1,6 @@
+use std::fmt;
+use std::fmt::Write;
+
 /// Defines whether a term in a query must be present,
 /// should be present or must not be present.
 #[derive(Debug, Clone, Hash, Copy, Eq, PartialEq)]
@@ -18,32 +21,38 @@ impl Occur {
     /// - `Should` => '?',
     /// - `Must` => '+'
     /// - `Not` => '-'
-    pub fn to_char(self) -> char {
+    fn to_char(self) -> char {
         match self {
             Occur::Should => '?',
             Occur::Must => '+',
             Occur::MustNot => '-',
         }
     }
-}
 
-/// Compose two occur values.
+    /// Compose two occur values.
-pub fn compose_occur(left: Occur, right: Occur) -> Occur {
+    pub fn compose(left: Occur, right: Occur) -> Occur {
-    match left {
+        match left {
-        Occur::Should => right,
+            Occur::Should => right,
-        Occur::Must => {
+            Occur::Must => {
-            if right == Occur::MustNot {
+                if right == Occur::MustNot {
-                Occur::MustNot
+                    Occur::MustNot
-            } else {
+                } else {
-                Occur::Must
+                    Occur::Must
+                }
             }
-        }
+            Occur::MustNot => {
-        Occur::MustNot => {
+                if right == Occur::MustNot {
-            if right == Occur::MustNot {
+                    Occur::Must
-                Occur::Must
+                } else {
-            } else {
+                    Occur::MustNot
-                Occur::MustNot
+                }
             }
         }
     }
 }
+
+impl fmt::Display for Occur {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        f.write_char(self.to_char())
+    }
+}

query-grammar/src/query_grammar.rs

@@ -1,6 +1,5 @@
 use super::user_input_ast::*;
-use crate::query::occur::Occur;
+use crate::Occur;
-use crate::query::query_parser::user_input_ast::UserInputBound;
 use combine::char::*;
 use combine::error::StreamError;
 use combine::stream::StreamErrorFor;

query-grammar/src/user_input_ast.rs

@@ -1,7 +1,7 @@
 use std::fmt;
 use std::fmt::{Debug, Formatter};
 
-use crate::query::Occur;
+use crate::Occur;
 
 #[derive(PartialEq)]
 pub enum UserInputLeaf {
@@ -151,7 +151,7 @@ impl fmt::Debug for UserInputAST {
                 Ok(())
             }
             UserInputAST::Unary(ref occur, ref subquery) => {
-                write!(formatter, "{}({:?})", occur.to_char(), subquery)
+                write!(formatter, "{}({:?})", occur, subquery)
             }
             UserInputAST::Leaf(ref subquery) => write!(formatter, "{:?}", subquery),
         }

src/collector/count_collector.rs

@@ -10,12 +10,10 @@ use crate::SegmentReader;
 /// documents match the query.
 ///
 /// ```rust
-/// #[macro_use]
-/// extern crate tantivy;
-/// use tantivy::schema::{Schema, TEXT};
-/// use tantivy::{Index, Result};
 /// use tantivy::collector::Count;
 /// use tantivy::query::QueryParser;
+/// use tantivy::schema::{Schema, TEXT};
+/// use tantivy::{doc, Index, Result};
 ///
 /// # fn main() { example().unwrap(); }
 /// fn example() -> Result<()> {

src/collector/facet_collector.rs

@@ -81,12 +81,10 @@ fn facet_depth(facet_bytes: &[u8]) -> usize {
 ///
 ///
 /// ```rust
-/// #[macro_use]
-/// extern crate tantivy;
-/// use tantivy::schema::{Facet, Schema, TEXT};
-/// use tantivy::{Index, Result};
 /// use tantivy::collector::FacetCollector;
 /// use tantivy::query::AllQuery;
+/// use tantivy::schema::{Facet, Schema, TEXT};
+/// use tantivy::{doc, Index, Result};
 ///
 /// # fn main() { example().unwrap(); }
 /// fn example() -> Result<()> {

src/collector/mod.rs

@@ -35,7 +35,6 @@ The resulting `Fruit` will then be a typed tuple with each collector's original
 in their respective position.
 
 ```rust
-# extern crate tantivy;
 # use tantivy::schema::*;
 # use tantivy::*;
 # use tantivy::query::*;

src/collector/multi_collector.rs

@@ -105,12 +105,10 @@ impl<TFruit: Fruit> FruitHandle<TFruit> {
 /// [Combining several collectors section of the collector documentation](./index.html#combining-several-collectors).
 ///
 /// ```rust
-/// #[macro_use]
-/// extern crate tantivy;
-/// use tantivy::schema::{Schema, TEXT};
-/// use tantivy::{Index, Result};
 /// use tantivy::collector::{Count, TopDocs, MultiCollector};
 /// use tantivy::query::QueryParser;
+/// use tantivy::schema::{Schema, TEXT};
+/// use tantivy::{doc, Index, Result};
 ///
 /// # fn main() { example().unwrap(); }
 /// fn example() -> Result<()> {

src/collector/top_score_collector.rs

@@ -23,13 +23,10 @@ use std::fmt;
 /// is `O(n log K)`.
 ///
 /// ```rust
-/// #[macro_use]
-/// extern crate tantivy;
-/// use tantivy::DocAddress;
-/// use tantivy::schema::{Schema, TEXT};
-/// use tantivy::{Index, Result};
 /// use tantivy::collector::TopDocs;
 /// use tantivy::query::QueryParser;
+/// use tantivy::schema::{Schema, TEXT};
+/// use tantivy::{doc, DocAddress, Index, Result};
 ///
 /// # fn main() { example().unwrap(); }
 /// fn example() -> Result<()> {
@@ -87,10 +84,8 @@ impl TopDocs {
     /// Set top-K to rank documents by a given fast field.
     ///
     /// ```rust
-    /// #[macro_use]
-    /// extern crate tantivy;
     /// # use tantivy::schema::{Schema, FAST, TEXT};
-    /// # use tantivy::{Index, Result, DocAddress};
+    /// # use tantivy::{doc, Index, Result, DocAddress};
     /// # use tantivy::query::{Query, QueryParser};
     /// use tantivy::Searcher;
     /// use tantivy::collector::TopDocs;
@@ -128,7 +123,7 @@ impl TopDocs {
     /// ///
     /// /// `field` is required to be a FAST field.
     /// fn docs_sorted_by_rating(searcher: &Searcher,
-    ///                          query: &Query,
+    ///                          query: &dyn Query,
     ///                          sort_by_field: Field)
     ///     -> Result<Vec<(u64, DocAddress)>> {
     ///
@@ -197,10 +192,8 @@ impl TopDocs {
     /// learning-to-rank model over various features
     ///
     /// ```rust
-    /// #[macro_use]
-    /// extern crate tantivy;
     /// # use tantivy::schema::{Schema, FAST, TEXT};
-    /// # use tantivy::{Index, DocAddress, DocId, Score};
+    /// # use tantivy::{doc, Index, DocAddress, DocId, Score};
     /// # use tantivy::query::QueryParser;
     /// use tantivy::SegmentReader;
     /// use tantivy::collector::TopDocs;
@@ -302,10 +295,8 @@ impl TopDocs {
     /// # Example
     ///
     /// ```rust
-    /// # #[macro_use]
-    /// # extern crate tantivy;
     /// # use tantivy::schema::{Schema, FAST, TEXT};
-    /// # use tantivy::{Index, DocAddress, DocId};
+    /// # use tantivy::{doc, Index, DocAddress, DocId};
     /// # use tantivy::query::QueryParser;
     /// use tantivy::SegmentReader;
     /// use tantivy::collector::TopDocs;

src/core/index.rs

@@ -216,8 +216,22 @@ impl Index {
         Index::open(mmap_directory)
     }
 
-    pub(crate) fn inventory(&self) -> &SegmentMetaInventory {
+    /// Returns the list of the segment metas tracked by the index.
-        &self.inventory
+    ///
+    /// Such segments can of course be part of the index,
+    /// but also they could be segments being currently built or in the middle of a merge
+    /// operation.
+    pub fn list_all_segment_metas(&self) -> Vec<SegmentMeta> {
+        self.inventory.all()
+    }
+
+    /// Creates a new segment_meta (Advanced user only).
+    ///
+    /// As long as the `SegmentMeta` lives, the files associated with the
+    /// `SegmentMeta` are guaranteed to not be garbage collected, regardless of
+    /// whether the segment is recorded as part of the index or not.
+    pub fn new_segment_meta(&self, segment_id: SegmentId, max_doc: u32) -> SegmentMeta {
+        self.inventory.new_segment_meta(segment_id, max_doc)
     }
 
     /// Open the index using the provided directory

src/core/index_meta.rs

@@ -30,7 +30,6 @@ impl SegmentMetaInventory {
             .collect::<Vec<_>>()
     }
 
-    #[doc(hidden)]
     pub fn new_segment_meta(&self, segment_id: SegmentId, max_doc: u32) -> SegmentMeta {
         let inner = InnerSegmentMeta {
             segment_id,

src/core/segment_id.rs

@@ -4,6 +4,8 @@ use uuid::Uuid;
 
 #[cfg(test)]
 use once_cell::sync::Lazy;
+use std::error::Error;
+use std::str::FromStr;
 #[cfg(test)]
 use std::sync::atomic;
 
@@ -52,15 +54,51 @@ impl SegmentId {
     /// and the rest is random.
     ///
     /// Picking the first 8 chars is ok to identify
-    /// segments in a display message.
+    /// segments in a display message (e.g. a5c4dfcb).
     pub fn short_uuid_string(&self) -> String {
         (&self.0.to_simple_ref().to_string()[..8]).to_string()
     }
 
     /// Returns a segment uuid string.
+    ///
+    /// It consists in 32 lowercase hexadecimal chars
+    /// (e.g. a5c4dfcbdfe645089129e308e26d5523)
     pub fn uuid_string(&self) -> String {
         self.0.to_simple_ref().to_string()
     }
+
+    /// Build a `SegmentId` string from the full uuid string.
+    ///
+    /// E.g. "a5c4dfcbdfe645089129e308e26d5523"
+    pub fn from_uuid_string(uuid_string: &str) -> Result<SegmentId, SegmentIdParseError> {
+        FromStr::from_str(uuid_string)
+    }
+}
+
+/// Error type used when parsing a `SegmentId` from a string fails.
+pub struct SegmentIdParseError(uuid::parser::ParseError);
+
+impl Error for SegmentIdParseError {}
+
+impl fmt::Debug for SegmentIdParseError {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        self.0.fmt(f)
+    }
+}
+
+impl fmt::Display for SegmentIdParseError {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        self.0.fmt(f)
+    }
+}
+
+impl FromStr for SegmentId {
+    type Err = SegmentIdParseError;
+
+    fn from_str(uuid_string: &str) -> Result<Self, SegmentIdParseError> {
+        let uuid = Uuid::parse_str(uuid_string).map_err(SegmentIdParseError)?;
+        Ok(SegmentId(uuid))
+    }
 }
 
 impl fmt::Debug for SegmentId {
@@ -80,3 +118,18 @@ impl Ord for SegmentId {
         self.0.as_bytes().cmp(other.0.as_bytes())
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::SegmentId;
+
+    #[test]
+    fn test_to_uuid_string() {
+        let full_uuid = "a5c4dfcbdfe645089129e308e26d5523";
+        let segment_id = SegmentId::from_uuid_string(full_uuid).unwrap();
+        assert_eq!(segment_id.uuid_string(), full_uuid);
+        assert_eq!(segment_id.short_uuid_string(), "a5c4dfcb");
+        // one extra char
+        assert!(SegmentId::from_uuid_string("a5c4dfcbdfe645089129e308e26d5523b").is_err());
+    }
+}

src/directory/mmap_directory.rs

@@ -265,7 +265,7 @@ impl MmapDirectoryInner {
             }
         }
         if let Some(watch_wrapper) = self.watcher.write().unwrap().as_mut() {
-            return Ok(watch_wrapper.watch(watch_callback));
+            Ok(watch_wrapper.watch(watch_callback))
         } else {
             unreachable!("At this point, watch wrapper is supposed to be initialized");
         }

src/fastfield/multivalued/writer.rs

@@ -5,8 +5,8 @@ use crate::postings::UnorderedTermId;
 use crate::schema::{Document, Field};
 use crate::termdict::TermOrdinal;
 use crate::DocId;
+use fnv::FnvHashMap;
 use itertools::Itertools;
-use std::collections::HashMap;
 use std::io;
 
 /// Writer for multi-valued (as in, more than one value per document)
@@ -102,7 +102,7 @@ impl MultiValueIntFastFieldWriter {
     pub fn serialize(
         &self,
         serializer: &mut FastFieldSerializer,
-        mapping_opt: Option<&HashMap<UnorderedTermId, TermOrdinal>>,
+        mapping_opt: Option<&FnvHashMap<UnorderedTermId, TermOrdinal>>,
     ) -> io::Result<()> {
         {
             // writing the offset index

src/fastfield/writer.rs

@@ -6,6 +6,7 @@ use crate::fastfield::{BytesFastFieldWriter, FastFieldSerializer};
 use crate::postings::UnorderedTermId;
 use crate::schema::{Cardinality, Document, Field, FieldType, Schema};
 use crate::termdict::TermOrdinal;
+use fnv::FnvHashMap;
 use std::collections::HashMap;
 use std::io;
 
@@ -116,7 +117,7 @@ impl FastFieldsWriter {
     pub fn serialize(
         &self,
         serializer: &mut FastFieldSerializer,
-        mapping: &HashMap<Field, HashMap<UnorderedTermId, TermOrdinal>>,
+        mapping: &HashMap<Field, FnvHashMap<UnorderedTermId, TermOrdinal>>,
     ) -> io::Result<()> {
         for field_writer in &self.single_value_writers {
             field_writer.serialize(serializer)?;

src/indexer/index_writer.rs

@@ -209,10 +209,7 @@ fn index_documents(
     assert!(num_docs > 0);
 
     let doc_opstamps: Vec<Opstamp> = segment_writer.finalize()?;
-    let segment_meta = segment
+    let segment_meta = segment.index().new_segment_meta(segment_id, num_docs);
-        .index()
-        .inventory()
-        .new_segment_meta(segment_id, num_docs);
 
     let last_docstamp: Opstamp = *(doc_opstamps.last().unwrap());
 
@@ -450,12 +447,10 @@ impl IndexWriter {
     /// by clearing and resubmitting necessary documents
     ///
     /// ```rust
-    /// #[macro_use]
-    /// extern crate tantivy;
-    /// use tantivy::query::QueryParser;
     /// use tantivy::collector::TopDocs;
+    /// use tantivy::query::QueryParser;
     /// use tantivy::schema::*;
-    /// use tantivy::Index;
+    /// use tantivy::{doc, Index};
     ///
     /// fn main() -> tantivy::Result<()> {
     ///     let mut schema_builder = Schema::builder();

src/indexer/segment_updater.rs

@@ -126,9 +126,7 @@ fn perform_merge(
 
     let num_docs = merger.write(segment_serializer)?;
 
-    let segment_meta = index
+    let segment_meta = index.new_segment_meta(merged_segment.id(), num_docs);
-        .inventory()
-        .new_segment_meta(merged_segment.id(), num_docs);
 
     let after_merge_segment_entry = SegmentEntry::new(segment_meta.clone(), delete_cursor, None);
     Ok(after_merge_segment_entry)
@@ -282,7 +280,7 @@ impl SegmentUpdater {
     fn list_files(&self) -> HashSet<PathBuf> {
         let mut files = HashSet::new();
         files.insert(META_FILEPATH.to_path_buf());
-        for segment_meta in self.0.index.inventory().all() {
+        for segment_meta in self.0.index.list_all_segment_metas() {
             files.extend(segment_meta.list_files());
         }
         files

src/lib.rs

@@ -1,5 +1,4 @@
 #![doc(html_logo_url = "http://fulmicoton.com/tantivy-logo/tantivy-logo.png")]
-#![recursion_limit = "100"]
 #![cfg_attr(all(feature = "unstable", test), feature(test))]
 #![cfg_attr(feature = "cargo-clippy", allow(clippy::module_inception))]
 #![doc(test(attr(allow(unused_variables), deny(warnings))))]
@@ -11,21 +10,12 @@
 //! Think `Lucene`, but in Rust.
 //!
 //! ```rust
-
-//! # extern crate tempfile;
-//! #
-//! #[macro_use]
-//! extern crate tantivy;
-//!
-//! // ...
-//!
 //! # use std::path::Path;
 //! # use tempfile::TempDir;
-//! # use tantivy::Index;
-//! # use tantivy::schema::*;
-//! # use tantivy::{Score, DocAddress};
 //! # use tantivy::collector::TopDocs;
 //! # use tantivy::query::QueryParser;
+//! # use tantivy::schema::*;
+//! # use tantivy::{doc, DocAddress, Index, Score};
 //! #
 //! # fn main() {
 //! #     // Let's create a temporary directory for the

src/macros.rs

@@ -22,11 +22,9 @@
 ///
 /// # Example
 ///
-/// ```
+/// ```rust
-/// #[macro_use]
-/// extern crate tantivy;
-///
 /// use tantivy::schema::{Schema, TEXT, FAST};
+/// use tantivy::doc;
 ///
 /// //...
 ///

src/postings/postings_writer.rs

@@ -12,6 +12,7 @@ use crate::tokenizer::TokenStream;
 use crate::tokenizer::{Token, MAX_TOKEN_LEN};
 use crate::DocId;
 use crate::Result;
+use fnv::FnvHashMap;
 use std::collections::HashMap;
 use std::io;
 use std::marker::PhantomData;
@@ -127,12 +128,12 @@ impl MultiFieldPostingsWriter {
     pub fn serialize(
         &self,
         serializer: &mut InvertedIndexSerializer,
-    ) -> Result<HashMap<Field, HashMap<UnorderedTermId, TermOrdinal>>> {
+    ) -> Result<HashMap<Field, FnvHashMap<UnorderedTermId, TermOrdinal>>> {
         let mut term_offsets: Vec<(&[u8], Addr, UnorderedTermId)> =
             self.term_index.iter().collect();
         term_offsets.sort_unstable_by_key(|&(k, _, _)| k);
 
-        let mut unordered_term_mappings: HashMap<Field, HashMap<UnorderedTermId, TermOrdinal>> =
+        let mut unordered_term_mappings: HashMap<Field, FnvHashMap<UnorderedTermId, TermOrdinal>> =
             HashMap::new();
 
         let field_offsets = make_field_partition(&term_offsets);
@@ -147,7 +148,7 @@ impl MultiFieldPostingsWriter {
                     let unordered_term_ids = term_offsets[start..stop]
                         .iter()
                         .map(|&(_, _, bucket)| bucket);
-                    let mapping: HashMap<UnorderedTermId, TermOrdinal> = unordered_term_ids
+                    let mapping: FnvHashMap<UnorderedTermId, TermOrdinal> = unordered_term_ids
                         .enumerate()
                         .map(|(term_ord, unord_term_id)| {
                             (unord_term_id as UnorderedTermId, term_ord as TermOrdinal)

src/postings/serializer.rs

@@ -141,10 +141,7 @@ impl<'a> FieldSerializer<'a> {
             FieldType::Str(ref text_options) => {
                 if let Some(text_indexing_options) = text_options.get_indexing_options() {
                     let index_option = text_indexing_options.index_option();
-                    (
+                    (index_option.has_freq(), index_option.has_positions())
-                        index_option.is_termfreq_enabled(),
-                        index_option.is_position_enabled(),
-                    )
                 } else {
                     (false, false)
                 }

src/query/automaton_weight.rs

@@ -8,15 +8,13 @@ use crate::termdict::{TermDictionary, TermStreamer};
 use crate::DocId;
 use crate::TantivyError;
 use crate::{Result, SkipResult};
+use std::sync::Arc;
 use tantivy_fst::Automaton;
 
 /// A weight struct for Fuzzy Term and Regex Queries
-pub struct AutomatonWeight<A>
+pub struct AutomatonWeight<A> {
-where
-    A: Automaton + Send + Sync + 'static,
-{
     field: Field,
-    automaton: A,
+    automaton: Arc<A>,
 }
 
 impl<A> AutomatonWeight<A>
@@ -24,12 +22,16 @@ where
     A: Automaton + Send + Sync + 'static,
 {
     /// Create a new AutomationWeight
-    pub fn new(field: Field, automaton: A) -> AutomatonWeight<A> {
+    pub fn new<IntoArcA: Into<Arc<A>>>(field: Field, automaton: IntoArcA) -> AutomatonWeight<A> {
-        AutomatonWeight { field, automaton }
+        AutomatonWeight {
+            field,
+            automaton: automaton.into(),
+        }
     }
 
     fn automaton_stream<'a>(&'a self, term_dict: &'a TermDictionary) -> TermStreamer<'a, &'a A> {
-        let term_stream_builder = term_dict.search(&self.automaton);
+        let automaton: &A = &*self.automaton;
+        let term_stream_builder = term_dict.search(automaton);
         term_stream_builder.into_stream()
     }
 }

src/query/fuzzy_query.rs

@@ -28,12 +28,10 @@ static LEV_BUILDER: Lazy<HashMap<(u8, bool), LevenshteinAutomatonBuilder>> = Laz
 /// containing a specific term that is within
 /// Levenshtein distance
 /// ```rust
-/// #[macro_use]
-/// extern crate tantivy;
-/// use tantivy::schema::{Schema, TEXT};
-/// use tantivy::{Index, Result, Term};
 /// use tantivy::collector::{Count, TopDocs};
 /// use tantivy::query::FuzzyTermQuery;
+/// use tantivy::schema::{Schema, TEXT};
+/// use tantivy::{doc, Index, Result, Term};
 ///
 /// # fn main() { example().unwrap(); }
 /// fn example() -> Result<()> {

src/query/intersection.rs

@@ -45,7 +45,7 @@ pub fn intersect_scorers(mut scorers: Vec<Box<dyn Scorer>>) -> Box<dyn Scorer> {
     })
 }
 
-/// Creates a `DocSet` that iterator through the intersection of two `DocSet`s.
+/// Creates a `DocSet` that iterate through the intersection of two or more `DocSet`s.
 pub struct Intersection<TDocSet: DocSet, TOtherDocSet: DocSet = Box<dyn Scorer>> {
     left: TDocSet,
     right: TDocSet,

src/query/intersection_two.rs

@@ -5,7 +5,7 @@ use Score;
 use SkipResult;
 
 
-/// Creates a `DocSet` that iterator through the intersection of two `DocSet`s.
+/// Creates a `DocSet` that iterate through the intersection of two `DocSet`s.
 pub struct IntersectionTwoTerms<TDocSet> {
     left: TDocSet,
     right: TDocSet

src/query/mod.rs

@@ -12,7 +12,6 @@ mod exclude;
 mod explanation;
 mod fuzzy_query;
 mod intersection;
-mod occur;
 mod phrase_query;
 mod query;
 mod query_parser;
@@ -43,7 +42,6 @@ pub use self::exclude::Exclude;
 pub use self::explanation::Explanation;
 pub use self::fuzzy_query::FuzzyTermQuery;
 pub use self::intersection::intersect_scorers;
-pub use self::occur::Occur;
 pub use self::phrase_query::PhraseQuery;
 pub use self::query::Query;
 pub use self::query_parser::QueryParser;
@@ -55,6 +53,7 @@ pub use self::scorer::ConstScorer;
 pub use self::scorer::Scorer;
 pub use self::term_query::TermQuery;
 pub use self::weight::Weight;
+pub use tantivy_query_grammar::Occur;
 
 #[cfg(test)]
 mod tests {

src/query/query_parser/mod.rs

@@ -1,6 +1,4 @@
-mod query_grammar;
 mod query_parser;
-mod user_input_ast;
 
 pub mod logical_ast;
 pub use self::query_parser::QueryParser;

src/query/query_parser/query_parser.rs

@@ -1,9 +1,5 @@
 use super::logical_ast::*;
-use super::query_grammar::parse_to_ast;
-use super::user_input_ast::*;
 use crate::core::Index;
-use crate::query::occur::compose_occur;
-use crate::query::query_parser::logical_ast::LogicalAST;
 use crate::query::AllQuery;
 use crate::query::BooleanQuery;
 use crate::query::EmptyQuery;
@@ -16,11 +12,11 @@ use crate::schema::IndexRecordOption;
 use crate::schema::{Field, Schema};
 use crate::schema::{FieldType, Term};
 use crate::tokenizer::TokenizerManager;
-use combine::Parser;
 use std::borrow::Cow;
 use std::num::{ParseFloatError, ParseIntError};
 use std::ops::Bound;
 use std::str::FromStr;
+use tantivy_query_grammar::{UserInputAST, UserInputBound, UserInputLeaf};
 
 /// Possible error that may happen when parsing a query.
 #[derive(Debug, PartialEq, Eq, Fail)]
@@ -222,9 +218,8 @@ impl QueryParser {
 
     /// Parse the user query into an AST.
     fn parse_query_to_logical_ast(&self, query: &str) -> Result<LogicalAST, QueryParserError> {
-        let (user_input_ast, _remaining) = parse_to_ast()
+        let user_input_ast =
-            .parse(query)
+            tantivy_query_grammar::parse_query(query).map_err(|_| QueryParserError::SyntaxError)?;
-            .map_err(|_| QueryParserError::SyntaxError)?;
         self.compute_logical_ast(user_input_ast)
     }
 
@@ -399,7 +394,7 @@ impl QueryParser {
                 let mut logical_sub_queries: Vec<(Occur, LogicalAST)> = Vec::new();
                 for sub_query in sub_queries {
                     let (occur, sub_ast) = self.compute_logical_ast_with_occur(sub_query)?;
-                    let new_occur = compose_occur(default_occur, occur);
+                    let new_occur = Occur::compose(default_occur, occur);
                     logical_sub_queries.push((new_occur, sub_ast));
                 }
                 Ok((Occur::Should, LogicalAST::Clause(logical_sub_queries)))
@@ -407,7 +402,7 @@ impl QueryParser {
             UserInputAST::Unary(left_occur, subquery) => {
                 let (right_occur, logical_sub_queries) =
                     self.compute_logical_ast_with_occur(*subquery)?;
-                Ok((compose_occur(left_occur, right_occur), logical_sub_queries))
+                Ok((Occur::compose(left_occur, right_occur), logical_sub_queries))
             }
             UserInputAST::Leaf(leaf) => {
                 let result_ast = self.compute_logical_ast_from_leaf(*leaf)?;

src/query/range_query.rs

@@ -38,14 +38,10 @@ fn map_bound<TFrom, TTo, Transform: Fn(&TFrom) -> TTo>(
 /// # Example
 ///
 /// ```rust
-///
-/// # #[macro_use]
-/// # extern crate tantivy;
-/// # use tantivy::Index;
-/// # use tantivy::schema::{Schema, INDEXED};
 /// # use tantivy::collector::Count;
-/// # use tantivy::Result;
 /// # use tantivy::query::RangeQuery;
+/// # use tantivy::schema::{Schema, INDEXED};
+/// # use tantivy::{doc, Index, Result};
 /// #
 /// # fn run() -> Result<()> {
 /// #     let mut schema_builder = Schema::builder();

src/query/regex_query.rs

@@ -4,22 +4,18 @@ use crate::schema::Field;
 use crate::Result;
 use crate::Searcher;
 use std::clone::Clone;
+use std::sync::Arc;
 use tantivy_fst::Regex;
 
-// A Regex Query matches all of the documents
+/// A Regex Query matches all of the documents
 /// containing a specific term that matches
-/// a regex pattern
+/// a regex pattern.
-/// A Fuzzy Query matches all of the documents
-/// containing a specific term that is within
-/// Levenshtein distance
 ///
 /// ```rust
-/// #[macro_use]
-/// extern crate tantivy;
-/// use tantivy::schema::{Schema, TEXT};
-/// use tantivy::{Index, Result, Term};
 /// use tantivy::collector::Count;
 /// use tantivy::query::RegexQuery;
+/// use tantivy::schema::{Schema, TEXT};
+/// use tantivy::{doc, Index, Result, Term};
 ///
 /// # fn main() { example().unwrap(); }
 /// fn example() -> Result<()> {
@@ -48,7 +44,7 @@ use tantivy_fst::Regex;
 ///     let searcher = reader.searcher();
 ///
 ///     let term = Term::from_field_text(title, "Diary");
-///     let query = RegexQuery::new("d[ai]{2}ry".to_string(), title);
+///     let query = RegexQuery::from_pattern("d[ai]{2}ry", title)?;
 ///     let count = searcher.search(&query, &Count)?;
 ///     assert_eq!(count, 3);
 ///     Ok(())
@@ -56,30 +52,34 @@ use tantivy_fst::Regex;
 /// ```
 #[derive(Debug, Clone)]
 pub struct RegexQuery {
-    regex_pattern: String,
+    regex: Arc<Regex>,
     field: Field,
 }
 
 impl RegexQuery {
-    /// Creates a new Fuzzy Query
+    /// Creates a new RegexQuery from a given pattern
-    pub fn new(regex_pattern: String, field: Field) -> RegexQuery {
+    pub fn from_pattern(regex_pattern: &str, field: Field) -> Result<Self> {
+        let regex = Regex::new(&regex_pattern)
+            .map_err(|_| TantivyError::InvalidArgument(regex_pattern.to_string()))?;
+        Ok(RegexQuery::from_regex(regex, field))
+    }
+
+    /// Creates a new RegexQuery from a fully built Regex
+    pub fn from_regex<T: Into<Arc<Regex>>>(regex: T, field: Field) -> Self {
         RegexQuery {
-            regex_pattern,
+            regex: regex.into(),
             field,
         }
     }
 
-    fn specialized_weight(&self) -> Result<AutomatonWeight<Regex>> {
+    fn specialized_weight(&self) -> AutomatonWeight<Regex> {
-        let automaton = Regex::new(&self.regex_pattern)
+        AutomatonWeight::new(self.field, self.regex.clone())
-            .map_err(|_| TantivyError::InvalidArgument(self.regex_pattern.clone()))?;
-
-        Ok(AutomatonWeight::new(self.field, automaton))
     }
 }
 
 impl Query for RegexQuery {
     fn weight(&self, _searcher: &Searcher, _scoring_enabled: bool) -> Result<Box<dyn Weight>> {
-        Ok(Box::new(self.specialized_weight()?))
+        Ok(Box::new(self.specialized_weight()))
     }
 }
 
@@ -87,13 +87,14 @@ impl Query for RegexQuery {
 mod test {
     use super::RegexQuery;
     use crate::collector::TopDocs;
-    use crate::schema::Schema;
     use crate::schema::TEXT;
+    use crate::schema::{Field, Schema};
     use crate::tests::assert_nearly_equals;
-    use crate::Index;
+    use crate::{Index, IndexReader};
+    use std::sync::Arc;
+    use tantivy_fst::Regex;
 
-    #[test]
+    fn build_test_index() -> (IndexReader, Field) {
-    pub fn test_regex_query() {
         let mut schema_builder = Schema::builder();
         let country_field = schema_builder.add_text_field("country", TEXT);
         let schema = schema_builder.build();
@@ -109,20 +110,65 @@ mod test {
             index_writer.commit().unwrap();
         }
         let reader = index.reader().unwrap();
+
+        (reader, country_field)
+    }
+
+    fn verify_regex_query(
+        query_matching_one: RegexQuery,
+        query_matching_zero: RegexQuery,
+        reader: IndexReader,
+    ) {
         let searcher = reader.searcher();
         {
-            let regex_query = RegexQuery::new("jap[ao]n".to_string(), country_field);
             let scored_docs = searcher
-                .search(&regex_query, &TopDocs::with_limit(2))
+                .search(&query_matching_one, &TopDocs::with_limit(2))
                 .unwrap();
             assert_eq!(scored_docs.len(), 1, "Expected only 1 document");
             let (score, _) = scored_docs[0];
             assert_nearly_equals(1f32, score);
         }
-        let regex_query = RegexQuery::new("jap[A-Z]n".to_string(), country_field);
         let top_docs = searcher
-            .search(&regex_query, &TopDocs::with_limit(2))
+            .search(&query_matching_zero, &TopDocs::with_limit(2))
             .unwrap();
         assert!(top_docs.is_empty(), "Expected ZERO document");
     }
+
+    #[test]
+    pub fn test_regex_query() {
+        let (reader, field) = build_test_index();
+
+        let matching_one = RegexQuery::from_pattern("jap[ao]n", field).unwrap();
+        let matching_zero = RegexQuery::from_pattern("jap[A-Z]n", field).unwrap();
+
+        verify_regex_query(matching_one, matching_zero, reader);
+    }
+
+    #[test]
+    pub fn test_construct_from_regex() {
+        let (reader, field) = build_test_index();
+
+        let matching_one = RegexQuery::from_regex(Regex::new("jap[ao]n").unwrap(), field);
+        let matching_zero = RegexQuery::from_regex(Regex::new("jap[A-Z]n").unwrap(), field);
+
+        verify_regex_query(matching_one, matching_zero, reader);
+    }
+
+    #[test]
+    pub fn test_construct_from_reused_regex() {
+        let r1 = Arc::new(Regex::new("jap[ao]n").unwrap());
+        let r2 = Arc::new(Regex::new("jap[A-Z]n").unwrap());
+
+        let (reader, field) = build_test_index();
+
+        let matching_one = RegexQuery::from_regex(r1.clone(), field);
+        let matching_zero = RegexQuery::from_regex(r2.clone(), field);
+
+        verify_regex_query(matching_one, matching_zero, reader.clone());
+
+        let matching_one = RegexQuery::from_regex(r1.clone(), field);
+        let matching_zero = RegexQuery::from_regex(r2.clone(), field);
+
+        verify_regex_query(matching_one, matching_zero, reader.clone());
+    }
 }

src/query/term_query/term_query.rs

@@ -20,12 +20,10 @@ use std::fmt;
 /// * `field norm` - number of tokens in the field.
 ///
 /// ```rust
-/// #[macro_use]
-/// extern crate tantivy;
-/// use tantivy::schema::{Schema, TEXT, IndexRecordOption};
-/// use tantivy::{Index, Result, Term};
 /// use tantivy::collector::{Count, TopDocs};
 /// use tantivy::query::TermQuery;
+/// use tantivy::schema::{Schema, TEXT, IndexRecordOption};
+/// use tantivy::{doc, Index, Result, Term};
 ///
 /// # fn main() { example().unwrap(); }
 /// fn example() -> Result<()> {

src/query/union.rs

@@ -28,7 +28,7 @@ where
     }
 }
 
-/// Creates a `DocSet` that iterator through the intersection of two `DocSet`s.
+/// Creates a `DocSet` that iterate through the union of two or more `DocSet`s.
 pub struct Union<TScorer, TScoreCombiner = DoNothingCombiner> {
     docsets: Vec<TScorer>,
     bitsets: Box<[TinySet; HORIZON_NUM_TINYBITSETS]>,

src/schema/index_record_option.rs

@@ -29,22 +29,6 @@ pub enum IndexRecordOption {
 }
 
 impl IndexRecordOption {
-    /// Returns true iff the term frequency will be encoded.
-    pub fn is_termfreq_enabled(self) -> bool {
-        match self {
-            IndexRecordOption::WithFreqsAndPositions | IndexRecordOption::WithFreqs => true,
-            _ => false,
-        }
-    }
-
-    /// Returns true iff the term positions within the document are stored as well.
-    pub fn is_position_enabled(self) -> bool {
-        match self {
-            IndexRecordOption::WithFreqsAndPositions => true,
-            _ => false,
-        }
-    }
-
     /// Returns true iff this option includes encoding
     /// term frequencies.
     pub fn has_freq(self) -> bool {

src/schema/schema.rs

@@ -301,28 +301,26 @@ impl Schema {
 
         let mut doc = Document::default();
         for (field_name, json_value) in json_obj.iter() {
-            match self.get_field(field_name) {
+            let field = self
-                Some(field) => {
+                .get_field(field_name)
-                    let field_entry = self.get_field_entry(field);
+                .ok_or_else(|| DocParsingError::NoSuchFieldInSchema(field_name.clone()))?;
-                    let field_type = field_entry.field_type();
+            let field_entry = self.get_field_entry(field);
-                    match *json_value {
+            let field_type = field_entry.field_type();
-                        JsonValue::Array(ref json_items) => {
+            match *json_value {
-                            for json_item in json_items {
+                JsonValue::Array(ref json_items) => {
-                                let value = field_type.value_from_json(json_item).map_err(|e| {
+                    for json_item in json_items {
-                                    DocParsingError::ValueError(field_name.clone(), e)
+                        let value = field_type
-                                })?;
+                            .value_from_json(json_item)
-                                doc.add(FieldValue::new(field, value));
+                            .map_err(|e| DocParsingError::ValueError(field_name.clone(), e))?;
-                            }
+                        doc.add(FieldValue::new(field, value));
-                        }
-                        _ => {
-                            let value = field_type
-                                .value_from_json(json_value)
-                                .map_err(|e| DocParsingError::ValueError(field_name.clone(), e))?;
-                            doc.add(FieldValue::new(field, value));
-                        }
                     }
                 }
-                None => return Err(DocParsingError::NoSuchFieldInSchema(field_name.clone())),
+                _ => {
+                    let value = field_type
+                        .value_from_json(json_value)
+                        .map_err(|e| DocParsingError::ValueError(field_name.clone(), e))?;
+                    doc.add(FieldValue::new(field, value));
+                }
             }
         }
         Ok(doc)

src/schema/term.rs

@@ -22,10 +22,10 @@ impl Term {
     /// Builds a term given a field, and a i64-value
     ///
     /// Assuming the term has a field id of 1, and a i64 value of 3234,
-    /// the Term will have 8 bytes.
+    /// the Term will have 12 bytes.
     ///
     /// The first four byte are dedicated to storing the field id as a u64.
-    /// The 4 following bytes are encoding the u64 value.
+    /// The 8 following bytes are encoding the u64 value.
     pub fn from_field_i64(field: Field, val: i64) -> Term {
         let val_u64: u64 = common::i64_to_u64(val);
         Term::from_field_u64(field, val_u64)
@@ -33,11 +33,11 @@ impl Term {
 
     /// Builds a term given a field, and a f64-value
     ///
-    /// Assuming the term has a field id of 1, and a u64 value of 3234,
+    /// Assuming the term has a field id of 1, and a f64 value of 1.5,
-    /// the Term will have 8 bytes. <= this is wrong
+    /// the Term will have 12 bytes.
     ///
     /// The first four byte are dedicated to storing the field id as a u64.
-    /// The 4 following bytes are encoding the u64 value.
+    /// The 8 following bytes are encoding the f64 as a u64 value.
     pub fn from_field_f64(field: Field, val: f64) -> Term {
         let val_u64: u64 = common::f64_to_u64(val);
         Term::from_field_u64(field, val_u64)
@@ -46,10 +46,10 @@ impl Term {
     /// Builds a term given a field, and a DateTime value
     ///
     /// Assuming the term has a field id of 1, and a timestamp i64 value of 3234,
-    /// the Term will have 8 bytes.
+    /// the Term will have 12 bytes.
     ///
     /// The first four byte are dedicated to storing the field id as a u64.
-    /// The 4 following bytes are encoding the DateTime as i64 timestamp value.
+    /// The 8 following bytes are encoding the DateTime as i64 timestamp value.
     pub fn from_field_date(field: Field, val: &DateTime) -> Term {
         let val_timestamp = val.timestamp();
         Term::from_field_i64(field, val_timestamp)
@@ -82,10 +82,10 @@ impl Term {
     /// Builds a term given a field, and a u64-value
     ///
     /// Assuming the term has a field id of 1, and a u64 value of 3234,
-    /// the Term will have 8 bytes.
+    /// the Term will have 12 bytes.
     ///
     /// The first four byte are dedicated to storing the field id as a u64.
-    /// The 4 following bytes are encoding the u64 value.
+    /// The 8 following bytes are encoding the u64 value.
     pub fn from_field_u64(field: Field, val: u64) -> Term {
         let mut term = Term(vec![0u8; INT_TERM_LEN]);
         term.set_field(field);
@@ -182,7 +182,7 @@ where
     ///
     /// # Panics
     /// ... or returns an invalid value
-    /// if the term is not a `i64` field.
+    /// if the term is not a `f64` field.
     pub fn get_f64(&self) -> f64 {
         common::u64_to_f64(BigEndian::read_u64(&self.0.as_ref()[4..]))
     }

src/snippet/mod.rs

@@ -213,11 +213,9 @@ fn select_best_fragment_combination(fragments: &[FragmentCandidate], text: &str)
 /// # Example
 ///
 /// ```rust
-/// # #[macro_use]
-/// # extern crate tantivy;
-/// # use tantivy::Index;
-/// # use tantivy::schema::{Schema, TEXT};
 /// # use tantivy::query::QueryParser;
+/// # use tantivy::schema::{Schema, TEXT};
+/// # use tantivy::{doc, Index};
 /// use tantivy::SnippetGenerator;
 ///
 /// # fn main() -> tantivy::Result<()> {

src/tokenizer/alphanum_only.rs

@@ -1,6 +1,5 @@
 //! # Example
-//! ```
+//! ```rust
-//! extern crate tantivy;
 //! use tantivy::tokenizer::*;
 //!
 //! # fn main() {

src/tokenizer/mod.rs

@@ -4,8 +4,7 @@
 //! You must define in your schema which tokenizer should be used for
 //! each of your fields :
 //!
-//! ```
+//! ```rust
-//! extern crate tantivy;
 //! use tantivy::schema::*;
 //!
 //! # fn main() {
@@ -65,8 +64,6 @@
 //! For instance, the `en_stem` is defined as follows.
 //!
 //! ```rust
-//! # extern crate tantivy;
-//!
 //! use tantivy::tokenizer::*;
 //!
 //! # fn main() {
@@ -80,8 +77,7 @@
 //! Once your tokenizer is defined, you need to
 //! register it with a name in your index's [`TokenizerManager`](./struct.TokenizerManager.html).
 //!
-//! ```
+//! ```rust
-//! # extern crate tantivy;
 //! # use tantivy::schema::Schema;
 //! # use tantivy::tokenizer::*;
 //! # use tantivy::Index;
@@ -101,8 +97,7 @@
 //!
 //! # Example
 //!
-//! ```
+//! ```rust
-//! extern crate tantivy;
 //! use tantivy::schema::{Schema, IndexRecordOption, TextOptions, TextFieldIndexing};
 //! use tantivy::tokenizer::*;
 //! use tantivy::Index;

src/tokenizer/ngram_tokenizer.rs

@@ -29,8 +29,7 @@ use super::{Token, TokenStream, Tokenizer};
 ///
 /// # Example
 ///
-/// ```
+/// ```rust
-/// # extern crate tantivy;
 /// use tantivy::tokenizer::*;
 /// # fn main() {
 /// let tokenizer = NgramTokenizer::new(2, 3, false);

src/tokenizer/remove_long.rs

@@ -1,6 +1,5 @@
 //! # Example
-//! ```
+//! ```rust
-//! extern crate tantivy;
 //! use tantivy::tokenizer::*;
 //!
 //! # fn main() {

src/tokenizer/stop_word_filter.rs

@@ -1,6 +1,5 @@
 //! # Example
-//! ```
+//! ```rust
-//! extern crate tantivy;
 //! use tantivy::tokenizer::*;
 //!
 //! # fn main() {