Integrated state into TermDict streamer

CHANGELOG.md

@@ -7,13 +7,10 @@ Tantivy 0.11.0
     - Better handling of whitespaces.
 - Closes #498 - add support for Elastic-style unbounded range queries for alphanumeric types eg. "title:>hello", "weight:>=70.5", "height:<200" (@petr-tik)
 - API change around `Box<BoxableTokenizer>`. See detail in #629
-- Avoid rebuilding Regex automaton whenever a regex query is reused. #630 (@brainlock)
- 
+
 ## How to update?
 
-- `Box<dyn BoxableTokenizer>` has been replaced by a `BoxedTokenizer` struct.
-- Regex are now compiled when the `RegexQuery` instance is built. As a result, it can now return
-an error and handling the `Result` is required.
+`Box<dyn BoxableTokenizer>` has been replaced by a `BoxedTokenizer` struct.
 
 Tantivy 0.10.1
 =====================

Cargo.toml

@@ -15,23 +15,23 @@ edition = "2018"
 [dependencies]
 base64 = "0.10.0"
 byteorder = "1.0"
-once_cell = "1.0"
-regex ={version = "1.3.0", default-features = false, features = ["std"]}
-tantivy-fst = "0.1"
+once_cell = "0.2"
+regex = "1.0"
+tantivy-fst = {git="https://github.com/tantivy-search/fst"}
 memmap = {version = "0.7", optional=true}
 lz4 = {version="1.20", optional=true}
 snap = {version="0.2"}
-derive_builder = "0.7"
 atomicwrites = {version="0.2.2", optional=true}
 tempfile = "3.0"
 log = "0.4"
+combine = ">=3.6.0,<4.0.0"
 serde = "1.0"
 serde_derive = "1.0"
 serde_json = "1.0"
 num_cpus = "1.2"
 fs2={version="0.4", optional=true}
 itertools = "0.8"
-levenshtein_automata = "0.1"
+levenshtein_automata = {version="0.1", features=["fst_automaton"]}
 notify = {version="4", optional=true}
 bit-set = "0.5"
 uuid = { version = "0.7.2", features = ["v4", "serde"] }
@@ -42,7 +42,6 @@ owning_ref = "0.4"
 stable_deref_trait = "1.0.0"
 rust-stemmers = "1.1"
 downcast-rs = { version="1.0" }
-tantivy-query-grammar = { path="./query-grammar" }
 bitpacking = {version="0.8", default-features = false, features=["bitpacker4x"]}
 census = "0.2"
 fnv = "1.0.6"
@@ -81,14 +80,10 @@ failpoints = ["fail/failpoints"]
 unstable = [] # useful for benches.
 wasm-bindgen = ["uuid/wasm-bindgen"]
 
-[workspace]
-members = ["query-grammar", "incremental-search"]
-
 [badges]
 travis-ci = { repository = "tantivy-search/tantivy" }
 
 [dev-dependencies.fail]
-version = "0.3"
 features = ["failpoints"]
 
 # Following the "fail" crate best practises, we isolate

Makefile

@@ -1,3 +0,0 @@
-test:
-	echo "Run test only... No examples."
-	cargo test --all --tests --lib

ci/script.sh

@@ -7,7 +7,7 @@ set -ex
 main() {
     if [ ! -z $CODECOV ]; then
         echo "Codecov"
-        cargo build --verbose && cargo coverage --verbose --all && bash <(curl -s https://codecov.io/bash) -s target/kcov
+        cargo build --verbose && cargo coverage --verbose && bash <(curl -s https://codecov.io/bash) -s target/kcov
     else
         echo "Build"
         cross build --target $TARGET
@@ -15,8 +15,7 @@ main() {
             return
         fi
         echo "Test"
-        cross test --target $TARGET --no-default-features --features mmap
-        cross test --target $TARGET --no-default-features --features mmap query-grammar
+        cross test --target $TARGET --no-default-features --features mmap -- --test-threads 1
     fi
     for example in $(ls examples/*.rs)
     do

examples/basic_search.rs

@@ -5,17 +5,20 @@
 //
 // We will :
 // - define our schema
-// - create an index in a directory
-// - index a few documents into our index
-// - search for the best document matching a basic query
-// - retrieve the best document's original content.
+// = create an index in a directory
+// - index few documents in our index
+// - search for the best document matchings "sea whale"
+// - retrieve the best document original content.
 
 // ---
 // Importing tantivy...
+#[macro_use]
+extern crate tantivy;
 use tantivy::collector::TopDocs;
 use tantivy::query::QueryParser;
 use tantivy::schema::*;
-use tantivy::{doc, Index, ReloadPolicy};
+use tantivy::Index;
+use tantivy::ReloadPolicy;
 use tempfile::TempDir;
 
 fn main() -> tantivy::Result<()> {
@@ -30,7 +33,7 @@ fn main() -> tantivy::Result<()> {
     // and for each field, its type and "the way it should
     // be indexed".
 
-    // First we need to define a schema ...
+    // first we need to define a schema ...
     let mut schema_builder = Schema::builder();
 
     // Our first field is title.
@@ -45,7 +48,7 @@ fn main() -> tantivy::Result<()> {
     //
     // `STORED` means that the field will also be saved
     // in a compressed, row-oriented key-value store.
-    // This store is useful for reconstructing the
+    // This store is useful to reconstruct the
     // documents that were selected during the search phase.
     schema_builder.add_text_field("title", TEXT | STORED);
 
@@ -54,7 +57,8 @@ fn main() -> tantivy::Result<()> {
     // need to be able to be able to retrieve it
     // for our application.
     //
-    // We can make our index lighter by omitting the `STORED` flag.
+    // We can make our index lighter and
+    // by omitting `STORED` flag.
     schema_builder.add_text_field("body", TEXT);
 
     let schema = schema_builder.build();
@@ -67,7 +71,7 @@ fn main() -> tantivy::Result<()> {
     // with our schema in the directory.
     let index = Index::create_in_dir(&index_path, schema.clone())?;
 
-    // To insert a document we will need an index writer.
+    // To insert document we need an index writer.
     // There must be only one writer at a time.
     // This single `IndexWriter` is already
     // multithreaded.
@@ -145,8 +149,8 @@ fn main() -> tantivy::Result<()> {
     // At this point our documents are not searchable.
     //
     //
-    // We need to call `.commit()` explicitly to force the
-    // `index_writer` to finish processing the documents in the queue,
+    // We need to call .commit() explicitly to force the
+    // index_writer to finish processing the documents in the queue,
     // flush the current index to the disk, and advertise
     // the existence of new documents.
     //
@@ -158,14 +162,14 @@ fn main() -> tantivy::Result<()> {
     // persistently indexed.
     //
     // In the scenario of a crash or a power failure,
-    // tantivy behaves as if it has rolled back to its last
+    // tantivy behaves as if has rolled back to its last
     // commit.
 
     // # Searching
     //
     // ### Searcher
     //
-    // A reader is required first in order to search an index.
+    // A reader is required to get search the index.
     // It acts as a `Searcher` pool that reloads itself,
     // depending on a `ReloadPolicy`.
     //
@@ -181,7 +185,7 @@ fn main() -> tantivy::Result<()> {
 
     // We now need to acquire a searcher.
     //
-    // A searcher points to a snapshotted, immutable version of the index.
+    // A searcher points to snapshotted, immutable version of the index.
     //
     // Some search experience might require more than
     // one query. Using the same searcher ensures that all of these queries will run on the
@@ -201,7 +205,7 @@ fn main() -> tantivy::Result<()> {
     // in both title and body.
     let query_parser = QueryParser::for_index(&index, vec![title, body]);
 
-    // `QueryParser` may fail if the query is not in the right
+    // QueryParser may fail if the query is not in the right
     // format. For user facing applications, this can be a problem.
     // A ticket has been opened regarding this problem.
     let query = query_parser.parse_query("sea whale")?;
@@ -217,7 +221,7 @@ fn main() -> tantivy::Result<()> {
     //
     // We are not interested in all of the documents but
     // only in the top 10. Keeping track of our top 10 best documents
-    // is the role of the `TopDocs` collector.
+    // is the role of the TopDocs.
 
     // We can now perform our query.
     let top_docs = searcher.search(&query, &TopDocs::with_limit(10))?;

examples/custom_collector.rs

@@ -9,12 +9,15 @@
 
 // ---
 // Importing tantivy...
+#[macro_use]
+extern crate tantivy;
 use tantivy::collector::{Collector, SegmentCollector};
 use tantivy::fastfield::FastFieldReader;
 use tantivy::query::QueryParser;
 use tantivy::schema::Field;
 use tantivy::schema::{Schema, FAST, INDEXED, TEXT};
-use tantivy::{doc, Index, SegmentReader, TantivyError};
+use tantivy::SegmentReader;
+use tantivy::{Index, TantivyError};
 
 #[derive(Default)]
 struct Stats {

examples/custom_tokenizer.rs

@@ -2,11 +2,14 @@
 //
 // In this example, we'll see how to define a tokenizer pipeline
 // by aligning a bunch of `TokenFilter`.
+
+#[macro_use]
+extern crate tantivy;
 use tantivy::collector::TopDocs;
 use tantivy::query::QueryParser;
 use tantivy::schema::*;
 use tantivy::tokenizer::NgramTokenizer;
-use tantivy::{doc, Index};
+use tantivy::Index;
 
 fn main() -> tantivy::Result<()> {
     // # Defining the schema

examples/deleting_updating_documents.rs

@@ -8,10 +8,13 @@
 //
 // ---
 // Importing tantivy...
+#[macro_use]
+extern crate tantivy;
 use tantivy::collector::TopDocs;
 use tantivy::query::TermQuery;
 use tantivy::schema::*;
-use tantivy::{doc, Index, IndexReader};
+use tantivy::Index;
+use tantivy::IndexReader;
 
 // A simple helper function to fetch a single document
 // given its id from our index.

examples/faceted_search.rs

@@ -12,10 +12,12 @@
 
 // ---
 // Importing tantivy...
+#[macro_use]
+extern crate tantivy;
 use tantivy::collector::FacetCollector;
 use tantivy::query::AllQuery;
 use tantivy::schema::*;
-use tantivy::{doc, Index};
+use tantivy::Index;
 use tempfile::TempDir;
 
 fn main() -> tantivy::Result<()> {

examples/integer_range_search.rs

@@ -2,10 +2,14 @@
 //
 // Below is an example of creating an indexed integer field in your schema
 // You can use RangeQuery to get a Count of all occurrences in a given range.
+
+#[macro_use]
+extern crate tantivy;
 use tantivy::collector::Count;
 use tantivy::query::RangeQuery;
 use tantivy::schema::{Schema, INDEXED};
-use tantivy::{doc, Index, Result};
+use tantivy::Index;
+use tantivy::Result;
 
 fn run() -> Result<()> {
     // For the sake of simplicity, this schema will only have 1 field

examples/iterating_docs_and_positions.rs

@@ -9,8 +9,11 @@
 
 // ---
 // Importing tantivy...
+#[macro_use]
+extern crate tantivy;
 use tantivy::schema::*;
-use tantivy::{doc, DocId, DocSet, Index, Postings};
+use tantivy::Index;
+use tantivy::{DocId, DocSet, Postings};
 
 fn main() -> tantivy::Result<()> {
     // We first create a schema for the sake of the

examples/multiple_producer.rs

@@ -25,11 +25,14 @@
 
 // ---
 // Importing tantivy...
+#[macro_use]
+extern crate tantivy;
 use std::sync::{Arc, RwLock};
 use std::thread;
 use std::time::Duration;
 use tantivy::schema::{Schema, STORED, TEXT};
-use tantivy::{doc, Index, IndexWriter, Opstamp};
+use tantivy::Opstamp;
+use tantivy::{Index, IndexWriter};
 
 fn main() -> tantivy::Result<()> {
     // # Defining the schema
@@ -46,9 +49,10 @@ fn main() -> tantivy::Result<()> {
     thread::spawn(move || {
         // we index 100 times the document... for the sake of the example.
         for i in 0..100 {
-            let opstamp = index_writer_clone_1
-                .read().unwrap() //< A read lock is sufficient here.
-                .add_document(
+            let opstamp = {
+                // A read lock is sufficient here.
+                let index_writer_rlock = index_writer_clone_1.read().unwrap();
+                index_writer_rlock.add_document(
                     doc!(
                         title => "Of Mice and Men",
                         body => "A few miles south of Soledad, the Salinas River drops in close to the hillside \
@@ -59,7 +63,8 @@ fn main() -> tantivy::Result<()> {
                         fresh and green with every spring, carrying in their lower leaf junctures the \
                         debris of the winter’s flooding; and sycamores with mottled, white, recumbent \
                         limbs and branches that arch over the pool"
-                    ));
+                    ))
+            };
             println!("add doc {} from thread 1 - opstamp {}", i, opstamp);
             thread::sleep(Duration::from_millis(20));
         }

examples/snippet.rs

@@ -7,10 +7,13 @@
 
 // ---
 // Importing tantivy...
+#[macro_use]
+extern crate tantivy;
 use tantivy::collector::TopDocs;
 use tantivy::query::QueryParser;
 use tantivy::schema::*;
-use tantivy::{doc, Index, Snippet, SnippetGenerator};
+use tantivy::Index;
+use tantivy::{Snippet, SnippetGenerator};
 use tempfile::TempDir;
 
 fn main() -> tantivy::Result<()> {

examples/stop_words.rs

@@ -11,11 +11,13 @@
 
 // ---
 // Importing tantivy...
+#[macro_use]
+extern crate tantivy;
 use tantivy::collector::TopDocs;
 use tantivy::query::QueryParser;
 use tantivy::schema::*;
 use tantivy::tokenizer::*;
-use tantivy::{doc, Index};
+use tantivy::Index;
 
 fn main() -> tantivy::Result<()> {
     // this example assumes you understand the content in `basic_search`

incremental-search/Cargo.toml

@@ -1,10 +0,0 @@
-[package]
-name = "incremental-search"
-version = "0.11.0"
-authors = ["Paul Masurel <paul.masurel@gmail.com>"]
-edition = "2018"
-
-
-[dependencies]
-derive_builder = "0.7"
-tantivy = {path = ".."}

incremental-search/src/bitset.rs

@@ -1,395 +0,0 @@
-use std::fmt;
-use std::u64;
-
-#[derive(Clone, Copy, Eq, PartialEq)]
-pub(crate) struct TinySet(u64);
-
-impl fmt::Debug for TinySet {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        self.into_iter().collect::<Vec<u32>>().fmt(f)
-    }
-}
-
-pub struct TinySetIterator(TinySet);
-impl Iterator for TinySetIterator {
-    type Item = u32;
-
-    fn next(&mut self) -> Option<Self::Item> {
-        self.0.pop_lowest()
-    }
-}
-
-impl IntoIterator for TinySet {
-    type Item = u32;
-    type IntoIter = TinySetIterator;
-    fn into_iter(self) -> Self::IntoIter {
-        TinySetIterator(self)
-    }
-}
-
-impl TinySet {
-    /// Returns an empty `TinySet`.
-    pub fn empty() -> TinySet {
-        TinySet(0u64)
-    }
-
-    /// Returns the complement of the set in `[0, 64[`.
-    fn complement(self) -> TinySet {
-        TinySet(!self.0)
-    }
-
-    /// Returns true iff the `TinySet` contains the element `el`.
-    pub fn contains(self, el: u32) -> bool {
-        !self.intersect(TinySet::singleton(el)).is_empty()
-    }
-
-    /// Returns the intersection of `self` and `other`
-    pub fn intersect(self, other: TinySet) -> TinySet {
-        TinySet(self.0 & other.0)
-    }
-
-    /// Creates a new `TinySet` containing only one element
-    /// within `[0; 64[`
-    #[inline(always)]
-    pub fn singleton(el: u32) -> TinySet {
-        TinySet(1u64 << u64::from(el))
-    }
-
-    /// Insert a new element within [0..64[
-    #[inline(always)]
-    pub fn insert(self, el: u32) -> TinySet {
-        self.union(TinySet::singleton(el))
-    }
-
-    /// Insert a new element within [0..64[
-    #[inline(always)]
-    pub fn insert_mut(&mut self, el: u32) -> bool {
-        let old = *self;
-        *self = old.insert(el);
-        old != *self
-    }
-
-    /// Returns the union of two tinysets
-    #[inline(always)]
-    pub fn union(self, other: TinySet) -> TinySet {
-        TinySet(self.0 | other.0)
-    }
-
-    /// Returns true iff the `TinySet` is empty.
-    #[inline(always)]
-    pub fn is_empty(self) -> bool {
-        self.0 == 0u64
-    }
-
-    /// Returns the lowest element in the `TinySet`
-    /// and removes it.
-    #[inline(always)]
-    pub fn pop_lowest(&mut self) -> Option<u32> {
-        if self.is_empty() {
-            None
-        } else {
-            let lowest = self.0.trailing_zeros() as u32;
-            self.0 ^= TinySet::singleton(lowest).0;
-            Some(lowest)
-        }
-    }
-
-    /// Returns a `TinySet` than contains all values up
-    /// to limit excluded.
-    ///
-    /// The limit is assumed to be strictly lower than 64.
-    pub fn range_lower(upper_bound: u32) -> TinySet {
-        TinySet((1u64 << u64::from(upper_bound % 64u32)) - 1u64)
-    }
-
-    /// Returns a `TinySet` that contains all values greater
-    /// or equal to the given limit, included. (and up to 63)
-    ///
-    /// The limit is assumed to be strictly lower than 64.
-    pub fn range_greater_or_equal(from_included: u32) -> TinySet {
-        TinySet::range_lower(from_included).complement()
-    }
-
-    pub fn clear(&mut self) {
-        self.0 = 0u64;
-    }
-
-    pub fn len(self) -> u32 {
-        self.0.count_ones()
-    }
-}
-
-#[derive(Clone)]
-pub struct BitSet {
-    tinysets: Box<[TinySet]>,
-    len: usize, //< Technically it should be u32, but we
-    // count multiple inserts.
-    // `usize` guards us from overflow.
-    max_value: u32,
-}
-
-fn num_buckets(max_val: u32) -> u32 {
-    (max_val + 63u32) / 64u32
-}
-
-impl BitSet {
-    /// Create a new `BitSet` that may contain elements
-    /// within `[0, max_val[`.
-    pub fn with_max_value(max_value: u32) -> BitSet {
-        let num_buckets = num_buckets(max_value);
-        let tinybisets = vec![TinySet::empty(); num_buckets as usize].into_boxed_slice();
-        BitSet {
-            tinysets: tinybisets,
-            len: 0,
-            max_value,
-        }
-    }
-
-    /// Removes all elements from the `BitSet`.
-    pub fn clear(&mut self) {
-        for tinyset in self.tinysets.iter_mut() {
-            *tinyset = TinySet::empty();
-        }
-    }
-
-    /// Returns the number of elements in the `BitSet`.
-    pub fn len(&self) -> usize {
-        self.len
-    }
-
-    /// Inserts an element in the `BitSet`
-    pub fn insert(&mut self, el: u32) {
-        // we do not check saturated els.
-        let higher = el / 64u32;
-        let lower = el % 64u32;
-        self.len += if self.tinysets[higher as usize].insert_mut(lower) {
-            1
-        } else {
-            0
-        };
-    }
-
-    /// Returns true iff the elements is in the `BitSet`.
-    pub fn contains(&self, el: u32) -> bool {
-        self.tinyset(el / 64u32).contains(el % 64)
-    }
-
-    /// Returns the first non-empty `TinySet` associated to a bucket lower
-    /// or greater than bucket.
-    ///
-    /// Reminder: the tiny set with the bucket `bucket`, represents the
-    /// elements from `bucket * 64` to `(bucket+1) * 64`.
-    pub(crate) fn first_non_empty_bucket(&self, bucket: u32) -> Option<u32> {
-        self.tinysets[bucket as usize..]
-            .iter()
-            .cloned()
-            .position(|tinyset| !tinyset.is_empty())
-            .map(|delta_bucket| bucket + delta_bucket as u32)
-    }
-
-    pub fn max_value(&self) -> u32 {
-        self.max_value
-    }
-
-    /// Returns the tiny bitset representing the
-    /// the set restricted to the number range from
-    /// `bucket * 64` to `(bucket + 1) * 64`.
-    pub(crate) fn tinyset(&self, bucket: u32) -> TinySet {
-        self.tinysets[bucket as usize]
-    }
-}
-
-#[cfg(test)]
-mod tests {
-
-    use super::BitSet;
-    use super::TinySet;
-    use crate::docset::DocSet;
-    use crate::query::BitSetDocSet;
-    use crate::tests;
-    use crate::tests::generate_nonunique_unsorted;
-    use std::collections::BTreeSet;
-    use std::collections::HashSet;
-
-    #[test]
-    fn test_tiny_set() {
-        assert!(TinySet::empty().is_empty());
-        {
-            let mut u = TinySet::empty().insert(1u32);
-            assert_eq!(u.pop_lowest(), Some(1u32));
-            assert!(u.pop_lowest().is_none())
-        }
-        {
-            let mut u = TinySet::empty().insert(1u32).insert(1u32);
-            assert_eq!(u.pop_lowest(), Some(1u32));
-            assert!(u.pop_lowest().is_none())
-        }
-        {
-            let mut u = TinySet::empty().insert(2u32);
-            assert_eq!(u.pop_lowest(), Some(2u32));
-            u.insert_mut(1u32);
-            assert_eq!(u.pop_lowest(), Some(1u32));
-            assert!(u.pop_lowest().is_none());
-        }
-        {
-            let mut u = TinySet::empty().insert(63u32);
-            assert_eq!(u.pop_lowest(), Some(63u32));
-            assert!(u.pop_lowest().is_none());
-        }
-    }
-
-    #[test]
-    fn test_bitset() {
-        let test_against_hashset = |els: &[u32], max_value: u32| {
-            let mut hashset: HashSet<u32> = HashSet::new();
-            let mut bitset = BitSet::with_max_value(max_value);
-            for &el in els {
-                assert!(el < max_value);
-                hashset.insert(el);
-                bitset.insert(el);
-            }
-            for el in 0..max_value {
-                assert_eq!(hashset.contains(&el), bitset.contains(el));
-            }
-            assert_eq!(bitset.max_value(), max_value);
-        };
-
-        test_against_hashset(&[], 0);
-        test_against_hashset(&[], 1);
-        test_against_hashset(&[0u32], 1);
-        test_against_hashset(&[0u32], 100);
-        test_against_hashset(&[1u32, 2u32], 4);
-        test_against_hashset(&[99u32], 100);
-        test_against_hashset(&[63u32], 64);
-        test_against_hashset(&[62u32, 63u32], 64);
-    }
-
-    #[test]
-    fn test_bitset_large() {
-        let arr = generate_nonunique_unsorted(100_000, 5_000);
-        let mut btreeset: BTreeSet<u32> = BTreeSet::new();
-        let mut bitset = BitSet::with_max_value(100_000);
-        for el in arr {
-            btreeset.insert(el);
-            bitset.insert(el);
-        }
-        for i in 0..100_000 {
-            assert_eq!(btreeset.contains(&i), bitset.contains(i));
-        }
-        assert_eq!(btreeset.len(), bitset.len());
-        let mut bitset_docset = BitSetDocSet::from(bitset);
-        for el in btreeset.into_iter() {
-            bitset_docset.advance();
-            assert_eq!(bitset_docset.doc(), el);
-        }
-        assert!(!bitset_docset.advance());
-    }
-
-    #[test]
-    fn test_bitset_num_buckets() {
-        use super::num_buckets;
-        assert_eq!(num_buckets(0u32), 0);
-        assert_eq!(num_buckets(1u32), 1);
-        assert_eq!(num_buckets(64u32), 1);
-        assert_eq!(num_buckets(65u32), 2);
-        assert_eq!(num_buckets(128u32), 2);
-        assert_eq!(num_buckets(129u32), 3);
-    }
-
-    #[test]
-    fn test_tinyset_range() {
-        assert_eq!(
-            TinySet::range_lower(3).into_iter().collect::<Vec<u32>>(),
-            [0, 1, 2]
-        );
-        assert!(TinySet::range_lower(0).is_empty());
-        assert_eq!(
-            TinySet::range_lower(63).into_iter().collect::<Vec<u32>>(),
-            (0u32..63u32).collect::<Vec<_>>()
-        );
-        assert_eq!(
-            TinySet::range_lower(1).into_iter().collect::<Vec<u32>>(),
-            [0]
-        );
-        assert_eq!(
-            TinySet::range_lower(2).into_iter().collect::<Vec<u32>>(),
-            [0, 1]
-        );
-        assert_eq!(
-            TinySet::range_greater_or_equal(3)
-                .into_iter()
-                .collect::<Vec<u32>>(),
-            (3u32..64u32).collect::<Vec<_>>()
-        );
-    }
-
-    #[test]
-    fn test_bitset_len() {
-        let mut bitset = BitSet::with_max_value(1_000);
-        assert_eq!(bitset.len(), 0);
-        bitset.insert(3u32);
-        assert_eq!(bitset.len(), 1);
-        bitset.insert(103u32);
-        assert_eq!(bitset.len(), 2);
-        bitset.insert(3u32);
-        assert_eq!(bitset.len(), 2);
-        bitset.insert(103u32);
-        assert_eq!(bitset.len(), 2);
-        bitset.insert(104u32);
-        assert_eq!(bitset.len(), 3);
-    }
-
-    #[test]
-    fn test_bitset_clear() {
-        let mut bitset = BitSet::with_max_value(1_000);
-        let els = tests::sample(1_000, 0.01f64);
-        for &el in &els {
-            bitset.insert(el);
-        }
-        assert!(els.iter().all(|el| bitset.contains(*el)));
-        bitset.clear();
-        for el in 0u32..1000u32 {
-            assert!(!bitset.contains(el));
-        }
-    }
-}
-
-#[cfg(all(test, feature = "unstable"))]
-mod bench {
-
-    use super::BitSet;
-    use super::TinySet;
-    use test;
-
-    #[bench]
-    fn bench_tinyset_pop(b: &mut test::Bencher) {
-        b.iter(|| {
-            let mut tinyset = TinySet::singleton(test::black_box(31u32));
-            tinyset.pop_lowest();
-            tinyset.pop_lowest();
-            tinyset.pop_lowest();
-            tinyset.pop_lowest();
-            tinyset.pop_lowest();
-            tinyset.pop_lowest();
-        });
-    }
-
-    #[bench]
-    fn bench_tinyset_sum(b: &mut test::Bencher) {
-        let tiny_set = TinySet::empty().insert(10u32).insert(14u32).insert(21u32);
-        b.iter(|| {
-            assert_eq!(test::black_box(tiny_set).into_iter().sum::<u32>(), 45u32);
-        });
-    }
-
-    #[bench]
-    fn bench_tinyarr_sum(b: &mut test::Bencher) {
-        let v = [10u32, 14u32, 21u32];
-        b.iter(|| test::black_box(v).iter().cloned().sum::<u32>());
-    }
-
-    #[bench]
-    fn bench_bitset_initialize(b: &mut test::Bencher) {
-        b.iter(|| BitSet::with_max_value(1_000_000));
-    }
-}

incremental-search/src/lib.rs

@@ -1,266 +0,0 @@
-use tantivy::query::{BooleanQuery, FuzzyTermQuery, EmptyQuery};
-use derive_builder::Builder;
-use std::str::FromStr;
-use tantivy::query::{FuzzyConfiguration, FuzzyConfigurationBuilder, Query, Occur};
-use tantivy::schema::Field;
-use tantivy::{Searcher, TantivyError, DocAddress, Term, Document};
-use tantivy::collector::TopDocs;
-use std::ops::Deref;
-
-
-#[derive(Debug)]
-pub struct IncrementalSearchQuery {
-    pub terms: Vec<String>,
-    pub last_is_prefix: bool,
-}
-
-impl IncrementalSearchQuery {
-    pub fn fuzzy_configurations(&self) -> Vec<FuzzyConfigurations> {
-        if self.terms.is_empty() {
-            return Vec::default();
-        }
-        let single_term_confs: Vec<FuzzyConfigurationBuilder> = (0u8..3u8)
-            .map(|d: u8| {
-                let mut builder = FuzzyConfigurationBuilder::default();
-                builder.distance(d).transposition_cost_one(true);
-                builder
-            })
-            .collect();
-        let mut configurations: Vec<Vec<FuzzyConfigurationBuilder>> = single_term_confs
-            .iter()
-            .map(|conf| vec![conf.clone()])
-            .collect();
-        let mut new_configurations = Vec::new();
-        for _ in 1..self.terms.len() {
-            new_configurations.clear();
-            for single_term_conf in &single_term_confs {
-                for configuration in &configurations {
-                    let mut new_configuration: Vec<FuzzyConfigurationBuilder> = configuration.clone();
-                    new_configuration.push(single_term_conf.clone());
-                    new_configurations.push(new_configuration);
-                }
-            }
-            std::mem::swap(&mut configurations, &mut new_configurations);
-        }
-        if self.last_is_prefix {
-            for configuration in &mut configurations {
-                if let Some(last_conf) = configuration.last_mut() {
-                    last_conf.prefix(true);
-                }
-            }
-        }
-        let mut fuzzy_configurations: Vec<FuzzyConfigurations> = configurations
-            .into_iter()
-            .map(FuzzyConfigurations::from)
-            .collect();
-        fuzzy_configurations.sort_by(|left, right| left.cost.partial_cmp(&right.cost).unwrap());
-        fuzzy_configurations
-    }
-
-    fn search_query(&self, fields: &[Field], configurations: FuzzyConfigurations) -> Box<dyn Query> {
-        if self.terms.is_empty() {
-            Box::new(EmptyQuery)
-        } else if self.terms.len() == 1 {
-            build_query_for_fields(fields, &self.terms[0], &configurations.configurations[0])
-        } else {
-            Box::new(BooleanQuery::from(self.terms.iter()
-                .zip(configurations.configurations.iter())
-                .map(|(term, configuration)|
-                         (Occur::Must, build_query_for_fields(fields, &term, &configuration))
-                )
-                .collect::<Vec<_>>()))
-        }
-    }
-}
-
-#[derive(Debug)]
-pub struct FuzzyConfigurations {
-    configurations: Vec<FuzzyConfiguration>,
-    cost: f64,
-}
-
-
-fn compute_cost(fuzzy_confs: &[FuzzyConfiguration]) -> f64 {
-    fuzzy_confs
-        .iter()
-        .map(|fuzzy_conf| {
-            let weight = if fuzzy_conf.prefix { 30f64 } else { 5f64 };
-            weight * f64::from(fuzzy_conf.distance)
-        })
-        .sum()
-}
-
-impl From<Vec<FuzzyConfigurationBuilder>> for FuzzyConfigurations {
-    fn from(fuzzy_conf_builder: Vec<FuzzyConfigurationBuilder>) -> FuzzyConfigurations {
-        let configurations = fuzzy_conf_builder
-            .into_iter()
-            .map(|conf| conf.build().unwrap())
-            .collect::<Vec<FuzzyConfiguration>>();
-        let cost = compute_cost(&configurations);
-        FuzzyConfigurations {
-            configurations,
-            cost,
-        }
-    }
-}
-
-#[derive(Debug)]
-pub struct ParseIncrementalQueryError;
-
-impl Into<TantivyError> for ParseIncrementalQueryError {
-    fn into(self) -> TantivyError {
-        TantivyError::InvalidArgument(format!("Invalid query: {:?}", self))
-    }
-}
-
-impl FromStr for IncrementalSearchQuery {
-    type Err = ParseIncrementalQueryError;
-
-    fn from_str(query_str: &str) -> Result<Self, Self::Err> {
-        let terms: Vec<String> = query_str
-            .split_whitespace()
-            .map(ToString::to_string)
-            .collect();
-        Ok(IncrementalSearchQuery {
-            terms,
-            last_is_prefix: query_str
-                .chars()
-                .last()
-                .map(|c| !c.is_whitespace())
-                .unwrap_or(false),
-        })
-    }
-}
-
-fn build_query_for_fields(fields: &[Field], term_text: &str, conf: &FuzzyConfiguration) -> Box<dyn Query> {
-    assert!(fields.len() > 0);
-    if fields.len() > 1 {
-        let term_queries: Vec<(Occur, Box<dyn Query>)> = fields
-            .iter()
-            .map(|&field| {
-                let term = Term::from_field_text(field, term_text);
-                let query = FuzzyTermQuery::new_from_configuration(term, conf.clone());
-                let boxed_query: Box<dyn Query> = Box::new(query);
-                (Occur::Must, boxed_query)
-            })
-            .collect();
-        Box::new(BooleanQuery::from(term_queries))
-    } else {
-        let term = Term::from_field_text(fields[0], term_text);
-        Box::new( FuzzyTermQuery::new_from_configuration(term, conf.clone()))
-    }
-
-}
-
-pub struct IncrementalSearchResult {
-    pub docs: Vec<Document>
-}
-
-#[derive(Builder, Default)]
-pub struct IncrementalSearch {
-    nhits: usize,
-    #[builder(default)]
-    search_fields: Vec<Field>,
-    #[builder(default)]
-    return_fields: Vec<Field>,
-}
-
-impl IncrementalSearch {
-
-    pub fn search<S: Deref<Target=Searcher>>(
-        &self,
-        query: &str,
-        searcher: &S,
-    ) -> tantivy::Result<IncrementalSearchResult> {
-        let searcher = searcher.deref();
-        let inc_search_query: IncrementalSearchQuery =
-            FromStr::from_str(query).map_err(Into::<TantivyError>::into)?;
-
-        let mut results: Vec<DocAddress> = Vec::default();
-        let mut remaining = self.nhits;
-        for fuzzy_conf in inc_search_query.fuzzy_configurations() {
-            if remaining == 0 {
-                break;
-            }
-            let query = inc_search_query.search_query(&self.search_fields[..], fuzzy_conf);
-            let new_docs = searcher.search(query.as_ref(), &TopDocs::with_limit(remaining))?;
-            // TODO(pmasurel) remove already added docs.
-            results.extend(new_docs.into_iter()
-                .map(|(_, doc_address)| doc_address));
-            remaining = self.nhits - results.len();
-            if remaining == 0 {
-                break;
-            }
-        }
-        let docs: Vec<Document> = results.into_iter()
-            .map(|doc_address: DocAddress| searcher.doc(doc_address))
-            .collect::<tantivy::Result<_>>()?;
-        Ok(IncrementalSearchResult {
-            docs
-        })
-    }
-}
-#[cfg(test)]
-mod tests {
-    use tantivy::doc;
-    use crate::{IncrementalSearch, IncrementalSearchBuilder, IncrementalSearchQuery};
-    use std::str::FromStr;
-    use tantivy::schema::{SchemaBuilder, TEXT, STORED};
-    use tantivy::Index;
-
-    #[test]
-    fn test_incremental_search() {
-        let incremental_search = IncrementalSearchBuilder::default()
-            .nhits(10)
-            .build()
-            .unwrap();
-    }
-
-    #[test]
-    fn test_incremental_search_query_parse_empty() {
-        let query = IncrementalSearchQuery::from_str("").unwrap();
-        assert_eq!(query.terms, Vec::<String>::new());
-        assert!(!query.last_is_prefix);
-    }
-
-    #[test]
-    fn test_incremental_search_query_parse_trailing_whitespace() {
-        let query = IncrementalSearchQuery::from_str("hello happy tax pa ").unwrap();
-        assert_eq!(query.terms, vec!["hello", "happy", "tax", "pa"]);
-        assert!(!query.last_is_prefix);
-    }
-
-    #[test]
-    fn test_incremental_search_query_parse_unicode_whitespace() {
-        let query = IncrementalSearchQuery::from_str("hello  happy tax pa ").unwrap();
-        assert_eq!(query.terms, vec!["hello", "happy", "tax", "pa"]);
-        assert!(!query.last_is_prefix);
-    }
-
-    #[test]
-    fn test_incremental_search_query_parse() {
-        let query = IncrementalSearchQuery::from_str("hello happy tax pa").unwrap();
-        assert_eq!(query.terms, vec!["hello", "happy", "tax", "pa"]);
-        assert!(query.last_is_prefix);
-    }
-
-    #[test]
-    fn test_blop() {
-        let mut schema_builder = SchemaBuilder::new();
-        let body = schema_builder.add_text_field("body", TEXT | STORED);
-        let schema = schema_builder.build();
-        let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_with_num_threads(1, 30_000_000).unwrap();
-        index_writer.add_document(doc!(body=> "hello happy tax payer"));
-        index_writer.commit().unwrap();
-        let reader = index.reader().unwrap();
-        let searcher = reader.searcher();
-        let incremental_search: IncrementalSearch = IncrementalSearchBuilder::default()
-            .nhits(1)
-            .search_fields(vec![body])
-            .build()
-            .unwrap();
-        let top_docs = incremental_search.search("hello hapy t", &searcher).unwrap();
-        assert_eq!(top_docs.docs.len(), 1);
-    }
-}

query-grammar/Cargo.toml

@@ -1,16 +0,0 @@
-[package]
-name = "tantivy-query-grammar"
-version = "0.11.0"
-authors = ["Paul Masurel <paul.masurel@gmail.com>"]
-license = "MIT"
-categories = ["database-implementations", "data-structures"]
-description = """Search engine library"""
-documentation = "https://tantivy-search.github.io/tantivy/tantivy/index.html"
-homepage = "https://github.com/tantivy-search/tantivy"
-repository = "https://github.com/tantivy-search/tantivy"
-readme = "README.md"
-keywords = ["search", "information", "retrieval"]
-edition = "2018"
-
-[dependencies]
-combine = ">=3.6.0,<4.0.0"

query-grammar/src/lib.rs

@@ -1,17 +0,0 @@
-#![recursion_limit = "100"]
-
-mod occur;
-mod query_grammar;
-mod user_input_ast;
-use combine::parser::Parser;
-
-pub use crate::occur::Occur;
-use crate::query_grammar::parse_to_ast;
-pub use crate::user_input_ast::{UserInputAST, UserInputBound, UserInputLeaf, UserInputLiteral};
-
-pub struct Error;
-
-pub fn parse_query(query: &str) -> Result<UserInputAST, Error> {
-    let (user_input_ast, _remaining) = parse_to_ast().parse(query).map_err(|_| Error)?;
-    Ok(user_input_ast)
-}

src/collector/count_collector.rs

@@ -10,10 +10,12 @@ use crate::SegmentReader;
 /// documents match the query.
 ///
 /// ```rust
+/// #[macro_use]
+/// extern crate tantivy;
+/// use tantivy::schema::{Schema, TEXT};
+/// use tantivy::{Index, Result};
 /// use tantivy::collector::Count;
 /// use tantivy::query::QueryParser;
-/// use tantivy::schema::{Schema, TEXT};
-/// use tantivy::{doc, Index, Result};
 ///
 /// # fn main() { example().unwrap(); }
 /// fn example() -> Result<()> {

src/collector/facet_collector.rs

@@ -81,10 +81,12 @@ fn facet_depth(facet_bytes: &[u8]) -> usize {
 ///
 ///
 /// ```rust
+/// #[macro_use]
+/// extern crate tantivy;
+/// use tantivy::schema::{Facet, Schema, TEXT};
+/// use tantivy::{Index, Result};
 /// use tantivy::collector::FacetCollector;
 /// use tantivy::query::AllQuery;
-/// use tantivy::schema::{Facet, Schema, TEXT};
-/// use tantivy::{doc, Index, Result};
 ///
 /// # fn main() { example().unwrap(); }
 /// fn example() -> Result<()> {

src/collector/mod.rs

@@ -35,6 +35,7 @@ The resulting `Fruit` will then be a typed tuple with each collector's original
 in their respective position.
 
 ```rust
+# extern crate tantivy;
 # use tantivy::schema::*;
 # use tantivy::*;
 # use tantivy::query::*;

src/collector/multi_collector.rs

@@ -105,10 +105,12 @@ impl<TFruit: Fruit> FruitHandle<TFruit> {
 /// [Combining several collectors section of the collector documentation](./index.html#combining-several-collectors).
 ///
 /// ```rust
+/// #[macro_use]
+/// extern crate tantivy;
+/// use tantivy::schema::{Schema, TEXT};
+/// use tantivy::{Index, Result};
 /// use tantivy::collector::{Count, TopDocs, MultiCollector};
 /// use tantivy::query::QueryParser;
-/// use tantivy::schema::{Schema, TEXT};
-/// use tantivy::{doc, Index, Result};
 ///
 /// # fn main() { example().unwrap(); }
 /// fn example() -> Result<()> {

src/collector/top_score_collector.rs

@@ -23,10 +23,13 @@ use std::fmt;
 /// is `O(n log K)`.
 ///
 /// ```rust
+/// #[macro_use]
+/// extern crate tantivy;
+/// use tantivy::DocAddress;
+/// use tantivy::schema::{Schema, TEXT};
+/// use tantivy::{Index, Result};
 /// use tantivy::collector::TopDocs;
 /// use tantivy::query::QueryParser;
-/// use tantivy::schema::{Schema, TEXT};
-/// use tantivy::{doc, DocAddress, Index, Result};
 ///
 /// # fn main() { example().unwrap(); }
 /// fn example() -> Result<()> {
@@ -84,8 +87,10 @@ impl TopDocs {
     /// Set top-K to rank documents by a given fast field.
     ///
     /// ```rust
+    /// #[macro_use]
+    /// extern crate tantivy;
     /// # use tantivy::schema::{Schema, FAST, TEXT};
-    /// # use tantivy::{doc, Index, Result, DocAddress};
+    /// # use tantivy::{Index, Result, DocAddress};
     /// # use tantivy::query::{Query, QueryParser};
     /// use tantivy::Searcher;
     /// use tantivy::collector::TopDocs;
@@ -123,7 +128,7 @@ impl TopDocs {
     /// ///
     /// /// `field` is required to be a FAST field.
     /// fn docs_sorted_by_rating(searcher: &Searcher,
-    ///                          query: &dyn Query,
+    ///                          query: &Query,
     ///                          sort_by_field: Field)
     ///     -> Result<Vec<(u64, DocAddress)>> {
     ///
@@ -192,8 +197,10 @@ impl TopDocs {
     /// learning-to-rank model over various features
     ///
     /// ```rust
+    /// #[macro_use]
+    /// extern crate tantivy;
     /// # use tantivy::schema::{Schema, FAST, TEXT};
-    /// # use tantivy::{doc, Index, DocAddress, DocId, Score};
+    /// # use tantivy::{Index, DocAddress, DocId, Score};
     /// # use tantivy::query::QueryParser;
     /// use tantivy::SegmentReader;
     /// use tantivy::collector::TopDocs;
@@ -295,8 +302,10 @@ impl TopDocs {
     /// # Example
     ///
     /// ```rust
+    /// # #[macro_use]
+    /// # extern crate tantivy;
     /// # use tantivy::schema::{Schema, FAST, TEXT};
-    /// # use tantivy::{doc, Index, DocAddress, DocId};
+    /// # use tantivy::{Index, DocAddress, DocId};
     /// # use tantivy::query::QueryParser;
     /// use tantivy::SegmentReader;
     /// use tantivy::collector::TopDocs;

src/core/index.rs

@@ -216,22 +216,8 @@ impl Index {
         Index::open(mmap_directory)
     }
 
-    /// Returns the list of the segment metas tracked by the index.
-    ///
-    /// Such segments can of course be part of the index,
-    /// but also they could be segments being currently built or in the middle of a merge
-    /// operation.
-    pub fn list_all_segment_metas(&self) -> Vec<SegmentMeta> {
-        self.inventory.all()
-    }
-
-    /// Creates a new segment_meta (Advanced user only).
-    ///
-    /// As long as the `SegmentMeta` lives, the files associated with the
-    /// `SegmentMeta` are guaranteed to not be garbage collected, regardless of
-    /// whether the segment is recorded as part of the index or not.
-    pub fn new_segment_meta(&self, segment_id: SegmentId, max_doc: u32) -> SegmentMeta {
-        self.inventory.new_segment_meta(segment_id, max_doc)
+    pub(crate) fn inventory(&self) -> &SegmentMetaInventory {
+        &self.inventory
     }
 
     /// Open the index using the provided directory

src/core/index_meta.rs

@@ -30,6 +30,7 @@ impl SegmentMetaInventory {
             .collect::<Vec<_>>()
     }
 
+    #[doc(hidden)]
     pub fn new_segment_meta(&self, segment_id: SegmentId, max_doc: u32) -> SegmentMeta {
         let inner = InnerSegmentMeta {
             segment_id,

src/core/segment_id.rs

@@ -4,8 +4,6 @@ use uuid::Uuid;
 
 #[cfg(test)]
 use once_cell::sync::Lazy;
-use std::error::Error;
-use std::str::FromStr;
 #[cfg(test)]
 use std::sync::atomic;
 
@@ -54,51 +52,15 @@ impl SegmentId {
     /// and the rest is random.
     ///
     /// Picking the first 8 chars is ok to identify
-    /// segments in a display message (e.g. a5c4dfcb).
+    /// segments in a display message.
     pub fn short_uuid_string(&self) -> String {
         (&self.0.to_simple_ref().to_string()[..8]).to_string()
     }
 
     /// Returns a segment uuid string.
-    ///
-    /// It consists in 32 lowercase hexadecimal chars
-    /// (e.g. a5c4dfcbdfe645089129e308e26d5523)
     pub fn uuid_string(&self) -> String {
         self.0.to_simple_ref().to_string()
     }
-
-    /// Build a `SegmentId` string from the full uuid string.
-    ///
-    /// E.g. "a5c4dfcbdfe645089129e308e26d5523"
-    pub fn from_uuid_string(uuid_string: &str) -> Result<SegmentId, SegmentIdParseError> {
-        FromStr::from_str(uuid_string)
-    }
-}
-
-/// Error type used when parsing a `SegmentId` from a string fails.
-pub struct SegmentIdParseError(uuid::parser::ParseError);
-
-impl Error for SegmentIdParseError {}
-
-impl fmt::Debug for SegmentIdParseError {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        self.0.fmt(f)
-    }
-}
-
-impl fmt::Display for SegmentIdParseError {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        self.0.fmt(f)
-    }
-}
-
-impl FromStr for SegmentId {
-    type Err = SegmentIdParseError;
-
-    fn from_str(uuid_string: &str) -> Result<Self, SegmentIdParseError> {
-        let uuid = Uuid::parse_str(uuid_string).map_err(SegmentIdParseError)?;
-        Ok(SegmentId(uuid))
-    }
 }
 
 impl fmt::Debug for SegmentId {
@@ -118,18 +80,3 @@ impl Ord for SegmentId {
         self.0.as_bytes().cmp(other.0.as_bytes())
     }
 }
-
-#[cfg(test)]
-mod tests {
-    use super::SegmentId;
-
-    #[test]
-    fn test_to_uuid_string() {
-        let full_uuid = "a5c4dfcbdfe645089129e308e26d5523";
-        let segment_id = SegmentId::from_uuid_string(full_uuid).unwrap();
-        assert_eq!(segment_id.uuid_string(), full_uuid);
-        assert_eq!(segment_id.short_uuid_string(), "a5c4dfcb");
-        // one extra char
-        assert!(SegmentId::from_uuid_string("a5c4dfcbdfe645089129e308e26d5523b").is_err());
-    }
-}

src/indexer/index_writer.rs

@@ -209,7 +209,10 @@ fn index_documents(
     assert!(num_docs > 0);
 
     let doc_opstamps: Vec<Opstamp> = segment_writer.finalize()?;
-    let segment_meta = segment.index().new_segment_meta(segment_id, num_docs);
+    let segment_meta = segment
+        .index()
+        .inventory()
+        .new_segment_meta(segment_id, num_docs);
 
     let last_docstamp: Opstamp = *(doc_opstamps.last().unwrap());
 
@@ -447,10 +450,12 @@ impl IndexWriter {
     /// by clearing and resubmitting necessary documents
     ///
     /// ```rust
-    /// use tantivy::collector::TopDocs;
+    /// #[macro_use]
+    /// extern crate tantivy;
     /// use tantivy::query::QueryParser;
+    /// use tantivy::collector::TopDocs;
     /// use tantivy::schema::*;
-    /// use tantivy::{doc, Index};
+    /// use tantivy::Index;
     ///
     /// fn main() -> tantivy::Result<()> {
     ///     let mut schema_builder = Schema::builder();

src/indexer/segment_updater.rs

@@ -126,7 +126,9 @@ fn perform_merge(
 
     let num_docs = merger.write(segment_serializer)?;
 
-    let segment_meta = index.new_segment_meta(merged_segment.id(), num_docs);
+    let segment_meta = index
+        .inventory()
+        .new_segment_meta(merged_segment.id(), num_docs);
 
     let after_merge_segment_entry = SegmentEntry::new(segment_meta.clone(), delete_cursor, None);
     Ok(after_merge_segment_entry)
@@ -280,7 +282,7 @@ impl SegmentUpdater {
     fn list_files(&self) -> HashSet<PathBuf> {
         let mut files = HashSet::new();
         files.insert(META_FILEPATH.to_path_buf());
-        for segment_meta in self.0.index.list_all_segment_metas() {
+        for segment_meta in self.0.index.inventory().all() {
             files.extend(segment_meta.list_files());
         }
         files

src/lib.rs

@@ -1,4 +1,5 @@
 #![doc(html_logo_url = "http://fulmicoton.com/tantivy-logo/tantivy-logo.png")]
+#![recursion_limit = "100"]
 #![cfg_attr(all(feature = "unstable", test), feature(test))]
 #![cfg_attr(feature = "cargo-clippy", allow(clippy::module_inception))]
 #![doc(test(attr(allow(unused_variables), deny(warnings))))]
@@ -10,12 +11,21 @@
 //! Think `Lucene`, but in Rust.
 //!
 //! ```rust
+
+//! # extern crate tempfile;
+//! #
+//! #[macro_use]
+//! extern crate tantivy;
+//!
+//! // ...
+//!
 //! # use std::path::Path;
 //! # use tempfile::TempDir;
+//! # use tantivy::Index;
+//! # use tantivy::schema::*;
+//! # use tantivy::{Score, DocAddress};
 //! # use tantivy::collector::TopDocs;
 //! # use tantivy::query::QueryParser;
-//! # use tantivy::schema::*;
-//! # use tantivy::{doc, DocAddress, Index, Score};
 //! #
 //! # fn main() {
 //! #     // Let's create a temporary directory for the

src/macros.rs

@@ -22,9 +22,11 @@
 ///
 /// # Example
 ///
-/// ```rust
+/// ```
+/// #[macro_use]
+/// extern crate tantivy;
+///
 /// use tantivy::schema::{Schema, TEXT, FAST};
-/// use tantivy::doc;
 ///
 /// //...
 ///

src/query/automaton_weight.rs

@@ -8,30 +8,30 @@ use crate::termdict::{TermDictionary, TermStreamer};
 use crate::DocId;
 use crate::TantivyError;
 use crate::{Result, SkipResult};
-use std::sync::Arc;
 use tantivy_fst::Automaton;
 
 /// A weight struct for Fuzzy Term and Regex Queries
-pub struct AutomatonWeight<A> {
+pub struct AutomatonWeight<A>
+where
+    A: Automaton + Send + Sync + 'static,
+    A::State: Clone + Default + Sized,
+{
     field: Field,
-    automaton: Arc<A>,
+    automaton: A,
 }
 
 impl<A> AutomatonWeight<A>
 where
     A: Automaton + Send + Sync + 'static,
+    A::State: Clone + Default + Sized,
 {
     /// Create a new AutomationWeight
-    pub fn new<IntoArcA: Into<Arc<A>>>(field: Field, automaton: IntoArcA) -> AutomatonWeight<A> {
-        AutomatonWeight {
-            field,
-            automaton: automaton.into(),
-        }
+    pub fn new(field: Field, automaton: A) -> AutomatonWeight<A> {
+        AutomatonWeight { field, automaton }
     }
 
     fn automaton_stream<'a>(&'a self, term_dict: &'a TermDictionary) -> TermStreamer<'a, &'a A> {
-        let automaton: &A = &*self.automaton;
-        let term_stream_builder = term_dict.search(automaton);
+        let term_stream_builder = term_dict.search(&self.automaton);
         term_stream_builder.into_stream()
     }
 }
@@ -39,6 +39,7 @@ where
 impl<A> Weight for AutomatonWeight<A>
 where
     A: Automaton + Send + Sync + 'static,
+    A::State: Clone + Default + Sized,
 {
     fn scorer(&self, reader: &SegmentReader) -> Result<Box<dyn Scorer>> {
         let max_doc = reader.max_doc();

src/query/fuzzy_query.rs

@@ -1,14 +1,12 @@
 use crate::error::TantivyError::InvalidArgument;
 use crate::query::{AutomatonWeight, Query, Weight};
 use crate::schema::Term;
-use crate::termdict::WrappedDFA;
 use crate::Result;
 use crate::Searcher;
-use levenshtein_automata::{Distance, LevenshteinAutomatonBuilder, DFA};
+use levenshtein_automata::{LevenshteinAutomatonBuilder, DFA};
 use once_cell::sync::Lazy;
 use std::collections::HashMap;
 use std::ops::Range;
-use derive_builder::Builder;
 
 /// A range of Levenshtein distances that we will build DFAs for our terms
 /// The computation is exponential, so best keep it to low single digits
@@ -26,46 +24,16 @@ static LEV_BUILDER: Lazy<HashMap<(u8, bool), LevenshteinAutomatonBuilder>> = Laz
     lev_builder_cache
 });
 
-
-#[derive(Builder, Default, Clone, Debug)]
-pub struct FuzzyConfiguration {
-    /// How many changes are we going to allow
-    pub distance: u8,
-    /// Should a transposition cost 1 or 2?
-    #[builder(default)]
-    pub transposition_cost_one: bool,
-    #[builder(default)]
-    pub prefix: bool,
-    /// If true, only the term with a levenshtein of exactly `distance` will match.
-    /// If false, terms at a distance `<=` to `distance` will match.
-    #[builder(default)]
-    pub exact_distance: bool,
-}
-
-fn build_dfa(fuzzy_configuration: &FuzzyConfiguration, term_text: &str) -> Result<DFA> {
-    let automaton_builder = LEV_BUILDER
-        .get(&(fuzzy_configuration.distance, fuzzy_configuration.transposition_cost_one))
-        .ok_or_else(|| {
-            InvalidArgument(format!(
-                "Levenshtein distance of {} is not allowed. Choose a value in the {:?} range",
-                fuzzy_configuration.distance, VALID_LEVENSHTEIN_DISTANCE_RANGE
-            ))
-        })?;
-    if fuzzy_configuration.prefix {
-        Ok(automaton_builder.build_prefix_dfa(term_text))
-    } else {
-        Ok(automaton_builder.build_dfa(term_text))
-    }
-}
-
 /// A Fuzzy Query matches all of the documents
 /// containing a specific term that is within
 /// Levenshtein distance
 /// ```rust
+/// #[macro_use]
+/// extern crate tantivy;
+/// use tantivy::schema::{Schema, TEXT};
+/// use tantivy::{Index, Result, Term};
 /// use tantivy::collector::{Count, TopDocs};
 /// use tantivy::query::FuzzyTermQuery;
-/// use tantivy::schema::{Schema, TEXT};
-/// use tantivy::{doc, Index, Result, Term};
 ///
 /// # fn main() { example().unwrap(); }
 /// fn example() -> Result<()> {
@@ -75,19 +43,32 @@ fn build_dfa(fuzzy_configuration: &FuzzyConfiguration, term_text: &str) -> Resul
 ///     let index = Index::create_in_ram(schema);
 ///     {
 ///         let mut index_writer = index.writer(3_000_000)?;
-///         index_writer.add_document(doc!(title => "The Name of the Wind"));
-///         index_writer.add_document(doc!(title => "The Diary of Muadib"));
-///         index_writer.add_document(doc!(title => "A Dairy Cow"));
-///         index_writer.add_document(doc!(title => "The Diary of a Young Girl"));
+///         index_writer.add_document(doc!(
+///             title => "The Name of the Wind",
+///         ));
+///         index_writer.add_document(doc!(
+///             title => "The Diary of Muadib",
+///         ));
+///         index_writer.add_document(doc!(
+///             title => "A Dairy Cow",
+///         ));
+///         index_writer.add_document(doc!(
+///             title => "The Diary of a Young Girl",
+///         ));
 ///         index_writer.commit().unwrap();
 ///     }
 ///     let reader = index.reader()?;
 ///     let searcher = reader.searcher();
-///     let term = Term::from_field_text(title, "Diary");
-///     let query = FuzzyTermQuery::new(term, 1, true);
-///     let (top_docs, count) = searcher.search(&query, &(TopDocs::with_limit(2), Count)).unwrap();
-///     assert_eq!(count, 2);
-///     assert_eq!(top_docs.len(), 2);
+///
+///     {
+///
+///         let term = Term::from_field_text(title, "Diary");
+///         let query = FuzzyTermQuery::new(term, 1, true);
+///         let (top_docs, count) = searcher.search(&query, &(TopDocs::with_limit(2), Count)).unwrap();
+///         assert_eq!(count, 2);
+///         assert_eq!(top_docs.len(), 2);
+///     }
+///
 ///     Ok(())
 /// }
 /// ```
@@ -95,58 +76,54 @@ fn build_dfa(fuzzy_configuration: &FuzzyConfiguration, term_text: &str) -> Resul
 pub struct FuzzyTermQuery {
     /// What term are we searching
     term: Term,
-    configuration: FuzzyConfiguration
+    /// How many changes are we going to allow
+    distance: u8,
+    /// Should a transposition cost 1 or 2?
+    transposition_cost_one: bool,
+    ///
+    prefix: bool,
 }
 
 impl FuzzyTermQuery {
-    pub fn new_from_configuration(term: Term, configuration: FuzzyConfiguration) -> FuzzyTermQuery {
-        FuzzyTermQuery {
-            term,
-            configuration
-        }
-    }
-
     /// Creates a new Fuzzy Query
     pub fn new(term: Term, distance: u8, transposition_cost_one: bool) -> FuzzyTermQuery {
         FuzzyTermQuery {
             term,
-            configuration: FuzzyConfiguration {
-                distance,
-                transposition_cost_one,
-                prefix: false,
-                exact_distance: false
+            distance,
+            transposition_cost_one,
+            prefix: false,
+        }
+    }
+
+    /// Creates a new Fuzzy Query that treats transpositions as cost one rather than two
+    pub fn new_prefix(term: Term, distance: u8, transposition_cost_one: bool) -> FuzzyTermQuery {
+        FuzzyTermQuery {
+            term,
+            distance,
+            transposition_cost_one,
+            prefix: true,
+        }
+    }
+
+    fn specialized_weight(&self) -> Result<AutomatonWeight<DFA>> {
+        // LEV_BUILDER is a HashMap, whose `get` method returns an Option
+        match LEV_BUILDER.get(&(self.distance, false)) {
+            // Unwrap the option and build the Ok(AutomatonWeight)
+            Some(automaton_builder) => {
+                let automaton = automaton_builder.build_dfa(self.term.text());
+                Ok(AutomatonWeight::new(self.term.field(), automaton))
             }
+            None => Err(InvalidArgument(format!(
+                "Levenshtein distance of {} is not allowed. Choose a value in the {:?} range",
+                self.distance, VALID_LEVENSHTEIN_DISTANCE_RANGE
+            ))),
         }
     }
 }
 
 impl Query for FuzzyTermQuery {
     fn weight(&self, _searcher: &Searcher, _scoring_enabled: bool) -> Result<Box<dyn Weight>> {
-        let dfa = build_dfa(&self.configuration, self.term.text())?;
-        // TODO optimize for distance = 0 and possibly prefix
-        if self.configuration.exact_distance {
-            let target_distance = self.configuration.distance;
-            let wrapped_dfa = WrappedDFA {
-                dfa,
-                condition: move |distance: Distance| distance == Distance::Exact(target_distance),
-            };
-            Ok(Box::new(AutomatonWeight::new(
-                self.term.field(),
-                wrapped_dfa,
-            )))
-        } else {
-            let wrapped_dfa = WrappedDFA {
-                dfa,
-                condition: move |distance: Distance| match distance {
-                    Distance::Exact(_) => true,
-                    Distance::AtLeast(_) => false,
-                },
-            };
-            Ok(Box::new(AutomatonWeight::new(
-                self.term.field(),
-                wrapped_dfa,
-            )))
-        }
+        Ok(Box::new(self.specialized_weight()?))
     }
 }
 
@@ -159,7 +136,6 @@ mod test {
     use crate::tests::assert_nearly_equals;
     use crate::Index;
     use crate::Term;
-    use super::FuzzyConfigurationBuilder;
 
     #[test]
     pub fn test_fuzzy_term() {
@@ -181,6 +157,7 @@ mod test {
         let searcher = reader.searcher();
         {
             let term = Term::from_field_text(country_field, "japon");
+
             let fuzzy_query = FuzzyTermQuery::new(term, 1, true);
             let top_docs = searcher
                 .search(&fuzzy_query, &TopDocs::with_limit(2))
@@ -189,73 +166,5 @@ mod test {
             let (score, _) = top_docs[0];
             assert_nearly_equals(1f32, score);
         }
-        {
-            let term = Term::from_field_text(country_field, "japon");
-            let fuzzy_conf = FuzzyConfigurationBuilder::default()
-                .distance(2)
-                .exact_distance(true)
-                .build()
-                .unwrap();
-            let fuzzy_query = FuzzyTermQuery::new_from_configuration(term, fuzzy_conf);
-            let top_docs = searcher
-                .search(&fuzzy_query, &TopDocs::with_limit(2))
-                .unwrap();
-            assert!(top_docs.is_empty());
-        }
-        {
-            let term = Term::from_field_text(country_field, "japon");
-            let fuzzy_conf = FuzzyConfigurationBuilder::default()
-                .distance(1)
-                .exact_distance(true)
-                .build()
-                .unwrap();
-            let fuzzy_query = FuzzyTermQuery::new_from_configuration(term, fuzzy_conf);
-            let top_docs = searcher
-                .search(&fuzzy_query, &TopDocs::with_limit(2))
-                .unwrap();
-            assert_eq!(top_docs.len(), 1);
-        }
-        {
-            let term = Term::from_field_text(country_field, "jpp");
-            let fuzzy_conf = FuzzyConfigurationBuilder::default()
-                .distance(1)
-                .prefix(true)
-                .build()
-                .unwrap();
-            let fuzzy_query = FuzzyTermQuery::new_from_configuration(term, fuzzy_conf);
-            let top_docs = searcher
-                .search(&fuzzy_query, &TopDocs::with_limit(2))
-                .unwrap();
-            assert_eq!(top_docs.len(), 1);
-        }
-        {
-            let term = Term::from_field_text(country_field, "jpaan");
-            let fuzzy_conf = FuzzyConfigurationBuilder::default()
-                .distance(1)
-                .exact_distance(true)
-                .transposition_cost_one(true)
-                .build()
-                .unwrap();
-            let fuzzy_query = FuzzyTermQuery::new_from_configuration(term, fuzzy_conf);
-            let top_docs = searcher
-                .search(&fuzzy_query, &TopDocs::with_limit(2))
-                .unwrap();
-            assert_eq!(top_docs.len(), 1);
-        }
-        {
-            let term = Term::from_field_text(country_field, "jpaan");
-            let fuzzy_conf = FuzzyConfigurationBuilder::default()
-                .distance(2)
-                .exact_distance(true)
-                .transposition_cost_one(false)
-                .build()
-                .unwrap();
-            let fuzzy_query = FuzzyTermQuery::new_from_configuration(term, fuzzy_conf);
-            let top_docs = searcher
-                .search(&fuzzy_query, &TopDocs::with_limit(2))
-                .unwrap();
-            assert_eq!(top_docs.len(), 1);
-        }
     }
-
 }

src/query/mod.rs

@@ -12,6 +12,7 @@ mod exclude;
 mod explanation;
 mod fuzzy_query;
 mod intersection;
+mod occur;
 mod phrase_query;
 mod query;
 mod query_parser;
@@ -40,8 +41,9 @@ pub use self::boolean_query::BooleanQuery;
 pub use self::empty_query::{EmptyQuery, EmptyScorer, EmptyWeight};
 pub use self::exclude::Exclude;
 pub use self::explanation::Explanation;
-pub use self::fuzzy_query::{FuzzyTermQuery, FuzzyConfiguration, FuzzyConfigurationBuilder};
+pub use self::fuzzy_query::FuzzyTermQuery;
 pub use self::intersection::intersect_scorers;
+pub use self::occur::Occur;
 pub use self::phrase_query::PhraseQuery;
 pub use self::query::Query;
 pub use self::query_parser::QueryParser;
@@ -53,7 +55,6 @@ pub use self::scorer::ConstScorer;
 pub use self::scorer::Scorer;
 pub use self::term_query::TermQuery;
 pub use self::weight::Weight;
-pub use tantivy_query_grammar::Occur;
 
 #[cfg(test)]
 mod tests {

src/query/occur.rs

@@ -25,24 +25,24 @@ impl Occur {
             Occur::MustNot => '-',
         }
     }
+}
 
-    /// Compose two occur values.
-    pub fn compose(left: Occur, right: Occur) -> Occur {
-        match left {
-            Occur::Should => right,
-            Occur::Must => {
-                if right == Occur::MustNot {
-                    Occur::MustNot
-                } else {
-                    Occur::Must
-                }
+/// Compose two occur values.
+pub fn compose_occur(left: Occur, right: Occur) -> Occur {
+    match left {
+        Occur::Should => right,
+        Occur::Must => {
+            if right == Occur::MustNot {
+                Occur::MustNot
+            } else {
+                Occur::Must
             }
-            Occur::MustNot => {
-                if right == Occur::MustNot {
-                    Occur::Must
-                } else {
-                    Occur::MustNot
-                }
+        }
+        Occur::MustNot => {
+            if right == Occur::MustNot {
+                Occur::Must
+            } else {
+                Occur::MustNot
             }
         }
     }

src/query/query_parser/mod.rs

@@ -1,4 +1,6 @@
+mod query_grammar;
 mod query_parser;
+mod user_input_ast;
 
 pub mod logical_ast;
 pub use self::query_parser::QueryParser;

src/query/query_parser/query_grammar.rs

@@ -1,5 +1,6 @@
 use super::user_input_ast::*;
-use crate::Occur;
+use crate::query::occur::Occur;
+use crate::query::query_parser::user_input_ast::UserInputBound;
 use combine::char::*;
 use combine::error::StreamError;
 use combine::stream::StreamErrorFor;

src/query/query_parser/query_parser.rs

@@ -1,5 +1,9 @@
 use super::logical_ast::*;
+use super::query_grammar::parse_to_ast;
+use super::user_input_ast::*;
 use crate::core::Index;
+use crate::query::occur::compose_occur;
+use crate::query::query_parser::logical_ast::LogicalAST;
 use crate::query::AllQuery;
 use crate::query::BooleanQuery;
 use crate::query::EmptyQuery;
@@ -12,11 +16,11 @@ use crate::schema::IndexRecordOption;
 use crate::schema::{Field, Schema};
 use crate::schema::{FieldType, Term};
 use crate::tokenizer::TokenizerManager;
+use combine::Parser;
 use std::borrow::Cow;
 use std::num::{ParseFloatError, ParseIntError};
 use std::ops::Bound;
 use std::str::FromStr;
-use tantivy_query_grammar::{UserInputAST, UserInputBound, UserInputLeaf};
 
 /// Possible error that may happen when parsing a query.
 #[derive(Debug, PartialEq, Eq, Fail)]
@@ -218,8 +222,9 @@ impl QueryParser {
 
     /// Parse the user query into an AST.
     fn parse_query_to_logical_ast(&self, query: &str) -> Result<LogicalAST, QueryParserError> {
-        let user_input_ast =
-            tantivy_query_grammar::parse_query(query).map_err(|_| QueryParserError::SyntaxError)?;
+        let (user_input_ast, _remaining) = parse_to_ast()
+            .parse(query)
+            .map_err(|_| QueryParserError::SyntaxError)?;
         self.compute_logical_ast(user_input_ast)
     }
 
@@ -394,7 +399,7 @@ impl QueryParser {
                 let mut logical_sub_queries: Vec<(Occur, LogicalAST)> = Vec::new();
                 for sub_query in sub_queries {
                     let (occur, sub_ast) = self.compute_logical_ast_with_occur(sub_query)?;
-                    let new_occur = Occur::compose(default_occur, occur);
+                    let new_occur = compose_occur(default_occur, occur);
                     logical_sub_queries.push((new_occur, sub_ast));
                 }
                 Ok((Occur::Should, LogicalAST::Clause(logical_sub_queries)))
@@ -402,7 +407,7 @@ impl QueryParser {
             UserInputAST::Unary(left_occur, subquery) => {
                 let (right_occur, logical_sub_queries) =
                     self.compute_logical_ast_with_occur(*subquery)?;
-                Ok((Occur::compose(left_occur, right_occur), logical_sub_queries))
+                Ok((compose_occur(left_occur, right_occur), logical_sub_queries))
             }
             UserInputAST::Leaf(leaf) => {
                 let result_ast = self.compute_logical_ast_from_leaf(*leaf)?;

src/query/query_parser/user_input_ast.rs

@@ -1,7 +1,7 @@
 use std::fmt;
 use std::fmt::{Debug, Formatter};
 
-use crate::Occur;
+use crate::query::Occur;
 
 #[derive(PartialEq)]
 pub enum UserInputLeaf {

src/query/range_query.rs

@@ -38,10 +38,14 @@ fn map_bound<TFrom, TTo, Transform: Fn(&TFrom) -> TTo>(
 /// # Example
 ///
 /// ```rust
-/// # use tantivy::collector::Count;
-/// # use tantivy::query::RangeQuery;
+///
+/// # #[macro_use]
+/// # extern crate tantivy;
+/// # use tantivy::Index;
 /// # use tantivy::schema::{Schema, INDEXED};
-/// # use tantivy::{doc, Index, Result};
+/// # use tantivy::collector::Count;
+/// # use tantivy::Result;
+/// # use tantivy::query::RangeQuery;
 /// #
 /// # fn run() -> Result<()> {
 /// #     let mut schema_builder = Schema::builder();

src/query/regex_query.rs

@@ -4,18 +4,22 @@ use crate::schema::Field;
 use crate::Result;
 use crate::Searcher;
 use std::clone::Clone;
-use std::sync::Arc;
 use tantivy_fst::Regex;
 
-/// A Regex Query matches all of the documents
+// A Regex Query matches all of the documents
 /// containing a specific term that matches
-/// a regex pattern.
+/// a regex pattern
+/// A Fuzzy Query matches all of the documents
+/// containing a specific term that is within
+/// Levenshtein distance
 ///
 /// ```rust
+/// #[macro_use]
+/// extern crate tantivy;
+/// use tantivy::schema::{Schema, TEXT};
+/// use tantivy::{Index, Result, Term};
 /// use tantivy::collector::Count;
 /// use tantivy::query::RegexQuery;
-/// use tantivy::schema::{Schema, TEXT};
-/// use tantivy::{doc, Index, Result, Term};
 ///
 /// # fn main() { example().unwrap(); }
 /// fn example() -> Result<()> {
@@ -44,7 +48,7 @@ use tantivy_fst::Regex;
 ///     let searcher = reader.searcher();
 ///
 ///     let term = Term::from_field_text(title, "Diary");
-///     let query = RegexQuery::from_pattern("d[ai]{2}ry", title)?;
+///     let query = RegexQuery::new("d[ai]{2}ry".to_string(), title);
 ///     let count = searcher.search(&query, &Count)?;
 ///     assert_eq!(count, 3);
 ///     Ok(())
@@ -52,34 +56,30 @@ use tantivy_fst::Regex;
 /// ```
 #[derive(Debug, Clone)]
 pub struct RegexQuery {
-    regex: Arc<Regex>,
+    regex_pattern: String,
     field: Field,
 }
 
 impl RegexQuery {
-    /// Creates a new RegexQuery from a given pattern
-    pub fn from_pattern(regex_pattern: &str, field: Field) -> Result<Self> {
-        let regex = Regex::new(&regex_pattern)
-            .map_err(|_| TantivyError::InvalidArgument(regex_pattern.to_string()))?;
-        Ok(RegexQuery::from_regex(regex, field))
-    }
-
-    /// Creates a new RegexQuery from a fully built Regex
-    pub fn from_regex<T: Into<Arc<Regex>>>(regex: T, field: Field) -> Self {
+    /// Creates a new Fuzzy Query
+    pub fn new(regex_pattern: String, field: Field) -> RegexQuery {
         RegexQuery {
-            regex: regex.into(),
+            regex_pattern,
             field,
         }
     }
 
-    fn specialized_weight(&self) -> AutomatonWeight<Regex> {
-        AutomatonWeight::new(self.field, self.regex.clone())
+    fn specialized_weight(&self) -> Result<AutomatonWeight<Regex>> {
+        let automaton = Regex::new(&self.regex_pattern)
+            .map_err(|_| TantivyError::InvalidArgument(self.regex_pattern.clone()))?;
+
+        Ok(AutomatonWeight::new(self.field, automaton))
     }
 }
 
 impl Query for RegexQuery {
     fn weight(&self, _searcher: &Searcher, _scoring_enabled: bool) -> Result<Box<dyn Weight>> {
-        Ok(Box::new(self.specialized_weight()))
+        Ok(Box::new(self.specialized_weight()?))
     }
 }
 
@@ -87,14 +87,13 @@ impl Query for RegexQuery {
 mod test {
     use super::RegexQuery;
     use crate::collector::TopDocs;
+    use crate::schema::Schema;
     use crate::schema::TEXT;
-    use crate::schema::{Field, Schema};
     use crate::tests::assert_nearly_equals;
-    use crate::{Index, IndexReader};
-    use std::sync::Arc;
-    use tantivy_fst::Regex;
+    use crate::Index;
 
-    fn build_test_index() -> (IndexReader, Field) {
+    #[test]
+    pub fn test_regex_query() {
         let mut schema_builder = Schema::builder();
         let country_field = schema_builder.add_text_field("country", TEXT);
         let schema = schema_builder.build();
@@ -110,65 +109,20 @@ mod test {
             index_writer.commit().unwrap();
         }
         let reader = index.reader().unwrap();
-
-        (reader, country_field)
-    }
-
-    fn verify_regex_query(
-        query_matching_one: RegexQuery,
-        query_matching_zero: RegexQuery,
-        reader: IndexReader,
-    ) {
         let searcher = reader.searcher();
         {
+            let regex_query = RegexQuery::new("jap[ao]n".to_string(), country_field);
             let scored_docs = searcher
-                .search(&query_matching_one, &TopDocs::with_limit(2))
+                .search(&regex_query, &TopDocs::with_limit(2))
                 .unwrap();
             assert_eq!(scored_docs.len(), 1, "Expected only 1 document");
             let (score, _) = scored_docs[0];
             assert_nearly_equals(1f32, score);
         }
+        let regex_query = RegexQuery::new("jap[A-Z]n".to_string(), country_field);
         let top_docs = searcher
-            .search(&query_matching_zero, &TopDocs::with_limit(2))
+            .search(&regex_query, &TopDocs::with_limit(2))
             .unwrap();
         assert!(top_docs.is_empty(), "Expected ZERO document");
     }
-
-    #[test]
-    pub fn test_regex_query() {
-        let (reader, field) = build_test_index();
-
-        let matching_one = RegexQuery::from_pattern("jap[ao]n", field).unwrap();
-        let matching_zero = RegexQuery::from_pattern("jap[A-Z]n", field).unwrap();
-
-        verify_regex_query(matching_one, matching_zero, reader);
-    }
-
-    #[test]
-    pub fn test_construct_from_regex() {
-        let (reader, field) = build_test_index();
-
-        let matching_one = RegexQuery::from_regex(Regex::new("jap[ao]n").unwrap(), field);
-        let matching_zero = RegexQuery::from_regex(Regex::new("jap[A-Z]n").unwrap(), field);
-
-        verify_regex_query(matching_one, matching_zero, reader);
-    }
-
-    #[test]
-    pub fn test_construct_from_reused_regex() {
-        let r1 = Arc::new(Regex::new("jap[ao]n").unwrap());
-        let r2 = Arc::new(Regex::new("jap[A-Z]n").unwrap());
-
-        let (reader, field) = build_test_index();
-
-        let matching_one = RegexQuery::from_regex(r1.clone(), field);
-        let matching_zero = RegexQuery::from_regex(r2.clone(), field);
-
-        verify_regex_query(matching_one, matching_zero, reader.clone());
-
-        let matching_one = RegexQuery::from_regex(r1.clone(), field);
-        let matching_zero = RegexQuery::from_regex(r2.clone(), field);
-
-        verify_regex_query(matching_one, matching_zero, reader.clone());
-    }
 }

src/query/term_query/term_query.rs

@@ -20,10 +20,12 @@ use std::fmt;
 /// * `field norm` - number of tokens in the field.
 ///
 /// ```rust
+/// #[macro_use]
+/// extern crate tantivy;
+/// use tantivy::schema::{Schema, TEXT, IndexRecordOption};
+/// use tantivy::{Index, Result, Term};
 /// use tantivy::collector::{Count, TopDocs};
 /// use tantivy::query::TermQuery;
-/// use tantivy::schema::{Schema, TEXT, IndexRecordOption};
-/// use tantivy::{doc, Index, Result, Term};
 ///
 /// # fn main() { example().unwrap(); }
 /// fn example() -> Result<()> {

src/schema/schema.rs

@@ -301,26 +301,28 @@ impl Schema {
 
         let mut doc = Document::default();
         for (field_name, json_value) in json_obj.iter() {
-            let field = self
-                .get_field(field_name)
-                .ok_or_else(|| DocParsingError::NoSuchFieldInSchema(field_name.clone()))?;
-            let field_entry = self.get_field_entry(field);
-            let field_type = field_entry.field_type();
-            match *json_value {
-                JsonValue::Array(ref json_items) => {
-                    for json_item in json_items {
-                        let value = field_type
-                            .value_from_json(json_item)
-                            .map_err(|e| DocParsingError::ValueError(field_name.clone(), e))?;
-                        doc.add(FieldValue::new(field, value));
+            match self.get_field(field_name) {
+                Some(field) => {
+                    let field_entry = self.get_field_entry(field);
+                    let field_type = field_entry.field_type();
+                    match *json_value {
+                        JsonValue::Array(ref json_items) => {
+                            for json_item in json_items {
+                                let value = field_type.value_from_json(json_item).map_err(|e| {
+                                    DocParsingError::ValueError(field_name.clone(), e)
+                                })?;
+                                doc.add(FieldValue::new(field, value));
+                            }
+                        }
+                        _ => {
+                            let value = field_type
+                                .value_from_json(json_value)
+                                .map_err(|e| DocParsingError::ValueError(field_name.clone(), e))?;
+                            doc.add(FieldValue::new(field, value));
+                        }
                     }
                 }
-                _ => {
-                    let value = field_type
-                        .value_from_json(json_value)
-                        .map_err(|e| DocParsingError::ValueError(field_name.clone(), e))?;
-                    doc.add(FieldValue::new(field, value));
-                }
+                None => return Err(DocParsingError::NoSuchFieldInSchema(field_name.clone())),
             }
         }
         Ok(doc)

src/snippet/mod.rs

@@ -213,9 +213,11 @@ fn select_best_fragment_combination(fragments: &[FragmentCandidate], text: &str)
 /// # Example
 ///
 /// ```rust
-/// # use tantivy::query::QueryParser;
+/// # #[macro_use]
+/// # extern crate tantivy;
+/// # use tantivy::Index;
 /// # use tantivy::schema::{Schema, TEXT};
-/// # use tantivy::{doc, Index};
+/// # use tantivy::query::QueryParser;
 /// use tantivy::SnippetGenerator;
 ///
 /// # fn main() -> tantivy::Result<()> {

src/termdict/mod.rs

@@ -31,43 +31,14 @@ mod termdict;
 pub use self::merger::TermMerger;
 pub use self::streamer::{TermStreamer, TermStreamerBuilder};
 pub use self::termdict::{TermDictionary, TermDictionaryBuilder};
-use levenshtein_automata::{Distance, DFA, SINK_STATE};
-use tantivy_fst::Automaton;
-
-pub(crate) struct WrappedDFA<Cond> {
-    pub dfa: DFA,
-    pub condition: Cond,
-}
-
-impl<Cond: Fn(Distance) -> bool> Automaton for WrappedDFA<Cond> {
-    type State = u32;
-
-    fn start(&self) -> Self::State {
-        self.dfa.initial_state()
-    }
-
-    fn is_match(&self, state: &Self::State) -> bool {
-        let distance = self.dfa.distance(*state);
-        (self.condition)(distance)
-    }
-
-    fn can_match(&self, state: &Self::State) -> bool {
-        *state != SINK_STATE
-    }
-
-    fn accept(&self, state: &Self::State, byte: u8) -> Self::State {
-        self.dfa.transition(*state, byte)
-    }
-}
 
 #[cfg(test)]
 mod tests {
-    use super::{TermDictionary, TermDictionaryBuilder, TermStreamer, WrappedDFA};
+    use super::{TermDictionary, TermDictionaryBuilder, TermStreamer};
     use crate::core::Index;
     use crate::directory::{Directory, RAMDirectory, ReadOnlySource};
     use crate::postings::TermInfo;
     use crate::schema::{Document, FieldType, Schema, TEXT};
-    use levenshtein_automata::Distance;
     use std::path::PathBuf;
     use std::str;
 
@@ -452,14 +423,9 @@ mod tests {
 
         // We can now build an entire dfa.
         let lev_automaton_builder = LevenshteinAutomatonBuilder::new(2, true);
-        let wrapped_dfa = WrappedDFA {
-            dfa: lev_automaton_builder.build_dfa("Spaen"),
-            condition: |distance| match distance {
-                Distance::Exact(_) => true,
-                Distance::AtLeast(_) => false,
-            },
-        };
-        let mut range = term_dict.search(wrapped_dfa).into_stream();
+        let automaton = lev_automaton_builder.build_dfa("Spaen");
+
+        let mut range = term_dict.search(automaton).into_stream();
 
         // get the first finding
         assert!(range.advance());

src/termdict/streamer.rs

@@ -2,7 +2,7 @@ use super::TermDictionary;
 use crate::postings::TermInfo;
 use crate::termdict::TermOrdinal;
 use tantivy_fst::automaton::AlwaysMatch;
-use tantivy_fst::map::{Stream, StreamBuilder};
+use tantivy_fst::map::{Stream, StreamBuilder, StreamWithState};
 use tantivy_fst::Automaton;
 use tantivy_fst::{IntoStreamer, Streamer};
 
@@ -11,6 +11,7 @@ use tantivy_fst::{IntoStreamer, Streamer};
 pub struct TermStreamerBuilder<'a, A = AlwaysMatch>
 where
     A: Automaton,
+    A::State: Clone,
 {
     fst_map: &'a TermDictionary,
     stream_builder: StreamBuilder<'a, A>,
@@ -19,6 +20,7 @@ where
 impl<'a, A> TermStreamerBuilder<'a, A>
 where
     A: Automaton,
+    A::State: Clone + Default + Sized,
 {
     pub(crate) fn new(fst_map: &'a TermDictionary, stream_builder: StreamBuilder<'a, A>) -> Self {
         TermStreamerBuilder {
@@ -56,10 +58,11 @@ where
     pub fn into_stream(self) -> TermStreamer<'a, A> {
         TermStreamer {
             fst_map: self.fst_map,
-            stream: self.stream_builder.into_stream(),
+            stream: self.stream_builder.with_state().into_stream(),
             term_ord: 0u64,
             current_key: Vec::with_capacity(100),
             current_value: TermInfo::default(),
+            state: Default::default(),
         }
     }
 }
@@ -69,27 +72,31 @@ where
 pub struct TermStreamer<'a, A = AlwaysMatch>
 where
     A: Automaton,
+    A::State: Clone + Default + Sized,
 {
     fst_map: &'a TermDictionary,
-    stream: Stream<'a, A>,
+    stream: StreamWithState<'a, A>,
     term_ord: TermOrdinal,
     current_key: Vec<u8>,
     current_value: TermInfo,
+    state: A::State,
 }
 
 impl<'a, A> TermStreamer<'a, A>
 where
     A: Automaton,
+    A::State: Clone + Default + Sized,
 {
     /// Advance position the stream on the next item.
     /// Before the first call to `.advance()`, the stream
     /// is an unitialized state.
     pub fn advance(&mut self) -> bool {
-        if let Some((term, term_ord)) = self.stream.next() {
+        if let Some((term, term_ord, state)) = self.stream.next() {
             self.current_key.clear();
             self.current_key.extend_from_slice(term);
             self.term_ord = term_ord;
             self.current_value = self.fst_map.term_info_from_ord(term_ord);
+            self.state = state;
             true
         } else {
             false
@@ -118,6 +125,10 @@ where
         &self.current_key
     }
 
+    pub fn state(&self) -> &A::State {
+        &self.state
+    }
+
     /// Accesses the current value.
     ///
     /// Calling `.value()` after the end of the stream will return the

src/termdict/termdict.rs

@@ -197,7 +197,11 @@ impl TermDictionary {
 
     /// Returns a search builder, to stream all of the terms
     /// within the Automaton
-    pub fn search<'a, A: Automaton + 'a>(&'a self, automaton: A) -> TermStreamerBuilder<'a, A> {
+    pub fn search<'a, A>(&'a self, automaton: A) -> TermStreamerBuilder<'a, A>
+    where
+        A: Automaton + 'a,
+        A::State: Clone + Default + Sized,
+    {
         let stream_builder = self.fst_index.search(automaton);
         TermStreamerBuilder::<A>::new(self, stream_builder)
     }

src/tokenizer/alphanum_only.rs

@@ -1,5 +1,6 @@
 //! # Example
-//! ```rust
+//! ```
+//! extern crate tantivy;
 //! use tantivy::tokenizer::*;
 //!
 //! # fn main() {

src/tokenizer/mod.rs

@@ -4,7 +4,8 @@
 //! You must define in your schema which tokenizer should be used for
 //! each of your fields :
 //!
-//! ```rust
+//! ```
+//! extern crate tantivy;
 //! use tantivy::schema::*;
 //!
 //! # fn main() {
@@ -64,6 +65,8 @@
 //! For instance, the `en_stem` is defined as follows.
 //!
 //! ```rust
+//! # extern crate tantivy;
+//!
 //! use tantivy::tokenizer::*;
 //!
 //! # fn main() {
@@ -77,7 +80,8 @@
 //! Once your tokenizer is defined, you need to
 //! register it with a name in your index's [`TokenizerManager`](./struct.TokenizerManager.html).
 //!
-//! ```rust
+//! ```
+//! # extern crate tantivy;
 //! # use tantivy::schema::Schema;
 //! # use tantivy::tokenizer::*;
 //! # use tantivy::Index;
@@ -97,7 +101,8 @@
 //!
 //! # Example
 //!
-//! ```rust
+//! ```
+//! extern crate tantivy;
 //! use tantivy::schema::{Schema, IndexRecordOption, TextOptions, TextFieldIndexing};
 //! use tantivy::tokenizer::*;
 //! use tantivy::Index;

src/tokenizer/ngram_tokenizer.rs

@@ -29,7 +29,8 @@ use super::{Token, TokenStream, Tokenizer};
 ///
 /// # Example
 ///
-/// ```rust
+/// ```
+/// # extern crate tantivy;
 /// use tantivy::tokenizer::*;
 /// # fn main() {
 /// let tokenizer = NgramTokenizer::new(2, 3, false);

src/tokenizer/remove_long.rs

@@ -1,5 +1,6 @@
 //! # Example
-//! ```rust
+//! ```
+//! extern crate tantivy;
 //! use tantivy::tokenizer::*;
 //!
 //! # fn main() {

src/tokenizer/stop_word_filter.rs

@@ -1,5 +1,6 @@
 //! # Example
-//! ```rust
+//! ```
+//! extern crate tantivy;
 //! use tantivy::tokenizer::*;
 //!
 //! # fn main() {