Added convert to static [u8]

CHANGELOG.md

@@ -1,21 +1,9 @@
-Tantivy 0.6
+Tantivy 0.5.2
 ==========================
+
 - Removed C code. Tantivy is now pure Rust.
 - BM25
 - Approximate field norms encoded over 1 byte.
-- Compiles on stable rust
-- Add &[u8] fastfield for associating arbitrary bytes to each document (@jason-wolfe) (#270)
-    - Completely uncompressed
-    - Internally: One u64 fast field for indexes, one fast field for the bytes themselves.
-- Add NGram token support (@drusellers)
-- Add Stopword Filter support (@drusellers)
-
-Tantivy 0.5.2
-===========================
-- bugfix #274
-- bugfix #280
-- bugfix #289
-
 
 Tantivy 0.5.1
 ==========================
@@ -93,7 +81,7 @@ Tantivy 0.3
 Special thanks to @Kodraus @lnicola @Ameobea @manuel-woelker @celaus
 for their contribution to this release.
 
-Thanks also to everyone in tantivy gitter chat
+Thanks also to everyone in tantivy gitter chat 
 for their advise and company :)
 
 https://gitter.im/tantivy-search/tantivy
@@ -101,9 +89,9 @@ https://gitter.im/tantivy-search/tantivy
 
 Warning:
 
-Tantivy 0.3 is NOT backward compatible with tantivy 0.2
+Tantivy 0.3 is NOT backward compatible with tantivy 0.2 
 code and index format.
-You should not expect backward compatibility before
+You should not expect backward compatibility before 
 tantivy 1.0.
 
 
@@ -129,7 +117,7 @@ Thanks to @KodrAus ! (#108)
   the natural ordering.
 - Building binary targets for tantivy-cli (Thanks to @KodrAus)
 - Misc invisible bug fixes, and code cleanup.
-- Use
+- Use 
 
 
 

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "tantivy"
-version = "0.6.0-dev"
+version = "0.5.1"
 authors = ["Paul Masurel <paul.masurel@gmail.com>"]
 license = "MIT"
 categories = ["database-implementations", "data-structures"]
@@ -12,14 +12,12 @@ readme = "README.md"
 keywords = ["search", "information", "retrieval"]
 
 [dependencies]
-base64 = "0.9.1"
 byteorder = "1.0"
 lazy_static = "0.2.1"
 tinysegmenter = "0.1.0"
 regex = "0.2"
-fst = {version="0.3", default-features=false}
+fst = {version="0.2", default-features=false}
 atomicwrites = {version="0.1", optional=true}
-tempfile = "2.1"
 log = "0.3.6"
 combine = "2.2"
 tempdir = "0.3"
@@ -28,8 +26,6 @@ serde_derive = "1.0"
 serde_json = "1.0"
 num_cpus = "1.2"
 itertools = "0.5.9"
-levenshtein_automata = {version="0.1", features=["fst_automaton"]}
-lz4 = "1.20"
 bit-set = "0.4.0"
 uuid = { version = "0.6", features = ["v4", "serde"] }
 chan = "0.1"
@@ -40,16 +36,17 @@ error-chain = "0.8"
 owning_ref = "0.3"
 stable_deref_trait = "1.0.0"
 rust-stemmers = "0.1.0"
-downcast = { version="0.9" }
+downcast = { version="0.9", features = ["nightly"]}
 matches = "0.1"
-bitpacking = "0.4"
-fnv = "1.0.6"
+snap = "0.2"
+bitpacking = {path = "../bitpacking"}
 
 [target.'cfg(windows)'.dependencies]
 winapi = "0.2"
 
 [dev-dependencies]
 rand = "0.3"
+tempfile = "2.1"
 env_logger = "0.4"
 
 [profile.release]
@@ -58,11 +55,12 @@ debug = false
 lto = true
 debug-assertions = false
 
+
 [features]
 default = ["mmap"]
-simd = ["bitpacking/simd"]
+streamdict = []
 mmap = ["fst/mmap", "atomicwrites"]
-unstable = ["simd"]
+
 
 [badges]
 travis-ci = { repository = "tantivy-search/tantivy" }
@@ -71,5 +69,11 @@ travis-ci = { repository = "tantivy-search/tantivy" }
 name = "simple_search"
 required-features = ["mmap"]
 
-[[example]]
-name = "custom_tokenizer"
+
+[[bin]]
+name = "convert_to_static"
+path = "./bin/convert_to_static.rs"
+
+[[bin]]
+name = "test_static_dir"
+path = "./bin/test_static_dir.rs"

README.md

@@ -43,12 +43,15 @@ It will walk you through getting a wikipedia search engine up and running in a f
 
 ## Development
 
-Tantivy now compiles on stable rust.
+Tantivy requires Rust Nightly because it uses requires the features [`box_syntax`](https://doc.rust-lang.org/stable/unstable-book/language-features/box-syntax.html), [`optin_builtin_traits`](https://github.com/rust-lang/rfcs/blob/master/text/0019-opt-in-builtin-traits.md), [`conservative_impl_trait`](https://github.com/rust-lang/rfcs/blob/master/text/1522-conservative-impl-trait.md),
+and [simd](https://github.com/rust-lang/rust/issues/27731).
+
+
 To check out and run test, you can simply run :
 
     git clone git@github.com:tantivy-search/tantivy.git
     cd tantivy
-    cargo build
+    cargo +nightly build
 
 
 ## Note on release build and performance

bin/convert_to_static.rs

@@ -0,0 +1,20 @@
+use std::env;
+use std::path::PathBuf;
+use std::fs::File;
+use std::io::Write;
+extern crate tantivy;
+use tantivy::directory::write_static_from_directory;
+
+fn main() {
+    // Prints each argument on a separate line
+    let  mut args = env::args();
+    args.next().unwrap();
+    let directory_path= args.next().expect("Expect 2 args.<directory_path> <outputfile>");
+    let output_path = args.next().expect("Expect 2 args.<directory_path> <outputfile>");
+    println!("{} => {}", directory_path, output_path);
+    let buffer = write_static_from_directory(&PathBuf::from(directory_path)).unwrap();
+    println!("Read all");
+    let mut output = File::create(output_path).unwrap();
+    output.write_all(&buffer[..]).unwrap();
+    output.flush().unwrap();
+}

bin/test_static_dir.rs

@@ -0,0 +1,51 @@
+use std::env;
+use std::path::PathBuf;
+use std::fs::File;
+use std::io::Write;
+extern crate tantivy;
+use tantivy::directory::{StaticDirectory, write_static_from_directory};
+use tantivy::Index;
+use tantivy::query::QueryParser;
+use tantivy::collector::TopCollector;
+
+
+static DATA: &'static [u8] = include_bytes!("output.bin");
+
+fn run() -> tantivy::Result<()> {
+    // Prints each argument on a separate line
+    let directory = StaticDirectory::open(DATA).unwrap();
+    let index = Index::open_directory(directory).unwrap();
+    index.load_searchers().unwrap();
+    let searcher = index.searcher();
+
+    let schema = index.schema();
+    let title = schema.get_field("title").unwrap();
+    let body = schema.get_field("body").unwrap();
+
+    let query_parser = QueryParser::for_index(&index, vec![title, body]);
+    let query = query_parser.parse_query("sea whale")?;
+
+    let mut top_collector = TopCollector::with_limit(10);
+
+    searcher.search(&*query, &mut top_collector)?;
+
+    let doc_addresses = top_collector.docs();
+
+    // The actual documents still need to be
+    // retrieved from Tantivy's store.
+    //
+    // Since the body field was not configured as stored,
+    // the document returned will only contain
+    // a title.
+
+    for doc_address in doc_addresses {
+        let retrieved_doc = searcher.doc(&doc_address)?;
+        println!("{}", schema.to_json(&retrieved_doc));
+    }
+    Ok(())
+}
+
+
+fn main() {
+    run().unwrap();
+}

examples/custom_tokenizer.rs

@@ -1,226 +0,0 @@
-extern crate tantivy;
-extern crate tempdir;
-
-#[macro_use]
-extern crate serde_json;
-
-use std::path::Path;
-use tantivy::collector::TopCollector;
-use tantivy::query::QueryParser;
-use tantivy::schema::*;
-use tantivy::tokenizer::NgramTokenizer;
-use tantivy::Index;
-use tempdir::TempDir;
-
-fn main() {
-    // Let's create a temporary directory for the
-    // sake of this example
-    if let Ok(dir) = TempDir::new("tantivy_token_example_dir") {
-        run_example(dir.path()).unwrap();
-        dir.close().unwrap();
-    }
-}
-
-fn run_example(index_path: &Path) -> tantivy::Result<()> {
-    // # Defining the schema
-    //
-    // The Tantivy index requires a very strict schema.
-    // The schema declares which fields are in the index,
-    // and for each field, its type and "the way it should
-    // be indexed".
-
-    // first we need to define a schema ...
-    let mut schema_builder = SchemaBuilder::default();
-
-    // Our first field is title.
-    // In this example we want to use NGram searching
-    // we will set that to 3 characters, so any three
-    // char in the title should be findable.
-    let text_field_indexing = TextFieldIndexing::default()
-        .set_tokenizer("ngram3")
-        .set_index_option(IndexRecordOption::WithFreqsAndPositions);
-    let text_options = TextOptions::default()
-        .set_indexing_options(text_field_indexing)
-        .set_stored();
-    schema_builder.add_text_field("title", text_options);
-
-    // Our second field is body.
-    // We want full-text search for it, but we do not
-    // need to be able to be able to retrieve it
-    // for our application.
-    //
-    // We can make our index lighter and
-    // by omitting `STORED` flag.
-    schema_builder.add_text_field("body", TEXT);
-
-    let schema = schema_builder.build();
-
-    // # Indexing documents
-    //
-    // Let's create a brand new index.
-    //
-    // This will actually just save a meta.json
-    // with our schema in the directory.
-    let index = Index::create(index_path, schema.clone())?;
-
-    // here we are registering our custome tokenizer
-    // this will store tokens of 3 characters each
-    index
-        .tokenizers()
-        .register("ngram3", NgramTokenizer::new(3, 3, false));
-
-    // To insert document we need an index writer.
-    // There must be only one writer at a time.
-    // This single `IndexWriter` is already
-    // multithreaded.
-    //
-    // Here we use a buffer of 50MB per thread. Using a bigger
-    // heap for the indexer can increase its throughput.
-    let mut index_writer = index.writer(50_000_000)?;
-
-    // Let's index our documents!
-    // We first need a handle on the title and the body field.
-
-    // ### Create a document "manually".
-    //
-    // We can create a document manually, by setting the fields
-    // one by one in a Document object.
-    let title = schema.get_field("title").unwrap();
-    let body = schema.get_field("body").unwrap();
-
-    let mut old_man_doc = Document::default();
-    old_man_doc.add_text(title, "The Old Man and the Sea");
-    old_man_doc.add_text(
-        body,
-        "He was an old man who fished alone in a skiff in the Gulf Stream and \
-         he had gone eighty-four days now without taking a fish.",
-    );
-
-    // ... and add it to the `IndexWriter`.
-    index_writer.add_document(old_man_doc);
-
-    // ### Create a document directly from json.
-    //
-    // Alternatively, we can use our schema to parse a
-    // document object directly from json.
-    // The document is a string, but we use the `json` macro
-    // from `serde_json` for the convenience of multi-line support.
-    let json = json!({
-       "title": "Of Mice and Men",
-       "body": "A few miles south of Soledad, the Salinas River drops in close to the hillside \
-                bank and runs deep and green. The water is warm too, for it has slipped twinkling \
-                over the yellow sands in the sunlight before reaching the narrow pool. On one \
-                side of the river the golden foothill slopes curve up to the strong and rocky \
-                Gabilan Mountains, but on the valley side the water is lined with trees—willows \
-                fresh and green with every spring, carrying in their lower leaf junctures the \
-                debris of the winter’s flooding; and sycamores with mottled, white, recumbent \
-                limbs and branches that arch over the pool"
-    });
-    let mice_and_men_doc = schema.parse_document(&json.to_string())?;
-
-    index_writer.add_document(mice_and_men_doc);
-
-    // Multi-valued field are allowed, they are
-    // expressed in JSON by an array.
-    // The following document has two titles.
-    let json = json!({
-       "title": ["Frankenstein", "The Modern Prometheus"],
-       "body": "You will rejoice to hear that no disaster has accompanied the commencement of an \
-                enterprise which you have regarded with such evil forebodings.  I arrived here \
-                yesterday, and my first task is to assure my dear sister of my welfare and \
-                increasing confidence in the success of my undertaking."
-    });
-    let frankenstein_doc = schema.parse_document(&json.to_string())?;
-
-    index_writer.add_document(frankenstein_doc);
-
-    // This is an example, so we will only index 3 documents
-    // here. You can check out tantivy's tutorial to index
-    // the English wikipedia. Tantivy's indexing is rather fast.
-    // Indexing 5 million articles of the English wikipedia takes
-    // around 4 minutes on my computer!
-
-    // ### Committing
-    //
-    // At this point our documents are not searchable.
-    //
-    //
-    // We need to call .commit() explicitly to force the
-    // index_writer to finish processing the documents in the queue,
-    // flush the current index to the disk, and advertise
-    // the existence of new documents.
-    //
-    // This call is blocking.
-    index_writer.commit()?;
-
-    // If `.commit()` returns correctly, then all of the
-    // documents that have been added are guaranteed to be
-    // persistently indexed.
-    //
-    // In the scenario of a crash or a power failure,
-    // tantivy behaves as if has rolled back to its last
-    // commit.
-
-    // # Searching
-    //
-    // Let's search our index. Start by reloading
-    // searchers in the index. This should be done
-    // after every commit().
-    index.load_searchers()?;
-
-    // Afterwards create one (or more) searchers.
-    //
-    // You should create a searcher
-    // every time you start a "search query".
-    let searcher = index.searcher();
-
-    // The query parser can interpret human queries.
-    // Here, if the user does not specify which
-    // field they want to search, tantivy will search
-    // in both title and body.
-    let query_parser = QueryParser::for_index(&index, vec![title, body]);
-
-    // here we want to get a hit on the 'ken' in Frankenstein
-    let query = query_parser.parse_query("ken")?;
-
-    // A query defines a set of documents, as
-    // well as the way they should be scored.
-    //
-    // A query created by the query parser is scored according
-    // to a metric called Tf-Idf, and will consider
-    // any document matching at least one of our terms.
-
-    // ### Collectors
-    //
-    // We are not interested in all of the documents but
-    // only in the top 10. Keeping track of our top 10 best documents
-    // is the role of the TopCollector.
-    let mut top_collector = TopCollector::with_limit(10);
-
-    // We can now perform our query.
-    searcher.search(&*query, &mut top_collector)?;
-
-    // Our top collector now contains the 10
-    // most relevant doc ids...
-    let doc_addresses = top_collector.docs();
-
-    // The actual documents still need to be
-    // retrieved from Tantivy's store.
-    //
-    // Since the body field was not configured as stored,
-    // the document returned will only contain
-    // a title.
-
-    for doc_address in doc_addresses {
-        let retrieved_doc = searcher.doc(&doc_address)?;
-        println!("{}", schema.to_json(&retrieved_doc));
-    }
-
-    // Wait for indexing and merging threads to shut down.
-    // Usually this isn't needed, but in `main` we try to
-    // delete the temporary directory and that fails on
-    // Windows if the files are still open.
-    index_writer.wait_merging_threads()?;
-
-    Ok(())
-}

examples/simple_search.rs

@@ -5,11 +5,11 @@ extern crate tempdir;
 extern crate serde_json;
 
 use std::path::Path;
+use tempdir::TempDir;
+use tantivy::Index;
+use tantivy::schema::*;
 use tantivy::collector::TopCollector;
 use tantivy::query::QueryParser;
-use tantivy::schema::*;
-use tantivy::Index;
-use tempdir::TempDir;
 
 fn main() {
     // Let's create a temporary directory for the

src/collector/chained_collector.rs

@@ -1,9 +1,9 @@
-use collector::Collector;
-use DocId;
 use Result;
-use Score;
+use collector::Collector;
 use SegmentLocalId;
 use SegmentReader;
+use DocId;
+use Score;
 
 /// Collector that does nothing.
 /// This is used in the chain Collector and will hopefully

src/collector/count_collector.rs

@@ -1,9 +1,9 @@
 use super::Collector;
 use DocId;
-use Result;
 use Score;
-use SegmentLocalId;
+use Result;
 use SegmentReader;
+use SegmentLocalId;
 
 /// `CountCollector` collector only counts how many
 /// documents match the query.

src/collector/facet_collector.rs

@@ -1,25 +1,27 @@
+use std::mem;
 use collector::Collector;
-use docset::SkipResult;
 use fastfield::FacetReader;
-use schema::Facet;
 use schema::Field;
 use std::cell::UnsafeCell;
-use std::collections::btree_map;
+use schema::Facet;
 use std::collections::BTreeMap;
-use std::collections::BTreeSet;
 use std::collections::BinaryHeap;
 use std::collections::Bound;
-use std::iter::Peekable;
-use std::mem;
-use std::{u64, usize};
+use termdict::TermDictionary;
+use termdict::TermStreamer;
+use termdict::TermStreamerBuilder;
+use std::collections::BTreeSet;
 use termdict::TermMerger;
+use docset::SkipResult;
+use std::{usize, u64};
+use std::iter::Peekable;
 
-use std::cmp::Ordering;
 use DocId;
 use Result;
 use Score;
-use SegmentLocalId;
 use SegmentReader;
+use SegmentLocalId;
+use std::cmp::Ordering;
 
 struct Hit<'a> {
     count: u64,
@@ -379,10 +381,8 @@ impl FacetCollector {
                 })
                 .sum();
             if count > 0u64 {
-                let bytes: Vec<u8> = facet_merger.key().to_owned();
-                // may create an corrupted facet if the term dicitonary is corrupted
-                let facet = unsafe { Facet::from_encoded(bytes) };
-                facet_counts.insert(facet, count);
+                let bytes = facet_merger.key().to_owned();
+                facet_counts.insert(Facet::from_encoded(bytes), count);
             }
         }
         FacetCounts { facet_counts }
@@ -432,20 +432,9 @@ pub struct FacetCounts {
     facet_counts: BTreeMap<Facet, u64>,
 }
 
-pub struct FacetChildIterator<'a> {
-    underlying: btree_map::Range<'a, Facet, u64>,
-}
-
-impl<'a> Iterator for FacetChildIterator<'a> {
-    type Item = (&'a Facet, u64);
-
-    fn next(&mut self) -> Option<Self::Item> {
-        self.underlying.next().map(|(facet, count)| (facet, *count))
-    }
-}
-
 impl FacetCounts {
-    pub fn get<T>(&self, facet_from: T) -> FacetChildIterator
+    #[allow(needless_lifetimes)] //< compiler fails if we remove the lifetime
+    pub fn get<'a, T>(&'a self, facet_from: T) -> impl Iterator<Item = (&'a Facet, u64)>
     where
         Facet: From<T>,
     {
@@ -454,13 +443,15 @@ impl FacetCounts {
         let right_bound = if facet.is_root() {
             Bound::Unbounded
         } else {
-            let mut facet_after_bytes: Vec<u8> = facet.encoded_bytes().to_owned();
+            let mut facet_after_bytes = facet.encoded_bytes().to_owned();
             facet_after_bytes.push(1u8);
-            let facet_after = unsafe { Facet::from_encoded(facet_after_bytes) }; // ok logic
+            let facet_after = Facet::from_encoded(facet_after_bytes);
             Bound::Excluded(facet_after)
         };
-        let underlying: btree_map::Range<_, _> = self.facet_counts.range((left_bound, right_bound));
-        FacetChildIterator { underlying }
+
+        self.facet_counts
+            .range((left_bound, right_bound))
+            .map(|(facet, count)| (facet, *count))
     }
 
     pub fn top_k<T>(&self, facet: T, k: usize) -> Vec<(&Facet, u64)>
@@ -492,13 +483,14 @@ impl FacetCounts {
 
 #[cfg(test)]
 mod tests {
-    use super::{FacetCollector, FacetCounts};
+    use test::Bencher;
     use core::Index;
-    use query::AllQuery;
-    use rand::{thread_rng, Rng};
-    use schema::Field;
     use schema::{Document, Facet, SchemaBuilder};
+    use query::AllQuery;
+    use super::{FacetCollector, FacetCounts};
     use std::iter;
+    use schema::Field;
+    use rand::{thread_rng, Rng};
 
     #[test]
     fn test_facet_collector_drilldown() {
@@ -553,10 +545,8 @@ mod tests {
     }
 
     #[test]
-    #[should_panic(
-        expected = "Tried to add a facet which is a descendant of \
-                    an already added facet."
-    )]
+    #[should_panic(expected = "Tried to add a facet which is a descendant of \
+                               an already added facet.")]
     fn test_misused_facet_collector() {
         let mut facet_collector = FacetCollector::for_field(Field(0));
         facet_collector.add_facet(Facet::from("/country"));
@@ -614,19 +604,6 @@ mod tests {
         }
     }
 
-}
-
-#[cfg(all(test, feature = "unstable"))]
-mod bench {
-
-    use collector::FacetCollector;
-    use query::AllQuery;
-    use rand::{thread_rng, Rng};
-    use schema::Facet;
-    use schema::SchemaBuilder;
-    use test::Bencher;
-    use Index;
-
     #[bench]
     fn bench_facet_collector(b: &mut Bencher) {
         let mut schema_builder = SchemaBuilder::new();

src/collector/mod.rs

@@ -2,11 +2,11 @@
 Defines how the documents matching a search query should be processed.
 */
 
-use DocId;
-use Result;
-use Score;
-use SegmentLocalId;
 use SegmentReader;
+use SegmentLocalId;
+use DocId;
+use Score;
+use Result;
 
 mod count_collector;
 pub use self::count_collector::CountCollector;
@@ -89,13 +89,13 @@ impl<'a, C: Collector> Collector for &'a mut C {
 pub mod tests {
 
     use super::*;
-    use core::SegmentReader;
-    use fastfield::BytesFastFieldReader;
-    use fastfield::FastFieldReader;
-    use schema::Field;
+    use test::Bencher;
     use DocId;
     use Score;
+    use core::SegmentReader;
     use SegmentLocalId;
+    use fastfield::FastFieldReader;
+    use schema::Field;
 
     /// Stores all of the doc ids.
     /// This collector is only used for tests.
@@ -186,52 +186,6 @@ pub mod tests {
         }
     }
 
-    /// Collects in order all of the fast field bytes for all of the
-    /// docs in the `DocSet`
-    ///
-    /// This collector is mainly useful for tests.
-    pub struct BytesFastFieldTestCollector {
-        vals: Vec<u8>,
-        field: Field,
-        ff_reader: Option<BytesFastFieldReader>,
-    }
-
-    impl BytesFastFieldTestCollector {
-        pub fn for_field(field: Field) -> BytesFastFieldTestCollector {
-            BytesFastFieldTestCollector {
-                vals: Vec::new(),
-                field,
-                ff_reader: None,
-            }
-        }
-
-        pub fn vals(self) -> Vec<u8> {
-            self.vals
-        }
-    }
-
-    impl Collector for BytesFastFieldTestCollector {
-        fn set_segment(&mut self, _segment_local_id: u32, segment: &SegmentReader) -> Result<()> {
-            self.ff_reader = Some(segment.bytes_fast_field_reader(self.field)?);
-            Ok(())
-        }
-
-        fn collect(&mut self, doc: u32, _score: f32) {
-            let val = self.ff_reader.as_ref().unwrap().get_val(doc);
-            self.vals.extend(val);
-        }
-
-        fn requires_scoring(&self) -> bool {
-            false
-        }
-    }
-}
-
-#[cfg(all(test, feature = "unstable"))]
-mod bench {
-    use collector::{Collector, CountCollector};
-    use test::Bencher;
-
     #[bench]
     fn build_collector(b: &mut Bencher) {
         b.iter(|| {

src/collector/multi_collector.rs

@@ -1,9 +1,9 @@
 use super::Collector;
 use DocId;
-use Result;
 use Score;
-use SegmentLocalId;
+use Result;
 use SegmentReader;
+use SegmentLocalId;
 
 /// Multicollector makes it possible to collect on more than one collector.
 /// It should only be used for use cases where the Collector types is unknown

src/collector/top_collector.rs

@@ -1,12 +1,12 @@
 use super::Collector;
-use std::cmp::Ordering;
-use std::collections::BinaryHeap;
-use DocAddress;
-use DocId;
-use Result;
-use Score;
-use SegmentLocalId;
 use SegmentReader;
+use SegmentLocalId;
+use DocAddress;
+use Result;
+use std::collections::BinaryHeap;
+use std::cmp::Ordering;
+use DocId;
+use Score;
 
 // Rust heap is a max-heap and we need a min heap.
 #[derive(Clone, Copy)]
@@ -135,9 +135,9 @@ impl Collector for TopCollector {
 mod tests {
 
     use super::*;
-    use collector::Collector;
     use DocId;
     use Score;
+    use collector::Collector;
 
     #[test]
     fn test_top_collector_not_at_capacity() {

src/common/bitpacker.rs

@@ -1,6 +1,6 @@
-use common::serialize::BinarySerializable;
-use std::io;
 use std::io::Write;
+use std::io;
+use common::serialize::BinarySerializable;
 use std::mem;
 use std::ops::Deref;
 use std::ptr;
@@ -106,8 +106,7 @@ where
                 addr + 8 <= data.len(),
                 "The fast field field should have been padded with 7 bytes."
             );
-            let val_unshifted_unmasked: u64 =
-                unsafe { ptr::read_unaligned(data[addr..].as_ptr() as *const u64) };
+            let val_unshifted_unmasked: u64 = unsafe { ptr::read_unaligned(data[addr..].as_ptr() as *const u64) };
             let val_shifted = (val_unshifted_unmasked >> bit_shift) as u64;
             val_shifted & mask
         } else {
@@ -142,8 +141,7 @@ where
             for output_val in output.iter_mut() {
                 let addr = addr_in_bits >> 3;
                 let bit_shift = addr_in_bits & 7;
-                let val_unshifted_unmasked: u64 =
-                    unsafe { ptr::read_unaligned(data[addr..].as_ptr() as *const u64) };
+                let val_unshifted_unmasked: u64 = unsafe { ptr::read_unaligned(data[addr..].as_ptr() as *const u64) };
                 let val_shifted = (val_unshifted_unmasked >> bit_shift) as u64;
                 *output_val = val_shifted & mask;
                 addr_in_bits += num_bits;

src/common/bitset.rs

@@ -202,14 +202,15 @@ impl BitSet {
 #[cfg(test)]
 mod tests {
 
+    extern crate test;
+    use tests;
+    use std::collections::HashSet;
     use super::BitSet;
     use super::TinySet;
-    use docset::DocSet;
-    use query::BitSetDocSet;
-    use std::collections::BTreeSet;
-    use std::collections::HashSet;
-    use tests;
     use tests::generate_nonunique_unsorted;
+    use std::collections::BTreeSet;
+    use query::BitSetDocSet;
+    use docset::DocSet;
 
     #[test]
     fn test_tiny_set() {
@@ -352,14 +353,6 @@ mod tests {
             assert!(!bitset.contains(el));
         }
     }
-}
-
-#[cfg(all(test, feature = "unstable"))]
-mod bench {
-
-    use super::BitSet;
-    use super::TinySet;
-    use test;
 
     #[bench]
     fn bench_tinyset_pop(b: &mut test::Bencher) {
@@ -392,4 +385,5 @@ mod bench {
     fn bench_bitset_initialize(b: &mut test::Bencher) {
         b.iter(|| BitSet::with_max_value(1_000_000));
     }
+
 }

src/common/composite_file.rs

@@ -1,12 +1,12 @@
-use common::BinarySerializable;
-use common::CountingWriter;
-use common::VInt;
-use directory::ReadOnlySource;
-use directory::WritePtr;
-use schema::Field;
-use std::collections::HashMap;
 use std::io::Write;
+use common::CountingWriter;
+use std::collections::HashMap;
+use schema::Field;
+use common::VInt;
+use directory::WritePtr;
 use std::io::{self, Read};
+use directory::ReadOnlySource;
+use common::BinarySerializable;
 
 #[derive(Eq, PartialEq, Hash, Copy, Ord, PartialOrd, Clone, Debug)]
 pub struct FileAddr {
@@ -30,7 +30,10 @@ impl BinarySerializable for FileAddr {
     fn deserialize<R: Read>(reader: &mut R) -> io::Result<Self> {
         let field = Field::deserialize(reader)?;
         let idx = VInt::deserialize(reader)?.0 as usize;
-        Ok(FileAddr { field, idx })
+        Ok(FileAddr {
+            field,
+            idx,
+        })
     }
 }
 
@@ -163,7 +166,7 @@ impl CompositeFile {
     /// to a given `Field` and stored in a `CompositeFile`.
     pub fn open_read_with_idx(&self, field: Field, idx: usize) -> Option<ReadOnlySource> {
         self.offsets_index
-            .get(&FileAddr { field, idx })
+            .get(&FileAddr { field, idx, })
             .map(|&(from, to)| self.data.slice(from, to))
     }
 }
@@ -171,12 +174,12 @@ impl CompositeFile {
 #[cfg(test)]
 mod test {
 
+    use std::io::Write;
     use super::{CompositeFile, CompositeWrite};
-    use common::BinarySerializable;
-    use common::VInt;
     use directory::{Directory, RAMDirectory};
     use schema::Field;
-    use std::io::Write;
+    use common::VInt;
+    use common::BinarySerializable;
     use std::path::Path;
 
     #[test]

src/common/counting_writer.rs

@@ -1,5 +1,5 @@
-use std::io;
 use std::io::Write;
+use std::io;
 
 pub struct CountingWriter<W> {
     underlying: W,

src/common/mod.rs

@@ -1,16 +1,16 @@
+    mod serialize;
+mod vint;
+mod counting_writer;
+mod composite_file;
 pub mod bitpacker;
 mod bitset;
-mod composite_file;
-mod counting_writer;
-mod serialize;
-mod vint;
 
-pub use self::bitset::BitSet;
-pub(crate) use self::bitset::TinySet;
 pub(crate) use self::composite_file::{CompositeFile, CompositeWrite};
-pub use self::counting_writer::CountingWriter;
 pub use self::serialize::{BinarySerializable, FixedSize};
 pub use self::vint::VInt;
+pub use self::counting_writer::CountingWriter;
+pub use self::bitset::BitSet;
+pub(crate) use self::bitset::TinySet;
 pub use byteorder::LittleEndian as Endianness;
 
 use std::io;
@@ -104,8 +104,8 @@ pub fn u64_to_i64(val: u64) -> i64 {
 #[cfg(test)]
 pub(crate) mod test {
 
-    pub use super::serialize::test::fixed_size_test;
     use super::{compute_num_bits, i64_to_u64, u64_to_i64};
+    pub use super::serialize::test::fixed_size_test;
 
     fn test_i64_converter_helper(val: i64) {
         assert_eq!(u64_to_i64(i64_to_u64(val)), val);

src/common/serialize.rs

@@ -1,10 +1,10 @@
 use byteorder::{ReadBytesExt, WriteBytesExt};
 use common::Endianness;
-use common::VInt;
 use std::fmt;
-use std::io;
-use std::io::Read;
 use std::io::Write;
+use std::io::Read;
+use std::io;
+use common::VInt;
 
 /// Trait for a simple binary serialization.
 pub trait BinarySerializable: fmt::Debug + Sized {
@@ -135,8 +135,8 @@ impl BinarySerializable for String {
 #[cfg(test)]
 pub mod test {
 
-    use super::*;
     use common::VInt;
+    use super::*;
 
     pub fn fixed_size_test<O: BinarySerializable + FixedSize + Default>() {
         let mut buffer = Vec::new();

src/common/vint.rs

@@ -1,7 +1,7 @@
 use super::BinarySerializable;
 use std::io;
-use std::io::Read;
 use std::io::Write;
+use std::io::Read;
 
 ///   Wrapper over a `u64` that serializes as a variable int.
 #[derive(Debug, Eq, PartialEq)]

src/compression/mod.rs

@@ -8,8 +8,10 @@ const COMPRESSED_BLOCK_MAX_SIZE: usize = COMPRESSION_BLOCK_SIZE * 4 + 1;
 
 pub use self::stream::CompressedIntStream;
 
+
 use bitpacking::{BitPacker, BitPacker4x};
 
+
 /// Returns the size in bytes of a compressed block, given `num_bits`.
 pub fn compressed_block_size(num_bits: u8) -> usize {
     1 + (num_bits as usize) * COMPRESSION_BLOCK_SIZE / 8
@@ -33,21 +35,19 @@ impl BlockEncoder {
     pub fn compress_block_sorted(&mut self, block: &[u32], offset: u32) -> &[u8] {
         let num_bits = self.bitpacker.num_bits_sorted(offset, block);
         self.output[0] = num_bits;
-        let written_size =
-            1 + self.bitpacker
-                .compress_sorted(offset, block, &mut self.output[1..], num_bits);
+        let written_size = 1 + self.bitpacker.compress_sorted(offset, block, &mut self.output[1..], num_bits);
         &self.output[..written_size]
     }
 
     pub fn compress_block_unsorted(&mut self, block: &[u32]) -> &[u8] {
         let num_bits = self.bitpacker.num_bits(block);
         self.output[0] = num_bits;
-        let written_size = 1 + self.bitpacker
-            .compress(block, &mut self.output[1..], num_bits);
+        let written_size = 1 + self.bitpacker.compress(block, &mut self.output[1..], num_bits);
         &self.output[..written_size]
     }
 }
 
+
 pub struct BlockDecoder {
     bitpacker: BitPacker4x,
     pub output: [u32; COMPRESSION_BLOCK_SIZE + 1],
@@ -68,23 +68,17 @@ impl BlockDecoder {
             output_len: 0,
         }
     }
-
+    
     pub fn uncompress_block_sorted(&mut self, compressed_data: &[u8], offset: u32) -> usize {
         let num_bits = compressed_data[0];
         self.output_len = COMPRESSION_BLOCK_SIZE;
-        1 + self.bitpacker.decompress_sorted(
-            offset,
-            &compressed_data[1..],
-            &mut self.output,
-            num_bits,
-        )
+        1 + self.bitpacker.decompress_sorted(offset, &compressed_data[1..], &mut self.output, num_bits)
     }
 
     pub fn uncompress_block_unsorted<'a>(&mut self, compressed_data: &'a [u8]) -> usize {
         let num_bits = compressed_data[0];
         self.output_len = COMPRESSION_BLOCK_SIZE;
-        1 + self.bitpacker
-            .decompress(&compressed_data[1..], &mut self.output, num_bits)
+        1 + self.bitpacker.decompress(&compressed_data[1..], &mut self.output, num_bits)
     }
 
     #[inline]
@@ -180,6 +174,8 @@ impl VIntDecoder for BlockDecoder {
 pub mod tests {
 
     use super::*;
+    use tests;
+    use test::Bencher;
 
     #[test]
     fn test_encode_sorted_block() {
@@ -268,34 +264,11 @@ pub mod tests {
             }
         }
     }
-}
-
-#[cfg(all(test, feature = "unstable"))]
-mod bench {
-
-    use super::*;
-    use rand::Rng;
-    use rand::SeedableRng;
-    use rand::XorShiftRng;
-    use test::Bencher;
-
-    fn generate_array_with_seed(n: usize, ratio: f32, seed_val: u32) -> Vec<u32> {
-        let seed: &[u32; 4] = &[1, 2, 3, seed_val];
-        let mut rng: XorShiftRng = XorShiftRng::from_seed(*seed);
-        (0..u32::max_value())
-            .filter(|_| rng.next_f32() < ratio)
-            .take(n)
-            .collect()
-    }
-
-    pub fn generate_array(n: usize, ratio: f32) -> Vec<u32> {
-        generate_array_with_seed(n, ratio, 4)
-    }
 
     #[bench]
     fn bench_compress(b: &mut Bencher) {
         let mut encoder = BlockEncoder::new();
-        let data = generate_array(COMPRESSION_BLOCK_SIZE, 0.1);
+        let data = tests::generate_array(COMPRESSION_BLOCK_SIZE, 0.1);
         b.iter(|| {
             encoder.compress_block_sorted(&data, 0u32);
         });
@@ -304,7 +277,7 @@ mod bench {
     #[bench]
     fn bench_uncompress(b: &mut Bencher) {
         let mut encoder = BlockEncoder::new();
-        let data = generate_array(COMPRESSION_BLOCK_SIZE, 0.1);
+        let data = tests::generate_array(COMPRESSION_BLOCK_SIZE, 0.1);
         let compressed = encoder.compress_block_sorted(&data, 0u32);
         let mut decoder = BlockDecoder::new();
         b.iter(|| {
@@ -331,7 +304,7 @@ mod bench {
     #[bench]
     fn bench_compress_vint(b: &mut Bencher) {
         let mut encoder = BlockEncoder::new();
-        let data = generate_array(NUM_INTS_BENCH_VINT, 0.001);
+        let data = tests::generate_array(NUM_INTS_BENCH_VINT, 0.001);
         b.iter(|| {
             encoder.compress_vint_sorted(&data, 0u32);
         });
@@ -340,11 +313,12 @@ mod bench {
     #[bench]
     fn bench_uncompress_vint(b: &mut Bencher) {
         let mut encoder = BlockEncoder::new();
-        let data = generate_array(NUM_INTS_BENCH_VINT, 0.001);
+        let data = tests::generate_array(NUM_INTS_BENCH_VINT, 0.001);
         let compressed = encoder.compress_vint_sorted(&data, 0u32);
         let mut decoder = BlockDecoder::new();
         b.iter(|| {
             decoder.uncompress_vint_sorted(compressed, 0u32, NUM_INTS_BENCH_VINT);
         });
     }
+
 }

src/compression/stream.rs

@@ -1,6 +1,6 @@
-use compression::compressed_block_size;
 use compression::BlockDecoder;
 use compression::COMPRESSION_BLOCK_SIZE;
+use compression::compressed_block_size;
 use directory::{ReadOnlySource, SourceRead};
 
 /// Reads a stream of compressed ints.
@@ -13,7 +13,7 @@ pub struct CompressedIntStream {
     buffer: SourceRead,
 
     block_decoder: BlockDecoder,
-    cached_addr: usize,      // address of the currently decoded block
+    cached_addr: usize, // address of the currently decoded block
     cached_next_addr: usize, // address following the currently decoded block
 
     addr: usize, // address of the block associated to the current position
@@ -42,8 +42,7 @@ impl CompressedIntStream {
             // no need to read.
             self.cached_next_addr
         } else {
-            let next_addr = addr + self.block_decoder
-                .uncompress_block_unsorted(self.buffer.slice_from(addr));
+            let next_addr = addr + self.block_decoder.uncompress_block_unsorted(self.buffer.slice_from(addr));
             self.cached_addr = addr;
             self.cached_next_addr = next_addr;
             next_addr
@@ -102,8 +101,8 @@ pub mod tests {
 
     use super::CompressedIntStream;
     use compression::compressed_block_size;
-    use compression::BlockEncoder;
     use compression::COMPRESSION_BLOCK_SIZE;
+    use compression::BlockEncoder;
     use directory::ReadOnlySource;
 
     fn create_stream_buffer() -> ReadOnlySource {

src/core/index.rs

@@ -1,32 +1,34 @@
-use core::SegmentId;
+use Result;
 use error::{ErrorKind, ResultExt};
-use schema::Schema;
 use serde_json;
+use schema::Schema;
+use std::sync::Arc;
 use std::borrow::BorrowMut;
 use std::fmt;
-use std::sync::Arc;
-use Result;
+use core::SegmentId;
 
-use super::pool::LeasedItem;
-use super::pool::Pool;
-use super::segment::create_segment;
-use super::segment::Segment;
-use core::searcher::Searcher;
-use core::IndexMeta;
-use core::SegmentMeta;
-use core::SegmentReader;
-use core::META_FILEPATH;
-use directory::ManagedDirectory;
-#[cfg(feature = "mmap")]
+
+#[cfg(feature="mmap")]
 use directory::MmapDirectory;
 use directory::{Directory, RAMDirectory};
 use indexer::index_writer::open_index_writer;
-use indexer::segment_updater::save_new_metas;
-use indexer::DirectoryLock;
+use core::searcher::Searcher;
+use std::convert::From;
 use num_cpus;
+use super::segment::Segment;
+use core::SegmentReader;
+use super::pool::Pool;
+use core::SegmentMeta;
+use super::pool::LeasedItem;
 use std::path::Path;
-use tokenizer::TokenizerManager;
+use core::IndexMeta;
+use indexer::DirectoryLock;
 use IndexWriter;
+use directory::ManagedDirectory;
+use core::META_FILEPATH;
+use super::segment::create_segment;
+use indexer::segment_updater::save_new_metas;
+use tokenizer::TokenizerManager;
 
 const NUM_SEARCHERS: usize = 12;
 
@@ -63,7 +65,7 @@ impl Index {
     /// The index will use the `MMapDirectory`.
     ///
     /// If a previous index was in this directory, then its meta file will be destroyed.
-    #[cfg(feature = "mmap")]
+    #[cfg(feature="mmap")]
     pub fn create<P: AsRef<Path>>(directory_path: P, schema: Schema) -> Result<Index> {
         let mmap_directory = MmapDirectory::open(directory_path)?;
         let directory = ManagedDirectory::new(mmap_directory)?;
@@ -83,7 +85,8 @@ impl Index {
     ///
     /// The temp directory is only used for testing the `MmapDirectory`.
     /// For other unit tests, prefer the `RAMDirectory`, see: `create_in_ram`.
-    #[cfg(feature = "mmap")]
+    #[cfg(feature="mmap")]
+    #[cfg(test)]
     pub fn create_from_tempdir(schema: Schema) -> Result<Index> {
         let mmap_directory = MmapDirectory::create_from_tempdir()?;
         let directory = ManagedDirectory::new(mmap_directory)?;
@@ -103,20 +106,6 @@ impl Index {
         Ok(index)
     }
 
-    /// Open the index using the provided directory
-    pub fn open_directory<D: Directory>(directory: D) -> Result<Index> {
-        let directory = ManagedDirectory::new(directory)?;
-        let metas = load_metas(&directory)?;
-        Index::create_from_metas(directory, &metas)
-    }
-
-    /// Opens a new directory from an index path.
-    #[cfg(feature = "mmap")]
-    pub fn open<P: AsRef<Path>>(directory_path: P) -> Result<Index> {
-        let mmap_directory = MmapDirectory::open(directory_path)?;
-        Index::open_directory(mmap_directory)
-    }
-
     /// Create a new index from a directory.
     pub fn from_directory(mut directory: ManagedDirectory, schema: Schema) -> Result<Index> {
         save_new_metas(schema.clone(), 0, directory.borrow_mut())?;
@@ -124,6 +113,22 @@ impl Index {
         Index::create_from_metas(directory, &metas)
     }
 
+    /// Opens a new directory from an index path.
+    #[cfg(feature="mmap")]
+    pub fn open<P: AsRef<Path>>(directory_path: P) -> Result<Index> {
+        let mmap_directory = MmapDirectory::open(directory_path)?;
+        let directory = ManagedDirectory::new(mmap_directory)?;
+        let metas = load_metas(&directory)?;
+        Index::create_from_metas(directory, &metas)
+    }
+
+    pub fn open_directory<TDirectory: Directory>(directory: TDirectory) -> Result<Index> {
+        let directory = ManagedDirectory::new(directory)?;
+        let metas = load_metas(&directory)?;
+        Index::create_from_metas(directory, &metas)
+    }
+
+
     /// Reads the index meta file from the directory.
     pub fn load_metas(&self) -> Result<IndexMeta> {
         load_metas(self.directory())
@@ -226,9 +231,8 @@ impl Index {
             .iter()
             .map(SegmentReader::open)
             .collect::<Result<_>>()?;
-        let schema = self.schema();
         let searchers = (0..NUM_SEARCHERS)
-            .map(|_| Searcher::new(schema.clone(), segment_readers.clone()))
+            .map(|_| Searcher::from(segment_readers.clone()))
             .collect();
         self.searcher_pool.publish_new_generation(searchers);
         Ok(())

src/core/index_meta.rs

@@ -1,7 +1,7 @@
-use core::SegmentMeta;
 use schema::Schema;
-use serde_json;
+use core::SegmentMeta;
 use std::fmt;
+use serde_json;
 
 /// Meta information about the `Index`.
 ///
@@ -45,9 +45,9 @@ impl fmt::Debug for IndexMeta {
 #[cfg(test)]
 mod tests {
 
+    use serde_json;
     use super::IndexMeta;
     use schema::{SchemaBuilder, TEXT};
-    use serde_json;
 
     #[test]
     fn test_serialize_metas() {

src/core/inverted_index_reader.rs

@@ -1,13 +1,13 @@
-use common::BinarySerializable;
-use compression::CompressedIntStream;
 use directory::{ReadOnlySource, SourceRead};
-use postings::FreqReadingOption;
-use postings::TermInfo;
+use termdict::{TermDictionary, TermDictionaryImpl};
 use postings::{BlockSegmentPostings, SegmentPostings};
-use schema::FieldType;
+use postings::TermInfo;
 use schema::IndexRecordOption;
 use schema::Term;
-use termdict::TermDictionary;
+use compression::CompressedIntStream;
+use postings::FreqReadingOption;
+use common::BinarySerializable;
+use schema::FieldType;
 
 /// The inverted index reader is in charge of accessing
 /// the inverted index associated to a specific field.
@@ -23,16 +23,16 @@ use termdict::TermDictionary;
 /// `InvertedIndexReader` are created by calling
 /// the `SegmentReader`'s [`.inverted_index(...)`] method
 pub struct InvertedIndexReader {
-    termdict: TermDictionary,
+    termdict: TermDictionaryImpl,
     postings_source: ReadOnlySource,
     positions_source: ReadOnlySource,
     record_option: IndexRecordOption,
-    total_num_tokens: u64,
+    total_num_tokens: u64
 }
 
 impl InvertedIndexReader {
     pub(crate) fn new(
-        termdict: TermDictionary,
+        termdict: TermDictionaryImpl,
         postings_source: ReadOnlySource,
         positions_source: ReadOnlySource,
         record_option: IndexRecordOption,
@@ -45,7 +45,7 @@ impl InvertedIndexReader {
             postings_source: postings_source.slice_from(8),
             positions_source,
             record_option,
-            total_num_tokens,
+            total_num_tokens
         }
     }
 
@@ -56,11 +56,11 @@ impl InvertedIndexReader {
             .get_index_record_option()
             .unwrap_or(IndexRecordOption::Basic);
         InvertedIndexReader {
-            termdict: TermDictionary::empty(field_type),
+            termdict:    TermDictionaryImpl::empty(field_type),
             postings_source: ReadOnlySource::empty(),
             positions_source: ReadOnlySource::empty(),
             record_option,
-            total_num_tokens: 0u64,
+            total_num_tokens: 0u64
         }
     }
 
@@ -70,7 +70,7 @@ impl InvertedIndexReader {
     }
 
     /// Return the term dictionary datastructure.
-    pub fn terms(&self) -> &TermDictionary {
+    pub fn terms(&self) -> &TermDictionaryImpl {
         &self.termdict
     }
 
@@ -149,6 +149,8 @@ impl InvertedIndexReader {
         self.total_num_tokens
     }
 
+
+
     /// Returns the segment postings associated with the term, and with the given option,
     /// or `None` if the term has never been encountered and indexed.
     ///
@@ -164,15 +166,12 @@ impl InvertedIndexReader {
         Some(self.read_postings_from_terminfo(&term_info, option))
     }
 
-    pub(crate) fn read_postings_no_deletes(
-        &self,
-        term: &Term,
-        option: IndexRecordOption,
-    ) -> Option<SegmentPostings> {
+    pub(crate) fn read_postings_no_deletes(&self, term: &Term, option: IndexRecordOption) -> Option<SegmentPostings> {
         let term_info = get!(self.get_term_info(term));
         Some(self.read_postings_from_terminfo(&term_info, option))
     }
 
+
     /// Returns the number of documents containing the term.
     pub fn doc_freq(&self, term: &Term) -> u32 {
         self.get_term_info(term)
@@ -180,3 +179,6 @@ impl InvertedIndexReader {
             .unwrap_or(0u32)
     }
 }
+
+
+

src/core/mod.rs

@@ -1,24 +1,24 @@
-pub mod index;
-mod index_meta;
-mod inverted_index_reader;
-mod pool;
 pub mod searcher;
-mod segment;
-mod segment_component;
-mod segment_id;
-mod segment_meta;
+pub mod index;
 mod segment_reader;
+mod segment_id;
+mod segment_component;
+mod segment;
+mod index_meta;
+mod pool;
+mod segment_meta;
+mod inverted_index_reader;
 
-pub use self::index::Index;
-pub use self::index_meta::IndexMeta;
 pub use self::inverted_index_reader::InvertedIndexReader;
 pub use self::searcher::Searcher;
-pub use self::segment::Segment;
-pub use self::segment::SerializableSegment;
 pub use self::segment_component::SegmentComponent;
 pub use self::segment_id::SegmentId;
-pub use self::segment_meta::SegmentMeta;
 pub use self::segment_reader::SegmentReader;
+pub use self::segment::Segment;
+pub use self::segment::SerializableSegment;
+pub use self::index::Index;
+pub use self::segment_meta::SegmentMeta;
+pub use self::index_meta::IndexMeta;
 
 use std::path::PathBuf;
 

src/core/pool.rs

@@ -1,8 +1,8 @@
-use crossbeam::sync::MsQueue;
-use std::mem;
-use std::ops::{Deref, DerefMut};
 use std::sync::atomic::AtomicUsize;
 use std::sync::atomic::Ordering;
+use std::mem;
+use std::ops::{Deref, DerefMut};
+use crossbeam::sync::MsQueue;
 use std::sync::Arc;
 
 pub struct GenerationItem<T> {
@@ -114,8 +114,8 @@ impl<T> Drop for LeasedItem<T> {
 #[cfg(test)]
 mod tests {
 
-    use super::Pool;
     use std::iter;
+    use super::Pool;
 
     #[test]
     fn test_pool() {

src/core/searcher.rs

@@ -1,15 +1,14 @@
-use collector::Collector;
-use core::InvertedIndexReader;
-use core::SegmentReader;
-use query::Query;
-use schema::Document;
-use schema::Schema;
-use schema::{Field, Term};
-use std::fmt;
-use std::sync::Arc;
-use termdict::TermMerger;
-use DocAddress;
 use Result;
+use core::SegmentReader;
+use schema::Document;
+use collector::Collector;
+use query::Query;
+use DocAddress;
+use schema::{Field, Term};
+use termdict::{TermDictionary, TermMerger};
+use std::sync::Arc;
+use std::fmt;
+use core::InvertedIndexReader;
 
 /// Holds a list of `SegmentReader`s ready for search.
 ///
@@ -17,18 +16,10 @@ use Result;
 /// the destruction of the `Searcher`.
 ///
 pub struct Searcher {
-    schema: Schema,
     segment_readers: Vec<SegmentReader>,
 }
 
 impl Searcher {
-    /// Creates a new `Searcher`
-    pub(crate) fn new(schema: Schema, segment_readers: Vec<SegmentReader>) -> Searcher {
-        Searcher {
-            schema,
-            segment_readers,
-        }
-    }
     /// Fetches a document from tantivy's store given a `DocAddress`.
     ///
     /// The searcher uses the segment ordinal to route the
@@ -39,11 +30,6 @@ impl Searcher {
         segment_reader.doc(doc_id)
     }
 
-    /// Access the schema associated to the index of this searcher.
-    pub fn schema(&self) -> &Schema {
-        &self.schema
-    }
-
     /// Returns the overall number of documents in the index.
     pub fn num_docs(&self) -> u64 {
         self.segment_readers
@@ -106,6 +92,12 @@ impl FieldSearcher {
     }
 }
 
+impl From<Vec<SegmentReader>> for Searcher {
+    fn from(segment_readers: Vec<SegmentReader>) -> Searcher {
+        Searcher { segment_readers }
+    }
+}
+
 impl fmt::Debug for Searcher {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         let segment_ids = self.segment_readers

src/core/segment.rs

@@ -1,16 +1,16 @@
-use super::SegmentComponent;
-use core::Index;
-use core::SegmentId;
-use core::SegmentMeta;
-use directory::error::{OpenReadError, OpenWriteError};
-use directory::Directory;
-use directory::{FileProtection, ReadOnlySource, WritePtr};
-use indexer::segment_serializer::SegmentSerializer;
+use Result;
+use std::path::PathBuf;
 use schema::Schema;
 use std::fmt;
-use std::path::PathBuf;
+use core::SegmentId;
+use directory::{FileProtection, ReadOnlySource, WritePtr};
+use indexer::segment_serializer::SegmentSerializer;
+use super::SegmentComponent;
+use core::Index;
 use std::result;
-use Result;
+use directory::Directory;
+use core::SegmentMeta;
+use directory::error::{OpenReadError, OpenWriteError};
 
 /// A segment is a piece of the index.
 #[derive(Clone)]
@@ -111,8 +111,8 @@ mod tests {
 
     use core::SegmentComponent;
     use directory::Directory;
-    use schema::SchemaBuilder;
     use std::collections::HashSet;
+    use schema::SchemaBuilder;
     use Index;
 
     #[test]

src/core/segment_component.rs

@@ -1,5 +1,3 @@
-use std::slice;
-
 /// Enum describing each component of a tantivy segment.
 /// Each component is stored in its own file,
 /// using the pattern `segment_uuid`.`component_extension`,
@@ -28,7 +26,7 @@ pub enum SegmentComponent {
 
 impl SegmentComponent {
     /// Iterates through the components.
-    pub fn iterator() -> slice::Iter<'static, SegmentComponent> {
+    pub fn iterator() -> impl Iterator<Item = &'static SegmentComponent> {
         static SEGMENT_COMPONENTS: [SegmentComponent; 7] = [
             SegmentComponent::POSTINGS,
             SegmentComponent::POSITIONS,

src/core/segment_id.rs

@@ -1,6 +1,6 @@
-use std::cmp::{Ord, Ordering};
-use std::fmt;
 use uuid::Uuid;
+use std::fmt;
+use std::cmp::{Ord, Ordering};
 
 #[cfg(test)]
 use std::sync::atomic;

src/core/segment_meta.rs

@@ -1,7 +1,7 @@
-use super::SegmentComponent;
 use core::SegmentId;
-use std::collections::HashSet;
+use super::SegmentComponent;
 use std::path::PathBuf;
+use std::collections::HashSet;
 
 #[derive(Clone, Debug, Serialize, Deserialize)]
 struct DeleteMeta {

src/core/segment_reader.rs

@@ -1,30 +1,31 @@
-use common::CompositeFile;
-use common::HasLen;
-use core::InvertedIndexReader;
+use Result;
 use core::Segment;
-use core::SegmentComponent;
 use core::SegmentId;
+use core::SegmentComponent;
+use std::sync::RwLock;
+use common::HasLen;
 use core::SegmentMeta;
-use error::ErrorKind;
-use fastfield::DeleteBitSet;
-use fastfield::FacetReader;
-use fastfield::FastFieldReader;
 use fastfield::{self, FastFieldNotAvailableError};
-use fastfield::{BytesFastFieldReader, FastValue, MultiValueIntFastFieldReader};
-use fieldnorm::FieldNormReader;
-use schema::Cardinality;
+use fastfield::DeleteBitSet;
+use store::StoreReader;
 use schema::Document;
+use DocId;
+use std::sync::Arc;
+use std::collections::HashMap;
+use common::CompositeFile;
+use std::fmt;
+use core::InvertedIndexReader;
 use schema::Field;
 use schema::FieldType;
+use error::ErrorKind;
+use termdict::TermDictionaryImpl;
+use fastfield::FacetReader;
+use fastfield::FastFieldReader;
 use schema::Schema;
-use std::collections::HashMap;
-use std::fmt;
-use std::sync::Arc;
-use std::sync::RwLock;
-use store::StoreReader;
 use termdict::TermDictionary;
-use DocId;
-use Result;
+use fastfield::{FastValue, MultiValueIntFastFieldReader};
+use schema::Cardinality;
+use fieldnorm::FieldNormReader;
 
 /// Entry point to access all of the datastructures of the `Segment`
 ///
@@ -75,11 +76,6 @@ impl SegmentReader {
         self.segment_meta.num_docs()
     }
 
-    /// Returns the schema of the index this segment belongs to.
-    pub fn schema(&self) -> &Schema {
-        &self.schema
-    }
-
     /// Return the number of documents that have been
     /// deleted in the segment.
     pub fn num_deleted_docs(&self) -> DocId {
@@ -109,25 +105,12 @@ impl SegmentReader {
     ) -> fastfield::Result<FastFieldReader<Item>> {
         let field_entry = self.schema.get_field_entry(field);
         if Item::fast_field_cardinality(field_entry.field_type()) == Some(Cardinality::SingleValue)
-        {
-            self.fast_fields_composite
-                .open_read(field)
-                .ok_or_else(|| FastFieldNotAvailableError::new(field_entry))
-                .map(FastFieldReader::open)
-        } else {
-            Err(FastFieldNotAvailableError::new(field_entry))
-        }
-    }
-
-    pub(crate) fn fast_field_reader_with_idx<Item: FastValue>(
-        &self,
-        field: Field,
-        idx: usize,
-    ) -> fastfield::Result<FastFieldReader<Item>> {
-        if let Some(ff_source) = self.fast_fields_composite.open_read_with_idx(field, idx) {
-            Ok(FastFieldReader::open(ff_source))
-        } else {
-            let field_entry = self.schema.get_field_entry(field);
+            {
+                self.fast_fields_composite
+                    .open_read(field)
+                    .ok_or_else(|| FastFieldNotAvailableError::new(field_entry))
+                    .map(FastFieldReader::open)
+            } else {
             Err(FastFieldNotAvailableError::new(field_entry))
         }
     }
@@ -140,32 +123,21 @@ impl SegmentReader {
     ) -> fastfield::Result<MultiValueIntFastFieldReader<Item>> {
         let field_entry = self.schema.get_field_entry(field);
         if Item::fast_field_cardinality(field_entry.field_type()) == Some(Cardinality::MultiValues)
-        {
-            let idx_reader = self.fast_field_reader_with_idx(field, 0)?;
-            let vals_reader = self.fast_field_reader_with_idx(field, 1)?;
-            Ok(MultiValueIntFastFieldReader::open(idx_reader, vals_reader))
-        } else {
+            {
+                let idx_reader = self.fast_fields_composite
+                    .open_read_with_idx(field, 0)
+                    .ok_or_else(|| FastFieldNotAvailableError::new(field_entry))
+                    .map(FastFieldReader::open)?;
+                let vals_reader = self.fast_fields_composite
+                    .open_read_with_idx(field, 1)
+                    .ok_or_else(|| FastFieldNotAvailableError::new(field_entry))
+                    .map(FastFieldReader::open)?;
+                Ok(MultiValueIntFastFieldReader::open(idx_reader, vals_reader))
+            } else {
             Err(FastFieldNotAvailableError::new(field_entry))
         }
     }
 
-    /// Accessor to the `BytesFastFieldReader` associated to a given `Field`.
-    pub fn bytes_fast_field_reader(&self, field: Field) -> fastfield::Result<BytesFastFieldReader> {
-        let field_entry = self.schema.get_field_entry(field);
-        match field_entry.field_type() {
-            &FieldType::Bytes => {}
-            _ => return Err(FastFieldNotAvailableError::new(field_entry)),
-        }
-        let idx_reader = self.fast_fields_composite
-            .open_read_with_idx(field, 0)
-            .ok_or_else(|| FastFieldNotAvailableError::new(field_entry))
-            .map(FastFieldReader::open)?;
-        let values = self.fast_fields_composite
-            .open_read_with_idx(field, 1)
-            .ok_or_else(|| FastFieldNotAvailableError::new(field_entry))?;
-        Ok(BytesFastFieldReader::open(idx_reader, values))
-    }
-
     /// Accessor to the `FacetReader` associated to a given `Field`.
     pub fn facet_reader(&self, field: Field) -> Result<FacetReader> {
         let field_entry = self.schema.get_field_entry(field);
@@ -185,7 +157,7 @@ impl SegmentReader {
                 field_entry.name()
             ))
         })?;
-        let termdict = TermDictionary::from_source(termdict_source);
+        let termdict = TermDictionaryImpl::from_source(termdict_source);
         let facet_reader = FacetReader::new(term_ords_reader, termdict);
         Ok(facet_reader)
     }
@@ -199,14 +171,12 @@ impl SegmentReader {
     /// They are simply stored as a fast field, serialized in
     /// the `.fieldnorm` file of the segment.
     pub fn get_fieldnorms_reader(&self, field: Field) -> FieldNormReader {
-        if let Some(fieldnorm_source) = self.fieldnorms_composite.open_read(field) {
+        if let Some(fieldnorm_source) = self.fieldnorms_composite
+            .open_read(field) {
             FieldNormReader::open(fieldnorm_source)
         } else {
             let field_name = self.schema.get_field_name(field);
-            let err_msg = format!(
-                "Field norm not found for field {:?}. Was it market as indexed during indexing.",
-                field_name
-            );
+            let err_msg=  format!("Field norm not found for field {:?}. Was it market as indexed during indexing.", field_name);
             panic!(err_msg);
         }
     }
@@ -241,12 +211,13 @@ impl SegmentReader {
         let fieldnorms_data = segment.open_read(SegmentComponent::FIELDNORMS)?;
         let fieldnorms_composite = CompositeFile::open(&fieldnorms_data)?;
 
-        let delete_bitset_opt = if segment.meta().has_deletes() {
-            let delete_data = segment.open_read(SegmentComponent::DELETE)?;
-            Some(DeleteBitSet::open(delete_data))
-        } else {
-            None
-        };
+        let delete_bitset_opt =
+            if segment.meta().has_deletes() {
+                let delete_data = segment.open_read(SegmentComponent::DELETE)?;
+                Some(DeleteBitSet::open(delete_data))
+            } else {
+                None
+            };
 
         let schema = segment.schema();
         Ok(SegmentReader {
@@ -310,7 +281,7 @@ impl SegmentReader {
             .expect("Index corrupted. Failed to open field positions in composite file.");
 
         let inv_idx_reader = Arc::new(InvertedIndexReader::new(
-            TermDictionary::from_source(termdict_source),
+            TermDictionaryImpl::from_source(termdict_source),
             postings_source,
             positions_source,
             record_option,
@@ -352,11 +323,6 @@ impl SegmentReader {
             .map(|delete_set| delete_set.is_deleted(doc))
             .unwrap_or(false)
     }
-
-    /// Returns an iterator that will iterate over the alive document ids
-    pub fn doc_ids_alive(&self) -> SegmentReaderAliveDocsIterator {
-        SegmentReaderAliveDocsIterator::new(&self)
-    }
 }
 
 impl fmt::Debug for SegmentReader {
@@ -364,90 +330,3 @@ impl fmt::Debug for SegmentReader {
         write!(f, "SegmentReader({:?})", self.segment_id)
     }
 }
-
-/// Implements the iterator trait to allow easy iteration
-/// over non-deleted ("alive") DocIds in a SegmentReader
-pub struct SegmentReaderAliveDocsIterator<'a> {
-    reader: &'a SegmentReader,
-    max_doc: DocId,
-    current: DocId,
-}
-
-impl<'a> SegmentReaderAliveDocsIterator<'a> {
-    pub fn new(reader: &'a SegmentReader) -> SegmentReaderAliveDocsIterator<'a> {
-        SegmentReaderAliveDocsIterator {
-            reader: reader,
-            max_doc: reader.max_doc(),
-            current: 0,
-        }
-    }
-}
-
-impl<'a> Iterator for SegmentReaderAliveDocsIterator<'a> {
-    type Item = DocId;
-
-    fn next(&mut self) -> Option<Self::Item> {
-        // TODO: Use TinySet (like in BitSetDocSet) to speed this process up
-        if self.current >= self.max_doc {
-            return None;
-        }
-
-        // find the next alive doc id
-        while self.reader.is_deleted(self.current) {
-            self.current += 1;
-
-            if self.current >= self.max_doc {
-                return None;
-            }
-        }
-
-        // capture the current alive DocId
-        let result = Some(self.current);
-
-        // move down the chain
-        self.current += 1;
-
-        result
-    }
-}
-
-#[cfg(test)]
-mod test {
-    use core::Index;
-    use DocId;
-    use schema::{TEXT, STORED, Term, SchemaBuilder};
-
-    #[test]
-    fn test_alive_docs_iterator() {
-        let mut schema_builder = SchemaBuilder::new();
-        schema_builder.add_text_field("name", TEXT | STORED);
-        let schema = schema_builder.build();
-        let index = Index::create_in_ram(schema.clone());
-        let name = schema.get_field("name").unwrap();
-
-        {
-            let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
-            index_writer.add_document(doc!(name => "tantivy"));
-            index_writer.add_document(doc!(name => "horse"));
-            index_writer.add_document(doc!(name => "jockey"));
-            index_writer.add_document(doc!(name => "cap"));
-
-            // we should now have one segment with two docs
-            index_writer.commit().unwrap();
-        }
-
-        {
-            let mut index_writer2 = index.writer(50_000_000).unwrap();
-            index_writer2.delete_term(Term::from_field_text(name, "horse"));
-            index_writer2.delete_term(Term::from_field_text(name, "cap"));
-
-            // ok, now we should have a deleted doc
-            index_writer2.commit().unwrap();
-        }
-
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
-        let docs: Vec<DocId> = searcher.segment_reader(0).doc_ids_alive().collect();
-        assert_eq!(vec![0u32, 2u32],  docs);
-    }
-}

src/datastruct/skip/mod.rs

@@ -1,10 +1,10 @@
 #![allow(dead_code)]
 
-mod skiplist;
 mod skiplist_builder;
+mod skiplist;
 
-pub use self::skiplist::SkipList;
 pub use self::skiplist_builder::SkipListBuilder;
+pub use self::skiplist::SkipList;
 
 #[cfg(test)]
 mod tests {

src/datastruct/skip/skiplist.rs

@@ -1,6 +1,6 @@
 use common::{BinarySerializable, VInt};
-use std::cmp::max;
 use std::marker::PhantomData;
+use std::cmp::max;
 
 static EMPTY: [u8; 0] = [];
 

src/datastruct/skip/skiplist_builder.rs

@@ -1,7 +1,7 @@
-use common::{is_power_of_2, BinarySerializable, VInt};
-use std::io;
 use std::io::Write;
+use common::{BinarySerializable, VInt, is_power_of_2};
 use std::marker::PhantomData;
+use std::io;
 
 struct LayerBuilder<T: BinarySerializable> {
     period_mask: usize,

src/datastruct/stacker/expull.rs

@@ -1,5 +1,5 @@
-use super::heap::{Heap, HeapAllocable};
 use std::mem;
+use super::heap::{Heap, HeapAllocable};
 
 #[inline]
 pub fn is_power_of_2(val: u32) -> bool {
@@ -99,8 +99,12 @@ impl<'a> Iterator for ExpUnrolledLinkedListIterator<'a> {
 #[cfg(test)]
 mod tests {
 
-    use super::super::heap::Heap;
     use super::*;
+    use super::super::heap::Heap;
+    use test::Bencher;
+
+    const NUM_STACK: usize = 10_000;
+    const STACK_SIZE: u32 = 1000;
 
     #[test]
     fn test_stack() {
@@ -120,17 +124,6 @@ mod tests {
         }
     }
 
-}
-
-#[cfg(all(test, feature = "unstable"))]
-mod bench {
-    use super::ExpUnrolledLinkedList;
-    use super::Heap;
-    use test::Bencher;
-
-    const NUM_STACK: usize = 10_000;
-    const STACK_SIZE: u32 = 1000;
-
     #[bench]
     fn bench_push_vec(bench: &mut Bencher) {
         bench.iter(|| {

src/datastruct/stacker/hashmap.rs

@@ -1,53 +1,54 @@
-use super::heap::{BytesRef, Heap, HeapAllocable};
-use postings::UnorderedTermId;
 use std::iter;
 use std::mem;
-use std::slice;
+use postings::UnorderedTermId;
+use super::heap::{BytesRef, Heap, HeapAllocable};
 
 mod murmurhash2 {
 
     const SEED: u32 = 3_242_157_231u32;
-    const M: u32 = 0x5bd1_e995;
 
     #[inline(always)]
     pub fn murmurhash2(key: &[u8]) -> u32 {
         let mut key_ptr: *const u32 = key.as_ptr() as *const u32;
+        let m: u32 = 0x5bd1_e995;
+        let r = 24;
         let len = key.len() as u32;
-        let mut h: u32 = SEED ^ len;
 
+        let mut h: u32 = SEED ^ len;
         let num_blocks = len >> 2;
         for _ in 0..num_blocks {
-            let mut k: u32 = unsafe { *key_ptr }; // ok because of num_blocks definition
-            k = k.wrapping_mul(M);
-            k ^= k >> 24;
-            k = k.wrapping_mul(M);
-            h = h.wrapping_mul(M);
+            let mut k: u32 = unsafe { *key_ptr };
+            k = k.wrapping_mul(m);
+            k ^= k >> r;
+            k = k.wrapping_mul(m);
+            k = k.wrapping_mul(m);
             h ^= k;
             key_ptr = key_ptr.wrapping_offset(1);
         }
 
         // Handle the last few bytes of the input array
-        let remaining: &[u8] = &key[key.len() & !3..];
-        match remaining.len() {
+        let remaining = len & 3;
+        let key_ptr_u8: *const u8 = key_ptr as *const u8;
+        match remaining {
             3 => {
-                h ^= u32::from(remaining[2]) << 16;
-                h ^= u32::from(remaining[1]) << 8;
-                h ^= u32::from(remaining[0]);
-                h = h.wrapping_mul(M);
+                h ^= unsafe { u32::from(*key_ptr_u8.wrapping_offset(2)) } << 16;
+                h ^= unsafe { u32::from(*key_ptr_u8.wrapping_offset(1)) } << 8;
+                h ^= unsafe { u32::from(*key_ptr_u8) };
+                h = h.wrapping_mul(m);
             }
             2 => {
-                h ^= u32::from(remaining[1]) << 8;
-                h ^= u32::from(remaining[0]);
-                h = h.wrapping_mul(M);
+                h ^= unsafe { u32::from(*key_ptr_u8.wrapping_offset(1)) } << 8;
+                h ^= unsafe { u32::from(*key_ptr_u8) };
+                h = h.wrapping_mul(m);
             }
             1 => {
-                h ^= u32::from(remaining[0]);
-                h = h.wrapping_mul(M);
+                h ^= unsafe { u32::from(*key_ptr_u8) };
+                h = h.wrapping_mul(m);
             }
             _ => {}
         }
         h ^= h >> 13;
-        h = h.wrapping_mul(M);
+        h = h.wrapping_mul(m);
         h ^ (h >> 15)
     }
 }
@@ -116,7 +117,11 @@ struct QuadraticProbing {
 
 impl QuadraticProbing {
     fn compute(hash: usize, mask: usize) -> QuadraticProbing {
-        QuadraticProbing { hash, i: 0, mask }
+        QuadraticProbing {
+            hash,
+            i: 0,
+            mask,
+        }
     }
 
     #[inline]
@@ -126,23 +131,6 @@ impl QuadraticProbing {
     }
 }
 
-pub struct Iter<'a: 'b, 'b> {
-    hashmap: &'b TermHashMap<'a>,
-    inner: slice::Iter<'a, usize>,
-}
-
-impl<'a, 'b> Iterator for Iter<'a, 'b> {
-    type Item = (&'b [u8], u32, UnorderedTermId);
-
-    fn next(&mut self) -> Option<Self::Item> {
-        self.inner.next().cloned().map(move |bucket: usize| {
-            let kv = self.hashmap.table[bucket];
-            let (key, offset): (&'b [u8], u32) = self.hashmap.get_key_value(kv.key_value_addr);
-            (key, offset, bucket as UnorderedTermId)
-        })
-    }
-}
-
 impl<'a> TermHashMap<'a> {
     pub fn new(num_bucket_power_of_2: usize, heap: &'a Heap) -> TermHashMap<'a> {
         let table_size = 1 << num_bucket_power_of_2;
@@ -173,16 +161,16 @@ impl<'a> TermHashMap<'a> {
     pub fn set_bucket(&mut self, hash: u32, key_value_addr: BytesRef, bucket: usize) {
         self.occupied.push(bucket);
         self.table[bucket] = KeyValue {
-            key_value_addr,
-            hash,
+            key_value_addr, hash
         };
     }
 
-    pub fn iter<'b: 'a>(&'b self) -> Iter<'a, 'b> {
-        Iter {
-            inner: self.occupied.iter(),
-            hashmap: &self,
-        }
+    pub fn iter<'b: 'a>(&'b self) -> impl Iterator<Item = (&'a [u8], u32, UnorderedTermId)> + 'b {
+        self.occupied.iter().cloned().map(move |bucket: usize| {
+            let kv = self.table[bucket];
+            let (key, offset) = self.get_key_value(kv.key_value_addr);
+            (key, offset, bucket as UnorderedTermId)
+        })
     }
 
     pub fn get_or_create<S: AsRef<[u8]>, V: HeapAllocable>(
@@ -214,32 +202,15 @@ impl<'a> TermHashMap<'a> {
     }
 }
 
-#[cfg(all(test, feature = "unstable"))]
-mod bench {
-    use super::murmurhash2::murmurhash2;
-    use test::Bencher;
-
-    #[bench]
-    fn bench_murmurhash2(b: &mut Bencher) {
-        let keys: [&'static str; 3] = ["wer qwe qwe qwe ", "werbq weqweqwe2 ", "weraq weqweqwe3 "];
-        b.iter(|| {
-            let mut s = 0;
-            for &key in &keys {
-                s ^= murmurhash2(key.as_bytes());
-            }
-            s
-        });
-    }
-}
-
 #[cfg(test)]
 mod tests {
 
+    use super::*;
     use super::super::heap::{Heap, HeapAllocable};
     use super::murmurhash2::murmurhash2;
-    use super::split_memory;
-    use super::*;
+    use test::Bencher;
     use std::collections::HashSet;
+    use super::split_memory;
 
     struct TestValue {
         val: u32,
@@ -310,17 +281,6 @@ mod tests {
         }
     }
 
-    #[test]
-    fn test_murmur_against_reference_impl() {
-        assert_eq!(murmurhash2("".as_bytes()), 3632506080);
-        assert_eq!(murmurhash2("a".as_bytes()), 455683869);
-        assert_eq!(murmurhash2("ab".as_bytes()), 2448092234);
-        assert_eq!(murmurhash2("abc".as_bytes()), 2066295634);
-        assert_eq!(murmurhash2("abcd".as_bytes()), 2588571162);
-        assert_eq!(murmurhash2("abcde".as_bytes()), 2988696942);
-        assert_eq!(murmurhash2("abcdefghijklmnop".as_bytes()), 2350868870);
-    }
-
     #[test]
     fn test_murmur_collisions() {
         let mut set: HashSet<u32> = HashSet::default();
@@ -332,4 +292,18 @@ mod tests {
         assert_eq!(set.len(), 10_000);
     }
 
+    #[bench]
+    fn bench_murmurhash_2(b: &mut Bencher) {
+        let keys: Vec<&'static str> =
+            vec!["wer qwe qwe qwe ", "werbq weqweqwe2 ", "weraq weqweqwe3 "];
+        b.iter(|| {
+            keys.iter()
+                .map(|&s| s.as_bytes())
+                .map(murmurhash2::murmurhash2)
+                .map(|h| h as u64)
+                .last()
+                .unwrap()
+        });
+    }
+
 }

src/datastruct/stacker/heap.rs

@@ -1,7 +1,7 @@
-use byteorder::{ByteOrder, NativeEndian};
 use std::cell::UnsafeCell;
 use std::mem;
 use std::ptr;
+use byteorder::{ByteOrder, NativeEndian};
 
 /// `BytesRef` refers to a slice in tantivy's custom `Heap`.
 ///

src/datastruct/stacker/mod.rs

@@ -1,10 +1,10 @@
-mod expull;
 pub(crate) mod hashmap;
 mod heap;
+mod expull;
 
+pub use self::heap::{Heap, HeapAllocable};
 pub use self::expull::ExpUnrolledLinkedList;
 pub use self::hashmap::TermHashMap;
-pub use self::heap::{Heap, HeapAllocable};
 
 #[test]
 fn test_unrolled_linked_list() {

src/directory/directory.rs

@@ -1,11 +1,11 @@
+use std::marker::Send;
+use std::fmt;
+use std::path::Path;
 use directory::error::{DeleteError, OpenReadError, OpenWriteError};
 use directory::{ReadOnlySource, WritePtr};
-use std::fmt;
-use std::io;
-use std::marker::Send;
-use std::marker::Sync;
-use std::path::Path;
 use std::result;
+use std::io;
+use std::marker::Sync;
 
 /// Write-once read many (WORM) abstraction for where
 /// tantivy's data should be stored.

src/directory/error.rs

@@ -1,7 +1,7 @@
 use std::error::Error as StdError;
-use std::fmt;
-use std::io;
 use std::path::PathBuf;
+use std::io;
+use std::fmt;
 
 /// General IO error with an optional path to the offending file.
 #[derive(Debug)]

src/directory/managed_directory.rs

@@ -1,18 +1,18 @@
-use core::MANAGED_FILEPATH;
+use std::path::{Path, PathBuf};
+use serde_json;
 use directory::error::{DeleteError, IOError, OpenReadError, OpenWriteError};
 use directory::{ReadOnlySource, WritePtr};
-use error::{ErrorKind, Result, ResultExt};
-use serde_json;
-use std::collections::HashMap;
-use std::collections::HashSet;
-use std::fmt;
-use std::io;
-use std::io::Write;
-use std::path::{Path, PathBuf};
 use std::result;
-use std::sync::RwLockWriteGuard;
-use std::sync::{Arc, RwLock};
+use std::io;
 use Directory;
+use std::sync::{Arc, RwLock};
+use std::collections::HashSet;
+use std::sync::RwLockWriteGuard;
+use std::io::Write;
+use core::MANAGED_FILEPATH;
+use std::collections::HashMap;
+use std::fmt;
+use error::{ErrorKind, Result, ResultExt};
 
 /// Wrapper of directories that keeps track of files created by Tantivy.
 ///
@@ -86,7 +86,7 @@ impl ManagedDirectory {
                 let managed_files: HashSet<PathBuf> = serde_json::from_str(&managed_files_json)
                     .chain_err(|| ErrorKind::CorruptedFile(MANAGED_FILEPATH.clone()))?;
                 Ok(ManagedDirectory {
-                    directory: Box::new(directory),
+                    directory: box directory,
                     meta_informations: Arc::new(RwLock::new(MetaInformation {
                         managed_paths: managed_files,
                         protected_files: HashMap::default(),
@@ -94,7 +94,7 @@ impl ManagedDirectory {
                 })
             }
             Err(OpenReadError::FileDoesNotExist(_)) => Ok(ManagedDirectory {
-                directory: Box::new(directory),
+                directory: box directory,
                 meta_informations: Arc::default(),
             }),
             Err(OpenReadError::IOError(e)) => Err(From::from(e)),
@@ -265,7 +265,7 @@ impl Directory for ManagedDirectory {
     }
 
     fn box_clone(&self) -> Box<Directory> {
-        Box::new(self.clone())
+        box self.clone()
     }
 }
 
@@ -282,10 +282,10 @@ impl Clone for ManagedDirectory {
 mod tests {
 
     use super::*;
-    #[cfg(feature = "mmap")]
+    #[cfg(feature="mmap")]
     use directory::MmapDirectory;
-    use std::io::Write;
     use std::path::Path;
+    use std::io::Write;
     use tempdir::TempDir;
 
     lazy_static! {
@@ -294,7 +294,7 @@ mod tests {
     }
 
     #[test]
-    #[cfg(feature = "mmap")]
+    #[cfg(feature="mmap")]
     fn test_managed_directory() {
         let tempdir = TempDir::new("index").unwrap();
         let tempdir_path = PathBuf::from(tempdir.path());
@@ -343,7 +343,7 @@ mod tests {
     }
 
     #[test]
-    #[cfg(feature = "mmap ")]
+    #[cfg(feature="mmap ")]
     fn test_managed_directory_gc_while_mmapped() {
         let tempdir = TempDir::new("index").unwrap();
         let tempdir_path = PathBuf::from(tempdir.path());
@@ -373,7 +373,7 @@ mod tests {
     }
 
     #[test]
-    #[cfg(feature = "mmap")]
+    #[cfg(feature="mmap")]
     fn test_managed_directory_protect() {
         let tempdir = TempDir::new("index").unwrap();
         let tempdir_path = PathBuf::from(tempdir.path());

src/directory/mmap_directory.rs

@@ -1,17 +1,17 @@
 use atomicwrites;
 use common::make_io_err;
-use directory::error::{DeleteError, IOError, OpenDirectoryError, OpenReadError, OpenWriteError};
-use directory::shared_vec_slice::SharedVecSlice;
 use directory::Directory;
+use directory::error::{DeleteError, IOError, OpenDirectoryError, OpenReadError, OpenWriteError};
 use directory::ReadOnlySource;
+use directory::shared_vec_slice::SharedVecSlice;
 use directory::WritePtr;
 use fst::raw::MmapReadOnly;
 use std::collections::hash_map::Entry as HashMapEntry;
 use std::collections::HashMap;
 use std::convert::From;
 use std::fmt;
-use std::fs::OpenOptions;
 use std::fs::{self, File};
+use std::fs::OpenOptions;
 use std::io::{self, Seek, SeekFrom};
 use std::io::{BufWriter, Read, Write};
 use std::path::{Path, PathBuf};
@@ -40,11 +40,9 @@ fn open_mmap(full_path: &Path) -> result::Result<Option<MmapReadOnly>, OpenReadE
         // instead.
         return Ok(None);
     }
-    unsafe {
-        MmapReadOnly::open(&file)
-            .map(Some)
-            .map_err(|e| From::from(IOError::with_path(full_path.to_owned(), e)))
-    }
+    MmapReadOnly::open(&file)
+        .map(Some)
+        .map_err(|e| From::from(IOError::with_path(full_path.to_owned(), e)))
 }
 
 #[derive(Default, Clone, Debug, Serialize, Deserialize)]

src/directory/mod.rs

@@ -4,29 +4,32 @@ WORM directory abstraction.
 
 */
 
-#[cfg(feature = "mmap")]
+#[cfg(feature="mmap")]
 mod mmap_directory;
 
-mod directory;
-mod managed_directory;
 mod ram_directory;
+mod directory;
 mod read_only_source;
 mod shared_vec_slice;
+mod managed_directory;
+mod static_directory;
 
 /// Errors specific to the directory module.
 pub mod error;
 
 use std::io::{BufWriter, Seek, Write};
 
+pub use self::static_directory::StaticDirectory;
+pub use self::static_directory::write_static_from_directory;
+pub use self::read_only_source::ReadOnlySource;
 pub use self::directory::Directory;
 pub use self::ram_directory::RAMDirectory;
-pub use self::read_only_source::ReadOnlySource;
 
-#[cfg(feature = "mmap")]
+#[cfg(feature="mmap")]
 pub use self::mmap_directory::MmapDirectory;
 
-pub(crate) use self::managed_directory::{FileProtection, ManagedDirectory};
 pub(crate) use self::read_only_source::SourceRead;
+pub(crate) use self::managed_directory::{FileProtection, ManagedDirectory};
 
 /// Synonym of Seek + Write
 pub trait SeekableWrite: Seek + Write {}
@@ -42,8 +45,8 @@ pub type WritePtr = BufWriter<Box<SeekableWrite>>;
 mod tests {
 
     use super::*;
-    use std::io::{Seek, SeekFrom, Write};
     use std::path::Path;
+    use std::io::{Seek, SeekFrom, Write};
 
     lazy_static! {
         static ref TEST_PATH: &'static Path = Path::new("some_path_for_test");
@@ -56,7 +59,7 @@ mod tests {
     }
 
     #[test]
-    #[cfg(feature = "mmap")]
+    #[cfg(feature="mmap")]
     fn test_mmap_directory() {
         let mut mmap_directory = MmapDirectory::create_from_tempdir().unwrap();
         test_directory(&mut mmap_directory);

src/directory/ram_directory.rs

@@ -1,14 +1,14 @@
-use super::shared_vec_slice::SharedVecSlice;
-use common::make_io_err;
-use directory::error::{DeleteError, IOError, OpenReadError, OpenWriteError};
-use directory::WritePtr;
-use directory::{Directory, ReadOnlySource};
 use std::collections::HashMap;
 use std::fmt;
 use std::io::{self, BufWriter, Cursor, Seek, SeekFrom, Write};
 use std::path::{Path, PathBuf};
 use std::result;
 use std::sync::{Arc, RwLock};
+use common::make_io_err;
+use directory::{Directory, ReadOnlySource};
+use directory::error::{DeleteError, IOError, OpenReadError, OpenWriteError};
+use directory::WritePtr;
+use super::shared_vec_slice::SharedVecSlice;
 
 /// Writer associated with the `RAMDirectory`
 ///

src/directory/read_only_source.rs

@@ -1,11 +1,13 @@
+#[cfg(feature="mmap")]
+use fst::raw::MmapReadOnly;
+use std::ops::Deref;
 use super::shared_vec_slice::SharedVecSlice;
 use common::HasLen;
-#[cfg(feature = "mmap")]
-use fst::raw::MmapReadOnly;
-use stable_deref_trait::{CloneStableDeref, StableDeref};
-use std::io::{self, Read};
-use std::ops::Deref;
 use std::slice;
+use std::io::{self, Read};
+use stable_deref_trait::{CloneStableDeref, StableDeref};
+
+const EMPTY_SLICE: [u8; 0] = [];
 
 /// Read object that represents files in tantivy.
 ///
@@ -15,10 +17,12 @@ use std::slice;
 /// hold by this object should never be altered or destroyed.
 pub enum ReadOnlySource {
     /// Mmap source of data
-    #[cfg(feature = "mmap")]
+    #[cfg(feature="mmap")]
     Mmap(MmapReadOnly),
     /// Wrapping a `Vec<u8>`
     Anonymous(SharedVecSlice),
+    /// Wrapping a static slice
+    Static(&'static [u8])
 }
 
 unsafe impl StableDeref for ReadOnlySource {}
@@ -35,15 +39,16 @@ impl Deref for ReadOnlySource {
 impl ReadOnlySource {
     /// Creates an empty ReadOnlySource
     pub fn empty() -> ReadOnlySource {
-        ReadOnlySource::Anonymous(SharedVecSlice::empty())
+        ReadOnlySource::Static(&EMPTY_SLICE)
     }
 
     /// Returns the data underlying the ReadOnlySource object.
     pub fn as_slice(&self) -> &[u8] {
         match *self {
-            #[cfg(feature = "mmap")]
-            ReadOnlySource::Mmap(ref mmap_read_only) => mmap_read_only.as_slice(),
+            #[cfg(feature="mmap")]
+            ReadOnlySource::Mmap(ref mmap_read_only) => unsafe { mmap_read_only.as_slice() },
             ReadOnlySource::Anonymous(ref shared_vec) => shared_vec.as_slice(),
+            ReadOnlySource::Static(data) => data,
         }
     }
 
@@ -66,14 +71,9 @@ impl ReadOnlySource {
     /// 1KB slice is remaining, the whole `500MBs`
     /// are retained in memory.
     pub fn slice(&self, from_offset: usize, to_offset: usize) -> ReadOnlySource {
-        assert!(
-            from_offset <= to_offset,
-            "Requested negative slice [{}..{}]",
-            from_offset,
-            to_offset
-        );
+        assert!(from_offset <= to_offset, "Requested negative slice [{}..{}]", from_offset, to_offset);
         match *self {
-            #[cfg(feature = "mmap")]
+            #[cfg(feature="mmap")]
             ReadOnlySource::Mmap(ref mmap_read_only) => {
                 let sliced_mmap = mmap_read_only.range(from_offset, to_offset - from_offset);
                 ReadOnlySource::Mmap(sliced_mmap)
@@ -81,6 +81,9 @@ impl ReadOnlySource {
             ReadOnlySource::Anonymous(ref shared_vec) => {
                 ReadOnlySource::Anonymous(shared_vec.slice(from_offset, to_offset))
             }
+            ReadOnlySource::Static(data) => {
+                ReadOnlySource::Static(&data[from_offset..to_offset])
+            }
         }
     }
 
@@ -121,6 +124,12 @@ impl From<Vec<u8>> for ReadOnlySource {
     }
 }
 
+impl From<&'static [u8]> for ReadOnlySource {
+    fn from(data: &'static [u8]) -> ReadOnlySource {
+        ReadOnlySource::Static(data)
+    }
+}
+
 /// Acts as a owning cursor over the data backed up by a `ReadOnlySource`
 pub(crate) struct SourceRead {
     _data_owner: ReadOnlySource,
@@ -135,11 +144,13 @@ impl SourceRead {
 
     pub fn slice_from(&self, start: usize) -> &[u8] {
         &self.cursor[start..]
+
     }
 
     pub fn get(&self, idx: usize) -> u8 {
         self.cursor[idx]
     }
+
 }
 
 impl AsRef<[u8]> for SourceRead {

src/directory/static_directory.rs

@@ -0,0 +1,123 @@
+use std::collections::HashMap;
+use Directory;
+use std::path::PathBuf;
+use directory::ReadOnlySource;
+use std::io::BufWriter;
+use directory::error::{DeleteError, OpenReadError, OpenWriteError};
+use std::path::Path;
+use std::fmt::{Formatter, Debug, self};
+use Result as TantivyResult;
+use directory::SeekableWrite;
+use std::io;
+use std::fs;
+use common::Endianness;
+use common::BinarySerializable;
+use common::VInt;
+use byteorder::ByteOrder;
+use std::str;
+use std::fs::File;
+use std::io::{Read, Write};
+use std::ffi::OsString;
+
+#[derive(Clone)]
+pub struct StaticDirectory {
+    files: HashMap<PathBuf, &'static [u8]>,
+}
+
+impl Debug for StaticDirectory {
+    fn fmt(&self, f: &mut Formatter) -> Result<(), fmt::Error> {
+        write!(f, "StaticDirectory[{} files]", self.files.len())?;
+        Ok(())
+    }
+}
+
+impl StaticDirectory {
+    pub fn open(mut data: &'static [u8]) -> TantivyResult<StaticDirectory> {
+        assert!(data.len() > 8);
+        let footer_len_offset = data.len() - 8;
+        let body_len = Endianness::read_u64(&data[footer_len_offset..]) as usize;
+        let mut body = &data[..body_len];
+        let mut footer = &data[body_len..footer_len_offset];
+        let num_files = VInt::deserialize(&mut footer)?.0 as usize;
+        let mut files = HashMap::new();
+        for _ in 0..num_files {
+            let filename_len = VInt::deserialize(&mut footer)?.0 as usize;
+            let filename = &footer[..filename_len];
+            footer = &footer[filename_len..];
+            let data_len = VInt::deserialize(&mut footer)?.0 as usize;
+            let file_data = &body[..data_len];
+            body = &body[data_len..];
+            let filename_str = str::from_utf8(filename).expect("Invalid UTF8");
+            let filename = PathBuf::from(filename_str);
+            println!("{:?} {:?}", filename, data_len);
+            files.insert(filename, file_data);
+        }
+        Ok(StaticDirectory {
+            files
+        })
+    }
+}
+
+impl Directory for StaticDirectory {
+    fn open_read(&self, path: &Path) -> Result<ReadOnlySource, OpenReadError> {
+        if let Some(static_data) = self.files.get(path) {
+            Ok(ReadOnlySource::from(*static_data))
+        } else {
+            Err(OpenReadError::FileDoesNotExist(path.to_owned()))
+        }
+    }
+
+    fn delete(&self, path: &Path) -> Result<(), DeleteError> {
+        unimplemented!("Static directory is read-only !")
+    }
+
+    fn exists(&self, path: &Path) -> bool {
+        self.files.contains_key(path)
+    }
+
+    fn open_write(&mut self, path: &Path) -> Result<BufWriter<Box<SeekableWrite>>, OpenWriteError> {
+        unimplemented!("Static directory is read-only !")
+    }
+
+    fn atomic_read(&self, path: &Path) -> Result<Vec<u8>, OpenReadError> {
+        if let Some(static_data) = self.files.get(path) {
+            Ok(static_data.to_vec())
+        } else {
+            Err(OpenReadError::FileDoesNotExist(path.to_owned()))
+        }
+    }
+
+    fn atomic_write(&mut self, path: &Path, data: &[u8]) -> io::Result<()> {
+        unimplemented!("Static directory is read-only !")
+    }
+
+    fn box_clone(&self) -> Box<Directory> {
+        box self.clone()
+    }
+}
+
+pub fn write_static_from_directory(directory_path: &Path) -> TantivyResult<Vec<u8>> {
+    assert!(directory_path.is_dir());
+    let mut file_data: Vec<(OsString, usize)> = Vec::new();
+    let mut write: Vec<u8> = Vec::new();
+    for entry in fs::read_dir(directory_path)? {
+        let entry = entry?;
+        let path = entry.path();
+        if path.is_file() {
+            info!("Appending {}", path.to_string_lossy());
+            let mut open_file = File::open(&path)?;
+            let file_len = open_file.read_to_end(&mut write)?;
+            file_data.push((entry.file_name(), file_len));
+        }
+    }
+    // write footer
+    let body_len = write.len();
+    VInt(file_data.len() as u64).serialize(&mut write)?;
+    for (filename, filelen) in file_data {
+        VInt(filename.len() as u64).serialize(&mut write)?;
+        write.write_all(filename.to_string_lossy().as_bytes())?;
+        VInt(filelen as u64).serialize(&mut write)?;
+    }
+    (body_len as u64).serialize(&mut write)?;
+    Ok(write)
+}

src/docset.rs

@@ -1,8 +1,8 @@
-use common::BitSet;
+use DocId;
 use std::borrow::Borrow;
 use std::borrow::BorrowMut;
 use std::cmp::Ordering;
-use DocId;
+use common::BitSet;
 
 /// Expresses the outcome of a call to `DocSet`'s `.skip_next(...)`.
 #[derive(PartialEq, Eq, Debug)]

src/error.rs

@@ -2,13 +2,13 @@
 
 use std::io;
 
-use directory::error::{IOError, OpenDirectoryError, OpenReadError, OpenWriteError};
-use fastfield::FastFieldNotAvailableError;
-use query;
-use schema;
-use serde_json;
 use std::path::PathBuf;
 use std::sync::PoisonError;
+use directory::error::{IOError, OpenDirectoryError, OpenReadError, OpenWriteError};
+use query;
+use schema;
+use fastfield::FastFieldNotAvailableError;
+use serde_json;
 
 error_chain!(
     errors {
@@ -48,10 +48,10 @@ error_chain!(
             description("an error occurred in a thread")
             display("an error occurred in a thread: '{}'", err)
         }
-        /// An Error appeared related to the schema.
-        SchemaError(message: String) {
-            description("the schema is not matching expectations.")
-            display("Schema error: '{}'", message)
+        /// An Error appeared related to the lack of a field.
+        SchemaError(field: String) {
+            description("a schema field is missing")
+            display("a schema field is missing: '{}'", field)
         }
         /// Tried to access a fastfield reader for a field not configured accordingly.
         FastFieldError(err: FastFieldNotAvailableError) {

src/fastfield/bytes/mod.rs

@@ -1,38 +0,0 @@
-mod reader;
-mod writer;
-
-pub use self::reader::BytesFastFieldReader;
-pub use self::writer::BytesFastFieldWriter;
-
-#[cfg(test)]
-mod tests {
-    use schema::SchemaBuilder;
-    use Index;
-
-    #[test]
-    fn test_bytes() {
-        let mut schema_builder = SchemaBuilder::default();
-        let field = schema_builder.add_bytes_field("bytesfield");
-        let schema = schema_builder.build();
-        let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
-        index_writer.add_document(doc!(field=>vec![0u8, 1, 2, 3]));
-        index_writer.add_document(doc!(field=>vec![]));
-        index_writer.add_document(doc!(field=>vec![255u8]));
-        index_writer.add_document(doc!(field=>vec![1u8, 3, 5, 7, 9]));
-        index_writer.add_document(doc!(field=>vec![0u8; 1000]));
-        assert!(index_writer.commit().is_ok());
-
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
-        let reader = searcher.segment_reader(0);
-        let bytes_reader = reader.bytes_fast_field_reader(field).unwrap();
-
-        assert_eq!(bytes_reader.get_val(0), &[0u8, 1, 2, 3]);
-        assert!(bytes_reader.get_val(1).is_empty());
-        assert_eq!(bytes_reader.get_val(2), &[255u8]);
-        assert_eq!(bytes_reader.get_val(3), &[1u8, 3, 5, 7, 9]);
-        let long = vec![0u8; 1000];
-        assert_eq!(bytes_reader.get_val(4), long.as_slice());
-    }
-}

src/fastfield/bytes/reader.rs

@@ -1,37 +0,0 @@
-use owning_ref::OwningRef;
-
-use directory::ReadOnlySource;
-use fastfield::FastFieldReader;
-use DocId;
-
-/// Reader for byte array fast fields
-///
-/// The reader is implemented as a `u64` fast field and a separate collection of bytes.
-///
-/// The `vals_reader` will access the concatenated list of all values for all documents.
-///
-/// The `idx_reader` associates, for each document, the index of its first value.
-///
-/// Reading the value for a document is done by reading the start index for it,
-/// and the start index for the next document, and keeping the bytes in between.
-pub struct BytesFastFieldReader {
-    idx_reader: FastFieldReader<u64>,
-    values: OwningRef<ReadOnlySource, [u8]>,
-}
-
-impl BytesFastFieldReader {
-    pub(crate) fn open(
-        idx_reader: FastFieldReader<u64>,
-        values_source: ReadOnlySource,
-    ) -> BytesFastFieldReader {
-        let values = OwningRef::new(values_source).map(|source| &source[..]);
-        BytesFastFieldReader { idx_reader, values }
-    }
-
-    /// Returns the bytes associated to the given `doc`
-    pub fn get_val(&self, doc: DocId) -> &[u8] {
-        let start = self.idx_reader.get(doc) as usize;
-        let stop = self.idx_reader.get(doc + 1) as usize;
-        &self.values[start..stop]
-    }
-}

src/fastfield/bytes/writer.rs

@@ -1,96 +0,0 @@
-use std::io;
-
-use fastfield::serializer::FastFieldSerializer;
-use schema::{Document, Field, Value};
-use DocId;
-
-/// Writer for byte array (as in, any number of bytes per document) fast fields
-///
-/// This `BytesFastFieldWriter` is only useful for advanced user.
-/// The normal way to get your associated bytes in your index
-/// is to
-/// - declare your field with fast set to `Cardinality::SingleValue`
-/// in your schema
-/// - add your document simply by calling `.add_document(...)` with associating bytes to the field.
-///
-/// The `BytesFastFieldWriter` can be acquired from the
-/// fast field writer by calling
-/// [`.get_bytes_writer(...)`](./struct.FastFieldsWriter.html#method.get_bytes_writer).
-///
-/// Once acquired, writing is done by calling `.add_document_val(&[u8])`
-/// once per document, even if there are no bytes associated to it.
-pub struct BytesFastFieldWriter {
-    field: Field,
-    vals: Vec<u8>,
-    doc_index: Vec<u64>,
-}
-
-impl BytesFastFieldWriter {
-    /// Creates a new `BytesFastFieldWriter`
-    pub fn new(field: Field) -> Self {
-        BytesFastFieldWriter {
-            field,
-            vals: Vec::new(),
-            doc_index: Vec::new(),
-        }
-    }
-
-    /// Access the field associated to the `BytesFastFieldWriter`
-    pub fn field(&self) -> Field {
-        self.field
-    }
-
-    /// Finalize the current document.
-    pub(crate) fn next_doc(&mut self) {
-        self.doc_index.push(self.vals.len() as u64);
-    }
-
-    /// Shift to the next document and add all of the
-    /// matching field values present in the document.
-    pub fn add_document(&mut self, doc: &Document) {
-        self.next_doc();
-        for field_value in doc.field_values() {
-            if field_value.field() == self.field {
-                if let &Value::Bytes(ref bytes) = field_value.value() {
-                    self.vals.extend_from_slice(bytes);
-                } else {
-                    panic!(
-                        "Bytes field contained non-Bytes Value!. Field {:?} = {:?}",
-                        self.field, field_value
-                    );
-                }
-            }
-        }
-    }
-
-    /// Register the bytes associated to a document.
-    ///
-    /// The method returns the `DocId` of the document that was
-    /// just written.
-    pub fn add_document_val(&mut self, val: &[u8]) -> DocId {
-        let doc = self.doc_index.len() as DocId;
-        self.next_doc();
-        self.vals.extend_from_slice(val);
-        doc
-    }
-
-    /// Serializes the fast field values by pushing them to the `FastFieldSerializer`.
-    pub fn serialize(&self, serializer: &mut FastFieldSerializer) -> io::Result<()> {
-        {
-            // writing the offset index
-            let mut doc_index_serializer =
-                serializer.new_u64_fast_field_with_idx(self.field, 0, self.vals.len() as u64, 0)?;
-            for &offset in &self.doc_index {
-                doc_index_serializer.add_val(offset)?;
-            }
-            doc_index_serializer.add_val(self.vals.len() as u64)?;
-            doc_index_serializer.close_field()?;
-        }
-        {
-            // writing the values themselves
-            let mut value_serializer = serializer.new_bytes_fast_field_with_idx(self.field, 1)?;
-            value_serializer.write_all(&self.vals)?;
-        }
-        Ok(())
-    }
-}

src/fastfield/delete.rs

@@ -1,10 +1,10 @@
 use bit_set::BitSet;
-use common::HasLen;
-use directory::ReadOnlySource;
 use directory::WritePtr;
-use std::io;
 use std::io::Write;
+use std::io;
+use directory::ReadOnlySource;
 use DocId;
+use common::HasLen;
 
 /// Write a delete `BitSet`
 ///
@@ -62,8 +62,10 @@ impl DeleteBitSet {
             b & (1u8 << shift) != 0
         }
     }
+
 }
 
+
 impl HasLen for DeleteBitSet {
     fn len(&self) -> usize {
         self.len
@@ -72,10 +74,10 @@ impl HasLen for DeleteBitSet {
 
 #[cfg(test)]
 mod tests {
-    use super::*;
+    use std::path::PathBuf;
     use bit_set::BitSet;
     use directory::*;
-    use std::path::PathBuf;
+    use super::*;
 
     fn test_delete_bitset_helper(bitset: &BitSet) {
         let test_path = PathBuf::from("test");

src/fastfield/error.rs

@@ -1,5 +1,5 @@
-use schema::FieldEntry;
 use std::result;
+use schema::FieldEntry;
 
 /// `FastFieldNotAvailableError` is returned when the
 /// user requested for a fast field reader, and the field was not

src/fastfield/facet_reader.rs

@@ -1,8 +1,8 @@
 use super::MultiValueIntFastFieldReader;
-use schema::Facet;
-use termdict::TermDictionary;
-use termdict::TermOrdinal;
 use DocId;
+use termdict::TermOrdinal;
+use schema::Facet;
+use termdict::{TermDictionary, TermDictionaryImpl};
 
 /// The facet reader makes it possible to access the list of
 /// facets associated to a given document in a specific
@@ -19,7 +19,7 @@ use DocId;
 /// only makes sense for a given segment.
 pub struct FacetReader {
     term_ords: MultiValueIntFastFieldReader<u64>,
-    term_dict: TermDictionary,
+    term_dict: TermDictionaryImpl,
 }
 
 impl FacetReader {
@@ -28,11 +28,11 @@ impl FacetReader {
     /// A facet reader just wraps :
     /// - a `MultiValueIntFastFieldReader` that makes it possible to
     /// access the list of facet ords for a given document.
-    /// - a `TermDictionary` that helps associating a facet to
+    /// - a `TermDictionaryImpl` that helps associating a facet to
     /// an ordinal and vice versa.
     pub fn new(
         term_ords: MultiValueIntFastFieldReader<u64>,
-        term_dict: TermDictionary,
+        term_dict: TermDictionaryImpl,
     ) -> FacetReader {
         FacetReader {
             term_ords,
@@ -50,7 +50,7 @@ impl FacetReader {
     }
 
     /// Accessor for the facet term dictionary.
-    pub fn facet_dict(&self) -> &TermDictionary {
+    pub fn facet_dict(&self) -> &TermDictionaryImpl {
         &self.term_dict
     }
 

src/fastfield/mod.rs

@@ -23,28 +23,26 @@ values stored.
 Read access performance is comparable to that of an array lookup.
 */
 
-pub use self::bytes::{BytesFastFieldReader, BytesFastFieldWriter};
-pub use self::delete::write_delete_bitset;
-pub use self::delete::DeleteBitSet;
-pub use self::error::{FastFieldNotAvailableError, Result};
-pub use self::facet_reader::FacetReader;
-pub use self::multivalued::{MultiValueIntFastFieldReader, MultiValueIntFastFieldWriter};
-pub use self::reader::FastFieldReader;
-pub use self::serializer::FastFieldSerializer;
-pub use self::writer::{FastFieldsWriter, IntFastFieldWriter};
 use common;
 use schema::Cardinality;
 use schema::FieldType;
 use schema::Value;
+pub use self::delete::DeleteBitSet;
+pub use self::delete::write_delete_bitset;
+pub use self::error::{FastFieldNotAvailableError, Result};
+pub use self::facet_reader::FacetReader;
+pub use self::multivalued::MultiValueIntFastFieldReader;
+pub use self::reader::FastFieldReader;
+pub use self::serializer::FastFieldSerializer;
+pub use self::writer::{FastFieldsWriter, IntFastFieldWriter};
 
-mod bytes;
-mod delete;
+mod reader;
+mod writer;
+mod serializer;
 mod error;
+mod delete;
 mod facet_reader;
 mod multivalued;
-mod reader;
-mod serializer;
-mod writer;
 
 /// Trait for types that are allowed for fast fields: (u64 or i64).
 pub trait FastValue: Default + Clone + Copy {
@@ -123,27 +121,31 @@ fn value_to_u64(value: &Value) -> u64 {
 #[cfg(test)]
 mod tests {
 
-    use super::*;
     use common::CompositeFile;
     use directory::{Directory, RAMDirectory, WritePtr};
     use fastfield::FastFieldReader;
     use rand::Rng;
     use rand::SeedableRng;
     use rand::XorShiftRng;
-    use schema::Document;
-    use schema::Field;
-    use schema::FAST;
     use schema::{Schema, SchemaBuilder};
+    use schema::Document;
+    use schema::FAST;
+    use schema::Field;
     use std::collections::HashMap;
     use std::path::Path;
+    use super::*;
+    use test;
+    use test::Bencher;
 
     lazy_static! {
-        pub static ref SCHEMA: Schema = {
+        static ref SCHEMA: Schema = {
             let mut schema_builder = SchemaBuilder::default();
             schema_builder.add_u64_field("field", FAST);
             schema_builder.build()
         };
-        pub static ref FIELD: Field = { SCHEMA.get_field("field").unwrap() };
+        static ref FIELD: Field = {
+            SCHEMA.get_field("field").unwrap()
+        };
     }
 
     #[test]
@@ -367,7 +369,7 @@ mod tests {
         }
     }
 
-    pub fn generate_permutation() -> Vec<u64> {
+    fn generate_permutation() -> Vec<u64> {
         let seed: &[u32; 4] = &[1, 2, 3, 4];
         let mut rng = XorShiftRng::from_seed(*seed);
         let mut permutation: Vec<u64> = (0u64..1_000_000u64).collect();
@@ -407,27 +409,13 @@ mod tests {
         }
     }
 
-}
-
-#[cfg(all(test, feature = "unstable"))]
-mod bench {
-    use super::tests::FIELD;
-    use super::tests::{generate_permutation, SCHEMA};
-    use super::*;
-    use common::CompositeFile;
-    use directory::{Directory, RAMDirectory, WritePtr};
-    use fastfield::FastFieldReader;
-    use std::collections::HashMap;
-    use std::path::Path;
-    use test::{self, Bencher};
-
     #[bench]
     fn bench_intfastfield_linear_veclookup(b: &mut Bencher) {
         let permutation = generate_permutation();
         b.iter(|| {
             let n = test::black_box(7000u32);
             let mut a = 0u64;
-            for i in (0u32..n / 7).map(|v| v * 7) {
+            for i in Iterator::step_by(0u32..n, 7) {
                 a ^= permutation[i as usize];
             }
             a
@@ -473,7 +461,7 @@ mod bench {
             b.iter(|| {
                 let n = test::black_box(7000u32);
                 let mut a = 0u64;
-                for i in (0u32..n / 7).map(|val| val * 7) {
+                for i in Iterator::step_by(0u32..n, 7) {
                     a ^= fast_field_reader.get(i);
                 }
                 a
@@ -514,5 +502,4 @@ mod bench {
             });
         }
     }
-
 }

src/fastfield/multivalued/mod.rs

@@ -1,15 +1,15 @@
-mod reader;
 mod writer;
+mod reader;
 
-pub use self::reader::MultiValueIntFastFieldReader;
 pub use self::writer::MultiValueIntFastFieldWriter;
+pub use self::reader::MultiValueIntFastFieldReader;
 
 #[cfg(test)]
 mod tests {
 
+    use schema::SchemaBuilder;
     use schema::Cardinality;
     use schema::IntOptions;
-    use schema::SchemaBuilder;
     use Index;
 
     #[test]

src/fastfield/multivalued/reader.rs

@@ -1,5 +1,5 @@
-use fastfield::{FastFieldReader, FastValue};
 use DocId;
+use fastfield::{FastFieldReader, FastValue};
 
 /// Reader for a multivalued `u64` fast field.
 ///
@@ -26,20 +26,13 @@ impl<Item: FastValue> MultiValueIntFastFieldReader<Item> {
         }
     }
 
-    /// Returns `(start, stop)`, such that the values associated
-    /// to the given document are `start..stop`.
-    fn range(&self, doc: DocId) -> (u64, u64) {
-        let start = self.idx_reader.get(doc);
-        let stop = self.idx_reader.get(doc + 1);
-        (start, stop)
-    }
-
     /// Returns the array of values associated to the given `doc`.
     pub fn get_vals(&self, doc: DocId, vals: &mut Vec<Item>) {
-        let (start, stop) = self.range(doc);
+        let start = self.idx_reader.get(doc) as u32;
+        let stop = self.idx_reader.get(doc + 1) as u32;
         let len = (stop - start) as usize;
         vals.resize(len, Item::default());
-        self.vals_reader.get_range(start as u32, &mut vals[..]);
+        self.vals_reader.get_range(start, &mut vals[..]);
     }
 }
 

src/fastfield/multivalued/writer.rs

@@ -1,35 +1,12 @@
+use fastfield::FastFieldSerializer;
 use fastfield::serializer::FastSingleFieldSerializer;
 use fastfield::value_to_u64;
-use fastfield::FastFieldSerializer;
-use itertools::Itertools;
+use std::collections::HashMap;
 use postings::UnorderedTermId;
 use schema::{Document, Field};
-use std::collections::HashMap;
 use std::io;
-use termdict::TermOrdinal;
-use DocId;
+use itertools::Itertools;
 
-/// Writer for multi-valued (as in, more than one value per document)
-/// int fast field.
-///
-/// This `Writer` is only useful for advanced user.
-/// The normal way to get your multivalued int in your index
-/// is to
-/// - declare your field with fast set to `Cardinality::MultiValues`
-/// in your schema
-/// - add your document simply by calling `.add_document(...)`.
-///
-/// The `MultiValueIntFastFieldWriter` can be acquired from the
-/// fastfield writer, by calling [`.get_multivalue_writer(...)`](./struct.FastFieldsWriter.html#method.get_multivalue_writer).
-///
-/// Once acquired, writing is done by calling calls to
-/// `.add_document_vals(&[u64])` once per document.
-///
-/// The serializer makes it possible to remap all of the values
-/// that were pushed to the writer using a mapping.
-/// This makes it possible to push unordered term ids,
-/// during indexing and remap them to their respective
-/// term ids when the segment is getting serialized.
 pub struct MultiValueIntFastFieldWriter {
     field: Field,
     vals: Vec<u64>,
@@ -39,7 +16,7 @@ pub struct MultiValueIntFastFieldWriter {
 
 impl MultiValueIntFastFieldWriter {
     /// Creates a new `IntFastFieldWriter`
-    pub(crate) fn new(field: Field, is_facet: bool) -> Self {
+    pub fn new(field: Field, is_facet: bool) -> Self {
         MultiValueIntFastFieldWriter {
             field,
             vals: Vec::new(),
@@ -48,26 +25,24 @@ impl MultiValueIntFastFieldWriter {
         }
     }
 
-    /// Access the field associated to the `MultiValueIntFastFieldWriter`
     pub fn field(&self) -> Field {
         self.field
     }
 
-    /// Finalize the current document.
-    pub(crate) fn next_doc(&mut self) {
+    pub fn next_doc(&mut self) {
         self.doc_index.push(self.vals.len() as u64);
     }
 
-    /// Pushes a new value to the current document.
-    pub(crate) fn add_val(&mut self, val: UnorderedTermId) {
+    /// Records a new value.
+    ///
+    /// The n-th value being recorded is implicitely
+    /// associated to the document with the `DocId` n.
+    /// (Well, `n-1` actually because of 0-indexing)
+    pub fn add_val(&mut self, val: UnorderedTermId) {
         self.vals.push(val);
     }
 
-    /// Shift to the next document and adds
-    /// all of the matching field values present in the document.
     pub fn add_document(&mut self, doc: &Document) {
-        self.next_doc();
-        // facets are indexed in the `SegmentWriter` as we encode their unordered id.
         if !self.is_facet {
             for field_value in doc.field_values() {
                 if field_value.field() == self.field {
@@ -77,17 +52,6 @@ impl MultiValueIntFastFieldWriter {
         }
     }
 
-    /// Register all of the values associated to a document.
-    ///
-    /// The method returns the `DocId` of the document that was
-    /// just written.
-    pub fn add_document_vals(&mut self, vals: &[UnorderedTermId]) -> DocId {
-        let doc = self.doc_index.len() as DocId;
-        self.next_doc();
-        self.vals.extend_from_slice(vals);
-        doc
-    }
-
     /// Serializes fast field values by pushing them to the `FastFieldSerializer`.
     ///
     /// HashMap makes it possible to remap them before serializing.
@@ -102,7 +66,7 @@ impl MultiValueIntFastFieldWriter {
     pub fn serialize(
         &self,
         serializer: &mut FastFieldSerializer,
-        mapping_opt: Option<&HashMap<UnorderedTermId, TermOrdinal>>,
+        mapping_opt: Option<&HashMap<UnorderedTermId, usize>>,
     ) -> io::Result<()> {
         {
             // writing the offset index
@@ -126,13 +90,13 @@ impl MultiValueIntFastFieldWriter {
                         1,
                     )?;
                     for val in &self.vals {
-                        let remapped_val = *mapping.get(val).expect("Missing term ordinal");
+                        let remapped_val = *mapping.get(val).expect("Missing term ordinal") as u64;
                         value_serializer.add_val(remapped_val)?;
                     }
                 }
                 None => {
                     let val_min_max = self.vals.iter().cloned().minmax();
-                    let (val_min, val_max) = val_min_max.into_option().unwrap_or((0u64, 0u64));
+                    let (val_min, val_max) = val_min_max.into_option().unwrap_or((0u64, 0));
                     value_serializer =
                         serializer.new_u64_fast_field_with_idx(self.field, val_min, val_max, 1)?;
                     for &val in &self.vals {

src/fastfield/reader.rs

@@ -1,19 +1,19 @@
-use super::FastValue;
-use common::bitpacker::BitUnpacker;
-use common::compute_num_bits;
 use common::BinarySerializable;
+use common::bitpacker::BitUnpacker;
 use common::CompositeFile;
-use directory::ReadOnlySource;
+use common::compute_num_bits;
 use directory::{Directory, RAMDirectory, WritePtr};
+use directory::ReadOnlySource;
+use DocId;
 use fastfield::{FastFieldSerializer, FastFieldsWriter};
 use owning_ref::OwningRef;
-use schema::SchemaBuilder;
 use schema::FAST;
+use schema::SchemaBuilder;
 use std::collections::HashMap;
 use std::marker::PhantomData;
 use std::mem;
 use std::path::Path;
-use DocId;
+use super::FastValue;
 
 /// Trait for accessing a fastfield.
 ///
@@ -67,20 +67,12 @@ impl<Item: FastValue> FastFieldReader<Item> {
     /// associated with the `DocId` going from
     /// `start` to `start + output.len()`.
     ///
-    /// Regardless of the type of `Item`, this method works
-    /// - transmuting the output array
-    /// - extracting the `Item`s as if they were `u64`
-    /// - possibly converting the `u64` value to the right type.
-    ///
     /// # Panics
     ///
     /// May panic if `start + output.len()` is greater than
     /// the segment's `maxdoc`.
-    ///
-    // TODO change start to `u64`.
-    // For multifastfield, start is an index in a second fastfield, not a `DocId`
     pub fn get_range(&self, start: u32, output: &mut [Item]) {
-        let output_u64: &mut [u64] = unsafe { mem::transmute(output) }; // ok: Item is either `u64` or `i64`
+        let output_u64: &mut [u64] = unsafe { mem::transmute(output) };
         self.bit_unpacker.get_range(start, output_u64);
         for out in output_u64.iter_mut() {
             *out = Item::from_u64(*out + self.min_value_u64).as_u64();

src/fastfield/serializer.rs

@@ -1,10 +1,10 @@
-use common::bitpacker::BitPacker;
-use common::compute_num_bits;
 use common::BinarySerializable;
-use common::CompositeWrite;
-use common::CountingWriter;
 use directory::WritePtr;
 use schema::Field;
+use common::bitpacker::BitPacker;
+use common::compute_num_bits;
+use common::CountingWriter;
+use common::CompositeWrite;
 use std::io::{self, Write};
 
 /// `FastFieldSerializer` is in charge of serializing
@@ -61,16 +61,6 @@ impl FastFieldSerializer {
         FastSingleFieldSerializer::open(field_write, min_value, max_value)
     }
 
-    /// Start serializing a new [u8] fast field
-    pub fn new_bytes_fast_field_with_idx(
-        &mut self,
-        field: Field,
-        idx: usize,
-    ) -> io::Result<FastBytesFieldSerializer<CountingWriter<WritePtr>>> {
-        let field_write = self.composite_write.for_field_with_idx(field, idx);
-        FastBytesFieldSerializer::open(field_write)
-    }
-
     /// Closes the serializer
     ///
     /// After this call the data must be persistently save on disk.
@@ -87,20 +77,11 @@ pub struct FastSingleFieldSerializer<'a, W: Write + 'a> {
 }
 
 impl<'a, W: Write> FastSingleFieldSerializer<'a, W> {
-    /// Creates a new fast field serializer.
-    ///
-    /// The serializer in fact encode the values by bitpacking
-    /// `(val - min_value)`.
-    ///
-    /// It requires a `min_value` and a `max_value` to compute
-    /// compute the minimum number of bits required to encode
-    /// values.
     fn open(
         write: &'a mut W,
         min_value: u64,
         max_value: u64,
     ) -> io::Result<FastSingleFieldSerializer<'a, W>> {
-        assert!(min_value <= max_value);
         min_value.serialize(write)?;
         let amplitude = max_value - min_value;
         amplitude.serialize(write)?;
@@ -126,21 +107,3 @@ impl<'a, W: Write> FastSingleFieldSerializer<'a, W> {
         self.bit_packer.close(&mut self.write)
     }
 }
-
-pub struct FastBytesFieldSerializer<'a, W: Write + 'a> {
-    write: &'a mut W,
-}
-
-impl<'a, W: Write> FastBytesFieldSerializer<'a, W> {
-    fn open(write: &'a mut W) -> io::Result<FastBytesFieldSerializer<'a, W>> {
-        Ok(FastBytesFieldSerializer { write })
-    }
-
-    pub fn write_all(&mut self, vals: &[u8]) -> io::Result<()> {
-        self.write.write_all(vals)
-    }
-
-    pub fn flush(&mut self) -> io::Result<()> {
-        self.write.flush()
-    }
-}

src/fastfield/writer.rs

@@ -1,19 +1,18 @@
-use super::multivalued::MultiValueIntFastFieldWriter;
-use common;
-use common::BinarySerializable;
-use common::VInt;
-use fastfield::{BytesFastFieldWriter, FastFieldSerializer};
-use postings::UnorderedTermId;
-use schema::{Cardinality, Document, Field, FieldType, Schema};
-use std::collections::HashMap;
+use schema::{Cardinality, Document, Field, Schema};
+use fastfield::FastFieldSerializer;
 use std::io;
-use termdict::TermOrdinal;
+use schema::FieldType;
+use common;
+use common::VInt;
+use std::collections::HashMap;
+use postings::UnorderedTermId;
+use super::multivalued::MultiValueIntFastFieldWriter;
+use common::BinarySerializable;
 
 /// The fastfieldswriter regroup all of the fast field writers.
 pub struct FastFieldsWriter {
     single_value_writers: Vec<IntFastFieldWriter>,
     multi_values_writers: Vec<MultiValueIntFastFieldWriter>,
-    bytes_value_writers: Vec<BytesFastFieldWriter>,
 }
 
 impl FastFieldsWriter {
@@ -21,7 +20,6 @@ impl FastFieldsWriter {
     pub fn from_schema(schema: &Schema) -> FastFieldsWriter {
         let mut single_value_writers = Vec::new();
         let mut multi_values_writers = Vec::new();
-        let mut bytes_value_writers = Vec::new();
 
         for (field_id, field_entry) in schema.fields().iter().enumerate() {
             let field = Field(field_id as u32);
@@ -49,17 +47,12 @@ impl FastFieldsWriter {
                     let fast_field_writer = MultiValueIntFastFieldWriter::new(field, true);
                     multi_values_writers.push(fast_field_writer);
                 }
-                FieldType::Bytes => {
-                    let fast_field_writer = BytesFastFieldWriter::new(field);
-                    bytes_value_writers.push(fast_field_writer);
-                }
                 _ => {}
             }
         }
         FastFieldsWriter {
             single_value_writers,
             multi_values_writers,
-            bytes_value_writers,
         }
     }
 
@@ -75,36 +68,24 @@ impl FastFieldsWriter {
     ///
     /// Returns None if the field does not exist, or is not
     /// configured as a multivalued fastfield in the schema.
-    pub fn get_multivalue_writer(
+    pub(crate) fn get_multivalue_writer(
         &mut self,
         field: Field,
     ) -> Option<&mut MultiValueIntFastFieldWriter> {
         // TODO optimize
+        // TODO expose for users
         self.multi_values_writers
             .iter_mut()
             .find(|multivalue_writer| multivalue_writer.field() == field)
     }
 
-    /// Returns the bytes fast field writer for the given field.
-    ///
-    /// Returns None if the field does not exist, or is not
-    /// configured as a bytes fastfield in the schema.
-    pub fn get_bytes_writer(&mut self, field: Field) -> Option<&mut BytesFastFieldWriter> {
-        // TODO optimize
-        self.bytes_value_writers
-            .iter_mut()
-            .find(|field_writer| field_writer.field() == field)
-    }
-
     /// Indexes all of the fastfields of a new document.
     pub fn add_document(&mut self, doc: &Document) {
         for field_writer in &mut self.single_value_writers {
             field_writer.add_document(doc);
         }
         for field_writer in &mut self.multi_values_writers {
-            field_writer.add_document(doc);
-        }
-        for field_writer in &mut self.bytes_value_writers {
+            field_writer.next_doc();
             field_writer.add_document(doc);
         }
     }
@@ -114,7 +95,7 @@ impl FastFieldsWriter {
     pub fn serialize(
         &self,
         serializer: &mut FastFieldSerializer,
-        mapping: &HashMap<Field, HashMap<UnorderedTermId, TermOrdinal>>,
+        mapping: &HashMap<Field, HashMap<UnorderedTermId, usize>>,
     ) -> io::Result<()> {
         for field_writer in &self.single_value_writers {
             field_writer.serialize(serializer)?;
@@ -123,9 +104,6 @@ impl FastFieldsWriter {
             let field = field_writer.field();
             field_writer.serialize(serializer, mapping.get(&field))?;
         }
-        for field_writer in &self.bytes_value_writers {
-            field_writer.serialize(serializer)?;
-        }
         Ok(())
     }
 }

src/fieldnorm/code.rs

@@ -1,8 +1,10 @@
+
 #[inline(always)]
 pub fn id_to_fieldnorm(id: u8) -> u32 {
     FIELD_NORMS_TABLE[id as usize]
 }
 
+
 #[inline(always)]
 pub fn fieldnorm_to_id(fieldnorm: u32) -> u8 {
     FIELD_NORMS_TABLE
@@ -10,34 +12,45 @@ pub fn fieldnorm_to_id(fieldnorm: u32) -> u8 {
         .unwrap_or_else(|idx| idx - 1) as u8
 }
 
+
 pub const FIELD_NORMS_TABLE: [u32; 256] = [
-    0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25,
-    26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 42, 44, 46, 48, 50, 52, 54, 56, 60,
-    64, 68, 72, 76, 80, 84, 88, 96, 104, 112, 120, 128, 136, 144, 152, 168, 184, 200, 216, 232,
-    248, 264, 280, 312, 344, 376, 408, 440, 472, 504, 536, 600, 664, 728, 792, 856, 920, 984, 1048,
-    1176, 1304, 1432, 1560, 1688, 1816, 1944, 2072, 2328, 2584, 2840, 3096, 3352, 3608, 3864, 4120,
-    4632, 5144, 5656, 6168, 6680, 7192, 7704, 8216, 9240, 10264, 11288, 12312, 13336, 14360, 15384,
-    16408, 18456, 20504, 22552, 24600, 26648, 28696, 30744, 32792, 36888, 40984, 45080, 49176,
-    53272, 57368, 61464, 65560, 73752, 81944, 90136, 98328, 106520, 114712, 122904, 131096, 147480,
-    163864, 180248, 196632, 213016, 229400, 245784, 262168, 294936, 327704, 360472, 393240, 426008,
-    458776, 491544, 524312, 589848, 655384, 720920, 786456, 851992, 917528, 983064, 1048600,
-    1179672, 1310744, 1441816, 1572888, 1703960, 1835032, 1966104, 2097176, 2359320, 2621464,
-    2883608, 3145752, 3407896, 3670040, 3932184, 4194328, 4718616, 5242904, 5767192, 6291480,
-    6815768, 7340056, 7864344, 8388632, 9437208, 10485784, 11534360, 12582936, 13631512, 14680088,
-    15728664, 16777240, 18874392, 20971544, 23068696, 25165848, 27263000, 29360152, 31457304,
-    33554456, 37748760, 41943064, 46137368, 50331672, 54525976, 58720280, 62914584, 67108888,
-    75497496, 83886104, 92274712, 100663320, 109051928, 117440536, 125829144, 134217752, 150994968,
-    167772184, 184549400, 201326616, 218103832, 234881048, 251658264, 268435480, 301989912,
-    335544344, 369098776, 402653208, 436207640, 469762072, 503316504, 536870936, 603979800,
-    671088664, 738197528, 805306392, 872415256, 939524120, 1006632984, 1073741848, 1207959576,
-    1342177304, 1476395032, 1610612760, 1744830488, 1879048216, 2013265944,
+    0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
+    16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
+    32, 33, 34, 35, 36, 37, 38, 39, 40, 42, 44, 46, 48, 50, 52, 54,
+    56, 60, 64, 68, 72, 76, 80, 84, 88, 96, 104, 112, 120, 128, 136, 144,
+    152, 168, 184, 200, 216, 232, 248, 264, 280, 312, 344, 376, 408, 440, 472, 504,
+    536, 600, 664, 728, 792, 856, 920, 984,
+    1048, 1176, 1304, 1432, 1560, 1688, 1816, 1944,
+    2072, 2328, 2584, 2840, 3096, 3352, 3608, 3864, 4120,
+    4632, 5144, 5656, 6168, 6680, 7192, 7704, 8216, 9240,
+    10264, 11288, 12312, 13336, 14360, 15384,
+    16408, 18456, 20504, 22552, 24600, 26648, 28696, 30744,
+    32792, 36888, 40984, 45080, 49176, 53272, 57368, 61464,
+    65560, 73752, 81944, 90136, 98328, 106520, 114712, 122904, 131096, 147480,
+    163864,  180248,  196632,  213016,  229400,  245784,  262168,
+    294936,  327704,  360472,  393240,  426008,  458776,
+    491544, 524312, 589848, 655384, 720920, 786456, 851992, 917528,
+    983064, 1048600, 1179672, 1310744, 1441816, 1572888, 1703960, 1835032,
+    1966104, 2097176, 2359320, 2621464, 2883608, 3145752, 3407896, 3670040, 3932184,
+    4194328, 4718616, 5242904, 5767192, 6291480, 6815768, 7340056, 7864344, 8388632, 9437208,
+    10485784, 11534360, 12582936, 13631512, 14680088, 15728664, 16777240, 18874392, 20971544,
+    23068696, 25165848, 27263000, 29360152, 31457304, 33554456, 37748760, 41943064,
+    46137368, 50331672, 54525976, 58720280, 62914584, 67108888, 75497496, 83886104,
+    92274712, 100663320, 109051928, 117440536, 125829144, 134217752, 150994968, 167772184,
+    184549400, 201326616, 218103832, 234881048, 251658264, 268435480, 301989912, 335544344,
+    369098776, 402653208, 436207640, 469762072, 503316504, 536870936, 603979800, 671088664,
+    738197528, 805306392, 872415256, 939524120, 1006632984, 1073741848, 1207959576, 1342177304,
+    1476395032, 1610612760, 1744830488, 1879048216, 2013265944
 ];
 
+
+
 #[cfg(test)]
 mod tests {
 
     use super::{fieldnorm_to_id, id_to_fieldnorm, FIELD_NORMS_TABLE};
 
+
     #[test]
     fn test_decode_code() {
         assert_eq!(fieldnorm_to_id(0), 0);
@@ -90,4 +103,4 @@ mod tests {
             assert_eq!(FIELD_NORMS_TABLE[i], decode_fieldnorm_byte(i as u8));
         }
     }
-}
+}

src/fieldnorm/mod.rs

@@ -17,12 +17,13 @@
 //!
 //! This trick is used by the [BM25 similarity]().
 mod code;
-mod reader;
 mod serializer;
 mod writer;
+mod reader;
 
 pub use self::reader::FieldNormReader;
-pub use self::serializer::FieldNormsSerializer;
 pub use self::writer::FieldNormsWriter;
+pub use self::serializer::FieldNormsSerializer;
 
 use self::code::{fieldnorm_to_id, id_to_fieldnorm};
+

src/fieldnorm/reader.rs

@@ -1,7 +1,8 @@
-use super::{fieldnorm_to_id, id_to_fieldnorm};
+use super::{id_to_fieldnorm, fieldnorm_to_id};
 use directory::ReadOnlySource;
 use DocId;
 
+
 /// Reads the fieldnorm associated to a document.
 /// The fieldnorm represents the length associated to
 /// a given Field of a given document.
@@ -20,13 +21,16 @@ use DocId;
 /// precompute computationally expensive functions of the fieldnorm
 /// in a very short array.
 pub struct FieldNormReader {
-    data: ReadOnlySource,
+    data: ReadOnlySource
 }
 
 impl FieldNormReader {
+
     /// Opens a field norm reader given its data source.
     pub fn open(data: ReadOnlySource) -> Self {
-        FieldNormReader { data }
+        FieldNormReader {
+            data
+        }
     }
 
     /// Returns the `fieldnorm` associated to a doc id.
@@ -67,13 +71,12 @@ impl FieldNormReader {
 #[cfg(test)]
 impl From<Vec<u32>> for FieldNormReader {
     fn from(field_norms: Vec<u32>) -> FieldNormReader {
-        let field_norms_id = field_norms
-            .into_iter()
+        let field_norms_id = field_norms.into_iter()
             .map(FieldNormReader::fieldnorm_to_id)
             .collect::<Vec<u8>>();
         let field_norms_data = ReadOnlySource::from(field_norms_id);
         FieldNormReader {
-            data: field_norms_data,
+            data: field_norms_data
         }
     }
-}
+}

src/fieldnorm/serializer.rs

@@ -1,24 +1,26 @@
-use common::CompositeWrite;
 use directory::WritePtr;
-use schema::Field;
 use std::io;
+use common::CompositeWrite;
+use schema::Field;
 use std::io::Write;
 
-/// The fieldnorms serializer is in charge of
-/// the serialization of field norms for all fields.
+
 pub struct FieldNormsSerializer {
     composite_write: CompositeWrite,
 }
 
 impl FieldNormsSerializer {
+
     /// Constructor
     pub fn from_write(write: WritePtr) -> io::Result<FieldNormsSerializer> {
         // just making room for the pointer to header.
         let composite_write = CompositeWrite::wrap(write);
-        Ok(FieldNormsSerializer { composite_write })
+        Ok(FieldNormsSerializer {
+            composite_write
+        })
     }
 
-    /// Serialize the given field
+
     pub fn serialize_field(&mut self, field: Field, fieldnorms_data: &[u8]) -> io::Result<()> {
         let write = self.composite_write.for_field(field);
         write.write_all(fieldnorms_data)?;
@@ -26,9 +28,10 @@ impl FieldNormsSerializer {
         Ok(())
     }
 
-    /// Clean up / flush / close
     pub fn close(self) -> io::Result<()> {
         self.composite_write.close()?;
         Ok(())
     }
+
 }
+

src/fieldnorm/writer.rs

@@ -1,36 +1,30 @@
 use DocId;
 
-use super::fieldnorm_to_id;
-use super::FieldNormsSerializer;
 use schema::Field;
-use schema::Schema;
+use super::FieldNormsSerializer;
 use std::io;
+use schema::Schema;
+use super::fieldnorm_to_id;
 
-/// The `FieldNormsWriter` is in charge of tracking the fieldnorm byte
-/// of each document for each field with field norms.
-///
-/// `FieldNormsWriter` stores a Vec<u8> for each tracked field, using a
-/// byte per document per field.
 pub struct FieldNormsWriter {
     fields: Vec<Field>,
-    fieldnorms_buffer: Vec<Vec<u8>>,
+    fieldnorms_buffer: Vec<Vec<u8>>
 }
 
 impl FieldNormsWriter {
-    /// Returns the fields that should have field norms computed
-    /// according to the given schema.
-    pub(crate) fn fields_with_fieldnorm(schema: &Schema) -> Vec<Field> {
+
+    pub fn fields_with_fieldnorm(schema: &Schema) -> Vec<Field> {
         schema
             .fields()
             .iter()
             .enumerate()
-            .filter(|&(_, field_entry)| field_entry.is_indexed())
+            .filter(|&(_, field_entry)| {
+                field_entry.is_indexed()
+            })
             .map(|(field, _)| Field(field as u32))
             .collect::<Vec<Field>>()
     }
 
-    /// Initialize with state for tracking the field norm fields
-    /// specified in the schema.
     pub fn for_schema(schema: &Schema) -> FieldNormsWriter {
         let fields = FieldNormsWriter::fields_with_fieldnorm(schema);
         let max_field = fields
@@ -41,40 +35,26 @@ impl FieldNormsWriter {
             .unwrap_or(0);
         FieldNormsWriter {
             fields,
-            fieldnorms_buffer: (0..max_field).map(|_| Vec::new()).collect::<Vec<_>>(),
+            fieldnorms_buffer: (0..max_field)
+                .map(|_| Vec::new())
+                .collect::<Vec<_>>()
         }
     }
 
-    /// Ensure that all documents in 0..max_doc have a byte associated with them
-    /// in each of the fieldnorm vectors.
-    ///
-    /// Will extend with 0-bytes for documents that have not been seen.
     pub fn fill_up_to_max_doc(&mut self, max_doc: DocId) {
         for &field in self.fields.iter() {
             self.fieldnorms_buffer[field.0 as usize].resize(max_doc as usize, 0u8);
         }
     }
 
-    /// Set the fieldnorm byte for the given document for the given field.
-    ///
-    /// Will internally convert the u32 `fieldnorm` value to the appropriate byte
-    /// to approximate the field norm in less space.
-    ///
-    /// * doc       - the document id
-    /// * field     - the field being set
-    /// * fieldnorm - the number of terms present in document `doc` in field `field`
     pub fn record(&mut self, doc: DocId, field: Field, fieldnorm: u32) {
         let fieldnorm_buffer: &mut Vec<u8> = &mut self.fieldnorms_buffer[field.0 as usize];
-        assert!(
-            fieldnorm_buffer.len() <= doc as usize,
-            "Cannot register a given fieldnorm twice"
-        );
+        assert!(fieldnorm_buffer.len() <= doc as usize, "Cannot register a given fieldnorm twice");
         // we fill intermediary `DocId` as  having a fieldnorm of 0.
         fieldnorm_buffer.resize(doc as usize + 1, 0u8);
         fieldnorm_buffer[doc as usize] = fieldnorm_to_id(fieldnorm);
     }
 
-    /// Serialize the seen fieldnorm values to the serializer for all fields.
     pub fn serialize(&self, fieldnorms_serializer: &mut FieldNormsSerializer) -> io::Result<()> {
         for &field in self.fields.iter() {
             let fieldnorm_values: &[u8] = &self.fieldnorms_buffer[field.0 as usize][..];
@@ -82,4 +62,4 @@ impl FieldNormsWriter {
         }
         Ok(())
     }
-}
+}

src/functional_test.rs

@@ -1,10 +1,10 @@
-use rand::thread_rng;
 use std::collections::HashSet;
+use rand::thread_rng;
 
-use rand::distributions::{IndependentSample, Range};
 use schema::*;
 use Index;
 use Searcher;
+use rand::distributions::{IndependentSample, Range};
 
 fn check_index_content(searcher: &Searcher, vals: &HashSet<u64>) {
     assert!(searcher.segment_readers().len() < 20);
@@ -13,7 +13,7 @@ fn check_index_content(searcher: &Searcher, vals: &HashSet<u64>) {
 
 #[test]
 #[ignore]
-#[cfg(feature = "mmap")]
+#[cfg(feature="mmap")]
 fn test_indexing() {
     let mut schema_builder = SchemaBuilder::default();
 

src/indexer/delete_queue.rs

@@ -1,7 +1,7 @@
 use super::operation::DeleteOperation;
+use std::sync::{Arc, RwLock};
 use std::mem;
 use std::ops::DerefMut;
-use std::sync::{Arc, RwLock};
 
 // The DeleteQueue is similar in conceptually to a multiple
 // consumer single producer broadcast channel.

src/indexer/directory_lock.rs

@@ -1,6 +1,6 @@
-use core::LOCKFILE_FILEPATH;
-use directory::error::OpenWriteError;
 use Directory;
+use directory::error::OpenWriteError;
+use core::LOCKFILE_FILEPATH;
 
 /// The directory lock is a mechanism used to
 /// prevent the creation of two [`IndexWriter`](struct.IndexWriter.html)

src/indexer/index_writer.rs

@@ -1,6 +1,3 @@
-use super::operation::AddOperation;
-use super::segment_updater::SegmentUpdater;
-use super::PreparedCommit;
 use bit_set::BitSet;
 use chan;
 use core::Index;
@@ -9,28 +6,32 @@ use core::SegmentComponent;
 use core::SegmentId;
 use core::SegmentMeta;
 use core::SegmentReader;
-use datastruct::stacker::hashmap::split_memory;
+use indexer::stamper::Stamper;
 use datastruct::stacker::Heap;
 use directory::FileProtection;
-use docset::DocSet;
 use error::{Error, ErrorKind, Result, ResultExt};
 use fastfield::write_delete_bitset;
-use futures::sync::oneshot::Receiver;
 use indexer::delete_queue::{DeleteCursor, DeleteQueue};
+use futures::Canceled;
+use datastruct::stacker::hashmap::split_memory;
+use futures::Future;
 use indexer::doc_opstamp_mapping::DocToOpstampMapping;
-use indexer::operation::DeleteOperation;
-use indexer::stamper::Stamper;
-use indexer::DirectoryLock;
 use indexer::MergePolicy;
+use indexer::operation::DeleteOperation;
 use indexer::SegmentEntry;
 use indexer::SegmentWriter;
-use schema::Document;
+use docset::DocSet;
 use schema::IndexRecordOption;
+use schema::Document;
 use schema::Term;
 use std::mem;
 use std::mem::swap;
-use std::thread;
 use std::thread::JoinHandle;
+use indexer::DirectoryLock;
+use super::operation::AddOperation;
+use super::segment_updater::SegmentUpdater;
+use super::PreparedCommit;
+use std::thread;
 
 // Size of the margin for the heap. A segment is closed when the remaining memory
 // in the heap goes below MARGIN_IN_BYTES.
@@ -80,6 +81,10 @@ pub struct IndexWriter {
     committed_opstamp: u64,
 }
 
+// IndexWriter cannot be sent to another thread.
+impl !Send for IndexWriter {}
+impl !Sync for IndexWriter {}
+
 /// Open a new index writer. Attempts to acquire a lockfile.
 ///
 /// The lockfile should be deleted on drop, but it is possible
@@ -195,6 +200,7 @@ pub fn advance_deletes(
     target_opstamp: u64,
 ) -> Result<Option<FileProtection>> {
     let mut file_protect: Option<FileProtection> = None;
+
     {
         if let Some(previous_opstamp) = segment_entry.meta().delete_opstamp() {
             // We are already up-to-date here.
@@ -235,6 +241,7 @@ pub fn advance_deletes(
         }
     }
     segment_entry.set_meta(segment.meta().clone());
+
     Ok(file_protect)
 }
 
@@ -441,7 +448,10 @@ impl IndexWriter {
     }
 
     /// Merges a given list of segments
-    pub fn merge(&mut self, segment_ids: &[SegmentId]) -> Receiver<SegmentMeta> {
+    pub fn merge(
+        &mut self,
+        segment_ids: &[SegmentId],
+    ) -> impl Future<Item = SegmentMeta, Error = Canceled> {
         self.segment_updater.start_merge(segment_ids)
     }
 
@@ -637,12 +647,12 @@ impl IndexWriter {
 #[cfg(test)]
 mod tests {
 
-    use env_logger;
-    use error::*;
     use indexer::NoMergePolicy;
     use schema::{self, Document};
     use Index;
     use Term;
+    use error::*;
+    use env_logger;
 
     #[test]
     fn test_lockfile_stops_duplicates() {
@@ -665,7 +675,7 @@ mod tests {
             "LogMergePolicy { min_merge_size: 8, min_layer_size: 10000, \
              level_log_size: 0.75 }"
         );
-        let merge_policy = Box::new(NoMergePolicy::default());
+        let merge_policy = box NoMergePolicy::default();
         index_writer.set_merge_policy(merge_policy);
         assert_eq!(
             format!("{:?}", index_writer.get_merge_policy()),

src/indexer/log_merge_policy.rs

@@ -82,7 +82,7 @@ impl MergePolicy for LogMergePolicy {
     }
 
     fn box_clone(&self) -> Box<MergePolicy> {
-        Box::new(self.clone())
+        box self.clone()
     }
 }
 
@@ -99,8 +99,8 @@ impl Default for LogMergePolicy {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use core::{SegmentId, SegmentMeta};
     use indexer::merge_policy::MergePolicy;
+    use core::{SegmentId, SegmentMeta};
 
     fn test_merge_policy() -> LogMergePolicy {
         let mut log_merge_policy = LogMergePolicy::default();

src/indexer/merge_policy.rs

@@ -1,7 +1,7 @@
 use core::SegmentId;
 use core::SegmentMeta;
-use std::fmt::Debug;
 use std::marker;
+use std::fmt::Debug;
 
 /// Set of segment suggested for a merge.
 #[derive(Debug, Clone)]
@@ -37,7 +37,7 @@ impl MergePolicy for NoMergePolicy {
     }
 
     fn box_clone(&self) -> Box<MergePolicy> {
-        Box::new(NoMergePolicy)
+        box NoMergePolicy
     }
 }
 
@@ -69,7 +69,7 @@ pub mod tests {
         }
 
         fn box_clone(&self) -> Box<MergePolicy> {
-            Box::new(MergeWheneverPossible)
+            box MergeWheneverPossible
         }
     }
 }

src/indexer/mod.rs

@@ -1,29 +1,29 @@
-pub mod delete_queue;
-mod directory_lock;
-mod doc_opstamp_mapping;
 pub mod index_writer;
-mod log_merge_policy;
-pub mod merge_policy;
-pub mod merger;
-pub mod operation;
-mod prepared_commit;
-mod segment_entry;
-mod segment_manager;
-mod segment_register;
 pub mod segment_serializer;
-pub mod segment_updater;
+pub mod merger;
+pub mod merge_policy;
+mod log_merge_policy;
+mod segment_register;
 mod segment_writer;
+mod segment_manager;
+pub mod delete_queue;
+pub mod segment_updater;
+mod directory_lock;
+mod segment_entry;
+mod doc_opstamp_mapping;
+pub mod operation;
 mod stamper;
+mod prepared_commit;
 
-pub(crate) use self::directory_lock::DirectoryLock;
+pub use self::prepared_commit::PreparedCommit;
+pub use self::segment_entry::{SegmentEntry, SegmentState};
+pub use self::segment_serializer::SegmentSerializer;
+pub use self::segment_writer::SegmentWriter;
 pub use self::index_writer::IndexWriter;
 pub use self::log_merge_policy::LogMergePolicy;
 pub use self::merge_policy::{MergeCandidate, MergePolicy, NoMergePolicy};
-pub use self::prepared_commit::PreparedCommit;
-pub use self::segment_entry::{SegmentEntry, SegmentState};
 pub use self::segment_manager::SegmentManager;
-pub use self::segment_serializer::SegmentSerializer;
-pub use self::segment_writer::SegmentWriter;
+pub(crate) use self::directory_lock::DirectoryLock;
 
 /// Alias for the default merge policy, which is the `LogMergePolicy`.
 pub type DefaultMergePolicy = LogMergePolicy;

src/indexer/prepared_commit.rs

@@ -1,5 +1,5 @@
-use super::IndexWriter;
 use Result;
+use super::IndexWriter;
 
 /// A prepared commit
 pub struct PreparedCommit<'a> {
@@ -13,7 +13,7 @@ impl<'a> PreparedCommit<'a> {
         PreparedCommit {
             index_writer,
             payload: None,
-            opstamp,
+            opstamp
         }
     }
 

src/indexer/segment_entry.rs

@@ -1,7 +1,7 @@
-use bit_set::BitSet;
-use core::SegmentId;
 use core::SegmentMeta;
+use bit_set::BitSet;
 use indexer::delete_queue::DeleteCursor;
+use core::SegmentId;
 use std::fmt;
 
 #[derive(Clone, Copy, PartialEq, Eq, Debug)]

src/indexer/segment_manager.rs

@@ -1,14 +1,14 @@
 use super::segment_register::SegmentRegister;
-use core::SegmentId;
+use std::sync::RwLock;
 use core::SegmentMeta;
 use core::{LOCKFILE_FILEPATH, META_FILEPATH};
-use indexer::delete_queue::DeleteCursor;
+use core::SegmentId;
 use indexer::SegmentEntry;
-use std::collections::hash_set::HashSet;
-use std::fmt::{self, Debug, Formatter};
 use std::path::PathBuf;
-use std::sync::RwLock;
+use std::collections::hash_set::HashSet;
 use std::sync::{RwLockReadGuard, RwLockWriteGuard};
+use std::fmt::{self, Debug, Formatter};
+use indexer::delete_queue::DeleteCursor;
 
 #[derive(Default)]
 struct SegmentRegisters {

src/indexer/segment_register.rs

@@ -1,10 +1,10 @@
 use core::SegmentId;
-use core::SegmentMeta;
-use indexer::delete_queue::DeleteCursor;
-use indexer::segment_entry::SegmentEntry;
 use std::collections::HashMap;
+use core::SegmentMeta;
 use std::fmt;
 use std::fmt::{Debug, Formatter};
+use indexer::segment_entry::SegmentEntry;
+use indexer::delete_queue::DeleteCursor;
 
 /// The segment register keeps track
 /// of the list of segment, their size as well
@@ -113,11 +113,11 @@ impl SegmentRegister {
 
 #[cfg(test)]
 mod tests {
-    use super::*;
+    use indexer::SegmentState;
     use core::SegmentId;
     use core::SegmentMeta;
     use indexer::delete_queue::*;
-    use indexer::SegmentState;
+    use super::*;
 
     fn segment_ids(segment_register: &SegmentRegister) -> Vec<SegmentId> {
         segment_register

src/indexer/segment_serializer.rs

@@ -3,9 +3,9 @@ use Result;
 use core::Segment;
 use core::SegmentComponent;
 use fastfield::FastFieldSerializer;
+use store::StoreWriter;
 use fieldnorm::FieldNormsSerializer;
 use postings::InvertedIndexSerializer;
-use store::StoreWriter;
 
 /// Segment serializer is in charge of laying out on disk
 /// the data accumulated and sorted by the `SegmentWriter`.
@@ -47,7 +47,7 @@ impl SegmentSerializer {
     }
 
     /// Accessor to the field norm serializer.
-    pub fn get_fieldnorms_serializer(&mut self) -> &mut FieldNormsSerializer {
+    pub fn  get_fieldnorms_serializer(&mut self) -> &mut FieldNormsSerializer {
         &mut self.fieldnorms_serializer
     }
 

src/indexer/segment_updater.rs

@@ -1,40 +1,40 @@
-use super::segment_manager::{get_mergeable_segments, SegmentManager};
 use core::Index;
 use core::IndexMeta;
+use core::META_FILEPATH;
 use core::Segment;
 use core::SegmentId;
 use core::SegmentMeta;
 use core::SerializableSegment;
-use core::META_FILEPATH;
 use directory::Directory;
-use directory::FileProtection;
-use error::{Error, ErrorKind, Result};
-use futures::oneshot;
-use futures::sync::oneshot::Receiver;
-use futures::Future;
-use futures_cpupool::CpuFuture;
-use futures_cpupool::CpuPool;
-use indexer::delete_queue::DeleteCursor;
-use indexer::index_writer::advance_deletes;
-use indexer::merger::IndexMerger;
 use indexer::stamper::Stamper;
+use error::{Error, ErrorKind, Result};
+use futures_cpupool::CpuPool;
+use futures::Future;
+use futures::Canceled;
+use futures::oneshot;
+use directory::FileProtection;
+use indexer::{DefaultMergePolicy, MergePolicy};
+use indexer::index_writer::advance_deletes;
 use indexer::MergeCandidate;
+use indexer::merger::IndexMerger;
 use indexer::SegmentEntry;
 use indexer::SegmentSerializer;
-use indexer::{DefaultMergePolicy, MergePolicy};
-use schema::Schema;
+use futures_cpupool::CpuFuture;
 use serde_json;
+use indexer::delete_queue::DeleteCursor;
+use schema::Schema;
 use std::borrow::BorrowMut;
 use std::collections::HashMap;
 use std::io::Write;
 use std::mem;
 use std::ops::DerefMut;
-use std::sync::atomic::Ordering;
-use std::sync::atomic::{AtomicBool, AtomicUsize};
 use std::sync::Arc;
+use std::sync::atomic::{AtomicBool, AtomicUsize};
+use std::sync::atomic::Ordering;
 use std::sync::RwLock;
 use std::thread;
 use std::thread::JoinHandle;
+use super::segment_manager::{get_mergeable_segments, SegmentManager};
 
 /// Save the index meta file.
 /// This operation is atomic :
@@ -171,7 +171,7 @@ impl SegmentUpdater {
             pool: CpuPool::new(1),
             index,
             segment_manager,
-            merge_policy: RwLock::new(Box::new(DefaultMergePolicy::default())),
+            merge_policy: RwLock::new(box DefaultMergePolicy::default()),
             merging_thread_id: AtomicUsize::default(),
             merging_threads: RwLock::new(HashMap::new()),
             generation: AtomicUsize::default(),
@@ -283,7 +283,10 @@ impl SegmentUpdater {
         }).wait()
     }
 
-    pub fn start_merge(&self, segment_ids: &[SegmentId]) -> Receiver<SegmentMeta> {
+    pub fn start_merge(
+        &self,
+        segment_ids: &[SegmentId],
+    ) -> impl Future<Item = SegmentMeta, Error = Canceled> {
         self.0.segment_manager.start_merge(segment_ids);
         let segment_updater_clone = self.clone();
 
@@ -358,9 +361,7 @@ impl SegmentUpdater {
         let committed_merge_candidates = merge_policy.compute_merge_candidates(&committed_segments);
         merge_candidates.extend_from_slice(&committed_merge_candidates[..]);
         for MergeCandidate(segment_metas) in merge_candidates {
-            if let Err(e) = self.start_merge(&segment_metas).fuse().poll() {
-                error!("The merge task failed quickly after starting: {:?}", e);
-            }
+            self.start_merge(&segment_metas);
         }
     }
 
@@ -479,9 +480,9 @@ impl SegmentUpdater {
 #[cfg(test)]
 mod tests {
 
-    use indexer::merge_policy::tests::MergeWheneverPossible;
-    use schema::*;
     use Index;
+    use schema::*;
+    use indexer::merge_policy::tests::MergeWheneverPossible;
 
     #[test]
     fn test_delete_during_merge() {
@@ -493,7 +494,7 @@ mod tests {
 
         // writing the segment
         let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
-        index_writer.set_merge_policy(Box::new(MergeWheneverPossible));
+        index_writer.set_merge_policy(box MergeWheneverPossible);
 
         {
             for _ in 0..100 {

src/indexer/segment_writer.rs

@@ -1,23 +1,23 @@
-use super::operation::AddOperation;
-use core::Segment;
-use core::SerializableSegment;
-use datastruct::stacker::Heap;
-use fastfield::FastFieldsWriter;
-use fieldnorm::FieldNormsWriter;
-use indexer::index_writer::MARGIN_IN_BYTES;
-use indexer::segment_serializer::SegmentSerializer;
-use postings::MultiFieldPostingsWriter;
-use schema::FieldType;
-use schema::Schema;
-use schema::Term;
-use schema::Value;
+use Result;
+use DocId;
 use std::io;
 use std::str;
+use schema::Schema;
+use schema::Term;
+use core::Segment;
+use core::SerializableSegment;
+use fastfield::FastFieldsWriter;
+use schema::FieldType;
+use indexer::segment_serializer::SegmentSerializer;
+use datastruct::stacker::Heap;
+use indexer::index_writer::MARGIN_IN_BYTES;
+use super::operation::AddOperation;
+use postings::MultiFieldPostingsWriter;
 use tokenizer::BoxedTokenizer;
 use tokenizer::FacetTokenizer;
 use tokenizer::{TokenStream, Tokenizer};
-use DocId;
-use Result;
+use schema::Value;
+use fieldnorm::FieldNormsWriter;
 
 /// A `SegmentWriter` is in charge of creating segment index from a
 /// documents.
@@ -35,6 +35,7 @@ pub struct SegmentWriter<'a> {
     tokenizers: Vec<Option<Box<BoxedTokenizer>>>,
 }
 
+
 impl<'a> SegmentWriter<'a> {
     /// Creates a new `SegmentWriter`
     ///
@@ -138,7 +139,8 @@ impl<'a> SegmentWriter<'a> {
                             }
                         })
                         .collect();
-                    let mut term = Term::for_field(field); // we set the Term
+                    let mut term = unsafe { Term::with_capacity(100) };
+                    term.set_field(field);
                     for facet_bytes in facets {
                         let mut unordered_term_id_opt = None;
                         let fake_str = unsafe { str::from_utf8_unchecked(facet_bytes) };
@@ -177,7 +179,8 @@ impl<'a> SegmentWriter<'a> {
                     } else {
                         0
                     };
-                    self.fieldnorms_writer.record(doc_id, field, num_tokens);
+                    self.fieldnorms_writer
+                        .record(doc_id, field, num_tokens);
                 }
                 FieldType::U64(ref int_option) => {
                     if int_option.is_indexed() {
@@ -201,9 +204,6 @@ impl<'a> SegmentWriter<'a> {
                         }
                     }
                 }
-                FieldType::Bytes => {
-                    // Do nothing. Bytes only supports fast fields.
-                }
             }
         }
         doc.filter_fields(|field| schema.get_field_entry(field).is_stored());

src/indexer/stamper.rs

@@ -1,66 +1,15 @@
-// AtomicU64 have not landed in stable.
-// For the moment let's just use AtomicUsize on
-// x86/64 bit platform, and a mutex on other platform.
+use std::sync::atomic::{AtomicUsize, Ordering};
+use std::sync::Arc;
 
-#[cfg(target = "x86_64")]
-mod archicture_impl {
+#[derive(Clone, Default)]
+pub struct Stamper(Arc<AtomicUsize>);
 
-    use std::sync::atomic::{AtomicUsize, Ordering};
-    use std::sync::Arc;
+impl Stamper {
+    pub fn new(first_opstamp: u64) -> Stamper {
+        Stamper(Arc::new(AtomicUsize::new(first_opstamp as usize)))
+    }
 
-    #[derive(Clone, Default)]
-    pub struct Stamper(Arc<AtomicU64>);
-
-    impl Stamper {
-        pub fn new(first_opstamp: u64) -> Stamper {
-            Stamper(Arc::new(AtomicU64::new(first_opstamp)))
-        }
-
-        pub fn stamp(&self) -> u64 {
-            self.0.fetch_add(1u64, Ordering::SeqCst) as u64
-        }
-    }
-}
-
-#[cfg(not(target = "x86_64"))]
-mod archicture_impl {
-
-    use std::sync::{Arc, Mutex};
-
-    #[derive(Clone, Default)]
-    pub struct Stamper(Arc<Mutex<u64>>);
-
-    impl Stamper {
-        pub fn new(first_opstamp: u64) -> Stamper {
-            Stamper(Arc::new(Mutex::new(first_opstamp)))
-        }
-
-        pub fn stamp(&self) -> u64 {
-            let mut guard = self.0.lock().expect("Failed to lock the stamper");
-            let previous_val = *guard;
-            *guard = previous_val + 1;
-            previous_val
-        }
-    }
-}
-
-pub use self::archicture_impl::Stamper;
-
-#[cfg(test)]
-mod test {
-
-    use super::Stamper;
-
-    #[test]
-    fn test_stamper() {
-        let stamper = Stamper::new(7u64);
-        assert_eq!(stamper.stamp(), 7u64);
-        assert_eq!(stamper.stamp(), 8u64);
-
-        let stamper_clone = stamper.clone();
-        assert_eq!(stamper.stamp(), 9u64);
-
-        assert_eq!(stamper.stamp(), 10u64);
-        assert_eq!(stamper_clone.stamp(), 11u64);
+    pub fn stamp(&self) -> u64 {
+        self.0.fetch_add(1, Ordering::SeqCst) as u64
     }
 }

src/lib.rs

@@ -1,7 +1,14 @@
 #![doc(html_logo_url = "http://fulmicoton.com/tantivy-logo/tantivy-logo.png")]
 #![cfg_attr(feature = "cargo-clippy", allow(module_inception))]
 #![cfg_attr(feature = "cargo-clippy", allow(inline_always))]
-#![cfg_attr(all(feature = "unstable", test), feature(test))]
+#![feature(box_syntax)]
+#![feature(optin_builtin_traits)]
+#![feature(conservative_impl_trait)]
+#![feature(collections_range)]
+#![feature(integer_atomics)]
+#![feature(drain_filter)]
+#![cfg_attr(test, feature(test))]
+#![cfg_attr(test, feature(iterator_step_by))]
 #![doc(test(attr(allow(unused_variables), deny(warnings))))]
 #![allow(unknown_lints)]
 #![allow(new_without_default)]
@@ -116,40 +123,36 @@ extern crate lazy_static;
 #[macro_use]
 extern crate serde_derive;
 
-#[cfg_attr(test, macro_use)]
-extern crate serde_json;
-
 #[macro_use]
 extern crate log;
 
 #[macro_use]
 extern crate error_chain;
 
-#[cfg(feature = "mmap")]
+#[cfg(feature="mmap")]
 extern crate atomicwrites;
-extern crate base64;
 extern crate bit_set;
-extern crate bitpacking;
 extern crate byteorder;
 extern crate chan;
 extern crate combine;
 extern crate crossbeam;
-extern crate fnv;
 extern crate fst;
 extern crate futures;
 extern crate futures_cpupool;
 extern crate itertools;
-extern crate levenshtein_automata;
-extern crate lz4;
+extern crate snap;
 extern crate num_cpus;
 extern crate owning_ref;
 extern crate regex;
 extern crate rust_stemmers;
 extern crate serde;
+extern crate serde_json;
 extern crate stable_deref_trait;
 extern crate tempdir;
+#[cfg(test)]
 extern crate tempfile;
 extern crate uuid;
+extern crate bitpacking;
 
 #[cfg(test)]
 #[macro_use]
@@ -163,8 +166,7 @@ extern crate winapi;
 
 #[cfg(test)]
 extern crate rand;
-
-#[cfg(all(test, feature = "unstable"))]
+#[cfg(test)]
 extern crate test;
 
 extern crate tinysegmenter;
@@ -183,36 +185,36 @@ pub use error::{Error, ErrorKind, ResultExt};
 /// Tantivy result.
 pub type Result<T> = std::result::Result<T, Error>;
 
-mod common;
-mod compression;
 mod core;
+mod compression;
 mod indexer;
+mod common;
 
-mod datastruct;
 #[allow(unused_doc_comment)]
 mod error;
 pub mod tokenizer;
+mod datastruct;
 
-pub mod collector;
+pub mod termdict;
+pub mod store;
+pub mod query;
 pub mod directory;
+pub mod collector;
+pub mod postings;
+pub mod schema;
 pub mod fastfield;
 pub mod fieldnorm;
-pub mod postings;
-pub mod query;
-pub mod schema;
-pub mod store;
-pub mod termdict;
 
 mod docset;
 pub use self::docset::{DocSet, SkipResult};
 
-pub use core::SegmentComponent;
-pub use core::{Index, Searcher, Segment, SegmentId, SegmentMeta};
-pub use core::{InvertedIndexReader, SegmentReader};
 pub use directory::Directory;
+pub use core::{Index, Searcher, Segment, SegmentId, SegmentMeta};
 pub use indexer::IndexWriter;
-pub use postings::Postings;
 pub use schema::{Document, Term};
+pub use core::{InvertedIndexReader, SegmentReader};
+pub use postings::Postings;
+pub use core::SegmentComponent;
 
 pub use common::{i64_to_u64, u64_to_i64};
 
@@ -228,10 +230,10 @@ pub fn version() -> &'static str {
 
 /// Defines tantivy's merging strategy
 pub mod merge_policy {
-    pub use indexer::DefaultMergePolicy;
-    pub use indexer::LogMergePolicy;
     pub use indexer::MergePolicy;
+    pub use indexer::LogMergePolicy;
     pub use indexer::NoMergePolicy;
+    pub use indexer::DefaultMergePolicy;
 }
 
 /// A `u32` identifying a document within a segment.
@@ -280,29 +282,33 @@ pub struct DocAddress(pub SegmentLocalId, pub DocId);
 mod tests {
 
     use collector::tests::TestCollector;
-    use core::SegmentReader;
-    use docset::DocSet;
-    use query::BooleanQuery;
-    use rand::distributions::{IndependentSample, Range};
-    use rand::{Rng, SeedableRng, XorShiftRng};
-    use schema::*;
     use Index;
+    use core::SegmentReader;
+    use query::BooleanQuery;
+    use schema::*;
+    use docset::DocSet;
     use IndexWriter;
     use Postings;
+    use rand::{Rng, SeedableRng, XorShiftRng};
+    use rand::distributions::{IndependentSample, Range};
 
     pub fn assert_nearly_equals(expected: f32, val: f32) {
-        assert!(
-            nearly_equals(val, expected),
-            "Got {}, expected {}.",
-            val,
-            expected
-        );
+        assert!(nearly_equals(val, expected), "Got {}, expected {}.", val, expected);
     }
 
     pub fn nearly_equals(a: f32, b: f32) -> bool {
         (a - b).abs() < 0.0005 * (a + b).abs()
     }
 
+    fn generate_array_with_seed(n: usize, ratio: f32, seed_val: u32) -> Vec<u32> {
+        let seed: &[u32; 4] = &[1, 2, 3, seed_val];
+        let mut rng: XorShiftRng = XorShiftRng::from_seed(*seed);
+        (0..u32::max_value())
+            .filter(|_| rng.next_f32() < ratio)
+            .take(n)
+            .collect()
+    }
+
     pub fn generate_nonunique_unsorted(max_value: u32, n_elems: usize) -> Vec<u32> {
         let seed: &[u32; 4] = &[1, 2, 3, 4];
         let mut rng: XorShiftRng = XorShiftRng::from_seed(*seed);
@@ -312,6 +318,10 @@ mod tests {
             .collect::<Vec<u32>>()
     }
 
+    pub fn generate_array(n: usize, ratio: f32) -> Vec<u32> {
+        generate_array_with_seed(n, ratio, 4)
+    }
+
     pub fn sample_with_seed(n: u32, ratio: f32, seed_val: u32) -> Vec<u32> {
         let seed: &[u32; 4] = &[1, 2, 3, seed_val];
         let mut rng: XorShiftRng = XorShiftRng::from_seed(*seed);
@@ -323,7 +333,7 @@ mod tests {
     }
 
     #[test]
-    #[cfg(feature = "mmap")]
+    #[cfg(feature="mmap")]
     fn test_indexing() {
         let mut schema_builder = SchemaBuilder::default();
         let text_field = schema_builder.add_text_field("text", TEXT);
@@ -449,6 +459,7 @@ mod tests {
         }
     }
 
+
     fn advance_undeleted(docset: &mut DocSet, reader: &SegmentReader) -> bool {
         while docset.advance() {
             if !reader.is_deleted(docset.doc()) {

src/postings/mod.rs

@@ -6,18 +6,20 @@ Postings module (also called inverted index)
 ///
 /// Postings, also called inverted lists, is the key datastructure
 /// to full-text search.
+
 mod postings;
-mod postings_writer;
 mod recorder;
-mod segment_postings;
 mod serializer;
+mod postings_writer;
 mod term_info;
+mod segment_postings;
 
-pub(crate) use self::postings_writer::MultiFieldPostingsWriter;
+use self::recorder::{NothingRecorder, Recorder, TFAndPositionRecorder, TermFrequencyRecorder};
 pub use self::serializer::{FieldSerializer, InvertedIndexSerializer};
+pub(crate) use self::postings_writer::MultiFieldPostingsWriter;
 
-pub use self::postings::Postings;
 pub use self::term_info::TermInfo;
+pub use self::postings::Postings;
 
 pub use self::segment_postings::{BlockSegmentPostings, SegmentPostings};
 
@@ -36,22 +38,25 @@ pub(crate) enum FreqReadingOption {
 pub mod tests {
 
     use super::*;
-    use core::Index;
-    use core::SegmentComponent;
-    use core::SegmentReader;
-    use datastruct::stacker::Heap;
     use docset::{DocSet, SkipResult};
-    use fieldnorm::FieldNormReader;
-    use indexer::operation::AddOperation;
-    use indexer::SegmentWriter;
-    use query::Scorer;
-    use rand::{Rng, SeedableRng, XorShiftRng};
-    use schema::Field;
-    use schema::IndexRecordOption;
-    use schema::{Document, SchemaBuilder, Term, INT_INDEXED, STRING, TEXT};
-    use std::iter;
     use DocId;
     use Score;
+    use query::Intersection;
+    use query::Scorer;
+    use schema::{Document, SchemaBuilder, Term, INT_INDEXED, STRING, TEXT};
+    use core::SegmentComponent;
+    use indexer::SegmentWriter;
+    use core::SegmentReader;
+    use core::Index;
+    use schema::IndexRecordOption;
+    use std::iter;
+    use datastruct::stacker::Heap;
+    use schema::Field;
+    use test::{self, Bencher};
+    use indexer::operation::AddOperation;
+    use tests;
+    use rand::{Rng, SeedableRng, XorShiftRng};
+    use fieldnorm::FieldNormReader;
 
     #[test]
     pub fn test_position_write() {
@@ -122,6 +127,7 @@ pub mod tests {
             assert_eq!(&[0, 5], &positions[..]);
         }
         {
+
             let mut postings = inverted_index
                 .read_postings(&term, IndexRecordOption::WithFreqsAndPositions)
                 .unwrap();
@@ -200,14 +206,13 @@ pub mod tests {
         {
             let segment_reader = SegmentReader::open(&segment).unwrap();
             {
-                let fieldnorm_reader = segment_reader.get_fieldnorms_reader(text_field);
+                let fieldnorm_reader = segment_reader.get_fieldnorms_reader(text_field) ;
                 assert_eq!(fieldnorm_reader.fieldnorm(0), 8 + 5);
                 assert_eq!(fieldnorm_reader.fieldnorm(1), 2);
                 for i in 2..1000 {
                     assert_eq!(
                         fieldnorm_reader.fieldnorm_id(i),
-                        FieldNormReader::fieldnorm_to_id(i + 1)
-                    );
+                        FieldNormReader::fieldnorm_to_id(i + 1) );
                 }
             }
             {
@@ -444,7 +449,7 @@ pub mod tests {
         // delete everything else
         {
             let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
-            index_writer.delete_term(term_1);
+                index_writer.delete_term(term_1);
 
             assert!(index_writer.commit().is_ok());
         }
@@ -474,23 +479,23 @@ pub mod tests {
     }
 
     lazy_static! {
-        pub static ref TERM_A: Term = {
+        static ref TERM_A: Term = {
             let field = Field(0);
             Term::from_field_text(field, "a")
         };
-        pub static ref TERM_B: Term = {
+        static ref TERM_B: Term = {
             let field = Field(0);
             Term::from_field_text(field, "b")
         };
-        pub static ref TERM_C: Term = {
+        static ref TERM_C: Term = {
             let field = Field(0);
             Term::from_field_text(field, "c")
         };
-        pub static ref TERM_D: Term = {
+        static ref TERM_D: Term = {
             let field = Field(0);
             Term::from_field_text(field, "d")
         };
-        pub static ref INDEX: Index = {
+        static ref INDEX: Index = {
             let mut schema_builder = SchemaBuilder::default();
             let text_field = schema_builder.add_text_field("text", STRING);
             let schema = schema_builder.build();
@@ -502,7 +507,7 @@ pub mod tests {
             let posting_list_size = 1_000_000;
             {
                 let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
-                for _ in 0..posting_list_size {
+                for _ in 0 .. posting_list_size {
                     let mut doc = Document::default();
                     if rng.gen_weighted_bool(15) {
                         doc.add_text(text_field, "a");
@@ -525,85 +530,6 @@ pub mod tests {
         };
     }
 
-    /// Wraps a given docset, and forward alls call but the
-    /// `.skip_next(...)`. This is useful to test that a specialized
-    /// implementation of `.skip_next(...)` is consistent
-    /// with the default implementation.
-    pub(crate) struct UnoptimizedDocSet<TDocSet: DocSet>(TDocSet);
-
-    impl<TDocSet: DocSet> UnoptimizedDocSet<TDocSet> {
-        pub fn wrap(docset: TDocSet) -> UnoptimizedDocSet<TDocSet> {
-            UnoptimizedDocSet(docset)
-        }
-    }
-
-    impl<TDocSet: DocSet> DocSet for UnoptimizedDocSet<TDocSet> {
-        fn advance(&mut self) -> bool {
-            self.0.advance()
-        }
-
-        fn doc(&self) -> DocId {
-            self.0.doc()
-        }
-
-        fn size_hint(&self) -> u32 {
-            self.0.size_hint()
-        }
-    }
-
-    impl<TScorer: Scorer> Scorer for UnoptimizedDocSet<TScorer> {
-        fn score(&mut self) -> Score {
-            self.0.score()
-        }
-    }
-
-    pub fn test_skip_against_unoptimized<F: Fn() -> Box<DocSet>>(
-        postings_factory: F,
-        targets: Vec<u32>,
-    ) {
-        for target in targets {
-            let mut postings_opt = postings_factory();
-            let mut postings_unopt = UnoptimizedDocSet::wrap(postings_factory());
-            let skip_result_opt = postings_opt.skip_next(target);
-            let skip_result_unopt = postings_unopt.skip_next(target);
-            assert_eq!(
-                skip_result_unopt, skip_result_opt,
-                "Failed while skipping to {}",
-                target
-            );
-            match skip_result_opt {
-                SkipResult::Reached => assert_eq!(postings_opt.doc(), target),
-                SkipResult::OverStep => assert!(postings_opt.doc() > target),
-                SkipResult::End => {
-                    return;
-                }
-            }
-            while postings_opt.advance() {
-                assert!(postings_unopt.advance());
-                assert_eq!(
-                    postings_opt.doc(),
-                    postings_unopt.doc(),
-                    "Failed while skipping to {}",
-                    target
-                );
-            }
-            assert!(!postings_unopt.advance());
-        }
-    }
-
-}
-
-#[cfg(all(test, feature = "unstable"))]
-mod bench {
-
-    use super::tests::*;
-    use docset::SkipResult;
-    use query::Intersection;
-    use schema::IndexRecordOption;
-    use test::{self, Bencher};
-    use tests;
-    use DocSet;
-
     #[bench]
     fn bench_segment_postings(b: &mut Bencher) {
         let searcher = INDEX.searcher();
@@ -720,4 +646,71 @@ mod bench {
             s
         });
     }
+
+    /// Wraps a given docset, and forward alls call but the
+    /// `.skip_next(...)`. This is useful to test that a specialized
+    /// implementation of `.skip_next(...)` is consistent
+    /// with the default implementation.
+    pub(crate) struct UnoptimizedDocSet<TDocSet: DocSet>(TDocSet);
+
+    impl<TDocSet: DocSet> UnoptimizedDocSet<TDocSet> {
+        pub fn wrap(docset: TDocSet) -> UnoptimizedDocSet<TDocSet> {
+            UnoptimizedDocSet(docset)
+        }
+    }
+
+    impl<TDocSet: DocSet> DocSet for UnoptimizedDocSet<TDocSet> {
+        fn advance(&mut self) -> bool {
+            self.0.advance()
+        }
+
+        fn doc(&self) -> DocId {
+            self.0.doc()
+        }
+
+        fn size_hint(&self) -> u32 {
+            self.0.size_hint()
+        }
+    }
+
+    impl<TScorer: Scorer> Scorer for UnoptimizedDocSet<TScorer> {
+        fn score(&mut self) -> Score {
+            self.0.score()
+        }
+    }
+
+    pub fn test_skip_against_unoptimized<F: Fn() -> Box<DocSet>>(
+        postings_factory: F,
+        targets: Vec<u32>,
+    ) {
+        for target in targets {
+            let mut postings_opt = postings_factory();
+            let mut postings_unopt = UnoptimizedDocSet::wrap(postings_factory());
+            let skip_result_opt = postings_opt.skip_next(target);
+            let skip_result_unopt = postings_unopt.skip_next(target);
+            assert_eq!(
+                skip_result_unopt, skip_result_opt,
+                "Failed while skipping to {}",
+                target
+            );
+            match skip_result_opt {
+                SkipResult::Reached => assert_eq!(postings_opt.doc(), target),
+                SkipResult::OverStep => assert!(postings_opt.doc() > target),
+                SkipResult::End => {
+                    return;
+                }
+            }
+            while postings_opt.advance() {
+                assert!(postings_unopt.advance());
+                assert_eq!(
+                    postings_opt.doc(),
+                    postings_unopt.doc(),
+                    "Failed while skipping to {}",
+                    target
+                );
+            }
+            assert!(!postings_unopt.advance());
+        }
+    }
+
 }

src/postings/postings_writer.rs

@@ -1,18 +1,21 @@
-use datastruct::stacker::{Heap, TermHashMap};
-use postings::recorder::{NothingRecorder, Recorder, TFAndPositionRecorder, TermFrequencyRecorder};
-use postings::UnorderedTermId;
+use DocId;
+use schema::Term;
 use postings::{FieldSerializer, InvertedIndexSerializer};
-use schema::IndexRecordOption;
-use schema::{Field, FieldEntry, FieldType, Schema, Term};
-use std::collections::HashMap;
 use std::io;
+use std::collections::HashMap;
+use postings::Recorder;
+use Result;
+use schema::{Field, Schema};
 use std::marker::PhantomData;
 use std::ops::DerefMut;
-use termdict::TermOrdinal;
+use datastruct::stacker::{Heap, TermHashMap};
+use postings::{NothingRecorder, TFAndPositionRecorder, TermFrequencyRecorder};
+use schema::FieldEntry;
+use schema::FieldType;
 use tokenizer::Token;
 use tokenizer::TokenStream;
-use DocId;
-use Result;
+use schema::IndexRecordOption;
+use postings::UnorderedTermId;
 
 fn posting_from_field_entry<'a>(
     field_entry: &FieldEntry,
@@ -36,17 +39,11 @@ fn posting_from_field_entry<'a>(
         FieldType::U64(_) | FieldType::I64(_) | FieldType::HierarchicalFacet => {
             SpecializedPostingsWriter::<NothingRecorder>::new_boxed(heap)
         }
-        FieldType::Bytes => {
-            // FieldType::Bytes cannot actually be indexed.
-            // TODO fix during the indexer refactoring described in #276
-            SpecializedPostingsWriter::<NothingRecorder>::new_boxed(heap)
-        }
     }
 }
 
 pub struct MultiFieldPostingsWriter<'a> {
     heap: &'a Heap,
-    schema: Schema,
     term_index: TermHashMap<'a>,
     per_field_postings_writers: Vec<Box<PostingsWriter + 'a>>,
 }
@@ -61,8 +58,8 @@ impl<'a> MultiFieldPostingsWriter<'a> {
             .iter()
             .map(|field_entry| posting_from_field_entry(field_entry, heap))
             .collect();
+
         MultiFieldPostingsWriter {
-            schema: schema.clone(),
             heap,
             term_index,
             per_field_postings_writers,
@@ -86,7 +83,7 @@ impl<'a> MultiFieldPostingsWriter<'a> {
     pub fn serialize(
         &self,
         serializer: &mut InvertedIndexSerializer,
-    ) -> Result<HashMap<Field, HashMap<UnorderedTermId, TermOrdinal>>> {
+    ) -> Result<HashMap<Field, HashMap<UnorderedTermId, usize>>> {
         let mut term_offsets: Vec<(&[u8], u32, UnorderedTermId)> = self.term_index.iter().collect();
         term_offsets.sort_by_key(|&(k, _, _)| k);
 
@@ -97,10 +94,8 @@ impl<'a> MultiFieldPostingsWriter<'a> {
             .map(|(key, _, _)| Term::wrap(key).field())
             .enumerate();
 
-        let mut unordered_term_mappings: HashMap<
-            Field,
-            HashMap<UnorderedTermId, TermOrdinal>,
-        > = HashMap::new();
+        let mut unordered_term_mappings: HashMap<Field, HashMap<UnorderedTermId, usize>> =
+            HashMap::new();
 
         let mut prev_field = Field(u32::max_value());
         for (offset, field) in term_offsets_it {
@@ -115,30 +110,20 @@ impl<'a> MultiFieldPostingsWriter<'a> {
             let (field, start) = offsets[i];
             let (_, stop) = offsets[i + 1];
 
-            let field_entry = self.schema.get_field_entry(field);
-
-            match field_entry.field_type() {
-                &FieldType::Str(_) | &FieldType::HierarchicalFacet => {
-                    // populating the (unordered term ord) -> (ordered term ord) mapping
-                    // for the field.
-                    let mut unordered_term_ids = term_offsets[start..stop]
-                        .iter()
-                        .map(|&(_, _, bucket)| bucket);
-                    let mut mapping: HashMap<UnorderedTermId, TermOrdinal> = unordered_term_ids
-                        .enumerate()
-                        .map(|(term_ord, unord_term_id)| {
-                            (unord_term_id as UnorderedTermId, term_ord as TermOrdinal)
-                        })
-                        .collect();
-                    unordered_term_mappings.insert(field, mapping);
-                }
-                &FieldType::U64(_) | &FieldType::I64(_) => {}
-                &FieldType::Bytes => {}
+            // populating the unordered term ord -> ordered term ord mapping
+            // for the field.
+            let mut mapping = HashMap::new();
+            for (term_ord, term_unord_id) in term_offsets[start..stop]
+                .iter()
+                .map(|&(_, _, bucket)| bucket)
+                .enumerate()
+            {
+                mapping.insert(term_unord_id, term_ord);
             }
+            unordered_term_mappings.insert(field, mapping);
 
             let postings_writer = &self.per_field_postings_writers[field.0 as usize];
-            let mut field_serializer =
-                serializer.new_field(field, postings_writer.total_num_tokens())?;
+            let mut field_serializer = serializer.new_field(field, postings_writer.total_num_tokens())?;
             postings_writer.serialize(
                 &term_offsets[start..stop],
                 &mut field_serializer,
@@ -194,7 +179,8 @@ pub trait PostingsWriter {
         token_stream: &mut TokenStream,
         heap: &Heap,
     ) -> u32 {
-        let mut term = Term::for_field(field);
+        let mut term = unsafe { Term::with_capacity(100) };
+        term.set_field(field);
         let num_tokens = {
             let mut sink = |token: &Token| {
                 term.set_text(token.text.as_str());

src/postings/recorder.rs

@@ -1,7 +1,7 @@
-use datastruct::stacker::{ExpUnrolledLinkedList, Heap, HeapAllocable};
-use postings::FieldSerializer;
-use std::{self, io};
 use DocId;
+use std::{self, io};
+use postings::FieldSerializer;
+use datastruct::stacker::{ExpUnrolledLinkedList, Heap, HeapAllocable};
 
 const EMPTY_ARRAY: [u32; 0] = [0u32; 0];
 const POSITION_END: u32 = std::u32::MAX;

src/postings/segment_postings.rs

@@ -2,15 +2,15 @@ use compression::{BlockDecoder, CompressedIntStream, VIntDecoder, COMPRESSION_BL
 use DocId;
 
 use common::BitSet;
-use common::CountingWriter;
 use common::HasLen;
-use compression::compressed_block_size;
-use directory::{ReadOnlySource, SourceRead};
+use postings::Postings;
 use docset::{DocSet, SkipResult};
 use fst::Streamer;
-use postings::serializer::PostingsSerializer;
+use compression::compressed_block_size;
+use directory::{ReadOnlySource, SourceRead};
 use postings::FreqReadingOption;
-use postings::Postings;
+use postings::serializer::PostingsSerializer;
+use common::CountingWriter;
 
 struct PositionComputer {
     // store the amount of position int
@@ -84,13 +84,9 @@ impl SegmentPostings {
             for &doc in docs {
                 postings_serializer.write_doc(doc, 1u32).unwrap();
             }
-            postings_serializer
-                .close_term()
-                .expect("In memory Serialization should never fail.");
+            postings_serializer.close_term().expect("In memory Serialization should never fail.");
         }
-        let (buffer, _) = counting_writer
-            .finish()
-            .expect("Serializing in a buffer should never fail.");
+        let (buffer , _) = counting_writer.finish().expect("Serializing in a buffer should never fail.");
         let data = ReadOnlySource::from(buffer);
         let block_segment_postings = BlockSegmentPostings::from_data(
             docs.len(),
@@ -102,6 +98,7 @@ impl SegmentPostings {
 }
 
 impl SegmentPostings {
+
     /// Reads a Segment postings from an &[u8]
     ///
     /// * `len` - number of document in the posting lists.
@@ -128,7 +125,7 @@ fn exponential_search(target: u32, mut start: usize, arr: &[u32]) -> (usize, usi
     loop {
         let new = start + jump;
         if new >= end {
-            return (start, end);
+            return (start, end)
         }
         if arr[new] > target {
             return (start, new);
@@ -166,8 +163,7 @@ impl DocSet for SegmentPostings {
                 if self.position_computer.is_some() {
                     let freqs_skipped = &self.block_cursor.freqs()[self.cur..];
                     let sum_freq: u32 = freqs_skipped.iter().sum();
-                    self.position_computer
-                        .as_mut()
+                    self.position_computer.as_mut()
                         .unwrap()
                         .add_skip(sum_freq as usize);
                 }
@@ -202,8 +198,7 @@ impl DocSet for SegmentPostings {
         if self.position_computer.is_some() {
             let freqs_skipped = &self.block_cursor.freqs()[self.cur..start];
             let sum_freqs: u32 = freqs_skipped.iter().sum();
-            self.position_computer
-                .as_mut()
+            self.position_computer.as_mut()
                 .unwrap()
                 .add_skip(sum_freqs as usize);
         }
@@ -216,6 +211,7 @@ impl DocSet for SegmentPostings {
         }
     }
 
+
     // goes to the next element.
     // next needs to be called a first time to point to the correct element.
     #[inline]
@@ -266,6 +262,7 @@ impl DocSet for SegmentPostings {
     }
 }
 
+
 impl HasLen for SegmentPostings {
     fn len(&self) -> usize {
         self.block_cursor.doc_freq()
@@ -279,11 +276,16 @@ impl Postings for SegmentPostings {
 
     fn positions_with_offset(&mut self, offset: u32, output: &mut Vec<u32>) {
         if self.position_computer.is_some() {
-            output.resize(self.term_freq() as usize, 0u32);
-            self.position_computer
-                .as_mut()
-                .unwrap()
-                .positions_with_offset(offset, &mut output[..])
+            let prev_capacity = output.capacity();
+            let term_freq = self.term_freq() as usize;
+            if term_freq > prev_capacity {
+                let additional_len = term_freq - output.len();
+                output.reserve(additional_len);
+            }
+            unsafe {
+                output.set_len(term_freq);
+                self.position_computer.as_mut().unwrap().positions_with_offset(offset, &mut output[..])
+            }
         } else {
             output.clear();
         }
@@ -471,16 +473,16 @@ impl<'b> Streamer<'b> for BlockSegmentPostings {
 #[cfg(test)]
 mod tests {
 
-    use super::BlockSegmentPostings;
-    use super::SegmentPostings;
-    use common::HasLen;
-    use core::Index;
     use docset::DocSet;
+    use super::SegmentPostings;
+    use schema::SchemaBuilder;
+    use core::Index;
+    use schema::INT_INDEXED;
+    use schema::Term;
     use fst::Streamer;
     use schema::IndexRecordOption;
-    use schema::SchemaBuilder;
-    use schema::Term;
-    use schema::INT_INDEXED;
+    use common::HasLen;
+    use super::BlockSegmentPostings;
 
     #[test]
     fn test_empty_segment_postings() {
@@ -568,3 +570,4 @@ mod tests {
         assert_eq!(block_segments.docs(), &[1, 3, 5]);
     }
 }
+

src/postings/serializer.rs

@@ -1,16 +1,20 @@
-use super::TermInfo;
-use common::BinarySerializable;
-use common::{CompositeWrite, CountingWriter};
-use compression::VIntEncoder;
-use compression::{BlockEncoder, COMPRESSION_BLOCK_SIZE};
-use core::Segment;
-use directory::WritePtr;
-use schema::Schema;
-use schema::{Field, FieldEntry, FieldType};
-use std::io::{self, Write};
-use termdict::{TermDictionaryBuilder, TermOrdinal};
-use DocId;
 use Result;
+use termdict::TermDictionaryBuilderImpl;
+use super::TermInfo;
+use schema::Field;
+use schema::FieldEntry;
+use schema::FieldType;
+use schema::Schema;
+use directory::WritePtr;
+use compression::{BlockEncoder, COMPRESSION_BLOCK_SIZE};
+use DocId;
+use core::Segment;
+use std::io::{self, Write};
+use compression::VIntEncoder;
+use common::BinarySerializable;
+use common::CountingWriter;
+use common::CompositeWrite;
+use termdict::TermDictionaryBuilder;
 
 /// `PostingsSerializer` is in charge of serializing
 /// postings on disk, in the
@@ -81,11 +85,7 @@ impl InvertedIndexSerializer {
     /// a given field.
     ///
     /// Loads the indexing options for the given field.
-    pub fn new_field(
-        &mut self,
-        field: Field,
-        total_num_tokens: u64,
-    ) -> io::Result<FieldSerializer> {
+    pub fn new_field(&mut self, field: Field, total_num_tokens: u64) -> io::Result<FieldSerializer> {
         let field_entry: &FieldEntry = self.schema.get_field_entry(field);
         let term_dictionary_write = self.terms_write.for_field(field);
         let postings_write = self.postings_write.for_field(field);
@@ -111,12 +111,11 @@ impl InvertedIndexSerializer {
 /// The field serializer is in charge of
 /// the serialization of a specific field.
 pub struct FieldSerializer<'a> {
-    term_dictionary_builder: TermDictionaryBuilder<&'a mut CountingWriter<WritePtr>>,
+    term_dictionary_builder: TermDictionaryBuilderImpl<&'a mut CountingWriter<WritePtr>>,
     postings_serializer: PostingsSerializer<&'a mut CountingWriter<WritePtr>>,
     positions_serializer_opt: Option<PositionSerializer<&'a mut CountingWriter<WritePtr>>>,
     current_term_info: TermInfo,
     term_open: bool,
-    num_terms: TermOrdinal,
 }
 
 impl<'a> FieldSerializer<'a> {
@@ -126,6 +125,7 @@ impl<'a> FieldSerializer<'a> {
         postings_write: &'a mut CountingWriter<WritePtr>,
         positions_write: &'a mut CountingWriter<WritePtr>,
     ) -> io::Result<FieldSerializer<'a>> {
+
         let (term_freq_enabled, position_enabled): (bool, bool) = match field_type {
             FieldType::Str(ref text_options) => {
                 if let Some(text_indexing_options) = text_options.get_indexing_options() {
@@ -141,7 +141,7 @@ impl<'a> FieldSerializer<'a> {
             _ => (false, false),
         };
         let term_dictionary_builder =
-            TermDictionaryBuilder::new(term_dictionary_write, field_type)?;
+            TermDictionaryBuilderImpl::new(term_dictionary_write, field_type)?;
         let postings_serializer = PostingsSerializer::new(postings_write, term_freq_enabled);
         let positions_serializer_opt = if position_enabled {
             Some(PositionSerializer::new(positions_write))
@@ -155,7 +155,6 @@ impl<'a> FieldSerializer<'a> {
             positions_serializer_opt,
             current_term_info: TermInfo::default(),
             term_open: false,
-            num_terms: TermOrdinal::default(),
         })
     }
 
@@ -176,7 +175,7 @@ impl<'a> FieldSerializer<'a> {
     /// * term - the term. It needs to come after the previous term according
     ///   to the lexicographical order.
     /// * doc_freq - return the number of document containing the term.
-    pub fn new_term(&mut self, term: &[u8]) -> io::Result<TermOrdinal> {
+    pub fn new_term(&mut self, term: &[u8]) -> io::Result<()> {
         assert!(
             !self.term_open,
             "Called new_term, while the previous term was not closed."
@@ -184,10 +183,7 @@ impl<'a> FieldSerializer<'a> {
         self.term_open = true;
         self.postings_serializer.clear();
         self.current_term_info = self.current_term_info();
-        self.term_dictionary_builder.insert_key(term)?;
-        let term_ordinal = self.num_terms;
-        self.num_terms += 1;
-        Ok(term_ordinal)
+        self.term_dictionary_builder.insert_key(term)
     }
 
     /// Serialize the information that a document contains the current term,

src/query/all_query.rs

@@ -1,10 +1,12 @@
-use core::Searcher;
+use query::Query;
+use query::Weight;
+use query::Scorer;
 use core::SegmentReader;
 use docset::DocSet;
-use query::{Query, Scorer, Weight};
-use DocId;
 use Result;
 use Score;
+use DocId;
+use core::Searcher;
 
 /// Query that matches all of the documents.
 ///
@@ -14,7 +16,7 @@ pub struct AllQuery;
 
 impl Query for AllQuery {
     fn weight(&self, _: &Searcher, _: bool) -> Result<Box<Weight>> {
-        Ok(Box::new(AllWeight))
+        Ok(box AllWeight)
     }
 }
 
@@ -23,47 +25,29 @@ pub struct AllWeight;
 
 impl Weight for AllWeight {
     fn scorer(&self, reader: &SegmentReader) -> Result<Box<Scorer>> {
-        Ok(Box::new(AllScorer {
-            state: State::NotStarted,
+        Ok(box AllScorer {
+            started: false,
             doc: 0u32,
             max_doc: reader.max_doc(),
-        }))
+        })
     }
 }
 
-enum State {
-    NotStarted,
-    Started,
-    Finished,
-}
-
 /// Scorer associated to the `AllQuery` query.
 pub struct AllScorer {
-    state: State,
+    started: bool,
     doc: DocId,
     max_doc: DocId,
 }
 
 impl DocSet for AllScorer {
     fn advance(&mut self) -> bool {
-        match self.state {
-            State::NotStarted => {
-                self.state = State::Started;
-                self.doc = 0;
-            }
-            State::Started => {
-                self.doc += 1u32;
-            }
-            State::Finished => {
-                return false;
-            }
-        }
-        if self.doc < self.max_doc {
-            return true;
+        if self.started {
+            self.doc += 1u32;
         } else {
-            self.state = State::Finished;
-            return false;
+            self.started = true;
         }
+        self.doc < self.max_doc
     }
 
     fn doc(&self) -> DocId {
@@ -80,46 +64,3 @@ impl Scorer for AllScorer {
         1f32
     }
 }
-
-#[cfg(test)]
-mod tests {
-
-    use super::AllQuery;
-    use query::Query;
-    use schema::{SchemaBuilder, TEXT};
-    use Index;
-
-    #[test]
-    fn test_all_query() {
-        let mut schema_builder = SchemaBuilder::default();
-        let field = schema_builder.add_text_field("text", TEXT);
-        let schema = schema_builder.build();
-        let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_with_num_threads(1, 10_000_000).unwrap();
-        index_writer.add_document(doc!(field=>"aaa"));
-        index_writer.add_document(doc!(field=>"bbb"));
-        index_writer.commit().unwrap();
-        index_writer.add_document(doc!(field=>"ccc"));
-        index_writer.commit().unwrap();
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
-        let weight = AllQuery.weight(&searcher, false).unwrap();
-        {
-            let reader = searcher.segment_reader(0);
-            let mut scorer = weight.scorer(reader).unwrap();
-            assert!(scorer.advance());
-            assert_eq!(scorer.doc(), 0u32);
-            assert!(scorer.advance());
-            assert_eq!(scorer.doc(), 1u32);
-            assert!(!scorer.advance());
-        }
-        {
-            let reader = searcher.segment_reader(1);
-            let mut scorer = weight.scorer(reader).unwrap();
-            assert!(scorer.advance());
-            assert_eq!(scorer.doc(), 0u32);
-            assert!(!scorer.advance());
-        }
-    }
-
-}

src/query/bitset/mod.rs

@@ -1,7 +1,7 @@
 use common::{BitSet, TinySet};
+use DocId;
 use docset::{DocSet, SkipResult};
 use std::cmp::Ordering;
-use DocId;
 
 /// A `BitSetDocSet` makes it possible to iterate through a bitset as if it was a `DocSet`.
 ///
@@ -120,10 +120,12 @@ impl DocSet for BitSetDocSet {
 
 #[cfg(test)]
 mod tests {
-    use super::BitSetDocSet;
+    use DocId;
     use common::BitSet;
     use docset::{DocSet, SkipResult};
-    use DocId;
+    use super::BitSetDocSet;
+    extern crate test;
+    use tests;
 
     fn create_docbitset(docs: &[DocId], max_doc: DocId) -> BitSetDocSet {
         let mut docset = BitSet::with_max_value(max_doc);
@@ -217,17 +219,6 @@ mod tests {
         }
     }
 
-}
-
-#[cfg(all(test, feature = "unstable"))]
-mod bench {
-
-    use super::BitSet;
-    use super::BitSetDocSet;
-    use test;
-    use tests;
-    use DocSet;
-
     #[bench]
     fn bench_bitset_1pct_insert(b: &mut test::Bencher) {
         use tests;

src/query/bm25.rs

@@ -1,7 +1,7 @@
 use fieldnorm::FieldNormReader;
-use Score;
-use Searcher;
 use Term;
+use Searcher;
+use Score;
 
 const K1: f32 = 1.2;
 const B: f32 = 0.75;
@@ -11,6 +11,7 @@ fn idf(doc_freq: u64, doc_count: u64) -> f32 {
     (1f32 + x).ln()
 }
 
+
 fn cached_tf_component(fieldnorm: u32, average_fieldnorm: f32) -> f32 {
     K1 * (1f32 - B + B * fieldnorm as f32 / average_fieldnorm)
 }
@@ -31,10 +32,11 @@ pub struct BM25Weight {
 }
 
 impl BM25Weight {
+
     pub fn null() -> BM25Weight {
         BM25Weight {
             weight: 0f32,
-            cache: [1f32; 256],
+            cache: [1f32; 256]
         }
     }
 
@@ -42,11 +44,7 @@ impl BM25Weight {
         assert!(!terms.is_empty(), "BM25 requires at least one term");
         let field = terms[0].field();
         for term in &terms[1..] {
-            assert_eq!(
-                term.field(),
-                field,
-                "All terms must belong to the same field."
-            );
+            assert_eq!(term.field(), field, "All terms must belong to the same field.");
         }
 
         let mut total_num_tokens = 0u64;
@@ -58,8 +56,7 @@ impl BM25Weight {
         }
         let average_fieldnorm = total_num_tokens as f32 / total_num_docs as f32;
 
-        let idf = terms
-            .iter()
+        let idf = terms.iter()
             .map(|term| {
                 let term_doc_freq = searcher.doc_freq(term);
                 idf(term_doc_freq, total_num_docs)
@@ -86,12 +83,12 @@ impl BM25Weight {
 #[cfg(test)]
 mod tests {
 
-    use super::idf;
     use tests::assert_nearly_equals;
+    use super::idf;
 
     #[test]
     fn test_idf() {
-        assert_nearly_equals(idf(1, 2), 0.6931472);
+        assert_nearly_equals(idf(1, 2),  0.6931472);
     }
 
 }

src/query/boolean_query/boolean_query.rs

@@ -1,12 +1,12 @@
-use super::boolean_weight::BooleanWeight;
-use query::Occur;
-use query::Query;
-use query::TermQuery;
-use query::Weight;
-use schema::IndexRecordOption;
-use schema::Term;
 use Result;
+use super::boolean_weight::BooleanWeight;
+use query::Weight;
 use Searcher;
+use query::Query;
+use schema::Term;
+use query::TermQuery;
+use schema::IndexRecordOption;
+use query::Occur;
 
 /// The boolean query combines a set of queries
 ///
@@ -37,7 +37,7 @@ impl Query for BooleanQuery {
                 Ok((*occur, subquery.weight(searcher, scoring_enabled)?))
             })
             .collect::<Result<_>>()?;
-        Ok(Box::new(BooleanWeight::new(sub_weights, scoring_enabled)))
+        Ok(box BooleanWeight::new(sub_weights, scoring_enabled))
     }
 }
 
@@ -48,8 +48,7 @@ impl BooleanQuery {
         let occur_term_queries: Vec<(Occur, Box<Query>)> = terms
             .into_iter()
             .map(|term| {
-                let term_query: Box<Query> =
-                    Box::new(TermQuery::new(term, IndexRecordOption::WithFreqs));
+                let term_query: Box<Query> = box TermQuery::new(term, IndexRecordOption::WithFreqs);
                 (Occur::Should, term_query)
             })
             .collect();

src/query/boolean_query/boolean_weight.rs

@@ -1,18 +1,19 @@
+use query::Weight;
 use core::SegmentReader;
-use downcast::Downcast;
-use query::intersect_scorers;
-use query::score_combiner::{DoNothingCombiner, ScoreCombiner, SumWithCoordsCombiner};
-use query::term_query::TermScorer;
+use query::Union;
+use std::collections::HashMap;
 use query::EmptyScorer;
+use query::Scorer;
+use downcast::Downcast;
+use std::borrow::Borrow;
 use query::Exclude;
 use query::Occur;
 use query::RequiredOptionalScorer;
-use query::Scorer;
-use query::Union;
-use query::Weight;
-use std::borrow::Borrow;
-use std::collections::HashMap;
+use query::score_combiner::{DoNothingCombiner, ScoreCombiner, SumWithCoordsCombiner};
 use Result;
+use query::intersect_scorers;
+use query::term_query::TermScorer;
+
 
 fn scorer_union<TScoreCombiner>(scorers: Vec<Box<Scorer>>) -> Box<Scorer>
 where
@@ -33,13 +34,14 @@ where
                 .into_iter()
                 .map(|scorer| *Downcast::<TermScorer>::downcast(scorer).unwrap())
                 .collect();
-            let scorer: Box<Scorer> = Box::new(Union::<TermScorer, TScoreCombiner>::from(scorers));
+            let scorer: Box<Scorer> = box Union::<TermScorer, TScoreCombiner>::from(scorers);
             return scorer;
         }
     }
 
-    let scorer: Box<Scorer> = Box::new(Union::<_, TScoreCombiner>::from(scorers));
+    let scorer: Box<Scorer> = box Union::<_, TScoreCombiner>::from(scorers);
     return scorer;
+
 }
 
 pub struct BooleanWeight {
@@ -76,17 +78,17 @@ impl BooleanWeight {
             .remove(&Occur::MustNot)
             .map(scorer_union::<TScoreCombiner>);
 
-        let must_scorer_opt: Option<Box<Scorer>> = per_occur_scorers
-            .remove(&Occur::Must)
-            .map(intersect_scorers);
+        let must_scorer_opt: Option<Box<Scorer>> =
+            per_occur_scorers.remove(&Occur::Must)
+                .map(intersect_scorers);
 
         let positive_scorer: Box<Scorer> = match (should_scorer_opt, must_scorer_opt) {
             (Some(should_scorer), Some(must_scorer)) => {
                 if self.scoring_enabled {
-                    Box::new(RequiredOptionalScorer::<_, _, TScoreCombiner>::new(
+                    box RequiredOptionalScorer::<_, _, TScoreCombiner>::new(
                         must_scorer,
                         should_scorer,
-                    ))
+                    )
                 } else {
                     must_scorer
                 }
@@ -94,12 +96,12 @@ impl BooleanWeight {
             (None, Some(must_scorer)) => must_scorer,
             (Some(should_scorer), None) => should_scorer,
             (None, None) => {
-                return Ok(Box::new(EmptyScorer));
+                return Ok(box EmptyScorer);
             }
         };
 
         if let Some(exclude_scorer) = exclude_scorer_opt {
-            Ok(Box::new(Exclude::new(positive_scorer, exclude_scorer)))
+            Ok(box Exclude::new(positive_scorer, exclude_scorer))
         } else {
             Ok(positive_scorer)
         }
@@ -109,11 +111,11 @@ impl BooleanWeight {
 impl Weight for BooleanWeight {
     fn scorer(&self, reader: &SegmentReader) -> Result<Box<Scorer>> {
         if self.weights.is_empty() {
-            Ok(Box::new(EmptyScorer))
+            Ok(box EmptyScorer)
         } else if self.weights.len() == 1 {
             let &(occur, ref weight) = &self.weights[0];
             if occur == Occur::MustNot {
-                Ok(Box::new(EmptyScorer))
+                Ok(box EmptyScorer)
             } else {
                 weight.scorer(reader)
             }

src/query/boolean_query/mod.rs

@@ -7,19 +7,19 @@ pub use self::boolean_query::BooleanQuery;
 mod tests {
 
     use super::*;
-    use collector::tests::TestCollector;
-    use downcast::Downcast;
-    use query::score_combiner::SumWithCoordsCombiner;
-    use query::term_query::TermScorer;
-    use query::Intersection;
     use query::Occur;
     use query::Query;
+    use query::TermQuery;
+    use query::Intersection;
+    use query::Scorer;
+    use collector::tests::TestCollector;
+    use Index;
+    use downcast::Downcast;
+    use schema::*;
     use query::QueryParser;
     use query::RequiredOptionalScorer;
-    use query::Scorer;
-    use query::TermQuery;
-    use schema::*;
-    use Index;
+    use query::score_combiner::SumWithCoordsCombiner;
+    use query::term_query::TermScorer;
 
     fn aux_test_helper() -> (Index, Field) {
         let mut schema_builder = SchemaBuilder::default();
@@ -110,7 +110,6 @@ mod tests {
             let query = query_parser.parse_query("+a b").unwrap();
             let weight = query.weight(&*searcher, false).unwrap();
             let scorer = weight.scorer(searcher.segment_reader(0u32)).unwrap();
-            println!("{:?}", scorer.type_name());
             assert!(Downcast::<TermScorer>::is_type(&*scorer));
         }
     }
@@ -124,7 +123,7 @@ mod tests {
                 Term::from_field_text(text_field, text),
                 IndexRecordOption::Basic,
             );
-            let query: Box<Query> = Box::new(term_query);
+            let query: Box<Query> = box term_query;
             query
         };
 
@@ -171,6 +170,7 @@ mod tests {
         }
     }
 
+
     #[test]
     pub fn test_intersection_score() {
         let (index, text_field) = aux_test_helper();
@@ -180,7 +180,7 @@ mod tests {
                 Term::from_field_text(text_field, text),
                 IndexRecordOption::Basic,
             );
-            let query: Box<Query> = Box::new(term_query);
+            let query: Box<Query> = box term_query;
             query
         };
 
@@ -192,10 +192,7 @@ mod tests {
         };
 
         {
-            let boolean_query = BooleanQuery::from(vec![
-                (Occur::Must, make_term_query("a")),
-                (Occur::Must, make_term_query("b")),
-            ]);
+            let boolean_query = BooleanQuery::from(vec![(Occur::Must, make_term_query("a")), (Occur::Must, make_term_query("b"))]);
             assert_eq!(score_docs(&boolean_query), vec![0.977973, 0.84699446]);
         }
     }

src/query/exclude.rs

@@ -1,7 +1,7 @@
-use docset::{DocSet, SkipResult};
 use query::Scorer;
-use DocId;
+use docset::{DocSet, SkipResult};
 use Score;
+use DocId;
 
 #[derive(Clone, Copy, Debug)]
 enum State {
@@ -129,10 +129,10 @@ where
 #[cfg(test)]
 mod tests {
 
-    use super::*;
-    use postings::tests::test_skip_against_unoptimized;
-    use query::VecDocSet;
     use tests::sample_with_seed;
+    use postings::tests::test_skip_against_unoptimized;
+    use super::*;
+    use query::VecDocSet;
 
     #[test]
     fn test_exclude() {
@@ -151,10 +151,10 @@ mod tests {
     fn test_exclude_skip() {
         test_skip_against_unoptimized(
             || {
-                Box::new(Exclude::new(
+                box Exclude::new(
                     VecDocSet::from(vec![1, 2, 5, 8, 10, 15, 24]),
                     VecDocSet::from(vec![1, 2, 3, 10, 16, 24]),
-                ))
+                )
             },
             vec![1, 2, 5, 8, 10, 15, 24],
         );
@@ -167,10 +167,10 @@ mod tests {
         let sample_skip = sample_with_seed(10_000, 0.005, 3);
         test_skip_against_unoptimized(
             || {
-                Box::new(Exclude::new(
+                box Exclude::new(
                     VecDocSet::from(sample_include.clone()),
                     VecDocSet::from(sample_exclude.clone()),
-                ))
+                )
             },
             sample_skip,
         );

src/query/fuzzy_query.rs

@@ -1,146 +0,0 @@
-use common::BitSet;
-use core::SegmentReader;
-use fst::Automaton;
-use levenshtein_automata::{LevenshteinAutomatonBuilder, DFA};
-use query::BitSetDocSet;
-use query::ConstScorer;
-use query::{Query, Scorer, Weight};
-use schema::{Field, IndexRecordOption, Term};
-use termdict::{TermDictionary, TermStreamer};
-use std::collections::HashMap;
-use Result;
-use Searcher;
-
-
-lazy_static! {
-    static ref LEV_BUILDER: HashMap<(u8, bool), LevenshteinAutomatonBuilder> = {
-        let mut lev_builder_cache = HashMap::new();
-        // TODO make population lazy on a `(distance, val)` basis
-        for distance in 0..3 {
-            for &transposition in [false, true].iter() {
-                let lev_automaton_builder = LevenshteinAutomatonBuilder::new(distance, transposition);
-                lev_builder_cache.insert((distance, transposition), lev_automaton_builder);
-            }
-        }
-        lev_builder_cache
-    };
-}
-
-/// A Fuzzy Query matches all of the documents
-/// containing a specific term that is with in
-/// Levenshtein distance
-#[derive(Debug)]
-pub struct FuzzyQuery {
-    term: Term,
-    distance: u8,
-    // TODO handle transposition optionally
-}
-
-impl FuzzyQuery {
-    /// Creates a new Fuzzy Query
-    pub fn new(term: Term, distance: u8) -> FuzzyQuery {
-        FuzzyQuery { term, distance }
-    }
-
-    pub fn specialized_weight(&self) -> Result<AutomatonWeight<DFA>> {
-        let automaton = LEV_BUILDER.get(&(self.distance, false))
-            .unwrap() // TODO return an error
-            .build_dfa(self.term.text());
-        Ok(AutomatonWeight {
-            term: self.term.clone(),
-            field: self.term.field(),
-            automaton,
-        })
-    }
-}
-
-impl Query for FuzzyQuery {
-    fn weight(&self, _searcher: &Searcher, _scoring_enabled: bool) -> Result<Box<Weight>> {
-        Ok(Box::new(self.specialized_weight()?))
-    }
-}
-
-pub struct AutomatonWeight<A>
-where A: Automaton
-{
-    term: Term,
-    field: Field,
-    automaton: A,
-}
-
-impl<A> AutomatonWeight<A>
-where
-    A: Automaton,
-{
-    fn automaton_stream<'a>(&'a self, term_dict: &'a TermDictionary) -> TermStreamer<'a, &'a A> {
-        let term_stream_builder = term_dict.search(&self.automaton);
-        term_stream_builder.into_stream()
-    }
-}
-
-impl<A> Weight for AutomatonWeight<A>
-where
-    A: Automaton,
-{
-    fn scorer(&self, reader: &SegmentReader) -> Result<Box<Scorer>> {
-        let max_doc = reader.max_doc();
-        let mut doc_bitset = BitSet::with_max_value(max_doc);
-
-        let inverted_index = reader.inverted_index(self.field);
-        let term_dict = inverted_index.terms();
-        let mut term_stream = self.automaton_stream(term_dict);
-        while term_stream.advance() {
-            let term_info = term_stream.value();
-            let mut block_segment_postings = inverted_index
-                .read_block_postings_from_terminfo(term_info, IndexRecordOption::Basic);
-            while block_segment_postings.advance() {
-                for &doc in block_segment_postings.docs() {
-                    doc_bitset.insert(doc);
-                }
-            }
-        }
-        let doc_bitset = BitSetDocSet::from(doc_bitset);
-        Ok(Box::new(ConstScorer::new(doc_bitset)))
-    }
-}
-
-#[cfg(test)]
-mod test {
-    use super::FuzzyQuery;
-    use schema::SchemaBuilder;
-    use Index;
-    use collector::TopCollector;
-    use schema::TEXT;
-    use Term;
-    use tests::assert_nearly_equals;
-
-    #[test]
-    pub fn test_automaton_weight() {
-        let mut schema_builder = SchemaBuilder::new();
-        let country_field = schema_builder.add_text_field("country", TEXT);
-        let schema = schema_builder.build();
-        let index = Index::create_in_ram(schema);
-        {
-            let mut index_writer = index.writer_with_num_threads(1, 10_000_000).unwrap();
-            index_writer.add_document(doc!(
-                country_field => "Japan",
-            ));
-            index_writer.add_document(doc!(
-                country_field => "Korea",
-            ));
-            index_writer.commit().unwrap();
-        }
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
-        {
-            let mut collector = TopCollector::with_limit(2);
-            let term = Term::from_field_text(country_field, "Japon");
-            let fuzzy_query = FuzzyQuery::new(term, 2);
-            searcher.search(&fuzzy_query, &mut collector).unwrap();
-            let scored_docs = collector.score_docs();
-            assert_eq!(scored_docs.len(), 1);
-            let (score, _) = scored_docs[0];
-            assert_nearly_equals(0.77802235, score);
-        }
-    }
-}

src/query/intersection.rs

@@ -1,11 +1,11 @@
 use docset::{DocSet, SkipResult};
-use downcast::Downcast;
-use query::term_query::TermScorer;
-use query::EmptyScorer;
 use query::Scorer;
-use std::borrow::Borrow;
+use query::EmptyScorer;
 use DocId;
+use downcast::Downcast;
+use std::borrow::Borrow;
 use Score;
+use query::term_query::TermScorer;
 
 /// Returns the intersection scorer.
 ///
@@ -22,7 +22,7 @@ pub fn intersect_scorers(mut scorers: Vec<Box<Scorer>>) -> Box<Scorer> {
     let second_rarest_opt = scorers.pop();
     scorers.reverse();
     match (rarest_opt, second_rarest_opt) {
-        (None, None) => Box::new(EmptyScorer),
+        (None, None) => box EmptyScorer,
         (Some(single_docset), None) => single_docset,
         (Some(left), Some(right)) => {
             {
@@ -32,33 +32,31 @@ pub fn intersect_scorers(mut scorers: Vec<Box<Scorer>>) -> Box<Scorer> {
                 }) {
                     let left = *Downcast::<TermScorer>::downcast(left).unwrap();
                     let right = *Downcast::<TermScorer>::downcast(right).unwrap();
-                    return Box::new(Intersection {
+                    return box Intersection {
                         left,
                         right,
                         others: scorers,
-                        num_docsets,
-                    });
+                        num_docsets
+                    }
                 }
             }
-            return Box::new(Intersection {
+            return box Intersection {
                 left,
                 right,
                 others: scorers,
-                num_docsets,
-            });
-        }
-        _ => {
-            unreachable!();
+                num_docsets
+            }
         }
+        _ => { unreachable!(); }
     }
 }
 
 /// Creates a `DocSet` that iterator through the intersection of two `DocSet`s.
-pub struct Intersection<TDocSet: DocSet, TOtherDocSet: DocSet = Box<Scorer>> {
+pub struct Intersection<TDocSet: DocSet, TOtherDocSet: DocSet=Box<Scorer>> {
     left: TDocSet,
     right: TDocSet,
     others: Vec<TOtherDocSet>,
-    num_docsets: usize,
+    num_docsets: usize
 }
 
 impl<TDocSet: DocSet> Intersection<TDocSet, TDocSet> {
@@ -73,17 +71,18 @@ impl<TDocSet: DocSet> Intersection<TDocSet, TDocSet> {
             left,
             right,
             others: docsets,
-            num_docsets,
+            num_docsets
         }
     }
 }
 
+
 impl<TDocSet: DocSet> Intersection<TDocSet, TDocSet> {
     pub(crate) fn docset_mut_specialized(&mut self, ord: usize) -> &mut TDocSet {
         match ord {
             0 => &mut self.left,
             1 => &mut self.right,
-            n => &mut self.others[n - 2],
+            n => &mut self.others[n - 2]
         }
     }
 }
@@ -93,7 +92,7 @@ impl<TDocSet: DocSet, TOtherDocSet: DocSet> Intersection<TDocSet, TOtherDocSet>
         match ord {
             0 => &mut self.left,
             1 => &mut self.right,
-            n => &mut self.others[n - 2],
+            n => &mut self.others[n - 2]
         }
     }
 }
@@ -115,30 +114,23 @@ impl<TDocSet: DocSet, TOtherDocSet: DocSet> DocSet for Intersection<TDocSet, TOt
             // of the two rarest `DocSet` in the intersection.
             loop {
                 match right.skip_next(candidate) {
-                    SkipResult::Reached => {
-                        break;
-                    }
+                    SkipResult::Reached => { break; }
                     SkipResult::OverStep => {
                         candidate = right.doc();
                         other_candidate_ord = usize::max_value();
                     }
-                    SkipResult::End => {
-                        return false;
-                    }
+                    SkipResult::End => { return false; }
                 }
 
                 match left.skip_next(candidate) {
-                    SkipResult::Reached => {
-                        break;
-                    }
+                    SkipResult::Reached => { break; }
                     SkipResult::OverStep => {
                         candidate = left.doc();
                         other_candidate_ord = usize::max_value();
                     }
-                    SkipResult::End => {
-                        return false;
-                    }
+                    SkipResult::End => { return false; }
                 }
+
             }
             // test the remaining scorers;
             for (ord, docset) in self.others.iter_mut().enumerate() {
@@ -155,22 +147,16 @@ impl<TDocSet: DocSet, TOtherDocSet: DocSet> DocSet for Intersection<TDocSet, TOt
                             // let's update our candidate.
                             candidate = docset.doc();
                             match left.skip_next(candidate) {
-                                SkipResult::Reached => {
-                                    other_candidate_ord = ord;
-                                }
+                                SkipResult::Reached => { other_candidate_ord = ord; }
                                 SkipResult::OverStep => {
                                     candidate = left.doc();
                                     other_candidate_ord = usize::max_value();
                                 }
-                                SkipResult::End => {
-                                    return false;
-                                }
+                                SkipResult::End => { return false; }
                             }
                             continue 'outer;
                         }
-                        SkipResult::End => {
-                            return false;
-                        }
+                        SkipResult::End => { return false; }
                     }
                 }
             }
@@ -178,7 +164,9 @@ impl<TDocSet: DocSet, TOtherDocSet: DocSet> DocSet for Intersection<TDocSet, TOt
         }
     }
 
+
     fn skip_next(&mut self, target: DocId) -> SkipResult {
+
         // We optimize skipping by skipping every single member
         // of the intersection to target.
         let mut current_target: DocId = target;
@@ -223,22 +211,18 @@ impl<TDocSet: DocSet, TOtherDocSet: DocSet> DocSet for Intersection<TDocSet, TOt
 }
 
 impl<TScorer, TOtherScorer> Scorer for Intersection<TScorer, TOtherScorer>
-where
-    TScorer: Scorer,
-    TOtherScorer: Scorer,
-{
+where TScorer: Scorer, TOtherScorer: Scorer {
     fn score(&mut self) -> Score {
-        self.left.score() + self.right.score()
-            + self.others.iter_mut().map(Scorer::score).sum::<Score>()
+        self.left.score() + self.right.score() + self.others.iter_mut().map(Scorer::score).sum::<Score>()
     }
 }
 
 #[cfg(test)]
 mod tests {
-    use super::Intersection;
     use docset::{DocSet, SkipResult};
-    use postings::tests::test_skip_against_unoptimized;
+    use super::Intersection;
     use query::VecDocSet;
+    use postings::tests::test_skip_against_unoptimized;
 
     #[test]
     fn test_intersection() {
@@ -287,7 +271,7 @@ mod tests {
             || {
                 let left = VecDocSet::from(vec![4]);
                 let right = VecDocSet::from(vec![2, 5]);
-                Box::new(Intersection::new(vec![left, right]))
+                box Intersection::new(vec![left, right])
             },
             vec![0, 2, 4, 5, 6],
         );
@@ -297,19 +281,19 @@ mod tests {
                 let mut right = VecDocSet::from(vec![2, 5, 10]);
                 left.advance();
                 right.advance();
-                Box::new(Intersection::new(vec![left, right]))
+                box Intersection::new(vec![left, right])
             },
             vec![0, 1, 2, 3, 4, 5, 6, 7, 10, 11],
         );
         test_skip_against_unoptimized(
             || {
-                Box::new(Intersection::new(vec![
+                box Intersection::new(vec![
                     VecDocSet::from(vec![1, 4, 5, 6]),
                     VecDocSet::from(vec![1, 2, 5, 6]),
                     VecDocSet::from(vec![1, 4, 5, 6]),
                     VecDocSet::from(vec![1, 5, 6]),
                     VecDocSet::from(vec![2, 4, 5, 7, 8]),
-                ]))
+                ])
             },
             vec![0, 1, 2, 3, 4, 5, 6, 7, 10, 11],
         );

src/query/mod.rs

@@ -2,23 +2,22 @@
 Query
 */
 
+mod query;
+mod boolean_query;
+mod scorer;
+mod occur;
+mod weight;
+mod term_query;
+mod query_parser;
+mod phrase_query;
 mod all_query;
 mod bitset;
-mod bm25;
-mod boolean_query;
-mod exclude;
-mod fuzzy_query;
-mod intersection;
-mod occur;
-mod phrase_query;
-mod query;
-mod query_parser;
 mod range_query;
-mod reqopt_scorer;
-mod scorer;
-mod term_query;
+mod exclude;
 mod union;
-mod weight;
+mod intersection;
+mod reqopt_scorer;
+mod bm25;
 
 #[cfg(test)]
 mod vec_docset;
@@ -31,20 +30,20 @@ pub use self::union::Union;
 #[cfg(test)]
 pub use self::vec_docset::VecDocSet;
 
-pub use self::all_query::{AllQuery, AllScorer, AllWeight};
+pub use self::reqopt_scorer::RequiredOptionalScorer;
+pub use self::exclude::Exclude;
 pub use self::bitset::BitSetDocSet;
 pub use self::boolean_query::BooleanQuery;
-pub use self::exclude::Exclude;
-pub use self::intersection::intersect_scorers;
 pub use self::occur::Occur;
 pub use self::phrase_query::PhraseQuery;
-pub use self::query::Query;
-pub use self::query_parser::QueryParser;
 pub use self::query_parser::QueryParserError;
-pub use self::range_query::RangeQuery;
-pub use self::reqopt_scorer::RequiredOptionalScorer;
-pub use self::scorer::ConstScorer;
+pub use self::query_parser::QueryParser;
+pub use self::query::Query;
 pub use self::scorer::EmptyScorer;
 pub use self::scorer::Scorer;
 pub use self::term_query::TermQuery;
 pub use self::weight::Weight;
+pub use self::all_query::{AllQuery, AllScorer, AllWeight};
+pub use self::range_query::RangeQuery;
+pub use self::scorer::ConstScorer;
+pub use self::intersection::intersect_scorers;