Added logging when token is dropped.

* Clippy comments

Clippy complaints that about the cast of &[u32] to a *const __m128i,
because of the lack of alignment constraints.

This commit passes the OutputBuffer object (which enforces proper
    alignment) instead of `&[u32]`.

* Clippy. Block alignment

* Code simplification

* Added comment. Code simplification

* Removed the extraneous freq block len hack.

.travis.yml

@@ -68,6 +68,11 @@ cache: cargo
 before_cache:
   # Travis can't cache files that are not readable by "others"
   - chmod -R a+r $HOME/.cargo
+  - find ./target/debug -type f -maxdepth 1 -delete
+  - rm -f  ./target/.rustc_info.json
+  - rm -fr ./target/debug/{deps,.fingerprint}/tantivy*
+  - rm -r target/debug/examples/
+  - ls -1 examples/ | sed -e 's/\.rs$//' | xargs -I "{}" find target/* -name "*{}*" -type f -delete
 
 #branches:
 #  only:
@@ -77,4 +82,4 @@ before_cache:
 
 notifications:
   email:
-    on_success: never
+    on_success: never

CHANGELOG.md

@@ -1,11 +1,69 @@
+Tantivy 0.10.0
+====================
+
+
+Minor
+---------
+- Small simplification of the code. 
+Calling .freq() or .doc() when .advance() has never 
+on segment postings should panic from now on.
+- Tokens exceeding `u16::max_value() - 4` chars are discarded silently instead of panicking.
+
+
 Tantivy 0.9.0
 =====================
 *0.9.0 index format is not compatible with the 
 previous index format.*
+- MAJOR BUGFIX : 
+  Some `Mmap` objects were being leaked, and would never get released. (@fulmicoton)
 - Removed most unsafe (@fulmicoton)
 - Indexer memory footprint improved. (VInt comp, inlining the first block. (@fulmicoton)
 - Stemming in other language possible (@pentlander)
 - Segments with no docs are deleted earlier (@barrotsteindev)
+- Added grouped add and delete operations. 
+  They are guaranteed to happen together (i.e. they cannot be split by a commit). 
+  In addition, adds are guaranteed to happen on the same segment. (@elbow-jason)
+- Removed `INT_STORED` and `INT_INDEXED`. It is now possible to use `STORED` and `INDEXED`
+  for int fields. (@fulmicoton)
+- Added DateTime field (@barrotsteindev)
+- Added IndexReader. By default, index is reloaded automatically upon new commits (@fulmicoton)
+- SIMD linear search within blocks (@fulmicoton)
+
+## How to update ?
+
+tantivy 0.9 brought some API breaking change.
+To update from tantivy 0.8, you will need to go through the following steps.
+
+- `schema::INT_INDEXED` and `schema::INT_STORED`  should be replaced by `schema::INDEXED` and `schema::INT_STORED`.
+- The index now does not hold the pool of searcher anymore. You are required to create an intermediary object called 
+`IndexReader` for this. 
+    
+    ```rust
+    // create the reader. You typically need to create 1 reader for the entire
+    // lifetime of you program.
+    let reader = index.reader()?;
+    
+    // Acquire a searcher (previously `index.searcher()`) is now written:
+    let searcher = reader.searcher();
+    
+    // With the default setting of the reader, you are not required to 
+    // call `index.load_searchers()` anymore.
+    //
+    // The IndexReader will pick up that change automatically, regardless
+    // of whether the update was done in a different process or not.
+    // If this behavior is not wanted, you can create your reader with 
+    // the `ReloadPolicy::Manual`, and manually decide when to reload the index
+    // by calling `reader.reload()?`.
+  
+    ```
+
+
+Tantivy 0.8.2
+=====================
+Fixing build for x86_64 platforms. (#496)
+No need to update from 0.8.1 if tantivy
+is building on your platform.
+
 
 Tantivy 0.8.1
 =====================

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "tantivy"
-version = "0.9.0-dev"
+version = "0.10.0-dev"
 authors = ["Paul Masurel <paul.masurel@gmail.com>"]
 license = "MIT"
 categories = ["database-implementations", "data-structures"]
@@ -16,14 +16,14 @@ base64 = "0.10.0"
 byteorder = "1.0"
 lazy_static = "1"
 regex = "1.0"
-fst = {version="0.3", default-features=false}
-fst-regex = { version="0.2" }
+tantivy-fst = "0.1"
+memmap = {version = "0.7", optional=true}
 lz4 = {version="1.20", optional=true}
 snap = {version="0.2"}
 atomicwrites = {version="0.2.2", optional=true}
 tempfile = "3.0"
 log = "0.4"
-combine = "3"
+combine = ">=3.6.0,<4.0.0"
 tempdir = "0.3"
 serde = "1.0"
 serde_derive = "1.0"
@@ -32,8 +32,9 @@ num_cpus = "1.2"
 fs2={version="0.4", optional=true}
 itertools = "0.8"
 levenshtein_automata = {version="0.1", features=["fst_automaton"]}
+notify = {version="4", optional=true}
 bit-set = "0.5"
-uuid = { version = "0.7", features = ["v4", "serde"] }
+uuid = { version = "0.7.2", features = ["v4", "serde"] }
 crossbeam = "0.5"
 futures = "0.1"
 futures-cpupool = "0.1"
@@ -41,7 +42,6 @@ owning_ref = "0.4"
 stable_deref_trait = "1.0.0"
 rust-stemmers = "1.1"
 downcast-rs = { version="1.0" }
-matches = "0.1"
 bitpacking = "0.6"
 census = "0.2"
 fnv = "1.0.6"
@@ -51,6 +51,7 @@ htmlescape = "0.3.1"
 fail = "0.2"
 scoped-pool = "1.0"
 murmurhash32 = "0.2"
+chrono = "0.4"
 
 [target.'cfg(windows)'.dependencies]
 winapi = "0.2"
@@ -58,6 +59,8 @@ winapi = "0.2"
 [dev-dependencies]
 rand = "0.6"
 maplit = "1"
+matches = "0.1.8"
+time = "0.1.42"
 
 [profile.release]
 opt-level = 3
@@ -71,12 +74,11 @@ overflow-checks = true
 [features]
 # by default no-fail is disabled. We manually enable it when running test.
 default = ["mmap", "no_fail"]
-mmap = ["fst/mmap", "atomicwrites", "fs2"]
+mmap = ["atomicwrites", "fs2", "memmap", "notify"]
 lz4-compression = ["lz4"]
 no_fail = ["fail/no_fail"]
 unstable = [] # useful for benches.
+wasm-bindgen = ["uuid/wasm-bindgen"]
 
 [badges]
 travis-ci = { repository = "tantivy-search/tantivy" }
-
-

README.md

@@ -17,6 +17,7 @@
 [![](https://sourcerer.io/fame/fulmicoton/tantivy-search/tantivy/images/6)](https://sourcerer.io/fame/fulmicoton/tantivy-search/tantivy/links/6)
 [![](https://sourcerer.io/fame/fulmicoton/tantivy-search/tantivy/images/7)](https://sourcerer.io/fame/fulmicoton/tantivy-search/tantivy/links/7)
 
+[![Become a patron](https://c5.patreon.com/external/logo/become_a_patron_button.png)](https://www.patreon.com/fulmicoton)
 
 
 **Tantivy** is a **full text search engine library** written in rust.
@@ -27,9 +28,18 @@ to build such a search engine.
 
 Tantivy is, in fact, strongly inspired by Lucene's design.
 
+# Benchmark
+
+Tantivy is typically faster than Lucene, but the results will depend on 
+the nature of the queries in your workload.
+
+The following [benchmark](https://tantivy-search.github.io/bench/) break downs 
+performance for different type of queries / collection.
+
 # Features
 
 - Full-text search
+- Configurable tokenizer. (stemming available for 17 latin languages. Third party support for Chinese ([tantivy-jieba](https://crates.io/crates/tantivy-jieba) and [cang-jie](https://crates.io/crates/cang-jie)) and [Japanese](https://crates.io/crates/tantivy-tokenizer-tiny-segmenter)
 - Fast (check out the :racehorse: :sparkles: [benchmark](https://tantivy-search.github.io/bench/) :sparkles: :racehorse:)
 - Tiny startup time (<10ms), perfect for command line tools
 - BM25 scoring (the same as lucene)
@@ -41,6 +51,7 @@ Tantivy is, in fact, strongly inspired by Lucene's design.
 - SIMD integer compression when the platform/CPU includes the SSE2 instruction set.
 - Single valued and multivalued u64 and i64 fast fields (equivalent of doc values in Lucene)
 - `&[u8]` fast fields
+- Text, i64, u64, dates and hierarchical facet fields
 - LZ4 compressed document store
 - Range queries
 - Faceted search
@@ -85,6 +96,14 @@ To check out and run tests, you can simply run :
 Some tests will not run with just `cargo test` because of `fail-rs`.
 To run the tests exhaustively, run `./run-tests.sh`. 
 
-# Contribute
+# How can I support this project ?
 
-Send me an email (paul.masurel at gmail.com) if you want to contribute to tantivy.
+There are many ways to support this project. 
+
+- If you use tantivy, tell us about your experience on [gitter](https://gitter.im/tantivy-search/tantivy) or by email (paul.masurel@gmail.com)
+- Report bugs
+- Write a blog post
+- Complete documentation
+- Contribute code (you can join [our gitter](https://gitter.im/tantivy-search/tantivy) )
+- Talk about tantivy around you
+- Drop a word on on [![Say Thanks!](https://img.shields.io/badge/Say%20Thanks-!-1EAEDB.svg)](https://saythanks.io/to/fulmicoton) or even [![Become a patron](https://c5.patreon.com/external/logo/become_a_patron_button.png)](https://www.patreon.com/fulmicoton)

examples/basic_search.rs

@@ -20,6 +20,7 @@ use tantivy::collector::TopDocs;
 use tantivy::query::QueryParser;
 use tantivy::schema::*;
 use tantivy::Index;
+use tantivy::ReloadPolicy;
 use tempdir::TempDir;
 
 fn main() -> tantivy::Result<()> {
@@ -170,24 +171,33 @@ fn main() -> tantivy::Result<()> {
     //
     // ### Searcher
     //
-    // Let's search our index. Start by reloading
-    // searchers in the index. This should be done
-    // after every `commit()`.
-    index.load_searchers()?;
+    // A reader is required to get search the index.
+    // It acts as a `Searcher` pool that reloads itself,
+    // depending on a `ReloadPolicy`.
+    //
+    // For a search server you will typically create one reader for the entire lifetime of your
+    // program, and acquire a new searcher for every single request.
+    //
+    // In the code below, we rely on the 'ON_COMMIT' policy: the reader
+    // will reload the index automatically after each commit.
+    let reader = index
+        .reader_builder()
+        .reload_policy(ReloadPolicy::OnCommit)
+        .try_into()?;
 
     // We now need to acquire a searcher.
-    // Some search experience might require more than
-    // one query.
     //
-    // The searcher ensure that we get to work
-    // with a consistent version of the index.
+    // A searcher points to snapshotted, immutable version of the index.
+    //
+    // Some search experience might require more than
+    // one query. Using the same searcher ensures that all of these queries will run on the
+    // same version of the index.
     //
     // Acquiring a `searcher` is very cheap.
     //
-    // You should acquire a searcher every time you
-    // start processing a request and
+    // You should acquire a searcher every time you start processing a request and
     // and release it right after your query is finished.
-    let searcher = index.searcher();
+    let searcher = reader.searcher();
 
     // ### Query
 
@@ -224,7 +234,6 @@ fn main() -> tantivy::Result<()> {
     // Since the body field was not configured as stored,
     // the document returned will only contain
     // a title.
-
     for (_score, doc_address) in top_docs {
         let retrieved_doc = searcher.doc(doc_address)?;
         println!("{}", schema.to_json(&retrieved_doc));

examples/custom_collector.rs

@@ -17,7 +17,7 @@ use tantivy::collector::{Collector, SegmentCollector};
 use tantivy::fastfield::FastFieldReader;
 use tantivy::query::QueryParser;
 use tantivy::schema::Field;
-use tantivy::schema::{Schema, FAST, INT_INDEXED, TEXT};
+use tantivy::schema::{Schema, FAST, INDEXED, TEXT};
 use tantivy::Index;
 use tantivy::SegmentReader;
 
@@ -137,7 +137,7 @@ fn main() -> tantivy::Result<()> {
     // products, and with a name, a description, and a price.
     let product_name = schema_builder.add_text_field("name", TEXT);
     let product_description = schema_builder.add_text_field("description", TEXT);
-    let price = schema_builder.add_u64_field("price", INT_INDEXED | FAST);
+    let price = schema_builder.add_u64_field("price", INDEXED | FAST);
     let schema = schema_builder.build();
 
     // # Indexing documents
@@ -170,9 +170,9 @@ fn main() -> tantivy::Result<()> {
         price => 5_200u64
     ));
     index_writer.commit()?;
-    index.load_searchers()?;
 
-    let searcher = index.searcher();
+    let reader = index.reader()?;
+    let searcher = reader.searcher();
     let query_parser = QueryParser::for_index(&index, vec![product_name, product_description]);
 
     // here we want to get a hit on the 'ken' in Frankenstein

examples/custom_tokenizer.rs

@@ -91,9 +91,9 @@ fn main() -> tantivy::Result<()> {
                 increasing confidence in the success of my undertaking."#
     ));
     index_writer.commit()?;
-    index.load_searchers()?;
 
-    let searcher = index.searcher();
+    let reader = index.reader()?;
+    let searcher = reader.searcher();
 
     // The query parser can interpret human queries.
     // Here, if the user does not specify which

examples/deleting_updating_documents.rs

@@ -14,12 +14,16 @@ use tantivy::collector::TopDocs;
 use tantivy::query::TermQuery;
 use tantivy::schema::*;
 use tantivy::Index;
+use tantivy::IndexReader;
 
 // A simple helper function to fetch a single document
 // given its id from our index.
 // It will be helpful to check our work.
-fn extract_doc_given_isbn(index: &Index, isbn_term: &Term) -> tantivy::Result<Option<Document>> {
-    let searcher = index.searcher();
+fn extract_doc_given_isbn(
+    reader: &IndexReader,
+    isbn_term: &Term,
+) -> tantivy::Result<Option<Document>> {
+    let searcher = reader.searcher();
 
     // This is the simplest query you can think of.
     // It matches all of the documents containing a specific term.
@@ -85,12 +89,12 @@ fn main() -> tantivy::Result<()> {
        isbn => "978-9176370711",
     ));
     index_writer.commit()?;
-    index.load_searchers()?;
+    let reader = index.reader()?;
 
     let frankenstein_isbn = Term::from_field_text(isbn, "978-9176370711");
 
     // Oops our frankenstein doc seems mispelled
-    let frankenstein_doc_misspelled = extract_doc_given_isbn(&index, &frankenstein_isbn)?.unwrap();
+    let frankenstein_doc_misspelled = extract_doc_given_isbn(&reader, &frankenstein_isbn)?.unwrap();
     assert_eq!(
         schema.to_json(&frankenstein_doc_misspelled),
         r#"{"isbn":["978-9176370711"],"title":["Frankentein"]}"#,
@@ -129,10 +133,10 @@ fn main() -> tantivy::Result<()> {
     // Everything happened as if the document was updated.
     index_writer.commit()?;
     // We reload our searcher to make our change available to clients.
-    index.load_searchers()?;
+    reader.reload()?;
 
     // No more typo!
-    let frankenstein_new_doc = extract_doc_given_isbn(&index, &frankenstein_isbn)?.unwrap();
+    let frankenstein_new_doc = extract_doc_given_isbn(&reader, &frankenstein_isbn)?.unwrap();
     assert_eq!(
         schema.to_json(&frankenstein_new_doc),
         r#"{"isbn":["978-9176370711"],"title":["Frankenstein"]}"#,

examples/faceted_search.rs

@@ -55,9 +55,9 @@ fn main() -> tantivy::Result<()> {
 
     index_writer.commit()?;
 
-    index.load_searchers()?;
+    let reader = index.reader()?;
 
-    let searcher = index.searcher();
+    let searcher = reader.searcher();
 
     let mut facet_collector = FacetCollector::for_field(tags);
     facet_collector.add_facet("/pools");

examples/integer_range_search.rs

@@ -7,17 +7,19 @@
 extern crate tantivy;
 use tantivy::collector::Count;
 use tantivy::query::RangeQuery;
-use tantivy::schema::{Schema, INT_INDEXED};
+use tantivy::schema::{Schema, INDEXED};
 use tantivy::Index;
 use tantivy::Result;
 
 fn run() -> Result<()> {
     // For the sake of simplicity, this schema will only have 1 field
     let mut schema_builder = Schema::builder();
-    // INT_INDEXED is shorthand for such fields
-    let year_field = schema_builder.add_u64_field("year", INT_INDEXED);
+
+    // `INDEXED` is a short-hand to indicate that our field should be "searchable".
+    let year_field = schema_builder.add_u64_field("year", INDEXED);
     let schema = schema_builder.build();
     let index = Index::create_in_ram(schema);
+    let reader = index.reader()?;
     {
         let mut index_writer = index.writer_with_num_threads(1, 6_000_000)?;
         for year in 1950u64..2019u64 {
@@ -26,8 +28,8 @@ fn run() -> Result<()> {
         index_writer.commit()?;
         // The index will be a range of years
     }
-    index.load_searchers()?;
-    let searcher = index.searcher();
+    reader.reload()?;
+    let searcher = reader.searcher();
     // The end is excluded i.e. here we are searching up to 1969
     let docs_in_the_sixties = RangeQuery::new_u64(year_field, 1960..1970);
     // Uses a Count collector to sum the total number of docs in the range

examples/iterating_docs_and_positions.rs

@@ -33,9 +33,9 @@ fn main() -> tantivy::Result<()> {
     index_writer.add_document(doc!(title => "The modern Promotheus"));
     index_writer.commit()?;
 
-    index.load_searchers()?;
+    let reader = index.reader()?;
 
-    let searcher = index.searcher();
+    let searcher = reader.searcher();
 
     // A tantivy index is actually a collection of segments.
     // Similarly, a searcher just wraps a list `segment_reader`.

examples/snippet.rs

@@ -48,9 +48,8 @@ fn main() -> tantivy::Result<()> {
     // ...
     index_writer.commit()?;
 
-    index.load_searchers()?;
-
-    let searcher = index.searcher();
+    let reader = index.reader()?;
+    let searcher = reader.searcher();
     let query_parser = QueryParser::for_index(&index, vec![title, body]);
     let query = query_parser.parse_query("sycamore spring")?;
 

examples/stop_words.rs

@@ -96,9 +96,9 @@ fn main() -> tantivy::Result<()> {
 
     index_writer.commit()?;
 
-    index.load_searchers()?;
+    let reader = index.reader()?;
 
-    let searcher = index.searcher();
+    let searcher = reader.searcher();
 
     let query_parser = QueryParser::for_index(&index, vec![title, body]);
 

examples/working_with_json.rs

@@ -12,7 +12,7 @@ fn main() -> tantivy::Result<()> {
     let mut schema_builder = Schema::builder();
     schema_builder.add_text_field("title", TEXT | STORED);
     schema_builder.add_text_field("body", TEXT);
-    schema_builder.add_u64_field("year", INT_INDEXED);
+    schema_builder.add_u64_field("year", INDEXED);
     let schema = schema_builder.build();
 
     // Let's assume we have a json-serialized document.

src/collector/count_collector.rs

@@ -40,8 +40,8 @@ use SegmentReader;
 ///         index_writer.commit().unwrap();
 ///     }
 ///
-///     index.load_searchers()?;
-///     let searcher = index.searcher();
+///     let reader = index.reader()?;
+///     let searcher = reader.searcher();
 ///
 ///     {
 ///         let query_parser = QueryParser::for_index(&index, vec![title]);

src/collector/facet_collector.rs

@@ -122,17 +122,16 @@ fn facet_depth(facet_bytes: &[u8]) -> usize {
 ///             facet => Facet::from("/lang/en"),
 ///             facet => Facet::from("/category/biography")
 ///         ));
-///         index_writer.commit().unwrap();
+///         index_writer.commit()?;
 ///     }
-///
-///     index.load_searchers()?;
-///     let searcher = index.searcher();
+///     let reader = index.reader()?;
+///     let searcher = reader.searcher();
 ///
 ///     {
 ///			let mut facet_collector = FacetCollector::for_field(facet);
 ///         facet_collector.add_facet("/lang");
 ///         facet_collector.add_facet("/category");
-///         let facet_counts = searcher.search(&AllQuery, &facet_collector).unwrap();
+///         let facet_counts = searcher.search(&AllQuery, &facet_collector)?;
 ///
 ///         // This lists all of the facet counts
 ///         let facets: Vec<(&Facet, u64)> = facet_counts
@@ -147,7 +146,7 @@ fn facet_depth(facet_bytes: &[u8]) -> usize {
 ///     {
 ///			let mut facet_collector = FacetCollector::for_field(facet);
 ///         facet_collector.add_facet("/category/fiction");
-///         let facet_counts = searcher.search(&AllQuery, &facet_collector).unwrap();
+///         let facet_counts = searcher.search(&AllQuery, &facet_collector)?;
 ///
 ///         // This lists all of the facet counts
 ///         let facets: Vec<(&Facet, u64)> = facet_counts
@@ -163,7 +162,7 @@ fn facet_depth(facet_bytes: &[u8]) -> usize {
 ///    {
 ///			let mut facet_collector = FacetCollector::for_field(facet);
 ///         facet_collector.add_facet("/category/fiction");
-///         let facet_counts = searcher.search(&AllQuery, &facet_collector).unwrap();
+///         let facet_counts = searcher.search(&AllQuery, &facet_collector)?;
 ///
 ///         // This lists all of the facet counts
 ///         let facets: Vec<(&Facet, u64)> = facet_counts.top_k("/category/fiction", 1);
@@ -483,8 +482,8 @@ mod tests {
             index_writer.add_document(doc);
         }
         index_writer.commit().unwrap();
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        let reader = index.reader().unwrap();
+        let searcher = reader.searcher();
         let mut facet_collector = FacetCollector::for_field(facet_field);
         facet_collector.add_facet(Facet::from("/top1"));
         let counts = searcher.search(&AllQuery, &facet_collector).unwrap();
@@ -532,8 +531,8 @@ mod tests {
             facet_field => Facet::from_text(&"/subjects/B/b"),
         ));
         index_writer.commit().unwrap();
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        let reader = index.reader().unwrap();
+        let searcher = reader.searcher();
         assert_eq!(searcher.num_docs(), 1);
         let mut facet_collector = FacetCollector::for_field(facet_field);
         facet_collector.add_facet("/subjects");
@@ -579,9 +578,7 @@ mod tests {
             index_writer.add_document(doc);
         }
         index_writer.commit().unwrap();
-        index.load_searchers().unwrap();
-
-        let searcher = index.searcher();
+        let searcher = index.reader().unwrap().searcher();
 
         let mut facet_collector = FacetCollector::for_field(facet_field);
         facet_collector.add_facet("/facet");
@@ -635,8 +632,7 @@ mod bench {
             index_writer.add_document(doc);
         }
         index_writer.commit().unwrap();
-        index.load_searchers().unwrap();
-
+        let reader = index.reader().unwrap();
         b.iter(|| {
             let searcher = index.searcher();
             let facet_collector = FacetCollector::for_field(facet_field);

src/collector/int_facet_collector.rs

@@ -101,8 +101,7 @@ mod tests {
             assert_eq!(index_writer.commit().unwrap(), 10u64);
         }
 
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        let searcher = index.reader().searcher();
         let mut ffvf_i64: IntFacetCollector<I64FastFieldReader> = IntFacetCollector::new(num_field_i64);
         let mut ffvf_u64: IntFacetCollector<U64FastFieldReader> = IntFacetCollector::new(num_field_u64);
 

src/collector/mod.rs

@@ -53,9 +53,9 @@ use tantivy::collector::{Count, TopDocs};
 #     index_writer.add_document(doc!(
 #        title => "The Diary of Muadib",
 #     ));
-#     index_writer.commit().unwrap();
-#     index.load_searchers()?;
-#     let searcher = index.searcher();
+#     index_writer.commit()?;
+#     let reader = index.reader()?;
+#     let searcher = reader.searcher();
 #     let query_parser = QueryParser::for_index(&index, vec![title]);
 #     let query = query_parser.parse_query("diary")?;
 let (doc_count, top_docs): (usize, Vec<(Score, DocAddress)>) =

src/collector/multi_collector.rs

@@ -134,8 +134,8 @@ impl<TFruit: Fruit> FruitHandle<TFruit> {
 ///         index_writer.commit().unwrap();
 ///     }
 ///
-///     index.load_searchers()?;
-///     let searcher = index.searcher();
+///     let reader = index.reader()?;
+///     let searcher = reader.searcher();
 ///
 ///     let mut collectors = MultiCollector::new();
 ///     let top_docs_handle = collectors.add_collector(TopDocs::with_limit(2));
@@ -278,8 +278,7 @@ mod tests {
             index_writer.add_document(doc!(text=>"abc"));
             index_writer.commit().unwrap();
         }
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        let searcher = index.reader().unwrap().searcher();
         let term = Term::from_field_text(text, "abc");
         let query = TermQuery::new(term, IndexRecordOption::Basic);
 

src/collector/top_field_collector.rs

@@ -23,15 +23,16 @@ use SegmentReader;
 /// # use tantivy::schema::{Schema, Field, FAST, TEXT};
 /// # use tantivy::{Index, Result, DocAddress};
 /// # use tantivy::query::{Query, QueryParser};
+/// use tantivy::Searcher;
 /// use tantivy::collector::TopDocs;
 ///
-/// # fn main() {
+/// # fn main() -> tantivy::Result<()> {
 /// #   let mut schema_builder = Schema::builder();
 /// #   let title = schema_builder.add_text_field("title", TEXT);
 /// #   let rating = schema_builder.add_u64_field("rating", FAST);
 /// #   let schema = schema_builder.build();
 /// #   let index = Index::create_in_ram(schema);
-/// #   let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
+/// #   let mut index_writer = index.writer_with_num_threads(1, 3_000_000)?;
 /// #   index_writer.add_document(doc!(
 /// #       title => "The Name of the Wind",
 /// #       rating => 92u64,
@@ -39,13 +40,14 @@ use SegmentReader;
 /// #   index_writer.add_document(doc!(title => "The Diary of Muadib", rating => 97u64));
 /// #   index_writer.add_document(doc!(title => "A Dairy Cow", rating => 63u64));
 /// #   index_writer.add_document(doc!(title => "The Diary of a Young Girl", rating => 80u64));
-/// #   index_writer.commit().unwrap();
-/// #   index.load_searchers().unwrap();
-///	#   let query = QueryParser::for_index(&index, vec![title]).parse_query("diary").unwrap();
-/// #   let top_docs = docs_sorted_by_rating(&index, &query, rating).unwrap();
+/// #   index_writer.commit()?;
+/// #   let reader = index.reader()?;
+/// #   let query = QueryParser::for_index(&index, vec![title]).parse_query("diary")?;
+/// #   let top_docs = docs_sorted_by_rating(&reader.searcher(), &query, rating)?;
 /// #   assert_eq!(top_docs,
 /// #            vec![(97u64, DocAddress(0u32, 1)),
 /// #                 (80u64, DocAddress(0u32, 3))]);
+/// #   Ok(())
 /// # }
 /// #
 /// /// Searches the document matching the given query, and
@@ -53,7 +55,9 @@ use SegmentReader;
 /// /// given in argument.
 /// ///
 /// /// `field` is required to be a FAST field.
-/// fn docs_sorted_by_rating(index: &Index, query: &Query, sort_by_field: Field)
+/// fn docs_sorted_by_rating(searcher: &Searcher,
+///                          query: &Query,
+///                          sort_by_field: Field)
 ///     -> Result<Vec<(u64, DocAddress)>> {
 ///
 ///     // This is where we build our collector!
@@ -61,8 +65,7 @@ use SegmentReader;
 ///
 ///     // ... and here is our documents. Not this is a simple vec.
 ///     // The `u64` in the pair is the value of our fast field for each documents.
-///     index.searcher()
-///          .search(query, &top_docs_by_rating)
+///     searcher.search(query, &top_docs_by_rating)
 /// }
 /// ```
 pub struct TopDocsByField<T> {
@@ -76,6 +79,12 @@ impl<T: FastValue + PartialOrd + Clone> TopDocsByField<T> {
     /// The given field name must be a fast field, otherwise the collector have an error while
     /// collecting results.
     ///
+    /// This constructor is crate-private. Client are supposed to call
+    /// build `TopDocsByField`  object using the `TopDocs` API.
+    ///
+    /// e.g.:
+    ///   `TopDocs::with_limit(2).order_by_field(sort_by_field)`
+    ///
     /// # Panics
     /// The method panics if limit is 0
     pub(crate) fn new(field: Field, limit: usize) -> TopDocsByField<T> {
@@ -171,7 +180,7 @@ mod tests {
                 size => 16u64,
             ));
         });
-        let searcher = index.searcher();
+        let searcher = index.reader().unwrap().searcher();
 
         let top_collector = TopDocs::with_limit(4).order_by_field(size);
         let top_docs: Vec<(u64, DocAddress)> = searcher.search(&query, &top_collector).unwrap();
@@ -198,7 +207,7 @@ mod tests {
                 size => 12u64,
             ));
         });
-        let searcher = index.searcher();
+        let searcher = index.reader().unwrap().searcher();
         let top_collector: TopDocsByField<u64> = TopDocs::with_limit(4).order_by_field(Field(2));
         let segment_reader = searcher.segment_reader(0u32);
         top_collector
@@ -218,7 +227,7 @@ mod tests {
                 size => 12u64,
             ));
         });
-        let searcher = index.searcher();
+        let searcher = index.reader().unwrap().searcher();
         let segment = searcher.segment_reader(0);
         let top_collector: TopDocsByField<u64> = TopDocs::with_limit(4).order_by_field(size);
         assert_matches!(
@@ -241,8 +250,6 @@ mod tests {
         let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
         doc_adder(&mut index_writer);
         index_writer.commit().unwrap();
-        index.load_searchers().unwrap();
-
         let query_parser = QueryParser::for_index(&index, vec![query_field]);
         let query = query_parser.parse_query(query).unwrap();
         (index, query)

src/collector/top_score_collector.rs

@@ -51,8 +51,8 @@ use SegmentReader;
 ///         index_writer.commit().unwrap();
 ///     }
 ///
-///     index.load_searchers()?;
-///     let searcher = index.searcher();
+///     let reader = index.reader()?;
+///     let searcher = reader.searcher();
 ///
 ///     let query_parser = QueryParser::for_index(&index, vec![title]);
 ///     let query = query_parser.parse_query("diary")?;
@@ -148,7 +148,6 @@ mod tests {
             index_writer.add_document(doc!(text_field=>"I like Droopy"));
             assert!(index_writer.commit().is_ok());
         }
-        index.load_searchers().unwrap();
         index
     }
 
@@ -159,6 +158,8 @@ mod tests {
         let query_parser = QueryParser::for_index(&index, vec![field]);
         let text_query = query_parser.parse_query("droopy tax").unwrap();
         let score_docs: Vec<(Score, DocAddress)> = index
+            .reader()
+            .unwrap()
             .searcher()
             .search(&text_query, &TopDocs::with_limit(4))
             .unwrap();
@@ -179,6 +180,8 @@ mod tests {
         let query_parser = QueryParser::for_index(&index, vec![field]);
         let text_query = query_parser.parse_query("droopy tax").unwrap();
         let score_docs: Vec<(Score, DocAddress)> = index
+            .reader()
+            .unwrap()
             .searcher()
             .search(&text_query, &TopDocs::with_limit(2))
             .unwrap();

src/common/bitpacker.rs

@@ -80,7 +80,7 @@ where
             (1u64 << num_bits) - 1u64
         };
         BitUnpacker {
-            num_bits: num_bits as u64,
+            num_bits: u64::from(num_bits),
             mask,
             data,
         }

src/common/mod.rs

@@ -13,7 +13,10 @@ pub use self::serialize::{BinarySerializable, FixedSize};
 pub use self::vint::{read_u32_vint, serialize_vint_u32, write_u32_vint, VInt};
 pub use byteorder::LittleEndian as Endianness;
 
-use std::io;
+/// Segment's max doc must be `< MAX_DOC_LIMIT`.
+///
+/// We do not allow segments with more than
+pub const MAX_DOC_LIMIT: u32 = 1 << 31;
 
 /// Computes the number of bits that will be used for bitpacking.
 ///
@@ -52,11 +55,6 @@ pub(crate) fn is_power_of_2(n: usize) -> bool {
     (n > 0) && (n & (n - 1) == 0)
 }
 
-/// Create a default io error given a string.
-pub(crate) fn make_io_err(msg: String) -> io::Error {
-    io::Error::new(io::ErrorKind::Other, msg)
-}
-
 /// Has length trait
 pub trait HasLen {
     /// Return length
@@ -134,4 +132,11 @@ pub(crate) mod test {
         assert_eq!(compute_num_bits(256), 9u8);
         assert_eq!(compute_num_bits(5_000_000_000), 33u8);
     }
+
+    #[test]
+    fn test_max_doc() {
+        // this is the first time I write a unit test for a constant.
+        assert!(((super::MAX_DOC_LIMIT - 1) as i32) >= 0);
+        assert!((super::MAX_DOC_LIMIT as i32) < 0);
+    }
 }

src/core/executor.rs

@@ -123,15 +123,14 @@ mod tests {
         }
     }
 
-}
-
-#[test]
-fn test_map_multithread() {
-    let result: Vec<usize> = Executor::multi_thread(3, "search-test")
-        .map(|i| Ok(i * 2), 0..10)
-        .unwrap();
-    assert_eq!(result.len(), 10);
-    for i in 0..10 {
-        assert_eq!(result[i], i * 2);
+    #[test]
+    fn test_map_multithread() {
+        let result: Vec<usize> = Executor::multi_thread(3, "search-test")
+            .map(|i| Ok(i * 2), 0..10)
+            .unwrap();
+        assert_eq!(result.len(), 10);
+        for i in 0..10 {
+            assert_eq!(result[i], i * 2);
+        }
     }
 }

src/core/index.rs

@@ -1,19 +1,14 @@
-use super::pool::LeasedItem;
-use super::pool::Pool;
 use super::segment::create_segment;
 use super::segment::Segment;
-use core::searcher::Searcher;
 use core::Executor;
 use core::IndexMeta;
 use core::SegmentId;
 use core::SegmentMeta;
-use core::SegmentReader;
 use core::META_FILEPATH;
 use directory::ManagedDirectory;
 #[cfg(feature = "mmap")]
 use directory::MmapDirectory;
 use directory::INDEX_WRITER_LOCK;
-use directory::META_LOCK;
 use directory::{Directory, RAMDirectory};
 use error::DataCorruption;
 use error::TantivyError;
@@ -21,14 +16,16 @@ use indexer::index_writer::open_index_writer;
 use indexer::index_writer::HEAP_SIZE_MIN;
 use indexer::segment_updater::save_new_metas;
 use num_cpus;
+use reader::IndexReader;
+use reader::IndexReaderBuilder;
 use schema::Field;
 use schema::FieldType;
 use schema::Schema;
 use serde_json;
 use std::borrow::BorrowMut;
 use std::fmt;
+#[cfg(feature = "mmap")]
 use std::path::Path;
-use std::sync::atomic::{AtomicUsize, Ordering};
 use std::sync::Arc;
 use tokenizer::BoxedTokenizer;
 use tokenizer::TokenizerManager;
@@ -49,11 +46,10 @@ fn load_metas(directory: &Directory) -> Result<IndexMeta> {
 }
 
 /// Search Index
+#[derive(Clone)]
 pub struct Index {
     directory: ManagedDirectory,
     schema: Schema,
-    num_searchers: Arc<AtomicUsize>,
-    searcher_pool: Arc<Pool<Searcher>>,
     executor: Arc<Executor>,
     tokenizers: TokenizerManager,
 }
@@ -158,16 +154,12 @@ impl Index {
     /// Creates a new index given a directory and an `IndexMeta`.
     fn create_from_metas(directory: ManagedDirectory, metas: &IndexMeta) -> Result<Index> {
         let schema = metas.schema.clone();
-        let n_cpus = num_cpus::get();
         let index = Index {
             directory,
             schema,
-            num_searchers: Arc::new(AtomicUsize::new(n_cpus)),
-            searcher_pool: Arc::new(Pool::new()),
             tokenizers: TokenizerManager::default(),
             executor: Arc::new(Executor::single_thread()),
         };
-        index.load_searchers()?;
         Ok(index)
     }
 
@@ -197,6 +189,22 @@ impl Index {
         }
     }
 
+    /// Create a default `IndexReader` for the given index.
+    ///
+    /// See [`Index.reader_builder()`](#method.reader_builder).
+    pub fn reader(&self) -> Result<IndexReader> {
+        self.reader_builder().try_into()
+    }
+
+    /// Create a `IndexReader` for the given index.
+    ///
+    /// Most project should create at most one reader for a given index.
+    /// This method is typically called only once per `Index` instance,
+    /// over the lifetime of most problem.
+    pub fn reader_builder(&self) -> IndexReaderBuilder {
+        IndexReaderBuilder::new(self.clone())
+    }
+
     /// Opens a new directory from an index path.
     #[cfg(feature = "mmap")]
     pub fn open_in_dir<P: AsRef<Path>>(directory_path: P) -> Result<Index> {
@@ -335,53 +343,6 @@ impl Index {
             .map(|segment_meta| segment_meta.id())
             .collect())
     }
-
-    /// Sets the number of searchers to use
-    ///
-    /// Only works after the next call to `load_searchers`
-    pub fn set_num_searchers(&mut self, num_searchers: usize) {
-        self.num_searchers.store(num_searchers, Ordering::Release);
-    }
-
-    /// Update searchers so that they reflect the state of the last
-    /// `.commit()`.
-    ///
-    /// If indexing happens in the same process as searching,
-    /// you most likely want to call `.load_searchers()` right after each
-    /// successful call to `.commit()`.
-    ///
-    /// If indexing and searching happen in different processes, the way to
-    /// get the freshest `index` at all time, is to watch `meta.json` and
-    /// call `load_searchers` whenever a changes happen.
-    pub fn load_searchers(&self) -> Result<()> {
-        let _meta_lock = self.directory().acquire_lock(&META_LOCK)?;
-        let searchable_segments = self.searchable_segments()?;
-        let segment_readers: Vec<SegmentReader> = searchable_segments
-            .iter()
-            .map(SegmentReader::open)
-            .collect::<Result<_>>()?;
-        let schema = self.schema();
-        let num_searchers: usize = self.num_searchers.load(Ordering::Acquire);
-        let searchers = (0..num_searchers)
-            .map(|_| Searcher::new(schema.clone(), self.clone(), segment_readers.clone()))
-            .collect();
-        self.searcher_pool.publish_new_generation(searchers);
-        Ok(())
-    }
-
-    /// Returns a searcher
-    ///
-    /// This method should be called every single time a search
-    /// query is performed.
-    /// The searchers are taken from a pool of `num_searchers` searchers.
-    /// If no searcher is available
-    /// this may block.
-    ///
-    /// The same searcher must be used for a given query, as it ensures
-    /// the use of a consistent segment set.
-    pub fn searcher(&self) -> LeasedItem<Searcher> {
-        self.searcher_pool.acquire()
-    }
 }
 
 impl fmt::Debug for Index {
@@ -390,29 +351,22 @@ impl fmt::Debug for Index {
     }
 }
 
-impl Clone for Index {
-    fn clone(&self) -> Index {
-        Index {
-            directory: self.directory.clone(),
-            schema: self.schema.clone(),
-            num_searchers: Arc::clone(&self.num_searchers),
-            searcher_pool: Arc::clone(&self.searcher_pool),
-            tokenizers: self.tokenizers.clone(),
-            executor: self.executor.clone(),
-        }
-    }
-}
-
 #[cfg(test)]
 mod tests {
     use directory::RAMDirectory;
-    use schema::{Schema, INT_INDEXED, TEXT};
+    use schema::Field;
+    use schema::{Schema, INDEXED, TEXT};
+    use std::thread;
+    use std::time::Duration;
     use Index;
+    use IndexReader;
+    use IndexWriter;
+    use ReloadPolicy;
 
     #[test]
     fn test_indexer_for_field() {
         let mut schema_builder = Schema::builder();
-        let num_likes_field = schema_builder.add_u64_field("num_likes", INT_INDEXED);
+        let num_likes_field = schema_builder.add_u64_field("num_likes", INDEXED);
         let body_field = schema_builder.add_text_field("body", TEXT);
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
@@ -470,7 +424,117 @@ mod tests {
 
     fn throw_away_schema() -> Schema {
         let mut schema_builder = Schema::builder();
-        let _ = schema_builder.add_u64_field("num_likes", INT_INDEXED);
+        let _ = schema_builder.add_u64_field("num_likes", INDEXED);
         schema_builder.build()
     }
+
+    #[test]
+    fn test_index_on_commit_reload_policy() {
+        let schema = throw_away_schema();
+        let field = schema.get_field("num_likes").unwrap();
+        let index = Index::create_in_ram(schema);
+        let reader = index
+            .reader_builder()
+            .reload_policy(ReloadPolicy::OnCommit)
+            .try_into()
+            .unwrap();
+        assert_eq!(reader.searcher().num_docs(), 0);
+        let mut writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
+        test_index_on_commit_reload_policy_aux(field, &mut writer, &reader);
+    }
+
+    #[cfg(feature = "mmap")]
+    mod mmap_specific {
+
+        use super::*;
+        use std::path::PathBuf;
+        use tempdir::TempDir;
+
+        #[test]
+        fn test_index_on_commit_reload_policy_mmap() {
+            let schema = throw_away_schema();
+            let field = schema.get_field("num_likes").unwrap();
+            let tempdir = TempDir::new("index").unwrap();
+            let tempdir_path = PathBuf::from(tempdir.path());
+            let index = Index::create_in_dir(&tempdir_path, schema).unwrap();
+            let mut writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
+            writer.commit().unwrap();
+            let reader = index
+                .reader_builder()
+                .reload_policy(ReloadPolicy::OnCommit)
+                .try_into()
+                .unwrap();
+            assert_eq!(reader.searcher().num_docs(), 0);
+            test_index_on_commit_reload_policy_aux(field, &mut writer, &reader);
+        }
+
+        #[test]
+        fn test_index_manual_policy_mmap() {
+            let schema = throw_away_schema();
+            let field = schema.get_field("num_likes").unwrap();
+            let index = Index::create_from_tempdir(schema).unwrap();
+            let mut writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
+            writer.commit().unwrap();
+            let reader = index
+                .reader_builder()
+                .reload_policy(ReloadPolicy::Manual)
+                .try_into()
+                .unwrap();
+            assert_eq!(reader.searcher().num_docs(), 0);
+            writer.add_document(doc!(field=>1u64));
+            writer.commit().unwrap();
+            thread::sleep(Duration::from_millis(500));
+            assert_eq!(reader.searcher().num_docs(), 0);
+            reader.reload().unwrap();
+            assert_eq!(reader.searcher().num_docs(), 1);
+        }
+
+        #[test]
+        fn test_index_on_commit_reload_policy_different_directories() {
+            let schema = throw_away_schema();
+            let field = schema.get_field("num_likes").unwrap();
+            let tempdir = TempDir::new("index").unwrap();
+            let tempdir_path = PathBuf::from(tempdir.path());
+            let write_index = Index::create_in_dir(&tempdir_path, schema).unwrap();
+            let read_index = Index::open_in_dir(&tempdir_path).unwrap();
+            let reader = read_index
+                .reader_builder()
+                .reload_policy(ReloadPolicy::OnCommit)
+                .try_into()
+                .unwrap();
+            assert_eq!(reader.searcher().num_docs(), 0);
+            let mut writer = write_index.writer_with_num_threads(1, 3_000_000).unwrap();
+            test_index_on_commit_reload_policy_aux(field, &mut writer, &reader);
+        }
+    }
+
+    fn test_index_on_commit_reload_policy_aux(
+        field: Field,
+        writer: &mut IndexWriter,
+        reader: &IndexReader,
+    ) {
+        assert_eq!(reader.searcher().num_docs(), 0);
+        writer.add_document(doc!(field=>1u64));
+        writer.commit().unwrap();
+        let mut count = 0;
+        for _ in 0..100 {
+            count = reader.searcher().num_docs();
+            if count > 0 {
+                break;
+            }
+            thread::sleep(Duration::from_millis(100));
+        }
+        assert_eq!(count, 1);
+        writer.add_document(doc!(field=>2u64));
+        writer.commit().unwrap();
+        let mut count = 0;
+        for _ in 0..10 {
+            count = reader.searcher().num_docs();
+            if count > 1 {
+                break;
+            }
+            thread::sleep(Duration::from_millis(100));
+        }
+        assert_eq!(count, 2);
+    }
 }

src/core/mod.rs

@@ -2,7 +2,6 @@ mod executor;
 pub mod index;
 mod index_meta;
 mod inverted_index_reader;
-mod pool;
 pub mod searcher;
 mod segment;
 mod segment_component;
@@ -25,6 +24,7 @@ pub use self::segment_reader::SegmentReader;
 use std::path::PathBuf;
 
 lazy_static! {
+
     /// The meta file contains all the information about the list of segments and the schema
     /// of the index.
     pub static ref META_FILEPATH: PathBuf = PathBuf::from("meta.json");

src/core/segment_id.rs

@@ -19,7 +19,7 @@ pub struct SegmentId(Uuid);
 #[cfg(test)]
 lazy_static! {
     static ref AUTO_INC_COUNTER: atomic::AtomicUsize = atomic::AtomicUsize::default();
-    static ref EMPTY_ARR: [u8; 8] = [0u8; 8];
+    static ref ZERO_ARRAY: [u8; 8] = [0u8; 8];
 }
 
 // During tests, we generate the segment id in a autoincrement manner
@@ -30,7 +30,7 @@ lazy_static! {
 #[cfg(test)]
 fn create_uuid() -> Uuid {
     let new_auto_inc_id = (*AUTO_INC_COUNTER).fetch_add(1, atomic::Ordering::SeqCst);
-    Uuid::from_fields(new_auto_inc_id as u32, 0, 0, &*EMPTY_ARR).unwrap()
+    Uuid::from_fields(new_auto_inc_id as u32, 0, 0, &*ZERO_ARRAY).unwrap()
 }
 
 #[cfg(not(test))]

src/core/segment_reader.rs

@@ -477,9 +477,7 @@ mod test {
             // ok, now we should have a deleted doc
             index_writer2.commit().unwrap();
         }
-
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        let searcher = index.reader().unwrap().searcher();
         let docs: Vec<DocId> = searcher.segment_reader(0).doc_ids_alive().collect();
         assert_eq!(vec![0u32, 2u32], docs);
     }

src/directory/directory.rs

@@ -1,6 +1,8 @@
 use directory::directory_lock::Lock;
 use directory::error::LockError;
 use directory::error::{DeleteError, OpenReadError, OpenWriteError};
+use directory::WatchCallback;
+use directory::WatchHandle;
 use directory::{ReadOnlySource, WritePtr};
 use std::fmt;
 use std::io;
@@ -187,6 +189,22 @@ pub trait Directory: DirectoryClone + fmt::Debug + Send + Sync + 'static {
             }
         }
     }
+
+    /// Registers a callback that will be called whenever a change on the `meta.json`
+    /// using the `atomic_write` API is detected.
+    ///
+    /// The behavior when using `.watch()` on a file using `.open_write(...)` is, on the other
+    /// hand, undefined.
+    ///
+    /// The file will be watched for the lifetime of the returned `WatchHandle`. The caller is
+    /// required to keep it.
+    /// It does not override previous callbacks. When the file is modified, all callback that are
+    /// registered (and whose `WatchHandle` is still alive) are triggered.
+    ///
+    /// Internally, tantivy only uses this API to detect new commits to implement the
+    /// `OnCommit` `ReloadPolicy`. Not implementing watch in a `Directory` only prevents the
+    /// `OnCommit` `ReloadPolicy` to work properly.
+    fn watch(&self, watch_callback: WatchCallback) -> WatchHandle;
 }
 
 /// DirectoryClone

src/directory/directory_lock.rs

@@ -43,7 +43,7 @@ lazy_static! {
         is_blocking: false
     };
     /// The meta lock file is here to protect the segment files being opened by
-    /// `.load_searchers()` from being garbage collected.
+    /// `IndexReader::reload()` from being garbage collected.
     /// It makes it possible for another process to safely consume
     /// our index in-writing. Ideally, we may have prefered `RWLock` semantics
     /// here, but it is difficult to achieve on Windows.

src/directory/error.rs

@@ -73,6 +73,14 @@ pub enum OpenDirectoryError {
     DoesNotExist(PathBuf),
     /// The path exists but is not a directory.
     NotADirectory(PathBuf),
+    /// IoError
+    IoError(io::Error),
+}
+
+impl From<io::Error> for OpenDirectoryError {
+    fn from(io_err: io::Error) -> Self {
+        OpenDirectoryError::IoError(io_err)
+    }
 }
 
 impl fmt::Display for OpenDirectoryError {
@@ -84,6 +92,11 @@ impl fmt::Display for OpenDirectoryError {
             OpenDirectoryError::NotADirectory(ref path) => {
                 write!(f, "the path '{:?}' exists but is not a directory", path)
             }
+            OpenDirectoryError::IoError(ref err) => write!(
+                f,
+                "IOError while trying to open/create the directory. {:?}",
+                err
+            ),
         }
     }
 }

src/directory/managed_directory.rs

@@ -4,6 +4,7 @@ use directory::DirectoryLock;
 use directory::Lock;
 use directory::META_LOCK;
 use directory::{ReadOnlySource, WritePtr};
+use directory::{WatchCallback, WatchHandle};
 use error::DataCorruption;
 use serde_json;
 use std::collections::HashSet;
@@ -241,6 +242,10 @@ impl Directory for ManagedDirectory {
     fn acquire_lock(&self, lock: &Lock) -> result::Result<DirectoryLock, LockError> {
         self.directory.acquire_lock(lock)
     }
+
+    fn watch(&self, watch_callback: WatchCallback) -> WatchHandle {
+        self.directory.watch(watch_callback)
+    }
 }
 
 impl Clone for ManagedDirectory {
@@ -255,95 +260,98 @@ impl Clone for ManagedDirectory {
 #[cfg(test)]
 mod tests {
 
-    use super::*;
     #[cfg(feature = "mmap")]
-    use directory::MmapDirectory;
-    use std::io::Write;
-    use std::path::Path;
-    use tempdir::TempDir;
+    mod mmap_specific {
 
-    lazy_static! {
-        static ref TEST_PATH1: &'static Path = Path::new("some_path_for_test");
-        static ref TEST_PATH2: &'static Path = Path::new("some_path_for_test2");
-    }
+        use super::super::*;
+        use std::path::Path;
+        use tempdir::TempDir;
 
-    #[test]
-    #[cfg(feature = "mmap")]
-    fn test_managed_directory() {
-        let tempdir = TempDir::new("index").unwrap();
-        let tempdir_path = PathBuf::from(tempdir.path());
-        {
-            let mmap_directory = MmapDirectory::open(&tempdir_path).unwrap();
-            let mut managed_directory = ManagedDirectory::wrap(mmap_directory).unwrap();
+        lazy_static! {
+            static ref TEST_PATH1: &'static Path = Path::new("some_path_for_test");
+            static ref TEST_PATH2: &'static Path = Path::new("some_path_for_test2");
+        }
+
+        use directory::MmapDirectory;
+        use std::io::Write;
+
+        #[test]
+        fn test_managed_directory() {
+            let tempdir = TempDir::new("index").unwrap();
+            let tempdir_path = PathBuf::from(tempdir.path());
             {
-                let mut write_file = managed_directory.open_write(*TEST_PATH1).unwrap();
-                write_file.flush().unwrap();
+                let mmap_directory = MmapDirectory::open(&tempdir_path).unwrap();
+                let mut managed_directory = ManagedDirectory::wrap(mmap_directory).unwrap();
+                {
+                    let mut write_file = managed_directory.open_write(*TEST_PATH1).unwrap();
+                    write_file.flush().unwrap();
+                }
+                {
+                    managed_directory
+                        .atomic_write(*TEST_PATH2, &vec![0u8, 1u8])
+                        .unwrap();
+                }
+                {
+                    assert!(managed_directory.exists(*TEST_PATH1));
+                    assert!(managed_directory.exists(*TEST_PATH2));
+                }
+                {
+                    let living_files: HashSet<PathBuf> =
+                        [TEST_PATH1.to_owned()].into_iter().cloned().collect();
+                    managed_directory.garbage_collect(|| living_files);
+                }
+                {
+                    assert!(managed_directory.exists(*TEST_PATH1));
+                    assert!(!managed_directory.exists(*TEST_PATH2));
+                }
             }
             {
-                managed_directory
-                    .atomic_write(*TEST_PATH2, &vec![0u8, 1u8])
-                    .unwrap();
-            }
-            {
-                assert!(managed_directory.exists(*TEST_PATH1));
-                assert!(managed_directory.exists(*TEST_PATH2));
-            }
-            {
-                let living_files: HashSet<PathBuf> =
-                    [TEST_PATH1.to_owned()].into_iter().cloned().collect();
-                managed_directory.garbage_collect(|| living_files);
-            }
-            {
-                assert!(managed_directory.exists(*TEST_PATH1));
-                assert!(!managed_directory.exists(*TEST_PATH2));
+                let mmap_directory = MmapDirectory::open(&tempdir_path).unwrap();
+                let mut managed_directory = ManagedDirectory::wrap(mmap_directory).unwrap();
+                {
+                    assert!(managed_directory.exists(*TEST_PATH1));
+                    assert!(!managed_directory.exists(*TEST_PATH2));
+                }
+                {
+                    let living_files: HashSet<PathBuf> = HashSet::new();
+                    managed_directory.garbage_collect(|| living_files);
+                }
+                {
+                    assert!(!managed_directory.exists(*TEST_PATH1));
+                    assert!(!managed_directory.exists(*TEST_PATH2));
+                }
             }
         }
-        {
+
+        #[test]
+        fn test_managed_directory_gc_while_mmapped() {
+            let tempdir = TempDir::new("index").unwrap();
+            let tempdir_path = PathBuf::from(tempdir.path());
+            let living_files = HashSet::new();
+
             let mmap_directory = MmapDirectory::open(&tempdir_path).unwrap();
             let mut managed_directory = ManagedDirectory::wrap(mmap_directory).unwrap();
-            {
-                assert!(managed_directory.exists(*TEST_PATH1));
-                assert!(!managed_directory.exists(*TEST_PATH2));
-            }
-            {
-                let living_files: HashSet<PathBuf> = HashSet::new();
-                managed_directory.garbage_collect(|| living_files);
-            }
-            {
-                assert!(!managed_directory.exists(*TEST_PATH1));
-                assert!(!managed_directory.exists(*TEST_PATH2));
-            }
-        }
-    }
-
-    #[test]
-    #[cfg(feature = "mmap ")]
-    fn test_managed_directory_gc_while_mmapped() {
-        let tempdir = TempDir::new("index").unwrap();
-        let tempdir_path = PathBuf::from(tempdir.path());
-        let living_files = HashSet::new();
-
-        let mmap_directory = MmapDirectory::open(&tempdir_path).unwrap();
-        let mut managed_directory = ManagedDirectory::wrap(mmap_directory).unwrap();
-        managed_directory
-            .atomic_write(*TEST_PATH1, &vec![0u8, 1u8])
-            .unwrap();
-        assert!(managed_directory.exists(*TEST_PATH1));
-
-        let _mmap_read = managed_directory.open_read(*TEST_PATH1).unwrap();
-        managed_directory.garbage_collect(|| living_files.clone());
-        if cfg!(target_os = "windows") {
-            // On Windows, gc should try and fail the file as it is mmapped.
+            managed_directory
+                .atomic_write(*TEST_PATH1, &vec![0u8, 1u8])
+                .unwrap();
             assert!(managed_directory.exists(*TEST_PATH1));
-            // unmap should happen here.
-            drop(_mmap_read);
-            // The file should still be in the list of managed file and
-            // eventually be deleted once mmap is released.
-            managed_directory.garbage_collect(|| living_files);
-            assert!(!managed_directory.exists(*TEST_PATH1));
-        } else {
-            assert!(!managed_directory.exists(*TEST_PATH1));
+
+            let _mmap_read = managed_directory.open_read(*TEST_PATH1).unwrap();
+            managed_directory.garbage_collect(|| living_files.clone());
+            if cfg!(target_os = "windows") {
+                // On Windows, gc should try and fail the file as it is mmapped.
+                assert!(managed_directory.exists(*TEST_PATH1));
+                // unmap should happen here.
+                drop(_mmap_read);
+                // The file should still be in the list of managed file and
+                // eventually be deleted once mmap is released.
+                managed_directory.garbage_collect(|| living_files);
+                assert!(!managed_directory.exists(*TEST_PATH1));
+            } else {
+                assert!(!managed_directory.exists(*TEST_PATH1));
+            }
         }
+
     }
 
 }

src/directory/mmap_directory.rs

@@ -1,18 +1,24 @@
 extern crate fs2;
+extern crate notify;
 
 use self::fs2::FileExt;
+use self::notify::RawEvent;
+use self::notify::RecursiveMode;
+use self::notify::Watcher;
 use atomicwrites;
-use common::make_io_err;
+use core::META_FILEPATH;
 use directory::error::LockError;
 use directory::error::{DeleteError, IOError, OpenDirectoryError, OpenReadError, OpenWriteError};
-use directory::shared_vec_slice::SharedVecSlice;
+use directory::read_only_source::BoxedData;
 use directory::Directory;
 use directory::DirectoryLock;
 use directory::Lock;
 use directory::ReadOnlySource;
+use directory::WatchCallback;
+use directory::WatchCallbackList;
+use directory::WatchHandle;
 use directory::WritePtr;
-use fst::raw::MmapReadOnly;
-use std::collections::hash_map::Entry as HashMapEntry;
+use memmap::Mmap;
 use std::collections::HashMap;
 use std::convert::From;
 use std::fmt;
@@ -22,14 +28,22 @@ use std::io::{self, Seek, SeekFrom};
 use std::io::{BufWriter, Read, Write};
 use std::path::{Path, PathBuf};
 use std::result;
+use std::sync::mpsc::{channel, Receiver, Sender};
 use std::sync::Arc;
+use std::sync::Mutex;
 use std::sync::RwLock;
+use std::sync::Weak;
+use std::thread;
 use tempdir::TempDir;
 
+/// Create a default io error given a string.
+pub(crate) fn make_io_err(msg: String) -> io::Error {
+    io::Error::new(io::ErrorKind::Other, msg)
+}
+
 /// Returns None iff the file exists, can be read, but is empty (and hence
-/// cannot be mmapped).
-///
-fn open_mmap(full_path: &Path) -> result::Result<Option<MmapReadOnly>, OpenReadError> {
+/// cannot be mmapped)
+fn open_mmap(full_path: &Path) -> result::Result<Option<Mmap>, OpenReadError> {
     let file = File::open(full_path).map_err(|e| {
         if e.kind() == io::ErrorKind::NotFound {
             OpenReadError::FileDoesNotExist(full_path.to_owned())
@@ -48,7 +62,7 @@ fn open_mmap(full_path: &Path) -> result::Result<Option<MmapReadOnly>, OpenReadE
         return Ok(None);
     }
     unsafe {
-        MmapReadOnly::open(&file)
+        memmap::Mmap::map(&file)
             .map(Some)
             .map_err(|e| From::from(IOError::with_path(full_path.to_owned(), e)))
     }
@@ -71,7 +85,7 @@ pub struct CacheInfo {
 
 struct MmapCache {
     counters: CacheCounters,
-    cache: HashMap<PathBuf, MmapReadOnly>,
+    cache: HashMap<PathBuf, Weak<BoxedData>>,
 }
 
 impl Default for MmapCache {
@@ -84,12 +98,7 @@ impl Default for MmapCache {
 }
 
 impl MmapCache {
-    /// Removes a `MmapReadOnly` entry from the mmap cache.
-    fn discard_from_cache(&mut self, full_path: &Path) -> bool {
-        self.cache.remove(full_path).is_some()
-    }
-
-    fn get_info(&mut self) -> CacheInfo {
+    fn get_info(&self) -> CacheInfo {
         let paths: Vec<PathBuf> = self.cache.keys().cloned().collect();
         CacheInfo {
             counters: self.counters.clone(),
@@ -97,26 +106,108 @@ impl MmapCache {
         }
     }
 
-    fn get_mmap(&mut self, full_path: &Path) -> Result<Option<MmapReadOnly>, OpenReadError> {
-        Ok(match self.cache.entry(full_path.to_owned()) {
-            HashMapEntry::Occupied(occupied_entry) => {
-                let mmap = occupied_entry.get();
+    fn remove_weak_ref(&mut self) {
+        let keys_to_remove: Vec<PathBuf> = self
+            .cache
+            .iter()
+            .filter(|(_, mmap_weakref)| mmap_weakref.upgrade().is_none())
+            .map(|(key, _)| key.clone())
+            .collect();
+        for key in keys_to_remove {
+            self.cache.remove(&key);
+        }
+    }
+
+    // Returns None if the file exists but as a len of 0 (and hence is not mmappable).
+    fn get_mmap(&mut self, full_path: &Path) -> Result<Option<Arc<BoxedData>>, OpenReadError> {
+        if let Some(mmap_weak) = self.cache.get(full_path) {
+            if let Some(mmap_arc) = mmap_weak.upgrade() {
                 self.counters.hit += 1;
-                Some(mmap.clone())
-            }
-            HashMapEntry::Vacant(vacant_entry) => {
-                self.counters.miss += 1;
-                if let Some(mmap) = open_mmap(full_path)? {
-                    vacant_entry.insert(mmap.clone());
-                    Some(mmap)
-                } else {
-                    None
-                }
+                return Ok(Some(mmap_arc));
             }
+        }
+        self.cache.remove(full_path);
+        self.counters.miss += 1;
+        Ok(if let Some(mmap) = open_mmap(full_path)? {
+            let mmap_arc: Arc<BoxedData> = Arc::new(Box::new(mmap));
+            let mmap_weak = Arc::downgrade(&mmap_arc);
+            self.cache.insert(full_path.to_owned(), mmap_weak);
+            Some(mmap_arc)
+        } else {
+            None
         })
     }
 }
 
+struct InnerWatcherWrapper {
+    _watcher: Mutex<notify::RecommendedWatcher>,
+    watcher_router: WatchCallbackList,
+}
+
+impl InnerWatcherWrapper {
+    pub fn new(path: &Path) -> Result<(Self, Receiver<notify::RawEvent>), notify::Error> {
+        let (tx, watcher_recv): (Sender<RawEvent>, Receiver<RawEvent>) = channel();
+        // We need to initialize the
+        let mut watcher = notify::raw_watcher(tx)?;
+        watcher.watch(path, RecursiveMode::Recursive)?;
+        let inner = InnerWatcherWrapper {
+            _watcher: Mutex::new(watcher),
+            watcher_router: Default::default(),
+        };
+        Ok((inner, watcher_recv))
+    }
+}
+
+#[derive(Clone)]
+pub(crate) struct WatcherWrapper {
+    inner: Arc<InnerWatcherWrapper>,
+}
+
+impl WatcherWrapper {
+    pub fn new(path: &Path) -> Result<Self, OpenDirectoryError> {
+        let (inner, watcher_recv) = InnerWatcherWrapper::new(path).map_err(|err| match err {
+            notify::Error::PathNotFound => OpenDirectoryError::DoesNotExist(path.to_owned()),
+            _ => {
+                panic!("Unknown error while starting watching directory {:?}", path);
+            }
+        })?;
+        let watcher_wrapper = WatcherWrapper {
+            inner: Arc::new(inner),
+        };
+        let watcher_wrapper_clone = watcher_wrapper.clone();
+        thread::Builder::new()
+            .name("meta-file-watch-thread".to_string())
+            .spawn(move || {
+                loop {
+                    match watcher_recv.recv().map(|evt| evt.path) {
+                        Ok(Some(changed_path)) => {
+                            // ... Actually subject to false positive.
+                            // We might want to be more accurate than this at one point.
+                            if let Some(filename) = changed_path.file_name() {
+                                if filename == *META_FILEPATH {
+                                    watcher_wrapper_clone.inner.watcher_router.broadcast();
+                                }
+                            }
+                        }
+                        Ok(None) => {
+                            // not an event we are interested in.
+                        }
+                        Err(_e) => {
+                            // the watch send channel was dropped
+                            break;
+                        }
+                    }
+                }
+            })
+            .expect("Failed to spawn thread to watch meta.json");
+        Ok(watcher_wrapper)
+    }
+
+    pub fn watch(&mut self, watch_callback: WatchCallback) -> WatchHandle {
+        self.inner.watcher_router.subscribe(watch_callback)
+    }
+}
+
 /// Directory storing data in files, read via mmap.
 ///
 /// The Mmap object are cached to limit the
@@ -131,31 +222,62 @@ impl MmapCache {
 /// On Windows the semantics are again different.
 #[derive(Clone)]
 pub struct MmapDirectory {
+    inner: Arc<MmapDirectoryInner>,
+}
+
+struct MmapDirectoryInner {
     root_path: PathBuf,
-    mmap_cache: Arc<RwLock<MmapCache>>,
-    _temp_directory: Arc<Option<TempDir>>,
+    mmap_cache: RwLock<MmapCache>,
+    _temp_directory: Option<TempDir>,
+    watcher: RwLock<WatcherWrapper>,
+}
+
+impl MmapDirectoryInner {
+    fn new(
+        root_path: PathBuf,
+        temp_directory: Option<TempDir>,
+    ) -> Result<MmapDirectoryInner, OpenDirectoryError> {
+        let watch_wrapper = WatcherWrapper::new(&root_path)?;
+        let mmap_directory_inner = MmapDirectoryInner {
+            root_path,
+            mmap_cache: Default::default(),
+            _temp_directory: temp_directory,
+            watcher: RwLock::new(watch_wrapper),
+        };
+        Ok(mmap_directory_inner)
+    }
+
+    fn watch(&self, watch_callback: WatchCallback) -> WatchHandle {
+        let mut wlock = self.watcher.write().unwrap();
+        wlock.watch(watch_callback)
+    }
 }
 
 impl fmt::Debug for MmapDirectory {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "MmapDirectory({:?})", self.root_path)
+        write!(f, "MmapDirectory({:?})", self.inner.root_path)
     }
 }
 
 impl MmapDirectory {
+    fn new(
+        root_path: PathBuf,
+        temp_directory: Option<TempDir>,
+    ) -> Result<MmapDirectory, OpenDirectoryError> {
+        let inner = MmapDirectoryInner::new(root_path, temp_directory)?;
+        Ok(MmapDirectory {
+            inner: Arc::new(inner),
+        })
+    }
+
     /// Creates a new MmapDirectory in a temporary directory.
     ///
     /// This is mostly useful to test the MmapDirectory itself.
     /// For your unit tests, prefer the RAMDirectory.
-    pub fn create_from_tempdir() -> io::Result<MmapDirectory> {
-        let tempdir = TempDir::new("index")?;
+    pub fn create_from_tempdir() -> Result<MmapDirectory, OpenDirectoryError> {
+        let tempdir = TempDir::new("index").map_err(OpenDirectoryError::IoError)?;
         let tempdir_path = PathBuf::from(tempdir.path());
-        let directory = MmapDirectory {
-            root_path: tempdir_path,
-            mmap_cache: Arc::new(RwLock::new(MmapCache::default())),
-            _temp_directory: Arc::new(Some(tempdir)),
-        };
-        Ok(directory)
+        MmapDirectory::new(tempdir_path, Some(tempdir))
     }
 
     /// Opens a MmapDirectory in a directory.
@@ -173,18 +295,14 @@ impl MmapDirectory {
                 directory_path,
             )))
         } else {
-            Ok(MmapDirectory {
-                root_path: PathBuf::from(directory_path),
-                mmap_cache: Arc::new(RwLock::new(MmapCache::default())),
-                _temp_directory: Arc::new(None),
-            })
+            Ok(MmapDirectory::new(PathBuf::from(directory_path), None)?)
         }
     }
 
     /// Joins a relative_path to the directory `root_path`
     /// to create a proper complete `filepath`.
     fn resolve_path(&self, relative_path: &Path) -> PathBuf {
-        self.root_path.join(relative_path)
+        self.inner.root_path.join(relative_path)
     }
 
     /// Sync the root directory.
@@ -209,7 +327,7 @@ impl MmapDirectory {
                 .custom_flags(winbase::FILE_FLAG_BACKUP_SEMANTICS);
         }
 
-        let fd = open_opts.open(&self.root_path)?;
+        let fd = open_opts.open(&self.inner.root_path)?;
         fd.sync_all()?;
         Ok(())
     }
@@ -219,9 +337,15 @@ impl MmapDirectory {
     ///
     /// The `MmapDirectory` embeds a `MmapDirectory`
     /// to avoid multiplying the `mmap` system calls.
-    pub fn get_cache_info(&mut self) -> CacheInfo {
-        self.mmap_cache
+    pub fn get_cache_info(&self) -> CacheInfo {
+        self.inner
+            .mmap_cache
             .write()
+            .expect("mmap cache lock is poisoned")
+            .remove_weak_ref();
+        self.inner
+            .mmap_cache
+            .read()
             .expect("Mmap cache lock is poisoned.")
             .get_info()
     }
@@ -274,7 +398,7 @@ impl Directory for MmapDirectory {
         debug!("Open Read {:?}", path);
         let full_path = self.resolve_path(path);
 
-        let mut mmap_cache = self.mmap_cache.write().map_err(|_| {
+        let mut mmap_cache = self.inner.mmap_cache.write().map_err(|_| {
             let msg = format!(
                 "Failed to acquired write lock \
                  on mmap cache while reading {:?}",
@@ -282,11 +406,34 @@ impl Directory for MmapDirectory {
             );
             IOError::with_path(path.to_owned(), make_io_err(msg))
         })?;
-
         Ok(mmap_cache
             .get_mmap(&full_path)?
-            .map(ReadOnlySource::Mmap)
-            .unwrap_or_else(|| ReadOnlySource::Anonymous(SharedVecSlice::empty())))
+            .map(ReadOnlySource::from)
+            .unwrap_or_else(ReadOnlySource::empty))
+    }
+
+    /// Any entry associated to the path in the mmap will be
+    /// removed before the file is deleted.
+    fn delete(&self, path: &Path) -> result::Result<(), DeleteError> {
+        debug!("Deleting file {:?}", path);
+        let full_path = self.resolve_path(path);
+        match fs::remove_file(&full_path) {
+            Ok(_) => self
+                .sync_directory()
+                .map_err(|e| IOError::with_path(path.to_owned(), e).into()),
+            Err(e) => {
+                if e.kind() == io::ErrorKind::NotFound {
+                    Err(DeleteError::FileDoesNotExist(path.to_owned()))
+                } else {
+                    Err(IOError::with_path(path.to_owned(), e).into())
+                }
+            }
+        }
+    }
+
+    fn exists(&self, path: &Path) -> bool {
+        let full_path = self.resolve_path(path);
+        full_path.exists()
     }
 
     fn open_write(&mut self, path: &Path) -> Result<WritePtr, OpenWriteError> {
@@ -319,44 +466,6 @@ impl Directory for MmapDirectory {
         Ok(BufWriter::new(Box::new(writer)))
     }
 
-    /// Any entry associated to the path in the mmap will be
-    /// removed before the file is deleted.
-    fn delete(&self, path: &Path) -> result::Result<(), DeleteError> {
-        debug!("Deleting file {:?}", path);
-        let full_path = self.resolve_path(path);
-        let mut mmap_cache = self.mmap_cache.write().map_err(|_| {
-            let msg = format!(
-                "Failed to acquired write lock \
-                 on mmap cache while deleting {:?}",
-                path
-            );
-            IOError::with_path(path.to_owned(), make_io_err(msg))
-        })?;
-        mmap_cache.discard_from_cache(path);
-
-        // Removing the entry in the MMap cache.
-        // The munmap will appear on Drop,
-        // when the last reference is gone.
-        mmap_cache.cache.remove(&full_path);
-        match fs::remove_file(&full_path) {
-            Ok(_) => self
-                .sync_directory()
-                .map_err(|e| IOError::with_path(path.to_owned(), e).into()),
-            Err(e) => {
-                if e.kind() == io::ErrorKind::NotFound {
-                    Err(DeleteError::FileDoesNotExist(path.to_owned()))
-                } else {
-                    Err(IOError::with_path(path.to_owned(), e).into())
-                }
-            }
-        }
-    }
-
-    fn exists(&self, path: &Path) -> bool {
-        let full_path = self.resolve_path(path);
-        full_path.exists()
-    }
-
     fn atomic_read(&self, path: &Path) -> Result<Vec<u8>, OpenReadError> {
         let full_path = self.resolve_path(path);
         let mut buffer = Vec::new();
@@ -403,6 +512,10 @@ impl Directory for MmapDirectory {
             _file: file,
         })))
     }
+
+    fn watch(&self, watch_callback: WatchCallback) -> WatchHandle {
+        self.inner.watch(watch_callback)
+    }
 }
 
 #[cfg(test)]
@@ -412,6 +525,13 @@ mod tests {
     // The following tests are specific to the MmapDirectory
 
     use super::*;
+    use schema::{Schema, SchemaBuilder, TEXT};
+    use std::fs;
+    use std::sync::atomic::{AtomicUsize, Ordering};
+    use std::thread;
+    use std::time::Duration;
+    use Index;
+    use ReloadPolicy;
 
     #[test]
     fn test_open_non_existant_path() {
@@ -436,7 +556,7 @@ mod tests {
 
     #[test]
     fn test_cache() {
-        let content = "abc".as_bytes();
+        let content = b"abc";
 
         // here we test if the cache releases
         // mmaps correctly.
@@ -452,26 +572,104 @@ mod tests {
                 w.flush().unwrap();
             }
         }
-        {
-            for (i, path) in paths.iter().enumerate() {
-                let _r = mmap_directory.open_read(path).unwrap();
-                assert_eq!(mmap_directory.get_cache_info().mmapped.len(), i + 1);
-            }
-            for path in paths.iter() {
-                let _r = mmap_directory.open_read(path).unwrap();
-                assert_eq!(mmap_directory.get_cache_info().mmapped.len(), num_paths);
-            }
-            for (i, path) in paths.iter().enumerate() {
-                mmap_directory.delete(path).unwrap();
-                assert_eq!(
-                    mmap_directory.get_cache_info().mmapped.len(),
-                    num_paths - i - 1
-                );
-            }
+
+        let mut keep = vec![];
+        for (i, path) in paths.iter().enumerate() {
+            keep.push(mmap_directory.open_read(path).unwrap());
+            assert_eq!(mmap_directory.get_cache_info().mmapped.len(), i + 1);
+        }
+        assert_eq!(mmap_directory.get_cache_info().counters.hit, 0);
+        assert_eq!(mmap_directory.get_cache_info().counters.miss, 10);
+        assert_eq!(mmap_directory.get_cache_info().mmapped.len(), 10);
+        for path in paths.iter() {
+            let _r = mmap_directory.open_read(path).unwrap();
+            assert_eq!(mmap_directory.get_cache_info().mmapped.len(), num_paths);
         }
         assert_eq!(mmap_directory.get_cache_info().counters.hit, 10);
         assert_eq!(mmap_directory.get_cache_info().counters.miss, 10);
+        assert_eq!(mmap_directory.get_cache_info().mmapped.len(), 10);
+
+        for path in paths.iter() {
+            let _r = mmap_directory.open_read(path).unwrap();
+            assert_eq!(mmap_directory.get_cache_info().mmapped.len(), 10);
+        }
+
+        assert_eq!(mmap_directory.get_cache_info().counters.hit, 20);
+        assert_eq!(mmap_directory.get_cache_info().counters.miss, 10);
+        assert_eq!(mmap_directory.get_cache_info().mmapped.len(), 10);
+        drop(keep);
+        for path in paths.iter() {
+            let _r = mmap_directory.open_read(path).unwrap();
+            assert_eq!(mmap_directory.get_cache_info().mmapped.len(), 1);
+        }
+        assert_eq!(mmap_directory.get_cache_info().counters.hit, 20);
+        assert_eq!(mmap_directory.get_cache_info().counters.miss, 20);
+        assert_eq!(mmap_directory.get_cache_info().mmapped.len(), 0);
+
+        for path in &paths {
+            mmap_directory.delete(path).unwrap();
+        }
+        assert_eq!(mmap_directory.get_cache_info().counters.hit, 20);
+        assert_eq!(mmap_directory.get_cache_info().counters.miss, 20);
+        assert_eq!(mmap_directory.get_cache_info().mmapped.len(), 0);
+        for path in paths.iter() {
+            assert!(mmap_directory.open_read(path).is_err());
+        }
+        assert_eq!(mmap_directory.get_cache_info().counters.hit, 20);
+        assert_eq!(mmap_directory.get_cache_info().counters.miss, 30);
         assert_eq!(mmap_directory.get_cache_info().mmapped.len(), 0);
     }
 
+    #[test]
+    fn test_watch_wrapper() {
+        let counter: Arc<AtomicUsize> = Default::default();
+        let counter_clone = counter.clone();
+        let tmp_dir: TempDir = tempdir::TempDir::new("test_watch_wrapper").unwrap();
+        let tmp_dirpath = tmp_dir.path().to_owned();
+        let mut watch_wrapper = WatcherWrapper::new(&tmp_dirpath).unwrap();
+        let tmp_file = tmp_dirpath.join("coucou");
+        let _handle = watch_wrapper.watch(Box::new(move || {
+            counter_clone.fetch_add(1, Ordering::SeqCst);
+        }));
+        assert_eq!(counter.load(Ordering::SeqCst), 0);
+        fs::write(&tmp_file, b"whateverwilldo").unwrap();
+        thread::sleep(Duration::new(0, 1_000u32));
+    }
+
+    #[test]
+    fn test_mmap_released() {
+        let mmap_directory = MmapDirectory::create_from_tempdir().unwrap();
+        let mut schema_builder: SchemaBuilder = Schema::builder();
+        let text_field = schema_builder.add_text_field("text", TEXT);
+        let schema = schema_builder.build();
+        {
+            let index = Index::create(mmap_directory.clone(), schema).unwrap();
+            let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
+            for _num_commits in 0..16 {
+                for _ in 0..10 {
+                    index_writer.add_document(doc!(text_field=>"abc"));
+                }
+                index_writer.commit().unwrap();
+            }
+            let reader = index
+                .reader_builder()
+                .reload_policy(ReloadPolicy::Manual)
+                .try_into()
+                .unwrap();
+            for _ in 0..30 {
+                index_writer.add_document(doc!(text_field=>"abc"));
+                index_writer.commit().unwrap();
+                reader.reload().unwrap();
+            }
+            index_writer.wait_merging_threads().unwrap();
+            reader.reload().unwrap();
+            let num_segments = reader.searcher().segment_readers().len();
+            assert_eq!(num_segments, 4);
+            assert_eq!(
+                num_segments * 7,
+                mmap_directory.get_cache_info().mmapped.len()
+            );
+        }
+        assert_eq!(mmap_directory.get_cache_info().mmapped.len(), 0);
+    }
 }

src/directory/mod.rs

@@ -12,7 +12,7 @@ mod directory_lock;
 mod managed_directory;
 mod ram_directory;
 mod read_only_source;
-mod shared_vec_slice;
+mod watch_event_router;
 
 /// Errors specific to the directory module.
 pub mod error;
@@ -22,6 +22,8 @@ pub use self::directory::{Directory, DirectoryClone};
 pub use self::directory_lock::{Lock, INDEX_WRITER_LOCK, META_LOCK};
 pub use self::ram_directory::RAMDirectory;
 pub use self::read_only_source::ReadOnlySource;
+pub(crate) use self::watch_event_router::WatchCallbackList;
+pub use self::watch_event_router::{WatchCallback, WatchHandle};
 use std::io::{BufWriter, Seek, Write};
 
 #[cfg(feature = "mmap")]

src/directory/ram_directory.rs

@@ -1,8 +1,8 @@
-use super::shared_vec_slice::SharedVecSlice;
-use common::make_io_err;
-use directory::error::{DeleteError, IOError, OpenReadError, OpenWriteError};
+use core::META_FILEPATH;
+use directory::error::{DeleteError, OpenReadError, OpenWriteError};
+use directory::WatchCallbackList;
 use directory::WritePtr;
-use directory::{Directory, ReadOnlySource};
+use directory::{Directory, ReadOnlySource, WatchCallback, WatchHandle};
 use std::collections::HashMap;
 use std::fmt;
 use std::io::{self, BufWriter, Cursor, Seek, SeekFrom, Write};
@@ -22,13 +22,13 @@ use std::sync::{Arc, RwLock};
 ///
 struct VecWriter {
     path: PathBuf,
-    shared_directory: InnerDirectory,
+    shared_directory: RAMDirectory,
     data: Cursor<Vec<u8>>,
     is_flushed: bool,
 }
 
 impl VecWriter {
-    fn new(path_buf: PathBuf, shared_directory: InnerDirectory) -> VecWriter {
+    fn new(path_buf: PathBuf, shared_directory: RAMDirectory) -> VecWriter {
         VecWriter {
             path: path_buf,
             data: Cursor::new(Vec::new()),
@@ -64,75 +64,44 @@ impl Write for VecWriter {
 
     fn flush(&mut self) -> io::Result<()> {
         self.is_flushed = true;
-        self.shared_directory
-            .write(self.path.clone(), self.data.get_ref())?;
+        let mut fs = self.shared_directory.fs.write().unwrap();
+        fs.write(self.path.clone(), self.data.get_ref());
         Ok(())
     }
 }
 
-#[derive(Clone)]
-struct InnerDirectory(Arc<RwLock<HashMap<PathBuf, Arc<Vec<u8>>>>>);
+#[derive(Default)]
+struct InnerDirectory {
+    fs: HashMap<PathBuf, ReadOnlySource>,
+    watch_router: WatchCallbackList,
+}
 
 impl InnerDirectory {
-    fn new() -> InnerDirectory {
-        InnerDirectory(Arc::new(RwLock::new(HashMap::new())))
-    }
-
-    fn write(&self, path: PathBuf, data: &[u8]) -> io::Result<bool> {
-        let mut map = self.0.write().map_err(|_| {
-            make_io_err(format!(
-                "Failed to lock the directory, when trying to write {:?}",
-                path
-            ))
-        })?;
-        let prev_value = map.insert(path, Arc::new(Vec::from(data)));
-        Ok(prev_value.is_some())
+    fn write(&mut self, path: PathBuf, data: &[u8]) -> bool {
+        let data = ReadOnlySource::new(Vec::from(data));
+        self.fs.insert(path, data).is_some()
     }
 
     fn open_read(&self, path: &Path) -> Result<ReadOnlySource, OpenReadError> {
-        self.0
-            .read()
-            .map_err(|_| {
-                let msg = format!(
-                    "Failed to acquire read lock for the \
-                     directory when trying to read {:?}",
-                    path
-                );
-                let io_err = make_io_err(msg);
-                OpenReadError::IOError(IOError::with_path(path.to_owned(), io_err))
-            })
-            .and_then(|readable_map| {
-                readable_map
-                    .get(path)
-                    .ok_or_else(|| OpenReadError::FileDoesNotExist(PathBuf::from(path)))
-                    .map(Arc::clone)
-                    .map(|data| ReadOnlySource::Anonymous(SharedVecSlice::new(data)))
-            })
+        self.fs
+            .get(path)
+            .ok_or_else(|| OpenReadError::FileDoesNotExist(PathBuf::from(path)))
+            .map(|el| el.clone())
     }
 
-    fn delete(&self, path: &Path) -> result::Result<(), DeleteError> {
-        self.0
-            .write()
-            .map_err(|_| {
-                let msg = format!(
-                    "Failed to acquire write lock for the \
-                     directory when trying to delete {:?}",
-                    path
-                );
-                let io_err = make_io_err(msg);
-                DeleteError::IOError(IOError::with_path(path.to_owned(), io_err))
-            })
-            .and_then(|mut writable_map| match writable_map.remove(path) {
-                Some(_) => Ok(()),
-                None => Err(DeleteError::FileDoesNotExist(PathBuf::from(path))),
-            })
+    fn delete(&mut self, path: &Path) -> result::Result<(), DeleteError> {
+        match self.fs.remove(path) {
+            Some(_) => Ok(()),
+            None => Err(DeleteError::FileDoesNotExist(PathBuf::from(path))),
+        }
     }
 
     fn exists(&self, path: &Path) -> bool {
-        self.0
-            .read()
-            .expect("Failed to get read lock directory.")
-            .contains_key(path)
+        self.fs.contains_key(path)
+    }
+
+    fn watch(&mut self, watch_handle: WatchCallback) -> WatchHandle {
+        self.watch_router.subscribe(watch_handle)
     }
 }
 
@@ -147,33 +116,36 @@ impl fmt::Debug for RAMDirectory {
 /// It is mainly meant for unit testing.
 /// Writes are only made visible upon flushing.
 ///
-#[derive(Clone)]
+#[derive(Clone, Default)]
 pub struct RAMDirectory {
-    fs: InnerDirectory,
+    fs: Arc<RwLock<InnerDirectory>>,
 }
 
 impl RAMDirectory {
     /// Constructor
     pub fn create() -> RAMDirectory {
-        RAMDirectory {
-            fs: InnerDirectory::new(),
-        }
+        Self::default()
     }
 }
 
 impl Directory for RAMDirectory {
     fn open_read(&self, path: &Path) -> result::Result<ReadOnlySource, OpenReadError> {
-        self.fs.open_read(path)
+        self.fs.read().unwrap().open_read(path)
+    }
+
+    fn delete(&self, path: &Path) -> result::Result<(), DeleteError> {
+        self.fs.write().unwrap().delete(path)
+    }
+
+    fn exists(&self, path: &Path) -> bool {
+        self.fs.read().unwrap().exists(path)
     }
 
     fn open_write(&mut self, path: &Path) -> Result<WritePtr, OpenWriteError> {
+        let mut fs = self.fs.write().unwrap();
         let path_buf = PathBuf::from(path);
-        let vec_writer = VecWriter::new(path_buf.clone(), self.fs.clone());
-
-        let exists = self
-            .fs
-            .write(path_buf.clone(), &Vec::new())
-            .map_err(|err| IOError::with_path(path.to_owned(), err))?;
+        let vec_writer = VecWriter::new(path_buf.clone(), self.clone());
+        let exists = fs.write(path_buf.clone(), &[]);
         // force the creation of the file to mimic the MMap directory.
         if exists {
             Err(OpenWriteError::FileAlreadyExists(path_buf))
@@ -182,17 +154,8 @@ impl Directory for RAMDirectory {
         }
     }
 
-    fn delete(&self, path: &Path) -> result::Result<(), DeleteError> {
-        self.fs.delete(path)
-    }
-
-    fn exists(&self, path: &Path) -> bool {
-        self.fs.exists(path)
-    }
-
     fn atomic_read(&self, path: &Path) -> Result<Vec<u8>, OpenReadError> {
-        let read = self.open_read(path)?;
-        Ok(read.as_slice().to_owned())
+        Ok(self.open_read(path)?.as_slice().to_owned())
     }
 
     fn atomic_write(&mut self, path: &Path, data: &[u8]) -> io::Result<()> {
@@ -201,10 +164,20 @@ impl Directory for RAMDirectory {
             msg.unwrap_or("Undefined".to_string())
         )));
         let path_buf = PathBuf::from(path);
-        let mut vec_writer = VecWriter::new(path_buf.clone(), self.fs.clone());
-        self.fs.write(path_buf, &Vec::new())?;
+
+        // Reserve the path to prevent calls to .write() to succeed.
+        self.fs.write().unwrap().write(path_buf.clone(), &[]);
+
+        let mut vec_writer = VecWriter::new(path_buf.clone(), self.clone());
         vec_writer.write_all(data)?;
         vec_writer.flush()?;
+        if path == Path::new(&*META_FILEPATH) {
+            self.fs.write().unwrap().watch_router.broadcast();
+        }
         Ok(())
     }
+
+    fn watch(&self, watch_callback: WatchCallback) -> WatchHandle {
+        self.fs.write().unwrap().watch(watch_callback)
+    }
 }

src/directory/read_only_source.rs

@@ -1,9 +1,9 @@
-use super::shared_vec_slice::SharedVecSlice;
 use common::HasLen;
-#[cfg(feature = "mmap")]
-use fst::raw::MmapReadOnly;
 use stable_deref_trait::{CloneStableDeref, StableDeref};
 use std::ops::Deref;
+use std::sync::Arc;
+
+pub type BoxedData = Box<Deref<Target = [u8]> + Send + Sync + 'static>;
 
 /// Read object that represents files in tantivy.
 ///
@@ -11,12 +11,10 @@ use std::ops::Deref;
 /// the data in the form of a constant read-only `&[u8]`.
 /// Whatever happens to the directory file, the data
 /// hold by this object should never be altered or destroyed.
-pub enum ReadOnlySource {
-    /// Mmap source of data
-    #[cfg(feature = "mmap")]
-    Mmap(MmapReadOnly),
-    /// Wrapping a `Vec<u8>`
-    Anonymous(SharedVecSlice),
+pub struct ReadOnlySource {
+    data: Arc<BoxedData>,
+    start: usize,
+    stop: usize,
 }
 
 unsafe impl StableDeref for ReadOnlySource {}
@@ -30,19 +28,38 @@ impl Deref for ReadOnlySource {
     }
 }
 
+impl From<Arc<BoxedData>> for ReadOnlySource {
+    fn from(data: Arc<BoxedData>) -> Self {
+        let len = data.len();
+        ReadOnlySource {
+            data,
+            start: 0,
+            stop: len,
+        }
+    }
+}
+
 impl ReadOnlySource {
+    pub(crate) fn new<D>(data: D) -> ReadOnlySource
+    where
+        D: Deref<Target = [u8]> + Send + Sync + 'static,
+    {
+        let len = data.len();
+        ReadOnlySource {
+            data: Arc::new(Box::new(data)),
+            start: 0,
+            stop: len,
+        }
+    }
+
     /// Creates an empty ReadOnlySource
     pub fn empty() -> ReadOnlySource {
-        ReadOnlySource::Anonymous(SharedVecSlice::empty())
+        ReadOnlySource::new(&[][..])
     }
 
     /// Returns the data underlying the ReadOnlySource object.
     pub fn as_slice(&self) -> &[u8] {
-        match *self {
-            #[cfg(feature = "mmap")]
-            ReadOnlySource::Mmap(ref mmap_read_only) => mmap_read_only.as_slice(),
-            ReadOnlySource::Anonymous(ref shared_vec) => shared_vec.as_slice(),
-        }
+        &self.data[self.start..self.stop]
     }
 
     /// Splits into 2 `ReadOnlySource`, at the offset given
@@ -63,22 +80,18 @@ impl ReadOnlySource {
     /// worth of data in anonymous memory, and only a
     /// 1KB slice is remaining, the whole `500MBs`
     /// are retained in memory.
-    pub fn slice(&self, from_offset: usize, to_offset: usize) -> ReadOnlySource {
+    pub fn slice(&self, start: usize, stop: usize) -> ReadOnlySource {
         assert!(
-            from_offset <= to_offset,
+            start <= stop,
             "Requested negative slice [{}..{}]",
-            from_offset,
-            to_offset
+            start,
+            stop
         );
-        match *self {
-            #[cfg(feature = "mmap")]
-            ReadOnlySource::Mmap(ref mmap_read_only) => {
-                let sliced_mmap = mmap_read_only.range(from_offset, to_offset - from_offset);
-                ReadOnlySource::Mmap(sliced_mmap)
-            }
-            ReadOnlySource::Anonymous(ref shared_vec) => {
-                ReadOnlySource::Anonymous(shared_vec.slice(from_offset, to_offset))
-            }
+        assert!(stop <= self.len());
+        ReadOnlySource {
+            data: self.data.clone(),
+            start: self.start + start,
+            stop: self.start + stop,
         }
     }
 
@@ -87,8 +100,7 @@ impl ReadOnlySource {
     ///
     /// Equivalent to `.slice(from_offset, self.len())`
     pub fn slice_from(&self, from_offset: usize) -> ReadOnlySource {
-        let len = self.len();
-        self.slice(from_offset, len)
+        self.slice(from_offset, self.len())
     }
 
     /// Like `.slice(...)` but enforcing only the `to`
@@ -102,19 +114,18 @@ impl ReadOnlySource {
 
 impl HasLen for ReadOnlySource {
     fn len(&self) -> usize {
-        self.as_slice().len()
+        self.stop - self.start
     }
 }
 
 impl Clone for ReadOnlySource {
     fn clone(&self) -> Self {
-        self.slice(0, self.len())
+        self.slice_from(0)
     }
 }
 
 impl From<Vec<u8>> for ReadOnlySource {
     fn from(data: Vec<u8>) -> ReadOnlySource {
-        let shared_data = SharedVecSlice::from(data);
-        ReadOnlySource::Anonymous(shared_data)
+        ReadOnlySource::new(data)
     }
 }

src/directory/shared_vec_slice.rs

@@ -1,41 +0,0 @@
-use std::sync::Arc;
-
-#[derive(Clone)]
-pub struct SharedVecSlice {
-    pub data: Arc<Vec<u8>>,
-    pub start: usize,
-    pub len: usize,
-}
-
-impl SharedVecSlice {
-    pub fn empty() -> SharedVecSlice {
-        SharedVecSlice::new(Arc::new(Vec::new()))
-    }
-
-    pub fn new(data: Arc<Vec<u8>>) -> SharedVecSlice {
-        let data_len = data.len();
-        SharedVecSlice {
-            data,
-            start: 0,
-            len: data_len,
-        }
-    }
-
-    pub fn as_slice(&self) -> &[u8] {
-        &self.data[self.start..self.start + self.len]
-    }
-
-    pub fn slice(&self, from_offset: usize, to_offset: usize) -> SharedVecSlice {
-        SharedVecSlice {
-            data: Arc::clone(&self.data),
-            start: self.start + from_offset,
-            len: to_offset - from_offset,
-        }
-    }
-}
-
-impl From<Vec<u8>> for SharedVecSlice {
-    fn from(data: Vec<u8>) -> SharedVecSlice {
-        SharedVecSlice::new(Arc::new(data))
-    }
-}

src/directory/tests.rs

@@ -1,7 +1,13 @@
 use super::*;
 use std::io::{Seek, SeekFrom, Write};
+use std::mem;
 use std::path::{Path, PathBuf};
+use std::sync::atomic::AtomicUsize;
+use std::sync::atomic::Ordering;
+use std::sync::Arc;
+use std::thread;
 use std::time;
+use std::time::Duration;
 
 lazy_static! {
     static ref TEST_PATH: &'static Path = Path::new("some_path_for_test");
@@ -30,19 +36,18 @@ fn ram_directory_panics_if_flush_forgotten() {
 
 fn test_simple(directory: &mut Directory) {
     {
-        {
-            let mut write_file = directory.open_write(*TEST_PATH).unwrap();
-            assert!(directory.exists(*TEST_PATH));
-            write_file.write_all(&[4]).unwrap();
-            write_file.write_all(&[3]).unwrap();
-            write_file.write_all(&[7, 3, 5]).unwrap();
-            write_file.flush().unwrap();
-        }
+        let mut write_file = directory.open_write(*TEST_PATH).unwrap();
+        assert!(directory.exists(*TEST_PATH));
+        write_file.write_all(&[4]).unwrap();
+        write_file.write_all(&[3]).unwrap();
+        write_file.write_all(&[7, 3, 5]).unwrap();
+        write_file.flush().unwrap();
+    }
+    {
         let read_file = directory.open_read(*TEST_PATH).unwrap();
         let data: &[u8] = &*read_file;
         assert_eq!(data, &[4u8, 3u8, 7u8, 3u8, 5u8]);
     }
-
     assert!(directory.delete(*TEST_PATH).is_ok());
     assert!(!directory.exists(*TEST_PATH));
 }
@@ -121,6 +126,41 @@ fn test_directory(directory: &mut Directory) {
     test_directory_delete(directory);
     test_lock_non_blocking(directory);
     test_lock_blocking(directory);
+    test_watch(directory);
+}
+
+fn test_watch(directory: &mut Directory) {
+    let counter: Arc<AtomicUsize> = Default::default();
+    let counter_clone = counter.clone();
+    let watch_callback = Box::new(move || {
+        counter_clone.fetch_add(1, Ordering::SeqCst);
+    });
+    assert!(directory
+        .atomic_write(Path::new("meta.json"), b"random_test_data")
+        .is_ok());
+    thread::sleep(Duration::new(0, 10_000));
+    assert_eq!(0, counter.load(Ordering::SeqCst));
+
+    let watch_handle = directory.watch(watch_callback);
+    for i in 0..10 {
+        assert_eq!(i, counter.load(Ordering::SeqCst));
+        assert!(directory
+            .atomic_write(Path::new("meta.json"), b"random_test_data_2")
+            .is_ok());
+        for _ in 0..100 {
+            if counter.load(Ordering::SeqCst) > i {
+                break;
+            }
+            thread::sleep(Duration::from_millis(10));
+        }
+        assert_eq!(i + 1, counter.load(Ordering::SeqCst));
+    }
+    mem::drop(watch_handle);
+    assert!(directory
+        .atomic_write(Path::new("meta.json"), b"random_test_data")
+        .is_ok());
+    thread::sleep(Duration::from_millis(200));
+    assert_eq!(10, counter.load(Ordering::SeqCst));
 }
 
 fn test_lock_non_blocking(directory: &mut Directory) {

src/directory/watch_event_router.rs

@@ -0,0 +1,156 @@
+use std::sync::Arc;
+use std::sync::RwLock;
+use std::sync::Weak;
+
+/// Type alias for callbacks registered when watching files of a `Directory`.
+pub type WatchCallback = Box<Fn() -> () + Sync + Send>;
+
+/// Helper struct to implement the watch method in `Directory` implementations.
+///
+/// It registers callbacks (See `.subscribe(...)`) and
+/// calls them upon calls to `.broadcast(...)`.
+#[derive(Default)]
+pub struct WatchCallbackList {
+    router: RwLock<Vec<Weak<WatchCallback>>>,
+}
+
+/// Controls how long a directory should watch for a file change.
+///
+/// After all the clones of `WatchHandle` are dropped, the associated will not be called when a
+/// file change is detected.
+#[must_use = "This `WatchHandle` controls the lifetime of the watch and should therefore be used."]
+#[derive(Clone)]
+pub struct WatchHandle(Arc<WatchCallback>);
+
+impl WatchCallbackList {
+    /// Suscribes a new callback and returns a handle that controls the lifetime of the callback.
+    pub fn subscribe(&self, watch_callback: WatchCallback) -> WatchHandle {
+        let watch_callback_arc = Arc::new(watch_callback);
+        let watch_callback_weak = Arc::downgrade(&watch_callback_arc);
+        self.router.write().unwrap().push(watch_callback_weak);
+        WatchHandle(watch_callback_arc)
+    }
+
+    fn list_callback(&self) -> Vec<Arc<WatchCallback>> {
+        let mut callbacks = vec![];
+        let mut router_wlock = self.router.write().unwrap();
+        let mut i = 0;
+        while i < router_wlock.len() {
+            if let Some(watch) = router_wlock[i].upgrade() {
+                callbacks.push(watch);
+                i += 1;
+            } else {
+                router_wlock.swap_remove(i);
+            }
+        }
+        callbacks
+    }
+
+    /// Triggers all callbacks
+    pub fn broadcast(&self) {
+        let callbacks = self.list_callback();
+        let spawn_res = std::thread::Builder::new()
+            .name("watch-callbacks".to_string())
+            .spawn(move || {
+                for callback in callbacks {
+                    callback();
+                }
+            });
+        if let Err(err) = spawn_res {
+            error!(
+                "Failed to spawn thread to call watch callbacks. Cause: {:?}",
+                err
+            );
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use directory::WatchCallbackList;
+    use std::mem;
+    use std::sync::atomic::{AtomicUsize, Ordering};
+    use std::sync::Arc;
+    use std::thread;
+    use std::time::Duration;
+
+    const WAIT_TIME: u64 = 20;
+
+    #[test]
+    fn test_watch_event_router_simple() {
+        let watch_event_router = WatchCallbackList::default();
+        let counter: Arc<AtomicUsize> = Default::default();
+        let counter_clone = counter.clone();
+        let inc_callback = Box::new(move || {
+            counter_clone.fetch_add(1, Ordering::SeqCst);
+        });
+        watch_event_router.broadcast();
+        assert_eq!(0, counter.load(Ordering::SeqCst));
+        let handle_a = watch_event_router.subscribe(inc_callback);
+        thread::sleep(Duration::from_millis(WAIT_TIME));
+        assert_eq!(0, counter.load(Ordering::SeqCst));
+        watch_event_router.broadcast();
+        thread::sleep(Duration::from_millis(WAIT_TIME));
+        assert_eq!(1, counter.load(Ordering::SeqCst));
+        watch_event_router.broadcast();
+        watch_event_router.broadcast();
+        watch_event_router.broadcast();
+        thread::sleep(Duration::from_millis(WAIT_TIME));
+        assert_eq!(4, counter.load(Ordering::SeqCst));
+        mem::drop(handle_a);
+        watch_event_router.broadcast();
+        thread::sleep(Duration::from_millis(WAIT_TIME));
+        assert_eq!(4, counter.load(Ordering::SeqCst));
+    }
+
+    #[test]
+    fn test_watch_event_router_multiple_callback_same_key() {
+        let watch_event_router = WatchCallbackList::default();
+        let counter: Arc<AtomicUsize> = Default::default();
+        let inc_callback = |inc: usize| {
+            let counter_clone = counter.clone();
+            Box::new(move || {
+                counter_clone.fetch_add(inc, Ordering::SeqCst);
+            })
+        };
+        let handle_a = watch_event_router.subscribe(inc_callback(1));
+        let handle_a2 = watch_event_router.subscribe(inc_callback(10));
+        thread::sleep(Duration::from_millis(WAIT_TIME));
+        assert_eq!(0, counter.load(Ordering::SeqCst));
+        watch_event_router.broadcast();
+        watch_event_router.broadcast();
+        thread::sleep(Duration::from_millis(WAIT_TIME));
+        assert_eq!(22, counter.load(Ordering::SeqCst));
+        mem::drop(handle_a);
+        watch_event_router.broadcast();
+        thread::sleep(Duration::from_millis(WAIT_TIME));
+        assert_eq!(32, counter.load(Ordering::SeqCst));
+        mem::drop(handle_a2);
+        watch_event_router.broadcast();
+        watch_event_router.broadcast();
+        thread::sleep(Duration::from_millis(WAIT_TIME));
+        assert_eq!(32, counter.load(Ordering::SeqCst));
+    }
+
+    #[test]
+    fn test_watch_event_router_multiple_callback_different_key() {
+        let watch_event_router = WatchCallbackList::default();
+        let counter: Arc<AtomicUsize> = Default::default();
+        let counter_clone = counter.clone();
+        let inc_callback = Box::new(move || {
+            counter_clone.fetch_add(1, Ordering::SeqCst);
+        });
+        let handle_a = watch_event_router.subscribe(inc_callback);
+        assert_eq!(0, counter.load(Ordering::SeqCst));
+        watch_event_router.broadcast();
+        watch_event_router.broadcast();
+        thread::sleep(Duration::from_millis(WAIT_TIME));
+        assert_eq!(2, counter.load(Ordering::SeqCst));
+        thread::sleep(Duration::from_millis(WAIT_TIME));
+        mem::drop(handle_a);
+        watch_event_router.broadcast();
+        thread::sleep(Duration::from_millis(WAIT_TIME));
+        assert_eq!(2, counter.load(Ordering::SeqCst));
+    }
+
+}

src/docset.rs

@@ -1,4 +1,5 @@
 use common::BitSet;
+use fastfield::DeleteBitSet;
 use std::borrow::Borrow;
 use std::borrow::BorrowMut;
 use std::cmp::Ordering;
@@ -95,9 +96,23 @@ pub trait DocSet {
     }
 
     /// Returns the number documents matching.
-    ///
     /// Calling this method consumes the `DocSet`.
-    fn count(&mut self) -> u32 {
+    fn count(&mut self, delete_bitset: &DeleteBitSet) -> u32 {
+        let mut count = 0u32;
+        while self.advance() {
+            if !delete_bitset.is_deleted(self.doc()) {
+                count += 1u32;
+            }
+        }
+        count
+    }
+
+    /// Returns the count of documents, deleted or not.
+    /// Calling this method consumes the `DocSet`.
+    ///
+    /// Of course, the result is an upper bound of the result
+    /// given by `count()`.
+    fn count_including_deleted(&mut self) -> u32 {
         let mut count = 0u32;
         while self.advance() {
             count += 1u32;
@@ -127,9 +142,14 @@ impl<TDocSet: DocSet + ?Sized> DocSet for Box<TDocSet> {
         unboxed.size_hint()
     }
 
-    fn count(&mut self) -> u32 {
+    fn count(&mut self, delete_bitset: &DeleteBitSet) -> u32 {
         let unboxed: &mut TDocSet = self.borrow_mut();
-        unboxed.count()
+        unboxed.count(delete_bitset)
+    }
+
+    fn count_including_deleted(&mut self) -> u32 {
+        let unboxed: &mut TDocSet = self.borrow_mut();
+        unboxed.count_including_deleted()
     }
 
     fn append_to_bitset(&mut self, bitset: &mut BitSet) {

src/error.rs

@@ -162,6 +162,7 @@ impl From<OpenDirectoryError> for TantivyError {
             OpenDirectoryError::NotADirectory(directory_path) => {
                 TantivyError::InvalidArgument(format!("{:?} is not a directory", directory_path))
             }
+            OpenDirectoryError::IoError(err) => TantivyError::IOError(IOError::from(err)),
         }
     }
 }

src/fastfield/bytes/mod.rs

@@ -22,9 +22,7 @@ mod tests {
         index_writer.add_document(doc!(field=>vec![1u8, 3, 5, 7, 9]));
         index_writer.add_document(doc!(field=>vec![0u8; 1000]));
         assert!(index_writer.commit().is_ok());
-
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        let searcher = index.reader().unwrap().searcher();
         let reader = searcher.segment_reader(0);
         let bytes_reader = reader.bytes_fast_field_reader(field).unwrap();
 

src/fastfield/multivalued/mod.rs

@@ -7,7 +7,13 @@ pub use self::writer::MultiValueIntFastFieldWriter;
 #[cfg(test)]
 mod tests {
 
+    extern crate time;
+
+    use self::time::Duration;
+    use collector::TopDocs;
+    use query::QueryParser;
     use schema::Cardinality;
+    use schema::Facet;
     use schema::IntOptions;
     use schema::Schema;
     use Index;
@@ -28,11 +34,12 @@ mod tests {
         index_writer.add_document(doc!(field=>5u64, field=>20u64,field=>1u64));
         assert!(index_writer.commit().is_ok());
 
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
-        let reader = searcher.segment_reader(0);
+        let searcher = index.reader().unwrap().searcher();
+        let segment_reader = searcher.segment_reader(0);
         let mut vals = Vec::new();
-        let multi_value_reader = reader.multi_fast_field_reader::<u64>(field).unwrap();
+        let multi_value_reader = segment_reader
+            .multi_fast_field_reader::<u64>(field)
+            .unwrap();
         {
             multi_value_reader.get_vals(2, &mut vals);
             assert_eq!(&vals, &[4u64]);
@@ -47,6 +54,133 @@ mod tests {
         }
     }
 
+    #[test]
+    fn test_multivalued_date() {
+        let mut schema_builder = Schema::builder();
+        let date_field = schema_builder.add_date_field(
+            "multi_date_field",
+            IntOptions::default()
+                .set_fast(Cardinality::MultiValues)
+                .set_indexed()
+                .set_stored(),
+        );
+        let time_i =
+            schema_builder.add_i64_field("time_stamp_i", IntOptions::default().set_stored());
+        let schema = schema_builder.build();
+        let index = Index::create_in_ram(schema);
+        let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
+        let first_time_stamp = chrono::Utc::now();
+        index_writer.add_document(
+            doc!(date_field=>first_time_stamp, date_field=>first_time_stamp, time_i=>1i64),
+        );
+        index_writer.add_document(doc!(time_i=>0i64));
+        // add one second
+        index_writer
+            .add_document(doc!(date_field=>first_time_stamp + Duration::seconds(1), time_i=>2i64));
+        // add another second
+        let two_secs_ahead = first_time_stamp + Duration::seconds(2);
+        index_writer.add_document(doc!(date_field=>two_secs_ahead, date_field=>two_secs_ahead,date_field=>two_secs_ahead, time_i=>3i64));
+        assert!(index_writer.commit().is_ok());
+
+        let reader = index.reader().unwrap();
+        let searcher = reader.searcher();
+        let reader = searcher.segment_reader(0);
+        assert_eq!(reader.num_docs(), 4);
+
+        {
+            let parser = QueryParser::for_index(&index, vec![date_field]);
+            let query = parser
+                .parse_query(&format!("\"{}\"", first_time_stamp.to_rfc3339()).to_string())
+                .expect("could not parse query");
+            let results = searcher
+                .search(&query, &TopDocs::with_limit(5))
+                .expect("could not query index");
+
+            assert_eq!(results.len(), 1);
+            for (_score, doc_address) in results {
+                let retrieved_doc = searcher.doc(doc_address).expect("cannot fetch doc");
+                assert_eq!(
+                    retrieved_doc
+                        .get_first(date_field)
+                        .expect("cannot find value")
+                        .date_value()
+                        .timestamp(),
+                    first_time_stamp.timestamp()
+                );
+                assert_eq!(
+                    retrieved_doc
+                        .get_first(time_i)
+                        .expect("cannot find value")
+                        .i64_value(),
+                    1i64
+                );
+            }
+        }
+
+        {
+            let parser = QueryParser::for_index(&index, vec![date_field]);
+            let query = parser
+                .parse_query(&format!("\"{}\"", two_secs_ahead.to_rfc3339()).to_string())
+                .expect("could not parse query");
+            let results = searcher
+                .search(&query, &TopDocs::with_limit(5))
+                .expect("could not query index");
+
+            assert_eq!(results.len(), 1);
+
+            for (_score, doc_address) in results {
+                let retrieved_doc = searcher.doc(doc_address).expect("cannot fetch doc");
+                assert_eq!(
+                    retrieved_doc
+                        .get_first(date_field)
+                        .expect("cannot find value")
+                        .date_value()
+                        .timestamp(),
+                    two_secs_ahead.timestamp()
+                );
+                assert_eq!(
+                    retrieved_doc
+                        .get_first(time_i)
+                        .expect("cannot find value")
+                        .i64_value(),
+                    3i64
+                );
+            }
+        }
+
+        // TODO: support Date range queries
+        //        {
+        //            let parser = QueryParser::for_index(&index, vec![date_field]);
+        //            let range_q = format!("\"{}\"..\"{}\"",
+        //                                  (first_time_stamp + Duration::seconds(1)).to_rfc3339(),
+        //                                  (first_time_stamp + Duration::seconds(3)).to_rfc3339()
+        //            );
+        //            let query = parser.parse_query(&range_q)
+        //                .expect("could not parse query");
+        //            let results = searcher.search(&query, &TopDocs::with_limit(5))
+        //                .expect("could not query index");
+        //
+        //
+        //            assert_eq!(results.len(), 2);
+        //            for (i, doc_pair) in results.iter().enumerate() {
+        //                let retrieved_doc = searcher.doc(doc_pair.1).expect("cannot fetch doc");
+        //                let offset_sec = match i {
+        //                    0 => 1,
+        //                    1 => 3,
+        //                    _ => panic!("should not have more than 2 docs")
+        //                };
+        //                let time_i_val = match i {
+        //                    0 => 2,
+        //                    1 => 3,
+        //                    _ => panic!("should not have more than 2 docs")
+        //                };
+        //                assert_eq!(retrieved_doc.get_first(date_field).expect("cannot find value").date_value().timestamp(),
+        //                           (first_time_stamp + Duration::seconds(offset_sec)).timestamp());
+        //                assert_eq!(retrieved_doc.get_first(time_i).expect("cannot find value").i64_value(), time_i_val);
+        //            }
+        //        }
+    }
+
     #[test]
     fn test_multivalued_i64() {
         let mut schema_builder = Schema::builder();
@@ -63,8 +197,7 @@ mod tests {
         index_writer.add_document(doc!(field=> -5i64, field => -20i64, field=>1i64));
         assert!(index_writer.commit().is_ok());
 
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        let searcher = index.reader().unwrap().searcher();
         let reader = searcher.segment_reader(0);
         let mut vals = Vec::new();
         let multi_value_reader = reader.multi_fast_field_reader::<i64>(field).unwrap();
@@ -85,4 +218,17 @@ mod tests {
             assert_eq!(&vals, &[-5i64, -20i64, 1i64]);
         }
     }
+    #[test]
+    #[ignore]
+    fn test_many_facets() {
+        let mut schema_builder = Schema::builder();
+        let field = schema_builder.add_facet_field("facetfield");
+        let schema = schema_builder.build();
+        let index = Index::create_in_ram(schema);
+        let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
+        for i in 0..100_000 {
+            index_writer.add_document(doc!(field=> Facet::from(format!("/lang/{}", i).as_str())));
+        }
+        assert!(index_writer.commit().is_ok());
+    }
 }

src/fastfield/multivalued/reader.rs

@@ -75,8 +75,7 @@ mod tests {
             index_writer.add_document(doc);
         }
         index_writer.commit().expect("Commit failed");
-        index.load_searchers().expect("Reloading searchers");
-        let searcher = index.searcher();
+        let searcher = index.reader().unwrap().searcher();
         let segment_reader = searcher.segment_reader(0);
         let mut facet_reader = segment_reader.facet_reader(facet_field).unwrap();
 

src/fastfield/multivalued/writer.rs

@@ -32,7 +32,7 @@ use DocId;
 /// term ids when the segment is getting serialized.
 pub struct MultiValueIntFastFieldWriter {
     field: Field,
-    vals: Vec<u64>,
+    vals: Vec<UnorderedTermId>,
     doc_index: Vec<u64>,
     is_facet: bool,
 }

src/fastfield/reader.rs

@@ -59,7 +59,7 @@ impl<Item: FastValue> FastFieldReader<Item> {
     /// May panic if `doc` is greater than the segment
     // `maxdoc`.
     pub fn get(&self, doc: DocId) -> Item {
-        self.get_u64(doc as u64)
+        self.get_u64(u64::from(doc))
     }
 
     pub(crate) fn get_u64(&self, doc: u64) -> Item {
@@ -98,7 +98,7 @@ impl<Item: FastValue> FastFieldReader<Item> {
     /// May panic if `start + output.len()` is greater than
     /// the segment's `maxdoc`.
     pub fn get_range(&self, start: DocId, output: &mut [Item]) {
-        self.get_range_u64(start as u64, output);
+        self.get_range_u64(u64::from(start), output);
     }
 
     /// Returns the minimum value for this fast field.

src/functional_test.rs

@@ -13,15 +13,15 @@ fn check_index_content(searcher: &Searcher, vals: &HashSet<u64>) {
 
 #[test]
 #[ignore]
-#[cfg(feature = "mmap")]
 fn test_indexing() {
     let mut schema_builder = Schema::builder();
 
-    let id_field = schema_builder.add_u64_field("id", INT_INDEXED);
-    let multiples_field = schema_builder.add_u64_field("multiples", INT_INDEXED);
+    let id_field = schema_builder.add_u64_field("id", INDEXED);
+    let multiples_field = schema_builder.add_u64_field("multiples", INDEXED);
     let schema = schema_builder.build();
 
     let index = Index::create_from_tempdir(schema).unwrap();
+    let reader = index.reader().unwrap();
 
     let mut rng = thread_rng();
 
@@ -36,8 +36,8 @@ fn test_indexing() {
             index_writer.commit().expect("Commit failed");
             committed_docs.extend(&uncommitted_docs);
             uncommitted_docs.clear();
-            index.load_searchers().unwrap();
-            let searcher = index.searcher();
+            reader.reload().unwrap();
+            let searcher = reader.searcher();
             // check that everything is correct.
             check_index_content(&searcher, &committed_docs);
         } else {

src/indexer/index_writer.rs

@@ -44,8 +44,8 @@ pub const HEAP_SIZE_MAX: usize = u32::max_value() as usize - MARGIN_IN_BYTES;
 // reaches `PIPELINE_MAX_SIZE_IN_DOCS`
 const PIPELINE_MAX_SIZE_IN_DOCS: usize = 10_000;
 
-type DocumentSender = channel::Sender<Vec<AddOperation>>;
-type DocumentReceiver = channel::Receiver<Vec<AddOperation>>;
+type OperationSender = channel::Sender<Vec<AddOperation>>;
+type OperationReceiver = channel::Receiver<Vec<AddOperation>>;
 
 /// Split the thread memory budget into
 /// - the heap size
@@ -85,8 +85,8 @@ pub struct IndexWriter {
 
     workers_join_handle: Vec<JoinHandle<Result<()>>>,
 
-    document_receiver: DocumentReceiver,
-    document_sender: DocumentSender,
+    operation_receiver: OperationReceiver,
+    operation_sender: OperationSender,
 
     segment_updater: SegmentUpdater,
 
@@ -133,7 +133,7 @@ pub fn open_index_writer(
         let err_msg = format!("The heap size per thread cannot exceed {}", HEAP_SIZE_MAX);
         return Err(TantivyError::InvalidArgument(err_msg));
     }
-    let (document_sender, document_receiver): (DocumentSender, DocumentReceiver) =
+    let (document_sender, document_receiver): (OperationSender, OperationReceiver) =
         channel::bounded(PIPELINE_MAX_SIZE_IN_DOCS);
 
     let delete_queue = DeleteQueue::new();
@@ -151,8 +151,8 @@ pub fn open_index_writer(
         heap_size_in_bytes_per_thread,
         index: index.clone(),
 
-        document_receiver,
-        document_sender,
+        operation_receiver: document_receiver,
+        operation_sender: document_sender,
 
         segment_updater,
 
@@ -335,7 +335,7 @@ impl IndexWriter {
     pub fn wait_merging_threads(mut self) -> Result<()> {
         // this will stop the indexing thread,
         // dropping the last reference to the segment_updater.
-        drop(self.document_sender);
+        drop(self.operation_sender);
 
         let former_workers_handles = mem::replace(&mut self.workers_join_handle, vec![]);
         for join_handle in former_workers_handles {
@@ -384,7 +384,7 @@ impl IndexWriter {
     /// The thread consumes documents from the pipeline.
     ///
     fn add_indexing_worker(&mut self) -> Result<()> {
-        let document_receiver_clone = self.document_receiver.clone();
+        let document_receiver_clone = self.operation_receiver.clone();
         let mut segment_updater = self.segment_updater.clone();
 
         let generation = self.generation;
@@ -479,11 +479,11 @@ impl IndexWriter {
     /// when no documents are remaining.
     ///
     /// Returns the former segment_ready channel.
-    fn recreate_document_channel(&mut self) -> DocumentReceiver {
-        let (document_sender, document_receiver): (DocumentSender, DocumentReceiver) =
+    fn recreate_document_channel(&mut self) -> OperationReceiver {
+        let (document_sender, document_receiver): (OperationSender, OperationReceiver) =
             channel::bounded(PIPELINE_MAX_SIZE_IN_DOCS);
-        mem::replace(&mut self.document_sender, document_sender);
-        mem::replace(&mut self.document_receiver, document_receiver)
+        mem::replace(&mut self.operation_sender, document_sender);
+        mem::replace(&mut self.operation_receiver, document_receiver)
     }
 
     /// Rollback to the last commit
@@ -501,7 +501,7 @@ impl IndexWriter {
         // segment updates will be ignored.
         self.segment_updater.kill();
 
-        let document_receiver = self.document_receiver.clone();
+        let document_receiver = self.operation_receiver.clone();
 
         // take the directory lock to create a new index_writer.
         let directory_lock = self
@@ -648,7 +648,7 @@ impl IndexWriter {
     pub fn add_document(&mut self, document: Document) -> u64 {
         let opstamp = self.stamper.stamp();
         let add_operation = AddOperation { opstamp, document };
-        let send_result = self.document_sender.send(vec![add_operation]);
+        let send_result = self.operation_sender.send(vec![add_operation]);
         if let Err(e) = send_result {
             panic!("Failed to index document. Sending to indexing channel failed. This probably means all of the indexing threads have panicked. {:?}", e);
         }
@@ -666,7 +666,7 @@ impl IndexWriter {
         let Range { start, end } = self.stamper.stamps(count + 1u64);
         let last_opstamp = end - 1;
         let stamps = Range {
-            start: start,
+            start,
             end: last_opstamp,
         };
         (last_opstamp, stamps)
@@ -675,16 +675,16 @@ impl IndexWriter {
     /// Runs a group of document operations ensuring that the operations are
     /// assigned contigous u64 opstamps and that add operations of the same
     /// group are flushed into the same segment.
-    /// 
+    ///
     /// If the indexing pipeline is full, this call may block.
-    /// 
+    ///
     /// Each operation of the given `user_operations` will receive an in-order,
     /// contiguous u64 opstamp. The entire batch itself is also given an
-    /// opstamp that is 1 greater than the last given operation. This 
+    /// opstamp that is 1 greater than the last given operation. This
     /// `batch_opstamp` is the return value of `run`. An empty group of
     /// `user_operations`, an empty `Vec<UserOperation>`, still receives
     /// a valid opstamp even though no changes were _actually_ made to the index.
-    /// 
+    ///
     /// Like adds and deletes (see `IndexWriter.add_document` and
     /// `IndexWriter.delete_term`), the changes made by calling `run` will be
     /// visible to readers only after calling `commit()`.
@@ -700,22 +700,16 @@ impl IndexWriter {
         for (user_op, opstamp) in user_operations.into_iter().zip(stamps) {
             match user_op {
                 UserOperation::Delete(term) => {
-                    let delete_operation = DeleteOperation {
-                        opstamp: opstamp,
-                        term: term,
-                    };
+                    let delete_operation = DeleteOperation { opstamp, term };
                     self.delete_queue.push(delete_operation);
                 }
-                UserOperation::Add(doc) => {
-                    let add_operation = AddOperation {
-                        opstamp: opstamp,
-                        document: doc,
-                    };
+                UserOperation::Add(document) => {
+                    let add_operation = AddOperation { opstamp, document };
                     adds.push(add_operation);
                 }
             }
         }
-        let send_result = self.document_sender.send(adds);
+        let send_result = self.operation_sender.send(adds);
         if let Err(e) = send_result {
             panic!("Failed to index document. Sending to indexing channel failed. This probably means all of the indexing threads have panicked. {:?}", e);
         };
@@ -729,13 +723,14 @@ mod tests {
 
     use super::super::operation::UserOperation;
     use super::initial_table_size;
+    use collector::TopDocs;
     use directory::error::LockError;
     use error::*;
     use indexer::NoMergePolicy;
-    use schema::{self, Document, IndexRecordOption};
-    use query::{TermQuery};
-    use collector::TopDocs;
+    use query::TermQuery;
+    use schema::{self, IndexRecordOption};
     use Index;
+    use ReloadPolicy;
     use Term;
 
     #[test]
@@ -763,6 +758,11 @@ mod tests {
         let mut schema_builder = schema::Schema::builder();
         let text_field = schema_builder.add_text_field("text", schema::TEXT);
         let index = Index::create_in_ram(schema_builder.build());
+        let reader = index
+            .reader_builder()
+            .reload_policy(ReloadPolicy::Manual)
+            .try_into()
+            .unwrap();
         let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
         let a_term = Term::from_field_text(text_field, "a");
         let b_term = Term::from_field_text(text_field, "b");
@@ -775,15 +775,15 @@ mod tests {
 
         index_writer.run(operations);
         index_writer.commit().expect("failed to commit");
-        index.load_searchers().expect("failed to load searchers");
-        
+        reader.reload().expect("failed to load searchers");
+
         let a_term = Term::from_field_text(text_field, "a");
         let b_term = Term::from_field_text(text_field, "b");
-        
+
         let a_query = TermQuery::new(a_term, IndexRecordOption::Basic);
         let b_query = TermQuery::new(b_term, IndexRecordOption::Basic);
 
-        let searcher = index.searcher();
+        let searcher = reader.searcher();
 
         let a_docs = searcher
             .search(&a_query, &TopDocs::with_limit(1))
@@ -870,9 +870,13 @@ mod tests {
         let mut schema_builder = schema::Schema::builder();
         let text_field = schema_builder.add_text_field("text", schema::TEXT);
         let index = Index::create_in_ram(schema_builder.build());
-
+        let reader = index
+            .reader_builder()
+            .reload_policy(ReloadPolicy::Manual)
+            .try_into()
+            .unwrap();
         let num_docs_containing = |s: &str| {
-            let searcher = index.searcher();
+            let searcher = reader.searcher();
             let term = Term::from_field_text(text_field, s);
             searcher.doc_freq(&term)
         };
@@ -882,7 +886,6 @@ mod tests {
             let mut index_writer = index.writer(3_000_000).unwrap();
             index_writer.add_document(doc!(text_field=>"a"));
             index_writer.rollback().unwrap();
-
             assert_eq!(index_writer.commit_opstamp(), 0u64);
             assert_eq!(num_docs_containing("a"), 0);
             {
@@ -890,13 +893,13 @@ mod tests {
                 index_writer.add_document(doc!(text_field=>"c"));
             }
             assert!(index_writer.commit().is_ok());
-            index.load_searchers().unwrap();
+            reader.reload().unwrap();
             assert_eq!(num_docs_containing("a"), 0);
             assert_eq!(num_docs_containing("b"), 1);
             assert_eq!(num_docs_containing("c"), 1);
         }
-        index.load_searchers().unwrap();
-        index.searcher();
+        reader.reload().unwrap();
+        reader.searcher();
     }
 
     #[test]
@@ -904,32 +907,33 @@ mod tests {
         let mut schema_builder = schema::Schema::builder();
         let text_field = schema_builder.add_text_field("text", schema::TEXT);
         let index = Index::create_in_ram(schema_builder.build());
+        let reader = index
+            .reader_builder()
+            .reload_policy(ReloadPolicy::Manual)
+            .try_into()
+            .unwrap();
         let num_docs_containing = |s: &str| {
-            let searcher = index.searcher();
             let term_a = Term::from_field_text(text_field, s);
-            searcher.doc_freq(&term_a)
+            reader.searcher().doc_freq(&term_a)
         };
         {
             // writing the segment
             let mut index_writer = index.writer(12_000_000).unwrap();
             // create 8 segments with 100 tiny docs
             for _doc in 0..100 {
-                let mut doc = Document::default();
-                doc.add_text(text_field, "a");
-                index_writer.add_document(doc);
+                index_writer.add_document(doc!(text_field=>"a"));
             }
             index_writer.commit().expect("commit failed");
             for _doc in 0..100 {
-                let mut doc = Document::default();
-                doc.add_text(text_field, "a");
-                index_writer.add_document(doc);
+                index_writer.add_document(doc!(text_field=>"a"));
             }
-            // this should create 8 segments and trigger a merge.
+            //  this should create 8 segments and trigger a merge.
             index_writer.commit().expect("commit failed");
             index_writer
                 .wait_merging_threads()
                 .expect("waiting merging thread failed");
-            index.load_searchers().unwrap();
+
+            reader.reload().unwrap();
 
             assert_eq!(num_docs_containing("a"), 200);
             assert!(index.searchable_segments().unwrap().len() < 8);
@@ -996,11 +1000,15 @@ mod tests {
             }
             index_writer.commit().unwrap();
         }
-        index.load_searchers().unwrap();
         let num_docs_containing = |s: &str| {
-            let searcher = index.searcher();
             let term_a = Term::from_field_text(text_field, s);
-            searcher.doc_freq(&term_a)
+            index
+                .reader_builder()
+                .reload_policy(ReloadPolicy::Manual)
+                .try_into()
+                .unwrap()
+                .searcher()
+                .doc_freq(&term_a)
         };
         assert_eq!(num_docs_containing("a"), 0);
         assert_eq!(num_docs_containing("b"), 100);
@@ -1008,9 +1016,9 @@ mod tests {
 
     #[test]
     fn test_hashmap_size() {
-        assert_eq!(initial_table_size(100_000), 12);
-        assert_eq!(initial_table_size(1_000_000), 15);
-        assert_eq!(initial_table_size(10_000_000), 18);
+        assert_eq!(initial_table_size(100_000), 11);
+        assert_eq!(initial_table_size(1_000_000), 14);
+        assert_eq!(initial_table_size(10_000_000), 17);
         assert_eq!(initial_table_size(1_000_000_000), 19);
     }
 
@@ -1032,11 +1040,9 @@ mod tests {
             index_writer.add_document(doc!(text_field => "b"));
         }
         assert!(index_writer.commit().is_err());
-        index.load_searchers().unwrap();
         let num_docs_containing = |s: &str| {
-            let searcher = index.searcher();
             let term_a = Term::from_field_text(text_field, s);
-            searcher.doc_freq(&term_a)
+            index.reader().unwrap().searcher().doc_freq(&term_a)
         };
         assert_eq!(num_docs_containing("a"), 100);
         assert_eq!(num_docs_containing("b"), 0);

src/indexer/merger.rs

@@ -1,3 +1,4 @@
+use common::MAX_DOC_LIMIT;
 use core::Segment;
 use core::SegmentReader;
 use core::SerializableSegment;
@@ -23,6 +24,7 @@ use termdict::TermMerger;
 use termdict::TermOrdinal;
 use DocId;
 use Result;
+use TantivyError;
 
 fn compute_total_num_tokens(readers: &[SegmentReader], field: Field) -> u64 {
     let mut total_tokens = 0u64;
@@ -150,6 +152,14 @@ impl IndexMerger {
                 readers.push(reader);
             }
         }
+        if max_doc >= MAX_DOC_LIMIT {
+            let err_msg = format!(
+                "The segment resulting from this merge would have {} docs,\
+                 which exceeds the limit {}.",
+                max_doc, MAX_DOC_LIMIT
+            );
+            return Err(TantivyError::InvalidArgument(err_msg));
+        }
         Ok(IndexMerger {
             schema,
             readers,
@@ -194,17 +204,17 @@ impl IndexMerger {
                         fast_field_serializer,
                     )?;
                 }
-                FieldType::U64(ref options) | FieldType::I64(ref options) => {
-                    match options.get_fastfield_cardinality() {
-                        Some(Cardinality::SingleValue) => {
-                            self.write_single_fast_field(field, fast_field_serializer)?;
-                        }
-                        Some(Cardinality::MultiValues) => {
-                            self.write_multi_fast_field(field, fast_field_serializer)?;
-                        }
-                        None => {}
+                FieldType::U64(ref options)
+                | FieldType::I64(ref options)
+                | FieldType::Date(ref options) => match options.get_fastfield_cardinality() {
+                    Some(Cardinality::SingleValue) => {
+                        self.write_single_fast_field(field, fast_field_serializer)?;
                     }
-                }
+                    Some(Cardinality::MultiValues) => {
+                        self.write_multi_fast_field(field, fast_field_serializer)?;
+                    }
+                    None => {}
+                },
                 FieldType::Str(_) => {
                     // We don't handle str fast field for the moment
                     // They can be implemented using what is done
@@ -654,7 +664,7 @@ mod tests {
     use schema::IntOptions;
     use schema::Term;
     use schema::TextFieldIndexing;
-    use schema::INT_INDEXED;
+    use schema::INDEXED;
     use std::io::Cursor;
     use DocAddress;
     use IndexWriter;
@@ -671,11 +681,13 @@ mod tests {
             )
             .set_stored();
         let text_field = schema_builder.add_text_field("text", text_fieldtype);
+        let date_field = schema_builder.add_date_field("date", INDEXED);
         let score_fieldtype = schema::IntOptions::default().set_fast(Cardinality::SingleValue);
         let score_field = schema_builder.add_u64_field("score", score_fieldtype);
         let bytes_score_field = schema_builder.add_bytes_field("score_bytes");
         let index = Index::create_in_ram(schema_builder.build());
-
+        let reader = index.reader().unwrap();
+        let curr_time = chrono::Utc::now();
         let add_score_bytes = |doc: &mut Document, score: u32| {
             let mut bytes = Vec::new();
             bytes
@@ -692,6 +704,7 @@ mod tests {
                     let mut doc = Document::default();
                     doc.add_text(text_field, "af b");
                     doc.add_u64(score_field, 3);
+                    doc.add_date(date_field, &curr_time);
                     add_score_bytes(&mut doc, 3);
                     index_writer.add_document(doc);
                 }
@@ -717,6 +730,7 @@ mod tests {
                 {
                     let mut doc = Document::default();
                     doc.add_text(text_field, "af b");
+                    doc.add_date(date_field, &curr_time);
                     doc.add_u64(score_field, 11);
                     add_score_bytes(&mut doc, 11);
                     index_writer.add_document(doc);
@@ -744,8 +758,8 @@ mod tests {
             index_writer.wait_merging_threads().unwrap();
         }
         {
-            index.load_searchers().unwrap();
-            let searcher = index.searcher();
+            reader.reload().unwrap();
+            let searcher = reader.searcher();
             let get_doc_ids = |terms: Vec<Term>| {
                 let query = BooleanQuery::new_multiterms_query(terms);
                 let top_docs = searcher.search(&query, &TestCollector).unwrap();
@@ -774,6 +788,10 @@ mod tests {
                         DocAddress(0, 4)
                     ]
                 );
+                assert_eq!(
+                    get_doc_ids(vec![Term::from_field_date(date_field, &curr_time)]),
+                    vec![DocAddress(0, 0), DocAddress(0, 3)]
+                );
             }
             {
                 let doc = searcher.doc(DocAddress(0, 0)).unwrap();
@@ -837,7 +855,7 @@ mod tests {
         let bytes_score_field = schema_builder.add_bytes_field("score_bytes");
         let index = Index::create_in_ram(schema_builder.build());
         let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
-
+        let reader = index.reader().unwrap();
         let search_term = |searcher: &Searcher, term: Term| {
             let collector = FastFieldTestCollector::for_field(score_field);
             let bytes_collector = BytesFastFieldTestCollector::for_field(bytes_score_field);
@@ -874,8 +892,8 @@ mod tests {
                 bytes_score_field => vec![0u8, 0, 0, 3],
             ));
             index_writer.commit().expect("committed");
-            index.load_searchers().unwrap();
-            let ref searcher = *index.searcher();
+            reader.reload().unwrap();
+            let searcher = reader.searcher();
             assert_eq!(searcher.num_docs(), 2);
             assert_eq!(searcher.segment_readers()[0].num_docs(), 2);
             assert_eq!(searcher.segment_readers()[0].max_doc(), 3);
@@ -921,8 +939,8 @@ mod tests {
                 bytes_score_field => vec![0u8, 0, 27, 88],
             ));
             index_writer.commit().expect("committed");
-            index.load_searchers().unwrap();
-            let searcher = index.searcher();
+            reader.reload().unwrap();
+            let searcher = reader.searcher();
 
             assert_eq!(searcher.segment_readers().len(), 2);
             assert_eq!(searcher.num_docs(), 3);
@@ -983,8 +1001,8 @@ mod tests {
                 .expect("Failed to initiate merge")
                 .wait()
                 .expect("Merging failed");
-            index.load_searchers().unwrap();
-            let searcher = index.searcher();
+            reader.reload().unwrap();
+            let searcher = reader.searcher();
             assert_eq!(searcher.segment_readers().len(), 1);
             assert_eq!(searcher.num_docs(), 3);
             assert_eq!(searcher.segment_readers()[0].num_docs(), 3);
@@ -1029,8 +1047,8 @@ mod tests {
             index_writer.delete_term(Term::from_field_text(text_field, "c"));
             index_writer.commit().unwrap();
 
-            index.load_searchers().unwrap();
-            let searcher = index.searcher();
+            reader.reload().unwrap();
+            let searcher = reader.searcher();
             assert_eq!(searcher.segment_readers().len(), 1);
             assert_eq!(searcher.num_docs(), 2);
             assert_eq!(searcher.segment_readers()[0].num_docs(), 2);
@@ -1080,9 +1098,9 @@ mod tests {
                 .expect("Failed to initiate merge")
                 .wait()
                 .expect("Merging failed");
-            index.load_searchers().unwrap();
+            reader.reload().unwrap();
 
-            let ref searcher = *index.searcher();
+            let searcher = reader.searcher();
             assert_eq!(searcher.segment_readers().len(), 1);
             assert_eq!(searcher.num_docs(), 2);
             assert_eq!(searcher.segment_readers()[0].num_docs(), 2);
@@ -1130,9 +1148,9 @@ mod tests {
             let segment_ids = index
                 .searchable_segment_ids()
                 .expect("Searchable segments failed.");
-            index.load_searchers().unwrap();
+            reader.reload().unwrap();
 
-            let ref searcher = *index.searcher();
+            let searcher = reader.searcher();
             assert!(segment_ids.is_empty());
             assert!(searcher.segment_readers().is_empty());
             assert_eq!(searcher.num_docs(), 0);
@@ -1144,6 +1162,7 @@ mod tests {
         let mut schema_builder = schema::Schema::builder();
         let facet_field = schema_builder.add_facet_field("facet");
         let index = Index::create_in_ram(schema_builder.build());
+        let reader = index.reader().unwrap();
         {
             let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
             let index_doc = |index_writer: &mut IndexWriter, doc_facets: &[&str]| {
@@ -1173,9 +1192,9 @@ mod tests {
             index_writer.commit().expect("committed");
         }
 
-        index.load_searchers().unwrap();
+        reader.reload().unwrap();
         let test_searcher = |expected_num_docs: usize, expected: &[(&str, u64)]| {
-            let searcher = index.searcher();
+            let searcher = reader.searcher();
             let mut facet_collector = FacetCollector::for_field(facet_field);
             facet_collector.add_facet(Facet::from("/top"));
             let (count, facet_counts) = searcher
@@ -1217,7 +1236,7 @@ mod tests {
                 .wait()
                 .expect("Merging failed");
             index_writer.wait_merging_threads().unwrap();
-            index.load_searchers().unwrap();
+            reader.reload().unwrap();
             test_searcher(
                 11,
                 &[
@@ -1238,7 +1257,7 @@ mod tests {
             let facet_term = Term::from_facet(facet_field, &facet);
             index_writer.delete_term(facet_term);
             index_writer.commit().unwrap();
-            index.load_searchers().unwrap();
+            reader.reload().unwrap();
             test_searcher(
                 9,
                 &[
@@ -1256,15 +1275,15 @@ mod tests {
     #[test]
     fn test_bug_merge() {
         let mut schema_builder = schema::Schema::builder();
-        let int_field = schema_builder.add_u64_field("intvals", INT_INDEXED);
+        let int_field = schema_builder.add_u64_field("intvals", INDEXED);
         let index = Index::create_in_ram(schema_builder.build());
         let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
         index_writer.add_document(doc!(int_field => 1u64));
         index_writer.commit().expect("commit failed");
         index_writer.add_document(doc!(int_field => 1u64));
         index_writer.commit().expect("commit failed");
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        let reader = index.reader().unwrap();
+        let searcher = reader.searcher();
         assert_eq!(searcher.num_docs(), 2);
         index_writer.delete_term(Term::from_field_u64(int_field, 1));
         let segment_ids = index
@@ -1275,10 +1294,10 @@ mod tests {
             .expect("Failed to initiate merge")
             .wait()
             .expect("Merging failed");
-        index.load_searchers().unwrap();
+        reader.reload().unwrap();
         // commit has not been called yet. The document should still be
         // there.
-        assert_eq!(index.searcher().num_docs(), 2);
+        assert_eq!(reader.searcher().num_docs(), 2);
     }
 
     #[test]
@@ -1289,7 +1308,7 @@ mod tests {
             .set_indexed();
         let int_field = schema_builder.add_u64_field("intvals", int_options);
         let index = Index::create_in_ram(schema_builder.build());
-
+        let reader = index.reader().unwrap();
         {
             let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
             let mut doc = Document::default();
@@ -1310,8 +1329,8 @@ mod tests {
                 .expect("Merging failed");
 
             // assert delete has not been committed
-            index.load_searchers().unwrap();
-            let searcher = index.searcher();
+            reader.reload().expect("failed to load searcher 1");
+            let searcher = reader.searcher();
             assert_eq!(searcher.num_docs(), 2);
 
             index_writer.commit().unwrap();
@@ -1319,13 +1338,13 @@ mod tests {
             index_writer.wait_merging_threads().unwrap();
         }
 
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        reader.reload().unwrap();
+        let searcher = reader.searcher();
         assert_eq!(searcher.num_docs(), 0);
     }
 
     #[test]
-    fn test_merge_multivalued_int_fields() {
+    fn test_merge_multivalued_int_fields_simple() {
         let mut schema_builder = schema::Schema::builder();
         let int_options = IntOptions::default()
             .set_fast(Cardinality::MultiValues)
@@ -1342,7 +1361,6 @@ mod tests {
                 }
                 index_writer.add_document(doc);
             };
-
             index_doc(&mut index_writer, &[1, 2]);
             index_doc(&mut index_writer, &[1, 2, 3]);
             index_doc(&mut index_writer, &[4, 5]);
@@ -1351,19 +1369,14 @@ mod tests {
             index_doc(&mut index_writer, &[3]);
             index_doc(&mut index_writer, &[17]);
             index_writer.commit().expect("committed");
-
             index_doc(&mut index_writer, &[20]);
             index_writer.commit().expect("committed");
-
             index_doc(&mut index_writer, &[28, 27]);
             index_doc(&mut index_writer, &[1_000]);
-
             index_writer.commit().expect("committed");
         }
-        index.load_searchers().unwrap();
-
-        let searcher = index.searcher();
-
+        let reader = index.reader().unwrap();
+        let searcher = reader.searcher();
         let mut vals: Vec<u64> = Vec::new();
 
         {
@@ -1429,13 +1442,14 @@ mod tests {
                 .expect("Failed to initiate merge")
                 .wait()
                 .expect("Merging failed");
-            index_writer.wait_merging_threads().unwrap();
+            index_writer
+                .wait_merging_threads()
+                .expect("Wait for merging threads");
         }
-
-        index.load_searchers().unwrap();
+        reader.reload().expect("Load searcher");
 
         {
-            let searcher = index.searcher();
+            let searcher = reader.searcher();
             println!(
                 "{:?}",
                 searcher

src/indexer/segment_updater.rs

@@ -62,7 +62,7 @@ pub fn save_new_metas(schema: Schema, directory: &mut Directory) -> Result<()> {
 /// Save the index meta file.
 /// This operation is atomic:
 /// Either
-//  - it fails, in which case an error is returned,
+///  - it fails, in which case an error is returned,
 /// and the `meta.json` remains untouched,
 /// - it success, and `meta.json` is written
 /// and flushed.
@@ -420,6 +420,7 @@ impl SegmentUpdater {
             })
             .collect::<Vec<_>>();
         merge_candidates.extend(committed_merge_candidates.into_iter());
+
         for merge_operation in merge_candidates {
             match self.start_merge_impl(merge_operation) {
                 Ok(merge_future) => {
@@ -565,9 +566,8 @@ mod tests {
             index_writer.delete_term(term);
             assert!(index_writer.commit().is_ok());
         }
-
-        index.load_searchers().unwrap();
-        assert_eq!(index.searcher().num_docs(), 302);
+        let reader = index.reader().unwrap();
+        assert_eq!(reader.searcher().num_docs(), 302);
 
         {
             index_writer
@@ -575,9 +575,9 @@ mod tests {
                 .expect("waiting for merging threads");
         }
 
-        index.load_searchers().unwrap();
-        assert_eq!(index.searcher().segment_readers().len(), 1);
-        assert_eq!(index.searcher().num_docs(), 302);
+        reader.reload().unwrap();
+        assert_eq!(reader.searcher().segment_readers().len(), 1);
+        assert_eq!(reader.searcher().num_docs(), 302);
     }
 
     #[test]
@@ -636,18 +636,18 @@ mod tests {
                 .expect("waiting for merging threads");
         }
 
-        index.load_searchers().unwrap();
-        assert_eq!(index.searcher().num_docs(), 0);
+        let reader = index.reader().unwrap();
+        assert_eq!(reader.searcher().num_docs(), 0);
 
         let seg_ids = index
             .searchable_segment_ids()
             .expect("Searchable segments failed.");
         assert!(seg_ids.is_empty());
 
-        index.load_searchers().unwrap();
-        assert_eq!(index.searcher().num_docs(), 0);
+        reader.reload().unwrap();
+        assert_eq!(reader.searcher().num_docs(), 0);
         // empty segments should be erased
         assert!(index.searchable_segment_metas().unwrap().is_empty());
-        assert!(index.searcher().segment_readers().is_empty());
+        assert!(reader.searcher().segment_readers().is_empty());
     }
 }

src/indexer/segment_writer.rs

@@ -171,6 +171,17 @@ impl SegmentWriter {
                         }
                     }
                 }
+                FieldType::Date(ref int_option) => {
+                    if int_option.is_indexed() {
+                        for field_value in field_values {
+                            let term = Term::from_field_i64(
+                                field_value.field(),
+                                field_value.value().date_value().timestamp(),
+                            );
+                            self.multifield_postings.subscribe(doc_id, &term);
+                        }
+                    }
+                }
                 FieldType::I64(ref int_option) => {
                     if int_option.is_indexed() {
                         for field_value in field_values {

src/indexer/stamper.rs

@@ -67,7 +67,7 @@ impl Stamper {
     pub fn stamps(&self, n: u64) -> Range<u64> {
         let start = self.0.fetch_add(n, Ordering::SeqCst);
         Range {
-            start: start,
+            start,
             end: start + n,
         }
     }

src/lib.rs

@@ -75,9 +75,9 @@
 //!
 //! // # Searching
 //!
-//! index.load_searchers()?;
+//! let reader = index.reader()?;
 //!
-//! let searcher = index.searcher();
+//! let searcher = reader.searcher();
 //!
 //! let query_parser = QueryParser::for_index(&index, vec![title, body]);
 //!
@@ -132,13 +132,13 @@ extern crate byteorder;
 extern crate combine;
 extern crate crossbeam;
 extern crate fnv;
-extern crate fst;
-extern crate fst_regex;
 extern crate futures;
 extern crate futures_cpupool;
 extern crate htmlescape;
 extern crate itertools;
 extern crate levenshtein_automata;
+#[cfg(feature = "mmap")]
+extern crate memmap;
 extern crate num_cpus;
 extern crate owning_ref;
 extern crate regex;
@@ -146,6 +146,7 @@ extern crate rust_stemmers;
 extern crate scoped_pool;
 extern crate serde;
 extern crate stable_deref_trait;
+extern crate tantivy_fst;
 extern crate tempdir;
 extern crate tempfile;
 extern crate uuid;
@@ -173,6 +174,7 @@ extern crate downcast_rs;
 #[macro_use]
 extern crate fail;
 
+#[cfg(feature = "mmap")]
 #[cfg(test)]
 mod functional_test;
 
@@ -185,11 +187,15 @@ pub use error::TantivyError;
 pub use error::TantivyError as Error;
 
 extern crate census;
+pub extern crate chrono;
 extern crate owned_read;
 
 /// Tantivy result.
 pub type Result<T> = std::result::Result<T, error::TantivyError>;
 
+/// Tantivy DateTime
+pub type DateTime = chrono::DateTime<chrono::Utc>;
+
 mod common;
 mod core;
 mod indexer;
@@ -210,6 +216,9 @@ pub mod space_usage;
 pub mod store;
 pub mod termdict;
 
+mod reader;
+
+pub use self::reader::{IndexReader, IndexReaderBuilder, ReloadPolicy};
 mod snippet;
 pub use self::snippet::{Snippet, SnippetGenerator};
 
@@ -298,6 +307,7 @@ mod tests {
     use Index;
     use IndexWriter;
     use Postings;
+    use ReloadPolicy;
 
     pub fn assert_nearly_equals(expected: f32, val: f32) {
         assert!(
@@ -386,8 +396,8 @@ mod tests {
             index_writer.commit().unwrap();
         }
         {
-            index.load_searchers().unwrap();
-            let searcher = index.searcher();
+            let reader = index.reader().unwrap();
+            let searcher = reader.searcher();
             let term_a = Term::from_field_text(text_field, "a");
             assert_eq!(searcher.doc_freq(&term_a), 3);
             let term_b = Term::from_field_text(text_field, "b");
@@ -414,8 +424,8 @@ mod tests {
             index_writer.commit().unwrap();
         }
         {
-            index.load_searchers().unwrap();
-            let searcher = index.searcher();
+            let index_reader = index.reader().unwrap();
+            let searcher = index_reader.searcher();
             let reader = searcher.segment_reader(0);
             {
                 let fieldnorm_reader = reader.get_fieldnorms_reader(text_field);
@@ -450,8 +460,8 @@ mod tests {
             index_writer.commit().unwrap();
         }
         {
-            index.load_searchers().unwrap();
-            let searcher = index.searcher();
+            let reader = index.reader().unwrap();
+            let searcher = reader.searcher();
             let segment_reader: &SegmentReader = searcher.segment_reader(0);
             let fieldnorms_reader = segment_reader.get_fieldnorms_reader(text_field);
             assert_eq!(fieldnorms_reader.fieldnorm(0), 3);
@@ -479,6 +489,11 @@ mod tests {
         let term_c = Term::from_field_text(text_field, "c");
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
+        let reader = index
+            .reader_builder()
+            .reload_policy(ReloadPolicy::Manual)
+            .try_into()
+            .unwrap();
         {
             // writing the segment
             let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
@@ -500,10 +515,10 @@ mod tests {
             index_writer.commit().unwrap();
         }
         {
-            index.load_searchers().unwrap();
-            let searcher = index.searcher();
-            let reader = searcher.segment_reader(0);
-            let inverted_index = reader.inverted_index(text_field);
+            reader.reload().unwrap();
+            let searcher = reader.searcher();
+            let segment_reader = searcher.segment_reader(0);
+            let inverted_index = segment_reader.inverted_index(text_field);
             assert!(inverted_index
                 .read_postings(&term_abcd, IndexRecordOption::WithFreqsAndPositions)
                 .is_none());
@@ -511,19 +526,19 @@ mod tests {
                 let mut postings = inverted_index
                     .read_postings(&term_a, IndexRecordOption::WithFreqsAndPositions)
                     .unwrap();
-                assert!(advance_undeleted(&mut postings, reader));
+                assert!(advance_undeleted(&mut postings, segment_reader));
                 assert_eq!(postings.doc(), 5);
-                assert!(!advance_undeleted(&mut postings, reader));
+                assert!(!advance_undeleted(&mut postings, segment_reader));
             }
             {
                 let mut postings = inverted_index
                     .read_postings(&term_b, IndexRecordOption::WithFreqsAndPositions)
                     .unwrap();
-                assert!(advance_undeleted(&mut postings, reader));
+                assert!(advance_undeleted(&mut postings, segment_reader));
                 assert_eq!(postings.doc(), 3);
-                assert!(advance_undeleted(&mut postings, reader));
+                assert!(advance_undeleted(&mut postings, segment_reader));
                 assert_eq!(postings.doc(), 4);
-                assert!(!advance_undeleted(&mut postings, reader));
+                assert!(!advance_undeleted(&mut postings, segment_reader));
             }
         }
         {
@@ -536,10 +551,10 @@ mod tests {
             index_writer.rollback().unwrap();
         }
         {
-            index.load_searchers().unwrap();
-            let searcher = index.searcher();
-            let reader = searcher.segment_reader(0);
-            let inverted_index = reader.inverted_index(term_abcd.field());
+            reader.reload().unwrap();
+            let searcher = reader.searcher();
+            let seg_reader = searcher.segment_reader(0);
+            let inverted_index = seg_reader.inverted_index(term_abcd.field());
 
             assert!(inverted_index
                 .read_postings(&term_abcd, IndexRecordOption::WithFreqsAndPositions)
@@ -548,19 +563,19 @@ mod tests {
                 let mut postings = inverted_index
                     .read_postings(&term_a, IndexRecordOption::WithFreqsAndPositions)
                     .unwrap();
-                assert!(advance_undeleted(&mut postings, reader));
+                assert!(advance_undeleted(&mut postings, seg_reader));
                 assert_eq!(postings.doc(), 5);
-                assert!(!advance_undeleted(&mut postings, reader));
+                assert!(!advance_undeleted(&mut postings, seg_reader));
             }
             {
                 let mut postings = inverted_index
                     .read_postings(&term_b, IndexRecordOption::WithFreqsAndPositions)
                     .unwrap();
-                assert!(advance_undeleted(&mut postings, reader));
+                assert!(advance_undeleted(&mut postings, seg_reader));
                 assert_eq!(postings.doc(), 3);
-                assert!(advance_undeleted(&mut postings, reader));
+                assert!(advance_undeleted(&mut postings, seg_reader));
                 assert_eq!(postings.doc(), 4);
-                assert!(!advance_undeleted(&mut postings, reader));
+                assert!(!advance_undeleted(&mut postings, seg_reader));
             }
         }
         {
@@ -573,10 +588,10 @@ mod tests {
             index_writer.commit().unwrap();
         }
         {
-            index.load_searchers().unwrap();
-            let searcher = index.searcher();
-            let reader = searcher.segment_reader(0);
-            let inverted_index = reader.inverted_index(term_abcd.field());
+            reader.reload().unwrap();
+            let searcher = reader.searcher();
+            let segment_reader = searcher.segment_reader(0);
+            let inverted_index = segment_reader.inverted_index(term_abcd.field());
             assert!(inverted_index
                 .read_postings(&term_abcd, IndexRecordOption::WithFreqsAndPositions)
                 .is_none());
@@ -584,25 +599,25 @@ mod tests {
                 let mut postings = inverted_index
                     .read_postings(&term_a, IndexRecordOption::WithFreqsAndPositions)
                     .unwrap();
-                assert!(!advance_undeleted(&mut postings, reader));
+                assert!(!advance_undeleted(&mut postings, segment_reader));
             }
             {
                 let mut postings = inverted_index
                     .read_postings(&term_b, IndexRecordOption::WithFreqsAndPositions)
                     .unwrap();
-                assert!(advance_undeleted(&mut postings, reader));
+                assert!(advance_undeleted(&mut postings, segment_reader));
                 assert_eq!(postings.doc(), 3);
-                assert!(advance_undeleted(&mut postings, reader));
+                assert!(advance_undeleted(&mut postings, segment_reader));
                 assert_eq!(postings.doc(), 4);
-                assert!(!advance_undeleted(&mut postings, reader));
+                assert!(!advance_undeleted(&mut postings, segment_reader));
             }
             {
                 let mut postings = inverted_index
                     .read_postings(&term_c, IndexRecordOption::WithFreqsAndPositions)
                     .unwrap();
-                assert!(advance_undeleted(&mut postings, reader));
+                assert!(advance_undeleted(&mut postings, segment_reader));
                 assert_eq!(postings.doc(), 4);
-                assert!(!advance_undeleted(&mut postings, reader));
+                assert!(!advance_undeleted(&mut postings, segment_reader));
             }
         }
     }
@@ -610,15 +625,15 @@ mod tests {
     #[test]
     fn test_indexed_u64() {
         let mut schema_builder = Schema::builder();
-        let field = schema_builder.add_u64_field("value", INT_INDEXED);
+        let field = schema_builder.add_u64_field("value", INDEXED);
         let schema = schema_builder.build();
 
         let index = Index::create_in_ram(schema);
         let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
         index_writer.add_document(doc!(field=>1u64));
         index_writer.commit().unwrap();
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        let reader = index.reader().unwrap();
+        let searcher = reader.searcher();
         let term = Term::from_field_u64(field, 1u64);
         let mut postings = searcher
             .segment_reader(0)
@@ -633,7 +648,7 @@ mod tests {
     #[test]
     fn test_indexed_i64() {
         let mut schema_builder = Schema::builder();
-        let value_field = schema_builder.add_i64_field("value", INT_INDEXED);
+        let value_field = schema_builder.add_i64_field("value", INDEXED);
         let schema = schema_builder.build();
 
         let index = Index::create_in_ram(schema);
@@ -641,8 +656,8 @@ mod tests {
         let negative_val = -1i64;
         index_writer.add_document(doc!(value_field => negative_val));
         index_writer.commit().unwrap();
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        let reader = index.reader().unwrap();
+        let searcher = reader.searcher();
         let term = Term::from_field_i64(value_field, negative_val);
         let mut postings = searcher
             .segment_reader(0)
@@ -664,8 +679,8 @@ mod tests {
         let mut index_writer = index.writer_with_num_threads(2, 6_000_000).unwrap();
         index_writer.add_document(doc!(text_field=>"a"));
         assert!(index_writer.commit().is_ok());
-        assert!(index.load_searchers().is_ok());
-        let searcher = index.searcher();
+        let reader = index.reader().unwrap();
+        let searcher = reader.searcher();
         let segment_reader = searcher.segment_reader(0);
         segment_reader.inverted_index(absent_field); //< should not panic
     }
@@ -676,6 +691,11 @@ mod tests {
         let text_field = schema_builder.add_text_field("text", TEXT);
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
+        let reader = index
+            .reader_builder()
+            .reload_policy(ReloadPolicy::Manual)
+            .try_into()
+            .unwrap();
 
         // writing the segment
         let mut index_writer = index.writer_with_num_threads(2, 6_000_000).unwrap();
@@ -701,8 +721,8 @@ mod tests {
         remove_document(&mut index_writer, "38");
         remove_document(&mut index_writer, "34");
         index_writer.commit().unwrap();
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        reader.reload().unwrap();
+        let searcher = reader.searcher();
         assert_eq!(searcher.num_docs(), 6);
     }
 
@@ -722,8 +742,8 @@ mod tests {
             index_writer.commit().unwrap();
         }
         {
-            index.load_searchers().unwrap();
-            let searcher = index.searcher();
+            let index_reader = index.reader().unwrap();
+            let searcher = index_reader.searcher();
             let reader = searcher.segment_reader(0);
             let inverted_index = reader.inverted_index(text_field);
             let term_abcd = Term::from_field_text(text_field, "abcd");
@@ -747,7 +767,7 @@ mod tests {
         let text_field = schema_builder.add_text_field("text", TEXT);
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
-
+        let reader = index.reader().unwrap();
         {
             // writing the segment
             let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
@@ -757,8 +777,8 @@ mod tests {
             index_writer.commit().unwrap();
         }
         {
-            index.load_searchers().unwrap();
-            let searcher = index.searcher();
+            reader.reload().unwrap();
+            let searcher = reader.searcher();
             let get_doc_ids = |terms: Vec<Term>| {
                 let query = BooleanQuery::new_multiterms_query(terms);
                 let topdocs = searcher.search(&query, &TestCollector).unwrap();
@@ -800,25 +820,22 @@ mod tests {
         let text_field = schema_builder.add_text_field("text", TEXT);
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
-
+        let reader = index
+            .reader_builder()
+            .reload_policy(ReloadPolicy::Manual)
+            .try_into()
+            .unwrap();
+        assert_eq!(reader.searcher().num_docs(), 0u64);
         {
             // writing the segment
             let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
-            {
-                let doc = doc!(text_field=>"af b");
-                index_writer.add_document(doc);
-            }
-            {
-                let doc = doc!(text_field=>"a b c");
-                index_writer.add_document(doc);
-            }
-            {
-                let doc = doc!(text_field=>"a b c d");
-                index_writer.add_document(doc);
-            }
+            index_writer.add_document(doc!(text_field=>"af b"));
+            index_writer.add_document(doc!(text_field=>"a b c"));
+            index_writer.add_document(doc!(text_field=>"a b c d"));
             index_writer.commit().unwrap();
         }
-        index.searcher();
+        reader.reload().unwrap();
+        assert_eq!(reader.searcher().num_docs(), 3u64);
     }
 
     #[test]
@@ -845,7 +862,7 @@ mod tests {
         let fast_field_unsigned = schema_builder.add_u64_field("unsigned", FAST);
         let fast_field_signed = schema_builder.add_i64_field("signed", FAST);
         let text_field = schema_builder.add_text_field("text", TEXT);
-        let stored_int_field = schema_builder.add_u64_field("text", INT_STORED);
+        let stored_int_field = schema_builder.add_u64_field("text", STORED);
         let schema = schema_builder.build();
 
         let index = Index::create_in_ram(schema);
@@ -855,9 +872,8 @@ mod tests {
             index_writer.add_document(document);
             index_writer.commit().unwrap();
         }
-
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        let reader = index.reader().unwrap();
+        let searcher = reader.searcher();
         let segment_reader: &SegmentReader = searcher.segment_reader(0);
         {
             let fast_field_reader_res = segment_reader.fast_field_reader::<u64>(text_field);

src/macros.rs

@@ -61,7 +61,7 @@ macro_rules! doc(
     };
     // if there is a trailing comma retry with the trailing comma stripped.
     ($($field:expr => $value:expr),+ ,) => {
-        doc!( $( $field => $value ), *);
+        doc!( $( $field => $value ), *)
     };
 );
 

src/positions/reader.rs

@@ -151,7 +151,8 @@ impl PositionReader {
         if self.ahead != Some(0) {
             // the block currently available is not the block
             // for the current position
-            self.bit_packer.decompress(position_data, self.buffer.as_mut(), num_bits);
+            self.bit_packer
+                .decompress(position_data, self.buffer.as_mut(), num_bits);
             self.ahead = Some(0);
         }
         let block_len = compressed_block_size(num_bits);

src/positions/serializer.rs

@@ -1,9 +1,9 @@
 use bitpacking::BitPacker;
+use bitpacking::BitPacker4x;
 use common::BinarySerializable;
 use common::CountingWriter;
 use positions::{COMPRESSION_BLOCK_SIZE, LONG_SKIP_INTERVAL};
 use std::io::{self, Write};
-use bitpacking::BitPacker4x;
 
 pub struct PositionSerializer<W: io::Write> {
     bit_packer: BitPacker4x,
@@ -53,7 +53,9 @@ impl<W: io::Write> PositionSerializer<W> {
     fn flush_block(&mut self) -> io::Result<()> {
         let num_bits = self.bit_packer.num_bits(&self.block[..]);
         self.write_skiplist.write_all(&[num_bits])?;
-        let written_len = self.bit_packer.compress(&self.block[..], &mut self.buffer, num_bits);
+        let written_len = self
+            .bit_packer
+            .compress(&self.block[..], &mut self.buffer, num_bits);
         self.write_stream.write_all(&self.buffer[..written_len])?;
         self.block.clear();
         if (self.num_ints % LONG_SKIP_INTERVAL) == 0u64 {

src/postings/block_search.rs

@@ -0,0 +1,249 @@
+use postings::compression::AlignedBuffer;
+
+/// This modules define the logic used to search for a doc in a given
+/// block. (at most 128 docs)
+///
+/// Searching within a block is a hotspot when running intersection.
+/// so it was worth defining it in its own module.
+
+#[cfg(target_arch = "x86_64")]
+mod sse2 {
+    use postings::compression::{AlignedBuffer, COMPRESSION_BLOCK_SIZE};
+    use std::arch::x86_64::__m128i as DataType;
+    use std::arch::x86_64::_mm_add_epi32 as op_add;
+    use std::arch::x86_64::_mm_cmplt_epi32 as op_lt;
+    use std::arch::x86_64::_mm_load_si128 as op_load; // requires 128-bits alignment
+    use std::arch::x86_64::_mm_set1_epi32 as set1;
+    use std::arch::x86_64::_mm_setzero_si128 as set0;
+    use std::arch::x86_64::_mm_sub_epi32 as op_sub;
+    use std::arch::x86_64::{_mm_cvtsi128_si32, _mm_shuffle_epi32};
+
+    const MASK1: i32 = 78;
+    const MASK2: i32 = 177;
+
+    /// Performs an exhaustive linear search over the
+    ///
+    /// There is no early exit here. We simply count the
+    /// number of elements that are `< target`.
+    pub(crate) fn linear_search_sse2_128(arr: &AlignedBuffer, target: u32) -> usize {
+        unsafe {
+            let ptr = arr as *const AlignedBuffer as *const DataType;
+            let vkey = set1(target as i32);
+            let mut cnt = set0();
+            // We work over 4 `__m128i` at a time.
+            // A single `__m128i` actual contains 4 `u32`.
+            for i in 0..(COMPRESSION_BLOCK_SIZE as isize) / (4 * 4) {
+                let cmp1 = op_lt(op_load(ptr.offset(i * 4)), vkey);
+                let cmp2 = op_lt(op_load(ptr.offset(i * 4 + 1)), vkey);
+                let cmp3 = op_lt(op_load(ptr.offset(i * 4 + 2)), vkey);
+                let cmp4 = op_lt(op_load(ptr.offset(i * 4 + 3)), vkey);
+                let sum = op_add(op_add(cmp1, cmp2), op_add(cmp3, cmp4));
+                cnt = op_sub(cnt, sum);
+            }
+            cnt = op_add(cnt, _mm_shuffle_epi32(cnt, MASK1));
+            cnt = op_add(cnt, _mm_shuffle_epi32(cnt, MASK2));
+            _mm_cvtsi128_si32(cnt) as usize
+        }
+    }
+
+    #[cfg(test)]
+    mod test {
+        use super::linear_search_sse2_128;
+        use postings::compression::{AlignedBuffer, COMPRESSION_BLOCK_SIZE};
+
+        #[test]
+        fn test_linear_search_sse2_128_u32() {
+            let mut block = [0u32; COMPRESSION_BLOCK_SIZE];
+            for el in 0u32..128u32 {
+                block[el as usize] = el * 2 + 1 << 18;
+            }
+            let target = block[64] + 1;
+            assert_eq!(linear_search_sse2_128(&AlignedBuffer(block), target), 65);
+        }
+    }
+}
+
+/// This `linear search` browser exhaustively through the array.
+/// but the early exit is very difficult to predict.
+///
+/// Coupled with `exponential search` this function is likely
+/// to be called with the same `len`
+fn linear_search(arr: &[u32], target: u32) -> usize {
+    arr.iter().map(|&el| if el < target { 1 } else { 0 }).sum()
+}
+
+fn exponential_search(arr: &[u32], target: u32) -> (usize, usize) {
+    let end = arr.len();
+    let mut begin = 0;
+    for &pivot in &[1, 3, 7, 15, 31, 63] {
+        if pivot >= end {
+            break;
+        }
+        if arr[pivot] > target {
+            return (begin, pivot);
+        }
+        begin = pivot;
+    }
+    (begin, end)
+}
+
+fn galloping(block_docs: &[u32], target: u32) -> usize {
+    let (start, end) = exponential_search(&block_docs, target);
+    start + linear_search(&block_docs[start..end], target)
+}
+
+/// Tantivy may rely on SIMD instructions to search for a specific document within
+/// a given block.
+#[derive(Clone, Copy, PartialEq)]
+pub enum BlockSearcher {
+    #[cfg(target_arch = "x86_64")]
+    SSE2,
+    Scalar,
+}
+
+impl BlockSearcher {
+    /// Search the first index containing an element greater or equal to
+    /// the target.
+    ///
+    /// The results should be equivalent to
+    /// ```ignore
+    /// block[..]
+    //       .iter()
+    //       .take_while(|&&val| val < target)
+    //       .count()
+    /// ```
+    ///
+    /// The `start` argument is just used to hint that the response is
+    /// greater than beyond `start`. The implementation may or may not use
+    /// it for optimization.
+    ///
+    /// # Assumption
+    ///
+    /// The array len is > start.
+    /// The block is sorted
+    /// The target is assumed greater or equal to the `arr[start]`.
+    /// The target is assumed smaller or equal to the last element of the block.
+    ///
+    /// Currently the scalar implementation starts by an exponential search, and
+    /// then operates a linear search in the result subarray.
+    ///
+    /// If SSE2 instructions are available in the `(platform, running CPU)`,
+    /// then we use a different implementation that does an exhaustive linear search over
+    /// the full block whenever the block is full (`len == 128`). It is surprisingly faster, most likely because of the lack
+    /// of branch.
+    pub(crate) fn search_in_block(
+        self,
+        block_docs: &AlignedBuffer,
+        len: usize,
+        start: usize,
+        target: u32,
+    ) -> usize {
+        #[cfg(target_arch = "x86_64")]
+        {
+            use postings::compression::COMPRESSION_BLOCK_SIZE;
+            if self == BlockSearcher::SSE2 && len == COMPRESSION_BLOCK_SIZE {
+                return sse2::linear_search_sse2_128(block_docs, target);
+            }
+        }
+        start + galloping(&block_docs.0[start..len], target)
+    }
+}
+
+impl Default for BlockSearcher {
+    fn default() -> BlockSearcher {
+        #[cfg(target_arch = "x86_64")]
+        {
+            if is_x86_feature_detected!("sse2") {
+                return BlockSearcher::SSE2;
+            }
+        }
+        BlockSearcher::Scalar
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::exponential_search;
+    use super::linear_search;
+    use super::BlockSearcher;
+    use postings::compression::{AlignedBuffer, COMPRESSION_BLOCK_SIZE};
+
+    #[test]
+    fn test_linear_search() {
+        let len: usize = 50;
+        let arr: Vec<u32> = (0..len).map(|el| 1u32 + (el as u32) * 2).collect();
+        for target in 1..*arr.last().unwrap() {
+            let res = linear_search(&arr[..], target);
+            if res > 0 {
+                assert!(arr[res - 1] < target);
+            }
+            if res < len {
+                assert!(arr[res] >= target);
+            }
+        }
+    }
+
+    #[test]
+    fn test_exponentiel_search() {
+        assert_eq!(exponential_search(&[1, 2], 0), (0, 1));
+        assert_eq!(exponential_search(&[1, 2], 1), (0, 1));
+        assert_eq!(
+            exponential_search(&[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], 7),
+            (3, 7)
+        );
+    }
+
+    fn util_test_search_in_block(block_searcher: BlockSearcher, block: &[u32], target: u32) {
+        let cursor = search_in_block_trivial_but_slow(block, target);
+        assert!(block.len() < COMPRESSION_BLOCK_SIZE);
+        let mut output_buffer = [u32::max_value(); COMPRESSION_BLOCK_SIZE];
+        output_buffer[..block.len()].copy_from_slice(block);
+        for i in 0..cursor {
+            assert_eq!(
+                block_searcher.search_in_block(
+                    &AlignedBuffer(output_buffer),
+                    block.len(),
+                    i,
+                    target
+                ),
+                cursor
+            );
+        }
+    }
+
+    fn util_test_search_in_block_all(block_searcher: BlockSearcher, block: &[u32]) {
+        use std::collections::HashSet;
+        let mut targets = HashSet::new();
+        for (i, val) in block.iter().cloned().enumerate() {
+            if i > 0 {
+                targets.insert(val - 1);
+            }
+            targets.insert(val);
+        }
+        for target in targets {
+            util_test_search_in_block(block_searcher, block, target);
+        }
+    }
+
+    fn search_in_block_trivial_but_slow(block: &[u32], target: u32) -> usize {
+        block.iter().take_while(|&&val| val < target).count()
+    }
+
+    fn test_search_in_block_util(block_searcher: BlockSearcher) {
+        for len in 1u32..128u32 {
+            let v: Vec<u32> = (0..len).map(|i| i * 2).collect();
+            util_test_search_in_block_all(block_searcher, &v[..]);
+        }
+    }
+
+    #[test]
+    fn test_search_in_block_scalar() {
+        test_search_in_block_util(BlockSearcher::Scalar);
+    }
+
+    #[cfg(target_arch = "x86_64")]
+    #[test]
+    fn test_search_in_block_sse2() {
+        test_search_in_block_util(BlockSearcher::SSE2);
+    }
+}

src/postings/compression/mod.rs

@@ -43,9 +43,14 @@ impl BlockEncoder {
     }
 }
 
+/// We ensure that the OutputBuffer is align on 128 bits
+/// in order to run SSE2 linear search on it.
+#[repr(align(128))]
+pub(crate) struct AlignedBuffer(pub [u32; COMPRESSION_BLOCK_SIZE]);
+
 pub struct BlockDecoder {
     bitpacker: BitPacker4x,
-    pub output: [u32; COMPRESSION_BLOCK_SIZE + 1],
+    output: AlignedBuffer,
     pub output_len: usize,
 }
 
@@ -55,11 +60,9 @@ impl BlockDecoder {
     }
 
     pub fn with_val(val: u32) -> BlockDecoder {
-        let mut output = [val; COMPRESSION_BLOCK_SIZE + 1];
-        output[COMPRESSION_BLOCK_SIZE] = 0u32;
         BlockDecoder {
             bitpacker: BitPacker4x::new(),
-            output,
+            output: AlignedBuffer([val; COMPRESSION_BLOCK_SIZE]),
             output_len: 0,
         }
     }
@@ -72,23 +75,28 @@ impl BlockDecoder {
     ) -> usize {
         self.output_len = COMPRESSION_BLOCK_SIZE;
         self.bitpacker
-            .decompress_sorted(offset, &compressed_data, &mut self.output, num_bits)
+            .decompress_sorted(offset, &compressed_data, &mut self.output.0, num_bits)
     }
 
     pub fn uncompress_block_unsorted(&mut self, compressed_data: &[u8], num_bits: u8) -> usize {
         self.output_len = COMPRESSION_BLOCK_SIZE;
         self.bitpacker
-            .decompress(&compressed_data, &mut self.output, num_bits)
+            .decompress(&compressed_data, &mut self.output.0, num_bits)
     }
 
     #[inline]
     pub fn output_array(&self) -> &[u32] {
-        &self.output[..self.output_len]
+        &self.output.0[..self.output_len]
+    }
+
+    #[inline]
+    pub(crate) fn output_aligned(&self) -> (&AlignedBuffer, usize) {
+        (&self.output, self.output_len)
     }
 
     #[inline]
     pub fn output(&self, idx: usize) -> u32 {
-        self.output[idx]
+        self.output.0[idx]
     }
 }
 
@@ -159,12 +167,12 @@ impl VIntDecoder for BlockDecoder {
         num_els: usize,
     ) -> usize {
         self.output_len = num_els;
-        vint::uncompress_sorted(compressed_data, &mut self.output[..num_els], offset)
+        vint::uncompress_sorted(compressed_data, &mut self.output.0[..num_els], offset)
     }
 
     fn uncompress_vint_unsorted<'a>(&mut self, compressed_data: &'a [u8], num_els: usize) -> usize {
         self.output_len = num_els;
-        vint::uncompress_unsorted(compressed_data, &mut self.output[..num_els])
+        vint::uncompress_unsorted(compressed_data, &mut self.output.0[..num_els])
     }
 }
 

src/postings/mod.rs

@@ -2,6 +2,7 @@
 Postings module (also called inverted index)
 */
 
+mod block_search;
 pub(crate) mod compression;
 /// Postings module
 ///
@@ -16,6 +17,8 @@ mod skip;
 mod stacker;
 mod term_info;
 
+pub(crate) use self::block_search::BlockSearcher;
+
 pub(crate) use self::postings_writer::MultiFieldPostingsWriter;
 pub use self::serializer::{FieldSerializer, InvertedIndexSerializer};
 
@@ -31,7 +34,6 @@ pub(crate) use self::stacker::compute_table_size;
 pub use common::HasLen;
 
 pub(crate) const USE_SKIP_INFO_LIMIT: u32 = COMPRESSION_BLOCK_SIZE as u32;
-
 pub(crate) type UnorderedTermId = u64;
 
 #[cfg_attr(feature = "cargo-clippy", allow(clippy::enum_variant_names))]
@@ -53,13 +55,15 @@ pub mod tests {
     use fieldnorm::FieldNormReader;
     use indexer::operation::AddOperation;
     use indexer::SegmentWriter;
+    use merge_policy::NoMergePolicy;
     use query::Scorer;
     use rand::rngs::StdRng;
     use rand::{Rng, SeedableRng};
-    use schema::Field;
-    use schema::IndexRecordOption;
-    use schema::{Document, Schema, Term, INT_INDEXED, STRING, TEXT};
+    use schema::{Document, Schema, Term, INDEXED, STRING, TEXT};
+    use schema::{Field, TextOptions};
+    use schema::{IndexRecordOption, TextFieldIndexing};
     use std::iter;
+    use tokenizer::{SimpleTokenizer, MAX_TOKEN_LEN};
     use DocId;
     use Score;
 
@@ -101,14 +105,11 @@ pub mod tests {
         }
         index_writer.add_document(doc!(title => r#"abc be be be be abc"#));
         index_writer.commit().unwrap();
-        index.load_searchers().unwrap();
 
-        let searcher = index.searcher();
+        let searcher = index.reader().unwrap().searcher();
         let inverted_index = searcher.segment_reader(0u32).inverted_index(title);
         let term = Term::from_field_text(title, "abc");
-
         let mut positions = Vec::new();
-
         {
             let mut postings = inverted_index
                 .read_postings(&term, IndexRecordOption::WithFreqsAndPositions)
@@ -161,6 +162,52 @@ pub mod tests {
         }
     }
 
+    #[test]
+    pub fn test_drop_token_that_are_too_long() {
+        let ok_token_text: String = iter::repeat('A').take(MAX_TOKEN_LEN).collect();
+        let mut exceeding_token_text: String = iter::repeat('A').take(MAX_TOKEN_LEN + 1).collect();
+        exceeding_token_text.push_str(" hello");
+        let mut schema_builder = Schema::builder();
+        let text_options = TextOptions::default().set_indexing_options(
+            TextFieldIndexing::default()
+                .set_index_option(IndexRecordOption::WithFreqsAndPositions)
+                .set_tokenizer("simple_no_truncation"),
+        );
+        let text_field = schema_builder.add_text_field("text", text_options);
+        let schema = schema_builder.build();
+        let index = Index::create_in_ram(schema.clone());
+        index
+            .tokenizers()
+            .register("simple_no_truncation", SimpleTokenizer);
+        let reader = index.reader().unwrap();
+        let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
+        index_writer.set_merge_policy(Box::new(NoMergePolicy));
+        {
+            index_writer.add_document(doc!(text_field=>exceeding_token_text));
+            index_writer.commit().unwrap();
+            reader.reload().unwrap();
+            let searcher = reader.searcher();
+            let segment_reader = searcher.segment_reader(0u32);
+            let inverted_index = segment_reader.inverted_index(text_field);
+            assert_eq!(inverted_index.terms().num_terms(), 1);
+            let mut bytes = vec![];
+            assert!(inverted_index.terms().ord_to_term(0, &mut bytes));
+            assert_eq!(&bytes, b"hello");
+        }
+        {
+            index_writer.add_document(doc!(text_field=>ok_token_text.clone()));
+            index_writer.commit().unwrap();
+            reader.reload().unwrap();
+            let searcher = reader.searcher();
+            let segment_reader = searcher.segment_reader(1u32);
+            let inverted_index = segment_reader.inverted_index(text_field);
+            assert_eq!(inverted_index.terms().num_terms(), 1);
+            let mut bytes = vec![];
+            assert!(inverted_index.terms().ord_to_term(0, &mut bytes));
+            assert_eq!(&bytes[..], ok_token_text.as_bytes());
+        }
+    }
+
     #[test]
     pub fn test_position_and_fieldnorm1() {
         let mut positions = Vec::new();
@@ -293,9 +340,8 @@ pub mod tests {
             }
             assert!(index_writer.commit().is_ok());
         }
-        index.load_searchers().unwrap();
         let term_a = Term::from_field_text(text_field, "a");
-        let searcher = index.searcher();
+        let searcher = index.reader().unwrap().searcher();
         let segment_reader = searcher.segment_reader(0);
         let mut postings = segment_reader
             .inverted_index(text_field)
@@ -317,7 +363,7 @@ pub mod tests {
 
         let index = {
             let mut schema_builder = Schema::builder();
-            let value_field = schema_builder.add_u64_field("value", INT_INDEXED);
+            let value_field = schema_builder.add_u64_field("value", INDEXED);
             let schema = schema_builder.build();
 
             let index = Index::create_in_ram(schema);
@@ -332,10 +378,9 @@ pub mod tests {
                 }
                 assert!(index_writer.commit().is_ok());
             }
-            index.load_searchers().unwrap();
             index
         };
-        let searcher = index.searcher();
+        let searcher = index.reader().unwrap().searcher();
         let segment_reader = searcher.segment_reader(0);
 
         // check that the basic usage works
@@ -403,8 +448,7 @@ pub mod tests {
             index_writer.delete_term(term_0);
             assert!(index_writer.commit().is_ok());
         }
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        let searcher = index.reader().unwrap().searcher();
         let segment_reader = searcher.segment_reader(0);
 
         // make sure seeking still works
@@ -451,12 +495,9 @@ pub mod tests {
         {
             let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
             index_writer.delete_term(term_1);
-
             assert!(index_writer.commit().is_ok());
         }
-        index.load_searchers().unwrap();
-
-        let searcher = index.searcher();
+        let searcher = index.reader().unwrap().searcher();
 
         // finally, check that it's empty
         {
@@ -512,7 +553,6 @@ pub mod tests {
                 }
                 assert!(index_writer.commit().is_ok());
             }
-            index.load_searchers().unwrap();
             index
         };
     }

src/postings/postings_writer.rs

@@ -12,8 +12,8 @@ use std::io;
 use std::marker::PhantomData;
 use std::ops::DerefMut;
 use termdict::TermOrdinal;
-use tokenizer::Token;
 use tokenizer::TokenStream;
+use tokenizer::{Token, MAX_TOKEN_LEN};
 use DocId;
 use Result;
 
@@ -33,9 +33,10 @@ fn posting_from_field_entry(field_entry: &FieldEntry) -> Box<PostingsWriter> {
                 }
             })
             .unwrap_or_else(|| SpecializedPostingsWriter::<NothingRecorder>::new_boxed()),
-        FieldType::U64(_) | FieldType::I64(_) | FieldType::HierarchicalFacet => {
-            SpecializedPostingsWriter::<NothingRecorder>::new_boxed()
-        }
+        FieldType::U64(_)
+        | FieldType::I64(_)
+        | FieldType::Date(_)
+        | FieldType::HierarchicalFacet => SpecializedPostingsWriter::<NothingRecorder>::new_boxed(),
         FieldType::Bytes => {
             // FieldType::Bytes cannot actually be indexed.
             // TODO fix during the indexer refactoring described in #276
@@ -51,6 +52,31 @@ pub struct MultiFieldPostingsWriter {
     per_field_postings_writers: Vec<Box<PostingsWriter>>,
 }
 
+fn make_field_partition(
+    term_offsets: &[(&[u8], Addr, UnorderedTermId)],
+) -> Vec<(Field, usize, usize)> {
+    let term_offsets_it = term_offsets
+        .iter()
+        .map(|(key, _, _)| Term::wrap(key).field())
+        .enumerate();
+    let mut prev_field = Field(u32::max_value());
+    let mut fields = vec![];
+    let mut offsets = vec![];
+    for (offset, field) in term_offsets_it {
+        if field != prev_field {
+            prev_field = field;
+            fields.push(field);
+            offsets.push(offset);
+        }
+    }
+    offsets.push(term_offsets.len());
+    let mut field_offsets = vec![];
+    for i in 0..fields.len() {
+        field_offsets.push((fields[i], offsets[i], offsets[i + 1]));
+    }
+    field_offsets
+}
+
 impl MultiFieldPostingsWriter {
     /// Create a new `MultiFieldPostingsWriter` given
     /// a schema and a heap.
@@ -96,36 +122,16 @@ impl MultiFieldPostingsWriter {
         &self,
         serializer: &mut InvertedIndexSerializer,
     ) -> Result<HashMap<Field, HashMap<UnorderedTermId, TermOrdinal>>> {
-        let mut term_offsets: Vec<(&[u8], Addr, UnorderedTermId)> = self
-            .term_index
-            .iter()
-            .map(|(term_bytes, addr, bucket_id)| (term_bytes, addr, bucket_id as UnorderedTermId))
-            .collect();
+        let mut term_offsets: Vec<(&[u8], Addr, UnorderedTermId)> =
+            self.term_index.iter().collect();
         term_offsets.sort_unstable_by_key(|&(k, _, _)| k);
 
-        let mut offsets: Vec<(Field, usize)> = vec![];
-        let term_offsets_it = term_offsets
-            .iter()
-            .cloned()
-            .map(|(key, _, _)| Term::wrap(key).field())
-            .enumerate();
-
         let mut unordered_term_mappings: HashMap<Field, HashMap<UnorderedTermId, TermOrdinal>> =
             HashMap::new();
 
-        let mut prev_field = Field(u32::max_value());
-        for (offset, field) in term_offsets_it {
-            if field != prev_field {
-                offsets.push((field, offset));
-                prev_field = field;
-            }
-        }
-        offsets.push((Field(0), term_offsets.len()));
-
-        for i in 0..(offsets.len() - 1) {
-            let (field, start) = offsets[i];
-            let (_, stop) = offsets[i + 1];
+        let field_offsets = make_field_partition(&term_offsets);
 
+        for (field, start, stop) in field_offsets {
             let field_entry = self.schema.get_field_entry(field);
 
             match *field_entry.field_type() {
@@ -143,7 +149,7 @@ impl MultiFieldPostingsWriter {
                         .collect();
                     unordered_term_mappings.insert(field, mapping);
                 }
-                FieldType::U64(_) | FieldType::I64(_) => {}
+                FieldType::U64(_) | FieldType::I64(_) | FieldType::Date(_) => {}
                 FieldType::Bytes => {}
             }
 
@@ -204,8 +210,18 @@ pub trait PostingsWriter {
     ) -> u32 {
         let mut term = Term::for_field(field);
         let mut sink = |token: &Token| {
-            term.set_text(token.text.as_str());
-            self.subscribe(term_index, doc_id, token.position as u32, &term, heap);
+            // We skip all tokens with a len greater than u16.
+            if token.text.len() <= MAX_TOKEN_LEN {
+                term.set_text(token.text.as_str());
+                self.subscribe(term_index, doc_id, token.position as u32, &term, heap);
+            } else {
+                info!(
+                    "A token exceeding MAX_TOKEN_LEN ({}>{}) was dropped. Search for \
+                     MAX_TOKEN_LEN in the documentation for more information.",
+                    token.text.len(),
+                    MAX_TOKEN_LEN
+                );
+            }
         };
         token_stream.process(&mut sink)
     }

src/postings/recorder.rs

@@ -4,7 +4,6 @@ use postings::FieldSerializer;
 use std::io;
 use DocId;
 
-const EMPTY_ARRAY: [u32; 0] = [0u32; 0];
 const POSITION_END: u32 = 0;
 
 #[derive(Default)]
@@ -115,7 +114,7 @@ impl Recorder for NothingRecorder {
         let buffer = buffer_lender.lend_u8();
         self.stack.read_to_end(heap, buffer);
         for doc in VInt32Reader::new(&buffer[..]) {
-            serializer.write_doc(doc as u32, 0u32, &EMPTY_ARRAY)?;
+            serializer.write_doc(doc as u32, 0u32, &[][..])?;
         }
         Ok(())
     }
@@ -168,7 +167,7 @@ impl Recorder for TermFrequencyRecorder {
         let mut u32_it = VInt32Reader::new(&buffer[..]);
         while let Some(doc) = u32_it.next() {
             let term_freq = u32_it.next().unwrap_or(self.current_tf);
-            serializer.write_doc(doc as u32, term_freq, &EMPTY_ARRAY)?;
+            serializer.write_doc(doc as u32, term_freq, &[][..])?;
         }
 
         Ok(())

src/postings/segment_postings.rs

@@ -2,22 +2,21 @@ use common::BitSet;
 use common::HasLen;
 use common::{BinarySerializable, VInt};
 use docset::{DocSet, SkipResult};
-use fst::Streamer;
 use owned_read::OwnedRead;
 use positions::PositionReader;
-use postings::compression::compressed_block_size;
+use postings::compression::{compressed_block_size, AlignedBuffer};
 use postings::compression::{BlockDecoder, VIntDecoder, COMPRESSION_BLOCK_SIZE};
 use postings::serializer::PostingsSerializer;
+use postings::BlockSearcher;
 use postings::FreqReadingOption;
 use postings::Postings;
 use postings::SkipReader;
 use postings::USE_SKIP_INFO_LIMIT;
 use schema::IndexRecordOption;
 use std::cmp::Ordering;
+use tantivy_fst::Streamer;
 use DocId;
 
-const EMPTY_ARR: [u8; 0] = [];
-
 struct PositionComputer {
     // store the amount of position int
     // before reading positions.
@@ -62,6 +61,7 @@ pub struct SegmentPostings {
     block_cursor: BlockSegmentPostings,
     cur: usize,
     position_computer: Option<PositionComputer>,
+    block_searcher: BlockSearcher,
 }
 
 impl SegmentPostings {
@@ -72,6 +72,7 @@ impl SegmentPostings {
             block_cursor: empty_block_cursor,
             cur: COMPRESSION_BLOCK_SIZE,
             position_computer: None,
+            block_searcher: BlockSearcher::default(),
         }
     }
 
@@ -119,42 +120,33 @@ impl SegmentPostings {
             block_cursor: segment_block_postings,
             cur: COMPRESSION_BLOCK_SIZE, // cursor within the block
             position_computer: positions_stream_opt.map(PositionComputer::new),
+            block_searcher: BlockSearcher::default(),
         }
     }
 }
 
-fn linear_search(arr: &[u32], target: u32) -> usize {
-    arr.iter().map(|&el| if el < target { 1 } else { 0 }).sum()
-}
-
-fn exponential_search(arr: &[u32], target: u32) -> (usize, usize) {
-    let end = arr.len();
-    let mut begin = 0;
-    for &pivot in &[1, 3, 7, 15, 31, 63] {
-        if pivot >= end {
-            break;
-        }
-        if arr[pivot] > target {
-            return (begin, pivot);
-        }
-        begin = pivot;
-    }
-    (begin, end)
-}
-
-/// Search the first index containing an element greater or equal to the target.
-///
-/// # Assumption
-///
-/// The array is assumed non empty.
-/// The target is assumed greater or equal to the first element.
-/// The target is assumed smaller or equal to the last element.
-fn search_within_block(block_docs: &[u32], target: u32) -> usize {
-    let (start, end) = exponential_search(block_docs, target);
-    start + linear_search(&block_docs[start..end], target)
-}
-
 impl DocSet for SegmentPostings {
+    // goes to the next element.
+    // next needs to be called a first time to point to the correct element.
+    #[inline]
+    fn advance(&mut self) -> bool {
+        if self.position_computer.is_some() && self.cur < COMPRESSION_BLOCK_SIZE {
+            let term_freq = self.term_freq() as usize;
+            if let Some(position_computer) = self.position_computer.as_mut() {
+                position_computer.add_skip(term_freq);
+            }
+        }
+        self.cur += 1;
+        if self.cur >= self.block_cursor.block_len() {
+            self.cur = 0;
+            if !self.block_cursor.advance() {
+                self.cur = COMPRESSION_BLOCK_SIZE;
+                return false;
+            }
+        }
+        true
+    }
+
     fn skip_next(&mut self, target: DocId) -> SkipResult {
         if !self.advance() {
             return SkipResult::End;
@@ -177,7 +169,6 @@ impl DocSet for SegmentPostings {
 
         // skip blocks until one that might contain the target
         // check if we need to go to the next block
-        let need_positions = self.position_computer.is_some();
         let mut sum_freqs_skipped: u32 = 0;
         if !self
             .block_cursor
@@ -191,7 +182,7 @@ impl DocSet for SegmentPostings {
             // we are not in the right block.
             //
             // First compute all of the freqs skipped from the current block.
-            if need_positions {
+            if self.position_computer.is_some() {
                 sum_freqs_skipped = self.block_cursor.freqs()[self.cur..].iter().sum();
                 match self.block_cursor.skip_to(target) {
                     BlockSegmentPostingsSkipResult::Success(block_skip_freqs) => {
@@ -210,25 +201,21 @@ impl DocSet for SegmentPostings {
             self.cur = 0;
         }
 
-        // we're in the right block now, start with an exponential search
-        let block_docs = self.block_cursor.docs();
-        let new_cur = self
-            .cur
-            .wrapping_add(search_within_block(&block_docs[self.cur..], target));
+        let cur = self.cur;
 
-        if need_positions {
-            sum_freqs_skipped += self.block_cursor.freqs()[self.cur..new_cur]
-                .iter()
-                .sum::<u32>();
-            self.position_computer
-                .as_mut()
-                .unwrap()
-                .add_skip(sum_freqs_skipped as usize);
+        // we're in the right block now, start with an exponential search
+        let (output, len) = self.block_cursor.docs_aligned();
+        let new_cur = self
+            .block_searcher
+            .search_in_block(&output, len, cur, target);
+        if let Some(position_computer) = self.position_computer.as_mut() {
+            sum_freqs_skipped += self.block_cursor.freqs()[cur..new_cur].iter().sum::<u32>();
+            position_computer.add_skip(sum_freqs_skipped as usize);
         }
         self.cur = new_cur;
 
         // `doc` is now the first element >= `target`
-        let doc = block_docs[new_cur];
+        let doc = output.0[new_cur];
         debug_assert!(doc >= target);
         if doc == target {
             SkipResult::Reached
@@ -237,40 +224,25 @@ impl DocSet for SegmentPostings {
         }
     }
 
-    // goes to the next element.
-    // next needs to be called a first time to point to the correct element.
-    #[inline]
-    fn advance(&mut self) -> bool {
-        if self.position_computer.is_some() {
-            let term_freq = self.term_freq() as usize;
-            self.position_computer.as_mut().unwrap().add_skip(term_freq);
-        }
-        self.cur += 1;
-        if self.cur >= self.block_cursor.block_len() {
-            self.cur = 0;
-            if !self.block_cursor.advance() {
-                self.cur = COMPRESSION_BLOCK_SIZE;
-                return false;
-            }
-        }
-        true
-    }
-
-    fn size_hint(&self) -> u32 {
-        self.len() as u32
-    }
-
     /// Return the current document's `DocId`.
+    ///
+    /// # Panics
+    ///
+    /// Will panics if called without having called advance before.
     #[inline]
     fn doc(&self) -> DocId {
         let docs = self.block_cursor.docs();
         debug_assert!(
             self.cur < docs.len(),
-            "Have you forgotten to call `.advance()` at least once before calling .doc()."
+            "Have you forgotten to call `.advance()` at least once before calling `.doc()`                                      ."
         );
         docs[self.cur]
     }
 
+    fn size_hint(&self) -> u32 {
+        self.len() as u32
+    }
+
     fn append_to_bitset(&mut self, bitset: &mut BitSet) {
         // finish the current block
         if self.advance() {
@@ -294,17 +266,33 @@ impl HasLen for SegmentPostings {
 }
 
 impl Postings for SegmentPostings {
+    /// Returns the frequency associated to the current document.
+    /// If the schema is set up so that no frequency have been encoded,
+    /// this method should always return 1.
+    ///
+    /// # Panics
+    ///
+    /// Will panics if called without having called advance before.
     fn term_freq(&self) -> u32 {
+        debug_assert!(
+            // Here we do not use the len of `freqs()`
+            // because it is actually ok to request for the freq of doc
+            // even if no frequency were encoded for the field.
+            //
+            // In that case we hit the block just as if the frequency had been
+            // decoded. The block is simply prefilled by the value 1.
+            self.cur < COMPRESSION_BLOCK_SIZE,
+            "Have you forgotten to call `.advance()` at least once before calling \
+             `.term_freq()`."
+        );
         self.block_cursor.freq(self.cur)
     }
 
     fn positions_with_offset(&mut self, offset: u32, output: &mut Vec<u32>) {
-        if self.position_computer.is_some() {
-            output.resize(self.term_freq() as usize, 0u32);
-            self.position_computer
-                .as_mut()
-                .unwrap()
-                .positions_with_offset(offset, &mut output[..])
+        let term_freq = self.term_freq() as usize;
+        if let Some(position_comp) = self.position_computer.as_mut() {
+            output.resize(term_freq, 0u32);
+            position_comp.positions_with_offset(offset, &mut output[..]);
         } else {
             output.clear();
         }
@@ -369,7 +357,7 @@ impl BlockSegmentPostings {
         let (skip_data_opt, postings_data) = split_into_skips_and_postings(doc_freq, data);
         let skip_reader = match skip_data_opt {
             Some(skip_data) => SkipReader::new(skip_data, record_option),
-            None => SkipReader::new(OwnedRead::new(&EMPTY_ARR[..]), record_option),
+            None => SkipReader::new(OwnedRead::new(&[][..]), record_option),
         };
         let doc_freq = doc_freq as usize;
         let num_vint_docs = doc_freq % COMPRESSION_BLOCK_SIZE;
@@ -403,7 +391,7 @@ impl BlockSegmentPostings {
         if let Some(skip_data) = skip_data_opt {
             self.skip_reader.reset(skip_data);
         } else {
-            self.skip_reader.reset(OwnedRead::new(&EMPTY_ARR[..]))
+            self.skip_reader.reset(OwnedRead::new(&[][..]))
         }
         self.doc_offset = 0;
         self.doc_freq = doc_freq as usize;
@@ -426,6 +414,10 @@ impl BlockSegmentPostings {
         self.doc_decoder.output_array()
     }
 
+    pub(crate) fn docs_aligned(&self) -> (&AlignedBuffer, usize) {
+        self.doc_decoder.output_aligned()
+    }
+
     /// Return the document at index `idx` of the block.
     #[inline]
     pub fn doc(&self, idx: usize) -> u32 {
@@ -616,39 +608,21 @@ impl<'b> Streamer<'b> for BlockSegmentPostings {
 
 #[cfg(test)]
 mod tests {
-
-    use super::exponential_search;
-    use super::linear_search;
-    use super::search_within_block;
     use super::BlockSegmentPostings;
     use super::BlockSegmentPostingsSkipResult;
     use super::SegmentPostings;
     use common::HasLen;
     use core::Index;
     use docset::DocSet;
-    use fst::Streamer;
+    use postings::postings::Postings;
     use schema::IndexRecordOption;
     use schema::Schema;
     use schema::Term;
-    use schema::INT_INDEXED;
+    use schema::INDEXED;
+    use tantivy_fst::Streamer;
     use DocId;
     use SkipResult;
 
-    #[test]
-    fn test_linear_search() {
-        let len: usize = 50;
-        let arr: Vec<u32> = (0..len).map(|el| 1u32 + (el as u32) * 2).collect();
-        for target in 1..*arr.last().unwrap() {
-            let res = linear_search(&arr[..], target);
-            if res > 0 {
-                assert!(arr[res - 1] < target);
-            }
-            if res < len {
-                assert!(arr[res] >= target);
-            }
-        }
-    }
-
     #[test]
     fn test_empty_segment_postings() {
         let mut postings = SegmentPostings::empty();
@@ -657,6 +631,18 @@ mod tests {
         assert_eq!(postings.len(), 0);
     }
 
+    #[test]
+    #[should_panic(expected = "Have you forgotten to call `.advance()`")]
+    fn test_panic_if_doc_called_before_advance() {
+        SegmentPostings::empty().doc();
+    }
+
+    #[test]
+    #[should_panic(expected = "Have you forgotten to call `.advance()`")]
+    fn test_panic_if_freq_called_before_advance() {
+        SegmentPostings::empty().term_freq();
+    }
+
     #[test]
     fn test_empty_block_segment_postings() {
         let mut postings = BlockSegmentPostings::empty();
@@ -664,56 +650,6 @@ mod tests {
         assert_eq!(postings.doc_freq(), 0);
     }
 
-    fn search_within_block_trivial_but_slow(block: &[u32], target: u32) -> usize {
-        block
-            .iter()
-            .cloned()
-            .enumerate()
-            .filter(|&(_, ref val)| *val >= target)
-            .next()
-            .unwrap()
-            .0
-    }
-
-    #[test]
-    fn test_exponentiel_search() {
-        assert_eq!(exponential_search(&[1, 2], 0), (0, 1));
-        assert_eq!(exponential_search(&[1, 2], 1), (0, 1));
-        assert_eq!(
-            exponential_search(&[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], 7),
-            (3, 7)
-        );
-    }
-
-    fn util_test_search_within_block(block: &[u32], target: u32) {
-        assert_eq!(
-            search_within_block(block, target),
-            search_within_block_trivial_but_slow(block, target)
-        );
-    }
-
-    fn util_test_search_within_block_all(block: &[u32]) {
-        use std::collections::HashSet;
-        let mut targets = HashSet::new();
-        for (i, val) in block.iter().cloned().enumerate() {
-            if i > 0 {
-                targets.insert(val - 1);
-            }
-            targets.insert(val);
-        }
-        for target in targets {
-            util_test_search_within_block(block, target);
-        }
-    }
-
-    #[test]
-    fn test_search_within_block() {
-        for len in 1u32..128u32 {
-            let v: Vec<u32> = (0..len).map(|i| i * 2).collect();
-            util_test_search_within_block_all(&v[..]);
-        }
-    }
-
     #[test]
     fn test_block_segment_postings() {
         let mut block_segments = build_block_postings(&(0..100_000).collect::<Vec<u32>>());
@@ -762,7 +698,7 @@ mod tests {
 
     fn build_block_postings(docs: &[DocId]) -> BlockSegmentPostings {
         let mut schema_builder = Schema::builder();
-        let int_field = schema_builder.add_u64_field("id", INT_INDEXED);
+        let int_field = schema_builder.add_u64_field("id", INDEXED);
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
         let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
@@ -775,8 +711,7 @@ mod tests {
             last_doc = doc + 1;
         }
         index_writer.commit().unwrap();
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        let searcher = index.reader().unwrap().searcher();
         let segment_reader = searcher.segment_reader(0);
         let inverted_index = segment_reader.inverted_index(int_field);
         let term = Term::from_field_u64(int_field, 0u64);
@@ -833,7 +768,7 @@ mod tests {
     #[test]
     fn test_reset_block_segment_postings() {
         let mut schema_builder = Schema::builder();
-        let int_field = schema_builder.add_u64_field("id", INT_INDEXED);
+        let int_field = schema_builder.add_u64_field("id", INDEXED);
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
         let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
@@ -844,8 +779,7 @@ mod tests {
             index_writer.add_document(doc);
         }
         index_writer.commit().unwrap();
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        let searcher = index.reader().unwrap().searcher();
         let segment_reader = searcher.segment_reader(0);
 
         let mut block_segments;

src/postings/serializer.rs

@@ -14,7 +14,7 @@ use termdict::{TermDictionaryBuilder, TermOrdinal};
 use DocId;
 use Result;
 
-/// `PostingsSerializer` is in charge of serializing
+/// `InvertedIndexSerializer` is in charge of serializing
 /// postings on disk, in the
 /// * `.idx` (inverted index)
 /// * `.pos` (positions file)
@@ -54,7 +54,7 @@ pub struct InvertedIndexSerializer {
 }
 
 impl InvertedIndexSerializer {
-    /// Open a new `PostingsSerializer` for the given segment
+    /// Open a new `InvertedIndexSerializer` for the given segment
     fn create(
         terms_write: CompositeWrite<WritePtr>,
         postings_write: CompositeWrite<WritePtr>,

src/postings/stacker/term_hashmap.rs

@@ -5,12 +5,11 @@ use self::murmurhash32::murmurhash2;
 use super::{Addr, MemoryArena};
 use byteorder::{ByteOrder, NativeEndian};
 use postings::stacker::memory_arena::store;
+use postings::UnorderedTermId;
 use std::iter;
 use std::mem;
 use std::slice;
 
-pub type BucketId = usize;
-
 /// Returns the actual memory size in bytes
 /// required to create a table of size $2^num_bits$.
 pub fn compute_table_size(num_bits: usize) -> usize {
@@ -28,6 +27,7 @@ pub fn compute_table_size(num_bits: usize) -> usize {
 struct KeyValue {
     key_value_addr: Addr,
     hash: u32,
+    unordered_term_id: UnorderedTermId,
 }
 
 impl Default for KeyValue {
@@ -35,6 +35,7 @@ impl Default for KeyValue {
         KeyValue {
             key_value_addr: Addr::null_pointer(),
             hash: 0u32,
+            unordered_term_id: UnorderedTermId::default(),
         }
     }
 }
@@ -59,6 +60,7 @@ pub struct TermHashMap {
     pub heap: MemoryArena,
     mask: usize,
     occupied: Vec<usize>,
+    len: usize,
 }
 
 struct QuadraticProbing {
@@ -85,13 +87,13 @@ pub struct Iter<'a> {
 }
 
 impl<'a> Iterator for Iter<'a> {
-    type Item = (&'a [u8], Addr, BucketId);
+    type Item = (&'a [u8], Addr, UnorderedTermId);
 
     fn next(&mut self) -> Option<Self::Item> {
         self.inner.next().cloned().map(move |bucket: usize| {
             let kv = self.hashmap.table[bucket];
             let (key, offset): (&'a [u8], Addr) = self.hashmap.get_key_value(kv.key_value_addr);
-            (key, offset, bucket as BucketId)
+            (key, offset, kv.unordered_term_id)
         })
     }
 }
@@ -106,6 +108,7 @@ impl TermHashMap {
             heap,
             mask: table_size - 1,
             occupied: Vec::with_capacity(table_size / 2),
+            len: 0,
         }
     }
 
@@ -139,12 +142,16 @@ impl TermHashMap {
         }
     }
 
-    pub fn set_bucket(&mut self, hash: u32, key_value_addr: Addr, bucket: usize) {
+    fn set_bucket(&mut self, hash: u32, key_value_addr: Addr, bucket: usize) -> UnorderedTermId {
         self.occupied.push(bucket);
+        let unordered_term_id = self.len as UnorderedTermId;
+        self.len += 1;
         self.table[bucket] = KeyValue {
             key_value_addr,
             hash,
+            unordered_term_id,
         };
+        unordered_term_id
     }
 
     pub fn iter(&self) -> Iter {
@@ -184,7 +191,11 @@ impl TermHashMap {
     /// will be in charge of returning a default value.
     /// If the key already as an associated value, then it will be passed
     /// `Some(previous_value)`.
-    pub fn mutate_or_create<S, V, TMutator>(&mut self, key: S, mut updater: TMutator) -> BucketId
+    pub fn mutate_or_create<S, V, TMutator>(
+        &mut self,
+        key: S,
+        mut updater: TMutator,
+    ) -> UnorderedTermId
     where
         S: AsRef<[u8]>,
         V: Copy + 'static,
@@ -200,6 +211,7 @@ impl TermHashMap {
             let bucket = probe.next_probe();
             let kv: KeyValue = self.table[bucket];
             if kv.is_empty() {
+                // The key does not exists yet.
                 let val = updater(None);
                 let num_bytes =
                     std::mem::size_of::<u16>() + key_bytes.len() + std::mem::size_of::<V>();
@@ -211,8 +223,7 @@ impl TermHashMap {
                     data[2..stop].copy_from_slice(key_bytes);
                     store(&mut data[stop..], val);
                 }
-                self.set_bucket(hash, key_addr, bucket);
-                return bucket as BucketId;
+                return self.set_bucket(hash, key_addr, bucket);
             } else if kv.hash == hash {
                 if let Some(val_addr) =
                     self.get_value_addr_if_key_match(key_bytes, kv.key_value_addr)
@@ -220,7 +231,7 @@ impl TermHashMap {
                     let v = self.heap.read(val_addr);
                     let new_v = updater(Some(v));
                     self.heap.write_at(val_addr, new_v);
-                    return bucket as BucketId;
+                    return kv.unordered_term_id;
                 }
             }
         }

src/query/all_query.rs

@@ -101,8 +101,9 @@ mod tests {
         index_writer.commit().unwrap();
         index_writer.add_document(doc!(field=>"ccc"));
         index_writer.commit().unwrap();
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        let reader = index.reader().unwrap();
+        reader.reload().unwrap();
+        let searcher = reader.searcher();
         let weight = AllQuery.weight(&searcher, false).unwrap();
         {
             let reader = searcher.segment_reader(0);

src/query/automaton_weight.rs

@@ -1,10 +1,10 @@
 use common::BitSet;
 use core::SegmentReader;
-use fst::Automaton;
 use query::BitSetDocSet;
 use query::ConstScorer;
 use query::{Scorer, Weight};
 use schema::{Field, IndexRecordOption};
+use tantivy_fst::Automaton;
 use termdict::{TermDictionary, TermStreamer};
 use Result;
 

src/query/boolean_query/mod.rs

@@ -51,7 +51,6 @@ mod tests {
             }
             assert!(index_writer.commit().is_ok());
         }
-        index.load_searchers().unwrap();
         (index, text_field)
     }
 
@@ -60,7 +59,8 @@ mod tests {
         let (index, text_field) = aux_test_helper();
         let query_parser = QueryParser::for_index(&index, vec![text_field]);
         let query = query_parser.parse_query("(+a +b) d").unwrap();
-        assert_eq!(query.count(&*index.searcher()).unwrap(), 3);
+        let searcher = index.reader().unwrap().searcher();
+        assert_eq!(query.count(&searcher).unwrap(), 3);
     }
 
     #[test]
@@ -68,7 +68,7 @@ mod tests {
         let (index, text_field) = aux_test_helper();
         let query_parser = QueryParser::for_index(&index, vec![text_field]);
         let query = query_parser.parse_query("+a").unwrap();
-        let searcher = index.searcher();
+        let searcher = index.reader().unwrap().searcher();
         let weight = query.weight(&searcher, true).unwrap();
         let scorer = weight.scorer(searcher.segment_reader(0u32)).unwrap();
         assert!(scorer.is::<TermScorer>());
@@ -78,7 +78,7 @@ mod tests {
     pub fn test_boolean_termonly_intersection() {
         let (index, text_field) = aux_test_helper();
         let query_parser = QueryParser::for_index(&index, vec![text_field]);
-        let searcher = index.searcher();
+        let searcher = index.reader().unwrap().searcher();
         {
             let query = query_parser.parse_query("+a +b +c").unwrap();
             let weight = query.weight(&searcher, true).unwrap();
@@ -97,7 +97,7 @@ mod tests {
     pub fn test_boolean_reqopt() {
         let (index, text_field) = aux_test_helper();
         let query_parser = QueryParser::for_index(&index, vec![text_field]);
-        let searcher = index.searcher();
+        let searcher = index.reader().unwrap().searcher();
         {
             let query = query_parser.parse_query("+a b").unwrap();
             let weight = query.weight(&searcher, true).unwrap();
@@ -126,10 +126,13 @@ mod tests {
             query
         };
 
+        let reader = index.reader().unwrap();
+
         let matching_docs = |boolean_query: &Query| {
-            let searcher = index.searcher();
-            let test_docs = searcher.search(boolean_query, &TestCollector).unwrap();
-            test_docs
+            reader
+                .searcher()
+                .search(boolean_query, &TestCollector)
+                .unwrap()
                 .docs()
                 .iter()
                 .cloned()
@@ -185,10 +188,12 @@ mod tests {
             let query: Box<Query> = Box::new(term_query);
             query
         };
-
+        let reader = index.reader().unwrap();
         let score_docs = |boolean_query: &Query| {
-            let searcher = index.searcher();
-            let fruit = searcher.search(boolean_query, &TestCollector).unwrap();
+            let fruit = reader
+                .searcher()
+                .search(boolean_query, &TestCollector)
+                .unwrap();
             fruit.scores().to_vec()
         };
 

src/query/fuzzy_query.rs

@@ -52,9 +52,8 @@ lazy_static! {
 ///         ));
 ///         index_writer.commit().unwrap();
 ///     }
-///
-///     index.load_searchers()?;
-///     let searcher = index.searcher();
+///     let reader = index.reader()?;
+///     let searcher = reader.searcher();
 ///
 ///     {
 ///
@@ -141,8 +140,8 @@ mod test {
             ));
             index_writer.commit().unwrap();
         }
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        let reader = index.reader().unwrap();
+        let searcher = reader.searcher();
         {
             let term = Term::from_field_text(country_field, "japon");
 

src/query/intersection.rs

@@ -14,41 +14,35 @@ use Score;
 /// specialized implementation if the two
 /// shortest scorers are `TermScorer`s.
 pub fn intersect_scorers(mut scorers: Vec<Box<Scorer>>) -> Box<Scorer> {
+    if scorers.is_empty() {
+        return Box::new(EmptyScorer);
+    }
+    if scorers.len() == 1 {
+        return scorers.pop().unwrap();
+    }
+    // We know that we have at least 2 elements.
     let num_docsets = scorers.len();
     scorers.sort_by(|left, right| right.size_hint().cmp(&left.size_hint()));
-    let rarest_opt = scorers.pop();
-    let second_rarest_opt = scorers.pop();
+    let left = scorers.pop().unwrap();
+    let right = scorers.pop().unwrap();
     scorers.reverse();
-    match (rarest_opt, second_rarest_opt) {
-        (None, None) => Box::new(EmptyScorer),
-        (Some(single_docset), None) => single_docset,
-        (Some(left), Some(right)) => {
-            {
-                let all_term_scorers = [&left, &right]
-                    .iter()
-                    .all(|&scorer| scorer.is::<TermScorer>());
-                if all_term_scorers {
-                    let left = *(left.downcast::<TermScorer>().map_err(|_| ()).unwrap());
-                    let right = *(right.downcast::<TermScorer>().map_err(|_| ()).unwrap());
-                    return Box::new(Intersection {
-                        left,
-                        right,
-                        others: scorers,
-                        num_docsets,
-                    });
-                }
-            }
-            Box::new(Intersection {
-                left,
-                right,
-                others: scorers,
-                num_docsets,
-            })
-        }
-        _ => {
-            unreachable!();
-        }
+    let all_term_scorers = [&left, &right]
+        .iter()
+        .all(|&scorer| scorer.is::<TermScorer>());
+    if all_term_scorers {
+        return Box::new(Intersection {
+            left: *(left.downcast::<TermScorer>().map_err(|_| ()).unwrap()),
+            right: *(right.downcast::<TermScorer>().map_err(|_| ()).unwrap()),
+            others: scorers,
+            num_docsets,
+        });
     }
+    Box::new(Intersection {
+        left,
+        right,
+        others: scorers,
+        num_docsets,
+    })
 }
 
 /// Creates a `DocSet` that iterator through the intersection of two `DocSet`s.
@@ -124,7 +118,6 @@ impl<TDocSet: DocSet, TOtherDocSet: DocSet> DocSet for Intersection<TDocSet, TOt
                         return false;
                     }
                 }
-
                 match left.skip_next(candidate) {
                     SkipResult::Reached => {
                         break;
@@ -140,35 +133,36 @@ impl<TDocSet: DocSet, TOtherDocSet: DocSet> DocSet for Intersection<TDocSet, TOt
             }
             // test the remaining scorers;
             for (ord, docset) in self.others.iter_mut().enumerate() {
-                if ord != other_candidate_ord {
-                    // `candidate_ord` is already at the
-                    // right position.
-                    //
-                    // Calling `skip_next` would advance this docset
-                    // and miss it.
-                    match docset.skip_next(candidate) {
-                        SkipResult::Reached => {}
-                        SkipResult::OverStep => {
-                            // this is not in the intersection,
-                            // let's update our candidate.
-                            candidate = docset.doc();
-                            match left.skip_next(candidate) {
-                                SkipResult::Reached => {
-                                    other_candidate_ord = ord;
-                                }
-                                SkipResult::OverStep => {
-                                    candidate = left.doc();
-                                    other_candidate_ord = usize::max_value();
-                                }
-                                SkipResult::End => {
-                                    return false;
-                                }
+                if ord == other_candidate_ord {
+                    continue;
+                }
+                // `candidate_ord` is already at the
+                // right position.
+                //
+                // Calling `skip_next` would advance this docset
+                // and miss it.
+                match docset.skip_next(candidate) {
+                    SkipResult::Reached => {}
+                    SkipResult::OverStep => {
+                        // this is not in the intersection,
+                        // let's update our candidate.
+                        candidate = docset.doc();
+                        match left.skip_next(candidate) {
+                            SkipResult::Reached => {
+                                other_candidate_ord = ord;
+                            }
+                            SkipResult::OverStep => {
+                                candidate = left.doc();
+                                other_candidate_ord = usize::max_value();
+                            }
+                            SkipResult::End => {
+                                return false;
                             }
-                            continue 'outer;
-                        }
-                        SkipResult::End => {
-                            return false;
                         }
+                        continue 'outer;
+                    }
+                    SkipResult::End => {
+                        return false;
                     }
                 }
             }

src/query/phrase_query/mod.rs

@@ -31,7 +31,6 @@ mod tests {
             }
             assert!(index_writer.commit().is_ok());
         }
-        index.load_searchers().unwrap();
         index
     }
 
@@ -46,8 +45,7 @@ mod tests {
         ]);
         let schema = index.schema();
         let text_field = schema.get_field("text").unwrap();
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        let searcher = index.reader().unwrap().searcher();
         let test_query = |texts: Vec<&str>| {
             let terms: Vec<Term> = texts
                 .iter()
@@ -90,8 +88,7 @@ mod tests {
             index_writer.add_document(doc!(text_field=>"a b c"));
             assert!(index_writer.commit().is_ok());
         }
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        let searcher = index.reader().unwrap().searcher();
         let phrase_query = PhraseQuery::new(vec![
             Term::from_field_text(text_field, "a"),
             Term::from_field_text(text_field, "b"),
@@ -115,8 +112,7 @@ mod tests {
         let index = create_index(&["a b c", "a b c a b"]);
         let schema = index.schema();
         let text_field = schema.get_field("text").unwrap();
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        let searcher = index.reader().unwrap().searcher();
         let test_query = |texts: Vec<&str>| {
             let terms: Vec<Term> = texts
                 .iter()
@@ -148,8 +144,7 @@ mod tests {
             assert!(index_writer.commit().is_ok());
         }
 
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        let searcher = index.reader().unwrap().searcher();
         let test_query = |texts: Vec<&str>| {
             let terms: Vec<Term> = texts
                 .iter()
@@ -177,8 +172,7 @@ mod tests {
             index_writer.add_document(doc!(text_field=>"a b c d e f g h"));
             assert!(index_writer.commit().is_ok());
         }
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        let searcher = index.reader().unwrap().searcher();
         let test_query = |texts: Vec<(usize, &str)>| {
             let terms: Vec<(usize, Term)> = texts
                 .iter()

src/query/query_parser/query_grammar.rs

@@ -52,7 +52,7 @@ parser! {
             field_name: None,
             phrase,
         });
-        try(term_query)
+        attempt(term_query)
             .or(term_default_field)
             .map(UserInputLeaf::from)
     }
@@ -83,12 +83,12 @@ parser! {
         let lower_bound = {
             let excl = (char('{'), term_val()).map(|(_, w)| UserInputBound::Exclusive(w));
             let incl = (char('['), term_val()).map(|(_, w)| UserInputBound::Inclusive(w));
-            try(excl).or(incl)
+            attempt(excl).or(incl)
         };
         let upper_bound = {
             let excl = (term_val(), char('}')).map(|(w, _)| UserInputBound::Exclusive(w));
             let incl = (term_val(), char(']')).map(|(w, _)| UserInputBound::Inclusive(w));
-            try(excl).or(incl)
+            attempt(excl).or(incl)
         };
         (
             optional((field(), char(':')).map(|x| x.0)),
@@ -112,11 +112,11 @@ parser! {
         .or((char('+'), leaf()).map(|(_, expr)| expr.unary(Occur::Must) ))
         .or((char('('), parse_to_ast(), char(')')).map(|(_, expr, _)| expr))
         .or(char('*').map(|_| UserInputAST::from(UserInputLeaf::All) ))
-        .or(try(
+        .or(attempt(
             (string("NOT"), spaces1(), leaf()).map(|(_, _, expr)| expr.unary(Occur::MustNot))
             )
          )
-        .or(try(
+        .or(attempt(
             range().map(UserInputAST::from)
             )
         )
@@ -160,7 +160,7 @@ parser! {
     where [I: Stream<Item = char>]
     {
         (
-            try(
+            attempt(
                 chainl1(
                     leaf().map(Element::SingleEl),
                     binary_operand().map(|op: BinaryOperand|

src/query/query_parser/query_parser.rs

@@ -50,6 +50,8 @@ pub enum QueryParserError {
     /// The query contains a range query with a phrase as one of the bounds.
     /// Only terms can be used as bounds.
     RangeMustNotHavePhrase,
+    /// The format for the date field is not RFC 3339 compliant.
+    DateFormatError(chrono::ParseError),
 }
 
 impl From<ParseIntError> for QueryParserError {
@@ -58,6 +60,12 @@ impl From<ParseIntError> for QueryParserError {
     }
 }
 
+impl From<chrono::ParseError> for QueryParserError {
+    fn from(err: chrono::ParseError) -> QueryParserError {
+        QueryParserError::DateFormatError(err)
+    }
+}
+
 /// Recursively remove empty clause from the AST
 ///
 /// Returns `None` iff the `logical_ast` ended up being empty.
@@ -127,6 +135,8 @@ fn trim_ast(logical_ast: LogicalAST) -> Option<LogicalAST> {
 ///   a word lexicographically between `a` and `c` (inclusive lower bound, exclusive upper bound).
 ///   Inclusive bounds are `[]`, exclusive are `{}`.
 ///
+/// * date values: The query parser supports rfc3339 formatted dates. For example "2002-10-02T15:00:00.05Z"
+///
 /// *  all docs query: A plain `*` will match all documents in the index.
 ///
 #[derive(Clone)]
@@ -229,6 +239,13 @@ impl QueryParser {
                 let term = Term::from_field_i64(field, val);
                 Ok(vec![(0, term)])
             }
+            FieldType::Date(_) => match chrono::DateTime::parse_from_rfc3339(phrase) {
+                Ok(x) => Ok(vec![(
+                    0,
+                    Term::from_field_date(field, &x.with_timezone(&chrono::Utc)),
+                )]),
+                Err(e) => Err(QueryParserError::DateFormatError(e)),
+            },
             FieldType::U64(_) => {
                 let val: u64 = u64::from_str(phrase)?;
                 let term = Term::from_field_u64(field, val);
@@ -487,7 +504,7 @@ mod test {
     use query::Query;
     use schema::Field;
     use schema::{IndexRecordOption, TextFieldIndexing, TextOptions};
-    use schema::{Schema, Term, INT_INDEXED, STORED, STRING, TEXT};
+    use schema::{Schema, Term, INDEXED, STORED, STRING, TEXT};
     use tokenizer::{LowerCaser, SimpleTokenizer, StopWordFilter, Tokenizer, TokenizerManager};
     use Index;
 
@@ -501,13 +518,14 @@ mod test {
             .set_stored();
         let title = schema_builder.add_text_field("title", TEXT);
         let text = schema_builder.add_text_field("text", TEXT);
-        schema_builder.add_i64_field("signed", INT_INDEXED);
-        schema_builder.add_u64_field("unsigned", INT_INDEXED);
+        schema_builder.add_i64_field("signed", INDEXED);
+        schema_builder.add_u64_field("unsigned", INDEXED);
         schema_builder.add_text_field("notindexed_text", STORED);
         schema_builder.add_text_field("notindexed_u64", STORED);
         schema_builder.add_text_field("notindexed_i64", STORED);
         schema_builder.add_text_field("nottokenized", STRING);
         schema_builder.add_text_field("with_stop_words", text_options);
+        schema_builder.add_date_field("date", INDEXED);
         let schema = schema_builder.build();
         let default_fields = vec![title, text];
         let tokenizer_manager = TokenizerManager::default();
@@ -767,6 +785,18 @@ mod test {
         );
     }
 
+    #[test]
+    pub fn test_query_parser_expected_date() {
+        let query_parser = make_query_parser();
+        assert_matches!(
+            query_parser.parse_query("date:18a"),
+            Err(QueryParserError::DateFormatError(_))
+        );
+        assert!(query_parser
+            .parse_query("date:\"1985-04-12T23:20:50.52Z\"")
+            .is_ok());
+    }
+
     #[test]
     pub fn test_query_parser_not_empty_but_no_tokens() {
         let query_parser = make_query_parser();

src/query/range_query.rs

@@ -40,14 +40,14 @@ fn map_bound<TFrom, TTo, Transform: Fn(&TFrom) -> TTo>(
 /// # #[macro_use]
 /// # extern crate tantivy;
 /// # use tantivy::Index;
-/// # use tantivy::schema::{Schema, INT_INDEXED};
+/// # use tantivy::schema::{Schema, INDEXED};
 /// # use tantivy::collector::Count;
 /// # use tantivy::Result;
 /// # use tantivy::query::RangeQuery;
 /// #
 /// # fn run() -> Result<()> {
 /// #     let mut schema_builder = Schema::builder();
-/// #     let year_field = schema_builder.add_u64_field("year", INT_INDEXED);
+/// #     let year_field = schema_builder.add_u64_field("year", INDEXED);
 /// #     let schema = schema_builder.build();
 /// #
 /// #     let index = Index::create_in_ram(schema);
@@ -61,8 +61,8 @@ fn map_bound<TFrom, TTo, Transform: Fn(&TFrom) -> TTo>(
 /// #         }
 /// #         index_writer.commit().unwrap();
 /// #     }
-/// #   index.load_searchers()?;
-/// let searcher = index.searcher();
+/// #   let reader = index.reader()?;
+/// let searcher = reader.searcher();
 ///
 /// let docs_in_the_sixties = RangeQuery::new_u64(year_field, 1960..1970);
 ///
@@ -293,7 +293,7 @@ mod tests {
 
     use super::RangeQuery;
     use collector::Count;
-    use schema::{Document, Field, Schema, INT_INDEXED};
+    use schema::{Document, Field, Schema, INDEXED};
     use std::collections::Bound;
     use Index;
     use Result;
@@ -302,7 +302,7 @@ mod tests {
     fn test_range_query_simple() {
         fn run() -> Result<()> {
             let mut schema_builder = Schema::builder();
-            let year_field = schema_builder.add_u64_field("year", INT_INDEXED);
+            let year_field = schema_builder.add_u64_field("year", INDEXED);
             let schema = schema_builder.build();
 
             let index = Index::create_in_ram(schema);
@@ -316,8 +316,8 @@ mod tests {
                 }
                 index_writer.commit().unwrap();
             }
-            index.load_searchers().unwrap();
-            let searcher = index.searcher();
+            let reader = index.reader().unwrap();
+            let searcher = reader.searcher();
 
             let docs_in_the_sixties = RangeQuery::new_u64(year_field, 1960u64..1970u64);
 
@@ -335,7 +335,7 @@ mod tests {
         let int_field: Field;
         let schema = {
             let mut schema_builder = Schema::builder();
-            int_field = schema_builder.add_i64_field("intfield", INT_INDEXED);
+            int_field = schema_builder.add_i64_field("intfield", INDEXED);
             schema_builder.build()
         };
 
@@ -355,8 +355,8 @@ mod tests {
 
             index_writer.commit().unwrap();
         }
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        let reader = index.reader().unwrap();
+        let searcher = reader.searcher();
         let count_multiples =
             |range_query: RangeQuery| searcher.search(&range_query, &Count).unwrap();
 

src/query/regex_query.rs

@@ -1,8 +1,8 @@
 use error::TantivyError;
-use fst_regex::Regex;
 use query::{AutomatonWeight, Query, Weight};
 use schema::Field;
 use std::clone::Clone;
+use tantivy_fst::Regex;
 use Result;
 use Searcher;
 
@@ -44,8 +44,8 @@ use Searcher;
 ///         index_writer.commit().unwrap();
 ///     }
 ///
-///     index.load_searchers()?;
-///     let searcher = index.searcher();
+///     let reader = index.reader()?;
+///     let searcher = reader.searcher();
 ///
 ///     let term = Term::from_field_text(title, "Diary");
 ///     let query = RegexQuery::new("d[ai]{2}ry".to_string(), title);
@@ -108,8 +108,8 @@ mod test {
             ));
             index_writer.commit().unwrap();
         }
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        let reader = index.reader().unwrap();
+        let searcher = reader.searcher();
         {
             let regex_query = RegexQuery::new("jap[ao]n".to_string(), country_field);
             let scored_docs = searcher

src/query/term_query/mod.rs

@@ -32,9 +32,7 @@ mod tests {
             }
             assert!(index_writer.commit().is_ok());
         }
-
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        let searcher = index.reader().unwrap().searcher();
         let term_query = TermQuery::new(
             Term::from_field_text(text_field, "a"),
             IndexRecordOption::Basic,
@@ -65,8 +63,7 @@ mod tests {
             index_writer.add_document(doc!(left_field => "left4 left1"));
             index_writer.commit().unwrap();
         }
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        let searcher = index.reader().unwrap().searcher();
         {
             let term = Term::from_field_text(left_field, "left2");
             let term_query = TermQuery::new(term, IndexRecordOption::WithFreqs);
@@ -101,4 +98,20 @@ mod tests {
         }
     }
 
+    #[test]
+    fn test_term_query_count_when_there_are_deletes() {
+        let mut schema_builder = Schema::builder();
+        let text_field = schema_builder.add_text_field("text", TEXT);
+        let schema = schema_builder.build();
+        let index = Index::create_in_ram(schema);
+        let mut index_writer = index.writer_with_num_threads(1, 5_000_000).unwrap();
+        index_writer.add_document(doc!(text_field=>"a b"));
+        index_writer.add_document(doc!(text_field=>"a c"));
+        index_writer.delete_term(Term::from_field_text(text_field, "b"));
+        index_writer.commit().unwrap();
+        let term_a = Term::from_field_text(text_field, "a");
+        let term_query = TermQuery::new(term_a, IndexRecordOption::Basic);
+        let reader = index.reader().unwrap();
+        assert_eq!(term_query.count(&*reader.searcher()).unwrap(), 1);
+    }
 }

src/query/term_query/term_query.rs

@@ -48,9 +48,8 @@ use Term;
 ///         ));
 ///         index_writer.commit()?;
 ///     }
-///
-///     index.load_searchers()?;
-///     let searcher = index.searcher();
+///     let reader = index.reader()?;
+///     let searcher = reader.searcher();
 ///
 ///     let query = TermQuery::new(
 ///         Term::from_field_text(title, "diary"),

src/query/term_query/term_weight.rs

@@ -39,15 +39,15 @@ impl Weight for TermWeight {
     }
 
     fn count(&self, reader: &SegmentReader) -> Result<u32> {
-        if reader.num_deleted_docs() == 0 {
+        if let Some(delete_bitset) = reader.delete_bitset() {
+            Ok(self.scorer(reader)?.count(delete_bitset))
+        } else {
             let field = self.term.field();
             Ok(reader
                 .inverted_index(field)
                 .get_term_info(&self.term)
                 .map(|term_info| term_info.doc_freq)
                 .unwrap_or(0))
-        } else {
-            Ok(self.scorer(reader)?.count())
         }
     }
 }

src/query/union.rs

@@ -145,7 +145,7 @@ where
         }
     }
 
-    fn count(&mut self) -> u32 {
+    fn count_including_deleted(&mut self) -> u32 {
         let mut count = self.bitsets[self.cursor..HORIZON_NUM_TINYBITSETS]
             .iter()
             .map(|bitset| bitset.len())
@@ -163,6 +163,8 @@ where
         count
     }
 
+    // TODO implement `count` efficiently.
+
     fn skip_next(&mut self, target: DocId) -> SkipResult {
         if !self.advance() {
             return SkipResult::End;
@@ -300,7 +302,7 @@ mod tests {
             count += 1;
         }
         assert!(!union_expected.advance());
-        assert_eq!(count, make_union().count());
+        assert_eq!(count, make_union().count_including_deleted());
     }
 
     #[test]

src/query/vec_docset.rs

@@ -5,8 +5,6 @@ use docset::DocSet;
 use std::num::Wrapping;
 use DocId;
 
-const EMPTY_ARRAY: [u32; 0] = [];
-
 /// Simulate a `Postings` objects from a `VecPostings`.
 /// `VecPostings` only exist for testing purposes.
 ///

src/query/weight.rs

@@ -13,6 +13,11 @@ pub trait Weight: Send + Sync + 'static {
 
     /// Returns the number documents within the given `SegmentReader`.
     fn count(&self, reader: &SegmentReader) -> Result<u32> {
-        Ok(self.scorer(reader)?.count())
+        let mut scorer = self.scorer(reader)?;
+        if let Some(delete_bitset) = reader.delete_bitset() {
+            Ok(scorer.count(delete_bitset))
+        } else {
+            Ok(scorer.count_including_deleted())
+        }
     }
 }

src/reader/mod.rs

@@ -0,0 +1,187 @@
+mod pool;
+
+use self::pool::{LeasedItem, Pool};
+use core::Segment;
+use directory::Directory;
+use directory::WatchHandle;
+use directory::META_LOCK;
+use std::sync::Arc;
+use Index;
+use Result;
+use Searcher;
+use SegmentReader;
+
+/// Defines when a new version of the index should be reloaded.
+///
+/// Regardless of whether you search and index in the same process, tantivy does not necessarily
+/// reflects the change that are commited to your index. `ReloadPolicy` precisely helps you define
+/// when you want your index to be reloaded.
+#[derive(Clone, Copy)]
+pub enum ReloadPolicy {
+    /// The index is entirely reloaded manually.
+    /// All updates of the index should be manual.
+    ///
+    /// No change is reflected automatically. You are required to call `.load_seacher()` manually.
+    Manual,
+    /// The index is reloaded within milliseconds after a new commit is available.
+    /// This is made possible by watching changes in the `meta.json` file.
+    OnCommit, // TODO add NEAR_REAL_TIME(target_ms)
+}
+
+/// `IndexReader` builder
+///
+/// It makes it possible to set the following values.
+///
+/// - `num_searchers` (by default, the number of detected CPU threads):
+///
+///   When `num_searchers` queries are requested at the same time, the `num_searchers` will block
+///   until the one of the searcher in-use gets released.
+/// - `reload_policy` (by default `ReloadPolicy::OnCommit`):
+///
+///   See [`ReloadPolicy`](./enum.ReloadPolicy.html) for more details.
+#[derive(Clone)]
+pub struct IndexReaderBuilder {
+    num_searchers: usize,
+    reload_policy: ReloadPolicy,
+    index: Index,
+}
+
+impl IndexReaderBuilder {
+    pub(crate) fn new(index: Index) -> IndexReaderBuilder {
+        IndexReaderBuilder {
+            num_searchers: num_cpus::get(),
+            reload_policy: ReloadPolicy::OnCommit,
+            index,
+        }
+    }
+
+    /// Builds the reader.
+    ///
+    /// Building the reader is a non-trivial operation that requires
+    /// to open different segment readers. It may take hundreds of milliseconds
+    /// of time and it may return an error.
+    /// TODO(pmasurel) Use the `TryInto` trait once it is available in stable.
+    pub fn try_into(self) -> Result<IndexReader> {
+        let inner_reader = InnerIndexReader {
+            index: self.index,
+            num_searchers: self.num_searchers,
+            searcher_pool: Pool::new(),
+        };
+        inner_reader.reload()?;
+        let inner_reader_arc = Arc::new(inner_reader);
+        let watch_handle_opt: Option<WatchHandle>;
+        match self.reload_policy {
+            ReloadPolicy::Manual => {
+                // No need to set anything...
+                watch_handle_opt = None;
+            }
+            ReloadPolicy::OnCommit => {
+                let inner_reader_arc_clone = inner_reader_arc.clone();
+                let callback = move || {
+                    if let Err(err) = inner_reader_arc_clone.reload() {
+                        error!(
+                            "Error while loading searcher after commit was detected. {:?}",
+                            err
+                        );
+                    }
+                };
+                let watch_handle = inner_reader_arc.index.directory().watch(Box::new(callback));
+                watch_handle_opt = Some(watch_handle);
+            }
+        }
+        Ok(IndexReader {
+            inner: inner_reader_arc,
+            watch_handle_opt,
+        })
+    }
+
+    /// Sets the reload_policy.
+    ///
+    /// See [`ReloadPolicy`](./enum.ReloadPolicy.html) for more details.
+    pub fn reload_policy(mut self, reload_policy: ReloadPolicy) -> IndexReaderBuilder {
+        self.reload_policy = reload_policy;
+        self
+    }
+
+    /// Sets the number of `Searcher` in the searcher pool.
+    pub fn num_searchers(mut self, num_searchers: usize) -> IndexReaderBuilder {
+        self.num_searchers = num_searchers;
+        self
+    }
+}
+
+struct InnerIndexReader {
+    num_searchers: usize,
+    searcher_pool: Pool<Searcher>,
+    index: Index,
+}
+
+impl InnerIndexReader {
+    fn reload(&self) -> Result<()> {
+        let segment_readers: Vec<SegmentReader> = {
+            let _meta_lock = self.index.directory().acquire_lock(&META_LOCK)?;
+            let searchable_segments = self.searchable_segments()?;
+            searchable_segments
+                .iter()
+                .map(SegmentReader::open)
+                .collect::<Result<_>>()?
+        };
+        let schema = self.index.schema();
+        let searchers = (0..self.num_searchers)
+            .map(|_| Searcher::new(schema.clone(), self.index.clone(), segment_readers.clone()))
+            .collect();
+        self.searcher_pool.publish_new_generation(searchers);
+        Ok(())
+    }
+
+    /// Returns the list of segments that are searchable
+    fn searchable_segments(&self) -> Result<Vec<Segment>> {
+        self.index.searchable_segments()
+    }
+
+    fn searcher(&self) -> LeasedItem<Searcher> {
+        self.searcher_pool.acquire()
+    }
+}
+
+/// `IndexReader` is your entry point to read and search the index.
+///
+/// It controls when a new version of the index should be loaded and lends
+/// you instances of `Searcher` for the last loaded version.
+///
+/// `Clone` does not clone the different pool of searcher. `IndexReader`
+/// just wraps and `Arc`.
+#[derive(Clone)]
+pub struct IndexReader {
+    inner: Arc<InnerIndexReader>,
+    watch_handle_opt: Option<WatchHandle>,
+}
+
+impl IndexReader {
+    /// Update searchers so that they reflect the state of the last
+    /// `.commit()`.
+    ///
+    /// If you set up the `OnCommit` `ReloadPolicy` (which is the default)
+    /// every commit should be rapidly reflected on your `IndexReader` and you should
+    /// not need to call `reload()` at all.
+    ///
+    /// This automatic reload can take 10s of milliseconds to kick in however, and in unit tests
+    /// it can be nice to deterministically force the reload of searchers.
+    pub fn reload(&self) -> Result<()> {
+        self.inner.reload()
+    }
+
+    /// Returns a searcher
+    ///
+    /// This method should be called every single time a search
+    /// query is performed.
+    /// The searchers are taken from a pool of `num_searchers` searchers.
+    /// If no searcher is available
+    /// this may block.
+    ///
+    /// The same searcher must be used for a given query, as it ensures
+    /// the use of a consistent segment set.
+    pub fn searcher(&self) -> LeasedItem<Searcher> {
+        self.inner.searcher()
+    }
+}

src/reader/pool.rs

@@ -1,4 +1,5 @@
-use crossbeam::queue::MsQueue;
+use crossbeam::crossbeam_channel::unbounded;
+use crossbeam::{Receiver, RecvError, Sender};
 use std::ops::{Deref, DerefMut};
 use std::sync::atomic::AtomicUsize;
 use std::sync::atomic::Ordering;
@@ -9,6 +10,37 @@ pub struct GenerationItem<T> {
     item: T,
 }
 
+/// Queue implementation for the Object Pool below
+/// Uses the unbounded Linked-List type queue from crossbeam-channel
+/// Splits the Queue into sender and receiver
+struct Queue<T> {
+    sender: Sender<T>,
+    receiver: Receiver<T>,
+}
+
+impl<T> Queue<T> {
+    fn new() -> Self {
+        let (s, r) = unbounded();
+        Queue {
+            sender: s,
+            receiver: r,
+        }
+    }
+
+    /// Sender trait returns a Result type, which is ignored.
+    /// The Result is not handled at the moment
+    fn push(&self, elem: T) {
+        self.sender
+            .send(elem)
+            .expect("Sending an item to crossbeam-queue shouldn't fail");
+    }
+
+    /// Relies on the underlying crossbeam-channel Receiver
+    /// to block on empty queue
+    fn pop(&self) -> Result<T, RecvError> {
+        self.receiver.recv()
+    }
+}
 
 /// An object pool
 ///
@@ -16,14 +48,14 @@ pub struct GenerationItem<T> {
 /// Object are wrapped in a `LeasedItem` wrapper and are
 /// released automatically back into the pool on `Drop`.
 pub struct Pool<T> {
-    queue: Arc<MsQueue<GenerationItem<T>>>,
+    queue: Arc<Queue<GenerationItem<T>>>,
     freshest_generation: AtomicUsize,
     next_generation: AtomicUsize,
 }
 
 impl<T> Pool<T> {
     pub fn new() -> Pool<T> {
-        let queue = Arc::new(MsQueue::new());
+        let queue = Arc::new(Queue::new());
         Pool {
             queue,
             freshest_generation: AtomicUsize::default(),
@@ -77,7 +109,7 @@ impl<T> Pool<T> {
     pub fn acquire(&self) -> LeasedItem<T> {
         let generation = self.generation();
         loop {
-            let gen_item = self.queue.pop();
+            let gen_item = self.queue.pop().unwrap();
             if gen_item.generation >= generation {
                 return LeasedItem {
                     gen_item: Some(gen_item),
@@ -93,7 +125,7 @@ impl<T> Pool<T> {
 
 pub struct LeasedItem<T> {
     gen_item: Option<GenerationItem<T>>,
-    recycle_queue: Arc<MsQueue<GenerationItem<T>>>,
+    recycle_queue: Arc<Queue<GenerationItem<T>>>,
 }
 
 impl<T> Deref for LeasedItem<T> {
@@ -130,6 +162,7 @@ impl<T> Drop for LeasedItem<T> {
 mod tests {
 
     use super::Pool;
+    use super::Queue;
     use std::iter;
 
     #[test]
@@ -146,4 +179,47 @@ mod tests {
             assert_eq!(*pool.acquire(), 11);
         }
     }
+
+    #[test]
+    fn test_queue() {
+        let q = Queue::new();
+        let elem = 5;
+        q.push(elem);
+        let res = q.pop();
+        assert_eq!(res.unwrap(), elem);
+    }
+
+    #[test]
+    fn test_pool_dont_panic_on_empty_pop() {
+        // When the object pool is exhausted, it shouldn't panic on pop()
+        use std::sync::Arc;
+        use std::{thread, time};
+
+        // Wrap the pool in an Arc, same way as its used in `core/index.rs`
+        let pool = Arc::new(Pool::new());
+        // clone pools outside the move scope of each new thread
+        let pool1 = Arc::clone(&pool);
+        let pool2 = Arc::clone(&pool);
+        let elements_for_pool = vec![1, 2];
+        pool.publish_new_generation(elements_for_pool);
+
+        let mut threads = vec![];
+        let sleep_dur = time::Duration::from_millis(10);
+        // spawn one more thread than there are elements in the pool
+        threads.push(thread::spawn(move || {
+            // leasing to make sure it's not dropped before sleep is called
+            let _leased_searcher = &pool.acquire();
+            thread::sleep(sleep_dur);
+        }));
+        threads.push(thread::spawn(move || {
+            // leasing to make sure it's not dropped before sleep is called
+            let _leased_searcher = &pool1.acquire();
+            thread::sleep(sleep_dur);
+        }));
+        threads.push(thread::spawn(move || {
+            // leasing to make sure it's not dropped before sleep is called
+            let _leased_searcher = &pool2.acquire();
+            thread::sleep(sleep_dur);
+        }));
+    }
 }

src/schema/document.rs

@@ -3,6 +3,7 @@ use common::BinarySerializable;
 use common::VInt;
 use itertools::Itertools;
 use std::io::{self, Read, Write};
+use DateTime;
 
 /// Tantivy's Document is the object that can
 /// be indexed and then searched for.
@@ -82,11 +83,16 @@ impl Document {
         self.add(FieldValue::new(field, Value::U64(value)));
     }
 
-    /// Add a u64 field
+    /// Add a i64 field
     pub fn add_i64(&mut self, field: Field, value: i64) {
         self.add(FieldValue::new(field, Value::I64(value)));
     }
 
+    /// Add a date field
+    pub fn add_date(&mut self, field: Field, value: &DateTime) {
+        self.add(FieldValue::new(field, Value::Date(*value)));
+    }
+
     /// Add a bytes field
     pub fn add_bytes(&mut self, field: Field, value: Vec<u8>) {
         self.add(FieldValue::new(field, Value::Bytes(value)))

src/schema/field_entry.rs

@@ -48,6 +48,15 @@ impl FieldEntry {
         }
     }
 
+    /// Creates a new date field entry in the schema, given
+    /// a name, and some options.
+    pub fn new_date(field_name: String, field_type: IntOptions) -> FieldEntry {
+        FieldEntry {
+            name: field_name,
+            field_type: FieldType::Date(field_type),
+        }
+    }
+
     /// Creates a field entry for a facet.
     pub fn new_facet(field_name: String) -> FieldEntry {
         FieldEntry {
@@ -78,7 +87,9 @@ impl FieldEntry {
     pub fn is_indexed(&self) -> bool {
         match self.field_type {
             FieldType::Str(ref options) => options.get_indexing_options().is_some(),
-            FieldType::U64(ref options) | FieldType::I64(ref options) => options.is_indexed(),
+            FieldType::U64(ref options)
+            | FieldType::I64(ref options)
+            | FieldType::Date(ref options) => options.is_indexed(),
             FieldType::HierarchicalFacet => true,
             FieldType::Bytes => false,
         }
@@ -95,7 +106,9 @@ impl FieldEntry {
     /// Returns true iff the field is stored
     pub fn is_stored(&self) -> bool {
         match self.field_type {
-            FieldType::U64(ref options) | FieldType::I64(ref options) => options.is_stored(),
+            FieldType::U64(ref options)
+            | FieldType::I64(ref options)
+            | FieldType::Date(ref options) => options.is_stored(),
             FieldType::Str(ref options) => options.is_stored(),
             // TODO make stored hierarchical facet optional
             FieldType::HierarchicalFacet => true,
@@ -125,6 +138,10 @@ impl Serialize for FieldEntry {
                 s.serialize_field("type", "i64")?;
                 s.serialize_field("options", options)?;
             }
+            FieldType::Date(ref options) => {
+                s.serialize_field("type", "date")?;
+                s.serialize_field("options", options)?;
+            }
             FieldType::HierarchicalFacet => {
                 s.serialize_field("type", "hierarchical_facet")?;
             }
@@ -188,7 +205,7 @@ impl<'de> Deserialize<'de> for FieldEntry {
                                 "bytes" => {
                                     field_type = Some(FieldType::Bytes);
                                 }
-                                "text" | "u64" | "i64" => {
+                                "text" | "u64" | "i64" | "date" => {
                                     // These types require additional options to create a field_type
                                 }
                                 _ => panic!("unhandled type"),
@@ -205,6 +222,7 @@ impl<'de> Deserialize<'de> for FieldEntry {
                                 "text" => field_type = Some(FieldType::Str(map.next_value()?)),
                                 "u64" => field_type = Some(FieldType::U64(map.next_value()?)),
                                 "i64" => field_type = Some(FieldType::I64(map.next_value()?)),
+                                "date" => field_type = Some(FieldType::Date(map.next_value()?)),
                                 _ => {
                                     let msg = format!("Unrecognised type {}", ty);
                                     return Err(de::Error::custom(msg));

src/schema/field_type.rs

@@ -34,6 +34,8 @@ pub enum Type {
     U64,
     /// `i64`
     I64,
+    /// `date(i64) timestamp`
+    Date,
     /// `tantivy::schema::Facet`. Passed as a string in JSON.
     HierarchicalFacet,
     /// `Vec<u8>`
@@ -50,6 +52,8 @@ pub enum FieldType {
     U64(IntOptions),
     /// Signed 64-bits integers 64 field type configuration
     I64(IntOptions),
+    /// Signed 64-bits Date 64 field type configuration,
+    Date(IntOptions),
     /// Hierachical Facet
     HierarchicalFacet,
     /// Bytes (one per document)
@@ -63,6 +67,7 @@ impl FieldType {
             FieldType::Str(_) => Type::Str,
             FieldType::U64(_) => Type::U64,
             FieldType::I64(_) => Type::I64,
+            FieldType::Date(_) => Type::Date,
             FieldType::HierarchicalFacet => Type::HierarchicalFacet,
             FieldType::Bytes => Type::Bytes,
         }
@@ -75,6 +80,7 @@ impl FieldType {
             FieldType::U64(ref int_options) | FieldType::I64(ref int_options) => {
                 int_options.is_indexed()
             }
+            FieldType::Date(ref date_options) => date_options.is_indexed(),
             FieldType::HierarchicalFacet => true,
             FieldType::Bytes => false,
         }
@@ -89,7 +95,9 @@ impl FieldType {
             FieldType::Str(ref text_options) => text_options
                 .get_indexing_options()
                 .map(|indexing_options| indexing_options.index_option()),
-            FieldType::U64(ref int_options) | FieldType::I64(ref int_options) => {
+            FieldType::U64(ref int_options)
+            | FieldType::I64(ref int_options)
+            | FieldType::Date(ref int_options) => {
                 if int_options.is_indexed() {
                     Some(IndexRecordOption::Basic)
                 } else {
@@ -110,9 +118,9 @@ impl FieldType {
         match *json {
             JsonValue::String(ref field_text) => match *self {
                 FieldType::Str(_) => Ok(Value::Str(field_text.clone())),
-                FieldType::U64(_) | FieldType::I64(_) => Err(ValueParsingError::TypeError(
-                    format!("Expected an integer, got {:?}", json),
-                )),
+                FieldType::U64(_) | FieldType::I64(_) | FieldType::Date(_) => Err(
+                    ValueParsingError::TypeError(format!("Expected an integer, got {:?}", json)),
+                ),
                 FieldType::HierarchicalFacet => Ok(Value::Facet(Facet::from(field_text))),
                 FieldType::Bytes => decode(field_text).map(Value::Bytes).map_err(|_| {
                     ValueParsingError::InvalidBase64(format!(
@@ -122,7 +130,7 @@ impl FieldType {
                 }),
             },
             JsonValue::Number(ref field_val_num) => match *self {
-                FieldType::I64(_) => {
+                FieldType::I64(_) | FieldType::Date(_) => {
                     if let Some(field_val_i64) = field_val_num.as_i64() {
                         Ok(Value::I64(field_val_i64))
                     } else {

src/schema/flags.rs

@@ -0,0 +1,81 @@
+use schema::IntOptions;
+use schema::TextOptions;
+use std::ops::BitOr;
+
+#[derive(Clone)]
+pub struct StoredFlag;
+/// Flag to mark the field as stored.
+/// This flag can apply to any kind of field.
+///
+/// A stored fields of a document can be retrieved given its `DocId`.
+/// Stored field are stored together and LZ4 compressed.
+/// Reading the stored fields of a document is relatively slow.
+/// (~ 100 microsecs)
+///
+/// It should not be used during scoring or collection.
+pub const STORED: SchemaFlagList<StoredFlag, ()> = SchemaFlagList {
+    head: StoredFlag,
+    tail: (),
+};
+
+#[derive(Clone)]
+pub struct IndexedFlag;
+/// Flag to mark the field as indexed.
+///
+/// The `INDEXED` flag can only be used when building `IntOptions` (`u64` and `i64` fields)
+/// Of course, text fields can also be indexed... But this is expressed by using either the
+/// `STRING` (untokenized) or `TEXT` (tokenized with the english tokenizer) flags.
+pub const INDEXED: SchemaFlagList<IndexedFlag, ()> = SchemaFlagList {
+    head: IndexedFlag,
+    tail: (),
+};
+
+#[derive(Clone)]
+pub struct FastFlag;
+/// Flag to mark the field as a fast field (similar to Lucene's DocValues)
+///
+/// Fast fields can be random-accessed rapidly. Fields useful for scoring, filtering
+/// or collection should be mark as fast fields.
+/// The `FAST` flag can only be used when building `IntOptions` (`u64` and `i64` fields)
+pub const FAST: SchemaFlagList<FastFlag, ()> = SchemaFlagList {
+    head: FastFlag,
+    tail: (),
+};
+
+impl<Head, OldHead, OldTail> BitOr<SchemaFlagList<Head, ()>> for SchemaFlagList<OldHead, OldTail>
+where
+    Head: Clone,
+    OldHead: Clone,
+    OldTail: Clone,
+{
+    type Output = SchemaFlagList<Head, SchemaFlagList<OldHead, OldTail>>;
+
+    fn bitor(self, head: SchemaFlagList<Head, ()>) -> Self::Output {
+        SchemaFlagList {
+            head: head.head,
+            tail: self.clone(),
+        }
+    }
+}
+
+impl<T: Clone + Into<IntOptions>> BitOr<IntOptions> for SchemaFlagList<T, ()> {
+    type Output = IntOptions;
+
+    fn bitor(self, rhs: IntOptions) -> Self::Output {
+        self.head.into() | rhs
+    }
+}
+
+impl<T: Clone + Into<TextOptions>> BitOr<TextOptions> for SchemaFlagList<T, ()> {
+    type Output = TextOptions;
+
+    fn bitor(self, rhs: TextOptions) -> Self::Output {
+        self.head.into() | rhs
+    }
+}
+
+#[derive(Clone)]
+pub struct SchemaFlagList<Head: Clone, Tail: Clone> {
+    pub head: Head,
+    pub tail: Tail,
+}

src/schema/int_options.rs

@@ -1,3 +1,4 @@
+use schema::flags::{FastFlag, IndexedFlag, SchemaFlagList, StoredFlag};
 use std::ops::BitOr;
 
 /// Express whether a field is single-value or multi-valued.
@@ -85,41 +86,62 @@ impl Default for IntOptions {
     }
 }
 
-/// Shortcut for a u64 fast field.
-///
-/// Such a shortcut can be composed as follows `STORED | FAST | INT_INDEXED`
-pub const FAST: IntOptions = IntOptions {
-    indexed: false,
-    stored: false,
-    fast: Some(Cardinality::SingleValue),
-};
+impl From<()> for IntOptions {
+    fn from(_: ()) -> IntOptions {
+        IntOptions::default()
+    }
+}
 
-/// Shortcut for a u64 indexed field.
-///
-/// Such a shortcut can be composed as follows `STORED | FAST | INT_INDEXED`
-pub const INT_INDEXED: IntOptions = IntOptions {
-    indexed: true,
-    stored: false,
-    fast: None,
-};
+impl From<FastFlag> for IntOptions {
+    fn from(_: FastFlag) -> Self {
+        IntOptions {
+            indexed: false,
+            stored: false,
+            fast: Some(Cardinality::SingleValue),
+        }
+    }
+}
 
-/// Shortcut for a u64 stored field.
-///
-/// Such a shortcut can be composed as follows `STORED | FAST | INT_INDEXED`
-pub const INT_STORED: IntOptions = IntOptions {
-    indexed: false,
-    stored: true,
-    fast: None,
-};
+impl From<StoredFlag> for IntOptions {
+    fn from(_: StoredFlag) -> Self {
+        IntOptions {
+            indexed: false,
+            stored: true,
+            fast: None,
+        }
+    }
+}
 
-impl BitOr for IntOptions {
+impl From<IndexedFlag> for IntOptions {
+    fn from(_: IndexedFlag) -> Self {
+        IntOptions {
+            indexed: true,
+            stored: false,
+            fast: None,
+        }
+    }
+}
+
+impl<T: Into<IntOptions>> BitOr<T> for IntOptions {
     type Output = IntOptions;
 
-    fn bitor(self, other: IntOptions) -> IntOptions {
+    fn bitor(self, other: T) -> IntOptions {
         let mut res = IntOptions::default();
+        let other = other.into();
         res.indexed = self.indexed | other.indexed;
         res.stored = self.stored | other.stored;
         res.fast = self.fast.or(other.fast);
         res
     }
 }
+
+impl<Head, Tail> From<SchemaFlagList<Head, Tail>> for IntOptions
+where
+    Head: Clone,
+    Tail: Clone,
+    Self: BitOr<Output = Self> + From<Head> + From<Tail>,
+{
+    fn from(head_tail: SchemaFlagList<Head, Tail>) -> Self {
+        Self::from(head_tail.head) | Self::from(head_tail.tail)
+    }
+}

src/schema/mod.rs

@@ -33,7 +33,7 @@ let title_options = TextOptions::default()
     .set_indexing_options(TextFieldIndexing::default()
         .set_tokenizer("default")
         .set_index_option(IndexRecordOption::WithFreqsAndPositions));
-schema_builder.add_text_field("title_options", title_options);
+schema_builder.add_text_field("title", title_options);
 let schema = schema_builder.build();
 ```
 
@@ -53,23 +53,8 @@ The effect of each possible setting is described more in detail
 On the other hand setting the field as stored or not determines whether the field should be returned
 when [`searcher.doc(doc_address)`](../struct.Searcher.html#method.doc) is called.
 
-### Shortcuts
 
-For convenience, a few special values of `TextOptions`.
-They can be composed using the `|` operator.
-The example can be rewritten :
-
-
-```
-use tantivy::schema::*;
-let mut schema_builder = Schema::builder();
-schema_builder.add_text_field("title_options", TEXT | STORED);
-let schema = schema_builder.build();
-```
-
-
-
-## Setting a u64 field
+## Setting a u64 or a i64 field
 
 ### Example
 
@@ -98,6 +83,23 @@ u64 that are indexed as fast will be stored in a special data structure that wil
 make it possible to access the u64 value given the doc id rapidly. This is useful if the value of
 the field is required during scoring or collection for instance.
 
+
+### Shortcuts
+
+
+For convenience, it is possible to define your field indexing options by combining different flags
+using the  `|` operator.
+
+For instance, a schema containing the two fields defined in the example above could be rewritten :
+
+```
+use tantivy::schema::*;
+let mut schema_builder = Schema::builder();
+schema_builder.add_u64_field("num_stars", INDEXED | STORED);
+schema_builder.add_text_field("title", TEXT | STORED);
+let schema = schema_builder.build();
+```
+
 */
 
 mod document;
@@ -116,13 +118,15 @@ mod named_field_document;
 mod text_options;
 mod value;
 
+mod flags;
+
 pub use self::named_field_document::NamedFieldDocument;
 pub use self::schema::DocParsingError;
 pub use self::schema::{Schema, SchemaBuilder};
 pub use self::value::Value;
 
 pub use self::facet::Facet;
-pub use self::facet::FACET_SEP_BYTE;
+pub(crate) use self::facet::FACET_SEP_BYTE;
 
 pub use self::document::Document;
 pub use self::field::Field;
@@ -135,15 +139,12 @@ pub use self::field_value::FieldValue;
 pub use self::index_record_option::IndexRecordOption;
 pub use self::text_options::TextFieldIndexing;
 pub use self::text_options::TextOptions;
-pub use self::text_options::STORED;
 pub use self::text_options::STRING;
 pub use self::text_options::TEXT;
 
+pub use self::flags::{FAST, INDEXED, STORED};
 pub use self::int_options::Cardinality;
 pub use self::int_options::IntOptions;
-pub use self::int_options::FAST;
-pub use self::int_options::INT_INDEXED;
-pub use self::int_options::INT_STORED;
 
 use regex::Regex;
 

src/schema/schema.rs

@@ -52,9 +52,13 @@ impl SchemaBuilder {
     /// by the second one.
     /// The first field will get a field id
     /// but only the second one will be indexed
-    pub fn add_u64_field(&mut self, field_name_str: &str, field_options: IntOptions) -> Field {
+    pub fn add_u64_field<T: Into<IntOptions>>(
+        &mut self,
+        field_name_str: &str,
+        field_options: T,
+    ) -> Field {
         let field_name = String::from(field_name_str);
-        let field_entry = FieldEntry::new_u64(field_name, field_options);
+        let field_entry = FieldEntry::new_u64(field_name, field_options.into());
         self.add_field(field_entry)
     }
 
@@ -68,9 +72,35 @@ impl SchemaBuilder {
     /// by the second one.
     /// The first field will get a field id
     /// but only the second one will be indexed
-    pub fn add_i64_field(&mut self, field_name_str: &str, field_options: IntOptions) -> Field {
+    pub fn add_i64_field<T: Into<IntOptions>>(
+        &mut self,
+        field_name_str: &str,
+        field_options: T,
+    ) -> Field {
         let field_name = String::from(field_name_str);
-        let field_entry = FieldEntry::new_i64(field_name, field_options);
+        let field_entry = FieldEntry::new_i64(field_name, field_options.into());
+        self.add_field(field_entry)
+    }
+
+    /// Adds a new date field.
+    /// Returns the associated field handle
+    /// Internally, Tantivy simply stores dates as i64 UTC timestamps,
+    /// while the user supplies DateTime values for convenience.
+    ///
+    /// # Caution
+    ///
+    /// Appending two fields with the same name
+    /// will result in the shadowing of the first
+    /// by the second one.
+    /// The first field will get a field id
+    /// but only the second one will be indexed
+    pub fn add_date_field<T: Into<IntOptions>>(
+        &mut self,
+        field_name_str: &str,
+        field_options: T,
+    ) -> Field {
+        let field_name = String::from(field_name_str);
+        let field_entry = FieldEntry::new_date(field_name, field_options.into());
         self.add_field(field_entry)
     }
 
@@ -84,9 +114,13 @@ impl SchemaBuilder {
     /// by the second one.
     /// The first field will get a field id
     /// but only the second one will be indexed
-    pub fn add_text_field(&mut self, field_name_str: &str, field_options: TextOptions) -> Field {
+    pub fn add_text_field<T: Into<TextOptions>>(
+        &mut self,
+        field_name_str: &str,
+        field_options: T,
+    ) -> Field {
         let field_name = String::from(field_name_str);
-        let field_entry = FieldEntry::new_text(field_name, field_options);
+        let field_entry = FieldEntry::new_text(field_name, field_options.into());
         self.add_field(field_entry)
     }
 
@@ -178,15 +212,7 @@ impl Schema {
         SchemaBuilder::default()
     }
 
-    /// Returns the field options associated with a given name.
-    ///
-    /// # Panics
-    /// Panics if the field name does not exist.
-    /// It is meant as an helper for user who created
-    /// and control the content of their schema.
-    ///
-    /// If panicking is not an option for you,
-    /// you may use `get(&self, field_name: &str)`.
+    /// Returns the field option associated with a given name.
     pub fn get_field(&self, field_name: &str) -> Option<Field> {
         self.0.fields_map.get(field_name).cloned()
     }

src/schema/term.rs

@@ -5,6 +5,7 @@ use byteorder::{BigEndian, ByteOrder};
 use common;
 use schema::Facet;
 use std::str;
+use DateTime;
 
 /// Size (in bytes) of the buffer of a int field.
 const INT_TERM_LEN: usize = 4 + 8;
@@ -30,6 +31,18 @@ impl Term {
         Term::from_field_u64(field, val_u64)
     }
 
+    /// Builds a term given a field, and a DateTime value
+    ///
+    /// Assuming the term has a field id of 1, and a timestamp i64 value of 3234,
+    /// the Term will have 8 bytes.
+    ///
+    /// The first four byte are dedicated to storing the field id as a u64.
+    /// The 4 following bytes are encoding the DateTime as i64 timestamp value.
+    pub fn from_field_date(field: Field, val: &DateTime) -> Term {
+        let val_timestamp = val.timestamp();
+        Term::from_field_i64(field, val_timestamp)
+    }
+
     /// Creates a `Term` given a facet.
     pub fn from_facet(field: Field, facet: &Facet) -> Term {
         let bytes = facet.encoded_str().as_bytes();

src/schema/text_options.rs

@@ -1,3 +1,5 @@
+use schema::flags::SchemaFlagList;
+use schema::flags::StoredFlag;
 use schema::IndexRecordOption;
 use std::borrow::Cow;
 use std::ops::BitOr;
@@ -109,19 +111,11 @@ pub const TEXT: TextOptions = TextOptions {
     stored: false,
 };
 
-/// A stored fields of a document can be retrieved given its `DocId`.
-/// Stored field are stored together and LZ4 compressed.
-/// Reading the stored fields of a document is relatively slow.
-/// (100 microsecs)
-pub const STORED: TextOptions = TextOptions {
-    indexing: None,
-    stored: true,
-};
-
-impl BitOr for TextOptions {
+impl<T: Into<TextOptions>> BitOr<T> for TextOptions {
     type Output = TextOptions;
 
-    fn bitor(self, other: TextOptions) -> TextOptions {
+    fn bitor(self, other: T) -> TextOptions {
+        let other = other.into();
         let mut res = TextOptions::default();
         res.indexing = self.indexing.or(other.indexing);
         res.stored = self.stored | other.stored;
@@ -129,6 +123,32 @@ impl BitOr for TextOptions {
     }
 }
 
+impl From<()> for TextOptions {
+    fn from(_: ()) -> TextOptions {
+        TextOptions::default()
+    }
+}
+
+impl From<StoredFlag> for TextOptions {
+    fn from(_: StoredFlag) -> TextOptions {
+        TextOptions {
+            indexing: None,
+            stored: true,
+        }
+    }
+}
+
+impl<Head, Tail> From<SchemaFlagList<Head, Tail>> for TextOptions
+where
+    Head: Clone,
+    Tail: Clone,
+    Self: BitOr<Output = Self> + From<Head> + From<Tail>,
+{
+    fn from(head_tail: SchemaFlagList<Head, Tail>) -> Self {
+        Self::from(head_tail.head) | Self::from(head_tail.tail)
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use schema::*;

src/schema/value.rs

@@ -2,6 +2,7 @@ use schema::Facet;
 use serde::de::Visitor;
 use serde::{Deserialize, Deserializer, Serialize, Serializer};
 use std::fmt;
+use DateTime;
 
 /// Value represents the value of a any field.
 /// It is an enum over all over all of the possible field type.
@@ -13,6 +14,8 @@ pub enum Value {
     U64(u64),
     /// Signed 64-bits Integer `i64`
     I64(i64),
+    /// Signed 64-bits Date time stamp `date`
+    Date(DateTime),
     /// Hierarchical Facet
     Facet(Facet),
     /// Arbitrarily sized byte array
@@ -28,6 +31,7 @@ impl Serialize for Value {
             Value::Str(ref v) => serializer.serialize_str(v),
             Value::U64(u) => serializer.serialize_u64(u),
             Value::I64(u) => serializer.serialize_i64(u),
+            Value::Date(ref date) => serializer.serialize_i64(date.timestamp()),
             Value::Facet(ref facet) => facet.serialize(serializer),
             Value::Bytes(ref bytes) => serializer.serialize_bytes(bytes),
         }
@@ -102,6 +106,17 @@ impl Value {
             _ => panic!("This is not a text field."),
         }
     }
+
+    /// Returns the Date-value, provided the value is of the `Date` type.
+    ///
+    /// # Panics
+    /// If the value is not of type `Date`
+    pub fn date_value(&self) -> &DateTime {
+        match *self {
+            Value::Date(ref value) => value,
+            _ => panic!("This is not a date field."),
+        }
+    }
 }
 
 impl From<String> for Value {
@@ -122,6 +137,12 @@ impl From<i64> for Value {
     }
 }
 
+impl From<DateTime> for Value {
+    fn from(date_time: DateTime) -> Value {
+        Value::Date(date_time)
+    }
+}
+
 impl<'a> From<&'a str> for Value {
     fn from(s: &'a str) -> Value {
         Value::Str(s.to_string())
@@ -142,6 +163,7 @@ impl From<Vec<u8>> for Value {
 
 mod binary_serialize {
     use super::Value;
+    use chrono::{TimeZone, Utc};
     use common::BinarySerializable;
     use schema::Facet;
     use std::io::{self, Read, Write};
@@ -151,6 +173,7 @@ mod binary_serialize {
     const I64_CODE: u8 = 2;
     const HIERARCHICAL_FACET_CODE: u8 = 3;
     const BYTES_CODE: u8 = 4;
+    const DATE_CODE: u8 = 5;
 
     impl BinarySerializable for Value {
         fn serialize<W: Write>(&self, writer: &mut W) -> io::Result<()> {
@@ -167,6 +190,10 @@ mod binary_serialize {
                     I64_CODE.serialize(writer)?;
                     val.serialize(writer)
                 }
+                Value::Date(ref val) => {
+                    DATE_CODE.serialize(writer)?;
+                    val.timestamp().serialize(writer)
+                }
                 Value::Facet(ref facet) => {
                     HIERARCHICAL_FACET_CODE.serialize(writer)?;
                     facet.serialize(writer)
@@ -192,6 +219,10 @@ mod binary_serialize {
                     let value = i64::deserialize(reader)?;
                     Ok(Value::I64(value))
                 }
+                DATE_CODE => {
+                    let timestamp = i64::deserialize(reader)?;
+                    Ok(Value::Date(Utc.timestamp(timestamp, 0)))
+                }
                 HIERARCHICAL_FACET_CODE => Ok(Value::Facet(Facet::deserialize(reader)?)),
                 BYTES_CODE => Ok(Value::Bytes(Vec::<u8>::deserialize(reader)?)),
                 _ => Err(io::Error::new(

src/snippet/mod.rs

@@ -241,8 +241,8 @@ fn select_best_fragment_combination(fragments: &[FragmentCandidate], text: &str)
 /// #    let query_parser = QueryParser::for_index(&index, vec![text_field]);
 /// // ...
 /// let query = query_parser.parse_query("haleurs flamands").unwrap();
-/// # index.load_searchers()?;
-/// # let searcher = index.searcher();
+/// # let reader = index.reader()?;
+/// # let searcher = reader.searcher();
 /// let mut snippet_generator = SnippetGenerator::create(&searcher, &*query, text_field)?;
 /// snippet_generator.set_max_num_chars(100);
 /// let snippet = snippet_generator.snippet_from_doc(&doc);
@@ -528,9 +528,8 @@ Survey in 2016, 2017, and 2018."#;
             index_writer.add_document(doc!(text_field => "a"));
             index_writer.add_document(doc!(text_field => "a b"));
             index_writer.commit().unwrap();
-            index.load_searchers().unwrap();
         }
-        let searcher = index.searcher();
+        let searcher = index.reader().unwrap().searcher();
         let query_parser = QueryParser::for_index(&index, vec![text_field]);
         {
             let query = query_parser.parse_query("e").unwrap();
@@ -587,8 +586,7 @@ Survey in 2016, 2017, and 2018."#;
             }
             index_writer.commit().unwrap();
         }
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        let searcher = index.reader().unwrap().searcher();
         let query_parser = QueryParser::for_index(&index, vec![text_field]);
         let query = query_parser.parse_query("rust design").unwrap();
         let mut snippet_generator =

src/space_usage/mod.rs

@@ -295,8 +295,7 @@ mod test {
     use core::Index;
     use schema::Field;
     use schema::Schema;
-    use schema::STORED;
-    use schema::{FAST, INT_INDEXED, TEXT};
+    use schema::{FAST, INDEXED, STORED, TEXT};
     use space_usage::ByteCount;
     use space_usage::PerFieldSpaceUsage;
     use Term;
@@ -305,9 +304,8 @@ mod test {
     fn test_empty() {
         let schema = Schema::builder().build();
         let index = Index::create_in_ram(schema.clone());
-
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        let reader = index.reader().unwrap();
+        let searcher = reader.searcher();
         let searcher_space_usage = searcher.space_usage();
         assert_eq!(0, searcher_space_usage.total());
     }
@@ -332,7 +330,7 @@ mod test {
     #[test]
     fn test_fast_indexed() {
         let mut schema_builder = Schema::builder();
-        let name = schema_builder.add_u64_field("name", FAST | INT_INDEXED);
+        let name = schema_builder.add_u64_field("name", FAST | INDEXED);
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema.clone());
 
@@ -345,8 +343,8 @@ mod test {
             index_writer.commit().unwrap();
         }
 
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        let reader = index.reader().unwrap();
+        let searcher = reader.searcher();
         let searcher_space_usage = searcher.space_usage();
         assert!(searcher_space_usage.total() > 0);
         assert_eq!(1, searcher_space_usage.segments().len());
@@ -385,8 +383,8 @@ mod test {
             index_writer.commit().unwrap();
         }
 
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        let reader = index.reader().unwrap();
+        let searcher = reader.searcher();
         let searcher_space_usage = searcher.space_usage();
         assert!(searcher_space_usage.total() > 0);
         assert_eq!(1, searcher_space_usage.segments().len());
@@ -424,9 +422,8 @@ mod test {
             index_writer.add_document(doc!(name => "hello hi goodbye"));
             index_writer.commit().unwrap();
         }
-
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        let reader = index.reader().unwrap();
+        let searcher = reader.searcher();
         let searcher_space_usage = searcher.space_usage();
         assert!(searcher_space_usage.total() > 0);
         assert_eq!(1, searcher_space_usage.segments().len());
@@ -450,7 +447,7 @@ mod test {
     #[test]
     fn test_deletes() {
         let mut schema_builder = Schema::builder();
-        let name = schema_builder.add_u64_field("name", INT_INDEXED);
+        let name = schema_builder.add_u64_field("name", INDEXED);
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema.clone());
 
@@ -472,9 +469,8 @@ mod test {
             index_writer2.commit().unwrap();
         }
 
-        index.load_searchers().unwrap();
-
-        let searcher = index.searcher();
+        let reader = index.reader().unwrap();
+        let searcher = reader.searcher();
         let searcher_space_usage = searcher.space_usage();
         assert!(searcher_space_usage.total() > 0);
         assert_eq!(1, searcher_space_usage.segments().len());

src/termdict/mod.rs

@@ -159,8 +159,7 @@ mod tests {
                 index_writer.commit().unwrap();
             }
         }
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        let searcher = index.reader().unwrap().searcher();
 
         let field_searcher = searcher.field(text_field);
         let mut term_it = field_searcher.terms();

src/termdict/streamer.rs

@@ -1,9 +1,9 @@
 use super::TermDictionary;
-use fst::automaton::AlwaysMatch;
-use fst::map::{Stream, StreamBuilder};
-use fst::Automaton;
-use fst::{IntoStreamer, Streamer};
 use postings::TermInfo;
+use tantivy_fst::automaton::AlwaysMatch;
+use tantivy_fst::map::{Stream, StreamBuilder};
+use tantivy_fst::Automaton;
+use tantivy_fst::{IntoStreamer, Streamer};
 use termdict::TermOrdinal;
 
 /// `TermStreamerBuilder` is a helper object used to define

src/termdict/termdict.rs

@@ -3,15 +3,15 @@ use super::{TermStreamer, TermStreamerBuilder};
 use common::BinarySerializable;
 use common::CountingWriter;
 use directory::ReadOnlySource;
-use fst;
-use fst::raw::Fst;
-use fst::Automaton;
 use postings::TermInfo;
 use schema::FieldType;
 use std::io::{self, Write};
+use tantivy_fst;
+use tantivy_fst::raw::Fst;
+use tantivy_fst::Automaton;
 use termdict::TermOrdinal;
 
-fn convert_fst_error(e: fst::Error) -> io::Error {
+fn convert_fst_error(e: tantivy_fst::Error) -> io::Error {
     io::Error::new(io::ErrorKind::Other, e)
 }
 
@@ -19,7 +19,7 @@ fn convert_fst_error(e: fst::Error) -> io::Error {
 ///
 /// Inserting must be done in the order of the `keys`.
 pub struct TermDictionaryBuilder<W> {
-    fst_builder: fst::MapBuilder<W>,
+    fst_builder: tantivy_fst::MapBuilder<W>,
     term_info_store_writer: TermInfoStoreWriter,
     term_ord: u64,
 }
@@ -30,7 +30,7 @@ where
 {
     /// Creates a new `TermDictionaryBuilder`
     pub fn create(w: W, _field_type: &FieldType) -> io::Result<Self> {
-        let fst_builder = fst::MapBuilder::new(w).map_err(convert_fst_error)?;
+        let fst_builder = tantivy_fst::MapBuilder::new(w).map_err(convert_fst_error)?;
         Ok(TermDictionaryBuilder {
             fst_builder,
             term_info_store_writer: TermInfoStoreWriter::new(),
@@ -87,17 +87,9 @@ where
     }
 }
 
-fn open_fst_index(source: ReadOnlySource) -> fst::Map {
-    let fst = match source {
-        ReadOnlySource::Anonymous(data) => {
-            Fst::from_shared_bytes(data.data, data.start, data.len).expect("FST data is corrupted")
-        }
-        #[cfg(feature = "mmap")]
-        ReadOnlySource::Mmap(mmap_readonly) => {
-            Fst::from_mmap(mmap_readonly).expect("FST data is corrupted")
-        }
-    };
-    fst::Map::from(fst)
+fn open_fst_index(source: ReadOnlySource) -> tantivy_fst::Map<ReadOnlySource> {
+    let fst = Fst::new(source).expect("FST data is corrupted");
+    tantivy_fst::Map::from(fst)
 }
 
 /// The term dictionary contains all of the terms in
@@ -107,7 +99,7 @@ fn open_fst_index(source: ReadOnlySource) -> fst::Map {
 /// respective `TermOrdinal`. The `TermInfoStore` then makes it
 /// possible to fetch the associated `TermInfo`.
 pub struct TermDictionary {
-    fst_index: fst::Map,
+    fst_index: tantivy_fst::Map<ReadOnlySource>,
     term_info_store: TermInfoStore,
 }
 

src/tokenizer/mod.rs

@@ -97,6 +97,8 @@
 //! If you built your schema programmatically, a complete example
 //! could like this for instance.
 //!
+//! Note that tokens with a len greater or equal to [`MAX_TOKEN_LEN`](./constant.MAX_TOKEN_LEN.html).
+//!
 //! # Example
 //!
 //! ```
@@ -157,6 +159,13 @@ pub use self::tokenizer::BoxedTokenizer;
 pub use self::tokenizer::{Token, TokenFilter, TokenStream, Tokenizer};
 pub use self::tokenizer_manager::TokenizerManager;
 
+/// Maximum authorized len (in bytes) for a token.
+///
+/// Tokenizer are in charge of not emitting tokens larger than this value.
+/// Currently, if a faulty tokenizer implementation emits tokens with a length larger than
+/// `2^16 - 1 - 4`, the token will simply be ignored downstream.
+pub const MAX_TOKEN_LEN: usize = u16::max_value() as usize - 4;
+
 #[cfg(test)]
 pub mod tests {
     use super::{
@@ -228,27 +237,27 @@ pub mod tests {
     fn test_non_en_tokenizer() {
         let tokenizer_manager = TokenizerManager::default();
         tokenizer_manager.register(
-            "es_stem",
+            "el_stem",
             SimpleTokenizer
                 .filter(RemoveLongFilter::limit(40))
                 .filter(LowerCaser)
-                .filter(Stemmer::new(Language::Spanish)),
+                .filter(Stemmer::new(Language::Greek)),
         );
-        let en_tokenizer = tokenizer_manager.get("es_stem").unwrap();
+        let en_tokenizer = tokenizer_manager.get("el_stem").unwrap();
         let mut tokens: Vec<Token> = vec![];
         {
             let mut add_token = |token: &Token| {
                 tokens.push(token.clone());
             };
             en_tokenizer
-                .token_stream("Hola, feliz contribuyente!")
+                .token_stream("Καλημέρα, χαρούμενε φορολογούμενε!")
                 .process(&mut add_token);
         }
 
         assert_eq!(tokens.len(), 3);
-        assert_token(&tokens[0], 0, "hola", 0, 4);
-        assert_token(&tokens[1], 1, "feliz", 6, 11);
-        assert_token(&tokens[2], 2, "contribuyent", 12, 25);
+        assert_token(&tokens[0], 0, "καλημερ", 0, 16);
+        assert_token(&tokens[1], 1, "χαρουμεν", 18, 36);
+        assert_token(&tokens[2], 2, "φορολογουμεν", 37, 63);
     }
 
     #[test]

src/tokenizer/stemmer.rs

@@ -2,7 +2,6 @@
 
 use super::{Token, TokenFilter, TokenStream};
 use rust_stemmers::{self, Algorithm};
-use std::sync::Arc;
 
 /// Available stemmer languages.
 #[derive(Debug, Serialize, Deserialize, Eq, PartialEq, Copy, Clone)]
@@ -57,14 +56,14 @@ impl Language {
 /// Tokens are expected to be lowercased beforehand.
 #[derive(Clone)]
 pub struct Stemmer {
-    stemmer_algorithm: Arc<Algorithm>,
+    stemmer_algorithm: Algorithm,
 }
 
 impl Stemmer {
     /// Creates a new Stemmer `TokenFilter` for a given language algorithm.
     pub fn new(language: Language) -> Stemmer {
         Stemmer {
-            stemmer_algorithm: Arc::new(language.algorithm()),
+            stemmer_algorithm: language.algorithm(),
         }
     }
 }
@@ -83,7 +82,7 @@ where
     type ResultTokenStream = StemmerTokenStream<TailTokenStream>;
 
     fn transform(&self, token_stream: TailTokenStream) -> Self::ResultTokenStream {
-        let inner_stemmer = rust_stemmers::Stemmer::create(Algorithm::English);
+        let inner_stemmer = rust_stemmers::Stemmer::create(self.stemmer_algorithm);
         StemmerTokenStream::wrap(inner_stemmer, token_stream)
     }
 }