Fix version

Adding IndexReader to the API. Making it possible to watch for changes.

* Closes #500

.travis.yml

@@ -29,7 +29,7 @@ addons:
 matrix:
   include:
     # Android
-    - env: TARGET=aarch64-linux-android DISABLE_TESTS=1
+    - env: TARGET=aarch64-linux-android
     #- env: TARGET=arm-linux-androideabi DISABLE_TESTS=1
     #- env: TARGET=armv7-linux-androideabi DISABLE_TESTS=1
     #- env: TARGET=i686-linux-android DISABLE_TESTS=1

CHANGELOG.md

@@ -1,9 +1,28 @@
 Tantivy 0.9.0
 =====================
+*0.9.0 index format is not compatible with the 
+previous index format.*
+- MAJOR BUGFIX : 
+  Some `Mmap` objects were being leaked, and would never get released. (@fulmicoton)
 - Removed most unsafe (@fulmicoton)
 - Indexer memory footprint improved. (VInt comp, inlining the first block. (@fulmicoton)
 - Stemming in other language possible (@pentlander)
 - Segments with no docs are deleted earlier (@barrotsteindev)
+- Added grouped add and delete operations. 
+  They are guaranteed to happen together (i.e. they cannot be split by a commit). 
+  In addition, adds are guaranteed to happen on the same segment. (@elbow-jason)
+- Removed `INT_STORED` and `INT_INDEXED`. It is now possible to use `STORED` and `INDEXED`
+  for int fields. (@fulmicoton)
+- Added DateTime field (@barrotsteindev)
+- Added IndexReader. By default, index is reloaded automatically upon new commits (@fulmicoton)
+
+
+Tantivy 0.8.2
+=====================
+Fixing build for x86_64 platforms. (#496)
+No need to update from 0.8.1 if tantivy
+is building on your platform.
+
 
 Tantivy 0.8.1
 =====================

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "tantivy"
-version = "0.9.0-dev"
+version = "0.9.0"
 authors = ["Paul Masurel <paul.masurel@gmail.com>"]
 license = "MIT"
 categories = ["database-implementations", "data-structures"]
@@ -16,8 +16,8 @@ base64 = "0.10.0"
 byteorder = "1.0"
 lazy_static = "1"
 regex = "1.0"
-fst = {version="0.3", default-features=false}
-fst-regex = { version="0.2" }
+tantivy-fst = "0.1"
+memmap = {version = "0.7", optional=true}
 lz4 = {version="1.20", optional=true}
 snap = {version="0.2"}
 atomicwrites = {version="0.2.2", optional=true}
@@ -32,17 +32,17 @@ num_cpus = "1.2"
 fs2={version="0.4", optional=true}
 itertools = "0.8"
 levenshtein_automata = {version="0.1", features=["fst_automaton"]}
+notify = {version="4", optional=true}
 bit-set = "0.5"
-uuid = { version = "0.7", features = ["v4", "serde"] }
+uuid = { version = "0.7.2", features = ["v4", "serde"] }
 crossbeam = "0.5"
 futures = "0.1"
 futures-cpupool = "0.1"
 owning_ref = "0.4"
 stable_deref_trait = "1.0.0"
-rust-stemmers = "1"
-downcast = { version="0.9" }
-matches = "0.1"
-bitpacking = "0.5"
+rust-stemmers = "1.1"
+downcast-rs = { version="1.0" }
+bitpacking = "0.6"
 census = "0.2"
 fnv = "1.0.6"
 owned-read = "0.4"
@@ -51,6 +51,7 @@ htmlescape = "0.3.1"
 fail = "0.2"
 scoped-pool = "1.0"
 murmurhash32 = "0.2"
+chrono = "0.4"
 
 [target.'cfg(windows)'.dependencies]
 winapi = "0.2"
@@ -58,6 +59,8 @@ winapi = "0.2"
 [dev-dependencies]
 rand = "0.6"
 maplit = "1"
+matches = "0.1.8"
+time = "0.1.42"
 
 [profile.release]
 opt-level = 3
@@ -71,12 +74,11 @@ overflow-checks = true
 [features]
 # by default no-fail is disabled. We manually enable it when running test.
 default = ["mmap", "no_fail"]
-mmap = ["fst/mmap", "atomicwrites", "fs2"]
+mmap = ["atomicwrites", "fs2", "memmap", "notify"]
 lz4-compression = ["lz4"]
 no_fail = ["fail/no_fail"]
 unstable = [] # useful for benches.
+wasm-bindgen = ["uuid/wasm-bindgen"]
 
 [badges]
 travis-ci = { repository = "tantivy-search/tantivy" }
-
-

README.md

@@ -30,6 +30,7 @@ Tantivy is, in fact, strongly inspired by Lucene's design.
 # Features
 
 - Full-text search
+- Configurable tokenizer. (stemming available for 17 latin languages. Third party support for Chinese ([tantivy-jieba](https://crates.io/crates/tantivy-jieba) and [cang-jie](https://crates.io/crates/cang-jie)) and [Japanese](https://crates.io/crates/tantivy-tokenizer-tiny-segmenter)
 - Fast (check out the :racehorse: :sparkles: [benchmark](https://tantivy-search.github.io/bench/) :sparkles: :racehorse:)
 - Tiny startup time (<10ms), perfect for command line tools
 - BM25 scoring (the same as lucene)
@@ -41,6 +42,7 @@ Tantivy is, in fact, strongly inspired by Lucene's design.
 - SIMD integer compression when the platform/CPU includes the SSE2 instruction set.
 - Single valued and multivalued u64 and i64 fast fields (equivalent of doc values in Lucene)
 - `&[u8]` fast fields
+- Text, i64, u64, dates and hierarchical facet fields
 - LZ4 compressed document store
 - Range queries
 - Faceted search

examples/basic_search.rs

@@ -20,6 +20,7 @@ use tantivy::collector::TopDocs;
 use tantivy::query::QueryParser;
 use tantivy::schema::*;
 use tantivy::Index;
+use tantivy::ReloadPolicy;
 use tempdir::TempDir;
 
 fn main() -> tantivy::Result<()> {
@@ -170,24 +171,33 @@ fn main() -> tantivy::Result<()> {
     //
     // ### Searcher
     //
-    // Let's search our index. Start by reloading
-    // searchers in the index. This should be done
-    // after every `commit()`.
-    index.load_searchers()?;
+    // A reader is required to get search the index.
+    // It acts as a `Searcher` pool that reloads itself,
+    // depending on a `ReloadPolicy`.
+    //
+    // For a search server you will typically create one reader for the entire lifetime of your
+    // program, and acquire a new searcher for every single request.
+    //
+    // In the code below, we rely on the 'ON_COMMIT' policy: the reader
+    // will reload the index automatically after each commit.
+    let reader = index
+        .reader_builder()
+        .reload_policy(ReloadPolicy::OnCommit)
+        .try_into()?;
 
     // We now need to acquire a searcher.
-    // Some search experience might require more than
-    // one query.
     //
-    // The searcher ensure that we get to work
-    // with a consistent version of the index.
+    // A searcher points to snapshotted, immutable version of the index.
+    //
+    // Some search experience might require more than
+    // one query. Using the same searcher ensures that all of these queries will run on the
+    // same version of the index.
     //
     // Acquiring a `searcher` is very cheap.
     //
-    // You should acquire a searcher every time you
-    // start processing a request and
+    // You should acquire a searcher every time you start processing a request and
     // and release it right after your query is finished.
-    let searcher = index.searcher();
+    let searcher = reader.searcher();
 
     // ### Query
 
@@ -224,7 +234,6 @@ fn main() -> tantivy::Result<()> {
     // Since the body field was not configured as stored,
     // the document returned will only contain
     // a title.
-
     for (_score, doc_address) in top_docs {
         let retrieved_doc = searcher.doc(doc_address)?;
         println!("{}", schema.to_json(&retrieved_doc));

examples/custom_collector.rs

@@ -17,7 +17,7 @@ use tantivy::collector::{Collector, SegmentCollector};
 use tantivy::fastfield::FastFieldReader;
 use tantivy::query::QueryParser;
 use tantivy::schema::Field;
-use tantivy::schema::{Schema, FAST, INT_INDEXED, TEXT};
+use tantivy::schema::{Schema, FAST, INDEXED, TEXT};
 use tantivy::Index;
 use tantivy::SegmentReader;
 
@@ -137,7 +137,7 @@ fn main() -> tantivy::Result<()> {
     // products, and with a name, a description, and a price.
     let product_name = schema_builder.add_text_field("name", TEXT);
     let product_description = schema_builder.add_text_field("description", TEXT);
-    let price = schema_builder.add_u64_field("price", INT_INDEXED | FAST);
+    let price = schema_builder.add_u64_field("price", INDEXED | FAST);
     let schema = schema_builder.build();
 
     // # Indexing documents
@@ -170,9 +170,9 @@ fn main() -> tantivy::Result<()> {
         price => 5_200u64
     ));
     index_writer.commit()?;
-    index.load_searchers()?;
 
-    let searcher = index.searcher();
+    let reader = index.reader()?;
+    let searcher = reader.searcher();
     let query_parser = QueryParser::for_index(&index, vec![product_name, product_description]);
 
     // here we want to get a hit on the 'ken' in Frankenstein

examples/custom_tokenizer.rs

@@ -91,9 +91,9 @@ fn main() -> tantivy::Result<()> {
                 increasing confidence in the success of my undertaking."#
     ));
     index_writer.commit()?;
-    index.load_searchers()?;
 
-    let searcher = index.searcher();
+    let reader = index.reader()?;
+    let searcher = reader.searcher();
 
     // The query parser can interpret human queries.
     // Here, if the user does not specify which

examples/deleting_updating_documents.rs

@@ -14,12 +14,16 @@ use tantivy::collector::TopDocs;
 use tantivy::query::TermQuery;
 use tantivy::schema::*;
 use tantivy::Index;
+use tantivy::IndexReader;
 
 // A simple helper function to fetch a single document
 // given its id from our index.
 // It will be helpful to check our work.
-fn extract_doc_given_isbn(index: &Index, isbn_term: &Term) -> tantivy::Result<Option<Document>> {
-    let searcher = index.searcher();
+fn extract_doc_given_isbn(
+    reader: &IndexReader,
+    isbn_term: &Term,
+) -> tantivy::Result<Option<Document>> {
+    let searcher = reader.searcher();
 
     // This is the simplest query you can think of.
     // It matches all of the documents containing a specific term.
@@ -85,12 +89,12 @@ fn main() -> tantivy::Result<()> {
        isbn => "978-9176370711",
     ));
     index_writer.commit()?;
-    index.load_searchers()?;
+    let reader = index.reader()?;
 
     let frankenstein_isbn = Term::from_field_text(isbn, "978-9176370711");
 
     // Oops our frankenstein doc seems mispelled
-    let frankenstein_doc_misspelled = extract_doc_given_isbn(&index, &frankenstein_isbn)?.unwrap();
+    let frankenstein_doc_misspelled = extract_doc_given_isbn(&reader, &frankenstein_isbn)?.unwrap();
     assert_eq!(
         schema.to_json(&frankenstein_doc_misspelled),
         r#"{"isbn":["978-9176370711"],"title":["Frankentein"]}"#,
@@ -129,10 +133,10 @@ fn main() -> tantivy::Result<()> {
     // Everything happened as if the document was updated.
     index_writer.commit()?;
     // We reload our searcher to make our change available to clients.
-    index.load_searchers()?;
+    reader.reload()?;
 
     // No more typo!
-    let frankenstein_new_doc = extract_doc_given_isbn(&index, &frankenstein_isbn)?.unwrap();
+    let frankenstein_new_doc = extract_doc_given_isbn(&reader, &frankenstein_isbn)?.unwrap();
     assert_eq!(
         schema.to_json(&frankenstein_new_doc),
         r#"{"isbn":["978-9176370711"],"title":["Frankenstein"]}"#,

examples/faceted_search.rs

@@ -55,9 +55,9 @@ fn main() -> tantivy::Result<()> {
 
     index_writer.commit()?;
 
-    index.load_searchers()?;
+    let reader = index.reader()?;
 
-    let searcher = index.searcher();
+    let searcher = reader.searcher();
 
     let mut facet_collector = FacetCollector::for_field(tags);
     facet_collector.add_facet("/pools");

examples/integer_range_search.rs

@@ -0,0 +1,43 @@
+// # Searching a range on an indexed int field.
+//
+// Below is an example of creating an indexed integer field in your schema
+// You can use RangeQuery to get a Count of all occurrences in a given range.
+
+#[macro_use]
+extern crate tantivy;
+use tantivy::collector::Count;
+use tantivy::query::RangeQuery;
+use tantivy::schema::{Schema, INDEXED};
+use tantivy::Index;
+use tantivy::Result;
+
+fn run() -> Result<()> {
+    // For the sake of simplicity, this schema will only have 1 field
+    let mut schema_builder = Schema::builder();
+
+    // `INDEXED` is a short-hand to indicate that our field should be "searchable".
+    let year_field = schema_builder.add_u64_field("year", INDEXED);
+    let schema = schema_builder.build();
+    let index = Index::create_in_ram(schema);
+    let reader = index.reader()?;
+    {
+        let mut index_writer = index.writer_with_num_threads(1, 6_000_000)?;
+        for year in 1950u64..2019u64 {
+            index_writer.add_document(doc!(year_field => year));
+        }
+        index_writer.commit()?;
+        // The index will be a range of years
+    }
+    reader.reload()?;
+    let searcher = reader.searcher();
+    // The end is excluded i.e. here we are searching up to 1969
+    let docs_in_the_sixties = RangeQuery::new_u64(year_field, 1960..1970);
+    // Uses a Count collector to sum the total number of docs in the range
+    let num_60s_books = searcher.search(&docs_in_the_sixties, &Count)?;
+    assert_eq!(num_60s_books, 10);
+    Ok(())
+}
+
+fn main() {
+    run().unwrap()
+}

examples/iterating_docs_and_positions.rs

@@ -33,9 +33,9 @@ fn main() -> tantivy::Result<()> {
     index_writer.add_document(doc!(title => "The modern Promotheus"));
     index_writer.commit()?;
 
-    index.load_searchers()?;
+    let reader = index.reader()?;
 
-    let searcher = index.searcher();
+    let searcher = reader.searcher();
 
     // A tantivy index is actually a collection of segments.
     // Similarly, a searcher just wraps a list `segment_reader`.

examples/snippet.rs

@@ -48,9 +48,8 @@ fn main() -> tantivy::Result<()> {
     // ...
     index_writer.commit()?;
 
-    index.load_searchers()?;
-
-    let searcher = index.searcher();
+    let reader = index.reader()?;
+    let searcher = reader.searcher();
     let query_parser = QueryParser::for_index(&index, vec![title, body]);
     let query = query_parser.parse_query("sycamore spring")?;
 

examples/stop_words.rs

@@ -96,9 +96,9 @@ fn main() -> tantivy::Result<()> {
 
     index_writer.commit()?;
 
-    index.load_searchers()?;
+    let reader = index.reader()?;
 
-    let searcher = index.searcher();
+    let searcher = reader.searcher();
 
     let query_parser = QueryParser::for_index(&index, vec![title, body]);
 

examples/working_with_json.rs

@@ -12,7 +12,7 @@ fn main() -> tantivy::Result<()> {
     let mut schema_builder = Schema::builder();
     schema_builder.add_text_field("title", TEXT | STORED);
     schema_builder.add_text_field("body", TEXT);
-    schema_builder.add_u64_field("year", INT_INDEXED);
+    schema_builder.add_u64_field("year", INDEXED);
     let schema = schema_builder.build();
 
     // Let's assume we have a json-serialized document.

src/collector/count_collector.rs

@@ -40,8 +40,8 @@ use SegmentReader;
 ///         index_writer.commit().unwrap();
 ///     }
 ///
-///     index.load_searchers()?;
-///     let searcher = index.searcher();
+///     let reader = index.reader()?;
+///     let searcher = reader.searcher();
 ///
 ///     {
 ///         let query_parser = QueryParser::for_index(&index, vec![title]);

src/collector/facet_collector.rs

@@ -122,17 +122,16 @@ fn facet_depth(facet_bytes: &[u8]) -> usize {
 ///             facet => Facet::from("/lang/en"),
 ///             facet => Facet::from("/category/biography")
 ///         ));
-///         index_writer.commit().unwrap();
+///         index_writer.commit()?;
 ///     }
-///
-///     index.load_searchers()?;
-///     let searcher = index.searcher();
+///     let reader = index.reader()?;
+///     let searcher = reader.searcher();
 ///
 ///     {
 ///			let mut facet_collector = FacetCollector::for_field(facet);
 ///         facet_collector.add_facet("/lang");
 ///         facet_collector.add_facet("/category");
-///         let facet_counts = searcher.search(&AllQuery, &facet_collector).unwrap();
+///         let facet_counts = searcher.search(&AllQuery, &facet_collector)?;
 ///
 ///         // This lists all of the facet counts
 ///         let facets: Vec<(&Facet, u64)> = facet_counts
@@ -147,7 +146,7 @@ fn facet_depth(facet_bytes: &[u8]) -> usize {
 ///     {
 ///			let mut facet_collector = FacetCollector::for_field(facet);
 ///         facet_collector.add_facet("/category/fiction");
-///         let facet_counts = searcher.search(&AllQuery, &facet_collector).unwrap();
+///         let facet_counts = searcher.search(&AllQuery, &facet_collector)?;
 ///
 ///         // This lists all of the facet counts
 ///         let facets: Vec<(&Facet, u64)> = facet_counts
@@ -163,7 +162,7 @@ fn facet_depth(facet_bytes: &[u8]) -> usize {
 ///    {
 ///			let mut facet_collector = FacetCollector::for_field(facet);
 ///         facet_collector.add_facet("/category/fiction");
-///         let facet_counts = searcher.search(&AllQuery, &facet_collector).unwrap();
+///         let facet_counts = searcher.search(&AllQuery, &facet_collector)?;
 ///
 ///         // This lists all of the facet counts
 ///         let facets: Vec<(&Facet, u64)> = facet_counts.top_k("/category/fiction", 1);
@@ -483,8 +482,8 @@ mod tests {
             index_writer.add_document(doc);
         }
         index_writer.commit().unwrap();
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        let reader = index.reader().unwrap();
+        let searcher = reader.searcher();
         let mut facet_collector = FacetCollector::for_field(facet_field);
         facet_collector.add_facet(Facet::from("/top1"));
         let counts = searcher.search(&AllQuery, &facet_collector).unwrap();
@@ -532,8 +531,8 @@ mod tests {
             facet_field => Facet::from_text(&"/subjects/B/b"),
         ));
         index_writer.commit().unwrap();
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        let reader = index.reader().unwrap();
+        let searcher = reader.searcher();
         assert_eq!(searcher.num_docs(), 1);
         let mut facet_collector = FacetCollector::for_field(facet_field);
         facet_collector.add_facet("/subjects");
@@ -579,9 +578,7 @@ mod tests {
             index_writer.add_document(doc);
         }
         index_writer.commit().unwrap();
-        index.load_searchers().unwrap();
-
-        let searcher = index.searcher();
+        let searcher = index.reader().unwrap().searcher();
 
         let mut facet_collector = FacetCollector::for_field(facet_field);
         facet_collector.add_facet("/facet");
@@ -635,8 +632,7 @@ mod bench {
             index_writer.add_document(doc);
         }
         index_writer.commit().unwrap();
-        index.load_searchers().unwrap();
-
+        let reader = index.reader().unwrap();
         b.iter(|| {
             let searcher = index.searcher();
             let facet_collector = FacetCollector::for_field(facet_field);

src/collector/int_facet_collector.rs

@@ -101,8 +101,7 @@ mod tests {
             assert_eq!(index_writer.commit().unwrap(), 10u64);
         }
 
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        let searcher = index.reader().searcher();
         let mut ffvf_i64: IntFacetCollector<I64FastFieldReader> = IntFacetCollector::new(num_field_i64);
         let mut ffvf_u64: IntFacetCollector<U64FastFieldReader> = IntFacetCollector::new(num_field_u64);
 

src/collector/mod.rs

@@ -53,9 +53,9 @@ use tantivy::collector::{Count, TopDocs};
 #     index_writer.add_document(doc!(
 #        title => "The Diary of Muadib",
 #     ));
-#     index_writer.commit().unwrap();
-#     index.load_searchers()?;
-#     let searcher = index.searcher();
+#     index_writer.commit()?;
+#     let reader = index.reader()?;
+#     let searcher = reader.searcher();
 #     let query_parser = QueryParser::for_index(&index, vec![title]);
 #     let query = query_parser.parse_query("diary")?;
 let (doc_count, top_docs): (usize, Vec<(Score, DocAddress)>) =
@@ -85,7 +85,7 @@ See the `custom_collector` example.
 
 */
 
-use downcast;
+use downcast_rs;
 use DocId;
 use Result;
 use Score;
@@ -111,9 +111,9 @@ pub use self::facet_collector::FacetCollector;
 
 /// `Fruit` is the type for the result of our collection.
 /// e.g. `usize` for the `Count` collector.
-pub trait Fruit: Send + downcast::Any {}
+pub trait Fruit: Send + downcast_rs::Downcast {}
 
-impl<T> Fruit for T where T: Send + downcast::Any {}
+impl<T> Fruit for T where T: Send + downcast_rs::Downcast {}
 
 /// Collectors are in charge of collecting and retaining relevant
 /// information from the document found and scored by the query.
@@ -358,10 +358,7 @@ where
     }
 }
 
-#[allow(missing_docs)]
-mod downcast_impl {
-    downcast!(super::Fruit);
-}
+impl_downcast!(Fruit);
 
 #[cfg(test)]
 pub mod tests;

src/collector/multi_collector.rs

@@ -1,7 +1,6 @@
 use super::Collector;
 use super::SegmentCollector;
 use collector::Fruit;
-use downcast::Downcast;
 use std::marker::PhantomData;
 use DocId;
 use Result;
@@ -37,11 +36,11 @@ impl<TCollector: Collector> Collector for CollectorWrapper<TCollector> {
         let typed_fruit: Vec<TCollector::Fruit> = children
             .into_iter()
             .map(|untyped_fruit| {
-                Downcast::<TCollector::Fruit>::downcast(untyped_fruit)
+                untyped_fruit
+                    .downcast::<TCollector::Fruit>()
                     .map(|boxed_but_typed| *boxed_but_typed)
-                    .map_err(|e| {
-                        let err_msg = format!("Failed to cast child collector fruit. {:?}", e);
-                        TantivyError::InvalidArgument(err_msg)
+                    .map_err(|_| {
+                        TantivyError::InvalidArgument("Failed to cast child fruit.".to_string())
                     })
             })
             .collect::<Result<_>>()?;
@@ -89,7 +88,10 @@ pub struct FruitHandle<TFruit: Fruit> {
 impl<TFruit: Fruit> FruitHandle<TFruit> {
     pub fn extract(self, fruits: &mut MultiFruit) -> TFruit {
         let boxed_fruit = fruits.sub_fruits[self.pos].take().expect("");
-        *Downcast::<TFruit>::downcast(boxed_fruit).expect("Failed")
+        *boxed_fruit
+            .downcast::<TFruit>()
+            .map_err(|_| ())
+            .expect("Failed to downcast collector fruit.")
     }
 }
 
@@ -132,8 +134,8 @@ impl<TFruit: Fruit> FruitHandle<TFruit> {
 ///         index_writer.commit().unwrap();
 ///     }
 ///
-///     index.load_searchers()?;
-///     let searcher = index.searcher();
+///     let reader = index.reader()?;
+///     let searcher = reader.searcher();
 ///
 ///     let mut collectors = MultiCollector::new();
 ///     let top_docs_handle = collectors.add_collector(TopDocs::with_limit(2));
@@ -276,8 +278,7 @@ mod tests {
             index_writer.add_document(doc!(text=>"abc"));
             index_writer.commit().unwrap();
         }
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        let searcher = index.reader().unwrap().searcher();
         let term = Term::from_field_text(text, "abc");
         let query = TermQuery::new(term, IndexRecordOption::Basic);
 

src/collector/top_field_collector.rs

@@ -23,15 +23,16 @@ use SegmentReader;
 /// # use tantivy::schema::{Schema, Field, FAST, TEXT};
 /// # use tantivy::{Index, Result, DocAddress};
 /// # use tantivy::query::{Query, QueryParser};
+/// use tantivy::Searcher;
 /// use tantivy::collector::TopDocs;
 ///
-/// # fn main() {
+/// # fn main() -> tantivy::Result<()> {
 /// #   let mut schema_builder = Schema::builder();
 /// #   let title = schema_builder.add_text_field("title", TEXT);
 /// #   let rating = schema_builder.add_u64_field("rating", FAST);
 /// #   let schema = schema_builder.build();
 /// #   let index = Index::create_in_ram(schema);
-/// #   let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
+/// #   let mut index_writer = index.writer_with_num_threads(1, 3_000_000)?;
 /// #   index_writer.add_document(doc!(
 /// #       title => "The Name of the Wind",
 /// #       rating => 92u64,
@@ -39,13 +40,14 @@ use SegmentReader;
 /// #   index_writer.add_document(doc!(title => "The Diary of Muadib", rating => 97u64));
 /// #   index_writer.add_document(doc!(title => "A Dairy Cow", rating => 63u64));
 /// #   index_writer.add_document(doc!(title => "The Diary of a Young Girl", rating => 80u64));
-/// #   index_writer.commit().unwrap();
-/// #   index.load_searchers().unwrap();
-///	#   let query = QueryParser::for_index(&index, vec![title]).parse_query("diary").unwrap();
-/// #   let top_docs = docs_sorted_by_rating(&index, &query, rating).unwrap();
+/// #   index_writer.commit()?;
+/// #   let reader = index.reader()?;
+/// #   let query = QueryParser::for_index(&index, vec![title]).parse_query("diary")?;
+/// #   let top_docs = docs_sorted_by_rating(&reader.searcher(), &query, rating)?;
 /// #   assert_eq!(top_docs,
 /// #            vec![(97u64, DocAddress(0u32, 1)),
 /// #                 (80u64, DocAddress(0u32, 3))]);
+/// #   Ok(())
 /// # }
 /// #
 /// /// Searches the document matching the given query, and
@@ -53,7 +55,9 @@ use SegmentReader;
 /// /// given in argument.
 /// ///
 /// /// `field` is required to be a FAST field.
-/// fn docs_sorted_by_rating(index: &Index, query: &Query, sort_by_field: Field)
+/// fn docs_sorted_by_rating(searcher: &Searcher,
+///                          query: &Query,
+///                          sort_by_field: Field)
 ///     -> Result<Vec<(u64, DocAddress)>> {
 ///
 ///     // This is where we build our collector!
@@ -61,8 +65,7 @@ use SegmentReader;
 ///
 ///     // ... and here is our documents. Not this is a simple vec.
 ///     // The `u64` in the pair is the value of our fast field for each documents.
-///     index.searcher()
-///          .search(query, &top_docs_by_rating)
+///     searcher.search(query, &top_docs_by_rating)
 /// }
 /// ```
 pub struct TopDocsByField<T> {
@@ -76,6 +79,12 @@ impl<T: FastValue + PartialOrd + Clone> TopDocsByField<T> {
     /// The given field name must be a fast field, otherwise the collector have an error while
     /// collecting results.
     ///
+    /// This constructor is crate-private. Client are supposed to call
+    /// build `TopDocsByField`  object using the `TopDocs` API.
+    ///
+    /// e.g.:
+    ///   `TopDocs::with_limit(2).order_by_field(sort_by_field)`
+    ///
     /// # Panics
     /// The method panics if limit is 0
     pub(crate) fn new(field: Field, limit: usize) -> TopDocsByField<T> {
@@ -171,7 +180,7 @@ mod tests {
                 size => 16u64,
             ));
         });
-        let searcher = index.searcher();
+        let searcher = index.reader().unwrap().searcher();
 
         let top_collector = TopDocs::with_limit(4).order_by_field(size);
         let top_docs: Vec<(u64, DocAddress)> = searcher.search(&query, &top_collector).unwrap();
@@ -198,7 +207,7 @@ mod tests {
                 size => 12u64,
             ));
         });
-        let searcher = index.searcher();
+        let searcher = index.reader().unwrap().searcher();
         let top_collector: TopDocsByField<u64> = TopDocs::with_limit(4).order_by_field(Field(2));
         let segment_reader = searcher.segment_reader(0u32);
         top_collector
@@ -218,7 +227,7 @@ mod tests {
                 size => 12u64,
             ));
         });
-        let searcher = index.searcher();
+        let searcher = index.reader().unwrap().searcher();
         let segment = searcher.segment_reader(0);
         let top_collector: TopDocsByField<u64> = TopDocs::with_limit(4).order_by_field(size);
         assert_matches!(
@@ -241,8 +250,6 @@ mod tests {
         let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
         doc_adder(&mut index_writer);
         index_writer.commit().unwrap();
-        index.load_searchers().unwrap();
-
         let query_parser = QueryParser::for_index(&index, vec![query_field]);
         let query = query_parser.parse_query(query).unwrap();
         (index, query)

src/collector/top_score_collector.rs

@@ -51,8 +51,8 @@ use SegmentReader;
 ///         index_writer.commit().unwrap();
 ///     }
 ///
-///     index.load_searchers()?;
-///     let searcher = index.searcher();
+///     let reader = index.reader()?;
+///     let searcher = reader.searcher();
 ///
 ///     let query_parser = QueryParser::for_index(&index, vec![title]);
 ///     let query = query_parser.parse_query("diary")?;
@@ -148,7 +148,6 @@ mod tests {
             index_writer.add_document(doc!(text_field=>"I like Droopy"));
             assert!(index_writer.commit().is_ok());
         }
-        index.load_searchers().unwrap();
         index
     }
 
@@ -159,6 +158,8 @@ mod tests {
         let query_parser = QueryParser::for_index(&index, vec![field]);
         let text_query = query_parser.parse_query("droopy tax").unwrap();
         let score_docs: Vec<(Score, DocAddress)> = index
+            .reader()
+            .unwrap()
             .searcher()
             .search(&text_query, &TopDocs::with_limit(4))
             .unwrap();
@@ -179,6 +180,8 @@ mod tests {
         let query_parser = QueryParser::for_index(&index, vec![field]);
         let text_query = query_parser.parse_query("droopy tax").unwrap();
         let score_docs: Vec<(Score, DocAddress)> = index
+            .reader()
+            .unwrap()
             .searcher()
             .search(&text_query, &TopDocs::with_limit(2))
             .unwrap();

src/common/bitpacker.rs

@@ -64,7 +64,7 @@ pub struct BitUnpacker<Data>
 where
     Data: Deref<Target = [u8]>,
 {
-    num_bits: usize,
+    num_bits: u64,
     mask: u64,
     data: Data,
 }
@@ -80,13 +80,13 @@ where
             (1u64 << num_bits) - 1u64
         };
         BitUnpacker {
-            num_bits: num_bits as usize,
+            num_bits: u64::from(num_bits),
             mask,
             data,
         }
     }
 
-    pub fn get(&self, idx: usize) -> u64 {
+    pub fn get(&self, idx: u64) -> u64 {
         if self.num_bits == 0 {
             return 0u64;
         }
@@ -97,10 +97,10 @@ where
         let addr = addr_in_bits >> 3;
         let bit_shift = addr_in_bits & 7;
         debug_assert!(
-            addr + 8 <= data.len(),
+            addr + 8 <= data.len() as u64,
             "The fast field field should have been padded with 7 bytes."
         );
-        let val_unshifted_unmasked: u64 = LittleEndian::read_u64(&data[addr..]);
+        let val_unshifted_unmasked: u64 = LittleEndian::read_u64(&data[(addr as usize)..]);
         let val_shifted = (val_unshifted_unmasked >> bit_shift) as u64;
         val_shifted & mask
     }
@@ -129,7 +129,7 @@ mod test {
     fn test_bitpacker_util(len: usize, num_bits: u8) {
         let (bitunpacker, vals) = create_fastfield_bitpacker(len, num_bits);
         for (i, val) in vals.iter().enumerate() {
-            assert_eq!(bitunpacker.get(i), *val);
+            assert_eq!(bitunpacker.get(i as u64), *val);
         }
     }
 

src/common/composite_file.rs

@@ -39,7 +39,7 @@ impl BinarySerializable for FileAddr {
 /// A `CompositeWrite` is used to write a `CompositeFile`.
 pub struct CompositeWrite<W = WritePtr> {
     write: CountingWriter<W>,
-    offsets: HashMap<FileAddr, usize>,
+    offsets: HashMap<FileAddr, u64>,
 }
 
 impl<W: Write> CompositeWrite<W> {

src/common/counting_writer.rs

@@ -3,7 +3,7 @@ use std::io::Write;
 
 pub struct CountingWriter<W> {
     underlying: W,
-    written_bytes: usize,
+    written_bytes: u64,
 }
 
 impl<W: Write> CountingWriter<W> {
@@ -14,11 +14,11 @@ impl<W: Write> CountingWriter<W> {
         }
     }
 
-    pub fn written_bytes(&self) -> usize {
+    pub fn written_bytes(&self) -> u64 {
         self.written_bytes
     }
 
-    pub fn finish(mut self) -> io::Result<(W, usize)> {
+    pub fn finish(mut self) -> io::Result<(W, u64)> {
         self.flush()?;
         Ok((self.underlying, self.written_bytes))
     }
@@ -27,10 +27,16 @@ impl<W: Write> CountingWriter<W> {
 impl<W: Write> Write for CountingWriter<W> {
     fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
         let written_size = self.underlying.write(buf)?;
-        self.written_bytes += written_size;
+        self.written_bytes += written_size as u64;
         Ok(written_size)
     }
 
+    fn write_all(&mut self, buf: &[u8]) -> io::Result<()> {
+        self.underlying.write_all(buf)?;
+        self.written_bytes += buf.len() as u64;
+        Ok(())
+    }
+
     fn flush(&mut self) -> io::Result<()> {
         self.underlying.flush()
     }
@@ -48,8 +54,8 @@ mod test {
         let mut counting_writer = CountingWriter::wrap(buffer);
         let bytes = (0u8..10u8).collect::<Vec<u8>>();
         counting_writer.write_all(&bytes).unwrap();
-        let (w, len): (Vec<u8>, usize) = counting_writer.finish().unwrap();
-        assert_eq!(len, 10);
+        let (w, len): (Vec<u8>, u64) = counting_writer.finish().unwrap();
+        assert_eq!(len, 10u64);
         assert_eq!(w.len(), 10);
     }
 }

src/core/executor.rs

@@ -123,15 +123,14 @@ mod tests {
         }
     }
 
-}
-
-#[test]
-fn test_map_multithread() {
-    let result: Vec<usize> = Executor::multi_thread(3, "search-test")
-        .map(|i| Ok(i * 2), 0..10)
-        .unwrap();
-    assert_eq!(result.len(), 10);
-    for i in 0..10 {
-        assert_eq!(result[i], i * 2);
+    #[test]
+    fn test_map_multithread() {
+        let result: Vec<usize> = Executor::multi_thread(3, "search-test")
+            .map(|i| Ok(i * 2), 0..10)
+            .unwrap();
+        assert_eq!(result.len(), 10);
+        for i in 0..10 {
+            assert_eq!(result[i], i * 2);
+        }
     }
 }

src/core/index.rs

@@ -1,19 +1,14 @@
-use super::pool::LeasedItem;
-use super::pool::Pool;
 use super::segment::create_segment;
 use super::segment::Segment;
-use core::searcher::Searcher;
 use core::Executor;
 use core::IndexMeta;
 use core::SegmentId;
 use core::SegmentMeta;
-use core::SegmentReader;
 use core::META_FILEPATH;
 use directory::ManagedDirectory;
 #[cfg(feature = "mmap")]
 use directory::MmapDirectory;
 use directory::INDEX_WRITER_LOCK;
-use directory::META_LOCK;
 use directory::{Directory, RAMDirectory};
 use error::DataCorruption;
 use error::TantivyError;
@@ -21,6 +16,8 @@ use indexer::index_writer::open_index_writer;
 use indexer::index_writer::HEAP_SIZE_MIN;
 use indexer::segment_updater::save_new_metas;
 use num_cpus;
+use reader::IndexReader;
+use reader::IndexReaderBuilder;
 use schema::Field;
 use schema::FieldType;
 use schema::Schema;
@@ -28,7 +25,6 @@ use serde_json;
 use std::borrow::BorrowMut;
 use std::fmt;
 use std::path::Path;
-use std::sync::atomic::{AtomicUsize, Ordering};
 use std::sync::Arc;
 use tokenizer::BoxedTokenizer;
 use tokenizer::TokenizerManager;
@@ -49,11 +45,10 @@ fn load_metas(directory: &Directory) -> Result<IndexMeta> {
 }
 
 /// Search Index
+#[derive(Clone)]
 pub struct Index {
     directory: ManagedDirectory,
     schema: Schema,
-    num_searchers: Arc<AtomicUsize>,
-    searcher_pool: Arc<Pool<Searcher>>,
     executor: Arc<Executor>,
     tokenizers: TokenizerManager,
 }
@@ -111,7 +106,6 @@ impl Index {
     }
 
     /// Opens or creates a new index in the provided directory
-    #[cfg(feature = "mmap")]
     pub fn open_or_create<Dir: Directory>(dir: Dir, schema: Schema) -> Result<Index> {
         if Index::exists(&dir) {
             let index = Index::open(dir)?;
@@ -159,16 +153,12 @@ impl Index {
     /// Creates a new index given a directory and an `IndexMeta`.
     fn create_from_metas(directory: ManagedDirectory, metas: &IndexMeta) -> Result<Index> {
         let schema = metas.schema.clone();
-        let n_cpus = num_cpus::get();
         let index = Index {
             directory,
             schema,
-            num_searchers: Arc::new(AtomicUsize::new(n_cpus)),
-            searcher_pool: Arc::new(Pool::new()),
             tokenizers: TokenizerManager::default(),
             executor: Arc::new(Executor::single_thread()),
         };
-        index.load_searchers()?;
         Ok(index)
     }
 
@@ -198,6 +188,22 @@ impl Index {
         }
     }
 
+    /// Create a default `IndexReader` for the given index.
+    ///
+    /// See [`Index.reader_builder()`](#method.reader_builder).
+    pub fn reader(&self) -> Result<IndexReader> {
+        self.reader_builder().try_into()
+    }
+
+    /// Create a `IndexReader` for the given index.
+    ///
+    /// Most project should create at most one reader for a given index.
+    /// This method is typically called only once per `Index` instance,
+    /// over the lifetime of most problem.
+    pub fn reader_builder(&self) -> IndexReaderBuilder {
+        IndexReaderBuilder::new(self.clone())
+    }
+
     /// Opens a new directory from an index path.
     #[cfg(feature = "mmap")]
     pub fn open_in_dir<P: AsRef<Path>>(directory_path: P) -> Result<Index> {
@@ -336,53 +342,6 @@ impl Index {
             .map(|segment_meta| segment_meta.id())
             .collect())
     }
-
-    /// Sets the number of searchers to use
-    ///
-    /// Only works after the next call to `load_searchers`
-    pub fn set_num_searchers(&mut self, num_searchers: usize) {
-        self.num_searchers.store(num_searchers, Ordering::Release);
-    }
-
-    /// Update searchers so that they reflect the state of the last
-    /// `.commit()`.
-    ///
-    /// If indexing happens in the same process as searching,
-    /// you most likely want to call `.load_searchers()` right after each
-    /// successful call to `.commit()`.
-    ///
-    /// If indexing and searching happen in different processes, the way to
-    /// get the freshest `index` at all time, is to watch `meta.json` and
-    /// call `load_searchers` whenever a changes happen.
-    pub fn load_searchers(&self) -> Result<()> {
-        let _meta_lock = self.directory().acquire_lock(&META_LOCK)?;
-        let searchable_segments = self.searchable_segments()?;
-        let segment_readers: Vec<SegmentReader> = searchable_segments
-            .iter()
-            .map(SegmentReader::open)
-            .collect::<Result<_>>()?;
-        let schema = self.schema();
-        let num_searchers: usize = self.num_searchers.load(Ordering::Acquire);
-        let searchers = (0..num_searchers)
-            .map(|_| Searcher::new(schema.clone(), self.clone(), segment_readers.clone()))
-            .collect();
-        self.searcher_pool.publish_new_generation(searchers);
-        Ok(())
-    }
-
-    /// Returns a searcher
-    ///
-    /// This method should be called every single time a search
-    /// query is performed.
-    /// The searchers are taken from a pool of `num_searchers` searchers.
-    /// If no searcher is available
-    /// this may block.
-    ///
-    /// The same searcher must be used for a given query, as it ensures
-    /// the use of a consistent segment set.
-    pub fn searcher(&self) -> LeasedItem<Searcher> {
-        self.searcher_pool.acquire()
-    }
 }
 
 impl fmt::Debug for Index {
@@ -391,29 +350,24 @@ impl fmt::Debug for Index {
     }
 }
 
-impl Clone for Index {
-    fn clone(&self) -> Index {
-        Index {
-            directory: self.directory.clone(),
-            schema: self.schema.clone(),
-            num_searchers: Arc::clone(&self.num_searchers),
-            searcher_pool: Arc::clone(&self.searcher_pool),
-            tokenizers: self.tokenizers.clone(),
-            executor: self.executor.clone(),
-        }
-    }
-}
-
 #[cfg(test)]
 mod tests {
     use directory::RAMDirectory;
-    use schema::{Schema, INT_INDEXED, TEXT};
+    use schema::Field;
+    use schema::{Schema, INDEXED, TEXT};
+    use std::path::PathBuf;
+    use std::thread;
+    use std::time::Duration;
+    use tempdir::TempDir;
     use Index;
+    use IndexReader;
+    use IndexWriter;
+    use ReloadPolicy;
 
     #[test]
     fn test_indexer_for_field() {
         let mut schema_builder = Schema::builder();
-        let num_likes_field = schema_builder.add_u64_field("num_likes", INT_INDEXED);
+        let num_likes_field = schema_builder.add_u64_field("num_likes", INDEXED);
         let body_field = schema_builder.add_text_field("body", TEXT);
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
@@ -471,7 +425,109 @@ mod tests {
 
     fn throw_away_schema() -> Schema {
         let mut schema_builder = Schema::builder();
-        let _ = schema_builder.add_u64_field("num_likes", INT_INDEXED);
+        let _ = schema_builder.add_u64_field("num_likes", INDEXED);
         schema_builder.build()
     }
+
+    #[test]
+    fn test_index_on_commit_reload_policy() {
+        let schema = throw_away_schema();
+        let field = schema.get_field("num_likes").unwrap();
+        let index = Index::create_in_ram(schema);
+        let reader = index
+            .reader_builder()
+            .reload_policy(ReloadPolicy::OnCommit)
+            .try_into()
+            .unwrap();
+        assert_eq!(reader.searcher().num_docs(), 0);
+        let mut writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
+        test_index_on_commit_reload_policy_aux(field, &mut writer, &reader);
+    }
+
+    #[test]
+    fn test_index_on_commit_reload_policy_mmap() {
+        let schema = throw_away_schema();
+        let field = schema.get_field("num_likes").unwrap();
+        let tempdir = TempDir::new("index").unwrap();
+        let tempdir_path = PathBuf::from(tempdir.path());
+        let index = Index::create_in_dir(&tempdir_path, schema).unwrap();
+        let mut writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
+        writer.commit().unwrap();
+        let reader = index
+            .reader_builder()
+            .reload_policy(ReloadPolicy::OnCommit)
+            .try_into()
+            .unwrap();
+        assert_eq!(reader.searcher().num_docs(), 0);
+        test_index_on_commit_reload_policy_aux(field, &mut writer, &reader);
+    }
+
+    #[test]
+    fn test_index_manual_policy_mmap() {
+        let schema = throw_away_schema();
+        let field = schema.get_field("num_likes").unwrap();
+        let index = Index::create_from_tempdir(schema).unwrap();
+        let mut writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
+        writer.commit().unwrap();
+        let reader = index
+            .reader_builder()
+            .reload_policy(ReloadPolicy::Manual)
+            .try_into()
+            .unwrap();
+        assert_eq!(reader.searcher().num_docs(), 0);
+        writer.add_document(doc!(field=>1u64));
+        writer.commit().unwrap();
+        thread::sleep(Duration::from_millis(500));
+        assert_eq!(reader.searcher().num_docs(), 0);
+        reader.reload().unwrap();
+        assert_eq!(reader.searcher().num_docs(), 1);
+    }
+
+    #[test]
+    fn test_index_on_commit_reload_policy_different_directories() {
+        let schema = throw_away_schema();
+        let field = schema.get_field("num_likes").unwrap();
+        let tempdir = TempDir::new("index").unwrap();
+        let tempdir_path = PathBuf::from(tempdir.path());
+        let write_index = Index::create_in_dir(&tempdir_path, schema).unwrap();
+        let read_index = Index::open_in_dir(&tempdir_path).unwrap();
+        let reader = read_index
+            .reader_builder()
+            .reload_policy(ReloadPolicy::OnCommit)
+            .try_into()
+            .unwrap();
+        assert_eq!(reader.searcher().num_docs(), 0);
+        let mut writer = write_index.writer_with_num_threads(1, 3_000_000).unwrap();
+        test_index_on_commit_reload_policy_aux(field, &mut writer, &reader);
+    }
+
+    fn test_index_on_commit_reload_policy_aux(
+        field: Field,
+        writer: &mut IndexWriter,
+        reader: &IndexReader,
+    ) {
+        assert_eq!(reader.searcher().num_docs(), 0);
+        writer.add_document(doc!(field=>1u64));
+        writer.commit().unwrap();
+        let mut count = 0;
+        for _ in 0..100 {
+            count = reader.searcher().num_docs();
+            if count > 0 {
+                break;
+            }
+            thread::sleep(Duration::from_millis(100));
+        }
+        assert_eq!(count, 1);
+        writer.add_document(doc!(field=>2u64));
+        writer.commit().unwrap();
+        let mut count = 0;
+        for _ in 0..10 {
+            count = reader.searcher().num_docs();
+            if count > 1 {
+                break;
+            }
+            thread::sleep(Duration::from_millis(100));
+        }
+        assert_eq!(count, 2);
+    }
 }

src/core/mod.rs

@@ -2,7 +2,6 @@ mod executor;
 pub mod index;
 mod index_meta;
 mod inverted_index_reader;
-mod pool;
 pub mod searcher;
 mod segment;
 mod segment_component;
@@ -25,6 +24,7 @@ pub use self::segment_reader::SegmentReader;
 use std::path::PathBuf;
 
 lazy_static! {
+
     /// The meta file contains all the information about the list of segments and the schema
     /// of the index.
     pub static ref META_FILEPATH: PathBuf = PathBuf::from("meta.json");

src/core/segment_id.rs

@@ -19,7 +19,7 @@ pub struct SegmentId(Uuid);
 #[cfg(test)]
 lazy_static! {
     static ref AUTO_INC_COUNTER: atomic::AtomicUsize = atomic::AtomicUsize::default();
-    static ref EMPTY_ARR: [u8; 8] = [0u8; 8];
+    static ref ZERO_ARRAY: [u8; 8] = [0u8; 8];
 }
 
 // During tests, we generate the segment id in a autoincrement manner
@@ -30,7 +30,7 @@ lazy_static! {
 #[cfg(test)]
 fn create_uuid() -> Uuid {
     let new_auto_inc_id = (*AUTO_INC_COUNTER).fetch_add(1, atomic::Ordering::SeqCst);
-    Uuid::from_fields(new_auto_inc_id as u32, 0, 0, &*EMPTY_ARR).unwrap()
+    Uuid::from_fields(new_auto_inc_id as u32, 0, 0, &*ZERO_ARRAY).unwrap()
 }
 
 #[cfg(not(test))]

src/core/segment_reader.rs

@@ -477,9 +477,7 @@ mod test {
             // ok, now we should have a deleted doc
             index_writer2.commit().unwrap();
         }
-
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        let searcher = index.reader().unwrap().searcher();
         let docs: Vec<DocId> = searcher.segment_reader(0).doc_ids_alive().collect();
         assert_eq!(vec![0u32, 2u32], docs);
     }

src/directory/directory.rs

@@ -1,6 +1,8 @@
 use directory::directory_lock::Lock;
 use directory::error::LockError;
 use directory::error::{DeleteError, OpenReadError, OpenWriteError};
+use directory::WatchCallback;
+use directory::WatchHandle;
 use directory::{ReadOnlySource, WritePtr};
 use std::fmt;
 use std::io;
@@ -187,6 +189,22 @@ pub trait Directory: DirectoryClone + fmt::Debug + Send + Sync + 'static {
             }
         }
     }
+
+    /// Registers a callback that will be called whenever a change on the `meta.json`
+    /// using the `atomic_write` API is detected.
+    ///
+    /// The behavior when using `.watch()` on a file using `.open_write(...)` is, on the other
+    /// hand, undefined.
+    ///
+    /// The file will be watched for the lifetime of the returned `WatchHandle`. The caller is
+    /// required to keep it.
+    /// It does not override previous callbacks. When the file is modified, all callback that are
+    /// registered (and whose `WatchHandle` is still alive) are triggered.
+    ///
+    /// Internally, tantivy only uses this API to detect new commits to implement the
+    /// `OnCommit` `ReloadPolicy`. Not implementing watch in a `Directory` only prevents the
+    /// `OnCommit` `ReloadPolicy` to work properly.
+    fn watch(&self, watch_callback: WatchCallback) -> WatchHandle;
 }
 
 /// DirectoryClone

src/directory/directory_lock.rs

@@ -43,7 +43,7 @@ lazy_static! {
         is_blocking: false
     };
     /// The meta lock file is here to protect the segment files being opened by
-    /// `.load_searchers()` from being garbage collected.
+    /// `IndexReader::reload()` from being garbage collected.
     /// It makes it possible for another process to safely consume
     /// our index in-writing. Ideally, we may have prefered `RWLock` semantics
     /// here, but it is difficult to achieve on Windows.

src/directory/error.rs

@@ -73,6 +73,14 @@ pub enum OpenDirectoryError {
     DoesNotExist(PathBuf),
     /// The path exists but is not a directory.
     NotADirectory(PathBuf),
+    /// IoError
+    IoError(io::Error),
+}
+
+impl From<io::Error> for OpenDirectoryError {
+    fn from(io_err: io::Error) -> Self {
+        OpenDirectoryError::IoError(io_err)
+    }
 }
 
 impl fmt::Display for OpenDirectoryError {
@@ -84,6 +92,11 @@ impl fmt::Display for OpenDirectoryError {
             OpenDirectoryError::NotADirectory(ref path) => {
                 write!(f, "the path '{:?}' exists but is not a directory", path)
             }
+            OpenDirectoryError::IoError(ref err) => write!(
+                f,
+                "IOError while trying to open/create the directory. {:?}",
+                err
+            ),
         }
     }
 }

src/directory/managed_directory.rs

@@ -4,6 +4,7 @@ use directory::DirectoryLock;
 use directory::Lock;
 use directory::META_LOCK;
 use directory::{ReadOnlySource, WritePtr};
+use directory::{WatchCallback, WatchHandle};
 use error::DataCorruption;
 use serde_json;
 use std::collections::HashSet;
@@ -241,6 +242,10 @@ impl Directory for ManagedDirectory {
     fn acquire_lock(&self, lock: &Lock) -> result::Result<DirectoryLock, LockError> {
         self.directory.acquire_lock(lock)
     }
+
+    fn watch(&self, watch_callback: WatchCallback) -> WatchHandle {
+        self.directory.watch(watch_callback)
+    }
 }
 
 impl Clone for ManagedDirectory {

src/directory/mmap_directory.rs

@@ -1,18 +1,25 @@
 extern crate fs2;
+extern crate notify;
 
 use self::fs2::FileExt;
+use self::notify::RawEvent;
+use self::notify::RecursiveMode;
+use self::notify::Watcher;
 use atomicwrites;
 use common::make_io_err;
+use core::META_FILEPATH;
 use directory::error::LockError;
 use directory::error::{DeleteError, IOError, OpenDirectoryError, OpenReadError, OpenWriteError};
-use directory::shared_vec_slice::SharedVecSlice;
+use directory::read_only_source::BoxedData;
 use directory::Directory;
 use directory::DirectoryLock;
 use directory::Lock;
 use directory::ReadOnlySource;
+use directory::WatchCallback;
+use directory::WatchCallbackList;
+use directory::WatchHandle;
 use directory::WritePtr;
-use fst::raw::MmapReadOnly;
-use std::collections::hash_map::Entry as HashMapEntry;
+use memmap::Mmap;
 use std::collections::HashMap;
 use std::convert::From;
 use std::fmt;
@@ -22,14 +29,17 @@ use std::io::{self, Seek, SeekFrom};
 use std::io::{BufWriter, Read, Write};
 use std::path::{Path, PathBuf};
 use std::result;
+use std::sync::mpsc::{channel, Receiver, Sender};
 use std::sync::Arc;
+use std::sync::Mutex;
 use std::sync::RwLock;
+use std::sync::Weak;
+use std::thread;
 use tempdir::TempDir;
 
 /// Returns None iff the file exists, can be read, but is empty (and hence
-/// cannot be mmapped).
-///
-fn open_mmap(full_path: &Path) -> result::Result<Option<MmapReadOnly>, OpenReadError> {
+/// cannot be mmapped)
+fn open_mmap(full_path: &Path) -> result::Result<Option<Mmap>, OpenReadError> {
     let file = File::open(full_path).map_err(|e| {
         if e.kind() == io::ErrorKind::NotFound {
             OpenReadError::FileDoesNotExist(full_path.to_owned())
@@ -48,7 +58,7 @@ fn open_mmap(full_path: &Path) -> result::Result<Option<MmapReadOnly>, OpenReadE
         return Ok(None);
     }
     unsafe {
-        MmapReadOnly::open(&file)
+        memmap::Mmap::map(&file)
             .map(Some)
             .map_err(|e| From::from(IOError::with_path(full_path.to_owned(), e)))
     }
@@ -71,7 +81,7 @@ pub struct CacheInfo {
 
 struct MmapCache {
     counters: CacheCounters,
-    cache: HashMap<PathBuf, MmapReadOnly>,
+    cache: HashMap<PathBuf, Weak<BoxedData>>,
 }
 
 impl Default for MmapCache {
@@ -84,12 +94,7 @@ impl Default for MmapCache {
 }
 
 impl MmapCache {
-    /// Removes a `MmapReadOnly` entry from the mmap cache.
-    fn discard_from_cache(&mut self, full_path: &Path) -> bool {
-        self.cache.remove(full_path).is_some()
-    }
-
-    fn get_info(&mut self) -> CacheInfo {
+    fn get_info(&self) -> CacheInfo {
         let paths: Vec<PathBuf> = self.cache.keys().cloned().collect();
         CacheInfo {
             counters: self.counters.clone(),
@@ -97,26 +102,108 @@ impl MmapCache {
         }
     }
 
-    fn get_mmap(&mut self, full_path: &Path) -> Result<Option<MmapReadOnly>, OpenReadError> {
-        Ok(match self.cache.entry(full_path.to_owned()) {
-            HashMapEntry::Occupied(occupied_entry) => {
-                let mmap = occupied_entry.get();
+    fn remove_weak_ref(&mut self) {
+        let keys_to_remove: Vec<PathBuf> = self
+            .cache
+            .iter()
+            .filter(|(_, mmap_weakref)| mmap_weakref.upgrade().is_none())
+            .map(|(key, _)| key.clone())
+            .collect();
+        for key in keys_to_remove {
+            self.cache.remove(&key);
+        }
+    }
+
+    // Returns None if the file exists but as a len of 0 (and hence is not mmappable).
+    fn get_mmap(&mut self, full_path: &Path) -> Result<Option<Arc<BoxedData>>, OpenReadError> {
+        if let Some(mmap_weak) = self.cache.get(full_path) {
+            if let Some(mmap_arc) = mmap_weak.upgrade() {
                 self.counters.hit += 1;
-                Some(mmap.clone())
-            }
-            HashMapEntry::Vacant(vacant_entry) => {
-                self.counters.miss += 1;
-                if let Some(mmap) = open_mmap(full_path)? {
-                    vacant_entry.insert(mmap.clone());
-                    Some(mmap)
-                } else {
-                    None
-                }
+                return Ok(Some(mmap_arc));
             }
+        }
+        self.cache.remove(full_path);
+        self.counters.miss += 1;
+        Ok(if let Some(mmap) = open_mmap(full_path)? {
+            let mmap_arc: Arc<BoxedData> = Arc::new(Box::new(mmap));
+            let mmap_weak = Arc::downgrade(&mmap_arc);
+            self.cache.insert(full_path.to_owned(), mmap_weak);
+            Some(mmap_arc)
+        } else {
+            None
         })
     }
 }
 
+struct InnerWatcherWrapper {
+    _watcher: Mutex<notify::RecommendedWatcher>,
+    watcher_router: WatchCallbackList,
+}
+
+impl InnerWatcherWrapper {
+    pub fn new(path: &Path) -> Result<(Self, Receiver<notify::RawEvent>), notify::Error> {
+        let (tx, watcher_recv): (Sender<RawEvent>, Receiver<RawEvent>) = channel();
+        // We need to initialize the
+        let mut watcher = notify::raw_watcher(tx)?;
+        watcher.watch(path, RecursiveMode::Recursive)?;
+        let inner = InnerWatcherWrapper {
+            _watcher: Mutex::new(watcher),
+            watcher_router: Default::default(),
+        };
+        Ok((inner, watcher_recv))
+    }
+}
+
+#[derive(Clone)]
+pub(crate) struct WatcherWrapper {
+    inner: Arc<InnerWatcherWrapper>,
+}
+
+impl WatcherWrapper {
+    pub fn new(path: &Path) -> Result<Self, OpenDirectoryError> {
+        let (inner, watcher_recv) = InnerWatcherWrapper::new(path).map_err(|err| match err {
+            notify::Error::PathNotFound => OpenDirectoryError::DoesNotExist(path.to_owned()),
+            _ => {
+                panic!("Unknown error while starting watching directory {:?}", path);
+            }
+        })?;
+        let watcher_wrapper = WatcherWrapper {
+            inner: Arc::new(inner),
+        };
+        let watcher_wrapper_clone = watcher_wrapper.clone();
+        thread::Builder::new()
+            .name("meta-file-watch-thread".to_string())
+            .spawn(move || {
+                loop {
+                    match watcher_recv.recv().map(|evt| evt.path) {
+                        Ok(Some(changed_path)) => {
+                            // ... Actually subject to false positive.
+                            // We might want to be more accurate than this at one point.
+                            if let Some(filename) = changed_path.file_name() {
+                                if filename == *META_FILEPATH {
+                                    watcher_wrapper_clone.inner.watcher_router.broadcast();
+                                }
+                            }
+                        }
+                        Ok(None) => {
+                            // not an event we are interested in.
+                        }
+                        Err(_e) => {
+                            // the watch send channel was dropped
+                            break;
+                        }
+                    }
+                }
+            })
+            .expect("Failed to spawn thread to watch meta.json");
+        Ok(watcher_wrapper)
+    }
+
+    pub fn watch(&mut self, watch_callback: WatchCallback) -> WatchHandle {
+        self.inner.watcher_router.subscribe(watch_callback)
+    }
+}
+
 /// Directory storing data in files, read via mmap.
 ///
 /// The Mmap object are cached to limit the
@@ -131,31 +218,62 @@ impl MmapCache {
 /// On Windows the semantics are again different.
 #[derive(Clone)]
 pub struct MmapDirectory {
+    inner: Arc<MmapDirectoryInner>,
+}
+
+struct MmapDirectoryInner {
     root_path: PathBuf,
-    mmap_cache: Arc<RwLock<MmapCache>>,
-    _temp_directory: Arc<Option<TempDir>>,
+    mmap_cache: RwLock<MmapCache>,
+    _temp_directory: Option<TempDir>,
+    watcher: RwLock<WatcherWrapper>,
+}
+
+impl MmapDirectoryInner {
+    fn new(
+        root_path: PathBuf,
+        temp_directory: Option<TempDir>,
+    ) -> Result<MmapDirectoryInner, OpenDirectoryError> {
+        let watch_wrapper = WatcherWrapper::new(&root_path)?;
+        let mmap_directory_inner = MmapDirectoryInner {
+            root_path,
+            mmap_cache: Default::default(),
+            _temp_directory: temp_directory,
+            watcher: RwLock::new(watch_wrapper),
+        };
+        Ok(mmap_directory_inner)
+    }
+
+    fn watch(&self, watch_callback: WatchCallback) -> WatchHandle {
+        let mut wlock = self.watcher.write().unwrap();
+        wlock.watch(watch_callback)
+    }
 }
 
 impl fmt::Debug for MmapDirectory {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "MmapDirectory({:?})", self.root_path)
+        write!(f, "MmapDirectory({:?})", self.inner.root_path)
     }
 }
 
 impl MmapDirectory {
+    fn new(
+        root_path: PathBuf,
+        temp_directory: Option<TempDir>,
+    ) -> Result<MmapDirectory, OpenDirectoryError> {
+        let inner = MmapDirectoryInner::new(root_path, temp_directory)?;
+        Ok(MmapDirectory {
+            inner: Arc::new(inner),
+        })
+    }
+
     /// Creates a new MmapDirectory in a temporary directory.
     ///
     /// This is mostly useful to test the MmapDirectory itself.
     /// For your unit tests, prefer the RAMDirectory.
-    pub fn create_from_tempdir() -> io::Result<MmapDirectory> {
-        let tempdir = TempDir::new("index")?;
+    pub fn create_from_tempdir() -> Result<MmapDirectory, OpenDirectoryError> {
+        let tempdir = TempDir::new("index").map_err(OpenDirectoryError::IoError)?;
         let tempdir_path = PathBuf::from(tempdir.path());
-        let directory = MmapDirectory {
-            root_path: tempdir_path,
-            mmap_cache: Arc::new(RwLock::new(MmapCache::default())),
-            _temp_directory: Arc::new(Some(tempdir)),
-        };
-        Ok(directory)
+        MmapDirectory::new(tempdir_path, Some(tempdir))
     }
 
     /// Opens a MmapDirectory in a directory.
@@ -173,18 +291,14 @@ impl MmapDirectory {
                 directory_path,
             )))
         } else {
-            Ok(MmapDirectory {
-                root_path: PathBuf::from(directory_path),
-                mmap_cache: Arc::new(RwLock::new(MmapCache::default())),
-                _temp_directory: Arc::new(None),
-            })
+            Ok(MmapDirectory::new(PathBuf::from(directory_path), None)?)
         }
     }
 
     /// Joins a relative_path to the directory `root_path`
     /// to create a proper complete `filepath`.
     fn resolve_path(&self, relative_path: &Path) -> PathBuf {
-        self.root_path.join(relative_path)
+        self.inner.root_path.join(relative_path)
     }
 
     /// Sync the root directory.
@@ -209,7 +323,7 @@ impl MmapDirectory {
                 .custom_flags(winbase::FILE_FLAG_BACKUP_SEMANTICS);
         }
 
-        let fd = open_opts.open(&self.root_path)?;
+        let fd = open_opts.open(&self.inner.root_path)?;
         fd.sync_all()?;
         Ok(())
     }
@@ -219,9 +333,15 @@ impl MmapDirectory {
     ///
     /// The `MmapDirectory` embeds a `MmapDirectory`
     /// to avoid multiplying the `mmap` system calls.
-    pub fn get_cache_info(&mut self) -> CacheInfo {
-        self.mmap_cache
+    pub fn get_cache_info(&self) -> CacheInfo {
+        self.inner
+            .mmap_cache
             .write()
+            .expect("mmap cache lock is poisoned")
+            .remove_weak_ref();
+        self.inner
+            .mmap_cache
+            .read()
             .expect("Mmap cache lock is poisoned.")
             .get_info()
     }
@@ -274,7 +394,7 @@ impl Directory for MmapDirectory {
         debug!("Open Read {:?}", path);
         let full_path = self.resolve_path(path);
 
-        let mut mmap_cache = self.mmap_cache.write().map_err(|_| {
+        let mut mmap_cache = self.inner.mmap_cache.write().map_err(|_| {
             let msg = format!(
                 "Failed to acquired write lock \
                  on mmap cache while reading {:?}",
@@ -282,11 +402,34 @@ impl Directory for MmapDirectory {
             );
             IOError::with_path(path.to_owned(), make_io_err(msg))
         })?;
-
         Ok(mmap_cache
             .get_mmap(&full_path)?
-            .map(ReadOnlySource::Mmap)
-            .unwrap_or_else(|| ReadOnlySource::Anonymous(SharedVecSlice::empty())))
+            .map(ReadOnlySource::from)
+            .unwrap_or_else(ReadOnlySource::empty))
+    }
+
+    /// Any entry associated to the path in the mmap will be
+    /// removed before the file is deleted.
+    fn delete(&self, path: &Path) -> result::Result<(), DeleteError> {
+        debug!("Deleting file {:?}", path);
+        let full_path = self.resolve_path(path);
+        match fs::remove_file(&full_path) {
+            Ok(_) => self
+                .sync_directory()
+                .map_err(|e| IOError::with_path(path.to_owned(), e).into()),
+            Err(e) => {
+                if e.kind() == io::ErrorKind::NotFound {
+                    Err(DeleteError::FileDoesNotExist(path.to_owned()))
+                } else {
+                    Err(IOError::with_path(path.to_owned(), e).into())
+                }
+            }
+        }
+    }
+
+    fn exists(&self, path: &Path) -> bool {
+        let full_path = self.resolve_path(path);
+        full_path.exists()
     }
 
     fn open_write(&mut self, path: &Path) -> Result<WritePtr, OpenWriteError> {
@@ -319,44 +462,6 @@ impl Directory for MmapDirectory {
         Ok(BufWriter::new(Box::new(writer)))
     }
 
-    /// Any entry associated to the path in the mmap will be
-    /// removed before the file is deleted.
-    fn delete(&self, path: &Path) -> result::Result<(), DeleteError> {
-        debug!("Deleting file {:?}", path);
-        let full_path = self.resolve_path(path);
-        let mut mmap_cache = self.mmap_cache.write().map_err(|_| {
-            let msg = format!(
-                "Failed to acquired write lock \
-                 on mmap cache while deleting {:?}",
-                path
-            );
-            IOError::with_path(path.to_owned(), make_io_err(msg))
-        })?;
-        mmap_cache.discard_from_cache(path);
-
-        // Removing the entry in the MMap cache.
-        // The munmap will appear on Drop,
-        // when the last reference is gone.
-        mmap_cache.cache.remove(&full_path);
-        match fs::remove_file(&full_path) {
-            Ok(_) => self
-                .sync_directory()
-                .map_err(|e| IOError::with_path(path.to_owned(), e).into()),
-            Err(e) => {
-                if e.kind() == io::ErrorKind::NotFound {
-                    Err(DeleteError::FileDoesNotExist(path.to_owned()))
-                } else {
-                    Err(IOError::with_path(path.to_owned(), e).into())
-                }
-            }
-        }
-    }
-
-    fn exists(&self, path: &Path) -> bool {
-        let full_path = self.resolve_path(path);
-        full_path.exists()
-    }
-
     fn atomic_read(&self, path: &Path) -> Result<Vec<u8>, OpenReadError> {
         let full_path = self.resolve_path(path);
         let mut buffer = Vec::new();
@@ -403,6 +508,10 @@ impl Directory for MmapDirectory {
             _file: file,
         })))
     }
+
+    fn watch(&self, watch_callback: WatchCallback) -> WatchHandle {
+        self.inner.watch(watch_callback)
+    }
 }
 
 #[cfg(test)]
@@ -412,6 +521,13 @@ mod tests {
     // The following tests are specific to the MmapDirectory
 
     use super::*;
+    use schema::{Schema, SchemaBuilder, TEXT};
+    use std::fs;
+    use std::sync::atomic::{AtomicUsize, Ordering};
+    use std::thread;
+    use std::time::Duration;
+    use Index;
+    use ReloadPolicy;
 
     #[test]
     fn test_open_non_existant_path() {
@@ -436,7 +552,7 @@ mod tests {
 
     #[test]
     fn test_cache() {
-        let content = "abc".as_bytes();
+        let content = b"abc";
 
         // here we test if the cache releases
         // mmaps correctly.
@@ -452,26 +568,104 @@ mod tests {
                 w.flush().unwrap();
             }
         }
-        {
-            for (i, path) in paths.iter().enumerate() {
-                let _r = mmap_directory.open_read(path).unwrap();
-                assert_eq!(mmap_directory.get_cache_info().mmapped.len(), i + 1);
-            }
-            for path in paths.iter() {
-                let _r = mmap_directory.open_read(path).unwrap();
-                assert_eq!(mmap_directory.get_cache_info().mmapped.len(), num_paths);
-            }
-            for (i, path) in paths.iter().enumerate() {
-                mmap_directory.delete(path).unwrap();
-                assert_eq!(
-                    mmap_directory.get_cache_info().mmapped.len(),
-                    num_paths - i - 1
-                );
-            }
+
+        let mut keep = vec![];
+        for (i, path) in paths.iter().enumerate() {
+            keep.push(mmap_directory.open_read(path).unwrap());
+            assert_eq!(mmap_directory.get_cache_info().mmapped.len(), i + 1);
+        }
+        assert_eq!(mmap_directory.get_cache_info().counters.hit, 0);
+        assert_eq!(mmap_directory.get_cache_info().counters.miss, 10);
+        assert_eq!(mmap_directory.get_cache_info().mmapped.len(), 10);
+        for path in paths.iter() {
+            let _r = mmap_directory.open_read(path).unwrap();
+            assert_eq!(mmap_directory.get_cache_info().mmapped.len(), num_paths);
         }
         assert_eq!(mmap_directory.get_cache_info().counters.hit, 10);
         assert_eq!(mmap_directory.get_cache_info().counters.miss, 10);
+        assert_eq!(mmap_directory.get_cache_info().mmapped.len(), 10);
+
+        for path in paths.iter() {
+            let _r = mmap_directory.open_read(path).unwrap();
+            assert_eq!(mmap_directory.get_cache_info().mmapped.len(), 10);
+        }
+
+        assert_eq!(mmap_directory.get_cache_info().counters.hit, 20);
+        assert_eq!(mmap_directory.get_cache_info().counters.miss, 10);
+        assert_eq!(mmap_directory.get_cache_info().mmapped.len(), 10);
+        drop(keep);
+        for path in paths.iter() {
+            let _r = mmap_directory.open_read(path).unwrap();
+            assert_eq!(mmap_directory.get_cache_info().mmapped.len(), 1);
+        }
+        assert_eq!(mmap_directory.get_cache_info().counters.hit, 20);
+        assert_eq!(mmap_directory.get_cache_info().counters.miss, 20);
+        assert_eq!(mmap_directory.get_cache_info().mmapped.len(), 0);
+
+        for path in &paths {
+            mmap_directory.delete(path).unwrap();
+        }
+        assert_eq!(mmap_directory.get_cache_info().counters.hit, 20);
+        assert_eq!(mmap_directory.get_cache_info().counters.miss, 20);
+        assert_eq!(mmap_directory.get_cache_info().mmapped.len(), 0);
+        for path in paths.iter() {
+            assert!(mmap_directory.open_read(path).is_err());
+        }
+        assert_eq!(mmap_directory.get_cache_info().counters.hit, 20);
+        assert_eq!(mmap_directory.get_cache_info().counters.miss, 30);
         assert_eq!(mmap_directory.get_cache_info().mmapped.len(), 0);
     }
 
+    #[test]
+    fn test_watch_wrapper() {
+        let counter: Arc<AtomicUsize> = Default::default();
+        let counter_clone = counter.clone();
+        let tmp_dir: TempDir = tempdir::TempDir::new("test_watch_wrapper").unwrap();
+        let tmp_dirpath = tmp_dir.path().to_owned();
+        let mut watch_wrapper = WatcherWrapper::new(&tmp_dirpath).unwrap();
+        let tmp_file = tmp_dirpath.join("coucou");
+        let _handle = watch_wrapper.watch(Box::new(move || {
+            counter_clone.fetch_add(1, Ordering::SeqCst);
+        }));
+        assert_eq!(counter.load(Ordering::SeqCst), 0);
+        fs::write(&tmp_file, b"whateverwilldo").unwrap();
+        thread::sleep(Duration::new(0, 1_000u32));
+    }
+
+    #[test]
+    fn test_mmap_released() {
+        let mmap_directory = MmapDirectory::create_from_tempdir().unwrap();
+        let mut schema_builder: SchemaBuilder = Schema::builder();
+        let text_field = schema_builder.add_text_field("text", TEXT);
+        let schema = schema_builder.build();
+        {
+            let index = Index::create(mmap_directory.clone(), schema).unwrap();
+            let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
+            for _num_commits in 0..16 {
+                for _ in 0..10 {
+                    index_writer.add_document(doc!(text_field=>"abc"));
+                }
+                index_writer.commit().unwrap();
+            }
+            let reader = index
+                .reader_builder()
+                .reload_policy(ReloadPolicy::Manual)
+                .try_into()
+                .unwrap();
+            for _ in 0..30 {
+                index_writer.add_document(doc!(text_field=>"abc"));
+                index_writer.commit().unwrap();
+                reader.reload().unwrap();
+            }
+            index_writer.wait_merging_threads().unwrap();
+            reader.reload().unwrap();
+            let num_segments = reader.searcher().segment_readers().len();
+            assert_eq!(num_segments, 4);
+            assert_eq!(
+                num_segments * 7,
+                mmap_directory.get_cache_info().mmapped.len()
+            );
+        }
+        assert_eq!(mmap_directory.get_cache_info().mmapped.len(), 0);
+    }
 }

src/directory/mod.rs

@@ -12,7 +12,7 @@ mod directory_lock;
 mod managed_directory;
 mod ram_directory;
 mod read_only_source;
-mod shared_vec_slice;
+mod watch_event_router;
 
 /// Errors specific to the directory module.
 pub mod error;
@@ -22,6 +22,8 @@ pub use self::directory::{Directory, DirectoryClone};
 pub use self::directory_lock::{Lock, INDEX_WRITER_LOCK, META_LOCK};
 pub use self::ram_directory::RAMDirectory;
 pub use self::read_only_source::ReadOnlySource;
+pub(crate) use self::watch_event_router::WatchCallbackList;
+pub use self::watch_event_router::{WatchCallback, WatchHandle};
 use std::io::{BufWriter, Seek, Write};
 
 #[cfg(feature = "mmap")]

src/directory/ram_directory.rs

@@ -1,8 +1,8 @@
-use super::shared_vec_slice::SharedVecSlice;
-use common::make_io_err;
-use directory::error::{DeleteError, IOError, OpenReadError, OpenWriteError};
+use core::META_FILEPATH;
+use directory::error::{DeleteError, OpenReadError, OpenWriteError};
+use directory::WatchCallbackList;
 use directory::WritePtr;
-use directory::{Directory, ReadOnlySource};
+use directory::{Directory, ReadOnlySource, WatchCallback, WatchHandle};
 use std::collections::HashMap;
 use std::fmt;
 use std::io::{self, BufWriter, Cursor, Seek, SeekFrom, Write};
@@ -22,13 +22,13 @@ use std::sync::{Arc, RwLock};
 ///
 struct VecWriter {
     path: PathBuf,
-    shared_directory: InnerDirectory,
+    shared_directory: RAMDirectory,
     data: Cursor<Vec<u8>>,
     is_flushed: bool,
 }
 
 impl VecWriter {
-    fn new(path_buf: PathBuf, shared_directory: InnerDirectory) -> VecWriter {
+    fn new(path_buf: PathBuf, shared_directory: RAMDirectory) -> VecWriter {
         VecWriter {
             path: path_buf,
             data: Cursor::new(Vec::new()),
@@ -64,75 +64,44 @@ impl Write for VecWriter {
 
     fn flush(&mut self) -> io::Result<()> {
         self.is_flushed = true;
-        self.shared_directory
-            .write(self.path.clone(), self.data.get_ref())?;
+        let mut fs = self.shared_directory.fs.write().unwrap();
+        fs.write(self.path.clone(), self.data.get_ref());
         Ok(())
     }
 }
 
-#[derive(Clone)]
-struct InnerDirectory(Arc<RwLock<HashMap<PathBuf, Arc<Vec<u8>>>>>);
+#[derive(Default)]
+struct InnerDirectory {
+    fs: HashMap<PathBuf, ReadOnlySource>,
+    watch_router: WatchCallbackList,
+}
 
 impl InnerDirectory {
-    fn new() -> InnerDirectory {
-        InnerDirectory(Arc::new(RwLock::new(HashMap::new())))
-    }
-
-    fn write(&self, path: PathBuf, data: &[u8]) -> io::Result<bool> {
-        let mut map = self.0.write().map_err(|_| {
-            make_io_err(format!(
-                "Failed to lock the directory, when trying to write {:?}",
-                path
-            ))
-        })?;
-        let prev_value = map.insert(path, Arc::new(Vec::from(data)));
-        Ok(prev_value.is_some())
+    fn write(&mut self, path: PathBuf, data: &[u8]) -> bool {
+        let data = ReadOnlySource::new(Vec::from(data));
+        self.fs.insert(path, data).is_some()
     }
 
     fn open_read(&self, path: &Path) -> Result<ReadOnlySource, OpenReadError> {
-        self.0
-            .read()
-            .map_err(|_| {
-                let msg = format!(
-                    "Failed to acquire read lock for the \
-                     directory when trying to read {:?}",
-                    path
-                );
-                let io_err = make_io_err(msg);
-                OpenReadError::IOError(IOError::with_path(path.to_owned(), io_err))
-            })
-            .and_then(|readable_map| {
-                readable_map
-                    .get(path)
-                    .ok_or_else(|| OpenReadError::FileDoesNotExist(PathBuf::from(path)))
-                    .map(Arc::clone)
-                    .map(|data| ReadOnlySource::Anonymous(SharedVecSlice::new(data)))
-            })
+        self.fs
+            .get(path)
+            .ok_or_else(|| OpenReadError::FileDoesNotExist(PathBuf::from(path)))
+            .map(|el| el.clone())
     }
 
-    fn delete(&self, path: &Path) -> result::Result<(), DeleteError> {
-        self.0
-            .write()
-            .map_err(|_| {
-                let msg = format!(
-                    "Failed to acquire write lock for the \
-                     directory when trying to delete {:?}",
-                    path
-                );
-                let io_err = make_io_err(msg);
-                DeleteError::IOError(IOError::with_path(path.to_owned(), io_err))
-            })
-            .and_then(|mut writable_map| match writable_map.remove(path) {
-                Some(_) => Ok(()),
-                None => Err(DeleteError::FileDoesNotExist(PathBuf::from(path))),
-            })
+    fn delete(&mut self, path: &Path) -> result::Result<(), DeleteError> {
+        match self.fs.remove(path) {
+            Some(_) => Ok(()),
+            None => Err(DeleteError::FileDoesNotExist(PathBuf::from(path))),
+        }
     }
 
     fn exists(&self, path: &Path) -> bool {
-        self.0
-            .read()
-            .expect("Failed to get read lock directory.")
-            .contains_key(path)
+        self.fs.contains_key(path)
+    }
+
+    fn watch(&mut self, watch_handle: WatchCallback) -> WatchHandle {
+        self.watch_router.subscribe(watch_handle)
     }
 }
 
@@ -147,33 +116,36 @@ impl fmt::Debug for RAMDirectory {
 /// It is mainly meant for unit testing.
 /// Writes are only made visible upon flushing.
 ///
-#[derive(Clone)]
+#[derive(Clone, Default)]
 pub struct RAMDirectory {
-    fs: InnerDirectory,
+    fs: Arc<RwLock<InnerDirectory>>,
 }
 
 impl RAMDirectory {
     /// Constructor
     pub fn create() -> RAMDirectory {
-        RAMDirectory {
-            fs: InnerDirectory::new(),
-        }
+        Self::default()
     }
 }
 
 impl Directory for RAMDirectory {
     fn open_read(&self, path: &Path) -> result::Result<ReadOnlySource, OpenReadError> {
-        self.fs.open_read(path)
+        self.fs.read().unwrap().open_read(path)
+    }
+
+    fn delete(&self, path: &Path) -> result::Result<(), DeleteError> {
+        self.fs.write().unwrap().delete(path)
+    }
+
+    fn exists(&self, path: &Path) -> bool {
+        self.fs.read().unwrap().exists(path)
     }
 
     fn open_write(&mut self, path: &Path) -> Result<WritePtr, OpenWriteError> {
+        let mut fs = self.fs.write().unwrap();
         let path_buf = PathBuf::from(path);
-        let vec_writer = VecWriter::new(path_buf.clone(), self.fs.clone());
-
-        let exists = self
-            .fs
-            .write(path_buf.clone(), &Vec::new())
-            .map_err(|err| IOError::with_path(path.to_owned(), err))?;
+        let vec_writer = VecWriter::new(path_buf.clone(), self.clone());
+        let exists = fs.write(path_buf.clone(), &[]);
         // force the creation of the file to mimic the MMap directory.
         if exists {
             Err(OpenWriteError::FileAlreadyExists(path_buf))
@@ -182,17 +154,8 @@ impl Directory for RAMDirectory {
         }
     }
 
-    fn delete(&self, path: &Path) -> result::Result<(), DeleteError> {
-        self.fs.delete(path)
-    }
-
-    fn exists(&self, path: &Path) -> bool {
-        self.fs.exists(path)
-    }
-
     fn atomic_read(&self, path: &Path) -> Result<Vec<u8>, OpenReadError> {
-        let read = self.open_read(path)?;
-        Ok(read.as_slice().to_owned())
+        Ok(self.open_read(path)?.as_slice().to_owned())
     }
 
     fn atomic_write(&mut self, path: &Path, data: &[u8]) -> io::Result<()> {
@@ -201,10 +164,20 @@ impl Directory for RAMDirectory {
             msg.unwrap_or("Undefined".to_string())
         )));
         let path_buf = PathBuf::from(path);
-        let mut vec_writer = VecWriter::new(path_buf.clone(), self.fs.clone());
-        self.fs.write(path_buf, &Vec::new())?;
+
+        // Reserve the path to prevent calls to .write() to succeed.
+        self.fs.write().unwrap().write(path_buf.clone(), &[]);
+
+        let mut vec_writer = VecWriter::new(path_buf.clone(), self.clone());
         vec_writer.write_all(data)?;
         vec_writer.flush()?;
+        if path == Path::new(&*META_FILEPATH) {
+            self.fs.write().unwrap().watch_router.broadcast();
+        }
         Ok(())
     }
+
+    fn watch(&self, watch_callback: WatchCallback) -> WatchHandle {
+        self.fs.write().unwrap().watch(watch_callback)
+    }
 }

src/directory/read_only_source.rs

@@ -1,9 +1,9 @@
-use super::shared_vec_slice::SharedVecSlice;
 use common::HasLen;
-#[cfg(feature = "mmap")]
-use fst::raw::MmapReadOnly;
 use stable_deref_trait::{CloneStableDeref, StableDeref};
 use std::ops::Deref;
+use std::sync::Arc;
+
+pub type BoxedData = Box<Deref<Target = [u8]> + Send + Sync + 'static>;
 
 /// Read object that represents files in tantivy.
 ///
@@ -11,12 +11,10 @@ use std::ops::Deref;
 /// the data in the form of a constant read-only `&[u8]`.
 /// Whatever happens to the directory file, the data
 /// hold by this object should never be altered or destroyed.
-pub enum ReadOnlySource {
-    /// Mmap source of data
-    #[cfg(feature = "mmap")]
-    Mmap(MmapReadOnly),
-    /// Wrapping a `Vec<u8>`
-    Anonymous(SharedVecSlice),
+pub struct ReadOnlySource {
+    data: Arc<BoxedData>,
+    start: usize,
+    stop: usize,
 }
 
 unsafe impl StableDeref for ReadOnlySource {}
@@ -30,19 +28,38 @@ impl Deref for ReadOnlySource {
     }
 }
 
+impl From<Arc<BoxedData>> for ReadOnlySource {
+    fn from(data: Arc<BoxedData>) -> Self {
+        let len = data.len();
+        ReadOnlySource {
+            data,
+            start: 0,
+            stop: len,
+        }
+    }
+}
+
 impl ReadOnlySource {
+    pub(crate) fn new<D>(data: D) -> ReadOnlySource
+    where
+        D: Deref<Target = [u8]> + Send + Sync + 'static,
+    {
+        let len = data.len();
+        ReadOnlySource {
+            data: Arc::new(Box::new(data)),
+            start: 0,
+            stop: len,
+        }
+    }
+
     /// Creates an empty ReadOnlySource
     pub fn empty() -> ReadOnlySource {
-        ReadOnlySource::Anonymous(SharedVecSlice::empty())
+        ReadOnlySource::new(&[][..])
     }
 
     /// Returns the data underlying the ReadOnlySource object.
     pub fn as_slice(&self) -> &[u8] {
-        match *self {
-            #[cfg(feature = "mmap")]
-            ReadOnlySource::Mmap(ref mmap_read_only) => mmap_read_only.as_slice(),
-            ReadOnlySource::Anonymous(ref shared_vec) => shared_vec.as_slice(),
-        }
+        &self.data[self.start..self.stop]
     }
 
     /// Splits into 2 `ReadOnlySource`, at the offset given
@@ -63,22 +80,18 @@ impl ReadOnlySource {
     /// worth of data in anonymous memory, and only a
     /// 1KB slice is remaining, the whole `500MBs`
     /// are retained in memory.
-    pub fn slice(&self, from_offset: usize, to_offset: usize) -> ReadOnlySource {
+    pub fn slice(&self, start: usize, stop: usize) -> ReadOnlySource {
         assert!(
-            from_offset <= to_offset,
+            start <= stop,
             "Requested negative slice [{}..{}]",
-            from_offset,
-            to_offset
+            start,
+            stop
         );
-        match *self {
-            #[cfg(feature = "mmap")]
-            ReadOnlySource::Mmap(ref mmap_read_only) => {
-                let sliced_mmap = mmap_read_only.range(from_offset, to_offset - from_offset);
-                ReadOnlySource::Mmap(sliced_mmap)
-            }
-            ReadOnlySource::Anonymous(ref shared_vec) => {
-                ReadOnlySource::Anonymous(shared_vec.slice(from_offset, to_offset))
-            }
+        assert!(stop <= self.len());
+        ReadOnlySource {
+            data: self.data.clone(),
+            start: self.start + start,
+            stop: self.start + stop,
         }
     }
 
@@ -87,8 +100,7 @@ impl ReadOnlySource {
     ///
     /// Equivalent to `.slice(from_offset, self.len())`
     pub fn slice_from(&self, from_offset: usize) -> ReadOnlySource {
-        let len = self.len();
-        self.slice(from_offset, len)
+        self.slice(from_offset, self.len())
     }
 
     /// Like `.slice(...)` but enforcing only the `to`
@@ -102,19 +114,18 @@ impl ReadOnlySource {
 
 impl HasLen for ReadOnlySource {
     fn len(&self) -> usize {
-        self.as_slice().len()
+        self.stop - self.start
     }
 }
 
 impl Clone for ReadOnlySource {
     fn clone(&self) -> Self {
-        self.slice(0, self.len())
+        self.slice_from(0)
     }
 }
 
 impl From<Vec<u8>> for ReadOnlySource {
     fn from(data: Vec<u8>) -> ReadOnlySource {
-        let shared_data = SharedVecSlice::from(data);
-        ReadOnlySource::Anonymous(shared_data)
+        ReadOnlySource::new(data)
     }
 }

src/directory/shared_vec_slice.rs

@@ -1,41 +0,0 @@
-use std::sync::Arc;
-
-#[derive(Clone)]
-pub struct SharedVecSlice {
-    pub data: Arc<Vec<u8>>,
-    pub start: usize,
-    pub len: usize,
-}
-
-impl SharedVecSlice {
-    pub fn empty() -> SharedVecSlice {
-        SharedVecSlice::new(Arc::new(Vec::new()))
-    }
-
-    pub fn new(data: Arc<Vec<u8>>) -> SharedVecSlice {
-        let data_len = data.len();
-        SharedVecSlice {
-            data,
-            start: 0,
-            len: data_len,
-        }
-    }
-
-    pub fn as_slice(&self) -> &[u8] {
-        &self.data[self.start..self.start + self.len]
-    }
-
-    pub fn slice(&self, from_offset: usize, to_offset: usize) -> SharedVecSlice {
-        SharedVecSlice {
-            data: Arc::clone(&self.data),
-            start: self.start + from_offset,
-            len: to_offset - from_offset,
-        }
-    }
-}
-
-impl From<Vec<u8>> for SharedVecSlice {
-    fn from(data: Vec<u8>) -> SharedVecSlice {
-        SharedVecSlice::new(Arc::new(data))
-    }
-}

src/directory/tests.rs

@@ -1,7 +1,13 @@
 use super::*;
 use std::io::{Seek, SeekFrom, Write};
+use std::mem;
 use std::path::{Path, PathBuf};
+use std::sync::atomic::AtomicUsize;
+use std::sync::atomic::Ordering;
+use std::sync::Arc;
+use std::thread;
 use std::time;
+use std::time::Duration;
 
 lazy_static! {
     static ref TEST_PATH: &'static Path = Path::new("some_path_for_test");
@@ -30,19 +36,18 @@ fn ram_directory_panics_if_flush_forgotten() {
 
 fn test_simple(directory: &mut Directory) {
     {
-        {
-            let mut write_file = directory.open_write(*TEST_PATH).unwrap();
-            assert!(directory.exists(*TEST_PATH));
-            write_file.write_all(&[4]).unwrap();
-            write_file.write_all(&[3]).unwrap();
-            write_file.write_all(&[7, 3, 5]).unwrap();
-            write_file.flush().unwrap();
-        }
+        let mut write_file = directory.open_write(*TEST_PATH).unwrap();
+        assert!(directory.exists(*TEST_PATH));
+        write_file.write_all(&[4]).unwrap();
+        write_file.write_all(&[3]).unwrap();
+        write_file.write_all(&[7, 3, 5]).unwrap();
+        write_file.flush().unwrap();
+    }
+    {
         let read_file = directory.open_read(*TEST_PATH).unwrap();
         let data: &[u8] = &*read_file;
         assert_eq!(data, &[4u8, 3u8, 7u8, 3u8, 5u8]);
     }
-
     assert!(directory.delete(*TEST_PATH).is_ok());
     assert!(!directory.exists(*TEST_PATH));
 }
@@ -121,6 +126,41 @@ fn test_directory(directory: &mut Directory) {
     test_directory_delete(directory);
     test_lock_non_blocking(directory);
     test_lock_blocking(directory);
+    test_watch(directory);
+}
+
+fn test_watch(directory: &mut Directory) {
+    let counter: Arc<AtomicUsize> = Default::default();
+    let counter_clone = counter.clone();
+    let watch_callback = Box::new(move || {
+        counter_clone.fetch_add(1, Ordering::SeqCst);
+    });
+    assert!(directory
+        .atomic_write(Path::new("meta.json"), b"random_test_data")
+        .is_ok());
+    thread::sleep(Duration::new(0, 10_000));
+    assert_eq!(0, counter.load(Ordering::SeqCst));
+
+    let watch_handle = directory.watch(watch_callback);
+    for i in 0..10 {
+        assert_eq!(i, counter.load(Ordering::SeqCst));
+        assert!(directory
+            .atomic_write(Path::new("meta.json"), b"random_test_data_2")
+            .is_ok());
+        for _ in 0..100 {
+            if counter.load(Ordering::SeqCst) > i {
+                break;
+            }
+            thread::sleep(Duration::from_millis(10));
+        }
+        assert_eq!(i + 1, counter.load(Ordering::SeqCst));
+    }
+    mem::drop(watch_handle);
+    assert!(directory
+        .atomic_write(Path::new("meta.json"), b"random_test_data")
+        .is_ok());
+    thread::sleep(Duration::from_millis(200));
+    assert_eq!(10, counter.load(Ordering::SeqCst));
 }
 
 fn test_lock_non_blocking(directory: &mut Directory) {

src/directory/watch_event_router.rs

@@ -0,0 +1,156 @@
+use std::sync::Arc;
+use std::sync::RwLock;
+use std::sync::Weak;
+
+/// Type alias for callbacks registered when watching files of a `Directory`.
+pub type WatchCallback = Box<Fn() -> () + Sync + Send>;
+
+/// Helper struct to implement the watch method in `Directory` implementations.
+///
+/// It registers callbacks (See `.subscribe(...)`) and
+/// calls them upon calls to `.broadcast(...)`.
+#[derive(Default)]
+pub struct WatchCallbackList {
+    router: RwLock<Vec<Weak<WatchCallback>>>,
+}
+
+/// Controls how long a directory should watch for a file change.
+///
+/// After all the clones of `WatchHandle` are dropped, the associated will not be called when a
+/// file change is detected.
+#[must_use = "This `WatchHandle` controls the lifetime of the watch and should therefore be used."]
+#[derive(Clone)]
+pub struct WatchHandle(Arc<WatchCallback>);
+
+impl WatchCallbackList {
+    /// Suscribes a new callback and returns a handle that controls the lifetime of the callback.
+    pub fn subscribe(&self, watch_callback: WatchCallback) -> WatchHandle {
+        let watch_callback_arc = Arc::new(watch_callback);
+        let watch_callback_weak = Arc::downgrade(&watch_callback_arc);
+        self.router.write().unwrap().push(watch_callback_weak);
+        WatchHandle(watch_callback_arc)
+    }
+
+    fn list_callback(&self) -> Vec<Arc<WatchCallback>> {
+        let mut callbacks = vec![];
+        let mut router_wlock = self.router.write().unwrap();
+        let mut i = 0;
+        while i < router_wlock.len() {
+            if let Some(watch) = router_wlock[i].upgrade() {
+                callbacks.push(watch);
+                i += 1;
+            } else {
+                router_wlock.swap_remove(i);
+            }
+        }
+        callbacks
+    }
+
+    /// Triggers all callbacks
+    pub fn broadcast(&self) {
+        let callbacks = self.list_callback();
+        let spawn_res = std::thread::Builder::new()
+            .name("watch-callbacks".to_string())
+            .spawn(move || {
+                for callback in callbacks {
+                    callback();
+                }
+            });
+        if let Err(err) = spawn_res {
+            error!(
+                "Failed to spawn thread to call watch callbacks. Cause: {:?}",
+                err
+            );
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use directory::WatchCallbackList;
+    use std::mem;
+    use std::sync::atomic::{AtomicUsize, Ordering};
+    use std::sync::Arc;
+    use std::thread;
+    use std::time::Duration;
+
+    const WAIT_TIME: u64 = 20;
+
+    #[test]
+    fn test_watch_event_router_simple() {
+        let watch_event_router = WatchCallbackList::default();
+        let counter: Arc<AtomicUsize> = Default::default();
+        let counter_clone = counter.clone();
+        let inc_callback = Box::new(move || {
+            counter_clone.fetch_add(1, Ordering::SeqCst);
+        });
+        watch_event_router.broadcast();
+        assert_eq!(0, counter.load(Ordering::SeqCst));
+        let handle_a = watch_event_router.subscribe(inc_callback);
+        thread::sleep(Duration::from_millis(WAIT_TIME));
+        assert_eq!(0, counter.load(Ordering::SeqCst));
+        watch_event_router.broadcast();
+        thread::sleep(Duration::from_millis(WAIT_TIME));
+        assert_eq!(1, counter.load(Ordering::SeqCst));
+        watch_event_router.broadcast();
+        watch_event_router.broadcast();
+        watch_event_router.broadcast();
+        thread::sleep(Duration::from_millis(WAIT_TIME));
+        assert_eq!(4, counter.load(Ordering::SeqCst));
+        mem::drop(handle_a);
+        watch_event_router.broadcast();
+        thread::sleep(Duration::from_millis(WAIT_TIME));
+        assert_eq!(4, counter.load(Ordering::SeqCst));
+    }
+
+    #[test]
+    fn test_watch_event_router_multiple_callback_same_key() {
+        let watch_event_router = WatchCallbackList::default();
+        let counter: Arc<AtomicUsize> = Default::default();
+        let inc_callback = |inc: usize| {
+            let counter_clone = counter.clone();
+            Box::new(move || {
+                counter_clone.fetch_add(inc, Ordering::SeqCst);
+            })
+        };
+        let handle_a = watch_event_router.subscribe(inc_callback(1));
+        let handle_a2 = watch_event_router.subscribe(inc_callback(10));
+        thread::sleep(Duration::from_millis(WAIT_TIME));
+        assert_eq!(0, counter.load(Ordering::SeqCst));
+        watch_event_router.broadcast();
+        watch_event_router.broadcast();
+        thread::sleep(Duration::from_millis(WAIT_TIME));
+        assert_eq!(22, counter.load(Ordering::SeqCst));
+        mem::drop(handle_a);
+        watch_event_router.broadcast();
+        thread::sleep(Duration::from_millis(WAIT_TIME));
+        assert_eq!(32, counter.load(Ordering::SeqCst));
+        mem::drop(handle_a2);
+        watch_event_router.broadcast();
+        watch_event_router.broadcast();
+        thread::sleep(Duration::from_millis(WAIT_TIME));
+        assert_eq!(32, counter.load(Ordering::SeqCst));
+    }
+
+    #[test]
+    fn test_watch_event_router_multiple_callback_different_key() {
+        let watch_event_router = WatchCallbackList::default();
+        let counter: Arc<AtomicUsize> = Default::default();
+        let counter_clone = counter.clone();
+        let inc_callback = Box::new(move || {
+            counter_clone.fetch_add(1, Ordering::SeqCst);
+        });
+        let handle_a = watch_event_router.subscribe(inc_callback);
+        assert_eq!(0, counter.load(Ordering::SeqCst));
+        watch_event_router.broadcast();
+        watch_event_router.broadcast();
+        thread::sleep(Duration::from_millis(WAIT_TIME));
+        assert_eq!(2, counter.load(Ordering::SeqCst));
+        thread::sleep(Duration::from_millis(WAIT_TIME));
+        mem::drop(handle_a);
+        watch_event_router.broadcast();
+        thread::sleep(Duration::from_millis(WAIT_TIME));
+        assert_eq!(2, counter.load(Ordering::SeqCst));
+    }
+
+}

src/error.rs

@@ -162,6 +162,7 @@ impl From<OpenDirectoryError> for TantivyError {
             OpenDirectoryError::NotADirectory(directory_path) => {
                 TantivyError::InvalidArgument(format!("{:?} is not a directory", directory_path))
             }
+            OpenDirectoryError::IoError(err) => TantivyError::IOError(IOError::from(err)),
         }
     }
 }

src/fastfield/bytes/mod.rs

@@ -22,9 +22,7 @@ mod tests {
         index_writer.add_document(doc!(field=>vec![1u8, 3, 5, 7, 9]));
         index_writer.add_document(doc!(field=>vec![0u8; 1000]));
         assert!(index_writer.commit().is_ok());
-
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        let searcher = index.reader().unwrap().searcher();
         let reader = searcher.segment_reader(0);
         let bytes_reader = reader.bytes_fast_field_reader(field).unwrap();
 

src/fastfield/multivalued/mod.rs

@@ -7,7 +7,13 @@ pub use self::writer::MultiValueIntFastFieldWriter;
 #[cfg(test)]
 mod tests {
 
+    extern crate time;
+
+    use self::time::Duration;
+    use collector::TopDocs;
+    use query::QueryParser;
     use schema::Cardinality;
+    use schema::Facet;
     use schema::IntOptions;
     use schema::Schema;
     use Index;
@@ -28,11 +34,12 @@ mod tests {
         index_writer.add_document(doc!(field=>5u64, field=>20u64,field=>1u64));
         assert!(index_writer.commit().is_ok());
 
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
-        let reader = searcher.segment_reader(0);
+        let searcher = index.reader().unwrap().searcher();
+        let segment_reader = searcher.segment_reader(0);
         let mut vals = Vec::new();
-        let multi_value_reader = reader.multi_fast_field_reader::<u64>(field).unwrap();
+        let multi_value_reader = segment_reader
+            .multi_fast_field_reader::<u64>(field)
+            .unwrap();
         {
             multi_value_reader.get_vals(2, &mut vals);
             assert_eq!(&vals, &[4u64]);
@@ -47,6 +54,133 @@ mod tests {
         }
     }
 
+    #[test]
+    fn test_multivalued_date() {
+        let mut schema_builder = Schema::builder();
+        let date_field = schema_builder.add_date_field(
+            "multi_date_field",
+            IntOptions::default()
+                .set_fast(Cardinality::MultiValues)
+                .set_indexed()
+                .set_stored(),
+        );
+        let time_i =
+            schema_builder.add_i64_field("time_stamp_i", IntOptions::default().set_stored());
+        let schema = schema_builder.build();
+        let index = Index::create_in_ram(schema);
+        let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
+        let first_time_stamp = chrono::Utc::now();
+        index_writer.add_document(
+            doc!(date_field=>first_time_stamp, date_field=>first_time_stamp, time_i=>1i64),
+        );
+        index_writer.add_document(doc!(time_i=>0i64));
+        // add one second
+        index_writer
+            .add_document(doc!(date_field=>first_time_stamp + Duration::seconds(1), time_i=>2i64));
+        // add another second
+        let two_secs_ahead = first_time_stamp + Duration::seconds(2);
+        index_writer.add_document(doc!(date_field=>two_secs_ahead, date_field=>two_secs_ahead,date_field=>two_secs_ahead, time_i=>3i64));
+        assert!(index_writer.commit().is_ok());
+
+        let reader = index.reader().unwrap();
+        let searcher = reader.searcher();
+        let reader = searcher.segment_reader(0);
+        assert_eq!(reader.num_docs(), 4);
+
+        {
+            let parser = QueryParser::for_index(&index, vec![date_field]);
+            let query = parser
+                .parse_query(&format!("\"{}\"", first_time_stamp.to_rfc3339()).to_string())
+                .expect("could not parse query");
+            let results = searcher
+                .search(&query, &TopDocs::with_limit(5))
+                .expect("could not query index");
+
+            assert_eq!(results.len(), 1);
+            for (_score, doc_address) in results {
+                let retrieved_doc = searcher.doc(doc_address).expect("cannot fetch doc");
+                assert_eq!(
+                    retrieved_doc
+                        .get_first(date_field)
+                        .expect("cannot find value")
+                        .date_value()
+                        .timestamp(),
+                    first_time_stamp.timestamp()
+                );
+                assert_eq!(
+                    retrieved_doc
+                        .get_first(time_i)
+                        .expect("cannot find value")
+                        .i64_value(),
+                    1i64
+                );
+            }
+        }
+
+        {
+            let parser = QueryParser::for_index(&index, vec![date_field]);
+            let query = parser
+                .parse_query(&format!("\"{}\"", two_secs_ahead.to_rfc3339()).to_string())
+                .expect("could not parse query");
+            let results = searcher
+                .search(&query, &TopDocs::with_limit(5))
+                .expect("could not query index");
+
+            assert_eq!(results.len(), 1);
+
+            for (_score, doc_address) in results {
+                let retrieved_doc = searcher.doc(doc_address).expect("cannot fetch doc");
+                assert_eq!(
+                    retrieved_doc
+                        .get_first(date_field)
+                        .expect("cannot find value")
+                        .date_value()
+                        .timestamp(),
+                    two_secs_ahead.timestamp()
+                );
+                assert_eq!(
+                    retrieved_doc
+                        .get_first(time_i)
+                        .expect("cannot find value")
+                        .i64_value(),
+                    3i64
+                );
+            }
+        }
+
+        // TODO: support Date range queries
+        //        {
+        //            let parser = QueryParser::for_index(&index, vec![date_field]);
+        //            let range_q = format!("\"{}\"..\"{}\"",
+        //                                  (first_time_stamp + Duration::seconds(1)).to_rfc3339(),
+        //                                  (first_time_stamp + Duration::seconds(3)).to_rfc3339()
+        //            );
+        //            let query = parser.parse_query(&range_q)
+        //                .expect("could not parse query");
+        //            let results = searcher.search(&query, &TopDocs::with_limit(5))
+        //                .expect("could not query index");
+        //
+        //
+        //            assert_eq!(results.len(), 2);
+        //            for (i, doc_pair) in results.iter().enumerate() {
+        //                let retrieved_doc = searcher.doc(doc_pair.1).expect("cannot fetch doc");
+        //                let offset_sec = match i {
+        //                    0 => 1,
+        //                    1 => 3,
+        //                    _ => panic!("should not have more than 2 docs")
+        //                };
+        //                let time_i_val = match i {
+        //                    0 => 2,
+        //                    1 => 3,
+        //                    _ => panic!("should not have more than 2 docs")
+        //                };
+        //                assert_eq!(retrieved_doc.get_first(date_field).expect("cannot find value").date_value().timestamp(),
+        //                           (first_time_stamp + Duration::seconds(offset_sec)).timestamp());
+        //                assert_eq!(retrieved_doc.get_first(time_i).expect("cannot find value").i64_value(), time_i_val);
+        //            }
+        //        }
+    }
+
     #[test]
     fn test_multivalued_i64() {
         let mut schema_builder = Schema::builder();
@@ -63,8 +197,7 @@ mod tests {
         index_writer.add_document(doc!(field=> -5i64, field => -20i64, field=>1i64));
         assert!(index_writer.commit().is_ok());
 
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        let searcher = index.reader().unwrap().searcher();
         let reader = searcher.segment_reader(0);
         let mut vals = Vec::new();
         let multi_value_reader = reader.multi_fast_field_reader::<i64>(field).unwrap();
@@ -85,4 +218,17 @@ mod tests {
             assert_eq!(&vals, &[-5i64, -20i64, 1i64]);
         }
     }
+    #[test]
+    #[ignore]
+    fn test_many_facets() {
+        let mut schema_builder = Schema::builder();
+        let field = schema_builder.add_facet_field("facetfield");
+        let schema = schema_builder.build();
+        let index = Index::create_in_ram(schema);
+        let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
+        for i in 0..100_000 {
+            index_writer.add_document(doc!(field=> Facet::from(format!("/lang/{}", i).as_str())));
+        }
+        assert!(index_writer.commit().is_ok());
+    }
 }

src/fastfield/multivalued/reader.rs

@@ -39,7 +39,7 @@ impl<Item: FastValue> MultiValueIntFastFieldReader<Item> {
         let (start, stop) = self.range(doc);
         let len = (stop - start) as usize;
         vals.resize(len, Item::default());
-        self.vals_reader.get_range(start as u32, &mut vals[..]);
+        self.vals_reader.get_range_u64(start, &mut vals[..]);
     }
 }
 
@@ -75,8 +75,7 @@ mod tests {
             index_writer.add_document(doc);
         }
         index_writer.commit().expect("Commit failed");
-        index.load_searchers().expect("Reloading searchers");
-        let searcher = index.searcher();
+        let searcher = index.reader().unwrap().searcher();
         let segment_reader = searcher.segment_reader(0);
         let mut facet_reader = segment_reader.facet_reader(facet_field).unwrap();
 

src/fastfield/multivalued/writer.rs

@@ -32,7 +32,7 @@ use DocId;
 /// term ids when the segment is getting serialized.
 pub struct MultiValueIntFastFieldWriter {
     field: Field,
-    vals: Vec<u64>,
+    vals: Vec<UnorderedTermId>,
     doc_index: Vec<u64>,
     is_facet: bool,
 }

src/fastfield/reader.rs

@@ -59,7 +59,29 @@ impl<Item: FastValue> FastFieldReader<Item> {
     /// May panic if `doc` is greater than the segment
     // `maxdoc`.
     pub fn get(&self, doc: DocId) -> Item {
-        Item::from_u64(self.min_value_u64 + self.bit_unpacker.get(doc as usize))
+        self.get_u64(u64::from(doc))
+    }
+
+    pub(crate) fn get_u64(&self, doc: u64) -> Item {
+        Item::from_u64(self.min_value_u64 + self.bit_unpacker.get(doc))
+    }
+
+    /// Internally `multivalued` also use SingleValue Fast fields.
+    /// It works as follows... A first column contains the list of start index
+    /// for each document, a second column contains the actual values.
+    ///
+    /// The values associated to a given doc, are then
+    ///  `second_column[first_column.get(doc)..first_column.get(doc+1)]`.
+    ///
+    /// Which means single value fast field reader can be indexed internally with
+    /// something different from a `DocId`. For this use case, we want to use `u64`
+    /// values.
+    ///
+    /// See `get_range` for an actual documentation about this method.
+    pub(crate) fn get_range_u64(&self, start: u64, output: &mut [Item]) {
+        for (i, out) in output.iter_mut().enumerate() {
+            *out = self.get_u64(start + (i as u64));
+        }
     }
 
     /// Fills an output buffer with the fast field values
@@ -75,13 +97,8 @@ impl<Item: FastValue> FastFieldReader<Item> {
     ///
     /// May panic if `start + output.len()` is greater than
     /// the segment's `maxdoc`.
-    ///
-    // TODO change start to `u64`.
-    // For multifastfield, start is an index in a second fastfield, not a `DocId`
-    pub fn get_range(&self, start: u32, output: &mut [Item]) {
-        for (i, out) in output.iter_mut().enumerate() {
-            *out = self.get(start + i as u32);
-        }
+    pub fn get_range(&self, start: DocId, output: &mut [Item]) {
+        self.get_range_u64(u64::from(start), output);
     }
 
     /// Returns the minimum value for this fast field.

src/functional_test.rs

@@ -17,11 +17,12 @@ fn check_index_content(searcher: &Searcher, vals: &HashSet<u64>) {
 fn test_indexing() {
     let mut schema_builder = Schema::builder();
 
-    let id_field = schema_builder.add_u64_field("id", INT_INDEXED);
-    let multiples_field = schema_builder.add_u64_field("multiples", INT_INDEXED);
+    let id_field = schema_builder.add_u64_field("id", INDEXED);
+    let multiples_field = schema_builder.add_u64_field("multiples", INDEXED);
     let schema = schema_builder.build();
 
     let index = Index::create_from_tempdir(schema).unwrap();
+    let reader = index.reader().unwrap();
 
     let mut rng = thread_rng();
 
@@ -36,8 +37,8 @@ fn test_indexing() {
             index_writer.commit().expect("Commit failed");
             committed_docs.extend(&uncommitted_docs);
             uncommitted_docs.clear();
-            index.load_searchers().unwrap();
-            let searcher = index.searcher();
+            reader.reload().unwrap();
+            let searcher = reader.searcher();
             // check that everything is correct.
             check_index_content(&searcher, &committed_docs);
         } else {

src/indexer/index_writer.rs

@@ -1,4 +1,4 @@
-use super::operation::AddOperation;
+use super::operation::{AddOperation, UserOperation};
 use super::segment_updater::SegmentUpdater;
 use super::PreparedCommit;
 use bit_set::BitSet;
@@ -26,6 +26,7 @@ use schema::Document;
 use schema::IndexRecordOption;
 use schema::Term;
 use std::mem;
+use std::ops::Range;
 use std::sync::Arc;
 use std::thread;
 use std::thread::JoinHandle;
@@ -43,8 +44,8 @@ pub const HEAP_SIZE_MAX: usize = u32::max_value() as usize - MARGIN_IN_BYTES;
 // reaches `PIPELINE_MAX_SIZE_IN_DOCS`
 const PIPELINE_MAX_SIZE_IN_DOCS: usize = 10_000;
 
-type DocumentSender = channel::Sender<AddOperation>;
-type DocumentReceiver = channel::Receiver<AddOperation>;
+type OperationSender = channel::Sender<Vec<AddOperation>>;
+type OperationReceiver = channel::Receiver<Vec<AddOperation>>;
 
 /// Split the thread memory budget into
 /// - the heap size
@@ -84,8 +85,8 @@ pub struct IndexWriter {
 
     workers_join_handle: Vec<JoinHandle<Result<()>>>,
 
-    document_receiver: DocumentReceiver,
-    document_sender: DocumentSender,
+    operation_receiver: OperationReceiver,
+    operation_sender: OperationSender,
 
     segment_updater: SegmentUpdater,
 
@@ -132,7 +133,7 @@ pub fn open_index_writer(
         let err_msg = format!("The heap size per thread cannot exceed {}", HEAP_SIZE_MAX);
         return Err(TantivyError::InvalidArgument(err_msg));
     }
-    let (document_sender, document_receiver): (DocumentSender, DocumentReceiver) =
+    let (document_sender, document_receiver): (OperationSender, OperationReceiver) =
         channel::bounded(PIPELINE_MAX_SIZE_IN_DOCS);
 
     let delete_queue = DeleteQueue::new();
@@ -150,8 +151,8 @@ pub fn open_index_writer(
         heap_size_in_bytes_per_thread,
         index: index.clone(),
 
-        document_receiver,
-        document_sender,
+        operation_receiver: document_receiver,
+        operation_sender: document_sender,
 
         segment_updater,
 
@@ -258,7 +259,7 @@ pub fn advance_deletes(
             write_delete_bitset(&delete_bitset, &mut delete_file)?;
         }
     }
-    segment_entry.set_meta((*segment.meta()).clone());
+    segment_entry.set_meta(segment.meta().clone());
     Ok(())
 }
 
@@ -266,7 +267,7 @@ fn index_documents(
     memory_budget: usize,
     segment: &Segment,
     generation: usize,
-    document_iterator: &mut Iterator<Item = AddOperation>,
+    document_iterator: &mut Iterator<Item = Vec<AddOperation>>,
     segment_updater: &mut SegmentUpdater,
     mut delete_cursor: DeleteCursor,
 ) -> Result<bool> {
@@ -274,11 +275,11 @@ fn index_documents(
     let segment_id = segment.id();
     let table_size = initial_table_size(memory_budget);
     let mut segment_writer = SegmentWriter::for_segment(table_size, segment.clone(), &schema)?;
-    for doc in document_iterator {
-        segment_writer.add_document(doc, &schema)?;
-
+    for documents in document_iterator {
+        for doc in documents {
+            segment_writer.add_document(doc, &schema)?;
+        }
         let mem_usage = segment_writer.mem_usage();
-
         if mem_usage >= memory_budget - MARGIN_IN_BYTES {
             info!(
                 "Buffer limit reached, flushing segment with maxdoc={}.",
@@ -334,7 +335,7 @@ impl IndexWriter {
     pub fn wait_merging_threads(mut self) -> Result<()> {
         // this will stop the indexing thread,
         // dropping the last reference to the segment_updater.
-        drop(self.document_sender);
+        drop(self.operation_sender);
 
         let former_workers_handles = mem::replace(&mut self.workers_join_handle, vec![]);
         for join_handle in former_workers_handles {
@@ -383,7 +384,7 @@ impl IndexWriter {
     /// The thread consumes documents from the pipeline.
     ///
     fn add_indexing_worker(&mut self) -> Result<()> {
-        let document_receiver_clone = self.document_receiver.clone();
+        let document_receiver_clone = self.operation_receiver.clone();
         let mut segment_updater = self.segment_updater.clone();
 
         let generation = self.generation;
@@ -409,8 +410,12 @@ impl IndexWriter {
                     // this is a valid guarantee as the
                     // peeked document now belongs to
                     // our local iterator.
-                    if let Some(operation) = document_iterator.peek() {
-                        delete_cursor.skip_to(operation.opstamp);
+                    if let Some(operations) = document_iterator.peek() {
+                        if let Some(first) = operations.first() {
+                            delete_cursor.skip_to(first.opstamp);
+                        } else {
+                            return Ok(());
+                        }
                     } else {
                         // No more documents.
                         // Happens when there is a commit, or if the `IndexWriter`
@@ -474,11 +479,11 @@ impl IndexWriter {
     /// when no documents are remaining.
     ///
     /// Returns the former segment_ready channel.
-    fn recreate_document_channel(&mut self) -> DocumentReceiver {
-        let (document_sender, document_receiver): (DocumentSender, DocumentReceiver) =
+    fn recreate_document_channel(&mut self) -> OperationReceiver {
+        let (document_sender, document_receiver): (OperationSender, OperationReceiver) =
             channel::bounded(PIPELINE_MAX_SIZE_IN_DOCS);
-        mem::replace(&mut self.document_sender, document_sender);
-        mem::replace(&mut self.document_receiver, document_receiver)
+        mem::replace(&mut self.operation_sender, document_sender);
+        mem::replace(&mut self.operation_receiver, document_receiver)
     }
 
     /// Rollback to the last commit
@@ -496,7 +501,7 @@ impl IndexWriter {
         // segment updates will be ignored.
         self.segment_updater.kill();
 
-        let document_receiver = self.document_receiver.clone();
+        let document_receiver = self.operation_receiver.clone();
 
         // take the directory lock to create a new index_writer.
         let directory_lock = self
@@ -643,25 +648,168 @@ impl IndexWriter {
     pub fn add_document(&mut self, document: Document) -> u64 {
         let opstamp = self.stamper.stamp();
         let add_operation = AddOperation { opstamp, document };
-        let send_result = self.document_sender.send(add_operation);
+        let send_result = self.operation_sender.send(vec![add_operation]);
         if let Err(e) = send_result {
             panic!("Failed to index document. Sending to indexing channel failed. This probably means all of the indexing threads have panicked. {:?}", e);
         }
         opstamp
     }
+
+    /// Gets a range of stamps from the stamper and "pops" the last stamp
+    /// from the range returning a tuple of the last optstamp and the popped
+    /// range.
+    ///
+    /// The total number of stamps generated by this method is `count + 1`;
+    /// each operation gets a stamp from the `stamps` iterator and `last_opstamp`
+    /// is for the batch itself.
+    fn get_batch_opstamps(&mut self, count: u64) -> (u64, Range<u64>) {
+        let Range { start, end } = self.stamper.stamps(count + 1u64);
+        let last_opstamp = end - 1;
+        let stamps = Range {
+            start,
+            end: last_opstamp,
+        };
+        (last_opstamp, stamps)
+    }
+
+    /// Runs a group of document operations ensuring that the operations are
+    /// assigned contigous u64 opstamps and that add operations of the same
+    /// group are flushed into the same segment.
+    ///
+    /// If the indexing pipeline is full, this call may block.
+    ///
+    /// Each operation of the given `user_operations` will receive an in-order,
+    /// contiguous u64 opstamp. The entire batch itself is also given an
+    /// opstamp that is 1 greater than the last given operation. This
+    /// `batch_opstamp` is the return value of `run`. An empty group of
+    /// `user_operations`, an empty `Vec<UserOperation>`, still receives
+    /// a valid opstamp even though no changes were _actually_ made to the index.
+    ///
+    /// Like adds and deletes (see `IndexWriter.add_document` and
+    /// `IndexWriter.delete_term`), the changes made by calling `run` will be
+    /// visible to readers only after calling `commit()`.
+    pub fn run(&mut self, user_operations: Vec<UserOperation>) -> u64 {
+        let count = user_operations.len() as u64;
+        if count == 0 {
+            return self.stamper.stamp();
+        }
+        let (batch_opstamp, stamps) = self.get_batch_opstamps(count);
+
+        let mut adds: Vec<AddOperation> = Vec::new();
+
+        for (user_op, opstamp) in user_operations.into_iter().zip(stamps) {
+            match user_op {
+                UserOperation::Delete(term) => {
+                    let delete_operation = DeleteOperation { opstamp, term };
+                    self.delete_queue.push(delete_operation);
+                }
+                UserOperation::Add(document) => {
+                    let add_operation = AddOperation { opstamp, document };
+                    adds.push(add_operation);
+                }
+            }
+        }
+        let send_result = self.operation_sender.send(adds);
+        if let Err(e) = send_result {
+            panic!("Failed to index document. Sending to indexing channel failed. This probably means all of the indexing threads have panicked. {:?}", e);
+        };
+
+        batch_opstamp
+    }
 }
 
 #[cfg(test)]
 mod tests {
 
+    use super::super::operation::UserOperation;
     use super::initial_table_size;
+    use collector::TopDocs;
     use directory::error::LockError;
     use error::*;
     use indexer::NoMergePolicy;
-    use schema::{self, Document};
+    use query::TermQuery;
+    use schema::{self, IndexRecordOption};
     use Index;
+    use ReloadPolicy;
     use Term;
 
+    #[test]
+    fn test_operations_group() {
+        // an operations group with 2 items should cause 3 opstamps 0, 1, and 2.
+        let mut schema_builder = schema::Schema::builder();
+        let text_field = schema_builder.add_text_field("text", schema::TEXT);
+        let index = Index::create_in_ram(schema_builder.build());
+        let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
+        let operations = vec![
+            UserOperation::Add(doc!(text_field=>"a")),
+            UserOperation::Add(doc!(text_field=>"b")),
+        ];
+        let batch_opstamp1 = index_writer.run(operations);
+        assert_eq!(batch_opstamp1, 2u64);
+    }
+
+    #[test]
+    fn test_ordered_batched_operations() {
+        // * one delete for `doc!(field=>"a")`
+        // * one add for `doc!(field=>"a")`
+        // * one add for `doc!(field=>"b")`
+        // * one delete for `doc!(field=>"b")`
+        // after commit there is one doc with "a" and 0 doc with "b"
+        let mut schema_builder = schema::Schema::builder();
+        let text_field = schema_builder.add_text_field("text", schema::TEXT);
+        let index = Index::create_in_ram(schema_builder.build());
+        let reader = index
+            .reader_builder()
+            .reload_policy(ReloadPolicy::Manual)
+            .try_into()
+            .unwrap();
+        let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
+        let a_term = Term::from_field_text(text_field, "a");
+        let b_term = Term::from_field_text(text_field, "b");
+        let operations = vec![
+            UserOperation::Delete(a_term),
+            UserOperation::Add(doc!(text_field=>"a")),
+            UserOperation::Add(doc!(text_field=>"b")),
+            UserOperation::Delete(b_term),
+        ];
+
+        index_writer.run(operations);
+        index_writer.commit().expect("failed to commit");
+        reader.reload().expect("failed to load searchers");
+
+        let a_term = Term::from_field_text(text_field, "a");
+        let b_term = Term::from_field_text(text_field, "b");
+
+        let a_query = TermQuery::new(a_term, IndexRecordOption::Basic);
+        let b_query = TermQuery::new(b_term, IndexRecordOption::Basic);
+
+        let searcher = reader.searcher();
+
+        let a_docs = searcher
+            .search(&a_query, &TopDocs::with_limit(1))
+            .expect("search for a failed");
+
+        let b_docs = searcher
+            .search(&b_query, &TopDocs::with_limit(1))
+            .expect("search for b failed");
+
+        assert_eq!(a_docs.len(), 1);
+        assert_eq!(b_docs.len(), 0);
+    }
+
+    #[test]
+    fn test_empty_operations_group() {
+        let schema_builder = schema::Schema::builder();
+        let index = Index::create_in_ram(schema_builder.build());
+        let mut index_writer = index.writer(3_000_000).unwrap();
+        let operations1 = vec![];
+        let batch_opstamp1 = index_writer.run(operations1);
+        assert_eq!(batch_opstamp1, 0u64);
+        let operations2 = vec![];
+        let batch_opstamp2 = index_writer.run(operations2);
+        assert_eq!(batch_opstamp2, 1u64);
+    }
+
     #[test]
     fn test_lockfile_stops_duplicates() {
         let schema_builder = schema::Schema::builder();
@@ -722,9 +870,13 @@ mod tests {
         let mut schema_builder = schema::Schema::builder();
         let text_field = schema_builder.add_text_field("text", schema::TEXT);
         let index = Index::create_in_ram(schema_builder.build());
-
+        let reader = index
+            .reader_builder()
+            .reload_policy(ReloadPolicy::Manual)
+            .try_into()
+            .unwrap();
         let num_docs_containing = |s: &str| {
-            let searcher = index.searcher();
+            let searcher = reader.searcher();
             let term = Term::from_field_text(text_field, s);
             searcher.doc_freq(&term)
         };
@@ -734,7 +886,6 @@ mod tests {
             let mut index_writer = index.writer(3_000_000).unwrap();
             index_writer.add_document(doc!(text_field=>"a"));
             index_writer.rollback().unwrap();
-
             assert_eq!(index_writer.commit_opstamp(), 0u64);
             assert_eq!(num_docs_containing("a"), 0);
             {
@@ -742,13 +893,13 @@ mod tests {
                 index_writer.add_document(doc!(text_field=>"c"));
             }
             assert!(index_writer.commit().is_ok());
-            index.load_searchers().unwrap();
+            reader.reload().unwrap();
             assert_eq!(num_docs_containing("a"), 0);
             assert_eq!(num_docs_containing("b"), 1);
             assert_eq!(num_docs_containing("c"), 1);
         }
-        index.load_searchers().unwrap();
-        index.searcher();
+        reader.reload().unwrap();
+        reader.searcher();
     }
 
     #[test]
@@ -756,32 +907,33 @@ mod tests {
         let mut schema_builder = schema::Schema::builder();
         let text_field = schema_builder.add_text_field("text", schema::TEXT);
         let index = Index::create_in_ram(schema_builder.build());
+        let reader = index
+            .reader_builder()
+            .reload_policy(ReloadPolicy::Manual)
+            .try_into()
+            .unwrap();
         let num_docs_containing = |s: &str| {
-            let searcher = index.searcher();
             let term_a = Term::from_field_text(text_field, s);
-            searcher.doc_freq(&term_a)
+            reader.searcher().doc_freq(&term_a)
         };
         {
             // writing the segment
             let mut index_writer = index.writer(12_000_000).unwrap();
             // create 8 segments with 100 tiny docs
             for _doc in 0..100 {
-                let mut doc = Document::default();
-                doc.add_text(text_field, "a");
-                index_writer.add_document(doc);
+                index_writer.add_document(doc!(text_field=>"a"));
             }
             index_writer.commit().expect("commit failed");
             for _doc in 0..100 {
-                let mut doc = Document::default();
-                doc.add_text(text_field, "a");
-                index_writer.add_document(doc);
+                index_writer.add_document(doc!(text_field=>"a"));
             }
-            // this should create 8 segments and trigger a merge.
+            //  this should create 8 segments and trigger a merge.
             index_writer.commit().expect("commit failed");
             index_writer
                 .wait_merging_threads()
                 .expect("waiting merging thread failed");
-            index.load_searchers().unwrap();
+
+            reader.reload().unwrap();
 
             assert_eq!(num_docs_containing("a"), 200);
             assert!(index.searchable_segments().unwrap().len() < 8);
@@ -848,11 +1000,15 @@ mod tests {
             }
             index_writer.commit().unwrap();
         }
-        index.load_searchers().unwrap();
         let num_docs_containing = |s: &str| {
-            let searcher = index.searcher();
             let term_a = Term::from_field_text(text_field, s);
-            searcher.doc_freq(&term_a)
+            index
+                .reader_builder()
+                .reload_policy(ReloadPolicy::Manual)
+                .try_into()
+                .unwrap()
+                .searcher()
+                .doc_freq(&term_a)
         };
         assert_eq!(num_docs_containing("a"), 0);
         assert_eq!(num_docs_containing("b"), 100);
@@ -860,9 +1016,9 @@ mod tests {
 
     #[test]
     fn test_hashmap_size() {
-        assert_eq!(initial_table_size(100_000), 12);
-        assert_eq!(initial_table_size(1_000_000), 15);
-        assert_eq!(initial_table_size(10_000_000), 18);
+        assert_eq!(initial_table_size(100_000), 11);
+        assert_eq!(initial_table_size(1_000_000), 14);
+        assert_eq!(initial_table_size(10_000_000), 17);
         assert_eq!(initial_table_size(1_000_000_000), 19);
     }
 
@@ -884,11 +1040,9 @@ mod tests {
             index_writer.add_document(doc!(text_field => "b"));
         }
         assert!(index_writer.commit().is_err());
-        index.load_searchers().unwrap();
         let num_docs_containing = |s: &str| {
-            let searcher = index.searcher();
             let term_a = Term::from_field_text(text_field, s);
-            searcher.doc_freq(&term_a)
+            index.reader().unwrap().searcher().doc_freq(&term_a)
         };
         assert_eq!(num_docs_containing("a"), 100);
         assert_eq!(num_docs_containing("b"), 0);

src/indexer/merger.rs

@@ -194,17 +194,17 @@ impl IndexMerger {
                         fast_field_serializer,
                     )?;
                 }
-                FieldType::U64(ref options) | FieldType::I64(ref options) => {
-                    match options.get_fastfield_cardinality() {
-                        Some(Cardinality::SingleValue) => {
-                            self.write_single_fast_field(field, fast_field_serializer)?;
-                        }
-                        Some(Cardinality::MultiValues) => {
-                            self.write_multi_fast_field(field, fast_field_serializer)?;
-                        }
-                        None => {}
+                FieldType::U64(ref options)
+                | FieldType::I64(ref options)
+                | FieldType::Date(ref options) => match options.get_fastfield_cardinality() {
+                    Some(Cardinality::SingleValue) => {
+                        self.write_single_fast_field(field, fast_field_serializer)?;
                     }
-                }
+                    Some(Cardinality::MultiValues) => {
+                        self.write_multi_fast_field(field, fast_field_serializer)?;
+                    }
+                    None => {}
+                },
                 FieldType::Str(_) => {
                     // We don't handle str fast field for the moment
                     // They can be implemented using what is done
@@ -654,7 +654,7 @@ mod tests {
     use schema::IntOptions;
     use schema::Term;
     use schema::TextFieldIndexing;
-    use schema::INT_INDEXED;
+    use schema::INDEXED;
     use std::io::Cursor;
     use DocAddress;
     use IndexWriter;
@@ -671,11 +671,13 @@ mod tests {
             )
             .set_stored();
         let text_field = schema_builder.add_text_field("text", text_fieldtype);
+        let date_field = schema_builder.add_date_field("date", INDEXED);
         let score_fieldtype = schema::IntOptions::default().set_fast(Cardinality::SingleValue);
         let score_field = schema_builder.add_u64_field("score", score_fieldtype);
         let bytes_score_field = schema_builder.add_bytes_field("score_bytes");
         let index = Index::create_in_ram(schema_builder.build());
-
+        let reader = index.reader().unwrap();
+        let curr_time = chrono::Utc::now();
         let add_score_bytes = |doc: &mut Document, score: u32| {
             let mut bytes = Vec::new();
             bytes
@@ -692,6 +694,7 @@ mod tests {
                     let mut doc = Document::default();
                     doc.add_text(text_field, "af b");
                     doc.add_u64(score_field, 3);
+                    doc.add_date(date_field, &curr_time);
                     add_score_bytes(&mut doc, 3);
                     index_writer.add_document(doc);
                 }
@@ -717,6 +720,7 @@ mod tests {
                 {
                     let mut doc = Document::default();
                     doc.add_text(text_field, "af b");
+                    doc.add_date(date_field, &curr_time);
                     doc.add_u64(score_field, 11);
                     add_score_bytes(&mut doc, 11);
                     index_writer.add_document(doc);
@@ -744,8 +748,8 @@ mod tests {
             index_writer.wait_merging_threads().unwrap();
         }
         {
-            index.load_searchers().unwrap();
-            let searcher = index.searcher();
+            reader.reload().unwrap();
+            let searcher = reader.searcher();
             let get_doc_ids = |terms: Vec<Term>| {
                 let query = BooleanQuery::new_multiterms_query(terms);
                 let top_docs = searcher.search(&query, &TestCollector).unwrap();
@@ -774,6 +778,10 @@ mod tests {
                         DocAddress(0, 4)
                     ]
                 );
+                assert_eq!(
+                    get_doc_ids(vec![Term::from_field_date(date_field, &curr_time)]),
+                    vec![DocAddress(0, 0), DocAddress(0, 3)]
+                );
             }
             {
                 let doc = searcher.doc(DocAddress(0, 0)).unwrap();
@@ -837,7 +845,7 @@ mod tests {
         let bytes_score_field = schema_builder.add_bytes_field("score_bytes");
         let index = Index::create_in_ram(schema_builder.build());
         let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
-
+        let reader = index.reader().unwrap();
         let search_term = |searcher: &Searcher, term: Term| {
             let collector = FastFieldTestCollector::for_field(score_field);
             let bytes_collector = BytesFastFieldTestCollector::for_field(bytes_score_field);
@@ -874,8 +882,8 @@ mod tests {
                 bytes_score_field => vec![0u8, 0, 0, 3],
             ));
             index_writer.commit().expect("committed");
-            index.load_searchers().unwrap();
-            let ref searcher = *index.searcher();
+            reader.reload().unwrap();
+            let searcher = reader.searcher();
             assert_eq!(searcher.num_docs(), 2);
             assert_eq!(searcher.segment_readers()[0].num_docs(), 2);
             assert_eq!(searcher.segment_readers()[0].max_doc(), 3);
@@ -921,8 +929,8 @@ mod tests {
                 bytes_score_field => vec![0u8, 0, 27, 88],
             ));
             index_writer.commit().expect("committed");
-            index.load_searchers().unwrap();
-            let searcher = index.searcher();
+            reader.reload().unwrap();
+            let searcher = reader.searcher();
 
             assert_eq!(searcher.segment_readers().len(), 2);
             assert_eq!(searcher.num_docs(), 3);
@@ -983,8 +991,8 @@ mod tests {
                 .expect("Failed to initiate merge")
                 .wait()
                 .expect("Merging failed");
-            index.load_searchers().unwrap();
-            let searcher = index.searcher();
+            reader.reload().unwrap();
+            let searcher = reader.searcher();
             assert_eq!(searcher.segment_readers().len(), 1);
             assert_eq!(searcher.num_docs(), 3);
             assert_eq!(searcher.segment_readers()[0].num_docs(), 3);
@@ -1029,8 +1037,8 @@ mod tests {
             index_writer.delete_term(Term::from_field_text(text_field, "c"));
             index_writer.commit().unwrap();
 
-            index.load_searchers().unwrap();
-            let searcher = index.searcher();
+            reader.reload().unwrap();
+            let searcher = reader.searcher();
             assert_eq!(searcher.segment_readers().len(), 1);
             assert_eq!(searcher.num_docs(), 2);
             assert_eq!(searcher.segment_readers()[0].num_docs(), 2);
@@ -1080,9 +1088,9 @@ mod tests {
                 .expect("Failed to initiate merge")
                 .wait()
                 .expect("Merging failed");
-            index.load_searchers().unwrap();
+            reader.reload().unwrap();
 
-            let ref searcher = *index.searcher();
+            let searcher = reader.searcher();
             assert_eq!(searcher.segment_readers().len(), 1);
             assert_eq!(searcher.num_docs(), 2);
             assert_eq!(searcher.segment_readers()[0].num_docs(), 2);
@@ -1130,9 +1138,9 @@ mod tests {
             let segment_ids = index
                 .searchable_segment_ids()
                 .expect("Searchable segments failed.");
-            index.load_searchers().unwrap();
+            reader.reload().unwrap();
 
-            let ref searcher = *index.searcher();
+            let searcher = reader.searcher();
             assert!(segment_ids.is_empty());
             assert!(searcher.segment_readers().is_empty());
             assert_eq!(searcher.num_docs(), 0);
@@ -1144,6 +1152,7 @@ mod tests {
         let mut schema_builder = schema::Schema::builder();
         let facet_field = schema_builder.add_facet_field("facet");
         let index = Index::create_in_ram(schema_builder.build());
+        let reader = index.reader().unwrap();
         {
             let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
             let index_doc = |index_writer: &mut IndexWriter, doc_facets: &[&str]| {
@@ -1173,9 +1182,9 @@ mod tests {
             index_writer.commit().expect("committed");
         }
 
-        index.load_searchers().unwrap();
+        reader.reload().unwrap();
         let test_searcher = |expected_num_docs: usize, expected: &[(&str, u64)]| {
-            let searcher = index.searcher();
+            let searcher = reader.searcher();
             let mut facet_collector = FacetCollector::for_field(facet_field);
             facet_collector.add_facet(Facet::from("/top"));
             let (count, facet_counts) = searcher
@@ -1217,7 +1226,7 @@ mod tests {
                 .wait()
                 .expect("Merging failed");
             index_writer.wait_merging_threads().unwrap();
-            index.load_searchers().unwrap();
+            reader.reload().unwrap();
             test_searcher(
                 11,
                 &[
@@ -1238,7 +1247,7 @@ mod tests {
             let facet_term = Term::from_facet(facet_field, &facet);
             index_writer.delete_term(facet_term);
             index_writer.commit().unwrap();
-            index.load_searchers().unwrap();
+            reader.reload().unwrap();
             test_searcher(
                 9,
                 &[
@@ -1256,15 +1265,15 @@ mod tests {
     #[test]
     fn test_bug_merge() {
         let mut schema_builder = schema::Schema::builder();
-        let int_field = schema_builder.add_u64_field("intvals", INT_INDEXED);
+        let int_field = schema_builder.add_u64_field("intvals", INDEXED);
         let index = Index::create_in_ram(schema_builder.build());
         let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
         index_writer.add_document(doc!(int_field => 1u64));
         index_writer.commit().expect("commit failed");
         index_writer.add_document(doc!(int_field => 1u64));
         index_writer.commit().expect("commit failed");
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        let reader = index.reader().unwrap();
+        let searcher = reader.searcher();
         assert_eq!(searcher.num_docs(), 2);
         index_writer.delete_term(Term::from_field_u64(int_field, 1));
         let segment_ids = index
@@ -1275,10 +1284,10 @@ mod tests {
             .expect("Failed to initiate merge")
             .wait()
             .expect("Merging failed");
-        index.load_searchers().unwrap();
+        reader.reload().unwrap();
         // commit has not been called yet. The document should still be
         // there.
-        assert_eq!(index.searcher().num_docs(), 2);
+        assert_eq!(reader.searcher().num_docs(), 2);
     }
 
     #[test]
@@ -1289,7 +1298,7 @@ mod tests {
             .set_indexed();
         let int_field = schema_builder.add_u64_field("intvals", int_options);
         let index = Index::create_in_ram(schema_builder.build());
-
+        let reader = index.reader().unwrap();
         {
             let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
             let mut doc = Document::default();
@@ -1310,8 +1319,8 @@ mod tests {
                 .expect("Merging failed");
 
             // assert delete has not been committed
-            index.load_searchers().unwrap();
-            let searcher = index.searcher();
+            reader.reload().expect("failed to load searcher 1");
+            let searcher = reader.searcher();
             assert_eq!(searcher.num_docs(), 2);
 
             index_writer.commit().unwrap();
@@ -1319,13 +1328,13 @@ mod tests {
             index_writer.wait_merging_threads().unwrap();
         }
 
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        reader.reload().unwrap();
+        let searcher = reader.searcher();
         assert_eq!(searcher.num_docs(), 0);
     }
 
     #[test]
-    fn test_merge_multivalued_int_fields() {
+    fn test_merge_multivalued_int_fields_simple() {
         let mut schema_builder = schema::Schema::builder();
         let int_options = IntOptions::default()
             .set_fast(Cardinality::MultiValues)
@@ -1342,7 +1351,6 @@ mod tests {
                 }
                 index_writer.add_document(doc);
             };
-
             index_doc(&mut index_writer, &[1, 2]);
             index_doc(&mut index_writer, &[1, 2, 3]);
             index_doc(&mut index_writer, &[4, 5]);
@@ -1351,19 +1359,14 @@ mod tests {
             index_doc(&mut index_writer, &[3]);
             index_doc(&mut index_writer, &[17]);
             index_writer.commit().expect("committed");
-
             index_doc(&mut index_writer, &[20]);
             index_writer.commit().expect("committed");
-
             index_doc(&mut index_writer, &[28, 27]);
             index_doc(&mut index_writer, &[1_000]);
-
             index_writer.commit().expect("committed");
         }
-        index.load_searchers().unwrap();
-
-        let searcher = index.searcher();
-
+        let reader = index.reader().unwrap();
+        let searcher = reader.searcher();
         let mut vals: Vec<u64> = Vec::new();
 
         {
@@ -1429,13 +1432,14 @@ mod tests {
                 .expect("Failed to initiate merge")
                 .wait()
                 .expect("Merging failed");
-            index_writer.wait_merging_threads().unwrap();
+            index_writer
+                .wait_merging_threads()
+                .expect("Wait for merging threads");
         }
-
-        index.load_searchers().unwrap();
+        reader.reload().expect("Load searcher");
 
         {
-            let searcher = index.searcher();
+            let searcher = reader.searcher();
             println!(
                 "{:?}",
                 searcher

src/indexer/operation.rs

@@ -14,3 +14,10 @@ pub struct AddOperation {
     pub opstamp: u64,
     pub document: Document,
 }
+
+/// UserOperation is an enum type that encapsulates other operation types.
+#[derive(Eq, PartialEq, Debug)]
+pub enum UserOperation {
+    Add(Document),
+    Delete(Term),
+}

src/indexer/segment_updater.rs

@@ -62,7 +62,7 @@ pub fn save_new_metas(schema: Schema, directory: &mut Directory) -> Result<()> {
 /// Save the index meta file.
 /// This operation is atomic:
 /// Either
-//  - it fails, in which case an error is returned,
+///  - it fails, in which case an error is returned,
 /// and the `meta.json` remains untouched,
 /// - it success, and `meta.json` is written
 /// and flushed.
@@ -565,9 +565,8 @@ mod tests {
             index_writer.delete_term(term);
             assert!(index_writer.commit().is_ok());
         }
-
-        index.load_searchers().unwrap();
-        assert_eq!(index.searcher().num_docs(), 302);
+        let reader = index.reader().unwrap();
+        assert_eq!(reader.searcher().num_docs(), 302);
 
         {
             index_writer
@@ -575,9 +574,9 @@ mod tests {
                 .expect("waiting for merging threads");
         }
 
-        index.load_searchers().unwrap();
-        assert_eq!(index.searcher().segment_readers().len(), 1);
-        assert_eq!(index.searcher().num_docs(), 302);
+        reader.reload().unwrap();
+        assert_eq!(reader.searcher().segment_readers().len(), 1);
+        assert_eq!(reader.searcher().num_docs(), 302);
     }
 
     #[test]
@@ -636,18 +635,18 @@ mod tests {
                 .expect("waiting for merging threads");
         }
 
-        index.load_searchers().unwrap();
-        assert_eq!(index.searcher().num_docs(), 0);
+        let reader = index.reader().unwrap();
+        assert_eq!(reader.searcher().num_docs(), 0);
 
         let seg_ids = index
             .searchable_segment_ids()
             .expect("Searchable segments failed.");
         assert!(seg_ids.is_empty());
 
-        index.load_searchers().unwrap();
-        assert_eq!(index.searcher().num_docs(), 0);
+        reader.reload().unwrap();
+        assert_eq!(reader.searcher().num_docs(), 0);
         // empty segments should be erased
         assert!(index.searchable_segment_metas().unwrap().is_empty());
-        assert!(index.searcher().segment_readers().is_empty());
+        assert!(reader.searcher().segment_readers().is_empty());
     }
 }

src/indexer/segment_writer.rs

@@ -171,6 +171,17 @@ impl SegmentWriter {
                         }
                     }
                 }
+                FieldType::Date(ref int_option) => {
+                    if int_option.is_indexed() {
+                        for field_value in field_values {
+                            let term = Term::from_field_i64(
+                                field_value.field(),
+                                field_value.value().date_value().timestamp(),
+                            );
+                            self.multifield_postings.subscribe(doc_id, &term);
+                        }
+                    }
+                }
                 FieldType::I64(ref int_option) => {
                     if int_option.is_indexed() {
                         for field_value in field_values {

src/indexer/stamper.rs

@@ -1,3 +1,4 @@
+use std::ops::Range;
 use std::sync::atomic::Ordering;
 use std::sync::Arc;
 
@@ -60,6 +61,16 @@ impl Stamper {
     pub fn stamp(&self) -> u64 {
         self.0.fetch_add(1u64, Ordering::SeqCst) as u64
     }
+
+    /// Given a desired count `n`, `stamps` returns an iterator that
+    /// will supply `n` number of u64 stamps.
+    pub fn stamps(&self, n: u64) -> Range<u64> {
+        let start = self.0.fetch_add(n, Ordering::SeqCst);
+        Range {
+            start,
+            end: start + n,
+        }
+    }
 }
 
 #[cfg(test)]
@@ -78,5 +89,7 @@ mod test {
 
         assert_eq!(stamper.stamp(), 10u64);
         assert_eq!(stamper_clone.stamp(), 11u64);
+        assert_eq!(stamper.stamps(3u64), (12..15));
+        assert_eq!(stamper.stamp(), 15u64);
     }
 }

src/lib.rs

@@ -75,9 +75,9 @@
 //!
 //! // # Searching
 //!
-//! index.load_searchers()?;
+//! let reader = index.reader()?;
 //!
-//! let searcher = index.searcher();
+//! let searcher = reader.searcher();
 //!
 //! let query_parser = QueryParser::for_index(&index, vec![title, body]);
 //!
@@ -132,13 +132,13 @@ extern crate byteorder;
 extern crate combine;
 extern crate crossbeam;
 extern crate fnv;
-extern crate fst;
-extern crate fst_regex;
 extern crate futures;
 extern crate futures_cpupool;
 extern crate htmlescape;
 extern crate itertools;
 extern crate levenshtein_automata;
+#[cfg(feature = "mmap")]
+extern crate memmap;
 extern crate num_cpus;
 extern crate owning_ref;
 extern crate regex;
@@ -146,6 +146,7 @@ extern crate rust_stemmers;
 extern crate scoped_pool;
 extern crate serde;
 extern crate stable_deref_trait;
+extern crate tantivy_fst;
 extern crate tempdir;
 extern crate tempfile;
 extern crate uuid;
@@ -168,7 +169,7 @@ extern crate maplit;
 extern crate test;
 
 #[macro_use]
-extern crate downcast;
+extern crate downcast_rs;
 
 #[macro_use]
 extern crate fail;
@@ -185,11 +186,15 @@ pub use error::TantivyError;
 pub use error::TantivyError as Error;
 
 extern crate census;
+pub extern crate chrono;
 extern crate owned_read;
 
 /// Tantivy result.
 pub type Result<T> = std::result::Result<T, error::TantivyError>;
 
+/// Tantivy DateTime
+pub type DateTime = chrono::DateTime<chrono::Utc>;
+
 mod common;
 mod core;
 mod indexer;
@@ -210,6 +215,9 @@ pub mod space_usage;
 pub mod store;
 pub mod termdict;
 
+mod reader;
+
+pub use self::reader::{IndexReader, IndexReaderBuilder, ReloadPolicy};
 mod snippet;
 pub use self::snippet::{Snippet, SnippetGenerator};
 
@@ -298,6 +306,7 @@ mod tests {
     use Index;
     use IndexWriter;
     use Postings;
+    use ReloadPolicy;
 
     pub fn assert_nearly_equals(expected: f32, val: f32) {
         assert!(
@@ -386,8 +395,8 @@ mod tests {
             index_writer.commit().unwrap();
         }
         {
-            index.load_searchers().unwrap();
-            let searcher = index.searcher();
+            let reader = index.reader().unwrap();
+            let searcher = reader.searcher();
             let term_a = Term::from_field_text(text_field, "a");
             assert_eq!(searcher.doc_freq(&term_a), 3);
             let term_b = Term::from_field_text(text_field, "b");
@@ -414,8 +423,8 @@ mod tests {
             index_writer.commit().unwrap();
         }
         {
-            index.load_searchers().unwrap();
-            let searcher = index.searcher();
+            let index_reader = index.reader().unwrap();
+            let searcher = index_reader.searcher();
             let reader = searcher.segment_reader(0);
             {
                 let fieldnorm_reader = reader.get_fieldnorms_reader(text_field);
@@ -450,8 +459,8 @@ mod tests {
             index_writer.commit().unwrap();
         }
         {
-            index.load_searchers().unwrap();
-            let searcher = index.searcher();
+            let reader = index.reader().unwrap();
+            let searcher = reader.searcher();
             let segment_reader: &SegmentReader = searcher.segment_reader(0);
             let fieldnorms_reader = segment_reader.get_fieldnorms_reader(text_field);
             assert_eq!(fieldnorms_reader.fieldnorm(0), 3);
@@ -479,6 +488,11 @@ mod tests {
         let term_c = Term::from_field_text(text_field, "c");
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
+        let reader = index
+            .reader_builder()
+            .reload_policy(ReloadPolicy::Manual)
+            .try_into()
+            .unwrap();
         {
             // writing the segment
             let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
@@ -500,10 +514,10 @@ mod tests {
             index_writer.commit().unwrap();
         }
         {
-            index.load_searchers().unwrap();
-            let searcher = index.searcher();
-            let reader = searcher.segment_reader(0);
-            let inverted_index = reader.inverted_index(text_field);
+            reader.reload().unwrap();
+            let searcher = reader.searcher();
+            let segment_reader = searcher.segment_reader(0);
+            let inverted_index = segment_reader.inverted_index(text_field);
             assert!(inverted_index
                 .read_postings(&term_abcd, IndexRecordOption::WithFreqsAndPositions)
                 .is_none());
@@ -511,19 +525,19 @@ mod tests {
                 let mut postings = inverted_index
                     .read_postings(&term_a, IndexRecordOption::WithFreqsAndPositions)
                     .unwrap();
-                assert!(advance_undeleted(&mut postings, reader));
+                assert!(advance_undeleted(&mut postings, segment_reader));
                 assert_eq!(postings.doc(), 5);
-                assert!(!advance_undeleted(&mut postings, reader));
+                assert!(!advance_undeleted(&mut postings, segment_reader));
             }
             {
                 let mut postings = inverted_index
                     .read_postings(&term_b, IndexRecordOption::WithFreqsAndPositions)
                     .unwrap();
-                assert!(advance_undeleted(&mut postings, reader));
+                assert!(advance_undeleted(&mut postings, segment_reader));
                 assert_eq!(postings.doc(), 3);
-                assert!(advance_undeleted(&mut postings, reader));
+                assert!(advance_undeleted(&mut postings, segment_reader));
                 assert_eq!(postings.doc(), 4);
-                assert!(!advance_undeleted(&mut postings, reader));
+                assert!(!advance_undeleted(&mut postings, segment_reader));
             }
         }
         {
@@ -536,10 +550,10 @@ mod tests {
             index_writer.rollback().unwrap();
         }
         {
-            index.load_searchers().unwrap();
-            let searcher = index.searcher();
-            let reader = searcher.segment_reader(0);
-            let inverted_index = reader.inverted_index(term_abcd.field());
+            reader.reload().unwrap();
+            let searcher = reader.searcher();
+            let seg_reader = searcher.segment_reader(0);
+            let inverted_index = seg_reader.inverted_index(term_abcd.field());
 
             assert!(inverted_index
                 .read_postings(&term_abcd, IndexRecordOption::WithFreqsAndPositions)
@@ -548,19 +562,19 @@ mod tests {
                 let mut postings = inverted_index
                     .read_postings(&term_a, IndexRecordOption::WithFreqsAndPositions)
                     .unwrap();
-                assert!(advance_undeleted(&mut postings, reader));
+                assert!(advance_undeleted(&mut postings, seg_reader));
                 assert_eq!(postings.doc(), 5);
-                assert!(!advance_undeleted(&mut postings, reader));
+                assert!(!advance_undeleted(&mut postings, seg_reader));
             }
             {
                 let mut postings = inverted_index
                     .read_postings(&term_b, IndexRecordOption::WithFreqsAndPositions)
                     .unwrap();
-                assert!(advance_undeleted(&mut postings, reader));
+                assert!(advance_undeleted(&mut postings, seg_reader));
                 assert_eq!(postings.doc(), 3);
-                assert!(advance_undeleted(&mut postings, reader));
+                assert!(advance_undeleted(&mut postings, seg_reader));
                 assert_eq!(postings.doc(), 4);
-                assert!(!advance_undeleted(&mut postings, reader));
+                assert!(!advance_undeleted(&mut postings, seg_reader));
             }
         }
         {
@@ -573,10 +587,10 @@ mod tests {
             index_writer.commit().unwrap();
         }
         {
-            index.load_searchers().unwrap();
-            let searcher = index.searcher();
-            let reader = searcher.segment_reader(0);
-            let inverted_index = reader.inverted_index(term_abcd.field());
+            reader.reload().unwrap();
+            let searcher = reader.searcher();
+            let segment_reader = searcher.segment_reader(0);
+            let inverted_index = segment_reader.inverted_index(term_abcd.field());
             assert!(inverted_index
                 .read_postings(&term_abcd, IndexRecordOption::WithFreqsAndPositions)
                 .is_none());
@@ -584,25 +598,25 @@ mod tests {
                 let mut postings = inverted_index
                     .read_postings(&term_a, IndexRecordOption::WithFreqsAndPositions)
                     .unwrap();
-                assert!(!advance_undeleted(&mut postings, reader));
+                assert!(!advance_undeleted(&mut postings, segment_reader));
             }
             {
                 let mut postings = inverted_index
                     .read_postings(&term_b, IndexRecordOption::WithFreqsAndPositions)
                     .unwrap();
-                assert!(advance_undeleted(&mut postings, reader));
+                assert!(advance_undeleted(&mut postings, segment_reader));
                 assert_eq!(postings.doc(), 3);
-                assert!(advance_undeleted(&mut postings, reader));
+                assert!(advance_undeleted(&mut postings, segment_reader));
                 assert_eq!(postings.doc(), 4);
-                assert!(!advance_undeleted(&mut postings, reader));
+                assert!(!advance_undeleted(&mut postings, segment_reader));
             }
             {
                 let mut postings = inverted_index
                     .read_postings(&term_c, IndexRecordOption::WithFreqsAndPositions)
                     .unwrap();
-                assert!(advance_undeleted(&mut postings, reader));
+                assert!(advance_undeleted(&mut postings, segment_reader));
                 assert_eq!(postings.doc(), 4);
-                assert!(!advance_undeleted(&mut postings, reader));
+                assert!(!advance_undeleted(&mut postings, segment_reader));
             }
         }
     }
@@ -610,15 +624,15 @@ mod tests {
     #[test]
     fn test_indexed_u64() {
         let mut schema_builder = Schema::builder();
-        let field = schema_builder.add_u64_field("value", INT_INDEXED);
+        let field = schema_builder.add_u64_field("value", INDEXED);
         let schema = schema_builder.build();
 
         let index = Index::create_in_ram(schema);
         let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
         index_writer.add_document(doc!(field=>1u64));
         index_writer.commit().unwrap();
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        let reader = index.reader().unwrap();
+        let searcher = reader.searcher();
         let term = Term::from_field_u64(field, 1u64);
         let mut postings = searcher
             .segment_reader(0)
@@ -633,7 +647,7 @@ mod tests {
     #[test]
     fn test_indexed_i64() {
         let mut schema_builder = Schema::builder();
-        let value_field = schema_builder.add_i64_field("value", INT_INDEXED);
+        let value_field = schema_builder.add_i64_field("value", INDEXED);
         let schema = schema_builder.build();
 
         let index = Index::create_in_ram(schema);
@@ -641,8 +655,8 @@ mod tests {
         let negative_val = -1i64;
         index_writer.add_document(doc!(value_field => negative_val));
         index_writer.commit().unwrap();
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        let reader = index.reader().unwrap();
+        let searcher = reader.searcher();
         let term = Term::from_field_i64(value_field, negative_val);
         let mut postings = searcher
             .segment_reader(0)
@@ -664,8 +678,8 @@ mod tests {
         let mut index_writer = index.writer_with_num_threads(2, 6_000_000).unwrap();
         index_writer.add_document(doc!(text_field=>"a"));
         assert!(index_writer.commit().is_ok());
-        assert!(index.load_searchers().is_ok());
-        let searcher = index.searcher();
+        let reader = index.reader().unwrap();
+        let searcher = reader.searcher();
         let segment_reader = searcher.segment_reader(0);
         segment_reader.inverted_index(absent_field); //< should not panic
     }
@@ -676,6 +690,11 @@ mod tests {
         let text_field = schema_builder.add_text_field("text", TEXT);
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
+        let reader = index
+            .reader_builder()
+            .reload_policy(ReloadPolicy::Manual)
+            .try_into()
+            .unwrap();
 
         // writing the segment
         let mut index_writer = index.writer_with_num_threads(2, 6_000_000).unwrap();
@@ -701,8 +720,8 @@ mod tests {
         remove_document(&mut index_writer, "38");
         remove_document(&mut index_writer, "34");
         index_writer.commit().unwrap();
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        reader.reload().unwrap();
+        let searcher = reader.searcher();
         assert_eq!(searcher.num_docs(), 6);
     }
 
@@ -722,8 +741,8 @@ mod tests {
             index_writer.commit().unwrap();
         }
         {
-            index.load_searchers().unwrap();
-            let searcher = index.searcher();
+            let index_reader = index.reader().unwrap();
+            let searcher = index_reader.searcher();
             let reader = searcher.segment_reader(0);
             let inverted_index = reader.inverted_index(text_field);
             let term_abcd = Term::from_field_text(text_field, "abcd");
@@ -747,7 +766,7 @@ mod tests {
         let text_field = schema_builder.add_text_field("text", TEXT);
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
-
+        let reader = index.reader().unwrap();
         {
             // writing the segment
             let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
@@ -757,8 +776,8 @@ mod tests {
             index_writer.commit().unwrap();
         }
         {
-            index.load_searchers().unwrap();
-            let searcher = index.searcher();
+            reader.reload().unwrap();
+            let searcher = reader.searcher();
             let get_doc_ids = |terms: Vec<Term>| {
                 let query = BooleanQuery::new_multiterms_query(terms);
                 let topdocs = searcher.search(&query, &TestCollector).unwrap();
@@ -800,25 +819,22 @@ mod tests {
         let text_field = schema_builder.add_text_field("text", TEXT);
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
-
+        let reader = index
+            .reader_builder()
+            .reload_policy(ReloadPolicy::Manual)
+            .try_into()
+            .unwrap();
+        assert_eq!(reader.searcher().num_docs(), 0u64);
         {
             // writing the segment
             let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
-            {
-                let doc = doc!(text_field=>"af b");
-                index_writer.add_document(doc);
-            }
-            {
-                let doc = doc!(text_field=>"a b c");
-                index_writer.add_document(doc);
-            }
-            {
-                let doc = doc!(text_field=>"a b c d");
-                index_writer.add_document(doc);
-            }
+            index_writer.add_document(doc!(text_field=>"af b"));
+            index_writer.add_document(doc!(text_field=>"a b c"));
+            index_writer.add_document(doc!(text_field=>"a b c d"));
             index_writer.commit().unwrap();
         }
-        index.searcher();
+        reader.reload().unwrap();
+        assert_eq!(reader.searcher().num_docs(), 3u64);
     }
 
     #[test]
@@ -845,7 +861,7 @@ mod tests {
         let fast_field_unsigned = schema_builder.add_u64_field("unsigned", FAST);
         let fast_field_signed = schema_builder.add_i64_field("signed", FAST);
         let text_field = schema_builder.add_text_field("text", TEXT);
-        let stored_int_field = schema_builder.add_u64_field("text", INT_STORED);
+        let stored_int_field = schema_builder.add_u64_field("text", STORED);
         let schema = schema_builder.build();
 
         let index = Index::create_in_ram(schema);
@@ -855,9 +871,8 @@ mod tests {
             index_writer.add_document(document);
             index_writer.commit().unwrap();
         }
-
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        let reader = index.reader().unwrap();
+        let searcher = reader.searcher();
         let segment_reader: &SegmentReader = searcher.segment_reader(0);
         {
             let fast_field_reader_res = segment_reader.fast_field_reader::<u64>(text_field);

src/macros.rs

@@ -61,7 +61,7 @@ macro_rules! doc(
     };
     // if there is a trailing comma retry with the trailing comma stripped.
     ($($field:expr => $value:expr),+ ,) => {
-        doc!( $( $field => $value ), *);
+        doc!( $( $field => $value ), *)
     };
 );
 

src/positions/mod.rs

@@ -34,10 +34,6 @@ const COMPRESSION_BLOCK_SIZE: usize = BitPacker4x::BLOCK_LEN;
 const LONG_SKIP_IN_BLOCKS: usize = 1_024;
 const LONG_SKIP_INTERVAL: u64 = (LONG_SKIP_IN_BLOCKS * COMPRESSION_BLOCK_SIZE) as u64;
 
-lazy_static! {
-    static ref BIT_PACKER: BitPacker4x = BitPacker4x::new();
-}
-
 #[cfg(test)]
 pub mod tests {
 

src/positions/reader.rs

@@ -1,4 +1,23 @@
-use super::BIT_PACKER;
+/// Positions works as a long sequence of compressed block.
+/// All terms are chained one after the other.
+///
+/// When accessing the position of a term, we get a positions_idx from the `Terminfo`.
+/// This means we need to skip to the `nth` positions efficiently.
+///
+/// This is done thanks to two levels of skiping that we refer to in the code
+/// as `long_skip` and `short_skip`.
+///
+/// The `long_skip` makes it possible to skip every 1_024 compression blocks (= 131_072 positions).
+/// Skipping offset are simply stored one after as an offset stored over 8 bytes.
+///
+/// We find the number of long skips, as `n / long_skip`.
+///
+/// Blocks are compressed using bitpacking, so `skip_read` contains the number of bytes
+/// (values can go from 0bit to 32 bits) required to decompressed every block.
+///
+/// A given block obviously takes `(128 x  num_bit_for_the_block / num_bits_in_a_byte)`,
+/// so skipping a block without decompressing it is just a matter of advancing that many
+/// bytes.
 use bitpacking::{BitPacker, BitPacker4x};
 use common::{BinarySerializable, FixedSize};
 use directory::ReadOnlySource;
@@ -8,9 +27,65 @@ use positions::LONG_SKIP_INTERVAL;
 use positions::LONG_SKIP_IN_BLOCKS;
 use postings::compression::compressed_block_size;
 
+struct Positions {
+    bit_packer: BitPacker4x,
+    skip_source: ReadOnlySource,
+    position_source: ReadOnlySource,
+    long_skip_source: ReadOnlySource,
+}
+
+impl Positions {
+    pub fn new(position_source: ReadOnlySource, skip_source: ReadOnlySource) -> Positions {
+        let skip_len = skip_source.len();
+        let (body, footer) = skip_source.split(skip_len - u32::SIZE_IN_BYTES);
+        let num_long_skips = u32::deserialize(&mut footer.as_slice()).expect("Index corrupted");
+        let body_split = body.len() - u64::SIZE_IN_BYTES * (num_long_skips as usize);
+        let (skip_source, long_skip_source) = body.split(body_split);
+        Positions {
+            bit_packer: BitPacker4x::new(),
+            skip_source,
+            long_skip_source,
+            position_source,
+        }
+    }
+
+    /// Returns the offset of the block associated to the given `long_skip_id`.
+    ///
+    /// One `long_skip_id` means `LONG_SKIP_IN_BLOCKS` blocks.
+    fn long_skip(&self, long_skip_id: usize) -> u64 {
+        if long_skip_id == 0 {
+            return 0;
+        }
+        let long_skip_slice = self.long_skip_source.as_slice();
+        let mut long_skip_blocks: &[u8] = &long_skip_slice[(long_skip_id - 1) * 8..][..8];
+        u64::deserialize(&mut long_skip_blocks).expect("Index corrupted")
+    }
+
+    fn reader(&self, offset: u64) -> PositionReader {
+        let long_skip_id = (offset / LONG_SKIP_INTERVAL) as usize;
+        let small_skip = (offset % LONG_SKIP_INTERVAL) as usize;
+        let offset_num_bytes: u64 = self.long_skip(long_skip_id);
+        let mut position_read = OwnedRead::new(self.position_source.clone());
+        position_read.advance(offset_num_bytes as usize);
+        let mut skip_read = OwnedRead::new(self.skip_source.clone());
+        skip_read.advance(long_skip_id * LONG_SKIP_IN_BLOCKS);
+        let mut position_reader = PositionReader {
+            bit_packer: self.bit_packer,
+            skip_read,
+            position_read,
+            inner_offset: 0,
+            buffer: Box::new([0u32; 128]),
+            ahead: None,
+        };
+        position_reader.skip(small_skip);
+        position_reader
+    }
+}
+
 pub struct PositionReader {
     skip_read: OwnedRead,
     position_read: OwnedRead,
+    bit_packer: BitPacker4x,
     inner_offset: usize,
     buffer: Box<[u32; 128]>,
     ahead: Option<usize>, // if None, no block is loaded.
@@ -27,6 +102,7 @@ pub struct PositionReader {
 // If the requested number of els ends exactly at a given block, the next
 // block is not decompressed.
 fn read_impl(
+    bit_packer: BitPacker4x,
     mut position: &[u8],
     buffer: &mut [u32; 128],
     mut inner_offset: usize,
@@ -37,21 +113,23 @@ fn read_impl(
     let mut output_len = output.len();
     let mut ahead = 0;
     loop {
-        let available_len = 128 - inner_offset;
+        let available_len = COMPRESSION_BLOCK_SIZE - inner_offset;
+        // We have enough elements in the current block.
+        // Let's copy the requested elements in the output buffer,
+        // and return.
         if output_len <= available_len {
             output[output_start..].copy_from_slice(&buffer[inner_offset..][..output_len]);
             return ahead;
-        } else {
-            output[output_start..][..available_len].copy_from_slice(&buffer[inner_offset..]);
-            output_len -= available_len;
-            output_start += available_len;
-            inner_offset = 0;
-            let num_bits = num_bits[ahead];
-            BitPacker4x::new().decompress(position, &mut buffer[..], num_bits);
-            let block_len = compressed_block_size(num_bits);
-            position = &position[block_len..];
-            ahead += 1;
         }
+        output[output_start..][..available_len].copy_from_slice(&buffer[inner_offset..]);
+        output_len -= available_len;
+        output_start += available_len;
+        inner_offset = 0;
+        let num_bits = num_bits[ahead];
+        bit_packer.decompress(position, &mut buffer[..], num_bits);
+        let block_len = compressed_block_size(num_bits);
+        position = &position[block_len..];
+        ahead += 1;
     }
 }
 
@@ -61,35 +139,7 @@ impl PositionReader {
         skip_source: ReadOnlySource,
         offset: u64,
     ) -> PositionReader {
-        let skip_len = skip_source.len();
-        let (body, footer) = skip_source.split(skip_len - u32::SIZE_IN_BYTES);
-        let num_long_skips = u32::deserialize(&mut footer.as_slice()).expect("Index corrupted");
-        let body_split = body.len() - u64::SIZE_IN_BYTES * (num_long_skips as usize);
-        let (skip_body, long_skips) = body.split(body_split);
-        let long_skip_id = (offset / LONG_SKIP_INTERVAL) as usize;
-        let small_skip = (offset - (long_skip_id as u64) * (LONG_SKIP_INTERVAL as u64)) as usize;
-        let offset_num_bytes: u64 = {
-            if long_skip_id > 0 {
-                let mut long_skip_blocks: &[u8] =
-                    &long_skips.as_slice()[(long_skip_id - 1) * 8..][..8];
-                u64::deserialize(&mut long_skip_blocks).expect("Index corrupted") * 16
-            } else {
-                0
-            }
-        };
-        let mut position_read = OwnedRead::new(position_source);
-        position_read.advance(offset_num_bytes as usize);
-        let mut skip_read = OwnedRead::new(skip_body);
-        skip_read.advance(long_skip_id * LONG_SKIP_IN_BLOCKS);
-        let mut position_reader = PositionReader {
-            skip_read,
-            position_read,
-            inner_offset: 0,
-            buffer: Box::new([0u32; 128]),
-            ahead: None,
-        };
-        position_reader.skip(small_skip);
-        position_reader
+        Positions::new(position_source, skip_source).reader(offset)
     }
 
     /// Fills a buffer with the next `output.len()` integers.
@@ -101,10 +151,13 @@ impl PositionReader {
         if self.ahead != Some(0) {
             // the block currently available is not the block
             // for the current position
-            BIT_PACKER.decompress(position_data, self.buffer.as_mut(), num_bits);
+            self.bit_packer
+                .decompress(position_data, self.buffer.as_mut(), num_bits);
+            self.ahead = Some(0);
         }
         let block_len = compressed_block_size(num_bits);
         self.ahead = Some(read_impl(
+            self.bit_packer,
             &position_data[block_len..],
             self.buffer.as_mut(),
             self.inner_offset,
@@ -133,14 +186,13 @@ impl PositionReader {
             }
         });
 
-        let skip_len = self.skip_read.as_ref()[..num_blocks_to_advance]
+        let skip_len_in_bits = self.skip_read.as_ref()[..num_blocks_to_advance]
             .iter()
-            .cloned()
-            .map(|num_bit| num_bit as usize)
+            .map(|num_bits| *num_bits as usize)
             .sum::<usize>()
-            * (COMPRESSION_BLOCK_SIZE / 8);
-
+            * COMPRESSION_BLOCK_SIZE;
+        let skip_len_in_bytes = skip_len_in_bits / 8;
         self.skip_read.advance(num_blocks_to_advance);
-        self.position_read.advance(skip_len);
+        self.position_read.advance(skip_len_in_bytes);
     }
 }

src/positions/serializer.rs

@@ -1,29 +1,30 @@
-use super::BIT_PACKER;
 use bitpacking::BitPacker;
+use bitpacking::BitPacker4x;
 use common::BinarySerializable;
+use common::CountingWriter;
 use positions::{COMPRESSION_BLOCK_SIZE, LONG_SKIP_INTERVAL};
-use std::io;
+use std::io::{self, Write};
 
 pub struct PositionSerializer<W: io::Write> {
-    write_stream: W,
+    bit_packer: BitPacker4x,
+    write_stream: CountingWriter<W>,
     write_skiplist: W,
     block: Vec<u32>,
     buffer: Vec<u8>,
     num_ints: u64,
     long_skips: Vec<u64>,
-    cumulated_num_bits: u64,
 }
 
 impl<W: io::Write> PositionSerializer<W> {
     pub fn new(write_stream: W, write_skiplist: W) -> PositionSerializer<W> {
         PositionSerializer {
-            write_stream,
+            bit_packer: BitPacker4x::new(),
+            write_stream: CountingWriter::wrap(write_stream),
             write_skiplist,
             block: Vec::with_capacity(128),
             buffer: vec![0u8; 128 * 4],
             num_ints: 0u64,
             long_skips: Vec::new(),
-            cumulated_num_bits: 0u64,
         }
     }
 
@@ -50,14 +51,15 @@ impl<W: io::Write> PositionSerializer<W> {
     }
 
     fn flush_block(&mut self) -> io::Result<()> {
-        let num_bits = BIT_PACKER.num_bits(&self.block[..]);
-        self.cumulated_num_bits += u64::from(num_bits);
+        let num_bits = self.bit_packer.num_bits(&self.block[..]);
         self.write_skiplist.write_all(&[num_bits])?;
-        let written_len = BIT_PACKER.compress(&self.block[..], &mut self.buffer, num_bits);
+        let written_len = self
+            .bit_packer
+            .compress(&self.block[..], &mut self.buffer, num_bits);
         self.write_stream.write_all(&self.buffer[..written_len])?;
         self.block.clear();
         if (self.num_ints % LONG_SKIP_INTERVAL) == 0u64 {
-            self.long_skips.push(self.cumulated_num_bits);
+            self.long_skips.push(self.write_stream.written_bytes());
         }
         Ok(())
     }

src/postings/mod.rs

@@ -31,7 +31,6 @@ pub(crate) use self::stacker::compute_table_size;
 pub use common::HasLen;
 
 pub(crate) const USE_SKIP_INFO_LIMIT: u32 = COMPRESSION_BLOCK_SIZE as u32;
-
 pub(crate) type UnorderedTermId = u64;
 
 #[cfg_attr(feature = "cargo-clippy", allow(clippy::enum_variant_names))]
@@ -58,7 +57,7 @@ pub mod tests {
     use rand::{Rng, SeedableRng};
     use schema::Field;
     use schema::IndexRecordOption;
-    use schema::{Document, Schema, Term, INT_INDEXED, STRING, TEXT};
+    use schema::{Document, Schema, Term, INDEXED, STRING, TEXT};
     use std::iter;
     use DocId;
     use Score;
@@ -101,9 +100,8 @@ pub mod tests {
         }
         index_writer.add_document(doc!(title => r#"abc be be be be abc"#));
         index_writer.commit().unwrap();
-        index.load_searchers().unwrap();
 
-        let searcher = index.searcher();
+        let searcher = index.reader().unwrap().searcher();
         let inverted_index = searcher.segment_reader(0u32).inverted_index(title);
         let term = Term::from_field_text(title, "abc");
 
@@ -293,9 +291,8 @@ pub mod tests {
             }
             assert!(index_writer.commit().is_ok());
         }
-        index.load_searchers().unwrap();
         let term_a = Term::from_field_text(text_field, "a");
-        let searcher = index.searcher();
+        let searcher = index.reader().unwrap().searcher();
         let segment_reader = searcher.segment_reader(0);
         let mut postings = segment_reader
             .inverted_index(text_field)
@@ -317,7 +314,7 @@ pub mod tests {
 
         let index = {
             let mut schema_builder = Schema::builder();
-            let value_field = schema_builder.add_u64_field("value", INT_INDEXED);
+            let value_field = schema_builder.add_u64_field("value", INDEXED);
             let schema = schema_builder.build();
 
             let index = Index::create_in_ram(schema);
@@ -332,10 +329,9 @@ pub mod tests {
                 }
                 assert!(index_writer.commit().is_ok());
             }
-            index.load_searchers().unwrap();
             index
         };
-        let searcher = index.searcher();
+        let searcher = index.reader().unwrap().searcher();
         let segment_reader = searcher.segment_reader(0);
 
         // check that the basic usage works
@@ -403,8 +399,7 @@ pub mod tests {
             index_writer.delete_term(term_0);
             assert!(index_writer.commit().is_ok());
         }
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        let searcher = index.reader().unwrap().searcher();
         let segment_reader = searcher.segment_reader(0);
 
         // make sure seeking still works
@@ -451,12 +446,9 @@ pub mod tests {
         {
             let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
             index_writer.delete_term(term_1);
-
             assert!(index_writer.commit().is_ok());
         }
-        index.load_searchers().unwrap();
-
-        let searcher = index.searcher();
+        let searcher = index.reader().unwrap().searcher();
 
         // finally, check that it's empty
         {
@@ -512,7 +504,6 @@ pub mod tests {
                 }
                 assert!(index_writer.commit().is_ok());
             }
-            index.load_searchers().unwrap();
             index
         };
     }

src/postings/postings_writer.rs

@@ -33,9 +33,10 @@ fn posting_from_field_entry(field_entry: &FieldEntry) -> Box<PostingsWriter> {
                 }
             })
             .unwrap_or_else(|| SpecializedPostingsWriter::<NothingRecorder>::new_boxed()),
-        FieldType::U64(_) | FieldType::I64(_) | FieldType::HierarchicalFacet => {
-            SpecializedPostingsWriter::<NothingRecorder>::new_boxed()
-        }
+        FieldType::U64(_)
+        | FieldType::I64(_)
+        | FieldType::Date(_)
+        | FieldType::HierarchicalFacet => SpecializedPostingsWriter::<NothingRecorder>::new_boxed(),
         FieldType::Bytes => {
             // FieldType::Bytes cannot actually be indexed.
             // TODO fix during the indexer refactoring described in #276
@@ -51,6 +52,31 @@ pub struct MultiFieldPostingsWriter {
     per_field_postings_writers: Vec<Box<PostingsWriter>>,
 }
 
+fn make_field_partition(
+    term_offsets: &[(&[u8], Addr, UnorderedTermId)],
+) -> Vec<(Field, usize, usize)> {
+    let term_offsets_it = term_offsets
+        .iter()
+        .map(|(key, _, _)| Term::wrap(key).field())
+        .enumerate();
+    let mut prev_field = Field(u32::max_value());
+    let mut fields = vec![];
+    let mut offsets = vec![];
+    for (offset, field) in term_offsets_it {
+        if field != prev_field {
+            prev_field = field;
+            fields.push(field);
+            offsets.push(offset);
+        }
+    }
+    offsets.push(term_offsets.len());
+    let mut field_offsets = vec![];
+    for i in 0..fields.len() {
+        field_offsets.push((fields[i], offsets[i], offsets[i + 1]));
+    }
+    field_offsets
+}
+
 impl MultiFieldPostingsWriter {
     /// Create a new `MultiFieldPostingsWriter` given
     /// a schema and a heap.
@@ -96,36 +122,16 @@ impl MultiFieldPostingsWriter {
         &self,
         serializer: &mut InvertedIndexSerializer,
     ) -> Result<HashMap<Field, HashMap<UnorderedTermId, TermOrdinal>>> {
-        let mut term_offsets: Vec<(&[u8], Addr, UnorderedTermId)> = self
-            .term_index
-            .iter()
-            .map(|(term_bytes, addr, bucket_id)| (term_bytes, addr, bucket_id as UnorderedTermId))
-            .collect();
+        let mut term_offsets: Vec<(&[u8], Addr, UnorderedTermId)> =
+            self.term_index.iter().collect();
         term_offsets.sort_unstable_by_key(|&(k, _, _)| k);
 
-        let mut offsets: Vec<(Field, usize)> = vec![];
-        let term_offsets_it = term_offsets
-            .iter()
-            .cloned()
-            .map(|(key, _, _)| Term::wrap(key).field())
-            .enumerate();
-
         let mut unordered_term_mappings: HashMap<Field, HashMap<UnorderedTermId, TermOrdinal>> =
             HashMap::new();
 
-        let mut prev_field = Field(u32::max_value());
-        for (offset, field) in term_offsets_it {
-            if field != prev_field {
-                offsets.push((field, offset));
-                prev_field = field;
-            }
-        }
-        offsets.push((Field(0), term_offsets.len()));
-
-        for i in 0..(offsets.len() - 1) {
-            let (field, start) = offsets[i];
-            let (_, stop) = offsets[i + 1];
+        let field_offsets = make_field_partition(&term_offsets);
 
+        for (field, start, stop) in field_offsets {
             let field_entry = self.schema.get_field_entry(field);
 
             match *field_entry.field_type() {
@@ -143,7 +149,7 @@ impl MultiFieldPostingsWriter {
                         .collect();
                     unordered_term_mappings.insert(field, mapping);
                 }
-                FieldType::U64(_) | FieldType::I64(_) => {}
+                FieldType::U64(_) | FieldType::I64(_) | FieldType::Date(_) => {}
                 FieldType::Bytes => {}
             }
 

src/postings/recorder.rs

@@ -4,7 +4,6 @@ use postings::FieldSerializer;
 use std::io;
 use DocId;
 
-const EMPTY_ARRAY: [u32; 0] = [0u32; 0];
 const POSITION_END: u32 = 0;
 
 #[derive(Default)]
@@ -115,7 +114,7 @@ impl Recorder for NothingRecorder {
         let buffer = buffer_lender.lend_u8();
         self.stack.read_to_end(heap, buffer);
         for doc in VInt32Reader::new(&buffer[..]) {
-            serializer.write_doc(doc as u32, 0u32, &EMPTY_ARRAY)?;
+            serializer.write_doc(doc as u32, 0u32, &[][..])?;
         }
         Ok(())
     }
@@ -168,7 +167,7 @@ impl Recorder for TermFrequencyRecorder {
         let mut u32_it = VInt32Reader::new(&buffer[..]);
         while let Some(doc) = u32_it.next() {
             let term_freq = u32_it.next().unwrap_or(self.current_tf);
-            serializer.write_doc(doc as u32, term_freq, &EMPTY_ARRAY)?;
+            serializer.write_doc(doc as u32, term_freq, &[][..])?;
         }
 
         Ok(())

src/postings/segment_postings.rs

@@ -2,7 +2,6 @@ use common::BitSet;
 use common::HasLen;
 use common::{BinarySerializable, VInt};
 use docset::{DocSet, SkipResult};
-use fst::Streamer;
 use owned_read::OwnedRead;
 use positions::PositionReader;
 use postings::compression::compressed_block_size;
@@ -14,10 +13,9 @@ use postings::SkipReader;
 use postings::USE_SKIP_INFO_LIMIT;
 use schema::IndexRecordOption;
 use std::cmp::Ordering;
+use tantivy_fst::Streamer;
 use DocId;
 
-const EMPTY_ARR: [u8; 0] = [];
-
 struct PositionComputer {
     // store the amount of position int
     // before reading positions.
@@ -123,12 +121,17 @@ impl SegmentPostings {
     }
 }
 
-fn exponential_search(target: u32, arr: &[u32]) -> (usize, usize) {
+fn linear_search(arr: &[u32], target: u32) -> usize {
+    arr.iter().map(|&el| if el < target { 1 } else { 0 }).sum()
+}
+
+fn exponential_search(arr: &[u32], target: u32) -> (usize, usize) {
     let end = arr.len();
-    debug_assert!(arr.len() <= 128);
-    debug_assert!(target <= arr[end - 1]);
     let mut begin = 0;
-    for &pivot in [1,3,7,15,31,63].iter().take_while(|&&el| el < end) {
+    for &pivot in &[1, 3, 7, 15, 31, 63] {
+        if pivot >= end {
+            break;
+        }
         if arr[pivot] > target {
             return (begin, pivot);
         }
@@ -145,12 +148,8 @@ fn exponential_search(target: u32, arr: &[u32]) -> (usize, usize) {
 /// The target is assumed greater or equal to the first element.
 /// The target is assumed smaller or equal to the last element.
 fn search_within_block(block_docs: &[u32], target: u32) -> usize {
-    let (start, end) = exponential_search(target, block_docs);
-    start.wrapping_add(
-        block_docs[start..end]
-            .binary_search(&target)
-            .unwrap_or_else(|e| e),
-    )
+    let (start, end) = exponential_search(block_docs, target);
+    start + linear_search(&block_docs[start..end], target)
 }
 
 impl DocSet for SegmentPostings {
@@ -368,7 +367,7 @@ impl BlockSegmentPostings {
         let (skip_data_opt, postings_data) = split_into_skips_and_postings(doc_freq, data);
         let skip_reader = match skip_data_opt {
             Some(skip_data) => SkipReader::new(skip_data, record_option),
-            None => SkipReader::new(OwnedRead::new(&EMPTY_ARR[..]), record_option),
+            None => SkipReader::new(OwnedRead::new(&[][..]), record_option),
         };
         let doc_freq = doc_freq as usize;
         let num_vint_docs = doc_freq % COMPRESSION_BLOCK_SIZE;
@@ -402,7 +401,7 @@ impl BlockSegmentPostings {
         if let Some(skip_data) = skip_data_opt {
             self.skip_reader.reset(skip_data);
         } else {
-            self.skip_reader.reset(OwnedRead::new(&EMPTY_ARR[..]))
+            self.skip_reader.reset(OwnedRead::new(&[][..]))
         }
         self.doc_offset = 0;
         self.doc_freq = doc_freq as usize;
@@ -617,6 +616,7 @@ impl<'b> Streamer<'b> for BlockSegmentPostings {
 mod tests {
 
     use super::exponential_search;
+    use super::linear_search;
     use super::search_within_block;
     use super::BlockSegmentPostings;
     use super::BlockSegmentPostingsSkipResult;
@@ -624,14 +624,29 @@ mod tests {
     use common::HasLen;
     use core::Index;
     use docset::DocSet;
-    use fst::Streamer;
     use schema::IndexRecordOption;
     use schema::Schema;
     use schema::Term;
-    use schema::INT_INDEXED;
+    use schema::INDEXED;
+    use tantivy_fst::Streamer;
     use DocId;
     use SkipResult;
 
+    #[test]
+    fn test_linear_search() {
+        let len: usize = 50;
+        let arr: Vec<u32> = (0..len).map(|el| 1u32 + (el as u32) * 2).collect();
+        for target in 1..*arr.last().unwrap() {
+            let res = linear_search(&arr[..], target);
+            if res > 0 {
+                assert!(arr[res - 1] < target);
+            }
+            if res < len {
+                assert!(arr[res] >= target);
+            }
+        }
+    }
+
     #[test]
     fn test_empty_segment_postings() {
         let mut postings = SegmentPostings::empty();
@@ -660,10 +675,10 @@ mod tests {
 
     #[test]
     fn test_exponentiel_search() {
-        assert_eq!(exponential_search(0, &[1, 2]), (0, 1));
-        assert_eq!(exponential_search(1, &[1, 2]), (0, 1));
+        assert_eq!(exponential_search(&[1, 2], 0), (0, 1));
+        assert_eq!(exponential_search(&[1, 2], 1), (0, 1));
         assert_eq!(
-            exponential_search(7, &[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]),
+            exponential_search(&[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], 7),
             (3, 7)
         );
     }
@@ -745,7 +760,7 @@ mod tests {
 
     fn build_block_postings(docs: &[DocId]) -> BlockSegmentPostings {
         let mut schema_builder = Schema::builder();
-        let int_field = schema_builder.add_u64_field("id", INT_INDEXED);
+        let int_field = schema_builder.add_u64_field("id", INDEXED);
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
         let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
@@ -758,8 +773,7 @@ mod tests {
             last_doc = doc + 1;
         }
         index_writer.commit().unwrap();
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        let searcher = index.reader().unwrap().searcher();
         let segment_reader = searcher.segment_reader(0);
         let inverted_index = segment_reader.inverted_index(int_field);
         let term = Term::from_field_u64(int_field, 0u64);
@@ -816,7 +830,7 @@ mod tests {
     #[test]
     fn test_reset_block_segment_postings() {
         let mut schema_builder = Schema::builder();
-        let int_field = schema_builder.add_u64_field("id", INT_INDEXED);
+        let int_field = schema_builder.add_u64_field("id", INDEXED);
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
         let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
@@ -827,8 +841,7 @@ mod tests {
             index_writer.add_document(doc);
         }
         index_writer.commit().unwrap();
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        let searcher = index.reader().unwrap().searcher();
         let segment_reader = searcher.segment_reader(0);
 
         let mut block_segments;

src/postings/stacker/term_hashmap.rs

@@ -5,12 +5,11 @@ use self::murmurhash32::murmurhash2;
 use super::{Addr, MemoryArena};
 use byteorder::{ByteOrder, NativeEndian};
 use postings::stacker::memory_arena::store;
+use postings::UnorderedTermId;
 use std::iter;
 use std::mem;
 use std::slice;
 
-pub type BucketId = usize;
-
 /// Returns the actual memory size in bytes
 /// required to create a table of size $2^num_bits$.
 pub fn compute_table_size(num_bits: usize) -> usize {
@@ -28,6 +27,7 @@ pub fn compute_table_size(num_bits: usize) -> usize {
 struct KeyValue {
     key_value_addr: Addr,
     hash: u32,
+    unordered_term_id: UnorderedTermId,
 }
 
 impl Default for KeyValue {
@@ -35,6 +35,7 @@ impl Default for KeyValue {
         KeyValue {
             key_value_addr: Addr::null_pointer(),
             hash: 0u32,
+            unordered_term_id: UnorderedTermId::default(),
         }
     }
 }
@@ -59,6 +60,7 @@ pub struct TermHashMap {
     pub heap: MemoryArena,
     mask: usize,
     occupied: Vec<usize>,
+    len: usize,
 }
 
 struct QuadraticProbing {
@@ -85,13 +87,13 @@ pub struct Iter<'a> {
 }
 
 impl<'a> Iterator for Iter<'a> {
-    type Item = (&'a [u8], Addr, BucketId);
+    type Item = (&'a [u8], Addr, UnorderedTermId);
 
     fn next(&mut self) -> Option<Self::Item> {
         self.inner.next().cloned().map(move |bucket: usize| {
             let kv = self.hashmap.table[bucket];
             let (key, offset): (&'a [u8], Addr) = self.hashmap.get_key_value(kv.key_value_addr);
-            (key, offset, bucket as BucketId)
+            (key, offset, kv.unordered_term_id)
         })
     }
 }
@@ -106,6 +108,7 @@ impl TermHashMap {
             heap,
             mask: table_size - 1,
             occupied: Vec::with_capacity(table_size / 2),
+            len: 0,
         }
     }
 
@@ -139,12 +142,16 @@ impl TermHashMap {
         }
     }
 
-    pub fn set_bucket(&mut self, hash: u32, key_value_addr: Addr, bucket: usize) {
+    fn set_bucket(&mut self, hash: u32, key_value_addr: Addr, bucket: usize) -> UnorderedTermId {
         self.occupied.push(bucket);
+        let unordered_term_id = self.len as UnorderedTermId;
+        self.len += 1;
         self.table[bucket] = KeyValue {
             key_value_addr,
             hash,
+            unordered_term_id,
         };
+        unordered_term_id
     }
 
     pub fn iter(&self) -> Iter {
@@ -184,7 +191,11 @@ impl TermHashMap {
     /// will be in charge of returning a default value.
     /// If the key already as an associated value, then it will be passed
     /// `Some(previous_value)`.
-    pub fn mutate_or_create<S, V, TMutator>(&mut self, key: S, mut updater: TMutator) -> BucketId
+    pub fn mutate_or_create<S, V, TMutator>(
+        &mut self,
+        key: S,
+        mut updater: TMutator,
+    ) -> UnorderedTermId
     where
         S: AsRef<[u8]>,
         V: Copy + 'static,
@@ -200,6 +211,7 @@ impl TermHashMap {
             let bucket = probe.next_probe();
             let kv: KeyValue = self.table[bucket];
             if kv.is_empty() {
+                // The key does not exists yet.
                 let val = updater(None);
                 let num_bytes =
                     std::mem::size_of::<u16>() + key_bytes.len() + std::mem::size_of::<V>();
@@ -211,8 +223,7 @@ impl TermHashMap {
                     data[2..stop].copy_from_slice(key_bytes);
                     store(&mut data[stop..], val);
                 }
-                self.set_bucket(hash, key_addr, bucket);
-                return bucket as BucketId;
+                return self.set_bucket(hash, key_addr, bucket);
             } else if kv.hash == hash {
                 if let Some(val_addr) =
                     self.get_value_addr_if_key_match(key_bytes, kv.key_value_addr)
@@ -220,7 +231,7 @@ impl TermHashMap {
                     let v = self.heap.read(val_addr);
                     let new_v = updater(Some(v));
                     self.heap.write_at(val_addr, new_v);
-                    return bucket as BucketId;
+                    return kv.unordered_term_id;
                 }
             }
         }

src/query/all_query.rs

@@ -101,8 +101,9 @@ mod tests {
         index_writer.commit().unwrap();
         index_writer.add_document(doc!(field=>"ccc"));
         index_writer.commit().unwrap();
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        let reader = index.reader().unwrap();
+        reader.reload().unwrap();
+        let searcher = reader.searcher();
         let weight = AllQuery.weight(&searcher, false).unwrap();
         {
             let reader = searcher.segment_reader(0);

src/query/automaton_weight.rs

@@ -1,10 +1,10 @@
 use common::BitSet;
 use core::SegmentReader;
-use fst::Automaton;
 use query::BitSetDocSet;
 use query::ConstScorer;
 use query::{Scorer, Weight};
 use schema::{Field, IndexRecordOption};
+use tantivy_fst::Automaton;
 use termdict::{TermDictionary, TermStreamer};
 use Result;
 

src/query/boolean_query/boolean_weight.rs

@@ -1,5 +1,4 @@
 use core::SegmentReader;
-use downcast::Downcast;
 use query::intersect_scorers;
 use query::score_combiner::{DoNothingCombiner, ScoreCombiner, SumWithCoordsCombiner};
 use query::term_query::TermScorer;
@@ -10,7 +9,6 @@ use query::RequiredOptionalScorer;
 use query::Scorer;
 use query::Union;
 use query::Weight;
-use std::borrow::Borrow;
 use std::collections::HashMap;
 use Result;
 
@@ -24,14 +22,11 @@ where
     }
 
     {
-        let is_all_term_queries = scorers.iter().all(|scorer| {
-            let scorer_ref: &Scorer = scorer.borrow();
-            Downcast::<TermScorer>::is_type(scorer_ref)
-        });
+        let is_all_term_queries = scorers.iter().all(|scorer| scorer.is::<TermScorer>());
         if is_all_term_queries {
             let scorers: Vec<TermScorer> = scorers
                 .into_iter()
-                .map(|scorer| *Downcast::<TermScorer>::downcast(scorer).unwrap())
+                .map(|scorer| *(scorer.downcast::<TermScorer>().map_err(|_| ()).unwrap()))
                 .collect();
             let scorer: Box<Scorer> = Box::new(Union::<TermScorer, TScoreCombiner>::from(scorers));
             return scorer;

src/query/boolean_query/mod.rs

@@ -8,7 +8,6 @@ mod tests {
 
     use super::*;
     use collector::tests::TestCollector;
-    use downcast::Downcast;
     use query::score_combiner::SumWithCoordsCombiner;
     use query::term_query::TermScorer;
     use query::Intersection;
@@ -52,7 +51,6 @@ mod tests {
             }
             assert!(index_writer.commit().is_ok());
         }
-        index.load_searchers().unwrap();
         (index, text_field)
     }
 
@@ -61,7 +59,8 @@ mod tests {
         let (index, text_field) = aux_test_helper();
         let query_parser = QueryParser::for_index(&index, vec![text_field]);
         let query = query_parser.parse_query("(+a +b) d").unwrap();
-        assert_eq!(query.count(&*index.searcher()).unwrap(), 3);
+        let searcher = index.reader().unwrap().searcher();
+        assert_eq!(query.count(&searcher).unwrap(), 3);
     }
 
     #[test]
@@ -69,28 +68,28 @@ mod tests {
         let (index, text_field) = aux_test_helper();
         let query_parser = QueryParser::for_index(&index, vec![text_field]);
         let query = query_parser.parse_query("+a").unwrap();
-        let searcher = index.searcher();
+        let searcher = index.reader().unwrap().searcher();
         let weight = query.weight(&searcher, true).unwrap();
         let scorer = weight.scorer(searcher.segment_reader(0u32)).unwrap();
-        assert!(Downcast::<TermScorer>::is_type(&*scorer));
+        assert!(scorer.is::<TermScorer>());
     }
 
     #[test]
     pub fn test_boolean_termonly_intersection() {
         let (index, text_field) = aux_test_helper();
         let query_parser = QueryParser::for_index(&index, vec![text_field]);
-        let searcher = index.searcher();
+        let searcher = index.reader().unwrap().searcher();
         {
             let query = query_parser.parse_query("+a +b +c").unwrap();
             let weight = query.weight(&searcher, true).unwrap();
             let scorer = weight.scorer(searcher.segment_reader(0u32)).unwrap();
-            assert!(Downcast::<Intersection<TermScorer>>::is_type(&*scorer));
+            assert!(scorer.is::<Intersection<TermScorer>>());
         }
         {
             let query = query_parser.parse_query("+a +(b c)").unwrap();
             let weight = query.weight(&searcher, true).unwrap();
             let scorer = weight.scorer(searcher.segment_reader(0u32)).unwrap();
-            assert!(Downcast::<Intersection<Box<Scorer>>>::is_type(&*scorer));
+            assert!(scorer.is::<Intersection<Box<Scorer>>>());
         }
     }
 
@@ -98,21 +97,19 @@ mod tests {
     pub fn test_boolean_reqopt() {
         let (index, text_field) = aux_test_helper();
         let query_parser = QueryParser::for_index(&index, vec![text_field]);
-        let searcher = index.searcher();
+        let searcher = index.reader().unwrap().searcher();
         {
             let query = query_parser.parse_query("+a b").unwrap();
             let weight = query.weight(&searcher, true).unwrap();
             let scorer = weight.scorer(searcher.segment_reader(0u32)).unwrap();
-            assert!(Downcast::<
-                RequiredOptionalScorer<Box<Scorer>, Box<Scorer>, SumWithCoordsCombiner>,
-            >::is_type(&*scorer));
+            assert!(scorer
+                .is::<RequiredOptionalScorer<Box<Scorer>, Box<Scorer>, SumWithCoordsCombiner>>());
         }
         {
             let query = query_parser.parse_query("+a b").unwrap();
             let weight = query.weight(&searcher, false).unwrap();
             let scorer = weight.scorer(searcher.segment_reader(0u32)).unwrap();
-            println!("{:?}", scorer.type_name());
-            assert!(Downcast::<TermScorer>::is_type(&*scorer));
+            assert!(scorer.is::<TermScorer>());
         }
     }
 
@@ -129,10 +126,13 @@ mod tests {
             query
         };
 
+        let reader = index.reader().unwrap();
+
         let matching_docs = |boolean_query: &Query| {
-            let searcher = index.searcher();
-            let test_docs = searcher.search(boolean_query, &TestCollector).unwrap();
-            test_docs
+            reader
+                .searcher()
+                .search(boolean_query, &TestCollector)
+                .unwrap()
                 .docs()
                 .iter()
                 .cloned()
@@ -188,10 +188,12 @@ mod tests {
             let query: Box<Query> = Box::new(term_query);
             query
         };
-
+        let reader = index.reader().unwrap();
         let score_docs = |boolean_query: &Query| {
-            let searcher = index.searcher();
-            let fruit = searcher.search(boolean_query, &TestCollector).unwrap();
+            let fruit = reader
+                .searcher()
+                .search(boolean_query, &TestCollector)
+                .unwrap();
             fruit.scores().to_vec()
         };
 

src/query/fuzzy_query.rs

@@ -52,9 +52,8 @@ lazy_static! {
 ///         ));
 ///         index_writer.commit().unwrap();
 ///     }
-///
-///     index.load_searchers()?;
-///     let searcher = index.searcher();
+///     let reader = index.reader()?;
+///     let searcher = reader.searcher();
 ///
 ///     {
 ///
@@ -141,8 +140,8 @@ mod test {
             ));
             index_writer.commit().unwrap();
         }
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        let reader = index.reader().unwrap();
+        let searcher = reader.searcher();
         {
             let term = Term::from_field_text(country_field, "japon");
 

src/query/intersection.rs

@@ -1,9 +1,7 @@
 use docset::{DocSet, SkipResult};
-use downcast::Downcast;
 use query::term_query::TermScorer;
 use query::EmptyScorer;
 use query::Scorer;
-use std::borrow::Borrow;
 use DocId;
 use Score;
 
@@ -26,13 +24,12 @@ pub fn intersect_scorers(mut scorers: Vec<Box<Scorer>>) -> Box<Scorer> {
         (Some(single_docset), None) => single_docset,
         (Some(left), Some(right)) => {
             {
-                let all_term_scorers = [&left, &right].iter().all(|&scorer| {
-                    let scorer_ref: &Scorer = <Box<Scorer> as Borrow<Scorer>>::borrow(scorer);
-                    Downcast::<TermScorer>::is_type(scorer_ref)
-                });
+                let all_term_scorers = [&left, &right]
+                    .iter()
+                    .all(|&scorer| scorer.is::<TermScorer>());
                 if all_term_scorers {
-                    let left = *Downcast::<TermScorer>::downcast(left).unwrap();
-                    let right = *Downcast::<TermScorer>::downcast(right).unwrap();
+                    let left = *(left.downcast::<TermScorer>().map_err(|_| ()).unwrap());
+                    let right = *(right.downcast::<TermScorer>().map_err(|_| ()).unwrap());
                     return Box::new(Intersection {
                         left,
                         right,

src/query/phrase_query/mod.rs

@@ -31,7 +31,6 @@ mod tests {
             }
             assert!(index_writer.commit().is_ok());
         }
-        index.load_searchers().unwrap();
         index
     }
 
@@ -46,8 +45,7 @@ mod tests {
         ]);
         let schema = index.schema();
         let text_field = schema.get_field("text").unwrap();
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        let searcher = index.reader().unwrap().searcher();
         let test_query = |texts: Vec<&str>| {
             let terms: Vec<Term> = texts
                 .iter()
@@ -90,8 +88,7 @@ mod tests {
             index_writer.add_document(doc!(text_field=>"a b c"));
             assert!(index_writer.commit().is_ok());
         }
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        let searcher = index.reader().unwrap().searcher();
         let phrase_query = PhraseQuery::new(vec![
             Term::from_field_text(text_field, "a"),
             Term::from_field_text(text_field, "b"),
@@ -115,8 +112,7 @@ mod tests {
         let index = create_index(&["a b c", "a b c a b"]);
         let schema = index.schema();
         let text_field = schema.get_field("text").unwrap();
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        let searcher = index.reader().unwrap().searcher();
         let test_query = |texts: Vec<&str>| {
             let terms: Vec<Term> = texts
                 .iter()
@@ -148,8 +144,7 @@ mod tests {
             assert!(index_writer.commit().is_ok());
         }
 
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        let searcher = index.reader().unwrap().searcher();
         let test_query = |texts: Vec<&str>| {
             let terms: Vec<Term> = texts
                 .iter()
@@ -177,8 +172,7 @@ mod tests {
             index_writer.add_document(doc!(text_field=>"a b c d e f g h"));
             assert!(index_writer.commit().is_ok());
         }
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        let searcher = index.reader().unwrap().searcher();
         let test_query = |texts: Vec<(usize, &str)>| {
             let terms: Vec<(usize, Term)> = texts
                 .iter()

src/query/phrase_query/phrase_scorer.rs

@@ -43,7 +43,7 @@ impl<TPostings: Postings> DocSet for PostingsWithOffset<TPostings> {
 
 pub struct PhraseScorer<TPostings: Postings> {
     intersection_docset: Intersection<PostingsWithOffset<TPostings>, PostingsWithOffset<TPostings>>,
-    num_docsets: usize,
+    num_terms: usize,
     left: Vec<u32>,
     right: Vec<u32>,
     phrase_count: u32,
@@ -138,7 +138,7 @@ impl<TPostings: Postings> PhraseScorer<TPostings> {
             .collect::<Vec<_>>();
         PhraseScorer {
             intersection_docset: Intersection::new(postings_with_offsets),
-            num_docsets,
+            num_terms: num_docsets,
             left: Vec::with_capacity(100),
             right: Vec::with_capacity(100),
             phrase_count: 0u32,
@@ -165,7 +165,7 @@ impl<TPostings: Postings> PhraseScorer<TPostings> {
                 .positions(&mut self.left);
         }
         let mut intersection_len = self.left.len();
-        for i in 1..self.num_docsets - 1 {
+        for i in 1..self.num_terms - 1 {
             {
                 self.intersection_docset
                     .docset_mut_specialized(i)
@@ -178,7 +178,7 @@ impl<TPostings: Postings> PhraseScorer<TPostings> {
         }
 
         self.intersection_docset
-            .docset_mut_specialized(self.num_docsets - 1)
+            .docset_mut_specialized(self.num_terms - 1)
             .positions(&mut self.right);
         intersection_exists(&self.left[..intersection_len], &self.right[..])
     }
@@ -190,7 +190,7 @@ impl<TPostings: Postings> PhraseScorer<TPostings> {
                 .positions(&mut self.left);
         }
         let mut intersection_len = self.left.len();
-        for i in 1..self.num_docsets - 1 {
+        for i in 1..self.num_terms - 1 {
             {
                 self.intersection_docset
                     .docset_mut_specialized(i)
@@ -203,7 +203,7 @@ impl<TPostings: Postings> PhraseScorer<TPostings> {
         }
 
         self.intersection_docset
-            .docset_mut_specialized(self.num_docsets - 1)
+            .docset_mut_specialized(self.num_terms - 1)
             .positions(&mut self.right);
         intersection_count(&self.left[..intersection_len], &self.right[..]) as u32
     }

src/query/query.rs

@@ -1,6 +1,6 @@
 use super::Weight;
 use core::searcher::Searcher;
-use downcast;
+use downcast_rs;
 use std::collections::BTreeSet;
 use std::fmt;
 use Result;
@@ -39,7 +39,7 @@ use Term;
 ///
 /// When implementing a new type of `Query`, it is normal to implement a
 /// dedicated `Query`, `Weight` and `Scorer`.
-pub trait Query: QueryClone + downcast::Any + fmt::Debug {
+pub trait Query: QueryClone + downcast_rs::Downcast + fmt::Debug {
     /// Create the weight associated to a query.
     ///
     /// If scoring is not required, setting `scoring_enabled` to `false`
@@ -96,7 +96,4 @@ impl QueryClone for Box<Query> {
     }
 }
 
-#[allow(missing_docs)]
-mod downcast_impl {
-    downcast!(super::Query);
-}
+impl_downcast!(Query);

src/query/query_parser/query_grammar.rs

@@ -52,7 +52,7 @@ parser! {
             field_name: None,
             phrase,
         });
-        try(term_query)
+        attempt(term_query)
             .or(term_default_field)
             .map(UserInputLeaf::from)
     }
@@ -83,12 +83,12 @@ parser! {
         let lower_bound = {
             let excl = (char('{'), term_val()).map(|(_, w)| UserInputBound::Exclusive(w));
             let incl = (char('['), term_val()).map(|(_, w)| UserInputBound::Inclusive(w));
-            try(excl).or(incl)
+            attempt(excl).or(incl)
         };
         let upper_bound = {
             let excl = (term_val(), char('}')).map(|(w, _)| UserInputBound::Exclusive(w));
             let incl = (term_val(), char(']')).map(|(w, _)| UserInputBound::Inclusive(w));
-            try(excl).or(incl)
+            attempt(excl).or(incl)
         };
         (
             optional((field(), char(':')).map(|x| x.0)),
@@ -112,11 +112,11 @@ parser! {
         .or((char('+'), leaf()).map(|(_, expr)| expr.unary(Occur::Must) ))
         .or((char('('), parse_to_ast(), char(')')).map(|(_, expr, _)| expr))
         .or(char('*').map(|_| UserInputAST::from(UserInputLeaf::All) ))
-        .or(try(
+        .or(attempt(
             (string("NOT"), spaces1(), leaf()).map(|(_, _, expr)| expr.unary(Occur::MustNot))
             )
          )
-        .or(try(
+        .or(attempt(
             range().map(UserInputAST::from)
             )
         )
@@ -160,7 +160,7 @@ parser! {
     where [I: Stream<Item = char>]
     {
         (
-            try(
+            attempt(
                 chainl1(
                     leaf().map(Element::SingleEl),
                     binary_operand().map(|op: BinaryOperand|

src/query/query_parser/query_parser.rs

@@ -50,6 +50,8 @@ pub enum QueryParserError {
     /// The query contains a range query with a phrase as one of the bounds.
     /// Only terms can be used as bounds.
     RangeMustNotHavePhrase,
+    /// The format for the date field is not RFC 3339 compliant.
+    DateFormatError(chrono::ParseError),
 }
 
 impl From<ParseIntError> for QueryParserError {
@@ -58,6 +60,12 @@ impl From<ParseIntError> for QueryParserError {
     }
 }
 
+impl From<chrono::ParseError> for QueryParserError {
+    fn from(err: chrono::ParseError) -> QueryParserError {
+        QueryParserError::DateFormatError(err)
+    }
+}
+
 /// Recursively remove empty clause from the AST
 ///
 /// Returns `None` iff the `logical_ast` ended up being empty.
@@ -127,6 +135,8 @@ fn trim_ast(logical_ast: LogicalAST) -> Option<LogicalAST> {
 ///   a word lexicographically between `a` and `c` (inclusive lower bound, exclusive upper bound).
 ///   Inclusive bounds are `[]`, exclusive are `{}`.
 ///
+/// * date values: The query parser supports rfc3339 formatted dates. For example "2002-10-02T15:00:00.05Z"
+///
 /// *  all docs query: A plain `*` will match all documents in the index.
 ///
 #[derive(Clone)]
@@ -229,6 +239,13 @@ impl QueryParser {
                 let term = Term::from_field_i64(field, val);
                 Ok(vec![(0, term)])
             }
+            FieldType::Date(_) => match chrono::DateTime::parse_from_rfc3339(phrase) {
+                Ok(x) => Ok(vec![(
+                    0,
+                    Term::from_field_date(field, &x.with_timezone(&chrono::Utc)),
+                )]),
+                Err(e) => Err(QueryParserError::DateFormatError(e)),
+            },
             FieldType::U64(_) => {
                 let val: u64 = u64::from_str(phrase)?;
                 let term = Term::from_field_u64(field, val);
@@ -487,7 +504,7 @@ mod test {
     use query::Query;
     use schema::Field;
     use schema::{IndexRecordOption, TextFieldIndexing, TextOptions};
-    use schema::{Schema, Term, INT_INDEXED, STORED, STRING, TEXT};
+    use schema::{Schema, Term, INDEXED, STORED, STRING, TEXT};
     use tokenizer::{LowerCaser, SimpleTokenizer, StopWordFilter, Tokenizer, TokenizerManager};
     use Index;
 
@@ -501,13 +518,14 @@ mod test {
             .set_stored();
         let title = schema_builder.add_text_field("title", TEXT);
         let text = schema_builder.add_text_field("text", TEXT);
-        schema_builder.add_i64_field("signed", INT_INDEXED);
-        schema_builder.add_u64_field("unsigned", INT_INDEXED);
+        schema_builder.add_i64_field("signed", INDEXED);
+        schema_builder.add_u64_field("unsigned", INDEXED);
         schema_builder.add_text_field("notindexed_text", STORED);
         schema_builder.add_text_field("notindexed_u64", STORED);
         schema_builder.add_text_field("notindexed_i64", STORED);
         schema_builder.add_text_field("nottokenized", STRING);
         schema_builder.add_text_field("with_stop_words", text_options);
+        schema_builder.add_date_field("date", INDEXED);
         let schema = schema_builder.build();
         let default_fields = vec![title, text];
         let tokenizer_manager = TokenizerManager::default();
@@ -767,6 +785,18 @@ mod test {
         );
     }
 
+    #[test]
+    pub fn test_query_parser_expected_date() {
+        let query_parser = make_query_parser();
+        assert_matches!(
+            query_parser.parse_query("date:18a"),
+            Err(QueryParserError::DateFormatError(_))
+        );
+        assert!(query_parser
+            .parse_query("date:\"1985-04-12T23:20:50.52Z\"")
+            .is_ok());
+    }
+
     #[test]
     pub fn test_query_parser_not_empty_but_no_tokens() {
         let query_parser = make_query_parser();

src/query/range_query.rs

@@ -40,14 +40,14 @@ fn map_bound<TFrom, TTo, Transform: Fn(&TFrom) -> TTo>(
 /// # #[macro_use]
 /// # extern crate tantivy;
 /// # use tantivy::Index;
-/// # use tantivy::schema::{Schema, INT_INDEXED};
+/// # use tantivy::schema::{Schema, INDEXED};
 /// # use tantivy::collector::Count;
 /// # use tantivy::Result;
 /// # use tantivy::query::RangeQuery;
 /// #
 /// # fn run() -> Result<()> {
 /// #     let mut schema_builder = Schema::builder();
-/// #     let year_field = schema_builder.add_u64_field("year", INT_INDEXED);
+/// #     let year_field = schema_builder.add_u64_field("year", INDEXED);
 /// #     let schema = schema_builder.build();
 /// #
 /// #     let index = Index::create_in_ram(schema);
@@ -61,8 +61,8 @@ fn map_bound<TFrom, TTo, Transform: Fn(&TFrom) -> TTo>(
 /// #         }
 /// #         index_writer.commit().unwrap();
 /// #     }
-/// #   index.load_searchers()?;
-/// let searcher = index.searcher();
+/// #   let reader = index.reader()?;
+/// let searcher = reader.searcher();
 ///
 /// let docs_in_the_sixties = RangeQuery::new_u64(year_field, 1960..1970);
 ///
@@ -293,7 +293,7 @@ mod tests {
 
     use super::RangeQuery;
     use collector::Count;
-    use schema::{Document, Field, Schema, INT_INDEXED};
+    use schema::{Document, Field, Schema, INDEXED};
     use std::collections::Bound;
     use Index;
     use Result;
@@ -302,7 +302,7 @@ mod tests {
     fn test_range_query_simple() {
         fn run() -> Result<()> {
             let mut schema_builder = Schema::builder();
-            let year_field = schema_builder.add_u64_field("year", INT_INDEXED);
+            let year_field = schema_builder.add_u64_field("year", INDEXED);
             let schema = schema_builder.build();
 
             let index = Index::create_in_ram(schema);
@@ -316,8 +316,8 @@ mod tests {
                 }
                 index_writer.commit().unwrap();
             }
-            index.load_searchers().unwrap();
-            let searcher = index.searcher();
+            let reader = index.reader().unwrap();
+            let searcher = reader.searcher();
 
             let docs_in_the_sixties = RangeQuery::new_u64(year_field, 1960u64..1970u64);
 
@@ -335,7 +335,7 @@ mod tests {
         let int_field: Field;
         let schema = {
             let mut schema_builder = Schema::builder();
-            int_field = schema_builder.add_i64_field("intfield", INT_INDEXED);
+            int_field = schema_builder.add_i64_field("intfield", INDEXED);
             schema_builder.build()
         };
 
@@ -355,8 +355,8 @@ mod tests {
 
             index_writer.commit().unwrap();
         }
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        let reader = index.reader().unwrap();
+        let searcher = reader.searcher();
         let count_multiples =
             |range_query: RangeQuery| searcher.search(&range_query, &Count).unwrap();
 

src/query/regex_query.rs

@@ -1,8 +1,8 @@
 use error::TantivyError;
-use fst_regex::Regex;
 use query::{AutomatonWeight, Query, Weight};
 use schema::Field;
 use std::clone::Clone;
+use tantivy_fst::Regex;
 use Result;
 use Searcher;
 
@@ -44,8 +44,8 @@ use Searcher;
 ///         index_writer.commit().unwrap();
 ///     }
 ///
-///     index.load_searchers()?;
-///     let searcher = index.searcher();
+///     let reader = index.reader()?;
+///     let searcher = reader.searcher();
 ///
 ///     let term = Term::from_field_text(title, "Diary");
 ///     let query = RegexQuery::new("d[ai]{2}ry".to_string(), title);
@@ -108,8 +108,8 @@ mod test {
             ));
             index_writer.commit().unwrap();
         }
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        let reader = index.reader().unwrap();
+        let searcher = reader.searcher();
         {
             let regex_query = RegexQuery::new("jap[ao]n".to_string(), country_field);
             let scored_docs = searcher

src/query/scorer.rs

@@ -1,6 +1,6 @@
 use common::BitSet;
 use docset::{DocSet, SkipResult};
-use downcast;
+use downcast_rs;
 use std::ops::DerefMut;
 use DocId;
 use Score;
@@ -8,7 +8,7 @@ use Score;
 /// Scored set of documents matching a query within a specific segment.
 ///
 /// See [`Query`](./trait.Query.html).
-pub trait Scorer: downcast::Any + DocSet + 'static {
+pub trait Scorer: downcast_rs::Downcast + DocSet + 'static {
     /// Returns the score.
     ///
     /// This method will perform a bit of computation and is not cached.
@@ -23,10 +23,7 @@ pub trait Scorer: downcast::Any + DocSet + 'static {
     }
 }
 
-#[allow(missing_docs)]
-mod downcast_impl {
-    downcast!(super::Scorer);
-}
+impl_downcast!(Scorer);
 
 impl Scorer for Box<Scorer> {
     fn score(&mut self) -> Score {

src/query/term_query/mod.rs

@@ -32,9 +32,7 @@ mod tests {
             }
             assert!(index_writer.commit().is_ok());
         }
-
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        let searcher = index.reader().unwrap().searcher();
         let term_query = TermQuery::new(
             Term::from_field_text(text_field, "a"),
             IndexRecordOption::Basic,
@@ -65,8 +63,7 @@ mod tests {
             index_writer.add_document(doc!(left_field => "left4 left1"));
             index_writer.commit().unwrap();
         }
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        let searcher = index.reader().unwrap().searcher();
         {
             let term = Term::from_field_text(left_field, "left2");
             let term_query = TermQuery::new(term, IndexRecordOption::WithFreqs);

src/query/term_query/term_query.rs

@@ -48,9 +48,8 @@ use Term;
 ///         ));
 ///         index_writer.commit()?;
 ///     }
-///
-///     index.load_searchers()?;
-///     let searcher = index.searcher();
+///     let reader = index.reader()?;
+///     let searcher = reader.searcher();
 ///
 ///     let query = TermQuery::new(
 ///         Term::from_field_text(title, "diary"),

src/query/vec_docset.rs

@@ -5,8 +5,6 @@ use docset::DocSet;
 use std::num::Wrapping;
 use DocId;
 
-const EMPTY_ARRAY: [u32; 0] = [];
-
 /// Simulate a `Postings` objects from a `VecPostings`.
 /// `VecPostings` only exist for testing purposes.
 ///

src/reader/mod.rs

@@ -0,0 +1,187 @@
+mod pool;
+
+use self::pool::{LeasedItem, Pool};
+use core::Segment;
+use directory::Directory;
+use directory::WatchHandle;
+use directory::META_LOCK;
+use std::sync::Arc;
+use Index;
+use Result;
+use Searcher;
+use SegmentReader;
+
+/// Defines when a new version of the index should be reloaded.
+///
+/// Regardless of whether you search and index in the same process, tantivy does not necessarily
+/// reflects the change that are commited to your index. `ReloadPolicy` precisely helps you define
+/// when you want your index to be reloaded.
+#[derive(Clone, Copy)]
+pub enum ReloadPolicy {
+    /// The index is entirely reloaded manually.
+    /// All updates of the index should be manual.
+    ///
+    /// No change is reflected automatically. You are required to call `.load_seacher()` manually.
+    Manual,
+    /// The index is reloaded within milliseconds after a new commit is available.
+    /// This is made possible by watching changes in the `meta.json` file.
+    OnCommit, // TODO add NEAR_REAL_TIME(target_ms)
+}
+
+/// `IndexReader` builder
+///
+/// It makes it possible to set the following values.
+///
+/// - `num_searchers` (by default, the number of detected CPU threads):
+///
+///   When `num_searchers` queries are requested at the same time, the `num_searchers` will block
+///   until the one of the searcher in-use gets released.
+/// - `reload_policy` (by default `ReloadPolicy::OnCommit`):
+///
+///   See [`ReloadPolicy`](./enum.ReloadPolicy.html) for more details.
+#[derive(Clone)]
+pub struct IndexReaderBuilder {
+    num_searchers: usize,
+    reload_policy: ReloadPolicy,
+    index: Index,
+}
+
+impl IndexReaderBuilder {
+    pub(crate) fn new(index: Index) -> IndexReaderBuilder {
+        IndexReaderBuilder {
+            num_searchers: num_cpus::get(),
+            reload_policy: ReloadPolicy::OnCommit,
+            index,
+        }
+    }
+
+    /// Builds the reader.
+    ///
+    /// Building the reader is a non-trivial operation that requires
+    /// to open different segment readers. It may take hundreds of milliseconds
+    /// of time and it may return an error.
+    /// TODO(pmasurel) Use the `TryInto` trait once it is available in stable.
+    pub fn try_into(self) -> Result<IndexReader> {
+        let inner_reader = InnerIndexReader {
+            index: self.index,
+            num_searchers: self.num_searchers,
+            searcher_pool: Pool::new(),
+        };
+        inner_reader.reload()?;
+        let inner_reader_arc = Arc::new(inner_reader);
+        let watch_handle_opt: Option<WatchHandle>;
+        match self.reload_policy {
+            ReloadPolicy::Manual => {
+                // No need to set anything...
+                watch_handle_opt = None;
+            }
+            ReloadPolicy::OnCommit => {
+                let inner_reader_arc_clone = inner_reader_arc.clone();
+                let callback = move || {
+                    if let Err(err) = inner_reader_arc_clone.reload() {
+                        error!(
+                            "Error while loading searcher after commit was detected. {:?}",
+                            err
+                        );
+                    }
+                };
+                let watch_handle = inner_reader_arc.index.directory().watch(Box::new(callback));
+                watch_handle_opt = Some(watch_handle);
+            }
+        }
+        Ok(IndexReader {
+            inner: inner_reader_arc,
+            watch_handle_opt,
+        })
+    }
+
+    /// Sets the reload_policy.
+    ///
+    /// See [`ReloadPolicy`](./enum.ReloadPolicy.html) for more details.
+    pub fn reload_policy(mut self, reload_policy: ReloadPolicy) -> IndexReaderBuilder {
+        self.reload_policy = reload_policy;
+        self
+    }
+
+    /// Sets the number of `Searcher` in the searcher pool.
+    pub fn num_searchers(mut self, num_searchers: usize) -> IndexReaderBuilder {
+        self.num_searchers = num_searchers;
+        self
+    }
+}
+
+struct InnerIndexReader {
+    num_searchers: usize,
+    searcher_pool: Pool<Searcher>,
+    index: Index,
+}
+
+impl InnerIndexReader {
+    fn reload(&self) -> Result<()> {
+        let segment_readers: Vec<SegmentReader> = {
+            let _meta_lock = self.index.directory().acquire_lock(&META_LOCK)?;
+            let searchable_segments = self.searchable_segments()?;
+            searchable_segments
+                .iter()
+                .map(SegmentReader::open)
+                .collect::<Result<_>>()?
+        };
+        let schema = self.index.schema();
+        let searchers = (0..self.num_searchers)
+            .map(|_| Searcher::new(schema.clone(), self.index.clone(), segment_readers.clone()))
+            .collect();
+        self.searcher_pool.publish_new_generation(searchers);
+        Ok(())
+    }
+
+    /// Returns the list of segments that are searchable
+    fn searchable_segments(&self) -> Result<Vec<Segment>> {
+        self.index.searchable_segments()
+    }
+
+    fn searcher(&self) -> LeasedItem<Searcher> {
+        self.searcher_pool.acquire()
+    }
+}
+
+/// `IndexReader` is your entry point to read and search the index.
+///
+/// It controls when a new version of the index should be loaded and lends
+/// you instances of `Searcher` for the last loaded version.
+///
+/// `Clone` does not clone the different pool of searcher. `IndexReader`
+/// just wraps and `Arc`.
+#[derive(Clone)]
+pub struct IndexReader {
+    inner: Arc<InnerIndexReader>,
+    watch_handle_opt: Option<WatchHandle>,
+}
+
+impl IndexReader {
+    /// Update searchers so that they reflect the state of the last
+    /// `.commit()`.
+    ///
+    /// If you set up the `OnCommit` `ReloadPolicy` (which is the default)
+    /// every commit should be rapidly reflected on your `IndexReader` and you should
+    /// not need to call `reload()` at all.
+    ///
+    /// This automatic reload can take 10s of milliseconds to kick in however, and in unit tests
+    /// it can be nice to deterministically force the reload of searchers.
+    pub fn reload(&self) -> Result<()> {
+        self.inner.reload()
+    }
+
+    /// Returns a searcher
+    ///
+    /// This method should be called every single time a search
+    /// query is performed.
+    /// The searchers are taken from a pool of `num_searchers` searchers.
+    /// If no searcher is available
+    /// this may block.
+    ///
+    /// The same searcher must be used for a given query, as it ensures
+    /// the use of a consistent segment set.
+    pub fn searcher(&self) -> LeasedItem<Searcher> {
+        self.inner.searcher()
+    }
+}

src/reader/pool.rs

@@ -1,5 +1,5 @@
-use crossbeam::queue::MsQueue;
-use std::mem;
+use crossbeam::crossbeam_channel::unbounded;
+use crossbeam::{Receiver, RecvError, Sender};
 use std::ops::{Deref, DerefMut};
 use std::sync::atomic::AtomicUsize;
 use std::sync::atomic::Ordering;
@@ -10,15 +10,52 @@ pub struct GenerationItem<T> {
     item: T,
 }
 
+/// Queue implementation for the Object Pool below
+/// Uses the unbounded Linked-List type queue from crossbeam-channel
+/// Splits the Queue into sender and receiver
+struct Queue<T> {
+    sender: Sender<T>,
+    receiver: Receiver<T>,
+}
+
+impl<T> Queue<T> {
+    fn new() -> Self {
+        let (s, r) = unbounded();
+        Queue {
+            sender: s,
+            receiver: r,
+        }
+    }
+
+    /// Sender trait returns a Result type, which is ignored.
+    /// The Result is not handled at the moment
+    fn push(&self, elem: T) {
+        self.sender
+            .send(elem)
+            .expect("Sending an item to crossbeam-queue shouldn't fail");
+    }
+
+    /// Relies on the underlying crossbeam-channel Receiver
+    /// to block on empty queue
+    fn pop(&self) -> Result<T, RecvError> {
+        self.receiver.recv()
+    }
+}
+
+/// An object pool
+///
+/// This is used in tantivy to create a pool of `Searcher`.
+/// Object are wrapped in a `LeasedItem` wrapper and are
+/// released automatically back into the pool on `Drop`.
 pub struct Pool<T> {
-    queue: Arc<MsQueue<GenerationItem<T>>>,
+    queue: Arc<Queue<GenerationItem<T>>>,
     freshest_generation: AtomicUsize,
     next_generation: AtomicUsize,
 }
 
 impl<T> Pool<T> {
     pub fn new() -> Pool<T> {
-        let queue = Arc::new(MsQueue::new());
+        let queue = Arc::new(Queue::new());
         Pool {
             queue,
             freshest_generation: AtomicUsize::default(),
@@ -26,6 +63,10 @@ impl<T> Pool<T> {
         }
     }
 
+    /// Publishes a new generation of `Searcher`.
+    ///
+    /// After publish, all new `Searcher` acquired will be
+    /// of the new generation.
     pub fn publish_new_generation(&self, items: Vec<T>) {
         let next_generation = self.next_generation.fetch_add(1, Ordering::SeqCst) + 1;
         for item in items {
@@ -61,10 +102,14 @@ impl<T> Pool<T> {
         self.freshest_generation.load(Ordering::Acquire)
     }
 
+    /// Acquires a new searcher.
+    ///
+    /// If no searcher is available, this methods block until
+    /// a searcher is released.
     pub fn acquire(&self) -> LeasedItem<T> {
         let generation = self.generation();
         loop {
-            let gen_item = self.queue.pop();
+            let gen_item = self.queue.pop().unwrap();
             if gen_item.generation >= generation {
                 return LeasedItem {
                     gen_item: Some(gen_item),
@@ -80,7 +125,7 @@ impl<T> Pool<T> {
 
 pub struct LeasedItem<T> {
     gen_item: Option<GenerationItem<T>>,
-    recycle_queue: Arc<MsQueue<GenerationItem<T>>>,
+    recycle_queue: Arc<Queue<GenerationItem<T>>>,
 }
 
 impl<T> Deref for LeasedItem<T> {
@@ -107,9 +152,9 @@ impl<T> DerefMut for LeasedItem<T> {
 
 impl<T> Drop for LeasedItem<T> {
     fn drop(&mut self) {
-        let gen_item: GenerationItem<T> = mem::replace(&mut self.gen_item, None)
-            .expect("Unwrapping a leased item should never fail");
-        self.recycle_queue.push(gen_item);
+        if let Some(gen_item) = self.gen_item.take() {
+            self.recycle_queue.push(gen_item);
+        }
     }
 }
 
@@ -117,6 +162,7 @@ impl<T> Drop for LeasedItem<T> {
 mod tests {
 
     use super::Pool;
+    use super::Queue;
     use std::iter;
 
     #[test]
@@ -133,4 +179,47 @@ mod tests {
             assert_eq!(*pool.acquire(), 11);
         }
     }
+
+    #[test]
+    fn test_queue() {
+        let q = Queue::new();
+        let elem = 5;
+        q.push(elem);
+        let res = q.pop();
+        assert_eq!(res.unwrap(), elem);
+    }
+
+    #[test]
+    fn test_pool_dont_panic_on_empty_pop() {
+        // When the object pool is exhausted, it shouldn't panic on pop()
+        use std::sync::Arc;
+        use std::{thread, time};
+
+        // Wrap the pool in an Arc, same way as its used in `core/index.rs`
+        let pool = Arc::new(Pool::new());
+        // clone pools outside the move scope of each new thread
+        let pool1 = Arc::clone(&pool);
+        let pool2 = Arc::clone(&pool);
+        let elements_for_pool = vec![1, 2];
+        pool.publish_new_generation(elements_for_pool);
+
+        let mut threads = vec![];
+        let sleep_dur = time::Duration::from_millis(10);
+        // spawn one more thread than there are elements in the pool
+        threads.push(thread::spawn(move || {
+            // leasing to make sure it's not dropped before sleep is called
+            let _leased_searcher = &pool.acquire();
+            thread::sleep(sleep_dur);
+        }));
+        threads.push(thread::spawn(move || {
+            // leasing to make sure it's not dropped before sleep is called
+            let _leased_searcher = &pool1.acquire();
+            thread::sleep(sleep_dur);
+        }));
+        threads.push(thread::spawn(move || {
+            // leasing to make sure it's not dropped before sleep is called
+            let _leased_searcher = &pool2.acquire();
+            thread::sleep(sleep_dur);
+        }));
+    }
 }

src/schema/document.rs

@@ -3,6 +3,7 @@ use common::BinarySerializable;
 use common::VInt;
 use itertools::Itertools;
 use std::io::{self, Read, Write};
+use DateTime;
 
 /// Tantivy's Document is the object that can
 /// be indexed and then searched for.
@@ -82,11 +83,16 @@ impl Document {
         self.add(FieldValue::new(field, Value::U64(value)));
     }
 
-    /// Add a u64 field
+    /// Add a i64 field
     pub fn add_i64(&mut self, field: Field, value: i64) {
         self.add(FieldValue::new(field, Value::I64(value)));
     }
 
+    /// Add a date field
+    pub fn add_date(&mut self, field: Field, value: &DateTime) {
+        self.add(FieldValue::new(field, Value::Date(*value)));
+    }
+
     /// Add a bytes field
     pub fn add_bytes(&mut self, field: Field, value: Vec<u8>) {
         self.add(FieldValue::new(field, Value::Bytes(value)))

src/schema/field_entry.rs

@@ -48,6 +48,15 @@ impl FieldEntry {
         }
     }
 
+    /// Creates a new date field entry in the schema, given
+    /// a name, and some options.
+    pub fn new_date(field_name: String, field_type: IntOptions) -> FieldEntry {
+        FieldEntry {
+            name: field_name,
+            field_type: FieldType::Date(field_type),
+        }
+    }
+
     /// Creates a field entry for a facet.
     pub fn new_facet(field_name: String) -> FieldEntry {
         FieldEntry {
@@ -78,7 +87,9 @@ impl FieldEntry {
     pub fn is_indexed(&self) -> bool {
         match self.field_type {
             FieldType::Str(ref options) => options.get_indexing_options().is_some(),
-            FieldType::U64(ref options) | FieldType::I64(ref options) => options.is_indexed(),
+            FieldType::U64(ref options)
+            | FieldType::I64(ref options)
+            | FieldType::Date(ref options) => options.is_indexed(),
             FieldType::HierarchicalFacet => true,
             FieldType::Bytes => false,
         }
@@ -95,7 +106,9 @@ impl FieldEntry {
     /// Returns true iff the field is stored
     pub fn is_stored(&self) -> bool {
         match self.field_type {
-            FieldType::U64(ref options) | FieldType::I64(ref options) => options.is_stored(),
+            FieldType::U64(ref options)
+            | FieldType::I64(ref options)
+            | FieldType::Date(ref options) => options.is_stored(),
             FieldType::Str(ref options) => options.is_stored(),
             // TODO make stored hierarchical facet optional
             FieldType::HierarchicalFacet => true,
@@ -125,6 +138,10 @@ impl Serialize for FieldEntry {
                 s.serialize_field("type", "i64")?;
                 s.serialize_field("options", options)?;
             }
+            FieldType::Date(ref options) => {
+                s.serialize_field("type", "date")?;
+                s.serialize_field("options", options)?;
+            }
             FieldType::HierarchicalFacet => {
                 s.serialize_field("type", "hierarchical_facet")?;
             }
@@ -188,7 +205,7 @@ impl<'de> Deserialize<'de> for FieldEntry {
                                 "bytes" => {
                                     field_type = Some(FieldType::Bytes);
                                 }
-                                "text" | "u64" | "i64" => {
+                                "text" | "u64" | "i64" | "date" => {
                                     // These types require additional options to create a field_type
                                 }
                                 _ => panic!("unhandled type"),
@@ -205,6 +222,7 @@ impl<'de> Deserialize<'de> for FieldEntry {
                                 "text" => field_type = Some(FieldType::Str(map.next_value()?)),
                                 "u64" => field_type = Some(FieldType::U64(map.next_value()?)),
                                 "i64" => field_type = Some(FieldType::I64(map.next_value()?)),
+                                "date" => field_type = Some(FieldType::Date(map.next_value()?)),
                                 _ => {
                                     let msg = format!("Unrecognised type {}", ty);
                                     return Err(de::Error::custom(msg));

src/schema/field_type.rs

@@ -34,6 +34,8 @@ pub enum Type {
     U64,
     /// `i64`
     I64,
+    /// `date(i64) timestamp`
+    Date,
     /// `tantivy::schema::Facet`. Passed as a string in JSON.
     HierarchicalFacet,
     /// `Vec<u8>`
@@ -50,6 +52,8 @@ pub enum FieldType {
     U64(IntOptions),
     /// Signed 64-bits integers 64 field type configuration
     I64(IntOptions),
+    /// Signed 64-bits Date 64 field type configuration,
+    Date(IntOptions),
     /// Hierachical Facet
     HierarchicalFacet,
     /// Bytes (one per document)
@@ -63,6 +67,7 @@ impl FieldType {
             FieldType::Str(_) => Type::Str,
             FieldType::U64(_) => Type::U64,
             FieldType::I64(_) => Type::I64,
+            FieldType::Date(_) => Type::Date,
             FieldType::HierarchicalFacet => Type::HierarchicalFacet,
             FieldType::Bytes => Type::Bytes,
         }
@@ -75,6 +80,7 @@ impl FieldType {
             FieldType::U64(ref int_options) | FieldType::I64(ref int_options) => {
                 int_options.is_indexed()
             }
+            FieldType::Date(ref date_options) => date_options.is_indexed(),
             FieldType::HierarchicalFacet => true,
             FieldType::Bytes => false,
         }
@@ -89,7 +95,9 @@ impl FieldType {
             FieldType::Str(ref text_options) => text_options
                 .get_indexing_options()
                 .map(|indexing_options| indexing_options.index_option()),
-            FieldType::U64(ref int_options) | FieldType::I64(ref int_options) => {
+            FieldType::U64(ref int_options)
+            | FieldType::I64(ref int_options)
+            | FieldType::Date(ref int_options) => {
                 if int_options.is_indexed() {
                     Some(IndexRecordOption::Basic)
                 } else {
@@ -110,9 +118,9 @@ impl FieldType {
         match *json {
             JsonValue::String(ref field_text) => match *self {
                 FieldType::Str(_) => Ok(Value::Str(field_text.clone())),
-                FieldType::U64(_) | FieldType::I64(_) => Err(ValueParsingError::TypeError(
-                    format!("Expected an integer, got {:?}", json),
-                )),
+                FieldType::U64(_) | FieldType::I64(_) | FieldType::Date(_) => Err(
+                    ValueParsingError::TypeError(format!("Expected an integer, got {:?}", json)),
+                ),
                 FieldType::HierarchicalFacet => Ok(Value::Facet(Facet::from(field_text))),
                 FieldType::Bytes => decode(field_text).map(Value::Bytes).map_err(|_| {
                     ValueParsingError::InvalidBase64(format!(
@@ -122,7 +130,7 @@ impl FieldType {
                 }),
             },
             JsonValue::Number(ref field_val_num) => match *self {
-                FieldType::I64(_) => {
+                FieldType::I64(_) | FieldType::Date(_) => {
                     if let Some(field_val_i64) = field_val_num.as_i64() {
                         Ok(Value::I64(field_val_i64))
                     } else {

src/schema/flags.rs

@@ -0,0 +1,81 @@
+use schema::IntOptions;
+use schema::TextOptions;
+use std::ops::BitOr;
+
+#[derive(Clone)]
+pub struct StoredFlag;
+/// Flag to mark the field as stored.
+/// This flag can apply to any kind of field.
+///
+/// A stored fields of a document can be retrieved given its `DocId`.
+/// Stored field are stored together and LZ4 compressed.
+/// Reading the stored fields of a document is relatively slow.
+/// (~ 100 microsecs)
+///
+/// It should not be used during scoring or collection.
+pub const STORED: SchemaFlagList<StoredFlag, ()> = SchemaFlagList {
+    head: StoredFlag,
+    tail: (),
+};
+
+#[derive(Clone)]
+pub struct IndexedFlag;
+/// Flag to mark the field as indexed.
+///
+/// The `INDEXED` flag can only be used when building `IntOptions` (`u64` and `i64` fields)
+/// Of course, text fields can also be indexed... But this is expressed by using either the
+/// `STRING` (untokenized) or `TEXT` (tokenized with the english tokenizer) flags.
+pub const INDEXED: SchemaFlagList<IndexedFlag, ()> = SchemaFlagList {
+    head: IndexedFlag,
+    tail: (),
+};
+
+#[derive(Clone)]
+pub struct FastFlag;
+/// Flag to mark the field as a fast field (similar to Lucene's DocValues)
+///
+/// Fast fields can be random-accessed rapidly. Fields useful for scoring, filtering
+/// or collection should be mark as fast fields.
+/// The `FAST` flag can only be used when building `IntOptions` (`u64` and `i64` fields)
+pub const FAST: SchemaFlagList<FastFlag, ()> = SchemaFlagList {
+    head: FastFlag,
+    tail: (),
+};
+
+impl<Head, OldHead, OldTail> BitOr<SchemaFlagList<Head, ()>> for SchemaFlagList<OldHead, OldTail>
+where
+    Head: Clone,
+    OldHead: Clone,
+    OldTail: Clone,
+{
+    type Output = SchemaFlagList<Head, SchemaFlagList<OldHead, OldTail>>;
+
+    fn bitor(self, head: SchemaFlagList<Head, ()>) -> Self::Output {
+        SchemaFlagList {
+            head: head.head,
+            tail: self.clone(),
+        }
+    }
+}
+
+impl<T: Clone + Into<IntOptions>> BitOr<IntOptions> for SchemaFlagList<T, ()> {
+    type Output = IntOptions;
+
+    fn bitor(self, rhs: IntOptions) -> Self::Output {
+        self.head.into() | rhs
+    }
+}
+
+impl<T: Clone + Into<TextOptions>> BitOr<TextOptions> for SchemaFlagList<T, ()> {
+    type Output = TextOptions;
+
+    fn bitor(self, rhs: TextOptions) -> Self::Output {
+        self.head.into() | rhs
+    }
+}
+
+#[derive(Clone)]
+pub struct SchemaFlagList<Head: Clone, Tail: Clone> {
+    pub head: Head,
+    pub tail: Tail,
+}

src/schema/int_options.rs

@@ -1,3 +1,4 @@
+use schema::flags::{FastFlag, IndexedFlag, SchemaFlagList, StoredFlag};
 use std::ops::BitOr;
 
 /// Express whether a field is single-value or multi-valued.
@@ -85,41 +86,62 @@ impl Default for IntOptions {
     }
 }
 
-/// Shortcut for a u64 fast field.
-///
-/// Such a shortcut can be composed as follows `STORED | FAST | INT_INDEXED`
-pub const FAST: IntOptions = IntOptions {
-    indexed: false,
-    stored: false,
-    fast: Some(Cardinality::SingleValue),
-};
+impl From<()> for IntOptions {
+    fn from(_: ()) -> IntOptions {
+        IntOptions::default()
+    }
+}
 
-/// Shortcut for a u64 indexed field.
-///
-/// Such a shortcut can be composed as follows `STORED | FAST | INT_INDEXED`
-pub const INT_INDEXED: IntOptions = IntOptions {
-    indexed: true,
-    stored: false,
-    fast: None,
-};
+impl From<FastFlag> for IntOptions {
+    fn from(_: FastFlag) -> Self {
+        IntOptions {
+            indexed: false,
+            stored: false,
+            fast: Some(Cardinality::SingleValue),
+        }
+    }
+}
 
-/// Shortcut for a u64 stored field.
-///
-/// Such a shortcut can be composed as follows `STORED | FAST | INT_INDEXED`
-pub const INT_STORED: IntOptions = IntOptions {
-    indexed: false,
-    stored: true,
-    fast: None,
-};
+impl From<StoredFlag> for IntOptions {
+    fn from(_: StoredFlag) -> Self {
+        IntOptions {
+            indexed: false,
+            stored: true,
+            fast: None,
+        }
+    }
+}
 
-impl BitOr for IntOptions {
+impl From<IndexedFlag> for IntOptions {
+    fn from(_: IndexedFlag) -> Self {
+        IntOptions {
+            indexed: true,
+            stored: false,
+            fast: None,
+        }
+    }
+}
+
+impl<T: Into<IntOptions>> BitOr<T> for IntOptions {
     type Output = IntOptions;
 
-    fn bitor(self, other: IntOptions) -> IntOptions {
+    fn bitor(self, other: T) -> IntOptions {
         let mut res = IntOptions::default();
+        let other = other.into();
         res.indexed = self.indexed | other.indexed;
         res.stored = self.stored | other.stored;
         res.fast = self.fast.or(other.fast);
         res
     }
 }
+
+impl<Head, Tail> From<SchemaFlagList<Head, Tail>> for IntOptions
+where
+    Head: Clone,
+    Tail: Clone,
+    Self: BitOr<Output = Self> + From<Head> + From<Tail>,
+{
+    fn from(head_tail: SchemaFlagList<Head, Tail>) -> Self {
+        Self::from(head_tail.head) | Self::from(head_tail.tail)
+    }
+}

src/schema/mod.rs

@@ -33,7 +33,7 @@ let title_options = TextOptions::default()
     .set_indexing_options(TextFieldIndexing::default()
         .set_tokenizer("default")
         .set_index_option(IndexRecordOption::WithFreqsAndPositions));
-schema_builder.add_text_field("title_options", title_options);
+schema_builder.add_text_field("title", title_options);
 let schema = schema_builder.build();
 ```
 
@@ -53,23 +53,8 @@ The effect of each possible setting is described more in detail
 On the other hand setting the field as stored or not determines whether the field should be returned
 when [`searcher.doc(doc_address)`](../struct.Searcher.html#method.doc) is called.
 
-### Shortcuts
 
-For convenience, a few special values of `TextOptions`.
-They can be composed using the `|` operator.
-The example can be rewritten :
-
-
-```
-use tantivy::schema::*;
-let mut schema_builder = Schema::builder();
-schema_builder.add_text_field("title_options", TEXT | STORED);
-let schema = schema_builder.build();
-```
-
-
-
-## Setting a u64 field
+## Setting a u64 or a i64 field
 
 ### Example
 
@@ -98,6 +83,23 @@ u64 that are indexed as fast will be stored in a special data structure that wil
 make it possible to access the u64 value given the doc id rapidly. This is useful if the value of
 the field is required during scoring or collection for instance.
 
+
+### Shortcuts
+
+
+For convenience, it is possible to define your field indexing options by combining different flags
+using the  `|` operator.
+
+For instance, a schema containing the two fields defined in the example above could be rewritten :
+
+```
+use tantivy::schema::*;
+let mut schema_builder = Schema::builder();
+schema_builder.add_u64_field("num_stars", INDEXED | STORED);
+schema_builder.add_text_field("title", TEXT | STORED);
+let schema = schema_builder.build();
+```
+
 */
 
 mod document;
@@ -116,13 +118,15 @@ mod named_field_document;
 mod text_options;
 mod value;
 
+mod flags;
+
 pub use self::named_field_document::NamedFieldDocument;
 pub use self::schema::DocParsingError;
 pub use self::schema::{Schema, SchemaBuilder};
 pub use self::value::Value;
 
 pub use self::facet::Facet;
-pub use self::facet::FACET_SEP_BYTE;
+pub(crate) use self::facet::FACET_SEP_BYTE;
 
 pub use self::document::Document;
 pub use self::field::Field;
@@ -135,15 +139,12 @@ pub use self::field_value::FieldValue;
 pub use self::index_record_option::IndexRecordOption;
 pub use self::text_options::TextFieldIndexing;
 pub use self::text_options::TextOptions;
-pub use self::text_options::STORED;
 pub use self::text_options::STRING;
 pub use self::text_options::TEXT;
 
+pub use self::flags::{FAST, INDEXED, STORED};
 pub use self::int_options::Cardinality;
 pub use self::int_options::IntOptions;
-pub use self::int_options::FAST;
-pub use self::int_options::INT_INDEXED;
-pub use self::int_options::INT_STORED;
 
 use regex::Regex;
 

src/schema/schema.rs

@@ -52,9 +52,13 @@ impl SchemaBuilder {
     /// by the second one.
     /// The first field will get a field id
     /// but only the second one will be indexed
-    pub fn add_u64_field(&mut self, field_name_str: &str, field_options: IntOptions) -> Field {
+    pub fn add_u64_field<T: Into<IntOptions>>(
+        &mut self,
+        field_name_str: &str,
+        field_options: T,
+    ) -> Field {
         let field_name = String::from(field_name_str);
-        let field_entry = FieldEntry::new_u64(field_name, field_options);
+        let field_entry = FieldEntry::new_u64(field_name, field_options.into());
         self.add_field(field_entry)
     }
 
@@ -68,9 +72,35 @@ impl SchemaBuilder {
     /// by the second one.
     /// The first field will get a field id
     /// but only the second one will be indexed
-    pub fn add_i64_field(&mut self, field_name_str: &str, field_options: IntOptions) -> Field {
+    pub fn add_i64_field<T: Into<IntOptions>>(
+        &mut self,
+        field_name_str: &str,
+        field_options: T,
+    ) -> Field {
         let field_name = String::from(field_name_str);
-        let field_entry = FieldEntry::new_i64(field_name, field_options);
+        let field_entry = FieldEntry::new_i64(field_name, field_options.into());
+        self.add_field(field_entry)
+    }
+
+    /// Adds a new date field.
+    /// Returns the associated field handle
+    /// Internally, Tantivy simply stores dates as i64 UTC timestamps,
+    /// while the user supplies DateTime values for convenience.
+    ///
+    /// # Caution
+    ///
+    /// Appending two fields with the same name
+    /// will result in the shadowing of the first
+    /// by the second one.
+    /// The first field will get a field id
+    /// but only the second one will be indexed
+    pub fn add_date_field<T: Into<IntOptions>>(
+        &mut self,
+        field_name_str: &str,
+        field_options: T,
+    ) -> Field {
+        let field_name = String::from(field_name_str);
+        let field_entry = FieldEntry::new_date(field_name, field_options.into());
         self.add_field(field_entry)
     }
 
@@ -84,9 +114,13 @@ impl SchemaBuilder {
     /// by the second one.
     /// The first field will get a field id
     /// but only the second one will be indexed
-    pub fn add_text_field(&mut self, field_name_str: &str, field_options: TextOptions) -> Field {
+    pub fn add_text_field<T: Into<TextOptions>>(
+        &mut self,
+        field_name_str: &str,
+        field_options: T,
+    ) -> Field {
         let field_name = String::from(field_name_str);
-        let field_entry = FieldEntry::new_text(field_name, field_options);
+        let field_entry = FieldEntry::new_text(field_name, field_options.into());
         self.add_field(field_entry)
     }
 
@@ -178,15 +212,7 @@ impl Schema {
         SchemaBuilder::default()
     }
 
-    /// Returns the field options associated with a given name.
-    ///
-    /// # Panics
-    /// Panics if the field name does not exist.
-    /// It is meant as an helper for user who created
-    /// and control the content of their schema.
-    ///
-    /// If panicking is not an option for you,
-    /// you may use `get(&self, field_name: &str)`.
+    /// Returns the field option associated with a given name.
     pub fn get_field(&self, field_name: &str) -> Option<Field> {
         self.0.fields_map.get(field_name).cloned()
     }

src/schema/term.rs

@@ -5,6 +5,7 @@ use byteorder::{BigEndian, ByteOrder};
 use common;
 use schema::Facet;
 use std::str;
+use DateTime;
 
 /// Size (in bytes) of the buffer of a int field.
 const INT_TERM_LEN: usize = 4 + 8;
@@ -30,6 +31,18 @@ impl Term {
         Term::from_field_u64(field, val_u64)
     }
 
+    /// Builds a term given a field, and a DateTime value
+    ///
+    /// Assuming the term has a field id of 1, and a timestamp i64 value of 3234,
+    /// the Term will have 8 bytes.
+    ///
+    /// The first four byte are dedicated to storing the field id as a u64.
+    /// The 4 following bytes are encoding the DateTime as i64 timestamp value.
+    pub fn from_field_date(field: Field, val: &DateTime) -> Term {
+        let val_timestamp = val.timestamp();
+        Term::from_field_i64(field, val_timestamp)
+    }
+
     /// Creates a `Term` given a facet.
     pub fn from_facet(field: Field, facet: &Facet) -> Term {
         let bytes = facet.encoded_str().as_bytes();

src/schema/text_options.rs

@@ -1,3 +1,5 @@
+use schema::flags::SchemaFlagList;
+use schema::flags::StoredFlag;
 use schema::IndexRecordOption;
 use std::borrow::Cow;
 use std::ops::BitOr;
@@ -109,19 +111,11 @@ pub const TEXT: TextOptions = TextOptions {
     stored: false,
 };
 
-/// A stored fields of a document can be retrieved given its `DocId`.
-/// Stored field are stored together and LZ4 compressed.
-/// Reading the stored fields of a document is relatively slow.
-/// (100 microsecs)
-pub const STORED: TextOptions = TextOptions {
-    indexing: None,
-    stored: true,
-};
-
-impl BitOr for TextOptions {
+impl<T: Into<TextOptions>> BitOr<T> for TextOptions {
     type Output = TextOptions;
 
-    fn bitor(self, other: TextOptions) -> TextOptions {
+    fn bitor(self, other: T) -> TextOptions {
+        let other = other.into();
         let mut res = TextOptions::default();
         res.indexing = self.indexing.or(other.indexing);
         res.stored = self.stored | other.stored;
@@ -129,6 +123,32 @@ impl BitOr for TextOptions {
     }
 }
 
+impl From<()> for TextOptions {
+    fn from(_: ()) -> TextOptions {
+        TextOptions::default()
+    }
+}
+
+impl From<StoredFlag> for TextOptions {
+    fn from(_: StoredFlag) -> TextOptions {
+        TextOptions {
+            indexing: None,
+            stored: true,
+        }
+    }
+}
+
+impl<Head, Tail> From<SchemaFlagList<Head, Tail>> for TextOptions
+where
+    Head: Clone,
+    Tail: Clone,
+    Self: BitOr<Output = Self> + From<Head> + From<Tail>,
+{
+    fn from(head_tail: SchemaFlagList<Head, Tail>) -> Self {
+        Self::from(head_tail.head) | Self::from(head_tail.tail)
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use schema::*;

src/schema/value.rs

@@ -2,6 +2,7 @@ use schema::Facet;
 use serde::de::Visitor;
 use serde::{Deserialize, Deserializer, Serialize, Serializer};
 use std::fmt;
+use DateTime;
 
 /// Value represents the value of a any field.
 /// It is an enum over all over all of the possible field type.
@@ -13,6 +14,8 @@ pub enum Value {
     U64(u64),
     /// Signed 64-bits Integer `i64`
     I64(i64),
+    /// Signed 64-bits Date time stamp `date`
+    Date(DateTime),
     /// Hierarchical Facet
     Facet(Facet),
     /// Arbitrarily sized byte array
@@ -28,6 +31,7 @@ impl Serialize for Value {
             Value::Str(ref v) => serializer.serialize_str(v),
             Value::U64(u) => serializer.serialize_u64(u),
             Value::I64(u) => serializer.serialize_i64(u),
+            Value::Date(ref date) => serializer.serialize_i64(date.timestamp()),
             Value::Facet(ref facet) => facet.serialize(serializer),
             Value::Bytes(ref bytes) => serializer.serialize_bytes(bytes),
         }
@@ -102,6 +106,17 @@ impl Value {
             _ => panic!("This is not a text field."),
         }
     }
+
+    /// Returns the Date-value, provided the value is of the `Date` type.
+    ///
+    /// # Panics
+    /// If the value is not of type `Date`
+    pub fn date_value(&self) -> &DateTime {
+        match *self {
+            Value::Date(ref value) => value,
+            _ => panic!("This is not a date field."),
+        }
+    }
 }
 
 impl From<String> for Value {
@@ -122,6 +137,12 @@ impl From<i64> for Value {
     }
 }
 
+impl From<DateTime> for Value {
+    fn from(date_time: DateTime) -> Value {
+        Value::Date(date_time)
+    }
+}
+
 impl<'a> From<&'a str> for Value {
     fn from(s: &'a str) -> Value {
         Value::Str(s.to_string())
@@ -142,6 +163,7 @@ impl From<Vec<u8>> for Value {
 
 mod binary_serialize {
     use super::Value;
+    use chrono::{TimeZone, Utc};
     use common::BinarySerializable;
     use schema::Facet;
     use std::io::{self, Read, Write};
@@ -151,6 +173,7 @@ mod binary_serialize {
     const I64_CODE: u8 = 2;
     const HIERARCHICAL_FACET_CODE: u8 = 3;
     const BYTES_CODE: u8 = 4;
+    const DATE_CODE: u8 = 5;
 
     impl BinarySerializable for Value {
         fn serialize<W: Write>(&self, writer: &mut W) -> io::Result<()> {
@@ -167,6 +190,10 @@ mod binary_serialize {
                     I64_CODE.serialize(writer)?;
                     val.serialize(writer)
                 }
+                Value::Date(ref val) => {
+                    DATE_CODE.serialize(writer)?;
+                    val.timestamp().serialize(writer)
+                }
                 Value::Facet(ref facet) => {
                     HIERARCHICAL_FACET_CODE.serialize(writer)?;
                     facet.serialize(writer)
@@ -192,6 +219,10 @@ mod binary_serialize {
                     let value = i64::deserialize(reader)?;
                     Ok(Value::I64(value))
                 }
+                DATE_CODE => {
+                    let timestamp = i64::deserialize(reader)?;
+                    Ok(Value::Date(Utc.timestamp(timestamp, 0)))
+                }
                 HIERARCHICAL_FACET_CODE => Ok(Value::Facet(Facet::deserialize(reader)?)),
                 BYTES_CODE => Ok(Value::Bytes(Vec::<u8>::deserialize(reader)?)),
                 _ => Err(io::Error::new(

src/snippet/mod.rs

@@ -241,8 +241,8 @@ fn select_best_fragment_combination(fragments: &[FragmentCandidate], text: &str)
 /// #    let query_parser = QueryParser::for_index(&index, vec![text_field]);
 /// // ...
 /// let query = query_parser.parse_query("haleurs flamands").unwrap();
-/// # index.load_searchers()?;
-/// # let searcher = index.searcher();
+/// # let reader = index.reader()?;
+/// # let searcher = reader.searcher();
 /// let mut snippet_generator = SnippetGenerator::create(&searcher, &*query, text_field)?;
 /// snippet_generator.set_max_num_chars(100);
 /// let snippet = snippet_generator.snippet_from_doc(&doc);
@@ -528,9 +528,8 @@ Survey in 2016, 2017, and 2018."#;
             index_writer.add_document(doc!(text_field => "a"));
             index_writer.add_document(doc!(text_field => "a b"));
             index_writer.commit().unwrap();
-            index.load_searchers().unwrap();
         }
-        let searcher = index.searcher();
+        let searcher = index.reader().unwrap().searcher();
         let query_parser = QueryParser::for_index(&index, vec![text_field]);
         {
             let query = query_parser.parse_query("e").unwrap();
@@ -587,8 +586,7 @@ Survey in 2016, 2017, and 2018."#;
             }
             index_writer.commit().unwrap();
         }
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        let searcher = index.reader().unwrap().searcher();
         let query_parser = QueryParser::for_index(&index, vec![text_field]);
         let query = query_parser.parse_query("rust design").unwrap();
         let mut snippet_generator =

src/space_usage/mod.rs

@@ -295,8 +295,7 @@ mod test {
     use core::Index;
     use schema::Field;
     use schema::Schema;
-    use schema::STORED;
-    use schema::{FAST, INT_INDEXED, TEXT};
+    use schema::{FAST, INDEXED, STORED, TEXT};
     use space_usage::ByteCount;
     use space_usage::PerFieldSpaceUsage;
     use Term;
@@ -305,9 +304,8 @@ mod test {
     fn test_empty() {
         let schema = Schema::builder().build();
         let index = Index::create_in_ram(schema.clone());
-
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        let reader = index.reader().unwrap();
+        let searcher = reader.searcher();
         let searcher_space_usage = searcher.space_usage();
         assert_eq!(0, searcher_space_usage.total());
     }
@@ -332,7 +330,7 @@ mod test {
     #[test]
     fn test_fast_indexed() {
         let mut schema_builder = Schema::builder();
-        let name = schema_builder.add_u64_field("name", FAST | INT_INDEXED);
+        let name = schema_builder.add_u64_field("name", FAST | INDEXED);
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema.clone());
 
@@ -345,8 +343,8 @@ mod test {
             index_writer.commit().unwrap();
         }
 
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        let reader = index.reader().unwrap();
+        let searcher = reader.searcher();
         let searcher_space_usage = searcher.space_usage();
         assert!(searcher_space_usage.total() > 0);
         assert_eq!(1, searcher_space_usage.segments().len());
@@ -385,8 +383,8 @@ mod test {
             index_writer.commit().unwrap();
         }
 
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        let reader = index.reader().unwrap();
+        let searcher = reader.searcher();
         let searcher_space_usage = searcher.space_usage();
         assert!(searcher_space_usage.total() > 0);
         assert_eq!(1, searcher_space_usage.segments().len());
@@ -424,9 +422,8 @@ mod test {
             index_writer.add_document(doc!(name => "hello hi goodbye"));
             index_writer.commit().unwrap();
         }
-
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        let reader = index.reader().unwrap();
+        let searcher = reader.searcher();
         let searcher_space_usage = searcher.space_usage();
         assert!(searcher_space_usage.total() > 0);
         assert_eq!(1, searcher_space_usage.segments().len());
@@ -450,7 +447,7 @@ mod test {
     #[test]
     fn test_deletes() {
         let mut schema_builder = Schema::builder();
-        let name = schema_builder.add_u64_field("name", INT_INDEXED);
+        let name = schema_builder.add_u64_field("name", INDEXED);
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema.clone());
 
@@ -472,9 +469,8 @@ mod test {
             index_writer2.commit().unwrap();
         }
 
-        index.load_searchers().unwrap();
-
-        let searcher = index.searcher();
+        let reader = index.reader().unwrap();
+        let searcher = reader.searcher();
         let searcher_space_usage = searcher.space_usage();
         assert!(searcher_space_usage.total() > 0);
         assert_eq!(1, searcher_space_usage.segments().len());

src/termdict/mod.rs

@@ -159,8 +159,7 @@ mod tests {
                 index_writer.commit().unwrap();
             }
         }
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        let searcher = index.reader().unwrap().searcher();
 
         let field_searcher = searcher.field(text_field);
         let mut term_it = field_searcher.terms();

src/termdict/streamer.rs

@@ -1,9 +1,9 @@
 use super::TermDictionary;
-use fst::automaton::AlwaysMatch;
-use fst::map::{Stream, StreamBuilder};
-use fst::Automaton;
-use fst::{IntoStreamer, Streamer};
 use postings::TermInfo;
+use tantivy_fst::automaton::AlwaysMatch;
+use tantivy_fst::map::{Stream, StreamBuilder};
+use tantivy_fst::Automaton;
+use tantivy_fst::{IntoStreamer, Streamer};
 use termdict::TermOrdinal;
 
 /// `TermStreamerBuilder` is a helper object used to define

src/termdict/termdict.rs

@@ -3,15 +3,15 @@ use super::{TermStreamer, TermStreamerBuilder};
 use common::BinarySerializable;
 use common::CountingWriter;
 use directory::ReadOnlySource;
-use fst;
-use fst::raw::Fst;
-use fst::Automaton;
 use postings::TermInfo;
 use schema::FieldType;
 use std::io::{self, Write};
+use tantivy_fst;
+use tantivy_fst::raw::Fst;
+use tantivy_fst::Automaton;
 use termdict::TermOrdinal;
 
-fn convert_fst_error(e: fst::Error) -> io::Error {
+fn convert_fst_error(e: tantivy_fst::Error) -> io::Error {
     io::Error::new(io::ErrorKind::Other, e)
 }
 
@@ -19,7 +19,7 @@ fn convert_fst_error(e: fst::Error) -> io::Error {
 ///
 /// Inserting must be done in the order of the `keys`.
 pub struct TermDictionaryBuilder<W> {
-    fst_builder: fst::MapBuilder<W>,
+    fst_builder: tantivy_fst::MapBuilder<W>,
     term_info_store_writer: TermInfoStoreWriter,
     term_ord: u64,
 }
@@ -30,7 +30,7 @@ where
 {
     /// Creates a new `TermDictionaryBuilder`
     pub fn create(w: W, _field_type: &FieldType) -> io::Result<Self> {
-        let fst_builder = fst::MapBuilder::new(w).map_err(convert_fst_error)?;
+        let fst_builder = tantivy_fst::MapBuilder::new(w).map_err(convert_fst_error)?;
         Ok(TermDictionaryBuilder {
             fst_builder,
             term_info_store_writer: TermInfoStoreWriter::new(),
@@ -87,17 +87,9 @@ where
     }
 }
 
-fn open_fst_index(source: ReadOnlySource) -> fst::Map {
-    let fst = match source {
-        ReadOnlySource::Anonymous(data) => {
-            Fst::from_shared_bytes(data.data, data.start, data.len).expect("FST data is corrupted")
-        }
-        #[cfg(feature = "mmap")]
-        ReadOnlySource::Mmap(mmap_readonly) => {
-            Fst::from_mmap(mmap_readonly).expect("FST data is corrupted")
-        }
-    };
-    fst::Map::from(fst)
+fn open_fst_index(source: ReadOnlySource) -> tantivy_fst::Map<ReadOnlySource> {
+    let fst = Fst::new(source).expect("FST data is corrupted");
+    tantivy_fst::Map::from(fst)
 }
 
 /// The term dictionary contains all of the terms in
@@ -107,7 +99,7 @@ fn open_fst_index(source: ReadOnlySource) -> fst::Map {
 /// respective `TermOrdinal`. The `TermInfoStore` then makes it
 /// possible to fetch the associated `TermInfo`.
 pub struct TermDictionary {
-    fst_index: fst::Map,
+    fst_index: tantivy_fst::Map<ReadOnlySource>,
     term_info_store: TermInfoStore,
 }
 

src/tokenizer/stemmer.rs

@@ -15,6 +15,7 @@ pub enum Language {
     Finnish,
     French,
     German,
+    Greek,
     Hungarian,
     Italian,
     Portuguese,
@@ -37,6 +38,7 @@ impl Language {
             Finnish => Algorithm::Finnish,
             French => Algorithm::French,
             German => Algorithm::German,
+            Greek => Algorithm::Greek,
             Hungarian => Algorithm::Hungarian,
             Italian => Algorithm::Italian,
             Portuguese => Algorithm::Portuguese,