Fix version

Adding IndexReader to the API. Making it possible to watch for changes.

* Closes #500

.travis.yml

@@ -29,7 +29,7 @@ addons:
 matrix:
   include:
     # Android
-    - env: TARGET=aarch64-linux-android DISABLE_TESTS=1
+    - env: TARGET=aarch64-linux-android
     #- env: TARGET=arm-linux-androideabi DISABLE_TESTS=1
     #- env: TARGET=armv7-linux-androideabi DISABLE_TESTS=1
     #- env: TARGET=i686-linux-android DISABLE_TESTS=1

CHANGELOG.md

@@ -1,3 +1,37 @@
+Tantivy 0.9.0
+=====================
+*0.9.0 index format is not compatible with the 
+previous index format.*
+- MAJOR BUGFIX : 
+  Some `Mmap` objects were being leaked, and would never get released. (@fulmicoton)
+- Removed most unsafe (@fulmicoton)
+- Indexer memory footprint improved. (VInt comp, inlining the first block. (@fulmicoton)
+- Stemming in other language possible (@pentlander)
+- Segments with no docs are deleted earlier (@barrotsteindev)
+- Added grouped add and delete operations. 
+  They are guaranteed to happen together (i.e. they cannot be split by a commit). 
+  In addition, adds are guaranteed to happen on the same segment. (@elbow-jason)
+- Removed `INT_STORED` and `INT_INDEXED`. It is now possible to use `STORED` and `INDEXED`
+  for int fields. (@fulmicoton)
+- Added DateTime field (@barrotsteindev)
+- Added IndexReader. By default, index is reloaded automatically upon new commits (@fulmicoton)
+
+
+Tantivy 0.8.2
+=====================
+Fixing build for x86_64 platforms. (#496)
+No need to update from 0.8.1 if tantivy
+is building on your platform.
+
+
+Tantivy 0.8.1
+=====================
+Hotfix of #476.
+
+Merge was reflecting deletes before commit was passed. 
+Thanks @barrotsteindev  for reporting the bug.
+
+
 Tantivy 0.8.0
 =====================
 *No change in the index format*

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "tantivy"
-version = "0.8.0"
+version = "0.9.0"
 authors = ["Paul Masurel <paul.masurel@gmail.com>"]
 license = "MIT"
 categories = ["database-implementations", "data-structures"]
@@ -16,8 +16,8 @@ base64 = "0.10.0"
 byteorder = "1.0"
 lazy_static = "1"
 regex = "1.0"
-fst = {version="0.3", default-features=false}
-fst-regex = { version="0.2" }
+tantivy-fst = "0.1"
+memmap = {version = "0.7", optional=true}
 lz4 = {version="1.20", optional=true}
 snap = {version="0.2"}
 atomicwrites = {version="0.2.2", optional=true}
@@ -29,20 +29,21 @@ serde = "1.0"
 serde_derive = "1.0"
 serde_json = "1.0"
 num_cpus = "1.2"
+fs2={version="0.4", optional=true}
 itertools = "0.8"
 levenshtein_automata = {version="0.1", features=["fst_automaton"]}
+notify = {version="4", optional=true}
 bit-set = "0.5"
-uuid = { version = "0.7", features = ["v4", "serde"] }
+uuid = { version = "0.7.2", features = ["v4", "serde"] }
 crossbeam = "0.5"
 futures = "0.1"
 futures-cpupool = "0.1"
 owning_ref = "0.4"
 stable_deref_trait = "1.0.0"
-rust-stemmers = "1"
-downcast = { version="0.9" }
-matches = "0.1"
-bitpacking = "0.5"
-census = "0.1"
+rust-stemmers = "1.1"
+downcast-rs = { version="1.0" }
+bitpacking = "0.6"
+census = "0.2"
 fnv = "1.0.6"
 owned-read = "0.4"
 failure = "0.1"
@@ -50,6 +51,7 @@ htmlescape = "0.3.1"
 fail = "0.2"
 scoped-pool = "1.0"
 murmurhash32 = "0.2"
+chrono = "0.4"
 
 [target.'cfg(windows)'.dependencies]
 winapi = "0.2"
@@ -57,6 +59,8 @@ winapi = "0.2"
 [dev-dependencies]
 rand = "0.6"
 maplit = "1"
+matches = "0.1.8"
+time = "0.1.42"
 
 [profile.release]
 opt-level = 3
@@ -70,12 +74,11 @@ overflow-checks = true
 [features]
 # by default no-fail is disabled. We manually enable it when running test.
 default = ["mmap", "no_fail"]
-mmap = ["fst/mmap", "atomicwrites"]
+mmap = ["atomicwrites", "fs2", "memmap", "notify"]
 lz4-compression = ["lz4"]
 no_fail = ["fail/no_fail"]
 unstable = [] # useful for benches.
+wasm-bindgen = ["uuid/wasm-bindgen"]
 
 [badges]
 travis-ci = { repository = "tantivy-search/tantivy" }
-
-

README.md

@@ -30,6 +30,7 @@ Tantivy is, in fact, strongly inspired by Lucene's design.
 # Features
 
 - Full-text search
+- Configurable tokenizer. (stemming available for 17 latin languages. Third party support for Chinese ([tantivy-jieba](https://crates.io/crates/tantivy-jieba) and [cang-jie](https://crates.io/crates/cang-jie)) and [Japanese](https://crates.io/crates/tantivy-tokenizer-tiny-segmenter)
 - Fast (check out the :racehorse: :sparkles: [benchmark](https://tantivy-search.github.io/bench/) :sparkles: :racehorse:)
 - Tiny startup time (<10ms), perfect for command line tools
 - BM25 scoring (the same as lucene)
@@ -41,6 +42,7 @@ Tantivy is, in fact, strongly inspired by Lucene's design.
 - SIMD integer compression when the platform/CPU includes the SSE2 instruction set.
 - Single valued and multivalued u64 and i64 fast fields (equivalent of doc values in Lucene)
 - `&[u8]` fast fields
+- Text, i64, u64, dates and hierarchical facet fields
 - LZ4 compressed document store
 - Range queries
 - Faceted search

examples/basic_search.rs

@@ -20,6 +20,7 @@ use tantivy::collector::TopDocs;
 use tantivy::query::QueryParser;
 use tantivy::schema::*;
 use tantivy::Index;
+use tantivy::ReloadPolicy;
 use tempdir::TempDir;
 
 fn main() -> tantivy::Result<()> {
@@ -170,24 +171,33 @@ fn main() -> tantivy::Result<()> {
     //
     // ### Searcher
     //
-    // Let's search our index. Start by reloading
-    // searchers in the index. This should be done
-    // after every `commit()`.
-    index.load_searchers()?;
+    // A reader is required to get search the index.
+    // It acts as a `Searcher` pool that reloads itself,
+    // depending on a `ReloadPolicy`.
+    //
+    // For a search server you will typically create one reader for the entire lifetime of your
+    // program, and acquire a new searcher for every single request.
+    //
+    // In the code below, we rely on the 'ON_COMMIT' policy: the reader
+    // will reload the index automatically after each commit.
+    let reader = index
+        .reader_builder()
+        .reload_policy(ReloadPolicy::OnCommit)
+        .try_into()?;
 
     // We now need to acquire a searcher.
-    // Some search experience might require more than
-    // one query.
     //
-    // The searcher ensure that we get to work
-    // with a consistent version of the index.
+    // A searcher points to snapshotted, immutable version of the index.
+    //
+    // Some search experience might require more than
+    // one query. Using the same searcher ensures that all of these queries will run on the
+    // same version of the index.
     //
     // Acquiring a `searcher` is very cheap.
     //
-    // You should acquire a searcher every time you
-    // start processing a request and
+    // You should acquire a searcher every time you start processing a request and
     // and release it right after your query is finished.
-    let searcher = index.searcher();
+    let searcher = reader.searcher();
 
     // ### Query
 
@@ -224,7 +234,6 @@ fn main() -> tantivy::Result<()> {
     // Since the body field was not configured as stored,
     // the document returned will only contain
     // a title.
-
     for (_score, doc_address) in top_docs {
         let retrieved_doc = searcher.doc(doc_address)?;
         println!("{}", schema.to_json(&retrieved_doc));

examples/custom_collector.rs

@@ -17,7 +17,7 @@ use tantivy::collector::{Collector, SegmentCollector};
 use tantivy::fastfield::FastFieldReader;
 use tantivy::query::QueryParser;
 use tantivy::schema::Field;
-use tantivy::schema::{Schema, FAST, INT_INDEXED, TEXT};
+use tantivy::schema::{Schema, FAST, INDEXED, TEXT};
 use tantivy::Index;
 use tantivy::SegmentReader;
 
@@ -137,7 +137,7 @@ fn main() -> tantivy::Result<()> {
     // products, and with a name, a description, and a price.
     let product_name = schema_builder.add_text_field("name", TEXT);
     let product_description = schema_builder.add_text_field("description", TEXT);
-    let price = schema_builder.add_u64_field("price", INT_INDEXED | FAST);
+    let price = schema_builder.add_u64_field("price", INDEXED | FAST);
     let schema = schema_builder.build();
 
     // # Indexing documents
@@ -170,9 +170,9 @@ fn main() -> tantivy::Result<()> {
         price => 5_200u64
     ));
     index_writer.commit()?;
-    index.load_searchers()?;
 
-    let searcher = index.searcher();
+    let reader = index.reader()?;
+    let searcher = reader.searcher();
     let query_parser = QueryParser::for_index(&index, vec![product_name, product_description]);
 
     // here we want to get a hit on the 'ken' in Frankenstein

examples/custom_tokenizer.rs

@@ -91,9 +91,9 @@ fn main() -> tantivy::Result<()> {
                 increasing confidence in the success of my undertaking."#
     ));
     index_writer.commit()?;
-    index.load_searchers()?;
 
-    let searcher = index.searcher();
+    let reader = index.reader()?;
+    let searcher = reader.searcher();
 
     // The query parser can interpret human queries.
     // Here, if the user does not specify which

examples/deleting_updating_documents.rs

@@ -14,12 +14,16 @@ use tantivy::collector::TopDocs;
 use tantivy::query::TermQuery;
 use tantivy::schema::*;
 use tantivy::Index;
+use tantivy::IndexReader;
 
 // A simple helper function to fetch a single document
 // given its id from our index.
 // It will be helpful to check our work.
-fn extract_doc_given_isbn(index: &Index, isbn_term: &Term) -> tantivy::Result<Option<Document>> {
-    let searcher = index.searcher();
+fn extract_doc_given_isbn(
+    reader: &IndexReader,
+    isbn_term: &Term,
+) -> tantivy::Result<Option<Document>> {
+    let searcher = reader.searcher();
 
     // This is the simplest query you can think of.
     // It matches all of the documents containing a specific term.
@@ -85,12 +89,12 @@ fn main() -> tantivy::Result<()> {
        isbn => "978-9176370711",
     ));
     index_writer.commit()?;
-    index.load_searchers()?;
+    let reader = index.reader()?;
 
     let frankenstein_isbn = Term::from_field_text(isbn, "978-9176370711");
 
     // Oops our frankenstein doc seems mispelled
-    let frankenstein_doc_misspelled = extract_doc_given_isbn(&index, &frankenstein_isbn)?.unwrap();
+    let frankenstein_doc_misspelled = extract_doc_given_isbn(&reader, &frankenstein_isbn)?.unwrap();
     assert_eq!(
         schema.to_json(&frankenstein_doc_misspelled),
         r#"{"isbn":["978-9176370711"],"title":["Frankentein"]}"#,
@@ -129,10 +133,10 @@ fn main() -> tantivy::Result<()> {
     // Everything happened as if the document was updated.
     index_writer.commit()?;
     // We reload our searcher to make our change available to clients.
-    index.load_searchers()?;
+    reader.reload()?;
 
     // No more typo!
-    let frankenstein_new_doc = extract_doc_given_isbn(&index, &frankenstein_isbn)?.unwrap();
+    let frankenstein_new_doc = extract_doc_given_isbn(&reader, &frankenstein_isbn)?.unwrap();
     assert_eq!(
         schema.to_json(&frankenstein_new_doc),
         r#"{"isbn":["978-9176370711"],"title":["Frankenstein"]}"#,

examples/faceted_search.rs

@@ -55,9 +55,9 @@ fn main() -> tantivy::Result<()> {
 
     index_writer.commit()?;
 
-    index.load_searchers()?;
+    let reader = index.reader()?;
 
-    let searcher = index.searcher();
+    let searcher = reader.searcher();
 
     let mut facet_collector = FacetCollector::for_field(tags);
     facet_collector.add_facet("/pools");

examples/integer_range_search.rs

@@ -0,0 +1,43 @@
+// # Searching a range on an indexed int field.
+//
+// Below is an example of creating an indexed integer field in your schema
+// You can use RangeQuery to get a Count of all occurrences in a given range.
+
+#[macro_use]
+extern crate tantivy;
+use tantivy::collector::Count;
+use tantivy::query::RangeQuery;
+use tantivy::schema::{Schema, INDEXED};
+use tantivy::Index;
+use tantivy::Result;
+
+fn run() -> Result<()> {
+    // For the sake of simplicity, this schema will only have 1 field
+    let mut schema_builder = Schema::builder();
+
+    // `INDEXED` is a short-hand to indicate that our field should be "searchable".
+    let year_field = schema_builder.add_u64_field("year", INDEXED);
+    let schema = schema_builder.build();
+    let index = Index::create_in_ram(schema);
+    let reader = index.reader()?;
+    {
+        let mut index_writer = index.writer_with_num_threads(1, 6_000_000)?;
+        for year in 1950u64..2019u64 {
+            index_writer.add_document(doc!(year_field => year));
+        }
+        index_writer.commit()?;
+        // The index will be a range of years
+    }
+    reader.reload()?;
+    let searcher = reader.searcher();
+    // The end is excluded i.e. here we are searching up to 1969
+    let docs_in_the_sixties = RangeQuery::new_u64(year_field, 1960..1970);
+    // Uses a Count collector to sum the total number of docs in the range
+    let num_60s_books = searcher.search(&docs_in_the_sixties, &Count)?;
+    assert_eq!(num_60s_books, 10);
+    Ok(())
+}
+
+fn main() {
+    run().unwrap()
+}

examples/iterating_docs_and_positions.rs

@@ -33,9 +33,9 @@ fn main() -> tantivy::Result<()> {
     index_writer.add_document(doc!(title => "The modern Promotheus"));
     index_writer.commit()?;
 
-    index.load_searchers()?;
+    let reader = index.reader()?;
 
-    let searcher = index.searcher();
+    let searcher = reader.searcher();
 
     // A tantivy index is actually a collection of segments.
     // Similarly, a searcher just wraps a list `segment_reader`.

examples/snippet.rs

@@ -48,9 +48,8 @@ fn main() -> tantivy::Result<()> {
     // ...
     index_writer.commit()?;
 
-    index.load_searchers()?;
-
-    let searcher = index.searcher();
+    let reader = index.reader()?;
+    let searcher = reader.searcher();
     let query_parser = QueryParser::for_index(&index, vec![title, body]);
     let query = query_parser.parse_query("sycamore spring")?;
 

examples/stop_words.rs

@@ -96,9 +96,9 @@ fn main() -> tantivy::Result<()> {
 
     index_writer.commit()?;
 
-    index.load_searchers()?;
+    let reader = index.reader()?;
 
-    let searcher = index.searcher();
+    let searcher = reader.searcher();
 
     let query_parser = QueryParser::for_index(&index, vec![title, body]);
 

examples/working_with_json.rs

@@ -12,7 +12,7 @@ fn main() -> tantivy::Result<()> {
     let mut schema_builder = Schema::builder();
     schema_builder.add_text_field("title", TEXT | STORED);
     schema_builder.add_text_field("body", TEXT);
-    schema_builder.add_u64_field("year", INT_INDEXED);
+    schema_builder.add_u64_field("year", INDEXED);
     let schema = schema_builder.build();
 
     // Let's assume we have a json-serialized document.

src/collector/count_collector.rs

@@ -40,8 +40,8 @@ use SegmentReader;
 ///         index_writer.commit().unwrap();
 ///     }
 ///
-///     index.load_searchers()?;
-///     let searcher = index.searcher();
+///     let reader = index.reader()?;
+///     let searcher = reader.searcher();
 ///
 ///     {
 ///         let query_parser = QueryParser::for_index(&index, vec![title]);

src/collector/facet_collector.rs

@@ -122,17 +122,16 @@ fn facet_depth(facet_bytes: &[u8]) -> usize {
 ///             facet => Facet::from("/lang/en"),
 ///             facet => Facet::from("/category/biography")
 ///         ));
-///         index_writer.commit().unwrap();
+///         index_writer.commit()?;
 ///     }
-///
-///     index.load_searchers()?;
-///     let searcher = index.searcher();
+///     let reader = index.reader()?;
+///     let searcher = reader.searcher();
 ///
 ///     {
 ///			let mut facet_collector = FacetCollector::for_field(facet);
 ///         facet_collector.add_facet("/lang");
 ///         facet_collector.add_facet("/category");
-///         let facet_counts = searcher.search(&AllQuery, &facet_collector).unwrap();
+///         let facet_counts = searcher.search(&AllQuery, &facet_collector)?;
 ///
 ///         // This lists all of the facet counts
 ///         let facets: Vec<(&Facet, u64)> = facet_counts
@@ -147,7 +146,7 @@ fn facet_depth(facet_bytes: &[u8]) -> usize {
 ///     {
 ///			let mut facet_collector = FacetCollector::for_field(facet);
 ///         facet_collector.add_facet("/category/fiction");
-///         let facet_counts = searcher.search(&AllQuery, &facet_collector).unwrap();
+///         let facet_counts = searcher.search(&AllQuery, &facet_collector)?;
 ///
 ///         // This lists all of the facet counts
 ///         let facets: Vec<(&Facet, u64)> = facet_counts
@@ -163,7 +162,7 @@ fn facet_depth(facet_bytes: &[u8]) -> usize {
 ///    {
 ///			let mut facet_collector = FacetCollector::for_field(facet);
 ///         facet_collector.add_facet("/category/fiction");
-///         let facet_counts = searcher.search(&AllQuery, &facet_collector).unwrap();
+///         let facet_counts = searcher.search(&AllQuery, &facet_collector)?;
 ///
 ///         // This lists all of the facet counts
 ///         let facets: Vec<(&Facet, u64)> = facet_counts.top_k("/category/fiction", 1);
@@ -483,8 +482,8 @@ mod tests {
             index_writer.add_document(doc);
         }
         index_writer.commit().unwrap();
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        let reader = index.reader().unwrap();
+        let searcher = reader.searcher();
         let mut facet_collector = FacetCollector::for_field(facet_field);
         facet_collector.add_facet(Facet::from("/top1"));
         let counts = searcher.search(&AllQuery, &facet_collector).unwrap();
@@ -532,8 +531,8 @@ mod tests {
             facet_field => Facet::from_text(&"/subjects/B/b"),
         ));
         index_writer.commit().unwrap();
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        let reader = index.reader().unwrap();
+        let searcher = reader.searcher();
         assert_eq!(searcher.num_docs(), 1);
         let mut facet_collector = FacetCollector::for_field(facet_field);
         facet_collector.add_facet("/subjects");
@@ -579,9 +578,7 @@ mod tests {
             index_writer.add_document(doc);
         }
         index_writer.commit().unwrap();
-        index.load_searchers().unwrap();
-
-        let searcher = index.searcher();
+        let searcher = index.reader().unwrap().searcher();
 
         let mut facet_collector = FacetCollector::for_field(facet_field);
         facet_collector.add_facet("/facet");
@@ -635,8 +632,7 @@ mod bench {
             index_writer.add_document(doc);
         }
         index_writer.commit().unwrap();
-        index.load_searchers().unwrap();
-
+        let reader = index.reader().unwrap();
         b.iter(|| {
             let searcher = index.searcher();
             let facet_collector = FacetCollector::for_field(facet_field);

src/collector/int_facet_collector.rs

@@ -88,7 +88,7 @@ mod tests {
         let index = Index::create_in_ram(schema.clone());
 
         {
-            let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
+            let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
             {
                 for i in 0u64..10u64 {
                     index_writer.add_document(doc!(
@@ -101,8 +101,7 @@ mod tests {
             assert_eq!(index_writer.commit().unwrap(), 10u64);
         }
 
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        let searcher = index.reader().searcher();
         let mut ffvf_i64: IntFacetCollector<I64FastFieldReader> = IntFacetCollector::new(num_field_i64);
         let mut ffvf_u64: IntFacetCollector<U64FastFieldReader> = IntFacetCollector::new(num_field_u64);
 

src/collector/mod.rs

@@ -53,9 +53,9 @@ use tantivy::collector::{Count, TopDocs};
 #     index_writer.add_document(doc!(
 #        title => "The Diary of Muadib",
 #     ));
-#     index_writer.commit().unwrap();
-#     index.load_searchers()?;
-#     let searcher = index.searcher();
+#     index_writer.commit()?;
+#     let reader = index.reader()?;
+#     let searcher = reader.searcher();
 #     let query_parser = QueryParser::for_index(&index, vec![title]);
 #     let query = query_parser.parse_query("diary")?;
 let (doc_count, top_docs): (usize, Vec<(Score, DocAddress)>) =
@@ -85,7 +85,7 @@ See the `custom_collector` example.
 
 */
 
-use downcast;
+use downcast_rs;
 use DocId;
 use Result;
 use Score;
@@ -111,9 +111,9 @@ pub use self::facet_collector::FacetCollector;
 
 /// `Fruit` is the type for the result of our collection.
 /// e.g. `usize` for the `Count` collector.
-pub trait Fruit: Send + downcast::Any {}
+pub trait Fruit: Send + downcast_rs::Downcast {}
 
-impl<T> Fruit for T where T: Send + downcast::Any {}
+impl<T> Fruit for T where T: Send + downcast_rs::Downcast {}
 
 /// Collectors are in charge of collecting and retaining relevant
 /// information from the document found and scored by the query.
@@ -358,10 +358,7 @@ where
     }
 }
 
-#[allow(missing_docs)]
-mod downcast_impl {
-    downcast!(super::Fruit);
-}
+impl_downcast!(Fruit);
 
 #[cfg(test)]
 pub mod tests;

src/collector/multi_collector.rs

@@ -1,7 +1,6 @@
 use super::Collector;
 use super::SegmentCollector;
 use collector::Fruit;
-use downcast::Downcast;
 use std::marker::PhantomData;
 use DocId;
 use Result;
@@ -37,11 +36,11 @@ impl<TCollector: Collector> Collector for CollectorWrapper<TCollector> {
         let typed_fruit: Vec<TCollector::Fruit> = children
             .into_iter()
             .map(|untyped_fruit| {
-                Downcast::<TCollector::Fruit>::downcast(untyped_fruit)
+                untyped_fruit
+                    .downcast::<TCollector::Fruit>()
                     .map(|boxed_but_typed| *boxed_but_typed)
-                    .map_err(|e| {
-                        let err_msg = format!("Failed to cast child collector fruit. {:?}", e);
-                        TantivyError::InvalidArgument(err_msg)
+                    .map_err(|_| {
+                        TantivyError::InvalidArgument("Failed to cast child fruit.".to_string())
                     })
             })
             .collect::<Result<_>>()?;
@@ -89,7 +88,10 @@ pub struct FruitHandle<TFruit: Fruit> {
 impl<TFruit: Fruit> FruitHandle<TFruit> {
     pub fn extract(self, fruits: &mut MultiFruit) -> TFruit {
         let boxed_fruit = fruits.sub_fruits[self.pos].take().expect("");
-        *Downcast::<TFruit>::downcast(boxed_fruit).expect("Failed")
+        *boxed_fruit
+            .downcast::<TFruit>()
+            .map_err(|_| ())
+            .expect("Failed to downcast collector fruit.")
     }
 }
 
@@ -132,8 +134,8 @@ impl<TFruit: Fruit> FruitHandle<TFruit> {
 ///         index_writer.commit().unwrap();
 ///     }
 ///
-///     index.load_searchers()?;
-///     let searcher = index.searcher();
+///     let reader = index.reader()?;
+///     let searcher = reader.searcher();
 ///
 ///     let mut collectors = MultiCollector::new();
 ///     let top_docs_handle = collectors.add_collector(TopDocs::with_limit(2));
@@ -276,8 +278,7 @@ mod tests {
             index_writer.add_document(doc!(text=>"abc"));
             index_writer.commit().unwrap();
         }
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        let searcher = index.reader().unwrap().searcher();
         let term = Term::from_field_text(text, "abc");
         let query = TermQuery::new(term, IndexRecordOption::Basic);
 

src/collector/top_field_collector.rs

@@ -23,15 +23,16 @@ use SegmentReader;
 /// # use tantivy::schema::{Schema, Field, FAST, TEXT};
 /// # use tantivy::{Index, Result, DocAddress};
 /// # use tantivy::query::{Query, QueryParser};
+/// use tantivy::Searcher;
 /// use tantivy::collector::TopDocs;
 ///
-/// # fn main() {
+/// # fn main() -> tantivy::Result<()> {
 /// #   let mut schema_builder = Schema::builder();
 /// #   let title = schema_builder.add_text_field("title", TEXT);
 /// #   let rating = schema_builder.add_u64_field("rating", FAST);
 /// #   let schema = schema_builder.build();
 /// #   let index = Index::create_in_ram(schema);
-/// #   let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
+/// #   let mut index_writer = index.writer_with_num_threads(1, 3_000_000)?;
 /// #   index_writer.add_document(doc!(
 /// #       title => "The Name of the Wind",
 /// #       rating => 92u64,
@@ -39,13 +40,14 @@ use SegmentReader;
 /// #   index_writer.add_document(doc!(title => "The Diary of Muadib", rating => 97u64));
 /// #   index_writer.add_document(doc!(title => "A Dairy Cow", rating => 63u64));
 /// #   index_writer.add_document(doc!(title => "The Diary of a Young Girl", rating => 80u64));
-/// #   index_writer.commit().unwrap();
-/// #   index.load_searchers().unwrap();
-///	#   let query = QueryParser::for_index(&index, vec![title]).parse_query("diary").unwrap();
-/// #   let top_docs = docs_sorted_by_rating(&index, &query, rating).unwrap();
+/// #   index_writer.commit()?;
+/// #   let reader = index.reader()?;
+/// #   let query = QueryParser::for_index(&index, vec![title]).parse_query("diary")?;
+/// #   let top_docs = docs_sorted_by_rating(&reader.searcher(), &query, rating)?;
 /// #   assert_eq!(top_docs,
 /// #            vec![(97u64, DocAddress(0u32, 1)),
 /// #                 (80u64, DocAddress(0u32, 3))]);
+/// #   Ok(())
 /// # }
 /// #
 /// /// Searches the document matching the given query, and
@@ -53,7 +55,9 @@ use SegmentReader;
 /// /// given in argument.
 /// ///
 /// /// `field` is required to be a FAST field.
-/// fn docs_sorted_by_rating(index: &Index, query: &Query, sort_by_field: Field)
+/// fn docs_sorted_by_rating(searcher: &Searcher,
+///                          query: &Query,
+///                          sort_by_field: Field)
 ///     -> Result<Vec<(u64, DocAddress)>> {
 ///
 ///     // This is where we build our collector!
@@ -61,8 +65,7 @@ use SegmentReader;
 ///
 ///     // ... and here is our documents. Not this is a simple vec.
 ///     // The `u64` in the pair is the value of our fast field for each documents.
-///     index.searcher()
-///          .search(query, &top_docs_by_rating)
+///     searcher.search(query, &top_docs_by_rating)
 /// }
 /// ```
 pub struct TopDocsByField<T> {
@@ -76,6 +79,12 @@ impl<T: FastValue + PartialOrd + Clone> TopDocsByField<T> {
     /// The given field name must be a fast field, otherwise the collector have an error while
     /// collecting results.
     ///
+    /// This constructor is crate-private. Client are supposed to call
+    /// build `TopDocsByField`  object using the `TopDocs` API.
+    ///
+    /// e.g.:
+    ///   `TopDocs::with_limit(2).order_by_field(sort_by_field)`
+    ///
     /// # Panics
     /// The method panics if limit is 0
     pub(crate) fn new(field: Field, limit: usize) -> TopDocsByField<T> {
@@ -171,7 +180,7 @@ mod tests {
                 size => 16u64,
             ));
         });
-        let searcher = index.searcher();
+        let searcher = index.reader().unwrap().searcher();
 
         let top_collector = TopDocs::with_limit(4).order_by_field(size);
         let top_docs: Vec<(u64, DocAddress)> = searcher.search(&query, &top_collector).unwrap();
@@ -198,7 +207,7 @@ mod tests {
                 size => 12u64,
             ));
         });
-        let searcher = index.searcher();
+        let searcher = index.reader().unwrap().searcher();
         let top_collector: TopDocsByField<u64> = TopDocs::with_limit(4).order_by_field(Field(2));
         let segment_reader = searcher.segment_reader(0u32);
         top_collector
@@ -218,7 +227,7 @@ mod tests {
                 size => 12u64,
             ));
         });
-        let searcher = index.searcher();
+        let searcher = index.reader().unwrap().searcher();
         let segment = searcher.segment_reader(0);
         let top_collector: TopDocsByField<u64> = TopDocs::with_limit(4).order_by_field(size);
         assert_matches!(
@@ -241,8 +250,6 @@ mod tests {
         let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
         doc_adder(&mut index_writer);
         index_writer.commit().unwrap();
-        index.load_searchers().unwrap();
-
         let query_parser = QueryParser::for_index(&index, vec![query_field]);
         let query = query_parser.parse_query(query).unwrap();
         (index, query)

src/collector/top_score_collector.rs

@@ -51,8 +51,8 @@ use SegmentReader;
 ///         index_writer.commit().unwrap();
 ///     }
 ///
-///     index.load_searchers()?;
-///     let searcher = index.searcher();
+///     let reader = index.reader()?;
+///     let searcher = reader.searcher();
 ///
 ///     let query_parser = QueryParser::for_index(&index, vec![title]);
 ///     let query = query_parser.parse_query("diary")?;
@@ -142,13 +142,12 @@ mod tests {
         let index = Index::create_in_ram(schema);
         {
             // writing the segment
-            let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
+            let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
             index_writer.add_document(doc!(text_field=>"Hello happy tax payer."));
             index_writer.add_document(doc!(text_field=>"Droopy says hello happy tax payer"));
             index_writer.add_document(doc!(text_field=>"I like Droopy"));
             assert!(index_writer.commit().is_ok());
         }
-        index.load_searchers().unwrap();
         index
     }
 
@@ -159,6 +158,8 @@ mod tests {
         let query_parser = QueryParser::for_index(&index, vec![field]);
         let text_query = query_parser.parse_query("droopy tax").unwrap();
         let score_docs: Vec<(Score, DocAddress)> = index
+            .reader()
+            .unwrap()
             .searcher()
             .search(&text_query, &TopDocs::with_limit(4))
             .unwrap();
@@ -179,6 +180,8 @@ mod tests {
         let query_parser = QueryParser::for_index(&index, vec![field]);
         let text_query = query_parser.parse_query("droopy tax").unwrap();
         let score_docs: Vec<(Score, DocAddress)> = index
+            .reader()
+            .unwrap()
             .searcher()
             .search(&text_query, &TopDocs::with_limit(2))
             .unwrap();

src/common/bitpacker.rs

@@ -64,7 +64,7 @@ pub struct BitUnpacker<Data>
 where
     Data: Deref<Target = [u8]>,
 {
-    num_bits: usize,
+    num_bits: u64,
     mask: u64,
     data: Data,
 }
@@ -80,13 +80,13 @@ where
             (1u64 << num_bits) - 1u64
         };
         BitUnpacker {
-            num_bits: num_bits as usize,
+            num_bits: u64::from(num_bits),
             mask,
             data,
         }
     }
 
-    pub fn get(&self, idx: usize) -> u64 {
+    pub fn get(&self, idx: u64) -> u64 {
         if self.num_bits == 0 {
             return 0u64;
         }
@@ -97,10 +97,10 @@ where
         let addr = addr_in_bits >> 3;
         let bit_shift = addr_in_bits & 7;
         debug_assert!(
-            addr + 8 <= data.len(),
+            addr + 8 <= data.len() as u64,
             "The fast field field should have been padded with 7 bytes."
         );
-        let val_unshifted_unmasked: u64 = LittleEndian::read_u64(&data[addr..]);
+        let val_unshifted_unmasked: u64 = LittleEndian::read_u64(&data[(addr as usize)..]);
         let val_shifted = (val_unshifted_unmasked >> bit_shift) as u64;
         val_shifted & mask
     }
@@ -129,7 +129,7 @@ mod test {
     fn test_bitpacker_util(len: usize, num_bits: u8) {
         let (bitunpacker, vals) = create_fastfield_bitpacker(len, num_bits);
         for (i, val) in vals.iter().enumerate() {
-            assert_eq!(bitunpacker.get(i), *val);
+            assert_eq!(bitunpacker.get(i as u64), *val);
         }
     }
 

src/common/composite_file.rs

@@ -39,7 +39,7 @@ impl BinarySerializable for FileAddr {
 /// A `CompositeWrite` is used to write a `CompositeFile`.
 pub struct CompositeWrite<W = WritePtr> {
     write: CountingWriter<W>,
-    offsets: HashMap<FileAddr, usize>,
+    offsets: HashMap<FileAddr, u64>,
 }
 
 impl<W: Write> CompositeWrite<W> {

src/common/counting_writer.rs

@@ -3,7 +3,7 @@ use std::io::Write;
 
 pub struct CountingWriter<W> {
     underlying: W,
-    written_bytes: usize,
+    written_bytes: u64,
 }
 
 impl<W: Write> CountingWriter<W> {
@@ -14,11 +14,11 @@ impl<W: Write> CountingWriter<W> {
         }
     }
 
-    pub fn written_bytes(&self) -> usize {
+    pub fn written_bytes(&self) -> u64 {
         self.written_bytes
     }
 
-    pub fn finish(mut self) -> io::Result<(W, usize)> {
+    pub fn finish(mut self) -> io::Result<(W, u64)> {
         self.flush()?;
         Ok((self.underlying, self.written_bytes))
     }
@@ -27,10 +27,16 @@ impl<W: Write> CountingWriter<W> {
 impl<W: Write> Write for CountingWriter<W> {
     fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
         let written_size = self.underlying.write(buf)?;
-        self.written_bytes += written_size;
+        self.written_bytes += written_size as u64;
         Ok(written_size)
     }
 
+    fn write_all(&mut self, buf: &[u8]) -> io::Result<()> {
+        self.underlying.write_all(buf)?;
+        self.written_bytes += buf.len() as u64;
+        Ok(())
+    }
+
     fn flush(&mut self) -> io::Result<()> {
         self.underlying.flush()
     }
@@ -48,8 +54,8 @@ mod test {
         let mut counting_writer = CountingWriter::wrap(buffer);
         let bytes = (0u8..10u8).collect::<Vec<u8>>();
         counting_writer.write_all(&bytes).unwrap();
-        let (w, len): (Vec<u8>, usize) = counting_writer.finish().unwrap();
-        assert_eq!(len, 10);
+        let (w, len): (Vec<u8>, u64) = counting_writer.finish().unwrap();
+        assert_eq!(len, 10u64);
         assert_eq!(w.len(), 10);
     }
 }

src/common/vint.rs

@@ -69,8 +69,8 @@ pub fn serialize_vint_u32(val: u32) -> (u64, usize) {
 /// # May Panic
 /// If the payload does not start by a valid `vint`
 fn vint_len(data: &[u8]) -> usize {
-    for i in 0..5.min(data.len()) {
-        if data[i] >= STOP_BIT {
+    for (i, &val) in data.iter().enumerate().take(5) {
+        if val >= STOP_BIT {
             return i + 1;
         }
     }

src/core/executor.rs

@@ -123,15 +123,14 @@ mod tests {
         }
     }
 
-}
-
-#[test]
-fn test_map_multithread() {
-    let result: Vec<usize> = Executor::multi_thread(3, "search-test")
-        .map(|i| Ok(i * 2), 0..10)
-        .unwrap();
-    assert_eq!(result.len(), 10);
-    for i in 0..10 {
-        assert_eq!(result[i], i * 2);
+    #[test]
+    fn test_map_multithread() {
+        let result: Vec<usize> = Executor::multi_thread(3, "search-test")
+            .map(|i| Ok(i * 2), 0..10)
+            .unwrap();
+        assert_eq!(result.len(), 10);
+        for i in 0..10 {
+            assert_eq!(result[i], i * 2);
+        }
     }
 }

src/core/index.rs

@@ -1,25 +1,23 @@
-use super::pool::LeasedItem;
-use super::pool::Pool;
 use super::segment::create_segment;
 use super::segment::Segment;
-use core::searcher::Searcher;
 use core::Executor;
 use core::IndexMeta;
 use core::SegmentId;
 use core::SegmentMeta;
-use core::SegmentReader;
 use core::META_FILEPATH;
 use directory::ManagedDirectory;
 #[cfg(feature = "mmap")]
 use directory::MmapDirectory;
+use directory::INDEX_WRITER_LOCK;
 use directory::{Directory, RAMDirectory};
 use error::DataCorruption;
 use error::TantivyError;
 use indexer::index_writer::open_index_writer;
 use indexer::index_writer::HEAP_SIZE_MIN;
 use indexer::segment_updater::save_new_metas;
-use indexer::LockType;
 use num_cpus;
+use reader::IndexReader;
+use reader::IndexReaderBuilder;
 use schema::Field;
 use schema::FieldType;
 use schema::Schema;
@@ -27,7 +25,6 @@ use serde_json;
 use std::borrow::BorrowMut;
 use std::fmt;
 use std::path::Path;
-use std::sync::atomic::{AtomicUsize, Ordering};
 use std::sync::Arc;
 use tokenizer::BoxedTokenizer;
 use tokenizer::TokenizerManager;
@@ -48,11 +45,10 @@ fn load_metas(directory: &Directory) -> Result<IndexMeta> {
 }
 
 /// Search Index
+#[derive(Clone)]
 pub struct Index {
     directory: ManagedDirectory,
     schema: Schema,
-    num_searchers: Arc<AtomicUsize>,
-    searcher_pool: Arc<Pool<Searcher>>,
     executor: Arc<Executor>,
     tokenizers: TokenizerManager,
 }
@@ -110,7 +106,6 @@ impl Index {
     }
 
     /// Opens or creates a new index in the provided directory
-    #[cfg(feature = "mmap")]
     pub fn open_or_create<Dir: Directory>(dir: Dir, schema: Schema) -> Result<Index> {
         if Index::exists(&dir) {
             let index = Index::open(dir)?;
@@ -150,7 +145,7 @@ impl Index {
     ///
     /// This will overwrite existing meta.json
     fn from_directory(mut directory: ManagedDirectory, schema: Schema) -> Result<Index> {
-        save_new_metas(schema.clone(), 0, directory.borrow_mut())?;
+        save_new_metas(schema.clone(), directory.borrow_mut())?;
         let metas = IndexMeta::with_schema(schema);
         Index::create_from_metas(directory, &metas)
     }
@@ -158,16 +153,12 @@ impl Index {
     /// Creates a new index given a directory and an `IndexMeta`.
     fn create_from_metas(directory: ManagedDirectory, metas: &IndexMeta) -> Result<Index> {
         let schema = metas.schema.clone();
-        let n_cpus = num_cpus::get();
         let index = Index {
             directory,
             schema,
-            num_searchers: Arc::new(AtomicUsize::new(n_cpus)),
-            searcher_pool: Arc::new(Pool::new()),
             tokenizers: TokenizerManager::default(),
             executor: Arc::new(Executor::single_thread()),
         };
-        index.load_searchers()?;
         Ok(index)
     }
 
@@ -197,6 +188,22 @@ impl Index {
         }
     }
 
+    /// Create a default `IndexReader` for the given index.
+    ///
+    /// See [`Index.reader_builder()`](#method.reader_builder).
+    pub fn reader(&self) -> Result<IndexReader> {
+        self.reader_builder().try_into()
+    }
+
+    /// Create a `IndexReader` for the given index.
+    ///
+    /// Most project should create at most one reader for a given index.
+    /// This method is typically called only once per `Index` instance,
+    /// over the lifetime of most problem.
+    pub fn reader_builder(&self) -> IndexReaderBuilder {
+        IndexReaderBuilder::new(self.clone())
+    }
+
     /// Opens a new directory from an index path.
     #[cfg(feature = "mmap")]
     pub fn open_in_dir<P: AsRef<Path>>(directory_path: P) -> Result<Index> {
@@ -232,7 +239,8 @@ impl Index {
     /// Each thread will receive a budget of  `overall_heap_size_in_bytes / num_threads`.
     ///
     /// # Errors
-    /// If the lockfile already exists, returns `Error::FileAlreadyExists`.
+    /// If the lockfile already exists, returns `Error::DirectoryLockBusy` or an `Error::IOError`.
+    ///
     /// # Panics
     /// If the heap size per thread is too small, panics.
     pub fn writer_with_num_threads(
@@ -240,7 +248,21 @@ impl Index {
         num_threads: usize,
         overall_heap_size_in_bytes: usize,
     ) -> Result<IndexWriter> {
-        let directory_lock = LockType::IndexWriterLock.acquire_lock(&self.directory)?;
+        let directory_lock = self
+            .directory
+            .acquire_lock(&INDEX_WRITER_LOCK)
+            .map_err(|err| {
+                TantivyError::LockFailure(
+                    err,
+                    Some(
+                        "Failed to acquire index lock. If you are using\
+                         a regular directory, this means there is already an \
+                         `IndexWriter` working on this `Directory`, in this process \
+                         or in a different process."
+                            .to_string(),
+                    ),
+                )
+            })?;
         let heap_size_in_bytes_per_thread = overall_heap_size_in_bytes / num_threads;
         open_index_writer(
             self,
@@ -320,53 +342,6 @@ impl Index {
             .map(|segment_meta| segment_meta.id())
             .collect())
     }
-
-    /// Sets the number of searchers to use
-    ///
-    /// Only works after the next call to `load_searchers`
-    pub fn set_num_searchers(&mut self, num_searchers: usize) {
-        self.num_searchers.store(num_searchers, Ordering::Release);
-    }
-
-    /// Update searchers so that they reflect the state of the last
-    /// `.commit()`.
-    ///
-    /// If indexing happens in the same process as searching,
-    /// you most likely want to call `.load_searchers()` right after each
-    /// successful call to `.commit()`.
-    ///
-    /// If indexing and searching happen in different processes, the way to
-    /// get the freshest `index` at all time, is to watch `meta.json` and
-    /// call `load_searchers` whenever a changes happen.
-    pub fn load_searchers(&self) -> Result<()> {
-        let _meta_lock = LockType::MetaLock.acquire_lock(self.directory())?;
-        let searchable_segments = self.searchable_segments()?;
-        let segment_readers: Vec<SegmentReader> = searchable_segments
-            .iter()
-            .map(SegmentReader::open)
-            .collect::<Result<_>>()?;
-        let schema = self.schema();
-        let num_searchers: usize = self.num_searchers.load(Ordering::Acquire);
-        let searchers = (0..num_searchers)
-            .map(|_| Searcher::new(schema.clone(), self.clone(), segment_readers.clone()))
-            .collect();
-        self.searcher_pool.publish_new_generation(searchers);
-        Ok(())
-    }
-
-    /// Returns a searcher
-    ///
-    /// This method should be called every single time a search
-    /// query is performed.
-    /// The searchers are taken from a pool of `num_searchers` searchers.
-    /// If no searcher is available
-    /// this may block.
-    ///
-    /// The same searcher must be used for a given query, as it ensures
-    /// the use of a consistent segment set.
-    pub fn searcher(&self) -> LeasedItem<Searcher> {
-        self.searcher_pool.acquire()
-    }
 }
 
 impl fmt::Debug for Index {
@@ -375,29 +350,24 @@ impl fmt::Debug for Index {
     }
 }
 
-impl Clone for Index {
-    fn clone(&self) -> Index {
-        Index {
-            directory: self.directory.clone(),
-            schema: self.schema.clone(),
-            num_searchers: Arc::clone(&self.num_searchers),
-            searcher_pool: Arc::clone(&self.searcher_pool),
-            tokenizers: self.tokenizers.clone(),
-            executor: self.executor.clone(),
-        }
-    }
-}
-
 #[cfg(test)]
 mod tests {
     use directory::RAMDirectory;
-    use schema::{Schema, INT_INDEXED, TEXT};
+    use schema::Field;
+    use schema::{Schema, INDEXED, TEXT};
+    use std::path::PathBuf;
+    use std::thread;
+    use std::time::Duration;
+    use tempdir::TempDir;
     use Index;
+    use IndexReader;
+    use IndexWriter;
+    use ReloadPolicy;
 
     #[test]
     fn test_indexer_for_field() {
         let mut schema_builder = Schema::builder();
-        let num_likes_field = schema_builder.add_u64_field("num_likes", INT_INDEXED);
+        let num_likes_field = schema_builder.add_u64_field("num_likes", INDEXED);
         let body_field = schema_builder.add_text_field("body", TEXT);
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
@@ -455,7 +425,109 @@ mod tests {
 
     fn throw_away_schema() -> Schema {
         let mut schema_builder = Schema::builder();
-        let _ = schema_builder.add_u64_field("num_likes", INT_INDEXED);
+        let _ = schema_builder.add_u64_field("num_likes", INDEXED);
         schema_builder.build()
     }
+
+    #[test]
+    fn test_index_on_commit_reload_policy() {
+        let schema = throw_away_schema();
+        let field = schema.get_field("num_likes").unwrap();
+        let index = Index::create_in_ram(schema);
+        let reader = index
+            .reader_builder()
+            .reload_policy(ReloadPolicy::OnCommit)
+            .try_into()
+            .unwrap();
+        assert_eq!(reader.searcher().num_docs(), 0);
+        let mut writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
+        test_index_on_commit_reload_policy_aux(field, &mut writer, &reader);
+    }
+
+    #[test]
+    fn test_index_on_commit_reload_policy_mmap() {
+        let schema = throw_away_schema();
+        let field = schema.get_field("num_likes").unwrap();
+        let tempdir = TempDir::new("index").unwrap();
+        let tempdir_path = PathBuf::from(tempdir.path());
+        let index = Index::create_in_dir(&tempdir_path, schema).unwrap();
+        let mut writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
+        writer.commit().unwrap();
+        let reader = index
+            .reader_builder()
+            .reload_policy(ReloadPolicy::OnCommit)
+            .try_into()
+            .unwrap();
+        assert_eq!(reader.searcher().num_docs(), 0);
+        test_index_on_commit_reload_policy_aux(field, &mut writer, &reader);
+    }
+
+    #[test]
+    fn test_index_manual_policy_mmap() {
+        let schema = throw_away_schema();
+        let field = schema.get_field("num_likes").unwrap();
+        let index = Index::create_from_tempdir(schema).unwrap();
+        let mut writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
+        writer.commit().unwrap();
+        let reader = index
+            .reader_builder()
+            .reload_policy(ReloadPolicy::Manual)
+            .try_into()
+            .unwrap();
+        assert_eq!(reader.searcher().num_docs(), 0);
+        writer.add_document(doc!(field=>1u64));
+        writer.commit().unwrap();
+        thread::sleep(Duration::from_millis(500));
+        assert_eq!(reader.searcher().num_docs(), 0);
+        reader.reload().unwrap();
+        assert_eq!(reader.searcher().num_docs(), 1);
+    }
+
+    #[test]
+    fn test_index_on_commit_reload_policy_different_directories() {
+        let schema = throw_away_schema();
+        let field = schema.get_field("num_likes").unwrap();
+        let tempdir = TempDir::new("index").unwrap();
+        let tempdir_path = PathBuf::from(tempdir.path());
+        let write_index = Index::create_in_dir(&tempdir_path, schema).unwrap();
+        let read_index = Index::open_in_dir(&tempdir_path).unwrap();
+        let reader = read_index
+            .reader_builder()
+            .reload_policy(ReloadPolicy::OnCommit)
+            .try_into()
+            .unwrap();
+        assert_eq!(reader.searcher().num_docs(), 0);
+        let mut writer = write_index.writer_with_num_threads(1, 3_000_000).unwrap();
+        test_index_on_commit_reload_policy_aux(field, &mut writer, &reader);
+    }
+
+    fn test_index_on_commit_reload_policy_aux(
+        field: Field,
+        writer: &mut IndexWriter,
+        reader: &IndexReader,
+    ) {
+        assert_eq!(reader.searcher().num_docs(), 0);
+        writer.add_document(doc!(field=>1u64));
+        writer.commit().unwrap();
+        let mut count = 0;
+        for _ in 0..100 {
+            count = reader.searcher().num_docs();
+            if count > 0 {
+                break;
+            }
+            thread::sleep(Duration::from_millis(100));
+        }
+        assert_eq!(count, 1);
+        writer.add_document(doc!(field=>2u64));
+        writer.commit().unwrap();
+        let mut count = 0;
+        for _ in 0..10 {
+            count = reader.searcher().num_docs();
+            if count > 1 {
+                break;
+            }
+            thread::sleep(Duration::from_millis(100));
+        }
+        assert_eq!(count, 2);
+    }
 }

src/core/mod.rs

@@ -2,7 +2,6 @@ mod executor;
 pub mod index;
 mod index_meta;
 mod inverted_index_reader;
-mod pool;
 pub mod searcher;
 mod segment;
 mod segment_component;
@@ -25,6 +24,7 @@ pub use self::segment_reader::SegmentReader;
 use std::path::PathBuf;
 
 lazy_static! {
+
     /// The meta file contains all the information about the list of segments and the schema
     /// of the index.
     pub static ref META_FILEPATH: PathBuf = PathBuf::from("meta.json");

src/core/segment_component.rs

@@ -41,6 +41,6 @@ impl SegmentComponent {
             SegmentComponent::STORE,
             SegmentComponent::DELETE,
         ];
-        SEGMENT_COMPONENTS.into_iter()
+        SEGMENT_COMPONENTS.iter()
     }
 }

src/core/segment_id.rs

@@ -19,7 +19,7 @@ pub struct SegmentId(Uuid);
 #[cfg(test)]
 lazy_static! {
     static ref AUTO_INC_COUNTER: atomic::AtomicUsize = atomic::AtomicUsize::default();
-    static ref EMPTY_ARR: [u8; 8] = [0u8; 8];
+    static ref ZERO_ARRAY: [u8; 8] = [0u8; 8];
 }
 
 // During tests, we generate the segment id in a autoincrement manner
@@ -30,7 +30,7 @@ lazy_static! {
 #[cfg(test)]
 fn create_uuid() -> Uuid {
     let new_auto_inc_id = (*AUTO_INC_COUNTER).fetch_add(1, atomic::Ordering::SeqCst);
-    Uuid::from_fields(new_auto_inc_id as u32, 0, 0, &*EMPTY_ARR).unwrap()
+    Uuid::from_fields(new_auto_inc_id as u32, 0, 0, &*ZERO_ARRAY).unwrap()
 }
 
 #[cfg(not(test))]

src/core/segment_reader.rs

@@ -477,9 +477,7 @@ mod test {
             // ok, now we should have a deleted doc
             index_writer2.commit().unwrap();
         }
-
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        let searcher = index.reader().unwrap().searcher();
         let docs: Vec<DocId> = searcher.segment_reader(0).doc_ids_alive().collect();
         assert_eq!(vec![0u32, 2u32], docs);
     }

src/directory/directory.rs

@@ -1,11 +1,104 @@
+use directory::directory_lock::Lock;
+use directory::error::LockError;
 use directory::error::{DeleteError, OpenReadError, OpenWriteError};
+use directory::WatchCallback;
+use directory::WatchHandle;
 use directory::{ReadOnlySource, WritePtr};
 use std::fmt;
 use std::io;
+use std::io::Write;
 use std::marker::Send;
 use std::marker::Sync;
 use std::path::Path;
+use std::path::PathBuf;
 use std::result;
+use std::thread;
+use std::time::Duration;
+
+/// Retry the logic of acquiring locks is pretty simple.
+/// We just retry `n` times after a given `duratio`, both
+/// depending on the type of lock.
+struct RetryPolicy {
+    num_retries: usize,
+    wait_in_ms: u64,
+}
+
+impl RetryPolicy {
+    fn no_retry() -> RetryPolicy {
+        RetryPolicy {
+            num_retries: 0,
+            wait_in_ms: 0,
+        }
+    }
+
+    fn wait_and_retry(&mut self) -> bool {
+        if self.num_retries == 0 {
+            false
+        } else {
+            self.num_retries -= 1;
+            let wait_duration = Duration::from_millis(self.wait_in_ms);
+            thread::sleep(wait_duration);
+            true
+        }
+    }
+}
+
+/// The `DirectoryLock` is an object that represents a file lock.
+/// See  [`LockType`](struct.LockType.html)
+///
+/// It is transparently associated to a lock file, that gets deleted
+/// on `Drop.` The lock is released automatically on `Drop`.
+pub struct DirectoryLock(Box<Drop + Send + 'static>);
+
+struct DirectoryLockGuard {
+    directory: Box<Directory>,
+    path: PathBuf,
+}
+
+impl<T: Drop + Send + 'static> From<Box<T>> for DirectoryLock {
+    fn from(underlying: Box<T>) -> Self {
+        DirectoryLock(underlying)
+    }
+}
+
+impl Drop for DirectoryLockGuard {
+    fn drop(&mut self) {
+        if let Err(e) = self.directory.delete(&*self.path) {
+            error!("Failed to remove the lock file. {:?}", e);
+        }
+    }
+}
+
+enum TryAcquireLockError {
+    FileExists,
+    IOError(io::Error),
+}
+
+fn try_acquire_lock(
+    filepath: &Path,
+    directory: &mut Directory,
+) -> Result<DirectoryLock, TryAcquireLockError> {
+    let mut write = directory.open_write(filepath).map_err(|e| match e {
+        OpenWriteError::FileAlreadyExists(_) => TryAcquireLockError::FileExists,
+        OpenWriteError::IOError(io_error) => TryAcquireLockError::IOError(io_error.into()),
+    })?;
+    write.flush().map_err(TryAcquireLockError::IOError)?;
+    Ok(DirectoryLock::from(Box::new(DirectoryLockGuard {
+        directory: directory.box_clone(),
+        path: filepath.to_owned(),
+    })))
+}
+
+fn retry_policy(is_blocking: bool) -> RetryPolicy {
+    if is_blocking {
+        RetryPolicy {
+            num_retries: 100,
+            wait_in_ms: 100,
+        }
+    } else {
+        RetryPolicy::no_retry()
+    }
+}
 
 /// Write-once read many (WORM) abstraction for where
 /// tantivy's data should be stored.
@@ -73,6 +166,45 @@ pub trait Directory: DirectoryClone + fmt::Debug + Send + Sync + 'static {
     ///
     /// The file may or may not previously exist.
     fn atomic_write(&mut self, path: &Path, data: &[u8]) -> io::Result<()>;
+
+    /// Acquire a lock in the given directory.
+    ///
+    /// The method is blocking or not depending on the `Lock` object.
+    fn acquire_lock(&self, lock: &Lock) -> Result<DirectoryLock, LockError> {
+        let mut box_directory = self.box_clone();
+        let mut retry_policy = retry_policy(lock.is_blocking);
+        loop {
+            match try_acquire_lock(&lock.filepath, &mut *box_directory) {
+                Ok(result) => {
+                    return Ok(result);
+                }
+                Err(TryAcquireLockError::FileExists) => {
+                    if !retry_policy.wait_and_retry() {
+                        return Err(LockError::LockBusy);
+                    }
+                }
+                Err(TryAcquireLockError::IOError(io_error)) => {
+                    return Err(LockError::IOError(io_error));
+                }
+            }
+        }
+    }
+
+    /// Registers a callback that will be called whenever a change on the `meta.json`
+    /// using the `atomic_write` API is detected.
+    ///
+    /// The behavior when using `.watch()` on a file using `.open_write(...)` is, on the other
+    /// hand, undefined.
+    ///
+    /// The file will be watched for the lifetime of the returned `WatchHandle`. The caller is
+    /// required to keep it.
+    /// It does not override previous callbacks. When the file is modified, all callback that are
+    /// registered (and whose `WatchHandle` is still alive) are triggered.
+    ///
+    /// Internally, tantivy only uses this API to detect new commits to implement the
+    /// `OnCommit` `ReloadPolicy`. Not implementing watch in a `Directory` only prevents the
+    /// `OnCommit` `ReloadPolicy` to work properly.
+    fn watch(&self, watch_callback: WatchCallback) -> WatchHandle;
 }
 
 /// DirectoryClone

src/directory/directory_lock.rs

@@ -0,0 +1,56 @@
+use std::path::PathBuf;
+
+/// A directory lock.
+///
+/// A lock is associated to a specific path and some
+/// [`LockParams`](./enum.LockParams.html).
+/// Tantivy itself uses only two locks but client application
+/// can use the directory facility to define their own locks.
+/// - [INDEX_WRITER_LOCK](./struct.INDEX_WRITER_LOCK.html)
+/// - [META_LOCK](./struct.META_LOCK.html)
+///
+/// Check out these locks documentation for more information.
+///
+#[derive(Debug)]
+pub struct Lock {
+    /// The lock needs to be associated with its own file `path`.
+    /// Depending on the platform, the lock might rely on the creation
+    /// and deletion of this filepath.
+    pub filepath: PathBuf,
+    /// `lock_params` describes whether acquiring the lock is meant
+    /// to be a blocking operation or a non-blocking.
+    ///
+    /// Acquiring a blocking lock blocks until the lock is
+    /// available.
+    /// Acquiring a blocking lock returns rapidly, either successfully
+    /// or with an error signifying that someone is already holding
+    /// the lock.
+    pub is_blocking: bool,
+}
+
+lazy_static! {
+     /// Only one process should be able to write tantivy's index at a time.
+    /// This lock file, when present, is in charge of preventing other processes to open an IndexWriter.
+    ///
+    /// If the process is killed and this file remains, it is safe to remove it manually.
+    ///
+    /// Failing to acquire this lock usually means a misuse of tantivy's API,
+    /// (creating more than one instance of the `IndexWriter`), are a spurious
+    /// lock file remaining after a crash. In the latter case, removing the file after
+    /// checking no process running tantivy is running is safe.
+    pub static ref INDEX_WRITER_LOCK: Lock = Lock {
+        filepath: PathBuf::from(".tantivy-writer.lock"),
+        is_blocking: false
+    };
+    /// The meta lock file is here to protect the segment files being opened by
+    /// `IndexReader::reload()` from being garbage collected.
+    /// It makes it possible for another process to safely consume
+    /// our index in-writing. Ideally, we may have prefered `RWLock` semantics
+    /// here, but it is difficult to achieve on Windows.
+    ///
+    /// Opening segment readers is a very fast process.
+    pub static ref META_LOCK: Lock = Lock {
+        filepath: PathBuf::from(".tantivy-meta.lock"),
+        is_blocking: true
+    };
+}

src/directory/error.rs

@@ -3,6 +3,22 @@ use std::fmt;
 use std::io;
 use std::path::PathBuf;
 
+/// Error while trying to acquire a directory lock.
+#[derive(Debug, Fail)]
+pub enum LockError {
+    /// Failed to acquired a lock as it is already hold by another
+    /// client.
+    /// - In the context of a blocking lock, this means the lock was not released within some `timeout` period.
+    /// - In the context of a non-blocking lock, this means the lock was busy at the moment of the call.
+    #[fail(
+        display = "Could not acquire lock as it is already held, possibly by a different process."
+    )]
+    LockBusy,
+    /// Trying to acquire a lock failed with an `IOError`
+    #[fail(display = "Failed to acquire the lock due to an io:Error.")]
+    IOError(io::Error),
+}
+
 /// General IO error with an optional path to the offending file.
 #[derive(Debug)]
 pub struct IOError {
@@ -10,6 +26,12 @@ pub struct IOError {
     err: io::Error,
 }
 
+impl Into<io::Error> for IOError {
+    fn into(self) -> io::Error {
+        self.err
+    }
+}
+
 impl fmt::Display for IOError {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         match self.path {
@@ -51,6 +73,14 @@ pub enum OpenDirectoryError {
     DoesNotExist(PathBuf),
     /// The path exists but is not a directory.
     NotADirectory(PathBuf),
+    /// IoError
+    IoError(io::Error),
+}
+
+impl From<io::Error> for OpenDirectoryError {
+    fn from(io_err: io::Error) -> Self {
+        OpenDirectoryError::IoError(io_err)
+    }
 }
 
 impl fmt::Display for OpenDirectoryError {
@@ -62,6 +92,11 @@ impl fmt::Display for OpenDirectoryError {
             OpenDirectoryError::NotADirectory(ref path) => {
                 write!(f, "the path '{:?}' exists but is not a directory", path)
             }
+            OpenDirectoryError::IoError(ref err) => write!(
+                f,
+                "IOError while trying to open/create the directory. {:?}",
+                err
+            ),
         }
     }
 }

src/directory/managed_directory.rs

@@ -1,8 +1,11 @@
 use core::MANAGED_FILEPATH;
-use directory::error::{DeleteError, IOError, OpenReadError, OpenWriteError};
+use directory::error::{DeleteError, IOError, LockError, OpenReadError, OpenWriteError};
+use directory::DirectoryLock;
+use directory::Lock;
+use directory::META_LOCK;
 use directory::{ReadOnlySource, WritePtr};
+use directory::{WatchCallback, WatchHandle};
 use error::DataCorruption;
-use indexer::LockType;
 use serde_json;
 use std::collections::HashSet;
 use std::io;
@@ -92,6 +95,9 @@ impl ManagedDirectory {
     ///
     /// * `living_files` - List of files that are still used by the index.
     ///
+    /// The use a callback ensures that the list of living_files is computed
+    /// while we hold the lock on meta.
+    ///
     /// This method does not panick nor returns errors.
     /// If a file cannot be deleted (for permission reasons for instance)
     /// an error is simply logged, and the file remains in the list of managed
@@ -122,7 +128,7 @@ impl ManagedDirectory {
             // 2) writer change meta.json (for instance after a merge or a commit)
             // 3) gc kicks in.
             // 4) gc removes a file that was useful for process B, before process B opened it.
-            if let Ok(_meta_lock) = LockType::MetaLock.acquire_lock(self) {
+            if let Ok(_meta_lock) = self.acquire_lock(&META_LOCK) {
                 let living_files = get_living_files();
                 for managed_path in &meta_informations_rlock.managed_paths {
                     if !living_files.contains(managed_path) {
@@ -232,6 +238,14 @@ impl Directory for ManagedDirectory {
     fn exists(&self, path: &Path) -> bool {
         self.directory.exists(path)
     }
+
+    fn acquire_lock(&self, lock: &Lock) -> result::Result<DirectoryLock, LockError> {
+        self.directory.acquire_lock(lock)
+    }
+
+    fn watch(&self, watch_callback: WatchCallback) -> WatchHandle {
+        self.directory.watch(watch_callback)
+    }
 }
 
 impl Clone for ManagedDirectory {

src/directory/mmap_directory.rs

@@ -1,12 +1,25 @@
+extern crate fs2;
+extern crate notify;
+
+use self::fs2::FileExt;
+use self::notify::RawEvent;
+use self::notify::RecursiveMode;
+use self::notify::Watcher;
 use atomicwrites;
 use common::make_io_err;
+use core::META_FILEPATH;
+use directory::error::LockError;
 use directory::error::{DeleteError, IOError, OpenDirectoryError, OpenReadError, OpenWriteError};
-use directory::shared_vec_slice::SharedVecSlice;
+use directory::read_only_source::BoxedData;
 use directory::Directory;
+use directory::DirectoryLock;
+use directory::Lock;
 use directory::ReadOnlySource;
+use directory::WatchCallback;
+use directory::WatchCallbackList;
+use directory::WatchHandle;
 use directory::WritePtr;
-use fst::raw::MmapReadOnly;
-use std::collections::hash_map::Entry as HashMapEntry;
+use memmap::Mmap;
 use std::collections::HashMap;
 use std::convert::From;
 use std::fmt;
@@ -16,14 +29,17 @@ use std::io::{self, Seek, SeekFrom};
 use std::io::{BufWriter, Read, Write};
 use std::path::{Path, PathBuf};
 use std::result;
+use std::sync::mpsc::{channel, Receiver, Sender};
 use std::sync::Arc;
+use std::sync::Mutex;
 use std::sync::RwLock;
+use std::sync::Weak;
+use std::thread;
 use tempdir::TempDir;
 
 /// Returns None iff the file exists, can be read, but is empty (and hence
-/// cannot be mmapped).
-///
-fn open_mmap(full_path: &Path) -> result::Result<Option<MmapReadOnly>, OpenReadError> {
+/// cannot be mmapped)
+fn open_mmap(full_path: &Path) -> result::Result<Option<Mmap>, OpenReadError> {
     let file = File::open(full_path).map_err(|e| {
         if e.kind() == io::ErrorKind::NotFound {
             OpenReadError::FileDoesNotExist(full_path.to_owned())
@@ -42,7 +58,7 @@ fn open_mmap(full_path: &Path) -> result::Result<Option<MmapReadOnly>, OpenReadE
         return Ok(None);
     }
     unsafe {
-        MmapReadOnly::open(&file)
+        memmap::Mmap::map(&file)
             .map(Some)
             .map_err(|e| From::from(IOError::with_path(full_path.to_owned(), e)))
     }
@@ -65,7 +81,7 @@ pub struct CacheInfo {
 
 struct MmapCache {
     counters: CacheCounters,
-    cache: HashMap<PathBuf, MmapReadOnly>,
+    cache: HashMap<PathBuf, Weak<BoxedData>>,
 }
 
 impl Default for MmapCache {
@@ -78,12 +94,7 @@ impl Default for MmapCache {
 }
 
 impl MmapCache {
-    /// Removes a `MmapReadOnly` entry from the mmap cache.
-    fn discard_from_cache(&mut self, full_path: &Path) -> bool {
-        self.cache.remove(full_path).is_some()
-    }
-
-    fn get_info(&mut self) -> CacheInfo {
+    fn get_info(&self) -> CacheInfo {
         let paths: Vec<PathBuf> = self.cache.keys().cloned().collect();
         CacheInfo {
             counters: self.counters.clone(),
@@ -91,57 +102,178 @@ impl MmapCache {
         }
     }
 
-    fn get_mmap(&mut self, full_path: &Path) -> Result<Option<MmapReadOnly>, OpenReadError> {
-        Ok(match self.cache.entry(full_path.to_owned()) {
-            HashMapEntry::Occupied(occupied_entry) => {
-                let mmap = occupied_entry.get();
+    fn remove_weak_ref(&mut self) {
+        let keys_to_remove: Vec<PathBuf> = self
+            .cache
+            .iter()
+            .filter(|(_, mmap_weakref)| mmap_weakref.upgrade().is_none())
+            .map(|(key, _)| key.clone())
+            .collect();
+        for key in keys_to_remove {
+            self.cache.remove(&key);
+        }
+    }
+
+    // Returns None if the file exists but as a len of 0 (and hence is not mmappable).
+    fn get_mmap(&mut self, full_path: &Path) -> Result<Option<Arc<BoxedData>>, OpenReadError> {
+        if let Some(mmap_weak) = self.cache.get(full_path) {
+            if let Some(mmap_arc) = mmap_weak.upgrade() {
                 self.counters.hit += 1;
-                Some(mmap.clone())
-            }
-            HashMapEntry::Vacant(vacant_entry) => {
-                self.counters.miss += 1;
-                if let Some(mmap) = open_mmap(full_path)? {
-                    vacant_entry.insert(mmap.clone());
-                    Some(mmap)
-                } else {
-                    None
-                }
+                return Ok(Some(mmap_arc));
             }
+        }
+        self.cache.remove(full_path);
+        self.counters.miss += 1;
+        Ok(if let Some(mmap) = open_mmap(full_path)? {
+            let mmap_arc: Arc<BoxedData> = Arc::new(Box::new(mmap));
+            let mmap_weak = Arc::downgrade(&mmap_arc);
+            self.cache.insert(full_path.to_owned(), mmap_weak);
+            Some(mmap_arc)
+        } else {
+            None
         })
     }
 }
 
+struct InnerWatcherWrapper {
+    _watcher: Mutex<notify::RecommendedWatcher>,
+    watcher_router: WatchCallbackList,
+}
+
+impl InnerWatcherWrapper {
+    pub fn new(path: &Path) -> Result<(Self, Receiver<notify::RawEvent>), notify::Error> {
+        let (tx, watcher_recv): (Sender<RawEvent>, Receiver<RawEvent>) = channel();
+        // We need to initialize the
+        let mut watcher = notify::raw_watcher(tx)?;
+        watcher.watch(path, RecursiveMode::Recursive)?;
+        let inner = InnerWatcherWrapper {
+            _watcher: Mutex::new(watcher),
+            watcher_router: Default::default(),
+        };
+        Ok((inner, watcher_recv))
+    }
+}
+
+#[derive(Clone)]
+pub(crate) struct WatcherWrapper {
+    inner: Arc<InnerWatcherWrapper>,
+}
+
+impl WatcherWrapper {
+    pub fn new(path: &Path) -> Result<Self, OpenDirectoryError> {
+        let (inner, watcher_recv) = InnerWatcherWrapper::new(path).map_err(|err| match err {
+            notify::Error::PathNotFound => OpenDirectoryError::DoesNotExist(path.to_owned()),
+            _ => {
+                panic!("Unknown error while starting watching directory {:?}", path);
+            }
+        })?;
+        let watcher_wrapper = WatcherWrapper {
+            inner: Arc::new(inner),
+        };
+        let watcher_wrapper_clone = watcher_wrapper.clone();
+        thread::Builder::new()
+            .name("meta-file-watch-thread".to_string())
+            .spawn(move || {
+                loop {
+                    match watcher_recv.recv().map(|evt| evt.path) {
+                        Ok(Some(changed_path)) => {
+                            // ... Actually subject to false positive.
+                            // We might want to be more accurate than this at one point.
+                            if let Some(filename) = changed_path.file_name() {
+                                if filename == *META_FILEPATH {
+                                    watcher_wrapper_clone.inner.watcher_router.broadcast();
+                                }
+                            }
+                        }
+                        Ok(None) => {
+                            // not an event we are interested in.
+                        }
+                        Err(_e) => {
+                            // the watch send channel was dropped
+                            break;
+                        }
+                    }
+                }
+            })
+            .expect("Failed to spawn thread to watch meta.json");
+        Ok(watcher_wrapper)
+    }
+
+    pub fn watch(&mut self, watch_callback: WatchCallback) -> WatchHandle {
+        self.inner.watcher_router.subscribe(watch_callback)
+    }
+}
+
 /// Directory storing data in files, read via mmap.
 ///
 /// The Mmap object are cached to limit the
 /// system calls.
+///
+/// In the `MmapDirectory`, locks are implemented using the `fs2` crate definition of locks.
+///
+/// On MacOS & linux, it relies on `flock` (aka `BSD Lock`). These locks solve most of the
+/// problems related to POSIX Locks, but may their contract may not be respected on `NFS`
+/// depending on the implementation.
+///
+/// On Windows the semantics are again different.
 #[derive(Clone)]
 pub struct MmapDirectory {
+    inner: Arc<MmapDirectoryInner>,
+}
+
+struct MmapDirectoryInner {
     root_path: PathBuf,
-    mmap_cache: Arc<RwLock<MmapCache>>,
-    _temp_directory: Arc<Option<TempDir>>,
+    mmap_cache: RwLock<MmapCache>,
+    _temp_directory: Option<TempDir>,
+    watcher: RwLock<WatcherWrapper>,
+}
+
+impl MmapDirectoryInner {
+    fn new(
+        root_path: PathBuf,
+        temp_directory: Option<TempDir>,
+    ) -> Result<MmapDirectoryInner, OpenDirectoryError> {
+        let watch_wrapper = WatcherWrapper::new(&root_path)?;
+        let mmap_directory_inner = MmapDirectoryInner {
+            root_path,
+            mmap_cache: Default::default(),
+            _temp_directory: temp_directory,
+            watcher: RwLock::new(watch_wrapper),
+        };
+        Ok(mmap_directory_inner)
+    }
+
+    fn watch(&self, watch_callback: WatchCallback) -> WatchHandle {
+        let mut wlock = self.watcher.write().unwrap();
+        wlock.watch(watch_callback)
+    }
 }
 
 impl fmt::Debug for MmapDirectory {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "MmapDirectory({:?})", self.root_path)
+        write!(f, "MmapDirectory({:?})", self.inner.root_path)
     }
 }
 
 impl MmapDirectory {
+    fn new(
+        root_path: PathBuf,
+        temp_directory: Option<TempDir>,
+    ) -> Result<MmapDirectory, OpenDirectoryError> {
+        let inner = MmapDirectoryInner::new(root_path, temp_directory)?;
+        Ok(MmapDirectory {
+            inner: Arc::new(inner),
+        })
+    }
+
     /// Creates a new MmapDirectory in a temporary directory.
     ///
     /// This is mostly useful to test the MmapDirectory itself.
     /// For your unit tests, prefer the RAMDirectory.
-    pub fn create_from_tempdir() -> io::Result<MmapDirectory> {
-        let tempdir = TempDir::new("index")?;
+    pub fn create_from_tempdir() -> Result<MmapDirectory, OpenDirectoryError> {
+        let tempdir = TempDir::new("index").map_err(OpenDirectoryError::IoError)?;
         let tempdir_path = PathBuf::from(tempdir.path());
-        let directory = MmapDirectory {
-            root_path: tempdir_path,
-            mmap_cache: Arc::new(RwLock::new(MmapCache::default())),
-            _temp_directory: Arc::new(Some(tempdir)),
-        };
-        Ok(directory)
+        MmapDirectory::new(tempdir_path, Some(tempdir))
     }
 
     /// Opens a MmapDirectory in a directory.
@@ -159,18 +291,14 @@ impl MmapDirectory {
                 directory_path,
             )))
         } else {
-            Ok(MmapDirectory {
-                root_path: PathBuf::from(directory_path),
-                mmap_cache: Arc::new(RwLock::new(MmapCache::default())),
-                _temp_directory: Arc::new(None),
-            })
+            Ok(MmapDirectory::new(PathBuf::from(directory_path), None)?)
         }
     }
 
     /// Joins a relative_path to the directory `root_path`
     /// to create a proper complete `filepath`.
     fn resolve_path(&self, relative_path: &Path) -> PathBuf {
-        self.root_path.join(relative_path)
+        self.inner.root_path.join(relative_path)
     }
 
     /// Sync the root directory.
@@ -195,7 +323,7 @@ impl MmapDirectory {
                 .custom_flags(winbase::FILE_FLAG_BACKUP_SEMANTICS);
         }
 
-        let fd = open_opts.open(&self.root_path)?;
+        let fd = open_opts.open(&self.inner.root_path)?;
         fd.sync_all()?;
         Ok(())
     }
@@ -205,14 +333,35 @@ impl MmapDirectory {
     ///
     /// The `MmapDirectory` embeds a `MmapDirectory`
     /// to avoid multiplying the `mmap` system calls.
-    pub fn get_cache_info(&mut self) -> CacheInfo {
-        self.mmap_cache
+    pub fn get_cache_info(&self) -> CacheInfo {
+        self.inner
+            .mmap_cache
             .write()
+            .expect("mmap cache lock is poisoned")
+            .remove_weak_ref();
+        self.inner
+            .mmap_cache
+            .read()
             .expect("Mmap cache lock is poisoned.")
             .get_info()
     }
 }
 
+/// We rely on fs2 for file locking. On Windows & MacOS this
+/// uses BSD locks (`flock`). The lock is actually released when
+/// the `File` object is dropped and its associated file descriptor
+/// is closed.
+struct ReleaseLockFile {
+    _file: File,
+    path: PathBuf,
+}
+
+impl Drop for ReleaseLockFile {
+    fn drop(&mut self) {
+        debug!("Releasing lock {:?}", self.path);
+    }
+}
+
 /// This Write wraps a File, but has the specificity of
 /// call `sync_all` on flush.
 struct SafeFileWriter(File);
@@ -245,7 +394,7 @@ impl Directory for MmapDirectory {
         debug!("Open Read {:?}", path);
         let full_path = self.resolve_path(path);
 
-        let mut mmap_cache = self.mmap_cache.write().map_err(|_| {
+        let mut mmap_cache = self.inner.mmap_cache.write().map_err(|_| {
             let msg = format!(
                 "Failed to acquired write lock \
                  on mmap cache while reading {:?}",
@@ -253,11 +402,34 @@ impl Directory for MmapDirectory {
             );
             IOError::with_path(path.to_owned(), make_io_err(msg))
         })?;
-
         Ok(mmap_cache
             .get_mmap(&full_path)?
-            .map(ReadOnlySource::Mmap)
-            .unwrap_or_else(|| ReadOnlySource::Anonymous(SharedVecSlice::empty())))
+            .map(ReadOnlySource::from)
+            .unwrap_or_else(ReadOnlySource::empty))
+    }
+
+    /// Any entry associated to the path in the mmap will be
+    /// removed before the file is deleted.
+    fn delete(&self, path: &Path) -> result::Result<(), DeleteError> {
+        debug!("Deleting file {:?}", path);
+        let full_path = self.resolve_path(path);
+        match fs::remove_file(&full_path) {
+            Ok(_) => self
+                .sync_directory()
+                .map_err(|e| IOError::with_path(path.to_owned(), e).into()),
+            Err(e) => {
+                if e.kind() == io::ErrorKind::NotFound {
+                    Err(DeleteError::FileDoesNotExist(path.to_owned()))
+                } else {
+                    Err(IOError::with_path(path.to_owned(), e).into())
+                }
+            }
+        }
+    }
+
+    fn exists(&self, path: &Path) -> bool {
+        let full_path = self.resolve_path(path);
+        full_path.exists()
     }
 
     fn open_write(&mut self, path: &Path) -> Result<WritePtr, OpenWriteError> {
@@ -290,44 +462,6 @@ impl Directory for MmapDirectory {
         Ok(BufWriter::new(Box::new(writer)))
     }
 
-    /// Any entry associated to the path in the mmap will be
-    /// removed before the file is deleted.
-    fn delete(&self, path: &Path) -> result::Result<(), DeleteError> {
-        debug!("Deleting file {:?}", path);
-        let full_path = self.resolve_path(path);
-        let mut mmap_cache = self.mmap_cache.write().map_err(|_| {
-            let msg = format!(
-                "Failed to acquired write lock \
-                 on mmap cache while deleting {:?}",
-                path
-            );
-            IOError::with_path(path.to_owned(), make_io_err(msg))
-        })?;
-        mmap_cache.discard_from_cache(path);
-
-        // Removing the entry in the MMap cache.
-        // The munmap will appear on Drop,
-        // when the last reference is gone.
-        mmap_cache.cache.remove(&full_path);
-        match fs::remove_file(&full_path) {
-            Ok(_) => self
-                .sync_directory()
-                .map_err(|e| IOError::with_path(path.to_owned(), e).into()),
-            Err(e) => {
-                if e.kind() == io::ErrorKind::NotFound {
-                    Err(DeleteError::FileDoesNotExist(path.to_owned()))
-                } else {
-                    Err(IOError::with_path(path.to_owned(), e).into())
-                }
-            }
-        }
-    }
-
-    fn exists(&self, path: &Path) -> bool {
-        let full_path = self.resolve_path(path);
-        full_path.exists()
-    }
-
     fn atomic_read(&self, path: &Path) -> Result<Vec<u8>, OpenReadError> {
         let full_path = self.resolve_path(path);
         let mut buffer = Vec::new();
@@ -354,6 +488,30 @@ impl Directory for MmapDirectory {
         meta_file.write(|f| f.write_all(data))?;
         Ok(())
     }
+
+    fn acquire_lock(&self, lock: &Lock) -> Result<DirectoryLock, LockError> {
+        let full_path = self.resolve_path(&lock.filepath);
+        // We make sure that the file exists.
+        let file: File = OpenOptions::new()
+            .write(true)
+            .create(true) //< if the file does not exist yet, create it.
+            .open(&full_path)
+            .map_err(LockError::IOError)?;
+        if lock.is_blocking {
+            file.lock_exclusive().map_err(LockError::IOError)?;
+        } else {
+            file.try_lock_exclusive().map_err(|_| LockError::LockBusy)?
+        }
+        // dropping the file handle will release the lock.
+        Ok(DirectoryLock::from(Box::new(ReleaseLockFile {
+            path: lock.filepath.clone(),
+            _file: file,
+        })))
+    }
+
+    fn watch(&self, watch_callback: WatchCallback) -> WatchHandle {
+        self.inner.watch(watch_callback)
+    }
 }
 
 #[cfg(test)]
@@ -363,6 +521,13 @@ mod tests {
     // The following tests are specific to the MmapDirectory
 
     use super::*;
+    use schema::{Schema, SchemaBuilder, TEXT};
+    use std::fs;
+    use std::sync::atomic::{AtomicUsize, Ordering};
+    use std::thread;
+    use std::time::Duration;
+    use Index;
+    use ReloadPolicy;
 
     #[test]
     fn test_open_non_existant_path() {
@@ -387,7 +552,7 @@ mod tests {
 
     #[test]
     fn test_cache() {
-        let content = "abc".as_bytes();
+        let content = b"abc";
 
         // here we test if the cache releases
         // mmaps correctly.
@@ -403,26 +568,104 @@ mod tests {
                 w.flush().unwrap();
             }
         }
-        {
-            for (i, path) in paths.iter().enumerate() {
-                let _r = mmap_directory.open_read(path).unwrap();
-                assert_eq!(mmap_directory.get_cache_info().mmapped.len(), i + 1);
-            }
-            for path in paths.iter() {
-                let _r = mmap_directory.open_read(path).unwrap();
-                assert_eq!(mmap_directory.get_cache_info().mmapped.len(), num_paths);
-            }
-            for (i, path) in paths.iter().enumerate() {
-                mmap_directory.delete(path).unwrap();
-                assert_eq!(
-                    mmap_directory.get_cache_info().mmapped.len(),
-                    num_paths - i - 1
-                );
-            }
+
+        let mut keep = vec![];
+        for (i, path) in paths.iter().enumerate() {
+            keep.push(mmap_directory.open_read(path).unwrap());
+            assert_eq!(mmap_directory.get_cache_info().mmapped.len(), i + 1);
+        }
+        assert_eq!(mmap_directory.get_cache_info().counters.hit, 0);
+        assert_eq!(mmap_directory.get_cache_info().counters.miss, 10);
+        assert_eq!(mmap_directory.get_cache_info().mmapped.len(), 10);
+        for path in paths.iter() {
+            let _r = mmap_directory.open_read(path).unwrap();
+            assert_eq!(mmap_directory.get_cache_info().mmapped.len(), num_paths);
         }
         assert_eq!(mmap_directory.get_cache_info().counters.hit, 10);
         assert_eq!(mmap_directory.get_cache_info().counters.miss, 10);
+        assert_eq!(mmap_directory.get_cache_info().mmapped.len(), 10);
+
+        for path in paths.iter() {
+            let _r = mmap_directory.open_read(path).unwrap();
+            assert_eq!(mmap_directory.get_cache_info().mmapped.len(), 10);
+        }
+
+        assert_eq!(mmap_directory.get_cache_info().counters.hit, 20);
+        assert_eq!(mmap_directory.get_cache_info().counters.miss, 10);
+        assert_eq!(mmap_directory.get_cache_info().mmapped.len(), 10);
+        drop(keep);
+        for path in paths.iter() {
+            let _r = mmap_directory.open_read(path).unwrap();
+            assert_eq!(mmap_directory.get_cache_info().mmapped.len(), 1);
+        }
+        assert_eq!(mmap_directory.get_cache_info().counters.hit, 20);
+        assert_eq!(mmap_directory.get_cache_info().counters.miss, 20);
+        assert_eq!(mmap_directory.get_cache_info().mmapped.len(), 0);
+
+        for path in &paths {
+            mmap_directory.delete(path).unwrap();
+        }
+        assert_eq!(mmap_directory.get_cache_info().counters.hit, 20);
+        assert_eq!(mmap_directory.get_cache_info().counters.miss, 20);
+        assert_eq!(mmap_directory.get_cache_info().mmapped.len(), 0);
+        for path in paths.iter() {
+            assert!(mmap_directory.open_read(path).is_err());
+        }
+        assert_eq!(mmap_directory.get_cache_info().counters.hit, 20);
+        assert_eq!(mmap_directory.get_cache_info().counters.miss, 30);
         assert_eq!(mmap_directory.get_cache_info().mmapped.len(), 0);
     }
 
+    #[test]
+    fn test_watch_wrapper() {
+        let counter: Arc<AtomicUsize> = Default::default();
+        let counter_clone = counter.clone();
+        let tmp_dir: TempDir = tempdir::TempDir::new("test_watch_wrapper").unwrap();
+        let tmp_dirpath = tmp_dir.path().to_owned();
+        let mut watch_wrapper = WatcherWrapper::new(&tmp_dirpath).unwrap();
+        let tmp_file = tmp_dirpath.join("coucou");
+        let _handle = watch_wrapper.watch(Box::new(move || {
+            counter_clone.fetch_add(1, Ordering::SeqCst);
+        }));
+        assert_eq!(counter.load(Ordering::SeqCst), 0);
+        fs::write(&tmp_file, b"whateverwilldo").unwrap();
+        thread::sleep(Duration::new(0, 1_000u32));
+    }
+
+    #[test]
+    fn test_mmap_released() {
+        let mmap_directory = MmapDirectory::create_from_tempdir().unwrap();
+        let mut schema_builder: SchemaBuilder = Schema::builder();
+        let text_field = schema_builder.add_text_field("text", TEXT);
+        let schema = schema_builder.build();
+        {
+            let index = Index::create(mmap_directory.clone(), schema).unwrap();
+            let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
+            for _num_commits in 0..16 {
+                for _ in 0..10 {
+                    index_writer.add_document(doc!(text_field=>"abc"));
+                }
+                index_writer.commit().unwrap();
+            }
+            let reader = index
+                .reader_builder()
+                .reload_policy(ReloadPolicy::Manual)
+                .try_into()
+                .unwrap();
+            for _ in 0..30 {
+                index_writer.add_document(doc!(text_field=>"abc"));
+                index_writer.commit().unwrap();
+                reader.reload().unwrap();
+            }
+            index_writer.wait_merging_threads().unwrap();
+            reader.reload().unwrap();
+            let num_segments = reader.searcher().segment_readers().len();
+            assert_eq!(num_segments, 4);
+            assert_eq!(
+                num_segments * 7,
+                mmap_directory.get_cache_info().mmapped.len()
+            );
+        }
+        assert_eq!(mmap_directory.get_cache_info().mmapped.len(), 0);
+    }
 }

src/directory/mod.rs

@@ -8,19 +8,23 @@ WORM directory abstraction.
 mod mmap_directory;
 
 mod directory;
+mod directory_lock;
 mod managed_directory;
 mod ram_directory;
 mod read_only_source;
-mod shared_vec_slice;
+mod watch_event_router;
 
 /// Errors specific to the directory module.
 pub mod error;
 
-use std::io::{BufWriter, Seek, Write};
-
+pub use self::directory::DirectoryLock;
 pub use self::directory::{Directory, DirectoryClone};
+pub use self::directory_lock::{Lock, INDEX_WRITER_LOCK, META_LOCK};
 pub use self::ram_directory::RAMDirectory;
 pub use self::read_only_source::ReadOnlySource;
+pub(crate) use self::watch_event_router::WatchCallbackList;
+pub use self::watch_event_router::{WatchCallback, WatchHandle};
+use std::io::{BufWriter, Seek, Write};
 
 #[cfg(feature = "mmap")]
 pub use self::mmap_directory::MmapDirectory;
@@ -38,128 +42,4 @@ impl<T: Seek + Write> SeekableWrite for T {}
 pub type WritePtr = BufWriter<Box<SeekableWrite>>;
 
 #[cfg(test)]
-mod tests {
-
-    use super::*;
-    use std::io::{Seek, SeekFrom, Write};
-    use std::path::Path;
-
-    lazy_static! {
-        static ref TEST_PATH: &'static Path = Path::new("some_path_for_test");
-    }
-
-    #[test]
-    fn test_ram_directory() {
-        let mut ram_directory = RAMDirectory::create();
-        test_directory(&mut ram_directory);
-    }
-
-    #[test]
-    #[cfg(feature = "mmap")]
-    fn test_mmap_directory() {
-        let mut mmap_directory = MmapDirectory::create_from_tempdir().unwrap();
-        test_directory(&mut mmap_directory);
-    }
-
-    #[test]
-    #[should_panic]
-    fn ram_directory_panics_if_flush_forgotten() {
-        let mut ram_directory = RAMDirectory::create();
-        let mut write_file = ram_directory.open_write(*TEST_PATH).unwrap();
-        assert!(write_file.write_all(&[4]).is_ok());
-    }
-
-    fn test_simple(directory: &mut Directory) {
-        {
-            {
-                let mut write_file = directory.open_write(*TEST_PATH).unwrap();
-                assert!(directory.exists(*TEST_PATH));
-                write_file.write_all(&[4]).unwrap();
-                write_file.write_all(&[3]).unwrap();
-                write_file.write_all(&[7, 3, 5]).unwrap();
-                write_file.flush().unwrap();
-            }
-            let read_file = directory.open_read(*TEST_PATH).unwrap();
-            let data: &[u8] = &*read_file;
-            assert_eq!(data, &[4u8, 3u8, 7u8, 3u8, 5u8]);
-        }
-
-        assert!(directory.delete(*TEST_PATH).is_ok());
-        assert!(!directory.exists(*TEST_PATH));
-    }
-
-    fn test_seek(directory: &mut Directory) {
-        {
-            {
-                let mut write_file = directory.open_write(*TEST_PATH).unwrap();
-                write_file.write_all(&[4, 3, 7, 3, 5]).unwrap();
-                write_file.seek(SeekFrom::Start(0)).unwrap();
-                write_file.write_all(&[3, 1]).unwrap();
-                write_file.flush().unwrap();
-            }
-            let read_file = directory.open_read(*TEST_PATH).unwrap();
-            let data: &[u8] = &*read_file;
-            assert_eq!(data, &[3u8, 1u8, 7u8, 3u8, 5u8]);
-        }
-
-        assert!(directory.delete(*TEST_PATH).is_ok());
-    }
-
-    fn test_rewrite_forbidden(directory: &mut Directory) {
-        {
-            directory.open_write(*TEST_PATH).unwrap();
-            assert!(directory.exists(*TEST_PATH));
-        }
-        {
-            assert!(directory.open_write(*TEST_PATH).is_err());
-        }
-        assert!(directory.delete(*TEST_PATH).is_ok());
-    }
-
-    fn test_write_create_the_file(directory: &mut Directory) {
-        {
-            assert!(directory.open_read(*TEST_PATH).is_err());
-            let _w = directory.open_write(*TEST_PATH).unwrap();
-            assert!(directory.exists(*TEST_PATH));
-            assert!(directory.open_read(*TEST_PATH).is_ok());
-            assert!(directory.delete(*TEST_PATH).is_ok());
-        }
-    }
-
-    fn test_directory_delete(directory: &mut Directory) {
-        assert!(directory.open_read(*TEST_PATH).is_err());
-        let mut write_file = directory.open_write(*TEST_PATH).unwrap();
-        write_file.write_all(&[1, 2, 3, 4]).unwrap();
-        write_file.flush().unwrap();
-        {
-            let read_handle = directory.open_read(*TEST_PATH).unwrap();
-            {
-                assert_eq!(&*read_handle, &[1u8, 2u8, 3u8, 4u8]);
-
-                // Mapped files can't be deleted on Windows
-                if !cfg!(windows) {
-                    assert!(directory.delete(*TEST_PATH).is_ok());
-                    assert_eq!(&*read_handle, &[1u8, 2u8, 3u8, 4u8]);
-                }
-
-                assert!(directory.delete(Path::new("SomeOtherPath")).is_err());
-            }
-        }
-
-        if cfg!(windows) {
-            assert!(directory.delete(*TEST_PATH).is_ok());
-        }
-
-        assert!(directory.open_read(*TEST_PATH).is_err());
-        assert!(directory.delete(*TEST_PATH).is_err());
-    }
-
-    fn test_directory(directory: &mut Directory) {
-        test_simple(directory);
-        test_seek(directory);
-        test_rewrite_forbidden(directory);
-        test_write_create_the_file(directory);
-        test_directory_delete(directory);
-    }
-
-}
+mod tests;

src/directory/ram_directory.rs

@@ -1,8 +1,8 @@
-use super::shared_vec_slice::SharedVecSlice;
-use common::make_io_err;
-use directory::error::{DeleteError, IOError, OpenReadError, OpenWriteError};
+use core::META_FILEPATH;
+use directory::error::{DeleteError, OpenReadError, OpenWriteError};
+use directory::WatchCallbackList;
 use directory::WritePtr;
-use directory::{Directory, ReadOnlySource};
+use directory::{Directory, ReadOnlySource, WatchCallback, WatchHandle};
 use std::collections::HashMap;
 use std::fmt;
 use std::io::{self, BufWriter, Cursor, Seek, SeekFrom, Write};
@@ -22,13 +22,13 @@ use std::sync::{Arc, RwLock};
 ///
 struct VecWriter {
     path: PathBuf,
-    shared_directory: InnerDirectory,
+    shared_directory: RAMDirectory,
     data: Cursor<Vec<u8>>,
     is_flushed: bool,
 }
 
 impl VecWriter {
-    fn new(path_buf: PathBuf, shared_directory: InnerDirectory) -> VecWriter {
+    fn new(path_buf: PathBuf, shared_directory: RAMDirectory) -> VecWriter {
         VecWriter {
             path: path_buf,
             data: Cursor::new(Vec::new()),
@@ -64,75 +64,44 @@ impl Write for VecWriter {
 
     fn flush(&mut self) -> io::Result<()> {
         self.is_flushed = true;
-        self.shared_directory
-            .write(self.path.clone(), self.data.get_ref())?;
+        let mut fs = self.shared_directory.fs.write().unwrap();
+        fs.write(self.path.clone(), self.data.get_ref());
         Ok(())
     }
 }
 
-#[derive(Clone)]
-struct InnerDirectory(Arc<RwLock<HashMap<PathBuf, Arc<Vec<u8>>>>>);
+#[derive(Default)]
+struct InnerDirectory {
+    fs: HashMap<PathBuf, ReadOnlySource>,
+    watch_router: WatchCallbackList,
+}
 
 impl InnerDirectory {
-    fn new() -> InnerDirectory {
-        InnerDirectory(Arc::new(RwLock::new(HashMap::new())))
-    }
-
-    fn write(&self, path: PathBuf, data: &[u8]) -> io::Result<bool> {
-        let mut map = self.0.write().map_err(|_| {
-            make_io_err(format!(
-                "Failed to lock the directory, when trying to write {:?}",
-                path
-            ))
-        })?;
-        let prev_value = map.insert(path, Arc::new(Vec::from(data)));
-        Ok(prev_value.is_some())
+    fn write(&mut self, path: PathBuf, data: &[u8]) -> bool {
+        let data = ReadOnlySource::new(Vec::from(data));
+        self.fs.insert(path, data).is_some()
     }
 
     fn open_read(&self, path: &Path) -> Result<ReadOnlySource, OpenReadError> {
-        self.0
-            .read()
-            .map_err(|_| {
-                let msg = format!(
-                    "Failed to acquire read lock for the \
-                     directory when trying to read {:?}",
-                    path
-                );
-                let io_err = make_io_err(msg);
-                OpenReadError::IOError(IOError::with_path(path.to_owned(), io_err))
-            })
-            .and_then(|readable_map| {
-                readable_map
-                    .get(path)
-                    .ok_or_else(|| OpenReadError::FileDoesNotExist(PathBuf::from(path)))
-                    .map(Arc::clone)
-                    .map(|data| ReadOnlySource::Anonymous(SharedVecSlice::new(data)))
-            })
+        self.fs
+            .get(path)
+            .ok_or_else(|| OpenReadError::FileDoesNotExist(PathBuf::from(path)))
+            .map(|el| el.clone())
     }
 
-    fn delete(&self, path: &Path) -> result::Result<(), DeleteError> {
-        self.0
-            .write()
-            .map_err(|_| {
-                let msg = format!(
-                    "Failed to acquire write lock for the \
-                     directory when trying to delete {:?}",
-                    path
-                );
-                let io_err = make_io_err(msg);
-                DeleteError::IOError(IOError::with_path(path.to_owned(), io_err))
-            })
-            .and_then(|mut writable_map| match writable_map.remove(path) {
-                Some(_) => Ok(()),
-                None => Err(DeleteError::FileDoesNotExist(PathBuf::from(path))),
-            })
+    fn delete(&mut self, path: &Path) -> result::Result<(), DeleteError> {
+        match self.fs.remove(path) {
+            Some(_) => Ok(()),
+            None => Err(DeleteError::FileDoesNotExist(PathBuf::from(path))),
+        }
     }
 
     fn exists(&self, path: &Path) -> bool {
-        self.0
-            .read()
-            .expect("Failed to get read lock directory.")
-            .contains_key(path)
+        self.fs.contains_key(path)
+    }
+
+    fn watch(&mut self, watch_handle: WatchCallback) -> WatchHandle {
+        self.watch_router.subscribe(watch_handle)
     }
 }
 
@@ -147,33 +116,36 @@ impl fmt::Debug for RAMDirectory {
 /// It is mainly meant for unit testing.
 /// Writes are only made visible upon flushing.
 ///
-#[derive(Clone)]
+#[derive(Clone, Default)]
 pub struct RAMDirectory {
-    fs: InnerDirectory,
+    fs: Arc<RwLock<InnerDirectory>>,
 }
 
 impl RAMDirectory {
     /// Constructor
     pub fn create() -> RAMDirectory {
-        RAMDirectory {
-            fs: InnerDirectory::new(),
-        }
+        Self::default()
     }
 }
 
 impl Directory for RAMDirectory {
     fn open_read(&self, path: &Path) -> result::Result<ReadOnlySource, OpenReadError> {
-        self.fs.open_read(path)
+        self.fs.read().unwrap().open_read(path)
+    }
+
+    fn delete(&self, path: &Path) -> result::Result<(), DeleteError> {
+        self.fs.write().unwrap().delete(path)
+    }
+
+    fn exists(&self, path: &Path) -> bool {
+        self.fs.read().unwrap().exists(path)
     }
 
     fn open_write(&mut self, path: &Path) -> Result<WritePtr, OpenWriteError> {
+        let mut fs = self.fs.write().unwrap();
         let path_buf = PathBuf::from(path);
-        let vec_writer = VecWriter::new(path_buf.clone(), self.fs.clone());
-
-        let exists = self
-            .fs
-            .write(path_buf.clone(), &Vec::new())
-            .map_err(|err| IOError::with_path(path.to_owned(), err))?;
+        let vec_writer = VecWriter::new(path_buf.clone(), self.clone());
+        let exists = fs.write(path_buf.clone(), &[]);
         // force the creation of the file to mimic the MMap directory.
         if exists {
             Err(OpenWriteError::FileAlreadyExists(path_buf))
@@ -182,17 +154,8 @@ impl Directory for RAMDirectory {
         }
     }
 
-    fn delete(&self, path: &Path) -> result::Result<(), DeleteError> {
-        self.fs.delete(path)
-    }
-
-    fn exists(&self, path: &Path) -> bool {
-        self.fs.exists(path)
-    }
-
     fn atomic_read(&self, path: &Path) -> Result<Vec<u8>, OpenReadError> {
-        let read = self.open_read(path)?;
-        Ok(read.as_slice().to_owned())
+        Ok(self.open_read(path)?.as_slice().to_owned())
     }
 
     fn atomic_write(&mut self, path: &Path, data: &[u8]) -> io::Result<()> {
@@ -201,10 +164,20 @@ impl Directory for RAMDirectory {
             msg.unwrap_or("Undefined".to_string())
         )));
         let path_buf = PathBuf::from(path);
-        let mut vec_writer = VecWriter::new(path_buf.clone(), self.fs.clone());
-        self.fs.write(path_buf, &Vec::new())?;
+
+        // Reserve the path to prevent calls to .write() to succeed.
+        self.fs.write().unwrap().write(path_buf.clone(), &[]);
+
+        let mut vec_writer = VecWriter::new(path_buf.clone(), self.clone());
         vec_writer.write_all(data)?;
         vec_writer.flush()?;
+        if path == Path::new(&*META_FILEPATH) {
+            self.fs.write().unwrap().watch_router.broadcast();
+        }
         Ok(())
     }
+
+    fn watch(&self, watch_callback: WatchCallback) -> WatchHandle {
+        self.fs.write().unwrap().watch(watch_callback)
+    }
 }

src/directory/read_only_source.rs

@@ -1,9 +1,9 @@
-use super::shared_vec_slice::SharedVecSlice;
 use common::HasLen;
-#[cfg(feature = "mmap")]
-use fst::raw::MmapReadOnly;
 use stable_deref_trait::{CloneStableDeref, StableDeref};
 use std::ops::Deref;
+use std::sync::Arc;
+
+pub type BoxedData = Box<Deref<Target = [u8]> + Send + Sync + 'static>;
 
 /// Read object that represents files in tantivy.
 ///
@@ -11,12 +11,10 @@ use std::ops::Deref;
 /// the data in the form of a constant read-only `&[u8]`.
 /// Whatever happens to the directory file, the data
 /// hold by this object should never be altered or destroyed.
-pub enum ReadOnlySource {
-    /// Mmap source of data
-    #[cfg(feature = "mmap")]
-    Mmap(MmapReadOnly),
-    /// Wrapping a `Vec<u8>`
-    Anonymous(SharedVecSlice),
+pub struct ReadOnlySource {
+    data: Arc<BoxedData>,
+    start: usize,
+    stop: usize,
 }
 
 unsafe impl StableDeref for ReadOnlySource {}
@@ -30,19 +28,38 @@ impl Deref for ReadOnlySource {
     }
 }
 
+impl From<Arc<BoxedData>> for ReadOnlySource {
+    fn from(data: Arc<BoxedData>) -> Self {
+        let len = data.len();
+        ReadOnlySource {
+            data,
+            start: 0,
+            stop: len,
+        }
+    }
+}
+
 impl ReadOnlySource {
+    pub(crate) fn new<D>(data: D) -> ReadOnlySource
+    where
+        D: Deref<Target = [u8]> + Send + Sync + 'static,
+    {
+        let len = data.len();
+        ReadOnlySource {
+            data: Arc::new(Box::new(data)),
+            start: 0,
+            stop: len,
+        }
+    }
+
     /// Creates an empty ReadOnlySource
     pub fn empty() -> ReadOnlySource {
-        ReadOnlySource::Anonymous(SharedVecSlice::empty())
+        ReadOnlySource::new(&[][..])
     }
 
     /// Returns the data underlying the ReadOnlySource object.
     pub fn as_slice(&self) -> &[u8] {
-        match *self {
-            #[cfg(feature = "mmap")]
-            ReadOnlySource::Mmap(ref mmap_read_only) => mmap_read_only.as_slice(),
-            ReadOnlySource::Anonymous(ref shared_vec) => shared_vec.as_slice(),
-        }
+        &self.data[self.start..self.stop]
     }
 
     /// Splits into 2 `ReadOnlySource`, at the offset given
@@ -63,22 +80,18 @@ impl ReadOnlySource {
     /// worth of data in anonymous memory, and only a
     /// 1KB slice is remaining, the whole `500MBs`
     /// are retained in memory.
-    pub fn slice(&self, from_offset: usize, to_offset: usize) -> ReadOnlySource {
+    pub fn slice(&self, start: usize, stop: usize) -> ReadOnlySource {
         assert!(
-            from_offset <= to_offset,
+            start <= stop,
             "Requested negative slice [{}..{}]",
-            from_offset,
-            to_offset
+            start,
+            stop
         );
-        match *self {
-            #[cfg(feature = "mmap")]
-            ReadOnlySource::Mmap(ref mmap_read_only) => {
-                let sliced_mmap = mmap_read_only.range(from_offset, to_offset - from_offset);
-                ReadOnlySource::Mmap(sliced_mmap)
-            }
-            ReadOnlySource::Anonymous(ref shared_vec) => {
-                ReadOnlySource::Anonymous(shared_vec.slice(from_offset, to_offset))
-            }
+        assert!(stop <= self.len());
+        ReadOnlySource {
+            data: self.data.clone(),
+            start: self.start + start,
+            stop: self.start + stop,
         }
     }
 
@@ -87,8 +100,7 @@ impl ReadOnlySource {
     ///
     /// Equivalent to `.slice(from_offset, self.len())`
     pub fn slice_from(&self, from_offset: usize) -> ReadOnlySource {
-        let len = self.len();
-        self.slice(from_offset, len)
+        self.slice(from_offset, self.len())
     }
 
     /// Like `.slice(...)` but enforcing only the `to`
@@ -102,19 +114,18 @@ impl ReadOnlySource {
 
 impl HasLen for ReadOnlySource {
     fn len(&self) -> usize {
-        self.as_slice().len()
+        self.stop - self.start
     }
 }
 
 impl Clone for ReadOnlySource {
     fn clone(&self) -> Self {
-        self.slice(0, self.len())
+        self.slice_from(0)
     }
 }
 
 impl From<Vec<u8>> for ReadOnlySource {
     fn from(data: Vec<u8>) -> ReadOnlySource {
-        let shared_data = SharedVecSlice::from(data);
-        ReadOnlySource::Anonymous(shared_data)
+        ReadOnlySource::new(data)
     }
 }

src/directory/shared_vec_slice.rs

@@ -1,41 +0,0 @@
-use std::sync::Arc;
-
-#[derive(Clone)]
-pub struct SharedVecSlice {
-    pub data: Arc<Vec<u8>>,
-    pub start: usize,
-    pub len: usize,
-}
-
-impl SharedVecSlice {
-    pub fn empty() -> SharedVecSlice {
-        SharedVecSlice::new(Arc::new(Vec::new()))
-    }
-
-    pub fn new(data: Arc<Vec<u8>>) -> SharedVecSlice {
-        let data_len = data.len();
-        SharedVecSlice {
-            data,
-            start: 0,
-            len: data_len,
-        }
-    }
-
-    pub fn as_slice(&self) -> &[u8] {
-        &self.data[self.start..self.start + self.len]
-    }
-
-    pub fn slice(&self, from_offset: usize, to_offset: usize) -> SharedVecSlice {
-        SharedVecSlice {
-            data: Arc::clone(&self.data),
-            start: self.start + from_offset,
-            len: to_offset - from_offset,
-        }
-    }
-}
-
-impl From<Vec<u8>> for SharedVecSlice {
-    fn from(data: Vec<u8>) -> SharedVecSlice {
-        SharedVecSlice::new(Arc::new(data))
-    }
-}

src/directory/tests.rs

@@ -0,0 +1,222 @@
+use super::*;
+use std::io::{Seek, SeekFrom, Write};
+use std::mem;
+use std::path::{Path, PathBuf};
+use std::sync::atomic::AtomicUsize;
+use std::sync::atomic::Ordering;
+use std::sync::Arc;
+use std::thread;
+use std::time;
+use std::time::Duration;
+
+lazy_static! {
+    static ref TEST_PATH: &'static Path = Path::new("some_path_for_test");
+}
+
+#[test]
+fn test_ram_directory() {
+    let mut ram_directory = RAMDirectory::create();
+    test_directory(&mut ram_directory);
+}
+
+#[test]
+#[cfg(feature = "mmap")]
+fn test_mmap_directory() {
+    let mut mmap_directory = MmapDirectory::create_from_tempdir().unwrap();
+    test_directory(&mut mmap_directory);
+}
+
+#[test]
+#[should_panic]
+fn ram_directory_panics_if_flush_forgotten() {
+    let mut ram_directory = RAMDirectory::create();
+    let mut write_file = ram_directory.open_write(*TEST_PATH).unwrap();
+    assert!(write_file.write_all(&[4]).is_ok());
+}
+
+fn test_simple(directory: &mut Directory) {
+    {
+        let mut write_file = directory.open_write(*TEST_PATH).unwrap();
+        assert!(directory.exists(*TEST_PATH));
+        write_file.write_all(&[4]).unwrap();
+        write_file.write_all(&[3]).unwrap();
+        write_file.write_all(&[7, 3, 5]).unwrap();
+        write_file.flush().unwrap();
+    }
+    {
+        let read_file = directory.open_read(*TEST_PATH).unwrap();
+        let data: &[u8] = &*read_file;
+        assert_eq!(data, &[4u8, 3u8, 7u8, 3u8, 5u8]);
+    }
+    assert!(directory.delete(*TEST_PATH).is_ok());
+    assert!(!directory.exists(*TEST_PATH));
+}
+
+fn test_seek(directory: &mut Directory) {
+    {
+        {
+            let mut write_file = directory.open_write(*TEST_PATH).unwrap();
+            write_file.write_all(&[4, 3, 7, 3, 5]).unwrap();
+            write_file.seek(SeekFrom::Start(0)).unwrap();
+            write_file.write_all(&[3, 1]).unwrap();
+            write_file.flush().unwrap();
+        }
+        let read_file = directory.open_read(*TEST_PATH).unwrap();
+        let data: &[u8] = &*read_file;
+        assert_eq!(data, &[3u8, 1u8, 7u8, 3u8, 5u8]);
+    }
+
+    assert!(directory.delete(*TEST_PATH).is_ok());
+}
+
+fn test_rewrite_forbidden(directory: &mut Directory) {
+    {
+        directory.open_write(*TEST_PATH).unwrap();
+        assert!(directory.exists(*TEST_PATH));
+    }
+    {
+        assert!(directory.open_write(*TEST_PATH).is_err());
+    }
+    assert!(directory.delete(*TEST_PATH).is_ok());
+}
+
+fn test_write_create_the_file(directory: &mut Directory) {
+    {
+        assert!(directory.open_read(*TEST_PATH).is_err());
+        let _w = directory.open_write(*TEST_PATH).unwrap();
+        assert!(directory.exists(*TEST_PATH));
+        assert!(directory.open_read(*TEST_PATH).is_ok());
+        assert!(directory.delete(*TEST_PATH).is_ok());
+    }
+}
+
+fn test_directory_delete(directory: &mut Directory) {
+    assert!(directory.open_read(*TEST_PATH).is_err());
+    let mut write_file = directory.open_write(*TEST_PATH).unwrap();
+    write_file.write_all(&[1, 2, 3, 4]).unwrap();
+    write_file.flush().unwrap();
+    {
+        let read_handle = directory.open_read(*TEST_PATH).unwrap();
+        {
+            assert_eq!(&*read_handle, &[1u8, 2u8, 3u8, 4u8]);
+
+            // Mapped files can't be deleted on Windows
+            if !cfg!(windows) {
+                assert!(directory.delete(*TEST_PATH).is_ok());
+                assert_eq!(&*read_handle, &[1u8, 2u8, 3u8, 4u8]);
+            }
+
+            assert!(directory.delete(Path::new("SomeOtherPath")).is_err());
+        }
+    }
+
+    if cfg!(windows) {
+        assert!(directory.delete(*TEST_PATH).is_ok());
+    }
+
+    assert!(directory.open_read(*TEST_PATH).is_err());
+    assert!(directory.delete(*TEST_PATH).is_err());
+}
+
+fn test_directory(directory: &mut Directory) {
+    test_simple(directory);
+    test_seek(directory);
+    test_rewrite_forbidden(directory);
+    test_write_create_the_file(directory);
+    test_directory_delete(directory);
+    test_lock_non_blocking(directory);
+    test_lock_blocking(directory);
+    test_watch(directory);
+}
+
+fn test_watch(directory: &mut Directory) {
+    let counter: Arc<AtomicUsize> = Default::default();
+    let counter_clone = counter.clone();
+    let watch_callback = Box::new(move || {
+        counter_clone.fetch_add(1, Ordering::SeqCst);
+    });
+    assert!(directory
+        .atomic_write(Path::new("meta.json"), b"random_test_data")
+        .is_ok());
+    thread::sleep(Duration::new(0, 10_000));
+    assert_eq!(0, counter.load(Ordering::SeqCst));
+
+    let watch_handle = directory.watch(watch_callback);
+    for i in 0..10 {
+        assert_eq!(i, counter.load(Ordering::SeqCst));
+        assert!(directory
+            .atomic_write(Path::new("meta.json"), b"random_test_data_2")
+            .is_ok());
+        for _ in 0..100 {
+            if counter.load(Ordering::SeqCst) > i {
+                break;
+            }
+            thread::sleep(Duration::from_millis(10));
+        }
+        assert_eq!(i + 1, counter.load(Ordering::SeqCst));
+    }
+    mem::drop(watch_handle);
+    assert!(directory
+        .atomic_write(Path::new("meta.json"), b"random_test_data")
+        .is_ok());
+    thread::sleep(Duration::from_millis(200));
+    assert_eq!(10, counter.load(Ordering::SeqCst));
+}
+
+fn test_lock_non_blocking(directory: &mut Directory) {
+    {
+        let lock_a_res = directory.acquire_lock(&Lock {
+            filepath: PathBuf::from("a.lock"),
+            is_blocking: false,
+        });
+        assert!(lock_a_res.is_ok());
+        let lock_b_res = directory.acquire_lock(&Lock {
+            filepath: PathBuf::from("b.lock"),
+            is_blocking: false,
+        });
+        assert!(lock_b_res.is_ok());
+        let lock_a_res2 = directory.acquire_lock(&Lock {
+            filepath: PathBuf::from("a.lock"),
+            is_blocking: false,
+        });
+        assert!(lock_a_res2.is_err());
+    }
+    let lock_a_res = directory.acquire_lock(&Lock {
+        filepath: PathBuf::from("a.lock"),
+        is_blocking: false,
+    });
+    assert!(lock_a_res.is_ok());
+}
+
+fn test_lock_blocking(directory: &mut Directory) {
+    let lock_a_res = directory.acquire_lock(&Lock {
+        filepath: PathBuf::from("a.lock"),
+        is_blocking: true,
+    });
+    assert!(lock_a_res.is_ok());
+    std::thread::spawn(move || {
+        //< lock_a_res is sent to the thread.
+        std::thread::sleep(time::Duration::from_millis(10));
+        // explicitely droping lock_a_res. It would have been sufficient to just force it
+        // to be part of the move, but the intent seems clearer that way.
+        drop(lock_a_res);
+    });
+    {
+        // A non-blocking call should fail, as the thread is running and holding the lock.
+        let lock_a_res = directory.acquire_lock(&Lock {
+            filepath: PathBuf::from("a.lock"),
+            is_blocking: false,
+        });
+        assert!(lock_a_res.is_err());
+    }
+    {
+        // the blocking call should wait for at least 10ms.
+        let start = time::Instant::now();
+        let lock_a_res = directory.acquire_lock(&Lock {
+            filepath: PathBuf::from("a.lock"),
+            is_blocking: true,
+        });
+        assert!(lock_a_res.is_ok());
+        assert!(start.elapsed().subsec_millis() >= 10);
+    }
+}

src/directory/watch_event_router.rs

@@ -0,0 +1,156 @@
+use std::sync::Arc;
+use std::sync::RwLock;
+use std::sync::Weak;
+
+/// Type alias for callbacks registered when watching files of a `Directory`.
+pub type WatchCallback = Box<Fn() -> () + Sync + Send>;
+
+/// Helper struct to implement the watch method in `Directory` implementations.
+///
+/// It registers callbacks (See `.subscribe(...)`) and
+/// calls them upon calls to `.broadcast(...)`.
+#[derive(Default)]
+pub struct WatchCallbackList {
+    router: RwLock<Vec<Weak<WatchCallback>>>,
+}
+
+/// Controls how long a directory should watch for a file change.
+///
+/// After all the clones of `WatchHandle` are dropped, the associated will not be called when a
+/// file change is detected.
+#[must_use = "This `WatchHandle` controls the lifetime of the watch and should therefore be used."]
+#[derive(Clone)]
+pub struct WatchHandle(Arc<WatchCallback>);
+
+impl WatchCallbackList {
+    /// Suscribes a new callback and returns a handle that controls the lifetime of the callback.
+    pub fn subscribe(&self, watch_callback: WatchCallback) -> WatchHandle {
+        let watch_callback_arc = Arc::new(watch_callback);
+        let watch_callback_weak = Arc::downgrade(&watch_callback_arc);
+        self.router.write().unwrap().push(watch_callback_weak);
+        WatchHandle(watch_callback_arc)
+    }
+
+    fn list_callback(&self) -> Vec<Arc<WatchCallback>> {
+        let mut callbacks = vec![];
+        let mut router_wlock = self.router.write().unwrap();
+        let mut i = 0;
+        while i < router_wlock.len() {
+            if let Some(watch) = router_wlock[i].upgrade() {
+                callbacks.push(watch);
+                i += 1;
+            } else {
+                router_wlock.swap_remove(i);
+            }
+        }
+        callbacks
+    }
+
+    /// Triggers all callbacks
+    pub fn broadcast(&self) {
+        let callbacks = self.list_callback();
+        let spawn_res = std::thread::Builder::new()
+            .name("watch-callbacks".to_string())
+            .spawn(move || {
+                for callback in callbacks {
+                    callback();
+                }
+            });
+        if let Err(err) = spawn_res {
+            error!(
+                "Failed to spawn thread to call watch callbacks. Cause: {:?}",
+                err
+            );
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use directory::WatchCallbackList;
+    use std::mem;
+    use std::sync::atomic::{AtomicUsize, Ordering};
+    use std::sync::Arc;
+    use std::thread;
+    use std::time::Duration;
+
+    const WAIT_TIME: u64 = 20;
+
+    #[test]
+    fn test_watch_event_router_simple() {
+        let watch_event_router = WatchCallbackList::default();
+        let counter: Arc<AtomicUsize> = Default::default();
+        let counter_clone = counter.clone();
+        let inc_callback = Box::new(move || {
+            counter_clone.fetch_add(1, Ordering::SeqCst);
+        });
+        watch_event_router.broadcast();
+        assert_eq!(0, counter.load(Ordering::SeqCst));
+        let handle_a = watch_event_router.subscribe(inc_callback);
+        thread::sleep(Duration::from_millis(WAIT_TIME));
+        assert_eq!(0, counter.load(Ordering::SeqCst));
+        watch_event_router.broadcast();
+        thread::sleep(Duration::from_millis(WAIT_TIME));
+        assert_eq!(1, counter.load(Ordering::SeqCst));
+        watch_event_router.broadcast();
+        watch_event_router.broadcast();
+        watch_event_router.broadcast();
+        thread::sleep(Duration::from_millis(WAIT_TIME));
+        assert_eq!(4, counter.load(Ordering::SeqCst));
+        mem::drop(handle_a);
+        watch_event_router.broadcast();
+        thread::sleep(Duration::from_millis(WAIT_TIME));
+        assert_eq!(4, counter.load(Ordering::SeqCst));
+    }
+
+    #[test]
+    fn test_watch_event_router_multiple_callback_same_key() {
+        let watch_event_router = WatchCallbackList::default();
+        let counter: Arc<AtomicUsize> = Default::default();
+        let inc_callback = |inc: usize| {
+            let counter_clone = counter.clone();
+            Box::new(move || {
+                counter_clone.fetch_add(inc, Ordering::SeqCst);
+            })
+        };
+        let handle_a = watch_event_router.subscribe(inc_callback(1));
+        let handle_a2 = watch_event_router.subscribe(inc_callback(10));
+        thread::sleep(Duration::from_millis(WAIT_TIME));
+        assert_eq!(0, counter.load(Ordering::SeqCst));
+        watch_event_router.broadcast();
+        watch_event_router.broadcast();
+        thread::sleep(Duration::from_millis(WAIT_TIME));
+        assert_eq!(22, counter.load(Ordering::SeqCst));
+        mem::drop(handle_a);
+        watch_event_router.broadcast();
+        thread::sleep(Duration::from_millis(WAIT_TIME));
+        assert_eq!(32, counter.load(Ordering::SeqCst));
+        mem::drop(handle_a2);
+        watch_event_router.broadcast();
+        watch_event_router.broadcast();
+        thread::sleep(Duration::from_millis(WAIT_TIME));
+        assert_eq!(32, counter.load(Ordering::SeqCst));
+    }
+
+    #[test]
+    fn test_watch_event_router_multiple_callback_different_key() {
+        let watch_event_router = WatchCallbackList::default();
+        let counter: Arc<AtomicUsize> = Default::default();
+        let counter_clone = counter.clone();
+        let inc_callback = Box::new(move || {
+            counter_clone.fetch_add(1, Ordering::SeqCst);
+        });
+        let handle_a = watch_event_router.subscribe(inc_callback);
+        assert_eq!(0, counter.load(Ordering::SeqCst));
+        watch_event_router.broadcast();
+        watch_event_router.broadcast();
+        thread::sleep(Duration::from_millis(WAIT_TIME));
+        assert_eq!(2, counter.load(Ordering::SeqCst));
+        thread::sleep(Duration::from_millis(WAIT_TIME));
+        mem::drop(handle_a);
+        watch_event_router.broadcast();
+        thread::sleep(Duration::from_millis(WAIT_TIME));
+        assert_eq!(2, counter.load(Ordering::SeqCst));
+    }
+
+}

src/error.rs

@@ -2,9 +2,9 @@
 
 use std::io;
 
+use directory::error::LockError;
 use directory::error::{IOError, OpenDirectoryError, OpenReadError, OpenWriteError};
 use fastfield::FastFieldNotAvailableError;
-use indexer::LockType;
 use query;
 use schema;
 use serde_json;
@@ -57,11 +57,8 @@ pub enum TantivyError {
     #[fail(display = "Index already exists")]
     IndexAlreadyExists,
     /// Failed to acquire file lock
-    #[fail(
-        display = "Failed to acquire Lockfile: {:?}. Possible causes: another IndexWriter instance or panic during previous lock drop.",
-        _0
-    )]
-    LockFailure(LockType),
+    #[fail(display = "Failed to acquire Lockfile: {:?}. {:?}", _0, _1)]
+    LockFailure(LockError, Option<String>),
     /// IO Error.
     #[fail(display = "An IO error occurred: '{}'", _0)]
     IOError(#[cause] IOError),
@@ -100,6 +97,12 @@ impl From<FastFieldNotAvailableError> for TantivyError {
     }
 }
 
+impl From<LockError> for TantivyError {
+    fn from(lock_error: LockError) -> TantivyError {
+        TantivyError::LockFailure(lock_error, None)
+    }
+}
+
 impl From<IOError> for TantivyError {
     fn from(io_error: IOError) -> TantivyError {
         TantivyError::IOError(io_error)
@@ -159,6 +162,7 @@ impl From<OpenDirectoryError> for TantivyError {
             OpenDirectoryError::NotADirectory(directory_path) => {
                 TantivyError::InvalidArgument(format!("{:?} is not a directory", directory_path))
             }
+            OpenDirectoryError::IoError(err) => TantivyError::IOError(IOError::from(err)),
         }
     }
 }

src/fastfield/bytes/mod.rs

@@ -22,9 +22,7 @@ mod tests {
         index_writer.add_document(doc!(field=>vec![1u8, 3, 5, 7, 9]));
         index_writer.add_document(doc!(field=>vec![0u8; 1000]));
         assert!(index_writer.commit().is_ok());
-
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        let searcher = index.reader().unwrap().searcher();
         let reader = searcher.segment_reader(0);
         let bytes_reader = reader.bytes_fast_field_reader(field).unwrap();
 

src/fastfield/multivalued/mod.rs

@@ -7,7 +7,13 @@ pub use self::writer::MultiValueIntFastFieldWriter;
 #[cfg(test)]
 mod tests {
 
+    extern crate time;
+
+    use self::time::Duration;
+    use collector::TopDocs;
+    use query::QueryParser;
     use schema::Cardinality;
+    use schema::Facet;
     use schema::IntOptions;
     use schema::Schema;
     use Index;
@@ -28,11 +34,12 @@ mod tests {
         index_writer.add_document(doc!(field=>5u64, field=>20u64,field=>1u64));
         assert!(index_writer.commit().is_ok());
 
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
-        let reader = searcher.segment_reader(0);
+        let searcher = index.reader().unwrap().searcher();
+        let segment_reader = searcher.segment_reader(0);
         let mut vals = Vec::new();
-        let multi_value_reader = reader.multi_fast_field_reader::<u64>(field).unwrap();
+        let multi_value_reader = segment_reader
+            .multi_fast_field_reader::<u64>(field)
+            .unwrap();
         {
             multi_value_reader.get_vals(2, &mut vals);
             assert_eq!(&vals, &[4u64]);
@@ -47,6 +54,133 @@ mod tests {
         }
     }
 
+    #[test]
+    fn test_multivalued_date() {
+        let mut schema_builder = Schema::builder();
+        let date_field = schema_builder.add_date_field(
+            "multi_date_field",
+            IntOptions::default()
+                .set_fast(Cardinality::MultiValues)
+                .set_indexed()
+                .set_stored(),
+        );
+        let time_i =
+            schema_builder.add_i64_field("time_stamp_i", IntOptions::default().set_stored());
+        let schema = schema_builder.build();
+        let index = Index::create_in_ram(schema);
+        let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
+        let first_time_stamp = chrono::Utc::now();
+        index_writer.add_document(
+            doc!(date_field=>first_time_stamp, date_field=>first_time_stamp, time_i=>1i64),
+        );
+        index_writer.add_document(doc!(time_i=>0i64));
+        // add one second
+        index_writer
+            .add_document(doc!(date_field=>first_time_stamp + Duration::seconds(1), time_i=>2i64));
+        // add another second
+        let two_secs_ahead = first_time_stamp + Duration::seconds(2);
+        index_writer.add_document(doc!(date_field=>two_secs_ahead, date_field=>two_secs_ahead,date_field=>two_secs_ahead, time_i=>3i64));
+        assert!(index_writer.commit().is_ok());
+
+        let reader = index.reader().unwrap();
+        let searcher = reader.searcher();
+        let reader = searcher.segment_reader(0);
+        assert_eq!(reader.num_docs(), 4);
+
+        {
+            let parser = QueryParser::for_index(&index, vec![date_field]);
+            let query = parser
+                .parse_query(&format!("\"{}\"", first_time_stamp.to_rfc3339()).to_string())
+                .expect("could not parse query");
+            let results = searcher
+                .search(&query, &TopDocs::with_limit(5))
+                .expect("could not query index");
+
+            assert_eq!(results.len(), 1);
+            for (_score, doc_address) in results {
+                let retrieved_doc = searcher.doc(doc_address).expect("cannot fetch doc");
+                assert_eq!(
+                    retrieved_doc
+                        .get_first(date_field)
+                        .expect("cannot find value")
+                        .date_value()
+                        .timestamp(),
+                    first_time_stamp.timestamp()
+                );
+                assert_eq!(
+                    retrieved_doc
+                        .get_first(time_i)
+                        .expect("cannot find value")
+                        .i64_value(),
+                    1i64
+                );
+            }
+        }
+
+        {
+            let parser = QueryParser::for_index(&index, vec![date_field]);
+            let query = parser
+                .parse_query(&format!("\"{}\"", two_secs_ahead.to_rfc3339()).to_string())
+                .expect("could not parse query");
+            let results = searcher
+                .search(&query, &TopDocs::with_limit(5))
+                .expect("could not query index");
+
+            assert_eq!(results.len(), 1);
+
+            for (_score, doc_address) in results {
+                let retrieved_doc = searcher.doc(doc_address).expect("cannot fetch doc");
+                assert_eq!(
+                    retrieved_doc
+                        .get_first(date_field)
+                        .expect("cannot find value")
+                        .date_value()
+                        .timestamp(),
+                    two_secs_ahead.timestamp()
+                );
+                assert_eq!(
+                    retrieved_doc
+                        .get_first(time_i)
+                        .expect("cannot find value")
+                        .i64_value(),
+                    3i64
+                );
+            }
+        }
+
+        // TODO: support Date range queries
+        //        {
+        //            let parser = QueryParser::for_index(&index, vec![date_field]);
+        //            let range_q = format!("\"{}\"..\"{}\"",
+        //                                  (first_time_stamp + Duration::seconds(1)).to_rfc3339(),
+        //                                  (first_time_stamp + Duration::seconds(3)).to_rfc3339()
+        //            );
+        //            let query = parser.parse_query(&range_q)
+        //                .expect("could not parse query");
+        //            let results = searcher.search(&query, &TopDocs::with_limit(5))
+        //                .expect("could not query index");
+        //
+        //
+        //            assert_eq!(results.len(), 2);
+        //            for (i, doc_pair) in results.iter().enumerate() {
+        //                let retrieved_doc = searcher.doc(doc_pair.1).expect("cannot fetch doc");
+        //                let offset_sec = match i {
+        //                    0 => 1,
+        //                    1 => 3,
+        //                    _ => panic!("should not have more than 2 docs")
+        //                };
+        //                let time_i_val = match i {
+        //                    0 => 2,
+        //                    1 => 3,
+        //                    _ => panic!("should not have more than 2 docs")
+        //                };
+        //                assert_eq!(retrieved_doc.get_first(date_field).expect("cannot find value").date_value().timestamp(),
+        //                           (first_time_stamp + Duration::seconds(offset_sec)).timestamp());
+        //                assert_eq!(retrieved_doc.get_first(time_i).expect("cannot find value").i64_value(), time_i_val);
+        //            }
+        //        }
+    }
+
     #[test]
     fn test_multivalued_i64() {
         let mut schema_builder = Schema::builder();
@@ -63,8 +197,7 @@ mod tests {
         index_writer.add_document(doc!(field=> -5i64, field => -20i64, field=>1i64));
         assert!(index_writer.commit().is_ok());
 
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        let searcher = index.reader().unwrap().searcher();
         let reader = searcher.segment_reader(0);
         let mut vals = Vec::new();
         let multi_value_reader = reader.multi_fast_field_reader::<i64>(field).unwrap();
@@ -85,4 +218,17 @@ mod tests {
             assert_eq!(&vals, &[-5i64, -20i64, 1i64]);
         }
     }
+    #[test]
+    #[ignore]
+    fn test_many_facets() {
+        let mut schema_builder = Schema::builder();
+        let field = schema_builder.add_facet_field("facetfield");
+        let schema = schema_builder.build();
+        let index = Index::create_in_ram(schema);
+        let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
+        for i in 0..100_000 {
+            index_writer.add_document(doc!(field=> Facet::from(format!("/lang/{}", i).as_str())));
+        }
+        assert!(index_writer.commit().is_ok());
+    }
 }

src/fastfield/multivalued/reader.rs

@@ -39,7 +39,7 @@ impl<Item: FastValue> MultiValueIntFastFieldReader<Item> {
         let (start, stop) = self.range(doc);
         let len = (stop - start) as usize;
         vals.resize(len, Item::default());
-        self.vals_reader.get_range(start as u32, &mut vals[..]);
+        self.vals_reader.get_range_u64(start, &mut vals[..]);
     }
 }
 
@@ -75,8 +75,7 @@ mod tests {
             index_writer.add_document(doc);
         }
         index_writer.commit().expect("Commit failed");
-        index.load_searchers().expect("Reloading searchers");
-        let searcher = index.searcher();
+        let searcher = index.reader().unwrap().searcher();
         let segment_reader = searcher.segment_reader(0);
         let mut facet_reader = segment_reader.facet_reader(facet_field).unwrap();
 

src/fastfield/multivalued/writer.rs

@@ -32,7 +32,7 @@ use DocId;
 /// term ids when the segment is getting serialized.
 pub struct MultiValueIntFastFieldWriter {
     field: Field,
-    vals: Vec<u64>,
+    vals: Vec<UnorderedTermId>,
     doc_index: Vec<u64>,
     is_facet: bool,
 }

src/fastfield/reader.rs

@@ -59,7 +59,29 @@ impl<Item: FastValue> FastFieldReader<Item> {
     /// May panic if `doc` is greater than the segment
     // `maxdoc`.
     pub fn get(&self, doc: DocId) -> Item {
-        Item::from_u64(self.min_value_u64 + self.bit_unpacker.get(doc as usize))
+        self.get_u64(u64::from(doc))
+    }
+
+    pub(crate) fn get_u64(&self, doc: u64) -> Item {
+        Item::from_u64(self.min_value_u64 + self.bit_unpacker.get(doc))
+    }
+
+    /// Internally `multivalued` also use SingleValue Fast fields.
+    /// It works as follows... A first column contains the list of start index
+    /// for each document, a second column contains the actual values.
+    ///
+    /// The values associated to a given doc, are then
+    ///  `second_column[first_column.get(doc)..first_column.get(doc+1)]`.
+    ///
+    /// Which means single value fast field reader can be indexed internally with
+    /// something different from a `DocId`. For this use case, we want to use `u64`
+    /// values.
+    ///
+    /// See `get_range` for an actual documentation about this method.
+    pub(crate) fn get_range_u64(&self, start: u64, output: &mut [Item]) {
+        for (i, out) in output.iter_mut().enumerate() {
+            *out = self.get_u64(start + (i as u64));
+        }
     }
 
     /// Fills an output buffer with the fast field values
@@ -75,13 +97,8 @@ impl<Item: FastValue> FastFieldReader<Item> {
     ///
     /// May panic if `start + output.len()` is greater than
     /// the segment's `maxdoc`.
-    ///
-    // TODO change start to `u64`.
-    // For multifastfield, start is an index in a second fastfield, not a `DocId`
-    pub fn get_range(&self, start: u32, output: &mut [Item]) {
-        for (i, out) in output.iter_mut().enumerate() {
-            *out = self.get(start + i as u32);
-        }
+    pub fn get_range(&self, start: DocId, output: &mut [Item]) {
+        self.get_range_u64(u64::from(start), output);
     }
 
     /// Returns the minimum value for this fast field.

src/functional_test.rs

@@ -17,11 +17,12 @@ fn check_index_content(searcher: &Searcher, vals: &HashSet<u64>) {
 fn test_indexing() {
     let mut schema_builder = Schema::builder();
 
-    let id_field = schema_builder.add_u64_field("id", INT_INDEXED);
-    let multiples_field = schema_builder.add_u64_field("multiples", INT_INDEXED);
+    let id_field = schema_builder.add_u64_field("id", INDEXED);
+    let multiples_field = schema_builder.add_u64_field("multiples", INDEXED);
     let schema = schema_builder.build();
 
     let index = Index::create_from_tempdir(schema).unwrap();
+    let reader = index.reader().unwrap();
 
     let mut rng = thread_rng();
 
@@ -36,8 +37,8 @@ fn test_indexing() {
             index_writer.commit().expect("Commit failed");
             committed_docs.extend(&uncommitted_docs);
             uncommitted_docs.clear();
-            index.load_searchers().unwrap();
-            let searcher = index.searcher();
+            reader.reload().unwrap();
+            let searcher = reader.searcher();
             // check that everything is correct.
             check_index_content(&searcher, &committed_docs);
         } else {

src/indexer/directory_lock.rs

@@ -1,131 +0,0 @@
-use directory::error::OpenWriteError;
-use std::io::Write;
-use std::path::{Path, PathBuf};
-use std::thread;
-use std::time::Duration;
-use Directory;
-use TantivyError;
-
-#[derive(Debug, Clone, Copy)]
-pub enum LockType {
-    /// Only one process should be able to write tantivy's index at a time.
-    /// This lock file, when present, is in charge of preventing other processes to open an IndexWriter.
-    ///
-    /// If the process is killed and this file remains, it is safe to remove it manually.
-    ///
-    /// Failing to acquire this lock usually means a misuse of tantivy's API,
-    /// (creating more than one instance of the `IndexWriter`), are a spurious
-    /// lock file remaining after a crash. In the latter case, removing the file after
-    /// checking no process running tantivy is running is safe.
-    IndexWriterLock,
-    /// The meta lock file is here to protect the segment files being opened by
-    /// `.load_searchers()` from being garbage collected.
-    /// It makes it possible for another process to safely consume
-    /// our index in-writing. Ideally, we may have prefered `RWLock` semantics
-    /// here, but it is difficult to achieve on Windows.
-    ///
-    /// Opening segment readers is a very fast process.
-    /// Right now if the lock cannot be acquire on the first attempt, the logic
-    /// is very simplistic. We retry after `100ms` until we effectively
-    /// acquire the lock.
-    /// This lock should not have much contention in normal usage.
-    MetaLock,
-}
-
-/// Retry the logic of acquiring locks is pretty simple.
-/// We just retry `n` times after a given `duratio`, both
-/// depending on the type of lock.
-struct RetryPolicy {
-    num_retries: usize,
-    wait_in_ms: u64,
-}
-
-impl RetryPolicy {
-    fn no_retry() -> RetryPolicy {
-        RetryPolicy {
-            num_retries: 0,
-            wait_in_ms: 0,
-        }
-    }
-
-    fn wait_and_retry(&mut self) -> bool {
-        if self.num_retries == 0 {
-            false
-        } else {
-            self.num_retries -= 1;
-            let wait_duration = Duration::from_millis(self.wait_in_ms);
-            thread::sleep(wait_duration);
-            true
-        }
-    }
-}
-
-impl LockType {
-    fn retry_policy(self) -> RetryPolicy {
-        match self {
-            LockType::IndexWriterLock => RetryPolicy::no_retry(),
-            LockType::MetaLock => RetryPolicy {
-                num_retries: 100,
-                wait_in_ms: 100,
-            },
-        }
-    }
-
-    fn try_acquire_lock(self, directory: &mut Directory) -> Result<DirectoryLock, TantivyError> {
-        let path = self.filename();
-        let mut write = directory.open_write(path).map_err(|e| match e {
-            OpenWriteError::FileAlreadyExists(_) => TantivyError::LockFailure(self),
-            OpenWriteError::IOError(io_error) => TantivyError::IOError(io_error),
-        })?;
-        write.flush()?;
-        Ok(DirectoryLock {
-            directory: directory.box_clone(),
-            path: path.to_owned(),
-        })
-    }
-
-    /// Acquire a lock in the given directory.
-    pub fn acquire_lock(self, directory: &Directory) -> Result<DirectoryLock, TantivyError> {
-        let mut box_directory = directory.box_clone();
-        let mut retry_policy = self.retry_policy();
-        loop {
-            let lock_result = self.try_acquire_lock(&mut *box_directory);
-            match lock_result {
-                Ok(result) => {
-                    return Ok(result);
-                }
-                Err(TantivyError::LockFailure(ref filepath)) => {
-                    if !retry_policy.wait_and_retry() {
-                        return Err(TantivyError::LockFailure(filepath.to_owned()));
-                    }
-                }
-                Err(_) => {}
-            }
-        }
-    }
-
-    fn filename(&self) -> &Path {
-        match *self {
-            LockType::MetaLock => Path::new(".tantivy-meta.lock"),
-            LockType::IndexWriterLock => Path::new(".tantivy-indexer.lock"),
-        }
-    }
-}
-
-/// The `DirectoryLock` is an object that represents a file lock.
-/// See  [`LockType`](struct.LockType.html)
-///
-/// It is transparently associated to a lock file, that gets deleted
-/// on `Drop.` The lock is release automatically on `Drop`.
-pub struct DirectoryLock {
-    directory: Box<Directory>,
-    path: PathBuf,
-}
-
-impl Drop for DirectoryLock {
-    fn drop(&mut self) {
-        if let Err(e) = self.directory.delete(&*self.path) {
-            error!("Failed to remove the lock file. {:?}", e);
-        }
-    }
-}

src/indexer/index_writer.rs

@@ -1,4 +1,4 @@
-use super::operation::AddOperation;
+use super::operation::{AddOperation, UserOperation};
 use super::segment_updater::SegmentUpdater;
 use super::PreparedCommit;
 use bit_set::BitSet;
@@ -9,15 +9,15 @@ use core::SegmentId;
 use core::SegmentMeta;
 use core::SegmentReader;
 use crossbeam::channel;
+use directory::DirectoryLock;
 use docset::DocSet;
 use error::TantivyError;
 use fastfield::write_delete_bitset;
-use futures::sync::oneshot::Receiver;
+use futures::{Canceled, Future};
 use indexer::delete_queue::{DeleteCursor, DeleteQueue};
 use indexer::doc_opstamp_mapping::DocToOpstampMapping;
 use indexer::operation::DeleteOperation;
 use indexer::stamper::Stamper;
-use indexer::DirectoryLock;
 use indexer::MergePolicy;
 use indexer::SegmentEntry;
 use indexer::SegmentWriter;
@@ -26,6 +26,8 @@ use schema::Document;
 use schema::IndexRecordOption;
 use schema::Term;
 use std::mem;
+use std::ops::Range;
+use std::sync::Arc;
 use std::thread;
 use std::thread::JoinHandle;
 use Result;
@@ -42,8 +44,8 @@ pub const HEAP_SIZE_MAX: usize = u32::max_value() as usize - MARGIN_IN_BYTES;
 // reaches `PIPELINE_MAX_SIZE_IN_DOCS`
 const PIPELINE_MAX_SIZE_IN_DOCS: usize = 10_000;
 
-type DocumentSender = channel::Sender<AddOperation>;
-type DocumentReceiver = channel::Receiver<AddOperation>;
+type OperationSender = channel::Sender<Vec<AddOperation>>;
+type OperationReceiver = channel::Receiver<Vec<AddOperation>>;
 
 /// Split the thread memory budget into
 /// - the heap size
@@ -83,8 +85,8 @@ pub struct IndexWriter {
 
     workers_join_handle: Vec<JoinHandle<Result<()>>>,
 
-    document_receiver: DocumentReceiver,
-    document_sender: DocumentSender,
+    operation_receiver: OperationReceiver,
+    operation_sender: OperationSender,
 
     segment_updater: SegmentUpdater,
 
@@ -131,7 +133,7 @@ pub fn open_index_writer(
         let err_msg = format!("The heap size per thread cannot exceed {}", HEAP_SIZE_MAX);
         return Err(TantivyError::InvalidArgument(err_msg));
     }
-    let (document_sender, document_receiver): (DocumentSender, DocumentReceiver) =
+    let (document_sender, document_receiver): (OperationSender, OperationReceiver) =
         channel::bounded(PIPELINE_MAX_SIZE_IN_DOCS);
 
     let delete_queue = DeleteQueue::new();
@@ -149,8 +151,8 @@ pub fn open_index_writer(
         heap_size_in_bytes_per_thread,
         index: index.clone(),
 
-        document_receiver,
-        document_sender,
+        operation_receiver: document_receiver,
+        operation_sender: document_sender,
 
         segment_updater,
 
@@ -257,7 +259,7 @@ pub fn advance_deletes(
             write_delete_bitset(&delete_bitset, &mut delete_file)?;
         }
     }
-    segment_entry.set_meta((*segment.meta()).clone());
+    segment_entry.set_meta(segment.meta().clone());
     Ok(())
 }
 
@@ -265,7 +267,7 @@ fn index_documents(
     memory_budget: usize,
     segment: &Segment,
     generation: usize,
-    document_iterator: &mut Iterator<Item = AddOperation>,
+    document_iterator: &mut Iterator<Item = Vec<AddOperation>>,
     segment_updater: &mut SegmentUpdater,
     mut delete_cursor: DeleteCursor,
 ) -> Result<bool> {
@@ -273,11 +275,11 @@ fn index_documents(
     let segment_id = segment.id();
     let table_size = initial_table_size(memory_budget);
     let mut segment_writer = SegmentWriter::for_segment(table_size, segment.clone(), &schema)?;
-    for doc in document_iterator {
-        segment_writer.add_document(doc, &schema)?;
-
+    for documents in document_iterator {
+        for doc in documents {
+            segment_writer.add_document(doc, &schema)?;
+        }
         let mem_usage = segment_writer.mem_usage();
-
         if mem_usage >= memory_budget - MARGIN_IN_BYTES {
             info!(
                 "Buffer limit reached, flushing segment with maxdoc={}.",
@@ -333,7 +335,7 @@ impl IndexWriter {
     pub fn wait_merging_threads(mut self) -> Result<()> {
         // this will stop the indexing thread,
         // dropping the last reference to the segment_updater.
-        drop(self.document_sender);
+        drop(self.operation_sender);
 
         let former_workers_handles = mem::replace(&mut self.workers_join_handle, vec![]);
         for join_handle in former_workers_handles {
@@ -366,20 +368,23 @@ impl IndexWriter {
             .add_segment(self.generation, segment_entry);
     }
 
-    /// *Experimental & Advanced API* Creates a new segment.
-    /// and marks it as currently in write.
+    /// Creates a new segment.
     ///
     /// This method is useful only for users trying to do complex
     /// operations, like converting an index format to another.
+    ///
+    /// It is safe to start writing file associated to the new `Segment`.
+    /// These will not be garbage collected as long as an instance object of
+    /// `SegmentMeta` object associated to the new `Segment` is "alive".
     pub fn new_segment(&self) -> Segment {
-        self.segment_updater.new_segment()
+        self.index.new_segment()
     }
 
     /// Spawns a new worker thread for indexing.
     /// The thread consumes documents from the pipeline.
     ///
     fn add_indexing_worker(&mut self) -> Result<()> {
-        let document_receiver_clone = self.document_receiver.clone();
+        let document_receiver_clone = self.operation_receiver.clone();
         let mut segment_updater = self.segment_updater.clone();
 
         let generation = self.generation;
@@ -387,6 +392,7 @@ impl IndexWriter {
         let mut delete_cursor = self.delete_queue.cursor();
 
         let mem_budget = self.heap_size_in_bytes_per_thread;
+        let index = self.index.clone();
         let join_handle: JoinHandle<Result<()>> = thread::Builder::new()
             .name(format!(
                 "thrd-tantivy-index{}-gen{}",
@@ -404,15 +410,19 @@ impl IndexWriter {
                     // this is a valid guarantee as the
                     // peeked document now belongs to
                     // our local iterator.
-                    if let Some(operation) = document_iterator.peek() {
-                        delete_cursor.skip_to(operation.opstamp);
+                    if let Some(operations) = document_iterator.peek() {
+                        if let Some(first) = operations.first() {
+                            delete_cursor.skip_to(first.opstamp);
+                        } else {
+                            return Ok(());
+                        }
                     } else {
                         // No more documents.
                         // Happens when there is a commit, or if the `IndexWriter`
                         // was dropped.
                         return Ok(());
                     }
-                    let segment = segment_updater.new_segment();
+                    let segment = index.new_segment();
                     index_documents(
                         mem_budget,
                         &segment,
@@ -429,7 +439,7 @@ impl IndexWriter {
     }
 
     /// Accessor to the merge policy.
-    pub fn get_merge_policy(&self) -> Box<MergePolicy> {
+    pub fn get_merge_policy(&self) -> Arc<Box<MergePolicy>> {
         self.segment_updater.get_merge_policy()
     }
 
@@ -454,7 +464,10 @@ impl IndexWriter {
     /// Merges a given list of segments
     ///
     /// `segment_ids` is required to be non-empty.
-    pub fn merge(&mut self, segment_ids: &[SegmentId]) -> Result<Receiver<SegmentMeta>> {
+    pub fn merge(
+        &mut self,
+        segment_ids: &[SegmentId],
+    ) -> Result<impl Future<Item = SegmentMeta, Error = Canceled>> {
         self.segment_updater.start_merge(segment_ids)
     }
 
@@ -466,11 +479,11 @@ impl IndexWriter {
     /// when no documents are remaining.
     ///
     /// Returns the former segment_ready channel.
-    fn recreate_document_channel(&mut self) -> DocumentReceiver {
-        let (document_sender, document_receiver): (DocumentSender, DocumentReceiver) =
+    fn recreate_document_channel(&mut self) -> OperationReceiver {
+        let (document_sender, document_receiver): (OperationSender, OperationReceiver) =
             channel::bounded(PIPELINE_MAX_SIZE_IN_DOCS);
-        mem::replace(&mut self.document_sender, document_sender);
-        mem::replace(&mut self.document_receiver, document_receiver)
+        mem::replace(&mut self.operation_sender, document_sender);
+        mem::replace(&mut self.operation_receiver, document_receiver)
     }
 
     /// Rollback to the last commit
@@ -488,7 +501,7 @@ impl IndexWriter {
         // segment updates will be ignored.
         self.segment_updater.kill();
 
-        let document_receiver = self.document_receiver.clone();
+        let document_receiver = self.operation_receiver.clone();
 
         // take the directory lock to create a new index_writer.
         let directory_lock = self
@@ -557,8 +570,7 @@ impl IndexWriter {
         // and recreate a new one channels.
         self.recreate_document_channel();
 
-        let former_workers_join_handle =
-            mem::replace(&mut self.workers_join_handle, Vec::new());
+        let former_workers_join_handle = mem::replace(&mut self.workers_join_handle, Vec::new());
 
         for worker_handle in former_workers_join_handle {
             let indexing_worker_result = worker_handle
@@ -636,32 +648,176 @@ impl IndexWriter {
     pub fn add_document(&mut self, document: Document) -> u64 {
         let opstamp = self.stamper.stamp();
         let add_operation = AddOperation { opstamp, document };
-        let send_result = self.document_sender.send(add_operation);
+        let send_result = self.operation_sender.send(vec![add_operation]);
         if let Err(e) = send_result {
             panic!("Failed to index document. Sending to indexing channel failed. This probably means all of the indexing threads have panicked. {:?}", e);
         }
         opstamp
     }
+
+    /// Gets a range of stamps from the stamper and "pops" the last stamp
+    /// from the range returning a tuple of the last optstamp and the popped
+    /// range.
+    ///
+    /// The total number of stamps generated by this method is `count + 1`;
+    /// each operation gets a stamp from the `stamps` iterator and `last_opstamp`
+    /// is for the batch itself.
+    fn get_batch_opstamps(&mut self, count: u64) -> (u64, Range<u64>) {
+        let Range { start, end } = self.stamper.stamps(count + 1u64);
+        let last_opstamp = end - 1;
+        let stamps = Range {
+            start,
+            end: last_opstamp,
+        };
+        (last_opstamp, stamps)
+    }
+
+    /// Runs a group of document operations ensuring that the operations are
+    /// assigned contigous u64 opstamps and that add operations of the same
+    /// group are flushed into the same segment.
+    ///
+    /// If the indexing pipeline is full, this call may block.
+    ///
+    /// Each operation of the given `user_operations` will receive an in-order,
+    /// contiguous u64 opstamp. The entire batch itself is also given an
+    /// opstamp that is 1 greater than the last given operation. This
+    /// `batch_opstamp` is the return value of `run`. An empty group of
+    /// `user_operations`, an empty `Vec<UserOperation>`, still receives
+    /// a valid opstamp even though no changes were _actually_ made to the index.
+    ///
+    /// Like adds and deletes (see `IndexWriter.add_document` and
+    /// `IndexWriter.delete_term`), the changes made by calling `run` will be
+    /// visible to readers only after calling `commit()`.
+    pub fn run(&mut self, user_operations: Vec<UserOperation>) -> u64 {
+        let count = user_operations.len() as u64;
+        if count == 0 {
+            return self.stamper.stamp();
+        }
+        let (batch_opstamp, stamps) = self.get_batch_opstamps(count);
+
+        let mut adds: Vec<AddOperation> = Vec::new();
+
+        for (user_op, opstamp) in user_operations.into_iter().zip(stamps) {
+            match user_op {
+                UserOperation::Delete(term) => {
+                    let delete_operation = DeleteOperation { opstamp, term };
+                    self.delete_queue.push(delete_operation);
+                }
+                UserOperation::Add(document) => {
+                    let add_operation = AddOperation { opstamp, document };
+                    adds.push(add_operation);
+                }
+            }
+        }
+        let send_result = self.operation_sender.send(adds);
+        if let Err(e) = send_result {
+            panic!("Failed to index document. Sending to indexing channel failed. This probably means all of the indexing threads have panicked. {:?}", e);
+        };
+
+        batch_opstamp
+    }
 }
 
 #[cfg(test)]
 mod tests {
 
+    use super::super::operation::UserOperation;
     use super::initial_table_size;
+    use collector::TopDocs;
+    use directory::error::LockError;
     use error::*;
     use indexer::NoMergePolicy;
-    use schema::{self, Document};
+    use query::TermQuery;
+    use schema::{self, IndexRecordOption};
     use Index;
+    use ReloadPolicy;
     use Term;
 
+    #[test]
+    fn test_operations_group() {
+        // an operations group with 2 items should cause 3 opstamps 0, 1, and 2.
+        let mut schema_builder = schema::Schema::builder();
+        let text_field = schema_builder.add_text_field("text", schema::TEXT);
+        let index = Index::create_in_ram(schema_builder.build());
+        let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
+        let operations = vec![
+            UserOperation::Add(doc!(text_field=>"a")),
+            UserOperation::Add(doc!(text_field=>"b")),
+        ];
+        let batch_opstamp1 = index_writer.run(operations);
+        assert_eq!(batch_opstamp1, 2u64);
+    }
+
+    #[test]
+    fn test_ordered_batched_operations() {
+        // * one delete for `doc!(field=>"a")`
+        // * one add for `doc!(field=>"a")`
+        // * one add for `doc!(field=>"b")`
+        // * one delete for `doc!(field=>"b")`
+        // after commit there is one doc with "a" and 0 doc with "b"
+        let mut schema_builder = schema::Schema::builder();
+        let text_field = schema_builder.add_text_field("text", schema::TEXT);
+        let index = Index::create_in_ram(schema_builder.build());
+        let reader = index
+            .reader_builder()
+            .reload_policy(ReloadPolicy::Manual)
+            .try_into()
+            .unwrap();
+        let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
+        let a_term = Term::from_field_text(text_field, "a");
+        let b_term = Term::from_field_text(text_field, "b");
+        let operations = vec![
+            UserOperation::Delete(a_term),
+            UserOperation::Add(doc!(text_field=>"a")),
+            UserOperation::Add(doc!(text_field=>"b")),
+            UserOperation::Delete(b_term),
+        ];
+
+        index_writer.run(operations);
+        index_writer.commit().expect("failed to commit");
+        reader.reload().expect("failed to load searchers");
+
+        let a_term = Term::from_field_text(text_field, "a");
+        let b_term = Term::from_field_text(text_field, "b");
+
+        let a_query = TermQuery::new(a_term, IndexRecordOption::Basic);
+        let b_query = TermQuery::new(b_term, IndexRecordOption::Basic);
+
+        let searcher = reader.searcher();
+
+        let a_docs = searcher
+            .search(&a_query, &TopDocs::with_limit(1))
+            .expect("search for a failed");
+
+        let b_docs = searcher
+            .search(&b_query, &TopDocs::with_limit(1))
+            .expect("search for b failed");
+
+        assert_eq!(a_docs.len(), 1);
+        assert_eq!(b_docs.len(), 0);
+    }
+
+    #[test]
+    fn test_empty_operations_group() {
+        let schema_builder = schema::Schema::builder();
+        let index = Index::create_in_ram(schema_builder.build());
+        let mut index_writer = index.writer(3_000_000).unwrap();
+        let operations1 = vec![];
+        let batch_opstamp1 = index_writer.run(operations1);
+        assert_eq!(batch_opstamp1, 0u64);
+        let operations2 = vec![];
+        let batch_opstamp2 = index_writer.run(operations2);
+        assert_eq!(batch_opstamp2, 1u64);
+    }
+
     #[test]
     fn test_lockfile_stops_duplicates() {
         let schema_builder = schema::Schema::builder();
         let index = Index::create_in_ram(schema_builder.build());
-        let _index_writer = index.writer(40_000_000).unwrap();
-        match index.writer(40_000_000) {
-            Err(TantivyError::LockFailure(_)) => {}
-            _ => panic!("Expected FileAlreadyExists error"),
+        let _index_writer = index.writer(3_000_000).unwrap();
+        match index.writer(3_000_000) {
+            Err(TantivyError::LockFailure(LockError::LockBusy, _)) => {}
+            _ => panic!("Expected a `LockFailure` error"),
         }
     }
 
@@ -673,8 +829,7 @@ mod tests {
         match index.writer_with_num_threads(1, 3_000_000) {
             Err(err) => {
                 let err_msg = err.to_string();
-                assert!(err_msg.contains("Lockfile"));
-                assert!(err_msg.contains("Possible causes:"))
+                assert!(err_msg.contains("already an `IndexWriter`"));
             }
             _ => panic!("Expected LockfileAlreadyExists error"),
         }
@@ -684,7 +839,7 @@ mod tests {
     fn test_set_merge_policy() {
         let schema_builder = schema::Schema::builder();
         let index = Index::create_in_ram(schema_builder.build());
-        let index_writer = index.writer(40_000_000).unwrap();
+        let index_writer = index.writer(3_000_000).unwrap();
         assert_eq!(
             format!("{:?}", index_writer.get_merge_policy()),
             "LogMergePolicy { min_merge_size: 8, min_layer_size: 10000, \
@@ -703,11 +858,11 @@ mod tests {
         let schema_builder = schema::Schema::builder();
         let index = Index::create_in_ram(schema_builder.build());
         {
-            let _index_writer = index.writer(40_000_000).unwrap();
+            let _index_writer = index.writer(3_000_000).unwrap();
             // the lock should be released when the
             // index_writer leaves the scope.
         }
-        let _index_writer_two = index.writer(40_000_000).unwrap();
+        let _index_writer_two = index.writer(3_000_000).unwrap();
     }
 
     #[test]
@@ -715,9 +870,13 @@ mod tests {
         let mut schema_builder = schema::Schema::builder();
         let text_field = schema_builder.add_text_field("text", schema::TEXT);
         let index = Index::create_in_ram(schema_builder.build());
-
+        let reader = index
+            .reader_builder()
+            .reload_policy(ReloadPolicy::Manual)
+            .try_into()
+            .unwrap();
         let num_docs_containing = |s: &str| {
-            let searcher = index.searcher();
+            let searcher = reader.searcher();
             let term = Term::from_field_text(text_field, s);
             searcher.doc_freq(&term)
         };
@@ -727,21 +886,20 @@ mod tests {
             let mut index_writer = index.writer(3_000_000).unwrap();
             index_writer.add_document(doc!(text_field=>"a"));
             index_writer.rollback().unwrap();
-
             assert_eq!(index_writer.commit_opstamp(), 0u64);
             assert_eq!(num_docs_containing("a"), 0);
             {
                 index_writer.add_document(doc!(text_field=>"b"));
                 index_writer.add_document(doc!(text_field=>"c"));
             }
-            assert_eq!(index_writer.commit().unwrap(), 2u64);
-            index.load_searchers().unwrap();
+            assert!(index_writer.commit().is_ok());
+            reader.reload().unwrap();
             assert_eq!(num_docs_containing("a"), 0);
             assert_eq!(num_docs_containing("b"), 1);
             assert_eq!(num_docs_containing("c"), 1);
         }
-        index.load_searchers().unwrap();
-        index.searcher();
+        reader.reload().unwrap();
+        reader.searcher();
     }
 
     #[test]
@@ -749,32 +907,33 @@ mod tests {
         let mut schema_builder = schema::Schema::builder();
         let text_field = schema_builder.add_text_field("text", schema::TEXT);
         let index = Index::create_in_ram(schema_builder.build());
+        let reader = index
+            .reader_builder()
+            .reload_policy(ReloadPolicy::Manual)
+            .try_into()
+            .unwrap();
         let num_docs_containing = |s: &str| {
-            let searcher = index.searcher();
             let term_a = Term::from_field_text(text_field, s);
-            searcher.doc_freq(&term_a)
+            reader.searcher().doc_freq(&term_a)
         };
         {
             // writing the segment
             let mut index_writer = index.writer(12_000_000).unwrap();
             // create 8 segments with 100 tiny docs
             for _doc in 0..100 {
-                let mut doc = Document::default();
-                doc.add_text(text_field, "a");
-                index_writer.add_document(doc);
+                index_writer.add_document(doc!(text_field=>"a"));
             }
             index_writer.commit().expect("commit failed");
             for _doc in 0..100 {
-                let mut doc = Document::default();
-                doc.add_text(text_field, "a");
-                index_writer.add_document(doc);
+                index_writer.add_document(doc!(text_field=>"a"));
             }
-            // this should create 8 segments and trigger a merge.
+            //  this should create 8 segments and trigger a merge.
             index_writer.commit().expect("commit failed");
             index_writer
                 .wait_merging_threads()
                 .expect("waiting merging thread failed");
-            index.load_searchers().unwrap();
+
+            reader.reload().unwrap();
 
             assert_eq!(num_docs_containing("a"), 200);
             assert!(index.searchable_segments().unwrap().len() < 8);
@@ -797,7 +956,6 @@ mod tests {
             {
                 let mut prepared_commit = index_writer.prepare_commit().expect("commit failed");
                 prepared_commit.set_payload("first commit");
-                assert_eq!(prepared_commit.opstamp(), 100);
                 prepared_commit.commit().expect("commit failed");
             }
             {
@@ -831,7 +989,6 @@ mod tests {
             {
                 let mut prepared_commit = index_writer.prepare_commit().expect("commit failed");
                 prepared_commit.set_payload("first commit");
-                assert_eq!(prepared_commit.opstamp(), 100);
                 prepared_commit.abort().expect("commit failed");
             }
             {
@@ -843,11 +1000,15 @@ mod tests {
             }
             index_writer.commit().unwrap();
         }
-        index.load_searchers().unwrap();
         let num_docs_containing = |s: &str| {
-            let searcher = index.searcher();
             let term_a = Term::from_field_text(text_field, s);
-            searcher.doc_freq(&term_a)
+            index
+                .reader_builder()
+                .reload_policy(ReloadPolicy::Manual)
+                .try_into()
+                .unwrap()
+                .searcher()
+                .doc_freq(&term_a)
         };
         assert_eq!(num_docs_containing("a"), 0);
         assert_eq!(num_docs_containing("b"), 100);
@@ -855,9 +1016,9 @@ mod tests {
 
     #[test]
     fn test_hashmap_size() {
-        assert_eq!(initial_table_size(100_000), 12);
-        assert_eq!(initial_table_size(1_000_000), 15);
-        assert_eq!(initial_table_size(10_000_000), 18);
+        assert_eq!(initial_table_size(100_000), 11);
+        assert_eq!(initial_table_size(1_000_000), 14);
+        assert_eq!(initial_table_size(10_000_000), 17);
         assert_eq!(initial_table_size(1_000_000_000), 19);
     }
 
@@ -879,11 +1040,9 @@ mod tests {
             index_writer.add_document(doc!(text_field => "b"));
         }
         assert!(index_writer.commit().is_err());
-        index.load_searchers().unwrap();
         let num_docs_containing = |s: &str| {
-            let searcher = index.searcher();
             let term_a = Term::from_field_text(text_field, s);
-            searcher.doc_freq(&term_a)
+            index.reader().unwrap().searcher().doc_freq(&term_a)
         };
         assert_eq!(num_docs_containing("a"), 100);
         assert_eq!(num_docs_containing("b"), 0);

src/indexer/merge_operation.rs

@@ -0,0 +1,64 @@
+use census::{Inventory, TrackedObject};
+use std::collections::HashSet;
+use SegmentId;
+
+#[derive(Default)]
+pub struct MergeOperationInventory(Inventory<InnerMergeOperation>);
+
+impl MergeOperationInventory {
+    pub fn segment_in_merge(&self) -> HashSet<SegmentId> {
+        let mut segment_in_merge = HashSet::default();
+        for merge_op in self.0.list() {
+            for &segment_id in &merge_op.segment_ids {
+                segment_in_merge.insert(segment_id);
+            }
+        }
+        segment_in_merge
+    }
+}
+
+/// A `MergeOperation` has two role.
+/// It carries all of the information required to describe a merge :
+/// - `target_opstamp` is the opstamp up to which we want to consume the
+/// delete queue and reflect their deletes.
+/// - `segment_ids` is the list of segment to be merged.
+///
+/// The second role is to ensure keep track of the fact that these
+/// segments are in merge and avoid starting a merge operation that
+/// may conflict with this one.
+///
+/// This works by tracking merge operations. When considering computing
+/// merge candidates, we simply list tracked merge operations and remove
+/// their segments from possible merge candidates.
+pub struct MergeOperation {
+    inner: TrackedObject<InnerMergeOperation>,
+}
+
+struct InnerMergeOperation {
+    target_opstamp: u64,
+    segment_ids: Vec<SegmentId>,
+}
+
+impl MergeOperation {
+    pub fn new(
+        inventory: &MergeOperationInventory,
+        target_opstamp: u64,
+        segment_ids: Vec<SegmentId>,
+    ) -> MergeOperation {
+        let inner_merge_operation = InnerMergeOperation {
+            target_opstamp,
+            segment_ids,
+        };
+        MergeOperation {
+            inner: inventory.0.track(inner_merge_operation),
+        }
+    }
+
+    pub fn target_opstamp(&self) -> u64 {
+        self.inner.target_opstamp
+    }
+
+    pub fn segment_ids(&self) -> &[SegmentId] {
+        &self.inner.segment_ids[..]
+    }
+}

src/indexer/merge_policy.rs

@@ -11,7 +11,7 @@ pub struct MergeCandidate(pub Vec<SegmentId>);
 ///
 /// Every time a the list of segments changes, the segment updater
 /// asks the merge policy if some segments should be merged.
-pub trait MergePolicy: MergePolicyClone + marker::Send + marker::Sync + Debug {
+pub trait MergePolicy: marker::Send + marker::Sync + Debug {
     /// Given the list of segment metas, returns the list of merge candidates.
     ///
     /// This call happens on the segment updater thread, and will block
@@ -19,21 +19,6 @@ pub trait MergePolicy: MergePolicyClone + marker::Send + marker::Sync + Debug {
     fn compute_merge_candidates(&self, segments: &[SegmentMeta]) -> Vec<MergeCandidate>;
 }
 
-/// MergePolicyClone
-pub trait MergePolicyClone {
-    /// Returns a boxed clone of the MergePolicy.
-    fn box_clone(&self) -> Box<MergePolicy>;
-}
-
-impl<T> MergePolicyClone for T
-where
-    T: 'static + MergePolicy + Clone,
-{
-    fn box_clone(&self) -> Box<MergePolicy> {
-        Box::new(self.clone())
-    }
-}
-
 /// Never merge segments.
 #[derive(Debug, Clone)]
 pub struct NoMergePolicy;

src/indexer/merger.rs

@@ -194,17 +194,17 @@ impl IndexMerger {
                         fast_field_serializer,
                     )?;
                 }
-                FieldType::U64(ref options) | FieldType::I64(ref options) => {
-                    match options.get_fastfield_cardinality() {
-                        Some(Cardinality::SingleValue) => {
-                            self.write_single_fast_field(field, fast_field_serializer)?;
-                        }
-                        Some(Cardinality::MultiValues) => {
-                            self.write_multi_fast_field(field, fast_field_serializer)?;
-                        }
-                        None => {}
+                FieldType::U64(ref options)
+                | FieldType::I64(ref options)
+                | FieldType::Date(ref options) => match options.get_fastfield_cardinality() {
+                    Some(Cardinality::SingleValue) => {
+                        self.write_single_fast_field(field, fast_field_serializer)?;
                     }
-                }
+                    Some(Cardinality::MultiValues) => {
+                        self.write_multi_fast_field(field, fast_field_serializer)?;
+                    }
+                    None => {}
+                },
                 FieldType::Str(_) => {
                     // We don't handle str fast field for the moment
                     // They can be implemented using what is done
@@ -654,6 +654,7 @@ mod tests {
     use schema::IntOptions;
     use schema::Term;
     use schema::TextFieldIndexing;
+    use schema::INDEXED;
     use std::io::Cursor;
     use DocAddress;
     use IndexWriter;
@@ -670,11 +671,13 @@ mod tests {
             )
             .set_stored();
         let text_field = schema_builder.add_text_field("text", text_fieldtype);
+        let date_field = schema_builder.add_date_field("date", INDEXED);
         let score_fieldtype = schema::IntOptions::default().set_fast(Cardinality::SingleValue);
         let score_field = schema_builder.add_u64_field("score", score_fieldtype);
         let bytes_score_field = schema_builder.add_bytes_field("score_bytes");
         let index = Index::create_in_ram(schema_builder.build());
-
+        let reader = index.reader().unwrap();
+        let curr_time = chrono::Utc::now();
         let add_score_bytes = |doc: &mut Document, score: u32| {
             let mut bytes = Vec::new();
             bytes
@@ -691,6 +694,7 @@ mod tests {
                     let mut doc = Document::default();
                     doc.add_text(text_field, "af b");
                     doc.add_u64(score_field, 3);
+                    doc.add_date(date_field, &curr_time);
                     add_score_bytes(&mut doc, 3);
                     index_writer.add_document(doc);
                 }
@@ -716,6 +720,7 @@ mod tests {
                 {
                     let mut doc = Document::default();
                     doc.add_text(text_field, "af b");
+                    doc.add_date(date_field, &curr_time);
                     doc.add_u64(score_field, 11);
                     add_score_bytes(&mut doc, 11);
                     index_writer.add_document(doc);
@@ -743,8 +748,8 @@ mod tests {
             index_writer.wait_merging_threads().unwrap();
         }
         {
-            index.load_searchers().unwrap();
-            let searcher = index.searcher();
+            reader.reload().unwrap();
+            let searcher = reader.searcher();
             let get_doc_ids = |terms: Vec<Term>| {
                 let query = BooleanQuery::new_multiterms_query(terms);
                 let top_docs = searcher.search(&query, &TestCollector).unwrap();
@@ -773,6 +778,10 @@ mod tests {
                         DocAddress(0, 4)
                     ]
                 );
+                assert_eq!(
+                    get_doc_ids(vec![Term::from_field_date(date_field, &curr_time)]),
+                    vec![DocAddress(0, 0), DocAddress(0, 3)]
+                );
             }
             {
                 let doc = searcher.doc(DocAddress(0, 0)).unwrap();
@@ -835,8 +844,8 @@ mod tests {
         let score_field = schema_builder.add_u64_field("score", score_fieldtype);
         let bytes_score_field = schema_builder.add_bytes_field("score_bytes");
         let index = Index::create_in_ram(schema_builder.build());
-        let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
-
+        let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
+        let reader = index.reader().unwrap();
         let search_term = |searcher: &Searcher, term: Term| {
             let collector = FastFieldTestCollector::for_field(score_field);
             let bytes_collector = BytesFastFieldTestCollector::for_field(bytes_score_field);
@@ -873,8 +882,8 @@ mod tests {
                 bytes_score_field => vec![0u8, 0, 0, 3],
             ));
             index_writer.commit().expect("committed");
-            index.load_searchers().unwrap();
-            let ref searcher = *index.searcher();
+            reader.reload().unwrap();
+            let searcher = reader.searcher();
             assert_eq!(searcher.num_docs(), 2);
             assert_eq!(searcher.segment_readers()[0].num_docs(), 2);
             assert_eq!(searcher.segment_readers()[0].max_doc(), 3);
@@ -920,8 +929,8 @@ mod tests {
                 bytes_score_field => vec![0u8, 0, 27, 88],
             ));
             index_writer.commit().expect("committed");
-            index.load_searchers().unwrap();
-            let searcher = index.searcher();
+            reader.reload().unwrap();
+            let searcher = reader.searcher();
 
             assert_eq!(searcher.segment_readers().len(), 2);
             assert_eq!(searcher.num_docs(), 3);
@@ -982,8 +991,8 @@ mod tests {
                 .expect("Failed to initiate merge")
                 .wait()
                 .expect("Merging failed");
-            index.load_searchers().unwrap();
-            let ref searcher = *index.searcher();
+            reader.reload().unwrap();
+            let searcher = reader.searcher();
             assert_eq!(searcher.segment_readers().len(), 1);
             assert_eq!(searcher.num_docs(), 3);
             assert_eq!(searcher.segment_readers()[0].num_docs(), 3);
@@ -1028,8 +1037,8 @@ mod tests {
             index_writer.delete_term(Term::from_field_text(text_field, "c"));
             index_writer.commit().unwrap();
 
-            index.load_searchers().unwrap();
-            let ref searcher = *index.searcher();
+            reader.reload().unwrap();
+            let searcher = reader.searcher();
             assert_eq!(searcher.segment_readers().len(), 1);
             assert_eq!(searcher.num_docs(), 2);
             assert_eq!(searcher.segment_readers()[0].num_docs(), 2);
@@ -1079,9 +1088,9 @@ mod tests {
                 .expect("Failed to initiate merge")
                 .wait()
                 .expect("Merging failed");
-            index.load_searchers().unwrap();
+            reader.reload().unwrap();
 
-            let ref searcher = *index.searcher();
+            let searcher = reader.searcher();
             assert_eq!(searcher.segment_readers().len(), 1);
             assert_eq!(searcher.num_docs(), 2);
             assert_eq!(searcher.segment_readers()[0].num_docs(), 2);
@@ -1125,18 +1134,15 @@ mod tests {
         {
             // Test removing all docs
             index_writer.delete_term(Term::from_field_text(text_field, "g"));
+            index_writer.commit().unwrap();
             let segment_ids = index
                 .searchable_segment_ids()
                 .expect("Searchable segments failed.");
-            index_writer
-                .merge(&segment_ids)
-                .expect("Failed to initiate merge")
-                .wait()
-                .expect("Merging failed");
-            index.load_searchers().unwrap();
+            reader.reload().unwrap();
 
-            let ref searcher = *index.searcher();
-            assert_eq!(searcher.segment_readers().len(), 1);
+            let searcher = reader.searcher();
+            assert!(segment_ids.is_empty());
+            assert!(searcher.segment_readers().is_empty());
             assert_eq!(searcher.num_docs(), 0);
         }
     }
@@ -1146,8 +1152,9 @@ mod tests {
         let mut schema_builder = schema::Schema::builder();
         let facet_field = schema_builder.add_facet_field("facet");
         let index = Index::create_in_ram(schema_builder.build());
+        let reader = index.reader().unwrap();
         {
-            let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
+            let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
             let index_doc = |index_writer: &mut IndexWriter, doc_facets: &[&str]| {
                 let mut doc = Document::default();
                 for facet in doc_facets {
@@ -1175,9 +1182,9 @@ mod tests {
             index_writer.commit().expect("committed");
         }
 
-        index.load_searchers().unwrap();
+        reader.reload().unwrap();
         let test_searcher = |expected_num_docs: usize, expected: &[(&str, u64)]| {
-            let searcher = index.searcher();
+            let searcher = reader.searcher();
             let mut facet_collector = FacetCollector::for_field(facet_field);
             facet_collector.add_facet(Facet::from("/top"));
             let (count, facet_counts) = searcher
@@ -1212,14 +1219,14 @@ mod tests {
             let segment_ids = index
                 .searchable_segment_ids()
                 .expect("Searchable segments failed.");
-            let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
+            let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
             index_writer
                 .merge(&segment_ids)
                 .expect("Failed to initiate merge")
                 .wait()
                 .expect("Merging failed");
             index_writer.wait_merging_threads().unwrap();
-            index.load_searchers().unwrap();
+            reader.reload().unwrap();
             test_searcher(
                 11,
                 &[
@@ -1235,12 +1242,12 @@ mod tests {
 
         // Deleting one term
         {
-            let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
+            let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
             let facet = Facet::from_path(vec!["top", "a", "firstdoc"]);
             let facet_term = Term::from_facet(facet_field, &facet);
             index_writer.delete_term(facet_term);
             index_writer.commit().unwrap();
-            index.load_searchers().unwrap();
+            reader.reload().unwrap();
             test_searcher(
                 9,
                 &[
@@ -1255,6 +1262,34 @@ mod tests {
         }
     }
 
+    #[test]
+    fn test_bug_merge() {
+        let mut schema_builder = schema::Schema::builder();
+        let int_field = schema_builder.add_u64_field("intvals", INDEXED);
+        let index = Index::create_in_ram(schema_builder.build());
+        let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
+        index_writer.add_document(doc!(int_field => 1u64));
+        index_writer.commit().expect("commit failed");
+        index_writer.add_document(doc!(int_field => 1u64));
+        index_writer.commit().expect("commit failed");
+        let reader = index.reader().unwrap();
+        let searcher = reader.searcher();
+        assert_eq!(searcher.num_docs(), 2);
+        index_writer.delete_term(Term::from_field_u64(int_field, 1));
+        let segment_ids = index
+            .searchable_segment_ids()
+            .expect("Searchable segments failed.");
+        index_writer
+            .merge(&segment_ids)
+            .expect("Failed to initiate merge")
+            .wait()
+            .expect("Merging failed");
+        reader.reload().unwrap();
+        // commit has not been called yet. The document should still be
+        // there.
+        assert_eq!(reader.searcher().num_docs(), 2);
+    }
+
     #[test]
     fn test_merge_multivalued_int_fields_all_deleted() {
         let mut schema_builder = schema::Schema::builder();
@@ -1263,9 +1298,9 @@ mod tests {
             .set_indexed();
         let int_field = schema_builder.add_u64_field("intvals", int_options);
         let index = Index::create_in_ram(schema_builder.build());
-
+        let reader = index.reader().unwrap();
         {
-            let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
+            let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
             let mut doc = Document::default();
             doc.add_u64(int_field, 1);
             index_writer.add_document(doc.clone());
@@ -1273,31 +1308,33 @@ mod tests {
             index_writer.add_document(doc);
             index_writer.commit().expect("commit failed");
             index_writer.delete_term(Term::from_field_u64(int_field, 1));
-            index_writer.commit().expect("commit failed");
-        }
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
-        assert_eq!(searcher.num_docs(), 0);
-        // Merging the segments
-        {
+
             let segment_ids = index
                 .searchable_segment_ids()
                 .expect("Searchable segments failed.");
-            let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
             index_writer
                 .merge(&segment_ids)
                 .expect("Failed to initiate merge")
                 .wait()
                 .expect("Merging failed");
+
+            // assert delete has not been committed
+            reader.reload().expect("failed to load searcher 1");
+            let searcher = reader.searcher();
+            assert_eq!(searcher.num_docs(), 2);
+
+            index_writer.commit().unwrap();
+
             index_writer.wait_merging_threads().unwrap();
         }
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+
+        reader.reload().unwrap();
+        let searcher = reader.searcher();
         assert_eq!(searcher.num_docs(), 0);
     }
 
     #[test]
-    fn test_merge_multivalued_int_fields() {
+    fn test_merge_multivalued_int_fields_simple() {
         let mut schema_builder = schema::Schema::builder();
         let int_options = IntOptions::default()
             .set_fast(Cardinality::MultiValues)
@@ -1306,7 +1343,7 @@ mod tests {
         let index = Index::create_in_ram(schema_builder.build());
 
         {
-            let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
+            let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
             let index_doc = |index_writer: &mut IndexWriter, int_vals: &[u64]| {
                 let mut doc = Document::default();
                 for &val in int_vals {
@@ -1314,7 +1351,6 @@ mod tests {
                 }
                 index_writer.add_document(doc);
             };
-
             index_doc(&mut index_writer, &[1, 2]);
             index_doc(&mut index_writer, &[1, 2, 3]);
             index_doc(&mut index_writer, &[4, 5]);
@@ -1323,19 +1359,14 @@ mod tests {
             index_doc(&mut index_writer, &[3]);
             index_doc(&mut index_writer, &[17]);
             index_writer.commit().expect("committed");
-
             index_doc(&mut index_writer, &[20]);
             index_writer.commit().expect("committed");
-
             index_doc(&mut index_writer, &[28, 27]);
             index_doc(&mut index_writer, &[1_000]);
-
             index_writer.commit().expect("committed");
         }
-        index.load_searchers().unwrap();
-
-        let searcher = index.searcher();
-
+        let reader = index.reader().unwrap();
+        let searcher = reader.searcher();
         let mut vals: Vec<u64> = Vec::new();
 
         {
@@ -1395,19 +1426,20 @@ mod tests {
             let segment_ids = index
                 .searchable_segment_ids()
                 .expect("Searchable segments failed.");
-            let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
+            let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
             index_writer
                 .merge(&segment_ids)
                 .expect("Failed to initiate merge")
                 .wait()
                 .expect("Merging failed");
-            index_writer.wait_merging_threads().unwrap();
+            index_writer
+                .wait_merging_threads()
+                .expect("Wait for merging threads");
         }
-
-        index.load_searchers().unwrap();
+        reader.reload().expect("Load searcher");
 
         {
-            let searcher = index.searcher();
+            let searcher = reader.searcher();
             println!(
                 "{:?}",
                 searcher

src/indexer/mod.rs

@@ -1,8 +1,9 @@
 pub mod delete_queue;
-mod directory_lock;
+
 mod doc_opstamp_mapping;
 pub mod index_writer;
 mod log_merge_policy;
+mod merge_operation;
 pub mod merge_policy;
 pub mod merger;
 pub mod operation;
@@ -15,14 +16,12 @@ pub mod segment_updater;
 mod segment_writer;
 mod stamper;
 
-pub(crate) use self::directory_lock::DirectoryLock;
-pub use self::directory_lock::LockType;
-
 pub use self::index_writer::IndexWriter;
 pub use self::log_merge_policy::LogMergePolicy;
+pub use self::merge_operation::{MergeOperation, MergeOperationInventory};
 pub use self::merge_policy::{MergeCandidate, MergePolicy, NoMergePolicy};
 pub use self::prepared_commit::PreparedCommit;
-pub use self::segment_entry::{SegmentEntry, SegmentState};
+pub use self::segment_entry::SegmentEntry;
 pub use self::segment_manager::SegmentManager;
 pub use self::segment_serializer::SegmentSerializer;
 pub use self::segment_writer::SegmentWriter;

src/indexer/operation.rs

@@ -14,3 +14,10 @@ pub struct AddOperation {
     pub opstamp: u64,
     pub document: Document,
 }
+
+/// UserOperation is an enum type that encapsulates other operation types.
+#[derive(Eq, PartialEq, Debug)]
+pub enum UserOperation {
+    Add(Document),
+    Delete(Term),
+}

src/indexer/segment_entry.rs

@@ -4,21 +4,6 @@ use core::SegmentMeta;
 use indexer::delete_queue::DeleteCursor;
 use std::fmt;
 
-#[derive(Clone, Copy, PartialEq, Eq, Debug)]
-pub enum SegmentState {
-    Ready,
-    InMerge,
-}
-
-impl SegmentState {
-    pub fn letter_code(self) -> char {
-        match self {
-            SegmentState::InMerge => 'M',
-            SegmentState::Ready => 'R',
-        }
-    }
-}
-
 /// A segment entry describes the state of
 /// a given segment, at a given instant.
 ///
@@ -35,7 +20,6 @@ impl SegmentState {
 #[derive(Clone)]
 pub struct SegmentEntry {
     meta: SegmentMeta,
-    state: SegmentState,
     delete_bitset: Option<BitSet>,
     delete_cursor: DeleteCursor,
 }
@@ -49,7 +33,6 @@ impl SegmentEntry {
     ) -> SegmentEntry {
         SegmentEntry {
             meta: segment_meta,
-            state: SegmentState::Ready,
             delete_bitset,
             delete_cursor,
         }
@@ -72,14 +55,6 @@ impl SegmentEntry {
         &mut self.delete_cursor
     }
 
-    /// Return the `SegmentEntry`.
-    ///
-    /// The state describes whether the segment is available for
-    /// a merge or not.
-    pub fn state(&self) -> SegmentState {
-        self.state
-    }
-
     /// Returns the segment id.
     pub fn segment_id(&self) -> SegmentId {
         self.meta.id()
@@ -89,33 +64,10 @@ impl SegmentEntry {
     pub fn meta(&self) -> &SegmentMeta {
         &self.meta
     }
-
-    /// Mark the `SegmentEntry` as in merge.
-    ///
-    /// Only segments that are not already
-    /// in a merge are elligible for future merge.
-    pub fn start_merge(&mut self) {
-        self.state = SegmentState::InMerge;
-    }
-
-    /// Cancel a merge
-    ///
-    /// If a merge fails, it is important to switch
-    /// the segment back to a idle state, so that it
-    /// may be elligible for future merges.
-    pub fn cancel_merge(&mut self) {
-        self.state = SegmentState::Ready;
-    }
-
-    /// Returns true iff a segment should
-    /// be considered for a merge.
-    pub fn is_ready(&self) -> bool {
-        self.state == SegmentState::Ready
-    }
 }
 
 impl fmt::Debug for SegmentEntry {
     fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
-        write!(formatter, "SegmentEntry({:?}, {:?})", self.meta, self.state)
+        write!(formatter, "SegmentEntry({:?})", self.meta)
     }
 }

src/indexer/segment_manager.rs

@@ -16,7 +16,6 @@ use Result as TantivyResult;
 struct SegmentRegisters {
     uncommitted: SegmentRegister,
     committed: SegmentRegister,
-    writing: HashSet<SegmentId>,
 }
 
 /// The segment manager stores the list of segments
@@ -41,12 +40,17 @@ impl Debug for SegmentManager {
 }
 
 pub fn get_mergeable_segments(
+    in_merge_segment_ids: &HashSet<SegmentId>,
     segment_manager: &SegmentManager,
 ) -> (Vec<SegmentMeta>, Vec<SegmentMeta>) {
     let registers_lock = segment_manager.read();
     (
-        registers_lock.committed.get_mergeable_segments(),
-        registers_lock.uncommitted.get_mergeable_segments(),
+        registers_lock
+            .committed
+            .get_mergeable_segments(in_merge_segment_ids),
+        registers_lock
+            .uncommitted
+            .get_mergeable_segments(in_merge_segment_ids),
     )
 }
 
@@ -59,7 +63,6 @@ impl SegmentManager {
             registers: RwLock::new(SegmentRegisters {
                 uncommitted: SegmentRegister::default(),
                 committed: SegmentRegister::new(segment_metas, delete_cursor),
-                writing: HashSet::new(),
             }),
         }
     }
@@ -72,12 +75,6 @@ impl SegmentManager {
         segment_entries
     }
 
-    /// Returns the overall number of segments in the `SegmentManager`
-    pub fn num_segments(&self) -> usize {
-        let registers_lock = self.read();
-        registers_lock.committed.len() + registers_lock.uncommitted.len()
-    }
-
     /// List the files that are useful to the index.
     ///
     /// This does not include lock files, or files that are obsolete
@@ -106,6 +103,21 @@ impl SegmentManager {
             .expect("Failed to acquire write lock on SegmentManager.")
     }
 
+    /// Deletes all empty segments
+    fn remove_empty_segments(&self) {
+        let mut registers_lock = self.write();
+        registers_lock
+            .committed
+            .segment_entries()
+            .iter()
+            .filter(|segment| segment.meta().num_docs() == 0)
+            .for_each(|segment| {
+                registers_lock
+                    .committed
+                    .remove_segment(&segment.segment_id())
+            });
+    }
+
     pub fn commit(&self, segment_entries: Vec<SegmentEntry>) {
         let mut registers_lock = self.write();
         registers_lock.committed.clear();
@@ -121,25 +133,22 @@ impl SegmentManager {
     /// the `segment_ids` are not either all committed or all
     /// uncommitted.
     pub fn start_merge(&self, segment_ids: &[SegmentId]) -> TantivyResult<Vec<SegmentEntry>> {
-        let mut registers_lock = self.write();
+        let registers_lock = self.read();
         let mut segment_entries = vec![];
         if registers_lock.uncommitted.contains_all(segment_ids) {
             for segment_id in segment_ids {
                 let segment_entry = registers_lock.uncommitted
-                    .start_merge(segment_id)
+                    .get(segment_id)
                     .expect("Segment id not found {}. Should never happen because of the contains all if-block.");
                 segment_entries.push(segment_entry);
             }
         } else if registers_lock.committed.contains_all(segment_ids) {
             for segment_id in segment_ids {
                 let segment_entry = registers_lock.committed
-                    .start_merge(segment_id)
+                    .get(segment_id)
                     .expect("Segment id not found {}. Should never happen because of the contains all if-block.");
                 segment_entries.push(segment_entry);
             }
-            for segment_id in segment_ids {
-                registers_lock.committed.start_merge(segment_id);
-            }
         } else {
             let error_msg = "Merge operation sent for segments that are not \
                              all uncommited or commited."
@@ -149,50 +158,8 @@ impl SegmentManager {
         Ok(segment_entries)
     }
 
-    pub fn cancel_merge(
-        &self,
-        before_merge_segment_ids: &[SegmentId],
-        after_merge_segment_id: SegmentId,
-    ) {
-        let mut registers_lock = self.write();
-
-        // we mark all segments are ready for merge.
-        {
-            let target_segment_register: &mut SegmentRegister;
-            target_segment_register = {
-                if registers_lock
-                    .uncommitted
-                    .contains_all(before_merge_segment_ids)
-                {
-                    &mut registers_lock.uncommitted
-                } else if registers_lock
-                    .committed
-                    .contains_all(before_merge_segment_ids)
-                {
-                    &mut registers_lock.committed
-                } else {
-                    warn!("couldn't find segment in SegmentManager");
-                    return;
-                }
-            };
-            for segment_id in before_merge_segment_ids {
-                target_segment_register.cancel_merge(segment_id);
-            }
-        }
-
-        // ... and we make sure the target segment entry
-        // can be garbage collected.
-        registers_lock.writing.remove(&after_merge_segment_id);
-    }
-
-    pub fn write_segment(&self, segment_id: SegmentId) {
-        let mut registers_lock = self.write();
-        registers_lock.writing.insert(segment_id);
-    }
-
     pub fn add_segment(&self, segment_entry: SegmentEntry) {
         let mut registers_lock = self.write();
-        registers_lock.writing.remove(&segment_entry.segment_id());
         registers_lock.uncommitted.add_segment_entry(segment_entry);
     }
 
@@ -202,10 +169,6 @@ impl SegmentManager {
         after_merge_segment_entry: SegmentEntry,
     ) {
         let mut registers_lock = self.write();
-        registers_lock
-            .writing
-            .remove(&after_merge_segment_entry.segment_id());
-
         let target_register: &mut SegmentRegister = {
             if registers_lock
                 .uncommitted
@@ -229,6 +192,7 @@ impl SegmentManager {
     }
 
     pub fn committed_segment_metas(&self) -> Vec<SegmentMeta> {
+        self.remove_empty_segments();
         let registers_lock = self.read();
         registers_lock.committed.segment_metas()
     }

src/indexer/segment_register.rs

@@ -3,6 +3,7 @@ use core::SegmentMeta;
 use indexer::delete_queue::DeleteCursor;
 use indexer::segment_entry::SegmentEntry;
 use std::collections::HashMap;
+use std::collections::HashSet;
 use std::fmt::{self, Debug, Formatter};
 
 /// The segment register keeps track
@@ -21,8 +22,8 @@ pub struct SegmentRegister {
 impl Debug for SegmentRegister {
     fn fmt(&self, f: &mut Formatter) -> Result<(), fmt::Error> {
         write!(f, "SegmentRegister(")?;
-        for (k, v) in &self.segment_states {
-            write!(f, "{}:{}, ", k.short_uuid_string(), v.state().letter_code())?;
+        for k in self.segment_states.keys() {
+            write!(f, "{}, ", k.short_uuid_string())?;
         }
         write!(f, ")")?;
         Ok(())
@@ -34,14 +35,13 @@ impl SegmentRegister {
         self.segment_states.clear();
     }
 
-    pub fn len(&self) -> usize {
-        self.segment_states.len()
-    }
-
-    pub fn get_mergeable_segments(&self) -> Vec<SegmentMeta> {
+    pub fn get_mergeable_segments(
+        &self,
+        in_merge_segment_ids: &HashSet<SegmentId>,
+    ) -> Vec<SegmentMeta> {
         self.segment_states
             .values()
-            .filter(|segment_entry| segment_entry.is_ready())
+            .filter(|segment_entry| !in_merge_segment_ids.contains(&segment_entry.segment_id()))
             .map(|segment_entry| segment_entry.meta().clone())
             .collect()
     }
@@ -60,7 +60,7 @@ impl SegmentRegister {
         segment_ids
     }
 
-    pub fn contains_all(&mut self, segment_ids: &[SegmentId]) -> bool {
+    pub fn contains_all(&self, segment_ids: &[SegmentId]) -> bool {
         segment_ids
             .iter()
             .all(|segment_id| self.segment_states.contains_key(segment_id))
@@ -75,20 +75,8 @@ impl SegmentRegister {
         self.segment_states.remove(segment_id);
     }
 
-    pub fn cancel_merge(&mut self, segment_id: &SegmentId) {
-        self.segment_states
-            .get_mut(segment_id)
-            .expect("Received a merge notification for a segment that is not registered")
-            .cancel_merge();
-    }
-
-    pub fn start_merge(&mut self, segment_id: &SegmentId) -> Option<SegmentEntry> {
-        if let Some(segment_entry) = self.segment_states.get_mut(segment_id) {
-            segment_entry.start_merge();
-            Some(segment_entry.clone())
-        } else {
-            None
-        }
+    pub fn get(&self, segment_id: &SegmentId) -> Option<SegmentEntry> {
+        self.segment_states.get(segment_id).cloned()
     }
 
     pub fn new(segment_metas: Vec<SegmentMeta>, delete_cursor: &DeleteCursor) -> SegmentRegister {
@@ -100,11 +88,6 @@ impl SegmentRegister {
         }
         SegmentRegister { segment_states }
     }
-
-    #[cfg(test)]
-    pub fn segment_entry(&self, segment_id: &SegmentId) -> Option<SegmentEntry> {
-        self.segment_states.get(segment_id).cloned()
-    }
 }
 
 #[cfg(test)]
@@ -113,7 +96,6 @@ mod tests {
     use core::SegmentId;
     use core::SegmentMeta;
     use indexer::delete_queue::*;
-    use indexer::SegmentState;
 
     fn segment_ids(segment_register: &SegmentRegister) -> Vec<SegmentId> {
         segment_register
@@ -137,42 +119,12 @@ mod tests {
             let segment_entry = SegmentEntry::new(segment_meta, delete_queue.cursor(), None);
             segment_register.add_segment_entry(segment_entry);
         }
-        assert_eq!(
-            segment_register
-                .segment_entry(&segment_id_a)
-                .unwrap()
-                .state(),
-            SegmentState::Ready
-        );
         assert_eq!(segment_ids(&segment_register), vec![segment_id_a]);
         {
             let segment_meta = SegmentMeta::new(segment_id_b, 0u32);
             let segment_entry = SegmentEntry::new(segment_meta, delete_queue.cursor(), None);
             segment_register.add_segment_entry(segment_entry);
         }
-        assert_eq!(
-            segment_register
-                .segment_entry(&segment_id_b)
-                .unwrap()
-                .state(),
-            SegmentState::Ready
-        );
-        segment_register.start_merge(&segment_id_a);
-        segment_register.start_merge(&segment_id_b);
-        assert_eq!(
-            segment_register
-                .segment_entry(&segment_id_a)
-                .unwrap()
-                .state(),
-            SegmentState::InMerge
-        );
-        assert_eq!(
-            segment_register
-                .segment_entry(&segment_id_b)
-                .unwrap()
-                .state(),
-            SegmentState::InMerge
-        );
         segment_register.remove_segment(&segment_id_a);
         segment_register.remove_segment(&segment_id_b);
         {

src/indexer/segment_updater.rs

@@ -16,9 +16,10 @@ use futures_cpupool::CpuFuture;
 use futures_cpupool::CpuPool;
 use indexer::delete_queue::DeleteCursor;
 use indexer::index_writer::advance_deletes;
+use indexer::merge_operation::MergeOperationInventory;
 use indexer::merger::IndexMerger;
 use indexer::stamper::Stamper;
-use indexer::MergeCandidate;
+use indexer::MergeOperation;
 use indexer::SegmentEntry;
 use indexer::SegmentSerializer;
 use indexer::{DefaultMergePolicy, MergePolicy};
@@ -26,6 +27,7 @@ use schema::Schema;
 use serde_json;
 use std::borrow::BorrowMut;
 use std::collections::HashMap;
+use std::collections::HashSet;
 use std::io::Write;
 use std::mem;
 use std::ops::DerefMut;
@@ -45,33 +47,30 @@ use Result;
 /// and flushed.
 ///
 /// This method is not part of tantivy's public API
-pub fn save_new_metas(schema: Schema, opstamp: u64, directory: &mut Directory) -> Result<()> {
-    save_metas(vec![], schema, opstamp, None, directory)
+pub fn save_new_metas(schema: Schema, directory: &mut Directory) -> Result<()> {
+    save_metas(
+        &IndexMeta {
+            segments: Vec::new(),
+            schema,
+            opstamp: 0u64,
+            payload: None,
+        },
+        directory,
+    )
 }
 
 /// Save the index meta file.
 /// This operation is atomic:
 /// Either
-//  - it fails, in which case an error is returned,
+///  - it fails, in which case an error is returned,
 /// and the `meta.json` remains untouched,
 /// - it success, and `meta.json` is written
 /// and flushed.
 ///
 /// This method is not part of tantivy's public API
-pub fn save_metas(
-    segment_metas: Vec<SegmentMeta>,
-    schema: Schema,
-    opstamp: u64,
-    payload: Option<String>,
-    directory: &mut Directory,
-) -> Result<()> {
-    let metas = IndexMeta {
-        segments: segment_metas,
-        schema,
-        opstamp,
-        payload,
-    };
-    let mut buffer = serde_json::to_vec_pretty(&metas)?;
+fn save_metas(metas: &IndexMeta, directory: &mut Directory) -> Result<()> {
+    let mut buffer = serde_json::to_vec_pretty(metas)?;
+    // Just adding a new line at the end of the buffer.
     writeln!(&mut buffer)?;
     directory.atomic_write(&META_FILEPATH, &buffer[..])?;
     debug!("Saved metas {:?}", serde_json::to_string_pretty(&metas));
@@ -83,16 +82,21 @@ pub fn save_metas(
 //
 // All this processing happens on a single thread
 // consuming a common queue.
+//
+// We voluntarily pass a merge_operation ref to guarantee that
+// the merge_operation is alive during the process
 #[derive(Clone)]
 pub struct SegmentUpdater(Arc<InnerSegmentUpdater>);
 
 fn perform_merge(
+    merge_operation: &MergeOperation,
     index: &Index,
     mut segment_entries: Vec<SegmentEntry>,
-    mut merged_segment: Segment,
-    target_opstamp: u64,
 ) -> Result<SegmentEntry> {
+    let target_opstamp = merge_operation.target_opstamp();
+
     // first we need to apply deletes to our segment.
+    let mut merged_segment = index.new_segment();
 
     // TODO add logging
     let schema = index.schema();
@@ -126,15 +130,23 @@ fn perform_merge(
 }
 
 struct InnerSegmentUpdater {
+    // we keep a copy of the current active IndexMeta to
+    // avoid loading the file everytime we need it in the
+    // `SegmentUpdater`.
+    //
+    // This should be up to date as all update happen through
+    // the unique active `SegmentUpdater`.
+    active_metas: RwLock<Arc<IndexMeta>>,
     pool: CpuPool,
     index: Index,
     segment_manager: SegmentManager,
-    merge_policy: RwLock<Box<MergePolicy>>,
+    merge_policy: RwLock<Arc<Box<MergePolicy>>>,
     merging_thread_id: AtomicUsize,
     merging_threads: RwLock<HashMap<usize, JoinHandle<Result<()>>>>,
     generation: AtomicUsize,
     killed: AtomicBool,
     stamper: Stamper,
+    merge_operations: MergeOperationInventory,
 }
 
 impl SegmentUpdater {
@@ -149,32 +161,29 @@ impl SegmentUpdater {
             .name_prefix("segment_updater")
             .pool_size(1)
             .create();
+        let index_meta = index.load_metas()?;
         Ok(SegmentUpdater(Arc::new(InnerSegmentUpdater {
+            active_metas: RwLock::new(Arc::new(index_meta)),
             pool,
             index,
             segment_manager,
-            merge_policy: RwLock::new(Box::new(DefaultMergePolicy::default())),
+            merge_policy: RwLock::new(Arc::new(Box::new(DefaultMergePolicy::default()))),
             merging_thread_id: AtomicUsize::default(),
             merging_threads: RwLock::new(HashMap::new()),
             generation: AtomicUsize::default(),
             killed: AtomicBool::new(false),
             stamper,
+            merge_operations: Default::default(),
         })))
     }
 
-    pub fn new_segment(&self) -> Segment {
-        let new_segment = self.0.index.new_segment();
-        let segment_id = new_segment.id();
-        self.0.segment_manager.write_segment(segment_id);
-        new_segment
-    }
-
-    pub fn get_merge_policy(&self) -> Box<MergePolicy> {
-        self.0.merge_policy.read().unwrap().box_clone()
+    pub fn get_merge_policy(&self) -> Arc<Box<MergePolicy>> {
+        self.0.merge_policy.read().unwrap().clone()
     }
 
     pub fn set_merge_policy(&self, merge_policy: Box<MergePolicy>) {
-        *self.0.merge_policy.write().unwrap() = merge_policy;
+        let arc_merge_policy = Arc::new(merge_policy);
+        *self.0.merge_policy.write().unwrap() = arc_merge_policy;
     }
 
     fn get_merging_thread_id(&self) -> usize {
@@ -244,14 +253,15 @@ impl SegmentUpdater {
             //
             // Segment 1 from disk 1, Segment 1 from disk 2, etc.
             commited_segment_metas.sort_by_key(|segment_meta| -(segment_meta.max_doc() as i32));
-            save_metas(
-                commited_segment_metas,
-                index.schema(),
+            let index_meta = IndexMeta {
+                segments: commited_segment_metas,
+                schema: index.schema(),
                 opstamp,
-                commit_message,
-                directory.box_clone().borrow_mut(),
-            )
-            .expect("Could not save metas.");
+                payload: commit_message,
+            };
+            save_metas(&index_meta, directory.box_clone().borrow_mut())
+                .expect("Could not save metas.");
+            self.store_meta(&index_meta);
         }
     }
 
@@ -286,51 +296,62 @@ impl SegmentUpdater {
     }
 
     pub fn start_merge(&self, segment_ids: &[SegmentId]) -> Result<Receiver<SegmentMeta>> {
-        //let future_merged_segment = */
-        let segment_ids_vec = segment_ids.to_vec();
-        self.run_async(move |segment_updater| {
-            segment_updater.start_merge_impl(&segment_ids_vec[..])
-        })
-        .wait()?
+        let commit_opstamp = self.load_metas().opstamp;
+        let merge_operation = MergeOperation::new(
+            &self.0.merge_operations,
+            commit_opstamp,
+            segment_ids.to_vec(),
+        );
+        self.run_async(move |segment_updater| segment_updater.start_merge_impl(merge_operation))
+            .wait()?
+    }
+
+    fn store_meta(&self, index_meta: &IndexMeta) {
+        *self.0.active_metas.write().unwrap() = Arc::new(index_meta.clone());
+    }
+    fn load_metas(&self) -> Arc<IndexMeta> {
+        self.0.active_metas.read().unwrap().clone()
     }
 
     // `segment_ids` is required to be non-empty.
-    fn start_merge_impl(&self, segment_ids: &[SegmentId]) -> Result<Receiver<SegmentMeta>> {
-        assert!(!segment_ids.is_empty(), "Segment_ids cannot be empty.");
+    fn start_merge_impl(&self, merge_operation: MergeOperation) -> Result<Receiver<SegmentMeta>> {
+        assert!(
+            !merge_operation.segment_ids().is_empty(),
+            "Segment_ids cannot be empty."
+        );
 
         let segment_updater_clone = self.clone();
-        let segment_entries: Vec<SegmentEntry> = self.0.segment_manager.start_merge(segment_ids)?;
+        let segment_entries: Vec<SegmentEntry> = self
+            .0
+            .segment_manager
+            .start_merge(merge_operation.segment_ids())?;
 
-        let segment_ids_vec = segment_ids.to_vec();
+        //        let segment_ids_vec = merge_operation.segment_ids.to_vec();
 
         let merging_thread_id = self.get_merging_thread_id();
         info!(
             "Starting merge thread #{} - {:?}",
-            merging_thread_id, segment_ids
+            merging_thread_id,
+            merge_operation.segment_ids()
         );
         let (merging_future_send, merging_future_recv) = oneshot();
 
-        let target_opstamp = self.0.stamper.stamp();
-
         // first we need to apply deletes to our segment.
         let merging_join_handle = thread::Builder::new()
             .name(format!("mergingthread-{}", merging_thread_id))
             .spawn(move || {
                 // first we need to apply deletes to our segment.
-                let merged_segment = segment_updater_clone.new_segment();
-                let merged_segment_id = merged_segment.id();
                 let merge_result = perform_merge(
+                    &merge_operation,
                     &segment_updater_clone.0.index,
                     segment_entries,
-                    merged_segment,
-                    target_opstamp,
                 );
 
                 match merge_result {
                     Ok(after_merge_segment_entry) => {
                         let merged_segment_meta = after_merge_segment_entry.meta().clone();
                         segment_updater_clone
-                            .end_merge(segment_ids_vec, after_merge_segment_entry)
+                            .end_merge(merge_operation, after_merge_segment_entry)
                             .expect("Segment updater thread is corrupted.");
 
                         // the future may fail if the listener of the oneshot future
@@ -341,13 +362,18 @@ impl SegmentUpdater {
                         let _merging_future_res = merging_future_send.send(merged_segment_meta);
                     }
                     Err(e) => {
-                        warn!("Merge of {:?} was cancelled: {:?}", segment_ids_vec, e);
+                        warn!(
+                            "Merge of {:?} was cancelled: {:?}",
+                            merge_operation.segment_ids(),
+                            e
+                        );
                         // ... cancel merge
                         if cfg!(test) {
                             panic!("Merge failed.");
                         }
-                        segment_updater_clone.cancel_merge(&segment_ids_vec, merged_segment_id);
-                        // merging_future_send will be dropped, sending an error to the future.
+                        // As `merge_operation` will be dropped, the segment in merge state will
+                        // be available for merge again.
+                        // `merging_future_send` will be dropped, sending an error to the future.
                     }
                 }
                 segment_updater_clone
@@ -368,16 +394,34 @@ impl SegmentUpdater {
     }
 
     fn consider_merge_options(&self) {
+        let merge_segment_ids: HashSet<SegmentId> = self.0.merge_operations.segment_in_merge();
         let (committed_segments, uncommitted_segments) =
-            get_mergeable_segments(&self.0.segment_manager);
+            get_mergeable_segments(&merge_segment_ids, &self.0.segment_manager);
+
         // Committed segments cannot be merged with uncommitted_segments.
         // We therefore consider merges using these two sets of segments independently.
         let merge_policy = self.get_merge_policy();
-        let mut merge_candidates = merge_policy.compute_merge_candidates(&uncommitted_segments);
-        let committed_merge_candidates = merge_policy.compute_merge_candidates(&committed_segments);
-        merge_candidates.extend_from_slice(&committed_merge_candidates[..]);
-        for MergeCandidate(segment_metas) in merge_candidates {
-            match self.start_merge_impl(&segment_metas) {
+
+        let current_opstamp = self.0.stamper.stamp();
+        let mut merge_candidates: Vec<MergeOperation> = merge_policy
+            .compute_merge_candidates(&uncommitted_segments)
+            .into_iter()
+            .map(|merge_candidate| {
+                MergeOperation::new(&self.0.merge_operations, current_opstamp, merge_candidate.0)
+            })
+            .collect();
+
+        let commit_opstamp = self.load_metas().opstamp;
+        let committed_merge_candidates = merge_policy
+            .compute_merge_candidates(&committed_segments)
+            .into_iter()
+            .map(|merge_candidate| {
+                MergeOperation::new(&self.0.merge_operations, commit_opstamp, merge_candidate.0)
+            })
+            .collect::<Vec<_>>();
+        merge_candidates.extend(committed_merge_candidates.into_iter());
+        for merge_operation in merge_candidates {
+            match self.start_merge_impl(merge_operation) {
                 Ok(merge_future) => {
                     if let Err(e) = merge_future.fuse().poll() {
                         error!("The merge task failed quickly after starting: {:?}", e);
@@ -393,31 +437,16 @@ impl SegmentUpdater {
         }
     }
 
-    fn cancel_merge(
-        &self,
-        before_merge_segment_ids: &[SegmentId],
-        after_merge_segment_entry: SegmentId,
-    ) {
-        self.0
-            .segment_manager
-            .cancel_merge(before_merge_segment_ids, after_merge_segment_entry);
-    }
-
     fn end_merge(
         &self,
-        before_merge_segment_ids: Vec<SegmentId>,
+        merge_operation: MergeOperation,
         mut after_merge_segment_entry: SegmentEntry,
     ) -> Result<()> {
         self.run_async(move |segment_updater| {
             info!("End merge {:?}", after_merge_segment_entry.meta());
             let mut delete_cursor = after_merge_segment_entry.delete_cursor().clone();
             if let Some(delete_operation) = delete_cursor.get() {
-                let committed_opstamp = segment_updater
-                    .0
-                    .index
-                    .load_metas()
-                    .expect("Failed to read opstamp")
-                    .opstamp;
+                let committed_opstamp = segment_updater.load_metas().opstamp;
                 if delete_operation.opstamp < committed_opstamp {
                     let index = &segment_updater.0.index;
                     let segment = index.segment(after_merge_segment_entry.meta().clone());
@@ -426,16 +455,15 @@ impl SegmentUpdater {
                     {
                         error!(
                             "Merge of {:?} was cancelled (advancing deletes failed): {:?}",
-                            before_merge_segment_ids, e
+                            merge_operation.segment_ids(),
+                            e
                         );
-                        // ... cancel merge
                         if cfg!(test) {
                             panic!("Merge failed.");
                         }
-                        segment_updater.cancel_merge(
-                            &before_merge_segment_ids,
-                            after_merge_segment_entry.segment_id(),
-                        );
+                        // ... cancel merge
+                        // `merge_operations` are tracked. As it is dropped, the
+                        // the segment_ids will be available again for merge.
                         return;
                     }
                 }
@@ -443,11 +471,11 @@ impl SegmentUpdater {
             segment_updater
                 .0
                 .segment_manager
-                .end_merge(&before_merge_segment_ids, after_merge_segment_entry);
+                .end_merge(merge_operation.segment_ids(), after_merge_segment_entry);
             segment_updater.consider_merge_options();
             info!("save metas");
-            let previous_metas = segment_updater.0.index.load_metas().unwrap();
-            segment_updater.save_metas(previous_metas.opstamp, previous_metas.payload);
+            let previous_metas = segment_updater.load_metas();
+            segment_updater.save_metas(previous_metas.opstamp, previous_metas.payload.clone());
             segment_updater.garbage_collect_files_exec();
         })
         .wait()
@@ -469,32 +497,25 @@ impl SegmentUpdater {
     /// Obsolete files will eventually be cleaned up
     /// by the directory garbage collector.
     pub fn wait_merging_thread(&self) -> Result<()> {
-        let mut num_segments: usize;
         loop {
-            num_segments = self.0.segment_manager.num_segments();
-
-            let mut new_merging_threads = HashMap::new();
-            {
+            let merging_threads: HashMap<usize, JoinHandle<Result<()>>> = {
                 let mut merging_threads = self.0.merging_threads.write().unwrap();
-                mem::swap(&mut new_merging_threads, merging_threads.deref_mut());
+                mem::replace(merging_threads.deref_mut(), HashMap::new())
+            };
+            if merging_threads.is_empty() {
+                return Ok(());
             }
-            debug!("wait merging thread {}", new_merging_threads.len());
-            for (_, merging_thread_handle) in new_merging_threads {
+            debug!("wait merging thread {}", merging_threads.len());
+            for (_, merging_thread_handle) in merging_threads {
                 merging_thread_handle
                     .join()
                     .map(|_| ())
                     .map_err(|_| TantivyError::ErrorInThread("Merging thread failed.".into()))?;
             }
-            // Our merging thread may have queued their completed
+            // Our merging thread may have queued their completed merged segment.
+            // Let's wait for that too.
             self.run_async(move |_| {}).wait()?;
-
-            let new_num_segments = self.0.segment_manager.num_segments();
-
-            if new_num_segments >= num_segments {
-                break;
-            }
         }
-        Ok(())
     }
 }
 
@@ -514,7 +535,7 @@ mod tests {
         let index = Index::create_in_ram(schema);
 
         // writing the segment
-        let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
+        let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
         index_writer.set_merge_policy(Box::new(MergeWheneverPossible));
 
         {
@@ -544,9 +565,8 @@ mod tests {
             index_writer.delete_term(term);
             assert!(index_writer.commit().is_ok());
         }
-
-        index.load_searchers().unwrap();
-        assert_eq!(index.searcher().num_docs(), 302);
+        let reader = index.reader().unwrap();
+        assert_eq!(reader.searcher().num_docs(), 302);
 
         {
             index_writer
@@ -554,8 +574,79 @@ mod tests {
                 .expect("waiting for merging threads");
         }
 
-        index.load_searchers().unwrap();
-        assert_eq!(index.searcher().segment_readers().len(), 1);
-        assert_eq!(index.searcher().num_docs(), 302);
+        reader.reload().unwrap();
+        assert_eq!(reader.searcher().segment_readers().len(), 1);
+        assert_eq!(reader.searcher().num_docs(), 302);
+    }
+
+    #[test]
+    fn delete_all_docs() {
+        let mut schema_builder = Schema::builder();
+        let text_field = schema_builder.add_text_field("text", TEXT);
+        let schema = schema_builder.build();
+
+        let index = Index::create_in_ram(schema);
+
+        // writing the segment
+        let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
+
+        {
+            for _ in 0..100 {
+                index_writer.add_document(doc!(text_field=>"a"));
+                index_writer.add_document(doc!(text_field=>"b"));
+            }
+            assert!(index_writer.commit().is_ok());
+        }
+
+        {
+            for _ in 0..100 {
+                index_writer.add_document(doc!(text_field=>"c"));
+                index_writer.add_document(doc!(text_field=>"d"));
+            }
+            assert!(index_writer.commit().is_ok());
+        }
+
+        {
+            index_writer.add_document(doc!(text_field=>"e"));
+            index_writer.add_document(doc!(text_field=>"f"));
+            assert!(index_writer.commit().is_ok());
+        }
+
+        {
+            let seg_ids = index
+                .searchable_segment_ids()
+                .expect("Searchable segments failed.");
+            // docs exist, should have at least 1 segment
+            assert!(seg_ids.len() > 0);
+        }
+
+        {
+            let term_vals = vec!["a", "b", "c", "d", "e", "f"];
+            for term_val in term_vals {
+                let term = Term::from_field_text(text_field, term_val);
+                index_writer.delete_term(term);
+                assert!(index_writer.commit().is_ok());
+            }
+        }
+
+        {
+            index_writer
+                .wait_merging_threads()
+                .expect("waiting for merging threads");
+        }
+
+        let reader = index.reader().unwrap();
+        assert_eq!(reader.searcher().num_docs(), 0);
+
+        let seg_ids = index
+            .searchable_segment_ids()
+            .expect("Searchable segments failed.");
+        assert!(seg_ids.is_empty());
+
+        reader.reload().unwrap();
+        assert_eq!(reader.searcher().num_docs(), 0);
+        // empty segments should be erased
+        assert!(index.searchable_segment_metas().unwrap().is_empty());
+        assert!(reader.searcher().segment_readers().is_empty());
     }
 }

src/indexer/segment_writer.rs

@@ -171,6 +171,17 @@ impl SegmentWriter {
                         }
                     }
                 }
+                FieldType::Date(ref int_option) => {
+                    if int_option.is_indexed() {
+                        for field_value in field_values {
+                            let term = Term::from_field_i64(
+                                field_value.field(),
+                                field_value.value().date_value().timestamp(),
+                            );
+                            self.multifield_postings.subscribe(doc_id, &term);
+                        }
+                    }
+                }
                 FieldType::I64(ref int_option) => {
                     if int_option.is_indexed() {
                         for field_value in field_values {

src/indexer/stamper.rs

@@ -1,50 +1,77 @@
+use std::ops::Range;
+use std::sync::atomic::Ordering;
+use std::sync::Arc;
+
 // AtomicU64 have not landed in stable.
 // For the moment let's just use AtomicUsize on
 // x86/64 bit platform, and a mutex on other platform.
-
-#[cfg(target = "x86_64")]
+#[cfg(target_arch = "x86_64")]
 mod archicture_impl {
 
     use std::sync::atomic::{AtomicUsize, Ordering};
-    use std::sync::Arc;
 
-    #[derive(Clone, Default)]
-    pub struct Stamper(Arc<AtomicU64>);
+    #[derive(Default)]
+    pub struct AtomicU64Ersatz(AtomicUsize);
 
-    impl Stamper {
-        pub fn new(first_opstamp: u64) -> Stamper {
-            Stamper(Arc::new(AtomicU64::new(first_opstamp)))
+    impl AtomicU64Ersatz {
+        pub fn new(first_opstamp: u64) -> AtomicU64Ersatz {
+            AtomicU64Ersatz(AtomicUsize::new(first_opstamp as usize))
         }
 
-        pub fn stamp(&self) -> u64 {
-            self.0.fetch_add(1u64, Ordering::SeqCst) as u64
+        pub fn fetch_add(&self, val: u64, order: Ordering) -> u64 {
+            self.0.fetch_add(val as usize, order) as u64
         }
     }
 }
 
-#[cfg(not(target = "x86_64"))]
+#[cfg(not(target_arch = "x86_64"))]
 mod archicture_impl {
 
-    use std::sync::{Arc, Mutex};
+    use std::sync::atomic::Ordering;
+    /// Under other architecture, we rely on a mutex.
+    use std::sync::RwLock;
 
-    #[derive(Clone, Default)]
-    pub struct Stamper(Arc<Mutex<u64>>);
+    #[derive(Default)]
+    pub struct AtomicU64Ersatz(RwLock<u64>);
 
-    impl Stamper {
-        pub fn new(first_opstamp: u64) -> Stamper {
-            Stamper(Arc::new(Mutex::new(first_opstamp)))
+    impl AtomicU64Ersatz {
+        pub fn new(first_opstamp: u64) -> AtomicU64Ersatz {
+            AtomicU64Ersatz(RwLock::new(first_opstamp))
         }
 
-        pub fn stamp(&self) -> u64 {
-            let mut guard = self.0.lock().expect("Failed to lock the stamper");
-            let previous_val = *guard;
-            *guard = previous_val + 1;
+        pub fn fetch_add(&self, incr: u64, _order: Ordering) -> u64 {
+            let mut lock = self.0.write().unwrap();
+            let previous_val = *lock;
+            *lock = previous_val + incr;
             previous_val
         }
     }
 }
 
-pub use self::archicture_impl::Stamper;
+use self::archicture_impl::AtomicU64Ersatz;
+
+#[derive(Clone, Default)]
+pub struct Stamper(Arc<AtomicU64Ersatz>);
+
+impl Stamper {
+    pub fn new(first_opstamp: u64) -> Stamper {
+        Stamper(Arc::new(AtomicU64Ersatz::new(first_opstamp)))
+    }
+
+    pub fn stamp(&self) -> u64 {
+        self.0.fetch_add(1u64, Ordering::SeqCst) as u64
+    }
+
+    /// Given a desired count `n`, `stamps` returns an iterator that
+    /// will supply `n` number of u64 stamps.
+    pub fn stamps(&self, n: u64) -> Range<u64> {
+        let start = self.0.fetch_add(n, Ordering::SeqCst);
+        Range {
+            start,
+            end: start + n,
+        }
+    }
+}
 
 #[cfg(test)]
 mod test {
@@ -62,5 +89,7 @@ mod test {
 
         assert_eq!(stamper.stamp(), 10u64);
         assert_eq!(stamper_clone.stamp(), 11u64);
+        assert_eq!(stamper.stamps(3u64), (12..15));
+        assert_eq!(stamper.stamp(), 15u64);
     }
 }

src/lib.rs

@@ -75,9 +75,9 @@
 //!
 //! // # Searching
 //!
-//! index.load_searchers()?;
+//! let reader = index.reader()?;
 //!
-//! let searcher = index.searcher();
+//! let searcher = reader.searcher();
 //!
 //! let query_parser = QueryParser::for_index(&index, vec![title, body]);
 //!
@@ -129,25 +129,24 @@ extern crate base64;
 extern crate bit_set;
 extern crate bitpacking;
 extern crate byteorder;
-extern crate scoped_pool;
-
 extern crate combine;
-
 extern crate crossbeam;
 extern crate fnv;
-extern crate fst;
-extern crate fst_regex;
 extern crate futures;
 extern crate futures_cpupool;
 extern crate htmlescape;
 extern crate itertools;
 extern crate levenshtein_automata;
+#[cfg(feature = "mmap")]
+extern crate memmap;
 extern crate num_cpus;
 extern crate owning_ref;
 extern crate regex;
 extern crate rust_stemmers;
+extern crate scoped_pool;
 extern crate serde;
 extern crate stable_deref_trait;
+extern crate tantivy_fst;
 extern crate tempdir;
 extern crate tempfile;
 extern crate uuid;
@@ -170,7 +169,7 @@ extern crate maplit;
 extern crate test;
 
 #[macro_use]
-extern crate downcast;
+extern crate downcast_rs;
 
 #[macro_use]
 extern crate fail;
@@ -187,11 +186,15 @@ pub use error::TantivyError;
 pub use error::TantivyError as Error;
 
 extern crate census;
+pub extern crate chrono;
 extern crate owned_read;
 
 /// Tantivy result.
 pub type Result<T> = std::result::Result<T, error::TantivyError>;
 
+/// Tantivy DateTime
+pub type DateTime = chrono::DateTime<chrono::Utc>;
+
 mod common;
 mod core;
 mod indexer;
@@ -212,6 +215,9 @@ pub mod space_usage;
 pub mod store;
 pub mod termdict;
 
+mod reader;
+
+pub use self::reader::{IndexReader, IndexReaderBuilder, ReloadPolicy};
 mod snippet;
 pub use self::snippet::{Snippet, SnippetGenerator};
 
@@ -231,11 +237,7 @@ pub use common::{i64_to_u64, u64_to_i64};
 /// Expose the current version of tantivy, as well
 /// whether it was compiled with the simd compression.
 pub fn version() -> &'static str {
-    if cfg!(feature = "simdcompression") {
-        concat!(env!("CARGO_PKG_VERSION"), "-simd")
-    } else {
-        concat!(env!("CARGO_PKG_VERSION"), "-nosimd")
-    }
+    env!("CARGO_PKG_VERSION")
 }
 
 /// Defines tantivy's merging strategy
@@ -304,6 +306,7 @@ mod tests {
     use Index;
     use IndexWriter;
     use Postings;
+    use ReloadPolicy;
 
     pub fn assert_nearly_equals(expected: f32, val: f32) {
         assert!(
@@ -348,7 +351,7 @@ mod tests {
         let index = Index::create_from_tempdir(schema).unwrap();
         {
             // writing the segment
-            let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
+            let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
             {
                 let doc = doc!(text_field=>"af b");
                 index_writer.add_document(doc);
@@ -370,7 +373,7 @@ mod tests {
         let mut schema_builder = Schema::builder();
         let text_field = schema_builder.add_text_field("text", TEXT);
         let index = Index::create_in_ram(schema_builder.build());
-        let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
+        let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
         {
             index_writer.add_document(doc!(text_field=>"a b c"));
             index_writer.commit().unwrap();
@@ -392,8 +395,8 @@ mod tests {
             index_writer.commit().unwrap();
         }
         {
-            index.load_searchers().unwrap();
-            let searcher = index.searcher();
+            let reader = index.reader().unwrap();
+            let searcher = reader.searcher();
             let term_a = Term::from_field_text(text_field, "a");
             assert_eq!(searcher.doc_freq(&term_a), 3);
             let term_b = Term::from_field_text(text_field, "b");
@@ -412,7 +415,7 @@ mod tests {
         let text_field = schema_builder.add_text_field("text", TEXT);
         let index = Index::create_in_ram(schema_builder.build());
         {
-            let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
+            let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
             {
                 let doc = doc!(text_field=>"a b c");
                 index_writer.add_document(doc);
@@ -420,8 +423,8 @@ mod tests {
             index_writer.commit().unwrap();
         }
         {
-            index.load_searchers().unwrap();
-            let searcher = index.searcher();
+            let index_reader = index.reader().unwrap();
+            let searcher = index_reader.searcher();
             let reader = searcher.segment_reader(0);
             {
                 let fieldnorm_reader = reader.get_fieldnorms_reader(text_field);
@@ -440,7 +443,7 @@ mod tests {
         let text_field = schema_builder.add_text_field("text", TEXT);
         let index = Index::create_in_ram(schema_builder.build());
         {
-            let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
+            let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
             {
                 let doc = doc!(text_field=>"a b c");
                 index_writer.add_document(doc);
@@ -456,8 +459,8 @@ mod tests {
             index_writer.commit().unwrap();
         }
         {
-            index.load_searchers().unwrap();
-            let searcher = index.searcher();
+            let reader = index.reader().unwrap();
+            let searcher = reader.searcher();
             let segment_reader: &SegmentReader = searcher.segment_reader(0);
             let fieldnorms_reader = segment_reader.get_fieldnorms_reader(text_field);
             assert_eq!(fieldnorms_reader.fieldnorm(0), 3);
@@ -485,9 +488,14 @@ mod tests {
         let term_c = Term::from_field_text(text_field, "c");
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
+        let reader = index
+            .reader_builder()
+            .reload_policy(ReloadPolicy::Manual)
+            .try_into()
+            .unwrap();
         {
             // writing the segment
-            let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
+            let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
             // 0
             index_writer.add_document(doc!(text_field=>"a b"));
             // 1
@@ -506,10 +514,10 @@ mod tests {
             index_writer.commit().unwrap();
         }
         {
-            index.load_searchers().unwrap();
-            let searcher = index.searcher();
-            let reader = searcher.segment_reader(0);
-            let inverted_index = reader.inverted_index(text_field);
+            reader.reload().unwrap();
+            let searcher = reader.searcher();
+            let segment_reader = searcher.segment_reader(0);
+            let inverted_index = segment_reader.inverted_index(text_field);
             assert!(inverted_index
                 .read_postings(&term_abcd, IndexRecordOption::WithFreqsAndPositions)
                 .is_none());
@@ -517,24 +525,24 @@ mod tests {
                 let mut postings = inverted_index
                     .read_postings(&term_a, IndexRecordOption::WithFreqsAndPositions)
                     .unwrap();
-                assert!(advance_undeleted(&mut postings, reader));
+                assert!(advance_undeleted(&mut postings, segment_reader));
                 assert_eq!(postings.doc(), 5);
-                assert!(!advance_undeleted(&mut postings, reader));
+                assert!(!advance_undeleted(&mut postings, segment_reader));
             }
             {
                 let mut postings = inverted_index
                     .read_postings(&term_b, IndexRecordOption::WithFreqsAndPositions)
                     .unwrap();
-                assert!(advance_undeleted(&mut postings, reader));
+                assert!(advance_undeleted(&mut postings, segment_reader));
                 assert_eq!(postings.doc(), 3);
-                assert!(advance_undeleted(&mut postings, reader));
+                assert!(advance_undeleted(&mut postings, segment_reader));
                 assert_eq!(postings.doc(), 4);
-                assert!(!advance_undeleted(&mut postings, reader));
+                assert!(!advance_undeleted(&mut postings, segment_reader));
             }
         }
         {
             // writing the segment
-            let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
+            let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
             // 0
             index_writer.add_document(doc!(text_field=>"a b"));
             // 1
@@ -542,10 +550,10 @@ mod tests {
             index_writer.rollback().unwrap();
         }
         {
-            index.load_searchers().unwrap();
-            let searcher = index.searcher();
-            let reader = searcher.segment_reader(0);
-            let inverted_index = reader.inverted_index(term_abcd.field());
+            reader.reload().unwrap();
+            let searcher = reader.searcher();
+            let seg_reader = searcher.segment_reader(0);
+            let inverted_index = seg_reader.inverted_index(term_abcd.field());
 
             assert!(inverted_index
                 .read_postings(&term_abcd, IndexRecordOption::WithFreqsAndPositions)
@@ -554,24 +562,24 @@ mod tests {
                 let mut postings = inverted_index
                     .read_postings(&term_a, IndexRecordOption::WithFreqsAndPositions)
                     .unwrap();
-                assert!(advance_undeleted(&mut postings, reader));
+                assert!(advance_undeleted(&mut postings, seg_reader));
                 assert_eq!(postings.doc(), 5);
-                assert!(!advance_undeleted(&mut postings, reader));
+                assert!(!advance_undeleted(&mut postings, seg_reader));
             }
             {
                 let mut postings = inverted_index
                     .read_postings(&term_b, IndexRecordOption::WithFreqsAndPositions)
                     .unwrap();
-                assert!(advance_undeleted(&mut postings, reader));
+                assert!(advance_undeleted(&mut postings, seg_reader));
                 assert_eq!(postings.doc(), 3);
-                assert!(advance_undeleted(&mut postings, reader));
+                assert!(advance_undeleted(&mut postings, seg_reader));
                 assert_eq!(postings.doc(), 4);
-                assert!(!advance_undeleted(&mut postings, reader));
+                assert!(!advance_undeleted(&mut postings, seg_reader));
             }
         }
         {
             // writing the segment
-            let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
+            let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
             index_writer.add_document(doc!(text_field=>"a b"));
             index_writer.delete_term(Term::from_field_text(text_field, "c"));
             index_writer.rollback().unwrap();
@@ -579,10 +587,10 @@ mod tests {
             index_writer.commit().unwrap();
         }
         {
-            index.load_searchers().unwrap();
-            let searcher = index.searcher();
-            let reader = searcher.segment_reader(0);
-            let inverted_index = reader.inverted_index(term_abcd.field());
+            reader.reload().unwrap();
+            let searcher = reader.searcher();
+            let segment_reader = searcher.segment_reader(0);
+            let inverted_index = segment_reader.inverted_index(term_abcd.field());
             assert!(inverted_index
                 .read_postings(&term_abcd, IndexRecordOption::WithFreqsAndPositions)
                 .is_none());
@@ -590,25 +598,25 @@ mod tests {
                 let mut postings = inverted_index
                     .read_postings(&term_a, IndexRecordOption::WithFreqsAndPositions)
                     .unwrap();
-                assert!(!advance_undeleted(&mut postings, reader));
+                assert!(!advance_undeleted(&mut postings, segment_reader));
             }
             {
                 let mut postings = inverted_index
                     .read_postings(&term_b, IndexRecordOption::WithFreqsAndPositions)
                     .unwrap();
-                assert!(advance_undeleted(&mut postings, reader));
+                assert!(advance_undeleted(&mut postings, segment_reader));
                 assert_eq!(postings.doc(), 3);
-                assert!(advance_undeleted(&mut postings, reader));
+                assert!(advance_undeleted(&mut postings, segment_reader));
                 assert_eq!(postings.doc(), 4);
-                assert!(!advance_undeleted(&mut postings, reader));
+                assert!(!advance_undeleted(&mut postings, segment_reader));
             }
             {
                 let mut postings = inverted_index
                     .read_postings(&term_c, IndexRecordOption::WithFreqsAndPositions)
                     .unwrap();
-                assert!(advance_undeleted(&mut postings, reader));
+                assert!(advance_undeleted(&mut postings, segment_reader));
                 assert_eq!(postings.doc(), 4);
-                assert!(!advance_undeleted(&mut postings, reader));
+                assert!(!advance_undeleted(&mut postings, segment_reader));
             }
         }
     }
@@ -616,15 +624,15 @@ mod tests {
     #[test]
     fn test_indexed_u64() {
         let mut schema_builder = Schema::builder();
-        let field = schema_builder.add_u64_field("value", INT_INDEXED);
+        let field = schema_builder.add_u64_field("value", INDEXED);
         let schema = schema_builder.build();
 
         let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
+        let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
         index_writer.add_document(doc!(field=>1u64));
         index_writer.commit().unwrap();
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        let reader = index.reader().unwrap();
+        let searcher = reader.searcher();
         let term = Term::from_field_u64(field, 1u64);
         let mut postings = searcher
             .segment_reader(0)
@@ -639,16 +647,16 @@ mod tests {
     #[test]
     fn test_indexed_i64() {
         let mut schema_builder = Schema::builder();
-        let value_field = schema_builder.add_i64_field("value", INT_INDEXED);
+        let value_field = schema_builder.add_i64_field("value", INDEXED);
         let schema = schema_builder.build();
 
         let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
+        let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
         let negative_val = -1i64;
         index_writer.add_document(doc!(value_field => negative_val));
         index_writer.commit().unwrap();
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        let reader = index.reader().unwrap();
+        let searcher = reader.searcher();
         let term = Term::from_field_i64(value_field, negative_val);
         let mut postings = searcher
             .segment_reader(0)
@@ -667,11 +675,11 @@ mod tests {
         let absent_field = schema_builder.add_text_field("text", TEXT);
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_with_num_threads(2, 40_000_000).unwrap();
+        let mut index_writer = index.writer_with_num_threads(2, 6_000_000).unwrap();
         index_writer.add_document(doc!(text_field=>"a"));
         assert!(index_writer.commit().is_ok());
-        assert!(index.load_searchers().is_ok());
-        let searcher = index.searcher();
+        let reader = index.reader().unwrap();
+        let searcher = reader.searcher();
         let segment_reader = searcher.segment_reader(0);
         segment_reader.inverted_index(absent_field); //< should not panic
     }
@@ -682,9 +690,14 @@ mod tests {
         let text_field = schema_builder.add_text_field("text", TEXT);
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
+        let reader = index
+            .reader_builder()
+            .reload_policy(ReloadPolicy::Manual)
+            .try_into()
+            .unwrap();
 
         // writing the segment
-        let mut index_writer = index.writer_with_num_threads(2, 40_000_000).unwrap();
+        let mut index_writer = index.writer_with_num_threads(2, 6_000_000).unwrap();
 
         let add_document = |index_writer: &mut IndexWriter, val: &'static str| {
             let doc = doc!(text_field=>val);
@@ -707,8 +720,8 @@ mod tests {
         remove_document(&mut index_writer, "38");
         remove_document(&mut index_writer, "34");
         index_writer.commit().unwrap();
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        reader.reload().unwrap();
+        let searcher = reader.searcher();
         assert_eq!(searcher.num_docs(), 6);
     }
 
@@ -720,7 +733,7 @@ mod tests {
         let index = Index::create_in_ram(schema);
         {
             // writing the segment
-            let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
+            let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
             {
                 let doc = doc!(text_field=>"af af af bc bc");
                 index_writer.add_document(doc);
@@ -728,8 +741,8 @@ mod tests {
             index_writer.commit().unwrap();
         }
         {
-            index.load_searchers().unwrap();
-            let searcher = index.searcher();
+            let index_reader = index.reader().unwrap();
+            let searcher = index_reader.searcher();
             let reader = searcher.segment_reader(0);
             let inverted_index = reader.inverted_index(text_field);
             let term_abcd = Term::from_field_text(text_field, "abcd");
@@ -753,18 +766,18 @@ mod tests {
         let text_field = schema_builder.add_text_field("text", TEXT);
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
-
+        let reader = index.reader().unwrap();
         {
             // writing the segment
-            let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
+            let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
             index_writer.add_document(doc!(text_field=>"af af af b"));
             index_writer.add_document(doc!(text_field=>"a b c"));
             index_writer.add_document(doc!(text_field=>"a b c d"));
             index_writer.commit().unwrap();
         }
         {
-            index.load_searchers().unwrap();
-            let searcher = index.searcher();
+            reader.reload().unwrap();
+            let searcher = reader.searcher();
             let get_doc_ids = |terms: Vec<Term>| {
                 let query = BooleanQuery::new_multiterms_query(terms);
                 let topdocs = searcher.search(&query, &TestCollector).unwrap();
@@ -806,25 +819,22 @@ mod tests {
         let text_field = schema_builder.add_text_field("text", TEXT);
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
-
+        let reader = index
+            .reader_builder()
+            .reload_policy(ReloadPolicy::Manual)
+            .try_into()
+            .unwrap();
+        assert_eq!(reader.searcher().num_docs(), 0u64);
         {
             // writing the segment
-            let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
-            {
-                let doc = doc!(text_field=>"af b");
-                index_writer.add_document(doc);
-            }
-            {
-                let doc = doc!(text_field=>"a b c");
-                index_writer.add_document(doc);
-            }
-            {
-                let doc = doc!(text_field=>"a b c d");
-                index_writer.add_document(doc);
-            }
+            let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
+            index_writer.add_document(doc!(text_field=>"af b"));
+            index_writer.add_document(doc!(text_field=>"a b c"));
+            index_writer.add_document(doc!(text_field=>"a b c d"));
             index_writer.commit().unwrap();
         }
-        index.searcher();
+        reader.reload().unwrap();
+        assert_eq!(reader.searcher().num_docs(), 3u64);
     }
 
     #[test]
@@ -851,7 +861,7 @@ mod tests {
         let fast_field_unsigned = schema_builder.add_u64_field("unsigned", FAST);
         let fast_field_signed = schema_builder.add_i64_field("signed", FAST);
         let text_field = schema_builder.add_text_field("text", TEXT);
-        let stored_int_field = schema_builder.add_u64_field("text", INT_STORED);
+        let stored_int_field = schema_builder.add_u64_field("text", STORED);
         let schema = schema_builder.build();
 
         let index = Index::create_in_ram(schema);
@@ -861,9 +871,8 @@ mod tests {
             index_writer.add_document(document);
             index_writer.commit().unwrap();
         }
-
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        let reader = index.reader().unwrap();
+        let searcher = reader.searcher();
         let segment_reader: &SegmentReader = searcher.segment_reader(0);
         {
             let fast_field_reader_res = segment_reader.fast_field_reader::<u64>(text_field);

src/macros.rs

@@ -61,7 +61,7 @@ macro_rules! doc(
     };
     // if there is a trailing comma retry with the trailing comma stripped.
     ($($field:expr => $value:expr),+ ,) => {
-        doc!( $( $field => $value ), *);
+        doc!( $( $field => $value ), *)
     };
 );
 

src/positions/mod.rs

@@ -34,10 +34,6 @@ const COMPRESSION_BLOCK_SIZE: usize = BitPacker4x::BLOCK_LEN;
 const LONG_SKIP_IN_BLOCKS: usize = 1_024;
 const LONG_SKIP_INTERVAL: u64 = (LONG_SKIP_IN_BLOCKS * COMPRESSION_BLOCK_SIZE) as u64;
 
-lazy_static! {
-    static ref BIT_PACKER: BitPacker4x = BitPacker4x::new();
-}
-
 #[cfg(test)]
 pub mod tests {
 

src/positions/reader.rs

@@ -1,4 +1,23 @@
-use super::BIT_PACKER;
+/// Positions works as a long sequence of compressed block.
+/// All terms are chained one after the other.
+///
+/// When accessing the position of a term, we get a positions_idx from the `Terminfo`.
+/// This means we need to skip to the `nth` positions efficiently.
+///
+/// This is done thanks to two levels of skiping that we refer to in the code
+/// as `long_skip` and `short_skip`.
+///
+/// The `long_skip` makes it possible to skip every 1_024 compression blocks (= 131_072 positions).
+/// Skipping offset are simply stored one after as an offset stored over 8 bytes.
+///
+/// We find the number of long skips, as `n / long_skip`.
+///
+/// Blocks are compressed using bitpacking, so `skip_read` contains the number of bytes
+/// (values can go from 0bit to 32 bits) required to decompressed every block.
+///
+/// A given block obviously takes `(128 x  num_bit_for_the_block / num_bits_in_a_byte)`,
+/// so skipping a block without decompressing it is just a matter of advancing that many
+/// bytes.
 use bitpacking::{BitPacker, BitPacker4x};
 use common::{BinarySerializable, FixedSize};
 use directory::ReadOnlySource;
@@ -8,9 +27,65 @@ use positions::LONG_SKIP_INTERVAL;
 use positions::LONG_SKIP_IN_BLOCKS;
 use postings::compression::compressed_block_size;
 
+struct Positions {
+    bit_packer: BitPacker4x,
+    skip_source: ReadOnlySource,
+    position_source: ReadOnlySource,
+    long_skip_source: ReadOnlySource,
+}
+
+impl Positions {
+    pub fn new(position_source: ReadOnlySource, skip_source: ReadOnlySource) -> Positions {
+        let skip_len = skip_source.len();
+        let (body, footer) = skip_source.split(skip_len - u32::SIZE_IN_BYTES);
+        let num_long_skips = u32::deserialize(&mut footer.as_slice()).expect("Index corrupted");
+        let body_split = body.len() - u64::SIZE_IN_BYTES * (num_long_skips as usize);
+        let (skip_source, long_skip_source) = body.split(body_split);
+        Positions {
+            bit_packer: BitPacker4x::new(),
+            skip_source,
+            long_skip_source,
+            position_source,
+        }
+    }
+
+    /// Returns the offset of the block associated to the given `long_skip_id`.
+    ///
+    /// One `long_skip_id` means `LONG_SKIP_IN_BLOCKS` blocks.
+    fn long_skip(&self, long_skip_id: usize) -> u64 {
+        if long_skip_id == 0 {
+            return 0;
+        }
+        let long_skip_slice = self.long_skip_source.as_slice();
+        let mut long_skip_blocks: &[u8] = &long_skip_slice[(long_skip_id - 1) * 8..][..8];
+        u64::deserialize(&mut long_skip_blocks).expect("Index corrupted")
+    }
+
+    fn reader(&self, offset: u64) -> PositionReader {
+        let long_skip_id = (offset / LONG_SKIP_INTERVAL) as usize;
+        let small_skip = (offset % LONG_SKIP_INTERVAL) as usize;
+        let offset_num_bytes: u64 = self.long_skip(long_skip_id);
+        let mut position_read = OwnedRead::new(self.position_source.clone());
+        position_read.advance(offset_num_bytes as usize);
+        let mut skip_read = OwnedRead::new(self.skip_source.clone());
+        skip_read.advance(long_skip_id * LONG_SKIP_IN_BLOCKS);
+        let mut position_reader = PositionReader {
+            bit_packer: self.bit_packer,
+            skip_read,
+            position_read,
+            inner_offset: 0,
+            buffer: Box::new([0u32; 128]),
+            ahead: None,
+        };
+        position_reader.skip(small_skip);
+        position_reader
+    }
+}
+
 pub struct PositionReader {
     skip_read: OwnedRead,
     position_read: OwnedRead,
+    bit_packer: BitPacker4x,
     inner_offset: usize,
     buffer: Box<[u32; 128]>,
     ahead: Option<usize>, // if None, no block is loaded.
@@ -27,6 +102,7 @@ pub struct PositionReader {
 // If the requested number of els ends exactly at a given block, the next
 // block is not decompressed.
 fn read_impl(
+    bit_packer: BitPacker4x,
     mut position: &[u8],
     buffer: &mut [u32; 128],
     mut inner_offset: usize,
@@ -37,21 +113,23 @@ fn read_impl(
     let mut output_len = output.len();
     let mut ahead = 0;
     loop {
-        let available_len = 128 - inner_offset;
+        let available_len = COMPRESSION_BLOCK_SIZE - inner_offset;
+        // We have enough elements in the current block.
+        // Let's copy the requested elements in the output buffer,
+        // and return.
         if output_len <= available_len {
             output[output_start..].copy_from_slice(&buffer[inner_offset..][..output_len]);
             return ahead;
-        } else {
-            output[output_start..][..available_len].copy_from_slice(&buffer[inner_offset..]);
-            output_len -= available_len;
-            output_start += available_len;
-            inner_offset = 0;
-            let num_bits = num_bits[ahead];
-            BitPacker4x::new().decompress(position, &mut buffer[..], num_bits);
-            let block_len = compressed_block_size(num_bits);
-            position = &position[block_len..];
-            ahead += 1;
         }
+        output[output_start..][..available_len].copy_from_slice(&buffer[inner_offset..]);
+        output_len -= available_len;
+        output_start += available_len;
+        inner_offset = 0;
+        let num_bits = num_bits[ahead];
+        bit_packer.decompress(position, &mut buffer[..], num_bits);
+        let block_len = compressed_block_size(num_bits);
+        position = &position[block_len..];
+        ahead += 1;
     }
 }
 
@@ -61,35 +139,7 @@ impl PositionReader {
         skip_source: ReadOnlySource,
         offset: u64,
     ) -> PositionReader {
-        let skip_len = skip_source.len();
-        let (body, footer) = skip_source.split(skip_len - u32::SIZE_IN_BYTES);
-        let num_long_skips = u32::deserialize(&mut footer.as_slice()).expect("Index corrupted");
-        let body_split = body.len() - u64::SIZE_IN_BYTES * (num_long_skips as usize);
-        let (skip_body, long_skips) = body.split(body_split);
-        let long_skip_id = (offset / LONG_SKIP_INTERVAL) as usize;
-        let small_skip = (offset - (long_skip_id as u64) * (LONG_SKIP_INTERVAL as u64)) as usize;
-        let offset_num_bytes: u64 = {
-            if long_skip_id > 0 {
-                let mut long_skip_blocks: &[u8] =
-                    &long_skips.as_slice()[(long_skip_id - 1) * 8..][..8];
-                u64::deserialize(&mut long_skip_blocks).expect("Index corrupted") * 16
-            } else {
-                0
-            }
-        };
-        let mut position_read = OwnedRead::new(position_source);
-        position_read.advance(offset_num_bytes as usize);
-        let mut skip_read = OwnedRead::new(skip_body);
-        skip_read.advance(long_skip_id * LONG_SKIP_IN_BLOCKS);
-        let mut position_reader = PositionReader {
-            skip_read,
-            position_read,
-            inner_offset: 0,
-            buffer: Box::new([0u32; 128]),
-            ahead: None,
-        };
-        position_reader.skip(small_skip);
-        position_reader
+        Positions::new(position_source, skip_source).reader(offset)
     }
 
     /// Fills a buffer with the next `output.len()` integers.
@@ -101,10 +151,13 @@ impl PositionReader {
         if self.ahead != Some(0) {
             // the block currently available is not the block
             // for the current position
-            BIT_PACKER.decompress(position_data, self.buffer.as_mut(), num_bits);
+            self.bit_packer
+                .decompress(position_data, self.buffer.as_mut(), num_bits);
+            self.ahead = Some(0);
         }
         let block_len = compressed_block_size(num_bits);
         self.ahead = Some(read_impl(
+            self.bit_packer,
             &position_data[block_len..],
             self.buffer.as_mut(),
             self.inner_offset,
@@ -133,14 +186,13 @@ impl PositionReader {
             }
         });
 
-        let skip_len = self.skip_read.as_ref()[..num_blocks_to_advance]
+        let skip_len_in_bits = self.skip_read.as_ref()[..num_blocks_to_advance]
             .iter()
-            .cloned()
-            .map(|num_bit| num_bit as usize)
+            .map(|num_bits| *num_bits as usize)
             .sum::<usize>()
-            * (COMPRESSION_BLOCK_SIZE / 8);
-
+            * COMPRESSION_BLOCK_SIZE;
+        let skip_len_in_bytes = skip_len_in_bits / 8;
         self.skip_read.advance(num_blocks_to_advance);
-        self.position_read.advance(skip_len);
+        self.position_read.advance(skip_len_in_bytes);
     }
 }

src/positions/serializer.rs

@@ -1,29 +1,30 @@
-use super::BIT_PACKER;
 use bitpacking::BitPacker;
+use bitpacking::BitPacker4x;
 use common::BinarySerializable;
+use common::CountingWriter;
 use positions::{COMPRESSION_BLOCK_SIZE, LONG_SKIP_INTERVAL};
-use std::io;
+use std::io::{self, Write};
 
 pub struct PositionSerializer<W: io::Write> {
-    write_stream: W,
+    bit_packer: BitPacker4x,
+    write_stream: CountingWriter<W>,
     write_skiplist: W,
     block: Vec<u32>,
     buffer: Vec<u8>,
     num_ints: u64,
     long_skips: Vec<u64>,
-    cumulated_num_bits: u64,
 }
 
 impl<W: io::Write> PositionSerializer<W> {
     pub fn new(write_stream: W, write_skiplist: W) -> PositionSerializer<W> {
         PositionSerializer {
-            write_stream,
+            bit_packer: BitPacker4x::new(),
+            write_stream: CountingWriter::wrap(write_stream),
             write_skiplist,
             block: Vec::with_capacity(128),
             buffer: vec![0u8; 128 * 4],
             num_ints: 0u64,
             long_skips: Vec::new(),
-            cumulated_num_bits: 0u64,
         }
     }
 
@@ -50,14 +51,15 @@ impl<W: io::Write> PositionSerializer<W> {
     }
 
     fn flush_block(&mut self) -> io::Result<()> {
-        let num_bits = BIT_PACKER.num_bits(&self.block[..]);
-        self.cumulated_num_bits += u64::from(num_bits);
+        let num_bits = self.bit_packer.num_bits(&self.block[..]);
         self.write_skiplist.write_all(&[num_bits])?;
-        let written_len = BIT_PACKER.compress(&self.block[..], &mut self.buffer, num_bits);
+        let written_len = self
+            .bit_packer
+            .compress(&self.block[..], &mut self.buffer, num_bits);
         self.write_stream.write_all(&self.buffer[..written_len])?;
         self.block.clear();
         if (self.num_ints % LONG_SKIP_INTERVAL) == 0u64 {
-            self.long_skips.push(self.cumulated_num_bits);
+            self.long_skips.push(self.write_stream.written_bytes());
         }
         Ok(())
     }

src/postings/mod.rs

@@ -31,7 +31,6 @@ pub(crate) use self::stacker::compute_table_size;
 pub use common::HasLen;
 
 pub(crate) const USE_SKIP_INFO_LIMIT: u32 = COMPRESSION_BLOCK_SIZE as u32;
-
 pub(crate) type UnorderedTermId = u64;
 
 #[cfg_attr(feature = "cargo-clippy", allow(clippy::enum_variant_names))]
@@ -58,7 +57,7 @@ pub mod tests {
     use rand::{Rng, SeedableRng};
     use schema::Field;
     use schema::IndexRecordOption;
-    use schema::{Document, Schema, Term, INT_INDEXED, STRING, TEXT};
+    use schema::{Document, Schema, Term, INDEXED, STRING, TEXT};
     use std::iter;
     use DocId;
     use Score;
@@ -101,9 +100,8 @@ pub mod tests {
         }
         index_writer.add_document(doc!(title => r#"abc be be be be abc"#));
         index_writer.commit().unwrap();
-        index.load_searchers().unwrap();
 
-        let searcher = index.searcher();
+        let searcher = index.reader().unwrap().searcher();
         let inverted_index = searcher.segment_reader(0u32).inverted_index(title);
         let term = Term::from_field_text(title, "abc");
 
@@ -280,7 +278,7 @@ pub mod tests {
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
         {
-            let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
+            let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
             {
                 let mut doc = Document::default();
                 doc.add_text(text_field, "g b b d c g c");
@@ -293,9 +291,8 @@ pub mod tests {
             }
             assert!(index_writer.commit().is_ok());
         }
-        index.load_searchers().unwrap();
         let term_a = Term::from_field_text(text_field, "a");
-        let searcher = index.searcher();
+        let searcher = index.reader().unwrap().searcher();
         let segment_reader = searcher.segment_reader(0);
         let mut postings = segment_reader
             .inverted_index(text_field)
@@ -317,12 +314,12 @@ pub mod tests {
 
         let index = {
             let mut schema_builder = Schema::builder();
-            let value_field = schema_builder.add_u64_field("value", INT_INDEXED);
+            let value_field = schema_builder.add_u64_field("value", INDEXED);
             let schema = schema_builder.build();
 
             let index = Index::create_in_ram(schema);
             {
-                let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
+                let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
                 for i in 0..num_docs {
                     let mut doc = Document::default();
                     doc.add_u64(value_field, 2);
@@ -332,10 +329,9 @@ pub mod tests {
                 }
                 assert!(index_writer.commit().is_ok());
             }
-            index.load_searchers().unwrap();
             index
         };
-        let searcher = index.searcher();
+        let searcher = index.reader().unwrap().searcher();
         let segment_reader = searcher.segment_reader(0);
 
         // check that the basic usage works
@@ -399,12 +395,11 @@ pub mod tests {
 
         // delete some of the documents
         {
-            let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
+            let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
             index_writer.delete_term(term_0);
             assert!(index_writer.commit().is_ok());
         }
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        let searcher = index.reader().unwrap().searcher();
         let segment_reader = searcher.segment_reader(0);
 
         // make sure seeking still works
@@ -449,33 +444,19 @@ pub mod tests {
 
         // delete everything else
         {
-            let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
+            let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
             index_writer.delete_term(term_1);
-
             assert!(index_writer.commit().is_ok());
         }
-        index.load_searchers().unwrap();
-
-        let searcher = index.searcher();
-        let segment_reader = searcher.segment_reader(0);
+        let searcher = index.reader().unwrap().searcher();
 
         // finally, check that it's empty
         {
-            let mut segment_postings = segment_reader
-                .inverted_index(term_2.field())
-                .read_postings(&term_2, IndexRecordOption::Basic)
-                .unwrap();
-
-            assert_eq!(segment_postings.skip_next(0), SkipResult::Reached);
-            assert_eq!(segment_postings.doc(), 0);
-            assert!(segment_reader.is_deleted(0));
-
-            let mut segment_postings = segment_reader
-                .inverted_index(term_2.field())
-                .read_postings(&term_2, IndexRecordOption::Basic)
-                .unwrap();
-
-            assert_eq!(segment_postings.skip_next(num_docs), SkipResult::End);
+            let searchable_segment_ids = index
+                .searchable_segment_ids()
+                .expect("could not get index segment ids");
+            assert!(searchable_segment_ids.is_empty());
+            assert_eq!(searcher.num_docs(), 0);
         }
     }
 
@@ -506,7 +487,7 @@ pub mod tests {
             let index = Index::create_in_ram(schema);
             let posting_list_size = 1_000_000;
             {
-                let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
+                let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
                 for _ in 0..posting_list_size {
                     let mut doc = Document::default();
                     if rng.gen_bool(1f64 / 15f64) {
@@ -523,7 +504,6 @@ pub mod tests {
                 }
                 assert!(index_writer.commit().is_ok());
             }
-            index.load_searchers().unwrap();
             index
         };
     }

src/postings/postings_writer.rs

@@ -33,9 +33,10 @@ fn posting_from_field_entry(field_entry: &FieldEntry) -> Box<PostingsWriter> {
                 }
             })
             .unwrap_or_else(|| SpecializedPostingsWriter::<NothingRecorder>::new_boxed()),
-        FieldType::U64(_) | FieldType::I64(_) | FieldType::HierarchicalFacet => {
-            SpecializedPostingsWriter::<NothingRecorder>::new_boxed()
-        }
+        FieldType::U64(_)
+        | FieldType::I64(_)
+        | FieldType::Date(_)
+        | FieldType::HierarchicalFacet => SpecializedPostingsWriter::<NothingRecorder>::new_boxed(),
         FieldType::Bytes => {
             // FieldType::Bytes cannot actually be indexed.
             // TODO fix during the indexer refactoring described in #276
@@ -51,6 +52,31 @@ pub struct MultiFieldPostingsWriter {
     per_field_postings_writers: Vec<Box<PostingsWriter>>,
 }
 
+fn make_field_partition(
+    term_offsets: &[(&[u8], Addr, UnorderedTermId)],
+) -> Vec<(Field, usize, usize)> {
+    let term_offsets_it = term_offsets
+        .iter()
+        .map(|(key, _, _)| Term::wrap(key).field())
+        .enumerate();
+    let mut prev_field = Field(u32::max_value());
+    let mut fields = vec![];
+    let mut offsets = vec![];
+    for (offset, field) in term_offsets_it {
+        if field != prev_field {
+            prev_field = field;
+            fields.push(field);
+            offsets.push(offset);
+        }
+    }
+    offsets.push(term_offsets.len());
+    let mut field_offsets = vec![];
+    for i in 0..fields.len() {
+        field_offsets.push((fields[i], offsets[i], offsets[i + 1]));
+    }
+    field_offsets
+}
+
 impl MultiFieldPostingsWriter {
     /// Create a new `MultiFieldPostingsWriter` given
     /// a schema and a heap.
@@ -96,36 +122,16 @@ impl MultiFieldPostingsWriter {
         &self,
         serializer: &mut InvertedIndexSerializer,
     ) -> Result<HashMap<Field, HashMap<UnorderedTermId, TermOrdinal>>> {
-        let mut term_offsets: Vec<(&[u8], Addr, UnorderedTermId)> = self
-            .term_index
-            .iter()
-            .map(|(term_bytes, addr, bucket_id)| (term_bytes, addr, bucket_id as UnorderedTermId))
-            .collect();
+        let mut term_offsets: Vec<(&[u8], Addr, UnorderedTermId)> =
+            self.term_index.iter().collect();
         term_offsets.sort_unstable_by_key(|&(k, _, _)| k);
 
-        let mut offsets: Vec<(Field, usize)> = vec![];
-        let term_offsets_it = term_offsets
-            .iter()
-            .cloned()
-            .map(|(key, _, _)| Term::wrap(key).field())
-            .enumerate();
-
         let mut unordered_term_mappings: HashMap<Field, HashMap<UnorderedTermId, TermOrdinal>> =
             HashMap::new();
 
-        let mut prev_field = Field(u32::max_value());
-        for (offset, field) in term_offsets_it {
-            if field != prev_field {
-                offsets.push((field, offset));
-                prev_field = field;
-            }
-        }
-        offsets.push((Field(0), term_offsets.len()));
-
-        for i in 0..(offsets.len() - 1) {
-            let (field, start) = offsets[i];
-            let (_, stop) = offsets[i + 1];
+        let field_offsets = make_field_partition(&term_offsets);
 
+        for (field, start, stop) in field_offsets {
             let field_entry = self.schema.get_field_entry(field);
 
             match *field_entry.field_type() {
@@ -143,7 +149,7 @@ impl MultiFieldPostingsWriter {
                         .collect();
                     unordered_term_mappings.insert(field, mapping);
                 }
-                FieldType::U64(_) | FieldType::I64(_) => {}
+                FieldType::U64(_) | FieldType::I64(_) | FieldType::Date(_) => {}
                 FieldType::Bytes => {}
             }
 

src/postings/recorder.rs

@@ -4,7 +4,6 @@ use postings::FieldSerializer;
 use std::io;
 use DocId;
 
-const EMPTY_ARRAY: [u32; 0] = [0u32; 0];
 const POSITION_END: u32 = 0;
 
 #[derive(Default)]
@@ -115,7 +114,7 @@ impl Recorder for NothingRecorder {
         let buffer = buffer_lender.lend_u8();
         self.stack.read_to_end(heap, buffer);
         for doc in VInt32Reader::new(&buffer[..]) {
-            serializer.write_doc(doc as u32, 0u32, &EMPTY_ARRAY)?;
+            serializer.write_doc(doc as u32, 0u32, &[][..])?;
         }
         Ok(())
     }
@@ -168,7 +167,7 @@ impl Recorder for TermFrequencyRecorder {
         let mut u32_it = VInt32Reader::new(&buffer[..]);
         while let Some(doc) = u32_it.next() {
             let term_freq = u32_it.next().unwrap_or(self.current_tf);
-            serializer.write_doc(doc as u32, term_freq, &EMPTY_ARRAY)?;
+            serializer.write_doc(doc as u32, term_freq, &[][..])?;
         }
 
         Ok(())

src/postings/segment_postings.rs

@@ -2,7 +2,6 @@ use common::BitSet;
 use common::HasLen;
 use common::{BinarySerializable, VInt};
 use docset::{DocSet, SkipResult};
-use fst::Streamer;
 use owned_read::OwnedRead;
 use positions::PositionReader;
 use postings::compression::compressed_block_size;
@@ -14,10 +13,9 @@ use postings::SkipReader;
 use postings::USE_SKIP_INFO_LIMIT;
 use schema::IndexRecordOption;
 use std::cmp::Ordering;
+use tantivy_fst::Streamer;
 use DocId;
 
-const EMPTY_ARR: [u8; 0] = [];
-
 struct PositionComputer {
     // store the amount of position int
     // before reading positions.
@@ -123,22 +121,23 @@ impl SegmentPostings {
     }
 }
 
-fn exponential_search(target: u32, arr: &[u32]) -> (usize, usize) {
-    let mut start = 0;
+fn linear_search(arr: &[u32], target: u32) -> usize {
+    arr.iter().map(|&el| if el < target { 1 } else { 0 }).sum()
+}
+
+fn exponential_search(arr: &[u32], target: u32) -> (usize, usize) {
     let end = arr.len();
-    debug_assert!(target <= arr[end - 1]);
-    let mut jump = 1;
-    loop {
-        let new = start + jump;
-        if new >= end {
-            return (start, end);
+    let mut begin = 0;
+    for &pivot in &[1, 3, 7, 15, 31, 63] {
+        if pivot >= end {
+            break;
         }
-        if arr[new] > target {
-            return (start, new);
+        if arr[pivot] > target {
+            return (begin, pivot);
         }
-        start = new;
-        jump *= 2;
+        begin = pivot;
     }
+    (begin, end)
 }
 
 /// Search the first index containing an element greater or equal to the target.
@@ -149,12 +148,8 @@ fn exponential_search(target: u32, arr: &[u32]) -> (usize, usize) {
 /// The target is assumed greater or equal to the first element.
 /// The target is assumed smaller or equal to the last element.
 fn search_within_block(block_docs: &[u32], target: u32) -> usize {
-    let (start, end) = exponential_search(target, block_docs);
-    start.wrapping_add(
-        block_docs[start..end]
-            .binary_search(&target)
-            .unwrap_or_else(|e| e),
-    )
+    let (start, end) = exponential_search(block_docs, target);
+    start + linear_search(&block_docs[start..end], target)
 }
 
 impl DocSet for SegmentPostings {
@@ -372,7 +367,7 @@ impl BlockSegmentPostings {
         let (skip_data_opt, postings_data) = split_into_skips_and_postings(doc_freq, data);
         let skip_reader = match skip_data_opt {
             Some(skip_data) => SkipReader::new(skip_data, record_option),
-            None => SkipReader::new(OwnedRead::new(&EMPTY_ARR[..]), record_option),
+            None => SkipReader::new(OwnedRead::new(&[][..]), record_option),
         };
         let doc_freq = doc_freq as usize;
         let num_vint_docs = doc_freq % COMPRESSION_BLOCK_SIZE;
@@ -406,7 +401,7 @@ impl BlockSegmentPostings {
         if let Some(skip_data) = skip_data_opt {
             self.skip_reader.reset(skip_data);
         } else {
-            self.skip_reader.reset(OwnedRead::new(&EMPTY_ARR[..]))
+            self.skip_reader.reset(OwnedRead::new(&[][..]))
         }
         self.doc_offset = 0;
         self.doc_freq = doc_freq as usize;
@@ -621,6 +616,7 @@ impl<'b> Streamer<'b> for BlockSegmentPostings {
 mod tests {
 
     use super::exponential_search;
+    use super::linear_search;
     use super::search_within_block;
     use super::BlockSegmentPostings;
     use super::BlockSegmentPostingsSkipResult;
@@ -628,14 +624,29 @@ mod tests {
     use common::HasLen;
     use core::Index;
     use docset::DocSet;
-    use fst::Streamer;
     use schema::IndexRecordOption;
     use schema::Schema;
     use schema::Term;
-    use schema::INT_INDEXED;
+    use schema::INDEXED;
+    use tantivy_fst::Streamer;
     use DocId;
     use SkipResult;
 
+    #[test]
+    fn test_linear_search() {
+        let len: usize = 50;
+        let arr: Vec<u32> = (0..len).map(|el| 1u32 + (el as u32) * 2).collect();
+        for target in 1..*arr.last().unwrap() {
+            let res = linear_search(&arr[..], target);
+            if res > 0 {
+                assert!(arr[res - 1] < target);
+            }
+            if res < len {
+                assert!(arr[res] >= target);
+            }
+        }
+    }
+
     #[test]
     fn test_empty_segment_postings() {
         let mut postings = SegmentPostings::empty();
@@ -664,10 +675,10 @@ mod tests {
 
     #[test]
     fn test_exponentiel_search() {
-        assert_eq!(exponential_search(0, &[1, 2]), (0, 1));
-        assert_eq!(exponential_search(1, &[1, 2]), (0, 1));
+        assert_eq!(exponential_search(&[1, 2], 0), (0, 1));
+        assert_eq!(exponential_search(&[1, 2], 1), (0, 1));
         assert_eq!(
-            exponential_search(7, &[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]),
+            exponential_search(&[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], 7),
             (3, 7)
         );
     }
@@ -749,10 +760,10 @@ mod tests {
 
     fn build_block_postings(docs: &[DocId]) -> BlockSegmentPostings {
         let mut schema_builder = Schema::builder();
-        let int_field = schema_builder.add_u64_field("id", INT_INDEXED);
+        let int_field = schema_builder.add_u64_field("id", INDEXED);
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
+        let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
         let mut last_doc = 0u32;
         for &doc in docs {
             for _ in last_doc..doc {
@@ -762,8 +773,7 @@ mod tests {
             last_doc = doc + 1;
         }
         index_writer.commit().unwrap();
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        let searcher = index.reader().unwrap().searcher();
         let segment_reader = searcher.segment_reader(0);
         let inverted_index = segment_reader.inverted_index(int_field);
         let term = Term::from_field_u64(int_field, 0u64);
@@ -820,10 +830,10 @@ mod tests {
     #[test]
     fn test_reset_block_segment_postings() {
         let mut schema_builder = Schema::builder();
-        let int_field = schema_builder.add_u64_field("id", INT_INDEXED);
+        let int_field = schema_builder.add_u64_field("id", INDEXED);
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
+        let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
         // create two postings list, one containg even number,
         // the other containing odd numbers.
         for i in 0..6 {
@@ -831,8 +841,7 @@ mod tests {
             index_writer.add_document(doc);
         }
         index_writer.commit().unwrap();
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        let searcher = index.reader().unwrap().searcher();
         let segment_reader = searcher.segment_reader(0);
 
         let mut block_segments;

src/postings/stacker/term_hashmap.rs

@@ -5,12 +5,11 @@ use self::murmurhash32::murmurhash2;
 use super::{Addr, MemoryArena};
 use byteorder::{ByteOrder, NativeEndian};
 use postings::stacker::memory_arena::store;
+use postings::UnorderedTermId;
 use std::iter;
 use std::mem;
 use std::slice;
 
-pub type BucketId = usize;
-
 /// Returns the actual memory size in bytes
 /// required to create a table of size $2^num_bits$.
 pub fn compute_table_size(num_bits: usize) -> usize {
@@ -28,6 +27,7 @@ pub fn compute_table_size(num_bits: usize) -> usize {
 struct KeyValue {
     key_value_addr: Addr,
     hash: u32,
+    unordered_term_id: UnorderedTermId,
 }
 
 impl Default for KeyValue {
@@ -35,6 +35,7 @@ impl Default for KeyValue {
         KeyValue {
             key_value_addr: Addr::null_pointer(),
             hash: 0u32,
+            unordered_term_id: UnorderedTermId::default(),
         }
     }
 }
@@ -59,6 +60,7 @@ pub struct TermHashMap {
     pub heap: MemoryArena,
     mask: usize,
     occupied: Vec<usize>,
+    len: usize,
 }
 
 struct QuadraticProbing {
@@ -85,13 +87,13 @@ pub struct Iter<'a> {
 }
 
 impl<'a> Iterator for Iter<'a> {
-    type Item = (&'a [u8], Addr, BucketId);
+    type Item = (&'a [u8], Addr, UnorderedTermId);
 
     fn next(&mut self) -> Option<Self::Item> {
         self.inner.next().cloned().map(move |bucket: usize| {
             let kv = self.hashmap.table[bucket];
             let (key, offset): (&'a [u8], Addr) = self.hashmap.get_key_value(kv.key_value_addr);
-            (key, offset, bucket as BucketId)
+            (key, offset, kv.unordered_term_id)
         })
     }
 }
@@ -106,6 +108,7 @@ impl TermHashMap {
             heap,
             mask: table_size - 1,
             occupied: Vec::with_capacity(table_size / 2),
+            len: 0,
         }
     }
 
@@ -139,12 +142,16 @@ impl TermHashMap {
         }
     }
 
-    pub fn set_bucket(&mut self, hash: u32, key_value_addr: Addr, bucket: usize) {
+    fn set_bucket(&mut self, hash: u32, key_value_addr: Addr, bucket: usize) -> UnorderedTermId {
         self.occupied.push(bucket);
+        let unordered_term_id = self.len as UnorderedTermId;
+        self.len += 1;
         self.table[bucket] = KeyValue {
             key_value_addr,
             hash,
+            unordered_term_id,
         };
+        unordered_term_id
     }
 
     pub fn iter(&self) -> Iter {
@@ -184,7 +191,11 @@ impl TermHashMap {
     /// will be in charge of returning a default value.
     /// If the key already as an associated value, then it will be passed
     /// `Some(previous_value)`.
-    pub fn mutate_or_create<S, V, TMutator>(&mut self, key: S, mut updater: TMutator) -> BucketId
+    pub fn mutate_or_create<S, V, TMutator>(
+        &mut self,
+        key: S,
+        mut updater: TMutator,
+    ) -> UnorderedTermId
     where
         S: AsRef<[u8]>,
         V: Copy + 'static,
@@ -200,6 +211,7 @@ impl TermHashMap {
             let bucket = probe.next_probe();
             let kv: KeyValue = self.table[bucket];
             if kv.is_empty() {
+                // The key does not exists yet.
                 let val = updater(None);
                 let num_bytes =
                     std::mem::size_of::<u16>() + key_bytes.len() + std::mem::size_of::<V>();
@@ -211,8 +223,7 @@ impl TermHashMap {
                     data[2..stop].copy_from_slice(key_bytes);
                     store(&mut data[stop..], val);
                 }
-                self.set_bucket(hash, key_addr, bucket);
-                return bucket as BucketId;
+                return self.set_bucket(hash, key_addr, bucket);
             } else if kv.hash == hash {
                 if let Some(val_addr) =
                     self.get_value_addr_if_key_match(key_bytes, kv.key_value_addr)
@@ -220,7 +231,7 @@ impl TermHashMap {
                     let v = self.heap.read(val_addr);
                     let new_v = updater(Some(v));
                     self.heap.write_at(val_addr, new_v);
-                    return bucket as BucketId;
+                    return kv.unordered_term_id;
                 }
             }
         }

src/query/all_query.rs

@@ -101,8 +101,9 @@ mod tests {
         index_writer.commit().unwrap();
         index_writer.add_document(doc!(field=>"ccc"));
         index_writer.commit().unwrap();
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        let reader = index.reader().unwrap();
+        reader.reload().unwrap();
+        let searcher = reader.searcher();
         let weight = AllQuery.weight(&searcher, false).unwrap();
         {
             let reader = searcher.segment_reader(0);

src/query/automaton_weight.rs

@@ -1,10 +1,10 @@
 use common::BitSet;
 use core::SegmentReader;
-use fst::Automaton;
 use query::BitSetDocSet;
 use query::ConstScorer;
 use query::{Scorer, Weight};
 use schema::{Field, IndexRecordOption};
+use tantivy_fst::Automaton;
 use termdict::{TermDictionary, TermStreamer};
 use Result;
 

src/query/boolean_query/boolean_weight.rs

@@ -1,5 +1,4 @@
 use core::SegmentReader;
-use downcast::Downcast;
 use query::intersect_scorers;
 use query::score_combiner::{DoNothingCombiner, ScoreCombiner, SumWithCoordsCombiner};
 use query::term_query::TermScorer;
@@ -10,7 +9,6 @@ use query::RequiredOptionalScorer;
 use query::Scorer;
 use query::Union;
 use query::Weight;
-use std::borrow::Borrow;
 use std::collections::HashMap;
 use Result;
 
@@ -24,14 +22,11 @@ where
     }
 
     {
-        let is_all_term_queries = scorers.iter().all(|scorer| {
-            let scorer_ref: &Scorer = scorer.borrow();
-            Downcast::<TermScorer>::is_type(scorer_ref)
-        });
+        let is_all_term_queries = scorers.iter().all(|scorer| scorer.is::<TermScorer>());
         if is_all_term_queries {
             let scorers: Vec<TermScorer> = scorers
                 .into_iter()
-                .map(|scorer| *Downcast::<TermScorer>::downcast(scorer).unwrap())
+                .map(|scorer| *(scorer.downcast::<TermScorer>().map_err(|_| ()).unwrap()))
                 .collect();
             let scorer: Box<Scorer> = Box::new(Union::<TermScorer, TScoreCombiner>::from(scorers));
             return scorer;

src/query/boolean_query/mod.rs

@@ -8,7 +8,6 @@ mod tests {
 
     use super::*;
     use collector::tests::TestCollector;
-    use downcast::Downcast;
     use query::score_combiner::SumWithCoordsCombiner;
     use query::term_query::TermScorer;
     use query::Intersection;
@@ -29,7 +28,7 @@ mod tests {
         let index = Index::create_in_ram(schema);
         {
             // writing the segment
-            let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
+            let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
             {
                 let doc = doc!(text_field => "a b c");
                 index_writer.add_document(doc);
@@ -52,7 +51,6 @@ mod tests {
             }
             assert!(index_writer.commit().is_ok());
         }
-        index.load_searchers().unwrap();
         (index, text_field)
     }
 
@@ -61,7 +59,8 @@ mod tests {
         let (index, text_field) = aux_test_helper();
         let query_parser = QueryParser::for_index(&index, vec![text_field]);
         let query = query_parser.parse_query("(+a +b) d").unwrap();
-        assert_eq!(query.count(&*index.searcher()).unwrap(), 3);
+        let searcher = index.reader().unwrap().searcher();
+        assert_eq!(query.count(&searcher).unwrap(), 3);
     }
 
     #[test]
@@ -69,28 +68,28 @@ mod tests {
         let (index, text_field) = aux_test_helper();
         let query_parser = QueryParser::for_index(&index, vec![text_field]);
         let query = query_parser.parse_query("+a").unwrap();
-        let searcher = index.searcher();
+        let searcher = index.reader().unwrap().searcher();
         let weight = query.weight(&searcher, true).unwrap();
         let scorer = weight.scorer(searcher.segment_reader(0u32)).unwrap();
-        assert!(Downcast::<TermScorer>::is_type(&*scorer));
+        assert!(scorer.is::<TermScorer>());
     }
 
     #[test]
     pub fn test_boolean_termonly_intersection() {
         let (index, text_field) = aux_test_helper();
         let query_parser = QueryParser::for_index(&index, vec![text_field]);
-        let searcher = index.searcher();
+        let searcher = index.reader().unwrap().searcher();
         {
             let query = query_parser.parse_query("+a +b +c").unwrap();
             let weight = query.weight(&searcher, true).unwrap();
             let scorer = weight.scorer(searcher.segment_reader(0u32)).unwrap();
-            assert!(Downcast::<Intersection<TermScorer>>::is_type(&*scorer));
+            assert!(scorer.is::<Intersection<TermScorer>>());
         }
         {
             let query = query_parser.parse_query("+a +(b c)").unwrap();
             let weight = query.weight(&searcher, true).unwrap();
             let scorer = weight.scorer(searcher.segment_reader(0u32)).unwrap();
-            assert!(Downcast::<Intersection<Box<Scorer>>>::is_type(&*scorer));
+            assert!(scorer.is::<Intersection<Box<Scorer>>>());
         }
     }
 
@@ -98,21 +97,19 @@ mod tests {
     pub fn test_boolean_reqopt() {
         let (index, text_field) = aux_test_helper();
         let query_parser = QueryParser::for_index(&index, vec![text_field]);
-        let searcher = index.searcher();
+        let searcher = index.reader().unwrap().searcher();
         {
             let query = query_parser.parse_query("+a b").unwrap();
             let weight = query.weight(&searcher, true).unwrap();
             let scorer = weight.scorer(searcher.segment_reader(0u32)).unwrap();
-            assert!(Downcast::<
-                RequiredOptionalScorer<Box<Scorer>, Box<Scorer>, SumWithCoordsCombiner>,
-            >::is_type(&*scorer));
+            assert!(scorer
+                .is::<RequiredOptionalScorer<Box<Scorer>, Box<Scorer>, SumWithCoordsCombiner>>());
         }
         {
             let query = query_parser.parse_query("+a b").unwrap();
             let weight = query.weight(&searcher, false).unwrap();
             let scorer = weight.scorer(searcher.segment_reader(0u32)).unwrap();
-            println!("{:?}", scorer.type_name());
-            assert!(Downcast::<TermScorer>::is_type(&*scorer));
+            assert!(scorer.is::<TermScorer>());
         }
     }
 
@@ -129,10 +126,13 @@ mod tests {
             query
         };
 
+        let reader = index.reader().unwrap();
+
         let matching_docs = |boolean_query: &Query| {
-            let searcher = index.searcher();
-            let test_docs = searcher.search(boolean_query, &TestCollector).unwrap();
-            test_docs
+            reader
+                .searcher()
+                .search(boolean_query, &TestCollector)
+                .unwrap()
                 .docs()
                 .iter()
                 .cloned()
@@ -188,10 +188,12 @@ mod tests {
             let query: Box<Query> = Box::new(term_query);
             query
         };
-
+        let reader = index.reader().unwrap();
         let score_docs = |boolean_query: &Query| {
-            let searcher = index.searcher();
-            let fruit = searcher.search(boolean_query, &TestCollector).unwrap();
+            let fruit = reader
+                .searcher()
+                .search(boolean_query, &TestCollector)
+                .unwrap();
             fruit.scores().to_vec()
         };
 

src/query/fuzzy_query.rs

@@ -52,9 +52,8 @@ lazy_static! {
 ///         ));
 ///         index_writer.commit().unwrap();
 ///     }
-///
-///     index.load_searchers()?;
-///     let searcher = index.searcher();
+///     let reader = index.reader()?;
+///     let searcher = reader.searcher();
 ///
 ///     {
 ///
@@ -141,8 +140,8 @@ mod test {
             ));
             index_writer.commit().unwrap();
         }
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        let reader = index.reader().unwrap();
+        let searcher = reader.searcher();
         {
             let term = Term::from_field_text(country_field, "japon");
 

src/query/intersection.rs

@@ -1,9 +1,7 @@
 use docset::{DocSet, SkipResult};
-use downcast::Downcast;
 use query::term_query::TermScorer;
 use query::EmptyScorer;
 use query::Scorer;
-use std::borrow::Borrow;
 use DocId;
 use Score;
 
@@ -26,13 +24,12 @@ pub fn intersect_scorers(mut scorers: Vec<Box<Scorer>>) -> Box<Scorer> {
         (Some(single_docset), None) => single_docset,
         (Some(left), Some(right)) => {
             {
-                let all_term_scorers = [&left, &right].into_iter().all(|scorer| {
-                    let scorer_ref: &Scorer = (*scorer).borrow();
-                    Downcast::<TermScorer>::is_type(scorer_ref)
-                });
+                let all_term_scorers = [&left, &right]
+                    .iter()
+                    .all(|&scorer| scorer.is::<TermScorer>());
                 if all_term_scorers {
-                    let left = *Downcast::<TermScorer>::downcast(left).unwrap();
-                    let right = *Downcast::<TermScorer>::downcast(right).unwrap();
+                    let left = *(left.downcast::<TermScorer>().map_err(|_| ()).unwrap());
+                    let right = *(right.downcast::<TermScorer>().map_err(|_| ()).unwrap());
                     return Box::new(Intersection {
                         left,
                         right,

src/query/phrase_query/mod.rs

@@ -24,14 +24,13 @@ mod tests {
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
         {
-            let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
+            let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
             for &text in texts {
                 let doc = doc!(text_field=>text);
                 index_writer.add_document(doc);
             }
             assert!(index_writer.commit().is_ok());
         }
-        index.load_searchers().unwrap();
         index
     }
 
@@ -46,8 +45,7 @@ mod tests {
         ]);
         let schema = index.schema();
         let text_field = schema.get_field("text").unwrap();
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        let searcher = index.reader().unwrap().searcher();
         let test_query = |texts: Vec<&str>| {
             let terms: Vec<Term> = texts
                 .iter()
@@ -86,12 +84,11 @@ mod tests {
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
         {
-            let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
+            let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
             index_writer.add_document(doc!(text_field=>"a b c"));
             assert!(index_writer.commit().is_ok());
         }
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        let searcher = index.reader().unwrap().searcher();
         let phrase_query = PhraseQuery::new(vec![
             Term::from_field_text(text_field, "a"),
             Term::from_field_text(text_field, "b"),
@@ -115,8 +112,7 @@ mod tests {
         let index = create_index(&["a b c", "a b c a b"]);
         let schema = index.schema();
         let text_field = schema.get_field("text").unwrap();
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        let searcher = index.reader().unwrap().searcher();
         let test_query = |texts: Vec<&str>| {
             let terms: Vec<Term> = texts
                 .iter()
@@ -141,15 +137,14 @@ mod tests {
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
         {
-            let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
+            let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
             index_writer.add_document(doc!(text_field=>"b"));
             index_writer.add_document(doc!(text_field=>"a b"));
             index_writer.add_document(doc!(text_field=>"b a"));
             assert!(index_writer.commit().is_ok());
         }
 
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        let searcher = index.reader().unwrap().searcher();
         let test_query = |texts: Vec<&str>| {
             let terms: Vec<Term> = texts
                 .iter()
@@ -173,12 +168,11 @@ mod tests {
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
         {
-            let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
+            let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
             index_writer.add_document(doc!(text_field=>"a b c d e f g h"));
             assert!(index_writer.commit().is_ok());
         }
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        let searcher = index.reader().unwrap().searcher();
         let test_query = |texts: Vec<(usize, &str)>| {
             let terms: Vec<(usize, Term)> = texts
                 .iter()

src/query/phrase_query/phrase_scorer.rs

@@ -43,7 +43,7 @@ impl<TPostings: Postings> DocSet for PostingsWithOffset<TPostings> {
 
 pub struct PhraseScorer<TPostings: Postings> {
     intersection_docset: Intersection<PostingsWithOffset<TPostings>, PostingsWithOffset<TPostings>>,
-    num_docsets: usize,
+    num_terms: usize,
     left: Vec<u32>,
     right: Vec<u32>,
     phrase_count: u32,
@@ -138,7 +138,7 @@ impl<TPostings: Postings> PhraseScorer<TPostings> {
             .collect::<Vec<_>>();
         PhraseScorer {
             intersection_docset: Intersection::new(postings_with_offsets),
-            num_docsets,
+            num_terms: num_docsets,
             left: Vec::with_capacity(100),
             right: Vec::with_capacity(100),
             phrase_count: 0u32,
@@ -165,7 +165,7 @@ impl<TPostings: Postings> PhraseScorer<TPostings> {
                 .positions(&mut self.left);
         }
         let mut intersection_len = self.left.len();
-        for i in 1..self.num_docsets - 1 {
+        for i in 1..self.num_terms - 1 {
             {
                 self.intersection_docset
                     .docset_mut_specialized(i)
@@ -178,7 +178,7 @@ impl<TPostings: Postings> PhraseScorer<TPostings> {
         }
 
         self.intersection_docset
-            .docset_mut_specialized(self.num_docsets - 1)
+            .docset_mut_specialized(self.num_terms - 1)
             .positions(&mut self.right);
         intersection_exists(&self.left[..intersection_len], &self.right[..])
     }
@@ -190,7 +190,7 @@ impl<TPostings: Postings> PhraseScorer<TPostings> {
                 .positions(&mut self.left);
         }
         let mut intersection_len = self.left.len();
-        for i in 1..self.num_docsets - 1 {
+        for i in 1..self.num_terms - 1 {
             {
                 self.intersection_docset
                     .docset_mut_specialized(i)
@@ -203,7 +203,7 @@ impl<TPostings: Postings> PhraseScorer<TPostings> {
         }
 
         self.intersection_docset
-            .docset_mut_specialized(self.num_docsets - 1)
+            .docset_mut_specialized(self.num_terms - 1)
             .positions(&mut self.right);
         intersection_count(&self.left[..intersection_len], &self.right[..]) as u32
     }

src/query/query.rs

@@ -1,6 +1,6 @@
 use super::Weight;
 use core::searcher::Searcher;
-use downcast;
+use downcast_rs;
 use std::collections::BTreeSet;
 use std::fmt;
 use Result;
@@ -39,7 +39,7 @@ use Term;
 ///
 /// When implementing a new type of `Query`, it is normal to implement a
 /// dedicated `Query`, `Weight` and `Scorer`.
-pub trait Query: QueryClone + downcast::Any + fmt::Debug {
+pub trait Query: QueryClone + downcast_rs::Downcast + fmt::Debug {
     /// Create the weight associated to a query.
     ///
     /// If scoring is not required, setting `scoring_enabled` to `false`
@@ -96,7 +96,4 @@ impl QueryClone for Box<Query> {
     }
 }
 
-#[allow(missing_docs)]
-mod downcast_impl {
-    downcast!(super::Query);
-}
+impl_downcast!(Query);

src/query/query_parser/query_grammar.rs

@@ -52,7 +52,7 @@ parser! {
             field_name: None,
             phrase,
         });
-        try(term_query)
+        attempt(term_query)
             .or(term_default_field)
             .map(UserInputLeaf::from)
     }
@@ -83,12 +83,12 @@ parser! {
         let lower_bound = {
             let excl = (char('{'), term_val()).map(|(_, w)| UserInputBound::Exclusive(w));
             let incl = (char('['), term_val()).map(|(_, w)| UserInputBound::Inclusive(w));
-            try(excl).or(incl)
+            attempt(excl).or(incl)
         };
         let upper_bound = {
             let excl = (term_val(), char('}')).map(|(w, _)| UserInputBound::Exclusive(w));
             let incl = (term_val(), char(']')).map(|(w, _)| UserInputBound::Inclusive(w));
-            try(excl).or(incl)
+            attempt(excl).or(incl)
         };
         (
             optional((field(), char(':')).map(|x| x.0)),
@@ -112,11 +112,11 @@ parser! {
         .or((char('+'), leaf()).map(|(_, expr)| expr.unary(Occur::Must) ))
         .or((char('('), parse_to_ast(), char(')')).map(|(_, expr, _)| expr))
         .or(char('*').map(|_| UserInputAST::from(UserInputLeaf::All) ))
-        .or(try(
+        .or(attempt(
             (string("NOT"), spaces1(), leaf()).map(|(_, _, expr)| expr.unary(Occur::MustNot))
             )
          )
-        .or(try(
+        .or(attempt(
             range().map(UserInputAST::from)
             )
         )
@@ -160,7 +160,7 @@ parser! {
     where [I: Stream<Item = char>]
     {
         (
-            try(
+            attempt(
                 chainl1(
                     leaf().map(Element::SingleEl),
                     binary_operand().map(|op: BinaryOperand|

src/query/query_parser/query_parser.rs

@@ -50,6 +50,8 @@ pub enum QueryParserError {
     /// The query contains a range query with a phrase as one of the bounds.
     /// Only terms can be used as bounds.
     RangeMustNotHavePhrase,
+    /// The format for the date field is not RFC 3339 compliant.
+    DateFormatError(chrono::ParseError),
 }
 
 impl From<ParseIntError> for QueryParserError {
@@ -58,6 +60,12 @@ impl From<ParseIntError> for QueryParserError {
     }
 }
 
+impl From<chrono::ParseError> for QueryParserError {
+    fn from(err: chrono::ParseError) -> QueryParserError {
+        QueryParserError::DateFormatError(err)
+    }
+}
+
 /// Recursively remove empty clause from the AST
 ///
 /// Returns `None` iff the `logical_ast` ended up being empty.
@@ -127,6 +135,8 @@ fn trim_ast(logical_ast: LogicalAST) -> Option<LogicalAST> {
 ///   a word lexicographically between `a` and `c` (inclusive lower bound, exclusive upper bound).
 ///   Inclusive bounds are `[]`, exclusive are `{}`.
 ///
+/// * date values: The query parser supports rfc3339 formatted dates. For example "2002-10-02T15:00:00.05Z"
+///
 /// *  all docs query: A plain `*` will match all documents in the index.
 ///
 #[derive(Clone)]
@@ -229,6 +239,13 @@ impl QueryParser {
                 let term = Term::from_field_i64(field, val);
                 Ok(vec![(0, term)])
             }
+            FieldType::Date(_) => match chrono::DateTime::parse_from_rfc3339(phrase) {
+                Ok(x) => Ok(vec![(
+                    0,
+                    Term::from_field_date(field, &x.with_timezone(&chrono::Utc)),
+                )]),
+                Err(e) => Err(QueryParserError::DateFormatError(e)),
+            },
             FieldType::U64(_) => {
                 let val: u64 = u64::from_str(phrase)?;
                 let term = Term::from_field_u64(field, val);
@@ -487,7 +504,7 @@ mod test {
     use query::Query;
     use schema::Field;
     use schema::{IndexRecordOption, TextFieldIndexing, TextOptions};
-    use schema::{Schema, Term, INT_INDEXED, STORED, STRING, TEXT};
+    use schema::{Schema, Term, INDEXED, STORED, STRING, TEXT};
     use tokenizer::{LowerCaser, SimpleTokenizer, StopWordFilter, Tokenizer, TokenizerManager};
     use Index;
 
@@ -501,13 +518,14 @@ mod test {
             .set_stored();
         let title = schema_builder.add_text_field("title", TEXT);
         let text = schema_builder.add_text_field("text", TEXT);
-        schema_builder.add_i64_field("signed", INT_INDEXED);
-        schema_builder.add_u64_field("unsigned", INT_INDEXED);
+        schema_builder.add_i64_field("signed", INDEXED);
+        schema_builder.add_u64_field("unsigned", INDEXED);
         schema_builder.add_text_field("notindexed_text", STORED);
         schema_builder.add_text_field("notindexed_u64", STORED);
         schema_builder.add_text_field("notindexed_i64", STORED);
         schema_builder.add_text_field("nottokenized", STRING);
         schema_builder.add_text_field("with_stop_words", text_options);
+        schema_builder.add_date_field("date", INDEXED);
         let schema = schema_builder.build();
         let default_fields = vec![title, text];
         let tokenizer_manager = TokenizerManager::default();
@@ -767,6 +785,18 @@ mod test {
         );
     }
 
+    #[test]
+    pub fn test_query_parser_expected_date() {
+        let query_parser = make_query_parser();
+        assert_matches!(
+            query_parser.parse_query("date:18a"),
+            Err(QueryParserError::DateFormatError(_))
+        );
+        assert!(query_parser
+            .parse_query("date:\"1985-04-12T23:20:50.52Z\"")
+            .is_ok());
+    }
+
     #[test]
     pub fn test_query_parser_not_empty_but_no_tokens() {
         let query_parser = make_query_parser();

src/query/range_query.rs

@@ -40,14 +40,14 @@ fn map_bound<TFrom, TTo, Transform: Fn(&TFrom) -> TTo>(
 /// # #[macro_use]
 /// # extern crate tantivy;
 /// # use tantivy::Index;
-/// # use tantivy::schema::{Schema, INT_INDEXED};
+/// # use tantivy::schema::{Schema, INDEXED};
 /// # use tantivy::collector::Count;
 /// # use tantivy::Result;
 /// # use tantivy::query::RangeQuery;
 /// #
 /// # fn run() -> Result<()> {
 /// #     let mut schema_builder = Schema::builder();
-/// #     let year_field = schema_builder.add_u64_field("year", INT_INDEXED);
+/// #     let year_field = schema_builder.add_u64_field("year", INDEXED);
 /// #     let schema = schema_builder.build();
 /// #
 /// #     let index = Index::create_in_ram(schema);
@@ -61,8 +61,8 @@ fn map_bound<TFrom, TTo, Transform: Fn(&TFrom) -> TTo>(
 /// #         }
 /// #         index_writer.commit().unwrap();
 /// #     }
-/// #   index.load_searchers()?;
-/// let searcher = index.searcher();
+/// #   let reader = index.reader()?;
+/// let searcher = reader.searcher();
 ///
 /// let docs_in_the_sixties = RangeQuery::new_u64(year_field, 1960..1970);
 ///
@@ -293,7 +293,7 @@ mod tests {
 
     use super::RangeQuery;
     use collector::Count;
-    use schema::{Document, Field, Schema, INT_INDEXED};
+    use schema::{Document, Field, Schema, INDEXED};
     use std::collections::Bound;
     use Index;
     use Result;
@@ -302,7 +302,7 @@ mod tests {
     fn test_range_query_simple() {
         fn run() -> Result<()> {
             let mut schema_builder = Schema::builder();
-            let year_field = schema_builder.add_u64_field("year", INT_INDEXED);
+            let year_field = schema_builder.add_u64_field("year", INDEXED);
             let schema = schema_builder.build();
 
             let index = Index::create_in_ram(schema);
@@ -316,8 +316,8 @@ mod tests {
                 }
                 index_writer.commit().unwrap();
             }
-            index.load_searchers().unwrap();
-            let searcher = index.searcher();
+            let reader = index.reader().unwrap();
+            let searcher = reader.searcher();
 
             let docs_in_the_sixties = RangeQuery::new_u64(year_field, 1960u64..1970u64);
 
@@ -335,7 +335,7 @@ mod tests {
         let int_field: Field;
         let schema = {
             let mut schema_builder = Schema::builder();
-            int_field = schema_builder.add_i64_field("intfield", INT_INDEXED);
+            int_field = schema_builder.add_i64_field("intfield", INDEXED);
             schema_builder.build()
         };
 
@@ -355,8 +355,8 @@ mod tests {
 
             index_writer.commit().unwrap();
         }
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        let reader = index.reader().unwrap();
+        let searcher = reader.searcher();
         let count_multiples =
             |range_query: RangeQuery| searcher.search(&range_query, &Count).unwrap();
 

src/query/regex_query.rs

@@ -1,8 +1,8 @@
 use error::TantivyError;
-use fst_regex::Regex;
 use query::{AutomatonWeight, Query, Weight};
 use schema::Field;
 use std::clone::Clone;
+use tantivy_fst::Regex;
 use Result;
 use Searcher;
 
@@ -44,8 +44,8 @@ use Searcher;
 ///         index_writer.commit().unwrap();
 ///     }
 ///
-///     index.load_searchers()?;
-///     let searcher = index.searcher();
+///     let reader = index.reader()?;
+///     let searcher = reader.searcher();
 ///
 ///     let term = Term::from_field_text(title, "Diary");
 ///     let query = RegexQuery::new("d[ai]{2}ry".to_string(), title);
@@ -108,8 +108,8 @@ mod test {
             ));
             index_writer.commit().unwrap();
         }
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        let reader = index.reader().unwrap();
+        let searcher = reader.searcher();
         {
             let regex_query = RegexQuery::new("jap[ao]n".to_string(), country_field);
             let scored_docs = searcher

src/query/scorer.rs

@@ -1,6 +1,6 @@
 use common::BitSet;
 use docset::{DocSet, SkipResult};
-use downcast;
+use downcast_rs;
 use std::ops::DerefMut;
 use DocId;
 use Score;
@@ -8,7 +8,7 @@ use Score;
 /// Scored set of documents matching a query within a specific segment.
 ///
 /// See [`Query`](./trait.Query.html).
-pub trait Scorer: downcast::Any + DocSet + 'static {
+pub trait Scorer: downcast_rs::Downcast + DocSet + 'static {
     /// Returns the score.
     ///
     /// This method will perform a bit of computation and is not cached.
@@ -23,10 +23,7 @@ pub trait Scorer: downcast::Any + DocSet + 'static {
     }
 }
 
-#[allow(missing_docs)]
-mod downcast_impl {
-    downcast!(super::Scorer);
-}
+impl_downcast!(Scorer);
 
 impl Scorer for Box<Scorer> {
     fn score(&mut self) -> Score {

src/query/term_query/mod.rs

@@ -25,16 +25,14 @@ mod tests {
         let index = Index::create_in_ram(schema);
         {
             // writing the segment
-            let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
+            let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
             {
                 let doc = doc!(text_field => "a");
                 index_writer.add_document(doc);
             }
             assert!(index_writer.commit().is_ok());
         }
-
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        let searcher = index.reader().unwrap().searcher();
         let term_query = TermQuery::new(
             Term::from_field_text(text_field, "a"),
             IndexRecordOption::Basic,
@@ -65,8 +63,7 @@ mod tests {
             index_writer.add_document(doc!(left_field => "left4 left1"));
             index_writer.commit().unwrap();
         }
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
+        let searcher = index.reader().unwrap().searcher();
         {
             let term = Term::from_field_text(left_field, "left2");
             let term_query = TermQuery::new(term, IndexRecordOption::WithFreqs);

src/query/term_query/term_query.rs

@@ -48,9 +48,8 @@ use Term;
 ///         ));
 ///         index_writer.commit()?;
 ///     }
-///
-///     index.load_searchers()?;
-///     let searcher = index.searcher();
+///     let reader = index.reader()?;
+///     let searcher = reader.searcher();
 ///
 ///     let query = TermQuery::new(
 ///         Term::from_field_text(title, "diary"),

src/query/vec_docset.rs

@@ -5,8 +5,6 @@ use docset::DocSet;
 use std::num::Wrapping;
 use DocId;
 
-const EMPTY_ARRAY: [u32; 0] = [];
-
 /// Simulate a `Postings` objects from a `VecPostings`.
 /// `VecPostings` only exist for testing purposes.
 ///

src/reader/mod.rs

@@ -0,0 +1,187 @@
+mod pool;
+
+use self::pool::{LeasedItem, Pool};
+use core::Segment;
+use directory::Directory;
+use directory::WatchHandle;
+use directory::META_LOCK;
+use std::sync::Arc;
+use Index;
+use Result;
+use Searcher;
+use SegmentReader;
+
+/// Defines when a new version of the index should be reloaded.
+///
+/// Regardless of whether you search and index in the same process, tantivy does not necessarily
+/// reflects the change that are commited to your index. `ReloadPolicy` precisely helps you define
+/// when you want your index to be reloaded.
+#[derive(Clone, Copy)]
+pub enum ReloadPolicy {
+    /// The index is entirely reloaded manually.
+    /// All updates of the index should be manual.
+    ///
+    /// No change is reflected automatically. You are required to call `.load_seacher()` manually.
+    Manual,
+    /// The index is reloaded within milliseconds after a new commit is available.
+    /// This is made possible by watching changes in the `meta.json` file.
+    OnCommit, // TODO add NEAR_REAL_TIME(target_ms)
+}
+
+/// `IndexReader` builder
+///
+/// It makes it possible to set the following values.
+///
+/// - `num_searchers` (by default, the number of detected CPU threads):
+///
+///   When `num_searchers` queries are requested at the same time, the `num_searchers` will block
+///   until the one of the searcher in-use gets released.
+/// - `reload_policy` (by default `ReloadPolicy::OnCommit`):
+///
+///   See [`ReloadPolicy`](./enum.ReloadPolicy.html) for more details.
+#[derive(Clone)]
+pub struct IndexReaderBuilder {
+    num_searchers: usize,
+    reload_policy: ReloadPolicy,
+    index: Index,
+}
+
+impl IndexReaderBuilder {
+    pub(crate) fn new(index: Index) -> IndexReaderBuilder {
+        IndexReaderBuilder {
+            num_searchers: num_cpus::get(),
+            reload_policy: ReloadPolicy::OnCommit,
+            index,
+        }
+    }
+
+    /// Builds the reader.
+    ///
+    /// Building the reader is a non-trivial operation that requires
+    /// to open different segment readers. It may take hundreds of milliseconds
+    /// of time and it may return an error.
+    /// TODO(pmasurel) Use the `TryInto` trait once it is available in stable.
+    pub fn try_into(self) -> Result<IndexReader> {
+        let inner_reader = InnerIndexReader {
+            index: self.index,
+            num_searchers: self.num_searchers,
+            searcher_pool: Pool::new(),
+        };
+        inner_reader.reload()?;
+        let inner_reader_arc = Arc::new(inner_reader);
+        let watch_handle_opt: Option<WatchHandle>;
+        match self.reload_policy {
+            ReloadPolicy::Manual => {
+                // No need to set anything...
+                watch_handle_opt = None;
+            }
+            ReloadPolicy::OnCommit => {
+                let inner_reader_arc_clone = inner_reader_arc.clone();
+                let callback = move || {
+                    if let Err(err) = inner_reader_arc_clone.reload() {
+                        error!(
+                            "Error while loading searcher after commit was detected. {:?}",
+                            err
+                        );
+                    }
+                };
+                let watch_handle = inner_reader_arc.index.directory().watch(Box::new(callback));
+                watch_handle_opt = Some(watch_handle);
+            }
+        }
+        Ok(IndexReader {
+            inner: inner_reader_arc,
+            watch_handle_opt,
+        })
+    }
+
+    /// Sets the reload_policy.
+    ///
+    /// See [`ReloadPolicy`](./enum.ReloadPolicy.html) for more details.
+    pub fn reload_policy(mut self, reload_policy: ReloadPolicy) -> IndexReaderBuilder {
+        self.reload_policy = reload_policy;
+        self
+    }
+
+    /// Sets the number of `Searcher` in the searcher pool.
+    pub fn num_searchers(mut self, num_searchers: usize) -> IndexReaderBuilder {
+        self.num_searchers = num_searchers;
+        self
+    }
+}
+
+struct InnerIndexReader {
+    num_searchers: usize,
+    searcher_pool: Pool<Searcher>,
+    index: Index,
+}
+
+impl InnerIndexReader {
+    fn reload(&self) -> Result<()> {
+        let segment_readers: Vec<SegmentReader> = {
+            let _meta_lock = self.index.directory().acquire_lock(&META_LOCK)?;
+            let searchable_segments = self.searchable_segments()?;
+            searchable_segments
+                .iter()
+                .map(SegmentReader::open)
+                .collect::<Result<_>>()?
+        };
+        let schema = self.index.schema();
+        let searchers = (0..self.num_searchers)
+            .map(|_| Searcher::new(schema.clone(), self.index.clone(), segment_readers.clone()))
+            .collect();
+        self.searcher_pool.publish_new_generation(searchers);
+        Ok(())
+    }
+
+    /// Returns the list of segments that are searchable
+    fn searchable_segments(&self) -> Result<Vec<Segment>> {
+        self.index.searchable_segments()
+    }
+
+    fn searcher(&self) -> LeasedItem<Searcher> {
+        self.searcher_pool.acquire()
+    }
+}
+
+/// `IndexReader` is your entry point to read and search the index.
+///
+/// It controls when a new version of the index should be loaded and lends
+/// you instances of `Searcher` for the last loaded version.
+///
+/// `Clone` does not clone the different pool of searcher. `IndexReader`
+/// just wraps and `Arc`.
+#[derive(Clone)]
+pub struct IndexReader {
+    inner: Arc<InnerIndexReader>,
+    watch_handle_opt: Option<WatchHandle>,
+}
+
+impl IndexReader {
+    /// Update searchers so that they reflect the state of the last
+    /// `.commit()`.
+    ///
+    /// If you set up the `OnCommit` `ReloadPolicy` (which is the default)
+    /// every commit should be rapidly reflected on your `IndexReader` and you should
+    /// not need to call `reload()` at all.
+    ///
+    /// This automatic reload can take 10s of milliseconds to kick in however, and in unit tests
+    /// it can be nice to deterministically force the reload of searchers.
+    pub fn reload(&self) -> Result<()> {
+        self.inner.reload()
+    }
+
+    /// Returns a searcher
+    ///
+    /// This method should be called every single time a search
+    /// query is performed.
+    /// The searchers are taken from a pool of `num_searchers` searchers.
+    /// If no searcher is available
+    /// this may block.
+    ///
+    /// The same searcher must be used for a given query, as it ensures
+    /// the use of a consistent segment set.
+    pub fn searcher(&self) -> LeasedItem<Searcher> {
+        self.inner.searcher()
+    }
+}

src/reader/pool.rs

@@ -1,5 +1,5 @@
-use crossbeam::queue::MsQueue;
-use std::mem;
+use crossbeam::crossbeam_channel::unbounded;
+use crossbeam::{Receiver, RecvError, Sender};
 use std::ops::{Deref, DerefMut};
 use std::sync::atomic::AtomicUsize;
 use std::sync::atomic::Ordering;
@@ -10,15 +10,52 @@ pub struct GenerationItem<T> {
     item: T,
 }
 
+/// Queue implementation for the Object Pool below
+/// Uses the unbounded Linked-List type queue from crossbeam-channel
+/// Splits the Queue into sender and receiver
+struct Queue<T> {
+    sender: Sender<T>,
+    receiver: Receiver<T>,
+}
+
+impl<T> Queue<T> {
+    fn new() -> Self {
+        let (s, r) = unbounded();
+        Queue {
+            sender: s,
+            receiver: r,
+        }
+    }
+
+    /// Sender trait returns a Result type, which is ignored.
+    /// The Result is not handled at the moment
+    fn push(&self, elem: T) {
+        self.sender
+            .send(elem)
+            .expect("Sending an item to crossbeam-queue shouldn't fail");
+    }
+
+    /// Relies on the underlying crossbeam-channel Receiver
+    /// to block on empty queue
+    fn pop(&self) -> Result<T, RecvError> {
+        self.receiver.recv()
+    }
+}
+
+/// An object pool
+///
+/// This is used in tantivy to create a pool of `Searcher`.
+/// Object are wrapped in a `LeasedItem` wrapper and are
+/// released automatically back into the pool on `Drop`.
 pub struct Pool<T> {
-    queue: Arc<MsQueue<GenerationItem<T>>>,
+    queue: Arc<Queue<GenerationItem<T>>>,
     freshest_generation: AtomicUsize,
     next_generation: AtomicUsize,
 }
 
 impl<T> Pool<T> {
     pub fn new() -> Pool<T> {
-        let queue = Arc::new(MsQueue::new());
+        let queue = Arc::new(Queue::new());
         Pool {
             queue,
             freshest_generation: AtomicUsize::default(),
@@ -26,6 +63,10 @@ impl<T> Pool<T> {
         }
     }
 
+    /// Publishes a new generation of `Searcher`.
+    ///
+    /// After publish, all new `Searcher` acquired will be
+    /// of the new generation.
     pub fn publish_new_generation(&self, items: Vec<T>) {
         let next_generation = self.next_generation.fetch_add(1, Ordering::SeqCst) + 1;
         for item in items {
@@ -61,10 +102,14 @@ impl<T> Pool<T> {
         self.freshest_generation.load(Ordering::Acquire)
     }
 
+    /// Acquires a new searcher.
+    ///
+    /// If no searcher is available, this methods block until
+    /// a searcher is released.
     pub fn acquire(&self) -> LeasedItem<T> {
         let generation = self.generation();
         loop {
-            let gen_item = self.queue.pop();
+            let gen_item = self.queue.pop().unwrap();
             if gen_item.generation >= generation {
                 return LeasedItem {
                     gen_item: Some(gen_item),
@@ -80,7 +125,7 @@ impl<T> Pool<T> {
 
 pub struct LeasedItem<T> {
     gen_item: Option<GenerationItem<T>>,
-    recycle_queue: Arc<MsQueue<GenerationItem<T>>>,
+    recycle_queue: Arc<Queue<GenerationItem<T>>>,
 }
 
 impl<T> Deref for LeasedItem<T> {
@@ -107,9 +152,9 @@ impl<T> DerefMut for LeasedItem<T> {
 
 impl<T> Drop for LeasedItem<T> {
     fn drop(&mut self) {
-        let gen_item: GenerationItem<T> = mem::replace(&mut self.gen_item, None)
-            .expect("Unwrapping a leased item should never fail");
-        self.recycle_queue.push(gen_item);
+        if let Some(gen_item) = self.gen_item.take() {
+            self.recycle_queue.push(gen_item);
+        }
     }
 }
 
@@ -117,6 +162,7 @@ impl<T> Drop for LeasedItem<T> {
 mod tests {
 
     use super::Pool;
+    use super::Queue;
     use std::iter;
 
     #[test]
@@ -133,4 +179,47 @@ mod tests {
             assert_eq!(*pool.acquire(), 11);
         }
     }
+
+    #[test]
+    fn test_queue() {
+        let q = Queue::new();
+        let elem = 5;
+        q.push(elem);
+        let res = q.pop();
+        assert_eq!(res.unwrap(), elem);
+    }
+
+    #[test]
+    fn test_pool_dont_panic_on_empty_pop() {
+        // When the object pool is exhausted, it shouldn't panic on pop()
+        use std::sync::Arc;
+        use std::{thread, time};
+
+        // Wrap the pool in an Arc, same way as its used in `core/index.rs`
+        let pool = Arc::new(Pool::new());
+        // clone pools outside the move scope of each new thread
+        let pool1 = Arc::clone(&pool);
+        let pool2 = Arc::clone(&pool);
+        let elements_for_pool = vec![1, 2];
+        pool.publish_new_generation(elements_for_pool);
+
+        let mut threads = vec![];
+        let sleep_dur = time::Duration::from_millis(10);
+        // spawn one more thread than there are elements in the pool
+        threads.push(thread::spawn(move || {
+            // leasing to make sure it's not dropped before sleep is called
+            let _leased_searcher = &pool.acquire();
+            thread::sleep(sleep_dur);
+        }));
+        threads.push(thread::spawn(move || {
+            // leasing to make sure it's not dropped before sleep is called
+            let _leased_searcher = &pool1.acquire();
+            thread::sleep(sleep_dur);
+        }));
+        threads.push(thread::spawn(move || {
+            // leasing to make sure it's not dropped before sleep is called
+            let _leased_searcher = &pool2.acquire();
+            thread::sleep(sleep_dur);
+        }));
+    }
 }

src/schema/document.rs

@@ -3,6 +3,7 @@ use common::BinarySerializable;
 use common::VInt;
 use itertools::Itertools;
 use std::io::{self, Read, Write};
+use DateTime;
 
 /// Tantivy's Document is the object that can
 /// be indexed and then searched for.
@@ -82,11 +83,16 @@ impl Document {
         self.add(FieldValue::new(field, Value::U64(value)));
     }
 
-    /// Add a u64 field
+    /// Add a i64 field
     pub fn add_i64(&mut self, field: Field, value: i64) {
         self.add(FieldValue::new(field, Value::I64(value)));
     }
 
+    /// Add a date field
+    pub fn add_date(&mut self, field: Field, value: &DateTime) {
+        self.add(FieldValue::new(field, Value::Date(*value)));
+    }
+
     /// Add a bytes field
     pub fn add_bytes(&mut self, field: Field, value: Vec<u8>) {
         self.add(FieldValue::new(field, Value::Bytes(value)))

src/schema/field_entry.rs

@@ -48,6 +48,15 @@ impl FieldEntry {
         }
     }
 
+    /// Creates a new date field entry in the schema, given
+    /// a name, and some options.
+    pub fn new_date(field_name: String, field_type: IntOptions) -> FieldEntry {
+        FieldEntry {
+            name: field_name,
+            field_type: FieldType::Date(field_type),
+        }
+    }
+
     /// Creates a field entry for a facet.
     pub fn new_facet(field_name: String) -> FieldEntry {
         FieldEntry {
@@ -78,7 +87,9 @@ impl FieldEntry {
     pub fn is_indexed(&self) -> bool {
         match self.field_type {
             FieldType::Str(ref options) => options.get_indexing_options().is_some(),
-            FieldType::U64(ref options) | FieldType::I64(ref options) => options.is_indexed(),
+            FieldType::U64(ref options)
+            | FieldType::I64(ref options)
+            | FieldType::Date(ref options) => options.is_indexed(),
             FieldType::HierarchicalFacet => true,
             FieldType::Bytes => false,
         }
@@ -95,7 +106,9 @@ impl FieldEntry {
     /// Returns true iff the field is stored
     pub fn is_stored(&self) -> bool {
         match self.field_type {
-            FieldType::U64(ref options) | FieldType::I64(ref options) => options.is_stored(),
+            FieldType::U64(ref options)
+            | FieldType::I64(ref options)
+            | FieldType::Date(ref options) => options.is_stored(),
             FieldType::Str(ref options) => options.is_stored(),
             // TODO make stored hierarchical facet optional
             FieldType::HierarchicalFacet => true,
@@ -125,6 +138,10 @@ impl Serialize for FieldEntry {
                 s.serialize_field("type", "i64")?;
                 s.serialize_field("options", options)?;
             }
+            FieldType::Date(ref options) => {
+                s.serialize_field("type", "date")?;
+                s.serialize_field("options", options)?;
+            }
             FieldType::HierarchicalFacet => {
                 s.serialize_field("type", "hierarchical_facet")?;
             }
@@ -188,7 +205,7 @@ impl<'de> Deserialize<'de> for FieldEntry {
                                 "bytes" => {
                                     field_type = Some(FieldType::Bytes);
                                 }
-                                "text" | "u64" | "i64" => {
+                                "text" | "u64" | "i64" | "date" => {
                                     // These types require additional options to create a field_type
                                 }
                                 _ => panic!("unhandled type"),
@@ -205,6 +222,7 @@ impl<'de> Deserialize<'de> for FieldEntry {
                                 "text" => field_type = Some(FieldType::Str(map.next_value()?)),
                                 "u64" => field_type = Some(FieldType::U64(map.next_value()?)),
                                 "i64" => field_type = Some(FieldType::I64(map.next_value()?)),
+                                "date" => field_type = Some(FieldType::Date(map.next_value()?)),
                                 _ => {
                                     let msg = format!("Unrecognised type {}", ty);
                                     return Err(de::Error::custom(msg));

src/schema/field_type.rs

@@ -34,6 +34,8 @@ pub enum Type {
     U64,
     /// `i64`
     I64,
+    /// `date(i64) timestamp`
+    Date,
     /// `tantivy::schema::Facet`. Passed as a string in JSON.
     HierarchicalFacet,
     /// `Vec<u8>`
@@ -50,6 +52,8 @@ pub enum FieldType {
     U64(IntOptions),
     /// Signed 64-bits integers 64 field type configuration
     I64(IntOptions),
+    /// Signed 64-bits Date 64 field type configuration,
+    Date(IntOptions),
     /// Hierachical Facet
     HierarchicalFacet,
     /// Bytes (one per document)
@@ -63,6 +67,7 @@ impl FieldType {
             FieldType::Str(_) => Type::Str,
             FieldType::U64(_) => Type::U64,
             FieldType::I64(_) => Type::I64,
+            FieldType::Date(_) => Type::Date,
             FieldType::HierarchicalFacet => Type::HierarchicalFacet,
             FieldType::Bytes => Type::Bytes,
         }
@@ -75,6 +80,7 @@ impl FieldType {
             FieldType::U64(ref int_options) | FieldType::I64(ref int_options) => {
                 int_options.is_indexed()
             }
+            FieldType::Date(ref date_options) => date_options.is_indexed(),
             FieldType::HierarchicalFacet => true,
             FieldType::Bytes => false,
         }
@@ -89,7 +95,9 @@ impl FieldType {
             FieldType::Str(ref text_options) => text_options
                 .get_indexing_options()
                 .map(|indexing_options| indexing_options.index_option()),
-            FieldType::U64(ref int_options) | FieldType::I64(ref int_options) => {
+            FieldType::U64(ref int_options)
+            | FieldType::I64(ref int_options)
+            | FieldType::Date(ref int_options) => {
                 if int_options.is_indexed() {
                     Some(IndexRecordOption::Basic)
                 } else {
@@ -110,9 +118,9 @@ impl FieldType {
         match *json {
             JsonValue::String(ref field_text) => match *self {
                 FieldType::Str(_) => Ok(Value::Str(field_text.clone())),
-                FieldType::U64(_) | FieldType::I64(_) => Err(ValueParsingError::TypeError(
-                    format!("Expected an integer, got {:?}", json),
-                )),
+                FieldType::U64(_) | FieldType::I64(_) | FieldType::Date(_) => Err(
+                    ValueParsingError::TypeError(format!("Expected an integer, got {:?}", json)),
+                ),
                 FieldType::HierarchicalFacet => Ok(Value::Facet(Facet::from(field_text))),
                 FieldType::Bytes => decode(field_text).map(Value::Bytes).map_err(|_| {
                     ValueParsingError::InvalidBase64(format!(
@@ -122,7 +130,7 @@ impl FieldType {
                 }),
             },
             JsonValue::Number(ref field_val_num) => match *self {
-                FieldType::I64(_) => {
+                FieldType::I64(_) | FieldType::Date(_) => {
                     if let Some(field_val_i64) = field_val_num.as_i64() {
                         Ok(Value::I64(field_val_i64))
                     } else {

src/schema/flags.rs

@@ -0,0 +1,81 @@
+use schema::IntOptions;
+use schema::TextOptions;
+use std::ops::BitOr;
+
+#[derive(Clone)]
+pub struct StoredFlag;
+/// Flag to mark the field as stored.
+/// This flag can apply to any kind of field.
+///
+/// A stored fields of a document can be retrieved given its `DocId`.
+/// Stored field are stored together and LZ4 compressed.
+/// Reading the stored fields of a document is relatively slow.
+/// (~ 100 microsecs)
+///
+/// It should not be used during scoring or collection.
+pub const STORED: SchemaFlagList<StoredFlag, ()> = SchemaFlagList {
+    head: StoredFlag,
+    tail: (),
+};
+
+#[derive(Clone)]
+pub struct IndexedFlag;
+/// Flag to mark the field as indexed.
+///
+/// The `INDEXED` flag can only be used when building `IntOptions` (`u64` and `i64` fields)
+/// Of course, text fields can also be indexed... But this is expressed by using either the
+/// `STRING` (untokenized) or `TEXT` (tokenized with the english tokenizer) flags.
+pub const INDEXED: SchemaFlagList<IndexedFlag, ()> = SchemaFlagList {
+    head: IndexedFlag,
+    tail: (),
+};
+
+#[derive(Clone)]
+pub struct FastFlag;
+/// Flag to mark the field as a fast field (similar to Lucene's DocValues)
+///
+/// Fast fields can be random-accessed rapidly. Fields useful for scoring, filtering
+/// or collection should be mark as fast fields.
+/// The `FAST` flag can only be used when building `IntOptions` (`u64` and `i64` fields)
+pub const FAST: SchemaFlagList<FastFlag, ()> = SchemaFlagList {
+    head: FastFlag,
+    tail: (),
+};
+
+impl<Head, OldHead, OldTail> BitOr<SchemaFlagList<Head, ()>> for SchemaFlagList<OldHead, OldTail>
+where
+    Head: Clone,
+    OldHead: Clone,
+    OldTail: Clone,
+{
+    type Output = SchemaFlagList<Head, SchemaFlagList<OldHead, OldTail>>;
+
+    fn bitor(self, head: SchemaFlagList<Head, ()>) -> Self::Output {
+        SchemaFlagList {
+            head: head.head,
+            tail: self.clone(),
+        }
+    }
+}
+
+impl<T: Clone + Into<IntOptions>> BitOr<IntOptions> for SchemaFlagList<T, ()> {
+    type Output = IntOptions;
+
+    fn bitor(self, rhs: IntOptions) -> Self::Output {
+        self.head.into() | rhs
+    }
+}
+
+impl<T: Clone + Into<TextOptions>> BitOr<TextOptions> for SchemaFlagList<T, ()> {
+    type Output = TextOptions;
+
+    fn bitor(self, rhs: TextOptions) -> Self::Output {
+        self.head.into() | rhs
+    }
+}
+
+#[derive(Clone)]
+pub struct SchemaFlagList<Head: Clone, Tail: Clone> {
+    pub head: Head,
+    pub tail: Tail,
+}

src/schema/int_options.rs

@@ -1,3 +1,4 @@
+use schema::flags::{FastFlag, IndexedFlag, SchemaFlagList, StoredFlag};
 use std::ops::BitOr;
 
 /// Express whether a field is single-value or multi-valued.
@@ -85,41 +86,62 @@ impl Default for IntOptions {
     }
 }
 
-/// Shortcut for a u64 fast field.
-///
-/// Such a shortcut can be composed as follows `STORED | FAST | INT_INDEXED`
-pub const FAST: IntOptions = IntOptions {
-    indexed: false,
-    stored: false,
-    fast: Some(Cardinality::SingleValue),
-};
+impl From<()> for IntOptions {
+    fn from(_: ()) -> IntOptions {
+        IntOptions::default()
+    }
+}
 
-/// Shortcut for a u64 indexed field.
-///
-/// Such a shortcut can be composed as follows `STORED | FAST | INT_INDEXED`
-pub const INT_INDEXED: IntOptions = IntOptions {
-    indexed: true,
-    stored: false,
-    fast: None,
-};
+impl From<FastFlag> for IntOptions {
+    fn from(_: FastFlag) -> Self {
+        IntOptions {
+            indexed: false,
+            stored: false,
+            fast: Some(Cardinality::SingleValue),
+        }
+    }
+}
 
-/// Shortcut for a u64 stored field.
-///
-/// Such a shortcut can be composed as follows `STORED | FAST | INT_INDEXED`
-pub const INT_STORED: IntOptions = IntOptions {
-    indexed: false,
-    stored: true,
-    fast: None,
-};
+impl From<StoredFlag> for IntOptions {
+    fn from(_: StoredFlag) -> Self {
+        IntOptions {
+            indexed: false,
+            stored: true,
+            fast: None,
+        }
+    }
+}
 
-impl BitOr for IntOptions {
+impl From<IndexedFlag> for IntOptions {
+    fn from(_: IndexedFlag) -> Self {
+        IntOptions {
+            indexed: true,
+            stored: false,
+            fast: None,
+        }
+    }
+}
+
+impl<T: Into<IntOptions>> BitOr<T> for IntOptions {
     type Output = IntOptions;
 
-    fn bitor(self, other: IntOptions) -> IntOptions {
+    fn bitor(self, other: T) -> IntOptions {
         let mut res = IntOptions::default();
+        let other = other.into();
         res.indexed = self.indexed | other.indexed;
         res.stored = self.stored | other.stored;
         res.fast = self.fast.or(other.fast);
         res
     }
 }
+
+impl<Head, Tail> From<SchemaFlagList<Head, Tail>> for IntOptions
+where
+    Head: Clone,
+    Tail: Clone,
+    Self: BitOr<Output = Self> + From<Head> + From<Tail>,
+{
+    fn from(head_tail: SchemaFlagList<Head, Tail>) -> Self {
+        Self::from(head_tail.head) | Self::from(head_tail.tail)
+    }
+}

src/schema/mod.rs

@@ -33,7 +33,7 @@ let title_options = TextOptions::default()
     .set_indexing_options(TextFieldIndexing::default()
         .set_tokenizer("default")
         .set_index_option(IndexRecordOption::WithFreqsAndPositions));
-schema_builder.add_text_field("title_options", title_options);
+schema_builder.add_text_field("title", title_options);
 let schema = schema_builder.build();
 ```
 
@@ -53,23 +53,8 @@ The effect of each possible setting is described more in detail
 On the other hand setting the field as stored or not determines whether the field should be returned
 when [`searcher.doc(doc_address)`](../struct.Searcher.html#method.doc) is called.
 
-### Shortcuts
 
-For convenience, a few special values of `TextOptions`.
-They can be composed using the `|` operator.
-The example can be rewritten :
-
-
-```
-use tantivy::schema::*;
-let mut schema_builder = Schema::builder();
-schema_builder.add_text_field("title_options", TEXT | STORED);
-let schema = schema_builder.build();
-```
-
-
-
-## Setting a u64 field
+## Setting a u64 or a i64 field
 
 ### Example
 
@@ -98,6 +83,23 @@ u64 that are indexed as fast will be stored in a special data structure that wil
 make it possible to access the u64 value given the doc id rapidly. This is useful if the value of
 the field is required during scoring or collection for instance.
 
+
+### Shortcuts
+
+
+For convenience, it is possible to define your field indexing options by combining different flags
+using the  `|` operator.
+
+For instance, a schema containing the two fields defined in the example above could be rewritten :
+
+```
+use tantivy::schema::*;
+let mut schema_builder = Schema::builder();
+schema_builder.add_u64_field("num_stars", INDEXED | STORED);
+schema_builder.add_text_field("title", TEXT | STORED);
+let schema = schema_builder.build();
+```
+
 */
 
 mod document;
@@ -116,13 +118,15 @@ mod named_field_document;
 mod text_options;
 mod value;
 
+mod flags;
+
 pub use self::named_field_document::NamedFieldDocument;
 pub use self::schema::DocParsingError;
 pub use self::schema::{Schema, SchemaBuilder};
 pub use self::value::Value;
 
 pub use self::facet::Facet;
-pub use self::facet::FACET_SEP_BYTE;
+pub(crate) use self::facet::FACET_SEP_BYTE;
 
 pub use self::document::Document;
 pub use self::field::Field;
@@ -135,15 +139,12 @@ pub use self::field_value::FieldValue;
 pub use self::index_record_option::IndexRecordOption;
 pub use self::text_options::TextFieldIndexing;
 pub use self::text_options::TextOptions;
-pub use self::text_options::STORED;
 pub use self::text_options::STRING;
 pub use self::text_options::TEXT;
 
+pub use self::flags::{FAST, INDEXED, STORED};
 pub use self::int_options::Cardinality;
 pub use self::int_options::IntOptions;
-pub use self::int_options::FAST;
-pub use self::int_options::INT_INDEXED;
-pub use self::int_options::INT_STORED;
 
 use regex::Regex;
 

src/schema/schema.rs

@@ -52,9 +52,13 @@ impl SchemaBuilder {
     /// by the second one.
     /// The first field will get a field id
     /// but only the second one will be indexed
-    pub fn add_u64_field(&mut self, field_name_str: &str, field_options: IntOptions) -> Field {
+    pub fn add_u64_field<T: Into<IntOptions>>(
+        &mut self,
+        field_name_str: &str,
+        field_options: T,
+    ) -> Field {
         let field_name = String::from(field_name_str);
-        let field_entry = FieldEntry::new_u64(field_name, field_options);
+        let field_entry = FieldEntry::new_u64(field_name, field_options.into());
         self.add_field(field_entry)
     }
 
@@ -68,9 +72,35 @@ impl SchemaBuilder {
     /// by the second one.
     /// The first field will get a field id
     /// but only the second one will be indexed
-    pub fn add_i64_field(&mut self, field_name_str: &str, field_options: IntOptions) -> Field {
+    pub fn add_i64_field<T: Into<IntOptions>>(
+        &mut self,
+        field_name_str: &str,
+        field_options: T,
+    ) -> Field {
         let field_name = String::from(field_name_str);
-        let field_entry = FieldEntry::new_i64(field_name, field_options);
+        let field_entry = FieldEntry::new_i64(field_name, field_options.into());
+        self.add_field(field_entry)
+    }
+
+    /// Adds a new date field.
+    /// Returns the associated field handle
+    /// Internally, Tantivy simply stores dates as i64 UTC timestamps,
+    /// while the user supplies DateTime values for convenience.
+    ///
+    /// # Caution
+    ///
+    /// Appending two fields with the same name
+    /// will result in the shadowing of the first
+    /// by the second one.
+    /// The first field will get a field id
+    /// but only the second one will be indexed
+    pub fn add_date_field<T: Into<IntOptions>>(
+        &mut self,
+        field_name_str: &str,
+        field_options: T,
+    ) -> Field {
+        let field_name = String::from(field_name_str);
+        let field_entry = FieldEntry::new_date(field_name, field_options.into());
         self.add_field(field_entry)
     }
 
@@ -84,9 +114,13 @@ impl SchemaBuilder {
     /// by the second one.
     /// The first field will get a field id
     /// but only the second one will be indexed
-    pub fn add_text_field(&mut self, field_name_str: &str, field_options: TextOptions) -> Field {
+    pub fn add_text_field<T: Into<TextOptions>>(
+        &mut self,
+        field_name_str: &str,
+        field_options: T,
+    ) -> Field {
         let field_name = String::from(field_name_str);
-        let field_entry = FieldEntry::new_text(field_name, field_options);
+        let field_entry = FieldEntry::new_text(field_name, field_options.into());
         self.add_field(field_entry)
     }
 
@@ -178,15 +212,7 @@ impl Schema {
         SchemaBuilder::default()
     }
 
-    /// Returns the field options associated with a given name.
-    ///
-    /// # Panics
-    /// Panics if the field name does not exist.
-    /// It is meant as an helper for user who created
-    /// and control the content of their schema.
-    ///
-    /// If panicking is not an option for you,
-    /// you may use `get(&self, field_name: &str)`.
+    /// Returns the field option associated with a given name.
     pub fn get_field(&self, field_name: &str) -> Option<Field> {
         self.0.fields_map.get(field_name).cloned()
     }

src/schema/term.rs

@@ -5,6 +5,7 @@ use byteorder::{BigEndian, ByteOrder};
 use common;
 use schema::Facet;
 use std::str;
+use DateTime;
 
 /// Size (in bytes) of the buffer of a int field.
 const INT_TERM_LEN: usize = 4 + 8;
@@ -30,6 +31,18 @@ impl Term {
         Term::from_field_u64(field, val_u64)
     }
 
+    /// Builds a term given a field, and a DateTime value
+    ///
+    /// Assuming the term has a field id of 1, and a timestamp i64 value of 3234,
+    /// the Term will have 8 bytes.
+    ///
+    /// The first four byte are dedicated to storing the field id as a u64.
+    /// The 4 following bytes are encoding the DateTime as i64 timestamp value.
+    pub fn from_field_date(field: Field, val: &DateTime) -> Term {
+        let val_timestamp = val.timestamp();
+        Term::from_field_i64(field, val_timestamp)
+    }
+
     /// Creates a `Term` given a facet.
     pub fn from_facet(field: Field, facet: &Facet) -> Term {
         let bytes = facet.encoded_str().as_bytes();

src/schema/text_options.rs

@@ -1,3 +1,5 @@
+use schema::flags::SchemaFlagList;
+use schema::flags::StoredFlag;
 use schema::IndexRecordOption;
 use std::borrow::Cow;
 use std::ops::BitOr;
@@ -109,19 +111,11 @@ pub const TEXT: TextOptions = TextOptions {
     stored: false,
 };
 
-/// A stored fields of a document can be retrieved given its `DocId`.
-/// Stored field are stored together and LZ4 compressed.
-/// Reading the stored fields of a document is relatively slow.
-/// (100 microsecs)
-pub const STORED: TextOptions = TextOptions {
-    indexing: None,
-    stored: true,
-};
-
-impl BitOr for TextOptions {
+impl<T: Into<TextOptions>> BitOr<T> for TextOptions {
     type Output = TextOptions;
 
-    fn bitor(self, other: TextOptions) -> TextOptions {
+    fn bitor(self, other: T) -> TextOptions {
+        let other = other.into();
         let mut res = TextOptions::default();
         res.indexing = self.indexing.or(other.indexing);
         res.stored = self.stored | other.stored;
@@ -129,6 +123,32 @@ impl BitOr for TextOptions {
     }
 }
 
+impl From<()> for TextOptions {
+    fn from(_: ()) -> TextOptions {
+        TextOptions::default()
+    }
+}
+
+impl From<StoredFlag> for TextOptions {
+    fn from(_: StoredFlag) -> TextOptions {
+        TextOptions {
+            indexing: None,
+            stored: true,
+        }
+    }
+}
+
+impl<Head, Tail> From<SchemaFlagList<Head, Tail>> for TextOptions
+where
+    Head: Clone,
+    Tail: Clone,
+    Self: BitOr<Output = Self> + From<Head> + From<Tail>,
+{
+    fn from(head_tail: SchemaFlagList<Head, Tail>) -> Self {
+        Self::from(head_tail.head) | Self::from(head_tail.tail)
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use schema::*;