Add grouped operations (#493)

* [WIP] added UserOperation enum, added IndexWriter.run, and added MultiStamp

* removed MultiStamp in favor of std::ops::Range

* changed IndexWriter::run to return u64, Stamper::stamps to return a Range, added tests, and added docs

* changed delete_cursor skipping to use first operation's opstamp vice last. change index_writer test to use 1 thread

* added test for order batch of operations

* added a test comment

CHANGELOG.md

@@ -1,5 +1,22 @@
+Tantivy 0.9.0
+=====================
+*0.9.0 index format is not compatible with the 
+previous index format.*
+- Removed most unsafe (@fulmicoton)
+- Indexer memory footprint improved. (VInt comp, inlining the first block. (@fulmicoton)
+- Stemming in other language possible (@pentlander)
+- Segments with no docs are deleted earlier (@barrotsteindev)
+
 Tantivy 0.8.1
 =====================
+Hotfix of #476.
+
+Merge was reflecting deletes before commit was passed. 
+Thanks @barrotsteindev  for reporting the bug.
+
+
+Tantivy 0.8.0
+=====================
 *No change in the index format*
 - API Breaking change in the collector API. (@jwolfe, @fulmicoton)
 - Multithreaded search (@jwolfe, @fulmicoton) 

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "tantivy"
-version = "0.8.0-dev"
+version = "0.9.0-dev"
 authors = ["Paul Masurel <paul.masurel@gmail.com>"]
 license = "MIT"
 categories = ["database-implementations", "data-structures"]
@@ -29,7 +29,8 @@ serde = "1.0"
 serde_derive = "1.0"
 serde_json = "1.0"
 num_cpus = "1.2"
-itertools = "0.7"
+fs2={version="0.4", optional=true}
+itertools = "0.8"
 levenshtein_automata = {version="0.1", features=["fst_automaton"]}
 bit-set = "0.5"
 uuid = { version = "0.7", features = ["v4", "serde"] }
@@ -38,18 +39,18 @@ futures = "0.1"
 futures-cpupool = "0.1"
 owning_ref = "0.4"
 stable_deref_trait = "1.0.0"
-rust-stemmers = "1"
-downcast = { version="0.9" }
+rust-stemmers = "1.1"
+downcast-rs = { version="1.0" }
 matches = "0.1"
-bitpacking = "0.5"
-census = "0.1"
+bitpacking = "0.6"
+census = "0.2"
 fnv = "1.0.6"
 owned-read = "0.4"
 failure = "0.1"
 htmlescape = "0.3.1"
 fail = "0.2"
 scoped-pool = "1.0"
-aho-corasick = "0.6"
+murmurhash32 = "0.2"
 
 [target.'cfg(windows)'.dependencies]
 winapi = "0.2"
@@ -70,7 +71,7 @@ overflow-checks = true
 [features]
 # by default no-fail is disabled. We manually enable it when running test.
 default = ["mmap", "no_fail"]
-mmap = ["fst/mmap", "atomicwrites"]
+mmap = ["fst/mmap", "atomicwrites", "fs2"]
 lz4-compression = ["lz4"]
 no_fail = ["fail/no_fail"]
 unstable = [] # useful for benches.

README.md

@@ -21,7 +21,7 @@
 
 **Tantivy** is a **full text search engine library** written in rust.
 
-It is closer to [Apache Lucene](https://lucene.apache.org/) than to [Elastic Search](https://www.elastic.co/products/elasticsearch) and [Apache Solr](https://lucene.apache.org/solr/) in the sense it is not
+It is closer to [Apache Lucene](https://lucene.apache.org/) than to [Elasticsearch](https://www.elastic.co/products/elasticsearch) and [Apache Solr](https://lucene.apache.org/solr/) in the sense it is not
 an off-the-shelf search engine server, but rather a crate that can be used
 to build such a search engine.
 
@@ -76,7 +76,7 @@ It will walk you through getting a wikipedia search engine up and running in a f
 Tantivy compiles on stable rust but requires `Rust >= 1.27`.
 To check out and run tests, you can simply run :
 
-    git clone git@github.com:tantivy-search/tantivy.git
+    git clone https://github.com/tantivy-search/tantivy.git
     cd tantivy
     cargo build
 

examples/basic_search.rs

@@ -106,37 +106,37 @@ fn main() -> tantivy::Result<()> {
     // For convenience, tantivy also comes with a macro to
     // reduce the boilerplate above.
     index_writer.add_document(doc!(
-        title => "Of Mice and Men",
-        body => "A few miles south of Soledad, the Salinas River drops in close to the hillside \
-                bank and runs deep and green. The water is warm too, for it has slipped twinkling \
-                over the yellow sands in the sunlight before reaching the narrow pool. On one \
-                side of the river the golden foothill slopes curve up to the strong and rocky \
-                Gabilan Mountains, but on the valley side the water is lined with trees—willows \
-                fresh and green with every spring, carrying in their lower leaf junctures the \
-                debris of the winter’s flooding; and sycamores with mottled, white, recumbent \
-                limbs and branches that arch over the pool"
+    title => "Of Mice and Men",
+    body => "A few miles south of Soledad, the Salinas River drops in close to the hillside \
+            bank and runs deep and green. The water is warm too, for it has slipped twinkling \
+            over the yellow sands in the sunlight before reaching the narrow pool. On one \
+            side of the river the golden foothill slopes curve up to the strong and rocky \
+            Gabilan Mountains, but on the valley side the water is lined with trees—willows \
+            fresh and green with every spring, carrying in their lower leaf junctures the \
+            debris of the winter’s flooding; and sycamores with mottled, white, recumbent \
+            limbs and branches that arch over the pool"
     ));
 
     index_writer.add_document(doc!(
-        title => "Of Mice and Men",
-        body => "A few miles south of Soledad, the Salinas River drops in close to the hillside \
-                bank and runs deep and green. The water is warm too, for it has slipped twinkling \
-                over the yellow sands in the sunlight before reaching the narrow pool. On one \
-                side of the river the golden foothill slopes curve up to the strong and rocky \
-                Gabilan Mountains, but on the valley side the water is lined with trees—willows \
-                fresh and green with every spring, carrying in their lower leaf junctures the \
-                debris of the winter’s flooding; and sycamores with mottled, white, recumbent \
-                limbs and branches that arch over the pool"
+    title => "Of Mice and Men",
+    body => "A few miles south of Soledad, the Salinas River drops in close to the hillside \
+            bank and runs deep and green. The water is warm too, for it has slipped twinkling \
+            over the yellow sands in the sunlight before reaching the narrow pool. On one \
+            side of the river the golden foothill slopes curve up to the strong and rocky \
+            Gabilan Mountains, but on the valley side the water is lined with trees—willows \
+            fresh and green with every spring, carrying in their lower leaf junctures the \
+            debris of the winter’s flooding; and sycamores with mottled, white, recumbent \
+            limbs and branches that arch over the pool"
     ));
 
     // Multivalued field just need to be repeated.
     index_writer.add_document(doc!(
-       title => "Frankenstein",
-       title => "The Modern Prometheus",
-       body => "You will rejoice to hear that no disaster has accompanied the commencement of an \
-                enterprise which you have regarded with such evil forebodings.  I arrived here \
-                yesterday, and my first task is to assure my dear sister of my welfare and \
-                increasing confidence in the success of my undertaking."
+    title => "Frankenstein",
+    title => "The Modern Prometheus",
+    body => "You will rejoice to hear that no disaster has accompanied the commencement of an \
+             enterprise which you have regarded with such evil forebodings.  I arrived here \
+             yesterday, and my first task is to assure my dear sister of my welfare and \
+             increasing confidence in the success of my undertaking."
     ));
 
     // This is an example, so we will only index 3 documents

examples/custom_tokenizer.rs

@@ -68,12 +68,12 @@ fn main() -> tantivy::Result<()> {
     // heap for the indexer can increase its throughput.
     let mut index_writer = index.writer(50_000_000)?;
     index_writer.add_document(doc!(
-        title => "The Old Man and the Sea",
-        body => "He was an old man who fished alone in a skiff in the Gulf Stream and \
-         he had gone eighty-four days now without taking a fish."
+    title => "The Old Man and the Sea",
+    body => "He was an old man who fished alone in a skiff in the Gulf Stream and \
+     he had gone eighty-four days now without taking a fish."
     ));
     index_writer.add_document(doc!(
-       title => "Of Mice and Men",
+    title => "Of Mice and Men",
        body => r#"A few miles south of Soledad, the Salinas River drops in close to the hillside
                 bank and runs deep and green. The water is warm too, for it has slipped twinkling
                 over the yellow sands in the sunlight before reaching the narrow pool. On one
@@ -84,7 +84,7 @@ fn main() -> tantivy::Result<()> {
                 limbs and branches that arch over the pool"#
     ));
     index_writer.add_document(doc!(
-        title => "Frankenstein",
+    title => "Frankenstein",
         body => r#"You will rejoice to hear that no disaster has accompanied the commencement of an
                 enterprise which you have regarded with such evil forebodings.  I arrived here
                 yesterday, and my first task is to assure my dear sister of my welfare and

examples/integer_range_search.rs

@@ -0,0 +1,41 @@
+// # Searching a range on an indexed int field.
+//
+// Below is an example of creating an indexed integer field in your schema
+// You can use RangeQuery to get a Count of all occurrences in a given range.
+
+#[macro_use]
+extern crate tantivy;
+use tantivy::collector::Count;
+use tantivy::query::RangeQuery;
+use tantivy::schema::{Schema, INT_INDEXED};
+use tantivy::Index;
+use tantivy::Result;
+
+fn run() -> Result<()> {
+    // For the sake of simplicity, this schema will only have 1 field
+    let mut schema_builder = Schema::builder();
+    // INT_INDEXED is shorthand for such fields
+    let year_field = schema_builder.add_u64_field("year", INT_INDEXED);
+    let schema = schema_builder.build();
+    let index = Index::create_in_ram(schema);
+    {
+        let mut index_writer = index.writer_with_num_threads(1, 6_000_000)?;
+        for year in 1950u64..2019u64 {
+            index_writer.add_document(doc!(year_field => year));
+        }
+        index_writer.commit()?;
+        // The index will be a range of years
+    }
+    index.load_searchers()?;
+    let searcher = index.searcher();
+    // The end is excluded i.e. here we are searching up to 1969
+    let docs_in_the_sixties = RangeQuery::new_u64(year_field, 1960..1970);
+    // Uses a Count collector to sum the total number of docs in the range
+    let num_60s_books = searcher.search(&docs_in_the_sixties, &Count)?;
+    assert_eq!(num_60s_books, 10);
+    Ok(())
+}
+
+fn main() {
+    run().unwrap()
+}

examples/snippet.rs

@@ -35,15 +35,15 @@ fn main() -> tantivy::Result<()> {
 
     // we'll only need one doc for this example.
     index_writer.add_document(doc!(
-        title => "Of Mice and Men",
-        body => "A few miles south of Soledad, the Salinas River drops in close to the hillside \
-                bank and runs deep and green. The water is warm too, for it has slipped twinkling \
-                over the yellow sands in the sunlight before reaching the narrow pool. On one \
-                side of the river the golden foothill slopes curve up to the strong and rocky \
-                Gabilan Mountains, but on the valley side the water is lined with trees—willows \
-                fresh and green with every spring, carrying in their lower leaf junctures the \
-                debris of the winter’s flooding; and sycamores with mottled, white, recumbent \
-                limbs and branches that arch over the pool"
+    title => "Of Mice and Men",
+    body => "A few miles south of Soledad, the Salinas River drops in close to the hillside \
+            bank and runs deep and green. The water is warm too, for it has slipped twinkling \
+            over the yellow sands in the sunlight before reaching the narrow pool. On one \
+            side of the river the golden foothill slopes curve up to the strong and rocky \
+            Gabilan Mountains, but on the valley side the water is lined with trees—willows \
+            fresh and green with every spring, carrying in their lower leaf junctures the \
+            debris of the winter’s flooding; and sycamores with mottled, white, recumbent \
+            limbs and branches that arch over the pool"
     ));
     // ...
     index_writer.commit()?;
@@ -56,7 +56,7 @@ fn main() -> tantivy::Result<()> {
 
     let top_docs = searcher.search(&query, &TopDocs::with_limit(10))?;
 
-    let snippet_generator = SnippetGenerator::new(&searcher, &*query, body)?;
+    let snippet_generator = SnippetGenerator::create(&searcher, &*query, body)?;
 
     for (score, doc_address) in top_docs {
         let doc = searcher.doc(doc_address)?;

examples/stop_words.rs

@@ -72,26 +72,26 @@ fn main() -> tantivy::Result<()> {
     title => "The Old Man and the Sea",
     body => "He was an old man who fished alone in a skiff in the Gulf Stream and \
      he had gone eighty-four days now without taking a fish."
-  ));
+    ));
 
     index_writer.add_document(doc!(
-      title => "Of Mice and Men",
-      body => "A few miles south of Soledad, the Salinas River drops in close to the hillside \
-              bank and runs deep and green. The water is warm too, for it has slipped twinkling \
-              over the yellow sands in the sunlight before reaching the narrow pool. On one \
-              side of the river the golden foothill slopes curve up to the strong and rocky \
-              Gabilan Mountains, but on the valley side the water is lined with trees—willows \
-              fresh and green with every spring, carrying in their lower leaf junctures the \
-              debris of the winter’s flooding; and sycamores with mottled, white, recumbent \
-              limbs and branches that arch over the pool"
-  ));
+    title => "Of Mice and Men",
+    body => "A few miles south of Soledad, the Salinas River drops in close to the hillside \
+            bank and runs deep and green. The water is warm too, for it has slipped twinkling \
+            over the yellow sands in the sunlight before reaching the narrow pool. On one \
+            side of the river the golden foothill slopes curve up to the strong and rocky \
+            Gabilan Mountains, but on the valley side the water is lined with trees—willows \
+            fresh and green with every spring, carrying in their lower leaf junctures the \
+            debris of the winter’s flooding; and sycamores with mottled, white, recumbent \
+            limbs and branches that arch over the pool"
+    ));
 
     index_writer.add_document(doc!(
-       title => "Frankenstein",
-       body => "You will rejoice to hear that no disaster has accompanied the commencement of an \
-                enterprise which you have regarded with such evil forebodings.  I arrived here \
-                yesterday, and my first task is to assure my dear sister of my welfare and \
-                increasing confidence in the success of my undertaking."
+    title => "Frankenstein",
+    body => "You will rejoice to hear that no disaster has accompanied the commencement of an \
+             enterprise which you have regarded with such evil forebodings.  I arrived here \
+             yesterday, and my first task is to assure my dear sister of my welfare and \
+             increasing confidence in the success of my undertaking."
     ));
 
     index_writer.commit()?;

src/collector/facet_collector.rs

@@ -197,7 +197,7 @@ fn skip<'a, I: Iterator<Item = &'a Facet>>(
 ) -> SkipResult {
     loop {
         match collapse_it.peek() {
-            Some(facet_bytes) => match facet_bytes.encoded_bytes().cmp(target) {
+            Some(facet_bytes) => match facet_bytes.encoded_str().as_bytes().cmp(target) {
                 Ordering::Less => {}
                 Ordering::Greater => {
                     return SkipResult::OverStep;
@@ -369,7 +369,8 @@ impl SegmentCollector for FacetSegmentCollector {
             let mut facet = vec![];
             let facet_ord = self.collapse_facet_ords[collapsed_facet_ord];
             facet_dict.ord_to_term(facet_ord as u64, &mut facet);
-            facet_counts.insert(unsafe { Facet::from_encoded(facet) }, count);
+            // TODO
+            facet_counts.insert(Facet::from_encoded(facet).unwrap(), count);
         }
         FacetCounts { facet_counts }
     }
@@ -403,9 +404,9 @@ impl FacetCounts {
         let right_bound = if facet.is_root() {
             Bound::Unbounded
         } else {
-            let mut facet_after_bytes: Vec<u8> = facet.encoded_bytes().to_owned();
-            facet_after_bytes.push(1u8);
-            let facet_after = unsafe { Facet::from_encoded(facet_after_bytes) }; // ok logic
+            let mut facet_after_bytes: String = facet.encoded_str().to_owned();
+            facet_after_bytes.push('\u{1}');
+            let facet_after = Facet::from_encoded_string(facet_after_bytes);
             Bound::Excluded(facet_after)
         };
         let underlying: btree_map::Range<_, _> = self.facet_counts.range((left_bound, right_bound));
@@ -474,7 +475,8 @@ mod tests {
                 n /= 4;
                 let leaf = n % 5;
                 Facet::from(&format!("/top{}/mid{}/leaf{}", top, mid, leaf))
-            }).collect();
+            })
+            .collect();
         for i in 0..num_facets * 10 {
             let mut doc = Document::new();
             doc.add_facet(facet_field, facets[i % num_facets].clone());
@@ -500,18 +502,16 @@ mod tests {
                     ("/top1/mid2", 50),
                     ("/top1/mid3", 50),
                 ]
-                    .iter()
-                    .map(|&(facet_str, count)| (String::from(facet_str), count))
-                    .collect::<Vec<_>>()
+                .iter()
+                .map(|&(facet_str, count)| (String::from(facet_str), count))
+                .collect::<Vec<_>>()
             );
         }
     }
 
     #[test]
-    #[should_panic(
-        expected = "Tried to add a facet which is a descendant of \
-                    an already added facet."
-    )]
+    #[should_panic(expected = "Tried to add a facet which is a descendant of \
+                               an already added facet.")]
     fn test_misused_facet_collector() {
         let mut facet_collector = FacetCollector::for_field(Field(0));
         facet_collector.add_facet(Facet::from("/country"));
@@ -563,13 +563,15 @@ mod tests {
                 let facet = Facet::from(&format!("/facet/{}", c));
                 let doc = doc!(facet_field => facet);
                 iter::repeat(doc).take(count)
-            }).map(|mut doc| {
+            })
+            .map(|mut doc| {
                 doc.add_facet(
                     facet_field,
                     &format!("/facet/{}", thread_rng().sample(&uniform)),
                 );
                 doc
-            }).collect();
+            })
+            .collect();
         docs[..].shuffle(&mut thread_rng());
 
         let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();

src/collector/int_facet_collector.rs

@@ -88,7 +88,7 @@ mod tests {
         let index = Index::create_in_ram(schema.clone());
 
         {
-            let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
+            let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
             {
                 for i in 0u64..10u64 {
                     index_writer.add_document(doc!(

src/collector/mod.rs

@@ -85,7 +85,7 @@ See the `custom_collector` example.
 
 */
 
-use downcast;
+use downcast_rs;
 use DocId;
 use Result;
 use Score;
@@ -111,9 +111,9 @@ pub use self::facet_collector::FacetCollector;
 
 /// `Fruit` is the type for the result of our collection.
 /// e.g. `usize` for the `Count` collector.
-pub trait Fruit: Send + downcast::Any {}
+pub trait Fruit: Send + downcast_rs::Downcast {}
 
-impl<T> Fruit for T where T: Send + downcast::Any {}
+impl<T> Fruit for T where T: Send + downcast_rs::Downcast {}
 
 /// Collectors are in charge of collecting and retaining relevant
 /// information from the document found and scored by the query.
@@ -358,10 +358,7 @@ where
     }
 }
 
-#[allow(missing_docs)]
-mod downcast_impl {
-    downcast!(super::Fruit);
-}
+impl_downcast!(Fruit);
 
 #[cfg(test)]
 pub mod tests;

src/collector/multi_collector.rs

@@ -1,7 +1,6 @@
 use super::Collector;
 use super::SegmentCollector;
 use collector::Fruit;
-use downcast::Downcast;
 use std::marker::PhantomData;
 use DocId;
 use Result;
@@ -37,13 +36,14 @@ impl<TCollector: Collector> Collector for CollectorWrapper<TCollector> {
         let typed_fruit: Vec<TCollector::Fruit> = children
             .into_iter()
             .map(|untyped_fruit| {
-                Downcast::<TCollector::Fruit>::downcast(untyped_fruit)
+                untyped_fruit
+                    .downcast::<TCollector::Fruit>()
                     .map(|boxed_but_typed| *boxed_but_typed)
-                    .map_err(|e| {
-                        let err_msg = format!("Failed to cast child collector fruit. {:?}", e);
-                        TantivyError::InvalidArgument(err_msg)
+                    .map_err(|_| {
+                        TantivyError::InvalidArgument("Failed to cast child fruit.".to_string())
                     })
-            }).collect::<Result<_>>()?;
+            })
+            .collect::<Result<_>>()?;
         let merged_fruit = self.0.merge_fruits(typed_fruit)?;
         Ok(Box::new(merged_fruit))
     }
@@ -88,14 +88,20 @@ pub struct FruitHandle<TFruit: Fruit> {
 impl<TFruit: Fruit> FruitHandle<TFruit> {
     pub fn extract(self, fruits: &mut MultiFruit) -> TFruit {
         let boxed_fruit = fruits.sub_fruits[self.pos].take().expect("");
-        *Downcast::<TFruit>::downcast(boxed_fruit).expect("Failed")
+        *boxed_fruit
+            .downcast::<TFruit>()
+            .map_err(|_| ())
+            .expect("Failed to downcast collector fruit.")
     }
 }
 
 /// Multicollector makes it possible to collect on more than one collector.
 /// It should only be used for use cases where the Collector types is unknown
 /// at compile time.
-/// If the type of the collectors is known, you should prefer to use `ChainedCollector`.
+///
+/// If the type of the collectors is known, you can just group yours collectors
+/// in a tuple. See the
+/// [Combining several collectors section of the collector documentation](./index.html#combining-several-collectors).
 ///
 /// ```rust
 /// #[macro_use]
@@ -147,6 +153,8 @@ impl<TFruit: Fruit> FruitHandle<TFruit> {
 ///     Ok(())
 /// }
 /// ```
+#[allow(clippy::type_complexity)]
+#[derive(Default)]
 pub struct MultiCollector<'a> {
     collector_wrappers:
         Vec<Box<Collector<Child = Box<BoxableSegmentCollector>, Fruit = Box<Fruit>> + 'a>>,
@@ -154,10 +162,8 @@ pub struct MultiCollector<'a> {
 
 impl<'a> MultiCollector<'a> {
     /// Create a new `MultiCollector`
-    pub fn new() -> MultiCollector<'a> {
-        MultiCollector {
-            collector_wrappers: Vec::new(),
-        }
+    pub fn new() -> Self {
+        Default::default()
     }
 
     /// Add a new collector to our `MultiCollector`.
@@ -213,7 +219,8 @@ impl<'a> Collector for MultiCollector<'a> {
             .zip(segment_fruits_list)
             .map(|(child_collector, segment_fruits)| {
                 Ok(Some(child_collector.merge_fruits(segment_fruits)?))
-            }).collect::<Result<_>>()?;
+            })
+            .collect::<Result<_>>()?;
         Ok(MultiFruit { sub_fruits })
     }
 }

src/collector/top_collector.rs

@@ -84,11 +84,9 @@ where
             for (feature, doc) in child_fruit {
                 if top_collector.len() < self.limit {
                     top_collector.push(ComparableDoc { feature, doc });
-                } else {
-                    if let Some(mut head) = top_collector.peek_mut() {
-                        if head.feature < feature {
-                            *head = ComparableDoc { feature, doc };
-                        }
+                } else if let Some(mut head) = top_collector.peek_mut() {
+                    if head.feature < feature {
+                        *head = ComparableDoc { feature, doc };
                     }
                 }
             }
@@ -142,7 +140,8 @@ impl<T: PartialOrd + Clone> TopSegmentCollector<T> {
                     comparable_doc.feature,
                     DocAddress(segment_id, comparable_doc.doc),
                 )
-            }).collect()
+            })
+            .collect()
     }
 
     /// Return true iff at least K documents have gone through

src/collector/top_score_collector.rs

@@ -142,7 +142,7 @@ mod tests {
         let index = Index::create_in_ram(schema);
         {
             // writing the segment
-            let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
+            let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
             index_writer.add_document(doc!(text_field=>"Hello happy tax payer."));
             index_writer.add_document(doc!(text_field=>"Droopy says hello happy tax payer"));
             index_writer.add_document(doc!(text_field=>"I like Droopy"));

src/common/bitpacker.rs

@@ -1,9 +1,6 @@
-use common::serialize::BinarySerializable;
+use byteorder::{ByteOrder, LittleEndian, WriteBytesExt};
 use std::io;
-use std::io::Write;
-use std::mem;
 use std::ops::Deref;
-use std::ptr;
 
 pub(crate) struct BitPacker {
     mini_buffer: u64,
@@ -18,7 +15,7 @@ impl BitPacker {
         }
     }
 
-    pub fn write<TWrite: Write>(
+    pub fn write<TWrite: io::Write>(
         &mut self,
         val: u64,
         num_bits: u8,
@@ -28,14 +25,14 @@ impl BitPacker {
         let num_bits = num_bits as usize;
         if self.mini_buffer_written + num_bits > 64 {
             self.mini_buffer |= val_u64.wrapping_shl(self.mini_buffer_written as u32);
-            self.mini_buffer.serialize(output)?;
+            output.write_u64::<LittleEndian>(self.mini_buffer)?;
             self.mini_buffer = val_u64.wrapping_shr((64 - self.mini_buffer_written) as u32);
             self.mini_buffer_written = self.mini_buffer_written + num_bits - 64;
         } else {
             self.mini_buffer |= val_u64 << self.mini_buffer_written;
             self.mini_buffer_written += num_bits;
             if self.mini_buffer_written == 64 {
-                self.mini_buffer.serialize(output)?;
+                output.write_u64::<LittleEndian>(self.mini_buffer)?;
                 self.mini_buffer_written = 0;
                 self.mini_buffer = 0u64;
             }
@@ -43,17 +40,18 @@ impl BitPacker {
         Ok(())
     }
 
-    pub fn flush<TWrite: Write>(&mut self, output: &mut TWrite) -> io::Result<()> {
+    pub fn flush<TWrite: io::Write>(&mut self, output: &mut TWrite) -> io::Result<()> {
         if self.mini_buffer_written > 0 {
             let num_bytes = (self.mini_buffer_written + 7) / 8;
-            let arr: [u8; 8] = unsafe { mem::transmute::<u64, [u8; 8]>(self.mini_buffer.to_le()) };
+            let mut arr: [u8; 8] = [0u8; 8];
+            LittleEndian::write_u64(&mut arr, self.mini_buffer);
             output.write_all(&arr[..num_bytes])?;
             self.mini_buffer_written = 0;
         }
         Ok(())
     }
 
-    pub fn close<TWrite: Write>(&mut self, output: &mut TWrite) -> io::Result<()> {
+    pub fn close<TWrite: io::Write>(&mut self, output: &mut TWrite) -> io::Result<()> {
         self.flush(output)?;
         // Padding the write file to simplify reads.
         output.write_all(&[0u8; 7])?;
@@ -66,7 +64,7 @@ pub struct BitUnpacker<Data>
 where
     Data: Deref<Target = [u8]>,
 {
-    num_bits: usize,
+    num_bits: u64,
     mask: u64,
     data: Data,
 }
@@ -82,13 +80,13 @@ where
             (1u64 << num_bits) - 1u64
         };
         BitUnpacker {
-            num_bits: num_bits as usize,
+            num_bits: num_bits as u64,
             mask,
             data,
         }
     }
 
-    pub fn get(&self, idx: usize) -> u64 {
+    pub fn get(&self, idx: u64) -> u64 {
         if self.num_bits == 0 {
             return 0u64;
         }
@@ -99,42 +97,13 @@ where
         let addr = addr_in_bits >> 3;
         let bit_shift = addr_in_bits & 7;
         debug_assert!(
-            addr + 8 <= data.len(),
+            addr + 8 <= data.len() as u64,
             "The fast field field should have been padded with 7 bytes."
         );
-        #[cfg_attr(feature = "cargo-clippy", allow(clippy::cast_ptr_alignment))]
-        let val_unshifted_unmasked: u64 =
-            u64::from_le(unsafe { ptr::read_unaligned(data[addr..].as_ptr() as *const u64) });
+        let val_unshifted_unmasked: u64 = LittleEndian::read_u64(&data[(addr as usize)..]);
         let val_shifted = (val_unshifted_unmasked >> bit_shift) as u64;
         val_shifted & mask
     }
-
-    /// Reads a range of values from the fast field.
-    ///
-    /// The range of values read is from
-    /// `[start..start + output.len()[`
-    pub fn get_range(&self, start: u32, output: &mut [u64]) {
-        if self.num_bits == 0 {
-            for val in output.iter_mut() {
-                *val = 0u64;
-            }
-        } else {
-            let data: &[u8] = &*self.data;
-            let num_bits = self.num_bits;
-            let mask = self.mask;
-            let mut addr_in_bits = (start as usize) * num_bits;
-            for output_val in output.iter_mut() {
-                let addr = addr_in_bits >> 3;
-                let bit_shift = addr_in_bits & 7;
-                #[cfg_attr(feature = "cargo-clippy", allow(clippy::cast_ptr_alignment))]
-                let val_unshifted_unmasked: u64 =
-                    unsafe { ptr::read_unaligned(data[addr..].as_ptr() as *const u64) };
-                let val_shifted = (val_unshifted_unmasked >> bit_shift) as u64;
-                *output_val = val_shifted & mask;
-                addr_in_bits += num_bits;
-            }
-        }
-    }
 }
 
 #[cfg(test)]
@@ -160,7 +129,7 @@ mod test {
     fn test_bitpacker_util(len: usize, num_bits: u8) {
         let (bitunpacker, vals) = create_fastfield_bitpacker(len, num_bits);
         for (i, val) in vals.iter().enumerate() {
-            assert_eq!(bitunpacker.get(i), *val);
+            assert_eq!(bitunpacker.get(i as u64), *val);
         }
     }
 
@@ -172,17 +141,4 @@ mod test {
         test_bitpacker_util(6, 14);
         test_bitpacker_util(1000, 14);
     }
-
-    #[test]
-    fn test_bitpacker_range() {
-        let (bitunpacker, vals) = create_fastfield_bitpacker(100_000, 12);
-        let buffer_len = 100;
-        let mut buffer = vec![0u64; buffer_len];
-        for start in vec![0, 10, 20, 100, 1_000] {
-            bitunpacker.get_range(start as u32, &mut buffer[..]);
-            for i in 0..buffer_len {
-                assert_eq!(buffer[i], vals[start + i]);
-            }
-        }
-    }
 }

src/common/composite_file.rs

@@ -39,7 +39,7 @@ impl BinarySerializable for FileAddr {
 /// A `CompositeWrite` is used to write a `CompositeFile`.
 pub struct CompositeWrite<W = WritePtr> {
     write: CountingWriter<W>,
-    offsets: HashMap<FileAddr, usize>,
+    offsets: HashMap<FileAddr, u64>,
 }
 
 impl<W: Write> CompositeWrite<W> {

src/common/counting_writer.rs

@@ -3,7 +3,7 @@ use std::io::Write;
 
 pub struct CountingWriter<W> {
     underlying: W,
-    written_bytes: usize,
+    written_bytes: u64,
 }
 
 impl<W: Write> CountingWriter<W> {
@@ -14,11 +14,11 @@ impl<W: Write> CountingWriter<W> {
         }
     }
 
-    pub fn written_bytes(&self) -> usize {
+    pub fn written_bytes(&self) -> u64 {
         self.written_bytes
     }
 
-    pub fn finish(mut self) -> io::Result<(W, usize)> {
+    pub fn finish(mut self) -> io::Result<(W, u64)> {
         self.flush()?;
         Ok((self.underlying, self.written_bytes))
     }
@@ -27,10 +27,16 @@ impl<W: Write> CountingWriter<W> {
 impl<W: Write> Write for CountingWriter<W> {
     fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
         let written_size = self.underlying.write(buf)?;
-        self.written_bytes += written_size;
+        self.written_bytes += written_size as u64;
         Ok(written_size)
     }
 
+    fn write_all(&mut self, buf: &[u8]) -> io::Result<()> {
+        self.underlying.write_all(buf)?;
+        self.written_bytes += buf.len() as u64;
+        Ok(())
+    }
+
     fn flush(&mut self) -> io::Result<()> {
         self.underlying.flush()
     }
@@ -48,8 +54,8 @@ mod test {
         let mut counting_writer = CountingWriter::wrap(buffer);
         let bytes = (0u8..10u8).collect::<Vec<u8>>();
         counting_writer.write_all(&bytes).unwrap();
-        let (w, len): (Vec<u8>, usize) = counting_writer.finish().unwrap();
-        assert_eq!(len, 10);
+        let (w, len): (Vec<u8>, u64) = counting_writer.finish().unwrap();
+        assert_eq!(len, 10u64);
         assert_eq!(w.len(), 10);
     }
 }

src/common/mod.rs

@@ -10,7 +10,7 @@ pub(crate) use self::bitset::TinySet;
 pub(crate) use self::composite_file::{CompositeFile, CompositeWrite};
 pub use self::counting_writer::CountingWriter;
 pub use self::serialize::{BinarySerializable, FixedSize};
-pub use self::vint::VInt;
+pub use self::vint::{read_u32_vint, serialize_vint_u32, write_u32_vint, VInt};
 pub use byteorder::LittleEndian as Endianness;
 
 use std::io;

src/common/vint.rs

@@ -1,4 +1,5 @@
 use super::BinarySerializable;
+use byteorder::{ByteOrder, LittleEndian};
 use std::io;
 use std::io::Read;
 use std::io::Write;
@@ -9,6 +10,100 @@ pub struct VInt(pub u64);
 
 const STOP_BIT: u8 = 128;
 
+pub fn serialize_vint_u32(val: u32) -> (u64, usize) {
+    const START_2: u64 = 1 << 7;
+    const START_3: u64 = 1 << 14;
+    const START_4: u64 = 1 << 21;
+    const START_5: u64 = 1 << 28;
+
+    const STOP_1: u64 = START_2 - 1;
+    const STOP_2: u64 = START_3 - 1;
+    const STOP_3: u64 = START_4 - 1;
+    const STOP_4: u64 = START_5 - 1;
+
+    const MASK_1: u64 = 127;
+    const MASK_2: u64 = MASK_1 << 7;
+    const MASK_3: u64 = MASK_2 << 7;
+    const MASK_4: u64 = MASK_3 << 7;
+    const MASK_5: u64 = MASK_4 << 7;
+
+    let val = u64::from(val);
+    const STOP_BIT: u64 = 128u64;
+    match val {
+        0...STOP_1 => (val | STOP_BIT, 1),
+        START_2...STOP_2 => (
+            (val & MASK_1) | ((val & MASK_2) << 1) | (STOP_BIT << (8)),
+            2,
+        ),
+        START_3...STOP_3 => (
+            (val & MASK_1) | ((val & MASK_2) << 1) | ((val & MASK_3) << 2) | (STOP_BIT << (8 * 2)),
+            3,
+        ),
+        START_4...STOP_4 => (
+            (val & MASK_1)
+                | ((val & MASK_2) << 1)
+                | ((val & MASK_3) << 2)
+                | ((val & MASK_4) << 3)
+                | (STOP_BIT << (8 * 3)),
+            4,
+        ),
+        _ => (
+            (val & MASK_1)
+                | ((val & MASK_2) << 1)
+                | ((val & MASK_3) << 2)
+                | ((val & MASK_4) << 3)
+                | ((val & MASK_5) << 4)
+                | (STOP_BIT << (8 * 4)),
+            5,
+        ),
+    }
+}
+
+/// Returns the number of bytes covered by a
+/// serialized vint `u32`.
+///
+/// Expects a buffer data that starts
+/// by the serialized `vint`, scans at most 5 bytes ahead until
+/// it finds the vint final byte.
+///
+/// # May Panic
+/// If the payload does not start by a valid `vint`
+fn vint_len(data: &[u8]) -> usize {
+    for (i, &val) in data.iter().enumerate().take(5) {
+        if val >= STOP_BIT {
+            return i + 1;
+        }
+    }
+    panic!("Corrupted data. Invalid VInt 32");
+}
+
+/// Reads a vint `u32` from a buffer, and
+/// consumes its payload data.
+///
+/// # Panics
+///
+/// If the buffer does not start by a valid
+/// vint payload
+pub fn read_u32_vint(data: &mut &[u8]) -> u32 {
+    let vlen = vint_len(*data);
+    let mut result = 0u32;
+    let mut shift = 0u64;
+    for &b in &data[..vlen] {
+        result |= u32::from(b & 127u8) << shift;
+        shift += 7;
+    }
+    *data = &data[vlen..];
+    result
+}
+
+/// Write a `u32` as a vint payload.
+pub fn write_u32_vint<W: io::Write>(val: u32, writer: &mut W) -> io::Result<()> {
+    let (val, num_bytes) = serialize_vint_u32(val);
+    let mut buffer = [0u8; 8];
+    LittleEndian::write_u64(&mut buffer, val);
+    writer.write_all(&buffer[..num_bytes])
+}
+
 impl VInt {
     pub fn val(&self) -> u64 {
         self.0
@@ -24,7 +119,7 @@ impl VInt {
         output.extend(&buffer[0..num_bytes]);
     }
 
-    fn serialize_into(&self, buffer: &mut [u8; 10]) -> usize {
+    pub fn serialize_into(&self, buffer: &mut [u8; 10]) -> usize {
         let mut remaining = self.0;
         for (i, b) in buffer.iter_mut().enumerate() {
             let next_byte: u8 = (remaining % 128u64) as u8;
@@ -64,7 +159,7 @@ impl BinarySerializable for VInt {
                     return Err(io::Error::new(
                         io::ErrorKind::InvalidData,
                         "Reach end of buffer while reading VInt",
-                    ))
+                    ));
                 }
             }
         }
@@ -74,7 +169,9 @@ impl BinarySerializable for VInt {
 #[cfg(test)]
 mod tests {
 
+    use super::serialize_vint_u32;
     use super::VInt;
+    use byteorder::{ByteOrder, LittleEndian};
     use common::BinarySerializable;
 
     fn aux_test_vint(val: u64) {
@@ -108,4 +205,28 @@ mod tests {
         }
         aux_test_vint(10);
     }
+
+    fn aux_test_serialize_vint_u32(val: u32) {
+        let mut buffer = [0u8; 10];
+        let mut buffer2 = [0u8; 10];
+        let len_vint = VInt(val as u64).serialize_into(&mut buffer);
+        let (vint, len) = serialize_vint_u32(val);
+        assert_eq!(len, len_vint, "len wrong for val {}", val);
+        LittleEndian::write_u64(&mut buffer2, vint);
+        assert_eq!(&buffer[..len], &buffer2[..len], "array wrong for {}", val);
+    }
+
+    #[test]
+    fn test_vint_u32() {
+        aux_test_serialize_vint_u32(0);
+        aux_test_serialize_vint_u32(1);
+        aux_test_serialize_vint_u32(5);
+        for i in 1..3 {
+            let power_of_128 = 1u32 << (7 * i);
+            aux_test_serialize_vint_u32(power_of_128 - 1u32);
+            aux_test_serialize_vint_u32(power_of_128);
+            aux_test_serialize_vint_u32(power_of_128 + 1u32);
+        }
+        aux_test_serialize_vint_u32(u32::max_value());
+    }
 }

src/core/executor.rs

@@ -64,17 +64,18 @@ impl Executor {
                     // This is important as it makes it possible for the fruit_receiver iteration to
                     // terminate.
                 };
-                let mut results = Vec::with_capacity(num_fruits);
-                unsafe { results.set_len(num_fruits) };
-                let mut num_items = 0;
+                // This is lame, but safe.
+                let mut results_with_position = Vec::with_capacity(num_fruits);
                 for (pos, fruit_res) in fruit_receiver {
-                    results[pos] = fruit_res?;
-                    num_items += 1;
+                    let fruit = fruit_res?;
+                    results_with_position.push((pos, fruit));
                 }
-                // this checks ensures that we filled of this
-                // uninitialized memory.
-                assert_eq!(num_items, results.len());
-                Ok(results)
+                results_with_position.sort_by_key(|(pos, _)| *pos);
+                assert_eq!(results_with_position.len(), num_fruits);
+                Ok(results_with_position
+                    .into_iter()
+                    .map(|(_, fruit)| fruit)
+                    .collect::<Vec<_>>())
             }
         }
     }
@@ -94,7 +95,8 @@ mod tests {
                     panic!("panic should propagate");
                 },
                 vec![0].into_iter(),
-            ).unwrap();
+            )
+            .unwrap();
     }
 
     #[test]
@@ -106,7 +108,8 @@ mod tests {
                     panic!("panic should propagate");
                 },
                 vec![0].into_iter(),
-            ).unwrap();
+            )
+            .unwrap();
     }
 
     #[test]

src/core/index.rs

@@ -12,12 +12,14 @@ use core::META_FILEPATH;
 use directory::ManagedDirectory;
 #[cfg(feature = "mmap")]
 use directory::MmapDirectory;
+use directory::INDEX_WRITER_LOCK;
+use directory::META_LOCK;
 use directory::{Directory, RAMDirectory};
+use error::DataCorruption;
 use error::TantivyError;
 use indexer::index_writer::open_index_writer;
 use indexer::index_writer::HEAP_SIZE_MIN;
 use indexer::segment_updater::save_new_metas;
-use indexer::LockType;
 use num_cpus;
 use schema::Field;
 use schema::FieldType;
@@ -37,7 +39,13 @@ fn load_metas(directory: &Directory) -> Result<IndexMeta> {
     let meta_data = directory.atomic_read(&META_FILEPATH)?;
     let meta_string = String::from_utf8_lossy(&meta_data);
     serde_json::from_str(&meta_string)
-        .map_err(|_| TantivyError::CorruptedFile(META_FILEPATH.clone()))
+        .map_err(|e| {
+            DataCorruption::new(
+                META_FILEPATH.clone(),
+                format!("Meta file cannot be deserialized. {:?}.", e),
+            )
+        })
+        .map_err(From::from)
 }
 
 /// Search Index
@@ -103,7 +111,6 @@ impl Index {
     }
 
     /// Opens or creates a new index in the provided directory
-    #[cfg(feature = "mmap")]
     pub fn open_or_create<Dir: Directory>(dir: Dir, schema: Schema) -> Result<Index> {
         if Index::exists(&dir) {
             let index = Index::open(dir)?;
@@ -135,7 +142,7 @@ impl Index {
 
     /// Creates a new index given an implementation of the trait `Directory`
     pub fn create<Dir: Directory>(dir: Dir, schema: Schema) -> Result<Index> {
-        let directory = ManagedDirectory::new(dir)?;
+        let directory = ManagedDirectory::wrap(dir)?;
         Index::from_directory(directory, schema)
     }
 
@@ -143,7 +150,7 @@ impl Index {
     ///
     /// This will overwrite existing meta.json
     fn from_directory(mut directory: ManagedDirectory, schema: Schema) -> Result<Index> {
-        save_new_metas(schema.clone(), 0, directory.borrow_mut())?;
+        save_new_metas(schema.clone(), directory.borrow_mut())?;
         let metas = IndexMeta::with_schema(schema);
         Index::create_from_metas(directory, &metas)
     }
@@ -199,7 +206,7 @@ impl Index {
 
     /// Open the index using the provided directory
     pub fn open<D: Directory>(directory: D) -> Result<Index> {
-        let directory = ManagedDirectory::new(directory)?;
+        let directory = ManagedDirectory::wrap(directory)?;
         let metas = load_metas(&directory)?;
         Index::create_from_metas(directory, &metas)
     }
@@ -225,7 +232,8 @@ impl Index {
     /// Each thread will receive a budget of  `overall_heap_size_in_bytes / num_threads`.
     ///
     /// # Errors
-    /// If the lockfile already exists, returns `Error::FileAlreadyExists`.
+    /// If the lockfile already exists, returns `Error::DirectoryLockBusy` or an `Error::IOError`.
+    ///
     /// # Panics
     /// If the heap size per thread is too small, panics.
     pub fn writer_with_num_threads(
@@ -233,7 +241,21 @@ impl Index {
         num_threads: usize,
         overall_heap_size_in_bytes: usize,
     ) -> Result<IndexWriter> {
-        let directory_lock = LockType::IndexWriterLock.acquire_lock(&self.directory)?;
+        let directory_lock = self
+            .directory
+            .acquire_lock(&INDEX_WRITER_LOCK)
+            .map_err(|err| {
+                TantivyError::LockFailure(
+                    err,
+                    Some(
+                        "Failed to acquire index lock. If you are using\
+                         a regular directory, this means there is already an \
+                         `IndexWriter` working on this `Directory`, in this process \
+                         or in a different process."
+                            .to_string(),
+                    ),
+                )
+            })?;
         let heap_size_in_bytes_per_thread = overall_heap_size_in_bytes / num_threads;
         open_index_writer(
             self,
@@ -332,7 +354,7 @@ impl Index {
     /// get the freshest `index` at all time, is to watch `meta.json` and
     /// call `load_searchers` whenever a changes happen.
     pub fn load_searchers(&self) -> Result<()> {
-        let _meta_lock = LockType::MetaLock.acquire_lock(self.directory())?;
+        let _meta_lock = self.directory().acquire_lock(&META_LOCK)?;
         let searchable_segments = self.searchable_segments()?;
         let segment_readers: Vec<SegmentReader> = searchable_segments
             .iter()

src/core/inverted_index_reader.rs

@@ -32,10 +32,7 @@ pub struct InvertedIndexReader {
 }
 
 impl InvertedIndexReader {
-    #[cfg_attr(
-        feature = "cargo-clippy",
-        allow(clippy::needless_pass_by_value)
-    )] // for symetry
+    #[cfg_attr(feature = "cargo-clippy", allow(clippy::needless_pass_by_value))] // for symetry
     pub(crate) fn new(
         termdict: TermDictionary,
         postings_source: ReadOnlySource,

src/core/pool.rs

@@ -1,5 +1,4 @@
 use crossbeam::queue::MsQueue;
-use std::mem;
 use std::ops::{Deref, DerefMut};
 use std::sync::atomic::AtomicUsize;
 use std::sync::atomic::Ordering;
@@ -10,6 +9,12 @@ pub struct GenerationItem<T> {
     item: T,
 }
 
+
+/// An object pool
+///
+/// This is used in tantivy to create a pool of `Searcher`.
+/// Object are wrapped in a `LeasedItem` wrapper and are
+/// released automatically back into the pool on `Drop`.
 pub struct Pool<T> {
     queue: Arc<MsQueue<GenerationItem<T>>>,
     freshest_generation: AtomicUsize,
@@ -26,6 +31,10 @@ impl<T> Pool<T> {
         }
     }
 
+    /// Publishes a new generation of `Searcher`.
+    ///
+    /// After publish, all new `Searcher` acquired will be
+    /// of the new generation.
     pub fn publish_new_generation(&self, items: Vec<T>) {
         let next_generation = self.next_generation.fetch_add(1, Ordering::SeqCst) + 1;
         for item in items {
@@ -61,6 +70,10 @@ impl<T> Pool<T> {
         self.freshest_generation.load(Ordering::Acquire)
     }
 
+    /// Acquires a new searcher.
+    ///
+    /// If no searcher is available, this methods block until
+    /// a searcher is released.
     pub fn acquire(&self) -> LeasedItem<T> {
         let generation = self.generation();
         loop {
@@ -107,9 +120,9 @@ impl<T> DerefMut for LeasedItem<T> {
 
 impl<T> Drop for LeasedItem<T> {
     fn drop(&mut self) {
-        let gen_item: GenerationItem<T> = mem::replace(&mut self.gen_item, None)
-            .expect("Unwrapping a leased item should never fail");
-        self.recycle_queue.push(gen_item);
+        if let Some(gen_item) = self.gen_item.take() {
+            self.recycle_queue.push(gen_item);
+        }
     }
 }
 

src/core/searcher.rs

@@ -104,7 +104,8 @@ impl Searcher {
             .iter()
             .map(|segment_reader| {
                 u64::from(segment_reader.inverted_index(term.field()).doc_freq(term))
-            }).sum::<u64>()
+            })
+            .sum::<u64>()
     }
 
     /// Return the list of segment readers

src/core/segment_component.rs

@@ -41,6 +41,6 @@ impl SegmentComponent {
             SegmentComponent::STORE,
             SegmentComponent::DELETE,
         ];
-        SEGMENT_COMPONENTS.into_iter()
+        SEGMENT_COMPONENTS.iter()
     }
 }

src/directory/directory.rs

@@ -1,11 +1,102 @@
+use directory::directory_lock::Lock;
+use directory::error::LockError;
 use directory::error::{DeleteError, OpenReadError, OpenWriteError};
 use directory::{ReadOnlySource, WritePtr};
 use std::fmt;
 use std::io;
+use std::io::Write;
 use std::marker::Send;
 use std::marker::Sync;
 use std::path::Path;
+use std::path::PathBuf;
 use std::result;
+use std::thread;
+use std::time::Duration;
+
+/// Retry the logic of acquiring locks is pretty simple.
+/// We just retry `n` times after a given `duratio`, both
+/// depending on the type of lock.
+struct RetryPolicy {
+    num_retries: usize,
+    wait_in_ms: u64,
+}
+
+impl RetryPolicy {
+    fn no_retry() -> RetryPolicy {
+        RetryPolicy {
+            num_retries: 0,
+            wait_in_ms: 0,
+        }
+    }
+
+    fn wait_and_retry(&mut self) -> bool {
+        if self.num_retries == 0 {
+            false
+        } else {
+            self.num_retries -= 1;
+            let wait_duration = Duration::from_millis(self.wait_in_ms);
+            thread::sleep(wait_duration);
+            true
+        }
+    }
+}
+
+/// The `DirectoryLock` is an object that represents a file lock.
+/// See  [`LockType`](struct.LockType.html)
+///
+/// It is transparently associated to a lock file, that gets deleted
+/// on `Drop.` The lock is released automatically on `Drop`.
+pub struct DirectoryLock(Box<Drop + Send + 'static>);
+
+struct DirectoryLockGuard {
+    directory: Box<Directory>,
+    path: PathBuf,
+}
+
+impl<T: Drop + Send + 'static> From<Box<T>> for DirectoryLock {
+    fn from(underlying: Box<T>) -> Self {
+        DirectoryLock(underlying)
+    }
+}
+
+impl Drop for DirectoryLockGuard {
+    fn drop(&mut self) {
+        if let Err(e) = self.directory.delete(&*self.path) {
+            error!("Failed to remove the lock file. {:?}", e);
+        }
+    }
+}
+
+enum TryAcquireLockError {
+    FileExists,
+    IOError(io::Error),
+}
+
+fn try_acquire_lock(
+    filepath: &Path,
+    directory: &mut Directory,
+) -> Result<DirectoryLock, TryAcquireLockError> {
+    let mut write = directory.open_write(filepath).map_err(|e| match e {
+        OpenWriteError::FileAlreadyExists(_) => TryAcquireLockError::FileExists,
+        OpenWriteError::IOError(io_error) => TryAcquireLockError::IOError(io_error.into()),
+    })?;
+    write.flush().map_err(TryAcquireLockError::IOError)?;
+    Ok(DirectoryLock::from(Box::new(DirectoryLockGuard {
+        directory: directory.box_clone(),
+        path: filepath.to_owned(),
+    })))
+}
+
+fn retry_policy(is_blocking: bool) -> RetryPolicy {
+    if is_blocking {
+        RetryPolicy {
+            num_retries: 100,
+            wait_in_ms: 100,
+        }
+    } else {
+        RetryPolicy::no_retry()
+    }
+}
 
 /// Write-once read many (WORM) abstraction for where
 /// tantivy's data should be stored.
@@ -73,6 +164,29 @@ pub trait Directory: DirectoryClone + fmt::Debug + Send + Sync + 'static {
     ///
     /// The file may or may not previously exist.
     fn atomic_write(&mut self, path: &Path, data: &[u8]) -> io::Result<()>;
+
+    /// Acquire a lock in the given directory.
+    ///
+    /// The method is blocking or not depending on the `Lock` object.
+    fn acquire_lock(&self, lock: &Lock) -> Result<DirectoryLock, LockError> {
+        let mut box_directory = self.box_clone();
+        let mut retry_policy = retry_policy(lock.is_blocking);
+        loop {
+            match try_acquire_lock(&lock.filepath, &mut *box_directory) {
+                Ok(result) => {
+                    return Ok(result);
+                }
+                Err(TryAcquireLockError::FileExists) => {
+                    if !retry_policy.wait_and_retry() {
+                        return Err(LockError::LockBusy);
+                    }
+                }
+                Err(TryAcquireLockError::IOError(io_error)) => {
+                    return Err(LockError::IOError(io_error));
+                }
+            }
+        }
+    }
 }
 
 /// DirectoryClone

src/directory/directory_lock.rs

@@ -0,0 +1,56 @@
+use std::path::PathBuf;
+
+/// A directory lock.
+///
+/// A lock is associated to a specific path and some
+/// [`LockParams`](./enum.LockParams.html).
+/// Tantivy itself uses only two locks but client application
+/// can use the directory facility to define their own locks.
+/// - [INDEX_WRITER_LOCK](./struct.INDEX_WRITER_LOCK.html)
+/// - [META_LOCK](./struct.META_LOCK.html)
+///
+/// Check out these locks documentation for more information.
+///
+#[derive(Debug)]
+pub struct Lock {
+    /// The lock needs to be associated with its own file `path`.
+    /// Depending on the platform, the lock might rely on the creation
+    /// and deletion of this filepath.
+    pub filepath: PathBuf,
+    /// `lock_params` describes whether acquiring the lock is meant
+    /// to be a blocking operation or a non-blocking.
+    ///
+    /// Acquiring a blocking lock blocks until the lock is
+    /// available.
+    /// Acquiring a blocking lock returns rapidly, either successfully
+    /// or with an error signifying that someone is already holding
+    /// the lock.
+    pub is_blocking: bool,
+}
+
+lazy_static! {
+     /// Only one process should be able to write tantivy's index at a time.
+    /// This lock file, when present, is in charge of preventing other processes to open an IndexWriter.
+    ///
+    /// If the process is killed and this file remains, it is safe to remove it manually.
+    ///
+    /// Failing to acquire this lock usually means a misuse of tantivy's API,
+    /// (creating more than one instance of the `IndexWriter`), are a spurious
+    /// lock file remaining after a crash. In the latter case, removing the file after
+    /// checking no process running tantivy is running is safe.
+    pub static ref INDEX_WRITER_LOCK: Lock = Lock {
+        filepath: PathBuf::from(".tantivy-writer.lock"),
+        is_blocking: false
+    };
+    /// The meta lock file is here to protect the segment files being opened by
+    /// `.load_searchers()` from being garbage collected.
+    /// It makes it possible for another process to safely consume
+    /// our index in-writing. Ideally, we may have prefered `RWLock` semantics
+    /// here, but it is difficult to achieve on Windows.
+    ///
+    /// Opening segment readers is a very fast process.
+    pub static ref META_LOCK: Lock = Lock {
+        filepath: PathBuf::from(".tantivy-meta.lock"),
+        is_blocking: true
+    };
+}

src/directory/error.rs

@@ -3,6 +3,22 @@ use std::fmt;
 use std::io;
 use std::path::PathBuf;
 
+/// Error while trying to acquire a directory lock.
+#[derive(Debug, Fail)]
+pub enum LockError {
+    /// Failed to acquired a lock as it is already hold by another
+    /// client.
+    /// - In the context of a blocking lock, this means the lock was not released within some `timeout` period.
+    /// - In the context of a non-blocking lock, this means the lock was busy at the moment of the call.
+    #[fail(
+        display = "Could not acquire lock as it is already held, possibly by a different process."
+    )]
+    LockBusy,
+    /// Trying to acquire a lock failed with an `IOError`
+    #[fail(display = "Failed to acquire the lock due to an io:Error.")]
+    IOError(io::Error),
+}
+
 /// General IO error with an optional path to the offending file.
 #[derive(Debug)]
 pub struct IOError {
@@ -10,6 +26,12 @@ pub struct IOError {
     err: io::Error,
 }
 
+impl Into<io::Error> for IOError {
+    fn into(self) -> io::Error {
+        self.err
+    }
+}
+
 impl fmt::Display for IOError {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         match self.path {

src/directory/managed_directory.rs

@@ -1,8 +1,10 @@
 use core::MANAGED_FILEPATH;
-use directory::error::{DeleteError, IOError, OpenReadError, OpenWriteError};
+use directory::error::{DeleteError, IOError, LockError, OpenReadError, OpenWriteError};
+use directory::DirectoryLock;
+use directory::Lock;
+use directory::META_LOCK;
 use directory::{ReadOnlySource, WritePtr};
-use error::TantivyError;
-use indexer::LockType;
+use error::DataCorruption;
 use serde_json;
 use std::collections::HashSet;
 use std::io;
@@ -59,12 +61,17 @@ fn save_managed_paths(
 
 impl ManagedDirectory {
     /// Wraps a directory as managed directory.
-    pub fn new<Dir: Directory>(directory: Dir) -> Result<ManagedDirectory> {
+    pub fn wrap<Dir: Directory>(directory: Dir) -> Result<ManagedDirectory> {
         match directory.atomic_read(&MANAGED_FILEPATH) {
             Ok(data) => {
                 let managed_files_json = String::from_utf8_lossy(&data);
                 let managed_files: HashSet<PathBuf> = serde_json::from_str(&managed_files_json)
-                    .map_err(|_| TantivyError::CorruptedFile(MANAGED_FILEPATH.clone()))?;
+                    .map_err(|e| {
+                        DataCorruption::new(
+                            MANAGED_FILEPATH.clone(),
+                            format!("Managed file cannot be deserialized: {:?}. ", e),
+                        )
+                    })?;
                 Ok(ManagedDirectory {
                     directory: Box::new(directory),
                     meta_informations: Arc::new(RwLock::new(MetaInformation {
@@ -87,6 +94,9 @@ impl ManagedDirectory {
     ///
     /// * `living_files` - List of files that are still used by the index.
     ///
+    /// The use a callback ensures that the list of living_files is computed
+    /// while we hold the lock on meta.
+    ///
     /// This method does not panick nor returns errors.
     /// If a file cannot be deleted (for permission reasons for instance)
     /// an error is simply logged, and the file remains in the list of managed
@@ -117,7 +127,7 @@ impl ManagedDirectory {
             // 2) writer change meta.json (for instance after a merge or a commit)
             // 3) gc kicks in.
             // 4) gc removes a file that was useful for process B, before process B opened it.
-            if let Ok(_meta_lock) = LockType::MetaLock.acquire_lock(self) {
+            if let Ok(_meta_lock) = self.acquire_lock(&META_LOCK) {
                 let living_files = get_living_files();
                 for managed_path in &meta_informations_rlock.managed_paths {
                     if !living_files.contains(managed_path) {
@@ -227,6 +237,10 @@ impl Directory for ManagedDirectory {
     fn exists(&self, path: &Path) -> bool {
         self.directory.exists(path)
     }
+
+    fn acquire_lock(&self, lock: &Lock) -> result::Result<DirectoryLock, LockError> {
+        self.directory.acquire_lock(lock)
+    }
 }
 
 impl Clone for ManagedDirectory {
@@ -260,7 +274,7 @@ mod tests {
         let tempdir_path = PathBuf::from(tempdir.path());
         {
             let mmap_directory = MmapDirectory::open(&tempdir_path).unwrap();
-            let mut managed_directory = ManagedDirectory::new(mmap_directory).unwrap();
+            let mut managed_directory = ManagedDirectory::wrap(mmap_directory).unwrap();
             {
                 let mut write_file = managed_directory.open_write(*TEST_PATH1).unwrap();
                 write_file.flush().unwrap();
@@ -286,7 +300,7 @@ mod tests {
         }
         {
             let mmap_directory = MmapDirectory::open(&tempdir_path).unwrap();
-            let mut managed_directory = ManagedDirectory::new(mmap_directory).unwrap();
+            let mut managed_directory = ManagedDirectory::wrap(mmap_directory).unwrap();
             {
                 assert!(managed_directory.exists(*TEST_PATH1));
                 assert!(!managed_directory.exists(*TEST_PATH2));
@@ -310,7 +324,7 @@ mod tests {
         let living_files = HashSet::new();
 
         let mmap_directory = MmapDirectory::open(&tempdir_path).unwrap();
-        let mut managed_directory = ManagedDirectory::new(mmap_directory).unwrap();
+        let mut managed_directory = ManagedDirectory::wrap(mmap_directory).unwrap();
         managed_directory
             .atomic_write(*TEST_PATH1, &vec![0u8, 1u8])
             .unwrap();

src/directory/mmap_directory.rs

@@ -1,8 +1,14 @@
+extern crate fs2;
+
+use self::fs2::FileExt;
 use atomicwrites;
 use common::make_io_err;
+use directory::error::LockError;
 use directory::error::{DeleteError, IOError, OpenDirectoryError, OpenReadError, OpenWriteError};
 use directory::shared_vec_slice::SharedVecSlice;
 use directory::Directory;
+use directory::DirectoryLock;
+use directory::Lock;
 use directory::ReadOnlySource;
 use directory::WritePtr;
 use fst::raw::MmapReadOnly;
@@ -115,6 +121,14 @@ impl MmapCache {
 ///
 /// The Mmap object are cached to limit the
 /// system calls.
+///
+/// In the `MmapDirectory`, locks are implemented using the `fs2` crate definition of locks.
+///
+/// On MacOS & linux, it relies on `flock` (aka `BSD Lock`). These locks solve most of the
+/// problems related to POSIX Locks, but may their contract may not be respected on `NFS`
+/// depending on the implementation.
+///
+/// On Windows the semantics are again different.
 #[derive(Clone)]
 pub struct MmapDirectory {
     root_path: PathBuf,
@@ -213,6 +227,21 @@ impl MmapDirectory {
     }
 }
 
+/// We rely on fs2 for file locking. On Windows & MacOS this
+/// uses BSD locks (`flock`). The lock is actually released when
+/// the `File` object is dropped and its associated file descriptor
+/// is closed.
+struct ReleaseLockFile {
+    _file: File,
+    path: PathBuf,
+}
+
+impl Drop for ReleaseLockFile {
+    fn drop(&mut self) {
+        debug!("Releasing lock {:?}", self.path);
+    }
+}
+
 /// This Write wraps a File, but has the specificity of
 /// call `sync_all` on flush.
 struct SafeFileWriter(File);
@@ -354,6 +383,26 @@ impl Directory for MmapDirectory {
         meta_file.write(|f| f.write_all(data))?;
         Ok(())
     }
+
+    fn acquire_lock(&self, lock: &Lock) -> Result<DirectoryLock, LockError> {
+        let full_path = self.resolve_path(&lock.filepath);
+        // We make sure that the file exists.
+        let file: File = OpenOptions::new()
+            .write(true)
+            .create(true) //< if the file does not exist yet, create it.
+            .open(&full_path)
+            .map_err(LockError::IOError)?;
+        if lock.is_blocking {
+            file.lock_exclusive().map_err(LockError::IOError)?;
+        } else {
+            file.try_lock_exclusive().map_err(|_| LockError::LockBusy)?
+        }
+        // dropping the file handle will release the lock.
+        Ok(DirectoryLock::from(Box::new(ReleaseLockFile {
+            path: lock.filepath.clone(),
+            _file: file,
+        })))
+    }
 }
 
 #[cfg(test)]

src/directory/mod.rs

@@ -8,6 +8,7 @@ WORM directory abstraction.
 mod mmap_directory;
 
 mod directory;
+mod directory_lock;
 mod managed_directory;
 mod ram_directory;
 mod read_only_source;
@@ -16,11 +17,12 @@ mod shared_vec_slice;
 /// Errors specific to the directory module.
 pub mod error;
 
-use std::io::{BufWriter, Seek, Write};
-
+pub use self::directory::DirectoryLock;
 pub use self::directory::{Directory, DirectoryClone};
+pub use self::directory_lock::{Lock, INDEX_WRITER_LOCK, META_LOCK};
 pub use self::ram_directory::RAMDirectory;
 pub use self::read_only_source::ReadOnlySource;
+use std::io::{BufWriter, Seek, Write};
 
 #[cfg(feature = "mmap")]
 pub use self::mmap_directory::MmapDirectory;
@@ -38,128 +40,4 @@ impl<T: Seek + Write> SeekableWrite for T {}
 pub type WritePtr = BufWriter<Box<SeekableWrite>>;
 
 #[cfg(test)]
-mod tests {
-
-    use super::*;
-    use std::io::{Seek, SeekFrom, Write};
-    use std::path::Path;
-
-    lazy_static! {
-        static ref TEST_PATH: &'static Path = Path::new("some_path_for_test");
-    }
-
-    #[test]
-    fn test_ram_directory() {
-        let mut ram_directory = RAMDirectory::create();
-        test_directory(&mut ram_directory);
-    }
-
-    #[test]
-    #[cfg(feature = "mmap")]
-    fn test_mmap_directory() {
-        let mut mmap_directory = MmapDirectory::create_from_tempdir().unwrap();
-        test_directory(&mut mmap_directory);
-    }
-
-    #[test]
-    #[should_panic]
-    fn ram_directory_panics_if_flush_forgotten() {
-        let mut ram_directory = RAMDirectory::create();
-        let mut write_file = ram_directory.open_write(*TEST_PATH).unwrap();
-        assert!(write_file.write_all(&[4]).is_ok());
-    }
-
-    fn test_simple(directory: &mut Directory) {
-        {
-            {
-                let mut write_file = directory.open_write(*TEST_PATH).unwrap();
-                assert!(directory.exists(*TEST_PATH));
-                write_file.write_all(&[4]).unwrap();
-                write_file.write_all(&[3]).unwrap();
-                write_file.write_all(&[7, 3, 5]).unwrap();
-                write_file.flush().unwrap();
-            }
-            let read_file = directory.open_read(*TEST_PATH).unwrap();
-            let data: &[u8] = &*read_file;
-            assert_eq!(data, &[4u8, 3u8, 7u8, 3u8, 5u8]);
-        }
-
-        assert!(directory.delete(*TEST_PATH).is_ok());
-        assert!(!directory.exists(*TEST_PATH));
-    }
-
-    fn test_seek(directory: &mut Directory) {
-        {
-            {
-                let mut write_file = directory.open_write(*TEST_PATH).unwrap();
-                write_file.write_all(&[4, 3, 7, 3, 5]).unwrap();
-                write_file.seek(SeekFrom::Start(0)).unwrap();
-                write_file.write_all(&[3, 1]).unwrap();
-                write_file.flush().unwrap();
-            }
-            let read_file = directory.open_read(*TEST_PATH).unwrap();
-            let data: &[u8] = &*read_file;
-            assert_eq!(data, &[3u8, 1u8, 7u8, 3u8, 5u8]);
-        }
-
-        assert!(directory.delete(*TEST_PATH).is_ok());
-    }
-
-    fn test_rewrite_forbidden(directory: &mut Directory) {
-        {
-            directory.open_write(*TEST_PATH).unwrap();
-            assert!(directory.exists(*TEST_PATH));
-        }
-        {
-            assert!(directory.open_write(*TEST_PATH).is_err());
-        }
-        assert!(directory.delete(*TEST_PATH).is_ok());
-    }
-
-    fn test_write_create_the_file(directory: &mut Directory) {
-        {
-            assert!(directory.open_read(*TEST_PATH).is_err());
-            let _w = directory.open_write(*TEST_PATH).unwrap();
-            assert!(directory.exists(*TEST_PATH));
-            assert!(directory.open_read(*TEST_PATH).is_ok());
-            assert!(directory.delete(*TEST_PATH).is_ok());
-        }
-    }
-
-    fn test_directory_delete(directory: &mut Directory) {
-        assert!(directory.open_read(*TEST_PATH).is_err());
-        let mut write_file = directory.open_write(*TEST_PATH).unwrap();
-        write_file.write_all(&[1, 2, 3, 4]).unwrap();
-        write_file.flush().unwrap();
-        {
-            let read_handle = directory.open_read(*TEST_PATH).unwrap();
-            {
-                assert_eq!(&*read_handle, &[1u8, 2u8, 3u8, 4u8]);
-
-                // Mapped files can't be deleted on Windows
-                if !cfg!(windows) {
-                    assert!(directory.delete(*TEST_PATH).is_ok());
-                    assert_eq!(&*read_handle, &[1u8, 2u8, 3u8, 4u8]);
-                }
-
-                assert!(directory.delete(Path::new("SomeOtherPath")).is_err());
-            }
-        }
-
-        if cfg!(windows) {
-            assert!(directory.delete(*TEST_PATH).is_ok());
-        }
-
-        assert!(directory.open_read(*TEST_PATH).is_err());
-        assert!(directory.delete(*TEST_PATH).is_err());
-    }
-
-    fn test_directory(directory: &mut Directory) {
-        test_simple(directory);
-        test_seek(directory);
-        test_rewrite_forbidden(directory);
-        test_write_create_the_file(directory);
-        test_directory_delete(directory);
-    }
-
-}
+mod tests;

src/directory/ram_directory.rs

@@ -100,7 +100,8 @@ impl InnerDirectory {
                 );
                 let io_err = make_io_err(msg);
                 OpenReadError::IOError(IOError::with_path(path.to_owned(), io_err))
-            }).and_then(|readable_map| {
+            })
+            .and_then(|readable_map| {
                 readable_map
                     .get(path)
                     .ok_or_else(|| OpenReadError::FileDoesNotExist(PathBuf::from(path)))
@@ -120,7 +121,8 @@ impl InnerDirectory {
                 );
                 let io_err = make_io_err(msg);
                 DeleteError::IOError(IOError::with_path(path.to_owned(), io_err))
-            }).and_then(|mut writable_map| match writable_map.remove(path) {
+            })
+            .and_then(|mut writable_map| match writable_map.remove(path) {
                 Some(_) => Ok(()),
                 None => Err(DeleteError::FileDoesNotExist(PathBuf::from(path))),
             })

src/directory/tests.rs

@@ -0,0 +1,182 @@
+use super::*;
+use std::io::{Seek, SeekFrom, Write};
+use std::path::{Path, PathBuf};
+use std::time;
+
+lazy_static! {
+    static ref TEST_PATH: &'static Path = Path::new("some_path_for_test");
+}
+
+#[test]
+fn test_ram_directory() {
+    let mut ram_directory = RAMDirectory::create();
+    test_directory(&mut ram_directory);
+}
+
+#[test]
+#[cfg(feature = "mmap")]
+fn test_mmap_directory() {
+    let mut mmap_directory = MmapDirectory::create_from_tempdir().unwrap();
+    test_directory(&mut mmap_directory);
+}
+
+#[test]
+#[should_panic]
+fn ram_directory_panics_if_flush_forgotten() {
+    let mut ram_directory = RAMDirectory::create();
+    let mut write_file = ram_directory.open_write(*TEST_PATH).unwrap();
+    assert!(write_file.write_all(&[4]).is_ok());
+}
+
+fn test_simple(directory: &mut Directory) {
+    {
+        {
+            let mut write_file = directory.open_write(*TEST_PATH).unwrap();
+            assert!(directory.exists(*TEST_PATH));
+            write_file.write_all(&[4]).unwrap();
+            write_file.write_all(&[3]).unwrap();
+            write_file.write_all(&[7, 3, 5]).unwrap();
+            write_file.flush().unwrap();
+        }
+        let read_file = directory.open_read(*TEST_PATH).unwrap();
+        let data: &[u8] = &*read_file;
+        assert_eq!(data, &[4u8, 3u8, 7u8, 3u8, 5u8]);
+    }
+
+    assert!(directory.delete(*TEST_PATH).is_ok());
+    assert!(!directory.exists(*TEST_PATH));
+}
+
+fn test_seek(directory: &mut Directory) {
+    {
+        {
+            let mut write_file = directory.open_write(*TEST_PATH).unwrap();
+            write_file.write_all(&[4, 3, 7, 3, 5]).unwrap();
+            write_file.seek(SeekFrom::Start(0)).unwrap();
+            write_file.write_all(&[3, 1]).unwrap();
+            write_file.flush().unwrap();
+        }
+        let read_file = directory.open_read(*TEST_PATH).unwrap();
+        let data: &[u8] = &*read_file;
+        assert_eq!(data, &[3u8, 1u8, 7u8, 3u8, 5u8]);
+    }
+
+    assert!(directory.delete(*TEST_PATH).is_ok());
+}
+
+fn test_rewrite_forbidden(directory: &mut Directory) {
+    {
+        directory.open_write(*TEST_PATH).unwrap();
+        assert!(directory.exists(*TEST_PATH));
+    }
+    {
+        assert!(directory.open_write(*TEST_PATH).is_err());
+    }
+    assert!(directory.delete(*TEST_PATH).is_ok());
+}
+
+fn test_write_create_the_file(directory: &mut Directory) {
+    {
+        assert!(directory.open_read(*TEST_PATH).is_err());
+        let _w = directory.open_write(*TEST_PATH).unwrap();
+        assert!(directory.exists(*TEST_PATH));
+        assert!(directory.open_read(*TEST_PATH).is_ok());
+        assert!(directory.delete(*TEST_PATH).is_ok());
+    }
+}
+
+fn test_directory_delete(directory: &mut Directory) {
+    assert!(directory.open_read(*TEST_PATH).is_err());
+    let mut write_file = directory.open_write(*TEST_PATH).unwrap();
+    write_file.write_all(&[1, 2, 3, 4]).unwrap();
+    write_file.flush().unwrap();
+    {
+        let read_handle = directory.open_read(*TEST_PATH).unwrap();
+        {
+            assert_eq!(&*read_handle, &[1u8, 2u8, 3u8, 4u8]);
+
+            // Mapped files can't be deleted on Windows
+            if !cfg!(windows) {
+                assert!(directory.delete(*TEST_PATH).is_ok());
+                assert_eq!(&*read_handle, &[1u8, 2u8, 3u8, 4u8]);
+            }
+
+            assert!(directory.delete(Path::new("SomeOtherPath")).is_err());
+        }
+    }
+
+    if cfg!(windows) {
+        assert!(directory.delete(*TEST_PATH).is_ok());
+    }
+
+    assert!(directory.open_read(*TEST_PATH).is_err());
+    assert!(directory.delete(*TEST_PATH).is_err());
+}
+
+fn test_directory(directory: &mut Directory) {
+    test_simple(directory);
+    test_seek(directory);
+    test_rewrite_forbidden(directory);
+    test_write_create_the_file(directory);
+    test_directory_delete(directory);
+    test_lock_non_blocking(directory);
+    test_lock_blocking(directory);
+}
+
+fn test_lock_non_blocking(directory: &mut Directory) {
+    {
+        let lock_a_res = directory.acquire_lock(&Lock {
+            filepath: PathBuf::from("a.lock"),
+            is_blocking: false,
+        });
+        assert!(lock_a_res.is_ok());
+        let lock_b_res = directory.acquire_lock(&Lock {
+            filepath: PathBuf::from("b.lock"),
+            is_blocking: false,
+        });
+        assert!(lock_b_res.is_ok());
+        let lock_a_res2 = directory.acquire_lock(&Lock {
+            filepath: PathBuf::from("a.lock"),
+            is_blocking: false,
+        });
+        assert!(lock_a_res2.is_err());
+    }
+    let lock_a_res = directory.acquire_lock(&Lock {
+        filepath: PathBuf::from("a.lock"),
+        is_blocking: false,
+    });
+    assert!(lock_a_res.is_ok());
+}
+
+fn test_lock_blocking(directory: &mut Directory) {
+    let lock_a_res = directory.acquire_lock(&Lock {
+        filepath: PathBuf::from("a.lock"),
+        is_blocking: true,
+    });
+    assert!(lock_a_res.is_ok());
+    std::thread::spawn(move || {
+        //< lock_a_res is sent to the thread.
+        std::thread::sleep(time::Duration::from_millis(10));
+        // explicitely droping lock_a_res. It would have been sufficient to just force it
+        // to be part of the move, but the intent seems clearer that way.
+        drop(lock_a_res);
+    });
+    {
+        // A non-blocking call should fail, as the thread is running and holding the lock.
+        let lock_a_res = directory.acquire_lock(&Lock {
+            filepath: PathBuf::from("a.lock"),
+            is_blocking: false,
+        });
+        assert!(lock_a_res.is_err());
+    }
+    {
+        // the blocking call should wait for at least 10ms.
+        let start = time::Instant::now();
+        let lock_a_res = directory.acquire_lock(&Lock {
+            filepath: PathBuf::from("a.lock"),
+            is_blocking: true,
+        });
+        assert!(lock_a_res.is_ok());
+        assert!(start.elapsed().subsec_millis() >= 10);
+    }
+}

src/error.rs

@@ -2,15 +2,48 @@
 
 use std::io;
 
+use directory::error::LockError;
 use directory::error::{IOError, OpenDirectoryError, OpenReadError, OpenWriteError};
 use fastfield::FastFieldNotAvailableError;
-use indexer::LockType;
 use query;
 use schema;
 use serde_json;
+use std::fmt;
 use std::path::PathBuf;
 use std::sync::PoisonError;
 
+pub struct DataCorruption {
+    filepath: Option<PathBuf>,
+    comment: String,
+}
+
+impl DataCorruption {
+    pub fn new(filepath: PathBuf, comment: String) -> DataCorruption {
+        DataCorruption {
+            filepath: Some(filepath),
+            comment,
+        }
+    }
+
+    pub fn comment_only(comment: String) -> DataCorruption {
+        DataCorruption {
+            filepath: None,
+            comment,
+        }
+    }
+}
+
+impl fmt::Debug for DataCorruption {
+    fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
+        write!(f, "Data corruption: ")?;
+        if let Some(ref filepath) = &self.filepath {
+            write!(f, "(in file `{:?}`)", filepath)?;
+        }
+        write!(f, ": {}.", self.comment)?;
+        Ok(())
+    }
+}
+
 /// The library's failure based error enum
 #[derive(Debug, Fail)]
 pub enum TantivyError {
@@ -24,17 +57,14 @@ pub enum TantivyError {
     #[fail(display = "Index already exists")]
     IndexAlreadyExists,
     /// Failed to acquire file lock
-    #[fail(
-        display = "Failed to acquire Lockfile: {:?}. Possible causes: another IndexWriter instance or panic during previous lock drop.",
-        _0
-    )]
-    LockFailure(LockType),
+    #[fail(display = "Failed to acquire Lockfile: {:?}. {:?}", _0, _1)]
+    LockFailure(LockError, Option<String>),
     /// IO Error.
     #[fail(display = "An IO error occurred: '{}'", _0)]
     IOError(#[cause] IOError),
     /// Data corruption.
-    #[fail(display = "File contains corrupted data: '{:?}'", _0)]
-    CorruptedFile(PathBuf),
+    #[fail(display = "{:?}", _0)]
+    DataCorruption(DataCorruption),
     /// A thread holding the locked panicked and poisoned the lock.
     #[fail(display = "A thread holding the locked panicked and poisoned the lock")]
     Poisoned,
@@ -55,12 +85,24 @@ pub enum TantivyError {
     SystemError(String),
 }
 
+impl From<DataCorruption> for TantivyError {
+    fn from(data_corruption: DataCorruption) -> TantivyError {
+        TantivyError::DataCorruption(data_corruption)
+    }
+}
+
 impl From<FastFieldNotAvailableError> for TantivyError {
     fn from(fastfield_error: FastFieldNotAvailableError) -> TantivyError {
         TantivyError::FastFieldError(fastfield_error)
     }
 }
 
+impl From<LockError> for TantivyError {
+    fn from(lock_error: LockError) -> TantivyError {
+        TantivyError::LockFailure(lock_error, None)
+    }
+}
+
 impl From<IOError> for TantivyError {
     fn from(io_error: IOError) -> TantivyError {
         TantivyError::IOError(io_error)

src/fastfield/facet_reader.rs

@@ -1,5 +1,6 @@
 use super::MultiValueIntFastFieldReader;
 use schema::Facet;
+use std::str;
 use termdict::TermDictionary;
 use termdict::TermOrdinal;
 use DocId;
@@ -20,6 +21,7 @@ use DocId;
 pub struct FacetReader {
     term_ords: MultiValueIntFastFieldReader<u64>,
     term_dict: TermDictionary,
+    buffer: Vec<u8>,
 }
 
 impl FacetReader {
@@ -37,6 +39,7 @@ impl FacetReader {
         FacetReader {
             term_ords,
             term_dict,
+            buffer: vec![],
         }
     }
 
@@ -55,11 +58,18 @@ impl FacetReader {
     }
 
     /// Given a term ordinal returns the term associated to it.
-    pub fn facet_from_ord(&self, facet_ord: TermOrdinal, output: &mut Facet) {
+    pub fn facet_from_ord(
+        &mut self,
+        facet_ord: TermOrdinal,
+        output: &mut Facet,
+    ) -> Result<(), str::Utf8Error> {
         let found_term = self
             .term_dict
-            .ord_to_term(facet_ord as u64, output.inner_buffer_mut());
+            .ord_to_term(facet_ord as u64, &mut self.buffer);
         assert!(found_term, "Term ordinal {} no found.", facet_ord);
+        let facet_str = str::from_utf8(&self.buffer[..])?;
+        output.set_facet_str(facet_str);
+        Ok(())
     }
 
     /// Return the list of facet ordinals associated to a document.

src/fastfield/multivalued/reader.rs

@@ -39,7 +39,7 @@ impl<Item: FastValue> MultiValueIntFastFieldReader<Item> {
         let (start, stop) = self.range(doc);
         let len = (stop - start) as usize;
         vals.resize(len, Item::default());
-        self.vals_reader.get_range(start as u32, &mut vals[..]);
+        self.vals_reader.get_range_u64(start, &mut vals[..]);
     }
 }
 
@@ -82,20 +82,20 @@ mod tests {
 
         let mut facet = Facet::root();
         {
-            facet_reader.facet_from_ord(1, &mut facet);
+            facet_reader.facet_from_ord(1, &mut facet).unwrap();
             assert_eq!(facet, Facet::from("/category"));
         }
         {
-            facet_reader.facet_from_ord(2, &mut facet);
+            facet_reader.facet_from_ord(2, &mut facet).unwrap();
             assert_eq!(facet, Facet::from("/category/cat1"));
         }
         {
-            facet_reader.facet_from_ord(3, &mut facet);
+            facet_reader.facet_from_ord(3, &mut facet).unwrap();
             assert_eq!(format!("{}", facet), "/category/cat2");
             assert_eq!(facet, Facet::from("/category/cat2"));
         }
         {
-            facet_reader.facet_from_ord(4, &mut facet);
+            facet_reader.facet_from_ord(4, &mut facet).unwrap();
             assert_eq!(facet, Facet::from("/category/cat3"));
         }
 

src/fastfield/reader.rs

@@ -59,7 +59,29 @@ impl<Item: FastValue> FastFieldReader<Item> {
     /// May panic if `doc` is greater than the segment
     // `maxdoc`.
     pub fn get(&self, doc: DocId) -> Item {
-        Item::from_u64(self.min_value_u64 + self.bit_unpacker.get(doc as usize))
+        self.get_u64(doc as u64)
+    }
+
+    pub(crate) fn get_u64(&self, doc: u64) -> Item {
+        Item::from_u64(self.min_value_u64 + self.bit_unpacker.get(doc))
+    }
+
+    /// Internally `multivalued` also use SingleValue Fast fields.
+    /// It works as follows... A first column contains the list of start index
+    /// for each document, a second column contains the actual values.
+    ///
+    /// The values associated to a given doc, are then
+    ///  `second_column[first_column.get(doc)..first_column.get(doc+1)]`.
+    ///
+    /// Which means single value fast field reader can be indexed internally with
+    /// something different from a `DocId`. For this use case, we want to use `u64`
+    /// values.
+    ///
+    /// See `get_range` for an actual documentation about this method.
+    pub(crate) fn get_range_u64(&self, start: u64, output: &mut [Item]) {
+        for (i, out) in output.iter_mut().enumerate() {
+            *out = self.get_u64(start + (i as u64));
+        }
     }
 
     /// Fills an output buffer with the fast field values
@@ -75,16 +97,8 @@ impl<Item: FastValue> FastFieldReader<Item> {
     ///
     /// May panic if `start + output.len()` is greater than
     /// the segment's `maxdoc`.
-    ///
-    // TODO change start to `u64`.
-    // For multifastfield, start is an index in a second fastfield, not a `DocId`
-    pub fn get_range(&self, start: u32, output: &mut [Item]) {
-        // ok: Item is either `u64` or `i64`
-        let output_u64: &mut [u64] = unsafe { &mut *(output as *mut [Item] as *mut [u64]) };
-        self.bit_unpacker.get_range(start, output_u64);
-        for out in output_u64.iter_mut() {
-            *out = Item::from_u64(*out + self.min_value_u64).as_u64();
-        }
+    pub fn get_range(&self, start: DocId, output: &mut [Item]) {
+        self.get_range_u64(start as u64, output);
     }
 
     /// Returns the minimum value for this fast field.

src/indexer/delete_queue.rs

@@ -191,10 +191,7 @@ impl DeleteCursor {
         }
     }
 
-    #[cfg_attr(
-        feature = "cargo-clippy",
-        allow(clippy::wrong_self_convention)
-    )]
+    #[cfg_attr(feature = "cargo-clippy", allow(clippy::wrong_self_convention))]
     fn is_behind_opstamp(&mut self, target_opstamp: u64) -> bool {
         self.get()
             .map(|operation| operation.opstamp < target_opstamp)

src/indexer/directory_lock.rs

@@ -1,131 +0,0 @@
-use directory::error::OpenWriteError;
-use std::io::Write;
-use std::path::{Path, PathBuf};
-use std::thread;
-use std::time::Duration;
-use Directory;
-use TantivyError;
-
-#[derive(Debug, Clone, Copy)]
-pub enum LockType {
-    /// Only one process should be able to write tantivy's index at a time.
-    /// This lock file, when present, is in charge of preventing other processes to open an IndexWriter.
-    ///
-    /// If the process is killed and this file remains, it is safe to remove it manually.
-    ///
-    /// Failing to acquire this lock usually means a misuse of tantivy's API,
-    /// (creating more than one instance of the `IndexWriter`), are a spurious
-    /// lock file remaining after a crash. In the latter case, removing the file after
-    /// checking no process running tantivy is running is safe.
-    IndexWriterLock,
-    /// The meta lock file is here to protect the segment files being opened by
-    /// `.load_searchers()` from being garbage collected.
-    /// It makes it possible for another process to safely consume
-    /// our index in-writing. Ideally, we may have prefered `RWLock` semantics
-    /// here, but it is difficult to achieve on Windows.
-    ///
-    /// Opening segment readers is a very fast process.
-    /// Right now if the lock cannot be acquire on the first attempt, the logic
-    /// is very simplistic. We retry after `100ms` until we effectively
-    /// acquire the lock.
-    /// This lock should not have much contention in normal usage.
-    MetaLock,
-}
-
-/// Retry the logic of acquiring locks is pretty simple.
-/// We just retry `n` times after a given `duratio`, both
-/// depending on the type of lock.
-struct RetryPolicy {
-    num_retries: usize,
-    wait_in_ms: u64,
-}
-
-impl RetryPolicy {
-    fn no_retry() -> RetryPolicy {
-        RetryPolicy {
-            num_retries: 0,
-            wait_in_ms: 0,
-        }
-    }
-
-    fn wait_and_retry(&mut self) -> bool {
-        if self.num_retries == 0 {
-            false
-        } else {
-            self.num_retries -= 1;
-            let wait_duration = Duration::from_millis(self.wait_in_ms);
-            thread::sleep(wait_duration);
-            true
-        }
-    }
-}
-
-impl LockType {
-    fn retry_policy(self) -> RetryPolicy {
-        match self {
-            LockType::IndexWriterLock => RetryPolicy::no_retry(),
-            LockType::MetaLock => RetryPolicy {
-                num_retries: 100,
-                wait_in_ms: 100,
-            },
-        }
-    }
-
-    fn try_acquire_lock(self, directory: &mut Directory) -> Result<DirectoryLock, TantivyError> {
-        let path = self.filename();
-        let mut write = directory.open_write(path).map_err(|e| match e {
-            OpenWriteError::FileAlreadyExists(_) => TantivyError::LockFailure(self),
-            OpenWriteError::IOError(io_error) => TantivyError::IOError(io_error),
-        })?;
-        write.flush()?;
-        Ok(DirectoryLock {
-            directory: directory.box_clone(),
-            path: path.to_owned(),
-        })
-    }
-
-    /// Acquire a lock in the given directory.
-    pub fn acquire_lock(self, directory: &Directory) -> Result<DirectoryLock, TantivyError> {
-        let mut box_directory = directory.box_clone();
-        let mut retry_policy = self.retry_policy();
-        loop {
-            let lock_result = self.try_acquire_lock(&mut *box_directory);
-            match lock_result {
-                Ok(result) => {
-                    return Ok(result);
-                }
-                Err(TantivyError::LockFailure(ref filepath)) => {
-                    if !retry_policy.wait_and_retry() {
-                        return Err(TantivyError::LockFailure(filepath.to_owned()));
-                    }
-                }
-                Err(_) => {}
-            }
-        }
-    }
-
-    fn filename(&self) -> &Path {
-        match *self {
-            LockType::MetaLock => Path::new(".tantivy-meta.lock"),
-            LockType::IndexWriterLock => Path::new(".tantivy-indexer.lock"),
-        }
-    }
-}
-
-/// The `DirectoryLock` is an object that represents a file lock.
-/// See  [`LockType`](struct.LockType.html)
-///
-/// It is transparently associated to a lock file, that gets deleted
-/// on `Drop.` The lock is release automatically on `Drop`.
-pub struct DirectoryLock {
-    directory: Box<Directory>,
-    path: PathBuf,
-}
-
-impl Drop for DirectoryLock {
-    fn drop(&mut self) {
-        if let Err(e) = self.directory.delete(&*self.path) {
-            error!("Failed to remove the lock file. {:?}", e);
-        }
-    }
-}

src/indexer/index_writer.rs

@@ -1,4 +1,4 @@
-use super::operation::AddOperation;
+use super::operation::{AddOperation, UserOperation};
 use super::segment_updater::SegmentUpdater;
 use super::PreparedCommit;
 use bit_set::BitSet;
@@ -9,15 +9,15 @@ use core::SegmentId;
 use core::SegmentMeta;
 use core::SegmentReader;
 use crossbeam::channel;
+use directory::DirectoryLock;
 use docset::DocSet;
 use error::TantivyError;
 use fastfield::write_delete_bitset;
-use futures::sync::oneshot::Receiver;
+use futures::{Canceled, Future};
 use indexer::delete_queue::{DeleteCursor, DeleteQueue};
 use indexer::doc_opstamp_mapping::DocToOpstampMapping;
 use indexer::operation::DeleteOperation;
 use indexer::stamper::Stamper;
-use indexer::DirectoryLock;
 use indexer::MergePolicy;
 use indexer::SegmentEntry;
 use indexer::SegmentWriter;
@@ -26,7 +26,8 @@ use schema::Document;
 use schema::IndexRecordOption;
 use schema::Term;
 use std::mem;
-use std::mem::swap;
+use std::ops::Range;
+use std::sync::Arc;
 use std::thread;
 use std::thread::JoinHandle;
 use Result;
@@ -43,8 +44,8 @@ pub const HEAP_SIZE_MAX: usize = u32::max_value() as usize - MARGIN_IN_BYTES;
 // reaches `PIPELINE_MAX_SIZE_IN_DOCS`
 const PIPELINE_MAX_SIZE_IN_DOCS: usize = 10_000;
 
-type DocumentSender = channel::Sender<AddOperation>;
-type DocumentReceiver = channel::Receiver<AddOperation>;
+type DocumentSender = channel::Sender<Vec<AddOperation>>;
+type DocumentReceiver = channel::Receiver<Vec<AddOperation>>;
 
 /// Split the thread memory budget into
 /// - the heap size
@@ -52,16 +53,19 @@ type DocumentReceiver = channel::Receiver<AddOperation>;
 ///
 /// Returns (the heap size in bytes, the hash table size in number of bits)
 fn initial_table_size(per_thread_memory_budget: usize) -> usize {
+    assert!(per_thread_memory_budget > 1_000);
     let table_size_limit: usize = per_thread_memory_budget / 3;
-    (1..)
+    if let Some(limit) = (1..)
         .take_while(|num_bits: &usize| compute_table_size(*num_bits) < table_size_limit)
         .last()
-        .unwrap_or_else(|| {
-            panic!(
-                "Per thread memory is too small: {}",
-                per_thread_memory_budget
-            )
-        }).min(19) // we cap it at 512K
+    {
+        limit.min(19) // we cap it at 2^19 = 512K.
+    } else {
+        unreachable!(
+            "Per thread memory is too small: {}",
+            per_thread_memory_budget
+        );
+    }
 }
 
 /// `IndexWriter` is the user entry-point to add document to an index.
@@ -139,7 +143,7 @@ pub fn open_index_writer(
     let stamper = Stamper::new(current_opstamp);
 
     let segment_updater =
-        SegmentUpdater::new(index.clone(), stamper.clone(), &delete_queue.cursor())?;
+        SegmentUpdater::create(index.clone(), stamper.clone(), &delete_queue.cursor())?;
 
     let mut index_writer = IndexWriter {
         _directory_lock: Some(directory_lock),
@@ -255,7 +259,7 @@ pub fn advance_deletes(
             write_delete_bitset(&delete_bitset, &mut delete_file)?;
         }
     }
-    segment_entry.set_meta((*segment.meta()).clone());
+    segment_entry.set_meta(segment.meta().clone());
     Ok(())
 }
 
@@ -263,7 +267,7 @@ fn index_documents(
     memory_budget: usize,
     segment: &Segment,
     generation: usize,
-    document_iterator: &mut Iterator<Item = AddOperation>,
+    document_iterator: &mut Iterator<Item = Vec<AddOperation>>,
     segment_updater: &mut SegmentUpdater,
     mut delete_cursor: DeleteCursor,
 ) -> Result<bool> {
@@ -271,11 +275,11 @@ fn index_documents(
     let segment_id = segment.id();
     let table_size = initial_table_size(memory_budget);
     let mut segment_writer = SegmentWriter::for_segment(table_size, segment.clone(), &schema)?;
-    for doc in document_iterator {
-        segment_writer.add_document(doc, &schema)?;
-
+    for documents in document_iterator {
+        for doc in documents {
+            segment_writer.add_document(doc, &schema)?;
+        }
         let mem_usage = segment_writer.mem_usage();
-
         if mem_usage >= memory_budget - MARGIN_IN_BYTES {
             info!(
                 "Buffer limit reached, flushing segment with maxdoc={}.",
@@ -301,7 +305,7 @@ fn index_documents(
 
     let last_docstamp: u64 = *(doc_opstamps.last().unwrap());
 
-    let segment_entry: SegmentEntry = if delete_cursor.get().is_some() {
+    let delete_bitset_opt = if delete_cursor.get().is_some() {
         let doc_to_opstamps = DocToOpstampMapping::from(doc_opstamps);
         let segment_reader = SegmentReader::open(segment)?;
         let mut deleted_bitset = BitSet::with_capacity(num_docs as usize);
@@ -312,18 +316,17 @@ fn index_documents(
             &doc_to_opstamps,
             last_docstamp,
         )?;
-        SegmentEntry::new(segment_meta, delete_cursor, {
-            if may_have_deletes {
-                Some(deleted_bitset)
-            } else {
-                None
-            }
-        })
+        if may_have_deletes {
+            Some(deleted_bitset)
+        } else {
+            None
+        }
     } else {
         // if there are no delete operation in the queue, no need
         // to even open the segment.
-        SegmentEntry::new(segment_meta, delete_cursor, None)
+        None
     };
+    let segment_entry = SegmentEntry::new(segment_meta, delete_cursor, delete_bitset_opt);
     Ok(segment_updater.add_segment(generation, segment_entry))
 }
 
@@ -365,13 +368,16 @@ impl IndexWriter {
             .add_segment(self.generation, segment_entry);
     }
 
-    /// *Experimental & Advanced API* Creates a new segment.
-    /// and marks it as currently in write.
+    /// Creates a new segment.
     ///
     /// This method is useful only for users trying to do complex
     /// operations, like converting an index format to another.
+    ///
+    /// It is safe to start writing file associated to the new `Segment`.
+    /// These will not be garbage collected as long as an instance object of
+    /// `SegmentMeta` object associated to the new `Segment` is "alive".
     pub fn new_segment(&self) -> Segment {
-        self.segment_updater.new_segment()
+        self.index.new_segment()
     }
 
     /// Spawns a new worker thread for indexing.
@@ -386,11 +392,13 @@ impl IndexWriter {
         let mut delete_cursor = self.delete_queue.cursor();
 
         let mem_budget = self.heap_size_in_bytes_per_thread;
+        let index = self.index.clone();
         let join_handle: JoinHandle<Result<()>> = thread::Builder::new()
             .name(format!(
                 "thrd-tantivy-index{}-gen{}",
                 self.worker_id, generation
-            )).spawn(move || {
+            ))
+            .spawn(move || {
                 loop {
                     let mut document_iterator =
                         document_receiver_clone.clone().into_iter().peekable();
@@ -402,15 +410,19 @@ impl IndexWriter {
                     // this is a valid guarantee as the
                     // peeked document now belongs to
                     // our local iterator.
-                    if let Some(operation) = document_iterator.peek() {
-                        delete_cursor.skip_to(operation.opstamp);
+                    if let Some(operations) = document_iterator.peek() {
+                        if let Some(first) = operations.first() {
+                            delete_cursor.skip_to(first.opstamp);
+                        } else {
+                            return Ok(());
+                        }
                     } else {
                         // No more documents.
                         // Happens when there is a commit, or if the `IndexWriter`
                         // was dropped.
                         return Ok(());
                     }
-                    let segment = segment_updater.new_segment();
+                    let segment = index.new_segment();
                     index_documents(
                         mem_budget,
                         &segment,
@@ -427,7 +439,7 @@ impl IndexWriter {
     }
 
     /// Accessor to the merge policy.
-    pub fn get_merge_policy(&self) -> Box<MergePolicy> {
+    pub fn get_merge_policy(&self) -> Arc<Box<MergePolicy>> {
         self.segment_updater.get_merge_policy()
     }
 
@@ -452,7 +464,10 @@ impl IndexWriter {
     /// Merges a given list of segments
     ///
     /// `segment_ids` is required to be non-empty.
-    pub fn merge(&mut self, segment_ids: &[SegmentId]) -> Result<Receiver<SegmentMeta>> {
+    pub fn merge(
+        &mut self,
+        segment_ids: &[SegmentId],
+    ) -> Result<impl Future<Item = SegmentMeta, Error = Canceled>> {
         self.segment_updater.start_merge(segment_ids)
     }
 
@@ -465,13 +480,10 @@ impl IndexWriter {
     ///
     /// Returns the former segment_ready channel.
     fn recreate_document_channel(&mut self) -> DocumentReceiver {
-        let (mut document_sender, mut document_receiver): (
-            DocumentSender,
-            DocumentReceiver,
-        ) = channel::bounded(PIPELINE_MAX_SIZE_IN_DOCS);
-        swap(&mut self.document_sender, &mut document_sender);
-        swap(&mut self.document_receiver, &mut document_receiver);
-        document_receiver
+        let (document_sender, document_receiver): (DocumentSender, DocumentReceiver) =
+            channel::bounded(PIPELINE_MAX_SIZE_IN_DOCS);
+        mem::replace(&mut self.document_sender, document_sender);
+        mem::replace(&mut self.document_receiver, document_receiver)
     }
 
     /// Rollback to the last commit
@@ -558,17 +570,12 @@ impl IndexWriter {
         // and recreate a new one channels.
         self.recreate_document_channel();
 
-        let mut former_workers_join_handle = Vec::new();
-        swap(
-            &mut former_workers_join_handle,
-            &mut self.workers_join_handle,
-        );
+        let former_workers_join_handle = mem::replace(&mut self.workers_join_handle, Vec::new());
 
         for worker_handle in former_workers_join_handle {
             let indexing_worker_result = worker_handle
                 .join()
                 .map_err(|e| TantivyError::ErrorInThread(format!("{:?}", e)))?;
-
             indexing_worker_result?;
             // add a new worker for the next generation.
             self.add_indexing_worker()?;
@@ -641,32 +648,176 @@ impl IndexWriter {
     pub fn add_document(&mut self, document: Document) -> u64 {
         let opstamp = self.stamper.stamp();
         let add_operation = AddOperation { opstamp, document };
-        let send_result = self.document_sender.send(add_operation);
+        let send_result = self.document_sender.send(vec![add_operation]);
         if let Err(e) = send_result {
             panic!("Failed to index document. Sending to indexing channel failed. This probably means all of the indexing threads have panicked. {:?}", e);
         }
         opstamp
     }
+
+    /// Gets a range of stamps from the stamper and "pops" the last stamp
+    /// from the range returning a tuple of the last optstamp and the popped
+    /// range.
+    ///
+    /// The total number of stamps generated by this method is `count + 1`;
+    /// each operation gets a stamp from the `stamps` iterator and `last_opstamp`
+    /// is for the batch itself.
+    fn get_batch_opstamps(&mut self, count: u64) -> (u64, Range<u64>) {
+        let Range { start, end } = self.stamper.stamps(count + 1u64);
+        let last_opstamp = end - 1;
+        let stamps = Range {
+            start: start,
+            end: last_opstamp,
+        };
+        (last_opstamp, stamps)
+    }
+
+    /// Runs a group of document operations ensuring that the operations are
+    /// assigned contigous u64 opstamps and that add operations of the same
+    /// group are flushed into the same segment.
+    /// 
+    /// If the indexing pipeline is full, this call may block.
+    /// 
+    /// Each operation of the given `user_operations` will receive an in-order,
+    /// contiguous u64 opstamp. The entire batch itself is also given an
+    /// opstamp that is 1 greater than the last given operation. This 
+    /// `batch_opstamp` is the return value of `run`. An empty group of
+    /// `user_operations`, an empty `Vec<UserOperation>`, still receives
+    /// a valid opstamp even though no changes were _actually_ made to the index.
+    /// 
+    /// Like adds and deletes (see `IndexWriter.add_document` and
+    /// `IndexWriter.delete_term`), the changes made by calling `run` will be
+    /// visible to readers only after calling `commit()`.
+    pub fn run(&mut self, user_operations: Vec<UserOperation>) -> u64 {
+        let count = user_operations.len() as u64;
+        if count == 0 {
+            return self.stamper.stamp();
+        }
+        let (batch_opstamp, stamps) = self.get_batch_opstamps(count);
+
+        let mut adds: Vec<AddOperation> = Vec::new();
+
+        for (user_op, opstamp) in user_operations.into_iter().zip(stamps) {
+            match user_op {
+                UserOperation::Delete(term) => {
+                    let delete_operation = DeleteOperation {
+                        opstamp: opstamp,
+                        term: term,
+                    };
+                    self.delete_queue.push(delete_operation);
+                }
+                UserOperation::Add(doc) => {
+                    let add_operation = AddOperation {
+                        opstamp: opstamp,
+                        document: doc,
+                    };
+                    adds.push(add_operation);
+                }
+            }
+        }
+        let send_result = self.document_sender.send(adds);
+        if let Err(e) = send_result {
+            panic!("Failed to index document. Sending to indexing channel failed. This probably means all of the indexing threads have panicked. {:?}", e);
+        };
+
+        batch_opstamp
+    }
 }
 
 #[cfg(test)]
 mod tests {
 
+    use super::super::operation::UserOperation;
     use super::initial_table_size;
+    use directory::error::LockError;
     use error::*;
     use indexer::NoMergePolicy;
-    use schema::{self, Document};
+    use schema::{self, Document, IndexRecordOption};
+    use query::{TermQuery};
+    use collector::TopDocs;
     use Index;
     use Term;
 
+    #[test]
+    fn test_operations_group() {
+        // an operations group with 2 items should cause 3 opstamps 0, 1, and 2.
+        let mut schema_builder = schema::Schema::builder();
+        let text_field = schema_builder.add_text_field("text", schema::TEXT);
+        let index = Index::create_in_ram(schema_builder.build());
+        let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
+        let operations = vec![
+            UserOperation::Add(doc!(text_field=>"a")),
+            UserOperation::Add(doc!(text_field=>"b")),
+        ];
+        let batch_opstamp1 = index_writer.run(operations);
+        assert_eq!(batch_opstamp1, 2u64);
+    }
+
+    #[test]
+    fn test_ordered_batched_operations() {
+        // * one delete for `doc!(field=>"a")`
+        // * one add for `doc!(field=>"a")`
+        // * one add for `doc!(field=>"b")`
+        // * one delete for `doc!(field=>"b")`
+        // after commit there is one doc with "a" and 0 doc with "b"
+        let mut schema_builder = schema::Schema::builder();
+        let text_field = schema_builder.add_text_field("text", schema::TEXT);
+        let index = Index::create_in_ram(schema_builder.build());
+        let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
+        let a_term = Term::from_field_text(text_field, "a");
+        let b_term = Term::from_field_text(text_field, "b");
+        let operations = vec![
+            UserOperation::Delete(a_term),
+            UserOperation::Add(doc!(text_field=>"a")),
+            UserOperation::Add(doc!(text_field=>"b")),
+            UserOperation::Delete(b_term),
+        ];
+
+        index_writer.run(operations);
+        index_writer.commit().expect("failed to commit");
+        index.load_searchers().expect("failed to load searchers");
+        
+        let a_term = Term::from_field_text(text_field, "a");
+        let b_term = Term::from_field_text(text_field, "b");
+        
+        let a_query = TermQuery::new(a_term, IndexRecordOption::Basic);
+        let b_query = TermQuery::new(b_term, IndexRecordOption::Basic);
+
+        let searcher = index.searcher();
+
+        let a_docs = searcher
+            .search(&a_query, &TopDocs::with_limit(1))
+            .expect("search for a failed");
+
+        let b_docs = searcher
+            .search(&b_query, &TopDocs::with_limit(1))
+            .expect("search for b failed");
+
+        assert_eq!(a_docs.len(), 1);
+        assert_eq!(b_docs.len(), 0);
+    }
+
+    #[test]
+    fn test_empty_operations_group() {
+        let schema_builder = schema::Schema::builder();
+        let index = Index::create_in_ram(schema_builder.build());
+        let mut index_writer = index.writer(3_000_000).unwrap();
+        let operations1 = vec![];
+        let batch_opstamp1 = index_writer.run(operations1);
+        assert_eq!(batch_opstamp1, 0u64);
+        let operations2 = vec![];
+        let batch_opstamp2 = index_writer.run(operations2);
+        assert_eq!(batch_opstamp2, 1u64);
+    }
+
     #[test]
     fn test_lockfile_stops_duplicates() {
         let schema_builder = schema::Schema::builder();
         let index = Index::create_in_ram(schema_builder.build());
-        let _index_writer = index.writer(40_000_000).unwrap();
-        match index.writer(40_000_000) {
-            Err(TantivyError::LockFailure(_)) => {}
-            _ => panic!("Expected FileAlreadyExists error"),
+        let _index_writer = index.writer(3_000_000).unwrap();
+        match index.writer(3_000_000) {
+            Err(TantivyError::LockFailure(LockError::LockBusy, _)) => {}
+            _ => panic!("Expected a `LockFailure` error"),
         }
     }
 
@@ -678,8 +829,7 @@ mod tests {
         match index.writer_with_num_threads(1, 3_000_000) {
             Err(err) => {
                 let err_msg = err.to_string();
-                assert!(err_msg.contains("Lockfile"));
-                assert!(err_msg.contains("Possible causes:"))
+                assert!(err_msg.contains("already an `IndexWriter`"));
             }
             _ => panic!("Expected LockfileAlreadyExists error"),
         }
@@ -689,7 +839,7 @@ mod tests {
     fn test_set_merge_policy() {
         let schema_builder = schema::Schema::builder();
         let index = Index::create_in_ram(schema_builder.build());
-        let index_writer = index.writer(40_000_000).unwrap();
+        let index_writer = index.writer(3_000_000).unwrap();
         assert_eq!(
             format!("{:?}", index_writer.get_merge_policy()),
             "LogMergePolicy { min_merge_size: 8, min_layer_size: 10000, \
@@ -708,11 +858,11 @@ mod tests {
         let schema_builder = schema::Schema::builder();
         let index = Index::create_in_ram(schema_builder.build());
         {
-            let _index_writer = index.writer(40_000_000).unwrap();
+            let _index_writer = index.writer(3_000_000).unwrap();
             // the lock should be released when the
             // index_writer leaves the scope.
         }
-        let _index_writer_two = index.writer(40_000_000).unwrap();
+        let _index_writer_two = index.writer(3_000_000).unwrap();
     }
 
     #[test]
@@ -739,7 +889,7 @@ mod tests {
                 index_writer.add_document(doc!(text_field=>"b"));
                 index_writer.add_document(doc!(text_field=>"c"));
             }
-            assert_eq!(index_writer.commit().unwrap(), 2u64);
+            assert!(index_writer.commit().is_ok());
             index.load_searchers().unwrap();
             assert_eq!(num_docs_containing("a"), 0);
             assert_eq!(num_docs_containing("b"), 1);
@@ -802,7 +952,6 @@ mod tests {
             {
                 let mut prepared_commit = index_writer.prepare_commit().expect("commit failed");
                 prepared_commit.set_payload("first commit");
-                assert_eq!(prepared_commit.opstamp(), 100);
                 prepared_commit.commit().expect("commit failed");
             }
             {
@@ -836,7 +985,6 @@ mod tests {
             {
                 let mut prepared_commit = index_writer.prepare_commit().expect("commit failed");
                 prepared_commit.set_payload("first commit");
-                assert_eq!(prepared_commit.opstamp(), 100);
                 prepared_commit.abort().expect("commit failed");
             }
             {

src/indexer/merge_operation.rs

@@ -0,0 +1,64 @@
+use census::{Inventory, TrackedObject};
+use std::collections::HashSet;
+use SegmentId;
+
+#[derive(Default)]
+pub struct MergeOperationInventory(Inventory<InnerMergeOperation>);
+
+impl MergeOperationInventory {
+    pub fn segment_in_merge(&self) -> HashSet<SegmentId> {
+        let mut segment_in_merge = HashSet::default();
+        for merge_op in self.0.list() {
+            for &segment_id in &merge_op.segment_ids {
+                segment_in_merge.insert(segment_id);
+            }
+        }
+        segment_in_merge
+    }
+}
+
+/// A `MergeOperation` has two role.
+/// It carries all of the information required to describe a merge :
+/// - `target_opstamp` is the opstamp up to which we want to consume the
+/// delete queue and reflect their deletes.
+/// - `segment_ids` is the list of segment to be merged.
+///
+/// The second role is to ensure keep track of the fact that these
+/// segments are in merge and avoid starting a merge operation that
+/// may conflict with this one.
+///
+/// This works by tracking merge operations. When considering computing
+/// merge candidates, we simply list tracked merge operations and remove
+/// their segments from possible merge candidates.
+pub struct MergeOperation {
+    inner: TrackedObject<InnerMergeOperation>,
+}
+
+struct InnerMergeOperation {
+    target_opstamp: u64,
+    segment_ids: Vec<SegmentId>,
+}
+
+impl MergeOperation {
+    pub fn new(
+        inventory: &MergeOperationInventory,
+        target_opstamp: u64,
+        segment_ids: Vec<SegmentId>,
+    ) -> MergeOperation {
+        let inner_merge_operation = InnerMergeOperation {
+            target_opstamp,
+            segment_ids,
+        };
+        MergeOperation {
+            inner: inventory.0.track(inner_merge_operation),
+        }
+    }
+
+    pub fn target_opstamp(&self) -> u64 {
+        self.inner.target_opstamp
+    }
+
+    pub fn segment_ids(&self) -> &[SegmentId] {
+        &self.inner.segment_ids[..]
+    }
+}

src/indexer/merge_policy.rs

@@ -11,7 +11,7 @@ pub struct MergeCandidate(pub Vec<SegmentId>);
 ///
 /// Every time a the list of segments changes, the segment updater
 /// asks the merge policy if some segments should be merged.
-pub trait MergePolicy: MergePolicyClone + marker::Send + marker::Sync + Debug {
+pub trait MergePolicy: marker::Send + marker::Sync + Debug {
     /// Given the list of segment metas, returns the list of merge candidates.
     ///
     /// This call happens on the segment updater thread, and will block
@@ -19,21 +19,6 @@ pub trait MergePolicy: MergePolicyClone + marker::Send + marker::Sync + Debug {
     fn compute_merge_candidates(&self, segments: &[SegmentMeta]) -> Vec<MergeCandidate>;
 }
 
-/// MergePolicyClone
-pub trait MergePolicyClone {
-    /// Returns a boxed clone of the MergePolicy.
-    fn box_clone(&self) -> Box<MergePolicy>;
-}
-
-impl<T> MergePolicyClone for T
-where
-    T: 'static + MergePolicy + Clone,
-{
-    fn box_clone(&self) -> Box<MergePolicy> {
-        Box::new(self.clone())
-    }
-}
-
 /// Never merge segments.
 #[derive(Debug, Clone)]
 pub struct NoMergePolicy;

src/indexer/merger.rs

@@ -40,13 +40,15 @@ fn compute_total_num_tokens(readers: &[SegmentReader], field: Field) -> u64 {
             total_tokens += reader.inverted_index(field).total_num_tokens();
         }
     }
-    total_tokens + count
-        .iter()
-        .cloned()
-        .enumerate()
-        .map(|(fieldnorm_ord, count)| {
-            count as u64 * u64::from(FieldNormReader::id_to_fieldnorm(fieldnorm_ord as u8))
-        }).sum::<u64>()
+    total_tokens
+        + count
+            .iter()
+            .cloned()
+            .enumerate()
+            .map(|(fieldnorm_ord, count)| {
+                count as u64 * u64::from(FieldNormReader::id_to_fieldnorm(fieldnorm_ord as u8))
+            })
+            .sum::<u64>()
 }
 
 pub struct IndexMerger {
@@ -523,7 +525,8 @@ impl IndexMerger {
                         }
                     }
                     None
-                }).collect();
+                })
+                .collect();
 
             // At this point, `segment_postings` contains the posting list
             // of all of the segments containing the given term.
@@ -651,6 +654,7 @@ mod tests {
     use schema::IntOptions;
     use schema::Term;
     use schema::TextFieldIndexing;
+    use schema::INT_INDEXED;
     use std::io::Cursor;
     use DocAddress;
     use IndexWriter;
@@ -664,7 +668,8 @@ mod tests {
                 TextFieldIndexing::default()
                     .set_tokenizer("default")
                     .set_index_option(IndexRecordOption::WithFreqs),
-            ).set_stored();
+            )
+            .set_stored();
         let text_field = schema_builder.add_text_field("text", text_fieldtype);
         let score_fieldtype = schema::IntOptions::default().set_fast(Cardinality::SingleValue);
         let score_field = schema_builder.add_u64_field("score", score_fieldtype);
@@ -803,7 +808,8 @@ mod tests {
                         .search(
                             &query,
                             &BytesFastFieldTestCollector::for_field(bytes_score_field),
-                        ).expect("failed to search")
+                        )
+                        .expect("failed to search")
                 };
                 assert_eq!(
                     get_fast_vals(vec![Term::from_field_text(text_field, "a")]),
@@ -823,13 +829,14 @@ mod tests {
         let text_fieldtype = schema::TextOptions::default()
             .set_indexing_options(
                 TextFieldIndexing::default().set_index_option(IndexRecordOption::WithFreqs),
-            ).set_stored();
+            )
+            .set_stored();
         let text_field = schema_builder.add_text_field("text", text_fieldtype);
         let score_fieldtype = schema::IntOptions::default().set_fast(Cardinality::SingleValue);
         let score_field = schema_builder.add_u64_field("score", score_fieldtype);
         let bytes_score_field = schema_builder.add_bytes_field("score_bytes");
         let index = Index::create_in_ram(schema_builder.build());
-        let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
+        let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
 
         let search_term = |searcher: &Searcher, term: Term| {
             let collector = FastFieldTestCollector::for_field(score_field);
@@ -851,21 +858,21 @@ mod tests {
         {
             // a first commit
             index_writer.add_document(doc!(
-                    text_field => "a b d",
-                    score_field => 1u64,
-                    bytes_score_field => vec![0u8, 0, 0, 1],
-                ));
+                text_field => "a b d",
+                score_field => 1u64,
+                bytes_score_field => vec![0u8, 0, 0, 1],
+            ));
             index_writer.add_document(doc!(
-                    text_field => "b c",
-                    score_field => 2u64,
-                    bytes_score_field => vec![0u8, 0, 0, 2],
-                ));
+                text_field => "b c",
+                score_field => 2u64,
+                bytes_score_field => vec![0u8, 0, 0, 2],
+            ));
             index_writer.delete_term(Term::from_field_text(text_field, "c"));
             index_writer.add_document(doc!(
-                    text_field => "c d",
-                    score_field => 3u64,
-                    bytes_score_field => vec![0u8, 0, 0, 3],
-                ));
+                text_field => "c d",
+                score_field => 3u64,
+                bytes_score_field => vec![0u8, 0, 0, 3],
+            ));
             index_writer.commit().expect("committed");
             index.load_searchers().unwrap();
             let ref searcher = *index.searcher();
@@ -892,27 +899,27 @@ mod tests {
         {
             // a second commit
             index_writer.add_document(doc!(
-                    text_field => "a d e",
-                    score_field => 4_000u64,
-                    bytes_score_field => vec![0u8, 0, 0, 4],
-                ));
+                text_field => "a d e",
+                score_field => 4_000u64,
+                bytes_score_field => vec![0u8, 0, 0, 4],
+            ));
             index_writer.add_document(doc!(
-                    text_field => "e f",
-                    score_field => 5_000u64,
-                    bytes_score_field => vec![0u8, 0, 0, 5],
-                ));
+                text_field => "e f",
+                score_field => 5_000u64,
+                bytes_score_field => vec![0u8, 0, 0, 5],
+            ));
             index_writer.delete_term(Term::from_field_text(text_field, "a"));
             index_writer.delete_term(Term::from_field_text(text_field, "f"));
             index_writer.add_document(doc!(
-                    text_field => "f g",
-                    score_field => 6_000u64,
-                    bytes_score_field => vec![0u8, 0, 23, 112],
-                ));
+                text_field => "f g",
+                score_field => 6_000u64,
+                bytes_score_field => vec![0u8, 0, 23, 112],
+            ));
             index_writer.add_document(doc!(
-                    text_field => "g h",
-                    score_field => 7_000u64,
-                    bytes_score_field => vec![0u8, 0, 27, 88],
-                ));
+                text_field => "g h",
+                score_field => 7_000u64,
+                bytes_score_field => vec![0u8, 0, 27, 88],
+            ));
             index_writer.commit().expect("committed");
             index.load_searchers().unwrap();
             let searcher = index.searcher();
@@ -977,7 +984,7 @@ mod tests {
                 .wait()
                 .expect("Merging failed");
             index.load_searchers().unwrap();
-            let ref searcher = *index.searcher();
+            let searcher = index.searcher();
             assert_eq!(searcher.segment_readers().len(), 1);
             assert_eq!(searcher.num_docs(), 3);
             assert_eq!(searcher.segment_readers()[0].num_docs(), 3);
@@ -1023,7 +1030,7 @@ mod tests {
             index_writer.commit().unwrap();
 
             index.load_searchers().unwrap();
-            let ref searcher = *index.searcher();
+            let searcher = index.searcher();
             assert_eq!(searcher.segment_readers().len(), 1);
             assert_eq!(searcher.num_docs(), 2);
             assert_eq!(searcher.segment_readers()[0].num_docs(), 2);
@@ -1119,18 +1126,15 @@ mod tests {
         {
             // Test removing all docs
             index_writer.delete_term(Term::from_field_text(text_field, "g"));
+            index_writer.commit().unwrap();
             let segment_ids = index
                 .searchable_segment_ids()
                 .expect("Searchable segments failed.");
-            index_writer
-                .merge(&segment_ids)
-                .expect("Failed to initiate merge")
-                .wait()
-                .expect("Merging failed");
             index.load_searchers().unwrap();
 
             let ref searcher = *index.searcher();
-            assert_eq!(searcher.segment_readers().len(), 1);
+            assert!(segment_ids.is_empty());
+            assert!(searcher.segment_readers().is_empty());
             assert_eq!(searcher.num_docs(), 0);
         }
     }
@@ -1141,7 +1145,7 @@ mod tests {
         let facet_field = schema_builder.add_facet_field("facet");
         let index = Index::create_in_ram(schema_builder.build());
         {
-            let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
+            let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
             let index_doc = |index_writer: &mut IndexWriter, doc_facets: &[&str]| {
                 let mut doc = Document::default();
                 for facet in doc_facets {
@@ -1206,7 +1210,7 @@ mod tests {
             let segment_ids = index
                 .searchable_segment_ids()
                 .expect("Searchable segments failed.");
-            let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
+            let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
             index_writer
                 .merge(&segment_ids)
                 .expect("Failed to initiate merge")
@@ -1229,7 +1233,7 @@ mod tests {
 
         // Deleting one term
         {
-            let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
+            let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
             let facet = Facet::from_path(vec!["top", "a", "firstdoc"]);
             let facet_term = Term::from_facet(facet_field, &facet);
             index_writer.delete_term(facet_term);
@@ -1249,6 +1253,34 @@ mod tests {
         }
     }
 
+    #[test]
+    fn test_bug_merge() {
+        let mut schema_builder = schema::Schema::builder();
+        let int_field = schema_builder.add_u64_field("intvals", INT_INDEXED);
+        let index = Index::create_in_ram(schema_builder.build());
+        let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
+        index_writer.add_document(doc!(int_field => 1u64));
+        index_writer.commit().expect("commit failed");
+        index_writer.add_document(doc!(int_field => 1u64));
+        index_writer.commit().expect("commit failed");
+        index.load_searchers().unwrap();
+        let searcher = index.searcher();
+        assert_eq!(searcher.num_docs(), 2);
+        index_writer.delete_term(Term::from_field_u64(int_field, 1));
+        let segment_ids = index
+            .searchable_segment_ids()
+            .expect("Searchable segments failed.");
+        index_writer
+            .merge(&segment_ids)
+            .expect("Failed to initiate merge")
+            .wait()
+            .expect("Merging failed");
+        index.load_searchers().unwrap();
+        // commit has not been called yet. The document should still be
+        // there.
+        assert_eq!(index.searcher().num_docs(), 2);
+    }
+
     #[test]
     fn test_merge_multivalued_int_fields_all_deleted() {
         let mut schema_builder = schema::Schema::builder();
@@ -1259,7 +1291,7 @@ mod tests {
         let index = Index::create_in_ram(schema_builder.build());
 
         {
-            let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
+            let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
             let mut doc = Document::default();
             doc.add_u64(int_field, 1);
             index_writer.add_document(doc.clone());
@@ -1267,24 +1299,26 @@ mod tests {
             index_writer.add_document(doc);
             index_writer.commit().expect("commit failed");
             index_writer.delete_term(Term::from_field_u64(int_field, 1));
-            index_writer.commit().expect("commit failed");
-        }
-        index.load_searchers().unwrap();
-        let searcher = index.searcher();
-        assert_eq!(searcher.num_docs(), 0);
-        // Merging the segments
-        {
+
             let segment_ids = index
                 .searchable_segment_ids()
                 .expect("Searchable segments failed.");
-            let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
             index_writer
                 .merge(&segment_ids)
                 .expect("Failed to initiate merge")
                 .wait()
                 .expect("Merging failed");
+
+            // assert delete has not been committed
+            index.load_searchers().unwrap();
+            let searcher = index.searcher();
+            assert_eq!(searcher.num_docs(), 2);
+
+            index_writer.commit().unwrap();
+
             index_writer.wait_merging_threads().unwrap();
         }
+
         index.load_searchers().unwrap();
         let searcher = index.searcher();
         assert_eq!(searcher.num_docs(), 0);
@@ -1300,7 +1334,7 @@ mod tests {
         let index = Index::create_in_ram(schema_builder.build());
 
         {
-            let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
+            let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
             let index_doc = |index_writer: &mut IndexWriter, int_vals: &[u64]| {
                 let mut doc = Document::default();
                 for &val in int_vals {
@@ -1389,7 +1423,7 @@ mod tests {
             let segment_ids = index
                 .searchable_segment_ids()
                 .expect("Searchable segments failed.");
-            let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
+            let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
             index_writer
                 .merge(&segment_ids)
                 .expect("Failed to initiate merge")

src/indexer/mod.rs

@@ -1,8 +1,9 @@
 pub mod delete_queue;
-mod directory_lock;
+
 mod doc_opstamp_mapping;
 pub mod index_writer;
 mod log_merge_policy;
+mod merge_operation;
 pub mod merge_policy;
 pub mod merger;
 pub mod operation;
@@ -15,14 +16,12 @@ pub mod segment_updater;
 mod segment_writer;
 mod stamper;
 
-pub(crate) use self::directory_lock::DirectoryLock;
-pub use self::directory_lock::LockType;
-
 pub use self::index_writer::IndexWriter;
 pub use self::log_merge_policy::LogMergePolicy;
+pub use self::merge_operation::{MergeOperation, MergeOperationInventory};
 pub use self::merge_policy::{MergeCandidate, MergePolicy, NoMergePolicy};
 pub use self::prepared_commit::PreparedCommit;
-pub use self::segment_entry::{SegmentEntry, SegmentState};
+pub use self::segment_entry::SegmentEntry;
 pub use self::segment_manager::SegmentManager;
 pub use self::segment_serializer::SegmentSerializer;
 pub use self::segment_writer::SegmentWriter;

src/indexer/operation.rs

@@ -14,3 +14,10 @@ pub struct AddOperation {
     pub opstamp: u64,
     pub document: Document,
 }
+
+/// UserOperation is an enum type that encapsulates other operation types.
+#[derive(Eq, PartialEq, Debug)]
+pub enum UserOperation {
+    Add(Document),
+    Delete(Term),
+}

src/indexer/segment_entry.rs

@@ -4,21 +4,6 @@ use core::SegmentMeta;
 use indexer::delete_queue::DeleteCursor;
 use std::fmt;
 
-#[derive(Clone, Copy, PartialEq, Eq, Debug)]
-pub enum SegmentState {
-    Ready,
-    InMerge,
-}
-
-impl SegmentState {
-    pub fn letter_code(self) -> char {
-        match self {
-            SegmentState::InMerge => 'M',
-            SegmentState::Ready => 'R',
-        }
-    }
-}
-
 /// A segment entry describes the state of
 /// a given segment, at a given instant.
 ///
@@ -35,7 +20,6 @@ impl SegmentState {
 #[derive(Clone)]
 pub struct SegmentEntry {
     meta: SegmentMeta,
-    state: SegmentState,
     delete_bitset: Option<BitSet>,
     delete_cursor: DeleteCursor,
 }
@@ -49,7 +33,6 @@ impl SegmentEntry {
     ) -> SegmentEntry {
         SegmentEntry {
             meta: segment_meta,
-            state: SegmentState::Ready,
             delete_bitset,
             delete_cursor,
         }
@@ -72,14 +55,6 @@ impl SegmentEntry {
         &mut self.delete_cursor
     }
 
-    /// Return the `SegmentEntry`.
-    ///
-    /// The state describes whether the segment is available for
-    /// a merge or not.
-    pub fn state(&self) -> SegmentState {
-        self.state
-    }
-
     /// Returns the segment id.
     pub fn segment_id(&self) -> SegmentId {
         self.meta.id()
@@ -89,33 +64,10 @@ impl SegmentEntry {
     pub fn meta(&self) -> &SegmentMeta {
         &self.meta
     }
-
-    /// Mark the `SegmentEntry` as in merge.
-    ///
-    /// Only segments that are not already
-    /// in a merge are elligible for future merge.
-    pub fn start_merge(&mut self) {
-        self.state = SegmentState::InMerge;
-    }
-
-    /// Cancel a merge
-    ///
-    /// If a merge fails, it is important to switch
-    /// the segment back to a idle state, so that it
-    /// may be elligible for future merges.
-    pub fn cancel_merge(&mut self) {
-        self.state = SegmentState::Ready;
-    }
-
-    /// Returns true iff a segment should
-    /// be considered for a merge.
-    pub fn is_ready(&self) -> bool {
-        self.state == SegmentState::Ready
-    }
 }
 
 impl fmt::Debug for SegmentEntry {
     fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
-        write!(formatter, "SegmentEntry({:?}, {:?})", self.meta, self.state)
+        write!(formatter, "SegmentEntry({:?})", self.meta)
     }
 }

src/indexer/segment_manager.rs

@@ -16,7 +16,6 @@ use Result as TantivyResult;
 struct SegmentRegisters {
     uncommitted: SegmentRegister,
     committed: SegmentRegister,
-    writing: HashSet<SegmentId>,
 }
 
 /// The segment manager stores the list of segments
@@ -41,12 +40,17 @@ impl Debug for SegmentManager {
 }
 
 pub fn get_mergeable_segments(
+    in_merge_segment_ids: &HashSet<SegmentId>,
     segment_manager: &SegmentManager,
 ) -> (Vec<SegmentMeta>, Vec<SegmentMeta>) {
     let registers_lock = segment_manager.read();
     (
-        registers_lock.committed.get_mergeable_segments(),
-        registers_lock.uncommitted.get_mergeable_segments(),
+        registers_lock
+            .committed
+            .get_mergeable_segments(in_merge_segment_ids),
+        registers_lock
+            .uncommitted
+            .get_mergeable_segments(in_merge_segment_ids),
     )
 }
 
@@ -59,7 +63,6 @@ impl SegmentManager {
             registers: RwLock::new(SegmentRegisters {
                 uncommitted: SegmentRegister::default(),
                 committed: SegmentRegister::new(segment_metas, delete_cursor),
-                writing: HashSet::new(),
             }),
         }
     }
@@ -72,12 +75,6 @@ impl SegmentManager {
         segment_entries
     }
 
-    /// Returns the overall number of segments in the `SegmentManager`
-    pub fn num_segments(&self) -> usize {
-        let registers_lock = self.read();
-        registers_lock.committed.len() + registers_lock.uncommitted.len()
-    }
-
     /// List the files that are useful to the index.
     ///
     /// This does not include lock files, or files that are obsolete
@@ -106,6 +103,21 @@ impl SegmentManager {
             .expect("Failed to acquire write lock on SegmentManager.")
     }
 
+    /// Deletes all empty segments
+    fn remove_empty_segments(&self) {
+        let mut registers_lock = self.write();
+        registers_lock
+            .committed
+            .segment_entries()
+            .iter()
+            .filter(|segment| segment.meta().num_docs() == 0)
+            .for_each(|segment| {
+                registers_lock
+                    .committed
+                    .remove_segment(&segment.segment_id())
+            });
+    }
+
     pub fn commit(&self, segment_entries: Vec<SegmentEntry>) {
         let mut registers_lock = self.write();
         registers_lock.committed.clear();
@@ -121,25 +133,22 @@ impl SegmentManager {
     /// the `segment_ids` are not either all committed or all
     /// uncommitted.
     pub fn start_merge(&self, segment_ids: &[SegmentId]) -> TantivyResult<Vec<SegmentEntry>> {
-        let mut registers_lock = self.write();
+        let registers_lock = self.read();
         let mut segment_entries = vec![];
         if registers_lock.uncommitted.contains_all(segment_ids) {
             for segment_id in segment_ids {
                 let segment_entry = registers_lock.uncommitted
-                    .start_merge(segment_id)
+                    .get(segment_id)
                     .expect("Segment id not found {}. Should never happen because of the contains all if-block.");
                 segment_entries.push(segment_entry);
             }
         } else if registers_lock.committed.contains_all(segment_ids) {
             for segment_id in segment_ids {
                 let segment_entry = registers_lock.committed
-                    .start_merge(segment_id)
+                    .get(segment_id)
                     .expect("Segment id not found {}. Should never happen because of the contains all if-block.");
                 segment_entries.push(segment_entry);
             }
-            for segment_id in segment_ids {
-                registers_lock.committed.start_merge(segment_id);
-            }
         } else {
             let error_msg = "Merge operation sent for segments that are not \
                              all uncommited or commited."
@@ -149,50 +158,8 @@ impl SegmentManager {
         Ok(segment_entries)
     }
 
-    pub fn cancel_merge(
-        &self,
-        before_merge_segment_ids: &[SegmentId],
-        after_merge_segment_id: SegmentId,
-    ) {
-        let mut registers_lock = self.write();
-
-        // we mark all segments are ready for merge.
-        {
-            let target_segment_register: &mut SegmentRegister;
-            target_segment_register = {
-                if registers_lock
-                    .uncommitted
-                    .contains_all(before_merge_segment_ids)
-                {
-                    &mut registers_lock.uncommitted
-                } else if registers_lock
-                    .committed
-                    .contains_all(before_merge_segment_ids)
-                {
-                    &mut registers_lock.committed
-                } else {
-                    warn!("couldn't find segment in SegmentManager");
-                    return;
-                }
-            };
-            for segment_id in before_merge_segment_ids {
-                target_segment_register.cancel_merge(segment_id);
-            }
-        }
-
-        // ... and we make sure the target segment entry
-        // can be garbage collected.
-        registers_lock.writing.remove(&after_merge_segment_id);
-    }
-
-    pub fn write_segment(&self, segment_id: SegmentId) {
-        let mut registers_lock = self.write();
-        registers_lock.writing.insert(segment_id);
-    }
-
     pub fn add_segment(&self, segment_entry: SegmentEntry) {
         let mut registers_lock = self.write();
-        registers_lock.writing.remove(&segment_entry.segment_id());
         registers_lock.uncommitted.add_segment_entry(segment_entry);
     }
 
@@ -202,10 +169,6 @@ impl SegmentManager {
         after_merge_segment_entry: SegmentEntry,
     ) {
         let mut registers_lock = self.write();
-        registers_lock
-            .writing
-            .remove(&after_merge_segment_entry.segment_id());
-
         let target_register: &mut SegmentRegister = {
             if registers_lock
                 .uncommitted
@@ -229,6 +192,7 @@ impl SegmentManager {
     }
 
     pub fn committed_segment_metas(&self) -> Vec<SegmentMeta> {
+        self.remove_empty_segments();
         let registers_lock = self.read();
         registers_lock.committed.segment_metas()
     }

src/indexer/segment_register.rs

@@ -3,6 +3,7 @@ use core::SegmentMeta;
 use indexer::delete_queue::DeleteCursor;
 use indexer::segment_entry::SegmentEntry;
 use std::collections::HashMap;
+use std::collections::HashSet;
 use std::fmt::{self, Debug, Formatter};
 
 /// The segment register keeps track
@@ -21,8 +22,8 @@ pub struct SegmentRegister {
 impl Debug for SegmentRegister {
     fn fmt(&self, f: &mut Formatter) -> Result<(), fmt::Error> {
         write!(f, "SegmentRegister(")?;
-        for (k, v) in &self.segment_states {
-            write!(f, "{}:{}, ", k.short_uuid_string(), v.state().letter_code())?;
+        for k in self.segment_states.keys() {
+            write!(f, "{}, ", k.short_uuid_string())?;
         }
         write!(f, ")")?;
         Ok(())
@@ -34,14 +35,13 @@ impl SegmentRegister {
         self.segment_states.clear();
     }
 
-    pub fn len(&self) -> usize {
-        self.segment_states.len()
-    }
-
-    pub fn get_mergeable_segments(&self) -> Vec<SegmentMeta> {
+    pub fn get_mergeable_segments(
+        &self,
+        in_merge_segment_ids: &HashSet<SegmentId>,
+    ) -> Vec<SegmentMeta> {
         self.segment_states
             .values()
-            .filter(|segment_entry| segment_entry.is_ready())
+            .filter(|segment_entry| !in_merge_segment_ids.contains(&segment_entry.segment_id()))
             .map(|segment_entry| segment_entry.meta().clone())
             .collect()
     }
@@ -60,7 +60,7 @@ impl SegmentRegister {
         segment_ids
     }
 
-    pub fn contains_all(&mut self, segment_ids: &[SegmentId]) -> bool {
+    pub fn contains_all(&self, segment_ids: &[SegmentId]) -> bool {
         segment_ids
             .iter()
             .all(|segment_id| self.segment_states.contains_key(segment_id))
@@ -75,20 +75,8 @@ impl SegmentRegister {
         self.segment_states.remove(segment_id);
     }
 
-    pub fn cancel_merge(&mut self, segment_id: &SegmentId) {
-        self.segment_states
-            .get_mut(segment_id)
-            .expect("Received a merge notification for a segment that is not registered")
-            .cancel_merge();
-    }
-
-    pub fn start_merge(&mut self, segment_id: &SegmentId) -> Option<SegmentEntry> {
-        if let Some(segment_entry) = self.segment_states.get_mut(segment_id) {
-            segment_entry.start_merge();
-            Some(segment_entry.clone())
-        } else {
-            None
-        }
+    pub fn get(&self, segment_id: &SegmentId) -> Option<SegmentEntry> {
+        self.segment_states.get(segment_id).cloned()
     }
 
     pub fn new(segment_metas: Vec<SegmentMeta>, delete_cursor: &DeleteCursor) -> SegmentRegister {
@@ -100,11 +88,6 @@ impl SegmentRegister {
         }
         SegmentRegister { segment_states }
     }
-
-    #[cfg(test)]
-    pub fn segment_entry(&self, segment_id: &SegmentId) -> Option<SegmentEntry> {
-        self.segment_states.get(segment_id).cloned()
-    }
 }
 
 #[cfg(test)]
@@ -113,7 +96,6 @@ mod tests {
     use core::SegmentId;
     use core::SegmentMeta;
     use indexer::delete_queue::*;
-    use indexer::SegmentState;
 
     fn segment_ids(segment_register: &SegmentRegister) -> Vec<SegmentId> {
         segment_register
@@ -137,42 +119,12 @@ mod tests {
             let segment_entry = SegmentEntry::new(segment_meta, delete_queue.cursor(), None);
             segment_register.add_segment_entry(segment_entry);
         }
-        assert_eq!(
-            segment_register
-                .segment_entry(&segment_id_a)
-                .unwrap()
-                .state(),
-            SegmentState::Ready
-        );
         assert_eq!(segment_ids(&segment_register), vec![segment_id_a]);
         {
             let segment_meta = SegmentMeta::new(segment_id_b, 0u32);
             let segment_entry = SegmentEntry::new(segment_meta, delete_queue.cursor(), None);
             segment_register.add_segment_entry(segment_entry);
         }
-        assert_eq!(
-            segment_register
-                .segment_entry(&segment_id_b)
-                .unwrap()
-                .state(),
-            SegmentState::Ready
-        );
-        segment_register.start_merge(&segment_id_a);
-        segment_register.start_merge(&segment_id_b);
-        assert_eq!(
-            segment_register
-                .segment_entry(&segment_id_a)
-                .unwrap()
-                .state(),
-            SegmentState::InMerge
-        );
-        assert_eq!(
-            segment_register
-                .segment_entry(&segment_id_b)
-                .unwrap()
-                .state(),
-            SegmentState::InMerge
-        );
         segment_register.remove_segment(&segment_id_a);
         segment_register.remove_segment(&segment_id_b);
         {

src/indexer/segment_updater.rs

@@ -16,9 +16,10 @@ use futures_cpupool::CpuFuture;
 use futures_cpupool::CpuPool;
 use indexer::delete_queue::DeleteCursor;
 use indexer::index_writer::advance_deletes;
+use indexer::merge_operation::MergeOperationInventory;
 use indexer::merger::IndexMerger;
 use indexer::stamper::Stamper;
-use indexer::MergeCandidate;
+use indexer::MergeOperation;
 use indexer::SegmentEntry;
 use indexer::SegmentSerializer;
 use indexer::{DefaultMergePolicy, MergePolicy};
@@ -26,6 +27,7 @@ use schema::Schema;
 use serde_json;
 use std::borrow::BorrowMut;
 use std::collections::HashMap;
+use std::collections::HashSet;
 use std::io::Write;
 use std::mem;
 use std::ops::DerefMut;
@@ -45,8 +47,16 @@ use Result;
 /// and flushed.
 ///
 /// This method is not part of tantivy's public API
-pub fn save_new_metas(schema: Schema, opstamp: u64, directory: &mut Directory) -> Result<()> {
-    save_metas(vec![], schema, opstamp, None, directory)
+pub fn save_new_metas(schema: Schema, directory: &mut Directory) -> Result<()> {
+    save_metas(
+        &IndexMeta {
+            segments: Vec::new(),
+            schema,
+            opstamp: 0u64,
+            payload: None,
+        },
+        directory,
+    )
 }
 
 /// Save the index meta file.
@@ -58,20 +68,9 @@ pub fn save_new_metas(schema: Schema, opstamp: u64, directory: &mut Directory) -
 /// and flushed.
 ///
 /// This method is not part of tantivy's public API
-pub fn save_metas(
-    segment_metas: Vec<SegmentMeta>,
-    schema: Schema,
-    opstamp: u64,
-    payload: Option<String>,
-    directory: &mut Directory,
-) -> Result<()> {
-    let metas = IndexMeta {
-        segments: segment_metas,
-        schema,
-        opstamp,
-        payload,
-    };
-    let mut buffer = serde_json::to_vec_pretty(&metas)?;
+fn save_metas(metas: &IndexMeta, directory: &mut Directory) -> Result<()> {
+    let mut buffer = serde_json::to_vec_pretty(metas)?;
+    // Just adding a new line at the end of the buffer.
     writeln!(&mut buffer)?;
     directory.atomic_write(&META_FILEPATH, &buffer[..])?;
     debug!("Saved metas {:?}", serde_json::to_string_pretty(&metas));
@@ -83,16 +82,21 @@ pub fn save_metas(
 //
 // All this processing happens on a single thread
 // consuming a common queue.
+//
+// We voluntarily pass a merge_operation ref to guarantee that
+// the merge_operation is alive during the process
 #[derive(Clone)]
 pub struct SegmentUpdater(Arc<InnerSegmentUpdater>);
 
 fn perform_merge(
+    merge_operation: &MergeOperation,
     index: &Index,
     mut segment_entries: Vec<SegmentEntry>,
-    mut merged_segment: Segment,
-    target_opstamp: u64,
 ) -> Result<SegmentEntry> {
+    let target_opstamp = merge_operation.target_opstamp();
+
     // first we need to apply deletes to our segment.
+    let mut merged_segment = index.new_segment();
 
     // TODO add logging
     let schema = index.schema();
@@ -126,19 +130,27 @@ fn perform_merge(
 }
 
 struct InnerSegmentUpdater {
+    // we keep a copy of the current active IndexMeta to
+    // avoid loading the file everytime we need it in the
+    // `SegmentUpdater`.
+    //
+    // This should be up to date as all update happen through
+    // the unique active `SegmentUpdater`.
+    active_metas: RwLock<Arc<IndexMeta>>,
     pool: CpuPool,
     index: Index,
     segment_manager: SegmentManager,
-    merge_policy: RwLock<Box<MergePolicy>>,
+    merge_policy: RwLock<Arc<Box<MergePolicy>>>,
     merging_thread_id: AtomicUsize,
     merging_threads: RwLock<HashMap<usize, JoinHandle<Result<()>>>>,
     generation: AtomicUsize,
     killed: AtomicBool,
     stamper: Stamper,
+    merge_operations: MergeOperationInventory,
 }
 
 impl SegmentUpdater {
-    pub fn new(
+    pub fn create(
         index: Index,
         stamper: Stamper,
         delete_cursor: &DeleteCursor,
@@ -149,32 +161,29 @@ impl SegmentUpdater {
             .name_prefix("segment_updater")
             .pool_size(1)
             .create();
+        let index_meta = index.load_metas()?;
         Ok(SegmentUpdater(Arc::new(InnerSegmentUpdater {
+            active_metas: RwLock::new(Arc::new(index_meta)),
             pool,
             index,
             segment_manager,
-            merge_policy: RwLock::new(Box::new(DefaultMergePolicy::default())),
+            merge_policy: RwLock::new(Arc::new(Box::new(DefaultMergePolicy::default()))),
             merging_thread_id: AtomicUsize::default(),
             merging_threads: RwLock::new(HashMap::new()),
             generation: AtomicUsize::default(),
             killed: AtomicBool::new(false),
             stamper,
+            merge_operations: Default::default(),
         })))
     }
 
-    pub fn new_segment(&self) -> Segment {
-        let new_segment = self.0.index.new_segment();
-        let segment_id = new_segment.id();
-        self.0.segment_manager.write_segment(segment_id);
-        new_segment
-    }
-
-    pub fn get_merge_policy(&self) -> Box<MergePolicy> {
-        self.0.merge_policy.read().unwrap().box_clone()
+    pub fn get_merge_policy(&self) -> Arc<Box<MergePolicy>> {
+        self.0.merge_policy.read().unwrap().clone()
     }
 
     pub fn set_merge_policy(&self, merge_policy: Box<MergePolicy>) {
-        *self.0.merge_policy.write().unwrap() = merge_policy;
+        let arc_merge_policy = Arc::new(merge_policy);
+        *self.0.merge_policy.write().unwrap() = arc_merge_policy;
     }
 
     fn get_merging_thread_id(&self) -> usize {
@@ -195,7 +204,8 @@ impl SegmentUpdater {
                 segment_updater.0.segment_manager.add_segment(segment_entry);
                 segment_updater.consider_merge_options();
                 true
-            }).forget();
+            })
+            .forget();
             true
         } else {
             false
@@ -243,20 +253,23 @@ impl SegmentUpdater {
             //
             // Segment 1 from disk 1, Segment 1 from disk 2, etc.
             commited_segment_metas.sort_by_key(|segment_meta| -(segment_meta.max_doc() as i32));
-            save_metas(
-                commited_segment_metas,
-                index.schema(),
+            let index_meta = IndexMeta {
+                segments: commited_segment_metas,
+                schema: index.schema(),
                 opstamp,
-                commit_message,
-                directory.box_clone().borrow_mut(),
-            ).expect("Could not save metas.");
+                payload: commit_message,
+            };
+            save_metas(&index_meta, directory.box_clone().borrow_mut())
+                .expect("Could not save metas.");
+            self.store_meta(&index_meta);
         }
     }
 
     pub fn garbage_collect_files(&self) -> Result<()> {
         self.run_async(move |segment_updater| {
             segment_updater.garbage_collect_files_exec();
-        }).wait()
+        })
+        .wait()
     }
 
     fn garbage_collect_files_exec(&self) {
@@ -278,54 +291,67 @@ impl SegmentUpdater {
                 segment_updater.garbage_collect_files_exec();
                 segment_updater.consider_merge_options();
             }
-        }).wait()
+        })
+        .wait()
     }
 
     pub fn start_merge(&self, segment_ids: &[SegmentId]) -> Result<Receiver<SegmentMeta>> {
-        //let future_merged_segment = */
-        let segment_ids_vec = segment_ids.to_vec();
-        self.run_async(move |segment_updater| {
-            segment_updater.start_merge_impl(&segment_ids_vec[..])
-        }).wait()?
+        let commit_opstamp = self.load_metas().opstamp;
+        let merge_operation = MergeOperation::new(
+            &self.0.merge_operations,
+            commit_opstamp,
+            segment_ids.to_vec(),
+        );
+        self.run_async(move |segment_updater| segment_updater.start_merge_impl(merge_operation))
+            .wait()?
+    }
+
+    fn store_meta(&self, index_meta: &IndexMeta) {
+        *self.0.active_metas.write().unwrap() = Arc::new(index_meta.clone());
+    }
+    fn load_metas(&self) -> Arc<IndexMeta> {
+        self.0.active_metas.read().unwrap().clone()
     }
 
     // `segment_ids` is required to be non-empty.
-    fn start_merge_impl(&self, segment_ids: &[SegmentId]) -> Result<Receiver<SegmentMeta>> {
-        assert!(!segment_ids.is_empty(), "Segment_ids cannot be empty.");
+    fn start_merge_impl(&self, merge_operation: MergeOperation) -> Result<Receiver<SegmentMeta>> {
+        assert!(
+            !merge_operation.segment_ids().is_empty(),
+            "Segment_ids cannot be empty."
+        );
 
         let segment_updater_clone = self.clone();
-        let segment_entries: Vec<SegmentEntry> = self.0.segment_manager.start_merge(segment_ids)?;
+        let segment_entries: Vec<SegmentEntry> = self
+            .0
+            .segment_manager
+            .start_merge(merge_operation.segment_ids())?;
 
-        let segment_ids_vec = segment_ids.to_vec();
+        //        let segment_ids_vec = merge_operation.segment_ids.to_vec();
 
         let merging_thread_id = self.get_merging_thread_id();
         info!(
             "Starting merge thread #{} - {:?}",
-            merging_thread_id, segment_ids
+            merging_thread_id,
+            merge_operation.segment_ids()
         );
         let (merging_future_send, merging_future_recv) = oneshot();
 
-        let target_opstamp = self.0.stamper.stamp();
-
         // first we need to apply deletes to our segment.
         let merging_join_handle = thread::Builder::new()
             .name(format!("mergingthread-{}", merging_thread_id))
             .spawn(move || {
                 // first we need to apply deletes to our segment.
-                let merged_segment = segment_updater_clone.new_segment();
-                let merged_segment_id = merged_segment.id();
                 let merge_result = perform_merge(
+                    &merge_operation,
                     &segment_updater_clone.0.index,
                     segment_entries,
-                    merged_segment,
-                    target_opstamp,
                 );
 
                 match merge_result {
                     Ok(after_merge_segment_entry) => {
                         let merged_segment_meta = after_merge_segment_entry.meta().clone();
                         segment_updater_clone
-                            .end_merge(segment_ids_vec, after_merge_segment_entry)
+                            .end_merge(merge_operation, after_merge_segment_entry)
                             .expect("Segment updater thread is corrupted.");
 
                         // the future may fail if the listener of the oneshot future
@@ -336,13 +362,18 @@ impl SegmentUpdater {
                         let _merging_future_res = merging_future_send.send(merged_segment_meta);
                     }
                     Err(e) => {
-                        warn!("Merge of {:?} was cancelled: {:?}", segment_ids_vec, e);
+                        warn!(
+                            "Merge of {:?} was cancelled: {:?}",
+                            merge_operation.segment_ids(),
+                            e
+                        );
                         // ... cancel merge
                         if cfg!(test) {
                             panic!("Merge failed.");
                         }
-                        segment_updater_clone.cancel_merge(&segment_ids_vec, merged_segment_id);
-                        // merging_future_send will be dropped, sending an error to the future.
+                        // As `merge_operation` will be dropped, the segment in merge state will
+                        // be available for merge again.
+                        // `merging_future_send` will be dropped, sending an error to the future.
                     }
                 }
                 segment_updater_clone
@@ -352,7 +383,8 @@ impl SegmentUpdater {
                     .unwrap()
                     .remove(&merging_thread_id);
                 Ok(())
-            }).expect("Failed to spawn a thread.");
+            })
+            .expect("Failed to spawn a thread.");
         self.0
             .merging_threads
             .write()
@@ -362,16 +394,34 @@ impl SegmentUpdater {
     }
 
     fn consider_merge_options(&self) {
+        let merge_segment_ids: HashSet<SegmentId> = self.0.merge_operations.segment_in_merge();
         let (committed_segments, uncommitted_segments) =
-            get_mergeable_segments(&self.0.segment_manager);
+            get_mergeable_segments(&merge_segment_ids, &self.0.segment_manager);
+
         // Committed segments cannot be merged with uncommitted_segments.
         // We therefore consider merges using these two sets of segments independently.
         let merge_policy = self.get_merge_policy();
-        let mut merge_candidates = merge_policy.compute_merge_candidates(&uncommitted_segments);
-        let committed_merge_candidates = merge_policy.compute_merge_candidates(&committed_segments);
-        merge_candidates.extend_from_slice(&committed_merge_candidates[..]);
-        for MergeCandidate(segment_metas) in merge_candidates {
-            match self.start_merge_impl(&segment_metas) {
+
+        let current_opstamp = self.0.stamper.stamp();
+        let mut merge_candidates: Vec<MergeOperation> = merge_policy
+            .compute_merge_candidates(&uncommitted_segments)
+            .into_iter()
+            .map(|merge_candidate| {
+                MergeOperation::new(&self.0.merge_operations, current_opstamp, merge_candidate.0)
+            })
+            .collect();
+
+        let commit_opstamp = self.load_metas().opstamp;
+        let committed_merge_candidates = merge_policy
+            .compute_merge_candidates(&committed_segments)
+            .into_iter()
+            .map(|merge_candidate| {
+                MergeOperation::new(&self.0.merge_operations, commit_opstamp, merge_candidate.0)
+            })
+            .collect::<Vec<_>>();
+        merge_candidates.extend(committed_merge_candidates.into_iter());
+        for merge_operation in merge_candidates {
+            match self.start_merge_impl(merge_operation) {
                 Ok(merge_future) => {
                     if let Err(e) = merge_future.fuse().poll() {
                         error!("The merge task failed quickly after starting: {:?}", e);
@@ -387,31 +437,16 @@ impl SegmentUpdater {
         }
     }
 
-    fn cancel_merge(
-        &self,
-        before_merge_segment_ids: &[SegmentId],
-        after_merge_segment_entry: SegmentId,
-    ) {
-        self.0
-            .segment_manager
-            .cancel_merge(before_merge_segment_ids, after_merge_segment_entry);
-    }
-
     fn end_merge(
         &self,
-        before_merge_segment_ids: Vec<SegmentId>,
+        merge_operation: MergeOperation,
         mut after_merge_segment_entry: SegmentEntry,
     ) -> Result<()> {
         self.run_async(move |segment_updater| {
             info!("End merge {:?}", after_merge_segment_entry.meta());
             let mut delete_cursor = after_merge_segment_entry.delete_cursor().clone();
             if let Some(delete_operation) = delete_cursor.get() {
-                let committed_opstamp = segment_updater
-                    .0
-                    .index
-                    .load_metas()
-                    .expect("Failed to read opstamp")
-                    .opstamp;
+                let committed_opstamp = segment_updater.load_metas().opstamp;
                 if delete_operation.opstamp < committed_opstamp {
                     let index = &segment_updater.0.index;
                     let segment = index.segment(after_merge_segment_entry.meta().clone());
@@ -420,16 +455,15 @@ impl SegmentUpdater {
                     {
                         error!(
                             "Merge of {:?} was cancelled (advancing deletes failed): {:?}",
-                            before_merge_segment_ids, e
+                            merge_operation.segment_ids(),
+                            e
                         );
-                        // ... cancel merge
                         if cfg!(test) {
                             panic!("Merge failed.");
                         }
-                        segment_updater.cancel_merge(
-                            &before_merge_segment_ids,
-                            after_merge_segment_entry.segment_id(),
-                        );
+                        // ... cancel merge
+                        // `merge_operations` are tracked. As it is dropped, the
+                        // the segment_ids will be available again for merge.
                         return;
                     }
                 }
@@ -437,13 +471,14 @@ impl SegmentUpdater {
             segment_updater
                 .0
                 .segment_manager
-                .end_merge(&before_merge_segment_ids, after_merge_segment_entry);
+                .end_merge(merge_operation.segment_ids(), after_merge_segment_entry);
             segment_updater.consider_merge_options();
             info!("save metas");
-            let previous_metas = segment_updater.0.index.load_metas().unwrap();
-            segment_updater.save_metas(previous_metas.opstamp, previous_metas.payload);
+            let previous_metas = segment_updater.load_metas();
+            segment_updater.save_metas(previous_metas.opstamp, previous_metas.payload.clone());
             segment_updater.garbage_collect_files_exec();
-        }).wait()
+        })
+        .wait()
     }
 
     /// Wait for current merging threads.
@@ -462,32 +497,25 @@ impl SegmentUpdater {
     /// Obsolete files will eventually be cleaned up
     /// by the directory garbage collector.
     pub fn wait_merging_thread(&self) -> Result<()> {
-        let mut num_segments: usize;
         loop {
-            num_segments = self.0.segment_manager.num_segments();
-
-            let mut new_merging_threads = HashMap::new();
-            {
+            let merging_threads: HashMap<usize, JoinHandle<Result<()>>> = {
                 let mut merging_threads = self.0.merging_threads.write().unwrap();
-                mem::swap(&mut new_merging_threads, merging_threads.deref_mut());
+                mem::replace(merging_threads.deref_mut(), HashMap::new())
+            };
+            if merging_threads.is_empty() {
+                return Ok(());
             }
-            debug!("wait merging thread {}", new_merging_threads.len());
-            for (_, merging_thread_handle) in new_merging_threads {
+            debug!("wait merging thread {}", merging_threads.len());
+            for (_, merging_thread_handle) in merging_threads {
                 merging_thread_handle
                     .join()
                     .map(|_| ())
                     .map_err(|_| TantivyError::ErrorInThread("Merging thread failed.".into()))?;
             }
-            // Our merging thread may have queued their completed
+            // Our merging thread may have queued their completed merged segment.
+            // Let's wait for that too.
             self.run_async(move |_| {}).wait()?;
-
-            let new_num_segments = self.0.segment_manager.num_segments();
-
-            if new_num_segments >= num_segments {
-                break;
-            }
         }
-        Ok(())
     }
 }
 
@@ -507,7 +535,7 @@ mod tests {
         let index = Index::create_in_ram(schema);
 
         // writing the segment
-        let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
+        let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
         index_writer.set_merge_policy(Box::new(MergeWheneverPossible));
 
         {
@@ -551,4 +579,75 @@ mod tests {
         assert_eq!(index.searcher().segment_readers().len(), 1);
         assert_eq!(index.searcher().num_docs(), 302);
     }
+
+    #[test]
+    fn delete_all_docs() {
+        let mut schema_builder = Schema::builder();
+        let text_field = schema_builder.add_text_field("text", TEXT);
+        let schema = schema_builder.build();
+
+        let index = Index::create_in_ram(schema);
+
+        // writing the segment
+        let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
+
+        {
+            for _ in 0..100 {
+                index_writer.add_document(doc!(text_field=>"a"));
+                index_writer.add_document(doc!(text_field=>"b"));
+            }
+            assert!(index_writer.commit().is_ok());
+        }
+
+        {
+            for _ in 0..100 {
+                index_writer.add_document(doc!(text_field=>"c"));
+                index_writer.add_document(doc!(text_field=>"d"));
+            }
+            assert!(index_writer.commit().is_ok());
+        }
+
+        {
+            index_writer.add_document(doc!(text_field=>"e"));
+            index_writer.add_document(doc!(text_field=>"f"));
+            assert!(index_writer.commit().is_ok());
+        }
+
+        {
+            let seg_ids = index
+                .searchable_segment_ids()
+                .expect("Searchable segments failed.");
+            // docs exist, should have at least 1 segment
+            assert!(seg_ids.len() > 0);
+        }
+
+        {
+            let term_vals = vec!["a", "b", "c", "d", "e", "f"];
+            for term_val in term_vals {
+                let term = Term::from_field_text(text_field, term_val);
+                index_writer.delete_term(term);
+                assert!(index_writer.commit().is_ok());
+            }
+        }
+
+        {
+            index_writer
+                .wait_merging_threads()
+                .expect("waiting for merging threads");
+        }
+
+        index.load_searchers().unwrap();
+        assert_eq!(index.searcher().num_docs(), 0);
+
+        let seg_ids = index
+            .searchable_segment_ids()
+            .expect("Searchable segments failed.");
+        assert!(seg_ids.is_empty());
+
+        index.load_searchers().unwrap();
+        assert_eq!(index.searcher().num_docs(), 0);
+        // empty segments should be erased
+        assert!(index.searchable_segment_metas().unwrap().is_empty());
+        assert!(index.searcher().segment_readers().is_empty());
+    }
 }

src/indexer/segment_writer.rs

@@ -62,7 +62,8 @@ impl SegmentWriter {
                             segment.index().tokenizers().get(tokenizer_name)
                         }),
                     _ => None,
-                }).collect();
+                })
+                .collect();
         Ok(SegmentWriter {
             max_doc: 0,
             multifield_postings,
@@ -110,18 +111,18 @@ impl SegmentWriter {
             }
             match *field_options.field_type() {
                 FieldType::HierarchicalFacet => {
-                    let facets: Vec<&[u8]> = field_values
+                    let facets: Vec<&str> = field_values
                         .iter()
                         .flat_map(|field_value| match *field_value.value() {
-                            Value::Facet(ref facet) => Some(facet.encoded_bytes()),
+                            Value::Facet(ref facet) => Some(facet.encoded_str()),
                             _ => {
                                 panic!("Expected hierarchical facet");
                             }
-                        }).collect();
+                        })
+                        .collect();
                     let mut term = Term::for_field(field); // we set the Term
-                    for facet_bytes in facets {
+                    for fake_str in facets {
                         let mut unordered_term_id_opt = None;
-                        let fake_str = unsafe { str::from_utf8_unchecked(facet_bytes) };
                         FacetTokenizer.token_stream(fake_str).process(&mut |token| {
                             term.set_text(&token.text);
                             let unordered_term_id =
@@ -145,7 +146,8 @@ impl SegmentWriter {
                             .flat_map(|field_value| match *field_value.value() {
                                 Value::Str(ref text) => Some(text.as_str()),
                                 _ => None,
-                            }).collect();
+                            })
+                            .collect();
                         if texts.is_empty() {
                             0
                         } else {

src/indexer/stamper.rs

@@ -1,50 +1,77 @@
+use std::ops::Range;
+use std::sync::atomic::Ordering;
+use std::sync::Arc;
+
 // AtomicU64 have not landed in stable.
 // For the moment let's just use AtomicUsize on
 // x86/64 bit platform, and a mutex on other platform.
-
-#[cfg(target = "x86_64")]
+#[cfg(target_arch = "x86_64")]
 mod archicture_impl {
 
     use std::sync::atomic::{AtomicUsize, Ordering};
-    use std::sync::Arc;
 
-    #[derive(Clone, Default)]
-    pub struct Stamper(Arc<AtomicU64>);
+    #[derive(Default)]
+    pub struct AtomicU64Ersatz(AtomicUsize);
 
-    impl Stamper {
-        pub fn new(first_opstamp: u64) -> Stamper {
-            Stamper(Arc::new(AtomicU64::new(first_opstamp)))
+    impl AtomicU64Ersatz {
+        pub fn new(first_opstamp: u64) -> AtomicU64Ersatz {
+            AtomicU64Ersatz(AtomicUsize::new(first_opstamp as usize))
         }
 
-        pub fn stamp(&self) -> u64 {
-            self.0.fetch_add(1u64, Ordering::SeqCst) as u64
+        pub fn fetch_add(&self, val: u64, order: Ordering) -> u64 {
+            self.0.fetch_add(val as usize, order) as u64
         }
     }
 }
 
-#[cfg(not(target = "x86_64"))]
+#[cfg(not(target_arch = "x86_64"))]
 mod archicture_impl {
 
-    use std::sync::{Arc, Mutex};
+    use std::sync::atomic::Ordering;
+    /// Under other architecture, we rely on a mutex.
+    use std::sync::RwLock;
 
-    #[derive(Clone, Default)]
-    pub struct Stamper(Arc<Mutex<u64>>);
+    #[derive(Default)]
+    pub struct AtomicU64Ersatz(RwLock<u64>);
 
-    impl Stamper {
-        pub fn new(first_opstamp: u64) -> Stamper {
-            Stamper(Arc::new(Mutex::new(first_opstamp)))
+    impl AtomicU64Ersatz {
+        pub fn new(first_opstamp: u64) -> AtomicU64Ersatz {
+            AtomicU64Ersatz(RwLock::new(first_opstamp))
         }
 
-        pub fn stamp(&self) -> u64 {
-            let mut guard = self.0.lock().expect("Failed to lock the stamper");
-            let previous_val = *guard;
-            *guard = previous_val + 1;
+        pub fn fetch_add(&self, incr: u64, _order: Ordering) -> u64 {
+            let mut lock = self.0.write().unwrap();
+            let previous_val = *lock;
+            *lock = previous_val + incr;
             previous_val
         }
     }
 }
 
-pub use self::archicture_impl::Stamper;
+use self::archicture_impl::AtomicU64Ersatz;
+
+#[derive(Clone, Default)]
+pub struct Stamper(Arc<AtomicU64Ersatz>);
+
+impl Stamper {
+    pub fn new(first_opstamp: u64) -> Stamper {
+        Stamper(Arc::new(AtomicU64Ersatz::new(first_opstamp)))
+    }
+
+    pub fn stamp(&self) -> u64 {
+        self.0.fetch_add(1u64, Ordering::SeqCst) as u64
+    }
+
+    /// Given a desired count `n`, `stamps` returns an iterator that
+    /// will supply `n` number of u64 stamps.
+    pub fn stamps(&self, n: u64) -> Range<u64> {
+        let start = self.0.fetch_add(n, Ordering::SeqCst);
+        Range {
+            start: start,
+            end: start + n,
+        }
+    }
+}
 
 #[cfg(test)]
 mod test {
@@ -62,5 +89,7 @@ mod test {
 
         assert_eq!(stamper.stamp(), 10u64);
         assert_eq!(stamper_clone.stamp(), 11u64);
+        assert_eq!(stamper.stamps(3u64), (12..15));
+        assert_eq!(stamper.stamp(), 15u64);
     }
 }

src/lib.rs

@@ -1,6 +1,5 @@
 #![doc(html_logo_url = "http://fulmicoton.com/tantivy-logo/tantivy-logo.png")]
 #![cfg_attr(all(feature = "unstable", test), feature(test))]
-#![cfg_attr(feature = "cargo-clippy", feature(tool_lints))]
 #![cfg_attr(feature = "cargo-clippy", allow(clippy::module_inception))]
 #![doc(test(attr(allow(unused_variables), deny(warnings))))]
 #![warn(missing_docs)]
@@ -130,10 +129,7 @@ extern crate base64;
 extern crate bit_set;
 extern crate bitpacking;
 extern crate byteorder;
-extern crate scoped_pool;
-
 extern crate combine;
-
 extern crate crossbeam;
 extern crate fnv;
 extern crate fst;
@@ -147,6 +143,7 @@ extern crate num_cpus;
 extern crate owning_ref;
 extern crate regex;
 extern crate rust_stemmers;
+extern crate scoped_pool;
 extern crate serde;
 extern crate stable_deref_trait;
 extern crate tempdir;
@@ -171,7 +168,7 @@ extern crate maplit;
 extern crate test;
 
 #[macro_use]
-extern crate downcast;
+extern crate downcast_rs;
 
 #[macro_use]
 extern crate fail;
@@ -184,10 +181,7 @@ mod macros;
 
 pub use error::TantivyError;
 
-#[deprecated(
-    since = "0.7.0",
-    note = "please use `tantivy::TantivyError` instead"
-)]
+#[deprecated(since = "0.7.0", note = "please use `tantivy::TantivyError` instead")]
 pub use error::TantivyError as Error;
 
 extern crate census;
@@ -217,7 +211,7 @@ pub mod store;
 pub mod termdict;
 
 mod snippet;
-pub use self::snippet::{SnippetGenerator, Snippet};
+pub use self::snippet::{Snippet, SnippetGenerator};
 
 mod docset;
 pub use self::docset::{DocSet, SkipResult};
@@ -235,11 +229,7 @@ pub use common::{i64_to_u64, u64_to_i64};
 /// Expose the current version of tantivy, as well
 /// whether it was compiled with the simd compression.
 pub fn version() -> &'static str {
-    if cfg!(feature = "simdcompression") {
-        concat!(env!("CARGO_PKG_VERSION"), "-simd")
-    } else {
-        concat!(env!("CARGO_PKG_VERSION"), "-nosimd")
-    }
+    env!("CARGO_PKG_VERSION")
 }
 
 /// Defines tantivy's merging strategy
@@ -352,7 +342,7 @@ mod tests {
         let index = Index::create_from_tempdir(schema).unwrap();
         {
             // writing the segment
-            let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
+            let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
             {
                 let doc = doc!(text_field=>"af b");
                 index_writer.add_document(doc);
@@ -374,7 +364,7 @@ mod tests {
         let mut schema_builder = Schema::builder();
         let text_field = schema_builder.add_text_field("text", TEXT);
         let index = Index::create_in_ram(schema_builder.build());
-        let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
+        let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
         {
             index_writer.add_document(doc!(text_field=>"a b c"));
             index_writer.commit().unwrap();
@@ -416,7 +406,7 @@ mod tests {
         let text_field = schema_builder.add_text_field("text", TEXT);
         let index = Index::create_in_ram(schema_builder.build());
         {
-            let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
+            let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
             {
                 let doc = doc!(text_field=>"a b c");
                 index_writer.add_document(doc);
@@ -444,7 +434,7 @@ mod tests {
         let text_field = schema_builder.add_text_field("text", TEXT);
         let index = Index::create_in_ram(schema_builder.build());
         {
-            let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
+            let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
             {
                 let doc = doc!(text_field=>"a b c");
                 index_writer.add_document(doc);
@@ -491,7 +481,7 @@ mod tests {
         let index = Index::create_in_ram(schema);
         {
             // writing the segment
-            let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
+            let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
             // 0
             index_writer.add_document(doc!(text_field=>"a b"));
             // 1
@@ -514,11 +504,9 @@ mod tests {
             let searcher = index.searcher();
             let reader = searcher.segment_reader(0);
             let inverted_index = reader.inverted_index(text_field);
-            assert!(
-                inverted_index
-                    .read_postings(&term_abcd, IndexRecordOption::WithFreqsAndPositions)
-                    .is_none()
-            );
+            assert!(inverted_index
+                .read_postings(&term_abcd, IndexRecordOption::WithFreqsAndPositions)
+                .is_none());
             {
                 let mut postings = inverted_index
                     .read_postings(&term_a, IndexRecordOption::WithFreqsAndPositions)
@@ -540,7 +528,7 @@ mod tests {
         }
         {
             // writing the segment
-            let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
+            let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
             // 0
             index_writer.add_document(doc!(text_field=>"a b"));
             // 1
@@ -553,11 +541,9 @@ mod tests {
             let reader = searcher.segment_reader(0);
             let inverted_index = reader.inverted_index(term_abcd.field());
 
-            assert!(
-                inverted_index
-                    .read_postings(&term_abcd, IndexRecordOption::WithFreqsAndPositions)
-                    .is_none()
-            );
+            assert!(inverted_index
+                .read_postings(&term_abcd, IndexRecordOption::WithFreqsAndPositions)
+                .is_none());
             {
                 let mut postings = inverted_index
                     .read_postings(&term_a, IndexRecordOption::WithFreqsAndPositions)
@@ -579,7 +565,7 @@ mod tests {
         }
         {
             // writing the segment
-            let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
+            let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
             index_writer.add_document(doc!(text_field=>"a b"));
             index_writer.delete_term(Term::from_field_text(text_field, "c"));
             index_writer.rollback().unwrap();
@@ -591,11 +577,9 @@ mod tests {
             let searcher = index.searcher();
             let reader = searcher.segment_reader(0);
             let inverted_index = reader.inverted_index(term_abcd.field());
-            assert!(
-                inverted_index
-                    .read_postings(&term_abcd, IndexRecordOption::WithFreqsAndPositions)
-                    .is_none()
-            );
+            assert!(inverted_index
+                .read_postings(&term_abcd, IndexRecordOption::WithFreqsAndPositions)
+                .is_none());
             {
                 let mut postings = inverted_index
                     .read_postings(&term_a, IndexRecordOption::WithFreqsAndPositions)
@@ -630,7 +614,7 @@ mod tests {
         let schema = schema_builder.build();
 
         let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
+        let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
         index_writer.add_document(doc!(field=>1u64));
         index_writer.commit().unwrap();
         index.load_searchers().unwrap();
@@ -653,7 +637,7 @@ mod tests {
         let schema = schema_builder.build();
 
         let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
+        let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
         let negative_val = -1i64;
         index_writer.add_document(doc!(value_field => negative_val));
         index_writer.commit().unwrap();
@@ -677,7 +661,7 @@ mod tests {
         let absent_field = schema_builder.add_text_field("text", TEXT);
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_with_num_threads(2, 40_000_000).unwrap();
+        let mut index_writer = index.writer_with_num_threads(2, 6_000_000).unwrap();
         index_writer.add_document(doc!(text_field=>"a"));
         assert!(index_writer.commit().is_ok());
         assert!(index.load_searchers().is_ok());
@@ -694,7 +678,7 @@ mod tests {
         let index = Index::create_in_ram(schema);
 
         // writing the segment
-        let mut index_writer = index.writer_with_num_threads(2, 40_000_000).unwrap();
+        let mut index_writer = index.writer_with_num_threads(2, 6_000_000).unwrap();
 
         let add_document = |index_writer: &mut IndexWriter, val: &'static str| {
             let doc = doc!(text_field=>val);
@@ -730,7 +714,7 @@ mod tests {
         let index = Index::create_in_ram(schema);
         {
             // writing the segment
-            let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
+            let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
             {
                 let doc = doc!(text_field=>"af af af bc bc");
                 index_writer.add_document(doc);
@@ -743,11 +727,9 @@ mod tests {
             let reader = searcher.segment_reader(0);
             let inverted_index = reader.inverted_index(text_field);
             let term_abcd = Term::from_field_text(text_field, "abcd");
-            assert!(
-                inverted_index
-                    .read_postings(&term_abcd, IndexRecordOption::WithFreqsAndPositions)
-                    .is_none()
-            );
+            assert!(inverted_index
+                .read_postings(&term_abcd, IndexRecordOption::WithFreqsAndPositions)
+                .is_none());
             let term_af = Term::from_field_text(text_field, "af");
             let mut postings = inverted_index
                 .read_postings(&term_af, IndexRecordOption::WithFreqsAndPositions)
@@ -768,7 +750,7 @@ mod tests {
 
         {
             // writing the segment
-            let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
+            let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
             index_writer.add_document(doc!(text_field=>"af af af b"));
             index_writer.add_document(doc!(text_field=>"a b c"));
             index_writer.add_document(doc!(text_field=>"a b c d"));
@@ -821,7 +803,7 @@ mod tests {
 
         {
             // writing the segment
-            let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
+            let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
             {
                 let doc = doc!(text_field=>"af b");
                 index_writer.add_document(doc);

src/macros.rs

@@ -77,10 +77,10 @@ mod test {
         let likes = schema_builder.add_u64_field("num_u64", FAST);
         let _schema = schema_builder.build();
         let _doc = doc!(
-            title => "Life Aquatic",
-            author => "Wes Anderson",
-            likes => 4u64
-            );
+        title => "Life Aquatic",
+        author => "Wes Anderson",
+        likes => 4u64
+        );
     }
 
     #[test]
@@ -91,9 +91,9 @@ mod test {
         let likes = schema_builder.add_u64_field("num_u64", FAST);
         let _schema = schema_builder.build();
         let _doc = doc!(
-            title => "Life Aquatic",
-            author => "Wes Anderson",
-            likes => 4u64,
-            );
+        title => "Life Aquatic",
+        author => "Wes Anderson",
+        likes => 4u64,
+        );
     }
 }

src/positions/mod.rs

@@ -34,10 +34,6 @@ const COMPRESSION_BLOCK_SIZE: usize = BitPacker4x::BLOCK_LEN;
 const LONG_SKIP_IN_BLOCKS: usize = 1_024;
 const LONG_SKIP_INTERVAL: u64 = (LONG_SKIP_IN_BLOCKS * COMPRESSION_BLOCK_SIZE) as u64;
 
-lazy_static! {
-    static ref BIT_PACKER: BitPacker4x = BitPacker4x::new();
-}
-
 #[cfg(test)]
 pub mod tests {
 

src/positions/reader.rs

@@ -1,4 +1,23 @@
-use super::BIT_PACKER;
+/// Positions works as a long sequence of compressed block.
+/// All terms are chained one after the other.
+///
+/// When accessing the position of a term, we get a positions_idx from the `Terminfo`.
+/// This means we need to skip to the `nth` positions efficiently.
+///
+/// This is done thanks to two levels of skiping that we refer to in the code
+/// as `long_skip` and `short_skip`.
+///
+/// The `long_skip` makes it possible to skip every 1_024 compression blocks (= 131_072 positions).
+/// Skipping offset are simply stored one after as an offset stored over 8 bytes.
+///
+/// We find the number of long skips, as `n / long_skip`.
+///
+/// Blocks are compressed using bitpacking, so `skip_read` contains the number of bytes
+/// (values can go from 0bit to 32 bits) required to decompressed every block.
+///
+/// A given block obviously takes `(128 x  num_bit_for_the_block / num_bits_in_a_byte)`,
+/// so skipping a block without decompressing it is just a matter of advancing that many
+/// bytes.
 use bitpacking::{BitPacker, BitPacker4x};
 use common::{BinarySerializable, FixedSize};
 use directory::ReadOnlySource;
@@ -8,9 +27,65 @@ use positions::LONG_SKIP_INTERVAL;
 use positions::LONG_SKIP_IN_BLOCKS;
 use postings::compression::compressed_block_size;
 
+struct Positions {
+    bit_packer: BitPacker4x,
+    skip_source: ReadOnlySource,
+    position_source: ReadOnlySource,
+    long_skip_source: ReadOnlySource,
+}
+
+impl Positions {
+    pub fn new(position_source: ReadOnlySource, skip_source: ReadOnlySource) -> Positions {
+        let skip_len = skip_source.len();
+        let (body, footer) = skip_source.split(skip_len - u32::SIZE_IN_BYTES);
+        let num_long_skips = u32::deserialize(&mut footer.as_slice()).expect("Index corrupted");
+        let body_split = body.len() - u64::SIZE_IN_BYTES * (num_long_skips as usize);
+        let (skip_source, long_skip_source) = body.split(body_split);
+        Positions {
+            bit_packer: BitPacker4x::new(),
+            skip_source,
+            long_skip_source,
+            position_source,
+        }
+    }
+
+    /// Returns the offset of the block associated to the given `long_skip_id`.
+    ///
+    /// One `long_skip_id` means `LONG_SKIP_IN_BLOCKS` blocks.
+    fn long_skip(&self, long_skip_id: usize) -> u64 {
+        if long_skip_id == 0 {
+            return 0;
+        }
+        let long_skip_slice = self.long_skip_source.as_slice();
+        let mut long_skip_blocks: &[u8] = &long_skip_slice[(long_skip_id - 1) * 8..][..8];
+        u64::deserialize(&mut long_skip_blocks).expect("Index corrupted")
+    }
+
+    fn reader(&self, offset: u64) -> PositionReader {
+        let long_skip_id = (offset / LONG_SKIP_INTERVAL) as usize;
+        let small_skip = (offset % LONG_SKIP_INTERVAL) as usize;
+        let offset_num_bytes: u64 = self.long_skip(long_skip_id);
+        let mut position_read = OwnedRead::new(self.position_source.clone());
+        position_read.advance(offset_num_bytes as usize);
+        let mut skip_read = OwnedRead::new(self.skip_source.clone());
+        skip_read.advance(long_skip_id * LONG_SKIP_IN_BLOCKS);
+        let mut position_reader = PositionReader {
+            bit_packer: self.bit_packer,
+            skip_read,
+            position_read,
+            inner_offset: 0,
+            buffer: Box::new([0u32; 128]),
+            ahead: None,
+        };
+        position_reader.skip(small_skip);
+        position_reader
+    }
+}
+
 pub struct PositionReader {
     skip_read: OwnedRead,
     position_read: OwnedRead,
+    bit_packer: BitPacker4x,
     inner_offset: usize,
     buffer: Box<[u32; 128]>,
     ahead: Option<usize>, // if None, no block is loaded.
@@ -27,6 +102,7 @@ pub struct PositionReader {
 // If the requested number of els ends exactly at a given block, the next
 // block is not decompressed.
 fn read_impl(
+    bit_packer: BitPacker4x,
     mut position: &[u8],
     buffer: &mut [u32; 128],
     mut inner_offset: usize,
@@ -37,21 +113,23 @@ fn read_impl(
     let mut output_len = output.len();
     let mut ahead = 0;
     loop {
-        let available_len = 128 - inner_offset;
+        let available_len = COMPRESSION_BLOCK_SIZE - inner_offset;
+        // We have enough elements in the current block.
+        // Let's copy the requested elements in the output buffer,
+        // and return.
         if output_len <= available_len {
             output[output_start..].copy_from_slice(&buffer[inner_offset..][..output_len]);
             return ahead;
-        } else {
-            output[output_start..][..available_len].copy_from_slice(&buffer[inner_offset..]);
-            output_len -= available_len;
-            output_start += available_len;
-            inner_offset = 0;
-            let num_bits = num_bits[ahead];
-            BitPacker4x::new().decompress(position, &mut buffer[..], num_bits);
-            let block_len = compressed_block_size(num_bits);
-            position = &position[block_len..];
-            ahead += 1;
         }
+        output[output_start..][..available_len].copy_from_slice(&buffer[inner_offset..]);
+        output_len -= available_len;
+        output_start += available_len;
+        inner_offset = 0;
+        let num_bits = num_bits[ahead];
+        bit_packer.decompress(position, &mut buffer[..], num_bits);
+        let block_len = compressed_block_size(num_bits);
+        position = &position[block_len..];
+        ahead += 1;
     }
 }
 
@@ -61,35 +139,7 @@ impl PositionReader {
         skip_source: ReadOnlySource,
         offset: u64,
     ) -> PositionReader {
-        let skip_len = skip_source.len();
-        let (body, footer) = skip_source.split(skip_len - u32::SIZE_IN_BYTES);
-        let num_long_skips = u32::deserialize(&mut footer.as_slice()).expect("Index corrupted");
-        let body_split = body.len() - u64::SIZE_IN_BYTES * (num_long_skips as usize);
-        let (skip_body, long_skips) = body.split(body_split);
-        let long_skip_id = (offset / LONG_SKIP_INTERVAL) as usize;
-        let small_skip = (offset - (long_skip_id as u64) * (LONG_SKIP_INTERVAL as u64)) as usize;
-        let offset_num_bytes: u64 = {
-            if long_skip_id > 0 {
-                let mut long_skip_blocks: &[u8] =
-                    &long_skips.as_slice()[(long_skip_id - 1) * 8..][..8];
-                u64::deserialize(&mut long_skip_blocks).expect("Index corrupted") * 16
-            } else {
-                0
-            }
-        };
-        let mut position_read = OwnedRead::new(position_source);
-        position_read.advance(offset_num_bytes as usize);
-        let mut skip_read = OwnedRead::new(skip_body);
-        skip_read.advance(long_skip_id * LONG_SKIP_IN_BLOCKS);
-        let mut position_reader = PositionReader {
-            skip_read,
-            position_read,
-            inner_offset: 0,
-            buffer: Box::new([0u32; 128]),
-            ahead: None,
-        };
-        position_reader.skip(small_skip);
-        position_reader
+        Positions::new(position_source, skip_source).reader(offset)
     }
 
     /// Fills a buffer with the next `output.len()` integers.
@@ -101,10 +151,12 @@ impl PositionReader {
         if self.ahead != Some(0) {
             // the block currently available is not the block
             // for the current position
-            BIT_PACKER.decompress(position_data, self.buffer.as_mut(), num_bits);
+            self.bit_packer.decompress(position_data, self.buffer.as_mut(), num_bits);
+            self.ahead = Some(0);
         }
         let block_len = compressed_block_size(num_bits);
         self.ahead = Some(read_impl(
+            self.bit_packer,
             &position_data[block_len..],
             self.buffer.as_mut(),
             self.inner_offset,
@@ -133,14 +185,13 @@ impl PositionReader {
             }
         });
 
-        let skip_len = self.skip_read.as_ref()[..num_blocks_to_advance]
+        let skip_len_in_bits = self.skip_read.as_ref()[..num_blocks_to_advance]
             .iter()
-            .cloned()
-            .map(|num_bit| num_bit as usize)
+            .map(|num_bits| *num_bits as usize)
             .sum::<usize>()
-            * (COMPRESSION_BLOCK_SIZE / 8);
-
+            * COMPRESSION_BLOCK_SIZE;
+        let skip_len_in_bytes = skip_len_in_bits / 8;
         self.skip_read.advance(num_blocks_to_advance);
-        self.position_read.advance(skip_len);
+        self.position_read.advance(skip_len_in_bytes);
     }
 }

src/positions/serializer.rs

@@ -1,29 +1,30 @@
-use super::BIT_PACKER;
 use bitpacking::BitPacker;
 use common::BinarySerializable;
+use common::CountingWriter;
 use positions::{COMPRESSION_BLOCK_SIZE, LONG_SKIP_INTERVAL};
-use std::io;
+use std::io::{self, Write};
+use bitpacking::BitPacker4x;
 
 pub struct PositionSerializer<W: io::Write> {
-    write_stream: W,
+    bit_packer: BitPacker4x,
+    write_stream: CountingWriter<W>,
     write_skiplist: W,
     block: Vec<u32>,
     buffer: Vec<u8>,
     num_ints: u64,
     long_skips: Vec<u64>,
-    cumulated_num_bits: u64,
 }
 
 impl<W: io::Write> PositionSerializer<W> {
     pub fn new(write_stream: W, write_skiplist: W) -> PositionSerializer<W> {
         PositionSerializer {
-            write_stream,
+            bit_packer: BitPacker4x::new(),
+            write_stream: CountingWriter::wrap(write_stream),
             write_skiplist,
             block: Vec::with_capacity(128),
             buffer: vec![0u8; 128 * 4],
             num_ints: 0u64,
             long_skips: Vec::new(),
-            cumulated_num_bits: 0u64,
         }
     }
 
@@ -50,14 +51,13 @@ impl<W: io::Write> PositionSerializer<W> {
     }
 
     fn flush_block(&mut self) -> io::Result<()> {
-        let num_bits = BIT_PACKER.num_bits(&self.block[..]);
-        self.cumulated_num_bits += u64::from(num_bits);
+        let num_bits = self.bit_packer.num_bits(&self.block[..]);
         self.write_skiplist.write_all(&[num_bits])?;
-        let written_len = BIT_PACKER.compress(&self.block[..], &mut self.buffer, num_bits);
+        let written_len = self.bit_packer.compress(&self.block[..], &mut self.buffer, num_bits);
         self.write_stream.write_all(&self.buffer[..written_len])?;
         self.block.clear();
         if (self.num_ints % LONG_SKIP_INTERVAL) == 0u64 {
-            self.long_skips.push(self.cumulated_num_bits);
+            self.long_skips.push(self.write_stream.written_bytes());
         }
         Ok(())
     }

src/postings/mod.rs

@@ -221,12 +221,10 @@ pub mod tests {
             }
             {
                 let term_a = Term::from_field_text(text_field, "abcdef");
-                assert!(
-                    segment_reader
-                        .inverted_index(term_a.field())
-                        .read_postings(&term_a, IndexRecordOption::WithFreqsAndPositions)
-                        .is_none()
-                );
+                assert!(segment_reader
+                    .inverted_index(term_a.field())
+                    .read_postings(&term_a, IndexRecordOption::WithFreqsAndPositions)
+                    .is_none());
             }
             {
                 let term_a = Term::from_field_text(text_field, "a");
@@ -282,7 +280,7 @@ pub mod tests {
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
         {
-            let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
+            let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
             {
                 let mut doc = Document::default();
                 doc.add_text(text_field, "g b b d c g c");
@@ -324,7 +322,7 @@ pub mod tests {
 
             let index = Index::create_in_ram(schema);
             {
-                let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
+                let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
                 for i in 0..num_docs {
                     let mut doc = Document::default();
                     doc.add_u64(value_field, 2);
@@ -401,7 +399,7 @@ pub mod tests {
 
         // delete some of the documents
         {
-            let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
+            let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
             index_writer.delete_term(term_0);
             assert!(index_writer.commit().is_ok());
         }
@@ -451,7 +449,7 @@ pub mod tests {
 
         // delete everything else
         {
-            let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
+            let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
             index_writer.delete_term(term_1);
 
             assert!(index_writer.commit().is_ok());
@@ -459,25 +457,14 @@ pub mod tests {
         index.load_searchers().unwrap();
 
         let searcher = index.searcher();
-        let segment_reader = searcher.segment_reader(0);
 
         // finally, check that it's empty
         {
-            let mut segment_postings = segment_reader
-                .inverted_index(term_2.field())
-                .read_postings(&term_2, IndexRecordOption::Basic)
-                .unwrap();
-
-            assert_eq!(segment_postings.skip_next(0), SkipResult::Reached);
-            assert_eq!(segment_postings.doc(), 0);
-            assert!(segment_reader.is_deleted(0));
-
-            let mut segment_postings = segment_reader
-                .inverted_index(term_2.field())
-                .read_postings(&term_2, IndexRecordOption::Basic)
-                .unwrap();
-
-            assert_eq!(segment_postings.skip_next(num_docs), SkipResult::End);
+            let searchable_segment_ids = index
+                .searchable_segment_ids()
+                .expect("could not get index segment ids");
+            assert!(searchable_segment_ids.is_empty());
+            assert_eq!(searcher.num_docs(), 0);
         }
     }
 
@@ -508,7 +495,7 @@ pub mod tests {
             let index = Index::create_in_ram(schema);
             let posting_list_size = 1_000_000;
             {
-                let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
+                let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
                 for _ in 0..posting_list_size {
                     let mut doc = Document::default();
                     if rng.gen_bool(1f64 / 15f64) {

src/postings/postings_writer.rs

@@ -1,6 +1,8 @@
 use super::stacker::{Addr, MemoryArena, TermHashMap};
 
-use postings::recorder::{NothingRecorder, Recorder, TFAndPositionRecorder, TermFrequencyRecorder};
+use postings::recorder::{
+    BufferLender, NothingRecorder, Recorder, TFAndPositionRecorder, TermFrequencyRecorder,
+};
 use postings::UnorderedTermId;
 use postings::{FieldSerializer, InvertedIndexSerializer};
 use schema::IndexRecordOption;
@@ -29,7 +31,8 @@ fn posting_from_field_entry(field_entry: &FieldEntry) -> Box<PostingsWriter> {
                 IndexRecordOption::WithFreqsAndPositions => {
                     SpecializedPostingsWriter::<TFAndPositionRecorder>::new_boxed()
                 }
-            }).unwrap_or_else(|| SpecializedPostingsWriter::<NothingRecorder>::new_boxed()),
+            })
+            .unwrap_or_else(|| SpecializedPostingsWriter::<NothingRecorder>::new_boxed()),
         FieldType::U64(_) | FieldType::I64(_) | FieldType::HierarchicalFacet => {
             SpecializedPostingsWriter::<NothingRecorder>::new_boxed()
         }
@@ -107,10 +110,8 @@ impl MultiFieldPostingsWriter {
             .map(|(key, _, _)| Term::wrap(key).field())
             .enumerate();
 
-        let mut unordered_term_mappings: HashMap<
-            Field,
-            HashMap<UnorderedTermId, TermOrdinal>,
-        > = HashMap::new();
+        let mut unordered_term_mappings: HashMap<Field, HashMap<UnorderedTermId, TermOrdinal>> =
+            HashMap::new();
 
         let mut prev_field = Field(u32::max_value());
         for (offset, field) in term_offsets_it {
@@ -138,7 +139,8 @@ impl MultiFieldPostingsWriter {
                         .enumerate()
                         .map(|(term_ord, unord_term_id)| {
                             (unord_term_id as UnorderedTermId, term_ord as TermOrdinal)
-                        }).collect();
+                        })
+                        .collect();
                     unordered_term_mappings.insert(field, mapping);
                 }
                 FieldType::U64(_) | FieldType::I64(_) => {}
@@ -213,7 +215,7 @@ pub trait PostingsWriter {
 
 /// The `SpecializedPostingsWriter` is just here to remove dynamic
 /// dispatch to the recorder information.
-pub struct SpecializedPostingsWriter<Rec: Recorder + 'static> {
+pub(crate) struct SpecializedPostingsWriter<Rec: Recorder + 'static> {
     total_num_tokens: u64,
     _recorder_type: PhantomData<Rec>,
 }
@@ -245,8 +247,7 @@ impl<Rec: Recorder + 'static> PostingsWriter for SpecializedPostingsWriter<Rec>
         debug_assert!(term.as_slice().len() >= 4);
         self.total_num_tokens += 1;
         term_index.mutate_or_create(term, |opt_recorder: Option<Rec>| {
-            if opt_recorder.is_some() {
-                let mut recorder = opt_recorder.unwrap();
+            if let Some(mut recorder) = opt_recorder {
                 let current_doc = recorder.current_doc();
                 if current_doc != doc {
                     recorder.close_doc(heap);
@@ -255,7 +256,7 @@ impl<Rec: Recorder + 'static> PostingsWriter for SpecializedPostingsWriter<Rec>
                 recorder.record_position(position, heap);
                 recorder
             } else {
-                let mut recorder = Rec::new(heap);
+                let mut recorder = Rec::new();
                 recorder.new_doc(doc, heap);
                 recorder.record_position(position, heap);
                 recorder
@@ -270,10 +271,11 @@ impl<Rec: Recorder + 'static> PostingsWriter for SpecializedPostingsWriter<Rec>
         termdict_heap: &MemoryArena,
         heap: &MemoryArena,
     ) -> io::Result<()> {
+        let mut buffer_lender = BufferLender::default();
         for &(term_bytes, addr, _) in term_addrs {
-            let recorder: Rec = unsafe { termdict_heap.read(addr) };
+            let recorder: Rec = termdict_heap.read(addr);
             serializer.new_term(&term_bytes[4..])?;
-            recorder.serialize(serializer, heap)?;
+            recorder.serialize(&mut buffer_lender, serializer, heap)?;
             serializer.close_term()?;
         }
         Ok(())

src/postings/recorder.rs

@@ -1,10 +1,51 @@
 use super::stacker::{ExpUnrolledLinkedList, MemoryArena};
+use common::{read_u32_vint, write_u32_vint};
 use postings::FieldSerializer;
-use std::{self, io};
+use std::io;
 use DocId;
 
 const EMPTY_ARRAY: [u32; 0] = [0u32; 0];
-const POSITION_END: u32 = std::u32::MAX;
+const POSITION_END: u32 = 0;
+
+#[derive(Default)]
+pub(crate) struct BufferLender {
+    buffer_u8: Vec<u8>,
+    buffer_u32: Vec<u32>,
+}
+
+impl BufferLender {
+    pub fn lend_u8(&mut self) -> &mut Vec<u8> {
+        self.buffer_u8.clear();
+        &mut self.buffer_u8
+    }
+    pub fn lend_all(&mut self) -> (&mut Vec<u8>, &mut Vec<u32>) {
+        self.buffer_u8.clear();
+        self.buffer_u32.clear();
+        (&mut self.buffer_u8, &mut self.buffer_u32)
+    }
+}
+
+pub struct VInt32Reader<'a> {
+    data: &'a [u8],
+}
+
+impl<'a> VInt32Reader<'a> {
+    fn new(data: &'a [u8]) -> VInt32Reader<'a> {
+        VInt32Reader { data }
+    }
+}
+
+impl<'a> Iterator for VInt32Reader<'a> {
+    type Item = u32;
+
+    fn next(&mut self) -> Option<u32> {
+        if self.data.is_empty() {
+            None
+        } else {
+            Some(read_u32_vint(&mut self.data))
+        }
+    }
+}
 
 /// Recorder is in charge of recording relevant information about
 /// the presence of a term in a document.
@@ -15,9 +56,9 @@ const POSITION_END: u32 = std::u32::MAX;
 ///   * the document id
 ///   * the term frequency
 ///   * the term positions
-pub trait Recorder: Copy {
+pub(crate) trait Recorder: Copy + 'static {
     ///
-    fn new(heap: &mut MemoryArena) -> Self;
+    fn new() -> Self;
     /// Returns the current document
     fn current_doc(&self) -> u32;
     /// Starts recording information about a new document
@@ -29,7 +70,12 @@ pub trait Recorder: Copy {
     /// Close the document. It will help record the term frequency.
     fn close_doc(&mut self, heap: &mut MemoryArena);
     /// Pushes the postings information to the serializer.
-    fn serialize(&self, serializer: &mut FieldSerializer, heap: &MemoryArena) -> io::Result<()>;
+    fn serialize(
+        &self,
+        buffer_lender: &mut BufferLender,
+        serializer: &mut FieldSerializer,
+        heap: &MemoryArena,
+    ) -> io::Result<()>;
 }
 
 /// Only records the doc ids
@@ -40,9 +86,9 @@ pub struct NothingRecorder {
 }
 
 impl Recorder for NothingRecorder {
-    fn new(heap: &mut MemoryArena) -> Self {
+    fn new() -> Self {
         NothingRecorder {
-            stack: ExpUnrolledLinkedList::new(heap),
+            stack: ExpUnrolledLinkedList::new(),
             current_doc: u32::max_value(),
         }
     }
@@ -53,16 +99,23 @@ impl Recorder for NothingRecorder {
 
     fn new_doc(&mut self, doc: DocId, heap: &mut MemoryArena) {
         self.current_doc = doc;
-        self.stack.push(doc, heap);
+        let _ = write_u32_vint(doc, &mut self.stack.writer(heap));
     }
 
     fn record_position(&mut self, _position: u32, _heap: &mut MemoryArena) {}
 
     fn close_doc(&mut self, _heap: &mut MemoryArena) {}
 
-    fn serialize(&self, serializer: &mut FieldSerializer, heap: &MemoryArena) -> io::Result<()> {
-        for doc in self.stack.iter(heap) {
-            serializer.write_doc(doc, 0u32, &EMPTY_ARRAY)?;
+    fn serialize(
+        &self,
+        buffer_lender: &mut BufferLender,
+        serializer: &mut FieldSerializer,
+        heap: &MemoryArena,
+    ) -> io::Result<()> {
+        let buffer = buffer_lender.lend_u8();
+        self.stack.read_to_end(heap, buffer);
+        for doc in VInt32Reader::new(&buffer[..]) {
+            serializer.write_doc(doc as u32, 0u32, &EMPTY_ARRAY)?;
         }
         Ok(())
     }
@@ -77,9 +130,9 @@ pub struct TermFrequencyRecorder {
 }
 
 impl Recorder for TermFrequencyRecorder {
-    fn new(heap: &mut MemoryArena) -> Self {
+    fn new() -> Self {
         TermFrequencyRecorder {
-            stack: ExpUnrolledLinkedList::new(heap),
+            stack: ExpUnrolledLinkedList::new(),
             current_doc: u32::max_value(),
             current_tf: 0u32,
         }
@@ -91,7 +144,7 @@ impl Recorder for TermFrequencyRecorder {
 
     fn new_doc(&mut self, doc: DocId, heap: &mut MemoryArena) {
         self.current_doc = doc;
-        self.stack.push(doc, heap);
+        let _ = write_u32_vint(doc, &mut self.stack.writer(heap));
     }
 
     fn record_position(&mut self, _position: u32, _heap: &mut MemoryArena) {
@@ -100,24 +153,24 @@ impl Recorder for TermFrequencyRecorder {
 
     fn close_doc(&mut self, heap: &mut MemoryArena) {
         debug_assert!(self.current_tf > 0);
-        self.stack.push(self.current_tf, heap);
+        let _ = write_u32_vint(self.current_tf, &mut self.stack.writer(heap));
         self.current_tf = 0;
     }
 
-    fn serialize(&self, serializer: &mut FieldSerializer, heap: &MemoryArena) -> io::Result<()> {
-        // the last document has not been closed...
-        // its term freq is self.current_tf.
-        let mut doc_iter = self
-            .stack
-            .iter(heap)
-            .chain(Some(self.current_tf).into_iter());
-
-        while let Some(doc) = doc_iter.next() {
-            let term_freq = doc_iter
-                .next()
-                .expect("The IndexWriter recorded a doc without a term freq.");
-            serializer.write_doc(doc, term_freq, &EMPTY_ARRAY)?;
+    fn serialize(
+        &self,
+        buffer_lender: &mut BufferLender,
+        serializer: &mut FieldSerializer,
+        heap: &MemoryArena,
+    ) -> io::Result<()> {
+        let buffer = buffer_lender.lend_u8();
+        self.stack.read_to_end(heap, buffer);
+        let mut u32_it = VInt32Reader::new(&buffer[..]);
+        while let Some(doc) = u32_it.next() {
+            let term_freq = u32_it.next().unwrap_or(self.current_tf);
+            serializer.write_doc(doc as u32, term_freq, &EMPTY_ARRAY)?;
         }
+
         Ok(())
     }
 }
@@ -128,11 +181,10 @@ pub struct TFAndPositionRecorder {
     stack: ExpUnrolledLinkedList,
     current_doc: DocId,
 }
-
 impl Recorder for TFAndPositionRecorder {
-    fn new(heap: &mut MemoryArena) -> Self {
+    fn new() -> Self {
         TFAndPositionRecorder {
-            stack: ExpUnrolledLinkedList::new(heap),
+            stack: ExpUnrolledLinkedList::new(),
             current_doc: u32::max_value(),
         }
     }
@@ -143,33 +195,88 @@ impl Recorder for TFAndPositionRecorder {
 
     fn new_doc(&mut self, doc: DocId, heap: &mut MemoryArena) {
         self.current_doc = doc;
-        self.stack.push(doc, heap);
+        let _ = write_u32_vint(doc, &mut self.stack.writer(heap));
     }
 
     fn record_position(&mut self, position: u32, heap: &mut MemoryArena) {
-        self.stack.push(position, heap);
+        let _ = write_u32_vint(position + 1u32, &mut self.stack.writer(heap));
     }
 
     fn close_doc(&mut self, heap: &mut MemoryArena) {
-        self.stack.push(POSITION_END, heap);
+        let _ = write_u32_vint(POSITION_END, &mut self.stack.writer(heap));
     }
 
-    fn serialize(&self, serializer: &mut FieldSerializer, heap: &MemoryArena) -> io::Result<()> {
-        let mut doc_positions = Vec::with_capacity(100);
-        let mut positions_iter = self.stack.iter(heap);
-        while let Some(doc) = positions_iter.next() {
-            let mut prev_position = 0;
-            doc_positions.clear();
-            for position in &mut positions_iter {
-                if position == POSITION_END {
-                    break;
-                } else {
-                    doc_positions.push(position - prev_position);
-                    prev_position = position;
+    fn serialize(
+        &self,
+        buffer_lender: &mut BufferLender,
+        serializer: &mut FieldSerializer,
+        heap: &MemoryArena,
+    ) -> io::Result<()> {
+        let (buffer_u8, buffer_positions) = buffer_lender.lend_all();
+        self.stack.read_to_end(heap, buffer_u8);
+        let mut u32_it = VInt32Reader::new(&buffer_u8[..]);
+        while let Some(doc) = u32_it.next() {
+            let mut prev_position_plus_one = 1u32;
+            buffer_positions.clear();
+            loop {
+                match u32_it.next() {
+                    Some(POSITION_END) | None => {
+                        break;
+                    }
+                    Some(position_plus_one) => {
+                        let delta_position = position_plus_one - prev_position_plus_one;
+                        buffer_positions.push(delta_position);
+                        prev_position_plus_one = position_plus_one;
+                    }
                 }
             }
-            serializer.write_doc(doc, doc_positions.len() as u32, &doc_positions)?;
+            serializer.write_doc(doc, buffer_positions.len() as u32, &buffer_positions)?;
         }
         Ok(())
     }
 }
+
+#[cfg(test)]
+mod tests {
+
+    use super::write_u32_vint;
+    use super::BufferLender;
+    use super::VInt32Reader;
+
+    #[test]
+    fn test_buffer_lender() {
+        let mut buffer_lender = BufferLender::default();
+        {
+            let buf = buffer_lender.lend_u8();
+            assert!(buf.is_empty());
+            buf.push(1u8);
+        }
+        {
+            let buf = buffer_lender.lend_u8();
+            assert!(buf.is_empty());
+            buf.push(1u8);
+        }
+        {
+            let (_, buf) = buffer_lender.lend_all();
+            assert!(buf.is_empty());
+            buf.push(1u32);
+        }
+        {
+            let (_, buf) = buffer_lender.lend_all();
+            assert!(buf.is_empty());
+            buf.push(1u32);
+        }
+    }
+
+    #[test]
+    fn test_vint_u32() {
+        let mut buffer = vec![];
+        let vals = [0, 1, 324_234_234, u32::max_value()];
+        for &i in &vals {
+            assert!(write_u32_vint(i, &mut buffer).is_ok());
+        }
+        assert_eq!(buffer.len(), 1 + 1 + 5 + 5);
+        let res: Vec<u32> = VInt32Reader::new(&buffer[..]).collect();
+        assert_eq!(&res[..], &vals[..]);
+    }
+}

src/postings/segment_postings.rs

@@ -123,23 +123,23 @@ impl SegmentPostings {
     }
 }
 
-fn exponential_search(target: u32, arr: &[u32]) -> (usize, usize) {
-    let mut start = 0;
+fn linear_search(arr: &[u32], target: u32) -> usize {
+    arr.iter().map(|&el| if el < target { 1 } else { 0 }).sum()
+}
+
+fn exponential_search(arr: &[u32], target: u32) -> (usize, usize) {
     let end = arr.len();
-    debug_assert!(target >= arr[start]);
-    debug_assert!(target <= arr[end - 1]);
-    let mut jump = 1;
-    loop {
-        let new = start + jump;
-        if new >= end {
-            return (start, end);
+    let mut begin = 0;
+    for &pivot in &[1, 3, 7, 15, 31, 63] {
+        if pivot >= end {
+            break;
         }
-        if arr[new] > target {
-            return (start, new);
+        if arr[pivot] > target {
+            return (begin, pivot);
         }
-        start = new;
-        jump *= 2;
+        begin = pivot;
     }
+    (begin, end)
 }
 
 /// Search the first index containing an element greater or equal to the target.
@@ -150,12 +150,8 @@ fn exponential_search(target: u32, arr: &[u32]) -> (usize, usize) {
 /// The target is assumed greater or equal to the first element.
 /// The target is assumed smaller or equal to the last element.
 fn search_within_block(block_docs: &[u32], target: u32) -> usize {
-    let (start, end) = exponential_search(target, block_docs);
-    start.wrapping_add(
-        block_docs[start..end]
-            .binary_search(&target)
-            .unwrap_or_else(|e| e),
-    )
+    let (start, end) = exponential_search(block_docs, target);
+    start + linear_search(&block_docs[start..end], target)
 }
 
 impl DocSet for SegmentPostings {
@@ -216,11 +212,10 @@ impl DocSet for SegmentPostings {
 
         // we're in the right block now, start with an exponential search
         let block_docs = self.block_cursor.docs();
-
-        debug_assert!(target >= self.doc());
         let new_cur = self
             .cur
             .wrapping_add(search_within_block(&block_docs[self.cur..], target));
+
         if need_positions {
             sum_freqs_skipped += self.block_cursor.freqs()[self.cur..new_cur]
                 .iter()
@@ -533,7 +528,8 @@ impl BlockSegmentPostings {
                     } else {
                         BlockSegmentPostingsSkipResult::Terminated
                     }
-                }).unwrap_or(BlockSegmentPostingsSkipResult::Terminated);
+                })
+                .unwrap_or(BlockSegmentPostingsSkipResult::Terminated);
         }
         BlockSegmentPostingsSkipResult::Terminated
     }
@@ -621,6 +617,8 @@ impl<'b> Streamer<'b> for BlockSegmentPostings {
 #[cfg(test)]
 mod tests {
 
+    use super::exponential_search;
+    use super::linear_search;
     use super::search_within_block;
     use super::BlockSegmentPostings;
     use super::BlockSegmentPostingsSkipResult;
@@ -634,6 +632,22 @@ mod tests {
     use schema::Term;
     use schema::INT_INDEXED;
     use DocId;
+    use SkipResult;
+
+    #[test]
+    fn test_linear_search() {
+        let len: usize = 50;
+        let arr: Vec<u32> = (0..len).map(|el| 1u32 + (el as u32) * 2).collect();
+        for target in 1..*arr.last().unwrap() {
+            let res = linear_search(&arr[..], target);
+            if res > 0 {
+                assert!(arr[res - 1] < target);
+            }
+            if res < len {
+                assert!(arr[res] >= target);
+            }
+        }
+    }
 
     #[test]
     fn test_empty_segment_postings() {
@@ -661,6 +675,16 @@ mod tests {
             .0
     }
 
+    #[test]
+    fn test_exponentiel_search() {
+        assert_eq!(exponential_search(&[1, 2], 0), (0, 1));
+        assert_eq!(exponential_search(&[1, 2], 1), (0, 1));
+        assert_eq!(
+            exponential_search(&[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], 7),
+            (3, 7)
+        );
+    }
+
     fn util_test_search_within_block(block: &[u32], target: u32) {
         assert_eq!(
             search_within_block(block, target),
@@ -692,7 +716,7 @@ mod tests {
 
     #[test]
     fn test_block_segment_postings() {
-        let mut block_segments = build_block_postings((0..100_000).collect::<Vec<u32>>());
+        let mut block_segments = build_block_postings(&(0..100_000).collect::<Vec<u32>>());
         let mut offset: u32 = 0u32;
         // checking that the block before calling advance is empty
         assert!(block_segments.docs().is_empty());
@@ -706,14 +730,44 @@ mod tests {
         }
     }
 
-    fn build_block_postings(docs: Vec<DocId>) -> BlockSegmentPostings {
+    #[test]
+    fn test_skip_right_at_new_block() {
+        let mut doc_ids = (0..128).collect::<Vec<u32>>();
+        doc_ids.push(129);
+        doc_ids.push(130);
+        {
+            let block_segments = build_block_postings(&doc_ids);
+            let mut docset = SegmentPostings::from_block_postings(block_segments, None);
+            assert_eq!(docset.skip_next(128), SkipResult::OverStep);
+            assert_eq!(docset.doc(), 129);
+            assert!(docset.advance());
+            assert_eq!(docset.doc(), 130);
+            assert!(!docset.advance());
+        }
+        {
+            let block_segments = build_block_postings(&doc_ids);
+            let mut docset = SegmentPostings::from_block_postings(block_segments, None);
+            assert_eq!(docset.skip_next(129), SkipResult::Reached);
+            assert_eq!(docset.doc(), 129);
+            assert!(docset.advance());
+            assert_eq!(docset.doc(), 130);
+            assert!(!docset.advance());
+        }
+        {
+            let block_segments = build_block_postings(&doc_ids);
+            let mut docset = SegmentPostings::from_block_postings(block_segments, None);
+            assert_eq!(docset.skip_next(131), SkipResult::End);
+        }
+    }
+
+    fn build_block_postings(docs: &[DocId]) -> BlockSegmentPostings {
         let mut schema_builder = Schema::builder();
         let int_field = schema_builder.add_u64_field("id", INT_INDEXED);
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
+        let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
         let mut last_doc = 0u32;
-        for doc in docs {
+        for &doc in docs {
             for _ in last_doc..doc {
                 index_writer.add_document(doc!(int_field=>1u64));
             }
@@ -733,7 +787,7 @@ mod tests {
     #[test]
     fn test_block_segment_postings_skip() {
         for i in 0..4 {
-            let mut block_postings = build_block_postings(vec![3]);
+            let mut block_postings = build_block_postings(&[3]);
             assert_eq!(
                 block_postings.skip_to(i),
                 BlockSegmentPostingsSkipResult::Success(0u32)
@@ -743,7 +797,7 @@ mod tests {
                 BlockSegmentPostingsSkipResult::Terminated
             );
         }
-        let mut block_postings = build_block_postings(vec![3]);
+        let mut block_postings = build_block_postings(&[3]);
         assert_eq!(
             block_postings.skip_to(4u32),
             BlockSegmentPostingsSkipResult::Terminated
@@ -756,7 +810,7 @@ mod tests {
         for i in 0..1300 {
             docs.push((i * i / 100) + i);
         }
-        let mut block_postings = build_block_postings(docs.clone());
+        let mut block_postings = build_block_postings(&docs[..]);
         for i in vec![0, 424, 10000] {
             assert_eq!(
                 block_postings.skip_to(i),
@@ -782,7 +836,7 @@ mod tests {
         let int_field = schema_builder.add_u64_field("id", INT_INDEXED);
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
+        let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
         // create two postings list, one containg even number,
         // the other containing odd numbers.
         for i in 0..6 {

src/postings/serializer.rs

@@ -55,7 +55,7 @@ pub struct InvertedIndexSerializer {
 
 impl InvertedIndexSerializer {
     /// Open a new `PostingsSerializer` for the given segment
-    fn new(
+    fn create(
         terms_write: CompositeWrite<WritePtr>,
         postings_write: CompositeWrite<WritePtr>,
         positions_write: CompositeWrite<WritePtr>,
@@ -74,7 +74,7 @@ impl InvertedIndexSerializer {
     /// Open a new `PostingsSerializer` for the given segment
     pub fn open(segment: &mut Segment) -> Result<InvertedIndexSerializer> {
         use SegmentComponent::{POSITIONS, POSITIONSSKIP, POSTINGS, TERMS};
-        InvertedIndexSerializer::new(
+        InvertedIndexSerializer::create(
             CompositeWrite::wrap(segment.open_write(TERMS)?),
             CompositeWrite::wrap(segment.open_write(POSTINGS)?),
             CompositeWrite::wrap(segment.open_write(POSITIONS)?),
@@ -99,7 +99,7 @@ impl InvertedIndexSerializer {
         let positions_write = self.positions_write.for_field(field);
         let positionsidx_write = self.positionsidx_write.for_field(field);
         let field_type: FieldType = (*field_entry.field_type()).clone();
-        FieldSerializer::new(
+        FieldSerializer::create(
             &field_type,
             term_dictionary_write,
             postings_write,
@@ -130,7 +130,7 @@ pub struct FieldSerializer<'a> {
 }
 
 impl<'a> FieldSerializer<'a> {
-    fn new(
+    fn create(
         field_type: &FieldType,
         term_dictionary_write: &'a mut CountingWriter<WritePtr>,
         postings_write: &'a mut CountingWriter<WritePtr>,
@@ -152,7 +152,7 @@ impl<'a> FieldSerializer<'a> {
             _ => (false, false),
         };
         let term_dictionary_builder =
-            TermDictionaryBuilder::new(term_dictionary_write, &field_type)?;
+            TermDictionaryBuilder::create(term_dictionary_write, &field_type)?;
         let postings_serializer =
             PostingsSerializer::new(postings_write, term_freq_enabled, position_enabled);
         let positions_serializer_opt = if position_enabled {

src/postings/stacker/expull.rs

@@ -1,28 +1,37 @@
 use super::{Addr, MemoryArena};
 
-use common::is_power_of_2;
+use postings::stacker::memory_arena::load;
+use postings::stacker::memory_arena::store;
+use std::io;
 use std::mem;
 
 const MAX_BLOCK_LEN: u32 = 1u32 << 15;
+const FIRST_BLOCK: usize = 16;
+const INLINED_BLOCK_LEN: usize = FIRST_BLOCK + mem::size_of::<Addr>();
 
-const FIRST_BLOCK: u32 = 4u32;
+enum CapacityResult {
+    Available(u32),
+    NeedAlloc(u32),
+}
 
-#[inline]
-pub fn jump_needed(len: u32) -> Option<usize> {
+fn len_to_capacity(len: u32) -> CapacityResult {
     match len {
-        0...3 => None,
-        4...MAX_BLOCK_LEN => {
-            if is_power_of_2(len as usize) {
-                Some(len as usize)
+        0...15 => CapacityResult::Available(FIRST_BLOCK as u32 - len),
+        16...MAX_BLOCK_LEN => {
+            let cap = 1 << (32u32 - (len - 1u32).leading_zeros());
+            let available = cap - len;
+            if available == 0 {
+                CapacityResult::NeedAlloc(len)
             } else {
-                None
+                CapacityResult::Available(available)
             }
         }
         n => {
-            if n % MAX_BLOCK_LEN == 0 {
-                Some(MAX_BLOCK_LEN as usize)
+            let available = n % MAX_BLOCK_LEN;
+            if available == 0 {
+                CapacityResult::NeedAlloc(MAX_BLOCK_LEN)
             } else {
-                None
+                CapacityResult::Available(MAX_BLOCK_LEN - available)
             }
         }
     }
@@ -52,82 +61,119 @@ pub fn jump_needed(len: u32) -> Option<usize> {
 #[derive(Debug, Clone, Copy)]
 pub struct ExpUnrolledLinkedList {
     len: u32,
-    head: Addr,
     tail: Addr,
+    inlined_data: [u8; INLINED_BLOCK_LEN as usize],
+}
+
+pub struct ExpUnrolledLinkedListWriter<'a> {
+    eull: &'a mut ExpUnrolledLinkedList,
+    heap: &'a mut MemoryArena,
+}
+
+fn ensure_capacity<'a>(
+    eull: &'a mut ExpUnrolledLinkedList,
+    heap: &'a mut MemoryArena,
+) -> &'a mut [u8] {
+    if eull.len <= FIRST_BLOCK as u32 {
+        // We are still hitting the inline block.
+        if eull.len < FIRST_BLOCK as u32 {
+            return &mut eull.inlined_data[eull.len as usize..FIRST_BLOCK];
+        }
+        // We need to allocate a new block!
+        let new_block_addr: Addr = heap.allocate_space(FIRST_BLOCK + mem::size_of::<Addr>());
+        store(&mut eull.inlined_data[FIRST_BLOCK..], new_block_addr);
+        eull.tail = new_block_addr;
+        return heap.slice_mut(eull.tail, FIRST_BLOCK);
+    }
+    let len = match len_to_capacity(eull.len) {
+        CapacityResult::NeedAlloc(new_block_len) => {
+            let new_block_addr: Addr =
+                heap.allocate_space(new_block_len as usize + mem::size_of::<Addr>());
+            heap.write_at(eull.tail, new_block_addr);
+            eull.tail = new_block_addr;
+            new_block_len
+        }
+        CapacityResult::Available(available) => available,
+    };
+    heap.slice_mut(eull.tail, len as usize)
+}
+
+impl<'a> ExpUnrolledLinkedListWriter<'a> {
+    pub fn extend_from_slice(&mut self, mut buf: &[u8]) {
+        if buf.is_empty() {
+            // we need to cut early, because `ensure_capacity`
+            // allocates if there is no capacity at all right now.
+            return;
+        }
+        while !buf.is_empty() {
+            let add_len: usize;
+            {
+                let output_buf = ensure_capacity(self.eull, self.heap);
+                add_len = buf.len().min(output_buf.len());
+                output_buf[..add_len].copy_from_slice(&buf[..add_len]);
+            }
+            self.eull.len += add_len as u32;
+            self.eull.tail = self.eull.tail.offset(add_len as u32);
+            buf = &buf[add_len..];
+        }
+    }
+}
+
+impl<'a> io::Write for ExpUnrolledLinkedListWriter<'a> {
+    fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
+        // There is no use case to only write the capacity.
+        // This is not IO after all, so we write the whole
+        // buffer even if the contract of `.write` is looser.
+        self.extend_from_slice(buf);
+        Ok(buf.len())
+    }
+
+    fn write_all(&mut self, buf: &[u8]) -> io::Result<()> {
+        self.extend_from_slice(buf);
+        Ok(())
+    }
+
+    fn flush(&mut self) -> io::Result<()> {
+        Ok(())
+    }
 }
 
 impl ExpUnrolledLinkedList {
-    pub fn new(heap: &mut MemoryArena) -> ExpUnrolledLinkedList {
-        let addr = heap.allocate_space((FIRST_BLOCK as usize) * mem::size_of::<u32>());
+    pub fn new() -> ExpUnrolledLinkedList {
         ExpUnrolledLinkedList {
             len: 0u32,
-            head: addr,
-            tail: addr,
+            tail: Addr::null_pointer(),
+            inlined_data: [0u8; INLINED_BLOCK_LEN as usize],
         }
     }
 
-    pub fn iter<'a>(&self, heap: &'a MemoryArena) -> ExpUnrolledLinkedListIterator<'a> {
-        ExpUnrolledLinkedListIterator {
-            heap,
-            addr: self.head,
-            len: self.len,
-            consumed: 0,
-        }
+    #[inline(always)]
+    pub fn writer<'a>(&'a mut self, heap: &'a mut MemoryArena) -> ExpUnrolledLinkedListWriter<'a> {
+        ExpUnrolledLinkedListWriter { eull: self, heap }
     }
 
-    /// Appends a new element to the current stack.
-    ///
-    /// If the current block end is reached, a new block is allocated.
-    pub fn push(&mut self, val: u32, heap: &mut MemoryArena) {
-        self.len += 1;
-        if let Some(new_block_len) = jump_needed(self.len) {
-            // We need to allocate another block.
-            // We also allocate an extra `u32` to store the pointer
-            // to the future next block.
-            let new_block_size: usize = (new_block_len + 1) * mem::size_of::<u32>();
-            let new_block_addr: Addr = heap.allocate_space(new_block_size);
-            unsafe {
-                // logic
-                heap.write(self.tail, new_block_addr)
-            };
-            self.tail = new_block_addr;
+    pub fn read_to_end(&self, heap: &MemoryArena, output: &mut Vec<u8>) {
+        let len = self.len as usize;
+        if len <= FIRST_BLOCK {
+            output.extend_from_slice(&self.inlined_data[..len]);
+            return;
         }
-        unsafe {
-            // logic
-            heap.write(self.tail, val);
-            self.tail = self.tail.offset(mem::size_of::<u32>() as u32);
-        }
-    }
-}
-
-pub struct ExpUnrolledLinkedListIterator<'a> {
-    heap: &'a MemoryArena,
-    addr: Addr,
-    len: u32,
-    consumed: u32,
-}
-
-impl<'a> Iterator for ExpUnrolledLinkedListIterator<'a> {
-    type Item = u32;
-
-    fn next(&mut self) -> Option<u32> {
-        if self.consumed == self.len {
-            None
-        } else {
-            self.consumed += 1;
-            let addr: Addr = if jump_needed(self.consumed).is_some() {
-                unsafe {
-                    // logic
-                    self.heap.read(self.addr)
-                }
-            } else {
-                self.addr
-            };
-            self.addr = addr.offset(mem::size_of::<u32>() as u32);
-            Some(unsafe {
-                // logic
-                self.heap.read(addr)
-            })
+        output.extend_from_slice(&self.inlined_data[..FIRST_BLOCK]);
+        let mut cur = FIRST_BLOCK;
+        let mut addr = load(&self.inlined_data[FIRST_BLOCK..]);
+        loop {
+            let cap = match len_to_capacity(cur as u32) {
+                CapacityResult::Available(capacity) => capacity,
+                CapacityResult::NeedAlloc(capacity) => capacity,
+            } as usize;
+            let data = heap.slice(addr, cap);
+            if cur + cap >= len {
+                output.extend_from_slice(&data[..(len - cur)]);
+                return;
+            }
+            output.extend_from_slice(data);
+            cur += cap;
+            addr = heap.read(addr.offset(cap as u32));
         }
     }
 }
@@ -136,39 +182,126 @@ impl<'a> Iterator for ExpUnrolledLinkedListIterator<'a> {
 mod tests {
 
     use super::super::MemoryArena;
-    use super::jump_needed;
+    use super::len_to_capacity;
     use super::*;
+    use byteorder::{ByteOrder, LittleEndian, WriteBytesExt};
 
+    #[test]
     #[test]
     fn test_stack() {
         let mut heap = MemoryArena::new();
-        let mut stack = ExpUnrolledLinkedList::new(&mut heap);
-        stack.push(1u32, &mut heap);
-        stack.push(2u32, &mut heap);
-        stack.push(4u32, &mut heap);
-        stack.push(8u32, &mut heap);
+        let mut stack = ExpUnrolledLinkedList::new();
+        stack.writer(&mut heap).extend_from_slice(&[1u8]);
+        stack.writer(&mut heap).extend_from_slice(&[2u8]);
+        stack.writer(&mut heap).extend_from_slice(&[3u8, 4u8]);
+        stack.writer(&mut heap).extend_from_slice(&[5u8]);
         {
-            let mut it = stack.iter(&heap);
-            assert_eq!(it.next().unwrap(), 1u32);
-            assert_eq!(it.next().unwrap(), 2u32);
-            assert_eq!(it.next().unwrap(), 4u32);
-            assert_eq!(it.next().unwrap(), 8u32);
-            assert!(it.next().is_none());
+            let mut buffer = Vec::new();
+            stack.read_to_end(&heap, &mut buffer);
+            assert_eq!(&buffer[..], &[1u8, 2u8, 3u8, 4u8, 5u8]);
         }
     }
 
     #[test]
-    fn test_jump_if_needed() {
-        let mut block_len = 4u32;
-        let mut i = 0;
-        while i < 10_000_000 {
-            assert!(jump_needed(i + block_len - 1).is_none());
-            assert!(jump_needed(i + block_len + 1).is_none());
-            assert!(jump_needed(i + block_len).is_some());
-            let new_block_len = jump_needed(i + block_len).unwrap();
-            i += block_len;
-            block_len = new_block_len as u32;
+    fn test_stack_long() {
+        let mut heap = MemoryArena::new();
+        let mut stack = ExpUnrolledLinkedList::new();
+        let source: Vec<u32> = (0..100).collect();
+        for &el in &source {
+            assert!(stack
+                .writer(&mut heap)
+                .write_u32::<LittleEndian>(el)
+                .is_ok());
         }
+        let mut buffer = Vec::new();
+        stack.read_to_end(&heap, &mut buffer);
+        let mut result = vec![];
+        let mut remaining = &buffer[..];
+        while !remaining.is_empty() {
+            result.push(LittleEndian::read_u32(&remaining[..4]));
+            remaining = &remaining[4..];
+        }
+        assert_eq!(&result[..], &source[..]);
+    }
+
+    #[test]
+    fn test_stack_interlaced() {
+        let mut heap = MemoryArena::new();
+        let mut stack = ExpUnrolledLinkedList::new();
+        let mut stack2 = ExpUnrolledLinkedList::new();
+
+        let mut vec1: Vec<u8> = vec![];
+        let mut vec2: Vec<u8> = vec![];
+
+        for i in 0..9 {
+            assert!(stack.writer(&mut heap).write_u32::<LittleEndian>(i).is_ok());
+            assert!(vec1.write_u32::<LittleEndian>(i).is_ok());
+            if i % 2 == 0 {
+                assert!(stack2
+                    .writer(&mut heap)
+                    .write_u32::<LittleEndian>(i)
+                    .is_ok());
+                assert!(vec2.write_u32::<LittleEndian>(i).is_ok());
+            }
+        }
+        let mut res1 = vec![];
+        let mut res2 = vec![];
+        stack.read_to_end(&heap, &mut res1);
+        stack2.read_to_end(&heap, &mut res2);
+        assert_eq!(&vec1[..], &res1[..]);
+        assert_eq!(&vec2[..], &res2[..]);
+    }
+
+    #[test]
+    fn test_jump_if_needed() {
+        let mut available = 16u32;
+        for i in 0..10_000_000 {
+            match len_to_capacity(i) {
+                CapacityResult::NeedAlloc(cap) => {
+                    assert_eq!(available, 0, "Failed len={}: Expected 0 got {}", i, cap);
+                    available = cap;
+                }
+                CapacityResult::Available(cap) => {
+                    assert_eq!(
+                        available, cap,
+                        "Failed len={}: Expected {} Got {}",
+                        i, available, cap
+                    );
+                }
+            }
+            available -= 1;
+        }
+    }
+
+    #[test]
+    fn test_jump_if_needed_progression() {
+        let mut v = vec![];
+        for i in 0.. {
+            if v.len() >= 10 {
+                break;
+            }
+            match len_to_capacity(i) {
+                CapacityResult::NeedAlloc(cap) => {
+                    v.push((i, cap));
+                }
+                _ => {}
+            }
+        }
+        assert_eq!(
+            &v[..],
+            &[
+                (16, 16),
+                (32, 32),
+                (64, 64),
+                (128, 128),
+                (256, 256),
+                (512, 512),
+                (1024, 1024),
+                (2048, 2048),
+                (4096, 4096),
+                (8192, 8192)
+            ]
+        );
     }
 }
 
@@ -176,6 +309,7 @@ mod tests {
 mod bench {
     use super::super::MemoryArena;
     use super::ExpUnrolledLinkedList;
+    use byteorder::{NativeEndian, WriteBytesExt};
     use test::Bencher;
 
     const NUM_STACK: usize = 10_000;
@@ -203,13 +337,13 @@ mod bench {
             let mut heap = MemoryArena::new();
             let mut stacks = Vec::with_capacity(100);
             for _ in 0..NUM_STACK {
-                let mut stack = ExpUnrolledLinkedList::new(&mut heap);
+                let mut stack = ExpUnrolledLinkedList::new();
                 stacks.push(stack);
             }
             for s in 0..NUM_STACK {
                 for i in 0u32..STACK_SIZE {
                     let t = s * 392017 % NUM_STACK;
-                    stacks[t].push(i, &mut heap);
+                    let _ = stacks[t].writer(&mut heap).write_u32::<NativeEndian>(i);
                 }
             }
         });

src/postings/stacker/memory_arena.rs

@@ -37,7 +37,7 @@ const PAGE_SIZE: usize = 1 << NUM_BITS_PAGE_ADDR; // pages are 1 MB large
 /// page of memory.
 ///
 /// The last 20 bits are an address within this page of memory.
-#[derive(Clone, Copy, Debug)]
+#[derive(Copy, Clone, Debug)]
 pub struct Addr(u32);
 
 impl Addr {
@@ -69,32 +69,16 @@ impl Addr {
     }
 }
 
-/// Trait required for an object to be `storable`.
-///
-/// # Warning
-///
-/// Most of the time you should not implement this trait,
-/// and only use the `MemoryArena` with object implementing `Copy`.
-///
-/// `ArenaStorable` is used in `tantivy` to force
-/// a `Copy` object and a `slice` of data to be stored contiguously.
-pub trait ArenaStorable {
-    fn num_bytes(&self) -> usize;
-    unsafe fn write_into(self, arena: &mut MemoryArena, addr: Addr);
+pub fn store<Item: Copy + 'static>(dest: &mut [u8], val: Item) {
+    assert_eq!(dest.len(), std::mem::size_of::<Item>());
+    unsafe {
+        ptr::write_unaligned(dest.as_mut_ptr() as *mut Item, val);
+    }
 }
 
-impl<V> ArenaStorable for V
-where
-    V: Copy,
-{
-    fn num_bytes(&self) -> usize {
-        mem::size_of::<V>()
-    }
-
-    unsafe fn write_into(self, arena: &mut MemoryArena, addr: Addr) {
-        let dst_ptr = arena.get_mut_ptr(addr) as *mut V;
-        ptr::write_unaligned(dst_ptr, self);
-    }
+pub fn load<Item: Copy + 'static>(data: &[u8]) -> Item {
+    assert_eq!(data.len(), std::mem::size_of::<Item>());
+    unsafe { ptr::read_unaligned(data.as_ptr() as *const Item) }
 }
 
 /// The `MemoryArena`
@@ -126,47 +110,9 @@ impl MemoryArena {
         self.pages.len() * PAGE_SIZE
     }
 
-    /// Writes a slice at the given address, assuming the
-    /// memory was allocated beforehands.
-    ///
-    ///  # Panics
-    ///
-    /// May panic or corrupt the heap if he space was not
-    /// properly allocated beforehands.
-    pub fn write_bytes<B: AsRef<[u8]>>(&mut self, addr: Addr, data: B) {
-        let bytes = data.as_ref();
-        self.pages[addr.page_id()]
-            .get_mut_slice(addr.page_local_addr(), bytes.len())
-            .copy_from_slice(bytes);
-    }
-
-    /// Returns the `len` bytes starting at `addr`
-    ///
-    /// # Panics
-    ///
-    /// Panics if the memory has not been allocated beforehands.
-    pub fn read_slice(&self, addr: Addr, len: usize) -> &[u8] {
-        self.pages[addr.page_id()].get_slice(addr.page_local_addr(), len)
-    }
-
-    unsafe fn get_mut_ptr(&mut self, addr: Addr) -> *mut u8 {
-        self.pages[addr.page_id()].get_mut_ptr(addr.page_local_addr())
-    }
-
-    /// Stores an item's data in the heap
-    ///
-    /// It allocates the `Item` beforehands.
-    pub fn store<Item: ArenaStorable>(&mut self, val: Item) -> Addr {
-        let num_bytes = val.num_bytes();
-        let addr = self.allocate_space(num_bytes);
-        unsafe {
-            self.write(addr, val);
-        };
-        addr
-    }
-
-    pub unsafe fn write<Item: ArenaStorable>(&mut self, addr: Addr, val: Item) {
-        val.write_into(self, addr)
+    pub fn write_at<Item: Copy + 'static>(&mut self, addr: Addr, val: Item) {
+        let dest = self.slice_mut(addr, std::mem::size_of::<Item>());
+        store(dest, val);
     }
 
     /// Read an item in the heap at the given `address`.
@@ -174,9 +120,21 @@ impl MemoryArena {
     /// # Panics
     ///
     /// If the address is erroneous
-    pub unsafe fn read<Item: Copy>(&self, addr: Addr) -> Item {
-        let ptr = self.pages[addr.page_id()].get_ptr(addr.page_local_addr());
-        ptr::read_unaligned(ptr as *const Item)
+    pub fn read<Item: Copy + 'static>(&self, addr: Addr) -> Item {
+        load(self.slice(addr, mem::size_of::<Item>()))
+    }
+
+    pub fn slice(&self, addr: Addr, len: usize) -> &[u8] {
+        self.pages[addr.page_id()].slice(addr.page_local_addr(), len)
+    }
+
+    pub fn slice_from(&self, addr: Addr) -> &[u8] {
+        self.pages[addr.page_id()].slice_from(addr.page_local_addr())
+    }
+
+    #[inline(always)]
+    pub fn slice_mut(&mut self, addr: Addr, len: usize) -> &mut [u8] {
+        self.pages[addr.page_id()].slice_mut(addr.page_local_addr(), len)
     }
 
     /// Allocates `len` bytes and returns the allocated address.
@@ -197,14 +155,10 @@ struct Page {
 
 impl Page {
     fn new(page_id: usize) -> Page {
-        let mut data: Vec<u8> = Vec::with_capacity(PAGE_SIZE);
-        unsafe {
-            data.set_len(PAGE_SIZE);
-        } // avoid initializing page
         Page {
             page_id,
             len: 0,
-            data: data.into_boxed_slice(),
+            data: vec![0u8; PAGE_SIZE].into_boxed_slice(),
         }
     }
 
@@ -213,12 +167,16 @@ impl Page {
         len + self.len <= PAGE_SIZE
     }
 
-    fn get_mut_slice(&mut self, local_addr: usize, len: usize) -> &mut [u8] {
-        &mut self.data[local_addr..][..len]
+    fn slice(&self, local_addr: usize, len: usize) -> &[u8] {
+        &self.slice_from(local_addr)[..len]
     }
 
-    fn get_slice(&self, local_addr: usize, len: usize) -> &[u8] {
-        &self.data[local_addr..][..len]
+    fn slice_from(&self, local_addr: usize) -> &[u8] {
+        &self.data[local_addr..]
+    }
+
+    fn slice_mut(&mut self, local_addr: usize, len: usize) -> &mut [u8] {
+        &mut self.data[local_addr..][..len]
     }
 
     fn allocate_space(&mut self, len: usize) -> Option<Addr> {
@@ -230,16 +188,6 @@ impl Page {
             None
         }
     }
-
-    #[inline(always)]
-    pub(crate) unsafe fn get_ptr(&self, addr: usize) -> *const u8 {
-        self.data.as_ptr().add(addr)
-    }
-
-    #[inline(always)]
-    pub(crate) unsafe fn get_mut_ptr(&mut self, addr: usize) -> *mut u8 {
-        self.data.as_mut_ptr().add(addr)
-    }
 }
 
 #[cfg(test)]
@@ -254,13 +202,13 @@ mod tests {
         let b = b"happy tax payer";
 
         let addr_a = arena.allocate_space(a.len());
-        arena.write_bytes(addr_a, a);
+        arena.slice_mut(addr_a, a.len()).copy_from_slice(a);
 
         let addr_b = arena.allocate_space(b.len());
-        arena.write_bytes(addr_b, b);
+        arena.slice_mut(addr_b, b.len()).copy_from_slice(b);
 
-        assert_eq!(arena.read_slice(addr_a, a.len()), a);
-        assert_eq!(arena.read_slice(addr_b, b.len()), b);
+        assert_eq!(arena.slice(addr_a, a.len()), a);
+        assert_eq!(arena.slice(addr_b, b.len()), b);
     }
 
     #[derive(Clone, Copy, Debug, Eq, PartialEq)]
@@ -283,9 +231,15 @@ mod tests {
             b: 221,
             c: 12,
         };
-        let addr_a = arena.store(a);
-        let addr_b = arena.store(b);
-        assert_eq!(unsafe { arena.read::<MyTest>(addr_a) }, a);
-        assert_eq!(unsafe { arena.read::<MyTest>(addr_b) }, b);
+
+        let num_bytes = std::mem::size_of::<MyTest>();
+        let addr_a = arena.allocate_space(num_bytes);
+        arena.write_at(addr_a, a);
+
+        let addr_b = arena.allocate_space(num_bytes);
+        arena.write_at(addr_b, b);
+
+        assert_eq!(arena.read::<MyTest>(addr_a), a);
+        assert_eq!(arena.read::<MyTest>(addr_b), b);
     }
 }

src/postings/stacker/mod.rs

@@ -1,9 +1,7 @@
 mod expull;
 mod memory_arena;
-mod murmurhash2;
 mod term_hashmap;
 
 pub use self::expull::ExpUnrolledLinkedList;
-pub use self::memory_arena::{Addr, ArenaStorable, MemoryArena};
-use self::murmurhash2::murmurhash2;
+pub use self::memory_arena::{Addr, MemoryArena};
 pub use self::term_hashmap::{compute_table_size, TermHashMap};

src/postings/stacker/murmurhash2.rs

@@ -1,87 +0,0 @@
-use std::ptr;
-const SEED: u32 = 3_242_157_231u32;
-const M: u32 = 0x5bd1_e995;
-
-#[inline(always)]
-pub fn murmurhash2(key: &[u8]) -> u32 {
-    #[cfg_attr(feature = "cargo-clippy", allow(clippy::cast_ptr_alignment))]
-    let mut key_ptr: *const u32 = key.as_ptr() as *const u32;
-    let len = key.len() as u32;
-    let mut h: u32 = SEED ^ len;
-
-    let num_blocks = len >> 2;
-    for _ in 0..num_blocks {
-        let mut k: u32 = unsafe { ptr::read_unaligned(key_ptr) }; // ok because of num_blocks definition
-        k = k.wrapping_mul(M);
-        k ^= k >> 24;
-        k = k.wrapping_mul(M);
-        h = h.wrapping_mul(M);
-        h ^= k;
-        key_ptr = key_ptr.wrapping_offset(1);
-    }
-
-    // Handle the last few bytes of the input array
-    let remaining: &[u8] = &key[key.len() & !3..];
-    match remaining.len() {
-        3 => {
-            h ^= u32::from(remaining[2]) << 16;
-            h ^= u32::from(remaining[1]) << 8;
-            h ^= u32::from(remaining[0]);
-            h = h.wrapping_mul(M);
-        }
-        2 => {
-            h ^= u32::from(remaining[1]) << 8;
-            h ^= u32::from(remaining[0]);
-            h = h.wrapping_mul(M);
-        }
-        1 => {
-            h ^= u32::from(remaining[0]);
-            h = h.wrapping_mul(M);
-        }
-        _ => {}
-    }
-    h ^= h >> 13;
-    h = h.wrapping_mul(M);
-    h ^ (h >> 15)
-}
-
-#[cfg(test)]
-mod test {
-
-    use super::murmurhash2;
-    use std::collections::HashSet;
-
-    #[test]
-    fn test_murmur() {
-        let s1 = "abcdef";
-        let s2 = "abcdeg";
-        for i in 0..5 {
-            assert_eq!(
-                murmurhash2(&s1[i..5].as_bytes()),
-                murmurhash2(&s2[i..5].as_bytes())
-            );
-        }
-    }
-
-    #[test]
-    fn test_murmur_against_reference_impl() {
-        assert_eq!(murmurhash2("".as_bytes()), 3632506080);
-        assert_eq!(murmurhash2("a".as_bytes()), 455683869);
-        assert_eq!(murmurhash2("ab".as_bytes()), 2448092234);
-        assert_eq!(murmurhash2("abc".as_bytes()), 2066295634);
-        assert_eq!(murmurhash2("abcd".as_bytes()), 2588571162);
-        assert_eq!(murmurhash2("abcde".as_bytes()), 2988696942);
-        assert_eq!(murmurhash2("abcdefghijklmnop".as_bytes()), 2350868870);
-    }
-
-    #[test]
-    fn test_murmur_collisions() {
-        let mut set: HashSet<u32> = HashSet::default();
-        for i in 0..10_000 {
-            let s = format!("hash{}", i);
-            let hash = murmurhash2(s.as_bytes());
-            set.insert(hash);
-        }
-        assert_eq!(set.len(), 10_000);
-    }
-}

src/postings/stacker/term_hashmap.rs

@@ -1,37 +1,16 @@
-use super::murmurhash2;
-use super::{Addr, ArenaStorable, MemoryArena};
+extern crate murmurhash32;
+
+use self::murmurhash32::murmurhash2;
+
+use super::{Addr, MemoryArena};
+use byteorder::{ByteOrder, NativeEndian};
+use postings::stacker::memory_arena::store;
 use std::iter;
 use std::mem;
 use std::slice;
 
 pub type BucketId = usize;
 
-struct KeyBytesValue<'a, V> {
-    key: &'a [u8],
-    value: V,
-}
-
-impl<'a, V> KeyBytesValue<'a, V> {
-    fn new(key: &'a [u8], value: V) -> KeyBytesValue<'a, V> {
-        KeyBytesValue { key, value }
-    }
-}
-
-impl<'a, V> ArenaStorable for KeyBytesValue<'a, V>
-where
-    V: ArenaStorable,
-{
-    fn num_bytes(&self) -> usize {
-        0u16.num_bytes() + self.key.len() + self.value.num_bytes()
-    }
-
-    unsafe fn write_into(self, arena: &mut MemoryArena, addr: Addr) {
-        arena.write(addr, self.key.len() as u16);
-        arena.write_bytes(addr.offset(2), self.key);
-        arena.write(addr.offset(2 + self.key.len() as u32), self.value);
-    }
-}
-
 /// Returns the actual memory size in bytes
 /// required to create a table of size $2^num_bits$.
 pub fn compute_table_size(num_bits: usize) -> usize {
@@ -111,8 +90,7 @@ impl<'a> Iterator for Iter<'a> {
     fn next(&mut self) -> Option<Self::Item> {
         self.inner.next().cloned().map(move |bucket: usize| {
             let kv = self.hashmap.table[bucket];
-            let (key, offset): (&'a [u8], Addr) =
-                unsafe { self.hashmap.get_key_value(kv.key_value_addr) };
+            let (key, offset): (&'a [u8], Addr) = self.hashmap.get_key_value(kv.key_value_addr);
             (key, offset, bucket as BucketId)
         })
     }
@@ -143,12 +121,22 @@ impl TermHashMap {
         self.table.len() < self.occupied.len() * 3
     }
 
-    unsafe fn get_key_value(&self, addr: Addr) -> (&[u8], Addr) {
-        let key_bytes_len = self.heap.read::<u16>(addr) as usize;
-        let key_addr = addr.offset(2u32);
-        let key_bytes: &[u8] = self.heap.read_slice(key_addr, key_bytes_len);
-        let val_addr: Addr = key_addr.offset(key_bytes.len() as u32);
-        (key_bytes, val_addr)
+    #[inline(always)]
+    fn get_key_value(&self, addr: Addr) -> (&[u8], Addr) {
+        let data = self.heap.slice_from(addr);
+        let key_bytes_len = NativeEndian::read_u16(data) as usize;
+        let key_bytes: &[u8] = &data[2..][..key_bytes_len];
+        (key_bytes, addr.offset(2u32 + key_bytes_len as u32))
+    }
+
+    #[inline(always)]
+    fn get_value_addr_if_key_match(&self, target_key: &[u8], addr: Addr) -> Option<Addr> {
+        let (stored_key, value_addr) = self.get_key_value(addr);
+        if stored_key == target_key {
+            Some(value_addr)
+        } else {
+            None
+        }
     }
 
     pub fn set_bucket(&mut self, hash: u32, key_value_addr: Addr, bucket: usize) {
@@ -199,36 +187,39 @@ impl TermHashMap {
     pub fn mutate_or_create<S, V, TMutator>(&mut self, key: S, mut updater: TMutator) -> BucketId
     where
         S: AsRef<[u8]>,
-        V: Copy,
+        V: Copy + 'static,
         TMutator: FnMut(Option<V>) -> V,
     {
         if self.is_saturated() {
             self.resize();
         }
         let key_bytes: &[u8] = key.as_ref();
-        let hash = murmurhash2::murmurhash2(key.as_ref());
+        let hash = murmurhash2(key.as_ref());
         let mut probe = self.probe(hash);
         loop {
             let bucket = probe.next_probe();
             let kv: KeyValue = self.table[bucket];
             if kv.is_empty() {
                 let val = updater(None);
-                let key_addr = self.heap.store(KeyBytesValue::new(key_bytes, val));
+                let num_bytes =
+                    std::mem::size_of::<u16>() + key_bytes.len() + std::mem::size_of::<V>();
+                let key_addr = self.heap.allocate_space(num_bytes);
+                {
+                    let data = self.heap.slice_mut(key_addr, num_bytes);
+                    NativeEndian::write_u16(data, key_bytes.len() as u16);
+                    let stop = 2 + key_bytes.len();
+                    data[2..stop].copy_from_slice(key_bytes);
+                    store(&mut data[stop..], val);
+                }
                 self.set_bucket(hash, key_addr, bucket);
                 return bucket as BucketId;
             } else if kv.hash == hash {
-                let (key_matches, val_addr) = {
-                    let (stored_key, val_addr): (&[u8], Addr) =
-                        unsafe { self.get_key_value(kv.key_value_addr) };
-                    (stored_key == key_bytes, val_addr)
-                };
-                if key_matches {
-                    unsafe {
-                        // logic
-                        let v = self.heap.read(val_addr);
-                        let new_v = updater(Some(v));
-                        self.heap.write(val_addr, new_v);
-                    };
+                if let Some(val_addr) =
+                    self.get_value_addr_if_key_match(key_bytes, kv.key_value_addr)
+                {
+                    let v = self.heap.read(val_addr);
+                    let new_v = updater(Some(v));
+                    self.heap.write_at(val_addr, new_v);
                     return bucket as BucketId;
                 }
             }
@@ -236,24 +227,6 @@ impl TermHashMap {
     }
 }
 
-#[cfg(all(test, feature = "unstable"))]
-mod bench {
-    use super::murmurhash2::murmurhash2;
-    use test::Bencher;
-
-    #[bench]
-    fn bench_murmurhash2(b: &mut Bencher) {
-        let keys: [&'static str; 3] = ["wer qwe qwe qwe ", "werbq weqweqwe2 ", "weraq weqweqwe3 "];
-        b.iter(|| {
-            let mut s = 0;
-            for &key in &keys {
-                s ^= murmurhash2(key.as_bytes());
-            }
-            s
-        });
-    }
-}
-
 #[cfg(test)]
 mod tests {
 
@@ -285,10 +258,7 @@ mod tests {
         let mut vanilla_hash_map = HashMap::new();
         let mut iter_values = hash_map.iter();
         while let Some((key, addr, _)) = iter_values.next() {
-            let val: u32 = unsafe {
-                // test
-                hash_map.heap.read(addr)
-            };
+            let val: u32 = hash_map.heap.read(addr);
             vanilla_hash_map.insert(key.to_owned(), val);
         }
         assert_eq!(vanilla_hash_map.len(), 2);

src/query/bm25.rs

@@ -63,7 +63,8 @@ impl BM25Weight {
             .map(|term| {
                 let term_doc_freq = searcher.doc_freq(term);
                 idf(term_doc_freq, total_num_docs)
-            }).sum::<f32>();
+            })
+            .sum::<f32>();
         BM25Weight::new(idf, average_fieldnorm)
     }
 

src/query/boolean_query/boolean_query.rs

@@ -47,7 +47,8 @@ impl Query for BooleanQuery {
             .iter()
             .map(|&(ref occur, ref subquery)| {
                 Ok((*occur, subquery.weight(searcher, scoring_enabled)?))
-            }).collect::<Result<_>>()?;
+            })
+            .collect::<Result<_>>()?;
         Ok(Box::new(BooleanWeight::new(sub_weights, scoring_enabled)))
     }
 
@@ -68,7 +69,8 @@ impl BooleanQuery {
                 let term_query: Box<Query> =
                     Box::new(TermQuery::new(term, IndexRecordOption::WithFreqs));
                 (Occur::Should, term_query)
-            }).collect();
+            })
+            .collect();
         BooleanQuery::from(occur_term_queries)
     }
 

src/query/boolean_query/boolean_weight.rs

@@ -1,5 +1,4 @@
 use core::SegmentReader;
-use downcast::Downcast;
 use query::intersect_scorers;
 use query::score_combiner::{DoNothingCombiner, ScoreCombiner, SumWithCoordsCombiner};
 use query::term_query::TermScorer;
@@ -10,7 +9,6 @@ use query::RequiredOptionalScorer;
 use query::Scorer;
 use query::Union;
 use query::Weight;
-use std::borrow::Borrow;
 use std::collections::HashMap;
 use Result;
 
@@ -24,14 +22,11 @@ where
     }
 
     {
-        let is_all_term_queries = scorers.iter().all(|scorer| {
-            let scorer_ref: &Scorer = scorer.borrow();
-            Downcast::<TermScorer>::is_type(scorer_ref)
-        });
+        let is_all_term_queries = scorers.iter().all(|scorer| scorer.is::<TermScorer>());
         if is_all_term_queries {
             let scorers: Vec<TermScorer> = scorers
                 .into_iter()
-                .map(|scorer| *Downcast::<TermScorer>::downcast(scorer).unwrap())
+                .map(|scorer| *(scorer.downcast::<TermScorer>().map_err(|_| ()).unwrap()))
                 .collect();
             let scorer: Box<Scorer> = Box::new(Union::<TermScorer, TScoreCombiner>::from(scorers));
             return scorer;

src/query/boolean_query/mod.rs

@@ -8,7 +8,6 @@ mod tests {
 
     use super::*;
     use collector::tests::TestCollector;
-    use downcast::Downcast;
     use query::score_combiner::SumWithCoordsCombiner;
     use query::term_query::TermScorer;
     use query::Intersection;
@@ -29,7 +28,7 @@ mod tests {
         let index = Index::create_in_ram(schema);
         {
             // writing the segment
-            let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
+            let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
             {
                 let doc = doc!(text_field => "a b c");
                 index_writer.add_document(doc);
@@ -72,7 +71,7 @@ mod tests {
         let searcher = index.searcher();
         let weight = query.weight(&searcher, true).unwrap();
         let scorer = weight.scorer(searcher.segment_reader(0u32)).unwrap();
-        assert!(Downcast::<TermScorer>::is_type(&*scorer));
+        assert!(scorer.is::<TermScorer>());
     }
 
     #[test]
@@ -84,13 +83,13 @@ mod tests {
             let query = query_parser.parse_query("+a +b +c").unwrap();
             let weight = query.weight(&searcher, true).unwrap();
             let scorer = weight.scorer(searcher.segment_reader(0u32)).unwrap();
-            assert!(Downcast::<Intersection<TermScorer>>::is_type(&*scorer));
+            assert!(scorer.is::<Intersection<TermScorer>>());
         }
         {
             let query = query_parser.parse_query("+a +(b c)").unwrap();
             let weight = query.weight(&searcher, true).unwrap();
             let scorer = weight.scorer(searcher.segment_reader(0u32)).unwrap();
-            assert!(Downcast::<Intersection<Box<Scorer>>>::is_type(&*scorer));
+            assert!(scorer.is::<Intersection<Box<Scorer>>>());
         }
     }
 
@@ -103,16 +102,14 @@ mod tests {
             let query = query_parser.parse_query("+a b").unwrap();
             let weight = query.weight(&searcher, true).unwrap();
             let scorer = weight.scorer(searcher.segment_reader(0u32)).unwrap();
-            assert!(Downcast::<
-                RequiredOptionalScorer<Box<Scorer>, Box<Scorer>, SumWithCoordsCombiner>,
-            >::is_type(&*scorer));
+            assert!(scorer
+                .is::<RequiredOptionalScorer<Box<Scorer>, Box<Scorer>, SumWithCoordsCombiner>>());
         }
         {
             let query = query_parser.parse_query("+a b").unwrap();
             let weight = query.weight(&searcher, false).unwrap();
             let scorer = weight.scorer(searcher.segment_reader(0u32)).unwrap();
-            println!("{:?}", scorer.type_name());
-            assert!(Downcast::<TermScorer>::is_type(&*scorer));
+            assert!(scorer.is::<TermScorer>());
         }
     }
 

src/query/intersection.rs

@@ -1,9 +1,7 @@
 use docset::{DocSet, SkipResult};
-use downcast::Downcast;
 use query::term_query::TermScorer;
 use query::EmptyScorer;
 use query::Scorer;
-use std::borrow::Borrow;
 use DocId;
 use Score;
 
@@ -26,13 +24,12 @@ pub fn intersect_scorers(mut scorers: Vec<Box<Scorer>>) -> Box<Scorer> {
         (Some(single_docset), None) => single_docset,
         (Some(left), Some(right)) => {
             {
-                let all_term_scorers = [&left, &right].into_iter().all(|scorer| {
-                    let scorer_ref: &Scorer = (*scorer).borrow();
-                    Downcast::<TermScorer>::is_type(scorer_ref)
-                });
+                let all_term_scorers = [&left, &right]
+                    .iter()
+                    .all(|&scorer| scorer.is::<TermScorer>());
                 if all_term_scorers {
-                    let left = *Downcast::<TermScorer>::downcast(left).unwrap();
-                    let right = *Downcast::<TermScorer>::downcast(right).unwrap();
+                    let left = *(left.downcast::<TermScorer>().map_err(|_| ()).unwrap());
+                    let right = *(right.downcast::<TermScorer>().map_err(|_| ()).unwrap());
                     return Box::new(Intersection {
                         left,
                         right,

src/query/phrase_query/mod.rs

@@ -24,7 +24,7 @@ mod tests {
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
         {
-            let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
+            let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
             for &text in texts {
                 let doc = doc!(text_field=>text);
                 index_writer.add_document(doc);
@@ -86,7 +86,7 @@ mod tests {
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
         {
-            let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
+            let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
             index_writer.add_document(doc!(text_field=>"a b c"));
             assert!(index_writer.commit().is_ok());
         }
@@ -141,7 +141,7 @@ mod tests {
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
         {
-            let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
+            let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
             index_writer.add_document(doc!(text_field=>"b"));
             index_writer.add_document(doc!(text_field=>"a b"));
             index_writer.add_document(doc!(text_field=>"b a"));
@@ -173,7 +173,7 @@ mod tests {
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
         {
-            let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
+            let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
             index_writer.add_document(doc!(text_field=>"a b c d e f g h"));
             assert!(index_writer.commit().is_ok());
         }

src/query/phrase_query/phrase_scorer.rs

@@ -43,7 +43,7 @@ impl<TPostings: Postings> DocSet for PostingsWithOffset<TPostings> {
 
 pub struct PhraseScorer<TPostings: Postings> {
     intersection_docset: Intersection<PostingsWithOffset<TPostings>, PostingsWithOffset<TPostings>>,
-    num_docsets: usize,
+    num_terms: usize,
     left: Vec<u32>,
     right: Vec<u32>,
     phrase_count: u32,
@@ -134,10 +134,11 @@ impl<TPostings: Postings> PhraseScorer<TPostings> {
             .into_iter()
             .map(|(offset, postings)| {
                 PostingsWithOffset::new(postings, (max_offset - offset) as u32)
-            }).collect::<Vec<_>>();
+            })
+            .collect::<Vec<_>>();
         PhraseScorer {
             intersection_docset: Intersection::new(postings_with_offsets),
-            num_docsets,
+            num_terms: num_docsets,
             left: Vec::with_capacity(100),
             right: Vec::with_capacity(100),
             phrase_count: 0u32,
@@ -164,7 +165,7 @@ impl<TPostings: Postings> PhraseScorer<TPostings> {
                 .positions(&mut self.left);
         }
         let mut intersection_len = self.left.len();
-        for i in 1..self.num_docsets - 1 {
+        for i in 1..self.num_terms - 1 {
             {
                 self.intersection_docset
                     .docset_mut_specialized(i)
@@ -177,7 +178,7 @@ impl<TPostings: Postings> PhraseScorer<TPostings> {
         }
 
         self.intersection_docset
-            .docset_mut_specialized(self.num_docsets - 1)
+            .docset_mut_specialized(self.num_terms - 1)
             .positions(&mut self.right);
         intersection_exists(&self.left[..intersection_len], &self.right[..])
     }
@@ -189,7 +190,7 @@ impl<TPostings: Postings> PhraseScorer<TPostings> {
                 .positions(&mut self.left);
         }
         let mut intersection_len = self.left.len();
-        for i in 1..self.num_docsets - 1 {
+        for i in 1..self.num_terms - 1 {
             {
                 self.intersection_docset
                     .docset_mut_specialized(i)
@@ -202,7 +203,7 @@ impl<TPostings: Postings> PhraseScorer<TPostings> {
         }
 
         self.intersection_docset
-            .docset_mut_specialized(self.num_docsets - 1)
+            .docset_mut_specialized(self.num_terms - 1)
             .positions(&mut self.right);
         intersection_count(&self.left[..intersection_len], &self.right[..]) as u32
     }

src/query/query.rs

@@ -1,6 +1,6 @@
 use super::Weight;
 use core::searcher::Searcher;
-use downcast;
+use downcast_rs;
 use std::collections::BTreeSet;
 use std::fmt;
 use Result;
@@ -39,7 +39,7 @@ use Term;
 ///
 /// When implementing a new type of `Query`, it is normal to implement a
 /// dedicated `Query`, `Weight` and `Scorer`.
-pub trait Query: QueryClone + downcast::Any + fmt::Debug {
+pub trait Query: QueryClone + downcast_rs::Downcast + fmt::Debug {
     /// Create the weight associated to a query.
     ///
     /// If scoring is not required, setting `scoring_enabled` to `false`
@@ -96,7 +96,4 @@ impl QueryClone for Box<Query> {
     }
 }
 
-#[allow(missing_docs)]
-mod downcast_impl {
-    downcast!(super::Query);
-}
+impl_downcast!(Query);

src/query/query_parser/query_parser.rs

@@ -68,7 +68,8 @@ fn trim_ast(logical_ast: LogicalAST) -> Option<LogicalAST> {
                 .into_iter()
                 .flat_map(|(occur, child)| {
                     trim_ast(child).map(|trimmed_child| (occur, trimmed_child))
-                }).collect::<Vec<_>>();
+                })
+                .collect::<Vec<_>>();
             if trimmed_children.is_empty() {
                 None
             } else {
@@ -422,7 +423,8 @@ impl QueryParser {
                             lower: self.resolve_bound(field, &lower)?,
                             upper: self.resolve_bound(field, &upper)?,
                         })))
-                    }).collect::<Result<Vec<_>, QueryParserError>>()?;
+                    })
+                    .collect::<Result<Vec<_>, QueryParserError>>()?;
                 let result_ast = if clauses.len() == 1 {
                     clauses.pop().unwrap()
                 } else {
@@ -598,25 +600,19 @@ mod test {
         assert!(query_parser.parse_query("signed:2324").is_ok());
         assert!(query_parser.parse_query("signed:\"22\"").is_ok());
         assert!(query_parser.parse_query("signed:\"-2234\"").is_ok());
-        assert!(
-            query_parser
-                .parse_query("signed:\"-9999999999999\"")
-                .is_ok()
-        );
+        assert!(query_parser
+            .parse_query("signed:\"-9999999999999\"")
+            .is_ok());
         assert!(query_parser.parse_query("signed:\"a\"").is_err());
         assert!(query_parser.parse_query("signed:\"2a\"").is_err());
-        assert!(
-            query_parser
-                .parse_query("signed:\"18446744073709551615\"")
-                .is_err()
-        );
+        assert!(query_parser
+            .parse_query("signed:\"18446744073709551615\"")
+            .is_err());
         assert!(query_parser.parse_query("unsigned:\"2\"").is_ok());
         assert!(query_parser.parse_query("unsigned:\"-2\"").is_err());
-        assert!(
-            query_parser
-                .parse_query("unsigned:\"18446744073709551615\"")
-                .is_ok()
-        );
+        assert!(query_parser
+            .parse_query("unsigned:\"18446744073709551615\"")
+            .is_ok());
         test_parse_query_to_logical_ast_helper(
             "unsigned:2324",
             "Term([0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 9, 20])",

src/query/scorer.rs

@@ -1,6 +1,6 @@
 use common::BitSet;
 use docset::{DocSet, SkipResult};
-use downcast;
+use downcast_rs;
 use std::ops::DerefMut;
 use DocId;
 use Score;
@@ -8,7 +8,7 @@ use Score;
 /// Scored set of documents matching a query within a specific segment.
 ///
 /// See [`Query`](./trait.Query.html).
-pub trait Scorer: downcast::Any + DocSet + 'static {
+pub trait Scorer: downcast_rs::Downcast + DocSet + 'static {
     /// Returns the score.
     ///
     /// This method will perform a bit of computation and is not cached.
@@ -23,10 +23,7 @@ pub trait Scorer: downcast::Any + DocSet + 'static {
     }
 }
 
-#[allow(missing_docs)]
-mod downcast_impl {
-    downcast!(super::Scorer);
-}
+impl_downcast!(Scorer);
 
 impl Scorer for Box<Scorer> {
     fn score(&mut self) -> Score {

src/query/term_query/mod.rs

@@ -1,4 +1,3 @@
-
 mod term_query;
 mod term_scorer;
 mod term_weight;
@@ -26,7 +25,7 @@ mod tests {
         let index = Index::create_in_ram(schema);
         {
             // writing the segment
-            let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
+            let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
             {
                 let doc = doc!(text_field => "a");
                 index_writer.add_document(doc);

src/query/union.rs

@@ -55,7 +55,8 @@ where
                         None
                     }
                 },
-            ).collect();
+            )
+            .collect();
         Union {
             docsets: non_empty_docsets,
             bitsets: Box::new([TinySet::empty(); HORIZON_NUM_TINYBITSETS]),
@@ -214,10 +215,7 @@ where
 
             // The target is outside of the buffered horizon.
             // advance all docsets to a doc >= to the target.
-            #[cfg_attr(
-                feature = "cargo-clippy",
-                allow(clippy::clippy::collapsible_if)
-            )]
+            #[cfg_attr(feature = "cargo-clippy", allow(clippy::clippy::collapsible_if))]
             unordered_drain_filter(&mut self.docsets, |docset| {
                 if docset.doc() < target {
                     if docset.skip_next(target) == SkipResult::End {

src/schema/facet.rs

@@ -6,6 +6,7 @@ use std::borrow::Cow;
 use std::fmt::{self, Debug, Display, Formatter};
 use std::io::{self, Read, Write};
 use std::str;
+use std::string::FromUtf8Error;
 
 const SLASH_BYTE: u8 = b'/';
 const ESCAPE_BYTE: u8 = b'\\';
@@ -14,6 +15,10 @@ const ESCAPE_BYTE: u8 = b'\\';
 /// representation of facets.
 pub const FACET_SEP_BYTE: u8 = 0u8;
 
+/// `char` used as a level separation in the binary
+/// representation of facets. (It is the null codepoint.)
+pub const FACET_SEP_CHAR: char = '\u{0}';
+
 /// A Facet represent a point in a given hierarchy.
 ///
 /// They are typically represented similarly to a filepath.
@@ -26,18 +31,18 @@ pub const FACET_SEP_BYTE: u8 = 0u8;
 /// its facet. In the example above, `/electronics/tv_and_video/`
 /// and `/electronics`.
 #[derive(Clone, Eq, Hash, PartialEq, Ord, PartialOrd)]
-pub struct Facet(Vec<u8>);
+pub struct Facet(String);
 
 impl Facet {
     /// Returns a new instance of the "root facet"
     /// Equivalent to `/`.
     pub fn root() -> Facet {
-        Facet(vec![])
+        Facet("".to_string())
     }
 
     /// Returns true iff the facet is the root facet `/`.
     pub fn is_root(&self) -> bool {
-        self.encoded_bytes().is_empty()
+        self.encoded_str().is_empty()
     }
 
     /// Returns a binary representation of the facet.
@@ -49,13 +54,19 @@ impl Facet {
     /// This representation has the benefit of making it possible to
     /// express "being a child of a given facet" as a range over
     /// the term ordinals.
-    pub fn encoded_bytes(&self) -> &[u8] {
+    pub fn encoded_str(&self) -> &str {
         &self.0
     }
 
+    pub(crate) fn from_encoded_string(facet_string: String) -> Facet {
+        Facet(facet_string)
+    }
+
     /// Creates a `Facet` from its binary representation.
-    pub(crate) unsafe fn from_encoded(encoded_bytes: Vec<u8>) -> Facet {
-        Facet(encoded_bytes)
+    pub fn from_encoded(encoded_bytes: Vec<u8>) -> Result<Facet, FromUtf8Error> {
+        // facet bytes validation. `0u8` is used a separator but that is still legal utf-8
+        //Ok(Facet(String::from_utf8(encoded_bytes)?))
+        String::from_utf8(encoded_bytes).map(Facet)
     }
 
     /// Parse a text representation of a facet.
@@ -79,36 +90,37 @@ impl Facet {
         Path: IntoIterator,
         Path::Item: ToString,
     {
-        let mut facet_bytes: Vec<u8> = Vec::with_capacity(100);
+        let mut facet_string: String = String::with_capacity(100);
         let mut step_it = path.into_iter();
         if let Some(step) = step_it.next() {
-            facet_bytes.extend_from_slice(step.to_string().as_bytes());
+            facet_string.push_str(&step.to_string());
         }
         for step in step_it {
-            facet_bytes.push(FACET_SEP_BYTE);
-            facet_bytes.extend_from_slice(step.to_string().as_bytes());
+            facet_string.push(FACET_SEP_CHAR);
+            facet_string.push_str(&step.to_string());
         }
-        Facet(facet_bytes)
+        Facet(facet_string)
     }
 
     /// Accessor for the inner buffer of the `Facet`.
-    pub(crate) fn inner_buffer_mut(&mut self) -> &mut Vec<u8> {
-        &mut self.0
+    pub(crate) fn set_facet_str(&mut self, facet_str: &str) {
+        self.0.clear();
+        self.0.push_str(facet_str);
     }
 
     /// Returns `true` iff other is a subfacet of `self`.
     pub fn is_prefix_of(&self, other: &Facet) -> bool {
-        let self_bytes: &[u8] = self.encoded_bytes();
-        let other_bytes: &[u8] = other.encoded_bytes();
-        self_bytes.len() < other_bytes.len()
-            && other_bytes.starts_with(self_bytes)
-            && other_bytes[self_bytes.len()] == 0u8
+        let self_str = self.encoded_str();
+        let other_str = other.encoded_str();
+        self_str.len() < other_str.len()
+            && other_str.starts_with(self_str)
+            && other_str.as_bytes()[self_str.len()] == FACET_SEP_BYTE
     }
 }
 
-impl Borrow<[u8]> for Facet {
-    fn borrow(&self) -> &[u8] {
-        self.encoded_bytes()
+impl Borrow<str> for Facet {
+    fn borrow(&self) -> &str {
+        self.encoded_str()
     }
 }
 
@@ -120,45 +132,51 @@ impl<'a, T: ?Sized + AsRef<str>> From<&'a T> for Facet {
             Idle,
         }
         let path: &str = path_asref.as_ref();
-        let mut facet_encoded = Vec::new();
+        assert!(!path.is_empty());
+        assert!(path.starts_with('/'));
+        let mut facet_encoded = String::new();
         let mut state = State::Idle;
         let path_bytes = path.as_bytes();
-        for &c in &path_bytes[1..] {
+        let mut last_offset = 1;
+        for i in 1..path_bytes.len() {
+            let c = path_bytes[i];
             match (state, c) {
-                (State::Idle, ESCAPE_BYTE) => state = State::Escaped,
+                (State::Idle, ESCAPE_BYTE) => {
+                    facet_encoded.push_str(&path[last_offset..i]);
+                    last_offset = i + 1;
+                    state = State::Escaped
+                }
                 (State::Idle, SLASH_BYTE) => {
-                    facet_encoded.push(FACET_SEP_BYTE);
+                    facet_encoded.push_str(&path[last_offset..i]);
+                    facet_encoded.push(FACET_SEP_CHAR);
+                    last_offset = i + 1;
                 }
-                (State::Escaped, any_char) => {
+                (State::Escaped, _escaped_char) => {
                     state = State::Idle;
-                    facet_encoded.push(any_char);
-                }
-                (State::Idle, other_char) => {
-                    facet_encoded.push(other_char);
                 }
+                (State::Idle, _any_char) => {}
             }
         }
+        facet_encoded.push_str(&path[last_offset..]);
         Facet(facet_encoded)
     }
 }
 
 impl BinarySerializable for Facet {
     fn serialize<W: Write>(&self, writer: &mut W) -> io::Result<()> {
-        <Vec<u8> as BinarySerializable>::serialize(&self.0, writer)
+        <String as BinarySerializable>::serialize(&self.0, writer)
     }
 
     fn deserialize<R: Read>(reader: &mut R) -> io::Result<Self> {
-        let bytes = <Vec<u8> as BinarySerializable>::deserialize(reader)?;
-        Ok(Facet(bytes))
+        Ok(Facet(<String as BinarySerializable>::deserialize(reader)?))
     }
 }
 
 impl Display for Facet {
     fn fmt(&self, f: &mut Formatter) -> fmt::Result {
-        for step in self.0.split(|&b| b == FACET_SEP_BYTE) {
+        for step in self.0.split(FACET_SEP_CHAR) {
             write!(f, "/")?;
-            let step_str = unsafe { str::from_utf8_unchecked(step) };
-            write!(f, "{}", escape_slashes(step_str))?;
+            write!(f, "{}", escape_slashes(step))?;
         }
         Ok(())
     }

src/schema/schema.rs

@@ -232,12 +232,14 @@ impl Schema {
                     let field_entry = self.get_field_entry(field);
                     let field_type = field_entry.field_type();
                     match *json_value {
-                        JsonValue::Array(ref json_items) => for json_item in json_items {
-                            let value = field_type
-                                .value_from_json(json_item)
-                                .map_err(|e| DocParsingError::ValueError(field_name.clone(), e))?;
-                            doc.add(FieldValue::new(field, value));
-                        },
+                        JsonValue::Array(ref json_items) => {
+                            for json_item in json_items {
+                                let value = field_type.value_from_json(json_item).map_err(|e| {
+                                    DocParsingError::ValueError(field_name.clone(), e)
+                                })?;
+                                doc.add(FieldValue::new(field, value));
+                            }
+                        }
                         _ => {
                             let value = field_type
                                 .value_from_json(json_value)
@@ -446,7 +448,8 @@ mod tests {
                 "count": 4,
                 "popularity": 10
             }"#,
-                ).unwrap();
+                )
+                .unwrap();
             assert_eq!(doc.get_first(title_field).unwrap().text(), Some("my title"));
             assert_eq!(
                 doc.get_first(author_field).unwrap().text(),

src/schema/term.rs

@@ -32,7 +32,7 @@ impl Term {
 
     /// Creates a `Term` given a facet.
     pub fn from_facet(field: Field, facet: &Facet) -> Term {
-        let bytes = facet.encoded_bytes();
+        let bytes = facet.encoded_str().as_bytes();
         let buffer = Vec::with_capacity(4 + bytes.len());
         let mut term = Term(buffer);
         term.set_field(field);
@@ -68,12 +68,7 @@ impl Term {
         term
     }
 
-    /// Creates a new Term with an empty buffer,
-    /// but with a given capacity.
-    ///
-    /// It is declared unsafe, as the term content
-    /// is not initialized, and a call to `.field()`
-    /// would panic.
+    /// Creates a new Term for a given field.
     pub(crate) fn for_field(field: Field) -> Term {
         let mut term = Term(Vec::with_capacity(100));
         term.set_field(field);

src/snippet/mod.rs

@@ -192,7 +192,8 @@ fn select_best_fragment_combination(fragments: &[FragmentCandidate], text: &str)
                     item.start - fragment.start_offset,
                     item.stop - fragment.start_offset,
                 )
-            }).collect();
+            })
+            .collect();
         Snippet {
             fragments: fragment_text.to_string(),
             highlighted,
@@ -242,7 +243,7 @@ fn select_best_fragment_combination(fragments: &[FragmentCandidate], text: &str)
 /// let query = query_parser.parse_query("haleurs flamands").unwrap();
 /// # index.load_searchers()?;
 /// # let searcher = index.searcher();
-/// let mut snippet_generator = SnippetGenerator::new(&searcher, &*query, text_field)?;
+/// let mut snippet_generator = SnippetGenerator::create(&searcher, &*query, text_field)?;
 /// snippet_generator.set_max_num_chars(100);
 /// let snippet = snippet_generator.snippet_from_doc(&doc);
 /// let snippet_html: String = snippet.to_html();
@@ -259,7 +260,7 @@ pub struct SnippetGenerator {
 
 impl SnippetGenerator {
     /// Creates a new snippet generator
-    pub fn new(searcher: &Searcher, query: &Query, field: Field) -> Result<SnippetGenerator> {
+    pub fn create(searcher: &Searcher, query: &Query, field: Field) -> Result<SnippetGenerator> {
         let mut terms = BTreeSet::new();
         query.query_terms(&mut terms);
         let terms_text: BTreeMap<String, f32> = terms
@@ -273,7 +274,8 @@ impl SnippetGenerator {
                 } else {
                     None
                 }
-            }).collect();
+            })
+            .collect();
         let tokenizer = searcher.index().tokenizer_for_field(field)?;
         Ok(SnippetGenerator {
             terms_text,
@@ -521,7 +523,7 @@ Survey in 2016, 2017, and 2018."#;
         let index = Index::create_in_ram(schema);
         {
             // writing the segment
-            let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
+            let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
             index_writer.add_document(doc!(text_field => "a"));
             index_writer.add_document(doc!(text_field => "a"));
             index_writer.add_document(doc!(text_field => "a b"));
@@ -532,12 +534,14 @@ Survey in 2016, 2017, and 2018."#;
         let query_parser = QueryParser::for_index(&index, vec![text_field]);
         {
             let query = query_parser.parse_query("e").unwrap();
-            let snippet_generator = SnippetGenerator::new(&searcher, &*query, text_field).unwrap();
+            let snippet_generator =
+                SnippetGenerator::create(&searcher, &*query, text_field).unwrap();
             assert!(snippet_generator.terms_text().is_empty());
         }
         {
             let query = query_parser.parse_query("a").unwrap();
-            let snippet_generator = SnippetGenerator::new(&searcher, &*query, text_field).unwrap();
+            let snippet_generator =
+                SnippetGenerator::create(&searcher, &*query, text_field).unwrap();
             assert_eq!(
                 &btreemap!("a".to_string() => 0.25f32),
                 snippet_generator.terms_text()
@@ -545,7 +549,8 @@ Survey in 2016, 2017, and 2018."#;
         }
         {
             let query = query_parser.parse_query("a b").unwrap();
-            let snippet_generator = SnippetGenerator::new(&searcher, &*query, text_field).unwrap();
+            let snippet_generator =
+                SnippetGenerator::create(&searcher, &*query, text_field).unwrap();
             assert_eq!(
                 &btreemap!("a".to_string() => 0.25f32, "b".to_string() => 0.5),
                 snippet_generator.terms_text()
@@ -553,7 +558,8 @@ Survey in 2016, 2017, and 2018."#;
         }
         {
             let query = query_parser.parse_query("a b c").unwrap();
-            let snippet_generator = SnippetGenerator::new(&searcher, &*query, text_field).unwrap();
+            let snippet_generator =
+                SnippetGenerator::create(&searcher, &*query, text_field).unwrap();
             assert_eq!(
                 &btreemap!("a".to_string() => 0.25f32, "b".to_string() => 0.5),
                 snippet_generator.terms_text()
@@ -574,7 +580,7 @@ Survey in 2016, 2017, and 2018."#;
         let index = Index::create_in_ram(schema);
         {
             // writing the segment
-            let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
+            let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
             {
                 let doc = doc ! (text_field => TEST_TEXT);
                 index_writer.add_document(doc);
@@ -585,7 +591,8 @@ Survey in 2016, 2017, and 2018."#;
         let searcher = index.searcher();
         let query_parser = QueryParser::for_index(&index, vec![text_field]);
         let query = query_parser.parse_query("rust design").unwrap();
-        let mut snippet_generator = SnippetGenerator::new(&searcher, &*query, text_field).unwrap();
+        let mut snippet_generator =
+            SnippetGenerator::create(&searcher, &*query, text_field).unwrap();
         {
             let snippet = snippet_generator.snippet(TEST_TEXT);
             assert_eq!(snippet.to_html(), "imperative-procedural paradigms. <b>Rust</b> is syntactically similar to C++[according to whom?],\nbut its <b>designers</b> intend it to provide better memory safety");

src/space_usage/mod.rs

@@ -80,6 +80,7 @@ pub struct SegmentSpaceUsage {
 }
 
 impl SegmentSpaceUsage {
+    #[allow(clippy::too_many_arguments)]
     pub(crate) fn new(
         num_docs: u32,
         termdict: PerFieldSpaceUsage,

src/store/reader.rs

@@ -95,10 +95,7 @@ impl StoreReader {
     }
 }
 
-#[cfg_attr(
-    feature = "cargo-clippy",
-    allow(clippy::needless_pass_by_value)
-)]
+#[cfg_attr(feature = "cargo-clippy", allow(clippy::needless_pass_by_value))]
 fn split_source(data: ReadOnlySource) -> (ReadOnlySource, ReadOnlySource, DocId) {
     let data_len = data.len();
     let footer_offset = data_len - size_of::<u64>() - size_of::<u32>();

src/termdict/merger.rs

@@ -53,7 +53,8 @@ impl<'a> TermMerger<'a> {
                 .map(|(ord, streamer)| HeapItem {
                     streamer,
                     segment_ord: ord,
-                }).collect(),
+                })
+                .collect(),
         }
     }
 
@@ -122,10 +123,7 @@ impl<'a> TermMerger<'a> {
     }
 
     /// Iterates through terms
-    #[cfg_attr(
-        feature = "cargo-clippy",
-        allow(clippy::should_implement_trait)
-    )]
+    #[cfg_attr(feature = "cargo-clippy", allow(clippy::should_implement_trait))]
     pub fn next(&mut self) -> Option<Term<&[u8]>> {
         if self.advance() {
             Some(Term::wrap(self.current_streamers[0].streamer.key()))

src/termdict/mod.rs

@@ -66,7 +66,7 @@ mod tests {
             let write = directory.open_write(&path).unwrap();
             let field_type = FieldType::Str(TEXT);
             let mut term_dictionary_builder =
-                TermDictionaryBuilder::new(write, &field_type).unwrap();
+                TermDictionaryBuilder::create(write, &field_type).unwrap();
             for term in COUNTRIES.iter() {
                 term_dictionary_builder
                     .insert(term.as_bytes(), &make_term_info(0u64))
@@ -92,7 +92,7 @@ mod tests {
             let write = directory.open_write(&path).unwrap();
             let field_type = FieldType::Str(TEXT);
             let mut term_dictionary_builder =
-                TermDictionaryBuilder::new(write, &field_type).unwrap();
+                TermDictionaryBuilder::create(write, &field_type).unwrap();
             term_dictionary_builder
                 .insert("abc".as_bytes(), &make_term_info(34u64))
                 .unwrap();
@@ -133,7 +133,7 @@ mod tests {
         let text_field = schema_builder.add_text_field("text", TEXT);
         let index = Index::create_in_ram(schema_builder.build());
         {
-            let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
+            let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
             {
                 {
                     let mut doc = Document::default();
@@ -167,7 +167,7 @@ mod tests {
         let mut term_string = String::new();
         while term_it.advance() {
             //let term = Term::from_bytes(term_it.key());
-            term_string.push_str(unsafe { str::from_utf8_unchecked(term_it.key()) }); // ok test
+            term_string.push_str(str::from_utf8(term_it.key()).expect("test"));
         }
         assert_eq!(&*term_string, "abcdef");
     }
@@ -180,7 +180,7 @@ mod tests {
         let field_type = FieldType::Str(TEXT);
         let buffer: Vec<u8> = {
             let mut term_dictionary_builder =
-                TermDictionaryBuilder::new(vec![], &field_type).unwrap();
+                TermDictionaryBuilder::create(vec![], &field_type).unwrap();
             for &(ref id, ref i) in &ids {
                 term_dictionary_builder
                     .insert(id.as_bytes(), &make_term_info(*i as u64))
@@ -210,7 +210,7 @@ mod tests {
         let field_type = FieldType::Str(TEXT);
         let buffer: Vec<u8> = {
             let mut term_dictionary_builder =
-                TermDictionaryBuilder::new(vec![], &field_type).unwrap();
+                TermDictionaryBuilder::create(vec![], &field_type).unwrap();
             // term requires more than 16bits
             term_dictionary_builder
                 .insert("abcdefghijklmnopqrstuvwxy", &make_term_info(1))
@@ -245,7 +245,7 @@ mod tests {
         let field_type = FieldType::Str(TEXT);
         let buffer: Vec<u8> = {
             let mut term_dictionary_builder =
-                TermDictionaryBuilder::new(vec![], &field_type).unwrap();
+                TermDictionaryBuilder::create(vec![], &field_type).unwrap();
             for &(ref id, ref i) in &ids {
                 term_dictionary_builder
                     .insert(id.as_bytes(), &make_term_info(*i as u64))
@@ -314,7 +314,7 @@ mod tests {
         let field_type = FieldType::Str(TEXT);
         let buffer: Vec<u8> = {
             let mut term_dictionary_builder =
-                TermDictionaryBuilder::new(vec![], &field_type).unwrap();
+                TermDictionaryBuilder::create(vec![], &field_type).unwrap();
             term_dictionary_builder
                 .insert(&[], &make_term_info(1 as u64))
                 .unwrap();
@@ -338,7 +338,7 @@ mod tests {
         let field_type = FieldType::Str(TEXT);
         let buffer: Vec<u8> = {
             let mut term_dictionary_builder =
-                TermDictionaryBuilder::new(vec![], &field_type).unwrap();
+                TermDictionaryBuilder::create(vec![], &field_type).unwrap();
             for i in 0u8..10u8 {
                 let number_arr = [i; 1];
                 term_dictionary_builder
@@ -408,7 +408,7 @@ mod tests {
             let write = directory.open_write(&path).unwrap();
             let field_type = FieldType::Str(TEXT);
             let mut term_dictionary_builder =
-                TermDictionaryBuilder::new(write, &field_type).unwrap();
+                TermDictionaryBuilder::create(write, &field_type).unwrap();
             for term in COUNTRIES.iter() {
                 term_dictionary_builder
                     .insert(term.as_bytes(), &make_term_info(0u64))

src/termdict/streamer.rs

@@ -132,10 +132,7 @@ where
     }
 
     /// Return the next `(key, value)` pair.
-    #[cfg_attr(
-        feature = "cargo-clippy",
-        allow(clippy::should_implement_trait)
-    )]
+    #[cfg_attr(feature = "cargo-clippy", allow(clippy::should_implement_trait))]
     pub fn next(&mut self) -> Option<(&[u8], &TermInfo)> {
         if self.advance() {
             Some((self.key(), self.value()))

src/termdict/term_info_store.rs

@@ -1,4 +1,4 @@
-use byteorder::ByteOrder;
+use byteorder::{ByteOrder, LittleEndian};
 use common::bitpacker::BitPacker;
 use common::compute_num_bits;
 use common::Endianness;
@@ -7,7 +7,6 @@ use directory::ReadOnlySource;
 use postings::TermInfo;
 use std::cmp;
 use std::io::{self, Read, Write};
-use std::ptr;
 use termdict::TermOrdinal;
 
 const BLOCK_LEN: usize = 256;
@@ -88,13 +87,17 @@ fn extract_bits(data: &[u8], addr_bits: usize, num_bits: u8) -> u64 {
     assert!(num_bits <= 56);
     let addr_byte = addr_bits / 8;
     let bit_shift = (addr_bits % 8) as u64;
-    assert!(data.len() >= addr_byte + 7);
-    let val_unshifted_unmasked: u64 = unsafe {
-        // ok because the pointer is only accessed using `ptr::read_unaligned`
-        #[cfg_attr(feature = "cargo-clippy", allow(clippy::cast_ptr_alignment))]
-        let addr = data.as_ptr().add(addr_byte) as *const u64;
-        // ok thanks to the 7 byte padding
-        ptr::read_unaligned(addr)
+    let val_unshifted_unmasked: u64 = if data.len() >= addr_byte + 8 {
+        LittleEndian::read_u64(&data[addr_byte..][..8])
+    } else {
+        // the buffer is not large enough.
+        // Let's copy the few remaining bytes to a 8 byte buffer
+        // padded with 0s.
+        let mut buf = [0u8; 8];
+        let data_to_copy = &data[addr_byte..];
+        let nbytes = data_to_copy.len();
+        buf[..nbytes].copy_from_slice(data_to_copy);
+        LittleEndian::read_u64(&buf)
     };
     let val_shifted_unmasked = val_unshifted_unmasked >> bit_shift;
     let mask = (1u64 << u64::from(num_bits)) - 1;
@@ -246,7 +249,6 @@ impl TermInfoStoreWriter {
         self.num_terms.serialize(write)?;
         write.write_all(&self.buffer_block_metas)?;
         write.write_all(&self.buffer_term_infos)?;
-        write.write_all(&[0u8; 7])?;
         Ok(())
     }
 }

src/termdict/termdict.rs

@@ -29,7 +29,7 @@ where
     W: Write,
 {
     /// Creates a new `TermDictionaryBuilder`
-    pub fn new(w: W, _field_type: &FieldType) -> io::Result<Self> {
+    pub fn create(w: W, _field_type: &FieldType) -> io::Result<Self> {
         let fst_builder = fst::MapBuilder::new(w).map_err(convert_fst_error)?;
         Ok(TermDictionaryBuilder {
             fst_builder,
@@ -132,7 +132,7 @@ impl TermDictionary {
     /// Creates an empty term dictionary which contains no terms.
     pub fn empty(field_type: &FieldType) -> Self {
         let term_dictionary_data: Vec<u8> =
-            TermDictionaryBuilder::new(Vec::<u8>::new(), &field_type)
+            TermDictionaryBuilder::create(Vec::<u8>::new(), &field_type)
                 .expect("Creating a TermDictionaryBuilder in a Vec<u8> should never fail")
                 .finish()
                 .expect("Writing in a Vec<u8> should never fail");

src/tokenizer/char_processing/char_filter.rs

@@ -1,78 +0,0 @@
-extern crate aho_corasick;
-use self::aho_corasick::{AcAutomaton, Automaton};
-use std::mem;
-
-use super::{OffsetIncrements, OffsetIncrementsBuilder};
-
-pub trait CharMogrifier {
-    fn process_text(&mut self, text: &str, dest: &mut String, correction: &mut OffsetIncrementsBuilder);
-}
-
-pub struct CharFilter {
-    text: String,
-    buffer: String,
-    mogrifiers: Vec<Box<CharMogrifier>>
-}
-
-impl CharFilter {
-    fn process_text(&mut self, text: &str) {
-        self.text.clear();
-        self.text.push_str(text);
-        self.buffer.clear();
-        let mut offset_increment_builder = OffsetIncrements::builder();
-        for mogrifier in &mut self.mogrifiers {
-            mogrifier.process_text(&self.text,
-                                   &mut self.buffer,
-                                   &mut offset_increment_builder);
-            mem::swap(&mut self.text, &mut self.buffer);
-            offset_increment_builder.new_layer();
-        }
-    }
-}
-
-
-pub struct SubstringReplacer<'a> {
-    automaton: AcAutomaton<&'a str>,
-    replacements: Vec<&'a str>
-}
-
-
-impl SubstringReplacer<'static> {
-    fn new(from_tos: Vec<(&'static str, &'static str)>) -> SubstringReplacer<'static> {
-        let from_ptns: Vec<&'static str> = from_tos
-            .iter()
-            .map(|(from_str, _)| *from_str)
-            .collect();
-        let to_strs: Vec<&'static str> = from_tos
-            .iter()
-            .map(|(_, to_str)| *to_str)
-            .collect();
-        let automaton = AcAutomaton::new(from_ptns);
-        SubstringReplacer {
-            automaton,
-            replacements: to_strs
-        }
-    }
-}
-impl<'a> CharMogrifier for SubstringReplacer<'a> {
-    // correction is an array that goes from old_offset -> new_offset.
-    // correction len is `text.len() + 1`
-    fn process_text(&mut self, text: &str, dest: &mut String, correction: &mut OffsetIncrementsBuilder) {
-        let mut start = 0;
-        for m in self.automaton.find(text) {
-            dest.push_str(&text[start..m.start]);
-            let replacement = self.replacements[m.pati];
-            let previous_len = m.end - m.start;
-            correction.register_inc(m.end, (replacement.len() as isize) - (previous_len as isize));
-            dest.push_str(replacement);
-            start = m.end;
-        }
-        dest.push_str(&text[start..]);
-    }
-}
-
-
-// lowercasing
-// Unicode normalization*
-// '
-// accent simplification

src/tokenizer/char_processing/mod.rs

@@ -1,4 +0,0 @@
-mod char_filter;
-mod offset_increments;
-
-pub use self::offset_increments::{OffsetIncrements, OffsetIncrementsBuilder};

src/tokenizer/char_processing/offset_increments.rs

@@ -1,264 +0,0 @@
-/*!
-Stores an increasing mapping from naturals to naturals.
-
-`CharFilter`s may make the original text longer or shorter.
-Token's offset need to refer to their offset in the original
-text.
-
-This struct is in charge of doing an efficient book-keeping
-of these shift in offsets and provide a mapping
-from the transformed text to the original text.
-
-We define the inverse of an increasing mapping `f` as:
-g(i) = max {j | f(j) <= i}
-     != min {j | f(i) >= i}
-
-The name `inverse` is a bit misleading:
-this is not really an involution.
-
-Note that having a single definition has some bad side effects.
-For instance, when trying to convert a segment of chars to
-its offset in the original string, the reverse mapping may
-return an empty string.
-
-We could use a different definition of the reverse mapping
-when computing the lower bound and the upper bound of the segment,
-but then non-overlapping tokens could have overlapping origins.
-
-# Example
-
-```
-forward mapping
-[0,1,2,5,6,7]
-Encoded sparsely as (3, 2)
-
-reverse mapping
-[0,1,2,2,2,3,4,5]
-Encoded sparsely as [(3, -1), (4,-1), (5,-1)]
-```
-
-
-*/
-
-/// Builds a reverse mapping using a sparse representation of the
-/// forward mapping.
-pub struct OffsetIncrementsBuilder {
-    cumulated: isize,
-    incs: Vec<(usize, isize)>,
-}
-
-impl OffsetIncrementsBuilder {
-    /// We require
-    /// - `from_offset + delta >= 0`
-    /// There is no need to call this function if delta = 0.
-    pub fn register_inc(&mut self, from_offset: usize, delta: isize) {
-        let mut cumulated = self.cumulated;
-        let from_offset_isize = from_offset as isize;
-        let to_offset = (from_offset_isize + self.cumulated) as usize;
-        if delta > 0 {
-            for i in 0..delta as usize {
-                cumulated += 1;
-                self.incs.push((to_offset + i, -cumulated));
-            }
-        } else {
-            assert_eq!(delta, -1);
-            cumulated -= 1;
-            self.incs.push((to_offset + 1, -cumulated));
-        }
-        println!("incs {:?}", self.incs);
-        self.cumulated = cumulated;
-    }
-
-    pub fn new_layer(&self) {
-        panic!();
-    }
-
-    fn build(self) -> OffsetIncrements {
-        OffsetIncrements {
-            incs: self.incs
-        }
-    }
-}
-
-
-#[derive(Default)]
-pub struct OffsetIncrementsReader {
-    shifts: Vec<(usize, isize)>,
-    current_shift: isize,
-    idx: usize,
-}
-
-impl OffsetIncrementsReader {
-    fn new(shifts: Vec<(usize, isize)>) -> OffsetIncrementsReader {
-        OffsetIncrementsReader {
-            shifts,
-            current_shift: 0,
-            idx: 0,
-        }
-    }
-
-    fn convert_offset(&mut self, target: usize) -> usize {
-        while self.idx < self.shifts.len() {
-            let (offset, shift) = self.shifts[self.idx];
-            if offset > target {
-                break;
-            } else {
-                self.current_shift = shift;
-            }
-            self.idx += 1;
-        }
-        return (self.current_shift + target as isize) as usize;
-    }
-}
-
-pub struct OffsetIncrements {
-    incs: Vec<(usize, isize)>
-}
-
-impl OffsetIncrements {
-    pub fn builder() -> OffsetIncrementsBuilder {
-        OffsetIncrementsBuilder {
-            cumulated: 0,
-            incs: Vec::new(),
-        }
-    }
-
-    pub fn reader(&self) -> OffsetIncrementsReader {
-        OffsetIncrementsReader::new(self.incs.clone()) // TODO Fixme, no clone
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::OffsetIncrements;
-    use super::OffsetIncrementsReader;
-
-
-    #[test]
-    fn test_offset_increment_reader_empty() {
-        let mut reader = OffsetIncrementsReader::new(vec![]);
-        for i in 0..3 {
-            assert_eq!(reader.convert_offset(i), i);
-        }
-    }
-
-    #[test]
-    fn test_offset_increment_reader_step() {
-        let mut reader = OffsetIncrementsReader::new(vec![(1, 1), (3, 3), (6, 2), (7, 1), (8, 0), (9, -1)]);
-        assert_eq!(reader.convert_offset(0), 0);
-        assert_eq!(reader.convert_offset(1), 2);
-        assert_eq!(reader.convert_offset(2), 3);
-        assert_eq!(reader.convert_offset(3), 6);
-        assert_eq!(reader.convert_offset(4), 7);
-        assert_eq!(reader.convert_offset(5), 8);
-        assert_eq!(reader.convert_offset(6), 8);
-        assert_eq!(reader.convert_offset(7), 8);
-        assert_eq!(reader.convert_offset(8), 8);
-        assert_eq!(reader.convert_offset(9), 8);
-    }
-
-    #[test]
-    fn test_offset_increment_reader_step_neg() {
-        let mut reader = OffsetIncrementsReader::new(vec![(1, -1), (2, -2), (3, -3)]);
-        assert_eq!(reader.convert_offset(0), 0);
-        assert_eq!(reader.convert_offset(1), 0);
-        assert_eq!(reader.convert_offset(2), 0);
-        assert_eq!(reader.convert_offset(3), 0);
-        assert_eq!(reader.convert_offset(4), 1);
-        assert_eq!(reader.convert_offset(5), 2);
-        assert_eq!(reader.convert_offset(6), 3);
-        assert_eq!(reader.convert_offset(7), 4);
-    }
-
-
-    fn aux_test_increment(increments: OffsetIncrements, expected: &[usize]) {
-        let mut reader = increments.reader();
-        for (i, el) in expected.iter().cloned().enumerate() {
-            assert_eq!(reader.convert_offset(i), el);
-        }
-    }
-
-
-    fn assert_is_increasing(v: &[usize]) {
-        assert!(v.len() > 0);
-        assert_eq!(v[0], 0);
-        let mut prec = 0;
-        for &val in &v[1..] {
-            assert!(val >= prec);
-            prec = val;
-        }
-    }
-
-    fn is_inverse(fwd: &[usize], rev: &[usize]) {
-        assert_is_increasing(fwd);
-        assert_is_increasing(rev);
-        println!("fwd {:?} rev {:?}", fwd, rev);
-        for (i, &antecedant) in rev.iter().enumerate() {
-            let expected = fwd
-                .iter()
-                .enumerate()
-                .filter(|(_, v)| **v <= i)
-                .map(|(ord, _)| ord)
-                .last()
-                .unwrap();
-            println!("i {}", i);
-            assert_eq!(expected, antecedant);
-        }
-    }
-
-    #[test]
-    fn test_is_inverse() {
-        is_inverse(&[0,1,1,1,2], &[0, 3, 4]);
-    }
-
-
-    fn is_reciprocal(left: &[usize], right: &[usize]) {
-        is_inverse(left, right);
-    }
-
-    #[test]
-    fn test_offset_increments_shorten() {
-        {
-            let mut offset_increment_builder = OffsetIncrements::builder();
-            // abcd -> abd
-            offset_increment_builder.register_inc(2, -1);
-            aux_test_increment(offset_increment_builder.build(), &[0, 1, 2, 4]);
-        }
-        {
-            let mut offset_increment_builder = OffsetIncrements::builder();
-            // abcdefgh -> abcdfgh
-            offset_increment_builder.register_inc(4, -1);
-            aux_test_increment(offset_increment_builder.build(), &[0, 1, 2, 3, 4, 6]);
-        }
-        {
-            let mut offset_increment_builder = OffsetIncrements::builder();
-            // abcd -> bcd
-            offset_increment_builder.register_inc(0, -1);
-            aux_test_increment(offset_increment_builder.build(), &[0, 2, 3]);
-        }
-    }
-
-
-    #[test]
-    fn test_offset_increments_builder() {
-        {
-            let mut offset_increment_builder = OffsetIncrements::builder();
-            offset_increment_builder.register_inc(2, 1);
-            // [0, 1, 3, 4, 5]
-            aux_test_increment(offset_increment_builder.build(), &[0,1,1,2,3,4,5]);
-        }
-        {
-            let mut offset_increment_builder = OffsetIncrements::builder();
-            offset_increment_builder.register_inc(3, 2);
-            // [0, 1, 2, 4, 5, 6]
-            aux_test_increment(offset_increment_builder.build(), &[0,1,2,2,2,3,4,5]);
-        }
-        {
-            let mut offset_increment_builder = OffsetIncrements::builder();
-            // 0, 0, 1, 2, 2, 2
-            offset_increment_builder.register_inc(1, 1);
-            offset_increment_builder.register_inc(3, 3);
-            aux_test_increment(offset_increment_builder.build(), &[0,0,1,2,2,2,2,3,4]);
-        }
-    }
-}

src/tokenizer/facet_tokenizer.rs

@@ -1,6 +1,5 @@
 use super::{Token, TokenStream, Tokenizer};
 use schema::FACET_SEP_BYTE;
-use std::str;
 
 /// The `FacetTokenizer` process a `Facet` binary representation
 /// and emits a token for all of its parent.
@@ -57,12 +56,11 @@ impl<'a> TokenStream for FacetTokenStream<'a> {
                     .position(|b| b == FACET_SEP_BYTE)
                     .map(|pos| cursor + 1 + pos)
                 {
-                    let facet_part =
-                        unsafe { str::from_utf8_unchecked(&bytes[cursor..next_sep_pos]) };
+                    let facet_part = &self.text[cursor..next_sep_pos];
                     self.token.text.push_str(facet_part);
                     self.state = State::UpToPosition(next_sep_pos);
                 } else {
-                    let facet_part = unsafe { str::from_utf8_unchecked(&bytes[cursor..]) };
+                    let facet_part = &self.text[cursor..];
                     self.token.text.push_str(facet_part);
                     self.state = State::Terminated;
                 }
@@ -86,7 +84,6 @@ mod tests {
 
     use super::FacetTokenizer;
     use schema::Facet;
-    use std::str;
     use tokenizer::{Token, TokenStream, Tokenizer};
 
     #[test]
@@ -95,11 +92,11 @@ mod tests {
         let mut tokens = vec![];
         {
             let mut add_token = |token: &Token| {
-                let facet = unsafe { Facet::from_encoded(token.text.as_bytes().to_owned()) }; // ok test
+                let facet = Facet::from_encoded(token.text.as_bytes().to_owned()).unwrap();
                 tokens.push(format!("{}", facet));
             };
             FacetTokenizer
-                .token_stream(unsafe { str::from_utf8_unchecked(facet.encoded_bytes()) })
+                .token_stream(facet.encoded_str())
                 .process(&mut add_token);
         }
         assert_eq!(tokens.len(), 4);
@@ -115,11 +112,11 @@ mod tests {
         let mut tokens = vec![];
         {
             let mut add_token = |token: &Token| {
-                let facet = unsafe { Facet::from_encoded(token.text.as_bytes().to_owned()) }; // ok test
+                let facet = Facet::from_encoded(token.text.as_bytes().to_owned()).unwrap(); // ok test
                 tokens.push(format!("{}", facet));
             };
             FacetTokenizer
-                .token_stream(unsafe { str::from_utf8_unchecked(facet.encoded_bytes()) }) // ok test
+                .token_stream(facet.encoded_str()) // ok test
                 .process(&mut add_token);
         }
         assert_eq!(tokens.len(), 1);

src/tokenizer/lower_caser.rs

@@ -50,7 +50,6 @@ where
                 self.token_mut().text.make_ascii_lowercase();
             } else {
                 to_lowercase_unicode(&mut self.tail.token_mut().text, &mut self.buffer);
-
                 mem::swap(&mut self.tail.token_mut().text, &mut self.buffer);
             }
             true

src/tokenizer/mod.rs

@@ -73,7 +73,7 @@
 //! let en_stem = SimpleTokenizer
 //!     .filter(RemoveLongFilter::limit(40))
 //!     .filter(LowerCaser)
-//!     .filter(Stemmer::new());
+//!     .filter(Stemmer::new(Language::English));
 //! # }
 //! ```
 //!
@@ -140,7 +140,6 @@ mod stop_word_filter;
 mod token_stream_chain;
 mod tokenizer;
 mod tokenizer_manager;
-mod char_processing;
 
 pub use self::alphanum_only::AlphaNumOnlyFilter;
 pub use self::facet_tokenizer::FacetTokenizer;
@@ -149,7 +148,7 @@ pub use self::ngram_tokenizer::NgramTokenizer;
 pub use self::raw_tokenizer::RawTokenizer;
 pub use self::remove_long::RemoveLongFilter;
 pub use self::simple_tokenizer::SimpleTokenizer;
-pub use self::stemmer::Stemmer;
+pub use self::stemmer::{Language, Stemmer};
 pub use self::stop_word_filter::StopWordFilter;
 pub(crate) use self::token_stream_chain::TokenStreamChain;
 pub(crate) use self::tokenizer::box_tokenizer;
@@ -160,8 +159,10 @@ pub use self::tokenizer_manager::TokenizerManager;
 
 #[cfg(test)]
 pub mod tests {
-    use super::Token;
-    use super::TokenizerManager;
+    use super::{
+        Language, LowerCaser, RemoveLongFilter, SimpleTokenizer, Stemmer, Token, Tokenizer,
+        TokenizerManager,
+    };
 
     /// This is a function that can be used in tests and doc tests
     /// to assert a token's correctness.
@@ -215,6 +216,7 @@ pub mod tests {
                 .token_stream("Hello, happy tax payer!")
                 .process(&mut add_token);
         }
+
         assert_eq!(tokens.len(), 4);
         assert_token(&tokens[0], 0, "hello", 0, 5);
         assert_token(&tokens[1], 1, "happi", 7, 12);
@@ -222,6 +224,33 @@ pub mod tests {
         assert_token(&tokens[3], 3, "payer", 17, 22);
     }
 
+    #[test]
+    fn test_non_en_tokenizer() {
+        let tokenizer_manager = TokenizerManager::default();
+        tokenizer_manager.register(
+            "es_stem",
+            SimpleTokenizer
+                .filter(RemoveLongFilter::limit(40))
+                .filter(LowerCaser)
+                .filter(Stemmer::new(Language::Spanish)),
+        );
+        let en_tokenizer = tokenizer_manager.get("es_stem").unwrap();
+        let mut tokens: Vec<Token> = vec![];
+        {
+            let mut add_token = |token: &Token| {
+                tokens.push(token.clone());
+            };
+            en_tokenizer
+                .token_stream("Hola, feliz contribuyente!")
+                .process(&mut add_token);
+        }
+
+        assert_eq!(tokens.len(), 3);
+        assert_token(&tokens[0], 0, "hola", 0, 4);
+        assert_token(&tokens[1], 1, "feliz", 6, 11);
+        assert_token(&tokens[2], 2, "contribuyent", 12, 25);
+    }
+
     #[test]
     fn test_tokenizer_empty() {
         let tokenizer_manager = TokenizerManager::default();

src/tokenizer/stemmer.rs

@@ -4,22 +4,78 @@ use super::{Token, TokenFilter, TokenStream};
 use rust_stemmers::{self, Algorithm};
 use std::sync::Arc;
 
-/// `Stemmer` token filter. Currently only English is supported.
-/// Tokens are expected to be lowercased beforehands.
+/// Available stemmer languages.
+#[derive(Debug, Serialize, Deserialize, Eq, PartialEq, Copy, Clone)]
+#[allow(missing_docs)]
+pub enum Language {
+    Arabic,
+    Danish,
+    Dutch,
+    English,
+    Finnish,
+    French,
+    German,
+    Greek,
+    Hungarian,
+    Italian,
+    Portuguese,
+    Romanian,
+    Russian,
+    Spanish,
+    Swedish,
+    Tamil,
+    Turkish,
+}
+
+impl Language {
+    fn algorithm(self) -> Algorithm {
+        use self::Language::*;
+        match self {
+            Arabic => Algorithm::Arabic,
+            Danish => Algorithm::Danish,
+            Dutch => Algorithm::Dutch,
+            English => Algorithm::English,
+            Finnish => Algorithm::Finnish,
+            French => Algorithm::French,
+            German => Algorithm::German,
+            Greek => Algorithm::Greek,
+            Hungarian => Algorithm::Hungarian,
+            Italian => Algorithm::Italian,
+            Portuguese => Algorithm::Portuguese,
+            Romanian => Algorithm::Romanian,
+            Russian => Algorithm::Russian,
+            Spanish => Algorithm::Spanish,
+            Swedish => Algorithm::Swedish,
+            Tamil => Algorithm::Tamil,
+            Turkish => Algorithm::Turkish,
+        }
+    }
+}
+
+/// `Stemmer` token filter. Several languages are supported, see `Language` for the available
+/// languages.
+/// Tokens are expected to be lowercased beforehand.
 #[derive(Clone)]
 pub struct Stemmer {
     stemmer_algorithm: Arc<Algorithm>,
 }
 
 impl Stemmer {
-    /// Creates a new Stemmer `TokenFilter`.
-    pub fn new() -> Stemmer {
+    /// Creates a new Stemmer `TokenFilter` for a given language algorithm.
+    pub fn new(language: Language) -> Stemmer {
         Stemmer {
-            stemmer_algorithm: Arc::new(Algorithm::English),
+            stemmer_algorithm: Arc::new(language.algorithm()),
         }
     }
 }
 
+impl Default for Stemmer {
+    /// Creates a new Stemmer `TokenFilter` for English.
+    fn default() -> Self {
+        Stemmer::new(Language::English)
+    }
+}
+
 impl<TailTokenStream> TokenFilter<TailTokenStream> for Stemmer
 where
     TailTokenStream: TokenStream,

src/tokenizer/tokenizer.rs

@@ -64,7 +64,7 @@ pub trait Tokenizer<'a>: Sized + Clone {
     /// let en_stem = SimpleTokenizer
     ///     .filter(RemoveLongFilter::limit(40))
     ///     .filter(LowerCaser)
-    ///     .filter(Stemmer::new());
+    ///     .filter(Stemmer::default());
     /// # }
     /// ```
     ///

src/tokenizer/tokenizer_manager.rs

@@ -1,6 +1,7 @@
 use std::collections::HashMap;
 use std::sync::{Arc, RwLock};
 use tokenizer::box_tokenizer;
+use tokenizer::stemmer::Language;
 use tokenizer::BoxedTokenizer;
 use tokenizer::LowerCaser;
 use tokenizer::RawTokenizer;
@@ -71,7 +72,7 @@ impl Default for TokenizerManager {
             SimpleTokenizer
                 .filter(RemoveLongFilter::limit(40))
                 .filter(LowerCaser)
-                .filter(Stemmer::new()),
+                .filter(Stemmer::new(Language::English)),
         );
         manager
     }