Edited changelog and bumped version.

This branch is to be published as a hotfix for 5.1.0.

Closes #280
Closes #274
Closes #289

.gitignore

@@ -1,3 +1,4 @@
+*.swp
 target
 target/debug
 .vscode
@@ -8,4 +9,4 @@ benchmark
 cpp/simdcomp/bitpackingbenchmark
 *.bk
 .idea
-trace.dat
+trace.dat

.travis.yml

@@ -1,4 +1,6 @@
 language: rust
+sudo: required
+cache: cargo
 rust:
   - nightly
 env:
@@ -11,6 +13,7 @@ addons:
   apt:
     sources:
       - ubuntu-toolchain-r-test
+      - kalakris-cmake
     packages:
       - gcc-4.8
       - g++-4.8
@@ -18,19 +21,18 @@ addons:
       - libelf-dev
       - libdw-dev
       - binutils-dev
+      - cmake
 before_script:
-  - |
-    pip install 'travis-cargo<0.2' --user &&
-    export PATH=$HOME/.local/bin:$PATH
+  - export PATH=$HOME/.cargo/bin:$PATH
+  - cargo install cargo-update || echo "cargo-update already installed"
+  - cargo install cargo-travis || echo "cargo-travis already installed"
+  - cargo install-update -a # update outdated cached binaries
 script:
-  - |
-    travis-cargo build &&
-    travis-cargo test &&
-    travis-cargo bench &&
-    travis-cargo doc
+  - cargo build
+  - cargo test
+  - cargo test -- --ignored
   - cargo run --example simple_search
+  - cargo doc
 after_success:
-  - bash ./script/build-doc.sh
-  - travis-cargo doc-upload
-  - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then travis-cargo coveralls --no-sudo --verify; fi
-  - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then ./kcov/build/src/kcov --verify --coveralls-id=$TRAVIS_JOB_ID --include-path=`pwd`/src --exclude-path=`pwd`/cpp --exclude-pattern=/.cargo target/kcov target/debug/tantivy-*; fi
+  - cargo coveralls --exclude-pattern cpp/,src/functional_test.rs
+  - cargo doc-upload

.vimrc

@@ -0,0 +1,13 @@
+set wildignore+=*/examples/*
+
+set tabstop=2
+set shiftwidth=2
+set softtabstop=2
+set expandtab
+set nosmarttab
+
+set textwidth=100
+
+autocmd BufRead *.rs :setlocal tags=./rusty-tags.vi;/
+autocmd BufWritePost *.rs :silent! exec "!rusty-tags vi -o --quiet --start-dir=" . expand('%:p:h') . "&" | redraw!
+

CHANGELOG.md

@@ -1,3 +1,45 @@
+Tantivy 0.5.2
+===========================
+- bugfix #274
+- bugfix #280
+- bugfix #289
+
+Tantivy 0.5.1
+==========================
+- bugfix #254 : tantivy failed if no documents in a segment contained a specific field.
+
+
+Tantivy 0.5
+==========================
+- Faceting
+- RangeQuery
+- Configurable tokenization pipeline
+- Bugfix in PhraseQuery
+- Various query optimisation
+- Allowing very large indexes
+    - 64 bits file address
+    - Smarter encoding of the `TermInfo` objects
+
+
+
+Tantivy 0.4.3
+==========================
+
+- Bugfix race condition when deleting files. (#198)
+
+
+Tantivy 0.4.2
+==========================
+
+- Prevent usage of AVX2 instructions (#201)
+
+
+Tantivy 0.4.1
+==========================
+
+- Bugfix for non-indexed fields. (#199)
+
+
 Tantivy 0.4.0
 ==========================
 

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "tantivy"
-version = "0.4.0"
+version = "0.5.2"
 authors = ["Paul Masurel <paul.masurel@gmail.com>"]
 build = "build.rs"
 license = "MIT"
@@ -14,10 +14,10 @@ keywords = ["search", "information", "retrieval"]
 
 [dependencies]
 byteorder = "1.0"
-memmap = "0.4"
 lazy_static = "0.2.1"
+tinysegmenter = "0.1.0"
 regex = "0.2"
-fst = "0.1.37"
+fst = "0.2"
 atomicwrites = "0.1.3"
 tempfile = "2.1"
 log = "0.3.6"
@@ -26,22 +26,23 @@ tempdir = "0.3"
 serde = "1.0"
 serde_derive = "1.0"
 serde_json = "1.0"
-bincode = "0.8"
-libc = {version = "0.2.20", optional=true}
+libc = { version = "0.2.20", optional=true }
 num_cpus = "1.2"
 itertools = "0.5.9"
 lz4 = "1.20"
 bit-set = "0.4.0"
 time = "0.1"
-uuid = { version = "0.5", features = ["v4", "serde"] }
+uuid = { version = "0.6", features = ["v4", "serde"] }
 chan = "0.1"
-version = "2"
-crossbeam = "0.2"
+crossbeam = "0.3"
 futures = "0.1"
 futures-cpupool = "0.1"
 error-chain = "0.8"
 owning_ref = "0.3"
 stable_deref_trait = "1.0.0"
+rust-stemmers = "0.1.0"
+downcast = { version="0.9", features = ["nightly"]}
+matches = "0.1"
 
 [target.'cfg(windows)'.dependencies]
 winapi = "0.2"
@@ -51,7 +52,7 @@ rand = "0.3"
 env_logger = "0.4"
 
 [build-dependencies]
-gcc = {version = "0.3", optional=true}
+cc = { version="1.0.0", optional=true }
 
 [profile.release]
 opt-level = 3
@@ -62,7 +63,7 @@ debug-assertions = false
 
 [features]
 default = ["simdcompression"]
-simdcompression = ["libc", "gcc"]
+simdcompression = ["libc", "cc"]
 streamdict = []
 
 

README.md

@@ -38,12 +38,10 @@ It will walk you through getting a wikipedia search engine up and running in a f
     - [For the last released version](https://docs.rs/tantivy/)
     - [For the last master branch](https://tantivy-search.github.io/tantivy/tantivy/index.html)
 
-# Compiling 
+# Compiling
 
 Tantivy requires Rust Nightly because it uses requires the features [`box_syntax`](https://doc.rust-lang.org/stable/book/box-syntax-and-patterns.html), [`optin_builtin_traits`](https://github.com/rust-lang/rfcs/blob/master/text/0019-opt-in-builtin-traits.md), and [`conservative_impl_trait`](https://github.com/rust-lang/rfcs/blob/master/text/1522-conservative-impl-trait.md).
-By default, `tantivy` uses a git submodule called `simdcomp`.
-After cloning the repository, you will need to initialize and update
-the submodules. The project can then be built using `cargo`.
+The project can then be built using `cargo`.
 
     git clone git@github.com:tantivy-search/tantivy.git
     cd tantivy
@@ -54,9 +52,9 @@ Alternatively, if you are trying to compile `tantivy` without simd compression,
 you can disable this functionality. In this case, this submodule is not required
 and you can compile tantivy by using the `--no-default-features` flag.
 
-    cargo build --no-default-features 
+    cargo build --no-default-features
 
 
 # Contribute
 
-Send me an email (paul.masurel at gmail.com) if you want to contribute to tantivy. 
+Send me an email (paul.masurel at gmail.com) if you want to contribute to tantivy.

build.rs

@@ -1,9 +1,9 @@
 #[cfg(feature = "simdcompression")]
 mod build {
-    extern crate gcc;
+    extern crate cc;
 
     pub fn build() {
-        let mut config = gcc::Config::new();
+        let mut config = cc::Build::new();
         config
             .include("./cpp/simdcomp/include")
             .file("cpp/simdcomp/src/avxbitpacking.c")

examples/html/simple_search.html

@@ -30,10 +30,12 @@
               
             </div>
             
-            <div class="content"><div class='highlight'><pre><span class="hljs-keyword">extern</span> <span class="hljs-keyword">crate</span> rustc_serialize;
-<span class="hljs-keyword">extern</span> <span class="hljs-keyword">crate</span> tantivy;
+            <div class="content"><div class='highlight'><pre><span class="hljs-keyword">extern</span> <span class="hljs-keyword">crate</span> tantivy;
 <span class="hljs-keyword">extern</span> <span class="hljs-keyword">crate</span> tempdir;
 
+<span class="hljs-meta">#[macro_use]</span>
+<span class="hljs-keyword">extern</span> <span class="hljs-keyword">crate</span> serde_json;
+
 <span class="hljs-keyword">use</span> std::path::Path;
 <span class="hljs-keyword">use</span> tempdir::TempDir;
 <span class="hljs-keyword">use</span> tantivy::Index;
@@ -108,8 +110,8 @@ be indexed”.</p>
                 <a class="pilcrow" href="#section-5">&#182;</a>
               </div>
               <p>Our first field is title.
-We want full-text search for it, and we want to be able
-to retrieve the document after the search.</p>
+We want full-text search for it, and we also want 
+to be able to retrieve the document after the search.</p>
 <p>TEXT | STORED is some syntactic sugar to describe
 that.</p>
 <p><code>TEXT</code> means the field should be tokenized and indexed,
@@ -132,9 +134,12 @@ documents that were selected during the search phase.</p>
               <div class="pilwrap ">
                 <a class="pilcrow" href="#section-6">&#182;</a>
               </div>
-              <p>Our first field is body.
-We want full-text search for it, and we want to be able
-to retrieve the body after the search.</p>
+              <p>Our second field is body.
+We want full-text search for it, but we do not 
+need to be able to be able to retrieve it
+for our application. </p>
+<p>We can make our index lighter and 
+by omitting <code>STORED</code> flag.</p>
 
             </div>
             
@@ -158,7 +163,7 @@ with our schema in the directory.</p>
 
             </div>
             
-            <div class="content"><div class='highlight'><pre>    <span class="hljs-keyword">let</span> index = <span class="hljs-built_in">try!</span>(Index::create(index_path, schema.clone()));</pre></div></div>
+            <div class="content"><div class='highlight'><pre>    <span class="hljs-keyword">let</span> index = Index::create(index_path, schema.clone())?;</pre></div></div>
             
         </li>
         
@@ -178,7 +183,7 @@ heap for the indexer can increase its throughput.</p>
 
             </div>
             
-            <div class="content"><div class='highlight'><pre>    <span class="hljs-keyword">let</span> <span class="hljs-keyword">mut</span> index_writer = <span class="hljs-built_in">try!</span>(index.writer(<span class="hljs-number">50_000_000</span>));</pre></div></div>
+            <div class="content"><div class='highlight'><pre>    <span class="hljs-keyword">let</span> <span class="hljs-keyword">mut</span> index_writer = index.writer(<span class="hljs-number">50_000_000</span>)?;</pre></div></div>
             
         </li>
         
@@ -214,9 +219,11 @@ one by one in a Document object.</p>
 
     <span class="hljs-keyword">let</span> <span class="hljs-keyword">mut</span> old_man_doc = Document::<span class="hljs-keyword">default</span>();
     old_man_doc.add_text(title, <span class="hljs-string">"The Old Man and the Sea"</span>);
-    old_man_doc.add_text(body,
-                         <span class="hljs-string">"He was an old man who fished alone in a skiff in the Gulf Stream and \
-                          he had gone eighty-four days now without taking a fish."</span>);</pre></div></div>
+    old_man_doc.add_text(
+        body,
+        <span class="hljs-string">"He was an old man who fished alone in a skiff in the Gulf Stream and \
+                          he had gone eighty-four days now without taking a fish."</span>,
+    );</pre></div></div>
             
         </li>
         
@@ -243,16 +250,25 @@ one by one in a Document object.</p>
                 <a class="pilcrow" href="#section-12">&#182;</a>
               </div>
               <h3 id="create-a-document-directly-from-json-">Create a document directly from json.</h3>
-<p>Alternatively, we can use our schema to parse
-a document object directly from json.</p>
+<p>Alternatively, we can use our schema to parse a
+document object directly from json.
+The document is a string, but we use the <code>json</code> macro
+from <code>serde_json</code> for the convenience of multi-line support.</p>
 
             </div>
             
-            <div class="content"><div class='highlight'><pre>
-    <span class="hljs-keyword">let</span> mice_and_men_doc = <span class="hljs-built_in">try!</span>(schema.parse_document(r#<span class="hljs-string">"{
-       "</span>title<span class="hljs-string">": "</span>Of Mice and Men<span class="hljs-string">",
-       "</span>body<span class="hljs-string">": "</span>few miles south of Soledad, the Salinas River drops <span class="hljs-keyword">in</span> close to the hillside bank and runs deep and green. The water is warm too, <span class="hljs-keyword">for</span> it has slipped twinkling over the yellow sands <span class="hljs-keyword">in</span> the sunlight before reaching the narrow pool. On one side of the river the golden foothill slopes curve up to the strong and rocky Gabilan Mountains, but on the valley side the water is lined with trees—willows fresh and green with every spring, carrying <span class="hljs-keyword">in</span> their lower leaf junctures the debris of the winter’s flooding; and sycamores with mottled, white,recumbent limbs and branches that arch over the pool<span class="hljs-string">"  
-    }"</span>#));
+            <div class="content"><div class='highlight'><pre>    <span class="hljs-keyword">let</span> json = json!({
+       <span class="hljs-string">"title"</span>: <span class="hljs-string">"Of Mice and Men"</span>,
+       <span class="hljs-string">"body"</span>: <span class="hljs-string">"A few miles south of Soledad, the Salinas River drops in close to the hillside \
+                bank and runs deep and green. The water is warm too, for it has slipped twinkling \
+                over the yellow sands in the sunlight before reaching the narrow pool. On one \
+                side of the river the golden foothill slopes curve up to the strong and rocky \
+                Gabilan Mountains, but on the valley side the water is lined with trees—willows \
+                fresh and green with every spring, carrying in their lower leaf junctures the \
+                debris of the winter’s flooding; and sycamores with mottled, white, recumbent \
+                limbs and branches that arch over the pool"</span>
+    });
+    <span class="hljs-keyword">let</span> mice_and_men_doc = schema.parse_document(&amp;json.to_string())?;
 
     index_writer.add_document(mice_and_men_doc);</pre></div></div>
             
@@ -271,10 +287,15 @@ The following document has two titles.</p>
 
             </div>
             
-            <div class="content"><div class='highlight'><pre>    <span class="hljs-keyword">let</span> frankenstein_doc = <span class="hljs-built_in">try!</span>(schema.parse_document(r#<span class="hljs-string">"{
-       "</span>title<span class="hljs-string">": ["</span>Frankenstein<span class="hljs-string">", "</span>The Modern Promotheus<span class="hljs-string">"],
-       "</span>body<span class="hljs-string">": "</span>You will rejoice to hear that no disaster has accompanied the commencement of an enterprise which you have regarded with such evil forebodings.  I arrived here yesterday, and my first task is to assure my dear sister of my welfare and increasing confidence <span class="hljs-keyword">in</span> the success of my undertaking.<span class="hljs-string">"  
-    }"</span>#));
+            <div class="content"><div class='highlight'><pre>    <span class="hljs-keyword">let</span> json = json!({
+       <span class="hljs-string">"title"</span>: [<span class="hljs-string">"Frankenstein"</span>, <span class="hljs-string">"The Modern Prometheus"</span>],
+       <span class="hljs-string">"body"</span>: <span class="hljs-string">"You will rejoice to hear that no disaster has accompanied the commencement of an \
+                enterprise which you have regarded with such evil forebodings.  I arrived here \
+                yesterday, and my first task is to assure my dear sister of my welfare and \
+                increasing confidence in the success of my undertaking."</span>
+    });
+    <span class="hljs-keyword">let</span> frankenstein_doc = schema.parse_document(&amp;json.to_string())?;
+
     index_writer.add_document(frankenstein_doc);</pre></div></div>
             
         </li>
@@ -313,7 +334,7 @@ the existence of new documents.</p>
 
             </div>
             
-            <div class="content"><div class='highlight'><pre>    <span class="hljs-built_in">try!</span>(index_writer.commit());</pre></div></div>
+            <div class="content"><div class='highlight'><pre>    index_writer.commit()?;</pre></div></div>
             
         </li>
         
@@ -349,7 +370,7 @@ after every commit().</p>
 
             </div>
             
-            <div class="content"><div class='highlight'><pre>    <span class="hljs-built_in">try!</span>(index.load_searchers());</pre></div></div>
+            <div class="content"><div class='highlight'><pre>    index.load_searchers()?;</pre></div></div>
             
         </li>
         
@@ -384,7 +405,7 @@ in both title and body.</p>
 
             </div>
             
-            <div class="content"><div class='highlight'><pre>    <span class="hljs-keyword">let</span> query_parser = QueryParser::new(index.schema(), <span class="hljs-built_in">vec!</span>[title, body]);</pre></div></div>
+            <div class="content"><div class='highlight'><pre>    <span class="hljs-keyword">let</span> <span class="hljs-keyword">mut</span> query_parser = QueryParser::for_index(index, <span class="hljs-built_in">vec!</span>[title, body]);</pre></div></div>
             
         </li>
         
@@ -401,7 +422,7 @@ A ticket has been opened regarding this problem.</p>
 
             </div>
             
-            <div class="content"><div class='highlight'><pre>    <span class="hljs-keyword">let</span> query = <span class="hljs-built_in">try!</span>(query_parser.parse_query(<span class="hljs-string">"sea whale"</span>));</pre></div></div>
+            <div class="content"><div class='highlight'><pre>    <span class="hljs-keyword">let</span> query = query_parser.parse_query(<span class="hljs-string">"sea whale"</span>)?;</pre></div></div>
             
         </li>
         
@@ -451,7 +472,7 @@ is the role of the TopCollector.</p>
 
             </div>
             
-            <div class="content"><div class='highlight'><pre>    <span class="hljs-built_in">try!</span>(searcher.search(&amp;*query, &amp;<span class="hljs-keyword">mut</span> top_collector));</pre></div></div>
+            <div class="content"><div class='highlight'><pre>    searcher.search(&amp;*query, &amp;<span class="hljs-keyword">mut</span> top_collector)?;</pre></div></div>
             
         </li>
         
@@ -488,9 +509,27 @@ a title.</p>
             
             <div class="content"><div class='highlight'><pre>
     <span class="hljs-keyword">for</span> doc_address <span class="hljs-keyword">in</span> doc_addresses {
-        <span class="hljs-keyword">let</span> retrieved_doc = <span class="hljs-built_in">try!</span>(searcher.doc(&amp;doc_address));
+        <span class="hljs-keyword">let</span> retrieved_doc = searcher.doc(&amp;doc_address)?;
         <span class="hljs-built_in">println!</span>(<span class="hljs-string">"{}"</span>, schema.to_json(&amp;retrieved_doc));
-    }
+    }</pre></div></div>
+            
+        </li>
+        
+        
+        <li id="section-26">
+            <div class="annotation">
+              
+              <div class="pilwrap ">
+                <a class="pilcrow" href="#section-26">&#182;</a>
+              </div>
+              <p>Wait for indexing and merging threads to shut down.
+Usually this isn’t needed, but in <code>main</code> we try to
+delete the temporary directory and that fails on
+Windows if the files are still open.</p>
+
+            </div>
+            
+            <div class="content"><div class='highlight'><pre>    index_writer.wait_merging_threads()?;
 
     <span class="hljs-literal">Ok</span>(())
 }</pre></div></div>

examples/simple_search.rs

@@ -20,10 +20,7 @@ fn main() {
     }
 }
 
-
 fn run_example(index_path: &Path) -> tantivy::Result<()> {
-
-
     // # Defining the schema
     //
     // The Tantivy index requires a very strict schema.
@@ -31,13 +28,12 @@ fn run_example(index_path: &Path) -> tantivy::Result<()> {
     // and for each field, its type and "the way it should
     // be indexed".
 
-
     // first we need to define a schema ...
     let mut schema_builder = SchemaBuilder::default();
 
     // Our first field is title.
-    // We want full-text search for it, and we want to be able
-    // to retrieve the document after the search.
+    // We want full-text search for it, and we also want
+    // to be able to retrieve the document after the search.
     //
     // TEXT | STORED is some syntactic sugar to describe
     // that.
@@ -51,15 +47,17 @@ fn run_example(index_path: &Path) -> tantivy::Result<()> {
     // documents that were selected during the search phase.
     schema_builder.add_text_field("title", TEXT | STORED);
 
-    // Our first field is body.
-    // We want full-text search for it, and we want to be able
-    // to retrieve the body after the search.
+    // Our second field is body.
+    // We want full-text search for it, but we do not
+    // need to be able to be able to retrieve it
+    // for our application.
+    //
+    // We can make our index lighter and
+    // by omitting `STORED` flag.
     schema_builder.add_text_field("body", TEXT);
 
     let schema = schema_builder.build();
 
-
-
     // # Indexing documents
     //
     // Let's create a brand new index.
@@ -68,7 +66,6 @@ fn run_example(index_path: &Path) -> tantivy::Result<()> {
     // with our schema in the directory.
     let index = Index::create(index_path, schema.clone())?;
 
-
     // To insert document we need an index writer.
     // There must be only one writer at a time.
     // This single `IndexWriter` is already
@@ -81,7 +78,6 @@ fn run_example(index_path: &Path) -> tantivy::Result<()> {
     // Let's index our documents!
     // We first need a handle on the title and the body field.
 
-
     // ### Create a document "manually".
     //
     // We can create a document manually, by setting the fields
@@ -91,9 +87,11 @@ fn run_example(index_path: &Path) -> tantivy::Result<()> {
 
     let mut old_man_doc = Document::default();
     old_man_doc.add_text(title, "The Old Man and the Sea");
-    old_man_doc.add_text(body,
-                         "He was an old man who fished alone in a skiff in the Gulf Stream and \
-                          he had gone eighty-four days now without taking a fish.");
+    old_man_doc.add_text(
+        body,
+        "He was an old man who fished alone in a skiff in the Gulf Stream and \
+         he had gone eighty-four days now without taking a fish.",
+    );
 
     // ... and add it to the `IndexWriter`.
     index_writer.add_document(old_man_doc);
@@ -139,7 +137,6 @@ fn run_example(index_path: &Path) -> tantivy::Result<()> {
     // Indexing 5 million articles of the English wikipedia takes
     // around 4 minutes on my computer!
 
-
     // ### Committing
     //
     // At this point our documents are not searchable.
@@ -161,7 +158,6 @@ fn run_example(index_path: &Path) -> tantivy::Result<()> {
     // tantivy behaves as if has rolled back to its last
     // commit.
 
-
     // # Searching
     //
     // Let's search our index. Start by reloading
@@ -179,14 +175,13 @@ fn run_example(index_path: &Path) -> tantivy::Result<()> {
     // Here, if the user does not specify which
     // field they want to search, tantivy will search
     // in both title and body.
-    let query_parser = QueryParser::new(index.schema(), vec![title, body]);
+    let query_parser = QueryParser::for_index(&index, vec![title, body]);
 
     // QueryParser may fail if the query is not in the right
     // format. For user facing applications, this can be a problem.
     // A ticket has been opened regarding this problem.
     let query = query_parser.parse_query("sea whale")?;
 
-
     // A query defines a set of documents, as
     // well as the way they should be scored.
     //

rustfmt.toml

@@ -0,0 +1 @@
+use_try_shorthand = true

src/analyzer/mod.rs

@@ -1,84 +0,0 @@
-extern crate regex;
-
-use std::str::Chars;
-use std::ascii::AsciiExt;
-
-pub struct TokenIter<'a> {
-    chars: Chars<'a>,
-    term_buffer: String,
-}
-
-fn append_char_lowercase(c: char, term_buffer: &mut String) {
-    term_buffer.push(c.to_ascii_lowercase());
-}
-
-pub trait StreamingIterator<'a, T> {
-    fn next(&'a mut self) -> Option<T>;
-}
-
-impl<'a, 'b> TokenIter<'b> {
-    fn consume_token(&'a mut self) -> Option<&'a str> {
-        for c in &mut self.chars {
-            if c.is_alphanumeric() {
-                append_char_lowercase(c, &mut self.term_buffer);
-            } else {
-                break;
-            }
-        }
-        Some(&self.term_buffer)
-    }
-}
-
-
-impl<'a, 'b> StreamingIterator<'a, &'a str> for TokenIter<'b> {
-    #[inline]
-    fn next(&'a mut self) -> Option<&'a str> {
-        self.term_buffer.clear();
-        // skipping non-letter characters.
-        loop {
-            match self.chars.next() {
-                Some(c) => {
-                    if c.is_alphanumeric() {
-                        append_char_lowercase(c, &mut self.term_buffer);
-                        return self.consume_token();
-                    }
-                }
-                None => {
-                    return None;
-                }
-            }
-        }
-    }
-}
-
-pub struct SimpleTokenizer;
-
-
-impl SimpleTokenizer {
-    pub fn tokenize<'a>(&self, text: &'a str) -> TokenIter<'a> {
-        TokenIter {
-            term_buffer: String::new(),
-            chars: text.chars(),
-        }
-    }
-}
-
-
-#[test]
-fn test_tokenizer() {
-    let simple_tokenizer = SimpleTokenizer;
-    let mut term_reader = simple_tokenizer.tokenize("hello, happy tax payer!");
-    assert_eq!(term_reader.next().unwrap(), "hello");
-    assert_eq!(term_reader.next().unwrap(), "happy");
-    assert_eq!(term_reader.next().unwrap(), "tax");
-    assert_eq!(term_reader.next().unwrap(), "payer");
-    assert_eq!(term_reader.next(), None);
-}
-
-
-#[test]
-fn test_tokenizer_empty() {
-    let simple_tokenizer = SimpleTokenizer;
-    let mut term_reader = simple_tokenizer.tokenize("");
-    assert_eq!(term_reader.next(), None);
-}

src/collector/chained_collector.rs

@@ -5,7 +5,6 @@ use SegmentReader;
 use DocId;
 use Score;
 
-
 /// Collector that does nothing.
 /// This is used in the chain Collector and will hopefully
 /// be optimized away by the compiler.
@@ -17,6 +16,10 @@ impl Collector for DoNothingCollector {
     }
     #[inline]
     fn collect(&mut self, _doc: DocId, _score: Score) {}
+    #[inline]
+    fn requires_scoring(&self) -> bool {
+        false
+    }
 }
 
 /// Zero-cost abstraction used to collect on multiple collectors.
@@ -38,12 +41,13 @@ impl<Left: Collector, Right: Collector> ChainedCollector<Left, Right> {
 }
 
 impl<Left: Collector, Right: Collector> Collector for ChainedCollector<Left, Right> {
-    fn set_segment(&mut self,
-                   segment_local_id: SegmentLocalId,
-                   segment: &SegmentReader)
-                   -> Result<()> {
-        try!(self.left.set_segment(segment_local_id, segment));
-        try!(self.right.set_segment(segment_local_id, segment));
+    fn set_segment(
+        &mut self,
+        segment_local_id: SegmentLocalId,
+        segment: &SegmentReader,
+    ) -> Result<()> {
+        self.left.set_segment(segment_local_id, segment)?;
+        self.right.set_segment(segment_local_id, segment)?;
         Ok(())
     }
 
@@ -51,6 +55,10 @@ impl<Left: Collector, Right: Collector> Collector for ChainedCollector<Left, Rig
         self.left.collect(doc, score);
         self.right.collect(doc, score);
     }
+
+    fn requires_scoring(&self) -> bool {
+        self.left.requires_scoring() || self.right.requires_scoring()
+    }
 }
 
 /// Creates a `ChainedCollector`
@@ -61,7 +69,6 @@ pub fn chain() -> ChainedCollector<DoNothingCollector, DoNothingCollector> {
     }
 }
 
-
 #[cfg(test)]
 mod tests {
 

src/collector/count_collector.rs

@@ -7,6 +7,7 @@ use SegmentLocalId;
 
 /// `CountCollector` collector only counts how many
 /// documents match the query.
+#[derive(Default)]
 pub struct CountCollector {
     count: usize,
 }
@@ -19,12 +20,6 @@ impl CountCollector {
     }
 }
 
-impl Default for CountCollector {
-    fn default() -> CountCollector {
-        CountCollector { count: 0 }
-    }
-}
-
 impl Collector for CountCollector {
     fn set_segment(&mut self, _: SegmentLocalId, _: &SegmentReader) -> Result<()> {
         Ok(())
@@ -33,23 +28,27 @@ impl Collector for CountCollector {
     fn collect(&mut self, _: DocId, _: Score) {
         self.count += 1;
     }
+
+    fn requires_scoring(&self) -> bool {
+        false
+    }
 }
 
 #[cfg(test)]
 mod tests {
 
-    use super::*;
-    use test::Bencher;
-    use collector::Collector;
+    use collector::{Collector, CountCollector};
 
-    #[bench]
-    fn build_collector(b: &mut Bencher) {
-        b.iter(|| {
-                   let mut count_collector = CountCollector::default();
-                   for doc in 0..1_000_000 {
-                       count_collector.collect(doc, 1f32);
-                   }
-                   count_collector.count()
-               });
+    #[test]
+    fn test_count_collector() {
+        let mut count_collector = CountCollector::default();
+        assert_eq!(count_collector.count(), 0);
+        count_collector.collect(0u32, 1f32);
+        assert_eq!(count_collector.count(), 1);
+        assert_eq!(count_collector.count(), 1);
+        count_collector.collect(1u32, 1f32);
+        assert_eq!(count_collector.count(), 2);
+        assert!(!count_collector.requires_scoring());
     }
+
 }

src/collector/facet_collector.rs

@@ -1,117 +1,637 @@
-use std::cmp::Eq;
-use std::collections::HashMap;
-use std::hash::Hash;
-
+use std::mem;
 use collector::Collector;
-use fastfield::FastFieldReader;
+use fastfield::FacetReader;
 use schema::Field;
+use std::cell::UnsafeCell;
+use schema::Facet;
+use std::collections::BTreeMap;
+use std::collections::BinaryHeap;
+use std::collections::Bound;
+use termdict::TermDictionary;
+use termdict::TermStreamer;
+use termdict::TermStreamerBuilder;
+use std::collections::BTreeSet;
+use termdict::TermMerger;
+use docset::SkipResult;
+use std::{usize, u64};
+use std::iter::Peekable;
 
 use DocId;
 use Result;
 use Score;
 use SegmentReader;
 use SegmentLocalId;
+use std::cmp::Ordering;
 
-
-/// Facet collector  for i64/u64 fast field
-pub struct FacetCollector<T>
-    where T: FastFieldReader,
-          T::ValueType: Eq + Hash
-{
-    counters: HashMap<T::ValueType, u64>,
-    field: Field,
-    ff_reader: Option<T>,
+struct Hit<'a> {
+    count: u64,
+    facet: &'a Facet,
 }
 
+impl<'a> Eq for Hit<'a> {}
 
-impl<T> FacetCollector<T>
-    where T: FastFieldReader,
-          T::ValueType: Eq + Hash
-{
-    /// Creates a new facet collector for aggregating a given field.
-    pub fn new(field: Field) -> FacetCollector<T> {
-        FacetCollector {
-            counters: HashMap::new(),
-            field: field,
-            ff_reader: None,
-        }
+impl<'a> PartialEq<Hit<'a>> for Hit<'a> {
+    fn eq(&self, other: &Hit) -> bool {
+        self.count == other.count
     }
 }
 
+impl<'a> PartialOrd<Hit<'a>> for Hit<'a> {
+    fn partial_cmp(&self, other: &Hit) -> Option<Ordering> {
+        Some(self.cmp(other))
+    }
+}
 
-impl<T> Collector for FacetCollector<T>
-    where T: FastFieldReader,
-          T::ValueType: Eq + Hash
-{
+impl<'a> Ord for Hit<'a> {
+    fn cmp(&self, other: &Self) -> Ordering {
+        other.count.cmp(&self.count)
+    }
+}
+
+struct SegmentFacetCounter {
+    pub facet_reader: FacetReader,
+    pub facet_ords: Vec<u64>,
+    pub facet_counts: Vec<u64>,
+}
+
+fn facet_depth(facet_bytes: &[u8]) -> usize {
+    if facet_bytes.is_empty() {
+        0
+    } else {
+        facet_bytes.iter().cloned().filter(|b| *b == 0u8).count() + 1
+    }
+}
+
+/// Collector for faceting
+///
+/// The collector collects all facets. You need to configure it
+/// beforehand with the facet you want to extract.
+///
+/// This is done by calling `.add_facet(...)` with the root of the
+/// facet you want to extract as argument.
+///
+/// Facet counts will only be computed for the facet that are direct children
+/// of such a root facet.
+///
+/// For instance, if your index represents books, your hierarchy of facets
+/// may contain `category`, `language`.
+///
+/// The category facet may include `subcategories`. For instance, a book
+/// could belong to `/category/fiction/fantasy`.
+///
+/// If you request the facet counts for `/category`, the result will be
+/// the breakdown of counts for the direct children of `/category`
+/// (e.g. `/category/fiction`, `/category/biography`, `/category/personal_development`).
+///
+/// Once collection is finished, you can harvest its results in the form
+/// of a `FacetCounts` object, and extract your face                t counts from it.
+///
+/// This implementation assumes you are working with a number of facets that
+/// is much hundreds of time lower than your number of documents.
+///
+///
+/// ```rust
+/// #[macro_use]
+/// extern crate tantivy;
+/// use tantivy::schema::{Facet, SchemaBuilder, TEXT};
+/// use tantivy::{Index, Result};
+/// use tantivy::collector::FacetCollector;
+/// use tantivy::query::AllQuery;
+///
+/// # fn main() { example().unwrap(); }
+/// fn example() -> Result<()> {
+///     let mut schema_builder = SchemaBuilder::new();
+///
+///     // Facet have their own specific type.
+///     // It is not a bad practise to put all of your
+///     // facet information in the same field.
+///     let facet = schema_builder.add_facet_field("facet");
+///     let title = schema_builder.add_text_field("title", TEXT);
+///     let schema = schema_builder.build();
+///     let index = Index::create_in_ram(schema);
+///     {
+///         let mut index_writer = index.writer(3_000_000)?;
+///         // a document can be associated to any number of facets
+///         index_writer.add_document(doc!(
+///             title => "The Name of the Wind",
+///             facet => Facet::from("/lang/en"),
+///             facet => Facet::from("/category/fiction/fantasy")
+///         ));
+///         index_writer.add_document(doc!(
+///             title => "Dune",
+///             facet => Facet::from("/lang/en"),
+///             facet => Facet::from("/category/fiction/sci-fi")
+///         ));
+///         index_writer.add_document(doc!(
+///             title => "La Vénus d'Ille",
+///             facet => Facet::from("/lang/fr"),
+///             facet => Facet::from("/category/fiction/fantasy"),
+///             facet => Facet::from("/category/fiction/horror")
+///         ));
+///         index_writer.add_document(doc!(
+///             title => "The Diary of a Young Girl",
+///             facet => Facet::from("/lang/en"),
+///             facet => Facet::from("/category/biography")
+///         ));
+///         index_writer.commit().unwrap();
+///     }
+///
+///     index.load_searchers()?;
+///     let searcher = index.searcher();
+///
+///     {
+///			let mut facet_collector = FacetCollector::for_field(facet);
+///         facet_collector.add_facet("/lang");
+///         facet_collector.add_facet("/category");
+///         searcher.search(&AllQuery, &mut facet_collector).unwrap();
+///
+///         // this object contains count aggregate for all of the facets.
+///         let counts = facet_collector.harvest();
+///
+///         // This lists all of the facet counts
+///         let facets: Vec<(&Facet, u64)> = counts
+///             .get("/category")
+///             .collect();
+///         assert_eq!(facets, vec![
+///             (&Facet::from("/category/biography"), 1),
+///             (&Facet::from("/category/fiction"), 3)
+///         ]);
+///     }
+///
+///     {
+///			let mut facet_collector = FacetCollector::for_field(facet);
+///         facet_collector.add_facet("/category/fiction");
+///         searcher.search(&AllQuery, &mut facet_collector).unwrap();
+///
+///         // this object contains count aggregate for all of the facets.
+///         let counts = facet_collector.harvest();
+///
+///         // This lists all of the facet counts
+///         let facets: Vec<(&Facet, u64)> = counts
+///             .get("/category/fiction")
+///             .collect();
+///         assert_eq!(facets, vec![
+///             (&Facet::from("/category/fiction/fantasy"), 2),
+///             (&Facet::from("/category/fiction/horror"), 1),
+///             (&Facet::from("/category/fiction/sci-fi"), 1)
+///         ]);
+///     }
+///
+///    {
+///			let mut facet_collector = FacetCollector::for_field(facet);
+///         facet_collector.add_facet("/category/fiction");
+///         searcher.search(&AllQuery, &mut facet_collector).unwrap();
+///
+///         // this object contains count aggregate for all of the facets.
+///         let counts = facet_collector.harvest();
+///
+///         // This lists all of the facet counts
+///         let facets: Vec<(&Facet, u64)> = counts.top_k("/category/fiction", 1);
+///         assert_eq!(facets, vec![
+///             (&Facet::from("/category/fiction/fantasy"), 2)
+///         ]);
+///     }
+///
+///     Ok(())
+/// }
+/// ```
+pub struct FacetCollector {
+    facet_ords: Vec<u64>,
+    field: Field,
+    ff_reader: Option<UnsafeCell<FacetReader>>,
+    segment_counters: Vec<SegmentFacetCounter>,
+
+    // facet_ord -> collapse facet_id
+    current_segment_collapse_mapping: Vec<usize>,
+    // collapse facet_id -> count
+    current_segment_counts: Vec<u64>,
+    // collapse facet_id -> facet_ord
+    current_collapse_facet_ords: Vec<u64>,
+
+    facets: BTreeSet<Facet>,
+}
+
+fn skip<'a, I: Iterator<Item = &'a Facet>>(
+    target: &[u8],
+    collapse_it: &mut Peekable<I>,
+) -> SkipResult {
+    loop {
+        match collapse_it.peek() {
+            Some(facet_bytes) => match facet_bytes.encoded_bytes().cmp(target) {
+                Ordering::Less => {}
+                Ordering::Greater => {
+                    return SkipResult::OverStep;
+                }
+                Ordering::Equal => {
+                    return SkipResult::Reached;
+                }
+            },
+            None => {
+                return SkipResult::End;
+            }
+        }
+        collapse_it.next();
+    }
+}
+
+impl FacetCollector {
+    /// Create a facet collector to collect the facets
+    /// from a specific facet `Field`.
+    ///
+    /// This function does not check whether the field
+    /// is of the proper type.
+    pub fn for_field(field: Field) -> FacetCollector {
+        FacetCollector {
+            facet_ords: Vec::with_capacity(255),
+            segment_counters: Vec::new(),
+            field,
+            ff_reader: None,
+            facets: BTreeSet::new(),
+
+            current_segment_collapse_mapping: Vec::new(),
+            current_collapse_facet_ords: Vec::new(),
+            current_segment_counts: Vec::new(),
+        }
+    }
+
+    /// Adds a facet that we want to record counts
+    ///
+    /// Adding facet `Facet::from("/country")` for instance,
+    /// will record the counts of all of the direct children of the facet country
+    /// (e.g. `/country/FR`, `/country/UK`).
+    ///
+    /// Adding two facets within which one is the prefix of the other is forbidden.
+    /// If you need the correct number of unique documents for two such facets,
+    /// just add them in separate `FacetCollector`.
+    pub fn add_facet<T>(&mut self, facet_from: T)
+    where
+        Facet: From<T>,
+    {
+        let facet = Facet::from(facet_from);
+        for old_facet in &self.facets {
+            assert!(
+                !old_facet.is_prefix_of(&facet),
+                "Tried to add a facet which is a descendant of an already added facet."
+            );
+            assert!(
+                !facet.is_prefix_of(old_facet),
+                "Tried to add a facet which is an ancestor of an already added facet."
+            );
+        }
+        self.facets.insert(facet);
+    }
+
+    fn set_collapse_mapping(&mut self, facet_reader: &FacetReader) {
+        self.current_segment_collapse_mapping.clear();
+        self.current_collapse_facet_ords.clear();
+        self.current_segment_counts.clear();
+        let mut collapse_facet_it = self.facets.iter().peekable();
+        self.current_collapse_facet_ords.push(0);
+        let mut facet_streamer = facet_reader.facet_dict().range().into_stream();
+        if !facet_streamer.advance() {
+            return;
+        }
+        'outer: loop {
+            // at the begining of this loop, facet_streamer
+            // is positionned on a term that has not been processed yet.
+            let skip_result = skip(facet_streamer.key(), &mut collapse_facet_it);
+            match skip_result {
+                SkipResult::Reached => {
+                    // we reach a facet we decided to collapse.
+                    let collapse_depth = facet_depth(facet_streamer.key());
+                    let mut collapsed_id = 0;
+                    self.current_segment_collapse_mapping.push(0);
+                    while facet_streamer.advance() {
+                        let depth = facet_depth(facet_streamer.key());
+                        if depth <= collapse_depth {
+                            continue 'outer;
+                        }
+                        if depth == collapse_depth + 1 {
+                            collapsed_id = self.current_collapse_facet_ords.len();
+                            self.current_collapse_facet_ords
+                                .push(facet_streamer.term_ord());
+                            self.current_segment_collapse_mapping.push(collapsed_id);
+                        } else {
+                            self.current_segment_collapse_mapping.push(collapsed_id);
+                        }
+                    }
+                    break;
+                }
+                SkipResult::End | SkipResult::OverStep => {
+                    self.current_segment_collapse_mapping.push(0);
+                    if !facet_streamer.advance() {
+                        break;
+                    }
+                }
+            }
+        }
+    }
+
+    fn finalize_segment(&mut self) {
+        if self.ff_reader.is_some() {
+            self.segment_counters.push(SegmentFacetCounter {
+                facet_reader: self.ff_reader.take().unwrap().into_inner(),
+                facet_ords: mem::replace(&mut self.current_collapse_facet_ords, Vec::new()),
+                facet_counts: mem::replace(&mut self.current_segment_counts, Vec::new()),
+            });
+        }
+    }
+
+    /// Returns the results of the collection.
+    ///
+    /// This method does not just return the counters,
+    /// it also translates the facet ordinals of the last segment.
+    pub fn harvest(mut self) -> FacetCounts {
+        self.finalize_segment();
+
+        let collapsed_facet_ords: Vec<&[u64]> = self.segment_counters
+            .iter()
+            .map(|segment_counter| &segment_counter.facet_ords[..])
+            .collect();
+        let collapsed_facet_counts: Vec<&[u64]> = self.segment_counters
+            .iter()
+            .map(|segment_counter| &segment_counter.facet_counts[..])
+            .collect();
+
+        let facet_streams = self.segment_counters
+            .iter()
+            .map(|seg_counts| seg_counts.facet_reader.facet_dict().range().into_stream())
+            .collect::<Vec<_>>();
+
+        let mut facet_merger = TermMerger::new(facet_streams);
+        let mut facet_counts = BTreeMap::new();
+
+        while facet_merger.advance() {
+            let count = facet_merger
+                .current_kvs()
+                .iter()
+                .map(|it| {
+                    let seg_ord = it.segment_ord;
+                    let term_ord = it.streamer.term_ord();
+                    collapsed_facet_ords[seg_ord]
+                        .binary_search(&term_ord)
+                        .map(|collapsed_term_id| {
+                            if collapsed_term_id == 0 {
+                                0
+                            } else {
+                                collapsed_facet_counts[seg_ord][collapsed_term_id]
+                            }
+                        })
+                        .unwrap_or(0)
+                })
+                .sum();
+            if count > 0u64 {
+                let bytes = facet_merger.key().to_owned();
+                facet_counts.insert(Facet::from_encoded(bytes), count);
+            }
+        }
+        FacetCounts { facet_counts }
+    }
+}
+
+impl Collector for FacetCollector {
     fn set_segment(&mut self, _: SegmentLocalId, reader: &SegmentReader) -> Result<()> {
-        self.ff_reader = Some(reader.get_fast_field_reader(self.field)?);
+        self.finalize_segment();
+        let facet_reader = reader.facet_reader(self.field)?;
+        self.set_collapse_mapping(&facet_reader);
+        self.current_segment_counts
+            .resize(self.current_collapse_facet_ords.len(), 0);
+        self.ff_reader = Some(UnsafeCell::new(facet_reader));
         Ok(())
     }
 
     fn collect(&mut self, doc: DocId, _: Score) {
-        let val = self.ff_reader
-            .as_ref()
-            .expect("collect() was called before set_segment. This should never happen.")
-            .get(doc);
-        *(self.counters.entry(val).or_insert(0)) += 1;
+        let facet_reader: &mut FacetReader = unsafe {
+            &mut *self.ff_reader
+                .as_ref()
+                .expect("collect() was called before set_segment. This should never happen.")
+                .get()
+        };
+        facet_reader.facet_ords(doc, &mut self.facet_ords);
+        let mut previous_collapsed_ord: usize = usize::MAX;
+        for &facet_ord in &self.facet_ords {
+            let collapsed_ord = self.current_segment_collapse_mapping[facet_ord as usize];
+            self.current_segment_counts[collapsed_ord] += if collapsed_ord == previous_collapsed_ord
+            {
+                0
+            } else {
+                1
+            };
+            previous_collapsed_ord = collapsed_ord;
+        }
+    }
+
+    fn requires_scoring(&self) -> bool {
+        false
     }
 }
 
+/// Intermediary result of the `FacetCollector` that stores
+/// the facet counts for all the segments.
+pub struct FacetCounts {
+    facet_counts: BTreeMap<Facet, u64>,
+}
 
+impl FacetCounts {
+    #[allow(needless_lifetimes)] //< compiler fails if we remove the lifetime
+    pub fn get<'a, T>(&'a self, facet_from: T) -> impl Iterator<Item = (&'a Facet, u64)>
+    where
+        Facet: From<T>,
+    {
+        let facet = Facet::from(facet_from);
+        let left_bound = Bound::Excluded(facet.clone());
+        let right_bound = if facet.is_root() {
+            Bound::Unbounded
+        } else {
+            let mut facet_after_bytes = facet.encoded_bytes().to_owned();
+            facet_after_bytes.push(1u8);
+            let facet_after = Facet::from_encoded(facet_after_bytes);
+            Bound::Excluded(facet_after)
+        };
+
+        self.facet_counts
+            .range((left_bound, right_bound))
+            .map(|(facet, count)| (facet, *count))
+    }
+
+    pub fn top_k<T>(&self, facet: T, k: usize) -> Vec<(&Facet, u64)>
+    where
+        Facet: From<T>,
+    {
+        let mut heap = BinaryHeap::with_capacity(k);
+        let mut it = self.get(facet);
+
+        for (facet, count) in (&mut it).take(k) {
+            heap.push(Hit { count, facet });
+        }
+
+        let mut lowest_count: u64 = heap.peek().map(|hit| hit.count).unwrap_or(u64::MIN);
+        for (facet, count) in it {
+            if count > lowest_count {
+                lowest_count = count;
+                if let Some(mut head) = heap.peek_mut() {
+                    *head = Hit { count, facet };
+                }
+            }
+        }
+        heap.into_sorted_vec()
+            .into_iter()
+            .map(|hit| (hit.facet, hit.count))
+            .collect::<Vec<_>>()
+    }
+}
 
 #[cfg(test)]
 mod tests {
-
-    use collector::{chain, FacetCollector};
-    use query::QueryParser;
-    use fastfield::{I64FastFieldReader, U64FastFieldReader};
-    use schema::{self, FAST, STRING};
-    use Index;
+    use test::Bencher;
+    use core::Index;
+    use schema::{Document, Facet, SchemaBuilder};
+    use query::AllQuery;
+    use super::{FacetCollector, FacetCounts};
+    use std::iter;
+    use schema::Field;
+    use rand::{thread_rng, Rng};
 
     #[test]
-    // create 10 documents, set num field value to 0 or 1 for even/odd ones
-    // make sure we have facet counters correctly filled
-    fn test_facet_collector_results() {
-
-        let mut schema_builder = schema::SchemaBuilder::new();
-        let num_field_i64 = schema_builder.add_i64_field("num_i64", FAST);
-        let num_field_u64 = schema_builder.add_u64_field("num_u64", FAST);
-        let text_field = schema_builder.add_text_field("text", STRING);
+    fn test_facet_collector_drilldown() {
+        let mut schema_builder = SchemaBuilder::new();
+        let facet_field = schema_builder.add_facet_field("facet");
         let schema = schema_builder.build();
+        let index = Index::create_in_ram(schema);
 
-        let index = Index::create_in_ram(schema.clone());
-
-        {
-            let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
-            {
-                for i in 0u64..10u64 {
-                    index_writer.add_document(doc!(
-                        num_field_i64 => ((i as i64) % 3i64) as i64,
-                        num_field_u64 => (i % 2u64) as u64,
-                        text_field => "text"
-                    ));
-                }
-            }
-            assert_eq!(index_writer.commit().unwrap(), 10u64);
+        let mut index_writer = index.writer(3_000_000).unwrap();
+        let num_facets: usize = 3 * 4 * 5;
+        let facets: Vec<Facet> = (0..num_facets)
+            .map(|mut n| {
+                let top = n % 3;
+                n /= 3;
+                let mid = n % 4;
+                n /= 4;
+                let leaf = n % 5;
+                Facet::from(&format!("/top{}/mid{}/leaf{}", top, mid, leaf))
+            })
+            .collect();
+        for i in 0..num_facets * 10 {
+            let mut doc = Document::new();
+            doc.add_facet(facet_field, facets[i % num_facets].clone());
+            index_writer.add_document(doc);
         }
-
+        index_writer.commit().unwrap();
         index.load_searchers().unwrap();
         let searcher = index.searcher();
-        let mut ffvf_i64: FacetCollector<I64FastFieldReader> = FacetCollector::new(num_field_i64);
-        let mut ffvf_u64: FacetCollector<U64FastFieldReader> = FacetCollector::new(num_field_u64);
 
+        let mut facet_collector = FacetCollector::for_field(facet_field);
+        facet_collector.add_facet(Facet::from("/top1"));
+        searcher.search(&AllQuery, &mut facet_collector).unwrap();
+
+        let counts: FacetCounts = facet_collector.harvest();
         {
-            // perform the query
-            let mut facet_collectors = chain().push(&mut ffvf_i64).push(&mut ffvf_u64);
-            let query_parser = QueryParser::new(schema, vec![text_field]);
-            let query = query_parser.parse_query("text:text").unwrap();
-            query.search(&searcher, &mut facet_collectors).unwrap();
+            let facets: Vec<(String, u64)> = counts
+                .get("/top1")
+                .map(|(facet, count)| (facet.to_string(), count))
+                .collect();
+            assert_eq!(
+                facets,
+                [
+                    ("/top1/mid0", 50),
+                    ("/top1/mid1", 50),
+                    ("/top1/mid2", 50),
+                    ("/top1/mid3", 50),
+                ].iter()
+                    .map(|&(facet_str, count)| (String::from(facet_str), count))
+                    .collect::<Vec<_>>()
+            );
         }
+    }
 
-        assert_eq!(ffvf_u64.counters[&0], 5);
-        assert_eq!(ffvf_u64.counters[&1], 5);
-        assert_eq!(ffvf_i64.counters[&0], 4);
-        assert_eq!(ffvf_i64.counters[&1], 3);
+    #[test]
+    #[should_panic(expected = "Tried to add a facet which is a descendant of \
+                               an already added facet.")]
+    fn test_misused_facet_collector() {
+        let mut facet_collector = FacetCollector::for_field(Field(0));
+        facet_collector.add_facet(Facet::from("/country"));
+        facet_collector.add_facet(Facet::from("/country/europe"));
+    }
 
+    #[test]
+    fn test_non_used_facet_collector() {
+        let mut facet_collector = FacetCollector::for_field(Field(0));
+        facet_collector.add_facet(Facet::from("/country"));
+        facet_collector.add_facet(Facet::from("/countryeurope"));
+    }
+
+    #[test]
+    fn test_facet_collector_topk() {
+        let mut schema_builder = SchemaBuilder::new();
+        let facet_field = schema_builder.add_facet_field("facet");
+        let schema = schema_builder.build();
+        let index = Index::create_in_ram(schema);
+
+        let mut docs: Vec<Document> = vec![("a", 10), ("b", 100), ("c", 7), ("d", 12), ("e", 21)]
+            .into_iter()
+            .flat_map(|(c, count)| {
+                let facet = Facet::from(&format!("/facet_{}", c));
+                let doc = doc!(facet_field => facet);
+                iter::repeat(doc).take(count)
+            })
+            .collect();
+        thread_rng().shuffle(&mut docs[..]);
+
+        let mut index_writer = index.writer(3_000_000).unwrap();
+        for doc in docs {
+            index_writer.add_document(doc);
+        }
+        index_writer.commit().unwrap();
+        index.load_searchers().unwrap();
+
+        let searcher = index.searcher();
+
+        let mut facet_collector = FacetCollector::for_field(facet_field);
+        facet_collector.add_facet("/");
+        searcher.search(&AllQuery, &mut facet_collector).unwrap();
+
+        let counts: FacetCounts = facet_collector.harvest();
+        {
+            let facets: Vec<(&Facet, u64)> = counts.top_k("/", 3);
+            assert_eq!(
+                facets,
+                vec![
+                    (&Facet::from("/facet_b"), 100),
+                    (&Facet::from("/facet_e"), 21),
+                    (&Facet::from("/facet_d"), 12),
+                ]
+            );
+        }
+    }
+
+    #[bench]
+    fn bench_facet_collector(b: &mut Bencher) {
+        let mut schema_builder = SchemaBuilder::new();
+        let facet_field = schema_builder.add_facet_field("facet");
+        let schema = schema_builder.build();
+        let index = Index::create_in_ram(schema);
+
+        let mut docs = vec![];
+        for val in 0..50 {
+            let facet = Facet::from(&format!("/facet_{}", val));
+            for _ in 0..val * val {
+                docs.push(doc!(facet_field=>facet.clone()));
+            }
+        }
+        // 40425 docs
+        thread_rng().shuffle(&mut docs[..]);
+
+        let mut index_writer = index.writer(3_000_000).unwrap();
+        for doc in docs {
+            index_writer.add_document(doc);
+        }
+        index_writer.commit().unwrap();
+        index.load_searchers().unwrap();
+
+        b.iter(|| {
+            let searcher = index.searcher();
+            let mut facet_collector = FacetCollector::for_field(facet_field);
+            searcher.search(&AllQuery, &mut facet_collector).unwrap();
+        });
     }
 }

src/collector/int_facet_collector.rs

@@ -0,0 +1,123 @@
+use std::cmp::Eq;
+use std::collections::HashMap;
+use std::hash::Hash;
+
+use collector::Collector;
+use fastfield::FastFieldReader;
+use schema::Field;
+
+use DocId;
+use Result;
+use Score;
+use SegmentReader;
+use SegmentLocalId;
+
+
+/// Facet collector  for i64/u64 fast field
+pub struct IntFacetCollector<T>
+where
+    T: FastFieldReader,
+    T::ValueType: Eq + Hash,
+{
+    counters: HashMap<T::ValueType, u64>,
+    field: Field,
+    ff_reader: Option<T>,
+}
+
+
+impl<T> IntFacetCollector<T>
+where
+    T: FastFieldReader,
+    T::ValueType: Eq + Hash,
+{
+    /// Creates a new facet collector for aggregating a given field.
+    pub fn new(field: Field) -> IntFacetCollector<T> {
+        IntFacetCollector {
+            counters: HashMap::new(),
+            field: field,
+            ff_reader: None,
+        }
+    }
+}
+
+
+impl<T> Collector for IntFacetCollector<T>
+where
+    T: FastFieldReader,
+    T::ValueType: Eq + Hash,
+{
+    fn set_segment(&mut self, _: SegmentLocalId, reader: &SegmentReader) -> Result<()> {
+        self.ff_reader = Some(reader.get_fast_field_reader(self.field)?);
+        Ok(())
+    }
+
+    fn collect(&mut self, doc: DocId, _: Score) {
+        let val = self.ff_reader
+            .as_ref()
+            .expect(
+                "collect() was called before set_segment. \
+                This should never happen.",
+            )
+            .get(doc);
+        *(self.counters.entry(val).or_insert(0)) += 1;
+    }
+}
+
+
+
+#[cfg(test)]
+mod tests {
+
+    use collector::{chain, IntFacetCollector};
+    use query::QueryParser;
+    use fastfield::{I64FastFieldReader, U64FastFieldReader};
+    use schema::{self, FAST, STRING};
+    use Index;
+
+    #[test]
+    // create 10 documents, set num field value to 0 or 1 for even/odd ones
+    // make sure we have facet counters correctly filled
+    fn test_facet_collector_results() {
+
+        let mut schema_builder = schema::SchemaBuilder::new();
+        let num_field_i64 = schema_builder.add_i64_field("num_i64", FAST);
+        let num_field_u64 = schema_builder.add_u64_field("num_u64", FAST);
+        let text_field = schema_builder.add_text_field("text", STRING);
+        let schema = schema_builder.build();
+
+        let index = Index::create_in_ram(schema.clone());
+
+        {
+            let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
+            {
+                for i in 0u64..10u64 {
+                    index_writer.add_document(doc!(
+                        num_field_i64 => ((i as i64) % 3i64) as i64,
+                        num_field_u64 => (i % 2u64) as u64,
+                        text_field => "text"
+                    ));
+                }
+            }
+            assert_eq!(index_writer.commit().unwrap(), 10u64);
+        }
+
+        index.load_searchers().unwrap();
+        let searcher = index.searcher();
+        let mut ffvf_i64: IntFacetCollector<I64FastFieldReader> = IntFacetCollector::new(num_field_i64);
+        let mut ffvf_u64: IntFacetCollector<U64FastFieldReader> = IntFacetCollector::new(num_field_u64);
+
+        {
+            // perform the query
+            let mut facet_collectors = chain().push(&mut ffvf_i64).push(&mut ffvf_u64);
+            let mut query_parser = QueryParser::for_index(index, vec![text_field]);
+            let query = query_parser.parse_query("text:text").unwrap();
+            query.search(&searcher, &mut facet_collectors).unwrap();
+        }
+
+        assert_eq!(ffvf_u64.counters[&0], 5);
+        assert_eq!(ffvf_u64.counters[&1], 5);
+        assert_eq!(ffvf_i64.counters[&0], 4);
+        assert_eq!(ffvf_i64.counters[&1], 3);
+
+    }
+}

src/collector/mod.rs

@@ -1,3 +1,7 @@
+/*!
+Defines how the documents matching a search query should be processed.
+*/
+
 use SegmentReader;
 use SegmentLocalId;
 use DocId;
@@ -51,29 +55,36 @@ pub use self::chained_collector::chain;
 pub trait Collector {
     /// `set_segment` is called before beginning to enumerate
     /// on this segment.
-    fn set_segment(&mut self,
-                   segment_local_id: SegmentLocalId,
-                   segment: &SegmentReader)
-                   -> Result<()>;
+    fn set_segment(
+        &mut self,
+        segment_local_id: SegmentLocalId,
+        segment: &SegmentReader,
+    ) -> Result<()>;
     /// The query pushes the scored document to the collector via this method.
     fn collect(&mut self, doc: DocId, score: Score);
+
+    /// Returns true iff the collector requires to compute scores for documents.
+    fn requires_scoring(&self) -> bool;
 }
 
-
 impl<'a, C: Collector> Collector for &'a mut C {
-    fn set_segment(&mut self,
-                   segment_local_id: SegmentLocalId,
-                   segment: &SegmentReader)
-                   -> Result<()> {
+    fn set_segment(
+        &mut self,
+        segment_local_id: SegmentLocalId,
+        segment: &SegmentReader,
+    ) -> Result<()> {
         (*self).set_segment(segment_local_id, segment)
     }
     /// The query pushes the scored document to the collector via this method.
     fn collect(&mut self, doc: DocId, score: Score) {
-        (*self).collect(doc, score);
+        C::collect(self, doc, score)
+    }
+
+    fn requires_scoring(&self) -> bool {
+        C::requires_scoring(self)
     }
 }
 
-
 #[cfg(test)]
 pub mod tests {
 
@@ -83,7 +94,6 @@ pub mod tests {
     use Score;
     use core::SegmentReader;
     use SegmentLocalId;
-    use fastfield::U64FastFieldReader;
     use fastfield::FastFieldReader;
     use schema::Field;
 
@@ -124,11 +134,12 @@ pub mod tests {
         fn collect(&mut self, doc: DocId, _score: Score) {
             self.docs.push(doc + self.offset);
         }
+
+        fn requires_scoring(&self) -> bool {
+            false
+        }
     }
 
-
-
-
     /// Collects in order all of the fast fields for all of the
     /// doc in the `DocSet`
     ///
@@ -136,14 +147,14 @@ pub mod tests {
     pub struct FastFieldTestCollector {
         vals: Vec<u64>,
         field: Field,
-        ff_reader: Option<U64FastFieldReader>,
+        ff_reader: Option<FastFieldReader<u64>>,
     }
 
     impl FastFieldTestCollector {
         pub fn for_field(field: Field) -> FastFieldTestCollector {
             FastFieldTestCollector {
                 vals: Vec::new(),
-                field: field,
+                field,
                 ff_reader: None,
             }
         }
@@ -155,7 +166,7 @@ pub mod tests {
 
     impl Collector for FastFieldTestCollector {
         fn set_segment(&mut self, _: SegmentLocalId, reader: &SegmentReader) -> Result<()> {
-            self.ff_reader = Some(reader.get_fast_field_reader(self.field)?);
+            self.ff_reader = Some(reader.fast_field_reader(self.field)?);
             Ok(())
         }
 
@@ -163,18 +174,20 @@ pub mod tests {
             let val = self.ff_reader.as_ref().unwrap().get(doc);
             self.vals.push(val);
         }
+        fn requires_scoring(&self) -> bool {
+            false
+        }
     }
 
-
     #[bench]
     fn build_collector(b: &mut Bencher) {
         b.iter(|| {
-                   let mut count_collector = CountCollector::default();
-                   let docs: Vec<u32> = (0..1_000_000).collect();
-                   for doc in docs {
-                       count_collector.collect(doc, 1f32);
-                   }
-                   count_collector.count()
-               });
+            let mut count_collector = CountCollector::default();
+            let docs: Vec<u32> = (0..1_000_000).collect();
+            for doc in docs {
+                count_collector.collect(doc, 1f32);
+            }
+            count_collector.count()
+        });
     }
 }

src/collector/multi_collector.rs

@@ -5,7 +5,6 @@ use Result;
 use SegmentReader;
 use SegmentLocalId;
 
-
 /// Multicollector makes it possible to collect on more than one collector.
 /// It should only be used for use cases where the Collector types is unknown
 /// at compile time.
@@ -17,18 +16,18 @@ pub struct MultiCollector<'a> {
 impl<'a> MultiCollector<'a> {
     /// Constructor
     pub fn from(collectors: Vec<&'a mut Collector>) -> MultiCollector {
-        MultiCollector { collectors: collectors }
+        MultiCollector { collectors }
     }
 }
 
-
 impl<'a> Collector for MultiCollector<'a> {
-    fn set_segment(&mut self,
-                   segment_local_id: SegmentLocalId,
-                   segment: &SegmentReader)
-                   -> Result<()> {
+    fn set_segment(
+        &mut self,
+        segment_local_id: SegmentLocalId,
+        segment: &SegmentReader,
+    ) -> Result<()> {
         for collector in &mut self.collectors {
-            try!(collector.set_segment(segment_local_id, segment));
+            collector.set_segment(segment_local_id, segment)?;
         }
         Ok(())
     }
@@ -38,10 +37,13 @@ impl<'a> Collector for MultiCollector<'a> {
             collector.collect(doc, score);
         }
     }
+    fn requires_scoring(&self) -> bool {
+        self.collectors
+            .iter()
+            .any(|collector| collector.requires_scoring())
+    }
 }
 
-
-
 #[cfg(test)]
 mod tests {
 
@@ -53,8 +55,8 @@ mod tests {
         let mut top_collector = TopCollector::with_limit(2);
         let mut count_collector = CountCollector::default();
         {
-            let mut collectors = MultiCollector::from(vec![&mut top_collector,
-                                                           &mut count_collector]);
+            let mut collectors =
+                MultiCollector::from(vec![&mut top_collector, &mut count_collector]);
             collectors.collect(1, 0.2);
             collectors.collect(2, 0.1);
             collectors.collect(3, 0.5);

src/collector/top_collector.rs

@@ -39,7 +39,6 @@ impl PartialEq for GlobalScoredDoc {
 
 impl Eq for GlobalScoredDoc {}
 
-
 /// The Top Collector keeps track of the K documents
 /// with the best scores.
 ///
@@ -61,7 +60,7 @@ impl TopCollector {
             panic!("Limit must be strictly greater than 0.");
         }
         TopCollector {
-            limit: limit,
+            limit,
             heap: BinaryHeap::with_capacity(limit),
             segment_id: 0,
         }
@@ -108,10 +107,9 @@ impl Collector for TopCollector {
     fn collect(&mut self, doc: DocId, score: Score) {
         if self.at_capacity() {
             // It's ok to unwrap as long as a limit of 0 is forbidden.
-            let limit_doc: GlobalScoredDoc =
-                *self.heap
-                     .peek()
-                     .expect("Top collector with size 0 is forbidden");
+            let limit_doc: GlobalScoredDoc = *self.heap
+                .peek()
+                .expect("Top collector with size 0 is forbidden");
             if limit_doc.score < score {
                 let mut mut_head = self.heap
                     .peek_mut()
@@ -121,16 +119,18 @@ impl Collector for TopCollector {
             }
         } else {
             let wrapped_doc = GlobalScoredDoc {
-                score: score,
+                score,
                 doc_address: DocAddress(self.segment_id, doc),
             };
             self.heap.push(wrapped_doc);
         }
+    }
 
+    fn requires_scoring(&self) -> bool {
+        true
     }
 }
 
-
 #[cfg(test)]
 mod tests {
 
@@ -179,8 +179,6 @@ mod tests {
                 .collect();
             assert_eq!(docs, vec![7, 1, 5, 3]);
         }
-
-
     }
 
     #[test]

src/common/bitpacker.rs

@@ -4,60 +4,35 @@ use common::serialize::BinarySerializable;
 use std::mem;
 use std::ops::Deref;
 
-/// Computes the number of bits that will be used for bitpacking.
-///
-/// In general the target is the minimum number of bits
-/// required to express the amplitude given in argument.
-///
-/// e.g. If the amplitude is 10, we can store all ints on simply 4bits.
-///
-/// The logic is slightly more convoluted here as for optimization
-/// reasons, we want to ensure that a value spawns over at most 8 bytes
-/// of aligns bytes.
-///
-/// Spawning over 9 bytes is possible for instance, if we do
-/// bitpacking with an amplitude of 63 bits.
-/// In this case, the second int will start on bit
-/// 63 (which belongs to byte 7) and ends at byte 15;
-/// Hence 9 bytes (from byte 7 to byte 15 included).
-///
-/// To avoid this, we force the number of bits to 64bits
-/// when the result is greater than `64-8 = 56 bits`.
-///
-/// Note that this only affects rare use cases spawning over
-/// a very large range of values. Even in this case, it results
-/// in an extra cost of at most 12% compared to the optimal
-/// number of bits.
-pub fn compute_num_bits(amplitude: u64) -> u8 {
-    let amplitude = (64u32 - amplitude.leading_zeros()) as u8;
-    if amplitude <= 64 - 8 { amplitude } else { 64 }
-}
-
-pub struct BitPacker {
+pub(crate) struct BitPacker {
     mini_buffer: u64,
     mini_buffer_written: usize,
-    num_bits: usize,
 }
 
 impl BitPacker {
-    pub fn new(num_bits: usize) -> BitPacker {
+    pub fn new() -> BitPacker {
         BitPacker {
             mini_buffer: 0u64,
             mini_buffer_written: 0,
-            num_bits: num_bits,
         }
     }
 
-    pub fn write<TWrite: Write>(&mut self, val: u64, output: &mut TWrite) -> io::Result<()> {
+    pub fn write<TWrite: Write>(
+        &mut self,
+        val: u64,
+        num_bits: u8,
+        output: &mut TWrite,
+    ) -> io::Result<()> {
         let val_u64 = val as u64;
-        if self.mini_buffer_written + self.num_bits > 64 {
+        let num_bits = num_bits as usize;
+        if self.mini_buffer_written + num_bits > 64 {
             self.mini_buffer |= val_u64.wrapping_shl(self.mini_buffer_written as u32);
             self.mini_buffer.serialize(output)?;
             self.mini_buffer = val_u64.wrapping_shr((64 - self.mini_buffer_written) as u32);
-            self.mini_buffer_written = self.mini_buffer_written + (self.num_bits as usize) - 64;
+            self.mini_buffer_written = self.mini_buffer_written + num_bits - 64;
         } else {
             self.mini_buffer |= val_u64 << self.mini_buffer_written;
-            self.mini_buffer_written += self.num_bits;
+            self.mini_buffer_written += num_bits;
             if self.mini_buffer_written == 64 {
                 self.mini_buffer.serialize(output)?;
                 self.mini_buffer_written = 0;
@@ -67,7 +42,7 @@ impl BitPacker {
         Ok(())
     }
 
-    fn flush<TWrite: Write>(&mut self, output: &mut TWrite) -> io::Result<()> {
+    pub fn flush<TWrite: Write>(&mut self, output: &mut TWrite) -> io::Result<()> {
         if self.mini_buffer_written > 0 {
             let num_bytes = (self.mini_buffer_written + 7) / 8;
             let arr: [u8; 8] = unsafe { mem::transmute::<u64, [u8; 8]>(self.mini_buffer) };
@@ -85,10 +60,10 @@ impl BitPacker {
     }
 }
 
-
-
+#[derive(Clone)]
 pub struct BitUnpacker<Data>
-    where Data: Deref<Target = [u8]>
+where
+    Data: Deref<Target = [u8]>,
 {
     num_bits: usize,
     mask: u64,
@@ -96,24 +71,25 @@ pub struct BitUnpacker<Data>
 }
 
 impl<Data> BitUnpacker<Data>
-    where Data: Deref<Target = [u8]>
+where
+    Data: Deref<Target = [u8]>,
 {
-    pub fn new(data: Data, num_bits: usize) -> BitUnpacker<Data> {
+    pub fn new(data: Data, num_bits: u8) -> BitUnpacker<Data> {
         let mask: u64 = if num_bits == 64 {
             !0u64
         } else {
             (1u64 << num_bits) - 1u64
         };
         BitUnpacker {
-            num_bits: num_bits,
-            mask: mask,
-            data: data,
+            num_bits: num_bits as usize,
+            mask,
+            data,
         }
     }
 
     pub fn get(&self, idx: usize) -> u64 {
         if self.num_bits == 0 {
-            return 0;
+            return 0u64;
         }
         let data: &[u8] = &*self.data;
         let num_bits = self.num_bits;
@@ -121,17 +97,40 @@ impl<Data> BitUnpacker<Data>
         let addr_in_bits = idx * num_bits;
         let addr = addr_in_bits >> 3;
         let bit_shift = addr_in_bits & 7;
-        debug_assert!(addr + 8 <= data.len(),
-                      "The fast field field should have been padded with 7 bytes.");
-        let val_unshifted_unmasked: u64 = unsafe { *(data[addr..].as_ptr() as *const u64) };
-        let val_shifted = (val_unshifted_unmasked >> bit_shift) as u64;
-        (val_shifted & mask)
+        if cfg!(feature = "simdcompression") {
+            // for simdcompression,
+            // the bitpacker is only used for fastfields,
+            // and we expect them to be always padded.
+            debug_assert!(
+                addr + 8 <= data.len(),
+                "The fast field field should have been padded with 7 bytes."
+            );
+            let val_unshifted_unmasked: u64 = unsafe { *(data[addr..].as_ptr() as *const u64) };
+            let val_shifted = (val_unshifted_unmasked >> bit_shift) as u64;
+            val_shifted & mask
+        } else {
+            let val_unshifted_unmasked: u64 = if addr + 8 <= data.len() {
+                unsafe { *(data[addr..].as_ptr() as *const u64) }
+            } else {
+                let mut buffer = [0u8; 8];
+                for i in addr..data.len() {
+                    buffer[i - addr] += data[i];
+                }
+                unsafe { *(buffer[..].as_ptr() as *const u64) }
+            };
+            let val_shifted = val_unshifted_unmasked >> (bit_shift as u64);
+            val_shifted & mask
+        }
     }
 
+    /// Reads a range of values from the fast field.
+    ///
+    /// The range of values read is from
+    /// `[start..start + output.len()[`
     pub fn get_range(&self, start: u32, output: &mut [u64]) {
         if self.num_bits == 0 {
             for val in output.iter_mut() {
-                *val = 0;
+                *val = 0u64;
             }
         } else {
             let data: &[u8] = &*self.data;
@@ -147,46 +146,30 @@ impl<Data> BitUnpacker<Data>
                 addr_in_bits += num_bits;
             }
         }
-
     }
 }
 
-
-
-
 #[cfg(test)]
 mod test {
-    use super::{BitPacker, BitUnpacker, compute_num_bits};
+    use super::{BitPacker, BitUnpacker};
 
-    #[test]
-    fn test_compute_num_bits() {
-        assert_eq!(compute_num_bits(1), 1u8);
-        assert_eq!(compute_num_bits(0), 0u8);
-        assert_eq!(compute_num_bits(2), 2u8);
-        assert_eq!(compute_num_bits(3), 2u8);
-        assert_eq!(compute_num_bits(4), 3u8);
-        assert_eq!(compute_num_bits(255), 8u8);
-        assert_eq!(compute_num_bits(256), 9u8);
-        assert_eq!(compute_num_bits(5_000_000_000), 33u8);
-    }
-
-    fn create_fastfield_bitpacker(len: usize, num_bits: usize) -> (BitUnpacker<Vec<u8>>, Vec<u64>) {
+    fn create_fastfield_bitpacker(len: usize, num_bits: u8) -> (BitUnpacker<Vec<u8>>, Vec<u64>) {
         let mut data = Vec::new();
-        let mut bitpacker = BitPacker::new(num_bits);
-        let max_val: u64 = (1 << num_bits) - 1;
+        let mut bitpacker = BitPacker::new();
+        let max_val: u64 = (1u64 << num_bits as u64) - 1u64;
         let vals: Vec<u64> = (0u64..len as u64)
             .map(|i| if max_val == 0 { 0 } else { i % max_val })
             .collect();
         for &val in &vals {
-            bitpacker.write(val, &mut data).unwrap();
+            bitpacker.write(val, num_bits, &mut data).unwrap();
         }
         bitpacker.close(&mut data).unwrap();
-        assert_eq!(data.len(), (num_bits * len + 7) / 8 + 7);
+        assert_eq!(data.len(), ((num_bits as usize) * len + 7) / 8 + 7);
         let bitunpacker = BitUnpacker::new(data, num_bits);
         (bitunpacker, vals)
     }
 
-    fn test_bitpacker_util(len: usize, num_bits: usize) {
+    fn test_bitpacker_util(len: usize, num_bits: u8) {
         let (bitunpacker, vals) = create_fastfield_bitpacker(len, num_bits);
         for (i, val) in vals.iter().enumerate() {
             assert_eq!(bitunpacker.get(i), *val);

src/common/bitset.rs

@@ -0,0 +1,390 @@
+use std::fmt;
+
+#[derive(Clone, Copy, Eq, PartialEq)]
+pub(crate) struct TinySet(u64);
+
+impl fmt::Debug for TinySet {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        self.into_iter().collect::<Vec<u32>>().fmt(f)
+    }
+}
+
+pub struct TinySetIterator(TinySet);
+impl Iterator for TinySetIterator {
+    type Item = u32;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        self.0.pop_lowest()
+    }
+}
+
+impl IntoIterator for TinySet {
+    type Item = u32;
+    type IntoIter = TinySetIterator;
+    fn into_iter(self) -> Self::IntoIter {
+        TinySetIterator(self)
+    }
+}
+
+impl TinySet {
+    /// Returns an empty `TinySet`.
+    pub fn empty() -> TinySet {
+        TinySet(0u64)
+    }
+
+    /// Returns the complement of the set in `[0, 64[`.
+    fn complement(&self) -> TinySet {
+        TinySet(!self.0)
+    }
+
+    /// Returns true iff the `TinySet` contains the element `el`.
+    pub fn contains(&self, el: u32) -> bool {
+        !self.intersect(TinySet::singleton(el)).is_empty()
+    }
+
+    /// Returns the intersection of `self` and `other`
+    pub fn intersect(&self, other: TinySet) -> TinySet {
+        TinySet(self.0 & other.0)
+    }
+
+    /// Creates a new `TinySet` containing only one element
+    /// within `[0; 64[`
+    #[inline(always)]
+    pub fn singleton(el: u32) -> TinySet {
+        TinySet(1u64 << u64::from(el))
+    }
+
+    /// Insert a new element within [0..64[
+    #[inline(always)]
+    pub fn insert(self, el: u32) -> TinySet {
+        self.union(TinySet::singleton(el))
+    }
+
+    /// Insert a new element within [0..64[
+    #[inline(always)]
+    pub fn insert_mut(&mut self, el: u32) -> bool {
+        let old = *self;
+        *self = old.insert(el);
+        old != *self
+    }
+
+    /// Returns the union of two tinysets
+    #[inline(always)]
+    pub fn union(self, other: TinySet) -> TinySet {
+        TinySet(self.0 | other.0)
+    }
+
+    /// Returns true iff the `TinySet` is empty.
+    #[inline(always)]
+    pub fn is_empty(&self) -> bool {
+        self.0 == 0u64
+    }
+
+    /// Returns the lowest element in the `TinySet`
+    /// and removes it.
+    #[inline(always)]
+    pub fn pop_lowest(&mut self) -> Option<u32> {
+        if let Some(lowest) = self.lowest() {
+            self.0 ^= TinySet::singleton(lowest).0;
+            Some(lowest)
+        } else {
+            None
+        }
+    }
+
+    /// Returns the lowest element in the `TinySet`
+    /// (or None if the set is empty).
+    #[inline(always)]
+    pub fn lowest(&mut self) -> Option<u32> {
+        if self.is_empty() {
+            None
+        } else {
+            let least_significant_bit = self.0.trailing_zeros() as u32;
+            Some(least_significant_bit)
+        }
+    }
+
+    /// Returns a `TinySet` than contains all values up
+    /// to limit excluded.
+    ///
+    /// The limit is assumed to be strictly lower than 64.
+    pub fn range_lower(upper_bound: u32) -> TinySet {
+        TinySet((1u64 << u64::from(upper_bound % 64u32)) - 1u64)
+    }
+
+    /// Returns a `TinySet` that contains all values greater
+    /// or equal to the given limit, included. (and up to 63)
+    ///
+    /// The limit is assumed to be strictly lower than 64.
+    pub fn range_greater_or_equal(from_included: u32) -> TinySet {
+        TinySet::range_lower(from_included).complement()
+    }
+
+    pub fn clear(&mut self) {
+        self.0 = 0u64;
+    }
+
+    pub fn len(&self) -> u32 {
+        self.0.count_ones()
+    }
+}
+
+#[derive(Clone)]
+pub struct BitSet {
+    tinysets: Box<[TinySet]>,
+    len: usize, //< Technically it should be u32, but we
+    // count multiple inserts.
+    // `usize` guards us from overflow.
+    max_value: u32,
+}
+
+fn num_buckets(max_val: u32) -> u32 {
+    (max_val + 63u32) / 64u32
+}
+
+impl BitSet {
+    /// Create a new `BitSet` that may contain elements
+    /// within `[0, max_val[`.
+    pub fn with_max_value(max_value: u32) -> BitSet {
+        let num_buckets = num_buckets(max_value);
+        let tinybisets = vec![TinySet::empty(); num_buckets as usize].into_boxed_slice();
+        BitSet {
+            tinysets: tinybisets,
+            len: 0,
+            max_value,
+        }
+    }
+
+    /// Removes all elements from the `BitSet`.
+    pub fn clear(&mut self) {
+        for tinyset in self.tinysets.iter_mut() {
+            *tinyset = TinySet::empty();
+        }
+    }
+
+    /// Returns the number of elements in the `BitSet`.
+    pub fn len(&self) -> usize {
+        self.len
+    }
+
+    /// Inserts an element in the `BitSet`
+    pub fn insert(&mut self, el: u32) {
+        // we do not check saturated els.
+        let higher = el / 64u32;
+        let lower = el % 64u32;
+        self.len += if self.tinysets[higher as usize].insert_mut(lower) {
+            1
+        } else {
+            0
+        };
+    }
+
+    /// Returns true iff the elements is in the `BitSet`.
+    pub fn contains(&self, el: u32) -> bool {
+        self.tinyset(el / 64u32).contains(el % 64)
+    }
+
+    /// Returns the first non-empty `TinySet` associated to a bucket lower
+    /// or greater than bucket.
+    ///
+    /// Reminder: the tiny set with the bucket `bucket`, represents the
+    /// elements from `bucket * 64` to `(bucket+1) * 64`.
+    pub(crate) fn first_non_empty_bucket(&self, bucket: u32) -> Option<u32> {
+        self.tinysets[bucket as usize..]
+            .iter()
+            .cloned()
+            .position(|tinyset| !tinyset.is_empty())
+            .map(|delta_bucket| bucket + delta_bucket as u32)
+    }
+
+    pub fn max_value(&self) -> u32 {
+        self.max_value
+    }
+
+    /// Returns the tiny bitset representing the
+    /// the set restricted to the number range from
+    /// `bucket * 64` to `(bucket + 1) * 64`.
+    pub(crate) fn tinyset(&self, bucket: u32) -> TinySet {
+        self.tinysets[bucket as usize]
+    }
+}
+
+#[cfg(test)]
+mod tests {
+
+    extern crate test;
+    use tests;
+    use std::collections::HashSet;
+    use super::BitSet;
+    use super::TinySet;
+    use tests::generate_nonunique_unsorted;
+    use std::collections::BTreeSet;
+    use query::BitSetDocSet;
+    use docset::DocSet;
+
+    #[test]
+    fn test_tiny_set() {
+        assert!(TinySet::empty().is_empty());
+        {
+            let mut u = TinySet::empty().insert(1u32);
+            assert_eq!(u.pop_lowest(), Some(1u32));
+            assert!(u.pop_lowest().is_none())
+        }
+        {
+            let mut u = TinySet::empty().insert(1u32).insert(1u32);
+            assert_eq!(u.pop_lowest(), Some(1u32));
+            assert!(u.pop_lowest().is_none())
+        }
+        {
+            let mut u = TinySet::empty().insert(2u32);
+            assert_eq!(u.pop_lowest(), Some(2u32));
+            u.insert_mut(1u32);
+            assert_eq!(u.pop_lowest(), Some(1u32));
+            assert!(u.pop_lowest().is_none());
+        }
+        {
+            let mut u = TinySet::empty().insert(63u32);
+            assert_eq!(u.pop_lowest(), Some(63u32));
+            assert!(u.pop_lowest().is_none());
+        }
+    }
+
+    #[test]
+    fn test_bitset() {
+        let test_against_hashset = |els: &[u32], max_value: u32| {
+            let mut hashset: HashSet<u32> = HashSet::new();
+            let mut bitset = BitSet::with_max_value(max_value);
+            for &el in els {
+                assert!(el < max_value);
+                hashset.insert(el);
+                bitset.insert(el);
+            }
+            for el in 0..max_value {
+                assert_eq!(hashset.contains(&el), bitset.contains(el));
+            }
+            assert_eq!(bitset.max_value(), max_value);
+        };
+
+        test_against_hashset(&[], 0);
+        test_against_hashset(&[], 1);
+        test_against_hashset(&[0u32], 1);
+        test_against_hashset(&[0u32], 100);
+        test_against_hashset(&[1u32, 2u32], 4);
+        test_against_hashset(&[99u32], 100);
+        test_against_hashset(&[63u32], 64);
+        test_against_hashset(&[62u32, 63u32], 64);
+    }
+
+    #[test]
+    fn test_bitset_large() {
+        let arr = generate_nonunique_unsorted(1_000_000, 50_000);
+        let mut btreeset: BTreeSet<u32> = BTreeSet::new();
+        let mut bitset = BitSet::with_max_value(1_000_000);
+        for el in arr {
+            btreeset.insert(el);
+            bitset.insert(el);
+        }
+        for i in 0..1_000_000 {
+            assert_eq!(btreeset.contains(&i), bitset.contains(i));
+        }
+        assert_eq!(btreeset.len(), bitset.len());
+        let mut bitset_docset = BitSetDocSet::from(bitset);
+        for el in btreeset.into_iter() {
+            bitset_docset.advance();
+            assert_eq!(bitset_docset.doc(), el);
+        }
+        assert!(!bitset_docset.advance());
+    }
+
+    #[test]
+    fn test_bitset_num_buckets() {
+        use super::num_buckets;
+        assert_eq!(num_buckets(0u32), 0);
+        assert_eq!(num_buckets(1u32), 1);
+        assert_eq!(num_buckets(64u32), 1);
+        assert_eq!(num_buckets(65u32), 2);
+        assert_eq!(num_buckets(128u32), 2);
+        assert_eq!(num_buckets(129u32), 3);
+    }
+
+    #[test]
+    fn test_tinyset_range() {
+        assert_eq!(
+            TinySet::range_lower(3).into_iter().collect::<Vec<u32>>(),
+            [0, 1, 2]
+        );
+        assert!(TinySet::range_lower(0).is_empty());
+        assert_eq!(
+            TinySet::range_lower(63).into_iter().collect::<Vec<u32>>(),
+            (0u32..63u32).collect::<Vec<_>>()
+        );
+        assert_eq!(
+            TinySet::range_lower(1).into_iter().collect::<Vec<u32>>(),
+            [0]
+        );
+        assert_eq!(
+            TinySet::range_lower(2).into_iter().collect::<Vec<u32>>(),
+            [0, 1]
+        );
+        assert_eq!(
+            TinySet::range_greater_or_equal(3)
+                .into_iter()
+                .collect::<Vec<u32>>(),
+            (3u32..64u32).collect::<Vec<_>>()
+        );
+    }
+
+    #[test]
+    fn test_bitset_len() {
+        let mut bitset = BitSet::with_max_value(1_000);
+        assert_eq!(bitset.len(), 0);
+        bitset.insert(3u32);
+        assert_eq!(bitset.len(), 1);
+        bitset.insert(103u32);
+        assert_eq!(bitset.len(), 2);
+        bitset.insert(3u32);
+        assert_eq!(bitset.len(), 2);
+        bitset.insert(103u32);
+        assert_eq!(bitset.len(), 2);
+        bitset.insert(104u32);
+        assert_eq!(bitset.len(), 3);
+    }
+
+    #[test]
+    fn test_bitset_clear() {
+        let mut bitset = BitSet::with_max_value(1_000);
+        let els = tests::sample(1_000, 0.01f32);
+        for &el in &els {
+            bitset.insert(el);
+        }
+        assert!(els.iter().all(|el| bitset.contains(*el)));
+        bitset.clear();
+        for el in 0u32..1000u32 {
+            assert!(!bitset.contains(el));
+        }
+    }
+
+    #[bench]
+    fn bench_tinyset_pop(b: &mut test::Bencher) {
+        b.iter(|| test::black_box(TinySet::singleton(31u32)).pop_lowest());
+    }
+
+    #[bench]
+    fn bench_tinyset_sum(b: &mut test::Bencher) {
+        let tiny_set = TinySet::empty().insert(10u32).insert(14u32).insert(21u32);
+        b.iter(|| {
+            assert_eq!(test::black_box(tiny_set).into_iter().sum::<u32>(), 45u32);
+        });
+    }
+
+    #[bench]
+    fn bench_tinyarr_sum(b: &mut test::Bencher) {
+        let v = [10u32, 14u32, 21u32];
+        b.iter(|| test::black_box(v).iter().cloned().sum::<u32>());
+    }
+
+    #[bench]
+    fn bench_bitset_initialize(b: &mut test::Bencher) {
+        b.iter(|| BitSet::with_max_value(1_000_000));
+    }
+}

src/common/composite_file.rs

@@ -0,0 +1,225 @@
+use std::io::Write;
+use common::CountingWriter;
+use std::collections::HashMap;
+use schema::Field;
+use common::VInt;
+use directory::WritePtr;
+use std::io::{self, Read};
+use directory::ReadOnlySource;
+use common::BinarySerializable;
+
+#[derive(Eq, PartialEq, Hash, Copy, Ord, PartialOrd, Clone, Debug)]
+pub struct FileAddr {
+    field: Field,
+    idx: usize,
+}
+
+impl FileAddr {
+    fn new(field: Field, idx: usize) -> FileAddr {
+        FileAddr { field, idx }
+    }
+}
+
+impl BinarySerializable for FileAddr {
+    fn serialize<W: Write>(&self, writer: &mut W) -> io::Result<()> {
+        self.field.serialize(writer)?;
+        VInt(self.idx as u64).serialize(writer)?;
+        Ok(())
+    }
+
+    fn deserialize<R: Read>(reader: &mut R) -> io::Result<Self> {
+        let field = Field::deserialize(reader)?;
+        let idx = VInt::deserialize(reader)?.0 as usize;
+        Ok(FileAddr {
+            field,
+            idx,
+        })
+    }
+}
+
+/// A `CompositeWrite` is used to write a `CompositeFile`.
+pub struct CompositeWrite<W = WritePtr> {
+    write: CountingWriter<W>,
+    offsets: HashMap<FileAddr, usize>,
+}
+
+impl<W: Write> CompositeWrite<W> {
+    /// Crate a new API writer that writes a composite file
+    /// in a given write.
+    pub fn wrap(w: W) -> CompositeWrite<W> {
+        CompositeWrite {
+            write: CountingWriter::wrap(w),
+            offsets: HashMap::new(),
+        }
+    }
+
+    /// Start writing a new field.
+    pub fn for_field(&mut self, field: Field) -> &mut CountingWriter<W> {
+        self.for_field_with_idx(field, 0)
+    }
+
+    /// Start writing a new field.
+    pub fn for_field_with_idx(&mut self, field: Field, idx: usize) -> &mut CountingWriter<W> {
+        let offset = self.write.written_bytes();
+        let file_addr = FileAddr::new(field, idx);
+        assert!(!self.offsets.contains_key(&file_addr));
+        self.offsets.insert(file_addr, offset);
+        &mut self.write
+    }
+
+    /// Close the composite file.
+    ///
+    /// An index of the different field offsets
+    /// will be written as a footer.
+    pub fn close(mut self) -> io::Result<()> {
+        let footer_offset = self.write.written_bytes();
+        VInt(self.offsets.len() as u64).serialize(&mut self.write)?;
+
+        let mut offset_fields: Vec<_> = self.offsets
+            .iter()
+            .map(|(file_addr, offset)| (*offset, *file_addr))
+            .collect();
+
+        offset_fields.sort();
+
+        let mut prev_offset = 0;
+        for (offset, file_addr) in offset_fields {
+            VInt((offset - prev_offset) as u64).serialize(&mut self.write)?;
+            file_addr.serialize(&mut self.write)?;
+            prev_offset = offset;
+        }
+
+        let footer_len = (self.write.written_bytes() - footer_offset) as u32;
+        footer_len.serialize(&mut self.write)?;
+        self.write.flush()?;
+        Ok(())
+    }
+}
+
+/// A composite file is an abstraction to store a
+/// file partitioned by field.
+///
+/// The file needs to be written field by field.
+/// A footer describes the start and stop offsets
+/// for each field.
+#[derive(Clone)]
+pub struct CompositeFile {
+    data: ReadOnlySource,
+    offsets_index: HashMap<FileAddr, (usize, usize)>,
+}
+
+impl CompositeFile {
+    /// Opens a composite file stored in a given
+    /// `ReadOnlySource`.
+    pub fn open(data: &ReadOnlySource) -> io::Result<CompositeFile> {
+        let end = data.len();
+        let footer_len_data = data.slice_from(end - 4);
+        let footer_len = u32::deserialize(&mut footer_len_data.as_slice())? as usize;
+
+        let footer_start = end - 4 - footer_len;
+        let footer_data = data.slice(footer_start, footer_start + footer_len);
+        let mut footer_buffer = footer_data.as_slice();
+        let num_fields = VInt::deserialize(&mut footer_buffer)?.0 as usize;
+
+        let mut file_addrs = vec![];
+        let mut offsets = vec![];
+
+        let mut field_index = HashMap::new();
+
+        let mut offset = 0;
+        for _ in 0..num_fields {
+            offset += VInt::deserialize(&mut footer_buffer)?.0 as usize;
+            let file_addr = FileAddr::deserialize(&mut footer_buffer)?;
+            offsets.push(offset);
+            file_addrs.push(file_addr);
+        }
+        offsets.push(footer_start);
+        for i in 0..num_fields {
+            let file_addr = file_addrs[i];
+            let start_offset = offsets[i];
+            let end_offset = offsets[i + 1];
+            field_index.insert(file_addr, (start_offset, end_offset));
+        }
+
+        Ok(CompositeFile {
+            data: data.slice_to(footer_start),
+            offsets_index: field_index,
+        })
+    }
+
+    /// Returns a composite file that stores
+    /// no fields.
+    pub fn empty() -> CompositeFile {
+        CompositeFile {
+            offsets_index: HashMap::new(),
+            data: ReadOnlySource::empty(),
+        }
+    }
+
+    /// Returns the `ReadOnlySource` associated
+    /// to a given `Field` and stored in a `CompositeFile`.
+    pub fn open_read(&self, field: Field) -> Option<ReadOnlySource> {
+        self.open_read_with_idx(field, 0)
+    }
+
+    /// Returns the `ReadOnlySource` associated
+    /// to a given `Field` and stored in a `CompositeFile`.
+    pub fn open_read_with_idx(&self, field: Field, idx: usize) -> Option<ReadOnlySource> {
+        self.offsets_index
+            .get(&FileAddr { field, idx, })
+            .map(|&(from, to)| self.data.slice(from, to))
+    }
+}
+
+#[cfg(test)]
+mod test {
+
+    use std::io::Write;
+    use super::{CompositeFile, CompositeWrite};
+    use directory::{Directory, RAMDirectory};
+    use schema::Field;
+    use common::VInt;
+    use common::BinarySerializable;
+    use std::path::Path;
+
+    #[test]
+    fn test_composite_file() {
+        let path = Path::new("test_path");
+        let mut directory = RAMDirectory::create();
+        {
+            let w = directory.open_write(path).unwrap();
+            let mut composite_write = CompositeWrite::wrap(w);
+            {
+                let mut write_0 = composite_write.for_field(Field(0u32));
+                VInt(32431123u64).serialize(&mut write_0).unwrap();
+                write_0.flush().unwrap();
+            }
+
+            {
+                let mut write_4 = composite_write.for_field(Field(4u32));
+                VInt(2).serialize(&mut write_4).unwrap();
+                write_4.flush().unwrap();
+            }
+            composite_write.close().unwrap();
+        }
+        {
+            let r = directory.open_read(path).unwrap();
+            let composite_file = CompositeFile::open(&r).unwrap();
+            {
+                let file0 = composite_file.open_read(Field(0u32)).unwrap();
+                let mut file0_buf = file0.as_slice();
+                let payload_0 = VInt::deserialize(&mut file0_buf).unwrap().0;
+                assert_eq!(file0_buf.len(), 0);
+                assert_eq!(payload_0, 32431123u64);
+            }
+            {
+                let file4 = composite_file.open_read(Field(4u32)).unwrap();
+                let mut file4_buf = file4.as_slice();
+                let payload_4 = VInt::deserialize(&mut file4_buf).unwrap().0;
+                assert_eq!(file4_buf.len(), 0);
+                assert_eq!(payload_4, 2u64);
+            }
+        }
+    }
+
+}

src/common/counting_writer.rs

@@ -1,8 +1,7 @@
 use std::io::Write;
 use std::io;
 
-
-pub struct CountingWriter<W: Write> {
+pub struct CountingWriter<W> {
     underlying: W,
     written_bytes: usize,
 }
@@ -10,7 +9,7 @@ pub struct CountingWriter<W: Write> {
 impl<W: Write> CountingWriter<W> {
     pub fn wrap(underlying: W) -> CountingWriter<W> {
         CountingWriter {
-            underlying: underlying,
+            underlying,
             written_bytes: 0,
         }
     }
@@ -37,8 +36,6 @@ impl<W: Write> Write for CountingWriter<W> {
     }
 }
 
-
-
 #[cfg(test)]
 mod test {
 

src/common/mod.rs

@@ -2,22 +2,64 @@ mod serialize;
 mod timer;
 mod vint;
 mod counting_writer;
+mod composite_file;
 pub mod bitpacker;
+mod bitset;
 
-pub use self::serialize::BinarySerializable;
+pub(crate) use self::composite_file::{CompositeFile, CompositeWrite};
+pub use self::serialize::{BinarySerializable, FixedSize};
 pub use self::timer::Timing;
 pub use self::timer::TimerTree;
 pub use self::timer::OpenTimer;
 pub use self::vint::VInt;
 pub use self::counting_writer::CountingWriter;
+pub use self::bitset::BitSet;
+pub(crate) use self::bitset::TinySet;
+pub use byteorder::LittleEndian as Endianness;
 
 use std::io;
 
-/// Create a default io error given a string.
-pub fn make_io_err(msg: String) -> io::Error {
-    io::Error::new(io::ErrorKind::Other, msg)
+/// Computes the number of bits that will be used for bitpacking.
+///
+/// In general the target is the minimum number of bits
+/// required to express the amplitude given in argument.
+///
+/// e.g. If the amplitude is 10, we can store all ints on simply 4bits.
+///
+/// The logic is slightly more convoluted here as for optimization
+/// reasons, we want to ensure that a value spawns over at most 8 bytes
+/// of aligns bytes.
+///
+/// Spanning over 9 bytes is possible for instance, if we do
+/// bitpacking with an amplitude of 63 bits.
+/// In this case, the second int will start on bit
+/// 63 (which belongs to byte 7) and ends at byte 15;
+/// Hence 9 bytes (from byte 7 to byte 15 included).
+///
+/// To avoid this, we force the number of bits to 64bits
+/// when the result is greater than `64-8 = 56 bits`.
+///
+/// Note that this only affects rare use cases spawning over
+/// a very large range of values. Even in this case, it results
+/// in an extra cost of at most 12% compared to the optimal
+/// number of bits.
+pub(crate) fn compute_num_bits(n: u64) -> u8 {
+    let amplitude = (64u32 - n.leading_zeros()) as u8;
+    if amplitude <= 64 - 8 {
+        amplitude
+    } else {
+        64
+    }
 }
 
+pub(crate) fn is_power_of_2(n: usize) -> bool {
+    (n > 0) && (n & (n - 1) == 0)
+}
+
+/// Create a default io error given a string.
+pub(crate) fn make_io_err(msg: String) -> io::Error {
+    io::Error::new(io::ErrorKind::Other, msg)
+}
 
 /// Has length trait
 pub trait HasLen {
@@ -32,7 +74,6 @@ pub trait HasLen {
 
 const HIGHEST_BIT: u64 = 1 << 63;
 
-
 /// Maps a `i64` to `u64`
 ///
 /// For simplicity, tantivy internally handles `i64` as `u64`.
@@ -64,11 +105,11 @@ pub fn u64_to_i64(val: u64) -> i64 {
     (val ^ HIGHEST_BIT) as i64
 }
 
-
 #[cfg(test)]
-mod test {
+pub(crate) mod test {
 
-    use super::{i64_to_u64, u64_to_i64};
+    use super::{compute_num_bits, i64_to_u64, u64_to_i64};
+    pub use super::serialize::test::fixed_size_test;
 
     fn test_i64_converter_helper(val: i64) {
         assert_eq!(u64_to_i64(i64_to_u64(val)), val);
@@ -85,4 +126,16 @@ mod test {
             test_i64_converter_helper(i);
         }
     }
+
+    #[test]
+    fn test_compute_num_bits() {
+        assert_eq!(compute_num_bits(1), 1u8);
+        assert_eq!(compute_num_bits(0), 0u8);
+        assert_eq!(compute_num_bits(2), 2u8);
+        assert_eq!(compute_num_bits(3), 2u8);
+        assert_eq!(compute_num_bits(4), 3u8);
+        assert_eq!(compute_num_bits(255), 8u8);
+        assert_eq!(compute_num_bits(256), 9u8);
+        assert_eq!(compute_num_bits(5_000_000_000), 33u8);
+    }
 }

src/common/serialize.rs

@@ -1,18 +1,25 @@
 use byteorder::{ReadBytesExt, WriteBytesExt};
-use byteorder::LittleEndian as Endianness;
+use common::Endianness;
 use std::fmt;
 use std::io::Write;
 use std::io::Read;
 use std::io;
 use common::VInt;
 
-
-
+/// Trait for a simple binary serialization.
 pub trait BinarySerializable: fmt::Debug + Sized {
+    /// Serialize
     fn serialize<W: Write>(&self, writer: &mut W) -> io::Result<()>;
+    /// Deserialize
     fn deserialize<R: Read>(reader: &mut R) -> io::Result<Self>;
 }
 
+/// `FixedSize` marks a `BinarySerializable` as
+/// always serializing to the same size.
+pub trait FixedSize: BinarySerializable {
+    const SIZE_IN_BYTES: usize;
+}
+
 impl BinarySerializable for () {
     fn serialize<W: Write>(&self, _: &mut W) -> io::Result<()> {
         Ok(())
@@ -22,6 +29,10 @@ impl BinarySerializable for () {
     }
 }
 
+impl FixedSize for () {
+    const SIZE_IN_BYTES: usize = 0;
+}
+
 impl<T: BinarySerializable> BinarySerializable for Vec<T> {
     fn serialize<W: Write>(&self, writer: &mut W) -> io::Result<()> {
         VInt(self.len() as u64).serialize(writer)?;
@@ -41,7 +52,6 @@ impl<T: BinarySerializable> BinarySerializable for Vec<T> {
     }
 }
 
-
 impl<Left: BinarySerializable, Right: BinarySerializable> BinarySerializable for (Left, Right) {
     fn serialize<W: Write>(&self, write: &mut W) -> io::Result<()> {
         self.0.serialize(write)?;
@@ -62,6 +72,9 @@ impl BinarySerializable for u32 {
     }
 }
 
+impl FixedSize for u32 {
+    const SIZE_IN_BYTES: usize = 4;
+}
 
 impl BinarySerializable for u64 {
     fn serialize<W: Write>(&self, writer: &mut W) -> io::Result<()> {
@@ -72,6 +85,10 @@ impl BinarySerializable for u64 {
     }
 }
 
+impl FixedSize for u64 {
+    const SIZE_IN_BYTES: usize = 8;
+}
+
 impl BinarySerializable for i64 {
     fn serialize<W: Write>(&self, writer: &mut W) -> io::Result<()> {
         writer.write_i64::<Endianness>(*self)
@@ -81,6 +98,9 @@ impl BinarySerializable for i64 {
     }
 }
 
+impl FixedSize for i64 {
+    const SIZE_IN_BYTES: usize = 8;
+}
 
 impl BinarySerializable for u8 {
     fn serialize<W: Write>(&self, writer: &mut W) -> io::Result<()> {
@@ -91,6 +111,10 @@ impl BinarySerializable for u8 {
     }
 }
 
+impl FixedSize for u8 {
+    const SIZE_IN_BYTES: usize = 1;
+}
+
 impl BinarySerializable for String {
     fn serialize<W: Write>(&self, writer: &mut W) -> io::Result<()> {
         let data: &[u8] = self.as_bytes();
@@ -108,65 +132,79 @@ impl BinarySerializable for String {
     }
 }
 
-
 #[cfg(test)]
-mod test {
+pub mod test {
 
     use common::VInt;
     use super::*;
 
-    fn serialize_test<T: BinarySerializable + Eq>(v: T, num_bytes: usize) {
+    pub fn fixed_size_test<O: BinarySerializable + FixedSize + Default>() {
+        let mut buffer = Vec::new();
+        O::default().serialize(&mut buffer).unwrap();
+        assert_eq!(buffer.len(), O::SIZE_IN_BYTES);
+    }
+
+    fn serialize_test<T: BinarySerializable + Eq>(v: T) -> usize {
         let mut buffer: Vec<u8> = Vec::new();
-        if num_bytes != 0 {
-            v.serialize(&mut buffer).unwrap();
-            assert_eq!(buffer.len(), num_bytes);
-        } else {
-            v.serialize(&mut buffer).unwrap();
-        }
+        v.serialize(&mut buffer).unwrap();
+        let num_bytes = buffer.len();
         let mut cursor = &buffer[..];
         let deser = T::deserialize(&mut cursor).unwrap();
         assert_eq!(deser, v);
+        num_bytes
     }
 
     #[test]
     fn test_serialize_u8() {
-        serialize_test(3u8, 1);
-        serialize_test(5u8, 1);
+        fixed_size_test::<u8>();
     }
 
     #[test]
     fn test_serialize_u32() {
-        serialize_test(3u32, 4);
-        serialize_test(5u32, 4);
-        serialize_test(u32::max_value(), 4);
+        fixed_size_test::<u32>();
+        assert_eq!(4, serialize_test(3u32));
+        assert_eq!(4, serialize_test(5u32));
+        assert_eq!(4, serialize_test(u32::max_value()));
+    }
+
+    #[test]
+    fn test_serialize_i64() {
+        fixed_size_test::<i64>();
+    }
+
+    #[test]
+    fn test_serialize_u64() {
+        fixed_size_test::<u64>();
     }
 
     #[test]
     fn test_serialize_string() {
-        serialize_test(String::from(""), 1);
-        serialize_test(String::from("ぽよぽよ"), 1 + 3 * 4);
-        serialize_test(String::from("富士さん見える。"), 1 + 3 * 8);
+        assert_eq!(serialize_test(String::from("")), 1);
+        assert_eq!(serialize_test(String::from("ぽよぽよ")), 1 + 3 * 4);
+        assert_eq!(
+            serialize_test(String::from("富士さん見える。")),
+            1 + 3 * 8
+        );
     }
 
     #[test]
     fn test_serialize_vec() {
-        let v: Vec<u8> = Vec::new();
-        serialize_test(v, 1);
-        serialize_test(vec![1u32, 3u32], 1 + 4 * 2);
+        assert_eq!(serialize_test(Vec::<u8>::new()), 1);
+        assert_eq!(serialize_test(vec![1u32, 3u32]), 1 + 4 * 2);
     }
 
     #[test]
     fn test_serialize_vint() {
         for i in 0..10_000 {
-            serialize_test(VInt(i as u64), 0);
+            serialize_test(VInt(i as u64));
         }
-        serialize_test(VInt(7u64), 1);
-        serialize_test(VInt(127u64), 1);
-        serialize_test(VInt(128u64), 2);
-        serialize_test(VInt(129u64), 2);
-        serialize_test(VInt(1234u64), 2);
-        serialize_test(VInt(16_383), 2);
-        serialize_test(VInt(16_384), 3);
-        serialize_test(VInt(u64::max_value()), 10);
+        assert_eq!(serialize_test(VInt(7u64)), 1);
+        assert_eq!(serialize_test(VInt(127u64)), 1);
+        assert_eq!(serialize_test(VInt(128u64)), 2);
+        assert_eq!(serialize_test(VInt(129u64)), 2);
+        assert_eq!(serialize_test(VInt(1234u64)), 2);
+        assert_eq!(serialize_test(VInt(16_383u64)), 2);
+        assert_eq!(serialize_test(VInt(16_384u64)), 3);
+        assert_eq!(serialize_test(VInt(u64::max_value())), 10);
     }
 }

src/common/timer.rs

@@ -14,7 +14,7 @@ impl<'a> OpenTimer<'a> {
     /// when the `OpenTimer` is dropped.
     pub fn open(&mut self, name: &'static str) -> OpenTimer {
         OpenTimer {
-            name: name,
+            name,
             timer_tree: self.timer_tree,
             start: PreciseTime::now(),
             depth: self.depth + 1,
@@ -24,16 +24,14 @@ impl<'a> OpenTimer<'a> {
 
 impl<'a> Drop for OpenTimer<'a> {
     fn drop(&mut self) {
-        self.timer_tree
-            .timings
-            .push(Timing {
-                      name: self.name,
-                      duration: self.start
-                          .to(PreciseTime::now())
-                          .num_microseconds()
-                          .unwrap(),
-                      depth: self.depth,
-                  });
+        self.timer_tree.timings.push(Timing {
+            name: self.name,
+            duration: self.start
+                .to(PreciseTime::now())
+                .num_microseconds()
+                .unwrap(),
+            depth: self.depth,
+        });
     }
 }
 
@@ -60,7 +58,7 @@ impl TimerTree {
     /// Open a new named subtask
     pub fn open(&mut self, name: &'static str) -> OpenTimer {
         OpenTimer {
-            name: name,
+            name,
             timer_tree: self,
             start: PreciseTime::now(),
             depth: 0,
@@ -70,11 +68,12 @@ impl TimerTree {
 
 impl Default for TimerTree {
     fn default() -> TimerTree {
-        TimerTree { timings: Vec::new() }
+        TimerTree {
+            timings: Vec::new(),
+        }
     }
 }
 
-
 #[cfg(test)]
 mod tests {
 

src/common/vint.rs

@@ -3,8 +3,6 @@ use std::io;
 use std::io::Write;
 use std::io::Read;
 
-
-
 ///   Wrapper over a `u64` that serializes as a variable int.
 #[derive(Debug, Eq, PartialEq)]
 pub struct VInt(pub u64);
@@ -13,6 +11,10 @@ impl VInt {
     pub fn val(&self) -> u64 {
         self.0
     }
+
+    pub fn deserialize_u64<R: Read>(reader: &mut R) -> io::Result<u64> {
+        VInt::deserialize(reader).map(|vint| vint.0)
+    }
 }
 
 impl BinarySerializable for VInt {
@@ -31,7 +33,6 @@ impl BinarySerializable for VInt {
             }
             i += 1;
         }
-
     }
 
     fn deserialize<R: Read>(reader: &mut R) -> io::Result<Self> {
@@ -41,13 +42,18 @@ impl BinarySerializable for VInt {
         loop {
             match bytes.next() {
                 Some(Ok(b)) => {
-                    result += ((b % 128u8) as u64) << shift;
+                    result += u64::from(b % 128u8) << shift;
                     if b & 128u8 != 0u8 {
                         break;
                     }
                     shift += 7;
                 }
-                _ => return Err(io::Error::new(io::ErrorKind::InvalidData, "Reach end of buffer")),
+                _ => {
+                    return Err(io::Error::new(
+                        io::ErrorKind::InvalidData,
+                        "Reach end of buffer",
+                    ))
+                }
             }
         }
         Ok(VInt(result))

src/compression/composite.rs

@@ -1,170 +0,0 @@
-use super::{BlockEncoder, BlockDecoder};
-use super::NUM_DOCS_PER_BLOCK;
-use compression::{VIntEncoder, VIntDecoder};
-
-pub struct CompositeEncoder {
-    block_encoder: BlockEncoder,
-    output: Vec<u8>,
-}
-
-impl CompositeEncoder {
-    pub fn new() -> CompositeEncoder {
-        CompositeEncoder {
-            block_encoder: BlockEncoder::new(),
-            output: Vec::with_capacity(500_000),
-        }
-    }
-
-    pub fn compress_sorted(&mut self, vals: &[u32]) -> &[u8] {
-        self.output.clear();
-        let num_blocks = vals.len() / NUM_DOCS_PER_BLOCK;
-        let mut offset = 0u32;
-        for i in 0..num_blocks {
-            let vals_slice = &vals[i * NUM_DOCS_PER_BLOCK..(i + 1) * NUM_DOCS_PER_BLOCK];
-            let block_compressed = self.block_encoder.compress_block_sorted(vals_slice, offset);
-            offset = vals_slice[NUM_DOCS_PER_BLOCK - 1];
-            self.output.extend_from_slice(block_compressed);
-        }
-        let vint_compressed =
-            self.block_encoder
-                .compress_vint_sorted(&vals[num_blocks * NUM_DOCS_PER_BLOCK..], offset);
-        self.output.extend_from_slice(vint_compressed);
-        &self.output
-    }
-
-    pub fn compress_unsorted(&mut self, vals: &[u32]) -> &[u8] {
-        self.output.clear();
-        let num_blocks = vals.len() / NUM_DOCS_PER_BLOCK;
-        for i in 0..num_blocks {
-            let vals_slice = &vals[i * NUM_DOCS_PER_BLOCK..(i + 1) * NUM_DOCS_PER_BLOCK];
-            let block_compressed = self.block_encoder.compress_block_unsorted(vals_slice);
-            self.output.extend_from_slice(block_compressed);
-        }
-        let vint_compressed = self.block_encoder
-            .compress_vint_unsorted(&vals[num_blocks * NUM_DOCS_PER_BLOCK..]);
-        self.output.extend_from_slice(vint_compressed);
-        &self.output
-    }
-}
-
-
-pub struct CompositeDecoder {
-    block_decoder: BlockDecoder,
-    vals: Vec<u32>,
-}
-
-
-impl CompositeDecoder {
-    pub fn new() -> CompositeDecoder {
-        CompositeDecoder {
-            block_decoder: BlockDecoder::new(),
-            vals: Vec::with_capacity(500_000),
-        }
-    }
-
-    pub fn uncompress_sorted(&mut self,
-                             mut compressed_data: &[u8],
-                             uncompressed_len: usize)
-                             -> &[u32] {
-        if uncompressed_len > self.vals.capacity() {
-            let extra_capacity = uncompressed_len - self.vals.capacity();
-            self.vals.reserve(extra_capacity);
-        }
-        let mut offset = 0u32;
-        self.vals.clear();
-        let num_blocks = uncompressed_len / NUM_DOCS_PER_BLOCK;
-        for _ in 0..num_blocks {
-            compressed_data = self.block_decoder
-                .uncompress_block_sorted(compressed_data, offset);
-            offset = self.block_decoder.output(NUM_DOCS_PER_BLOCK - 1);
-            self.vals
-                .extend_from_slice(self.block_decoder.output_array());
-        }
-        self.block_decoder
-            .uncompress_vint_sorted(compressed_data,
-                                    offset,
-                                    uncompressed_len % NUM_DOCS_PER_BLOCK);
-        self.vals
-            .extend_from_slice(self.block_decoder.output_array());
-        &self.vals
-    }
-
-    pub fn uncompress_unsorted(&mut self,
-                               mut compressed_data: &[u8],
-                               uncompressed_len: usize)
-                               -> &[u32] {
-        self.vals.clear();
-        let num_blocks = uncompressed_len / NUM_DOCS_PER_BLOCK;
-        for _ in 0..num_blocks {
-            compressed_data = self.block_decoder
-                .uncompress_block_unsorted(compressed_data);
-            self.vals
-                .extend_from_slice(self.block_decoder.output_array());
-        }
-        self.block_decoder
-            .uncompress_vint_unsorted(compressed_data, uncompressed_len % NUM_DOCS_PER_BLOCK);
-        self.vals
-            .extend_from_slice(self.block_decoder.output_array());
-        &self.vals
-    }
-}
-
-impl Into<Vec<u32>> for CompositeDecoder {
-    fn into(self) -> Vec<u32> {
-        self.vals
-    }
-}
-
-
-#[cfg(test)]
-pub mod tests {
-
-    use test::Bencher;
-    use super::*;
-    use tests;
-
-    #[test]
-    fn test_composite_unsorted() {
-        let data = tests::generate_array(10_000, 0.1);
-        let mut encoder = CompositeEncoder::new();
-        let compressed = encoder.compress_unsorted(&data);
-        assert!(compressed.len() <= 19_794);
-        let mut decoder = CompositeDecoder::new();
-        let result = decoder.uncompress_unsorted(&compressed, data.len());
-        for i in 0..data.len() {
-            assert_eq!(data[i], result[i]);
-        }
-    }
-
-    #[test]
-    fn test_composite_sorted() {
-        let data = tests::generate_array(10_000, 0.1);
-        let mut encoder = CompositeEncoder::new();
-        let compressed = encoder.compress_sorted(&data);
-        assert!(compressed.len() <= 7_826);
-        let mut decoder = CompositeDecoder::new();
-        let result = decoder.uncompress_sorted(&compressed, data.len());
-        for i in 0..data.len() {
-            assert_eq!(data[i], result[i]);
-        }
-    }
-
-
-    const BENCH_NUM_INTS: usize = 99_968;
-
-    #[bench]
-    fn bench_compress(b: &mut Bencher) {
-        let mut encoder = CompositeEncoder::new();
-        let data = tests::generate_array(BENCH_NUM_INTS, 0.1);
-        b.iter(|| { encoder.compress_sorted(&data); });
-    }
-
-    #[bench]
-    fn bench_uncompress(b: &mut Bencher) {
-        let mut encoder = CompositeEncoder::new();
-        let data = tests::generate_array(BENCH_NUM_INTS, 0.1);
-        let compressed = encoder.compress_sorted(&data);
-        let mut decoder = CompositeDecoder::new();
-        b.iter(|| { decoder.uncompress_sorted(compressed, BENCH_NUM_INTS); });
-    }
-}

src/compression/mod.rs

@@ -1,52 +1,90 @@
 #![allow(dead_code)]
 
 
-mod composite;
-pub use self::composite::{CompositeEncoder, CompositeDecoder};
+mod stream;
 
+pub use self::stream::CompressedIntStream;
 
-#[cfg(not(feature="simdcompression"))]
+pub const COMPRESSION_BLOCK_SIZE: usize = 128;
+
+/// Returns the size in bytes of a compressed block, given `num_bits`.
+pub fn compressed_block_size(num_bits: u8) -> usize {
+    1 + (num_bits as usize) * 16
+}
+
+#[cfg(not(feature = "simdcompression"))]
 mod pack {
     mod compression_pack_nosimd;
-    pub use self::compression_pack_nosimd::*;
+    pub use self::compression_pack_nosimd::{BlockDecoder, BlockEncoder};
 }
 
-#[cfg(feature="simdcompression")]
+#[cfg(feature = "simdcompression")]
 mod pack {
     mod compression_pack_simd;
-    pub use self::compression_pack_simd::*;
+    pub use self::compression_pack_simd::{BlockDecoder, BlockEncoder};
 }
 
-pub use self::pack::{BlockEncoder, BlockDecoder};
+pub use self::pack::{BlockDecoder, BlockEncoder};
 
-#[cfg( any(not(feature="simdcompression"), target_env="msvc") )]
+#[cfg(any(not(feature = "simdcompression"), target_env = "msvc"))]
 mod vint {
     mod compression_vint_nosimd;
-    pub use self::compression_vint_nosimd::*;
+    pub(crate) use self::compression_vint_nosimd::*;
 }
 
-#[cfg( all(feature="simdcompression", not(target_env="msvc")) )]
+#[cfg(all(feature = "simdcompression", not(target_env = "msvc")))]
 mod vint {
     mod compression_vint_simd;
-    pub use self::compression_vint_simd::*;
+    pub(crate) use self::compression_vint_simd::*;
 }
 
-
 pub trait VIntEncoder {
+    /// Compresses an array of `u32` integers,
+    /// using [delta-encoding](https://en.wikipedia.org/wiki/Delta_encoding)
+    /// and variable bytes encoding.
+    ///
+    /// The method takes an array of ints to compress, and returns
+    /// a `&[u8]` representing the compressed data.
+    ///
+    /// The method also takes an offset to give the value of the
+    /// hypothetical previous element in the delta-encoding.
     fn compress_vint_sorted(&mut self, input: &[u32], offset: u32) -> &[u8];
+
+    /// Compresses an array of `u32` integers,
+    /// using variable bytes encoding.
+    ///
+    /// The method takes an array of ints to compress, and returns
+    /// a `&[u8]` representing the compressed data.
     fn compress_vint_unsorted(&mut self, input: &[u32]) -> &[u8];
 }
 
 pub trait VIntDecoder {
-    fn uncompress_vint_sorted<'a>(&mut self,
-                                  compressed_data: &'a [u8],
-                                  offset: u32,
-                                  num_els: usize)
-                                  -> &'a [u8];
-    fn uncompress_vint_unsorted<'a>(&mut self,
-                                    compressed_data: &'a [u8],
-                                    num_els: usize)
-                                    -> &'a [u8];
+    /// Uncompress an array of `u32` integers,
+    /// that were compressed using [delta-encoding](https://en.wikipedia.org/wiki/Delta_encoding)
+    /// and variable bytes encoding.
+    ///
+    /// The method takes a number of int to decompress, and returns
+    /// the amount of bytes that were read to decompress them.
+    ///
+    /// The method also takes an offset to give the value of the
+    /// hypothetical previous element in the delta-encoding.
+    ///
+    /// For instance, if delta encoded are `1, 3, 9`, and the
+    /// `offset` is 5, then the output will be:
+    /// `5 + 1 = 6, 6 + 3= 9, 9 + 9 = 18`
+    fn uncompress_vint_sorted<'a>(
+        &mut self,
+        compressed_data: &'a [u8],
+        offset: u32,
+        num_els: usize,
+    ) -> usize;
+
+    /// Uncompress an array of `u32s`, compressed using variable
+    /// byte encoding.
+    ///
+    /// The method takes a number of int to decompress, and returns
+    /// the amount of bytes that were read to decompress them.
+    fn uncompress_vint_unsorted<'a>(&mut self, compressed_data: &'a [u8], num_els: usize) -> usize;
 }
 
 impl VIntEncoder for BlockEncoder {
@@ -60,27 +98,22 @@ impl VIntEncoder for BlockEncoder {
 }
 
 impl VIntDecoder for BlockDecoder {
-    fn uncompress_vint_sorted<'a>(&mut self,
-                                  compressed_data: &'a [u8],
-                                  offset: u32,
-                                  num_els: usize)
-                                  -> &'a [u8] {
+    fn uncompress_vint_sorted<'a>(
+        &mut self,
+        compressed_data: &'a [u8],
+        offset: u32,
+        num_els: usize,
+    ) -> usize {
         self.output_len = num_els;
         vint::uncompress_sorted(compressed_data, &mut self.output[..num_els], offset)
     }
 
-    fn uncompress_vint_unsorted<'a>(&mut self,
-                                    compressed_data: &'a [u8],
-                                    num_els: usize)
-                                    -> &'a [u8] {
+    fn uncompress_vint_unsorted<'a>(&mut self, compressed_data: &'a [u8], num_els: usize) -> usize {
         self.output_len = num_els;
         vint::uncompress_unsorted(compressed_data, &mut self.output[..num_els])
     }
 }
 
-
-pub const NUM_DOCS_PER_BLOCK: usize = 128; //< should be a power of 2 to let the compiler optimize.
-
 #[cfg(test)]
 pub mod tests {
 
@@ -95,8 +128,8 @@ pub mod tests {
         let compressed_data = encoder.compress_block_sorted(&vals, 0);
         let mut decoder = BlockDecoder::new();
         {
-            let remaining_data = decoder.uncompress_block_sorted(compressed_data, 0);
-            assert_eq!(remaining_data.len(), 0);
+            let consumed_num_bytes = decoder.uncompress_block_sorted(compressed_data, 0);
+            assert_eq!(consumed_num_bytes, compressed_data.len());
         }
         for i in 0..128 {
             assert_eq!(vals[i], decoder.output(i));
@@ -110,8 +143,8 @@ pub mod tests {
         let compressed_data = encoder.compress_block_sorted(&vals, 10);
         let mut decoder = BlockDecoder::new();
         {
-            let remaining_data = decoder.uncompress_block_sorted(compressed_data, 10);
-            assert_eq!(remaining_data.len(), 0);
+            let consumed_num_bytes = decoder.uncompress_block_sorted(compressed_data, 10);
+            assert_eq!(consumed_num_bytes, compressed_data.len());
         }
         for i in 0..128 {
             assert_eq!(vals[i], decoder.output(i));
@@ -129,9 +162,9 @@ pub mod tests {
         compressed.push(173u8);
         let mut decoder = BlockDecoder::new();
         {
-            let remaining_data = decoder.uncompress_block_sorted(&compressed, 10);
-            assert_eq!(remaining_data.len(), 1);
-            assert_eq!(remaining_data[0], 173u8);
+            let consumed_num_bytes = decoder.uncompress_block_sorted(&compressed, 10);
+            assert_eq!(consumed_num_bytes, compressed.len() - 1);
+            assert_eq!(compressed[consumed_num_bytes], 173u8);
         }
         for i in 0..n {
             assert_eq!(vals[i], decoder.output(i));
@@ -149,16 +182,15 @@ pub mod tests {
         compressed.push(173u8);
         let mut decoder = BlockDecoder::new();
         {
-            let remaining_data = decoder.uncompress_block_unsorted(&compressed);
-            assert_eq!(remaining_data.len(), 1);
-            assert_eq!(remaining_data[0], 173u8);
+            let consumed_num_bytes = decoder.uncompress_block_unsorted(&compressed);
+            assert_eq!(consumed_num_bytes + 1, compressed.len());
+            assert_eq!(compressed[consumed_num_bytes], 173u8);
         }
         for i in 0..n {
             assert_eq!(vals[i], decoder.output(i));
         }
     }
 
-
     #[test]
     fn test_encode_vint() {
         {
@@ -169,31 +201,47 @@ pub mod tests {
                 let encoded_data = encoder.compress_vint_sorted(&input, *offset);
                 assert!(encoded_data.len() <= expected_length);
                 let mut decoder = BlockDecoder::new();
-                let remaining_data =
+                let consumed_num_bytes =
                     decoder.uncompress_vint_sorted(&encoded_data, *offset, input.len());
-                assert_eq!(0, remaining_data.len());
+                assert_eq!(consumed_num_bytes, encoded_data.len());
                 assert_eq!(input, decoder.output_array());
             }
         }
     }
 
-
     #[bench]
     fn bench_compress(b: &mut Bencher) {
         let mut encoder = BlockEncoder::new();
-        let data = tests::generate_array(NUM_DOCS_PER_BLOCK, 0.1);
-        b.iter(|| { encoder.compress_block_sorted(&data, 0u32); });
+        let data = tests::generate_array(COMPRESSION_BLOCK_SIZE, 0.1);
+        b.iter(|| {
+            encoder.compress_block_sorted(&data, 0u32);
+        });
     }
 
     #[bench]
     fn bench_uncompress(b: &mut Bencher) {
         let mut encoder = BlockEncoder::new();
-        let data = tests::generate_array(NUM_DOCS_PER_BLOCK, 0.1);
+        let data = tests::generate_array(COMPRESSION_BLOCK_SIZE, 0.1);
         let compressed = encoder.compress_block_sorted(&data, 0u32);
         let mut decoder = BlockDecoder::new();
-        b.iter(|| { decoder.uncompress_block_sorted(compressed, 0u32); });
+        b.iter(|| {
+            decoder.uncompress_block_sorted(compressed, 0u32);
+        });
     }
 
+    #[test]
+    fn test_all_docs_compression_numbits() {
+        for num_bits in 0..33 {
+            let mut data = [0u32; 128];
+            if num_bits > 0 {
+                data[0] = 1 << (num_bits - 1);
+            }
+            let mut encoder = BlockEncoder::new();
+            let compressed = encoder.compress_block_unsorted(&data);
+            assert_eq!(compressed[0] as usize, num_bits);
+            assert_eq!(compressed.len(), compressed_block_size(compressed[0]));
+        }
+    }
 
     const NUM_INTS_BENCH_VINT: usize = 10;
 
@@ -201,7 +249,9 @@ pub mod tests {
     fn bench_compress_vint(b: &mut Bencher) {
         let mut encoder = BlockEncoder::new();
         let data = tests::generate_array(NUM_INTS_BENCH_VINT, 0.001);
-        b.iter(|| { encoder.compress_vint_sorted(&data, 0u32); });
+        b.iter(|| {
+            encoder.compress_vint_sorted(&data, 0u32);
+        });
     }
 
     #[bench]
@@ -210,7 +260,9 @@ pub mod tests {
         let data = tests::generate_array(NUM_INTS_BENCH_VINT, 0.001);
         let compressed = encoder.compress_vint_sorted(&data, 0u32);
         let mut decoder = BlockDecoder::new();
-        b.iter(|| { decoder.uncompress_vint_sorted(compressed, 0u32, NUM_INTS_BENCH_VINT); });
+        b.iter(|| {
+            decoder.uncompress_vint_sorted(compressed, 0u32, NUM_INTS_BENCH_VINT);
+        });
     }
 
 }

src/compression/pack/compression_pack_nosimd.rs

@@ -1,16 +1,17 @@
-use common::bitpacker::compute_num_bits;
+use common::compute_num_bits;
 use common::bitpacker::{BitPacker, BitUnpacker};
+use common::CountingWriter;
 use std::cmp;
 use std::io::Write;
-use super::super::NUM_DOCS_PER_BLOCK;
+use super::super::{compressed_block_size, COMPRESSION_BLOCK_SIZE};
 
-const COMPRESSED_BLOCK_MAX_SIZE: usize = NUM_DOCS_PER_BLOCK * 4 + 1;
+const COMPRESSED_BLOCK_MAX_SIZE: usize = COMPRESSION_BLOCK_SIZE * 4 + 1;
 
-pub fn compress_sorted(vals: &mut [u32], mut output: &mut [u8], offset: u32) -> usize {
+pub fn compress_sorted(vals: &mut [u32], output: &mut [u8], offset: u32) -> usize {
     let mut max_delta = 0;
     {
         let mut local_offset = offset;
-        for i in 0..NUM_DOCS_PER_BLOCK {
+        for i in 0..COMPRESSION_BLOCK_SIZE {
             let val = vals[i];
             let delta = val - local_offset;
             max_delta = cmp::max(max_delta, delta);
@@ -18,24 +19,25 @@ pub fn compress_sorted(vals: &mut [u32], mut output: &mut [u8], offset: u32) ->
             local_offset = val;
         }
     }
-    let num_bits = compute_num_bits(max_delta);
-    output.write_all(&[num_bits]).unwrap();
-    let mut bit_packer = BitPacker::new(num_bits as usize);
+    let mut counting_writer = CountingWriter::wrap(output);
+    let num_bits = compute_num_bits(max_delta as u64);
+    counting_writer.write_all(&[num_bits]).unwrap();
+
+    let mut bit_packer = BitPacker::new();
     for val in vals {
-        bit_packer.write(*val, &mut output).unwrap();
+        bit_packer
+            .write(*val as u64, num_bits, &mut counting_writer)
+            .unwrap();
     }
-    1 +
-    bit_packer
-        .close(&mut output)
-        .expect("packing in memory should never fail")
+    let compressed_size = counting_writer.written_bytes();
+    assert_eq!(compressed_size, compressed_block_size(num_bits));
+    compressed_size
 }
 
-
-
 pub struct BlockEncoder {
     pub output: [u8; COMPRESSED_BLOCK_MAX_SIZE],
     pub output_len: usize,
-    input_buffer: [u32; NUM_DOCS_PER_BLOCK],
+    input_buffer: [u32; COMPRESSION_BLOCK_SIZE],
 }
 
 impl BlockEncoder {
@@ -43,7 +45,7 @@ impl BlockEncoder {
         BlockEncoder {
             output: [0u8; COMPRESSED_BLOCK_MAX_SIZE],
             output_len: 0,
-            input_buffer: [0u32; NUM_DOCS_PER_BLOCK],
+            input_buffer: [0u32; COMPRESSION_BLOCK_SIZE],
         }
     }
 
@@ -54,22 +56,33 @@ impl BlockEncoder {
     }
 
     pub fn compress_block_unsorted(&mut self, vals: &[u32]) -> &[u8] {
-        let compressed_size: usize = {
-            let mut output: &mut [u8] = &mut self.output;
+        let compressed_size = {
+            let output: &mut [u8] = &mut self.output;
             let max = vals.iter()
                 .cloned()
                 .max()
                 .expect("compress unsorted called with an empty array");
-            let num_bits = compute_num_bits(max);
-            output.write_all(&[num_bits]).unwrap();
-            let mut bit_packer = BitPacker::new(num_bits as usize);
+            let num_bits = compute_num_bits(max as u64);
+            let mut counting_writer = CountingWriter::wrap(output);
+            counting_writer.write_all(&[num_bits]).unwrap();
+            let mut bit_packer = BitPacker::new();
             for val in vals {
-                bit_packer.write(*val, &mut output).unwrap();
+                bit_packer
+                    .write(*val as u64, num_bits, &mut counting_writer)
+                    .unwrap();
             }
-            1 +
-            bit_packer
-                .close(&mut output)
-                .expect("packing in memory should never fail")
+            for _ in vals.len()..COMPRESSION_BLOCK_SIZE {
+                bit_packer
+                    .write(vals[0] as u64, num_bits, &mut counting_writer)
+                    .unwrap();
+            }
+            bit_packer.flush(&mut counting_writer).expect(
+                "Flushing the bitpacking \
+                 in an in RAM buffer should never fail",
+            );
+            // we avoid writing "closing", because we
+            // do not want 7 bytes of padding here.
+            counting_writer.written_bytes()
         };
         &self.output[..compressed_size]
     }
@@ -80,7 +93,6 @@ pub struct BlockDecoder {
     pub output_len: usize,
 }
 
-
 impl BlockDecoder {
     pub fn new() -> BlockDecoder {
         BlockDecoder::with_val(0u32)
@@ -93,34 +105,35 @@ impl BlockDecoder {
         }
     }
 
-    pub fn uncompress_block_sorted<'a>(&mut self,
-                                       compressed_data: &'a [u8],
-                                       mut offset: u32)
-                                       -> &'a [u8] {
+    pub fn uncompress_block_sorted<'a>(
+        &mut self,
+        compressed_data: &'a [u8],
+        mut offset: u32,
+    ) -> usize {
         let consumed_size = {
             let num_bits = compressed_data[0];
-            let bit_unpacker = BitUnpacker::new(&compressed_data[1..], num_bits as usize);
-            for i in 0..NUM_DOCS_PER_BLOCK {
+            let bit_unpacker = BitUnpacker::new(&compressed_data[1..], num_bits);
+            for i in 0..COMPRESSION_BLOCK_SIZE {
                 let delta = bit_unpacker.get(i);
-                let val = offset + delta;
+                let val = offset + delta as u32;
                 self.output[i] = val;
                 offset = val;
             }
-            1 + (num_bits as usize * NUM_DOCS_PER_BLOCK + 7) / 8
+            compressed_block_size(num_bits)
         };
-        self.output_len = NUM_DOCS_PER_BLOCK;
-        &compressed_data[consumed_size..]
+        self.output_len = COMPRESSION_BLOCK_SIZE;
+        consumed_size
     }
 
-    pub fn uncompress_block_unsorted<'a>(&mut self, compressed_data: &'a [u8]) -> &'a [u8] {
+    pub fn uncompress_block_unsorted<'a>(&mut self, compressed_data: &'a [u8]) -> usize {
         let num_bits = compressed_data[0];
-        let bit_unpacker = BitUnpacker::new(&compressed_data[1..], num_bits as usize);
-        for i in 0..NUM_DOCS_PER_BLOCK {
-            self.output[i] = bit_unpacker.get(i);
+        let bit_unpacker = BitUnpacker::new(&compressed_data[1..], num_bits);
+        for i in 0..COMPRESSION_BLOCK_SIZE {
+            self.output[i] = bit_unpacker.get(i) as u32;
         }
-        let consumed_size = 1 + (num_bits as usize * NUM_DOCS_PER_BLOCK + 7) / 8;
-        self.output_len = NUM_DOCS_PER_BLOCK;
-        &compressed_data[consumed_size..]
+        let consumed_size = 1 + (num_bits as usize * COMPRESSION_BLOCK_SIZE + 7) / 8;
+        self.output_len = COMPRESSION_BLOCK_SIZE;
+        consumed_size
     }
 
     #[inline]

src/compression/pack/compression_pack_simd.rs

@@ -1,6 +1,6 @@
-use super::super::NUM_DOCS_PER_BLOCK;
+use compression::COMPRESSION_BLOCK_SIZE;
 
-const COMPRESSED_BLOCK_MAX_SIZE: usize = NUM_DOCS_PER_BLOCK * 4 + 1;
+const COMPRESSED_BLOCK_MAX_SIZE: usize = COMPRESSION_BLOCK_SIZE * 4 + 1;
 
 mod simdcomp {
     use libc::size_t;
@@ -8,10 +8,11 @@ mod simdcomp {
     extern "C" {
         pub fn compress_sorted(data: *const u32, output: *mut u8, offset: u32) -> size_t;
 
-        pub fn uncompress_sorted(compressed_data: *const u8,
-                                 output: *mut u32,
-                                 offset: u32)
-                                 -> size_t;
+        pub fn uncompress_sorted(
+            compressed_data: *const u8,
+            output: *mut u32,
+            offset: u32,
+        ) -> size_t;
 
         pub fn compress_unsorted(data: *const u32, output: *mut u8) -> size_t;
 
@@ -35,7 +36,6 @@ fn uncompress_unsorted(compressed_data: &[u8], output: &mut [u32]) -> usize {
     unsafe { simdcomp::uncompress_unsorted(compressed_data.as_ptr(), output.as_mut_ptr()) }
 }
 
-
 pub struct BlockEncoder {
     pub output: [u8; COMPRESSED_BLOCK_MAX_SIZE],
     pub output_len: usize,
@@ -65,7 +65,6 @@ pub struct BlockDecoder {
     pub output_len: usize,
 }
 
-
 impl BlockDecoder {
     pub fn new() -> BlockDecoder {
         BlockDecoder::with_val(0u32)
@@ -78,19 +77,16 @@ impl BlockDecoder {
         }
     }
 
-    pub fn uncompress_block_sorted<'a>(&mut self,
-                                       compressed_data: &'a [u8],
-                                       offset: u32)
-                                       -> &'a [u8] {
+    pub fn uncompress_block_sorted(&mut self, compressed_data: &[u8], offset: u32) -> usize {
         let consumed_size = uncompress_sorted(compressed_data, &mut self.output, offset);
-        self.output_len = NUM_DOCS_PER_BLOCK;
-        &compressed_data[consumed_size..]
+        self.output_len = COMPRESSION_BLOCK_SIZE;
+        consumed_size
     }
 
-    pub fn uncompress_block_unsorted<'a>(&mut self, compressed_data: &'a [u8]) -> &'a [u8] {
+    pub fn uncompress_block_unsorted<'a>(&mut self, compressed_data: &'a [u8]) -> usize {
         let consumed_size = uncompress_unsorted(compressed_data, &mut self.output);
-        self.output_len = NUM_DOCS_PER_BLOCK;
-        &compressed_data[consumed_size..]
+        self.output_len = COMPRESSION_BLOCK_SIZE;
+        consumed_size
     }
 
     #[inline]
@@ -104,7 +100,6 @@ impl BlockDecoder {
     }
 }
 
-
 #[cfg(test)]
 mod tests {
 
@@ -117,4 +112,5 @@ mod tests {
         let compressed = encoder.compress_block_sorted(&data, 0u32);
         assert_eq!(compressed.len(), 17);
     }
+
 }

src/compression/stream.rs

@@ -0,0 +1,130 @@
+use compression::BlockDecoder;
+use compression::COMPRESSION_BLOCK_SIZE;
+use compression::compressed_block_size;
+use directory::{ReadOnlySource, SourceRead};
+
+/// Reads a stream of compressed ints.
+///
+/// Tantivy uses `CompressedIntStream` to read
+/// the position file.
+/// The `.skip(...)` makes it possible to avoid
+/// decompressing blocks that are not required.
+pub struct CompressedIntStream {
+    buffer: SourceRead,
+    block_decoder: BlockDecoder,
+    inner_offset: usize,
+}
+
+impl CompressedIntStream {
+    /// Opens a compressed int stream.
+    pub(crate) fn wrap(source: ReadOnlySource) -> CompressedIntStream {
+        CompressedIntStream {
+            buffer: SourceRead::from(source),
+            block_decoder: BlockDecoder::new(),
+            inner_offset: COMPRESSION_BLOCK_SIZE,
+        }
+    }
+
+    /// Fills a buffer with the next `output.len()` integers,
+    /// and advance the stream by that many els.
+    pub fn read(&mut self, output: &mut [u32]) {
+        let mut num_els: usize = output.len();
+        let mut start: usize = 0;
+        loop {
+            let available = COMPRESSION_BLOCK_SIZE - self.inner_offset;
+            if num_els >= available {
+                if available > 0 {
+                    let uncompressed_block =
+                        &self.block_decoder.output_array()[self.inner_offset..];
+                    output[start..][..available].clone_from_slice(uncompressed_block);
+                }
+                num_els -= available;
+                start += available;
+                let num_consumed_bytes = self.block_decoder
+                    .uncompress_block_unsorted(self.buffer.as_ref());
+                self.buffer.advance(num_consumed_bytes);
+                self.inner_offset = 0;
+            } else {
+                let uncompressed_block = &self.block_decoder.output_array()
+                    [self.inner_offset..self.inner_offset + num_els];
+                output[start..][..num_els].clone_from_slice(uncompressed_block);
+                self.inner_offset += num_els;
+                break;
+            }
+        }
+    }
+
+    /// Skip the next `skip_len` integer.
+    ///
+    /// If a full block is skipped, calling
+    /// `.skip(...)` will avoid decompressing it.
+    pub fn skip(&mut self, mut skip_len: usize) {
+        let available = COMPRESSION_BLOCK_SIZE - self.inner_offset;
+        if available >= skip_len {
+            self.inner_offset += skip_len;
+        } else {
+            skip_len -= available;
+            // entirely skip decompressing some blocks.
+            while skip_len >= COMPRESSION_BLOCK_SIZE {
+                skip_len -= COMPRESSION_BLOCK_SIZE;
+                let num_bits: u8 = self.buffer.as_ref()[0];
+                let block_len = compressed_block_size(num_bits);
+                self.buffer.advance(block_len);
+            }
+            let num_consumed_bytes = self.block_decoder
+                .uncompress_block_unsorted(self.buffer.as_ref());
+            self.buffer.advance(num_consumed_bytes);
+            self.inner_offset = skip_len;
+        }
+    }
+}
+
+#[cfg(test)]
+pub mod tests {
+
+    use super::CompressedIntStream;
+    use compression::compressed_block_size;
+    use compression::COMPRESSION_BLOCK_SIZE;
+    use compression::BlockEncoder;
+    use directory::ReadOnlySource;
+
+    fn create_stream_buffer() -> ReadOnlySource {
+        let mut buffer: Vec<u8> = vec![];
+        let mut encoder = BlockEncoder::new();
+        let vals: Vec<u32> = (0u32..1_025u32).collect();
+        for chunk in vals.chunks(COMPRESSION_BLOCK_SIZE) {
+            let compressed_block = encoder.compress_block_unsorted(chunk);
+            let num_bits = compressed_block[0];
+            assert_eq!(compressed_block_size(num_bits), compressed_block.len());
+            buffer.extend_from_slice(compressed_block);
+        }
+        if cfg!(simd) {
+            buffer.extend_from_slice(&[0u8; 7]);
+        }
+        ReadOnlySource::from(buffer)
+    }
+
+    #[test]
+    fn test_compressed_int_stream() {
+        let buffer = create_stream_buffer();
+        let mut stream = CompressedIntStream::wrap(buffer);
+        let mut block: [u32; COMPRESSION_BLOCK_SIZE] = [0u32; COMPRESSION_BLOCK_SIZE];
+
+        stream.read(&mut block[0..2]);
+        assert_eq!(block[0], 0);
+        assert_eq!(block[1], 1);
+        stream.skip(5);
+        stream.read(&mut block[0..3]);
+        assert_eq!(block[0], 7);
+        assert_eq!(block[1], 8);
+        assert_eq!(block[2], 9);
+        stream.skip(500);
+        stream.read(&mut block[0..3]);
+        assert_eq!(block[0], 510);
+        assert_eq!(block[1], 511);
+        assert_eq!(block[2], 512);
+        stream.skip(511);
+        stream.read(&mut block[..1]);
+        assert_eq!(block[0], 1024);
+    }
+}

src/compression/vint/compression_vint_nosimd.rs

@@ -1,6 +1,9 @@
-
 #[inline(always)]
-pub fn compress_sorted<'a>(input: &[u32], output: &'a mut [u8], mut offset: u32) -> &'a [u8] {
+pub(crate) fn compress_sorted<'a>(
+    input: &[u32],
+    output: &'a mut [u8],
+    mut offset: u32,
+) -> &'a [u8] {
     let mut byte_written = 0;
     for &v in input {
         let mut to_encode: u32 = v - offset;
@@ -22,7 +25,7 @@ pub fn compress_sorted<'a>(input: &[u32], output: &'a mut [u8], mut offset: u32)
 }
 
 #[inline(always)]
-pub fn compress_unsorted<'a>(input: &[u32], output: &'a mut [u8]) -> &'a [u8] {
+pub(crate) fn compress_unsorted<'a>(input: &[u32], output: &'a mut [u8]) -> &'a [u8] {
     let mut byte_written = 0;
     for &v in input {
         let mut to_encode: u32 = v;
@@ -43,10 +46,11 @@ pub fn compress_unsorted<'a>(input: &[u32], output: &'a mut [u8]) -> &'a [u8] {
 }
 
 #[inline(always)]
-pub fn uncompress_sorted<'a>(compressed_data: &'a [u8],
-                             output: &mut [u32],
-                             offset: u32)
-                             -> &'a [u8] {
+pub(crate) fn uncompress_sorted<'a>(
+    compressed_data: &'a [u8],
+    output: &mut [u32],
+    offset: u32,
+) -> usize {
     let mut read_byte = 0;
     let mut result = offset;
     let num_els = output.len();
@@ -63,11 +67,11 @@ pub fn uncompress_sorted<'a>(compressed_data: &'a [u8],
         }
         output[i] = result;
     }
-    &compressed_data[read_byte..]
+    read_byte
 }
 
 #[inline(always)]
-pub fn uncompress_unsorted<'a>(compressed_data: &'a [u8], output: &mut [u32]) -> &'a [u8] {
+pub(crate) fn uncompress_unsorted<'a>(compressed_data: &'a [u8], output: &mut [u32]) -> usize {
     let mut read_byte = 0;
     let num_els = output.len();
     for i in 0..num_els {
@@ -84,5 +88,5 @@ pub fn uncompress_unsorted<'a>(compressed_data: &'a [u8], output: &mut [u32]) ->
         }
         output[i] = result;
     }
-    &compressed_data[read_byte..]
+    read_byte
 }

src/compression/vint/compression_vint_simd.rs

@@ -1,44 +1,47 @@
-
 mod streamvbyte {
 
     use libc::size_t;
 
     extern "C" {
-        pub fn streamvbyte_delta_encode(data: *const u32,
-                                        num_els: u32,
-                                        output: *mut u8,
-                                        offset: u32)
-                                        -> size_t;
+        pub fn streamvbyte_delta_encode(
+            data: *const u32,
+            num_els: u32,
+            output: *mut u8,
+            offset: u32,
+        ) -> size_t;
 
-        pub fn streamvbyte_delta_decode(compressed_data: *const u8,
-                                        output: *mut u32,
-                                        num_els: u32,
-                                        offset: u32)
-                                        -> size_t;
+        pub fn streamvbyte_delta_decode(
+            compressed_data: *const u8,
+            output: *mut u32,
+            num_els: u32,
+            offset: u32,
+        ) -> size_t;
 
         pub fn streamvbyte_encode(data: *const u32, num_els: u32, output: *mut u8) -> size_t;
 
-        pub fn streamvbyte_decode(compressed_data: *const u8,
-                                  output: *mut u32,
-                                  num_els: usize)
-                                  -> size_t;
+        pub fn streamvbyte_decode(
+            compressed_data: *const u8,
+            output: *mut u32,
+            num_els: usize,
+        ) -> size_t;
     }
 }
 
-
 #[inline(always)]
-pub fn compress_sorted<'a>(input: &[u32], output: &'a mut [u8], offset: u32) -> &'a [u8] {
+pub(crate) fn compress_sorted<'a>(input: &[u32], output: &'a mut [u8], offset: u32) -> &'a [u8] {
     let compress_length = unsafe {
-        streamvbyte::streamvbyte_delta_encode(input.as_ptr(),
-                                              input.len() as u32,
-                                              output.as_mut_ptr(),
-                                              offset)
+        streamvbyte::streamvbyte_delta_encode(
+            input.as_ptr(),
+            input.len() as u32,
+            output.as_mut_ptr(),
+            offset,
+        )
     };
     &output[..compress_length]
 }
 
 #[inline(always)]
-pub fn compress_unsorted<'a>(input: &[u32], output: &'a mut [u8]) -> &'a [u8] {
+pub(crate) fn compress_unsorted<'a>(input: &[u32], output: &'a mut [u8]) -> &'a [u8] {
     let compress_length = unsafe {
         streamvbyte::streamvbyte_encode(input.as_ptr(), input.len() as u32, output.as_mut_ptr())
     };
@@ -46,23 +49,24 @@ pub fn compress_unsorted<'a>(input: &[u32], output: &'a mut [u8]) -> &'a [u8] {
 }
 
 #[inline(always)]
-pub fn uncompress_sorted<'a>(compressed_data: &'a [u8],
-                             output: &mut [u32],
-                             offset: u32)
-                             -> &'a [u8] {
-    let consumed_bytes = unsafe {
-        streamvbyte::streamvbyte_delta_decode(compressed_data.as_ptr(),
-                                              output.as_mut_ptr(),
-                                              output.len() as u32,
-                                              offset)
-    };
-    &compressed_data[consumed_bytes..]
+pub(crate) fn uncompress_sorted<'a>(
+    compressed_data: &'a [u8],
+    output: &mut [u32],
+    offset: u32,
+) -> usize {
+    unsafe {
+        streamvbyte::streamvbyte_delta_decode(
+            compressed_data.as_ptr(),
+            output.as_mut_ptr(),
+            output.len() as u32,
+            offset,
+        )
+    }
 }
 
 #[inline(always)]
-pub fn uncompress_unsorted<'a>(compressed_data: &'a [u8], output: &mut [u32]) -> &'a [u8] {
-    let consumed_bytes = unsafe {
+pub(crate) fn uncompress_unsorted<'a>(compressed_data: &'a [u8], output: &mut [u32]) -> usize {
+    unsafe {
         streamvbyte::streamvbyte_decode(compressed_data.as_ptr(), output.as_mut_ptr(), output.len())
-    };
-    &compressed_data[consumed_bytes..]
+    }
 }

src/core/index.rs

@@ -18,11 +18,13 @@ use core::SegmentMeta;
 use super::pool::LeasedItem;
 use std::path::Path;
 use core::IndexMeta;
+use indexer::DirectoryLock;
 use IndexWriter;
 use directory::ManagedDirectory;
 use core::META_FILEPATH;
 use super::segment::create_segment;
 use indexer::segment_updater::save_new_metas;
+use tokenizer::TokenizerManager;
 
 const NUM_SEARCHERS: usize = 12;
 
@@ -32,14 +34,14 @@ fn load_metas(directory: &Directory) -> Result<IndexMeta> {
     serde_json::from_str(&meta_string).chain_err(|| ErrorKind::CorruptedFile(META_FILEPATH.clone()))
 }
 
-/// Tantivy's Search Index
+/// Search Index
 pub struct Index {
     directory: ManagedDirectory,
     schema: Schema,
     searcher_pool: Arc<Pool<Searcher>>,
+    tokenizers: TokenizerManager,
 }
 
-
 impl Index {
     /// Creates a new index using the `RAMDirectory`.
     ///
@@ -48,9 +50,10 @@ impl Index {
     pub fn create_in_ram(schema: Schema) -> Index {
         let ram_directory = RAMDirectory::create();
         // unwrap is ok here
-        let directory = ManagedDirectory::new(ram_directory)
-            .expect("Creating a managed directory from a brand new RAM directory \
-                     should never fail.");
+        let directory = ManagedDirectory::new(ram_directory).expect(
+            "Creating a managed directory from a brand new RAM directory \
+             should never fail.",
+        );
         Index::from_directory(directory, schema).expect("Creating a RAMDirectory should never fail")
     }
 
@@ -58,12 +61,17 @@ impl Index {
     /// The index will use the `MMapDirectory`.
     ///
     /// If a previous index was in this directory, then its meta file will be destroyed.
-    pub fn create(directory_path: &Path, schema: Schema) -> Result<Index> {
+    pub fn create<P: AsRef<Path>>(directory_path: P, schema: Schema) -> Result<Index> {
         let mmap_directory = MmapDirectory::open(directory_path)?;
         let directory = ManagedDirectory::new(mmap_directory)?;
         Index::from_directory(directory, schema)
     }
 
+    /// Accessor for the tokenizer manager.
+    pub fn tokenizers(&self) -> &TokenizerManager {
+        &self.tokenizers
+    }
+
     /// Creates a new index in a temp directory.
     ///
     /// The index will use the `MMapDirectory` in a newly created directory.
@@ -79,37 +87,36 @@ impl Index {
     }
 
     /// Creates a new index given a directory and an `IndexMeta`.
-    fn create_from_metas(directory: ManagedDirectory, metas: IndexMeta) -> Result<Index> {
+    fn create_from_metas(directory: ManagedDirectory, metas: &IndexMeta) -> Result<Index> {
         let schema = metas.schema.clone();
         let index = Index {
-            directory: directory,
-            schema: schema,
+            directory,
+            schema,
             searcher_pool: Arc::new(Pool::new()),
+            tokenizers: TokenizerManager::default(),
         };
-        try!(index.load_searchers());
+        index.load_searchers()?;
         Ok(index)
     }
 
     /// Create a new index from a directory.
     pub fn from_directory(mut directory: ManagedDirectory, schema: Schema) -> Result<Index> {
         save_new_metas(schema.clone(), 0, directory.borrow_mut())?;
-        Index::create_from_metas(directory, IndexMeta::with_schema(schema))
+        let metas = IndexMeta::with_schema(schema);
+        Index::create_from_metas(directory, &metas)
     }
 
     /// Opens a new directory from an index path.
-    pub fn open(directory_path: &Path) -> Result<Index> {
+    pub fn open<P: AsRef<Path>>(directory_path: P) -> Result<Index> {
         let mmap_directory = MmapDirectory::open(directory_path)?;
         let directory = ManagedDirectory::new(mmap_directory)?;
-        let metas = try!(load_metas(&directory));
-        Index::create_from_metas(directory, metas)
+        let metas = load_metas(&directory)?;
+        Index::create_from_metas(directory, &metas)
     }
 
-    /// Returns the index opstamp.
-    ///
-    /// The opstamp is the number of documents that have been added
-    /// from the beginning of time, and until the moment of the last commit.
-    pub fn opstamp(&self) -> u64 {
-        load_metas(self.directory()).unwrap().opstamp
+    /// Reads the index meta file from the directory.
+    pub fn load_metas(&self) -> Result<IndexMeta> {
+        load_metas(self.directory())
     }
 
     /// Open a new index writer. Attempts to acquire a lockfile.
@@ -127,14 +134,15 @@ impl Index {
     /// If the lockfile already exists, returns `Error::FileAlreadyExists`.
     /// # Panics
     /// If the heap size per thread is too small, panics.
-    pub fn writer_with_num_threads(&self,
-                                   num_threads: usize,
-                                   heap_size_in_bytes: usize)
-                                   -> Result<IndexWriter> {
-        open_index_writer(self, num_threads, heap_size_in_bytes)
+    pub fn writer_with_num_threads(
+        &self,
+        num_threads: usize,
+        heap_size_in_bytes: usize,
+    ) -> Result<IndexWriter> {
+        let directory_lock = DirectoryLock::lock(self.directory().box_clone())?;
+        open_index_writer(self, num_threads, heap_size_in_bytes, directory_lock)
     }
 
-
     /// Creates a multithreaded writer
     /// It just calls `writer_with_num_threads` with the number of cores as `num_threads`
     ///
@@ -156,9 +164,9 @@ impl Index {
     /// Returns the list of segments that are searchable
     pub fn searchable_segments(&self) -> Result<Vec<Segment>> {
         Ok(self.searchable_segment_metas()?
-               .into_iter()
-               .map(|segment_meta| self.segment(segment_meta))
-               .collect())
+            .into_iter()
+            .map(|segment_meta| self.segment(segment_meta))
+            .collect())
     }
 
     #[doc(hidden)]
@@ -185,28 +193,29 @@ impl Index {
     /// Reads the meta.json and returns the list of
     /// `SegmentMeta` from the last commit.
     pub fn searchable_segment_metas(&self) -> Result<Vec<SegmentMeta>> {
-        Ok(load_metas(self.directory())?.segments)
+        Ok(self.load_metas()?.segments)
     }
 
     /// Returns the list of segment ids that are searchable.
     pub fn searchable_segment_ids(&self) -> Result<Vec<SegmentId>> {
         Ok(self.searchable_segment_metas()?
-               .iter()
-               .map(|segment_meta| segment_meta.id())
-               .collect())
+            .iter()
+            .map(|segment_meta| segment_meta.id())
+            .collect())
     }
 
     /// Creates a new generation of searchers after
+
     /// a change of the set of searchable indexes.
     ///
     /// This needs to be called when a new segment has been
     /// published or after a merge.
     pub fn load_searchers(&self) -> Result<()> {
         let searchable_segments = self.searchable_segments()?;
-        let segment_readers: Vec<SegmentReader> = try!(searchable_segments
-                                                           .into_iter()
-                                                           .map(SegmentReader::open)
-                                                           .collect());
+        let segment_readers: Vec<SegmentReader> = searchable_segments
+            .iter()
+            .map(SegmentReader::open)
+            .collect::<Result<_>>()?;
         let searchers = (0..NUM_SEARCHERS)
             .map(|_| Searcher::from(segment_readers.clone()))
             .collect();
@@ -229,7 +238,6 @@ impl Index {
     }
 }
 
-
 impl fmt::Debug for Index {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         write!(f, "Index({:?})", self.directory)
@@ -241,7 +249,8 @@ impl Clone for Index {
         Index {
             directory: self.directory.clone(),
             schema: self.schema.clone(),
-            searcher_pool: self.searcher_pool.clone(),
+            searcher_pool: Arc::clone(&self.searcher_pool),
+            tokenizers: self.tokenizers.clone(),
         }
     }
 }

src/core/index_meta.rs

@@ -1,27 +1,68 @@
 use schema::Schema;
 use core::SegmentMeta;
+use std::fmt;
+use serde_json;
 
 /// Meta information about the `Index`.
 ///
 /// This object is serialized on disk in the `meta.json` file.
 /// It keeps information about
 /// * the searchable segments,
-/// * the index docstamp
+/// * the index `docstamp`
 /// * the schema
 ///
-#[derive(Clone,Debug,Serialize, Deserialize)]
+#[derive(Clone, Serialize, Deserialize)]
 pub struct IndexMeta {
     pub segments: Vec<SegmentMeta>,
     pub schema: Schema,
     pub opstamp: u64,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub payload: Option<String>,
 }
 
 impl IndexMeta {
     pub fn with_schema(schema: Schema) -> IndexMeta {
         IndexMeta {
             segments: vec![],
-            schema: schema,
+            schema,
             opstamp: 0u64,
+            payload: None,
         }
     }
 }
+
+impl fmt::Debug for IndexMeta {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(
+            f,
+            "{}",
+            serde_json::ser::to_string(self)
+                .expect("JSON serialization for IndexMeta should never fail.")
+        )
+    }
+}
+
+#[cfg(test)]
+mod tests {
+
+    use serde_json;
+    use super::IndexMeta;
+    use schema::{SchemaBuilder, TEXT};
+
+    #[test]
+    fn test_serialize_metas() {
+        let schema = {
+            let mut schema_builder = SchemaBuilder::new();
+            schema_builder.add_text_field("text", TEXT);
+            schema_builder.build()
+        };
+        let index_metas = IndexMeta {
+            segments: Vec::new(),
+            schema: schema,
+            opstamp: 0u64,
+            payload: None,
+        };
+        let json = serde_json::ser::to_string(&index_metas).expect("serialization failed");
+        assert_eq!(json, r#"{"segments":[],"schema":[{"name":"text","type":"text","options":{"indexing":{"record":"position","tokenizer":"default"},"stored":false}}],"opstamp":0}"#);
+    }
+}

src/core/inverted_index_reader.rs

@@ -0,0 +1,166 @@
+use directory::{ReadOnlySource, SourceRead};
+use termdict::{TermDictionary, TermDictionaryImpl};
+use postings::{BlockSegmentPostings, SegmentPostings};
+use postings::TermInfo;
+use schema::IndexRecordOption;
+use schema::Term;
+use fastfield::DeleteBitSet;
+use compression::CompressedIntStream;
+use postings::FreqReadingOption;
+use schema::FieldType;
+
+/// The inverted index reader is in charge of accessing
+/// the inverted index associated to a specific field.
+///
+/// # Note
+///
+/// It is safe to delete the segment associated to
+/// an `InvertedIndexReader`. As long as it is open,
+/// the `ReadOnlySource` it is relying on should
+/// stay available.
+///
+///
+/// `InvertedIndexReader` are created by calling
+/// the `SegmentReader`'s [`.inverted_index(...)`] method
+pub struct InvertedIndexReader {
+    termdict: TermDictionaryImpl,
+    postings_source: ReadOnlySource,
+    positions_source: ReadOnlySource,
+    delete_bitset: DeleteBitSet,
+    record_option: IndexRecordOption,
+}
+
+impl InvertedIndexReader {
+    pub(crate) fn new(
+        termdict: TermDictionaryImpl,
+        postings_source: ReadOnlySource,
+        positions_source: ReadOnlySource,
+        delete_bitset: DeleteBitSet,
+        record_option: IndexRecordOption,
+    ) -> InvertedIndexReader {
+        InvertedIndexReader {
+            termdict,
+            postings_source,
+            positions_source,
+            delete_bitset,
+            record_option,
+        }
+    }
+
+    /// Creates an empty `InvertedIndexReader` object, which
+    /// contains no terms at all.
+    pub fn empty(field_type: FieldType) -> InvertedIndexReader {
+        let record_option = field_type
+            .get_index_record_option()
+            .unwrap_or(IndexRecordOption::Basic);
+        InvertedIndexReader::new(
+            TermDictionaryImpl::empty(field_type),
+            ReadOnlySource::empty(),
+            ReadOnlySource::empty(),
+            DeleteBitSet::empty(),
+            record_option,
+        )
+    }
+
+    /// Returns the term info associated with the term.
+    pub fn get_term_info(&self, term: &Term) -> Option<TermInfo> {
+        self.termdict.get(term.value_bytes())
+    }
+
+    /// Return the term dictionary datastructure.
+    pub fn terms(&self) -> &TermDictionaryImpl {
+        &self.termdict
+    }
+
+    /// Resets the block segment to another position of the postings
+    /// file.
+    ///
+    /// This is useful for enumerating through a list of terms,
+    /// and consuming the associated posting lists while avoiding
+    /// reallocating a `BlockSegmentPostings`.
+    ///
+    /// # Warning
+    ///
+    /// This does not reset the positions list.
+    pub fn reset_block_postings_from_terminfo(
+        &self,
+        term_info: &TermInfo,
+        block_postings: &mut BlockSegmentPostings,
+    ) {
+        let offset = term_info.postings_offset as usize;
+        let end_source = self.postings_source.len();
+        let postings_slice = self.postings_source.slice(offset, end_source);
+        let postings_reader = SourceRead::from(postings_slice);
+        block_postings.reset(term_info.doc_freq as usize, postings_reader);
+    }
+
+    /// Returns a block postings given a `term_info`.
+    /// This method is for an advanced usage only.
+    ///
+    /// Most user should prefer using `read_postings` instead.
+    pub fn read_block_postings_from_terminfo(
+        &self,
+        term_info: &TermInfo,
+        requested_option: IndexRecordOption,
+    ) -> BlockSegmentPostings {
+        let offset = term_info.postings_offset as usize;
+        let postings_data = self.postings_source.slice_from(offset);
+        let freq_reading_option = match (self.record_option, requested_option) {
+            (IndexRecordOption::Basic, _) => FreqReadingOption::NoFreq,
+            (_, IndexRecordOption::Basic) => FreqReadingOption::SkipFreq,
+            (_, _) => FreqReadingOption::ReadFreq,
+        };
+        BlockSegmentPostings::from_data(
+            term_info.doc_freq as usize,
+            SourceRead::from(postings_data),
+            freq_reading_option,
+        )
+    }
+
+    /// Returns a posting object given a `term_info`.
+    /// This method is for an advanced usage only.
+    ///
+    /// Most user should prefer using `read_postings` instead.
+    pub fn read_postings_from_terminfo(
+        &self,
+        term_info: &TermInfo,
+        option: IndexRecordOption,
+    ) -> SegmentPostings {
+        let block_postings = self.read_block_postings_from_terminfo(term_info, option);
+        let delete_bitset = self.delete_bitset.clone();
+        let position_stream = {
+            if option.has_positions() {
+                let position_offset = term_info.positions_offset;
+                let positions_source = self.positions_source.slice_from(position_offset as usize);
+                let mut stream = CompressedIntStream::wrap(positions_source);
+                stream.skip(term_info.positions_inner_offset as usize);
+                Some(stream)
+            } else {
+                None
+            }
+        };
+        SegmentPostings::from_block_postings(block_postings, delete_bitset, position_stream)
+    }
+
+    /// Returns the segment postings associated with the term, and with the given option,
+    /// or `None` if the term has never been encountered and indexed.
+    ///
+    /// If the field was not indexed with the indexing options that cover
+    /// the requested options, the returned `SegmentPostings` the method does not fail
+    /// and returns a `SegmentPostings` with as much information as possible.
+    ///
+    /// For instance, requesting `IndexRecordOption::Freq` for a
+    /// `TextIndexingOptions` that does not index position will return a `SegmentPostings`
+    /// with `DocId`s and frequencies.
+    pub fn read_postings(&self, term: &Term, option: IndexRecordOption) -> Option<SegmentPostings> {
+        let term_info = get!(self.get_term_info(term));
+        Some(self.read_postings_from_terminfo(&term_info, option))
+    }
+
+    /// Returns the number of documents containing the term.
+    pub fn doc_freq(&self, term: &Term) -> u32 {
+        self.get_term_info(term)
+            .map(|term_info| term_info.doc_freq)
+            .unwrap_or(0u32)
+    }
+}

src/core/mod.rs

@@ -7,7 +7,9 @@ mod segment;
 mod index_meta;
 mod pool;
 mod segment_meta;
+mod inverted_index_reader;
 
+pub use self::inverted_index_reader::InvertedIndexReader;
 pub use self::searcher::Searcher;
 pub use self::segment_component::SegmentComponent;
 pub use self::segment_id::SegmentId;
@@ -18,7 +20,6 @@ pub use self::index::Index;
 pub use self::segment_meta::SegmentMeta;
 pub use self::index_meta::IndexMeta;
 
-
 use std::path::PathBuf;
 
 lazy_static! {

src/core/pool.rs

@@ -10,44 +10,17 @@ pub struct GenerationItem<T> {
     item: T,
 }
 
-
-// See https://github.com/crossbeam-rs/crossbeam/issues/91
-struct NonLeakingMsQueue<T> {
-    underlying_queue: MsQueue<T>,
-}
-
-impl<T> Default for NonLeakingMsQueue<T> {
-    fn default() -> NonLeakingMsQueue<T> {
-        NonLeakingMsQueue { underlying_queue: MsQueue::new() }
-    }
-}
-
-impl<T> NonLeakingMsQueue<T> {
-    fn pop(&self) -> T {
-        self.underlying_queue.pop()
-    }
-
-    fn push(&self, el: T) {
-        self.underlying_queue.push(el);
-    }
-}
-
-impl<T> Drop for NonLeakingMsQueue<T> {
-    fn drop(&mut self) {
-        while let Some(_popped_item_to_be_dropped) = self.underlying_queue.try_pop() {}
-    }
-}
-
 pub struct Pool<T> {
-    queue: Arc<NonLeakingMsQueue<GenerationItem<T>>>,
+    queue: Arc<MsQueue<GenerationItem<T>>>,
     freshest_generation: AtomicUsize,
     next_generation: AtomicUsize,
 }
 
 impl<T> Pool<T> {
     pub fn new() -> Pool<T> {
+        let queue = Arc::new(MsQueue::new());
         Pool {
-            queue: Arc::default(),
+            queue,
             freshest_generation: AtomicUsize::default(),
             next_generation: AtomicUsize::default(),
         }
@@ -57,7 +30,7 @@ impl<T> Pool<T> {
         let next_generation = self.next_generation.fetch_add(1, Ordering::SeqCst) + 1;
         for item in items {
             let gen_item = GenerationItem {
-                item: item,
+                item,
                 generation: next_generation,
             };
             self.queue.push(gen_item);
@@ -68,7 +41,7 @@ impl<T> Pool<T> {
     /// At the exit of this method,
     /// - freshest_generation has a value greater or equal than generation
     /// - freshest_generation has a value that has been advertised
-    /// - freshest_generation has
+    /// - freshest_generation has)
     fn advertise_generation(&self, generation: usize) {
         // not optimal at all but the easiest to read proof.
         loop {
@@ -76,8 +49,11 @@ impl<T> Pool<T> {
             if former_generation >= generation {
                 break;
             }
-            self.freshest_generation
-                .compare_and_swap(former_generation, generation, Ordering::SeqCst);
+            self.freshest_generation.compare_and_swap(
+                former_generation,
+                generation,
+                Ordering::SeqCst,
+            );
         }
     }
 
@@ -91,21 +67,20 @@ impl<T> Pool<T> {
             let gen_item = self.queue.pop();
             if gen_item.generation >= generation {
                 return LeasedItem {
-                           gen_item: Some(gen_item),
-                           recycle_queue: self.queue.clone(),
-                       };
+                    gen_item: Some(gen_item),
+                    recycle_queue: Arc::clone(&self.queue),
+                };
             } else {
                 // this searcher is obsolete,
                 // removing it from the pool.
             }
         }
-
     }
 }
 
 pub struct LeasedItem<T> {
     gen_item: Option<GenerationItem<T>>,
-    recycle_queue: Arc<NonLeakingMsQueue<GenerationItem<T>>>,
+    recycle_queue: Arc<MsQueue<GenerationItem<T>>>,
 }
 
 impl<T> Deref for LeasedItem<T> {
@@ -113,18 +88,18 @@ impl<T> Deref for LeasedItem<T> {
 
     fn deref(&self) -> &T {
         &self.gen_item
-             .as_ref()
-             .expect("Unwrapping a leased item should never fail")
-             .item // unwrap is safe here
+            .as_ref()
+            .expect("Unwrapping a leased item should never fail")
+            .item // unwrap is safe here
     }
 }
 
 impl<T> DerefMut for LeasedItem<T> {
     fn deref_mut(&mut self) -> &mut T {
         &mut self.gen_item
-                 .as_mut()
-                 .expect("Unwrapping a mut leased item should never fail")
-                 .item // unwrap is safe here
+            .as_mut()
+            .expect("Unwrapping a mut leased item should never fail")
+            .item // unwrap is safe here
     }
 }
 
@@ -136,8 +111,6 @@ impl<T> Drop for LeasedItem<T> {
     }
 }
 
-
-
 #[cfg(test)]
 mod tests {
 

src/core/searcher.rs

@@ -6,11 +6,11 @@ use common::TimerTree;
 use query::Query;
 use DocId;
 use DocAddress;
-use schema::Term;
-use termdict::TermMerger;
+use schema::{Field, Term};
+use termdict::{TermDictionary, TermMerger};
+use std::sync::Arc;
 use std::fmt;
-use postings::TermInfo;
-
+use core::InvertedIndexReader;
 
 /// Holds a list of `SegmentReader`s ready for search.
 ///
@@ -21,7 +21,6 @@ pub struct Searcher {
     segment_readers: Vec<SegmentReader>,
 }
 
-
 impl Searcher {
     /// Fetches a document from tantivy's store given a `DocAddress`.
     ///
@@ -38,7 +37,7 @@ impl Searcher {
         self.segment_readers
             .iter()
             .map(|segment_reader| segment_reader.num_docs())
-            .fold(0u32, |acc, val| acc + val)
+            .sum::<u32>()
     }
 
     /// Return the overall number of documents containing
@@ -46,8 +45,8 @@ impl Searcher {
     pub fn doc_freq(&self, term: &Term) -> u32 {
         self.segment_readers
             .iter()
-            .map(|segment_reader| segment_reader.doc_freq(term))
-            .fold(0u32, |acc, val| acc + val)
+            .map(|segment_reader| segment_reader.inverted_index(term.field()).doc_freq(term))
+            .sum::<u32>()
     }
 
     /// Return the list of segment readers
@@ -65,23 +64,39 @@ impl Searcher {
         query.search(self, collector)
     }
 
-    /// Returns a Stream over all of the sorted unique terms of
-    /// the searcher.
-    ///
-    /// This includes all of the fields from all of the segment_readers.
-    /// See [`TermIterator`](struct.TermIterator.html).
-    ///
-    /// # Warning
-    /// This API is very likely to change in the future.
-    pub fn terms(&self) -> TermMerger<TermInfo> {
-        TermMerger::from(self.segment_readers())
+    /// Return the field searcher associated to a `Field`.
+    pub fn field(&self, field: Field) -> FieldSearcher {
+        let inv_index_readers = self.segment_readers
+            .iter()
+            .map(|segment_reader| segment_reader.inverted_index(field))
+            .collect::<Vec<_>>();
+        FieldSearcher::new(inv_index_readers)
     }
 }
 
+pub struct FieldSearcher {
+    inv_index_readers: Vec<Arc<InvertedIndexReader>>,
+}
+
+impl FieldSearcher {
+    fn new(inv_index_readers: Vec<Arc<InvertedIndexReader>>) -> FieldSearcher {
+        FieldSearcher { inv_index_readers }
+    }
+
+    /// Returns a Stream over all of the sorted unique terms of
+    /// for the given field.
+    pub fn terms(&self) -> TermMerger {
+        let term_streamers: Vec<_> = self.inv_index_readers
+            .iter()
+            .map(|inverted_index| inverted_index.terms().stream())
+            .collect();
+        TermMerger::new(term_streamers)
+    }
+}
 
 impl From<Vec<SegmentReader>> for Searcher {
     fn from(segment_readers: Vec<SegmentReader>) -> Searcher {
-        Searcher { segment_readers: segment_readers }
+        Searcher { segment_readers }
     }
 }
 

src/core/segment.rs

@@ -3,7 +3,7 @@ use std::path::PathBuf;
 use schema::Schema;
 use std::fmt;
 use core::SegmentId;
-use directory::{ReadOnlySource, WritePtr, FileProtection};
+use directory::{FileProtection, ReadOnlySource, WritePtr};
 use indexer::segment_serializer::SegmentSerializer;
 use super::SegmentComponent;
 use core::Index;
@@ -29,13 +29,15 @@ impl fmt::Debug for Segment {
 ///
 /// The function is here to make it private outside `tantivy`.
 pub fn create_segment(index: Index, meta: SegmentMeta) -> Segment {
-    Segment {
-        index: index,
-        meta: meta,
-    }
+    Segment { index, meta }
 }
 
 impl Segment {
+    /// Returns the index the segment belongs to.
+    pub fn index(&self) -> &Index {
+        &self.index
+    }
+
     /// Returns our index's schema.
     pub fn schema(&self) -> Schema {
         self.index.schema()
@@ -64,7 +66,6 @@ impl Segment {
         self.meta.relative_path(component)
     }
 
-
     /// Protects a specific component file from being deleted.
     ///
     /// Returns a FileProtection object. The file is guaranteed
@@ -76,20 +77,22 @@ impl Segment {
     }
 
     /// Open one of the component file for a *regular* read.
-    pub fn open_read(&self,
-                     component: SegmentComponent)
-                     -> result::Result<ReadOnlySource, OpenReadError> {
+    pub fn open_read(
+        &self,
+        component: SegmentComponent,
+    ) -> result::Result<ReadOnlySource, OpenReadError> {
         let path = self.relative_path(component);
-        let source = try!(self.index.directory().open_read(&path));
+        let source = self.index.directory().open_read(&path)?;
         Ok(source)
     }
 
     /// Open one of the component file for *regular* write.
-    pub fn open_write(&mut self,
-                      component: SegmentComponent)
-                      -> result::Result<WritePtr, OpenWriteError> {
+    pub fn open_write(
+        &mut self,
+        component: SegmentComponent,
+    ) -> result::Result<WritePtr, OpenWriteError> {
         let path = self.relative_path(component);
-        let write = try!(self.index.directory_mut().open_write(&path));
+        let write = self.index.directory_mut().open_write(&path)?;
         Ok(write)
     }
 }
@@ -125,11 +128,11 @@ mod tests {
         {
             let _file_protection = segment.protect_from_delete(SegmentComponent::POSTINGS);
             assert!(directory.exists(&*path));
-            directory.garbage_collect(living_files.clone());
+            directory.garbage_collect(|| living_files.clone());
             assert!(directory.exists(&*path));
         }
 
-        directory.garbage_collect(living_files);
+        directory.garbage_collect(|| living_files);
         assert!(!directory.exists(&*path));
     }
 

src/core/segment_component.rs

@@ -1,4 +1,3 @@
-
 /// Enum describing each component of a tantivy segment.
 /// Each component is stored in its own file,
 /// using the pattern `segment_uuid`.`component_extension`,
@@ -28,13 +27,15 @@ pub enum SegmentComponent {
 impl SegmentComponent {
     /// Iterates through the components.
     pub fn iterator() -> impl Iterator<Item = &'static SegmentComponent> {
-        static SEGMENT_COMPONENTS: [SegmentComponent; 7] = [SegmentComponent::POSTINGS,
-                                                            SegmentComponent::POSITIONS,
-                                                            SegmentComponent::FASTFIELDS,
-                                                            SegmentComponent::FIELDNORMS,
-                                                            SegmentComponent::TERMS,
-                                                            SegmentComponent::STORE,
-                                                            SegmentComponent::DELETE];
+        static SEGMENT_COMPONENTS: [SegmentComponent; 7] = [
+            SegmentComponent::POSTINGS,
+            SegmentComponent::POSITIONS,
+            SegmentComponent::FASTFIELDS,
+            SegmentComponent::FIELDNORMS,
+            SegmentComponent::TERMS,
+            SegmentComponent::STORE,
+            SegmentComponent::DELETE,
+        ];
         SEGMENT_COMPONENTS.into_iter()
     }
 }

src/core/segment_id.rs

@@ -1,11 +1,11 @@
 use uuid::Uuid;
 use std::fmt;
-use std::cmp::{Ordering, Ord};
+use std::cmp::{Ord, Ordering};
 
 #[cfg(test)]
 use std::sync::atomic;
 
-/// Tantivy `SegmentId`.
+/// Uuid identifying a segment.
 ///
 /// Tantivy's segment are identified
 /// by a UUID which is used to prefix the filenames
@@ -16,14 +16,12 @@ use std::sync::atomic;
 #[derive(Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
 pub struct SegmentId(Uuid);
 
-
 #[cfg(test)]
 lazy_static! {
     static ref AUTO_INC_COUNTER: atomic::AtomicUsize = atomic::AtomicUsize::default();
     static ref EMPTY_ARR: [u8; 8] = [0u8; 8];
 }
 
-
 // During tests, we generate the segment id in a autoincrement manner
 // for consistency of segment id between run.
 //
@@ -46,7 +44,6 @@ impl SegmentId {
         SegmentId(create_uuid())
     }
 
-
     /// Returns a shorter identifier of the segment.
     ///
     /// We are using UUID4, so only 6 bits are fixed,
@@ -70,7 +67,6 @@ impl fmt::Debug for SegmentId {
     }
 }
 
-
 impl PartialOrd for SegmentId {
     fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
         Some(self.cmp(other))

src/core/segment_meta.rs

@@ -25,7 +25,7 @@ impl SegmentMeta {
     /// a segment with no deletes and no documents.
     pub fn new(segment_id: SegmentId) -> SegmentMeta {
         SegmentMeta {
-            segment_id: segment_id,
+            segment_id,
             max_doc: 0,
             deletes: None,
         }
@@ -54,7 +54,6 @@ impl SegmentMeta {
         SegmentComponent::iterator()
             .map(|component| self.relative_path(*component))
             .collect::<HashSet<PathBuf>>()
-
     }
 
     /// Returns the relative path of a component of our segment.
@@ -64,16 +63,14 @@ impl SegmentMeta {
     pub fn relative_path(&self, component: SegmentComponent) -> PathBuf {
         let mut path = self.id().uuid_string();
         path.push_str(&*match component {
-                           SegmentComponent::POSITIONS => ".pos".to_string(),
-                           SegmentComponent::POSTINGS => ".idx".to_string(),
-                           SegmentComponent::TERMS => ".term".to_string(),
-                           SegmentComponent::STORE => ".store".to_string(),
-                           SegmentComponent::FASTFIELDS => ".fast".to_string(),
-                           SegmentComponent::FIELDNORMS => ".fieldnorm".to_string(),
-                           SegmentComponent::DELETE => {
-                               format!(".{}.del", self.delete_opstamp().unwrap_or(0))
-                           }
-                       });
+            SegmentComponent::POSITIONS => ".pos".to_string(),
+            SegmentComponent::POSTINGS => ".idx".to_string(),
+            SegmentComponent::TERMS => ".term".to_string(),
+            SegmentComponent::STORE => ".store".to_string(),
+            SegmentComponent::FASTFIELDS => ".fast".to_string(),
+            SegmentComponent::FIELDNORMS => ".fieldnorm".to_string(),
+            SegmentComponent::DELETE => format!(".{}.del", self.delete_opstamp().unwrap_or(0)),
+        });
         PathBuf::from(path)
     }
 
@@ -111,8 +108,8 @@ impl SegmentMeta {
     #[doc(hidden)]
     pub fn set_delete_meta(&mut self, num_deleted_docs: u32, opstamp: u64) {
         self.deletes = Some(DeleteMeta {
-                                num_deleted_docs: num_deleted_docs,
-                                opstamp: opstamp,
-                            });
+            num_deleted_docs,
+            opstamp,
+        });
     }
 }

src/core/segment_reader.rs

@@ -2,30 +2,29 @@ use Result;
 use core::Segment;
 use core::SegmentId;
 use core::SegmentComponent;
-use schema::Term;
+use std::sync::RwLock;
 use common::HasLen;
 use core::SegmentMeta;
 use fastfield::{self, FastFieldNotAvailableError};
 use fastfield::DeleteBitSet;
 use store::StoreReader;
 use schema::Document;
-use directory::ReadOnlySource;
 use DocId;
-use std::str;
-use termdict::TermDictionary;
-use std::cmp;
-use postings::TermInfo;
-use termdict::TermDictionaryImpl;
 use std::sync::Arc;
+use std::collections::HashMap;
+use common::CompositeFile;
 use std::fmt;
+use core::InvertedIndexReader;
 use schema::Field;
-use postings::SegmentPostingsOption;
-use postings::{SegmentPostings, BlockSegmentPostings};
-use fastfield::{FastFieldsReader, FastFieldReader, U64FastFieldReader};
+use schema::FieldType;
+use error::ErrorKind;
+use termdict::TermDictionaryImpl;
+use fastfield::FacetReader;
+use fastfield::FastFieldReader;
 use schema::Schema;
-use postings::FreqHandler;
-
-
+use termdict::TermDictionary;
+use fastfield::{FastValue, MultiValueIntFastFieldReader};
+use schema::Cardinality;
 
 /// Entry point to access all of the datastructures of the `Segment`
 ///
@@ -38,17 +37,23 @@ use postings::FreqHandler;
 /// The segment reader has a very low memory footprint,
 /// as close to all of the memory data is mmapped.
 ///
+///
+/// TODO fix not decoding docfreq
 #[derive(Clone)]
 pub struct SegmentReader {
+    inv_idx_reader_cache: Arc<RwLock<HashMap<Field, Arc<InvertedIndexReader>>>>,
+
     segment_id: SegmentId,
     segment_meta: SegmentMeta,
-    terms: Arc<TermDictionaryImpl>,
-    postings_data: ReadOnlySource,
+
+    termdict_composite: CompositeFile,
+    postings_composite: CompositeFile,
+    positions_composite: CompositeFile,
+    fast_fields_composite: CompositeFile,
+    fieldnorms_composite: CompositeFile,
+
     store_reader: StoreReader,
-    fast_fields_reader: Arc<FastFieldsReader>,
-    fieldnorms_reader: Arc<FastFieldsReader>,
     delete_bitset: DeleteBitSet,
-    positions_data: ReadOnlySource,
     schema: Schema,
 }
 
@@ -76,11 +81,6 @@ impl SegmentReader {
         self.delete_bitset.len() as DocId
     }
 
-    #[doc(hidden)]
-    pub fn fast_fields_reader(&self) -> &FastFieldsReader {
-        &*self.fast_fields_reader
-    }
-
     /// Accessor to a segment's fast field reader given a field.
     ///
     /// Returns the u64 fast value reader if the field
@@ -91,20 +91,76 @@ impl SegmentReader {
     ///
     /// # Panics
     /// May panic if the index is corrupted.
-    pub fn get_fast_field_reader<TFastFieldReader: FastFieldReader>
-        (&self,
-         field: Field)
-         -> fastfield::Result<TFastFieldReader> {
+    pub fn fast_field_reader<Item: FastValue>(
+        &self,
+        field: Field,
+    ) -> fastfield::Result<FastFieldReader<Item>> {
         let field_entry = self.schema.get_field_entry(field);
-        if !TFastFieldReader::is_enabled(field_entry.field_type()) {
-            Err(FastFieldNotAvailableError::new(field_entry))
+        if Item::fast_field_cardinality(field_entry.field_type()) == Some(Cardinality::SingleValue)
+        {
+            self.fast_fields_composite
+                .open_read(field)
+                .ok_or_else(|| FastFieldNotAvailableError::new(field_entry))
+                .map(FastFieldReader::open)
         } else {
-            Ok(self.fast_fields_reader
-                   .open_reader(field)
-                   .expect("Fast field file corrupted."))
+            Err(FastFieldNotAvailableError::new(field_entry))
         }
     }
 
+    pub(crate) fn fast_field_reader_with_idx<Item: FastValue>(
+        &self,
+        field: Field,
+        idx: usize
+    ) -> fastfield::Result<FastFieldReader<Item>> {
+        if let Some(ff_source) = self.fast_fields_composite.open_read_with_idx(field, idx) {
+            Ok(FastFieldReader::open(ff_source))
+        } else {
+            let field_entry = self.schema.get_field_entry(field);
+            Err(FastFieldNotAvailableError::new(field_entry))
+        }
+    }
+
+    /// Accessor to the `MultiValueIntFastFieldReader` associated to a given `Field`.
+    /// May panick if the field is not a multivalued fastfield of the type `Item`.
+    pub fn multi_fast_field_reader<Item: FastValue>(
+        &self,
+        field: Field,
+    ) -> fastfield::Result<MultiValueIntFastFieldReader<Item>> {
+        let field_entry = self.schema.get_field_entry(field);
+        if Item::fast_field_cardinality(field_entry.field_type()) == Some(Cardinality::MultiValues)
+        {
+            let idx_reader = self.fast_field_reader_with_idx(field, 0)?;
+            let vals_reader = self.fast_field_reader_with_idx(field, 1)?;
+            Ok(MultiValueIntFastFieldReader::open(idx_reader, vals_reader))
+        } else {
+            Err(FastFieldNotAvailableError::new(field_entry))
+        }
+    }
+
+    /// Accessor to the `FacetReader` associated to a given `Field`.
+    pub fn facet_reader(&self, field: Field) -> Result<FacetReader> {
+        let field_entry = self.schema.get_field_entry(field);
+        if field_entry.field_type() != &FieldType::HierarchicalFacet {
+            return Err(ErrorKind::InvalidArgument(format!(
+                "The field {:?} is not a \
+                 hierarchical facet.",
+                field_entry
+            )).into());
+        }
+        let term_ords_reader = self.multi_fast_field_reader(field)?;
+        let termdict_source = self.termdict_composite.open_read(field).ok_or_else(|| {
+            ErrorKind::InvalidArgument(format!(
+                "The field \"{}\" is a hierarchical \
+                 but this segment does not seem to have the field term \
+                 dictionary.",
+                field_entry.name()
+            ))
+        })?;
+        let termdict = TermDictionaryImpl::from_source(termdict_source);
+        let facet_reader = FacetReader::new(term_ords_reader, termdict);
+        Ok(facet_reader)
+    }
+
     /// Accessor to the segment's `Field norms`'s reader.
     ///
     /// Field norms are the length (in tokens) of the fields.
@@ -113,16 +169,10 @@ impl SegmentReader {
     ///
     /// They are simply stored as a fast field, serialized in
     /// the `.fieldnorm` file of the segment.
-    pub fn get_fieldnorms_reader(&self, field: Field) -> Option<U64FastFieldReader> {
-        self.fieldnorms_reader.open_reader(field)
-    }
-
-    /// Returns the number of documents containing the term.
-    pub fn doc_freq(&self, term: &Term) -> u32 {
-        match self.get_term_info(term) {
-            Some(term_info) => term_info.doc_freq,
-            None => 0,
-        }
+    pub fn get_fieldnorms_reader(&self, field: Field) -> Option<FastFieldReader<u64>> {
+        self.fieldnorms_composite
+            .open_read(field)
+            .map(FastFieldReader::open)
     }
 
     /// Accessor to the segment's `StoreReader`.
@@ -131,25 +181,29 @@ impl SegmentReader {
     }
 
     /// Open a new segment for reading.
-    pub fn open(segment: Segment) -> Result<SegmentReader> {
-
-        let source = segment.open_read(SegmentComponent::TERMS)?;
-        let terms = TermDictionaryImpl::from_source(source)?;
+    pub fn open(segment: &Segment) -> Result<SegmentReader> {
+        let termdict_source = segment.open_read(SegmentComponent::TERMS)?;
+        let termdict_composite = CompositeFile::open(&termdict_source)?;
 
         let store_source = segment.open_read(SegmentComponent::STORE)?;
         let store_reader = StoreReader::from_source(store_source);
 
-        let postings_shared_mmap = segment.open_read(SegmentComponent::POSTINGS)?;
+        let postings_source = segment.open_read(SegmentComponent::POSTINGS)?;
+        let postings_composite = CompositeFile::open(&postings_source)?;
 
-        let fast_field_data = segment.open_read(SegmentComponent::FASTFIELDS)?;
-        let fast_fields_reader = FastFieldsReader::from_source(fast_field_data)?;
+        let positions_composite = {
+            if let Ok(source) = segment.open_read(SegmentComponent::POSITIONS) {
+                CompositeFile::open(&source)?
+            } else {
+                CompositeFile::empty()
+            }
+        };
+
+        let fast_fields_data = segment.open_read(SegmentComponent::FASTFIELDS)?;
+        let fast_fields_composite = CompositeFile::open(&fast_fields_data)?;
 
         let fieldnorms_data = segment.open_read(SegmentComponent::FIELDNORMS)?;
-        let fieldnorms_reader = FastFieldsReader::from_source(fieldnorms_data)?;
-
-        let positions_data = segment
-            .open_read(SegmentComponent::POSITIONS)
-            .unwrap_or_else(|_| ReadOnlySource::empty());
+        let fieldnorms_composite = CompositeFile::open(&fieldnorms_data)?;
 
         let delete_bitset = if segment.meta().has_deletes() {
             let delete_data = segment.open_read(SegmentComponent::DELETE)?;
@@ -160,22 +214,81 @@ impl SegmentReader {
 
         let schema = segment.schema();
         Ok(SegmentReader {
-               segment_meta: segment.meta().clone(),
-               postings_data: postings_shared_mmap,
-               terms: Arc::new(terms),
-               segment_id: segment.id(),
-               store_reader: store_reader,
-               fast_fields_reader: Arc::new(fast_fields_reader),
-               fieldnorms_reader: Arc::new(fieldnorms_reader),
-               delete_bitset: delete_bitset,
-               positions_data: positions_data,
-               schema: schema,
-           })
+            inv_idx_reader_cache: Arc::new(RwLock::new(HashMap::new())),
+            segment_meta: segment.meta().clone(),
+            termdict_composite,
+            postings_composite,
+            fast_fields_composite,
+            fieldnorms_composite,
+            segment_id: segment.id(),
+            store_reader,
+            delete_bitset,
+            positions_composite,
+            schema,
+        })
     }
 
-    /// Return the term dictionary datastructure.
-    pub fn terms(&self) -> &TermDictionaryImpl {
-        &self.terms
+    /// Returns a field reader associated to the field given in argument.
+    /// If the field was not present in the index during indexing time,
+    /// the InvertedIndexReader is empty.
+    ///
+    /// The field reader is in charge of iterating through the
+    /// term dictionary associated to a specific field,
+    /// and opening the posting list associated to any term.
+    pub fn inverted_index(&self, field: Field) -> Arc<InvertedIndexReader> {
+        if let Some(inv_idx_reader) = self.inv_idx_reader_cache
+            .read()
+            .expect("Lock poisoned. This should never happen")
+            .get(&field)
+        {
+            return Arc::clone(inv_idx_reader);
+        }
+        let field_entry = self.schema.get_field_entry(field);
+        let field_type = field_entry.field_type();
+        let record_option_opt = field_type.get_index_record_option();
+
+        if record_option_opt.is_none() {
+            panic!("Field {:?} does not seem indexed.", field_entry.name());
+        }
+
+        let record_option = record_option_opt.unwrap();
+
+        let postings_source_opt = self.postings_composite.open_read(field);
+
+        if postings_source_opt.is_none() {
+            // no documents in the segment contained this field.
+            // As a result, no data is associated to the inverted index.
+            //
+            // Returns an empty inverted index.
+            return Arc::new(InvertedIndexReader::empty(field_type.clone()));
+        }
+
+        let postings_source = postings_source_opt.unwrap();
+
+        let termdict_source = self.termdict_composite
+            .open_read(field)
+            .expect("Failed to open field term dictionary in composite file. Is the field indexed");
+
+        let positions_source = self.positions_composite
+            .open_read(field)
+            .expect("Index corrupted. Failed to open field positions in composite file.");
+
+        let inv_idx_reader = Arc::new(InvertedIndexReader::new(
+            TermDictionaryImpl::from_source(termdict_source),
+            postings_source,
+            positions_source,
+            self.delete_bitset.clone(),
+            record_option,
+        ));
+
+        // by releasing the lock in between, we may end up opening the inverting index
+        // twice, but this is fine.
+        self.inv_idx_reader_cache
+            .write()
+            .expect("Field reader cache lock poisoned. This should never happen.")
+            .insert(field, Arc::clone(&inv_idx_reader));
+
+        inv_idx_reader
     }
 
     /// Returns the document (or to be accurate, its stored field)
@@ -186,90 +299,6 @@ impl SegmentReader {
         self.store_reader.get(doc_id)
     }
 
-
-    /// Returns the segment postings associated with the term, and with the given option,
-    /// or `None` if the term has never been encounterred and indexed.
-    ///
-    /// If the field was not indexed with the indexing options that cover
-    /// the requested options, the returned `SegmentPostings` the method does not fail
-    /// and returns a `SegmentPostings` with as much information as possible.
-    ///
-    /// For instance, requesting `SegmentPostingsOption::FreqAndPositions` for a
-    /// `TextIndexingOptions` that does not index position will return a `SegmentPostings`
-    /// with `DocId`s and frequencies.
-    pub fn read_postings(&self,
-                         term: &Term,
-                         option: SegmentPostingsOption)
-                         -> Option<SegmentPostings> {
-        let field = term.field();
-        let field_entry = self.schema.get_field_entry(field);
-        let term_info = get!(self.get_term_info(term));
-        let maximum_option = get!(field_entry.field_type().get_segment_postings_option());
-        let best_effort_option = cmp::min(maximum_option, option);
-        Some(self.read_postings_from_terminfo(&term_info, best_effort_option))
-    }
-
-
-    /// Returns a posting object given a `term_info`.
-    /// This method is for an advanced usage only.
-    ///
-    /// Most user should prefer using `read_postings` instead.
-    pub fn read_postings_from_terminfo(&self,
-                                       term_info: &TermInfo,
-                                       option: SegmentPostingsOption)
-                                       -> SegmentPostings {
-        let block_postings = self.read_block_postings_from_terminfo(term_info, option);
-        let delete_bitset = self.delete_bitset.clone();
-        SegmentPostings::from_block_postings(block_postings, delete_bitset)
-    }
-
-
-    /// Returns a block postings given a `term_info`.
-    /// This method is for an advanced usage only.
-    ///
-    /// Most user should prefer using `read_postings` instead.
-    pub fn read_block_postings_from_terminfo(&self,
-                                             term_info: &TermInfo,
-                                             option: SegmentPostingsOption)
-                                             -> BlockSegmentPostings {
-        let offset = term_info.postings_offset as usize;
-        let postings_data = &self.postings_data[offset..];
-        let freq_handler = match option {
-            SegmentPostingsOption::NoFreq => FreqHandler::new_without_freq(),
-            SegmentPostingsOption::Freq => FreqHandler::new_with_freq(),
-            SegmentPostingsOption::FreqAndPositions => {
-                let offset = term_info.positions_offset as usize;
-                let offseted_position_data = &self.positions_data[offset..];
-                FreqHandler::new_with_freq_and_position(offseted_position_data)
-            }
-        };
-        BlockSegmentPostings::from_data(term_info.doc_freq as usize, postings_data, freq_handler)
-    }
-
-
-    /// Resets the block segment to another position of the postings
-    /// file.
-    ///
-    /// This is useful for enumerating through a list of terms,
-    /// and consuming the associated posting lists while avoiding
-    /// reallocating a `BlockSegmentPostings`.
-    ///
-    /// # Warning
-    ///
-    /// This does not reset the positions list.
-    pub fn reset_block_postings_from_terminfo<'a>(&'a self,
-                                                  term_info: &TermInfo,
-                                                  block_postings: &mut BlockSegmentPostings<'a>) {
-        let offset = term_info.postings_offset as usize;
-        let postings_data: &'a [u8] = &self.postings_data[offset..];
-        block_postings.reset(term_info.doc_freq as usize, postings_data);
-    }
-
-    /// Returns the term info associated with the term.
-    pub fn get_term_info(&self, term: &Term) -> Option<TermInfo> {
-        self.terms.get(term.as_slice())
-    }
-
     /// Returns the segment id
     pub fn segment_id(&self) -> SegmentId {
         self.segment_id
@@ -281,7 +310,6 @@ impl SegmentReader {
         &self.delete_bitset
     }
 
-
     /// Returns true iff the `doc` is marked
     /// as deleted.
     pub fn is_deleted(&self, doc: DocId) -> bool {
@@ -289,7 +317,6 @@ impl SegmentReader {
     }
 }
 
-
 impl fmt::Debug for SegmentReader {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         write!(f, "SegmentReader({:?})", self.segment_id)

src/datastruct/mod.rs

@@ -1,4 +1,4 @@
 mod skip;
 pub mod stacker;
 
-pub use self::skip::{SkipListBuilder, SkipList};
+pub use self::skip::{SkipList, SkipListBuilder};

src/datastruct/skip/mod.rs

@@ -6,17 +6,15 @@ mod skiplist;
 pub use self::skiplist_builder::SkipListBuilder;
 pub use self::skiplist::SkipList;
 
-
-
 #[cfg(test)]
 mod tests {
 
-    use super::*;
+    use super::{SkipList, SkipListBuilder};
 
     #[test]
     fn test_skiplist() {
         let mut output: Vec<u8> = Vec::new();
-        let mut skip_list_builder: SkipListBuilder<u32> = SkipListBuilder::new(10);
+        let mut skip_list_builder: SkipListBuilder<u32> = SkipListBuilder::new(8);
         skip_list_builder.insert(2, &3).unwrap();
         skip_list_builder.write::<Vec<u8>>(&mut output).unwrap();
         let mut skip_list: SkipList<u32> = SkipList::from(output.as_slice());
@@ -26,7 +24,7 @@ mod tests {
     #[test]
     fn test_skiplist2() {
         let mut output: Vec<u8> = Vec::new();
-        let skip_list_builder: SkipListBuilder<u32> = SkipListBuilder::new(10);
+        let skip_list_builder: SkipListBuilder<u32> = SkipListBuilder::new(8);
         skip_list_builder.write::<Vec<u8>>(&mut output).unwrap();
         let mut skip_list: SkipList<u32> = SkipList::from(output.as_slice());
         assert_eq!(skip_list.next(), None);
@@ -73,7 +71,7 @@ mod tests {
     #[test]
     fn test_skiplist5() {
         let mut output: Vec<u8> = Vec::new();
-        let mut skip_list_builder: SkipListBuilder<()> = SkipListBuilder::new(3);
+        let mut skip_list_builder: SkipListBuilder<()> = SkipListBuilder::new(4);
         skip_list_builder.insert(2, &()).unwrap();
         skip_list_builder.insert(3, &()).unwrap();
         skip_list_builder.insert(5, &()).unwrap();
@@ -105,7 +103,7 @@ mod tests {
     #[test]
     fn test_skiplist7() {
         let mut output: Vec<u8> = Vec::new();
-        let mut skip_list_builder: SkipListBuilder<()> = SkipListBuilder::new(3);
+        let mut skip_list_builder: SkipListBuilder<()> = SkipListBuilder::new(4);
         for i in 0..1000 {
             skip_list_builder.insert(i, &()).unwrap();
         }
@@ -123,35 +121,48 @@ mod tests {
     #[test]
     fn test_skiplist8() {
         let mut output: Vec<u8> = Vec::new();
-        let mut skip_list_builder: SkipListBuilder<u32> = SkipListBuilder::new(10);
+        let mut skip_list_builder: SkipListBuilder<u64> = SkipListBuilder::new(8);
         skip_list_builder.insert(2, &3).unwrap();
         skip_list_builder.write::<Vec<u8>>(&mut output).unwrap();
-        assert_eq!(output.len(), 13);
+        assert_eq!(output.len(), 11);
         assert_eq!(output[0], 1u8 + 128u8);
     }
 
     #[test]
     fn test_skiplist9() {
         let mut output: Vec<u8> = Vec::new();
-        let mut skip_list_builder: SkipListBuilder<u32> = SkipListBuilder::new(3);
-        for i in 0..9 {
+        let mut skip_list_builder: SkipListBuilder<u64> = SkipListBuilder::new(4);
+        for i in 0..4 * 4 * 4 {
             skip_list_builder.insert(i, &i).unwrap();
         }
         skip_list_builder.write::<Vec<u8>>(&mut output).unwrap();
-        assert_eq!(output.len(), 117);
-        assert_eq!(output[0], 3u8 + 128u8);
+        assert_eq!(output.len(), 774);
+        assert_eq!(output[0], 4u8 + 128u8);
     }
 
     #[test]
     fn test_skiplist10() {
         // checking that void gets serialized to nothing.
         let mut output: Vec<u8> = Vec::new();
-        let mut skip_list_builder: SkipListBuilder<()> = SkipListBuilder::new(3);
-        for i in 0..9 {
+        let mut skip_list_builder: SkipListBuilder<()> = SkipListBuilder::new(4);
+        for i in 0..((4 * 4 * 4) - 1) {
             skip_list_builder.insert(i, &()).unwrap();
         }
         skip_list_builder.write::<Vec<u8>>(&mut output).unwrap();
-        assert_eq!(output.len(), 81);
+        assert_eq!(output.len(), 230);
+        assert_eq!(output[0], 128u8 + 3u8);
+    }
+
+    #[test]
+    fn test_skiplist11() {
+        // checking that void gets serialized to nothing.
+        let mut output: Vec<u8> = Vec::new();
+        let mut skip_list_builder: SkipListBuilder<()> = SkipListBuilder::new(4);
+        for i in 0..(4 * 4) {
+            skip_list_builder.insert(i, &()).unwrap();
+        }
+        skip_list_builder.write::<Vec<u8>>(&mut output).unwrap();
+        assert_eq!(output.len(), 65);
         assert_eq!(output[0], 128u8 + 3u8);
     }
 

src/datastruct/skip/skiplist.rs

@@ -1,6 +1,5 @@
-use common::BinarySerializable;
+use common::{BinarySerializable, VInt};
 use std::marker::PhantomData;
-use DocId;
 use std::cmp::max;
 
 static EMPTY: [u8; 0] = [];
@@ -8,21 +7,20 @@ static EMPTY: [u8; 0] = [];
 struct Layer<'a, T> {
     data: &'a [u8],
     cursor: &'a [u8],
-    next_id: DocId,
+    next_id: Option<u64>,
     _phantom_: PhantomData<T>,
 }
 
 impl<'a, T: BinarySerializable> Iterator for Layer<'a, T> {
-    type Item = (DocId, T);
+    type Item = (u64, T);
 
-    fn next(&mut self) -> Option<(DocId, T)> {
-        if self.next_id == u32::max_value() {
-            None
-        } else {
+    fn next(&mut self) -> Option<(u64, T)> {
+        if let Some(cur_id) = self.next_id {
             let cur_val = T::deserialize(&mut self.cursor).unwrap();
-            let cur_id = self.next_id;
-            self.next_id = u32::deserialize(&mut self.cursor).unwrap_or(u32::max_value());
+            self.next_id = VInt::deserialize_u64(&mut self.cursor).ok();
             Some((cur_id, cur_val))
+        } else {
+            None
         }
     }
 }
@@ -30,11 +28,11 @@ impl<'a, T: BinarySerializable> Iterator for Layer<'a, T> {
 impl<'a, T: BinarySerializable> From<&'a [u8]> for Layer<'a, T> {
     fn from(data: &'a [u8]) -> Layer<'a, T> {
         let mut cursor = data;
-        let next_id = u32::deserialize(&mut cursor).unwrap_or(u32::max_value());
+        let next_id = VInt::deserialize_u64(&mut cursor).ok();
         Layer {
-            data: data,
-            cursor: cursor,
-            next_id: next_id,
+            data,
+            cursor,
+            next_id,
             _phantom_: PhantomData,
         }
     }
@@ -45,14 +43,14 @@ impl<'a, T: BinarySerializable> Layer<'a, T> {
         Layer {
             data: &EMPTY,
             cursor: &EMPTY,
-            next_id: DocId::max_value(),
+            next_id: None,
             _phantom_: PhantomData,
         }
     }
 
     fn seek_offset(&mut self, offset: usize) {
         self.cursor = &self.data[offset..];
-        self.next_id = u32::deserialize(&mut self.cursor).unwrap_or(u32::max_value());
+        self.next_id = VInt::deserialize_u64(&mut self.cursor).ok();
     }
 
     // Returns the last element (key, val)
@@ -60,56 +58,61 @@ impl<'a, T: BinarySerializable> Layer<'a, T> {
     //
     // If there is no such element anymore,
     // returns None.
-    fn seek(&mut self, doc_id: DocId) -> Option<(DocId, T)> {
-        let mut val = None;
-        while self.next_id < doc_id {
-            match self.next() {
-                None => {
-                    break;
-                }
-                v => {
-                    val = v;
+    //
+    // If the element exists, it will be returned
+    // at the next call to `.next()`.
+    fn seek(&mut self, key: u64) -> Option<(u64, T)> {
+        let mut result: Option<(u64, T)> = None;
+        loop {
+            if let Some(next_id) = self.next_id {
+                if next_id < key {
+                    if let Some(v) = self.next() {
+                        result = Some(v);
+                        continue;
+                    }
                 }
             }
+            return result;
         }
-        val
     }
 }
 
-
 pub struct SkipList<'a, T: BinarySerializable> {
     data_layer: Layer<'a, T>,
-    skip_layers: Vec<Layer<'a, u32>>,
+    skip_layers: Vec<Layer<'a, u64>>,
 }
 
 impl<'a, T: BinarySerializable> Iterator for SkipList<'a, T> {
-    type Item = (DocId, T);
+    type Item = (u64, T);
 
-    fn next(&mut self) -> Option<(DocId, T)> {
+    fn next(&mut self) -> Option<(u64, T)> {
         self.data_layer.next()
     }
 }
 
 impl<'a, T: BinarySerializable> SkipList<'a, T> {
-    pub fn seek(&mut self, doc_id: DocId) -> Option<(DocId, T)> {
-        let mut next_layer_skip: Option<(DocId, u32)> = None;
+    pub fn seek(&mut self, key: u64) -> Option<(u64, T)> {
+        let mut next_layer_skip: Option<(u64, u64)> = None;
         for skip_layer in &mut self.skip_layers {
             if let Some((_, offset)) = next_layer_skip {
                 skip_layer.seek_offset(offset as usize);
             }
-            next_layer_skip = skip_layer.seek(doc_id);
+            next_layer_skip = skip_layer.seek(key);
         }
         if let Some((_, offset)) = next_layer_skip {
             self.data_layer.seek_offset(offset as usize);
         }
-        self.data_layer.seek(doc_id)
+        self.data_layer.seek(key)
     }
 }
 
-
 impl<'a, T: BinarySerializable> From<&'a [u8]> for SkipList<'a, T> {
     fn from(mut data: &'a [u8]) -> SkipList<'a, T> {
-        let offsets: Vec<u32> = Vec::deserialize(&mut data).unwrap();
+        let offsets: Vec<u64> = Vec::<VInt>::deserialize(&mut data)
+            .unwrap()
+            .into_iter()
+            .map(|el| el.0)
+            .collect();
         let num_layers = offsets.len();
         let layers_data: &[u8] = data;
         let data_layer: Layer<'a, T> = if num_layers == 0 {
@@ -123,8 +126,8 @@ impl<'a, T: BinarySerializable> From<&'a [u8]> for SkipList<'a, T> {
             .map(|(start, stop)| Layer::from(&layers_data[start..stop]))
             .collect();
         SkipList {
-            skip_layers: skip_layers,
-            data_layer: data_layer,
+            skip_layers,
+            data_layer,
         }
     }
 }

src/datastruct/skip/skiplist_builder.rs

@@ -1,13 +1,11 @@
 use std::io::Write;
-use common::BinarySerializable;
+use common::{BinarySerializable, VInt, is_power_of_2};
 use std::marker::PhantomData;
-use DocId;
 use std::io;
 
 struct LayerBuilder<T: BinarySerializable> {
-    period: usize,
+    period_mask: usize,
     buffer: Vec<u8>,
-    remaining: usize,
     len: usize,
     _phantom_: PhantomData<T>,
 }
@@ -23,48 +21,45 @@ impl<T: BinarySerializable> LayerBuilder<T> {
     }
 
     fn with_period(period: usize) -> LayerBuilder<T> {
+        assert!(is_power_of_2(period), "The period has to be a power of 2.");
         LayerBuilder {
-            period: period,
+            period_mask: (period - 1),
             buffer: Vec::new(),
-            remaining: period,
             len: 0,
             _phantom_: PhantomData,
         }
     }
 
-    fn insert(&mut self, doc_id: DocId, value: &T) -> io::Result<Option<(DocId, u32)>> {
-        self.remaining -= 1;
+    fn insert(&mut self, key: u64, value: &T) -> io::Result<Option<(u64, u64)>> {
         self.len += 1;
-        let offset = self.written_size() as u32;
-        doc_id.serialize(&mut self.buffer)?;
+        let offset = self.written_size() as u64;
+        VInt(key).serialize(&mut self.buffer)?;
         value.serialize(&mut self.buffer)?;
-        Ok(if self.remaining == 0 {
-               self.remaining = self.period;
-               Some((doc_id, offset))
-           } else {
-               None
-           })
+        let emit_skip_info = (self.period_mask & self.len) == 0;
+        if emit_skip_info {
+            Ok(Some((key, offset)))
+        } else {
+            Ok(None)
+        }
     }
 }
 
-
 pub struct SkipListBuilder<T: BinarySerializable> {
     period: usize,
     data_layer: LayerBuilder<T>,
-    skip_layers: Vec<LayerBuilder<u32>>,
+    skip_layers: Vec<LayerBuilder<u64>>,
 }
 
-
 impl<T: BinarySerializable> SkipListBuilder<T> {
     pub fn new(period: usize) -> SkipListBuilder<T> {
         SkipListBuilder {
-            period: period,
+            period,
             data_layer: LayerBuilder::with_period(period),
             skip_layers: Vec::new(),
         }
     }
 
-    fn get_skip_layer(&mut self, layer_id: usize) -> &mut LayerBuilder<u32> {
+    fn get_skip_layer(&mut self, layer_id: usize) -> &mut LayerBuilder<u64> {
         if layer_id == self.skip_layers.len() {
             let layer_builder = LayerBuilder::with_period(self.period);
             self.skip_layers.push(layer_builder);
@@ -72,15 +67,13 @@ impl<T: BinarySerializable> SkipListBuilder<T> {
         &mut self.skip_layers[layer_id]
     }
 
-    pub fn insert(&mut self, doc_id: DocId, dest: &T) -> io::Result<()> {
+    pub fn insert(&mut self, key: u64, dest: &T) -> io::Result<()> {
         let mut layer_id = 0;
-        let mut skip_pointer = try!(self.data_layer.insert(doc_id, dest));
+        let mut skip_pointer = self.data_layer.insert(key, dest)?;
         loop {
             skip_pointer = match skip_pointer {
-                Some((skip_doc_id, skip_offset)) => {
-                    try!(self.get_skip_layer(layer_id)
-                             .insert(skip_doc_id, &skip_offset))
-                }
+                Some((skip_doc_id, skip_offset)) => self.get_skip_layer(layer_id)
+                    .insert(skip_doc_id, &skip_offset)?,
                 None => {
                     return Ok(());
                 }
@@ -90,13 +83,11 @@ impl<T: BinarySerializable> SkipListBuilder<T> {
     }
 
     pub fn write<W: Write>(self, output: &mut W) -> io::Result<()> {
-        let mut size: u32 = 0;
-        let mut layer_sizes: Vec<u32> = Vec::new();
-        size += self.data_layer.buffer.len() as u32;
-        layer_sizes.push(size);
+        let mut size: u64 = self.data_layer.buffer.len() as u64;
+        let mut layer_sizes = vec![VInt(size)];
         for layer in self.skip_layers.iter().rev() {
-            size += layer.buffer.len() as u32;
-            layer_sizes.push(size);
+            size += layer.buffer.len() as u64;
+            layer_sizes.push(VInt(size));
         }
         layer_sizes.serialize(output)?;
         self.data_layer.write(output)?;

src/datastruct/stacker/expull.rs

@@ -1,7 +1,6 @@
 use std::mem;
 use super::heap::{Heap, HeapAllocable};
 
-
 #[inline]
 pub fn is_power_of_2(val: u32) -> bool {
     val & (val - 1) == 0
@@ -12,7 +11,6 @@ pub fn jump_needed(val: u32) -> bool {
     val > 3 && is_power_of_2(val)
 }
 
-
 #[derive(Debug, Clone)]
 pub struct ExpUnrolledLinkedList {
     len: u32,
@@ -26,7 +24,7 @@ pub struct ExpUnrolledLinkedList {
 impl ExpUnrolledLinkedList {
     pub fn iter<'a>(&self, addr: u32, heap: &'a Heap) -> ExpUnrolledLinkedListIterator<'a> {
         ExpUnrolledLinkedListIterator {
-            heap: heap,
+            heap,
             addr: addr + 2u32 * (mem::size_of::<u32>() as u32),
             len: self.len,
             consumed: 0,
@@ -51,7 +49,6 @@ impl ExpUnrolledLinkedList {
     }
 }
 
-
 impl HeapAllocable for u32 {
     fn with_addr(_addr: u32) -> u32 {
         0u32
@@ -96,14 +93,9 @@ impl<'a> Iterator for ExpUnrolledLinkedListIterator<'a> {
             self.addr = addr + mem::size_of::<u32>() as u32;
             Some(*self.heap.get_mut_ref(addr))
         }
-
     }
 }
 
-
-
-
-
 #[cfg(test)]
 mod tests {
 

src/datastruct/stacker/hashmap.rs

@@ -1,6 +1,7 @@
 use std::iter;
 use std::mem;
-use super::heap::{Heap, HeapAllocable, BytesRef};
+use postings::UnorderedTermId;
+use super::heap::{BytesRef, Heap, HeapAllocable};
 
 mod murmurhash2 {
 
@@ -9,7 +10,7 @@ mod murmurhash2 {
     #[inline(always)]
     pub fn murmurhash2(key: &[u8]) -> u32 {
         let mut key_ptr: *const u32 = key.as_ptr() as *const u32;
-        let m: u32 = 0x5bd1e995;
+        let m: u32 = 0x5bd1_e995;
         let r = 24;
         let len = key.len() as u32;
 
@@ -30,18 +31,18 @@ mod murmurhash2 {
         let key_ptr_u8: *const u8 = key_ptr as *const u8;
         match remaining {
             3 => {
-                h ^= unsafe { *key_ptr_u8.wrapping_offset(2) as u32 } << 16;
-                h ^= unsafe { *key_ptr_u8.wrapping_offset(1) as u32 } << 8;
-                h ^= unsafe { *key_ptr_u8 as u32 };
+                h ^= unsafe { u32::from(*key_ptr_u8.wrapping_offset(2)) } << 16;
+                h ^= unsafe { u32::from(*key_ptr_u8.wrapping_offset(1)) } << 8;
+                h ^= unsafe { u32::from(*key_ptr_u8) };
                 h = h.wrapping_mul(m);
             }
             2 => {
-                h ^= unsafe { *key_ptr_u8.wrapping_offset(1) as u32 } << 8;
-                h ^= unsafe { *key_ptr_u8 as u32 };
+                h ^= unsafe { u32::from(*key_ptr_u8.wrapping_offset(1)) } << 8;
+                h ^= unsafe { u32::from(*key_ptr_u8) };
                 h = h.wrapping_mul(m);
             }
             1 => {
-                h ^= unsafe { *key_ptr_u8 as u32 };
+                h ^= unsafe { u32::from(*key_ptr_u8) };
                 h = h.wrapping_mul(m);
             }
             _ => {}
@@ -52,9 +53,6 @@ mod murmurhash2 {
     }
 }
 
-
-
-
 /// Split the thread memory budget into
 /// - the heap size
 /// - the hash table "table" itself.
@@ -62,21 +60,20 @@ mod murmurhash2 {
 /// Returns (the heap size in bytes, the hash table size in number of bits)
 pub(crate) fn split_memory(per_thread_memory_budget: usize) -> (usize, usize) {
     let table_size_limit: usize = per_thread_memory_budget / 3;
-    let compute_table_size = |num_bits: usize| {
-        let table_size: usize = (1 << num_bits) * mem::size_of::<KeyValue>();
-        table_size * mem::size_of::<KeyValue>()
-    };
+    let compute_table_size = |num_bits: usize| (1 << num_bits) * mem::size_of::<KeyValue>();
     let table_num_bits: usize = (1..)
         .into_iter()
         .take_while(|num_bits: &usize| compute_table_size(*num_bits) < table_size_limit)
         .last()
-        .expect(&format!("Per thread memory is too small: {}", per_thread_memory_budget));
+        .expect(&format!(
+            "Per thread memory is too small: {}",
+            per_thread_memory_budget
+        ));
     let table_size = compute_table_size(table_num_bits);
     let heap_size = per_thread_memory_budget - table_size;
     (heap_size, table_num_bits)
 }
 
-
 /// `KeyValue` is the item stored in the hash table.
 /// The key is actually a `BytesRef` object stored in an external heap.
 /// The `value_addr` also points to an address in the heap.
@@ -85,7 +82,6 @@ pub(crate) fn split_memory(per_thread_memory_budget: usize) -> (usize, usize) {
 /// For this reason, the (start, stop) information is actually redundant
 /// and can be simplified in the future
 #[derive(Copy, Clone, Default)]
-#[repr(packed)]
 struct KeyValue {
     key_value_addr: BytesRef,
     hash: u32,
@@ -97,7 +93,6 @@ impl KeyValue {
     }
 }
 
-
 /// Customized `HashMap` with string keys
 ///
 /// This `HashMap` takes String as keys. Keys are
@@ -107,14 +102,13 @@ impl KeyValue {
 /// the computation of the hash of the key twice,
 /// or copying the key as long as there is no insert.
 ///
-pub struct HashMap<'a> {
+pub struct TermHashMap<'a> {
     table: Box<[KeyValue]>,
     heap: &'a Heap,
     mask: usize,
     occupied: Vec<usize>,
 }
 
-
 struct QuadraticProbing {
     hash: usize,
     i: usize,
@@ -124,9 +118,9 @@ struct QuadraticProbing {
 impl QuadraticProbing {
     fn compute(hash: usize, mask: usize) -> QuadraticProbing {
         QuadraticProbing {
-            hash: hash,
+            hash,
             i: 0,
-            mask: mask,
+            mask,
         }
     }
 
@@ -137,14 +131,13 @@ impl QuadraticProbing {
     }
 }
 
-
-impl<'a> HashMap<'a> {
-    pub fn new(num_bucket_power_of_2: usize, heap: &'a Heap) -> HashMap<'a> {
+impl<'a> TermHashMap<'a> {
+    pub fn new(num_bucket_power_of_2: usize, heap: &'a Heap) -> TermHashMap<'a> {
         let table_size = 1 << num_bucket_power_of_2;
         let table: Vec<KeyValue> = iter::repeat(KeyValue::default()).take(table_size).collect();
-        HashMap {
+        TermHashMap {
             table: table.into_boxed_slice(),
-            heap: heap,
+            heap,
             mask: table_size - 1,
             occupied: Vec::with_capacity(table_size / 2),
         }
@@ -165,26 +158,25 @@ impl<'a> HashMap<'a> {
         (key_bytes, expull_addr)
     }
 
-    pub fn set_bucket(&mut self, hash: u32, key_bytes_ref: BytesRef, bucket: usize) {
+    pub fn set_bucket(&mut self, hash: u32, key_value_addr: BytesRef, bucket: usize) {
         self.occupied.push(bucket);
         self.table[bucket] = KeyValue {
-            key_value_addr: key_bytes_ref,
-            hash: hash,
+            key_value_addr, hash
         };
     }
 
-    pub fn iter<'b: 'a>(&'b self) -> impl Iterator<Item = (&'a [u8], u32)> + 'b {
-        self.occupied
-            .iter()
-            .cloned()
-            .map(move |bucket: usize| {
-                     let kv = self.table[bucket];
-                     self.get_key_value(kv.key_value_addr)
-                 })
+    pub fn iter<'b: 'a>(&'b self) -> impl Iterator<Item = (&'a [u8], u32, UnorderedTermId)> + 'b {
+        self.occupied.iter().cloned().map(move |bucket: usize| {
+            let kv = self.table[bucket];
+            let (key, offset) = self.get_key_value(kv.key_value_addr);
+            (key, offset, bucket as UnorderedTermId)
+        })
     }
 
-
-    pub fn get_or_create<S: AsRef<[u8]>, V: HeapAllocable>(&mut self, key: S) -> &mut V {
+    pub fn get_or_create<S: AsRef<[u8]>, V: HeapAllocable>(
+        &mut self,
+        key: S,
+    ) -> (UnorderedTermId, &mut V) {
         let key_bytes: &[u8] = key.as_ref();
         let hash = murmurhash2::murmurhash2(key.as_ref());
         let mut probe = self.probe(hash);
@@ -196,18 +188,20 @@ impl<'a> HashMap<'a> {
                 let (addr, val): (u32, &mut V) = self.heap.allocate_object();
                 assert_eq!(addr, key_bytes_ref.addr() + 2 + key_bytes.len() as u32);
                 self.set_bucket(hash, key_bytes_ref, bucket);
-                return val;
+                return (bucket as UnorderedTermId, val);
             } else if kv.hash == hash {
                 let (stored_key, expull_addr): (&[u8], u32) = self.get_key_value(kv.key_value_addr);
                 if stored_key == key_bytes {
-                    return self.heap.get_mut_ref(expull_addr);
+                    return (
+                        bucket as UnorderedTermId,
+                        self.heap.get_mut_ref(expull_addr),
+                    );
                 }
             }
         }
     }
 }
 
-
 #[cfg(test)]
 mod tests {
 
@@ -218,7 +212,6 @@ mod tests {
     use std::collections::HashSet;
     use super::split_memory;
 
-
     struct TestValue {
         val: u32,
         _addr: u32,
@@ -235,42 +228,41 @@ mod tests {
 
     #[test]
     fn test_hashmap_size() {
-        assert_eq!(split_memory(100_000), (67232, 9));
-        assert_eq!(split_memory(1_000_000), (737856, 12));
-        assert_eq!(split_memory(10_000_000), (7902848, 15));
+        assert_eq!(split_memory(100_000), (67232, 12));
+        assert_eq!(split_memory(1_000_000), (737856, 15));
+        assert_eq!(split_memory(10_000_000), (7902848, 18));
     }
 
-
     #[test]
     fn test_hash_map() {
         let heap = Heap::with_capacity(2_000_000);
-        let mut hash_map: HashMap = HashMap::new(18, &heap);
+        let mut hash_map: TermHashMap = TermHashMap::new(18, &heap);
         {
-            let v: &mut TestValue = hash_map.get_or_create("abc");
+            let v: &mut TestValue = hash_map.get_or_create("abc").1;
             assert_eq!(v.val, 0u32);
             v.val = 3u32;
         }
         {
-            let v: &mut TestValue = hash_map.get_or_create("abcd");
+            let v: &mut TestValue = hash_map.get_or_create("abcd").1;
             assert_eq!(v.val, 0u32);
             v.val = 4u32;
         }
         {
-            let v: &mut TestValue = hash_map.get_or_create("abc");
+            let v: &mut TestValue = hash_map.get_or_create("abc").1;
             assert_eq!(v.val, 3u32);
         }
         {
-            let v: &mut TestValue = hash_map.get_or_create("abcd");
+            let v: &mut TestValue = hash_map.get_or_create("abcd").1;
             assert_eq!(v.val, 4u32);
         }
         let mut iter_values = hash_map.iter();
         {
-            let (_, addr) = iter_values.next().unwrap();
+            let (_, addr, _) = iter_values.next().unwrap();
             let val: &TestValue = heap.get_ref(addr);
             assert_eq!(val.val, 3u32);
         }
         {
-            let (_, addr) = iter_values.next().unwrap();
+            let (_, addr, _) = iter_values.next().unwrap();
             let val: &TestValue = heap.get_ref(addr);
             assert_eq!(val.val, 4u32);
         }
@@ -282,8 +274,10 @@ mod tests {
         let s1 = "abcdef";
         let s2 = "abcdeg";
         for i in 0..5 {
-            assert_eq!(murmurhash2(&s1[i..5].as_bytes()),
-                       murmurhash2(&s2[i..5].as_bytes()));
+            assert_eq!(
+                murmurhash2(&s1[i..5].as_bytes()),
+                murmurhash2(&s2[i..5].as_bytes())
+            );
         }
     }
 
@@ -303,14 +297,13 @@ mod tests {
         let keys: Vec<&'static str> =
             vec!["wer qwe qwe qwe ", "werbq weqweqwe2 ", "weraq weqweqwe3 "];
         b.iter(|| {
-                   keys.iter()
-                       .map(|&s| s.as_bytes())
-                       .map(murmurhash2::murmurhash2)
-                       .map(|h| h as u64)
-                       .last()
-                       .unwrap()
-               });
+            keys.iter()
+                .map(|&s| s.as_bytes())
+                .map(murmurhash2::murmurhash2)
+                .map(|h| h as u64)
+                .last()
+                .unwrap()
+        });
     }
 
-
 }

src/datastruct/stacker/heap.rs

@@ -1,7 +1,7 @@
 use std::cell::UnsafeCell;
 use std::mem;
 use std::ptr;
-use byteorder::{NativeEndian, ByteOrder};
+use byteorder::{ByteOrder, NativeEndian};
 
 /// `BytesRef` refers to a slice in tantivy's custom `Heap`.
 ///
@@ -40,7 +40,9 @@ pub struct Heap {
 impl Heap {
     /// Creates a new heap with a given capacity
     pub fn with_capacity(num_bytes: usize) -> Heap {
-        Heap { inner: UnsafeCell::new(InnerHeap::with_capacity(num_bytes)) }
+        Heap {
+            inner: UnsafeCell::new(InnerHeap::with_capacity(num_bytes)),
+        }
     }
 
     fn inner(&self) -> &mut InnerHeap {
@@ -102,7 +104,6 @@ impl Heap {
     }
 }
 
-
 struct InnerHeap {
     buffer: Vec<u8>,
     buffer_len: u32,
@@ -110,12 +111,11 @@ struct InnerHeap {
     next_heap: Option<Box<InnerHeap>>,
 }
 
-
 impl InnerHeap {
     pub fn with_capacity(num_bytes: usize) -> InnerHeap {
         let buffer: Vec<u8> = vec![0u8; num_bytes];
         InnerHeap {
-            buffer: buffer,
+            buffer,
             buffer_len: num_bytes as u32,
             next_heap: None,
             used: 0u32,
@@ -144,7 +144,10 @@ impl InnerHeap {
             addr
         } else {
             if self.next_heap.is_none() {
-                info!(r#"Exceeded heap size. The segment will be committed right after indexing this document."#,);
+                info!(
+                    r#"Exceeded heap size. The segment will be committed right
+                         after indexing this document."#,
+                );
                 self.next_heap = Some(Box::new(InnerHeap::with_capacity(self.buffer_len as usize)));
             }
             self.next_heap.as_mut().unwrap().allocate_space(num_bytes) + self.buffer_len

src/datastruct/stacker/mod.rs

@@ -4,7 +4,7 @@ mod expull;
 
 pub use self::heap::{Heap, HeapAllocable};
 pub use self::expull::ExpUnrolledLinkedList;
-pub use self::hashmap::HashMap;
+pub use self::hashmap::TermHashMap;
 
 #[test]
 fn test_unrolled_linked_list() {
@@ -16,15 +16,15 @@ fn test_unrolled_linked_list() {
         ks.push(2);
         ks.push(3);
         for k in (1..5).map(|k| k * 100) {
-            let mut hashmap: HashMap = HashMap::new(10, &heap);
+            let mut hashmap: TermHashMap = TermHashMap::new(10, &heap);
             for j in 0..k {
                 for i in 0..500 {
-                    let v: &mut ExpUnrolledLinkedList = hashmap.get_or_create(i.to_string());
+                    let v: &mut ExpUnrolledLinkedList = hashmap.get_or_create(i.to_string()).1;
                     v.push(i * j, &heap);
                 }
             }
             let mut map_addr: collections::HashMap<Vec<u8>, u32> = collections::HashMap::new();
-            for (key, addr) in hashmap.iter() {
+            for (key, addr, _) in hashmap.iter() {
                 map_addr.insert(Vec::from(key), addr);
             }
 
@@ -39,6 +39,5 @@ fn test_unrolled_linked_list() {
                 assert!(!it.next().is_some());
             }
         }
-
     }
 }

src/directory/directory.rs

@@ -1,7 +1,7 @@
 use std::marker::Send;
 use std::fmt;
 use std::path::Path;
-use directory::error::{OpenReadError, DeleteError, OpenWriteError};
+use directory::error::{DeleteError, OpenReadError, OpenWriteError};
 use directory::{ReadOnlySource, WritePtr};
 use std::result;
 use std::io;

src/directory/error.rs

@@ -33,17 +33,14 @@ impl IOError {
     pub(crate) fn with_path(path: PathBuf, err: io::Error) -> Self {
         IOError {
             path: Some(path),
-            err: err,
+            err,
         }
     }
 }
 
 impl From<io::Error> for IOError {
     fn from(err: io::Error) -> IOError {
-        IOError {
-            path: None,
-            err: err,
-        }
+        IOError { path: None, err }
     }
 }
 
@@ -102,11 +99,11 @@ impl fmt::Display for OpenWriteError {
             OpenWriteError::FileAlreadyExists(ref path) => {
                 write!(f, "the file '{:?}' already exists", path)
             }
-            OpenWriteError::IOError(ref err) => {
-                write!(f,
-                       "an io error occurred while opening a file for writing: '{}'",
-                       err)
-            }
+            OpenWriteError::IOError(ref err) => write!(
+                f,
+                "an io error occurred while opening a file for writing: '{}'",
+                err
+            ),
         }
     }
 }
@@ -146,11 +143,11 @@ impl fmt::Display for OpenReadError {
             OpenReadError::FileDoesNotExist(ref path) => {
                 write!(f, "the file '{:?}' does not exist", path)
             }
-            OpenReadError::IOError(ref err) => {
-                write!(f,
-                       "an io error occurred while opening a file for reading: '{}'",
-                       err)
-            }
+            OpenReadError::IOError(ref err) => write!(
+                f,
+                "an io error occurred while opening a file for reading: '{}'",
+                err
+            ),
         }
     }
 }
@@ -210,8 +207,7 @@ impl StdError for DeleteError {
 
     fn cause(&self) -> Option<&StdError> {
         match *self {
-            DeleteError::FileDoesNotExist(_) |
-            DeleteError::FileProtected(_) => None,
+            DeleteError::FileDoesNotExist(_) | DeleteError::FileProtected(_) => None,
             DeleteError::IOError(ref err) => Some(err),
         }
     }

src/directory/managed_directory.rs

@@ -1,6 +1,6 @@
 use std::path::{Path, PathBuf};
 use serde_json;
-use directory::error::{IOError, OpenReadError, DeleteError, OpenWriteError};
+use directory::error::{DeleteError, IOError, OpenReadError, OpenWriteError};
 use directory::{ReadOnlySource, WritePtr};
 use std::result;
 use std::io;
@@ -12,7 +12,7 @@ use std::io::Write;
 use core::MANAGED_FILEPATH;
 use std::collections::HashMap;
 use std::fmt;
-use error::{Result, ErrorKind, ResultExt};
+use error::{ErrorKind, Result, ResultExt};
 
 /// Wrapper of directories that keeps track of files created by Tantivy.
 ///
@@ -35,7 +35,6 @@ struct MetaInformation {
     protected_files: HashMap<PathBuf, usize>,
 }
 
-
 /// A `FileProtection` prevents the garbage collection of a file.
 ///
 /// See `ManagedDirectory.protect_file_from_delete`.
@@ -68,9 +67,10 @@ impl Drop for FileProtection {
 
 /// Saves the file containing the list of existing files
 /// that were created by tantivy.
-fn save_managed_paths(directory: &mut Directory,
-                      wlock: &RwLockWriteGuard<MetaInformation>)
-                      -> io::Result<()> {
+fn save_managed_paths(
+    directory: &mut Directory,
+    wlock: &RwLockWriteGuard<MetaInformation>,
+) -> io::Result<()> {
     let mut w = serde_json::to_vec(&wlock.managed_paths)?;
     write!(&mut w, "\n")?;
     directory.atomic_write(&MANAGED_FILEPATH, &w[..])?;
@@ -83,24 +83,20 @@ impl ManagedDirectory {
         match directory.atomic_read(&MANAGED_FILEPATH) {
             Ok(data) => {
                 let managed_files_json = String::from_utf8_lossy(&data);
-                let managed_files: HashSet<PathBuf> =
-                    serde_json::from_str(&managed_files_json)
-                        .chain_err(|| ErrorKind::CorruptedFile(MANAGED_FILEPATH.clone()))?;
+                let managed_files: HashSet<PathBuf> = serde_json::from_str(&managed_files_json)
+                    .chain_err(|| ErrorKind::CorruptedFile(MANAGED_FILEPATH.clone()))?;
                 Ok(ManagedDirectory {
-                       directory: box directory,
-                       meta_informations: Arc::new(RwLock::new(MetaInformation {
-                                                                   managed_paths: managed_files,
-                                                                   protected_files:
-                                                                       HashMap::default(),
-                                                               })),
-                   })
-            }
-            Err(OpenReadError::FileDoesNotExist(_)) => {
-                Ok(ManagedDirectory {
-                       directory: box directory,
-                       meta_informations: Arc::default(),
-                   })
+                    directory: box directory,
+                    meta_informations: Arc::new(RwLock::new(MetaInformation {
+                        managed_paths: managed_files,
+                        protected_files: HashMap::default(),
+                    })),
+                })
             }
+            Err(OpenReadError::FileDoesNotExist(_)) => Ok(ManagedDirectory {
+                directory: box directory,
+                meta_informations: Arc::default(),
+            }),
             Err(OpenReadError::IOError(e)) => Err(From::from(e)),
         }
     }
@@ -116,15 +112,25 @@ impl ManagedDirectory {
     /// If a file cannot be deleted (for permission reasons for instance)
     /// an error is simply logged, and the file remains in the list of managed
     /// files.
-    pub fn garbage_collect(&mut self, living_files: HashSet<PathBuf>) {
+    pub fn garbage_collect<L: FnOnce() -> HashSet<PathBuf>>(&mut self, get_living_files: L) {
         info!("Garbage collect");
         let mut files_to_delete = vec![];
         {
             // releasing the lock as .delete() will use it too.
-            let meta_informations_rlock =
-                self.meta_informations
-                    .read()
-                    .expect("Managed directory rlock poisoned in garbage collect.");
+            let meta_informations_rlock = self.meta_informations
+                .read()
+                .expect("Managed directory rlock poisoned in garbage collect.");
+
+            // It is crucial to get the living files after acquiring the
+            // read lock of meta informations. That way, we
+            // avoid the following scenario.
+            //
+            // 1) we get the list of living files.
+            // 2) someone creates a new file.
+            // 3) we start garbage collection and remove this file
+            // even though it is a living file.
+            let living_files = get_living_files();
+
             for managed_path in &meta_informations_rlock.managed_paths {
                 if !living_files.contains(managed_path) {
                     files_to_delete.push(managed_path.clone());
@@ -156,13 +162,11 @@ impl ManagedDirectory {
                                 // this is expected.
                             }
                         }
-
                     }
                 }
             }
         }
 
-
         if !deleted_files.is_empty() {
             // update the list of managed files by removing
             // the file that were removed.
@@ -179,10 +183,8 @@ impl ManagedDirectory {
                 error!("Failed to save the list of managed files.");
             }
         }
-
     }
 
-
     /// Protects a file from being garbage collected.
     ///
     /// The method returns a `FileProtection` object.
@@ -195,9 +197,9 @@ impl ManagedDirectory {
                 .write()
                 .expect("Managed file lock poisoned on protect");
             *meta_informations_wlock
-                 .protected_files
-                 .entry(pathbuf.clone())
-                 .or_insert(0) += 1;
+                .protected_files
+                .entry(pathbuf.clone())
+                .or_insert(0) += 1;
         }
         FileProtection {
             directory: self.clone(),
@@ -271,14 +273,11 @@ impl Clone for ManagedDirectory {
     fn clone(&self) -> ManagedDirectory {
         ManagedDirectory {
             directory: self.directory.box_clone(),
-            meta_informations: self.meta_informations.clone(),
+            meta_informations: Arc::clone(&self.meta_informations),
         }
     }
 }
 
-
-
-
 #[cfg(test)]
 mod tests {
 
@@ -316,7 +315,7 @@ mod tests {
             {
                 let living_files: HashSet<PathBuf> =
                     [TEST_PATH1.to_owned()].into_iter().cloned().collect();
-                managed_directory.garbage_collect(living_files);
+                managed_directory.garbage_collect(|| living_files);
             }
             {
                 assert!(managed_directory.exists(*TEST_PATH1));
@@ -332,7 +331,7 @@ mod tests {
             }
             {
                 let living_files: HashSet<PathBuf> = HashSet::new();
-                managed_directory.garbage_collect(living_files);
+                managed_directory.garbage_collect(|| living_files);
             }
             {
                 assert!(!managed_directory.exists(*TEST_PATH1));
@@ -355,7 +354,7 @@ mod tests {
         assert!(managed_directory.exists(*TEST_PATH1));
 
         let _mmap_read = managed_directory.open_read(*TEST_PATH1).unwrap();
-        managed_directory.garbage_collect(living_files.clone());
+        managed_directory.garbage_collect(|| living_files.clone());
         if cfg!(target_os = "windows") {
             // On Windows, gc should try and fail the file as it is mmapped.
             assert!(managed_directory.exists(*TEST_PATH1));
@@ -363,15 +362,13 @@ mod tests {
             drop(_mmap_read);
             // The file should still be in the list of managed file and
             // eventually be deleted once mmap is released.
-            managed_directory.garbage_collect(living_files);
+            managed_directory.garbage_collect(|| living_files);
             assert!(!managed_directory.exists(*TEST_PATH1));
         } else {
             assert!(!managed_directory.exists(*TEST_PATH1));
         }
-
     }
 
-
     #[test]
     fn test_managed_directory_protect() {
         let tempdir = TempDir::new("index").unwrap();
@@ -387,14 +384,12 @@ mod tests {
 
         {
             let _file_protection = managed_directory.protect_file_from_delete(*TEST_PATH1);
-            managed_directory.garbage_collect(living_files.clone());
+            managed_directory.garbage_collect(|| living_files.clone());
             assert!(managed_directory.exists(*TEST_PATH1));
         }
 
-        managed_directory.garbage_collect(living_files.clone());
+        managed_directory.garbage_collect(|| living_files.clone());
         assert!(!managed_directory.exists(*TEST_PATH1));
-
-
     }
 
 }

src/directory/mmap_directory.rs

@@ -1,12 +1,11 @@
 use atomicwrites;
 use common::make_io_err;
 use directory::Directory;
-use directory::error::{IOError, OpenWriteError, OpenReadError, DeleteError, OpenDirectoryError};
+use directory::error::{DeleteError, IOError, OpenDirectoryError, OpenReadError, OpenWriteError};
 use directory::ReadOnlySource;
 use directory::shared_vec_slice::SharedVecSlice;
 use directory::WritePtr;
 use fst::raw::MmapReadOnly;
-use memmap::{Mmap, Protection};
 use std::collections::hash_map::Entry as HashMapEntry;
 use std::collections::HashMap;
 use std::convert::From;
@@ -15,50 +14,47 @@ use std::fs::{self, File};
 use std::fs::OpenOptions;
 use std::io::{self, Seek, SeekFrom};
 use std::io::{BufWriter, Read, Write};
-use std::mem;
 use std::path::{Path, PathBuf};
 use std::result;
 use std::sync::Arc;
 use std::sync::RwLock;
-use std::sync::Weak;
 use tempdir::TempDir;
 
-fn open_mmap(full_path: &PathBuf) -> result::Result<Option<Arc<Mmap>>, OpenReadError> {
-    let file = File::open(&full_path)
-        .map_err(|e| if e.kind() == io::ErrorKind::NotFound {
-                     OpenReadError::FileDoesNotExist(full_path.clone())
-                 } else {
-                     OpenReadError::IOError(IOError::with_path(full_path.to_owned(), e))
-                 })?;
+/// Returns None iff the file exists, can be read, but is empty (and hence
+/// cannot be mmapped).
+///
+fn open_mmap(full_path: &Path) -> result::Result<Option<MmapReadOnly>, OpenReadError> {
+    let file = File::open(full_path).map_err(|e| {
+        if e.kind() == io::ErrorKind::NotFound {
+            OpenReadError::FileDoesNotExist(full_path.to_owned())
+        } else {
+            OpenReadError::IOError(IOError::with_path(full_path.to_owned(), e))
+        }
+    })?;
 
     let meta_data = file.metadata()
         .map_err(|e| IOError::with_path(full_path.to_owned(), e))?;
     if meta_data.len() == 0 {
         // if the file size is 0, it will not be possible
-        // to mmap the file, so we return an anonymous mmap_cache
+        // to mmap the file, so we return None
         // instead.
         return Ok(None);
     }
-    match Mmap::open(&file, Protection::Read) {
-        Ok(mmap) => Ok(Some(Arc::new(mmap))),
-        Err(e) => Err(IOError::with_path(full_path.to_owned(), e))?,
-    }
-
+    MmapReadOnly::open(&file)
+        .map(Some)
+        .map_err(|e| From::from(IOError::with_path(full_path.to_owned(), e)))
 }
 
-#[derive(Default,Clone,Debug,Serialize,Deserialize)]
+#[derive(Default, Clone, Debug, Serialize, Deserialize)]
 pub struct CacheCounters {
     // Number of time the cache prevents to call `mmap`
     pub hit: usize,
     // Number of time tantivy had to call `mmap`
     // as no entry was in the cache.
-    pub miss_empty: usize,
-    // Number of time tantivy had to call `mmap`
-    // as the entry in the cache was evinced.
-    pub miss_weak: usize,
+    pub miss: usize,
 }
 
-#[derive(Clone,Debug,Serialize,Deserialize)]
+#[derive(Clone, Debug, Serialize, Deserialize)]
 pub struct CacheInfo {
     pub counters: CacheCounters,
     pub mmapped: Vec<PathBuf>,
@@ -66,39 +62,25 @@ pub struct CacheInfo {
 
 struct MmapCache {
     counters: CacheCounters,
-    cache: HashMap<PathBuf, Weak<Mmap>>,
-    purge_weak_limit: usize,
+    cache: HashMap<PathBuf, MmapReadOnly>,
 }
 
-const STARTING_PURGE_WEAK_LIMIT: usize = 1_000;
-
 impl Default for MmapCache {
     fn default() -> MmapCache {
         MmapCache {
             counters: CacheCounters::default(),
             cache: HashMap::new(),
-            purge_weak_limit: STARTING_PURGE_WEAK_LIMIT,
         }
     }
 }
 
-
 impl MmapCache {
-    fn cleanup(&mut self) {
-        let previous_cache_size = self.cache.len();
-        let mut new_cache = HashMap::new();
-        mem::swap(&mut new_cache, &mut self.cache);
-        self.cache = new_cache
-            .into_iter()
-            .filter(|&(_, ref weak_ref)| weak_ref.upgrade().is_some())
-            .collect();
-        if self.cache.len() == previous_cache_size {
-            self.purge_weak_limit *= 2;
-        }
+    /// Removes a `MmapReadOnly` entry from the mmap cache.
+    fn discard_from_cache(&mut self, full_path: &Path) -> bool {
+        self.cache.remove(full_path).is_some()
     }
 
     fn get_info(&mut self) -> CacheInfo {
-        self.cleanup();
         let paths: Vec<PathBuf> = self.cache.keys().cloned().collect();
         CacheInfo {
             counters: self.counters.clone(),
@@ -106,38 +88,23 @@ impl MmapCache {
         }
     }
 
-    fn get_mmap(&mut self, full_path: PathBuf) -> Result<Option<Arc<Mmap>>, OpenReadError> {
-        // if we exceed this limit, then we go through the weak
-        // and remove those that are obsolete.
-        if self.cache.len() > self.purge_weak_limit {
-            self.cleanup();
-        }
-        Ok(match self.cache.entry(full_path.clone()) {
-               HashMapEntry::Occupied(mut occupied_entry) => {
-                   if let Some(mmap_arc) = occupied_entry.get().upgrade() {
-                       self.counters.hit += 1;
-                       Some(mmap_arc.clone())
-                   } else {
-                       // The entry exists but the weak ref has been destroyed.
-                       self.counters.miss_weak += 1;
-                       if let Some(mmap_arc) = open_mmap(&full_path)? {
-                           occupied_entry.insert(Arc::downgrade(&mmap_arc));
-                           Some(mmap_arc)
-                       } else {
-                           None
-                       }
-                   }
-               }
-               HashMapEntry::Vacant(vacant_entry) => {
-                   self.counters.miss_empty += 1;
-                   if let Some(mmap_arc) = open_mmap(&full_path)? {
-                       vacant_entry.insert(Arc::downgrade(&mmap_arc));
-                       Some(mmap_arc)
-                   } else {
-                       None
-                   }
-               }
-           })
+    fn get_mmap(&mut self, full_path: &Path) -> Result<Option<MmapReadOnly>, OpenReadError> {
+        Ok(match self.cache.entry(full_path.to_owned()) {
+            HashMapEntry::Occupied(occupied_entry) => {
+                let mmap = occupied_entry.get();
+                self.counters.hit += 1;
+                Some(mmap.clone())
+            }
+            HashMapEntry::Vacant(vacant_entry) => {
+                self.counters.miss += 1;
+                if let Some(mmap) = open_mmap(full_path)? {
+                    vacant_entry.insert(mmap.clone());
+                    Some(mmap)
+                } else {
+                    None
+                }
+            }
+        })
     }
 }
 
@@ -164,10 +131,10 @@ impl MmapDirectory {
     /// This is mostly useful to test the MmapDirectory itself.
     /// For your unit tests, prefer the RAMDirectory.
     pub fn create_from_tempdir() -> io::Result<MmapDirectory> {
-        let tempdir = try!(TempDir::new("index"));
+        let tempdir = TempDir::new("index")?;
         let tempdir_path = PathBuf::from(tempdir.path());
         let directory = MmapDirectory {
-            root_path: PathBuf::from(tempdir_path),
+            root_path: tempdir_path,
             mmap_cache: Arc::new(RwLock::new(MmapCache::default())),
             _temp_directory: Arc::new(Some(tempdir)),
         };
@@ -178,17 +145,22 @@ impl MmapDirectory {
     ///
     /// Returns an error if the `directory_path` does not
     /// exist or if it is not a directory.
-    pub fn open(directory_path: &Path) -> Result<MmapDirectory, OpenDirectoryError> {
+    pub fn open<P: AsRef<Path>>(directory_path: P) -> Result<MmapDirectory, OpenDirectoryError> {
+        let directory_path: &Path = directory_path.as_ref();
         if !directory_path.exists() {
-            Err(OpenDirectoryError::DoesNotExist(PathBuf::from(directory_path)))
+            Err(OpenDirectoryError::DoesNotExist(PathBuf::from(
+                directory_path,
+            )))
         } else if !directory_path.is_dir() {
-            Err(OpenDirectoryError::NotADirectory(PathBuf::from(directory_path)))
+            Err(OpenDirectoryError::NotADirectory(PathBuf::from(
+                directory_path,
+            )))
         } else {
             Ok(MmapDirectory {
-                   root_path: PathBuf::from(directory_path),
-                   mmap_cache: Arc::new(RwLock::new(MmapCache::default())),
-                   _temp_directory: Arc::new(None),
-               })
+                root_path: PathBuf::from(directory_path),
+                mmap_cache: Arc::new(RwLock::new(MmapCache::default())),
+                _temp_directory: Arc::new(None),
+            })
         }
     }
 
@@ -220,10 +192,11 @@ impl MmapDirectory {
                 .custom_flags(winbase::FILE_FLAG_BACKUP_SEMANTICS);
         }
 
-        let fd = try!(open_opts.open(&self.root_path));
-        try!(fd.sync_all());
+        let fd = open_opts.open(&self.root_path)?;
+        fd.sync_all()?;
         Ok(())
     }
+
     /// Returns some statistical information
     /// about the Mmap cache.
     ///
@@ -253,7 +226,7 @@ impl Write for SafeFileWriter {
     }
 
     fn flush(&mut self) -> io::Result<()> {
-        try!(self.0.flush());
+        self.0.flush()?;
         self.0.sync_all()
     }
 }
@@ -264,26 +237,24 @@ impl Seek for SafeFileWriter {
     }
 }
 
-
 impl Directory for MmapDirectory {
     fn open_read(&self, path: &Path) -> result::Result<ReadOnlySource, OpenReadError> {
         debug!("Open Read {:?}", path);
         let full_path = self.resolve_path(path);
 
-        let mut mmap_cache = self.mmap_cache
-            .write()
-            .map_err(|_| {
-                         let msg = format!("Failed to acquired write lock \
-                                            on mmap cache while reading {:?}",
-                                           path);
-                         IOError::with_path(path.to_owned(), make_io_err(msg))
-                     })?;
+        let mut mmap_cache = self.mmap_cache.write().map_err(|_| {
+            let msg = format!(
+                "Failed to acquired write lock \
+                 on mmap cache while reading {:?}",
+                path
+            );
+            IOError::with_path(path.to_owned(), make_io_err(msg))
+        })?;
 
         Ok(mmap_cache
-               .get_mmap(full_path)?
-               .map(MmapReadOnly::from)
-               .map(ReadOnlySource::Mmap)
-               .unwrap_or_else(|| ReadOnlySource::Anonymous(SharedVecSlice::empty())))
+            .get_mmap(&full_path)?
+            .map(ReadOnlySource::Mmap)
+            .unwrap_or_else(|| ReadOnlySource::Anonymous(SharedVecSlice::empty())))
     }
 
     fn open_write(&mut self, path: &Path) -> Result<WritePtr, OpenWriteError> {
@@ -295,12 +266,13 @@ impl Directory for MmapDirectory {
             .create_new(true)
             .open(full_path);
 
-        let mut file = open_res
-            .map_err(|err| if err.kind() == io::ErrorKind::AlreadyExists {
-                         OpenWriteError::FileAlreadyExists(path.to_owned())
-                     } else {
-                         IOError::with_path(path.to_owned(), err).into()
-                     })?;
+        let mut file = open_res.map_err(|err| {
+            if err.kind() == io::ErrorKind::AlreadyExists {
+                OpenWriteError::FileAlreadyExists(path.to_owned())
+            } else {
+                IOError::with_path(path.to_owned(), err).into()
+            }
+        })?;
 
         // making sure the file is created.
         file.flush()
@@ -315,26 +287,28 @@ impl Directory for MmapDirectory {
         Ok(BufWriter::new(Box::new(writer)))
     }
 
+    /// Any entry associated to the path in the mmap will be
+    /// removed before the file is deleted.
     fn delete(&self, path: &Path) -> result::Result<(), DeleteError> {
         debug!("Deleting file {:?}", path);
         let full_path = self.resolve_path(path);
-        let mut mmap_cache = self.mmap_cache
-            .write()
-            .map_err(|_| {
-                         let msg = format!("Failed to acquired write lock \
-                                            on mmap cache while deleting {:?}",
-                                           path);
-                         IOError::with_path(path.to_owned(), make_io_err(msg))
-                     })?;
+        let mut mmap_cache = self.mmap_cache.write().map_err(|_| {
+            let msg = format!(
+                "Failed to acquired write lock \
+                 on mmap cache while deleting {:?}",
+                path
+            );
+            IOError::with_path(path.to_owned(), make_io_err(msg))
+        })?;
+        mmap_cache.discard_from_cache(path);
+
         // Removing the entry in the MMap cache.
         // The munmap will appear on Drop,
         // when the last reference is gone.
         mmap_cache.cache.remove(&full_path);
         match fs::remove_file(&full_path) {
-            Ok(_) => {
-                self.sync_directory()
-                    .map_err(|e| IOError::with_path(path.to_owned(), e).into())
-            }
+            Ok(_) => self.sync_directory()
+                .map_err(|e| IOError::with_path(path.to_owned(), e).into()),
             Err(e) => {
                 if e.kind() == io::ErrorKind::NotFound {
                     Err(DeleteError::FileDoesNotExist(path.to_owned()))
@@ -367,14 +341,13 @@ impl Directory for MmapDirectory {
                 }
             }
         }
-
     }
 
     fn atomic_write(&mut self, path: &Path, data: &[u8]) -> io::Result<()> {
         debug!("Atomic Write {:?}", path);
         let full_path = self.resolve_path(path);
         let meta_file = atomicwrites::AtomicFile::new(full_path, atomicwrites::AllowOverwrite);
-        try!(meta_file.write(|f| f.write_all(data)));
+        meta_file.write(|f| f.write_all(data))?;
         Ok(())
     }
 
@@ -383,9 +356,6 @@ impl Directory for MmapDirectory {
     }
 }
 
-
-
-
 #[cfg(test)]
 mod tests {
 
@@ -417,7 +387,8 @@ mod tests {
         // here we test if the cache releases
         // mmaps correctly.
         let mut mmap_directory = MmapDirectory::create_from_tempdir().unwrap();
-        let paths: Vec<PathBuf> = (0..10)
+        let num_paths = 10;
+        let paths: Vec<PathBuf> = (0..num_paths)
             .map(|i| PathBuf::from(&*format!("file_{}", i)))
             .collect();
         {
@@ -428,52 +399,25 @@ mod tests {
             }
         }
         {
-            for path in &paths {
-                {
-                    let _r = mmap_directory.open_read(path).unwrap();
-                    assert_eq!(mmap_directory.get_cache_info().mmapped.len(), 1);
-                }
-                assert_eq!(mmap_directory.get_cache_info().mmapped.len(), 0);
-            }
-        }
-        assert_eq!(mmap_directory.get_cache_info().counters.miss_empty, 10);
-
-
-        {
-            // test weak miss
-            // the first pass create the weak refs.
-            for path in &paths {
-                let _r = mmap_directory.open_read(path).unwrap();
-            }
-            // ... the second hits the weak refs.
-            for path in &paths {
-                let _r = mmap_directory.open_read(path).unwrap();
-            }
-            let cache_info = mmap_directory.get_cache_info();
-            assert_eq!(cache_info.counters.miss_empty, 20);
-            assert_eq!(cache_info.counters.miss_weak, 10);
-        }
-
-        {
-            let mut saved_readmmaps = vec![];
-            // Keeps reference alive
             for (i, path) in paths.iter().enumerate() {
-                let r = mmap_directory.open_read(path).unwrap();
-                saved_readmmaps.push(r);
+                let _r = mmap_directory.open_read(path).unwrap();
                 assert_eq!(mmap_directory.get_cache_info().mmapped.len(), i + 1);
             }
-            let cache_info = mmap_directory.get_cache_info();
-            assert_eq!(cache_info.counters.miss_empty, 30);
-            assert_eq!(cache_info.counters.miss_weak, 10);
-            assert_eq!(cache_info.mmapped.len(), 10);
-
-            for saved_readmmap in saved_readmmaps {
-                assert_eq!(saved_readmmap.as_slice(), content);
+            for path in paths.iter() {
+                let _r = mmap_directory.open_read(path).unwrap();
+                assert_eq!(mmap_directory.get_cache_info().mmapped.len(), num_paths);
+            }
+            for (i, path) in paths.iter().enumerate() {
+                mmap_directory.delete(path).unwrap();
+                assert_eq!(
+                    mmap_directory.get_cache_info().mmapped.len(),
+                    num_paths - i - 1
+                );
             }
         }
-
+        assert_eq!(mmap_directory.get_cache_info().counters.hit, 10);
+        assert_eq!(mmap_directory.get_cache_info().counters.miss, 10);
         assert_eq!(mmap_directory.get_cache_info().mmapped.len(), 0);
-
     }
 
 }

src/directory/mod.rs

@@ -13,14 +13,15 @@ mod managed_directory;
 /// Errors specific to the directory module.
 pub mod error;
 
-use std::io::{Write, Seek};
+use std::io::{BufWriter, Seek, Write};
 
-use std::io::BufWriter;
 pub use self::read_only_source::ReadOnlySource;
 pub use self::directory::Directory;
 pub use self::ram_directory::RAMDirectory;
 pub use self::mmap_directory::MmapDirectory;
-pub use self::managed_directory::{ManagedDirectory, FileProtection};
+
+pub(crate) use self::read_only_source::SourceRead;
+pub(crate) use self::managed_directory::{FileProtection, ManagedDirectory};
 
 /// Synonym of Seek + Write
 pub trait SeekableWrite: Seek + Write {}
@@ -37,7 +38,7 @@ mod tests {
 
     use super::*;
     use std::path::Path;
-    use std::io::{Write, Seek, SeekFrom};
+    use std::io::{Seek, SeekFrom, Write};
 
     lazy_static! {
         static ref TEST_PATH: &'static Path = Path::new("some_path_for_test");
@@ -103,7 +104,6 @@ mod tests {
         {
             directory.open_write(*TEST_PATH).unwrap();
             assert!(directory.exists(*TEST_PATH));
-
         }
         {
             assert!(directory.open_write(*TEST_PATH).is_err());
@@ -116,9 +116,6 @@ mod tests {
             assert!(directory.open_read(*TEST_PATH).is_err());
             let _w = directory.open_write(*TEST_PATH).unwrap();
             assert!(directory.exists(*TEST_PATH));
-            if let Err(e) = directory.open_read(*TEST_PATH) {
-                println!("{:?}", e);
-            }
             assert!(directory.open_read(*TEST_PATH).is_ok());
             assert!(directory.delete(*TEST_PATH).is_ok());
         }

src/directory/ram_directory.rs

@@ -1,12 +1,12 @@
 use std::collections::HashMap;
 use std::fmt;
-use std::io::{self, BufWriter, Cursor, Write, Seek, SeekFrom};
+use std::io::{self, BufWriter, Cursor, Seek, SeekFrom, Write};
 use std::path::{Path, PathBuf};
 use std::result;
 use std::sync::{Arc, RwLock};
 use common::make_io_err;
 use directory::{Directory, ReadOnlySource};
-use directory::error::{IOError, OpenWriteError, OpenReadError, DeleteError};
+use directory::error::{DeleteError, IOError, OpenReadError, OpenWriteError};
 use directory::WritePtr;
 use super::shared_vec_slice::SharedVecSlice;
 
@@ -32,7 +32,7 @@ impl VecWriter {
         VecWriter {
             path: path_buf,
             data: Cursor::new(Vec::new()),
-            shared_directory: shared_directory,
+            shared_directory,
             is_flushed: true,
         }
     }
@@ -41,8 +41,10 @@ impl VecWriter {
 impl Drop for VecWriter {
     fn drop(&mut self) {
         if !self.is_flushed {
-            panic!("You forgot to flush {:?} before its writter got Drop. Do not rely on drop.",
-                   self.path)
+            panic!(
+                "You forgot to flush {:?} before its writter got Drop. Do not rely on drop.",
+                self.path
+            )
         }
     }
 }
@@ -56,14 +58,14 @@ impl Seek for VecWriter {
 impl Write for VecWriter {
     fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
         self.is_flushed = false;
-        try!(self.data.write_all(buf));
+        self.data.write_all(buf)?;
         Ok(buf.len())
     }
 
     fn flush(&mut self) -> io::Result<()> {
         self.is_flushed = true;
-        try!(self.shared_directory
-                 .write(self.path.clone(), self.data.get_ref()));
+        self.shared_directory
+            .write(self.path.clone(), self.data.get_ref())?;
         Ok(())
     }
 }
@@ -71,20 +73,18 @@ impl Write for VecWriter {
 #[derive(Clone)]
 struct InnerDirectory(Arc<RwLock<HashMap<PathBuf, Arc<Vec<u8>>>>>);
 
-
-
 impl InnerDirectory {
     fn new() -> InnerDirectory {
         InnerDirectory(Arc::new(RwLock::new(HashMap::new())))
     }
 
     fn write(&self, path: PathBuf, data: &[u8]) -> io::Result<bool> {
-        let mut map = try!(self.0
-                               .write()
-                               .map_err(|_| {
-            make_io_err(format!("Failed to lock the directory, when trying to write {:?}",
-                                path))
-        }));
+        let mut map = self.0.write().map_err(|_| {
+            make_io_err(format!(
+                "Failed to lock the directory, when trying to write {:?}",
+                path
+            ))
+        })?;
         let prev_value = map.insert(path, Arc::new(Vec::from(data)));
         Ok(prev_value.is_some())
     }
@@ -93,17 +93,20 @@ impl InnerDirectory {
         self.0
             .read()
             .map_err(|_| {
-                         let msg = format!("Failed to acquire read lock for the \
-                                            directory when trying to read {:?}",
-                                           path);
-                         let io_err = make_io_err(msg);
-                         OpenReadError::IOError(IOError::with_path(path.to_owned(), io_err))
-                     })
+                let msg = format!(
+                    "Failed to acquire read lock for the \
+                     directory when trying to read {:?}",
+                    path
+                );
+                let io_err = make_io_err(msg);
+                OpenReadError::IOError(IOError::with_path(path.to_owned(), io_err))
+            })
             .and_then(|readable_map| {
                 readable_map
                     .get(path)
                     .ok_or_else(|| OpenReadError::FileDoesNotExist(PathBuf::from(path)))
-                    .map(|data| ReadOnlySource::Anonymous(SharedVecSlice::new(data.clone())))
+                    .map(Arc::clone)
+                    .map(|data| ReadOnlySource::Anonymous(SharedVecSlice::new(data)))
             })
     }
 
@@ -111,16 +114,18 @@ impl InnerDirectory {
         self.0
             .write()
             .map_err(|_| {
-                         let msg = format!("Failed to acquire write lock for the \
-                                            directory when trying to delete {:?}",
-                                           path);
-                         let io_err = make_io_err(msg);
-                         DeleteError::IOError(IOError::with_path(path.to_owned(), io_err))
-                     })
+                let msg = format!(
+                    "Failed to acquire write lock for the \
+                     directory when trying to delete {:?}",
+                    path
+                );
+                let io_err = make_io_err(msg);
+                DeleteError::IOError(IOError::with_path(path.to_owned(), io_err))
+            })
             .and_then(|mut writable_map| match writable_map.remove(path) {
-                          Some(_) => Ok(()),
-                          None => Err(DeleteError::FileDoesNotExist(PathBuf::from(path))),
-                      })
+                Some(_) => Ok(()),
+                None => Err(DeleteError::FileDoesNotExist(PathBuf::from(path))),
+            })
     }
 
     fn exists(&self, path: &Path) -> bool {
@@ -137,7 +142,6 @@ impl fmt::Debug for RAMDirectory {
     }
 }
 
-
 /// A Directory storing everything in anonymous memory.
 ///
 /// It is mainly meant for unit testing.
@@ -151,7 +155,9 @@ pub struct RAMDirectory {
 impl RAMDirectory {
     /// Constructor
     pub fn create() -> RAMDirectory {
-        RAMDirectory { fs: InnerDirectory::new() }
+        RAMDirectory {
+            fs: InnerDirectory::new(),
+        }
     }
 }
 
@@ -180,7 +186,6 @@ impl Directory for RAMDirectory {
         self.fs.delete(path)
     }
 
-
     fn exists(&self, path: &Path) -> bool {
         self.fs.exists(path)
     }
@@ -193,9 +198,9 @@ impl Directory for RAMDirectory {
     fn atomic_write(&mut self, path: &Path, data: &[u8]) -> io::Result<()> {
         let path_buf = PathBuf::from(path);
         let mut vec_writer = VecWriter::new(path_buf.clone(), self.fs.clone());
-        try!(self.fs.write(path_buf, &Vec::new()));
-        try!(vec_writer.write_all(data));
-        try!(vec_writer.flush());
+        self.fs.write(path_buf, &Vec::new())?;
+        vec_writer.write_all(data)?;
+        vec_writer.flush()?;
         Ok(())
     }
 

src/directory/read_only_source.rs

@@ -2,7 +2,9 @@ use fst::raw::MmapReadOnly;
 use std::ops::Deref;
 use super::shared_vec_slice::SharedVecSlice;
 use common::HasLen;
-use stable_deref_trait::StableDeref;
+use std::slice;
+use std::io::{self, Read};
+use stable_deref_trait::{CloneStableDeref, StableDeref};
 
 /// Read object that represents files in tantivy.
 ///
@@ -18,6 +20,7 @@ pub enum ReadOnlySource {
 }
 
 unsafe impl StableDeref for ReadOnlySource {}
+unsafe impl CloneStableDeref for ReadOnlySource {}
 
 impl Deref for ReadOnlySource {
     type Target = [u8];
@@ -41,6 +44,14 @@ impl ReadOnlySource {
         }
     }
 
+    /// Splits into 2 `ReadOnlySource`, at the offset given
+    /// as an argument.
+    pub fn split(self, addr: usize) -> (ReadOnlySource, ReadOnlySource) {
+        let left = self.slice(0, addr);
+        let right = self.slice_from(addr);
+        (left, right)
+    }
+
     /// Creates a ReadOnlySource that is just a
     /// view over a slice of the data.
     ///
@@ -62,6 +73,23 @@ impl ReadOnlySource {
             }
         }
     }
+
+    /// Like `.slice(...)` but enforcing only the `from`
+    /// boundary.
+    ///
+    /// Equivalent to `.slice(from_offset, self.len())`
+    pub fn slice_from(&self, from_offset: usize) -> ReadOnlySource {
+        let len = self.len();
+        self.slice(from_offset, len)
+    }
+
+    /// Like `.slice(...)` but enforcing only the `to`
+    /// boundary.
+    ///
+    /// Equivalent to `.slice(0, to_offset)`
+    pub fn slice_to(&self, to_offset: usize) -> ReadOnlySource {
+        self.slice(0, to_offset)
+    }
 }
 
 impl HasLen for ReadOnlySource {
@@ -82,3 +110,41 @@ impl From<Vec<u8>> for ReadOnlySource {
         ReadOnlySource::Anonymous(shared_data)
     }
 }
+
+/// Acts as a owning cursor over the data backed up by a `ReadOnlySource`
+pub(crate) struct SourceRead {
+    _data_owner: ReadOnlySource,
+    cursor: &'static [u8],
+}
+
+impl SourceRead {
+    // Advance the cursor by a given number of bytes.
+    pub fn advance(&mut self, len: usize) {
+        self.cursor = &self.cursor[len..];
+    }
+}
+
+impl AsRef<[u8]> for SourceRead {
+    fn as_ref(&self) -> &[u8] {
+        self.cursor
+    }
+}
+
+impl From<ReadOnlySource> for SourceRead {
+    // Creates a new `SourceRead` from a given `ReadOnlySource`
+    fn from(source: ReadOnlySource) -> SourceRead {
+        let len = source.len();
+        let slice_ptr = source.as_slice().as_ptr();
+        let static_slice = unsafe { slice::from_raw_parts(slice_ptr, len) };
+        SourceRead {
+            _data_owner: source,
+            cursor: static_slice,
+        }
+    }
+}
+
+impl Read for SourceRead {
+    fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
+        self.cursor.read(buf)
+    }
+}

src/directory/shared_vec_slice.rs

@@ -1,6 +1,5 @@
 use std::sync::Arc;
 
-
 #[derive(Clone)]
 pub struct SharedVecSlice {
     pub data: Arc<Vec<u8>>,
@@ -16,7 +15,7 @@ impl SharedVecSlice {
     pub fn new(data: Arc<Vec<u8>>) -> SharedVecSlice {
         let data_len = data.len();
         SharedVecSlice {
-            data: data,
+            data,
             start: 0,
             len: data_len,
         }
@@ -28,14 +27,13 @@ impl SharedVecSlice {
 
     pub fn slice(&self, from_offset: usize, to_offset: usize) -> SharedVecSlice {
         SharedVecSlice {
-            data: self.data.clone(),
+            data: Arc::clone(&self.data),
             start: self.start + from_offset,
             len: to_offset - from_offset,
         }
     }
 }
 
-
 impl From<Vec<u8>> for SharedVecSlice {
     fn from(data: Vec<u8>) -> SharedVecSlice {
         SharedVecSlice::new(Arc::new(data))

src/docset.rs

@@ -2,7 +2,7 @@ use DocId;
 use std::borrow::Borrow;
 use std::borrow::BorrowMut;
 use std::cmp::Ordering;
-
+use common::BitSet;
 
 /// Expresses the outcome of a call to `DocSet`'s `.skip_next(...)`.
 #[derive(PartialEq, Eq, Debug)]
@@ -16,7 +16,6 @@ pub enum SkipResult {
     End,
 }
 
-
 /// Represents an iterable set of sorted doc ids.
 pub trait DocSet {
     /// Goes to the next element.
@@ -35,6 +34,9 @@ pub trait DocSet {
     /// More specifically, if the docset is already positionned on the target
     /// skipping will advance to the next position and return SkipResult::Overstep.
     ///
+    /// If `.skip_next()` oversteps, then the docset must be positionned correctly
+    /// on an existing document. In other words, `.doc()` should return the first document
+    /// greater than `DocId`.
     fn skip_next(&mut self, target: DocId) -> SkipResult {
         if !self.advance() {
             return SkipResult::End;
@@ -52,26 +54,58 @@ pub trait DocSet {
         }
     }
 
+    /// Fills a given mutable buffer with the next doc ids from the
+    /// `DocSet`
+    ///
+    /// If that many `DocId`s are available, the method should
+    /// fill the entire buffer and return the length of the buffer.
+    ///
+    /// If we reach the end of the `DocSet` before filling
+    /// it entirely, then the buffer is filled up to this point, and
+    /// return value is the number of elements that were filled.
+    ///
+    /// # Warning
+    ///
+    /// This method is only here for specific high-performance
+    /// use case where batching. The normal way to
+    /// go through the `DocId`'s is to call `.advance()`.
+    fn fill_buffer(&mut self, buffer: &mut [DocId]) -> usize {
+        for (i, buffer_val) in buffer.iter_mut().enumerate() {
+            if self.advance() {
+                *buffer_val = self.doc();
+            } else {
+                return i;
+            }
+        }
+        buffer.len()
+    }
+
     /// Returns the current document
     fn doc(&self) -> DocId;
 
-    /// Advances the cursor to the next document
-    /// None is returned if the iterator has `DocSet`
-    /// has already been entirely consumed.
-    fn next(&mut self) -> Option<DocId> {
-        if self.advance() {
-            Some(self.doc())
-        } else {
-            None
+    /// Returns a best-effort hint of the
+    /// length of the docset.
+    fn size_hint(&self) -> u32;
+
+    /// Appends all docs to a `bitset`.
+    fn append_to_bitset(&mut self, bitset: &mut BitSet) {
+        while self.advance() {
+            bitset.insert(self.doc());
         }
     }
 
-    /// Returns a best-effort hint of the
-    /// length of the docset.
-    fn size_hint(&self) -> usize;
+    /// Returns the number documents matching.
+    ///
+    /// Calling this method consumes the `DocSet`.
+    fn count(&mut self) -> u32 {
+        let mut count = 0u32;
+        while self.advance() {
+            count += 1u32;
+        }
+        count
+    }
 }
 
-
 impl<TDocSet: DocSet + ?Sized> DocSet for Box<TDocSet> {
     fn advance(&mut self) -> bool {
         let unboxed: &mut TDocSet = self.borrow_mut();
@@ -88,30 +122,18 @@ impl<TDocSet: DocSet + ?Sized> DocSet for Box<TDocSet> {
         unboxed.doc()
     }
 
-    fn size_hint(&self) -> usize {
+    fn size_hint(&self) -> u32 {
         let unboxed: &TDocSet = self.borrow();
         unboxed.size_hint()
     }
-}
 
-impl<'a, TDocSet: DocSet> DocSet for &'a mut TDocSet {
-    fn advance(&mut self) -> bool {
-        let unref: &mut TDocSet = *self;
-        unref.advance()
+    fn count(&mut self) -> u32 {
+        let unboxed: &mut TDocSet = self.borrow_mut();
+        unboxed.count()
     }
 
-    fn skip_next(&mut self, target: DocId) -> SkipResult {
-        let unref: &mut TDocSet = *self;
-        unref.skip_next(target)
-    }
-
-    fn doc(&self) -> DocId {
-        let unref: &TDocSet = *self;
-        unref.doc()
-    }
-
-    fn size_hint(&self) -> usize {
-        let unref: &TDocSet = *self;
-        unref.size_hint()
+    fn append_to_bitset(&mut self, bitset: &mut BitSet) {
+        let unboxed: &mut TDocSet = self.borrow_mut();
+        unboxed.append_to_bitset(bitset);
     }
 }

src/error.rs

@@ -4,7 +4,7 @@ use std::io;
 
 use std::path::PathBuf;
 use std::sync::PoisonError;
-use directory::error::{IOError, OpenReadError, OpenWriteError, OpenDirectoryError};
+use directory::error::{IOError, OpenDirectoryError, OpenReadError, OpenWriteError};
 use query;
 use schema;
 use fastfield::FastFieldNotAvailableError;
@@ -111,12 +111,9 @@ impl From<schema::DocParsingError> for Error {
 impl From<OpenWriteError> for Error {
     fn from(error: OpenWriteError) -> Error {
         match error {
-                OpenWriteError::FileAlreadyExists(filepath) => {
-                    ErrorKind::FileAlreadyExists(filepath)
-                }
-                OpenWriteError::IOError(io_error) => ErrorKind::IOError(io_error),
-            }
-            .into()
+            OpenWriteError::FileAlreadyExists(filepath) => ErrorKind::FileAlreadyExists(filepath),
+            OpenWriteError::IOError(io_error) => ErrorKind::IOError(io_error),
+        }.into()
     }
 }
 
@@ -126,10 +123,9 @@ impl From<OpenDirectoryError> for Error {
             OpenDirectoryError::DoesNotExist(directory_path) => {
                 ErrorKind::PathDoesNotExist(directory_path).into()
             }
-            OpenDirectoryError::NotADirectory(directory_path) => {
-                ErrorKind::InvalidArgument(format!("{:?} is not a directory", directory_path))
-                    .into()
-            }
+            OpenDirectoryError::NotADirectory(directory_path) => ErrorKind::InvalidArgument(
+                format!("{:?} is not a directory", directory_path),
+            ).into(),
         }
     }
 }

src/fastfield/delete.rs

@@ -38,7 +38,6 @@ pub struct DeleteBitSet {
     len: usize,
 }
 
-
 impl DeleteBitSet {
     /// Opens a delete bitset given its data source.
     pub fn open(data: ReadOnlySource) -> DeleteBitSet {
@@ -47,7 +46,7 @@ impl DeleteBitSet {
             .map(|b| b.count_ones() as usize)
             .sum();
         DeleteBitSet {
-            data: data,
+            data,
             len: num_deleted,
         }
     }
@@ -79,7 +78,6 @@ impl DeleteBitSet {
     }
 }
 
-
 impl HasLen for DeleteBitSet {
     fn len(&self) -> usize {
         self.len

src/fastfield/error.rs

@@ -14,10 +14,11 @@ impl FastFieldNotAvailableError {
     /// `field_entry` is the configuration of the field
     /// for which fast fields are not available.
     pub fn new(field_entry: &FieldEntry) -> FastFieldNotAvailableError {
-        FastFieldNotAvailableError { field_name: field_entry.name().to_string() }
+        FastFieldNotAvailableError {
+            field_name: field_entry.name().to_string(),
+        }
     }
 }
 
-
 /// Result when trying to access a fast field reader.
 pub type Result<R> = result::Result<R, FastFieldNotAvailableError>;

src/fastfield/facet_reader.rs

@@ -0,0 +1,68 @@
+use super::MultiValueIntFastFieldReader;
+use DocId;
+use termdict::TermOrdinal;
+use schema::Facet;
+use termdict::{TermDictionary, TermDictionaryImpl};
+
+/// The facet reader makes it possible to access the list of
+/// facets associated to a given document in a specific
+/// segment.
+///
+/// Rather than manipulating `Facet` object directly, the API
+/// exposes those in the form of list of `Facet` ordinal.
+///
+/// A segment ordinal can then be translated into a facet via
+/// `.facet_from_ord(...)`.
+///
+/// Facet ordinals are defined as their position in the sorted
+/// list of facets. This ordinal is segment local and
+/// only makes sense for a given segment.
+pub struct FacetReader {
+    term_ords: MultiValueIntFastFieldReader<u64>,
+    term_dict: TermDictionaryImpl,
+}
+
+impl FacetReader {
+    /// Creates a new `FacetReader`.
+    ///
+    /// A facet reader just wraps :
+    /// - a `MultiValueIntFastFieldReader` that makes it possible to
+    /// access the list of facet ords for a given document.
+    /// - a `TermDictionaryImpl` that helps associating a facet to
+    /// an ordinal and vice versa.
+    pub fn new(
+        term_ords: MultiValueIntFastFieldReader<u64>,
+        term_dict: TermDictionaryImpl,
+    ) -> FacetReader {
+        FacetReader {
+            term_ords,
+            term_dict,
+        }
+    }
+
+    /// Returns the size of the sets of facets in the segment.
+    /// This does not take in account the documents that may be marked
+    /// as deleted.
+    ///
+    /// `Facet` ordinals range from `0` to `num_facets() - 1`.
+    pub fn num_facets(&self) -> usize {
+        self.term_dict.num_terms()
+    }
+
+    /// Accessor for the facet term dictionary.
+    pub fn facet_dict(&self) -> &TermDictionaryImpl {
+        &self.term_dict
+    }
+
+    /// Given a term ordinal returns the term associated to it.
+    pub fn facet_from_ord(&self, facet_ord: TermOrdinal, output: &mut Facet) {
+        let found_term = self.term_dict
+            .ord_to_term(facet_ord as u64, output.inner_buffer_mut());
+        assert!(found_term, "Term ordinal {} no found.", facet_ord);
+    }
+
+    /// Return the list of facet ordinals associated to a document.
+    pub fn facet_ords(&mut self, doc: DocId, output: &mut Vec<u64>) {
+        self.term_ords.get_vals(doc, output);
+    }
+}

src/fastfield/mod.rs

@@ -1,5 +1,5 @@
 /*!
-Fast fields is a column oriented storage storage.
+Column oriented field storage for tantivy.
 
 It is the equivalent of `Lucene`'s `DocValues`.
 
@@ -9,49 +9,133 @@ It is designed for the fast random access of some document
 fields given a document id.
 
 `FastField` are useful when a field is required for all or most of
-the `DocSet` : for instance for scoring, grouping, filtering, or facetting.
+the `DocSet` : for instance for scoring, grouping, filtering, or faceting.
 
 
 Fields have to be declared as `FAST` in the  schema.
 Currently only 64-bits integers (signed or unsigned) are
 supported.
 
-They are stored in a bitpacked fashion so that their
+They are stored in a bit-packed fashion so that their
 memory usage is directly linear with the amplitude of the
 values stored.
 
 Read access performance is comparable to that of an array lookup.
 */
 
+use common;
+use schema::Cardinality;
+use schema::FieldType;
+use schema::Value;
+pub use self::delete::DeleteBitSet;
+pub use self::delete::write_delete_bitset;
+pub use self::error::{FastFieldNotAvailableError, Result};
+pub use self::facet_reader::FacetReader;
+pub use self::multivalued::MultiValueIntFastFieldReader;
+pub use self::reader::FastFieldReader;
+pub use self::serializer::FastFieldSerializer;
+pub use self::writer::{FastFieldsWriter, IntFastFieldWriter};
+
 mod reader;
 mod writer;
 mod serializer;
 mod error;
 mod delete;
+mod facet_reader;
+mod multivalued;
 
-pub use self::delete::write_delete_bitset;
-pub use self::delete::DeleteBitSet;
-pub use self::writer::{FastFieldsWriter, IntFastFieldWriter};
-pub use self::reader::{FastFieldsReader, U64FastFieldReader, I64FastFieldReader};
-pub use self::reader::FastFieldReader;
-pub use self::serializer::FastFieldSerializer;
-pub use self::error::{Result, FastFieldNotAvailableError};
+/// Trait for types that are allowed for fast fields: (u64 or i64).
+pub trait FastValue: Default + Clone + Copy {
+    /// Converts a value from u64
+    ///
+    /// Internally all fast field values are encoded as u64.
+    fn from_u64(val: u64) -> Self;
+
+    /// Converts a value to u64.
+    ///
+    /// Internally all fast field values are encoded as u64.
+    fn to_u64(&self) -> u64;
+
+    /// Returns the fast field cardinality that can be extracted from the given
+    /// `FieldType`.
+    ///
+    /// If the type is not a fast field, `None` is returned.
+    fn fast_field_cardinality(field_type: &FieldType) -> Option<Cardinality>;
+
+    /// Cast value to `u64`.
+    /// The value is just reinterpreted in memory.
+    fn as_u64(&self) -> u64;
+}
+
+impl FastValue for u64 {
+    fn from_u64(val: u64) -> Self {
+        val
+    }
+
+    fn to_u64(&self) -> u64 {
+        *self
+    }
+
+    fn as_u64(&self) -> u64 {
+        *self
+    }
+
+    fn fast_field_cardinality(field_type: &FieldType) -> Option<Cardinality> {
+        match *field_type {
+            FieldType::U64(ref integer_options) => integer_options.get_fastfield_cardinality(),
+            FieldType::HierarchicalFacet => Some(Cardinality::MultiValues),
+            _ => None,
+        }
+    }
+}
+
+impl FastValue for i64 {
+    fn from_u64(val: u64) -> Self {
+        common::u64_to_i64(val)
+    }
+
+    fn to_u64(&self) -> u64 {
+        common::i64_to_u64(*self)
+    }
+
+    fn fast_field_cardinality(field_type: &FieldType) -> Option<Cardinality> {
+        match *field_type {
+            FieldType::I64(ref integer_options) => integer_options.get_fastfield_cardinality(),
+            _ => None,
+        }
+    }
+
+    fn as_u64(&self) -> u64 {
+        *self as u64
+    }
+}
+
+fn value_to_u64(value: &Value) -> u64 {
+    match *value {
+        Value::U64(ref val) => *val,
+        Value::I64(ref val) => common::i64_to_u64(*val),
+        _ => panic!("Expected a u64/i64 field, got {:?} ", value),
+    }
+}
 
 #[cfg(test)]
 mod tests {
-    use super::*;
-    use schema::Field;
-    use std::path::Path;
-    use directory::{Directory, WritePtr, RAMDirectory};
-    use schema::Document;
-    use schema::{Schema, SchemaBuilder};
-    use schema::FAST;
-    use test::Bencher;
-    use test;
+
+    use common::CompositeFile;
+    use directory::{Directory, RAMDirectory, WritePtr};
     use fastfield::FastFieldReader;
     use rand::Rng;
     use rand::SeedableRng;
     use rand::XorShiftRng;
+    use schema::{Schema, SchemaBuilder};
+    use schema::Document;
+    use schema::FAST;
+    use schema::Field;
+    use std::collections::HashMap;
+    use std::path::Path;
+    use super::*;
+    use test;
+    use test::Bencher;
 
     lazy_static! {
         static ref SCHEMA: Schema = {
@@ -64,15 +148,9 @@ mod tests {
         };
     }
 
-    fn add_single_field_doc(fast_field_writers: &mut FastFieldsWriter, field: Field, value: u64) {
-        let mut doc = Document::default();
-        doc.add_u64(field, value);
-        fast_field_writers.add_document(&doc);
-    }
-
     #[test]
     pub fn test_fastfield() {
-        let test_fastfield = U64FastFieldReader::from(vec![100, 200, 300]);
+        let test_fastfield = FastFieldReader::<u64>::from(vec![100, 200, 300]);
         assert_eq!(test_fastfield.get(0), 100);
         assert_eq!(test_fastfield.get(1), 200);
         assert_eq!(test_fastfield.get(2), 300);
@@ -84,22 +162,24 @@ mod tests {
         let mut directory: RAMDirectory = RAMDirectory::create();
         {
             let write: WritePtr = directory.open_write(Path::new("test")).unwrap();
-            let mut serializer = FastFieldSerializer::new(write).unwrap();
+            let mut serializer = FastFieldSerializer::from_write(write).unwrap();
             let mut fast_field_writers = FastFieldsWriter::from_schema(&SCHEMA);
-            add_single_field_doc(&mut fast_field_writers, *FIELD, 13u64);
-            add_single_field_doc(&mut fast_field_writers, *FIELD, 14u64);
-            add_single_field_doc(&mut fast_field_writers, *FIELD, 2u64);
-            fast_field_writers.serialize(&mut serializer).unwrap();
+            fast_field_writers.add_document(&doc!(*FIELD=>13u64));
+            fast_field_writers.add_document(&doc!(*FIELD=>14u64));
+            fast_field_writers.add_document(&doc!(*FIELD=>2u64));
+            fast_field_writers
+                .serialize(&mut serializer, &HashMap::new())
+                .unwrap();
             serializer.close().unwrap();
         }
         let source = directory.open_read(&path).unwrap();
         {
-            assert_eq!(source.len(), 38 as usize);
+            assert_eq!(source.len(), 36 as usize);
         }
         {
-            let fast_field_readers = FastFieldsReader::from_source(source).unwrap();
-            let fast_field_reader: U64FastFieldReader =
-                fast_field_readers.open_reader(*FIELD).unwrap();
+            let composite_file = CompositeFile::open(&source).unwrap();
+            let field_source = composite_file.open_read(*FIELD).unwrap();
+            let fast_field_reader = FastFieldReader::<u64>::open(field_source);
             assert_eq!(fast_field_reader.get(0), 13u64);
             assert_eq!(fast_field_reader.get(1), 14u64);
             assert_eq!(fast_field_reader.get(2), 2u64);
@@ -112,28 +192,30 @@ mod tests {
         let mut directory: RAMDirectory = RAMDirectory::create();
         {
             let write: WritePtr = directory.open_write(Path::new("test")).unwrap();
-            let mut serializer = FastFieldSerializer::new(write).unwrap();
+            let mut serializer = FastFieldSerializer::from_write(write).unwrap();
             let mut fast_field_writers = FastFieldsWriter::from_schema(&SCHEMA);
-            add_single_field_doc(&mut fast_field_writers, *FIELD, 4u64);
-            add_single_field_doc(&mut fast_field_writers, *FIELD, 14_082_001u64);
-            add_single_field_doc(&mut fast_field_writers, *FIELD, 3_052u64);
-            add_single_field_doc(&mut fast_field_writers, *FIELD, 9002u64);
-            add_single_field_doc(&mut fast_field_writers, *FIELD, 15_001u64);
-            add_single_field_doc(&mut fast_field_writers, *FIELD, 777u64);
-            add_single_field_doc(&mut fast_field_writers, *FIELD, 1_002u64);
-            add_single_field_doc(&mut fast_field_writers, *FIELD, 1_501u64);
-            add_single_field_doc(&mut fast_field_writers, *FIELD, 215u64);
-            fast_field_writers.serialize(&mut serializer).unwrap();
+            fast_field_writers.add_document(&doc!(*FIELD=>4u64));
+            fast_field_writers.add_document(&doc!(*FIELD=>14_082_001u64));
+            fast_field_writers.add_document(&doc!(*FIELD=>3_052u64));
+            fast_field_writers.add_document(&doc!(*FIELD=>9_002u64));
+            fast_field_writers.add_document(&doc!(*FIELD=>15_001u64));
+            fast_field_writers.add_document(&doc!(*FIELD=>777u64));
+            fast_field_writers.add_document(&doc!(*FIELD=>1_002u64));
+            fast_field_writers.add_document(&doc!(*FIELD=>1_501u64));
+            fast_field_writers.add_document(&doc!(*FIELD=>215u64));
+            fast_field_writers
+                .serialize(&mut serializer, &HashMap::new())
+                .unwrap();
             serializer.close().unwrap();
         }
         let source = directory.open_read(&path).unwrap();
         {
-            assert_eq!(source.len(), 63 as usize);
+            assert_eq!(source.len(), 61 as usize);
         }
         {
-            let fast_field_readers = FastFieldsReader::from_source(source).unwrap();
-            let fast_field_reader: U64FastFieldReader =
-                fast_field_readers.open_reader(*FIELD).unwrap();
+            let fast_fields_composite = CompositeFile::open(&source).unwrap();
+            let data = fast_fields_composite.open_read(*FIELD).unwrap();
+            let fast_field_reader = FastFieldReader::<u64>::open(data);
             assert_eq!(fast_field_reader.get(0), 4u64);
             assert_eq!(fast_field_reader.get(1), 14_082_001u64);
             assert_eq!(fast_field_reader.get(2), 3_052u64);
@@ -151,25 +233,26 @@ mod tests {
         let path = Path::new("test");
         let mut directory: RAMDirectory = RAMDirectory::create();
 
-
         {
             let write: WritePtr = directory.open_write(Path::new("test")).unwrap();
-            let mut serializer = FastFieldSerializer::new(write).unwrap();
+            let mut serializer = FastFieldSerializer::from_write(write).unwrap();
             let mut fast_field_writers = FastFieldsWriter::from_schema(&SCHEMA);
             for _ in 0..10_000 {
-                add_single_field_doc(&mut fast_field_writers, *FIELD, 100_000u64);
+                fast_field_writers.add_document(&doc!(*FIELD=>100_000u64));
             }
-            fast_field_writers.serialize(&mut serializer).unwrap();
+            fast_field_writers
+                .serialize(&mut serializer, &HashMap::new())
+                .unwrap();
             serializer.close().unwrap();
         }
         let source = directory.open_read(&path).unwrap();
         {
-            assert_eq!(source.len(), 36 as usize);
+            assert_eq!(source.len(), 34 as usize);
         }
         {
-            let fast_field_readers = FastFieldsReader::from_source(source).unwrap();
-            let fast_field_reader: U64FastFieldReader =
-                fast_field_readers.open_reader(*FIELD).unwrap();
+            let fast_fields_composite = CompositeFile::open(&source).unwrap();
+            let data = fast_fields_composite.open_read(*FIELD).unwrap();
+            let fast_field_reader = FastFieldReader::<u64>::open(data);
             for doc in 0..10_000 {
                 assert_eq!(fast_field_reader.get(doc), 100_000u64);
             }
@@ -183,30 +266,32 @@ mod tests {
 
         {
             let write: WritePtr = directory.open_write(Path::new("test")).unwrap();
-            let mut serializer = FastFieldSerializer::new(write).unwrap();
+            let mut serializer = FastFieldSerializer::from_write(write).unwrap();
             let mut fast_field_writers = FastFieldsWriter::from_schema(&SCHEMA);
             // forcing the amplitude to be high
-            add_single_field_doc(&mut fast_field_writers, *FIELD, 0u64);
+            fast_field_writers.add_document(&doc!(*FIELD=>0u64));
             for i in 0u64..10_000u64 {
-                add_single_field_doc(&mut fast_field_writers,
-                                     *FIELD,
-                                     5_000_000_000_000_000_000u64 + i);
+                fast_field_writers.add_document(&doc!(*FIELD=>5_000_000_000_000_000_000u64 + i));
             }
-            fast_field_writers.serialize(&mut serializer).unwrap();
+            fast_field_writers
+                .serialize(&mut serializer, &HashMap::new())
+                .unwrap();
             serializer.close().unwrap();
         }
         let source = directory.open_read(&path).unwrap();
         {
-            assert_eq!(source.len(), 80044 as usize);
+            assert_eq!(source.len(), 80042 as usize);
         }
         {
-            let fast_field_readers = FastFieldsReader::from_source(source).unwrap();
-            let fast_field_reader: U64FastFieldReader =
-                fast_field_readers.open_reader(*FIELD).unwrap();
+            let fast_fields_composite = CompositeFile::open(&source).unwrap();
+            let data = fast_fields_composite.open_read(*FIELD).unwrap();
+            let fast_field_reader = FastFieldReader::<u64>::open(data);
             assert_eq!(fast_field_reader.get(0), 0u64);
             for doc in 1..10_001 {
-                assert_eq!(fast_field_reader.get(doc),
-                           5_000_000_000_000_000_000u64 + doc as u64 - 1u64);
+                assert_eq!(
+                    fast_field_reader.get(doc),
+                    5_000_000_000_000_000_000u64 + doc as u64 - 1u64
+                );
             }
         }
     }
@@ -221,24 +306,27 @@ mod tests {
         let schema = schema_builder.build();
         {
             let write: WritePtr = directory.open_write(Path::new("test")).unwrap();
-            let mut serializer = FastFieldSerializer::new(write).unwrap();
+            let mut serializer = FastFieldSerializer::from_write(write).unwrap();
             let mut fast_field_writers = FastFieldsWriter::from_schema(&schema);
             for i in -100i64..10_000i64 {
                 let mut doc = Document::default();
                 doc.add_i64(i64_field, i);
                 fast_field_writers.add_document(&doc);
             }
-            fast_field_writers.serialize(&mut serializer).unwrap();
+            fast_field_writers
+                .serialize(&mut serializer, &HashMap::new())
+                .unwrap();
             serializer.close().unwrap();
         }
         let source = directory.open_read(&path).unwrap();
         {
-            assert_eq!(source.len(), 17711 as usize);
+            assert_eq!(source.len(), 17709 as usize);
         }
         {
-            let fast_field_readers = FastFieldsReader::from_source(source).unwrap();
-            let fast_field_reader: I64FastFieldReader =
-                fast_field_readers.open_reader(i64_field).unwrap();
+            let fast_fields_composite = CompositeFile::open(&source).unwrap();
+            let data = fast_fields_composite.open_read(i64_field).unwrap();
+            let fast_field_reader = FastFieldReader::<i64>::open(data);
+
             assert_eq!(fast_field_reader.min_value(), -100i64);
             assert_eq!(fast_field_reader.max_value(), 9_999i64);
             for (doc, i) in (-100i64..10_000i64).enumerate() {
@@ -262,19 +350,21 @@ mod tests {
 
         {
             let write: WritePtr = directory.open_write(Path::new("test")).unwrap();
-            let mut serializer = FastFieldSerializer::new(write).unwrap();
+            let mut serializer = FastFieldSerializer::from_write(write).unwrap();
             let mut fast_field_writers = FastFieldsWriter::from_schema(&schema);
             let doc = Document::default();
             fast_field_writers.add_document(&doc);
-            fast_field_writers.serialize(&mut serializer).unwrap();
+            fast_field_writers
+                .serialize(&mut serializer, &HashMap::new())
+                .unwrap();
             serializer.close().unwrap();
         }
 
         let source = directory.open_read(&path).unwrap();
         {
-            let fast_field_readers = FastFieldsReader::from_source(source).unwrap();
-            let fast_field_reader: I64FastFieldReader =
-                fast_field_readers.open_reader(i64_field).unwrap();
+            let fast_fields_composite = CompositeFile::open(&source).unwrap();
+            let data = fast_fields_composite.open_read(i64_field).unwrap();
+            let fast_field_reader = FastFieldReader::<i64>::open(data);
             assert_eq!(fast_field_reader.get(0u32), 0i64);
         }
     }
@@ -295,19 +385,22 @@ mod tests {
         let mut directory = RAMDirectory::create();
         {
             let write: WritePtr = directory.open_write(Path::new("test")).unwrap();
-            let mut serializer = FastFieldSerializer::new(write).unwrap();
+            let mut serializer = FastFieldSerializer::from_write(write).unwrap();
             let mut fast_field_writers = FastFieldsWriter::from_schema(&SCHEMA);
-            for x in &permutation {
-                add_single_field_doc(&mut fast_field_writers, *FIELD, *x);
+            for &x in &permutation {
+                fast_field_writers.add_document(&doc!(*FIELD=>x));
             }
-            fast_field_writers.serialize(&mut serializer).unwrap();
+            fast_field_writers
+                .serialize(&mut serializer, &HashMap::new())
+                .unwrap();
             serializer.close().unwrap();
         }
         let source = directory.open_read(&path).unwrap();
         {
-            let fast_field_readers = FastFieldsReader::from_source(source).unwrap();
-            let fast_field_reader: U64FastFieldReader =
-                fast_field_readers.open_reader(*FIELD).unwrap();
+            let fast_fields_composite = CompositeFile::open(&source).unwrap();
+            let data = fast_fields_composite.open_read(*FIELD).unwrap();
+            let fast_field_reader = FastFieldReader::<u64>::open(data);
+
             let mut a = 0u64;
             for _ in 0..n {
                 assert_eq!(fast_field_reader.get(a as u32), permutation[a as usize]);
@@ -322,7 +415,7 @@ mod tests {
         b.iter(|| {
             let n = test::black_box(7000u32);
             let mut a = 0u64;
-            for i in Iterator::step_by((0u32..n), 7) {
+            for i in Iterator::step_by(0u32..n, 7) {
                 a ^= permutation[i as usize];
             }
             a
@@ -333,13 +426,13 @@ mod tests {
     fn bench_intfastfield_veclookup(b: &mut Bencher) {
         let permutation = generate_permutation();
         b.iter(|| {
-                   let n = test::black_box(1000u32);
-                   let mut a = 0u64;
-                   for _ in 0u32..n {
-                       a = permutation[a as usize];
-                   }
-                   a
-               });
+            let n = test::black_box(1000u32);
+            let mut a = 0u64;
+            for _ in 0u32..n {
+                a = permutation[a as usize];
+            }
+            a
+        });
     }
 
     #[bench]
@@ -349,23 +442,26 @@ mod tests {
         let mut directory: RAMDirectory = RAMDirectory::create();
         {
             let write: WritePtr = directory.open_write(Path::new("test")).unwrap();
-            let mut serializer = FastFieldSerializer::new(write).unwrap();
+            let mut serializer = FastFieldSerializer::from_write(write).unwrap();
             let mut fast_field_writers = FastFieldsWriter::from_schema(&SCHEMA);
-            for x in &permutation {
-                add_single_field_doc(&mut fast_field_writers, *FIELD, *x);
+            for &x in &permutation {
+                fast_field_writers.add_document(&doc!(*FIELD=>x));
             }
-            fast_field_writers.serialize(&mut serializer).unwrap();
+            fast_field_writers
+                .serialize(&mut serializer, &HashMap::new())
+                .unwrap();
             serializer.close().unwrap();
         }
         let source = directory.open_read(&path).unwrap();
         {
-            let fast_field_readers = FastFieldsReader::from_source(source).unwrap();
-            let fast_field_reader: U64FastFieldReader =
-                fast_field_readers.open_reader(*FIELD).unwrap();
+            let fast_fields_composite = CompositeFile::open(&source).unwrap();
+            let data = fast_fields_composite.open_read(*FIELD).unwrap();
+            let fast_field_reader = FastFieldReader::<u64>::open(data);
+
             b.iter(|| {
                 let n = test::black_box(7000u32);
                 let mut a = 0u64;
-                for i in Iterator::step_by((0u32..n), 7) {
+                for i in Iterator::step_by(0u32..n, 7) {
                     a ^= fast_field_reader.get(i);
                 }
                 a
@@ -380,27 +476,30 @@ mod tests {
         let mut directory: RAMDirectory = RAMDirectory::create();
         {
             let write: WritePtr = directory.open_write(Path::new("test")).unwrap();
-            let mut serializer = FastFieldSerializer::new(write).unwrap();
+            let mut serializer = FastFieldSerializer::from_write(write).unwrap();
             let mut fast_field_writers = FastFieldsWriter::from_schema(&SCHEMA);
-            for x in &permutation {
-                add_single_field_doc(&mut fast_field_writers, *FIELD, *x);
+            for &x in &permutation {
+                fast_field_writers.add_document(&doc!(*FIELD=>x));
             }
-            fast_field_writers.serialize(&mut serializer).unwrap();
+            fast_field_writers
+                .serialize(&mut serializer, &HashMap::new())
+                .unwrap();
             serializer.close().unwrap();
         }
         let source = directory.open_read(&path).unwrap();
         {
-            let fast_field_readers = FastFieldsReader::from_source(source).unwrap();
-            let fast_field_reader: U64FastFieldReader =
-                fast_field_readers.open_reader(*FIELD).unwrap();
+            let fast_fields_composite = CompositeFile::open(&source).unwrap();
+            let data = fast_fields_composite.open_read(*FIELD).unwrap();
+            let fast_field_reader = FastFieldReader::<u64>::open(data);
+
             b.iter(|| {
-                       let n = test::black_box(1000u32);
-                       let mut a = 0u32;
-                       for _ in 0u32..n {
-                           a = fast_field_reader.get(a) as u32;
-                       }
-                       a
-                   });
+                let n = test::black_box(1000u32);
+                let mut a = 0u32;
+                for _ in 0u32..n {
+                    a = fast_field_reader.get(a) as u32;
+                }
+                a
+            });
         }
     }
 }

src/fastfield/multivalued/mod.rs

@@ -0,0 +1,88 @@
+mod writer;
+mod reader;
+
+pub use self::writer::MultiValueIntFastFieldWriter;
+pub use self::reader::MultiValueIntFastFieldReader;
+
+#[cfg(test)]
+mod tests {
+
+    use schema::SchemaBuilder;
+    use schema::Cardinality;
+    use schema::IntOptions;
+    use Index;
+
+    #[test]
+    fn test_multivalued_u64() {
+        let mut schema_builder = SchemaBuilder::default();
+        let field = schema_builder.add_u64_field(
+            "multifield",
+            IntOptions::default().set_fast(Cardinality::MultiValues),
+        );
+        let schema = schema_builder.build();
+        let index = Index::create_in_ram(schema);
+        let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
+        index_writer.add_document(doc!(field=>1u64, field=>3u64));
+        index_writer.add_document(doc!());
+        index_writer.add_document(doc!(field=>4u64));
+        index_writer.add_document(doc!(field=>5u64, field=>20u64,field=>1u64));
+        assert!(index_writer.commit().is_ok());
+
+        index.load_searchers().unwrap();
+        let searcher = index.searcher();
+        let reader = searcher.segment_reader(0);
+        let mut vals = Vec::new();
+        let multi_value_reader = reader.multi_fast_field_reader::<u64>(field).unwrap();
+        {
+            multi_value_reader.get_vals(2, &mut vals);
+            assert_eq!(&vals, &[4u64]);
+        }
+        {
+            multi_value_reader.get_vals(0, &mut vals);
+            assert_eq!(&vals, &[1u64, 3u64]);
+        }
+        {
+            multi_value_reader.get_vals(1, &mut vals);
+            assert!(vals.is_empty());
+        }
+    }
+
+    #[test]
+    fn test_multivalued_i64() {
+        let mut schema_builder = SchemaBuilder::default();
+        let field = schema_builder.add_i64_field(
+            "multifield",
+            IntOptions::default().set_fast(Cardinality::MultiValues),
+        );
+        let schema = schema_builder.build();
+        let index = Index::create_in_ram(schema);
+        let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
+        index_writer.add_document(doc!(field=> 1i64, field => 3i64));
+        index_writer.add_document(doc!());
+        index_writer.add_document(doc!(field=> -4i64));
+        index_writer.add_document(doc!(field=> -5i64, field => -20i64, field=>1i64));
+        assert!(index_writer.commit().is_ok());
+
+        index.load_searchers().unwrap();
+        let searcher = index.searcher();
+        let reader = searcher.segment_reader(0);
+        let mut vals = Vec::new();
+        let multi_value_reader = reader.multi_fast_field_reader::<i64>(field).unwrap();
+        {
+            multi_value_reader.get_vals(2, &mut vals);
+            assert_eq!(&vals, &[-4i64]);
+        }
+        {
+            multi_value_reader.get_vals(0, &mut vals);
+            assert_eq!(&vals, &[1i64, 3i64]);
+        }
+        {
+            multi_value_reader.get_vals(1, &mut vals);
+            assert!(vals.is_empty());
+        }
+        {
+            multi_value_reader.get_vals(3, &mut vals);
+            assert_eq!(&vals, &[-5i64, -20i64, 1i64]);
+        }
+    }
+}

src/fastfield/multivalued/reader.rs

@@ -0,0 +1,127 @@
+use DocId;
+use fastfield::{FastFieldReader, FastValue};
+
+/// Reader for a multivalued `u64` fast field.
+///
+/// The reader is implemented as two `u64` fast field.
+///
+/// The `vals_reader` will access the concatenated list of all
+/// values for all reader.
+/// The `idx_reader` associated, for each document, the index of its first value.
+///
+#[derive(Clone)]
+pub struct MultiValueIntFastFieldReader<Item: FastValue> {
+    idx_reader: FastFieldReader<u64>,
+    vals_reader: FastFieldReader<Item>,
+}
+
+impl<Item: FastValue> MultiValueIntFastFieldReader<Item> {
+    pub(crate) fn open(
+        idx_reader: FastFieldReader<u64>,
+        vals_reader: FastFieldReader<Item>,
+    ) -> MultiValueIntFastFieldReader<Item> {
+        MultiValueIntFastFieldReader {
+            idx_reader,
+            vals_reader,
+        }
+    }
+
+    /// Returns `(start, stop)`, such that the values associated
+    /// to the given document are `start..stop`.
+    fn range(&self, doc: DocId) -> (u64, u64) {
+        let start = self.idx_reader.get(doc);
+        let stop = self.idx_reader.get(doc + 1);
+        (start, stop)
+    }
+
+    /// Returns the number of values associated to a given document.
+    pub fn num_vals(&self, doc: DocId) -> usize {
+        let (start, stop) = self.range(doc);
+        (stop - start) as usize
+    }
+
+    /// Returns the overall number of values associated to documents.
+    pub(crate) fn total_num_vals(&self) -> u64 {
+        self.idx_reader.max_value()
+    }
+
+    /// Returns the array of values associated to the given `doc`.
+    pub fn get_vals(&self, doc: DocId, vals: &mut Vec<Item>) {
+        let (start, stop) = self.range(doc);
+        let len = (stop - start) as usize;
+        vals.resize(len, Item::default());
+        self.vals_reader.get_range(start as u32, &mut vals[..]);
+    }
+}
+
+#[cfg(test)]
+mod tests {
+
+    use core::Index;
+    use schema::{Document, Facet, SchemaBuilder};
+
+    #[test]
+    fn test_multifastfield_reader() {
+        let mut schema_builder = SchemaBuilder::new();
+        let facet_field = schema_builder.add_facet_field("facets");
+        let schema = schema_builder.build();
+        let index = Index::create_in_ram(schema);
+        let mut index_writer = index
+            .writer_with_num_threads(1, 30_000_000)
+            .expect("Failed to create index writer.");
+        {
+            let mut doc = Document::new();
+            doc.add_facet(facet_field, "/category/cat2");
+            doc.add_facet(facet_field, "/category/cat1");
+            index_writer.add_document(doc);
+        }
+        {
+            let mut doc = Document::new();
+            doc.add_facet(facet_field, "/category/cat2");
+            index_writer.add_document(doc);
+        }
+        {
+            let mut doc = Document::new();
+            doc.add_facet(facet_field, "/category/cat3");
+            index_writer.add_document(doc);
+        }
+        index_writer.commit().expect("Commit failed");
+        index.load_searchers().expect("Reloading searchers");
+        let searcher = index.searcher();
+        let segment_reader = searcher.segment_reader(0);
+        let mut facet_reader = segment_reader.facet_reader(facet_field).unwrap();
+
+        let mut facet = Facet::root();
+        {
+            facet_reader.facet_from_ord(1, &mut facet);
+            assert_eq!(facet, Facet::from("/category"));
+        }
+        {
+            facet_reader.facet_from_ord(2, &mut facet);
+            assert_eq!(facet, Facet::from("/category/cat1"));
+        }
+        {
+            facet_reader.facet_from_ord(3, &mut facet);
+            assert_eq!(format!("{}", facet), "/category/cat2");
+            assert_eq!(facet, Facet::from("/category/cat2"));
+        }
+        {
+            facet_reader.facet_from_ord(4, &mut facet);
+            assert_eq!(facet, Facet::from("/category/cat3"));
+        }
+
+        let mut vals = Vec::new();
+        {
+            facet_reader.facet_ords(0, &mut vals);
+            assert_eq!(&vals[..], &[3, 2]);
+        }
+        {
+            facet_reader.facet_ords(1, &mut vals);
+            assert_eq!(&vals[..], &[3]);
+        }
+        {
+            facet_reader.facet_ords(2, &mut vals);
+            assert_eq!(&vals[..], &[4]);
+        }
+    }
+}

src/fastfield/multivalued/writer.rs

@@ -0,0 +1,112 @@
+use fastfield::FastFieldSerializer;
+use fastfield::serializer::FastSingleFieldSerializer;
+use fastfield::value_to_u64;
+use std::collections::HashMap;
+use postings::UnorderedTermId;
+use schema::{Document, Field};
+use std::io;
+use itertools::Itertools;
+use termdict::TermOrdinal;
+
+pub struct MultiValueIntFastFieldWriter {
+    field: Field,
+    vals: Vec<u64>,
+    doc_index: Vec<u64>,
+    is_facet: bool,
+}
+
+impl MultiValueIntFastFieldWriter {
+    /// Creates a new `IntFastFieldWriter`
+    pub fn new(field: Field, is_facet: bool) -> Self {
+        MultiValueIntFastFieldWriter {
+            field,
+            vals: Vec::new(),
+            doc_index: Vec::new(),
+            is_facet,
+        }
+    }
+
+    pub fn field(&self) -> Field {
+        self.field
+    }
+
+    pub fn next_doc(&mut self) {
+        self.doc_index.push(self.vals.len() as u64);
+    }
+
+    /// Records a new value.
+    ///
+    /// The n-th value being recorded is implicitely
+    /// associated to the document with the `DocId` n.
+    /// (Well, `n-1` actually because of 0-indexing)
+    pub fn add_val(&mut self, val: UnorderedTermId) {
+        self.vals.push(val);
+    }
+
+    pub fn add_document(&mut self, doc: &Document) {
+        if !self.is_facet {
+            for field_value in doc.field_values() {
+                if field_value.field() == self.field {
+                    self.add_val(value_to_u64(field_value.value()));
+                }
+            }
+        }
+    }
+
+    /// Serializes fast field values by pushing them to the `FastFieldSerializer`.
+    ///
+    /// HashMap makes it possible to remap them before serializing.
+    /// Specifically, string terms are first stored in the writer as their
+    /// position in the `IndexWriter`'s `HashMap`. This value is called
+    /// an `UnorderedTermId`.
+    ///
+    /// During the serialization of the segment, terms gets sorted and
+    /// `tantivy` builds a mapping to convert this `UnorderedTermId` into
+    /// term ordinals.
+    ///
+    pub fn serialize(
+        &self,
+        serializer: &mut FastFieldSerializer,
+        mapping_opt: Option<&HashMap<UnorderedTermId, TermOrdinal>>,
+    ) -> io::Result<()> {
+        {
+            // writing the offset index
+            let mut doc_index_serializer =
+                serializer.new_u64_fast_field_with_idx(self.field, 0, self.vals.len() as u64, 0)?;
+            for &offset in &self.doc_index {
+                doc_index_serializer.add_val(offset)?;
+            }
+            doc_index_serializer.add_val(self.vals.len() as u64)?;
+            doc_index_serializer.close_field()?;
+        }
+        {
+            // writing the values themselves.
+            let mut value_serializer: FastSingleFieldSerializer<_>;
+            match mapping_opt {
+                Some(mapping) => {
+                    value_serializer = serializer.new_u64_fast_field_with_idx(
+                        self.field,
+                        0u64,
+                        mapping.len() as u64,
+                        1,
+                    )?;
+                    for val in &self.vals {
+                        let remapped_val = *mapping.get(val).expect("Missing term ordinal");
+                        value_serializer.add_val(remapped_val)?;
+                    }
+                }
+                None => {
+                    let val_min_max = self.vals.iter().cloned().minmax();
+                    let (val_min, val_max) = val_min_max.into_option().unwrap_or((0u64, 0u64));
+                    value_serializer =
+                        serializer.new_u64_fast_field_with_idx(self.field, val_min, val_max, 1)?;
+                    for &val in &self.vals {
+                        value_serializer.add_val(val)?;
+                    }
+                }
+            }
+            value_serializer.close_field()?;
+        }
+        Ok(())
+    }
+}

src/fastfield/reader.rs

@@ -1,29 +1,55 @@
-use std::io;
-use std::collections::HashMap;
-use directory::ReadOnlySource;
 use common::BinarySerializable;
-use DocId;
-use schema::{Field, SchemaBuilder};
-use std::path::Path;
-use schema::FAST;
-use directory::{WritePtr, RAMDirectory, Directory};
-use fastfield::FastFieldSerializer;
-use fastfield::FastFieldsWriter;
-use common::bitpacker::compute_num_bits;
 use common::bitpacker::BitUnpacker;
-use schema::FieldType;
-use error::ResultExt;
-use std::mem;
-use common;
+use common::CompositeFile;
+use common::compute_num_bits;
+use directory::{Directory, RAMDirectory, WritePtr};
+use directory::ReadOnlySource;
+use DocId;
+use fastfield::{FastFieldSerializer, FastFieldsWriter};
 use owning_ref::OwningRef;
+use schema::FAST;
+use schema::SchemaBuilder;
+use std::collections::HashMap;
+use std::marker::PhantomData;
+use std::mem;
+use std::path::Path;
+use super::FastValue;
 
 /// Trait for accessing a fastfield.
 ///
 /// Depending on the field type, a different
 /// fast field is required.
-pub trait FastFieldReader: Sized {
-    /// Type of the value stored in the fastfield.
-    type ValueType;
+#[derive(Clone)]
+pub struct FastFieldReader<Item: FastValue> {
+    bit_unpacker: BitUnpacker<OwningRef<ReadOnlySource, [u8]>>,
+    min_value_u64: u64,
+    max_value_u64: u64,
+    _phantom: PhantomData<Item>,
+}
+
+impl<Item: FastValue> FastFieldReader<Item> {
+    /// Opens a fast field given a source.
+    pub fn open(data: ReadOnlySource) -> Self {
+        let min_value: u64;
+        let amplitude: u64;
+        {
+            let mut cursor = data.as_slice();
+            min_value =
+                u64::deserialize(&mut cursor).expect("Failed to read the min_value of fast field.");
+            amplitude =
+                u64::deserialize(&mut cursor).expect("Failed to read the amplitude of fast field.");
+        }
+        let max_value = min_value + amplitude;
+        let num_bits = compute_num_bits(amplitude);
+        let owning_ref = OwningRef::new(data).map(|data| &data[16..]);
+        let bit_unpacker = BitUnpacker::new(owning_ref, num_bits);
+        FastFieldReader {
+            min_value_u64: min_value,
+            max_value_u64: max_value,
+            bit_unpacker,
+            _phantom: PhantomData,
+        }
+    }
 
     /// Return the value associated to the given document.
     ///
@@ -33,7 +59,9 @@ pub trait FastFieldReader: Sized {
     ///
     /// May panic if `doc` is greater than the segment
     // `maxdoc`.
-    fn get(&self, doc: DocId) -> Self::ValueType;
+    pub fn get(&self, doc: DocId) -> Item {
+        Item::from_u64(self.min_value_u64 + self.bit_unpacker.get(doc as usize))
+    }
 
     /// Fills an output buffer with the fast field values
     /// associated with the `DocId` going from
@@ -43,32 +71,24 @@ pub trait FastFieldReader: Sized {
     ///
     /// May panic if `start + output.len()` is greater than
     /// the segment's `maxdoc`.
-    fn get_range(&self, start: u32, output: &mut [Self::ValueType]);
+    ///
+    // TODO change start to `u64`.
+    // For multifastfield, start is an index in a second fastfield, not a `DocId`
+    pub fn get_range(&self, start: u32, output: &mut [Item]) {
+        let output_u64: &mut [u64] = unsafe { mem::transmute(output) };
+        self.bit_unpacker.get_range(start, output_u64);
+        for out in output_u64.iter_mut() {
+            *out = Item::from_u64(*out + self.min_value_u64).as_u64();
+        }
+    }
 
-    /// Opens a fast field given a source.
-    fn open(source: ReadOnlySource) -> Self;
-
-    /// Returns true iff the given field_type makes
-    /// it possible to access the field values via a
-    /// fastfield.
-    fn is_enabled(field_type: &FieldType) -> bool;
-}
-
-/// `FastFieldReader` for unsigned 64-bits integers.
-pub struct U64FastFieldReader {
-    bit_unpacker: BitUnpacker<OwningRef<ReadOnlySource, [u8]>>,
-    min_value: u64,
-    max_value: u64,
-}
-
-impl U64FastFieldReader {
     /// Returns the minimum value for this fast field.
     ///
-    /// The min value does not take in account of possible
-    /// deleted document, and should be considered as a lower bound
-    /// of the actual minimum value.
-    pub fn min_value(&self) -> u64 {
-        self.min_value
+    /// The max value does not take in account of possible
+    /// deleted document, and should be considered as an upper bound
+    /// of the actual maximum value.
+    pub fn min_value(&self) -> Item {
+        Item::from_u64(self.min_value_u64)
     }
 
     /// Returns the maximum value for this fast field.
@@ -76,222 +96,45 @@ impl U64FastFieldReader {
     /// The max value does not take in account of possible
     /// deleted document, and should be considered as an upper bound
     /// of the actual maximum value.
-    pub fn max_value(&self) -> u64 {
-        self.max_value
+    pub fn max_value(&self) -> Item {
+        Item::from_u64(self.max_value_u64)
     }
 }
 
-impl FastFieldReader for U64FastFieldReader {
-    type ValueType = u64;
-
-    fn get(&self, doc: DocId) -> u64 {
-        self.min_value + self.bit_unpacker.get(doc as usize)
-    }
-
-    fn is_enabled(field_type: &FieldType) -> bool {
-        match *field_type {
-            FieldType::U64(ref integer_options) => integer_options.is_fast(),
-            _ => false,
-        }
-    }
-
-    fn get_range(&self, start: u32, output: &mut [Self::ValueType]) {
-        self.bit_unpacker.get_range(start, output);
-        for out in output.iter_mut() {
-            *out += self.min_value;
-        }
-    }
-
-    /// Opens a new fast field reader given a read only source.
-    ///
-    /// # Panics
-    /// Panics if the data is corrupted.
-    fn open(data: ReadOnlySource) -> U64FastFieldReader {
-        let min_value: u64;
-        let amplitude: u64;
-        {
-            let mut cursor = data.as_slice();
-            min_value = u64::deserialize(&mut cursor)
-                .expect("Failed to read the min_value of fast field.");
-            amplitude = u64::deserialize(&mut cursor)
-                .expect("Failed to read the amplitude of fast field.");
-
-        }
-        let max_value = min_value + amplitude;
-        let num_bits = compute_num_bits(amplitude);
-        let owning_ref = OwningRef::new(data).map(|data| &data[16..]);
-        let bit_unpacker = BitUnpacker::new(owning_ref, num_bits as usize);
-        U64FastFieldReader {
-            min_value: min_value,
-            max_value: max_value,
-            bit_unpacker: bit_unpacker,
-        }
-    }
-}
-
-
-impl From<Vec<u64>> for U64FastFieldReader {
-    fn from(vals: Vec<u64>) -> U64FastFieldReader {
+impl<Item: FastValue> From<Vec<Item>> for FastFieldReader<Item> {
+    fn from(vals: Vec<Item>) -> FastFieldReader<Item> {
         let mut schema_builder = SchemaBuilder::default();
         let field = schema_builder.add_u64_field("field", FAST);
         let schema = schema_builder.build();
-        let path = Path::new("test");
+        let path = Path::new("__dummy__");
         let mut directory: RAMDirectory = RAMDirectory::create();
         {
-            let write: WritePtr = directory.open_write(Path::new("test")).unwrap();
-            let mut serializer = FastFieldSerializer::new(write).unwrap();
+            let write: WritePtr = directory
+                .open_write(path)
+                .expect("With a RAMDirectory, this should never fail.");
+            let mut serializer = FastFieldSerializer::from_write(write)
+                .expect("With a RAMDirectory, this should never fail.");
             let mut fast_field_writers = FastFieldsWriter::from_schema(&schema);
-            for val in vals {
-                let mut fast_field_writer = fast_field_writers.get_field_writer(field).unwrap();
-                fast_field_writer.add_val(val);
+            {
+                let fast_field_writer = fast_field_writers
+                    .get_field_writer(field)
+                    .expect("With a RAMDirectory, this should never fail.");
+                for val in vals {
+                    fast_field_writer.add_val(val.to_u64());
+                }
             }
-            fast_field_writers.serialize(&mut serializer).unwrap();
+            fast_field_writers
+                .serialize(&mut serializer, &HashMap::new())
+                .unwrap();
             serializer.close().unwrap();
         }
-        directory
-            .open_read(path)
-            .chain_err(|| "Failed to open the file")
-            .and_then(|source| {
-                          FastFieldsReader::from_source(source)
-                              .chain_err(|| "Failed to read the file.")
-                      })
-            .and_then(|ff_readers| {
-                          ff_readers
-                              .open_reader(field)
-                              .ok_or_else(|| "Failed to find the requested field".into())
-                      })
-            .expect("This should never happen, please report.")
 
-    }
-}
-
-/// `FastFieldReader` for signed 64-bits integers.
-pub struct I64FastFieldReader {
-    underlying: U64FastFieldReader,
-}
-
-impl I64FastFieldReader {
-    /// Returns the minimum value for this fast field.
-    ///
-    /// The min value does not take in account of possible
-    /// deleted document, and should be considered as a lower bound
-    /// of the actual minimum value.
-    pub fn min_value(&self) -> i64 {
-        common::u64_to_i64(self.underlying.min_value())
-    }
-
-    /// Returns the maximum value for this fast field.
-    ///
-    /// The max value does not take in account of possible
-    /// deleted document, and should be considered as an upper bound
-    /// of the actual maximum value.
-    pub fn max_value(&self) -> i64 {
-        common::u64_to_i64(self.underlying.max_value())
-    }
-}
-
-impl FastFieldReader for I64FastFieldReader {
-    type ValueType = i64;
-
-    ///
-    ///
-    /// # Panics
-    ///
-    /// May panic or return wrong random result if `doc`
-    /// is greater or equal to the segment's `maxdoc`.
-    fn get(&self, doc: DocId) -> i64 {
-        common::u64_to_i64(self.underlying.get(doc))
-    }
-
-    ///
-    /// # Panics
-    ///
-    /// May panic or return wrong random result if `doc`
-    /// is greater or equal to the segment's `maxdoc`.
-    fn get_range(&self, start: u32, output: &mut [Self::ValueType]) {
-        let output_u64: &mut [u64] = unsafe { mem::transmute(output) };
-        self.underlying.get_range(start, output_u64);
-        for mut_val in output_u64.iter_mut() {
-            *mut_val ^= 1 << 63;
-        }
-    }
-
-    /// Opens a new fast field reader given a read only source.
-    ///
-    /// # Panics
-    /// Panics if the data is corrupted.
-    fn open(data: ReadOnlySource) -> I64FastFieldReader {
-        I64FastFieldReader { underlying: U64FastFieldReader::open(data) }
-    }
-
-    fn is_enabled(field_type: &FieldType) -> bool {
-        match *field_type {
-            FieldType::I64(ref integer_options) => integer_options.is_fast(),
-            _ => false,
-        }
-    }
-}
-
-
-
-/// The `FastFieldsReader` is the datastructure containing
-/// all of the fast fields' data.
-///
-/// It contains a mapping that associated these fields to
-/// the proper slice in the fastfield reader file.
-pub struct FastFieldsReader {
-    source: ReadOnlySource,
-    field_offsets: HashMap<Field, (u32, u32)>,
-}
-
-impl FastFieldsReader {
-    /// Opens a `FastFieldsReader`
-    ///
-    /// When opening the fast field reader, the
-    /// the list of the offset is read (as a footer of the
-    /// data file).
-    pub fn from_source(source: ReadOnlySource) -> io::Result<FastFieldsReader> {
-        let header_offset;
-        let field_offsets: Vec<(Field, u32)>;
-        {
-            let buffer = source.as_slice();
-            {
-                let mut cursor = buffer;
-                header_offset = u32::deserialize(&mut cursor)?;
-            }
-            {
-                let mut cursor = &buffer[header_offset as usize..];
-                field_offsets = Vec::deserialize(&mut cursor)?;
-            }
-        }
-        let mut end_offsets: Vec<u32> = field_offsets.iter().map(|&(_, offset)| offset).collect();
-        end_offsets.push(header_offset);
-        let mut field_offsets_map: HashMap<Field, (u32, u32)> = HashMap::new();
-        for (field_start_offsets, stop_offset) in
-            field_offsets.iter().zip(end_offsets.iter().skip(1)) {
-            let (field, start_offset) = *field_start_offsets;
-            field_offsets_map.insert(field, (start_offset, *stop_offset));
-        }
-        Ok(FastFieldsReader {
-               field_offsets: field_offsets_map,
-               source: source,
-           })
-    }
-
-    /// Returns the u64 fast value reader if the field
-    /// is a u64 field indexed as "fast".
-    ///
-    /// Return None if the field is not a u64 field
-    /// indexed with the fast option.
-    ///
-    /// # Panics
-    /// May panic if the index is corrupted.
-    pub fn open_reader<FFReader: FastFieldReader>(&self, field: Field) -> Option<FFReader> {
-        self.field_offsets
-            .get(&field)
-            .map(|&(start, stop)| {
-                     let field_source = self.source.slice(start as usize, stop as usize);
-                     FFReader::open(field_source)
-                 })
+        let source = directory.open_read(path).expect("Failed to open the file");
+        let composite_file =
+            CompositeFile::open(&source).expect("Failed to read the composite file");
+        let field_source = composite_file
+            .open_read(field)
+            .expect("File component not found");
+        FastFieldReader::open(field_source)
     }
 }

src/fastfield/serializer.rs

@@ -1,9 +1,11 @@
 use common::BinarySerializable;
 use directory::WritePtr;
 use schema::Field;
-use common::bitpacker::{compute_num_bits, BitPacker};
+use common::bitpacker::BitPacker;
+use common::compute_num_bits;
 use common::CountingWriter;
-use std::io::{self, Write, Seek, SeekFrom};
+use common::CompositeWrite;
+use std::io::{self, Write};
 
 /// `FastFieldSerializer` is in charge of serializing
 /// fastfields on disk.
@@ -26,85 +28,92 @@ use std::io::{self, Write, Seek, SeekFrom};
 /// * `close_field()`
 /// * `close()`
 pub struct FastFieldSerializer {
-    write: CountingWriter<WritePtr>,
-    fields: Vec<(Field, u32)>,
-    min_value: u64,
-    field_open: bool,
-    bit_packer: BitPacker,
+    composite_write: CompositeWrite<WritePtr>,
 }
 
-
 impl FastFieldSerializer {
     /// Constructor
-    pub fn new(write: WritePtr) -> io::Result<FastFieldSerializer> {
+    pub fn from_write(write: WritePtr) -> io::Result<FastFieldSerializer> {
         // just making room for the pointer to header.
-        let mut counting_writer = CountingWriter::wrap(write);
-        0u32.serialize(&mut counting_writer)?;
-        Ok(FastFieldSerializer {
-               write: counting_writer,
-               fields: Vec::new(),
-               min_value: 0,
-               field_open: false,
-               bit_packer: BitPacker::new(0),
-           })
+        let composite_write = CompositeWrite::wrap(write);
+        Ok(FastFieldSerializer { composite_write })
     }
 
     /// Start serializing a new u64 fast field
-    pub fn new_u64_fast_field(&mut self,
-                              field: Field,
-                              min_value: u64,
-                              max_value: u64)
-                              -> io::Result<()> {
-        if self.field_open {
-            return Err(io::Error::new(io::ErrorKind::Other, "Previous field not closed"));
-        }
-        self.min_value = min_value;
-        self.field_open = true;
-        self.fields.push((field, self.write.written_bytes() as u32));
-        let write = &mut self.write;
-        min_value.serialize(write)?;
-        let amplitude = max_value - min_value;
-        amplitude.serialize(write)?;
-        let num_bits = compute_num_bits(amplitude);
-        self.bit_packer = BitPacker::new(num_bits as usize);
-        Ok(())
+    pub fn new_u64_fast_field(
+        &mut self,
+        field: Field,
+        min_value: u64,
+        max_value: u64,
+    ) -> io::Result<FastSingleFieldSerializer<CountingWriter<WritePtr>>> {
+        self.new_u64_fast_field_with_idx(field, min_value, max_value, 0)
     }
 
-
-    /// Pushes a new value to the currently open u64 fast field.
-    pub fn add_val(&mut self, val: u64) -> io::Result<()> {
-        let val_to_write: u64 = val - self.min_value;
-        self.bit_packer.write(val_to_write, &mut self.write)?;
-        Ok(())
+    /// Start serializing a new u64 fast field
+    pub fn new_u64_fast_field_with_idx(
+        &mut self,
+        field: Field,
+        min_value: u64,
+        max_value: u64,
+        idx: usize,
+    ) -> io::Result<FastSingleFieldSerializer<CountingWriter<WritePtr>>> {
+        let field_write = self.composite_write.for_field_with_idx(field, idx);
+        FastSingleFieldSerializer::open(field_write, min_value, max_value)
     }
 
-    /// Close the u64 fast field.
-    pub fn close_field(&mut self) -> io::Result<()> {
-        if !self.field_open {
-            return Err(io::Error::new(io::ErrorKind::Other, "Current field is already closed"));
-        }
-        self.field_open = false;
-        // adding some padding to make sure we
-        // can read the last elements with our u64
-        // cursor
-        self.bit_packer.close(&mut self.write)?;
-        Ok(())
-    }
-
-
     /// Closes the serializer
     ///
     /// After this call the data must be persistently save on disk.
-    pub fn close(self) -> io::Result<usize> {
-        if self.field_open {
-            return Err(io::Error::new(io::ErrorKind::Other, "Last field not closed"));
-        }
-        let header_offset: usize = self.write.written_bytes() as usize;
-        let (mut write, written_size) = self.write.finish()?;
-        self.fields.serialize(&mut write)?;
-        write.seek(SeekFrom::Start(0))?;
-        (header_offset as u32).serialize(&mut write)?;
-        write.flush()?;
-        Ok(written_size)
+    pub fn close(self) -> io::Result<()> {
+        self.composite_write.close()
+    }
+}
+
+pub struct FastSingleFieldSerializer<'a, W: Write + 'a> {
+    bit_packer: BitPacker,
+    write: &'a mut W,
+    min_value: u64,
+    num_bits: u8,
+}
+
+impl<'a, W: Write> FastSingleFieldSerializer<'a, W> {
+
+    /// Creates a new fast field serializer.
+    ///
+    /// The serializer in fact encode the values by bitpacking
+    /// `(val - min_value)`.
+    ///
+    /// It requires a `min_value` and a `max_value` to compute
+    /// compute the minimum number of bits required to encode
+    /// values.
+    fn open(
+        write: &'a mut W,
+        min_value: u64,
+        max_value: u64,
+    ) -> io::Result<FastSingleFieldSerializer<'a, W>> {
+        assert!(min_value <= max_value);
+        min_value.serialize(write)?;
+        let amplitude = max_value - min_value;
+        amplitude.serialize(write)?;
+        let num_bits = compute_num_bits(amplitude);
+        let bit_packer = BitPacker::new();
+        Ok(FastSingleFieldSerializer {
+            write,
+            bit_packer,
+            min_value,
+            num_bits,
+        })
+    }
+
+    /// Pushes a new value to the currently open u64 fast field.
+    pub fn add_val(&mut self, val: u64) -> io::Result<()> {
+        let val_to_write: u64 = val - self.min_value;
+        self.bit_packer
+            .write(val_to_write, self.num_bits, &mut self.write)?;
+        Ok(())
+    }
+
+    pub fn close_field(mut self) -> io::Result<()> {
+        self.bit_packer.close(&mut self.write)
     }
 }

src/fastfield/writer.rs

@@ -1,82 +1,120 @@
-use schema::{Schema, Field, Document};
+use schema::{Cardinality, Document, Field, Schema};
 use fastfield::FastFieldSerializer;
 use std::io;
-use schema::Value;
 use DocId;
 use schema::FieldType;
 use common;
 use common::VInt;
+use std::collections::HashMap;
+use postings::UnorderedTermId;
+use super::multivalued::MultiValueIntFastFieldWriter;
 use common::BinarySerializable;
+use termdict::TermOrdinal;
 
 /// The fastfieldswriter regroup all of the fast field writers.
 pub struct FastFieldsWriter {
-    field_writers: Vec<IntFastFieldWriter>,
+    single_value_writers: Vec<IntFastFieldWriter>,
+    multi_values_writers: Vec<MultiValueIntFastFieldWriter>,
 }
 
 impl FastFieldsWriter {
     /// Create all `FastFieldWriter` required by the schema.
     pub fn from_schema(schema: &Schema) -> FastFieldsWriter {
-        let field_writers: Vec<IntFastFieldWriter> = schema
-            .fields()
-            .iter()
-            .enumerate()
-            .flat_map(|(field_id, field_entry)| {
-                let field = Field(field_id as u32);
-                match *field_entry.field_type() {
-                    FieldType::I64(ref int_options) => {
-                        if int_options.is_fast() {
+        let mut single_value_writers = Vec::new();
+        let mut multi_values_writers = Vec::new();
+
+        for (field_id, field_entry) in schema.fields().iter().enumerate() {
+            let field = Field(field_id as u32);
+            let default_value = if let FieldType::I64(_) = *field_entry.field_type() {
+                common::i64_to_u64(0i64)
+            } else {
+                0u64
+            };
+            match *field_entry.field_type() {
+                FieldType::I64(ref int_options) | FieldType::U64(ref int_options) => {
+                    match int_options.get_fastfield_cardinality() {
+                        Some(Cardinality::SingleValue) => {
                             let mut fast_field_writer = IntFastFieldWriter::new(field);
-                            fast_field_writer.set_val_if_missing(common::i64_to_u64(0i64));
-                            Some(fast_field_writer)
-                        } else {
-                            None
+                            fast_field_writer.set_val_if_missing(default_value);
+                            single_value_writers.push(fast_field_writer);
                         }
-                    }
-                    FieldType::U64(ref int_options) => {
-                        if int_options.is_fast() {
-                            Some(IntFastFieldWriter::new(field))
-                        } else {
-                            None
+                        Some(Cardinality::MultiValues) => {
+                            let fast_field_writer = MultiValueIntFastFieldWriter::new(field, false);
+                            multi_values_writers.push(fast_field_writer);
                         }
+                        None => {}
                     }
-                    _ => None,
                 }
-            })
-            .collect();
-        FastFieldsWriter { field_writers: field_writers }
+                FieldType::HierarchicalFacet => {
+                    let fast_field_writer = MultiValueIntFastFieldWriter::new(field, true);
+                    multi_values_writers.push(fast_field_writer);
+                }
+                _ => {}
+            }
+        }
+        FastFieldsWriter {
+            single_value_writers,
+            multi_values_writers,
+        }
     }
 
-    /// Returns a `FastFieldsWriter`
-    /// with a `IntFastFieldWriter` for each
+    /// Returns a `FastFieldsWriter with a `u64` `IntFastFieldWriter` for each
     /// of the field given in argument.
-    pub fn new(fields: Vec<Field>) -> FastFieldsWriter {
+    pub(crate) fn new(fields: Vec<Field>) -> FastFieldsWriter {
         FastFieldsWriter {
-            field_writers: fields.into_iter().map(IntFastFieldWriter::new).collect(),
+            single_value_writers: fields.into_iter().map(IntFastFieldWriter::new).collect(),
+            multi_values_writers: vec![],
         }
     }
 
     /// Get the `FastFieldWriter` associated to a field.
     pub fn get_field_writer(&mut self, field: Field) -> Option<&mut IntFastFieldWriter> {
         // TODO optimize
-        self.field_writers
+        self.single_value_writers
             .iter_mut()
-            .find(|field_writer| field_writer.field == field)
+            .find(|field_writer| field_writer.field() == field)
     }
 
+    /// Returns the fast field multi-value writer for the given field.
+    ///
+    /// Returns None if the field does not exist, or is not
+    /// configured as a multivalued fastfield in the schema.
+    pub(crate) fn get_multivalue_writer(
+        &mut self,
+        field: Field,
+    ) -> Option<&mut MultiValueIntFastFieldWriter> {
+        // TODO optimize
+        // TODO expose for users
+        self.multi_values_writers
+            .iter_mut()
+            .find(|multivalue_writer| multivalue_writer.field() == field)
+    }
 
     /// Indexes all of the fastfields of a new document.
     pub fn add_document(&mut self, doc: &Document) {
-        for field_writer in &mut self.field_writers {
+        for field_writer in &mut self.single_value_writers {
+            field_writer.add_document(doc);
+        }
+        for field_writer in &mut self.multi_values_writers {
+            field_writer.next_doc();
             field_writer.add_document(doc);
         }
     }
 
     /// Serializes all of the `FastFieldWriter`s by pushing them in
     /// order to the fast field serializer.
-    pub fn serialize(&self, serializer: &mut FastFieldSerializer) -> io::Result<()> {
-        for field_writer in &self.field_writers {
+    pub fn serialize(
+        &self,
+        serializer: &mut FastFieldSerializer,
+        mapping: &HashMap<Field, HashMap<UnorderedTermId, TermOrdinal>>,
+    ) -> io::Result<()> {
+        for field_writer in &self.single_value_writers {
             field_writer.serialize(serializer)?;
         }
+        for field_writer in &self.multi_values_writers {
+            let field = field_writer.field();
+            field_writer.serialize(serializer, mapping.get(&field))?;
+        }
         Ok(())
     }
 
@@ -85,7 +123,7 @@ impl FastFieldsWriter {
     ///
     /// The missing values will be filled with 0.
     pub fn fill_val_up_to(&mut self, doc: DocId) {
-        for field_writer in &mut self.field_writers {
+        for field_writer in &mut self.single_value_writers {
             field_writer.fill_val_up_to(doc);
         }
     }
@@ -119,7 +157,7 @@ impl IntFastFieldWriter {
     /// Creates a new `IntFastFieldWriter`
     pub fn new(field: Field) -> IntFastFieldWriter {
         IntFastFieldWriter {
-            field: field,
+            field,
             vals: Vec::new(),
             val_count: 0,
             val_if_missing: 0u64,
@@ -128,6 +166,11 @@ impl IntFastFieldWriter {
         }
     }
 
+    /// Returns the field that this writer is targetting.
+    pub fn field(&self) -> Field {
+        self.field
+    }
+
     /// Sets the default value.
     ///
     /// This default value is recorded for documents if
@@ -169,7 +212,6 @@ impl IntFastFieldWriter {
         self.val_count += 1;
     }
 
-
     /// Extract the value associated to the fast field for
     /// this document.
     ///
@@ -182,13 +224,7 @@ impl IntFastFieldWriter {
     /// only the first one is taken in account.
     fn extract_val(&self, doc: &Document) -> u64 {
         match doc.get_first(self.field) {
-            Some(v) => {
-                match *v {
-                    Value::U64(ref val) => *val,
-                    Value::I64(ref val) => common::i64_to_u64(*val),
-                    _ => panic!("Expected a u64field, got {:?} ", v),
-                }
-            }
+            Some(v) => super::value_to_u64(v),
             None => self.val_if_missing,
         }
     }
@@ -208,13 +244,13 @@ impl IntFastFieldWriter {
             (self.val_min, self.val_max)
         };
 
-        serializer.new_u64_fast_field(self.field, min, max)?;
+        let mut single_field_serializer = serializer.new_u64_fast_field(self.field, min, max)?;
 
         let mut cursor = self.vals.as_slice();
         while let Ok(VInt(val)) = VInt::deserialize(&mut cursor) {
-            serializer.add_val(val)?;
+            single_field_serializer.add_val(val)?;
         }
 
-        serializer.close_field()
+        single_field_serializer.close_field()
     }
 }

src/functional_test.rs

@@ -14,7 +14,6 @@ fn check_index_content(searcher: &Searcher, vals: &HashSet<u64>) {
 #[test]
 #[ignore]
 fn test_indexing() {
-
     let mut schema_builder = SchemaBuilder::default();
 
     let id_field = schema_builder.add_u64_field("id", INT_INDEXED);

src/indexer/delete_queue.rs

@@ -3,7 +3,6 @@ use std::sync::{Arc, RwLock};
 use std::mem;
 use std::ops::DerefMut;
 
-
 // The DeleteQueue is similar in conceptually to a multiple
 // consumer single producer broadcast channel.
 //
@@ -16,7 +15,7 @@ use std::ops::DerefMut;
 // - calling `delete_queue.cursor()` returns a cursor, that
 //   will include all future delete operation (and no past operations).
 // - cloning an existing cursor returns a new cursor, that
-//   is at the exact same position, and can now advance independantly
+//   is at the exact same position, and can now advance independently
 //   from the original cursor.
 #[derive(Default)]
 struct InnerDeleteQueue {
@@ -29,26 +28,25 @@ pub struct DeleteQueue {
     inner: Arc<RwLock<InnerDeleteQueue>>,
 }
 
-
 impl DeleteQueue {
     // Creates a new delete queue.
     pub fn new() -> DeleteQueue {
-
-        let delete_queue = DeleteQueue { inner: Arc::default() };
+        let delete_queue = DeleteQueue {
+            inner: Arc::default(),
+        };
 
         let next_block = NextBlock::from(delete_queue.clone());
         {
             let mut delete_queue_wlock = delete_queue.inner.write().unwrap();
             delete_queue_wlock.last_block = Some(Arc::new(Block {
-                                                              operations: Arc::default(),
-                                                              next: next_block,
-                                                          }));
+                operations: Arc::default(),
+                next: next_block,
+            }));
         }
 
         delete_queue
     }
 
-
     // Creates a new cursor that makes it possible to
     // consume future delete operations.
     //
@@ -59,9 +57,11 @@ impl DeleteQueue {
             .expect("Read lock poisoned when opening delete queue cursor")
             .last_block
             .clone()
-            .expect("Failed to unwrap last_block. This should never happen
+            .expect(
+                "Failed to unwrap last_block. This should never happen
                 as the Option<> is only here to make
-                initialization possible");
+                initialization possible",
+            );
         let operations_len = last_block.operations.len();
         DeleteCursor {
             block: last_block,
@@ -108,9 +108,9 @@ impl DeleteQueue {
         let next_block = NextBlock::from(self.clone());
         {
             self_wlock.last_block = Some(Arc::new(Block {
-                                                      operations: Arc::new(delete_operations),
-                                                      next: next_block,
-                                                  }));
+                operations: Arc::new(delete_operations),
+                next: next_block,
+            }));
         }
         self_wlock.last_block.clone()
     }
@@ -136,7 +136,7 @@ impl NextBlock {
                 .read()
                 .expect("Failed to acquire write lock in delete queue");
             if let InnerNextBlock::Closed(ref block) = *next_read_lock {
-                return Some(block.clone());
+                return Some(Arc::clone(block));
             }
         }
         let next_block;
@@ -146,20 +146,18 @@ impl NextBlock {
                 .expect("Failed to acquire write lock in delete queue");
             match *next_write_lock {
                 InnerNextBlock::Closed(ref block) => {
-                    return Some(block.clone());
+                    return Some(Arc::clone(block));
                 }
-                InnerNextBlock::Writer(ref writer) => {
-                    match writer.flush() {
-                        Some(flushed_next_block) => {
-                            next_block = flushed_next_block;
-                        }
-                        None => {
-                            return None;
-                        }
+                InnerNextBlock::Writer(ref writer) => match writer.flush() {
+                    Some(flushed_next_block) => {
+                        next_block = flushed_next_block;
                     }
-                }
+                    None => {
+                        return None;
+                    }
+                },
             }
-            *next_write_lock.deref_mut() = InnerNextBlock::Closed(next_block.clone());
+            *next_write_lock.deref_mut() = InnerNextBlock::Closed(Arc::clone(&next_block));
             Some(next_block)
         }
     }
@@ -170,14 +168,12 @@ struct Block {
     next: NextBlock,
 }
 
-
 #[derive(Clone)]
 pub struct DeleteCursor {
     block: Arc<Block>,
     pos: usize,
 }
 
-
 impl DeleteCursor {
     /// Skips operations and position it so that
     /// - either all of the delete operation currently in the
@@ -246,16 +242,11 @@ impl DeleteCursor {
     }
 }
 
-
-
-
-
-
 #[cfg(test)]
 mod tests {
 
-    use super::{DeleteQueue, DeleteOperation};
-    use schema::{Term, Field};
+    use super::{DeleteOperation, DeleteQueue};
+    use schema::{Field, Term};
 
     #[test]
     fn test_deletequeue() {
@@ -302,6 +293,5 @@ mod tests {
             operations_it.advance();
             assert!(operations_it.get().is_none());
         }
-
     }
 }

src/indexer/directory_lock.rs

@@ -2,8 +2,6 @@ use Directory;
 use directory::error::OpenWriteError;
 use core::LOCKFILE_FILEPATH;
 
-
-
 /// The directory lock is a mechanism used to
 /// prevent the creation of two [`IndexWriter`](struct.IndexWriter.html)
 ///
@@ -15,8 +13,8 @@ pub struct DirectoryLock {
 
 impl DirectoryLock {
     pub fn lock(mut directory: Box<Directory>) -> Result<DirectoryLock, OpenWriteError> {
-        try!(directory.open_write(&*LOCKFILE_FILEPATH));
-        Ok(DirectoryLock { directory: directory })
+        directory.open_write(&*LOCKFILE_FILEPATH)?;
+        Ok(DirectoryLock { directory })
     }
 }
 

src/indexer/doc_opstamp_mapping.rs

@@ -1,7 +1,6 @@
 use std::sync::Arc;
 use DocId;
 
-
 // Doc to opstamp is used to identify which
 // document should be deleted.
 //
@@ -56,8 +55,10 @@ mod tests {
     #[test]
     fn test_doc_to_opstamp_mapping_none() {
         let doc_to_opstamp_mapping = DocToOpstampMapping::None;
-        assert_eq!(doc_to_opstamp_mapping.compute_doc_limit(1),
-                   u32::max_value());
+        assert_eq!(
+            doc_to_opstamp_mapping.compute_doc_limit(1),
+            u32::max_value()
+        );
     }
 
     #[test]

src/indexer/index_writer.rs

@@ -10,7 +10,6 @@ use indexer::stamper::Stamper;
 use datastruct::stacker::Heap;
 use directory::FileProtection;
 use error::{Error, ErrorKind, Result, ResultExt};
-use Directory;
 use fastfield::write_delete_bitset;
 use indexer::delete_queue::{DeleteCursor, DeleteQueue};
 use futures::Canceled;
@@ -21,17 +20,17 @@ use indexer::MergePolicy;
 use indexer::operation::DeleteOperation;
 use indexer::SegmentEntry;
 use indexer::SegmentWriter;
-use postings::DocSet;
-use postings::SegmentPostingsOption;
+use docset::DocSet;
+use schema::IndexRecordOption;
 use schema::Document;
-use schema::Schema;
 use schema::Term;
 use std::mem;
 use std::mem::swap;
 use std::thread::JoinHandle;
-use super::directory_lock::DirectoryLock;
+use indexer::DirectoryLock;
 use super::operation::AddOperation;
 use super::segment_updater::SegmentUpdater;
+use super::PreparedCommit;
 use std::thread;
 
 // Size of the margin for the heap. A segment is closed when the remaining memory
@@ -52,12 +51,12 @@ type DocumentReceiver = chan::Receiver<AddOperation>;
 ///
 /// It manages a small number of indexing thread, as well as a shared
 /// indexing queue.
-/// Each indexing thread builds its own independant `Segment`, via
+/// Each indexing thread builds its own independent `Segment`, via
 /// a `SegmentWriter` object.
 pub struct IndexWriter {
     // the lock is just used to bind the
     // lifetime of the lock with that of the IndexWriter.
-    _directory_lock: DirectoryLock,
+    _directory_lock: Option<DirectoryLock>,
 
     index: Index,
 
@@ -86,8 +85,6 @@ pub struct IndexWriter {
 impl !Send for IndexWriter {}
 impl !Sync for IndexWriter {}
 
-
-
 /// Open a new index writer. Attempts to acquire a lockfile.
 ///
 /// The lockfile should be deleted on drop, but it is possible
@@ -102,49 +99,48 @@ impl !Sync for IndexWriter {}
 /// If the lockfile already exists, returns `Error::FileAlreadyExists`.
 /// # Panics
 /// If the heap size per thread is too small, panics.
-pub fn open_index_writer(index: &Index,
-                         num_threads: usize,
-                         heap_size_in_bytes_per_thread: usize)
-                         -> Result<IndexWriter> {
-
-    if heap_size_in_bytes_per_thread <= HEAP_SIZE_LIMIT as usize {
-        panic!(format!("The heap size per thread needs to be at least {}.",
-                       HEAP_SIZE_LIMIT));
+pub fn open_index_writer(
+    index: &Index,
+    num_threads: usize,
+    heap_size_in_bytes_per_thread: usize,
+    directory_lock: DirectoryLock,
+) -> Result<IndexWriter> {
+    if heap_size_in_bytes_per_thread < HEAP_SIZE_LIMIT as usize {
+        panic!(format!(
+            "The heap size per thread needs to be at least {}.",
+            HEAP_SIZE_LIMIT
+        ));
     }
-
-    let directory_lock = DirectoryLock::lock(index.directory().box_clone())?;
-
     let (document_sender, document_receiver): (DocumentSender, DocumentReceiver) =
         chan::sync(PIPELINE_MAX_SIZE_IN_DOCS);
 
-
     let delete_queue = DeleteQueue::new();
 
-    let current_opstamp = index.opstamp();
+    let current_opstamp = index.load_metas()?.opstamp;
 
     let stamper = Stamper::new(current_opstamp);
 
     let segment_updater =
-        SegmentUpdater::new(index.clone(), stamper.clone(), delete_queue.cursor())?;
+        SegmentUpdater::new(index.clone(), stamper.clone(), &delete_queue.cursor())?;
 
     let mut index_writer = IndexWriter {
-        _directory_lock: directory_lock,
+        _directory_lock: Some(directory_lock),
 
-        heap_size_in_bytes_per_thread: heap_size_in_bytes_per_thread,
+        heap_size_in_bytes_per_thread,
         index: index.clone(),
 
-        document_receiver: document_receiver,
-        document_sender: document_sender,
+        document_receiver,
+        document_sender,
 
-        segment_updater: segment_updater,
+        segment_updater,
 
         workers_join_handle: vec![],
-        num_threads: num_threads,
+        num_threads,
 
-        delete_queue: delete_queue,
+        delete_queue,
 
         committed_opstamp: current_opstamp,
-        stamper: stamper,
+        stamper,
 
         generation: 0,
 
@@ -154,15 +150,13 @@ pub fn open_index_writer(index: &Index,
     Ok(index_writer)
 }
 
-
-
-pub fn compute_deleted_bitset(delete_bitset: &mut BitSet,
-                              segment_reader: &SegmentReader,
-                              delete_cursor: &mut DeleteCursor,
-                              doc_opstamps: &DocToOpstampMapping,
-                              target_opstamp: u64)
-                              -> Result<bool> {
-
+pub fn compute_deleted_bitset(
+    delete_bitset: &mut BitSet,
+    segment_reader: &SegmentReader,
+    delete_cursor: &mut DeleteCursor,
+    doc_opstamps: &DocToOpstampMapping,
+    target_opstamp: u64,
+) -> Result<bool> {
     let mut might_have_changed = false;
 
     #[cfg_attr(feature = "cargo-clippy", allow(while_let_loop))]
@@ -177,8 +171,10 @@ pub fn compute_deleted_bitset(delete_bitset: &mut BitSet,
                 // Limit doc helps identify the first document
                 // that may be affected by the delete operation.
                 let limit_doc = doc_opstamps.compute_doc_limit(delete_op.opstamp);
+                let inverted_index = segment_reader.inverted_index(delete_op.term.field());
                 if let Some(mut docset) =
-                    segment_reader.read_postings(&delete_op.term, SegmentPostingsOption::NoFreq) {
+                    inverted_index.read_postings(&delete_op.term, IndexRecordOption::Basic)
+                {
                     while docset.advance() {
                         let deleted_doc = docset.doc();
                         if deleted_doc < limit_doc {
@@ -198,13 +194,12 @@ pub fn compute_deleted_bitset(delete_bitset: &mut BitSet,
 
 /// Advance delete for the given segment up
 /// to the target opstamp.
-pub fn advance_deletes(mut segment: Segment,
-                       segment_entry: &mut SegmentEntry,
-                       target_opstamp: u64)
-                       -> Result<Option<FileProtection>> {
-
+pub fn advance_deletes(
+    mut segment: Segment,
+    segment_entry: &mut SegmentEntry,
+    target_opstamp: u64,
+) -> Result<Option<FileProtection>> {
     let mut file_protect: Option<FileProtection> = None;
-
     {
         if let Some(previous_opstamp) = segment_entry.meta().delete_opstamp() {
             // We are already up-to-date here.
@@ -212,7 +207,7 @@ pub fn advance_deletes(mut segment: Segment,
                 return Ok(file_protect);
             }
         }
-        let segment_reader = SegmentReader::open(segment.clone())?;
+        let segment_reader = SegmentReader::open(&segment)?;
         let max_doc = segment_reader.max_doc();
 
         let mut delete_bitset: BitSet = match segment_entry.delete_bitset() {
@@ -222,11 +217,13 @@ pub fn advance_deletes(mut segment: Segment,
 
         let delete_cursor = segment_entry.delete_cursor();
 
-        compute_deleted_bitset(&mut delete_bitset,
-                               &segment_reader,
-                               delete_cursor,
-                               &DocToOpstampMapping::None,
-                               target_opstamp)?;
+        compute_deleted_bitset(
+            &mut delete_bitset,
+            &segment_reader,
+            delete_cursor,
+            &DocToOpstampMapping::None,
+            target_opstamp,
+        )?;
 
         for doc in 0u32..max_doc {
             if segment_reader.is_deleted(doc) {
@@ -243,30 +240,33 @@ pub fn advance_deletes(mut segment: Segment,
         }
     }
     segment_entry.set_meta(segment.meta().clone());
-
     Ok(file_protect)
 }
 
-fn index_documents(heap: &mut Heap,
-                   table_size: usize,
-                   segment: Segment,
-                   schema: &Schema,
-                   generation: usize,
-                   document_iterator: &mut Iterator<Item = AddOperation>,
-                   segment_updater: &mut SegmentUpdater,
-                   mut delete_cursor: DeleteCursor)
-                   -> Result<bool> {
+fn index_documents(
+    heap: &mut Heap,
+    table_size: usize,
+    segment: &Segment,
+    generation: usize,
+    document_iterator: &mut Iterator<Item = AddOperation>,
+    segment_updater: &mut SegmentUpdater,
+    mut delete_cursor: DeleteCursor,
+) -> Result<bool> {
     heap.clear();
+    let schema = segment.schema();
     let segment_id = segment.id();
-    let mut segment_writer = SegmentWriter::for_segment(heap, table_size, segment.clone(), schema)?;
+    let mut segment_writer =
+        SegmentWriter::for_segment(heap, table_size, segment.clone(), &schema)?;
     for doc in document_iterator {
-        try!(segment_writer.add_document(&doc, schema));
+        segment_writer.add_document(doc, &schema)?;
         // There is two possible conditions to close the segment.
         // One is the memory arena dedicated to the segment is
         // getting full.
         if segment_writer.is_buffer_full() {
-            info!("Buffer limit reached, flushing segment with maxdoc={}.",
-                  segment_writer.max_doc());
+            info!(
+                "Buffer limit reached, flushing segment with maxdoc={}.",
+                segment_writer.max_doc()
+            );
             break;
         }
         // The second is the term dictionary hash table
@@ -275,11 +275,18 @@ fn index_documents(heap: &mut Heap,
         // Tantivy does not resize its hashtable. When it reaches
         // capacity, we just stop indexing new document.
         if segment_writer.is_term_saturated() {
-            info!("Term dic saturated, flushing segment with maxdoc={}.",
-                  segment_writer.max_doc());
+            info!(
+                "Term dic saturated, flushing segment with maxdoc={}.",
+                segment_writer.max_doc()
+            );
             break;
         }
     }
+
+    if !segment_updater.is_alive() {
+        return Ok(false);
+    }
+
     let num_docs = segment_writer.max_doc();
 
     // this is ensured by the call to peek before starting
@@ -296,11 +303,13 @@ fn index_documents(heap: &mut Heap,
     let doc_to_opstamps = DocToOpstampMapping::from(doc_opstamps);
     let segment_reader = SegmentReader::open(segment)?;
     let mut deleted_bitset = BitSet::with_capacity(num_docs as usize);
-    let may_have_deletes = compute_deleted_bitset(&mut deleted_bitset,
-                                                  &segment_reader,
-                                                  &mut delete_cursor,
-                                                  &doc_to_opstamps,
-                                                  last_docstamp)?;
+    let may_have_deletes = compute_deleted_bitset(
+        &mut deleted_bitset,
+        &segment_reader,
+        &mut delete_cursor,
+        &doc_to_opstamps,
+        last_docstamp,
+    )?;
 
     let segment_entry = SegmentEntry::new(segment_meta, delete_cursor, {
         if may_have_deletes {
@@ -311,13 +320,11 @@ fn index_documents(heap: &mut Heap,
     });
 
     Ok(segment_updater.add_segment(generation, segment_entry))
-
 }
 
 impl IndexWriter {
     /// The index writer
     pub fn wait_merging_threads(mut self) -> Result<()> {
-
         // this will stop the indexing thread,
         // dropping the last reference to the segment_updater.
         drop(self.document_sender);
@@ -331,10 +338,9 @@ impl IndexWriter {
         }
         drop(self.workers_join_handle);
 
-        let result =
-            self.segment_updater
-                .wait_merging_thread()
-                .chain_err(|| ErrorKind::ErrorInThread("Failed to join merging thread.".into()));
+        let result = self.segment_updater
+            .wait_merging_thread()
+            .chain_err(|| ErrorKind::ErrorInThread("Failed to join merging thread.".into()));
 
         if let Err(ref e) = result {
             error!("Some merging thread failed {:?}", e);
@@ -351,7 +357,11 @@ impl IndexWriter {
             .add_segment(self.generation, segment_entry);
     }
 
-    #[doc(hidden)]
+    /// *Experimental & Advanced API* Creates a new segment.
+    /// and marks it as currently in write.
+    ///
+    /// This method is useful only for users trying to do complex
+    /// operations, like converting an index format to another.
     pub fn new_segment(&self) -> Segment {
         self.segment_updater.new_segment()
     }
@@ -360,7 +370,6 @@ impl IndexWriter {
     /// The thread consumes documents from the pipeline.
     ///
     fn add_indexing_worker(&mut self) -> Result<()> {
-        let schema = self.index.schema();
         let document_receiver_clone = self.document_receiver.clone();
         let mut segment_updater = self.segment_updater.clone();
         let (heap_size, table_size) = split_memory(self.heap_size_in_bytes_per_thread);
@@ -372,11 +381,12 @@ impl IndexWriter {
         let mut delete_cursor = self.delete_queue.cursor();
 
         let join_handle: JoinHandle<Result<()>> = thread::Builder::new()
-            .name(format!("indexing thread {} for gen {}", self.worker_id, generation))
+            .name(format!(
+                "indexing thread {} for gen {}",
+                self.worker_id, generation
+            ))
             .spawn(move || {
-
                 loop {
-
                     let mut document_iterator =
                         document_receiver_clone.clone().into_iter().peekable();
 
@@ -396,15 +406,15 @@ impl IndexWriter {
                         return Ok(());
                     }
                     let segment = segment_updater.new_segment();
-                    index_documents(&mut heap,
-                                    table_size,
-                                    segment,
-                                    &schema,
-                                    generation,
-                                    &mut document_iterator,
-                                    &mut segment_updater,
-                                    delete_cursor.clone())?;
-
+                    index_documents(
+                        &mut heap,
+                        table_size,
+                        &segment,
+                        generation,
+                        &mut document_iterator,
+                        &mut segment_updater,
+                        delete_cursor.clone(),
+                    )?;
                 }
             })?;
         self.worker_id += 1;
@@ -424,7 +434,7 @@ impl IndexWriter {
 
     fn start_workers(&mut self) -> Result<()> {
         for _ in 0..self.num_threads {
-            try!(self.add_indexing_worker());
+            self.add_indexing_worker()?;
         }
         Ok(())
     }
@@ -436,9 +446,10 @@ impl IndexWriter {
     }
 
     /// Merges a given list of segments
-    pub fn merge(&mut self,
-                 segment_ids: &[SegmentId])
-                 -> impl Future<Item = SegmentMeta, Error = Canceled> {
+    pub fn merge(
+        &mut self,
+        segment_ids: &[SegmentId],
+    ) -> impl Future<Item = SegmentMeta, Error = Canceled> {
         self.segment_updater.start_merge(segment_ids)
     }
 
@@ -451,9 +462,10 @@ impl IndexWriter {
     ///
     /// Returns the former segment_ready channel.
     fn recreate_document_channel(&mut self) -> DocumentReceiver {
-        let (mut document_sender, mut document_receiver): (DocumentSender,
-                                                           DocumentReceiver) =
-            chan::sync(PIPELINE_MAX_SIZE_IN_DOCS);
+        let (mut document_sender, mut document_receiver): (
+            DocumentSender,
+            DocumentReceiver,
+        ) = chan::sync(PIPELINE_MAX_SIZE_IN_DOCS);
         swap(&mut self.document_sender, &mut document_sender);
         swap(&mut self.document_receiver, &mut document_receiver);
         document_receiver
@@ -467,26 +479,102 @@ impl IndexWriter {
     /// state as it was after the last commit.
     ///
     /// The opstamp at the last commit is returned.
-    pub fn rollback(mut self) -> Result<IndexWriter> {
+    pub fn rollback(&mut self) -> Result<()> {
         info!("Rolling back to opstamp {}", self.committed_opstamp);
 
+        // marks the segment updater as killed. From now on, all
+        // segment updates will be ignored.
         self.segment_updater.kill();
 
+        let document_receiver = self.document_receiver.clone();
+
+        // take the directory lock to create a new index_writer.
+        let directory_lock = self._directory_lock
+            .take()
+            .expect("The IndexWriter does not have any lock. This is a bug, please report.");
+
+        let new_index_writer: IndexWriter = open_index_writer(
+            &self.index,
+            self.num_threads,
+            self.heap_size_in_bytes_per_thread,
+            directory_lock,
+        )?;
+
+        // the current `self` is dropped right away because of this call.
+        //
+        // This will drop the document queue, and the thread
+        // should terminate.
+        mem::replace(self, new_index_writer);
+
         // Drains the document receiver pipeline :
         // Workers don't need to index the pending documents.
-        let receiver_clone = self.document_receiver.clone();
-        let index = self.index.clone();
-        let num_threads = self.num_threads;
-        let heap_size_in_bytes_per_thread = self.heap_size_in_bytes_per_thread;
-        drop(self);
-        for _ in receiver_clone {}
-
-        let index_writer = open_index_writer(&index, num_threads, heap_size_in_bytes_per_thread)?;
-
-        Ok(index_writer)
+        //
+        // This will reach an end as the only document_sender
+        // was dropped with the index_writer.
+        for _ in document_receiver.clone() {}
 
+        Ok(())
     }
 
+    /// Prepares a commit.
+    ///
+    /// Calling `prepare_commit()` will cut the indexing
+    /// queue. All pending documents will be sent to the
+    /// indexing workers. They will then terminate, regardless
+    /// of the size of their current segment and flush their
+    /// work on disk.
+    ///
+    /// Once a commit is "prepared", you can either
+    /// call
+    /// * `.commit()`: to accept this commit
+    /// * `.abort()`: to cancel this commit.
+    ///
+    /// In the current implementation, `PreparedCommit` borrows
+    /// the `IndexWriter` mutably so we are guaranteed that no new
+    /// document can be added as long as it is committed or is
+    /// dropped.
+    ///
+    /// It is also possible to add a payload to the `commit`
+    /// using this API.
+    /// See [`PreparedCommit::set_payload()`](PreparedCommit.html)
+    pub fn prepare_commit(&mut self) -> Result<PreparedCommit> {
+        // Here, because we join all of the worker threads,
+        // all of the segment update for this commit have been
+        // sent.
+        //
+        // No document belonging to the next generation have been
+        // pushed too, because add_document can only happen
+        // on this thread.
+
+        // This will move uncommitted segments to the state of
+        // committed segments.
+        info!("Preparing commit");
+
+        // this will drop the current document channel
+        // and recreate a new one channels.
+        self.recreate_document_channel();
+
+        let mut former_workers_join_handle = Vec::new();
+        swap(
+            &mut former_workers_join_handle,
+            &mut self.workers_join_handle,
+        );
+
+        for worker_handle in former_workers_join_handle {
+            let indexing_worker_result = worker_handle
+                .join()
+                .map_err(|e| Error::from_kind(ErrorKind::ErrorInThread(format!("{:?}", e))))?;
+
+            indexing_worker_result?;
+            // add a new worker for the next generation.
+            self.add_indexing_worker()?;
+        }
+
+        let commit_opstamp = self.stamper.stamp();
+        let prepared_commit = PreparedCommit::new(self, commit_opstamp);
+        info!("Prepared commit {}", commit_opstamp);
+        Ok(prepared_commit)
+    }
 
     /// Commits all of the pending changes
     ///
@@ -503,45 +591,11 @@ impl IndexWriter {
     /// that made it in the commit.
     ///
     pub fn commit(&mut self) -> Result<u64> {
+        self.prepare_commit()?.commit()
+    }
 
-        // here, because we join all of the worker threads,
-        // all of the segment update for this commit have been
-        // sent.
-        //
-        // No document belonging to the next generation have been
-        // pushed too, because add_document can only happen
-        // on this thread.
-
-        // This will move uncommitted segments to the state of
-        // committed segments.
-        self.committed_opstamp = self.stamper.stamp();
-        info!("committing {}", self.committed_opstamp);
-
-        // this will drop the current document channel
-        // and recreate a new one channels.
-        self.recreate_document_channel();
-
-        let mut former_workers_join_handle = Vec::new();
-        swap(&mut former_workers_join_handle,
-             &mut self.workers_join_handle);
-
-        for worker_handle in former_workers_join_handle {
-            let indexing_worker_result =
-                worker_handle
-                    .join()
-                    .map_err(|e| Error::from_kind(ErrorKind::ErrorInThread(format!("{:?}", e))))?;
-
-            indexing_worker_result?;
-            // add a new worker for the next generation.
-            self.add_indexing_worker()?;
-        }
-
-
-
-        // wait for the segment update thread to have processed the info
-        self.segment_updater.commit(self.committed_opstamp)?;
-
-        Ok(self.committed_opstamp)
+    pub(crate) fn segment_updater(&self) -> &SegmentUpdater {
+        &self.segment_updater
     }
 
     /// Delete all documents containing a given term.
@@ -554,10 +608,7 @@ impl IndexWriter {
     /// only after calling `commit()`.
     pub fn delete_term(&mut self, term: Term) -> u64 {
         let opstamp = self.stamper.stamp();
-        let delete_operation = DeleteOperation {
-            opstamp: opstamp,
-            term: term,
-        };
+        let delete_operation = DeleteOperation { opstamp, term };
         self.delete_queue.push(delete_operation);
         opstamp
     }
@@ -585,18 +636,12 @@ impl IndexWriter {
     /// have been added since the creation of the index.
     pub fn add_document(&mut self, document: Document) -> u64 {
         let opstamp = self.stamper.stamp();
-        let add_operation = AddOperation {
-            opstamp: opstamp,
-            document: document,
-        };
+        let add_operation = AddOperation { opstamp, document };
         self.document_sender.send(add_operation);
         opstamp
     }
 }
 
-
-
-
 #[cfg(test)]
 mod tests {
 
@@ -623,13 +668,17 @@ mod tests {
         let schema_builder = schema::SchemaBuilder::default();
         let index = Index::create_in_ram(schema_builder.build());
         let index_writer = index.writer(40_000_000).unwrap();
-        assert_eq!(format!("{:?}", index_writer.get_merge_policy()),
-                   "LogMergePolicy { min_merge_size: 8, min_layer_size: 10000, \
-                    level_log_size: 0.75 }");
+        assert_eq!(
+            format!("{:?}", index_writer.get_merge_policy()),
+            "LogMergePolicy { min_merge_size: 8, min_layer_size: 10000, \
+             level_log_size: 0.75 }"
+        );
         let merge_policy = box NoMergePolicy::default();
         index_writer.set_merge_policy(merge_policy);
-        assert_eq!(format!("{:?}", index_writer.get_merge_policy()),
-                   "NoMergePolicy");
+        assert_eq!(
+            format!("{:?}", index_writer.get_merge_policy()),
+            "NoMergePolicy"
+        );
     }
 
     #[test]
@@ -652,33 +701,21 @@ mod tests {
 
         let num_docs_containing = |s: &str| {
             let searcher = index.searcher();
-            let term_a = Term::from_field_text(text_field, s);
-            searcher.doc_freq(&term_a)
+            let term = Term::from_field_text(text_field, s);
+            searcher.doc_freq(&term)
         };
 
         {
             // writing the segment
             let mut index_writer = index.writer_with_num_threads(3, 40_000_000).unwrap();
-            {
-                let mut doc = Document::default();
-                doc.add_text(text_field, "a");
-                index_writer.add_document(doc);
-            }
-
-            index_writer = index_writer.rollback().unwrap();
+            index_writer.add_document(doc!(text_field=>"a"));
+            index_writer.rollback().unwrap();
 
             assert_eq!(index_writer.commit_opstamp(), 0u64);
             assert_eq!(num_docs_containing("a"), 0);
-
             {
-                let mut doc = Document::default();
-                doc.add_text(text_field, "b");
-                index_writer.add_document(doc);
-            }
-            {
-                let mut doc = Document::default();
-                doc.add_text(text_field, "c");
-                index_writer.add_document(doc);
+                index_writer.add_document(doc!(text_field=>"b"));
+                index_writer.add_document(doc!(text_field=>"c"));
             }
             assert_eq!(index_writer.commit().unwrap(), 2u64);
             index.load_searchers().unwrap();
@@ -690,7 +727,6 @@ mod tests {
         index.searcher();
     }
 
-
     #[test]
     fn test_with_merges() {
         let _ = env_logger::init();
@@ -726,9 +762,81 @@ mod tests {
 
             assert_eq!(num_docs_containing("a"), 200);
             assert!(index.searchable_segments().unwrap().len() < 8);
-
         }
     }
 
+    #[test]
+    fn test_prepare_with_commit_message() {
+        let _ = env_logger::init();
+        let mut schema_builder = schema::SchemaBuilder::default();
+        let text_field = schema_builder.add_text_field("text", schema::TEXT);
+        let index = Index::create_in_ram(schema_builder.build());
+
+        {
+            // writing the segment
+            let mut index_writer = index.writer_with_num_threads(4, 4 * 30_000_000).unwrap();
+            // create 8 segments with 100 tiny docs
+            for _doc in 0..100 {
+                index_writer.add_document(doc!(text_field => "a"));
+            }
+            {
+                let mut prepared_commit = index_writer.prepare_commit().expect("commit failed");
+                prepared_commit.set_payload("first commit");
+                assert_eq!(prepared_commit.opstamp(), 100);
+                prepared_commit.commit().expect("commit failed");
+            }
+            {
+                let metas = index.load_metas().unwrap();
+                assert_eq!(metas.payload.unwrap(), "first commit");
+            }
+            for _doc in 0..100 {
+                index_writer.add_document(doc!(text_field => "a"));
+            }
+            index_writer.commit().unwrap();
+            {
+                let metas = index.load_metas().unwrap();
+                assert!(metas.payload.is_none());
+            }
+        }
+    }
+
+    #[test]
+    fn test_prepare_but_rollback() {
+        let _ = env_logger::init();
+        let mut schema_builder = schema::SchemaBuilder::default();
+        let text_field = schema_builder.add_text_field("text", schema::TEXT);
+        let index = Index::create_in_ram(schema_builder.build());
+
+        {
+            // writing the segment
+            let mut index_writer = index.writer_with_num_threads(4, 4 * 30_000_000).unwrap();
+            // create 8 segments with 100 tiny docs
+            for _doc in 0..100 {
+                index_writer.add_document(doc!(text_field => "a"));
+            }
+            {
+                let mut prepared_commit = index_writer.prepare_commit().expect("commit failed");
+                prepared_commit.set_payload("first commit");
+                assert_eq!(prepared_commit.opstamp(), 100);
+                prepared_commit.abort().expect("commit failed");
+            }
+            {
+                let metas = index.load_metas().unwrap();
+                assert!(metas.payload.is_none());
+            }
+            for _doc in 0..100 {
+                index_writer.add_document(doc!(text_field => "b"));
+            }
+            index_writer.commit().unwrap();
+        }
+        index.load_searchers().unwrap();
+        let num_docs_containing = |s: &str| {
+            let searcher = index.searcher();
+            let term_a = Term::from_field_text(text_field, s);
+            searcher.doc_freq(&term_a)
+        };
+        assert_eq!(num_docs_containing("a"), 0);
+        assert_eq!(num_docs_containing("b"), 100);
+    }
 
 }

src/indexer/log_merge_policy.rs

@@ -1,5 +1,4 @@
-extern crate itertools;
-use super::merge_policy::{MergePolicy, MergeCandidate};
+use super::merge_policy::{MergeCandidate, MergePolicy};
 use core::SegmentMeta;
 use std::cmp;
 use std::f64;
@@ -8,7 +7,6 @@ const DEFAULT_LEVEL_LOG_SIZE: f64 = 0.75;
 const DEFAULT_MIN_LAYER_SIZE: u32 = 10_000;
 const DEFAULT_MIN_MERGE_SIZE: usize = 8;
 
-
 /// `LogMergePolicy` tries tries to merge segments that have a similar number of
 /// documents.
 #[derive(Debug, Clone)]
@@ -58,11 +56,11 @@ impl MergePolicy for LogMergePolicy {
             .enumerate()
             .collect::<Vec<(usize, u32)>>();
 
-        size_sorted_tuples.sort_by(|x, y| y.cmp(x));
+        size_sorted_tuples.sort_by(|x, y| y.1.cmp(&(x.1)));
 
         let size_sorted_log_tuples: Vec<_> = size_sorted_tuples
             .into_iter()
-            .map(|(ind, num_docs)| (ind, (self.clip_min_size(num_docs) as f64).log2()))
+            .map(|(ind, num_docs)| (ind, f64::from(self.clip_min_size(num_docs)).log2()))
             .collect();
 
         let (first_ind, first_score) = size_sorted_log_tuples[0];
@@ -102,7 +100,7 @@ impl Default for LogMergePolicy {
 mod tests {
     use super::*;
     use indexer::merge_policy::MergePolicy;
-    use core::{SegmentMeta, SegmentId};
+    use core::{SegmentId, SegmentMeta};
 
     fn test_merge_policy() -> LogMergePolicy {
         let mut log_merge_policy = LogMergePolicy::default();
@@ -134,12 +132,23 @@ mod tests {
     #[test]
     fn test_log_merge_policy_levels() {
         // multiple levels all get merged correctly
-        let test_input = vec![seg_meta(10),
-                              seg_meta(10),
-                              seg_meta(10),
-                              seg_meta(1000),
-                              seg_meta(1000),
-                              seg_meta(1000)];
+        // 2 MergeCandidates expected:
+        // * one with the 6 * 10-docs segments
+        // * one with the 3 * 1000-docs segments
+        // no MergeCandidate expected for the 2 * 10_000-docs segments as min_merge_size=3
+        let test_input = vec![
+            seg_meta(10),
+            seg_meta(10),
+            seg_meta(10),
+            seg_meta(1000),
+            seg_meta(1000),
+            seg_meta(1000),
+            seg_meta(10000),
+            seg_meta(10000),
+            seg_meta(10),
+            seg_meta(10),
+            seg_meta(10),
+        ];
         let result_list = test_merge_policy().compute_merge_candidates(&test_input);
         assert_eq!(result_list.len(), 2);
     }
@@ -147,24 +156,28 @@ mod tests {
     #[test]
     fn test_log_merge_policy_within_levels() {
         // multiple levels all get merged correctly
-        let test_input = vec![seg_meta(10),
-                              seg_meta(11),
-                              seg_meta(12),
-                              seg_meta(1000),
-                              seg_meta(1000),
-                              seg_meta(1000)];
+        let test_input = vec![
+            seg_meta(10),   // log2(10) = ~3.32 (> 3.58 - 0.75)
+            seg_meta(11),   // log2(11) = ~3.46
+            seg_meta(12),   // log2(12) = ~3.58
+            seg_meta(800),  // log2(800) = ~9.64 (> 9.97 - 0.75)
+            seg_meta(1000), // log2(1000) = ~9.97
+            seg_meta(1000),
+        ]; // log2(1000) = ~9.97
         let result_list = test_merge_policy().compute_merge_candidates(&test_input);
         assert_eq!(result_list.len(), 2);
     }
     #[test]
     fn test_log_merge_policy_small_segments() {
-        // multiple levels all get merged correctly
-        let test_input = vec![seg_meta(1),
-                              seg_meta(1),
-                              seg_meta(1),
-                              seg_meta(2),
-                              seg_meta(2),
-                              seg_meta(2)];
+        // segments under min_layer_size are merged together
+        let test_input = vec![
+            seg_meta(1),
+            seg_meta(1),
+            seg_meta(1),
+            seg_meta(2),
+            seg_meta(2),
+            seg_meta(2),
+        ];
         let result_list = test_merge_policy().compute_merge_candidates(&test_input);
         assert_eq!(result_list.len(), 1);
     }

src/indexer/merge_policy.rs

@@ -3,12 +3,10 @@ use core::SegmentMeta;
 use std::marker;
 use std::fmt::Debug;
 
-
 /// Set of segment suggested for a merge.
 #[derive(Debug, Clone)]
 pub struct MergeCandidate(pub Vec<SegmentId>);
 
-
 /// The `MergePolicy` defines which segments should be merged.
 ///
 /// Every time a the list of segments changes, the segment updater
@@ -43,7 +41,6 @@ impl MergePolicy for NoMergePolicy {
     }
 }
 
-
 #[cfg(test)]
 pub mod tests {
 
@@ -51,7 +48,6 @@ pub mod tests {
     use core::SegmentId;
     use core::SegmentMeta;
 
-
     /// `MergePolicy` useful for test purposes.
     ///
     /// Everytime there is more than one segment,

src/indexer/mod.rs

@@ -13,14 +13,17 @@ mod segment_entry;
 mod doc_opstamp_mapping;
 pub mod operation;
 mod stamper;
+mod prepared_commit;
 
+pub use self::prepared_commit::PreparedCommit;
 pub use self::segment_entry::{SegmentEntry, SegmentState};
 pub use self::segment_serializer::SegmentSerializer;
 pub use self::segment_writer::SegmentWriter;
 pub use self::index_writer::IndexWriter;
 pub use self::log_merge_policy::LogMergePolicy;
-pub use self::merge_policy::{NoMergePolicy, MergeCandidate, MergePolicy};
+pub use self::merge_policy::{MergeCandidate, MergePolicy, NoMergePolicy};
 pub use self::segment_manager::SegmentManager;
+pub(crate) use self::directory_lock::DirectoryLock;
 
 /// Alias for the default merge policy, which is the `LogMergePolicy`.
 pub type DefaultMergePolicy = LogMergePolicy;

src/indexer/operation.rs

@@ -1,7 +1,6 @@
 use schema::Document;
 use schema::Term;
 
-
 /// Timestamped Delete operation.
 #[derive(Clone, Eq, PartialEq, Debug)]
 pub struct DeleteOperation {

src/indexer/prepared_commit.rs

@@ -0,0 +1,39 @@
+use Result;
+use super::IndexWriter;
+
+/// A prepared commit
+pub struct PreparedCommit<'a> {
+    index_writer: &'a mut IndexWriter,
+    payload: Option<String>,
+    opstamp: u64,
+}
+
+impl<'a> PreparedCommit<'a> {
+    pub(crate) fn new(index_writer: &'a mut IndexWriter, opstamp: u64) -> PreparedCommit {
+        PreparedCommit {
+            index_writer,
+            payload: None,
+            opstamp
+        }
+    }
+
+    pub fn opstamp(&self) -> u64 {
+        self.opstamp
+    }
+
+    pub fn set_payload(&mut self, payload: &str) {
+        self.payload = Some(payload.to_string())
+    }
+
+    pub fn abort(self) -> Result<()> {
+        self.index_writer.rollback()
+    }
+
+    pub fn commit(self) -> Result<u64> {
+        info!("committing {}", self.opstamp);
+        self.index_writer
+            .segment_updater()
+            .commit(self.opstamp, self.payload)?;
+        Ok(self.opstamp)
+    }
+}

src/indexer/segment_entry.rs

@@ -4,7 +4,6 @@ use indexer::delete_queue::DeleteCursor;
 use core::SegmentId;
 use std::fmt;
 
-
 #[derive(Clone, Copy, PartialEq, Eq, Debug)]
 pub enum SegmentState {
     Ready,
@@ -20,7 +19,6 @@ impl SegmentState {
     }
 }
 
-
 /// A segment entry describes the state of
 /// a given segment, at a given instant.
 ///
@@ -44,19 +42,19 @@ pub struct SegmentEntry {
 
 impl SegmentEntry {
     /// Create a new `SegmentEntry`
-    pub fn new(segment_meta: SegmentMeta,
-               delete_cursor: DeleteCursor,
-               delete_bitset: Option<BitSet>)
-               -> SegmentEntry {
+    pub fn new(
+        segment_meta: SegmentMeta,
+        delete_cursor: DeleteCursor,
+        delete_bitset: Option<BitSet>,
+    ) -> SegmentEntry {
         SegmentEntry {
             meta: segment_meta,
             state: SegmentState::Ready,
-            delete_bitset: delete_bitset,
-            delete_cursor: delete_cursor,
+            delete_bitset,
+            delete_cursor,
         }
     }
 
-
     /// Return a reference to the segment entry deleted bitset.
     ///
     /// `DocId` in this bitset are flagged as deleted.
@@ -69,7 +67,6 @@ impl SegmentEntry {
         self.meta = segment_meta;
     }
 
-
     /// Return a reference to the segment_entry's delete cursor
     pub fn delete_cursor(&mut self) -> &mut DeleteCursor {
         &mut self.delete_cursor
@@ -88,13 +85,11 @@ impl SegmentEntry {
         self.meta.id()
     }
 
-
     /// Accessor to the `SegmentMeta`
     pub fn meta(&self) -> &SegmentMeta {
         &self.meta
     }
 
-
     /// Mark the `SegmentEntry` as in merge.
     ///
     /// Only segments that are not already
@@ -112,7 +107,6 @@ impl SegmentEntry {
         self.state = SegmentState::Ready;
     }
 
-
     /// Returns true iff a segment should
     /// be considered for a merge.
     pub fn is_ready(&self) -> bool {

src/indexer/segment_manager.rs

@@ -1,7 +1,7 @@
 use super::segment_register::SegmentRegister;
 use std::sync::RwLock;
 use core::SegmentMeta;
-use core::{META_FILEPATH, LOCKFILE_FILEPATH};
+use core::{LOCKFILE_FILEPATH, META_FILEPATH};
 use core::SegmentId;
 use indexer::SegmentEntry;
 use std::path::PathBuf;
@@ -17,8 +17,6 @@ struct SegmentRegisters {
     writing: HashSet<SegmentId>,
 }
 
-
-
 /// The segment manager stores the list of segments
 /// as well as their state.
 ///
@@ -32,31 +30,35 @@ pub struct SegmentManager {
 impl Debug for SegmentManager {
     fn fmt(&self, f: &mut Formatter) -> Result<(), fmt::Error> {
         let lock = self.read();
-        write!(f,
-               "{{ uncommitted: {:?}, committed: {:?} }}",
-               lock.uncommitted,
-               lock.committed)
+        write!(
+            f,
+            "{{ uncommitted: {:?}, committed: {:?} }}",
+            lock.uncommitted, lock.committed
+        )
     }
 }
 
-pub fn get_mergeable_segments(segment_manager: &SegmentManager)
-                              -> (Vec<SegmentMeta>, Vec<SegmentMeta>) {
+pub fn get_mergeable_segments(
+    segment_manager: &SegmentManager,
+) -> (Vec<SegmentMeta>, Vec<SegmentMeta>) {
     let registers_lock = segment_manager.read();
-    (registers_lock.committed.get_mergeable_segments(),
-     registers_lock.uncommitted.get_mergeable_segments())
+    (
+        registers_lock.committed.get_mergeable_segments(),
+        registers_lock.uncommitted.get_mergeable_segments(),
+    )
 }
 
 impl SegmentManager {
-    pub fn from_segments(segment_metas: Vec<SegmentMeta>,
-                         delete_cursor: DeleteCursor)
-                         -> SegmentManager {
+    pub fn from_segments(
+        segment_metas: Vec<SegmentMeta>,
+        delete_cursor: &DeleteCursor,
+    ) -> SegmentManager {
         SegmentManager {
             registers: RwLock::new(SegmentRegisters {
-                                       uncommitted: SegmentRegister::default(),
-                                       committed: SegmentRegister::new(segment_metas,
-                                                                       delete_cursor),
-                                       writing: HashSet::new(),
-                                   }),
+                uncommitted: SegmentRegister::default(),
+                committed: SegmentRegister::new(segment_metas, delete_cursor),
+                writing: HashSet::new(),
+            }),
         }
     }
 
@@ -139,11 +141,11 @@ impl SegmentManager {
         }
     }
 
-
-    pub fn cancel_merge(&self,
-                        before_merge_segment_ids: &[SegmentId],
-                        after_merge_segment_id: SegmentId) {
-
+    pub fn cancel_merge(
+        &self,
+        before_merge_segment_ids: &[SegmentId],
+        after_merge_segment_id: SegmentId,
+    ) {
         let mut registers_lock = self.write();
 
         // we mark all segments are ready for merge.
@@ -151,12 +153,14 @@ impl SegmentManager {
             let target_segment_register: &mut SegmentRegister;
             target_segment_register = {
                 if registers_lock
-                       .uncommitted
-                       .contains_all(before_merge_segment_ids) {
+                    .uncommitted
+                    .contains_all(before_merge_segment_ids)
+                {
                     &mut registers_lock.uncommitted
                 } else if registers_lock
-                              .committed
-                              .contains_all(before_merge_segment_ids) {
+                    .committed
+                    .contains_all(before_merge_segment_ids)
+                {
                     &mut registers_lock.committed
                 } else {
                     warn!("couldn't find segment in SegmentManager");
@@ -173,7 +177,6 @@ impl SegmentManager {
         registers_lock.writing.remove(&after_merge_segment_id);
     }
 
-
     pub fn write_segment(&self, segment_id: SegmentId) {
         let mut registers_lock = self.write();
         registers_lock.writing.insert(segment_id);
@@ -185,23 +188,26 @@ impl SegmentManager {
         registers_lock.uncommitted.add_segment_entry(segment_entry);
     }
 
-    pub fn end_merge(&self,
-                     before_merge_segment_ids: &[SegmentId],
-                     after_merge_segment_entry: SegmentEntry) {
-
+    pub fn end_merge(
+        &self,
+        before_merge_segment_ids: &[SegmentId],
+        after_merge_segment_entry: SegmentEntry,
+    ) {
         let mut registers_lock = self.write();
         registers_lock
             .writing
             .remove(&after_merge_segment_entry.segment_id());
 
-        let mut target_register: &mut SegmentRegister = {
+        let target_register: &mut SegmentRegister = {
             if registers_lock
-                   .uncommitted
-                   .contains_all(before_merge_segment_ids) {
+                .uncommitted
+                .contains_all(before_merge_segment_ids)
+            {
                 &mut registers_lock.uncommitted
             } else if registers_lock
-                          .committed
-                          .contains_all(before_merge_segment_ids) {
+                .committed
+                .contains_all(before_merge_segment_ids)
+            {
                 &mut registers_lock.committed
             } else {
                 warn!("couldn't find segment in SegmentManager");
@@ -212,10 +218,6 @@ impl SegmentManager {
             target_register.remove_segment(segment_id);
         }
         target_register.add_segment_entry(after_merge_segment_entry);
-
-
-
-
     }
 
     pub fn committed_segment_metas(&self) -> Vec<SegmentMeta> {

src/indexer/segment_register.rs

@@ -19,19 +19,17 @@ pub struct SegmentRegister {
     segment_states: HashMap<SegmentId, SegmentEntry>,
 }
 
-
 impl Debug for SegmentRegister {
     fn fmt(&self, f: &mut Formatter) -> Result<(), fmt::Error> {
-        try!(write!(f, "SegmentRegister("));
+        write!(f, "SegmentRegister(")?;
         for (k, v) in &self.segment_states {
-            try!(write!(f, "{}:{}, ", k.short_uuid_string(), v.state().letter_code()));
+            write!(f, "{}:{}, ", k.short_uuid_string(), v.state().letter_code())?;
         }
-        try!(write!(f, ")"));
+        write!(f, ")")?;
         Ok(())
     }
 }
 
-
 impl SegmentRegister {
     pub fn clear(&mut self) {
         self.segment_states.clear();
@@ -102,18 +100,17 @@ impl SegmentRegister {
             .start_merge();
     }
 
-    pub fn new(segment_metas: Vec<SegmentMeta>, delete_cursor: DeleteCursor) -> SegmentRegister {
+    pub fn new(segment_metas: Vec<SegmentMeta>, delete_cursor: &DeleteCursor) -> SegmentRegister {
         let mut segment_states = HashMap::new();
         for segment_meta in segment_metas {
             let segment_id = segment_meta.id();
             let segment_entry = SegmentEntry::new(segment_meta, delete_cursor.clone(), None);
             segment_states.insert(segment_id, segment_entry);
         }
-        SegmentRegister { segment_states: segment_states }
+        SegmentRegister { segment_states }
     }
 }
 
-
 #[cfg(test)]
 mod tests {
     use indexer::SegmentState;
@@ -144,34 +141,42 @@ mod tests {
             let segment_entry = SegmentEntry::new(segment_meta, delete_queue.cursor(), None);
             segment_register.add_segment_entry(segment_entry);
         }
-        assert_eq!(segment_register
-                       .segment_entry(&segment_id_a)
-                       .unwrap()
-                       .state(),
-                   SegmentState::Ready);
+        assert_eq!(
+            segment_register
+                .segment_entry(&segment_id_a)
+                .unwrap()
+                .state(),
+            SegmentState::Ready
+        );
         assert_eq!(segment_ids(&segment_register), vec![segment_id_a]);
         {
             let segment_meta = SegmentMeta::new(segment_id_b);
             let segment_entry = SegmentEntry::new(segment_meta, delete_queue.cursor(), None);
             segment_register.add_segment_entry(segment_entry);
         }
-        assert_eq!(segment_register
-                       .segment_entry(&segment_id_b)
-                       .unwrap()
-                       .state(),
-                   SegmentState::Ready);
+        assert_eq!(
+            segment_register
+                .segment_entry(&segment_id_b)
+                .unwrap()
+                .state(),
+            SegmentState::Ready
+        );
         segment_register.start_merge(&segment_id_a);
         segment_register.start_merge(&segment_id_b);
-        assert_eq!(segment_register
-                       .segment_entry(&segment_id_a)
-                       .unwrap()
-                       .state(),
-                   SegmentState::InMerge);
-        assert_eq!(segment_register
-                       .segment_entry(&segment_id_b)
-                       .unwrap()
-                       .state(),
-                   SegmentState::InMerge);
+        assert_eq!(
+            segment_register
+                .segment_entry(&segment_id_a)
+                .unwrap()
+                .state(),
+            SegmentState::InMerge
+        );
+        assert_eq!(
+            segment_register
+                .segment_entry(&segment_id_b)
+                .unwrap()
+                .state(),
+            SegmentState::InMerge
+        );
         segment_register.remove_segment(&segment_id_a);
         segment_register.remove_segment(&segment_id_b);
         {

src/indexer/segment_serializer.rs

@@ -4,8 +4,7 @@ use core::Segment;
 use core::SegmentComponent;
 use fastfield::FastFieldSerializer;
 use store::StoreWriter;
-use postings::PostingsSerializer;
-
+use postings::InvertedIndexSerializer;
 
 /// Segment serializer is in charge of laying out on disk
 /// the data accumulated and sorted by the `SegmentWriter`.
@@ -13,31 +12,31 @@ pub struct SegmentSerializer {
     store_writer: StoreWriter,
     fast_field_serializer: FastFieldSerializer,
     fieldnorms_serializer: FastFieldSerializer,
-    postings_serializer: PostingsSerializer,
+    postings_serializer: InvertedIndexSerializer,
 }
 
 impl SegmentSerializer {
     /// Creates a new `SegmentSerializer`.
     pub fn for_segment(segment: &mut Segment) -> Result<SegmentSerializer> {
-        let store_write = try!(segment.open_write(SegmentComponent::STORE));
+        let store_write = segment.open_write(SegmentComponent::STORE)?;
 
-        let fast_field_write = try!(segment.open_write(SegmentComponent::FASTFIELDS));
-        let fast_field_serializer = try!(FastFieldSerializer::new(fast_field_write));
+        let fast_field_write = segment.open_write(SegmentComponent::FASTFIELDS)?;
+        let fast_field_serializer = FastFieldSerializer::from_write(fast_field_write)?;
 
-        let fieldnorms_write = try!(segment.open_write(SegmentComponent::FIELDNORMS));
-        let fieldnorms_serializer = try!(FastFieldSerializer::new(fieldnorms_write));
+        let fieldnorms_write = segment.open_write(SegmentComponent::FIELDNORMS)?;
+        let fieldnorms_serializer = FastFieldSerializer::from_write(fieldnorms_write)?;
 
-        let postings_serializer = try!(PostingsSerializer::open(segment));
+        let postings_serializer = InvertedIndexSerializer::open(segment)?;
         Ok(SegmentSerializer {
-               postings_serializer: postings_serializer,
-               store_writer: StoreWriter::new(store_write),
-               fast_field_serializer: fast_field_serializer,
-               fieldnorms_serializer: fieldnorms_serializer,
-           })
+            postings_serializer,
+            store_writer: StoreWriter::new(store_write),
+            fast_field_serializer,
+            fieldnorms_serializer,
+        })
     }
 
     /// Accessor to the `PostingsSerializer`.
-    pub fn get_postings_serializer(&mut self) -> &mut PostingsSerializer {
+    pub fn get_postings_serializer(&mut self) -> &mut InvertedIndexSerializer {
         &mut self.postings_serializer
     }
 

src/indexer/segment_updater.rs

@@ -13,7 +13,7 @@ use futures::Future;
 use futures::Canceled;
 use futures::oneshot;
 use directory::FileProtection;
-use indexer::{MergePolicy, DefaultMergePolicy};
+use indexer::{DefaultMergePolicy, MergePolicy};
 use indexer::index_writer::advance_deletes;
 use indexer::MergeCandidate;
 use indexer::merger::IndexMerger;
@@ -29,13 +29,12 @@ use std::io::Write;
 use std::mem;
 use std::ops::DerefMut;
 use std::sync::Arc;
-use std::sync::atomic::{AtomicUsize, AtomicBool};
+use std::sync::atomic::{AtomicBool, AtomicUsize};
 use std::sync::atomic::Ordering;
 use std::sync::RwLock;
 use std::thread;
 use std::thread::JoinHandle;
-use super::segment_manager::{SegmentManager, get_mergeable_segments};
-
+use super::segment_manager::{get_mergeable_segments, SegmentManager};
 
 /// Save the index meta file.
 /// This operation is atomic :
@@ -47,11 +46,9 @@ use super::segment_manager::{SegmentManager, get_mergeable_segments};
 ///
 /// This method is not part of tantivy's public API
 pub fn save_new_metas(schema: Schema, opstamp: u64, directory: &mut Directory) -> Result<()> {
-    save_metas(vec![], schema, opstamp, directory)
+    save_metas(vec![], schema, opstamp, None, directory)
 }
 
-
-
 /// Save the index meta file.
 /// This operation is atomic:
 /// Either
@@ -61,24 +58,26 @@ pub fn save_new_metas(schema: Schema, opstamp: u64, directory: &mut Directory) -
 /// and flushed.
 ///
 /// This method is not part of tantivy's public API
-pub fn save_metas(segment_metas: Vec<SegmentMeta>,
-                  schema: Schema,
-                  opstamp: u64,
-                  directory: &mut Directory)
-                  -> Result<()> {
+pub fn save_metas(
+    segment_metas: Vec<SegmentMeta>,
+    schema: Schema,
+    opstamp: u64,
+    payload: Option<String>,
+    directory: &mut Directory,
+) -> Result<()> {
     let metas = IndexMeta {
         segments: segment_metas,
-        schema: schema,
-        opstamp: opstamp,
+        schema,
+        opstamp,
+        payload,
     };
-    let mut w = try!(serde_json::to_vec_pretty(&metas));
-    try!(write!(&mut w, "\n"));
-    directory.atomic_write(&META_FILEPATH, &w[..])?;
+    let mut buffer = serde_json::to_vec_pretty(&metas)?;
+    write!(&mut buffer, "\n")?;
+    directory.atomic_write(&META_FILEPATH, &buffer[..])?;
     debug!("Saved metas {:?}", serde_json::to_string_pretty(&metas));
     Ok(())
 }
 
-
 // The segment update runner is in charge of processing all
 //  of the `SegmentUpdate`s.
 //
@@ -87,13 +86,12 @@ pub fn save_metas(segment_metas: Vec<SegmentMeta>,
 #[derive(Clone)]
 pub struct SegmentUpdater(Arc<InnerSegmentUpdater>);
 
-
-
-fn perform_merge(segment_ids: &[SegmentId],
-                 segment_updater: &SegmentUpdater,
-                 mut merged_segment: Segment,
-                 target_opstamp: u64)
-                 -> Result<SegmentEntry> {
+fn perform_merge(
+    segment_ids: &[SegmentId],
+    segment_updater: &SegmentUpdater,
+    mut merged_segment: Segment,
+    target_opstamp: u64,
+) -> Result<SegmentEntry> {
     // first we need to apply deletes to our segment.
     info!("Start merge: {:?}", segment_ids);
 
@@ -104,18 +102,21 @@ fn perform_merge(segment_ids: &[SegmentId],
     let mut file_protections: Vec<FileProtection> = vec![];
 
     for segment_id in segment_ids {
-        if let Some(mut segment_entry) =
-            segment_updater.0.segment_manager.segment_entry(segment_id) {
+        if let Some(mut segment_entry) = segment_updater.0.segment_manager.segment_entry(segment_id)
+        {
             let segment = index.segment(segment_entry.meta().clone());
             if let Some(file_protection) =
-                advance_deletes(segment, &mut segment_entry, target_opstamp)? {
+                advance_deletes(segment, &mut segment_entry, target_opstamp)?
+            {
                 file_protections.push(file_protection);
             }
             segment_entries.push(segment_entry);
         } else {
             error!("Error, had to abort merge as some of the segment is not managed anymore.");
-            let msg = format!("Segment {:?} requested for merge is not managed.",
-                              segment_id);
+            let msg = format!(
+                "Segment {:?} requested for merge is not managed.",
+                segment_id
+            );
             bail!(ErrorKind::InvalidArgument(msg));
         }
     }
@@ -127,7 +128,6 @@ fn perform_merge(segment_ids: &[SegmentId],
         .map(|segment_entry| index.segment(segment_entry.meta().clone()))
         .collect();
 
-
     // An IndexMerger is like a "view" of our merged segments.
     let merger: IndexMerger = IndexMerger::open(schema, &segments[..])?;
 
@@ -147,7 +147,6 @@ fn perform_merge(segment_ids: &[SegmentId],
     Ok(after_merge_segment_entry)
 }
 
-
 struct InnerSegmentUpdater {
     pool: CpuPool,
     index: Index,
@@ -161,23 +160,24 @@ struct InnerSegmentUpdater {
 }
 
 impl SegmentUpdater {
-    pub fn new(index: Index,
-               stamper: Stamper,
-               delete_cursor: DeleteCursor)
-               -> Result<SegmentUpdater> {
+    pub fn new(
+        index: Index,
+        stamper: Stamper,
+        delete_cursor: &DeleteCursor,
+    ) -> Result<SegmentUpdater> {
         let segments = index.searchable_segment_metas()?;
         let segment_manager = SegmentManager::from_segments(segments, delete_cursor);
         Ok(SegmentUpdater(Arc::new(InnerSegmentUpdater {
-                                       pool: CpuPool::new(1),
-                                       index: index,
-                                       segment_manager: segment_manager,
-                                       merge_policy: RwLock::new(box DefaultMergePolicy::default()),
-                                       merging_thread_id: AtomicUsize::default(),
-                                       merging_threads: RwLock::new(HashMap::new()),
-                                       generation: AtomicUsize::default(),
-                                       killed: AtomicBool::new(false),
-                                       stamper: stamper,
-                                   })))
+            pool: CpuPool::new(1),
+            index,
+            segment_manager,
+            merge_policy: RwLock::new(box DefaultMergePolicy::default()),
+            merging_thread_id: AtomicUsize::default(),
+            merging_threads: RwLock::new(HashMap::new()),
+            generation: AtomicUsize::default(),
+            killed: AtomicBool::new(false),
+            stamper,
+        })))
     }
 
     pub fn new_segment(&self) -> Segment {
@@ -199,23 +199,21 @@ impl SegmentUpdater {
         self.0.merging_thread_id.fetch_add(1, Ordering::SeqCst)
     }
 
-    fn run_async<T: 'static + Send, F: 'static + Send + FnOnce(SegmentUpdater) -> T>
-        (&self,
-         f: F)
-         -> CpuFuture<T, Error> {
+    fn run_async<T: 'static + Send, F: 'static + Send + FnOnce(SegmentUpdater) -> T>(
+        &self,
+        f: F,
+    ) -> CpuFuture<T, Error> {
         let me_clone = self.clone();
         self.0.pool.spawn_fn(move || Ok(f(me_clone)))
     }
 
-
     pub fn add_segment(&self, generation: usize, segment_entry: SegmentEntry) -> bool {
         if generation >= self.0.generation.load(Ordering::Acquire) {
             self.run_async(|segment_updater| {
-                               segment_updater.0.segment_manager.add_segment(segment_entry);
-                               segment_updater.consider_merge_options();
-                               true
-                           })
-                .forget();
+                segment_updater.0.segment_manager.add_segment(segment_entry);
+                segment_updater.consider_merge_options();
+                true
+            }).forget();
             true
         } else {
             false
@@ -226,11 +224,10 @@ impl SegmentUpdater {
         self.0.killed.store(true, Ordering::Release);
     }
 
-    fn is_alive(&self) -> bool {
+    pub fn is_alive(&self) -> bool {
         !self.0.killed.load(Ordering::Acquire)
     }
 
-
     /// Apply deletes up to the target opstamp to all segments.
     ///
     /// Tne method returns copies of the segment entries,
@@ -242,53 +239,54 @@ impl SegmentUpdater {
             advance_deletes(segment, segment_entry, target_opstamp)?;
         }
         Ok(segment_entries)
-
     }
 
-    pub fn save_metas(&self, opstamp: u64) {
+    pub fn save_metas(&self, opstamp: u64, commit_message: Option<String>) {
         if self.is_alive() {
             let index = &self.0.index;
             let directory = index.directory();
-            save_metas(self.0.segment_manager.committed_segment_metas(),
-                       index.schema(),
-                       opstamp,
-                       directory.box_clone().borrow_mut())
-                    .expect("Could not save metas.");
+            save_metas(
+                self.0.segment_manager.committed_segment_metas(),
+                index.schema(),
+                opstamp,
+                commit_message,
+                directory.box_clone().borrow_mut(),
+            ).expect("Could not save metas.");
         }
     }
 
     pub fn garbage_collect_files(&self) -> Result<()> {
         self.run_async(move |segment_updater| {
             segment_updater.garbage_collect_files_exec();
-        })
-        .wait()
+        }).wait()
     }
 
     fn garbage_collect_files_exec(&self) {
         info!("Running garbage collection");
-        let living_files = self.0.segment_manager.list_files();
         let mut index = self.0.index.clone();
-        index.directory_mut().garbage_collect(living_files);
+        index
+            .directory_mut()
+            .garbage_collect(|| self.0.segment_manager.list_files());
     }
 
-    pub fn commit(&self, opstamp: u64) -> Result<()> {
-        self.run_async(move |segment_updater| if segment_updater.is_alive() {
-                       let segment_entries = segment_updater
-                           .purge_deletes(opstamp)
-                           .expect("Failed purge deletes");
-                       segment_updater.0.segment_manager.commit(segment_entries);
-                       segment_updater.save_metas(opstamp);
-                       segment_updater.garbage_collect_files_exec();
-                       segment_updater.consider_merge_options();
-                   })
-            .wait()
+    pub fn commit(&self, opstamp: u64, payload: Option<String>) -> Result<()> {
+        self.run_async(move |segment_updater| {
+            if segment_updater.is_alive() {
+                let segment_entries = segment_updater
+                    .purge_deletes(opstamp)
+                    .expect("Failed purge deletes");
+                segment_updater.0.segment_manager.commit(segment_entries);
+                segment_updater.save_metas(opstamp, payload);
+                segment_updater.garbage_collect_files_exec();
+                segment_updater.consider_merge_options();
+            }
+        }).wait()
     }
 
-
-    pub fn start_merge(&self,
-                       segment_ids: &[SegmentId])
-                       -> impl Future<Item = SegmentMeta, Error = Canceled> {
-
+    pub fn start_merge(
+        &self,
+        segment_ids: &[SegmentId],
+    ) -> impl Future<Item = SegmentMeta, Error = Canceled> {
         self.0.segment_manager.start_merge(segment_ids);
         let segment_updater_clone = self.clone();
 
@@ -303,14 +301,15 @@ impl SegmentUpdater {
 
         let target_opstamp = self.0.stamper.stamp();
         let merging_join_handle = thread::spawn(move || {
-
             // first we need to apply deletes to our segment.
             let merged_segment = segment_updater_clone.new_segment();
             let merged_segment_id = merged_segment.id();
-            let merge_result = perform_merge(&segment_ids_vec,
-                                             &segment_updater_clone,
-                                             merged_segment,
-                                             target_opstamp);
+            let merge_result = perform_merge(
+                &segment_ids_vec,
+                &segment_updater_clone,
+                merged_segment,
+                target_opstamp,
+            );
 
             match merge_result {
                 Ok(after_merge_segment_entry) => {
@@ -352,7 +351,6 @@ impl SegmentUpdater {
         merging_future_recv
     }
 
-
     fn consider_merge_options(&self) {
         let (committed_segments, uncommitted_segments) =
             get_mergeable_segments(&self.0.segment_manager);
@@ -367,59 +365,73 @@ impl SegmentUpdater {
         }
     }
 
-    fn cancel_merge(&self,
-                    before_merge_segment_ids: &[SegmentId],
-                    after_merge_segment_entry: SegmentId) {
+    fn cancel_merge(
+        &self,
+        before_merge_segment_ids: &[SegmentId],
+        after_merge_segment_entry: SegmentId,
+    ) {
         self.0
             .segment_manager
             .cancel_merge(before_merge_segment_ids, after_merge_segment_entry);
     }
 
-
-    fn end_merge(&self,
-                 before_merge_segment_ids: Vec<SegmentId>,
-                 mut after_merge_segment_entry: SegmentEntry)
-                 -> Result<()> {
-
+    fn end_merge(
+        &self,
+        before_merge_segment_ids: Vec<SegmentId>,
+        mut after_merge_segment_entry: SegmentEntry,
+    ) -> Result<()> {
         self.run_async(move |segment_updater| {
             info!("End merge {:?}", after_merge_segment_entry.meta());
             let mut delete_cursor = after_merge_segment_entry.delete_cursor().clone();
             let mut _file_protection_opt = None;
             if let Some(delete_operation) = delete_cursor.get() {
-                let committed_opstamp = segment_updater.0.index.opstamp();
+                let committed_opstamp = segment_updater
+                    .0
+                    .index
+                    .load_metas()
+                    .expect("Failed to read opstamp")
+                    .opstamp;
                 if delete_operation.opstamp < committed_opstamp {
                     let index = &segment_updater.0.index;
                     let segment = index.segment(after_merge_segment_entry.meta().clone());
-                    match advance_deletes(segment,
-                                          &mut after_merge_segment_entry,
-                                          committed_opstamp) {
+                    match advance_deletes(
+                        segment,
+                        &mut after_merge_segment_entry,
+                        committed_opstamp,
+                    ) {
                         Ok(file_protection_opt_res) => {
                             _file_protection_opt = file_protection_opt_res;
                         }
                         Err(e) => {
-                            error!("Merge of {:?} was cancelled (advancing deletes failed): {:?}",
-                                   before_merge_segment_ids, e);
+                            error!(
+                                "Merge of {:?} was cancelled (advancing deletes failed): {:?}",
+                                before_merge_segment_ids, e
+                            );
                             // ... cancel merge
                             if cfg!(test) {
                                 panic!("Merge failed.");
                             }
-                            segment_updater.cancel_merge(&before_merge_segment_ids,
-                                                         after_merge_segment_entry.segment_id());
+                            segment_updater.cancel_merge(
+                                &before_merge_segment_ids,
+                                after_merge_segment_entry.segment_id(),
+                            );
                             return;
                         }
                     }
                 }
             }
-            segment_updater.0.segment_manager.end_merge(&before_merge_segment_ids,
-                                                        after_merge_segment_entry);
+            segment_updater
+                .0
+                .segment_manager
+                .end_merge(&before_merge_segment_ids, after_merge_segment_entry);
             segment_updater.consider_merge_options();
             info!("save metas");
-            segment_updater.save_metas(segment_updater.0.index.opstamp());
+            let previous_metas = segment_updater.0.index.load_metas().unwrap();
+            segment_updater.save_metas(previous_metas.opstamp, previous_metas.payload);
             segment_updater.garbage_collect_files_exec();
         }).wait()
     }
 
-
     /// Wait for current merging threads.
     ///
     /// Upon termination of the current merging threads,
@@ -436,10 +448,8 @@ impl SegmentUpdater {
     /// Obsolete files will eventually be cleaned up
     /// by the directory garbage collector.
     pub fn wait_merging_thread(&self) -> Result<()> {
-
         let mut num_segments: usize;
         loop {
-
             num_segments = self.0.segment_manager.num_segments();
 
             let mut new_merging_threads = HashMap::new();
@@ -467,9 +477,6 @@ impl SegmentUpdater {
     }
 }
 
-
-
-
 #[cfg(test)]
 mod tests {
 

src/indexer/segment_writer.rs

@@ -1,20 +1,24 @@
 use Result;
 use DocId;
 use std::io;
+use std::str;
 use schema::Schema;
 use schema::Term;
 use core::Segment;
 use core::SerializableSegment;
 use fastfield::FastFieldsWriter;
 use schema::Field;
-use schema::FieldValue;
 use schema::FieldType;
 use indexer::segment_serializer::SegmentSerializer;
+use std::collections::HashMap;
 use datastruct::stacker::Heap;
 use indexer::index_writer::MARGIN_IN_BYTES;
 use super::operation::AddOperation;
 use postings::MultiFieldPostingsWriter;
-
+use tokenizer::BoxedTokenizer;
+use tokenizer::FacetTokenizer;
+use tokenizer::{TokenStream, Tokenizer};
+use schema::Value;
 
 /// A `SegmentWriter` is in charge of creating segment index from a
 /// documents.
@@ -29,9 +33,9 @@ pub struct SegmentWriter<'a> {
     fast_field_writers: FastFieldsWriter,
     fieldnorms_writer: FastFieldsWriter,
     doc_opstamps: Vec<u64>,
+    tokenizers: Vec<Option<Box<BoxedTokenizer>>>,
 }
 
-
 fn create_fieldnorms_writer(schema: &Schema) -> FastFieldsWriter {
     let u64_fields: Vec<Field> = schema
         .fields()
@@ -43,7 +47,6 @@ fn create_fieldnorms_writer(schema: &Schema) -> FastFieldsWriter {
     FastFieldsWriter::new(u64_fields)
 }
 
-
 impl<'a> SegmentWriter<'a> {
     /// Creates a new `SegmentWriter`
     ///
@@ -54,22 +57,38 @@ impl<'a> SegmentWriter<'a> {
     /// the flushing behavior as a buffer limit
     /// - segment: The segment being written
     /// - schema
-    pub fn for_segment(heap: &'a Heap,
-                       table_bits: usize,
-                       mut segment: Segment,
-                       schema: &Schema)
-                       -> Result<SegmentWriter<'a>> {
+    pub fn for_segment(
+        heap: &'a Heap,
+        table_bits: usize,
+        mut segment: Segment,
+        schema: &Schema,
+    ) -> Result<SegmentWriter<'a>> {
         let segment_serializer = SegmentSerializer::for_segment(&mut segment)?;
         let multifield_postings = MultiFieldPostingsWriter::new(schema, table_bits, heap);
+        let tokenizers = schema
+            .fields()
+            .iter()
+            .map(|field_entry| field_entry.field_type())
+            .map(|field_type| match *field_type {
+                FieldType::Str(ref text_options) => text_options.get_indexing_options().and_then(
+                    |text_index_option| {
+                        let tokenizer_name = &text_index_option.tokenizer();
+                        segment.index().tokenizers().get(tokenizer_name)
+                    },
+                ),
+                _ => None,
+            })
+            .collect();
         Ok(SegmentWriter {
-               heap: heap,
-               max_doc: 0,
-               multifield_postings: multifield_postings,
-               fieldnorms_writer: create_fieldnorms_writer(schema),
-               segment_serializer: segment_serializer,
-               fast_field_writers: FastFieldsWriter::from_schema(schema),
-               doc_opstamps: Vec::with_capacity(1_000),
-           })
+            heap,
+            max_doc: 0,
+            multifield_postings,
+            fieldnorms_writer: create_fieldnorms_writer(schema),
+            segment_serializer,
+            fast_field_writers: FastFieldsWriter::from_schema(schema),
+            doc_opstamps: Vec::with_capacity(1_000),
+            tokenizers,
+        })
     }
 
     /// Lay on disk the current content of the `SegmentWriter`
@@ -77,10 +96,12 @@ impl<'a> SegmentWriter<'a> {
     /// Finalize consumes the `SegmentWriter`, so that it cannot
     /// be used afterwards.
     pub fn finalize(self) -> Result<Vec<u64>> {
-        write(&self.multifield_postings,
-              &self.fast_field_writers,
-              &self.fieldnorms_writer,
-              self.segment_serializer)?;
+        write(
+            &self.multifield_postings,
+            &self.fast_field_writers,
+            &self.fieldnorms_writer,
+            self.segment_serializer,
+        )?;
         Ok(self.doc_opstamps)
     }
 
@@ -95,7 +116,6 @@ impl<'a> SegmentWriter<'a> {
         self.heap.num_free_bytes() <= MARGIN_IN_BYTES
     }
 
-
     /// Return true if the term dictionary hashmap is reaching capacity.
     /// It is one of the condition that triggers a `SegmentWriter` to
     /// be finalized.
@@ -103,69 +123,110 @@ impl<'a> SegmentWriter<'a> {
         self.multifield_postings.is_term_saturated()
     }
 
-
     /// Indexes a new document
     ///
     /// As a user, you should rather use `IndexWriter`'s add_document.
-    pub fn add_document(&mut self,
-                        add_operation: &AddOperation,
-                        schema: &Schema)
-                        -> io::Result<()> {
+    pub fn add_document(&mut self, add_operation: AddOperation, schema: &Schema) -> io::Result<()> {
         let doc_id = self.max_doc;
-        let doc = &add_operation.document;
+        let mut doc = add_operation.document;
         self.doc_opstamps.push(add_operation.opstamp);
+
+        self.fast_field_writers.add_document(&doc);
+
         for (field, field_values) in doc.get_sorted_field_values() {
             let field_options = schema.get_field_entry(field);
+            if !field_options.is_indexed() {
+                continue;
+            }
             match *field_options.field_type() {
-                FieldType::Str(ref text_options) => {
-                    let num_tokens: u32 = if text_options.get_indexing_options().is_tokenized() {
-                        self.multifield_postings
-                            .index_text(doc_id, field, &field_values)
-                    } else {
-                        let num_field_values = field_values.len() as u32;
-                        for field_value in field_values {
-                            let term = Term::from_field_text(field, field_value.value().text());
-                            self.multifield_postings.suscribe(doc_id, &term);
+                FieldType::HierarchicalFacet => {
+                    let facets: Vec<&[u8]> = field_values
+                        .iter()
+                        .flat_map(|field_value| match *field_value.value() {
+                            Value::Facet(ref facet) => Some(facet.encoded_bytes()),
+                            _ => {
+                                panic!("Expected hierarchical facet");
+                            }
+                        })
+                        .collect();
+                    let mut term = unsafe { Term::with_capacity(100) };
+                    term.set_field(field);
+                    for facet_bytes in facets {
+                        let mut unordered_term_id_opt = None;
+                        let fake_str = unsafe { str::from_utf8_unchecked(facet_bytes) };
+                        FacetTokenizer.token_stream(fake_str).process(&mut |token| {
+                            term.set_text(&token.text);
+                            let unordered_term_id =
+                                self.multifield_postings.subscribe(doc_id, &term);
+                            unordered_term_id_opt = Some(unordered_term_id);
+                        });
+                        if let Some(unordered_term_id) = unordered_term_id_opt {
+                            self.fast_field_writers
+                                .get_multivalue_writer(field)
+                                .expect("multified writer for facet missing")
+                                .add_val(unordered_term_id);
                         }
-                        num_field_values
+                    }
+                }
+                FieldType::Str(_) => {
+                    let num_tokens = if let Some(ref mut tokenizer) =
+                        self.tokenizers[field.0 as usize]
+                    {
+                        let texts: Vec<&str> = field_values
+                            .iter()
+                            .flat_map(|field_value| match *field_value.value() {
+                                Value::Str(ref text) => Some(text.as_str()),
+                                _ => None,
+                            })
+                            .collect();
+                        if texts.is_empty() {
+                            0
+                        } else {
+                            let mut token_stream = tokenizer.token_stream_texts(&texts[..]);
+                            self.multifield_postings
+                                .index_text(doc_id, field, &mut token_stream)
+                        }
+                    } else {
+                        0
                     };
                     self.fieldnorms_writer
                         .get_field_writer(field)
-                        .map(|field_norms_writer| field_norms_writer.add_val(num_tokens as u64));
+                        .map(|field_norms_writer| {
+                            field_norms_writer.add_val(u64::from(num_tokens))
+                        });
                 }
                 FieldType::U64(ref int_option) => {
                     if int_option.is_indexed() {
                         for field_value in field_values {
-                            let term = Term::from_field_u64(field_value.field(),
-                                                            field_value.value().u64_value());
-                            self.multifield_postings.suscribe(doc_id, &term);
+                            let term = Term::from_field_u64(
+                                field_value.field(),
+                                field_value.value().u64_value(),
+                            );
+                            self.multifield_postings.subscribe(doc_id, &term);
                         }
                     }
                 }
                 FieldType::I64(ref int_option) => {
                     if int_option.is_indexed() {
                         for field_value in field_values {
-                            let term = Term::from_field_i64(field_value.field(),
-                                                            field_value.value().i64_value());
-                            self.multifield_postings.suscribe(doc_id, &term);
+                            let term = Term::from_field_i64(
+                                field_value.field(),
+                                field_value.value().i64_value(),
+                            );
+                            self.multifield_postings.subscribe(doc_id, &term);
                         }
                     }
                 }
             }
         }
         self.fieldnorms_writer.fill_val_up_to(doc_id);
-        self.fast_field_writers.add_document(doc);
-        let stored_fieldvalues: Vec<&FieldValue> = doc.field_values()
-            .iter()
-            .filter(|field_value| schema.get_field_entry(field_value.field()).is_stored())
-            .collect();
+        doc.filter_fields(|field| schema.get_field_entry(field).is_stored());
         let doc_writer = self.segment_serializer.get_store_writer();
-        try!(doc_writer.store(&stored_fieldvalues));
+        doc_writer.store(&doc)?;
         self.max_doc += 1;
         Ok(())
     }
 
-
     /// Max doc is
     /// - the number of documents in the segment assuming there is no deletes
     /// - the maximum document id (including deleted documents) + 1
@@ -188,16 +249,16 @@ impl<'a> SegmentWriter<'a> {
 }
 
 // This method is used as a trick to workaround the borrow checker
-fn write(multifield_postings: &MultiFieldPostingsWriter,
-         fast_field_writers: &FastFieldsWriter,
-         fieldnorms_writer: &FastFieldsWriter,
-         mut serializer: SegmentSerializer)
-         -> Result<()> {
-
-    try!(multifield_postings.serialize(serializer.get_postings_serializer()));
-    try!(fast_field_writers.serialize(serializer.get_fast_field_serializer()));
-    try!(fieldnorms_writer.serialize(serializer.get_fieldnorms_serializer()));
-    try!(serializer.close());
+fn write(
+    multifield_postings: &MultiFieldPostingsWriter,
+    fast_field_writers: &FastFieldsWriter,
+    fieldnorms_writer: &FastFieldsWriter,
+    mut serializer: SegmentSerializer,
+) -> Result<()> {
+    let term_ord_map = multifield_postings.serialize(serializer.get_postings_serializer())?;
+    fast_field_writers.serialize(serializer.get_fast_field_serializer(), &term_ord_map)?;
+    fieldnorms_writer.serialize(serializer.get_fieldnorms_serializer(), &HashMap::new())?;
+    serializer.close()?;
 
     Ok(())
 }
@@ -205,10 +266,12 @@ fn write(multifield_postings: &MultiFieldPostingsWriter,
 impl<'a> SerializableSegment for SegmentWriter<'a> {
     fn write(&self, serializer: SegmentSerializer) -> Result<u32> {
         let max_doc = self.max_doc;
-        write(&self.multifield_postings,
-              &self.fast_field_writers,
-              &self.fieldnorms_writer,
-              serializer)?;
+        write(
+            &self.multifield_postings,
+            &self.fast_field_writers,
+            &self.fieldnorms_writer,
+            serializer,
+        )?;
         Ok(max_doc)
     }
 }

src/indexer/stamper.rs

@@ -1,7 +1,6 @@
 use std::sync::atomic::{AtomicU64, Ordering};
 use std::sync::Arc;
 
-
 #[derive(Clone, Default)]
 pub struct Stamper(Arc<AtomicU64>);
 

src/lib.rs

@@ -1,20 +1,18 @@
 #![doc(html_logo_url = "http://fulmicoton.com/tantivy-logo/tantivy-logo.png")]
 #![cfg_attr(feature = "cargo-clippy", allow(module_inception))]
 #![cfg_attr(feature = "cargo-clippy", allow(inline_always))]
-
 #![feature(box_syntax)]
 #![feature(optin_builtin_traits)]
 #![feature(conservative_impl_trait)]
+#![feature(collections_range)]
 #![feature(integer_atomics)]
-
+#![feature(drain_filter)]
 #![cfg_attr(test, feature(test))]
 #![cfg_attr(test, feature(iterator_step_by))]
-
-
 #![doc(test(attr(allow(unused_variables), deny(warnings))))]
-
 #![allow(unknown_lints)]
-
+#![allow(new_without_default)]
+#![allow(decimal_literal_representation)]
 #![warn(missing_docs)]
 
 //! # `tantivy`
@@ -22,6 +20,98 @@
 //! Tantivy is a search engine library.
 //! Think `Lucene`, but in Rust.
 //!
+//! ```rust
+
+//! # extern crate tempdir;
+//! #
+//! #[macro_use]
+//! extern crate tantivy;
+//!
+//! // ...
+//!
+//! # use std::path::Path;
+//! # use tempdir::TempDir;
+//! # use tantivy::Index;
+//! # use tantivy::schema::*;
+//! # use tantivy::collector::TopCollector;
+//! # use tantivy::query::QueryParser;
+//! #
+//! # fn main() {
+//! #     // Let's create a temporary directory for the
+//! #     // sake of this example
+//! #     if let Ok(dir) = TempDir::new("tantivy_example_dir") {
+//! #         run_example(dir.path()).unwrap();
+//! #         dir.close().unwrap();
+//! #     }
+//! # }
+//! #
+//! # fn run_example(index_path: &Path) -> tantivy::Result<()> {
+//! // First we need to define a schema ...
+//!
+//! // `TEXT` means the field should be tokenized and indexed,
+//! // along with its term frequency and term positions.
+//! //
+//! // `STORED` means that the field will also be saved
+//! // in a compressed, row-oriented key-value store.
+//! // This store is useful to reconstruct the
+//! // documents that were selected during the search phase.
+//! let mut schema_builder = SchemaBuilder::default();
+//! let title = schema_builder.add_text_field("title", TEXT | STORED);
+//! let body = schema_builder.add_text_field("body", TEXT);
+//! let schema = schema_builder.build();
+//!
+//! // Indexing documents
+//!
+//! let index = Index::create(index_path, schema.clone())?;
+//!
+//! // Here we use a buffer of 100MB that will be split
+//! // between indexing threads.
+//! let mut index_writer = index.writer(100_000_000)?;
+//!
+//! // Let's index one documents!
+//! index_writer.add_document(doc!(
+//!     title => "The Old Man and the Sea",
+//!     body => "He was an old man who fished alone in a skiff in \
+//!             the Gulf Stream and he had gone eighty-four days \
+//!             now without taking a fish."
+//! ));
+//!
+//! // We need to call .commit() explicitly to force the
+//! // index_writer to finish processing the documents in the queue,
+//! // flush the current index to the disk, and advertise
+//! // the existence of new documents.
+//! index_writer.commit()?;
+//!
+//! // # Searching
+//!
+//! index.load_searchers()?;
+//!
+//! let searcher = index.searcher();
+//!
+//! let query_parser = QueryParser::for_index(&index, vec![title, body]);
+//!
+//! // QueryParser may fail if the query is not in the right
+//! // format. For user facing applications, this can be a problem.
+//! // A ticket has been opened regarding this problem.
+//! let query = query_parser.parse_query("sea whale")?;
+//!
+//! let mut top_collector = TopCollector::with_limit(10);
+//! searcher.search(&*query, &mut top_collector)?;
+//!
+//! // Our top collector now contains the 10
+//! // most relevant doc ids...
+//! let doc_addresses = top_collector.docs();
+//! for doc_address in doc_addresses {
+//!     let retrieved_doc = searcher.doc(&doc_address)?;
+//!     println!("{}", schema.to_json(&retrieved_doc));
+//! }
+//!
+//! # Ok(())
+//! # }
+//! ```
+//!
+//!
+//!
 //! A good place for you to get started is to check out
 //! the example code (
 //! [literate programming](http://fulmicoton.com/tantivy-examples/simple_search.html) /
@@ -39,53 +129,55 @@ extern crate log;
 #[macro_use]
 extern crate error_chain;
 
-#[macro_use]
-extern crate version;
-extern crate fst;
-extern crate byteorder;
-extern crate memmap;
-extern crate regex;
-extern crate tempfile;
 extern crate atomicwrites;
-extern crate tempdir;
-extern crate serde;
-extern crate bincode;
-extern crate serde_json;
-extern crate time;
-extern crate lz4;
-extern crate uuid;
-extern crate num_cpus;
-extern crate combine;
-extern crate itertools;
-extern crate chan;
-extern crate crossbeam;
 extern crate bit_set;
+extern crate byteorder;
+extern crate chan;
+extern crate combine;
+extern crate crossbeam;
+extern crate fst;
 extern crate futures;
 extern crate futures_cpupool;
+extern crate itertools;
+extern crate lz4;
+extern crate num_cpus;
 extern crate owning_ref;
+extern crate regex;
+extern crate rust_stemmers;
+extern crate serde;
+extern crate serde_json;
 extern crate stable_deref_trait;
+extern crate tempdir;
+extern crate tempfile;
+extern crate time;
+extern crate uuid;
+
+#[cfg(test)]
+#[macro_use]
+extern crate matches;
 
 #[cfg(test)]
 extern crate env_logger;
 
-#[cfg(feature="simdcompression")]
+#[cfg(feature = "simdcompression")]
 extern crate libc;
 
 #[cfg(windows)]
 extern crate winapi;
 
-#[cfg(test)]
-extern crate test;
 #[cfg(test)]
 extern crate rand;
+#[cfg(test)]
+extern crate test;
 
+extern crate tinysegmenter;
+
+#[macro_use]
+extern crate downcast;
 
 #[cfg(test)]
 mod functional_test;
 
-
-
-
 #[macro_use]
 mod macros;
 
@@ -98,43 +190,33 @@ mod core;
 mod compression;
 mod indexer;
 mod common;
+
+#[allow(unused_doc_comment)]
 mod error;
-mod analyzer;
+pub mod tokenizer;
 mod datastruct;
 
 pub mod termdict;
-
-/// Row-oriented, slow, compressed storage of documents
 pub mod store;
-
-/// Query module
 pub mod query;
-
 pub mod directory;
-
-/// Collector module
 pub mod collector;
-
-/// Postings module (also called inverted index)
 pub mod postings;
-
-/// Schema
 pub mod schema;
-
 pub mod fastfield;
 
+mod docset;
+pub use self::docset::{DocSet, SkipResult};
 
 pub use directory::Directory;
-pub use core::{Index, Segment, SegmentId, SegmentMeta, Searcher};
+pub use core::{Index, Searcher, Segment, SegmentId, SegmentMeta};
 pub use indexer::IndexWriter;
-pub use schema::{Term, Document};
-pub use core::SegmentReader;
+pub use schema::{Document, Term};
+pub use core::{InvertedIndexReader, SegmentReader};
 pub use self::common::TimerTree;
 
-pub use postings::DocSet;
 pub use postings::Postings;
 pub use core::SegmentComponent;
-pub use postings::SegmentPostingsOption;
 
 pub use common::{i64_to_u64, u64_to_i64};
 
@@ -142,9 +224,9 @@ pub use common::{i64_to_u64, u64_to_i64};
 /// whether it was compiled with the simd compression.
 pub fn version() -> &'static str {
     if cfg!(feature = "simdcompression") {
-        concat!(version!(), "-simd")
+        concat!(env!("CARGO_PKG_VERSION"), "-simd")
     } else {
-        concat!(version!(), "-nosimd")
+        concat!(env!("CARGO_PKG_VERSION"), "-nosimd")
     }
 }
 
@@ -156,15 +238,19 @@ pub mod merge_policy {
     pub use indexer::DefaultMergePolicy;
 }
 
-/// u32 identifying a document within a segment.
-/// Documents have their doc id assigned incrementally,
+/// A `u32` identifying a document within a segment.
+/// Documents have their `DocId` assigned incrementally,
 /// as they are added in the segment.
 pub type DocId = u32;
 
-/// f32 the score of a document.
+/// A f32 that represents the relevance of the document to the query
+///
+/// This is modelled internally as a `f32`. The
+/// larger the number, the more relevant the document
+/// to the search
 pub type Score = f32;
 
-/// A segment local id identifies a segment.
+/// A `SegmentLocalId` identifies a segment.
 /// It only makes sense for a given searcher.
 pub type SegmentLocalId = u32;
 
@@ -183,7 +269,6 @@ impl DocAddress {
     }
 }
 
-
 /// `DocAddress` contains all the necessary information
 /// to identify a document given a `Searcher` object.
 ///
@@ -195,7 +280,6 @@ impl DocAddress {
 #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
 pub struct DocAddress(pub SegmentLocalId, pub DocId);
 
-
 #[cfg(test)]
 mod tests {
 
@@ -203,14 +287,12 @@ mod tests {
     use Index;
     use core::SegmentReader;
     use query::BooleanQuery;
-    use postings::SegmentPostingsOption;
     use schema::*;
-    use DocSet;
+    use docset::DocSet;
     use IndexWriter;
-    use postings::SegmentPostingsOption::FreqAndPositions;
-    use fastfield::{FastFieldReader, U64FastFieldReader, I64FastFieldReader};
     use Postings;
-    use rand::{XorShiftRng, Rng, SeedableRng};
+    use rand::{Rng, SeedableRng, XorShiftRng};
+    use rand::distributions::{IndependentSample, Range};
 
     fn generate_array_with_seed(n: usize, ratio: f32, seed_val: u32) -> Vec<u32> {
         let seed: &[u32; 4] = &[1, 2, 3, seed_val];
@@ -221,11 +303,20 @@ mod tests {
             .collect()
     }
 
+    pub fn generate_nonunique_unsorted(max_value: u32, n_elems: usize) -> Vec<u32> {
+        let seed: &[u32; 4] = &[1, 2, 3, 4];
+        let mut rng: XorShiftRng = XorShiftRng::from_seed(*seed);
+        let between = Range::new(0u32, max_value);
+        (0..n_elems)
+            .map(|_| between.ind_sample(&mut rng))
+            .collect::<Vec<u32>>()
+    }
+
     pub fn generate_array(n: usize, ratio: f32) -> Vec<u32> {
         generate_array_with_seed(n, ratio, 4)
     }
 
-    fn sample_with_seed(n: u32, ratio: f32, seed_val: u32) -> Vec<u32> {
+    pub fn sample_with_seed(n: u32, ratio: f32, seed_val: u32) -> Vec<u32> {
         let seed: &[u32; 4] = &[1, 2, 3, seed_val];
         let mut rng: XorShiftRng = XorShiftRng::from_seed(*seed);
         (0..n).filter(|_| rng.next_f32() < ratio).collect()
@@ -258,11 +349,10 @@ mod tests {
             }
             assert!(index_writer.commit().is_ok());
         }
-
     }
 
     #[test]
-    fn test_docfreq() {
+    fn test_docfreq1() {
         let mut schema_builder = SchemaBuilder::default();
         let text_field = schema_builder.add_text_field("text", TEXT);
         let index = Index::create_in_ram(schema_builder.build());
@@ -301,7 +391,6 @@ mod tests {
         }
     }
 
-
     #[test]
     fn test_fieldnorm() {
         let mut schema_builder = SchemaBuilder::default();
@@ -334,7 +423,6 @@ mod tests {
         }
     }
 
-
     #[test]
     fn test_delete_postings1() {
         let mut schema_builder = SchemaBuilder::default();
@@ -390,15 +478,24 @@ mod tests {
             index.load_searchers().unwrap();
             let searcher = index.searcher();
             let reader = searcher.segment_reader(0);
-            assert!(reader.read_postings(&term_abcd, FreqAndPositions).is_none());
+            let inverted_index = reader.inverted_index(text_field);
+            assert!(
+                inverted_index
+                    .read_postings(&term_abcd, IndexRecordOption::WithFreqsAndPositions)
+                    .is_none()
+            );
             {
-                let mut postings = reader.read_postings(&term_a, FreqAndPositions).unwrap();
+                let mut postings = inverted_index
+                    .read_postings(&term_a, IndexRecordOption::WithFreqsAndPositions)
+                    .unwrap();
                 assert!(postings.advance());
                 assert_eq!(postings.doc(), 5);
                 assert!(!postings.advance());
             }
             {
-                let mut postings = reader.read_postings(&term_b, FreqAndPositions).unwrap();
+                let mut postings = inverted_index
+                    .read_postings(&term_b, IndexRecordOption::WithFreqsAndPositions)
+                    .unwrap();
                 assert!(postings.advance());
                 assert_eq!(postings.doc(), 3);
                 assert!(postings.advance());
@@ -424,16 +521,25 @@ mod tests {
             index.load_searchers().unwrap();
             let searcher = index.searcher();
             let reader = searcher.segment_reader(0);
+            let inverted_index = reader.inverted_index(term_abcd.field());
 
-            assert!(reader.read_postings(&term_abcd, FreqAndPositions).is_none());
+            assert!(
+                inverted_index
+                    .read_postings(&term_abcd, IndexRecordOption::WithFreqsAndPositions)
+                    .is_none()
+            );
             {
-                let mut postings = reader.read_postings(&term_a, FreqAndPositions).unwrap();
+                let mut postings = inverted_index
+                    .read_postings(&term_a, IndexRecordOption::WithFreqsAndPositions)
+                    .unwrap();
                 assert!(postings.advance());
                 assert_eq!(postings.doc(), 5);
                 assert!(!postings.advance());
             }
             {
-                let mut postings = reader.read_postings(&term_b, FreqAndPositions).unwrap();
+                let mut postings = inverted_index
+                    .read_postings(&term_b, IndexRecordOption::WithFreqsAndPositions)
+                    .unwrap();
                 assert!(postings.advance());
                 assert_eq!(postings.doc(), 3);
                 assert!(postings.advance());
@@ -451,7 +557,7 @@ mod tests {
             {
                 index_writer.delete_term(Term::from_field_text(text_field, "c"));
             }
-            index_writer = index_writer.rollback().unwrap();
+            index_writer.rollback().unwrap();
             index_writer.delete_term(Term::from_field_text(text_field, "a"));
             index_writer.commit().unwrap();
         }
@@ -459,13 +565,22 @@ mod tests {
             index.load_searchers().unwrap();
             let searcher = index.searcher();
             let reader = searcher.segment_reader(0);
-            assert!(reader.read_postings(&term_abcd, FreqAndPositions).is_none());
+            let inverted_index = reader.inverted_index(term_abcd.field());
+            assert!(
+                inverted_index
+                    .read_postings(&term_abcd, IndexRecordOption::WithFreqsAndPositions)
+                    .is_none()
+            );
             {
-                let mut postings = reader.read_postings(&term_a, FreqAndPositions).unwrap();
+                let mut postings = inverted_index
+                    .read_postings(&term_a, IndexRecordOption::WithFreqsAndPositions)
+                    .unwrap();
                 assert!(!postings.advance());
             }
             {
-                let mut postings = reader.read_postings(&term_b, FreqAndPositions).unwrap();
+                let mut postings = inverted_index
+                    .read_postings(&term_b, IndexRecordOption::WithFreqsAndPositions)
+                    .unwrap();
                 assert!(postings.advance());
                 assert_eq!(postings.doc(), 3);
                 assert!(postings.advance());
@@ -473,7 +588,9 @@ mod tests {
                 assert!(!postings.advance());
             }
             {
-                let mut postings = reader.read_postings(&term_c, FreqAndPositions).unwrap();
+                let mut postings = inverted_index
+                    .read_postings(&term_c, IndexRecordOption::WithFreqsAndPositions)
+                    .unwrap();
                 assert!(postings.advance());
                 assert_eq!(postings.doc(), 4);
                 assert!(!postings.advance());
@@ -481,7 +598,6 @@ mod tests {
         }
     }
 
-
     #[test]
     fn test_indexed_u64() {
         let mut schema_builder = SchemaBuilder::default();
@@ -497,7 +613,8 @@ mod tests {
         let term = Term::from_field_u64(field, 1u64);
         let mut postings = searcher
             .segment_reader(0)
-            .read_postings(&term, SegmentPostingsOption::NoFreq)
+            .inverted_index(term.field())
+            .read_postings(&term, IndexRecordOption::Basic)
             .unwrap();
         assert!(postings.advance());
         assert_eq!(postings.doc(), 0);
@@ -520,13 +637,30 @@ mod tests {
         let term = Term::from_field_i64(value_field, negative_val);
         let mut postings = searcher
             .segment_reader(0)
-            .read_postings(&term, SegmentPostingsOption::NoFreq)
+            .inverted_index(term.field())
+            .read_postings(&term, IndexRecordOption::Basic)
             .unwrap();
         assert!(postings.advance());
         assert_eq!(postings.doc(), 0);
         assert!(!postings.advance());
     }
 
+    #[test]
+    fn test_indexedfield_not_in_documents() {
+        let mut schema_builder = SchemaBuilder::default();
+        let text_field = schema_builder.add_text_field("text", TEXT);
+        let absent_field = schema_builder.add_text_field("text", TEXT);
+        let schema = schema_builder.build();
+        let index = Index::create_in_ram(schema);
+        let mut index_writer = index.writer_with_num_threads(2, 40_000_000).unwrap();
+        index_writer.add_document(doc!(text_field=>"a"));
+        assert!(index_writer.commit().is_ok());
+        assert!(index.load_searchers().is_ok());
+        let searcher = index.searcher();
+        let segment_reader = searcher.segment_reader(0);
+        segment_reader.inverted_index(absent_field); //< should not panic
+    }
+
     #[test]
     fn test_delete_postings2() {
         let mut schema_builder = SchemaBuilder::default();
@@ -582,10 +716,17 @@ mod tests {
             index.load_searchers().unwrap();
             let searcher = index.searcher();
             let reader = searcher.segment_reader(0);
+            let inverted_index = reader.inverted_index(text_field);
             let term_abcd = Term::from_field_text(text_field, "abcd");
-            assert!(reader.read_postings(&term_abcd, FreqAndPositions).is_none());
+            assert!(
+                inverted_index
+                    .read_postings(&term_abcd, IndexRecordOption::WithFreqsAndPositions)
+                    .is_none()
+            );
             let term_af = Term::from_field_text(text_field, "af");
-            let mut postings = reader.read_postings(&term_af, FreqAndPositions).unwrap();
+            let mut postings = inverted_index
+                .read_postings(&term_af, IndexRecordOption::WithFreqsAndPositions)
+                .unwrap();
             assert!(postings.advance());
             assert_eq!(postings.doc(), 0);
             assert_eq!(postings.term_freq(), 3);
@@ -627,29 +768,43 @@ mod tests {
                 collector.docs()
             };
             {
-                assert_eq!(get_doc_ids(vec![Term::from_field_text(text_field, "a")]),
-                           vec![1, 2]);
+                assert_eq!(
+                    get_doc_ids(vec![Term::from_field_text(text_field, "a")]),
+                    vec![1, 2]
+                );
             }
             {
-                assert_eq!(get_doc_ids(vec![Term::from_field_text(text_field, "af")]),
-                           vec![0]);
+                assert_eq!(
+                    get_doc_ids(vec![Term::from_field_text(text_field, "af")]),
+                    vec![0]
+                );
             }
             {
-                assert_eq!(get_doc_ids(vec![Term::from_field_text(text_field, "b")]),
-                           vec![0, 1, 2]);
+                assert_eq!(
+                    get_doc_ids(vec![Term::from_field_text(text_field, "b")]),
+                    vec![0, 1, 2]
+                );
             }
             {
-                assert_eq!(get_doc_ids(vec![Term::from_field_text(text_field, "c")]),
-                           vec![1, 2]);
+                assert_eq!(
+                    get_doc_ids(vec![Term::from_field_text(text_field, "c")]),
+                    vec![1, 2]
+                );
             }
             {
-                assert_eq!(get_doc_ids(vec![Term::from_field_text(text_field, "d")]),
-                           vec![2]);
+                assert_eq!(
+                    get_doc_ids(vec![Term::from_field_text(text_field, "d")]),
+                    vec![2]
+                );
             }
             {
-                assert_eq!(get_doc_ids(vec![Term::from_field_text(text_field, "b"),
-                                            Term::from_field_text(text_field, "a")]),
-                           vec![0, 1, 2]);
+                assert_eq!(
+                    get_doc_ids(vec![
+                        Term::from_field_text(text_field, "b"),
+                        Term::from_field_text(text_field, "a"),
+                    ]),
+                    vec![0, 1, 2]
+                );
             }
         }
     }
@@ -720,35 +875,29 @@ mod tests {
         let searcher = index.searcher();
         let segment_reader: &SegmentReader = searcher.segment_reader(0);
         {
-            let fast_field_reader_res =
-                segment_reader.get_fast_field_reader::<U64FastFieldReader>(text_field);
+            let fast_field_reader_res = segment_reader.fast_field_reader::<u64>(text_field);
             assert!(fast_field_reader_res.is_err());
         }
         {
-            let fast_field_reader_res =
-                segment_reader.get_fast_field_reader::<U64FastFieldReader>(stored_int_field);
+            let fast_field_reader_res = segment_reader.fast_field_reader::<u64>(stored_int_field);
             assert!(fast_field_reader_res.is_err());
         }
         {
-            let fast_field_reader_res =
-                segment_reader.get_fast_field_reader::<U64FastFieldReader>(fast_field_signed);
+            let fast_field_reader_res = segment_reader.fast_field_reader::<u64>(fast_field_signed);
             assert!(fast_field_reader_res.is_err());
         }
         {
-            let fast_field_reader_res =
-                segment_reader.get_fast_field_reader::<I64FastFieldReader>(fast_field_signed);
+            let fast_field_reader_res = segment_reader.fast_field_reader::<i64>(fast_field_signed);
             assert!(fast_field_reader_res.is_ok());
             let fast_field_reader = fast_field_reader_res.unwrap();
             assert_eq!(fast_field_reader.get(0), 4i64)
         }
 
         {
-            let fast_field_reader_res =
-                segment_reader.get_fast_field_reader::<I64FastFieldReader>(fast_field_signed);
+            let fast_field_reader_res = segment_reader.fast_field_reader::<i64>(fast_field_signed);
             assert!(fast_field_reader_res.is_ok());
             let fast_field_reader = fast_field_reader_res.unwrap();
             assert_eq!(fast_field_reader.get(0), 4i64)
         }
-
     }
 }

src/macros.rs

@@ -1,9 +1,7 @@
-
 macro_rules! get(
     ($e:expr) => (match $e { Some(e) => e, None => return None })
 );
 
-
 /// `doc!` is a shortcut that helps building `Document`
 /// objects.
 ///
@@ -56,7 +54,7 @@ macro_rules! doc(
             ($crate::Document::default())
         }
     }; // avoids a warning due to the useless `mut`.
-    ($($field:ident => $value:expr),*) => {
+    ($($field:expr => $value:expr),*) => {
         {
             let mut document = $crate::Document::default();
             $(
@@ -65,4 +63,41 @@ macro_rules! doc(
             document
         }
     };
+    // if there is a trailing comma retry with the trailing comma stripped.
+    ($($field:expr => $value:expr),+ ,) => {
+        doc!( $( $field => $value ), *);
+    };
 );
+
+#[cfg(test)]
+mod test {
+    use schema::{SchemaBuilder, FAST, TEXT};
+
+    #[test]
+    fn test_doc_basic() {
+        let mut schema_builder = SchemaBuilder::new();
+        let title = schema_builder.add_text_field("title", TEXT);
+        let author = schema_builder.add_text_field("text", TEXT);
+        let likes = schema_builder.add_u64_field("num_u64", FAST);
+        let _schema = schema_builder.build();
+        let _doc = doc!(
+            title => "Life Aquatic",
+            author => "Wes Anderson",
+            likes => 4u64
+            );
+    }
+
+    #[test]
+    fn test_doc_trailing_comma() {
+        let mut schema_builder = SchemaBuilder::new();
+        let title = schema_builder.add_text_field("title", TEXT);
+        let author = schema_builder.add_text_field("text", TEXT);
+        let likes = schema_builder.add_u64_field("num_u64", FAST);
+        let _schema = schema_builder.build();
+        let _doc = doc!(
+            title => "Life Aquatic",
+            author => "Wes Anderson",
+            likes => 4u64,
+            );
+    }
+}

src/postings/freq_handler.rs

@@ -1,125 +0,0 @@
-use compression::BlockDecoder;
-use common::VInt;
-use common::BinarySerializable;
-use compression::{CompositeDecoder, VIntDecoder};
-use postings::SegmentPostingsOption;
-use compression::NUM_DOCS_PER_BLOCK;
-
-
-/// `FreqHandler`  is in charge of decompressing
-/// frequencies and/or positions.
-pub struct FreqHandler {
-    freq_decoder: BlockDecoder,
-    positions: Vec<u32>,
-    option: SegmentPostingsOption,
-    positions_offsets: [usize; NUM_DOCS_PER_BLOCK + 1],
-}
-
-
-fn read_positions(data: &[u8]) -> Vec<u32> {
-    let mut composite_reader = CompositeDecoder::new();
-    let mut readable: &[u8] = data;
-    let uncompressed_len = VInt::deserialize(&mut readable).unwrap().0 as usize;
-    composite_reader.uncompress_unsorted(readable, uncompressed_len);
-    composite_reader.into()
-}
-
-
-
-impl FreqHandler {
-    /// Returns a `FreqHandler` that just decodes `DocId`s.
-    pub fn new_without_freq() -> FreqHandler {
-        FreqHandler {
-            freq_decoder: BlockDecoder::with_val(1u32),
-            positions: Vec::new(),
-            option: SegmentPostingsOption::NoFreq,
-            positions_offsets: [0; NUM_DOCS_PER_BLOCK + 1],
-        }
-    }
-
-    /// Returns a `FreqHandler` that decodes `DocId`s and term frequencies.
-    pub fn new_with_freq() -> FreqHandler {
-        FreqHandler {
-            freq_decoder: BlockDecoder::new(),
-            positions: Vec::new(),
-            option: SegmentPostingsOption::Freq,
-            positions_offsets: [0; NUM_DOCS_PER_BLOCK + 1],
-        }
-    }
-
-    /// Returns a `FreqHandler` that decodes `DocId`s, term frequencies, and term positions.
-    pub fn new_with_freq_and_position(position_data: &[u8]) -> FreqHandler {
-        let positions = read_positions(position_data);
-        FreqHandler {
-            freq_decoder: BlockDecoder::new(),
-            positions: positions,
-            option: SegmentPostingsOption::FreqAndPositions,
-            positions_offsets: [0; NUM_DOCS_PER_BLOCK + 1],
-        }
-    }
-
-    fn fill_positions_offset(&mut self) {
-        let mut cur_position: usize = self.positions_offsets[NUM_DOCS_PER_BLOCK];
-        let mut i: usize = 0;
-        self.positions_offsets[i] = cur_position;
-        let mut last_cur_position = cur_position;
-        for &doc_freq in self.freq_decoder.output_array() {
-            i += 1;
-            let mut cumulated_pos = 0u32;
-            // this next loop decodes delta positions into normal positions.
-            for j in last_cur_position..(last_cur_position + (doc_freq as usize)) {
-                cumulated_pos += self.positions[j];
-                self.positions[j] = cumulated_pos;
-            }
-            cur_position += doc_freq as usize;
-            self.positions_offsets[i] = cur_position;
-            last_cur_position = cur_position;
-        }
-    }
-
-
-    /// Accessor to term frequency
-    ///
-    /// idx is the offset of the current doc in the block.
-    /// It takes value between 0 and 128.
-    pub fn freq(&self, idx: usize) -> u32 {
-        self.freq_decoder.output(idx)
-    }
-
-    /// Accessor to the positions
-    ///
-    /// idx is the offset of the current doc in the block.
-    /// It takes value between 0 and 128.
-    pub fn positions(&self, idx: usize) -> &[u32] {
-        let start = self.positions_offsets[idx];
-        let stop = self.positions_offsets[idx + 1];
-        &self.positions[start..stop]
-    }
-
-    /// Decompresses a complete frequency block
-    pub fn read_freq_block<'a>(&mut self, data: &'a [u8]) -> &'a [u8] {
-        match self.option {
-            SegmentPostingsOption::NoFreq => data,
-            SegmentPostingsOption::Freq => self.freq_decoder.uncompress_block_unsorted(data),
-            SegmentPostingsOption::FreqAndPositions => {
-                let remaining: &'a [u8] = self.freq_decoder.uncompress_block_unsorted(data);
-                self.fill_positions_offset();
-                remaining
-            }
-        }
-    }
-
-    /// Decompresses an incomplete frequency block
-    pub fn read_freq_vint(&mut self, data: &[u8], num_els: usize) {
-        match self.option {
-            SegmentPostingsOption::NoFreq => {}
-            SegmentPostingsOption::Freq => {
-                self.freq_decoder.uncompress_vint_unsorted(data, num_els);
-            }
-            SegmentPostingsOption::FreqAndPositions => {
-                self.freq_decoder.uncompress_vint_unsorted(data, num_els);
-                self.fill_positions_offset();
-            }
-        }
-    }
-}

src/postings/intersection.rs

@@ -1,134 +0,0 @@
-use postings::DocSet;
-use postings::SkipResult;
-use DocId;
-
-/// Creates a `DocSet` that iterator through the intersection of two `DocSet`s.
-pub struct IntersectionDocSet<TDocSet: DocSet> {
-    docsets: Vec<TDocSet>,
-    finished: bool,
-    doc: DocId,
-}
-
-impl<TDocSet: DocSet> From<Vec<TDocSet>> for IntersectionDocSet<TDocSet> {
-    fn from(mut docsets: Vec<TDocSet>) -> IntersectionDocSet<TDocSet> {
-        assert!(docsets.len() >= 2);
-        docsets.sort_by_key(|docset| docset.size_hint());
-        IntersectionDocSet {
-            docsets: docsets,
-            finished: false,
-            doc: 0u32,
-        }
-    }
-}
-
-impl<TDocSet: DocSet> IntersectionDocSet<TDocSet> {
-    /// Returns an array to the underlying `DocSet`s of the intersection.
-    /// These `DocSet` are in the same position as the `IntersectionDocSet`,
-    /// so that user can access their `docfreq` and `positions`.
-    pub fn docsets(&self) -> &[TDocSet] {
-        &self.docsets[..]
-    }
-}
-
-
-impl<TDocSet: DocSet> DocSet for IntersectionDocSet<TDocSet> {
-    fn size_hint(&self) -> usize {
-        self.docsets
-            .iter()
-            .map(|docset| docset.size_hint())
-            .min()
-            .unwrap() // safe as docsets cannot be empty.
-    }
-
-    #[allow(never_loop)]
-    fn advance(&mut self) -> bool {
-        if self.finished {
-            return false;
-        }
-
-        let mut candidate_doc = self.doc;
-        let mut candidate_ord = self.docsets.len();
-
-        'outer: loop {
-
-            for (ord, docset) in self.docsets.iter_mut().enumerate() {
-                if ord != candidate_ord {
-                    // `candidate_ord` is already at the
-                    // right position.
-                    //
-                    // Calling `skip_next` would advance this docset
-                    // and miss it.
-                    match docset.skip_next(candidate_doc) {
-                        SkipResult::Reached => {}
-                        SkipResult::OverStep => {
-                            // this is not in the intersection,
-                            // let's update our candidate.
-                            candidate_doc = docset.doc();
-                            candidate_ord = ord;
-                            continue 'outer;
-                        }
-                        SkipResult::End => {
-                            self.finished = true;
-                            return false;
-                        }
-                    }
-                }
-            }
-
-            self.doc = candidate_doc;
-            return true;
-        }
-    }
-
-    fn doc(&self) -> DocId {
-        self.doc
-    }
-}
-
-
-#[cfg(test)]
-mod tests {
-
-    use postings::{DocSet, VecPostings, IntersectionDocSet};
-
-    #[test]
-    fn test_intersection() {
-        {
-            let left = VecPostings::from(vec![1, 3, 9]);
-            let right = VecPostings::from(vec![3, 4, 9, 18]);
-            let mut intersection = IntersectionDocSet::from(vec![left, right]);
-            assert!(intersection.advance());
-            assert_eq!(intersection.doc(), 3);
-            assert!(intersection.advance());
-            assert_eq!(intersection.doc(), 9);
-            assert!(!intersection.advance());
-        }
-        {
-            let a = VecPostings::from(vec![1, 3, 9]);
-            let b = VecPostings::from(vec![3, 4, 9, 18]);
-            let c = VecPostings::from(vec![1, 5, 9, 111]);
-            let mut intersection = IntersectionDocSet::from(vec![a, b, c]);
-            assert!(intersection.advance());
-            assert_eq!(intersection.doc(), 9);
-            assert!(!intersection.advance());
-        }
-    }
-
-    #[test]
-    fn test_intersection_zero() {
-        let left = VecPostings::from(vec![0]);
-        let right = VecPostings::from(vec![0]);
-        let mut intersection = IntersectionDocSet::from(vec![left, right]);
-        assert!(intersection.advance());
-        assert_eq!(intersection.doc(), 0);
-    }
-
-    #[test]
-    fn test_intersection_empty() {
-        let a = VecPostings::from(vec![1, 3]);
-        let b = VecPostings::from(vec![1, 4]);
-        let c = VecPostings::from(vec![3, 9]);
-        let mut intersection = IntersectionDocSet::from(vec![a, b, c]);
-        assert!(!intersection.advance());
-    }
-}

src/postings/mod.rs

@@ -1,57 +1,62 @@
+/*!
+Postings module (also called inverted index)
+*/
+
 /// Postings module
 ///
 /// Postings, also called inverted lists, is the key datastructure
 /// to full-text search.
 
-
 mod postings;
 mod recorder;
 mod serializer;
 mod postings_writer;
 mod term_info;
-mod vec_postings;
 mod segment_postings;
-mod intersection;
-mod freq_handler;
-mod docset;
-mod segment_postings_option;
 
-pub use self::docset::{SkipResult, DocSet};
-use self::recorder::{Recorder, NothingRecorder, TermFrequencyRecorder, TFAndPositionRecorder};
-pub use self::serializer::PostingsSerializer;
+use self::recorder::{NothingRecorder, Recorder, TFAndPositionRecorder, TermFrequencyRecorder};
+pub use self::serializer::{FieldSerializer, InvertedIndexSerializer};
 pub(crate) use self::postings_writer::MultiFieldPostingsWriter;
+
 pub use self::term_info::TermInfo;
 pub use self::postings::Postings;
 
-#[cfg(test)]
-pub use self::vec_postings::VecPostings;
+pub use self::segment_postings::{BlockSegmentPostings, SegmentPostings};
 
-pub use self::segment_postings::{SegmentPostings, BlockSegmentPostings};
-pub use self::intersection::IntersectionDocSet;
-pub use self::freq_handler::FreqHandler;
-pub use self::segment_postings_option::SegmentPostingsOption;
 pub use common::HasLen;
 
+pub(crate) type UnorderedTermId = u64;
+
+#[allow(enum_variant_names)]
+pub(crate) enum FreqReadingOption {
+    NoFreq,
+    SkipFreq,
+    ReadFreq,
+}
 
 #[cfg(test)]
-mod tests {
+pub mod tests {
 
     use super::*;
-    use schema::{Document, INT_INDEXED, TEXT, STRING, SchemaBuilder, Term};
+    use docset::{DocSet, SkipResult};
+    use DocId;
+    use Score;
+    use query::Intersection;
+    use query::Scorer;
+    use schema::{Document, SchemaBuilder, Term, INT_INDEXED, STRING, TEXT};
     use core::SegmentComponent;
     use indexer::SegmentWriter;
     use core::SegmentReader;
     use core::Index;
-    use postings::SegmentPostingsOption::FreqAndPositions;
+    use schema::IndexRecordOption;
     use std::iter;
     use datastruct::stacker::Heap;
-    use fastfield::FastFieldReader;
     use query::TermQuery;
     use schema::Field;
     use test::{self, Bencher};
     use indexer::operation::AddOperation;
     use tests;
-    use rand::{XorShiftRng, Rng, SeedableRng};
+    use rand::{Rng, SeedableRng, XorShiftRng};
 
     #[test]
     pub fn test_position_write() {
@@ -60,21 +65,90 @@ mod tests {
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
         let mut segment = index.new_segment();
-        let mut posting_serializer = PostingsSerializer::open(&mut segment).unwrap();
-        posting_serializer.new_field(text_field);
-        posting_serializer.new_term("abc".as_bytes()).unwrap();
-        for doc_id in 0u32..3u32 {
-            let positions = vec![1, 2, 3, 2];
-            posting_serializer.write_doc(doc_id, 2, &positions).unwrap();
+        let mut posting_serializer = InvertedIndexSerializer::open(&mut segment).unwrap();
+        {
+            let mut field_serializer = posting_serializer.new_field(text_field).unwrap();
+            field_serializer.new_term("abc".as_bytes()).unwrap();
+            for doc_id in 0u32..120u32 {
+                let delta_positions = vec![1, 2, 3, 2];
+                field_serializer
+                    .write_doc(doc_id, 2, &delta_positions)
+                    .unwrap();
+            }
+            field_serializer.close_term().unwrap();
         }
-        posting_serializer.close_term().unwrap();
         posting_serializer.close().unwrap();
         let read = segment.open_read(SegmentComponent::POSITIONS).unwrap();
-        assert!(read.len() <= 16);
+        assert!(read.len() <= 140);
     }
 
     #[test]
-    pub fn test_position_and_fieldnorm() {
+    pub fn test_skip_positions() {
+        let mut schema_builder = SchemaBuilder::new();
+        let title = schema_builder.add_text_field("title", TEXT);
+        let schema = schema_builder.build();
+        let index = Index::create_in_ram(schema);
+        let mut index_writer = index.writer_with_num_threads(1, 30_000_000).unwrap();
+        index_writer.add_document(doc!(title => r#"abc abc abc"#));
+        index_writer.add_document(doc!(title => r#"abc be be be be abc"#));
+        for _ in 0..1_000 {
+            index_writer.add_document(doc!(title => r#"abc abc abc"#));
+        }
+        index_writer.add_document(doc!(title => r#"abc be be be be abc"#));
+        index_writer.commit().unwrap();
+        index.load_searchers().unwrap();
+        let searcher = index.searcher();
+        let query = TermQuery::new(
+            Term::from_field_text(title, "abc"),
+            IndexRecordOption::WithFreqsAndPositions,
+        );
+        let weight = query.specialized_weight(&*searcher, true);
+        {
+            let mut scorer = weight
+                .specialized_scorer(searcher.segment_reader(0u32))
+                .unwrap();
+            scorer.advance();
+            assert_eq!(&[0, 1, 2], scorer.postings().positions());
+            scorer.advance();
+            assert_eq!(&[0, 5], scorer.postings().positions());
+        }
+        {
+            let mut scorer = weight
+                .specialized_scorer(searcher.segment_reader(0u32))
+                .unwrap();
+            scorer.advance();
+            scorer.advance();
+            assert_eq!(&[0, 5], scorer.postings().positions());
+        }
+        {
+            let mut scorer = weight
+                .specialized_scorer(searcher.segment_reader(0u32))
+                .unwrap();
+            assert_eq!(scorer.skip_next(1), SkipResult::Reached);
+            assert_eq!(scorer.doc(), 1);
+            assert_eq!(&[0, 5], scorer.postings().positions());
+        }
+        {
+            let mut scorer = weight
+                .specialized_scorer(searcher.segment_reader(0u32))
+                .unwrap();
+            assert_eq!(scorer.skip_next(1002), SkipResult::Reached);
+            assert_eq!(scorer.doc(), 1002);
+            assert_eq!(&[0, 5], scorer.postings().positions());
+        }
+        {
+            let mut scorer = weight
+                .specialized_scorer(searcher.segment_reader(0u32))
+                .unwrap();
+            assert_eq!(scorer.skip_next(100), SkipResult::Reached);
+            assert_eq!(scorer.skip_next(1002), SkipResult::Reached);
+            assert_eq!(scorer.doc(), 1002);
+            assert_eq!(&[0, 5], scorer.postings().positions());
+        }
+    }
+
+    #[test]
+    pub fn test_position_and_fieldnorm1() {
         let mut schema_builder = SchemaBuilder::default();
         let text_field = schema_builder.add_text_field("text", TEXT);
         let schema = schema_builder.build();
@@ -83,8 +157,8 @@ mod tests {
 
         let heap = Heap::with_capacity(10_000_000);
         {
-            let mut segment_writer = SegmentWriter::for_segment(&heap, 18, segment.clone(), &schema)
-                .unwrap();
+            let mut segment_writer =
+                SegmentWriter::for_segment(&heap, 18, segment.clone(), &schema).unwrap();
             {
                 let mut doc = Document::default();
                 // checking that position works if the field has two values
@@ -94,7 +168,7 @@ mod tests {
                     opstamp: 0u64,
                     document: doc,
                 };
-                segment_writer.add_document(&op, &schema).unwrap();
+                segment_writer.add_document(op, &schema).unwrap();
             }
             {
                 let mut doc = Document::default();
@@ -103,7 +177,7 @@ mod tests {
                     opstamp: 1u64,
                     document: doc,
                 };
-                segment_writer.add_document(&op, &schema).unwrap();
+                segment_writer.add_document(op, &schema).unwrap();
             }
             for i in 2..1000 {
                 let mut doc = Document::default();
@@ -114,12 +188,12 @@ mod tests {
                     opstamp: 2u64,
                     document: doc,
                 };
-                segment_writer.add_document(&op, &schema).unwrap();
+                segment_writer.add_document(op, &schema).unwrap();
             }
             segment_writer.finalize().unwrap();
         }
         {
-            let segment_reader = SegmentReader::open(segment).unwrap();
+            let segment_reader = SegmentReader::open(&segment).unwrap();
             {
                 let fieldnorm_reader = segment_reader.get_fieldnorms_reader(text_field).unwrap();
                 assert_eq!(fieldnorm_reader.get(0), 8 + 5);
@@ -130,20 +204,25 @@ mod tests {
             }
             {
                 let term_a = Term::from_field_text(text_field, "abcdef");
-                assert!(segment_reader
-                            .read_postings(&term_a, FreqAndPositions)
-                            .is_none());
+                assert!(
+                    segment_reader
+                        .inverted_index(term_a.field())
+                        .read_postings(&term_a, IndexRecordOption::WithFreqsAndPositions)
+                        .is_none()
+                );
             }
             {
                 let term_a = Term::from_field_text(text_field, "a");
                 let mut postings_a = segment_reader
-                    .read_postings(&term_a, FreqAndPositions)
+                    .inverted_index(term_a.field())
+                    .read_postings(&term_a, IndexRecordOption::WithFreqsAndPositions)
                     .unwrap();
                 assert_eq!(postings_a.len(), 1000);
                 assert!(postings_a.advance());
                 assert_eq!(postings_a.doc(), 0);
                 assert_eq!(postings_a.term_freq(), 6);
                 assert_eq!(postings_a.positions(), [0, 2, 4, 6, 7, 13]);
+                assert_eq!(postings_a.positions(), [0, 2, 4, 6, 7, 13]);
                 assert!(postings_a.advance());
                 assert_eq!(postings_a.doc(), 1u32);
                 assert_eq!(postings_a.term_freq(), 1);
@@ -158,7 +237,8 @@ mod tests {
             {
                 let term_e = Term::from_field_text(text_field, "e");
                 let mut postings_e = segment_reader
-                    .read_postings(&term_e, FreqAndPositions)
+                    .inverted_index(term_e.field())
+                    .read_postings(&term_e, IndexRecordOption::WithFreqsAndPositions)
                     .unwrap();
                 assert_eq!(postings_e.len(), 1000 - 2);
                 for i in 2u32..1000u32 {
@@ -197,11 +277,13 @@ mod tests {
             assert!(index_writer.commit().is_ok());
         }
         index.load_searchers().unwrap();
-        let term_query = TermQuery::new(Term::from_field_text(text_field, "a"),
-                                        SegmentPostingsOption::NoFreq);
+        let term_query = TermQuery::new(
+            Term::from_field_text(text_field, "a"),
+            IndexRecordOption::Basic,
+        );
         let searcher = index.searcher();
-        let mut term_weight = term_query.specialized_weight(&*searcher);
-        term_weight.segment_postings_options = SegmentPostingsOption::FreqAndPositions;
+        let mut term_weight = term_query.specialized_weight(&*searcher, true);
+        term_weight.index_record_option = IndexRecordOption::WithFreqsAndPositions;
         let segment_reader = &searcher.segment_readers()[0];
         let mut term_scorer = term_weight.specialized_scorer(segment_reader).unwrap();
         assert!(term_scorer.advance());
@@ -245,7 +327,8 @@ mod tests {
         for i in 0..num_docs - 1 {
             for j in i + 1..num_docs {
                 let mut segment_postings = segment_reader
-                    .read_postings(&term_2, SegmentPostingsOption::NoFreq)
+                    .inverted_index(term_2.field())
+                    .read_postings(&term_2, IndexRecordOption::Basic)
                     .unwrap();
 
                 assert_eq!(segment_postings.skip_next(i), SkipResult::Reached);
@@ -258,7 +341,8 @@ mod tests {
 
         {
             let mut segment_postings = segment_reader
-                .read_postings(&term_2, SegmentPostingsOption::NoFreq)
+                .inverted_index(term_2.field())
+                .read_postings(&term_2, IndexRecordOption::Basic)
                 .unwrap();
 
             // check that `skip_next` advances the iterator
@@ -278,7 +362,8 @@ mod tests {
         // check that filtering works
         {
             let mut segment_postings = segment_reader
-                .read_postings(&term_0, SegmentPostingsOption::NoFreq)
+                .inverted_index(term_0.field())
+                .read_postings(&term_0, IndexRecordOption::Basic)
                 .unwrap();
 
             for i in 0..num_docs / 2 {
@@ -287,7 +372,8 @@ mod tests {
             }
 
             let mut segment_postings = segment_reader
-                .read_postings(&term_0, SegmentPostingsOption::NoFreq)
+                .inverted_index(term_0.field())
+                .read_postings(&term_0, IndexRecordOption::Basic)
                 .unwrap();
 
             for i in 0..num_docs / 2 - 1 {
@@ -311,7 +397,8 @@ mod tests {
         // make sure seeking still works
         for i in 0..num_docs {
             let mut segment_postings = segment_reader
-                .read_postings(&term_2, SegmentPostingsOption::NoFreq)
+                .inverted_index(term_2.field())
+                .read_postings(&term_2, IndexRecordOption::Basic)
                 .unwrap();
 
             if i % 2 == 0 {
@@ -326,7 +413,8 @@ mod tests {
         // now try with a longer sequence
         {
             let mut segment_postings = segment_reader
-                .read_postings(&term_2, SegmentPostingsOption::NoFreq)
+                .inverted_index(term_2.field())
+                .read_postings(&term_2, IndexRecordOption::Basic)
                 .unwrap();
 
             let mut last = 2; // start from 5 to avoid seeking to 3 twice
@@ -361,20 +449,21 @@ mod tests {
         // finally, check that it's empty
         {
             let mut segment_postings = segment_reader
-                .read_postings(&term_2, SegmentPostingsOption::NoFreq)
+                .inverted_index(term_2.field())
+                .read_postings(&term_2, IndexRecordOption::Basic)
                 .unwrap();
 
             assert_eq!(segment_postings.skip_next(0), SkipResult::End);
 
             let mut segment_postings = segment_reader
-                .read_postings(&term_2, SegmentPostingsOption::NoFreq)
+                .inverted_index(term_2.field())
+                .read_postings(&term_2, IndexRecordOption::Basic)
                 .unwrap();
 
             assert_eq!(segment_postings.skip_next(num_docs), SkipResult::End);
         }
     }
 
-
     lazy_static! {
         static ref TERM_A: Term = {
             let field = Field(0);
@@ -433,11 +522,12 @@ mod tests {
         let segment_reader = searcher.segment_reader(0);
 
         b.iter(|| {
-                   let mut segment_postings = segment_reader
-                       .read_postings(&*TERM_A, SegmentPostingsOption::NoFreq)
-                       .unwrap();
-                   while segment_postings.advance() {}
-               });
+            let mut segment_postings = segment_reader
+                .inverted_index(TERM_A.field())
+                .read_postings(&*TERM_A, IndexRecordOption::Basic)
+                .unwrap();
+            while segment_postings.advance() {}
+        });
     }
 
     #[bench]
@@ -446,21 +536,27 @@ mod tests {
         let segment_reader = searcher.segment_reader(0);
         b.iter(|| {
             let segment_postings_a = segment_reader
-                .read_postings(&*TERM_A, SegmentPostingsOption::NoFreq)
+                .inverted_index(TERM_A.field())
+                .read_postings(&*TERM_A, IndexRecordOption::Basic)
                 .unwrap();
             let segment_postings_b = segment_reader
-                .read_postings(&*TERM_B, SegmentPostingsOption::NoFreq)
+                .inverted_index(TERM_B.field())
+                .read_postings(&*TERM_B, IndexRecordOption::Basic)
                 .unwrap();
             let segment_postings_c = segment_reader
-                .read_postings(&*TERM_C, SegmentPostingsOption::NoFreq)
+                .inverted_index(TERM_C.field())
+                .read_postings(&*TERM_C, IndexRecordOption::Basic)
                 .unwrap();
             let segment_postings_d = segment_reader
-                .read_postings(&*TERM_D, SegmentPostingsOption::NoFreq)
+                .inverted_index(TERM_D.field())
+                .read_postings(&*TERM_D, IndexRecordOption::Basic)
                 .unwrap();
-            let mut intersection = IntersectionDocSet::from(vec![segment_postings_a,
-                                                                 segment_postings_b,
-                                                                 segment_postings_c,
-                                                                 segment_postings_d]);
+            let mut intersection = Intersection::from(vec![
+                segment_postings_a,
+                segment_postings_b,
+                segment_postings_c,
+                segment_postings_d,
+            ]);
             while intersection.advance() {}
         });
     }
@@ -471,7 +567,8 @@ mod tests {
         let docs = tests::sample(segment_reader.num_docs(), p);
 
         let mut segment_postings = segment_reader
-            .read_postings(&*TERM_A, SegmentPostingsOption::NoFreq)
+            .inverted_index(TERM_A.field())
+            .read_postings(&*TERM_A, IndexRecordOption::Basic)
             .unwrap();
 
         let mut existing_docs = Vec::new();
@@ -487,7 +584,8 @@ mod tests {
 
         b.iter(|| {
             let mut segment_postings = segment_reader
-                .read_postings(&*TERM_A, SegmentPostingsOption::NoFreq)
+                .inverted_index(TERM_A.field())
+                .read_postings(&*TERM_A, IndexRecordOption::Basic)
                 .unwrap();
             for doc in &existing_docs {
                 if segment_postings.skip_next(*doc) == SkipResult::End {
@@ -524,7 +622,8 @@ mod tests {
         b.iter(|| {
             let n: u32 = test::black_box(17);
             let mut segment_postings = segment_reader
-                .read_postings(&*TERM_A, SegmentPostingsOption::NoFreq)
+                .inverted_index(TERM_A.field())
+                .read_postings(&*TERM_A, IndexRecordOption::Basic)
                 .unwrap();
             let mut s = 0u32;
             while segment_postings.advance() {
@@ -534,4 +633,70 @@ mod tests {
         });
     }
 
+    /// Wraps a given docset, and forward alls call but the
+    /// `.skip_next(...)`. This is useful to test that a specialized
+    /// implementation of `.skip_next(...)` is consistent
+    /// with the default implementation.
+    pub(crate) struct UnoptimizedDocSet<TDocSet: DocSet>(TDocSet);
+
+    impl<TDocSet: DocSet> UnoptimizedDocSet<TDocSet> {
+        pub fn wrap(docset: TDocSet) -> UnoptimizedDocSet<TDocSet> {
+            UnoptimizedDocSet(docset)
+        }
+    }
+
+    impl<TDocSet: DocSet> DocSet for UnoptimizedDocSet<TDocSet> {
+        fn advance(&mut self) -> bool {
+            self.0.advance()
+        }
+
+        fn doc(&self) -> DocId {
+            self.0.doc()
+        }
+
+        fn size_hint(&self) -> u32 {
+            self.0.size_hint()
+        }
+    }
+
+    impl<TScorer: Scorer> Scorer for UnoptimizedDocSet<TScorer> {
+        fn score(&mut self) -> Score {
+            self.0.score()
+        }
+    }
+
+    pub fn test_skip_against_unoptimized<F: Fn() -> Box<DocSet>>(
+        postings_factory: F,
+        targets: Vec<u32>,
+    ) {
+        for target in targets {
+            let mut postings_opt = postings_factory();
+            let mut postings_unopt = UnoptimizedDocSet::wrap(postings_factory());
+            let skip_result_opt = postings_opt.skip_next(target);
+            let skip_result_unopt = postings_unopt.skip_next(target);
+            assert_eq!(
+                skip_result_unopt, skip_result_opt,
+                "Failed while skipping to {}",
+                target
+            );
+            match skip_result_opt {
+                SkipResult::Reached => assert_eq!(postings_opt.doc(), target),
+                SkipResult::OverStep => assert!(postings_opt.doc() > target),
+                SkipResult::End => {
+                    return;
+                }
+            }
+            while postings_opt.advance() {
+                assert!(postings_unopt.advance());
+                assert_eq!(
+                    postings_opt.doc(),
+                    postings_unopt.doc(),
+                    "Failed while skipping to {}",
+                    target
+                );
+            }
+            assert!(!postings_unopt.advance());
+        }
+    }
+
 }

src/postings/postings.rs

@@ -1,5 +1,4 @@
-use std::borrow::Borrow;
-use postings::docset::DocSet;
+use docset::DocSet;
 
 /// Postings (also called inverted list)
 ///
@@ -18,27 +17,3 @@ pub trait Postings: DocSet {
     /// token ordinals.
     fn positions(&self) -> &[u32];
 }
-
-impl<TPostings: Postings> Postings for Box<TPostings> {
-    fn term_freq(&self) -> u32 {
-        let unboxed: &TPostings = self.borrow();
-        unboxed.term_freq()
-    }
-
-    fn positions(&self) -> &[u32] {
-        let unboxed: &TPostings = self.borrow();
-        unboxed.positions()
-    }
-}
-
-impl<'a, TPostings: Postings> Postings for &'a mut TPostings {
-    fn term_freq(&self) -> u32 {
-        let unref: &TPostings = *self;
-        unref.term_freq()
-    }
-
-    fn positions(&self) -> &[u32] {
-        let unref: &TPostings = *self;
-        unref.positions()
-    }
-}

src/postings/postings_writer.rs

@@ -1,45 +1,52 @@
 use DocId;
 use schema::Term;
-use schema::FieldValue;
-use postings::PostingsSerializer;
+use postings::{FieldSerializer, InvertedIndexSerializer};
 use std::io;
+use std::collections::HashMap;
 use postings::Recorder;
-use analyzer::SimpleTokenizer;
 use Result;
-use schema::{Schema, Field};
-use analyzer::StreamingIterator;
+use schema::{Field, Schema};
 use std::marker::PhantomData;
 use std::ops::DerefMut;
-use datastruct::stacker::{HashMap, Heap};
-use postings::{NothingRecorder, TermFrequencyRecorder, TFAndPositionRecorder};
+use datastruct::stacker::{Heap, TermHashMap};
+use postings::{NothingRecorder, TFAndPositionRecorder, TermFrequencyRecorder};
 use schema::FieldEntry;
 use schema::FieldType;
-use schema::TextIndexingOptions;
+use tokenizer::Token;
+use tokenizer::TokenStream;
+use schema::IndexRecordOption;
+use postings::UnorderedTermId;
+use termdict::TermOrdinal;
 
-fn posting_from_field_entry<'a>(field_entry: &FieldEntry,
-                                heap: &'a Heap)
-                                -> Box<PostingsWriter + 'a> {
+fn posting_from_field_entry<'a>(
+    field_entry: &FieldEntry,
+    heap: &'a Heap,
+) -> Box<PostingsWriter + 'a> {
     match *field_entry.field_type() {
-        FieldType::Str(ref text_options) => {
-            match text_options.get_indexing_options() {
-                TextIndexingOptions::TokenizedWithFreq => {
+        FieldType::Str(ref text_options) => text_options
+            .get_indexing_options()
+            .map(|indexing_options| match indexing_options.index_option() {
+                IndexRecordOption::Basic => {
+                    SpecializedPostingsWriter::<NothingRecorder>::new_boxed(heap)
+                }
+                IndexRecordOption::WithFreqs => {
                     SpecializedPostingsWriter::<TermFrequencyRecorder>::new_boxed(heap)
                 }
-                TextIndexingOptions::TokenizedWithFreqAndPosition => {
+                IndexRecordOption::WithFreqsAndPositions => {
                     SpecializedPostingsWriter::<TFAndPositionRecorder>::new_boxed(heap)
                 }
-                _ => SpecializedPostingsWriter::<NothingRecorder>::new_boxed(heap),
-            }
+            })
+            .unwrap_or_else(|| SpecializedPostingsWriter::<NothingRecorder>::new_boxed(heap)),
+        FieldType::U64(_) | FieldType::I64(_) | FieldType::HierarchicalFacet => {
+            SpecializedPostingsWriter::<NothingRecorder>::new_boxed(heap)
         }
-        FieldType::U64(_) |
-        FieldType::I64(_) => SpecializedPostingsWriter::<NothingRecorder>::new_boxed(heap),
     }
 }
 
-
 pub struct MultiFieldPostingsWriter<'a> {
     heap: &'a Heap,
-    term_index: HashMap<'a>,
+    schema: Schema,
+    term_index: TermHashMap<'a>,
     per_field_postings_writers: Vec<Box<PostingsWriter + 'a>>,
 }
 
@@ -47,48 +54,51 @@ impl<'a> MultiFieldPostingsWriter<'a> {
     /// Create a new `MultiFieldPostingsWriter` given
     /// a schema and a heap.
     pub fn new(schema: &Schema, table_bits: usize, heap: &'a Heap) -> MultiFieldPostingsWriter<'a> {
-        let term_index = HashMap::new(table_bits, heap);
+        let term_index = TermHashMap::new(table_bits, heap);
         let per_field_postings_writers: Vec<_> = schema
             .fields()
             .iter()
-            .map(|field_entry| {
-                posting_from_field_entry(field_entry, heap)
-            })
+            .map(|field_entry| posting_from_field_entry(field_entry, heap))
             .collect();
-
         MultiFieldPostingsWriter {
-            heap: heap,
-            term_index: term_index,
-            per_field_postings_writers: per_field_postings_writers,
+            schema: schema.clone(),
+            heap,
+            term_index,
+            per_field_postings_writers,
         }
     }
 
-    pub fn index_text(&mut self, doc: DocId, field: Field, field_values: &[&FieldValue]) -> u32 {
+    pub fn index_text(&mut self, doc: DocId, field: Field, token_stream: &mut TokenStream) -> u32 {
         let postings_writer = self.per_field_postings_writers[field.0 as usize].deref_mut();
-        postings_writer.index_text(&mut self.term_index, doc, field, field_values, self.heap)
+        postings_writer.index_text(&mut self.term_index, doc, field, token_stream, self.heap)
     }
 
-    pub fn suscribe(&mut self, doc: DocId, term: &Term) {
+    pub fn subscribe(&mut self, doc: DocId, term: &Term) -> UnorderedTermId {
         let postings_writer = self.per_field_postings_writers[term.field().0 as usize].deref_mut();
-        postings_writer.suscribe(&mut self.term_index, doc, 0u32, term, self.heap)
+        postings_writer.subscribe(&mut self.term_index, doc, 0u32, term, self.heap)
     }
 
-
     /// Serialize the inverted index.
     /// It pushes all term, one field at a time, towards the
     /// postings serializer.
     #[allow(needless_range_loop)]
-    pub fn serialize(&self, serializer: &mut PostingsSerializer) -> Result<()> {
-        let mut term_offsets: Vec<(&[u8], u32)> = self.term_index.iter().collect();
-        term_offsets.sort_by_key(|&(k, _v)| k);
+    pub fn serialize(
+        &self,
+        serializer: &mut InvertedIndexSerializer,
+    ) -> Result<HashMap<Field, HashMap<UnorderedTermId, TermOrdinal>>> {
+        let mut term_offsets: Vec<(&[u8], u32, UnorderedTermId)> = self.term_index.iter().collect();
+        term_offsets.sort_by_key(|&(k, _, _)| k);
 
         let mut offsets: Vec<(Field, usize)> = vec![];
         let term_offsets_it = term_offsets
             .iter()
             .cloned()
-            .map(|(key, _)| Term::wrap(key).field())
+            .map(|(key, _, _)| Term::wrap(key).field())
             .enumerate();
 
+        let mut unordered_term_mappings: HashMap<Field, HashMap<UnorderedTermId, TermOrdinal>> =
+            HashMap::new();
+
         let mut prev_field = Field(u32::max_value());
         for (offset, field) in term_offsets_it {
             if field != prev_field {
@@ -97,14 +107,39 @@ impl<'a> MultiFieldPostingsWriter<'a> {
             }
         }
         offsets.push((Field(0), term_offsets.len()));
+
         for i in 0..(offsets.len() - 1) {
             let (field, start) = offsets[i];
             let (_, stop) = offsets[i + 1];
+
+            let field_entry = self.schema.get_field_entry(field);
+
+            match field_entry.field_type() {
+                FieldType::Str(_) | FieldType::HierarchicalFacet => {
+                    // populating the (unordered term ord) -> (ordered term ord) mapping
+                    // for the field.
+                    let mut unordered_term_ids = term_offsets[start..stop]
+                        .iter()
+                        .map(|&(_, _, bucket)| bucket);
+                    let mut mapping: HashMap<UnorderedTermId, TermOrdinal> = unordered_term_ids
+                        .enumerate()
+                        .map(|(term_ord, unord_term_id)| (unord_term_id as UnorderedTermId, term_ord as TermOrdinal))
+                        .collect();
+                    unordered_term_mappings.insert(field, mapping);
+                }
+                FieldType::U64(_) | FieldType::I64(_) => {}
+            }
+
             let postings_writer = &self.per_field_postings_writers[field.0 as usize];
-            postings_writer
-                .serialize(field, &term_offsets[start..stop], serializer, self.heap)?;
+            let mut field_serializer = serializer.new_field(field)?;
+            postings_writer.serialize(
+                &term_offsets[start..stop],
+                &mut field_serializer,
+                self.heap,
+            )?;
+            field_serializer.close()?;
         }
-        Ok(())
+        Ok(unordered_term_mappings)
     }
 
     /// Return true iff the term dictionary is saturated.
@@ -113,7 +148,6 @@ impl<'a> MultiFieldPostingsWriter<'a> {
     }
 }
 
-
 /// The `PostingsWriter` is in charge of receiving documenting
 /// and building a `Segment` in anonymous memory.
 ///
@@ -126,49 +160,40 @@ pub trait PostingsWriter {
     /// * term - the term
     /// * heap - heap used to store the postings informations as well as the terms
     /// in the hashmap.
-    fn suscribe(&mut self,
-                term_index: &mut HashMap,
-                doc: DocId,
-                pos: u32,
-                term: &Term,
-                heap: &Heap);
+    fn subscribe(
+        &mut self,
+        term_index: &mut TermHashMap,
+        doc: DocId,
+        pos: u32,
+        term: &Term,
+        heap: &Heap,
+    ) -> UnorderedTermId;
 
     /// Serializes the postings on disk.
     /// The actual serialization format is handled by the `PostingsSerializer`.
-    fn serialize(&self,
-                 field: Field,
-                 term_addrs: &[(&[u8], u32)],
-                 serializer: &mut PostingsSerializer,
-                 heap: &Heap)
-                 -> io::Result<()>;
+    fn serialize(
+        &self,
+        term_addrs: &[(&[u8], u32, UnorderedTermId)],
+        serializer: &mut FieldSerializer,
+        heap: &Heap,
+    ) -> io::Result<()>;
 
-    /// Tokenize a text and suscribe all of its token.
-    fn index_text<'a>(&mut self,
-                      term_index: &mut HashMap,
-                      doc_id: DocId,
-                      field: Field,
-                      field_values: &[&'a FieldValue],
-                      heap: &Heap)
-                      -> u32 {
-        let mut pos = 0u32;
-        let mut num_tokens: u32 = 0u32;
+    /// Tokenize a text and subscribe all of its token.
+    fn index_text(
+        &mut self,
+        term_index: &mut TermHashMap,
+        doc_id: DocId,
+        field: Field,
+        token_stream: &mut TokenStream,
+        heap: &Heap,
+    ) -> u32 {
         let mut term = unsafe { Term::with_capacity(100) };
         term.set_field(field);
-        for field_value in field_values {
-            let mut tokens = SimpleTokenizer.tokenize(field_value.value().text());
-            // right now num_tokens and pos are redundant, but it should
-            // change when we get proper analyzers
-            while let Some(token) = tokens.next() {
-                term.set_text(token);
-                self.suscribe(term_index, doc_id, pos, &term, heap);
-                pos += 1u32;
-                num_tokens += 1u32;
-            }
-            pos += 1;
-            // THIS is to avoid phrase query accross field repetition.
-            // span queries might still match though :|
-        }
-        num_tokens
+        let mut sink = |token: &Token| {
+            term.set_text(token.text.as_str());
+            self.subscribe(term_index, doc_id, token.position as u32, &term, heap);
+        };
+        token_stream.process(&mut sink)
     }
 }
 
@@ -183,7 +208,7 @@ impl<'a, Rec: Recorder + 'static> SpecializedPostingsWriter<'a, Rec> {
     /// constructor
     pub fn new(heap: &'a Heap) -> SpecializedPostingsWriter<'a, Rec> {
         SpecializedPostingsWriter {
-            heap: heap,
+            heap,
             _recorder_type: PhantomData,
         }
     }
@@ -195,14 +220,16 @@ impl<'a, Rec: Recorder + 'static> SpecializedPostingsWriter<'a, Rec> {
 }
 
 impl<'a, Rec: Recorder + 'static> PostingsWriter for SpecializedPostingsWriter<'a, Rec> {
-    fn suscribe(&mut self,
-                term_index: &mut HashMap,
-                doc: DocId,
-                position: u32,
-                term: &Term,
-                heap: &Heap) {
+    fn subscribe(
+        &mut self,
+        term_index: &mut TermHashMap,
+        doc: DocId,
+        position: u32,
+        term: &Term,
+        heap: &Heap,
+    ) -> UnorderedTermId {
         debug_assert!(term.as_slice().len() >= 4);
-        let recorder: &mut Rec = term_index.get_or_create(term);
+        let (term_ord, recorder): (UnorderedTermId, &mut Rec) = term_index.get_or_create(term);
         let current_doc = recorder.current_doc();
         if current_doc != doc {
             if current_doc != u32::max_value() {
@@ -211,22 +238,21 @@ impl<'a, Rec: Recorder + 'static> PostingsWriter for SpecializedPostingsWriter<'
             recorder.new_doc(doc, heap);
         }
         recorder.record_position(position, heap);
+        term_ord
     }
 
-    fn serialize(&self,
-                 field: Field,
-                 term_addrs: &[(&[u8], u32)],
-                 serializer: &mut PostingsSerializer,
-                 heap: &Heap)
-                 -> io::Result<()> {
-        serializer.new_field(field);
-        for &(term_bytes, addr) in term_addrs {
+    fn serialize(
+        &self,
+        term_addrs: &[(&[u8], u32, UnorderedTermId)],
+        serializer: &mut FieldSerializer,
+        heap: &Heap,
+    ) -> io::Result<()> {
+        for &(term_bytes, addr, _) in term_addrs {
             let recorder: &mut Rec = self.heap.get_mut_ref(addr);
-            try!(serializer.new_term(term_bytes));
-            try!(recorder.serialize(addr, serializer, heap));
-            try!(serializer.close_term());
+            serializer.new_term(&term_bytes[4..])?;
+            recorder.serialize(addr, serializer, heap)?;
+            serializer.close_term()?;
         }
         Ok(())
     }
 }
-

src/postings/recorder.rs

@@ -1,10 +1,10 @@
 use DocId;
-use std::io;
-use postings::PostingsSerializer;
+use std::{self, io};
+use postings::FieldSerializer;
 use datastruct::stacker::{ExpUnrolledLinkedList, Heap, HeapAllocable};
 
 const EMPTY_ARRAY: [u32; 0] = [0u32; 0];
-const POSITION_END: u32 = 4294967295;
+const POSITION_END: u32 = std::u32::MAX;
 
 /// Recorder is in charge of recording relevant information about
 /// the presence of a term in a document.
@@ -27,15 +27,15 @@ pub trait Recorder: HeapAllocable {
     /// Close the document. It will help record the term frequency.
     fn close_doc(&mut self, heap: &Heap);
     /// Pushes the postings information to the serializer.
-    fn serialize(&self,
-                 self_addr: u32,
-                 serializer: &mut PostingsSerializer,
-                 heap: &Heap)
-                 -> io::Result<()>;
+    fn serialize(
+        &self,
+        self_addr: u32,
+        serializer: &mut FieldSerializer,
+        heap: &Heap,
+    ) -> io::Result<()>;
 }
 
 /// Only records the doc ids
-#[repr(C, packed)]
 pub struct NothingRecorder {
     stack: ExpUnrolledLinkedList,
     current_doc: DocId,
@@ -64,21 +64,20 @@ impl Recorder for NothingRecorder {
 
     fn close_doc(&mut self, _heap: &Heap) {}
 
-    fn serialize(&self,
-                 self_addr: u32,
-                 serializer: &mut PostingsSerializer,
-                 heap: &Heap)
-                 -> io::Result<()> {
+    fn serialize(
+        &self,
+        self_addr: u32,
+        serializer: &mut FieldSerializer,
+        heap: &Heap,
+    ) -> io::Result<()> {
         for doc in self.stack.iter(self_addr, heap) {
-            try!(serializer.write_doc(doc, 0u32, &EMPTY_ARRAY));
+            serializer.write_doc(doc, 0u32, &EMPTY_ARRAY)?;
         }
         Ok(())
     }
 }
 
-
 /// Recorder encoding document ids, and term frequencies
-#[repr(C, packed)]
 pub struct TermFrequencyRecorder {
     stack: ExpUnrolledLinkedList,
     current_doc: DocId,
@@ -115,12 +114,12 @@ impl Recorder for TermFrequencyRecorder {
         self.current_tf = 0;
     }
 
-
-    fn serialize(&self,
-                 self_addr: u32,
-                 serializer: &mut PostingsSerializer,
-                 heap: &Heap)
-                 -> io::Result<()> {
+    fn serialize(
+        &self,
+        self_addr: u32,
+        serializer: &mut FieldSerializer,
+        heap: &Heap,
+    ) -> io::Result<()> {
         // the last document has not been closed...
         // its term freq is self.current_tf.
         let mut doc_iter = self.stack
@@ -138,7 +137,6 @@ impl Recorder for TermFrequencyRecorder {
 }
 
 /// Recorder encoding term frequencies as well as positions.
-#[repr(C, packed)]
 pub struct TFAndPositionRecorder {
     stack: ExpUnrolledLinkedList,
     current_doc: DocId,
@@ -171,11 +169,12 @@ impl Recorder for TFAndPositionRecorder {
         self.stack.push(POSITION_END, heap);
     }
 
-    fn serialize(&self,
-                 self_addr: u32,
-                 serializer: &mut PostingsSerializer,
-                 heap: &Heap)
-                 -> io::Result<()> {
+    fn serialize(
+        &self,
+        self_addr: u32,
+        serializer: &mut FieldSerializer,
+        heap: &Heap,
+    ) -> io::Result<()> {
         let mut doc_positions = Vec::with_capacity(100);
         let mut positions_iter = self.stack.iter(self_addr, heap);
         while let Some(doc) = positions_iter.next() {
@@ -189,7 +188,7 @@ impl Recorder for TFAndPositionRecorder {
                     prev_position = position;
                 }
             }
-            try!(serializer.write_doc(doc, doc_positions.len() as u32, &doc_positions));
+            serializer.write_doc(doc, doc_positions.len() as u32, &doc_positions)?;
         }
         Ok(())
     }

src/postings/segment_postings.rs

@@ -1,65 +1,159 @@
-use compression::{NUM_DOCS_PER_BLOCK, BlockDecoder, VIntDecoder};
+use compression::{BlockDecoder, CompressedIntStream, VIntDecoder, COMPRESSION_BLOCK_SIZE};
 use DocId;
-use postings::{Postings, FreqHandler, DocSet, HasLen, SkipResult};
+
+use common::BitSet;
+use common::HasLen;
+use postings::Postings;
+use docset::{DocSet, SkipResult};
 use std::cmp;
-use fastfield::DeleteBitSet;
 use fst::Streamer;
+use compression::compressed_block_size;
+use fastfield::DeleteBitSet;
+use std::cell::UnsafeCell;
+use directory::{ReadOnlySource, SourceRead};
+use postings::FreqReadingOption;
+use postings::serializer::PostingsSerializer;
 
+const EMPTY_POSITIONS: [u32; 0] = [0u32; 0];
 
-const EMPTY_DATA: [u8; 0] = [0u8; 0];
+struct PositionComputer {
+    // store the amount of position int
+    // before reading positions.
+    //
+    // if none, position are already loaded in
+    // the positions vec.
+    position_to_skip: Option<usize>,
+    positions: Vec<u32>,
+    positions_stream: CompressedIntStream,
+}
 
+impl PositionComputer {
+    pub fn new(positions_stream: CompressedIntStream) -> PositionComputer {
+        PositionComputer {
+            position_to_skip: None,
+            positions: vec![],
+            positions_stream,
+        }
+    }
+
+    pub fn add_skip(&mut self, num_skip: usize) {
+        self.position_to_skip = Some(
+            self.position_to_skip
+                .map(|prev_skip| prev_skip + num_skip)
+                .unwrap_or(0),
+        );
+    }
+
+    pub fn positions(&mut self, term_freq: usize) -> &[u32] {
+        if let Some(num_skip) = self.position_to_skip {
+            self.positions.resize(term_freq, 0u32);
+            self.positions_stream.skip(num_skip);
+            self.positions_stream.read(&mut self.positions[..term_freq]);
+
+            let mut cum = 0u32;
+            for i in 0..term_freq as usize {
+                cum += self.positions[i];
+                self.positions[i] = cum;
+            }
+            self.position_to_skip = None;
+        }
+        &self.positions[..term_freq]
+    }
+}
 
 /// `SegmentPostings` represents the inverted list or postings associated to
 /// a term in a `Segment`.
 ///
 /// As we iterate through the `SegmentPostings`, the frequencies are optionally decoded.
 /// Positions on the other hand, are optionally entirely decoded upfront.
-pub struct SegmentPostings<'a> {
-    block_cursor: BlockSegmentPostings<'a>,
+pub struct SegmentPostings {
+    block_cursor: BlockSegmentPostings,
     cur: usize,
     delete_bitset: DeleteBitSet,
+    position_computer: Option<UnsafeCell<PositionComputer>>,
 }
 
-impl<'a> SegmentPostings<'a> {
+impl SegmentPostings {
+    /// Creates a segment postings object with the given documents
+    /// and no frequency encoded.
+    ///
+    /// This method is mostly useful for unit tests.
+    ///
+    /// It serializes the doc ids using tantivy's codec
+    /// and returns a `SegmentPostings` object that embeds a
+    /// buffer with the serialized data.
+    pub fn create_from_docs(docs: &[u32]) -> SegmentPostings {
+        let mut buffer = Vec::new();
+        {
+            let mut postings_serializer = PostingsSerializer::new(&mut buffer, false);
+            for &doc in docs {
+                postings_serializer.write_doc(doc, 1u32).unwrap();
+            }
+            postings_serializer.close_term().unwrap();
+        }
+        let data = ReadOnlySource::from(buffer);
+        let block_segment_postings = BlockSegmentPostings::from_data(
+            docs.len(),
+            SourceRead::from(data),
+            FreqReadingOption::NoFreq,
+        );
+        SegmentPostings::from_block_postings(block_segment_postings, DeleteBitSet::empty(), None)
+    }
+
     /// Reads a Segment postings from an &[u8]
     ///
     /// * `len` - number of document in the posting lists.
     /// * `data` - data array. The complete data is not necessarily used.
     /// * `freq_handler` - the freq handler is in charge of decoding
     ///   frequencies and/or positions
-    pub fn from_block_postings(segment_block_postings: BlockSegmentPostings<'a>,
-                               delete_bitset: DeleteBitSet)
-                               -> SegmentPostings<'a> {
+    pub fn from_block_postings(
+        segment_block_postings: BlockSegmentPostings,
+        delete_bitset: DeleteBitSet,
+        positions_stream_opt: Option<CompressedIntStream>,
+    ) -> SegmentPostings {
+        let position_computer =
+            positions_stream_opt.map(|stream| UnsafeCell::new(PositionComputer::new(stream)));
         SegmentPostings {
             block_cursor: segment_block_postings,
-            cur: NUM_DOCS_PER_BLOCK, // cursor within the block
-            delete_bitset: delete_bitset,
+            cur: COMPRESSION_BLOCK_SIZE, // cursor within the block
+            delete_bitset,
+            position_computer,
         }
     }
 
     /// Returns an empty segment postings object
-    pub fn empty() -> SegmentPostings<'static> {
+    pub fn empty() -> SegmentPostings {
         let empty_block_cursor = BlockSegmentPostings::empty();
         SegmentPostings {
             block_cursor: empty_block_cursor,
             delete_bitset: DeleteBitSet::empty(),
-            cur: NUM_DOCS_PER_BLOCK,
+            cur: COMPRESSION_BLOCK_SIZE,
+            position_computer: None,
+        }
+    }
+
+    fn position_add_skip<F: FnOnce() -> usize>(&self, num_skips_fn: F) {
+        if let Some(position_computer) = self.position_computer.as_ref() {
+            let num_skips = num_skips_fn();
+            unsafe {
+                (*position_computer.get()).add_skip(num_skips);
+            }
         }
     }
 }
 
-
-impl<'a> DocSet for SegmentPostings<'a> {
+impl DocSet for SegmentPostings {
     // goes to the next element.
     // next needs to be called a first time to point to the correct element.
     #[inline]
     fn advance(&mut self) -> bool {
         loop {
+            self.position_add_skip(|| self.term_freq() as usize);
             self.cur += 1;
             if self.cur >= self.block_cursor.block_len() {
                 self.cur = 0;
                 if !self.block_cursor.advance() {
-                    self.cur = NUM_DOCS_PER_BLOCK;
+                    self.cur = COMPRESSION_BLOCK_SIZE;
                     return false;
                 }
             }
@@ -69,12 +163,15 @@ impl<'a> DocSet for SegmentPostings<'a> {
         }
     }
 
-
     fn skip_next(&mut self, target: DocId) -> SkipResult {
         if !self.advance() {
             return SkipResult::End;
         }
 
+        // in the following, thanks to the call to advance above,
+        // we know that the position is not loaded and we need
+        // to skip every doc_freq we cross.
+
         // skip blocks until one that might contain the target
         loop {
             // check if we need to go to the next block
@@ -83,13 +180,25 @@ impl<'a> DocSet for SegmentPostings<'a> {
                 (block_docs[self.cur], block_docs[block_docs.len() - 1])
             };
             if target > last_doc_in_block {
+                // we add skip for the current term independantly,
+                // so that position_add_skip will decide if it should
+                // just set itself to Some(0) or effectively
+                // add the term freq.
+                //let num_skips: u32 = ;
+                self.position_add_skip(|| {
+                    let freqs_skipped = &self.block_cursor.freqs()[self.cur..];
+                    let sum_freq: u32 = freqs_skipped.iter().cloned().sum();
+                    sum_freq as usize
+                });
+
                 if !self.block_cursor.advance() {
                     return SkipResult::End;
                 }
+
                 self.cur = 0;
             } else {
                 if target < current_doc {
-                    // We've overpassed the target after the first `advance` call
+                    // We've passed the target after the first `advance` call
                     // or we're at the beginning of a block.
                     // Either way, we're on the first `DocId` greater than `target`
                     return SkipResult::OverStep;
@@ -135,6 +244,13 @@ impl<'a> DocSet for SegmentPostings<'a> {
 
             // `doc` is now >= `target`
             let doc = block_docs[start];
+
+            self.position_add_skip(|| {
+                let freqs_skipped = &self.block_cursor.freqs()[self.cur..start];
+                let sum_freqs: u32 = freqs_skipped.iter().sum();
+                sum_freqs as usize
+            });
+
             self.cur = start;
 
             if !self.delete_bitset.is_deleted(doc) {
@@ -152,32 +268,56 @@ impl<'a> DocSet for SegmentPostings<'a> {
         }
     }
 
-    fn size_hint(&self) -> usize {
-        self.len()
+    fn size_hint(&self) -> u32 {
+        self.len() as u32
     }
 
+    /// Return the current document's `DocId`.
     #[inline]
     fn doc(&self) -> DocId {
         let docs = self.block_cursor.docs();
-        assert!(self.cur < docs.len(),
-                "Have you forgotten to call `.advance()` at least once before calling .doc().");
+        debug_assert!(
+            self.cur < docs.len(),
+            "Have you forgotten to call `.advance()` at least once before calling .doc()."
+        );
         docs[self.cur]
     }
+
+    fn append_to_bitset(&mut self, bitset: &mut BitSet) {
+        // finish the current block
+        if self.advance() {
+            for &doc in &self.block_cursor.docs()[self.cur..] {
+                bitset.insert(doc);
+            }
+            // ... iterate through the remaining blocks.
+            while self.block_cursor.advance() {
+                for &doc in self.block_cursor.docs() {
+                    bitset.insert(doc);
+                }
+            }
+        }
+    }
 }
 
-impl<'a> HasLen for SegmentPostings<'a> {
+impl HasLen for SegmentPostings {
     fn len(&self) -> usize {
         self.block_cursor.doc_freq()
     }
 }
 
-impl<'a> Postings for SegmentPostings<'a> {
+impl Postings for SegmentPostings {
     fn term_freq(&self) -> u32 {
-        self.block_cursor.freq_handler().freq(self.cur)
+        self.block_cursor.freq(self.cur)
     }
 
     fn positions(&self) -> &[u32] {
-        self.block_cursor.freq_handler().positions(self.cur)
+        let term_freq = self.term_freq();
+        self.position_computer
+            .as_ref()
+            .map(|position_computer| unsafe {
+                (&mut *position_computer.get()).positions(term_freq as usize)
+            })
+            .unwrap_or(&EMPTY_POSITIONS[..])
     }
 }
 
@@ -188,31 +328,35 @@ impl<'a> Postings for SegmentPostings<'a> {
 ///
 /// While it is useful for some very specific high-performance
 /// use cases, you should prefer using `SegmentPostings` for most usage.
-pub struct BlockSegmentPostings<'a> {
-    block_decoder: BlockDecoder,
+pub struct BlockSegmentPostings {
+    doc_decoder: BlockDecoder,
+    freq_decoder: BlockDecoder,
+    freq_reading_option: FreqReadingOption,
+
     doc_freq: usize,
     doc_offset: DocId,
-    num_binpacked_blocks: usize,
+    num_bitpacked_blocks: usize,
     num_vint_docs: usize,
-    remaining_data: &'a [u8],
-    freq_handler: FreqHandler,
+    remaining_data: SourceRead,
 }
 
-impl<'a> BlockSegmentPostings<'a> {
-    pub(crate) fn from_data(doc_freq: usize,
-                            data: &'a [u8],
-                            freq_handler: FreqHandler)
-                            -> BlockSegmentPostings<'a> {
-        let num_binpacked_blocks: usize = (doc_freq as usize) / NUM_DOCS_PER_BLOCK;
-        let num_vint_docs = (doc_freq as usize) - NUM_DOCS_PER_BLOCK * num_binpacked_blocks;
+impl BlockSegmentPostings {
+    pub(crate) fn from_data(
+        doc_freq: usize,
+        data: SourceRead,
+        freq_reading_option: FreqReadingOption,
+    ) -> BlockSegmentPostings {
+        let num_bitpacked_blocks: usize = (doc_freq as usize) / COMPRESSION_BLOCK_SIZE;
+        let num_vint_docs = (doc_freq as usize) - COMPRESSION_BLOCK_SIZE * num_bitpacked_blocks;
         BlockSegmentPostings {
-            num_binpacked_blocks: num_binpacked_blocks,
-            num_vint_docs: num_vint_docs,
-            block_decoder: BlockDecoder::new(),
-            freq_handler: freq_handler,
+            num_bitpacked_blocks,
+            num_vint_docs,
+            doc_decoder: BlockDecoder::new(),
+            freq_decoder: BlockDecoder::with_val(1),
+            freq_reading_option,
             remaining_data: data,
             doc_offset: 0,
-            doc_freq: doc_freq,
+            doc_freq,
         }
     }
 
@@ -226,10 +370,10 @@ impl<'a> BlockSegmentPostings<'a> {
     // # Warning
     //
     // This does not reset the positions list.
-    pub(crate) fn reset(&mut self, doc_freq: usize, postings_data: &'a [u8]) {
-        let num_binpacked_blocks: usize = doc_freq / NUM_DOCS_PER_BLOCK;
-        let num_vint_docs = doc_freq & (NUM_DOCS_PER_BLOCK - 1);
-        self.num_binpacked_blocks = num_binpacked_blocks;
+    pub(crate) fn reset(&mut self, doc_freq: usize, postings_data: SourceRead) {
+        let num_binpacked_blocks: usize = doc_freq / COMPRESSION_BLOCK_SIZE;
+        let num_vint_docs = doc_freq & (COMPRESSION_BLOCK_SIZE - 1);
+        self.num_bitpacked_blocks = num_binpacked_blocks;
         self.num_vint_docs = num_vint_docs;
         self.remaining_data = postings_data;
         self.doc_offset = 0;
@@ -250,7 +394,25 @@ impl<'a> BlockSegmentPostings<'a> {
     /// returned by `.docs()` is empty.
     #[inline]
     pub fn docs(&self) -> &[DocId] {
-        self.block_decoder.output_array()
+        self.doc_decoder.output_array()
+    }
+
+    /// Return the document at index `idx` of the block.
+    #[inline]
+    pub fn doc(&self, idx: usize) -> u32 {
+        self.doc_decoder.output(idx)
+    }
+
+    /// Return the array of `term freq` in the block.
+    #[inline]
+    pub fn freqs(&self) -> &[u32] {
+        self.freq_decoder.output_array()
+    }
+
+    /// Return the frequency at index `idx` of the block.
+    #[inline]
+    pub fn freq(&self, idx: usize) -> u32 {
+        self.freq_decoder.output(idx)
     }
 
     /// Returns the length of the current block.
@@ -260,35 +422,47 @@ impl<'a> BlockSegmentPostings<'a> {
     /// of any number between 1 and `NUM_DOCS_PER_BLOCK - 1`
     #[inline]
     fn block_len(&self) -> usize {
-        self.block_decoder.output_len
-    }
-
-
-    /// Returns a reference to the frequency handler.
-    pub fn freq_handler(&self) -> &FreqHandler {
-        &self.freq_handler
+        self.doc_decoder.output_len
     }
 
     /// Advance to the next block.
     ///
     /// Returns false iff there was no remaining blocks.
     pub fn advance(&mut self) -> bool {
-        if self.num_binpacked_blocks > 0 {
-            self.remaining_data =
-                self.block_decoder
-                    .uncompress_block_sorted(self.remaining_data, self.doc_offset);
-            self.remaining_data = self.freq_handler.read_freq_block(self.remaining_data);
-            self.doc_offset = self.block_decoder.output(NUM_DOCS_PER_BLOCK - 1);
-            self.num_binpacked_blocks -= 1;
+        if self.num_bitpacked_blocks > 0 {
+            let num_consumed_bytes = self.doc_decoder
+                .uncompress_block_sorted(self.remaining_data.as_ref(), self.doc_offset);
+            self.remaining_data.advance(num_consumed_bytes);
+            match self.freq_reading_option {
+                FreqReadingOption::NoFreq => {}
+                FreqReadingOption::SkipFreq => {
+                    let num_bytes_to_skip = compressed_block_size(self.remaining_data.as_ref()[0]);
+                    self.remaining_data.advance(num_bytes_to_skip);
+                }
+                FreqReadingOption::ReadFreq => {
+                    let num_consumed_bytes = self.freq_decoder
+                        .uncompress_block_unsorted(self.remaining_data.as_ref());
+                    self.remaining_data.advance(num_consumed_bytes);
+                }
+            }
+            // it will be used as the next offset.
+            self.doc_offset = self.doc_decoder.output(COMPRESSION_BLOCK_SIZE - 1);
+            self.num_bitpacked_blocks -= 1;
             true
         } else if self.num_vint_docs > 0 {
-            self.remaining_data =
-                self.block_decoder
-                    .uncompress_vint_sorted(self.remaining_data,
-                                            self.doc_offset,
-                                            self.num_vint_docs);
-            self.freq_handler
-                .read_freq_vint(self.remaining_data, self.num_vint_docs);
+            let num_compressed_bytes = self.doc_decoder.uncompress_vint_sorted(
+                self.remaining_data.as_ref(),
+                self.doc_offset,
+                self.num_vint_docs,
+            );
+            self.remaining_data.advance(num_compressed_bytes);
+            match self.freq_reading_option {
+                FreqReadingOption::NoFreq | FreqReadingOption::SkipFreq => {}
+                FreqReadingOption::ReadFreq => {
+                    self.freq_decoder
+                        .uncompress_vint_unsorted(self.remaining_data.as_ref(), self.num_vint_docs);
+                }
+            }
             self.num_vint_docs = 0;
             true
         } else {
@@ -297,20 +471,23 @@ impl<'a> BlockSegmentPostings<'a> {
     }
 
     /// Returns an empty segment postings object
-    pub fn empty() -> BlockSegmentPostings<'static> {
+    pub fn empty() -> BlockSegmentPostings {
         BlockSegmentPostings {
-            num_binpacked_blocks: 0,
+            num_bitpacked_blocks: 0,
             num_vint_docs: 0,
-            block_decoder: BlockDecoder::new(),
-            freq_handler: FreqHandler::new_without_freq(),
-            remaining_data: &EMPTY_DATA,
+
+            doc_decoder: BlockDecoder::new(),
+            freq_decoder: BlockDecoder::with_val(1),
+            freq_reading_option: FreqReadingOption::NoFreq,
+
+            remaining_data: From::from(ReadOnlySource::empty()),
             doc_offset: 0,
             doc_freq: 0,
         }
     }
 }
 
-impl<'a, 'b> Streamer<'b> for BlockSegmentPostings<'a> {
+impl<'b> Streamer<'b> for BlockSegmentPostings {
     type Item = &'b [DocId];
 
     fn next(&'b mut self) -> Option<&'b [DocId]> {
@@ -325,14 +502,14 @@ impl<'a, 'b> Streamer<'b> for BlockSegmentPostings<'a> {
 #[cfg(test)]
 mod tests {
 
-    use DocSet;
+    use docset::DocSet;
     use super::SegmentPostings;
     use schema::SchemaBuilder;
     use core::Index;
     use schema::INT_INDEXED;
     use schema::Term;
     use fst::Streamer;
-    use postings::SegmentPostingsOption;
+    use schema::IndexRecordOption;
     use common::HasLen;
     use super::BlockSegmentPostings;
 
@@ -366,11 +543,11 @@ mod tests {
         index.load_searchers().unwrap();
         let searcher = index.searcher();
         let segment_reader = searcher.segment_reader(0);
+        let inverted_index = segment_reader.inverted_index(int_field);
         let term = Term::from_field_u64(int_field, 0u64);
-        let term_info = segment_reader.get_term_info(&term).unwrap();
+        let term_info = inverted_index.get_term_info(&term).unwrap();
         let mut block_segments =
-            segment_reader
-                .read_block_postings_from_terminfo(&term_info, SegmentPostingsOption::NoFreq);
+            inverted_index.read_block_postings_from_terminfo(&term_info, IndexRecordOption::Basic);
         let mut offset: u32 = 0u32;
         // checking that the block before calling advance is empty
         assert!(block_segments.docs().is_empty());
@@ -384,7 +561,6 @@ mod tests {
         }
     }
 
-
     #[test]
     fn test_reset_block_segment_postings() {
         let mut schema_builder = SchemaBuilder::default();
@@ -406,19 +582,20 @@ mod tests {
         let mut block_segments;
         {
             let term = Term::from_field_u64(int_field, 0u64);
-            let term_info = segment_reader.get_term_info(&term).unwrap();
-            block_segments =
-                segment_reader
-                    .read_block_postings_from_terminfo(&term_info, SegmentPostingsOption::NoFreq);
+            let inverted_index = segment_reader.inverted_index(int_field);
+            let term_info = inverted_index.get_term_info(&term).unwrap();
+            block_segments = inverted_index
+                .read_block_postings_from_terminfo(&term_info, IndexRecordOption::Basic);
         }
         assert!(block_segments.advance());
-        assert!(block_segments.docs() == &[0, 2, 4]);
+        assert_eq!(block_segments.docs(), &[0, 2, 4]);
         {
             let term = Term::from_field_u64(int_field, 1u64);
-            let term_info = segment_reader.get_term_info(&term).unwrap();
-            segment_reader.reset_block_postings_from_terminfo(&term_info, &mut block_segments);
+            let inverted_index = segment_reader.inverted_index(int_field);
+            let term_info = inverted_index.get_term_info(&term).unwrap();
+            inverted_index.reset_block_postings_from_terminfo(&term_info, &mut block_segments);
         }
         assert!(block_segments.advance());
-        assert!(block_segments.docs() == &[1, 3, 5]);
+        assert_eq!(block_segments.docs(), &[1, 3, 5]);
     }
 }

src/postings/segment_postings_option.rs

@@ -1,29 +0,0 @@
-
-
-/// Object describing the amount of information required when reading a postings.
-///
-/// Since decoding information is not free, this makes it possible to
-/// avoid this extra cost when the information is not required.
-/// For instance, positions are useful when running phrase queries
-/// but useless in other queries.
-#[derive(Clone, Copy, Debug, PartialEq, PartialOrd, Ord, Eq)]
-pub enum SegmentPostingsOption {
-    /// Only the doc ids are decoded
-    NoFreq,
-    /// DocIds and term frequencies are decoded
-    Freq,
-    /// DocIds, term frequencies and positions will be decoded.
-    FreqAndPositions,
-}
-
-#[cfg(test)]
-mod tests {
-
-    use super::SegmentPostingsOption;
-
-    #[test]
-    fn test_cmp_segment_postings_option() {
-        assert!(SegmentPostingsOption::FreqAndPositions > SegmentPostingsOption::Freq);
-        assert!(SegmentPostingsOption::Freq > SegmentPostingsOption::NoFreq);
-    }
-}