Added SegmentFruit

.github/FUNDING.yml

@@ -1,12 +0,0 @@
-# These are supported funding model platforms
-
-github: fulmicoton
-patreon: # Replace with a single Patreon username
-open_collective: # Replace with a single Open Collective username
-ko_fi: # Replace with a single Ko-fi username
-tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
-community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
-liberapay: # Replace with a single Liberapay username
-issuehunt: # Replace with a single IssueHunt username
-otechie: # Replace with a single Otechie username
-custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']

.gitignore

@@ -1,5 +1,4 @@
 tantivy.iml
-proptest-regressions
 *.swp
 target
 target/debug
@@ -12,4 +11,3 @@ cpp/simdcomp/bitpackingbenchmark
 *.bk
 .idea
 trace.dat
-cargo-timing*

.travis.yml

@@ -10,7 +10,7 @@ env:
   global:
     - CRATE_NAME=tantivy
     - TRAVIS_CARGO_NIGHTLY_FEATURE=""
-    # - secure: eC8HjTi1wgRVCsMAeXEXt8Ckr0YBSGOEnQkkW4/Nde/OZ9jJjz2nmP1ELQlDE7+czHub2QvYtDMG0parcHZDx/Kus0yvyn08y3g2rhGIiE7y8OCvQm1Mybu2D/p7enm6shXquQ6Z5KRfRq+18mHy80wy9ABMA/ukEZdvnfQ76/Een8/Lb0eHaDoXDXn3PqLVtByvSfQQ7OhS60dEScu8PWZ6/l1057P5NpdWbMExBE7Ro4zYXNhkJeGZx0nP/Bd4Jjdt1XfPzMEybV6NZ5xsTILUBFTmOOt603IsqKGov089NExqxYu5bD3K+S4MzF1Nd6VhomNPJqLDCfhlymJCUj5n5Ku4yidlhQbM4Ej9nGrBalJnhcjBjPua5tmMF2WCxP9muKn/2tIOu1/+wc0vMf9Yd3wKIkf5+FtUxCgs2O+NslWvmOMAMI/yD25m7hb4t1IwE/4Bk+GVcWJRWXbo0/m6ZUHzRzdjUY2a1qvw7C9udzdhg7gcnXwsKrSWi2NjMiIVw86l+Zim0nLpKIN41sxZHLaFRG63Ki8zQ/481LGn32awJ6i3sizKS0WD+N1DfR2qYMrwYHaMN0uR0OFXYTJkFvTFttAeUY3EKmRKAuMhmO2YRdSr4/j/G5E9HMc1gSGJj6PxgpQU7EpvxRsmoVAEJr0mszmOj9icGHep/FM=
+    - secure: eC8HjTi1wgRVCsMAeXEXt8Ckr0YBSGOEnQkkW4/Nde/OZ9jJjz2nmP1ELQlDE7+czHub2QvYtDMG0parcHZDx/Kus0yvyn08y3g2rhGIiE7y8OCvQm1Mybu2D/p7enm6shXquQ6Z5KRfRq+18mHy80wy9ABMA/ukEZdvnfQ76/Een8/Lb0eHaDoXDXn3PqLVtByvSfQQ7OhS60dEScu8PWZ6/l1057P5NpdWbMExBE7Ro4zYXNhkJeGZx0nP/Bd4Jjdt1XfPzMEybV6NZ5xsTILUBFTmOOt603IsqKGov089NExqxYu5bD3K+S4MzF1Nd6VhomNPJqLDCfhlymJCUj5n5Ku4yidlhQbM4Ej9nGrBalJnhcjBjPua5tmMF2WCxP9muKn/2tIOu1/+wc0vMf9Yd3wKIkf5+FtUxCgs2O+NslWvmOMAMI/yD25m7hb4t1IwE/4Bk+GVcWJRWXbo0/m6ZUHzRzdjUY2a1qvw7C9udzdhg7gcnXwsKrSWi2NjMiIVw86l+Zim0nLpKIN41sxZHLaFRG63Ki8zQ/481LGn32awJ6i3sizKS0WD+N1DfR2qYMrwYHaMN0uR0OFXYTJkFvTFttAeUY3EKmRKAuMhmO2YRdSr4/j/G5E9HMc1gSGJj6PxgpQU7EpvxRsmoVAEJr0mszmOj9icGHep/FM=
 
 addons:
   apt:
@@ -38,21 +38,20 @@ matrix:
     # Linux
     #- env: TARGET=aarch64-unknown-linux-gnu
     #- env: TARGET=i686-unknown-linux-gnu
-    - env: TARGET=x86_64-unknown-linux-gnu CODECOV=1 #UPLOAD_DOCS=1
+    - env: TARGET=x86_64-unknown-linux-gnu CODECOV=1
     # - env: TARGET=x86_64-unknown-linux-musl CODECOV=1
+
     # OSX
-    #- env: TARGET=x86_64-apple-darwin
-    #  os: osx
+    - env: TARGET=x86_64-apple-darwin
+      os: osx
 
 before_install:
   - set -e
   - rustup self update
-  - rustup component add rustfmt
 
 install:
   - sh ci/install.sh
   - source ~/.cargo/env || true
-  - env | grep "TRAVIS"
 
 before_script:
   - export PATH=$HOME/.cargo/bin:$PATH
@@ -61,25 +60,14 @@ before_script:
 
 script:
   - bash ci/script.sh
-  - cargo fmt --all -- --check
 
 before_deploy:
   - sh ci/before_deploy.sh
 
-after_success:
-  # Needs GH_TOKEN env var to be set in travis settings
-  - if [[ -v GH_TOKEN ]]; then echo "GH TOKEN IS SET"; else echo "GH TOKEN NOT SET"; fi
-  - if [[ -v UPLOAD_DOCS ]]; then cargo doc; cargo doc-upload; else echo "doc upload disabled."; fi
-
-#cache: cargo
-#before_cache:
-#  # Travis can't cache files that are not readable by "others"
-#  - chmod -R a+r $HOME/.cargo
-#  - find ./target/debug -type f -maxdepth 1 -delete
-#  - rm -f  ./target/.rustc_info.json
-#  - rm -fr ./target/debug/{deps,.fingerprint}/tantivy*
-#  - rm -r target/debug/examples/
-#  - ls -1 examples/ | sed -e 's/\.rs$//' | xargs -I "{}" find target/* -name "*{}*" -type f -delete
+cache: cargo
+before_cache:
+  # Travis can't cache files that are not readable by "others"
+  - chmod -R a+r $HOME/.cargo
 
 #branches:
 #  only:
@@ -89,4 +77,4 @@ after_success:
 
 notifications:
   email:
-    on_success: never
+    on_success: never

CHANGELOG.md

@@ -1,220 +1,5 @@
-Tantivy 0.14.0
-=========================
-- Remove dependency to atomicwrites #833. Implemented by @pmasurel upon suggestion and research from @asafigan). 
-- Migrated tantivy error from the now deprecated `failure` crate to `thiserror` #760. (@hirevo)
-- Switched to structure logging (via the `slog` crate). (@pmasurel)
-
-Tantivy 0.13.1
-===================
-Made `Query` and `Collector` `Send + Sync`.
-Updated misc dependency versions.
-
-Tantivy 0.13.0
-======================
-Tantivy 0.13 introduce a change in the index format that will require
-you to reindex your index (BlockWAND information are added in the skiplist). 
-The index size increase is minor as this information is only added for
-full blocks.
-If you have a massive index for which reindexing is not an option, please contact me
-so that we can discuss possible solutions.
-
-- Bugfix in `FuzzyTermQuery` not matching terms by prefix when it should (@Peachball)
-- Relaxed constraints on the custom/tweak score functions. At the segment level, they can be mut, and they are not required to be Sync + Send.
-- `MMapDirectory::open` does not return a `Result` anymore.
-- Change in the DocSet and Scorer API. (@fulmicoton). 
-A freshly created DocSet point directly to their first doc. A sentinel value called TERMINATED marks the end of a DocSet.
-`.advance()` returns the new DocId. `Scorer::skip(target)` has been replaced by `Scorer::seek(target)` and returns the resulting DocId.
-As a result, iterating through DocSet now looks as follows
-```rust
-let mut doc = docset.doc();
-while doc != TERMINATED {
-   // ...
-   doc = docset.advance();
-}
-```
-The change made it possible to greatly simplify a lot of the docset's code.
-- Misc internal optimization and introduction of the `Scorer::for_each_pruning` function. (@fulmicoton)
-- Added an offset option to the Top(.*)Collectors. (@robyoung)
-- Added Block WAND. Performance on TOP-K on term-unions should be greatly increased. (@fulmicoton, and special thanks 
-to the PISA team for answering all my questions!)
-
-Tantivy 0.12.0
-======================
-- Removing static dispatch in tokenizers for simplicity. (#762)
-- Added backward iteration for `TermDictionary` stream. (@halvorboe)
-- Fixed a performance issue when searching for the posting lists of a missing term (@audunhalland)
-- Added a configurable maximum number of docs (10M by default) for a segment to be considered for merge (@hntd187, landed by @halvorboe #713) 
-- Important Bugfix #777, causing tantivy to retain memory mapping. (diagnosed by @poljar)
-- Added support for field boosting. (#547, @fulmicoton)
-
-## How to update?
-
-Crates relying on custom tokenizer, or registering tokenizer in the manager will require some 
-minor changes. Check https://github.com/tantivy-search/tantivy/blob/master/examples/custom_tokenizer.rs
-to check for some code sample.
-
-Tantivy 0.11.3
-=======================
-- Fixed DateTime as a fast field (#735)
-
-Tantivy 0.11.2
-=======================
-- The future returned by `IndexWriter::merge` does not borrow `self` mutably anymore (#732)
-- Exposing a constructor for `WatchHandle` (#731)
-
-Tantivy 0.11.1
-=====================
-- Bug fix #729
-
-
-Tantivy 0.11.0
-=====================
-
-- Added f64 field. Internally reuse u64 code the same way i64 does (@fdb-hiroshima)
-- Various bugfixes in the query parser.
-    - Better handling of hyphens in query parser. (#609)
-    - Better handling of whitespaces.
-- Closes #498 - add support for Elastic-style unbounded range queries for alphanumeric types eg. "title:>hello", "weight:>=70.5", "height:<200" (@petr-tik)
-- API change around `Box<BoxableTokenizer>`. See detail in #629
-- Avoid rebuilding Regex automaton whenever a regex query is reused. #639 (@brainlock)
-- Add footer with some metadata to index files. #605 (@fdb-hiroshima)
-- Add a method to check the compatibility of the footer in the index with the running version of tantivy (@petr-tik)
-- TopDocs collector: ensure stable sorting on equal score. #671 (@brainlock)
-- Added handling of pre-tokenized text fields (#642), which will enable users to
-  load tokens created outside tantivy. See usage in examples/pre_tokenized_text. (@kkoziara)
-- Fix crash when committing multiple times with deleted documents. #681 (@brainlock)
-
-## How to update?
-
-- The index format is changed. You are required to reindex your data to use tantivy 0.11. 
-- `Box<dyn BoxableTokenizer>` has been replaced by a `BoxedTokenizer` struct.
-- Regex are now compiled when the `RegexQuery` instance is built. As a result, it can now return
-an error and handling the `Result` is required.
-- `tantivy::version()` now returns a `Version` object. This object implements `ToString()`
-
-Tantivy 0.10.2
-=====================
-
-- Closes #656. Solving memory leak.
-
-Tantivy 0.10.1
-=====================
-
-- Closes #544.  A few users experienced problems with the directory watching system.
-Avoid watching the mmap directory until someone effectively creates a reader that uses
-this functionality.
-
-
-Tantivy 0.10.0
-=====================
-
-*Tantivy 0.10.0 index format is compatible with the index format in 0.9.0.*
-
-- Added an API to easily tweak or entirely replace the 
- default score. See `TopDocs::tweak_score`and `TopScore::custom_score` (@pmasurel)
-- Added an ASCII folding filter (@drusellers)
-- Bugfix in `query.count` in presence of deletes (@pmasurel)
-- Added `.explain(...)` in `Query` and `Weight` to (@pmasurel)
-- Added an efficient way to `delete_all_documents` in `IndexWriter` (@petr-tik). 
-  All segments are simply removed.
-
-Minor
----------
-- Switched to Rust 2018 (@uvd)
-- Small simplification of the code. 
-Calling .freq() or .doc() when .advance() has never been called
-on segment postings should panic from now on.
-- Tokens exceeding `u16::max_value() - 4` chars are discarded silently instead of panicking.
-- Fast fields are now preloaded when the `SegmentReader` is created.
-- `IndexMeta` is now public.  (@hntd187)
-- `IndexWriter` `add_document`, `delete_term`. `IndexWriter` is `Sync`, making it possible to use it with a `
-Arc<RwLock<IndexWriter>>`. `add_document` and `delete_term` can 
-only require a read lock. (@pmasurel)
-- Introducing `Opstamp` as an expressive type alias for `u64`. (@petr-tik)
-- Stamper now relies on `AtomicU64` on all platforms (@petr-tik)
-- Bugfix - Files get deleted slightly earlier
-- Compilation resources improved (@fdb-hiroshima)
-
-## How to update?
-
-Your program should be usable as is.
-
-### Fast fields
-
-Fast fields used to be accessed directly from the `SegmentReader`.
-The API changed, you are now required to acquire your fast field reader via the
-`segment_reader.fast_fields()`, and use one of the typed method: 
-- `.u64()`, `.i64()` if your field is single-valued ;
-- `.u64s()`, `.i64s()` if your field is multi-valued ;
-- `.bytes()` if your field is bytes fast field.
-
-
-
-Tantivy 0.9.0
-=====================
-*0.9.0 index format is not compatible with the 
-previous index format.*
-- MAJOR BUGFIX : 
-  Some `Mmap` objects were being leaked, and would never get released. (@fulmicoton)
-- Removed most unsafe (@fulmicoton)
-- Indexer memory footprint improved. (VInt comp, inlining the first block. (@fulmicoton)
-- Stemming in other language possible (@pentlander)
-- Segments with no docs are deleted earlier (@barrotsteindev)
-- Added grouped add and delete operations. 
-  They are guaranteed to happen together (i.e. they cannot be split by a commit). 
-  In addition, adds are guaranteed to happen on the same segment. (@elbow-jason)
-- Removed `INT_STORED` and `INT_INDEXED`. It is now possible to use `STORED` and `INDEXED`
-  for int fields. (@fulmicoton)
-- Added DateTime field (@barrotsteindev)
-- Added IndexReader. By default, index is reloaded automatically upon new commits (@fulmicoton)
-- SIMD linear search within blocks (@fulmicoton)
-
-## How to update ?
-
-tantivy 0.9 brought some API breaking change.
-To update from tantivy 0.8, you will need to go through the following steps.
-
-- `schema::INT_INDEXED` and `schema::INT_STORED`  should be replaced by `schema::INDEXED` and `schema::INT_STORED`.
-- The index now does not hold the pool of searcher anymore. You are required to create an intermediary object called 
-`IndexReader` for this. 
-    
-    ```rust
-    // create the reader. You typically need to create 1 reader for the entire
-    // lifetime of you program.
-    let reader = index.reader()?;
-    
-    // Acquire a searcher (previously `index.searcher()`) is now written:
-    let searcher = reader.searcher();
-    
-    // With the default setting of the reader, you are not required to 
-    // call `index.load_searchers()` anymore.
-    //
-    // The IndexReader will pick up that change automatically, regardless
-    // of whether the update was done in a different process or not.
-    // If this behavior is not wanted, you can create your reader with 
-    // the `ReloadPolicy::Manual`, and manually decide when to reload the index
-    // by calling `reader.reload()?`.
-  
-    ```
-
-
-Tantivy 0.8.2
-=====================
-Fixing build for x86_64 platforms. (#496)
-No need to update from 0.8.1 if tantivy
-is building on your platform.
-
-
 Tantivy 0.8.1
 =====================
-Hotfix of #476.
-
-Merge was reflecting deletes before commit was passed. 
-Thanks @barrotsteindev  for reporting the bug.
-
-
-Tantivy 0.8.0
-=====================
 *No change in the index format*
 - API Breaking change in the collector API. (@jwolfe, @fulmicoton)
 - Multithreaded search (@jwolfe, @fulmicoton) 

Cargo.toml

@@ -1,68 +1,61 @@
 [package]
 name = "tantivy"
-version = "0.14.0-dev"
+version = "0.8.0-dev"
 authors = ["Paul Masurel <paul.masurel@gmail.com>"]
 license = "MIT"
 categories = ["database-implementations", "data-structures"]
 description = """Search engine library"""
-documentation = "https://docs.rs/tantivy/"
+documentation = "https://tantivy-search.github.io/tantivy/tantivy/index.html"
 homepage = "https://github.com/tantivy-search/tantivy"
 repository = "https://github.com/tantivy-search/tantivy"
 readme = "README.md"
 keywords = ["search", "information", "retrieval"]
-edition = "2018"
 
 [dependencies]
-base64 = "0.12"
-byteorder = "1"
-crc32fast = "1"
-once_cell = "1"
-regex ={version = "1", default-features = false, features = ["std"]}
-tantivy-fst = "0.3"
-memmap = {version = "0.7", optional=true}
-lz4 = {version="1", optional=true}
-snap = "1"
-tempfile = {version="3", optional=true}
-slog = "2.5"
-slog-stdlog = "4"
-serde = {version="1", features=["derive"]}
-serde_json = "1"
-num_cpus = "1"
-fs2={version="0.4", optional=true}
-levenshtein_automata = "0.2"
-notify = {version="4", optional=true}
-uuid = { version = "0.8", features = ["v4", "serde"] }
-crossbeam = "0.7"
-futures = {version = "0.3",  features=["thread-pool"] }
+base64 = "0.10.0"
+byteorder = "1.0"
+lazy_static = "1"
+regex = "1.0"
+fst = {version="0.3", default-features=false}
+fst-regex = { version="0.2" }
+lz4 = {version="1.20", optional=true}
+snap = {version="0.2"}
+atomicwrites = {version="0.2.2", optional=true}
+tempfile = "3.0"
+log = "0.4"
+combine = "3"
+tempdir = "0.3"
+serde = "1.0"
+serde_derive = "1.0"
+serde_json = "1.0"
+num_cpus = "1.2"
+itertools = "0.7"
+levenshtein_automata = {version="0.1", features=["fst_automaton"]}
+bit-set = "0.5"
+uuid = { version = "0.7", features = ["v4", "serde"] }
+crossbeam = "0.5"
+futures = "0.1"
+futures-cpupool = "0.1"
 owning_ref = "0.4"
-tantivy-query-grammar = { version="0.14.0-dev", path="./query-grammar" }
-stable_deref_trait = "1"
+stable_deref_trait = "1.0.0"
 rust-stemmers = "1"
-downcast-rs = "1"
-bitpacking = {version="0.8", default-features = false, features=["bitpacker4x"]}
-census = "0.4"
-fnv = "1"
+downcast = { version="0.9" }
+matches = "0.1"
+bitpacking = "0.5"
+census = "0.1"
+fnv = "1.0.6"
 owned-read = "0.4"
-thiserror = "1.0"
-htmlescape = "0.3"
-fail = "0.4"
-murmurhash32 = "0.2"
-chrono = "0.4"
-smallvec = "1"
-rayon = "1"
+failure = "0.1"
+htmlescape = "0.3.1"
+fail = "0.2"
+scoped-pool = "1.0"
 
 [target.'cfg(windows)'.dependencies]
-winapi = "0.3"
+winapi = "0.2"
 
 [dev-dependencies]
-rand = "0.7"
+rand = "0.6"
 maplit = "1"
-matches = "0.1.8"
-proptest = "0.10"
-
-[dev-dependencies.fail]
-version = "0.4"
-features = ["failpoints"]
 
 [profile.release]
 opt-level = 3
@@ -74,27 +67,14 @@ debug-assertions = true
 overflow-checks = true
 
 [features]
-default = ["mmap"]
-mmap = ["fs2", "tempfile", "memmap", "notify"]
+# by default no-fail is disabled. We manually enable it when running test.
+default = ["mmap", "no_fail"]
+mmap = ["fst/mmap", "atomicwrites"]
 lz4-compression = ["lz4"]
-failpoints = ["fail/failpoints"]
+no_fail = ["fail/no_fail"]
 unstable = [] # useful for benches.
-wasm-bindgen = ["uuid/wasm-bindgen"]
-
-[workspace]
-members = ["query-grammar"]
 
 [badges]
 travis-ci = { repository = "tantivy-search/tantivy" }
 
-# Following the "fail" crate best practises, we isolate
-# tests that define specific behavior in fail check points
-# in a different binary.
-#
-# We do that because, fail rely on a global definition of
-# failpoints behavior and hence, it is incompatible with
-# multithreading.
-[[test]]
-name = "failpoints"
-path = "tests/failpoints/mod.rs"
-required-features = ["fail/failpoints"]
+

Makefile

@@ -1,3 +0,0 @@
-test:
-	echo "Run test only... No examples."
-	cargo test --tests --lib

README.md

@@ -4,7 +4,6 @@
 [![Join the chat at https://gitter.im/tantivy-search/tantivy](https://badges.gitter.im/tantivy-search/tantivy.svg)](https://gitter.im/tantivy-search/tantivy?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
 [![Build status](https://ci.appveyor.com/api/projects/status/r7nb13kj23u8m9pj/branch/master?svg=true)](https://ci.appveyor.com/project/fulmicoton/tantivy/branch/master)
-[![Crates.io](https://img.shields.io/crates/v/tantivy.svg)](https://crates.io/crates/tantivy)
 [![Say Thanks!](https://img.shields.io/badge/Say%20Thanks-!-1EAEDB.svg)](https://saythanks.io/to/fulmicoton)
 
 ![Tantivy](https://tantivy-search.github.io/logo/tantivy-logo.png)
@@ -18,118 +17,74 @@
 [![](https://sourcerer.io/fame/fulmicoton/tantivy-search/tantivy/images/6)](https://sourcerer.io/fame/fulmicoton/tantivy-search/tantivy/links/6)
 [![](https://sourcerer.io/fame/fulmicoton/tantivy-search/tantivy/images/7)](https://sourcerer.io/fame/fulmicoton/tantivy-search/tantivy/links/7)
 
-[![Become a patron](https://c5.patreon.com/external/logo/become_a_patron_button.png)](https://www.patreon.com/fulmicoton)
 
 
-**Tantivy** is a **full text search engine library** written in Rust.
+**Tantivy** is a **full text search engine library** written in rust.
 
-It is closer to [Apache Lucene](https://lucene.apache.org/) than to [Elasticsearch](https://www.elastic.co/products/elasticsearch) or [Apache Solr](https://lucene.apache.org/solr/) in the sense it is not
+It is closer to [Apache Lucene](https://lucene.apache.org/) than to [Elastic Search](https://www.elastic.co/products/elasticsearch) and [Apache Solr](https://lucene.apache.org/solr/) in the sense it is not
 an off-the-shelf search engine server, but rather a crate that can be used
 to build such a search engine.
 
 Tantivy is, in fact, strongly inspired by Lucene's design.
 
-# Benchmark
-
-The following [benchmark](https://tantivy-search.github.io/bench/) break downs 
-performance for different type of queries / collection.
-
-Your mileage WILL vary depending on the nature of queries and their load.
-
 # Features
 
 - Full-text search
-- Configurable tokenizer (stemming available for 17 Latin languages with third party support for Chinese ([tantivy-jieba](https://crates.io/crates/tantivy-jieba) and [cang-jie](https://crates.io/crates/cang-jie)), Japanese ([lindera](https://github.com/lindera-morphology/lindera-tantivy) and [tantivy-tokenizer-tiny-segmente](https://crates.io/crates/tantivy-tokenizer-tiny-segmenter)) and Korean ([lindera](https://github.com/lindera-morphology/lindera-tantivy) + [lindera-ko-dic-builder](https://github.com/lindera-morphology/lindera-ko-dic-builder))
 - Fast (check out the :racehorse: :sparkles: [benchmark](https://tantivy-search.github.io/bench/) :sparkles: :racehorse:)
 - Tiny startup time (<10ms), perfect for command line tools
-- BM25 scoring (the same as Lucene)
-- Natural query language (e.g. `(michael AND jackson) OR "king of pop"`)
-- Phrase queries search (e.g. `"michael jackson"`)
+- BM25 scoring (the same as lucene)
+- Natural query language `(michael AND jackson) OR "king of pop"`
+- Phrase queries search (`"michael jackson"`)
 - Incremental indexing
 - Multithreaded indexing (indexing English Wikipedia takes < 3 minutes on my desktop)
 - Mmap directory
-- SIMD integer compression when the platform/CPU includes the SSE2 instruction set
-- Single valued and multivalued u64, i64, and f64 fast fields (equivalent of doc values in Lucene)
+- SIMD integer compression when the platform/CPU includes the SSE2 instruction set.
+- Single valued and multivalued u64 and i64 fast fields (equivalent of doc values in Lucene)
 - `&[u8]` fast fields
-- Text, i64, u64, f64, dates, and hierarchical facet fields
 - LZ4 compressed document store
 - Range queries
 - Faceted search
 - Configurable indexing (optional term frequency and position indexing)
 - Cheesy logo with a horse
 
-## Non-features
+# Non-features
 
-- Distributed search is out of the scope of Tantivy. That being said, Tantivy is a
+- Distributed search is out of the scope of tantivy. That being said, tantivy is meant as a
 library upon which one could build a distributed search. Serializable/mergeable collector state for instance, 
-are within the scope of Tantivy.
+are within the scope of tantivy.
 
 
+# Supported OS and compiler
+
+Tantivy works on stable rust (>= 1.27) and supports Linux, MacOS and Windows.
+
 # Getting started
 
-Tantivy works on stable Rust (>= 1.27) and supports Linux, MacOS, and Windows.
+- [tantivy's simple search example](http://fulmicoton.com/tantivy-examples/simple_search.html)
+- [tantivy-cli and its tutorial](https://github.com/tantivy-search/tantivy-cli).
+`tantivy-cli` is an actual command line interface that makes it easy for you to create a search engine,
+index documents and search via the CLI or a small server with a REST API.
+It will walk you through getting a wikipedia search engine up and running in a few minutes.
+- [reference doc]
+    - [For the last released version](https://docs.rs/tantivy/)
+    - [For the last master branch](https://tantivy-search.github.io/tantivy/tantivy/index.html)
 
-- [Tantivy's simple search example](https://tantivy-search.github.io/examples/basic_search.html)
-- [tantivy-cli and its tutorial](https://github.com/tantivy-search/tantivy-cli) - `tantivy-cli` is an actual command line interface that makes it easy for you to create a search engine,
-index documents, and search via the CLI or a small server with a REST API.
-It walks you through getting a wikipedia search engine up and running in a few minutes.
-- [Reference doc for the last released version](https://docs.rs/tantivy/)
+# Compiling
 
-# How can I support this project?
+## Development
 
-There are many ways to support this project. 
+Tantivy compiles on stable rust but requires `Rust >= 1.27`.
+To check out and run tests, you can simply run :
 
-- Use Tantivy and tell us about your experience on [Gitter](https://gitter.im/tantivy-search/tantivy) or by email (paul.masurel@gmail.com)
-- Report bugs
-- Write a blog post
-- Help with documentation by asking questions or submitting PRs
-- Contribute code (you can join [our Gitter](https://gitter.im/tantivy-search/tantivy))
-- Talk about Tantivy around you
-- Drop a word on on [![Say Thanks!](https://img.shields.io/badge/Say%20Thanks-!-1EAEDB.svg)](https://saythanks.io/to/fulmicoton) or even [![Become a patron](https://c5.patreon.com/external/logo/become_a_patron_button.png)](https://www.patreon.com/fulmicoton)
-
-# Contributing code
-
-We use the GitHub Pull Request workflow: reference a GitHub ticket and/or include a comprehensive commit message when opening a PR.
-
-## Clone and build locally
-
-Tantivy compiles on stable Rust but requires `Rust >= 1.27`.
-To check out and run tests, you can simply run:
-
-```bash
-    git clone https://github.com/tantivy-search/tantivy.git
+    git clone git@github.com:tantivy-search/tantivy.git
     cd tantivy
     cargo build
-```
 
-## Run tests
+## Running tests
 
 Some tests will not run with just `cargo test` because of `fail-rs`.
-To run the tests exhaustively, run `./run-tests.sh`.
+To run the tests exhaustively, run `./run-tests.sh`. 
 
-## Debug
+# Contribute
 
-You might find it useful to step through the programme with a debugger.
-
-### A failing test
-
-Make sure you haven't run `cargo clean` after the most recent `cargo test` or `cargo build` to guarantee that the `target/` directory exists. Use this bash script to find the name of the most recent debug build of Tantivy and run it under `rust-gdb`:
-
-```bash
-find target/debug/ -maxdepth 1 -executable -type f -name "tantivy*" -printf '%TY-%Tm-%Td %TT %p\n' | sort -r | cut -d " " -f 3 | xargs -I RECENT_DBG_TANTIVY rust-gdb RECENT_DBG_TANTIVY
-```
-
-Now that you are in `rust-gdb`, you can set breakpoints on lines and methods that match your source code and run the debug executable with flags that you normally pass to `cargo test` like this:
-
-```bash
-$gdb run --test-threads 1 --test $NAME_OF_TEST
-```
-
-### An example
-
-By default, `rustc` compiles everything in the `examples/` directory in debug mode. This makes it easy for you to make examples to reproduce bugs:
-
-```bash
-rust-gdb target/debug/examples/$EXAMPLE_NAME
-$ gdb run
-```
+Send me an email (paul.masurel at gmail.com) if you want to contribute to tantivy.

appveyor.yml

@@ -18,5 +18,5 @@ install:
 build: false
 
 test_script:
-  - REM SET RUST_LOG=tantivy,test & cargo test --all --verbose --no-default-features --features mmap
+  - REM SET RUST_LOG=tantivy,test & cargo test --verbose --no-default-features --features mmap -- --test-threads 1
   - REM SET RUST_BACKTRACE=1 & cargo build --examples

ci/script.sh

@@ -7,7 +7,7 @@ set -ex
 main() {
     if [ ! -z $CODECOV ]; then
         echo "Codecov"
-        cargo build --verbose && cargo coverage --verbose --all && bash <(curl -s https://codecov.io/bash) -s target/kcov
+        cargo build --verbose && cargo coverage --verbose && bash <(curl -s https://codecov.io/bash) -s target/kcov
     else
         echo "Build"
         cross build --target $TARGET
@@ -15,8 +15,7 @@ main() {
             return
         fi
         echo "Test"
-        cross test --target $TARGET --no-default-features --features mmap
-        cross test --target $TARGET --no-default-features --features mmap query-grammar
+        cross test --target $TARGET --no-default-features --features mmap -- --test-threads 1
     fi
     for example in $(ls examples/*.rs)
     do

examples/basic_search.rs

@@ -5,23 +5,27 @@
 //
 // We will :
 // - define our schema
-// - create an index in a directory
-// - index a few documents into our index
-// - search for the best document matching a basic query
-// - retrieve the best document's original content.
+// = create an index in a directory
+// - index few documents in our index
+// - search for the best document matchings "sea whale"
+// - retrieve the best document original content.
+
+extern crate tempdir;
 
 // ---
 // Importing tantivy...
+#[macro_use]
+extern crate tantivy;
 use tantivy::collector::TopDocs;
 use tantivy::query::QueryParser;
 use tantivy::schema::*;
-use tantivy::{doc, Index, ReloadPolicy};
-use tempfile::TempDir;
+use tantivy::Index;
+use tempdir::TempDir;
 
 fn main() -> tantivy::Result<()> {
     // Let's create a temporary directory for the
     // sake of this example
-    let index_path = TempDir::new()?;
+    let index_path = TempDir::new("tantivy_example_dir")?;
 
     // # Defining the schema
     //
@@ -30,7 +34,7 @@ fn main() -> tantivy::Result<()> {
     // and for each field, its type and "the way it should
     // be indexed".
 
-    // First we need to define a schema ...
+    // first we need to define a schema ...
     let mut schema_builder = Schema::builder();
 
     // Our first field is title.
@@ -45,7 +49,7 @@ fn main() -> tantivy::Result<()> {
     //
     // `STORED` means that the field will also be saved
     // in a compressed, row-oriented key-value store.
-    // This store is useful for reconstructing the
+    // This store is useful to reconstruct the
     // documents that were selected during the search phase.
     schema_builder.add_text_field("title", TEXT | STORED);
 
@@ -54,7 +58,8 @@ fn main() -> tantivy::Result<()> {
     // need to be able to be able to retrieve it
     // for our application.
     //
-    // We can make our index lighter by omitting the `STORED` flag.
+    // We can make our index lighter and
+    // by omitting `STORED` flag.
     schema_builder.add_text_field("body", TEXT);
 
     let schema = schema_builder.build();
@@ -67,7 +72,7 @@ fn main() -> tantivy::Result<()> {
     // with our schema in the directory.
     let index = Index::create_in_dir(&index_path, schema.clone())?;
 
-    // To insert a document we will need an index writer.
+    // To insert document we need an index writer.
     // There must be only one writer at a time.
     // This single `IndexWriter` is already
     // multithreaded.
@@ -112,6 +117,18 @@ fn main() -> tantivy::Result<()> {
             limbs and branches that arch over the pool"
     ));
 
+    index_writer.add_document(doc!(
+    title => "Of Mice and Men",
+    body => "A few miles south of Soledad, the Salinas River drops in close to the hillside \
+            bank and runs deep and green. The water is warm too, for it has slipped twinkling \
+            over the yellow sands in the sunlight before reaching the narrow pool. On one \
+            side of the river the golden foothill slopes curve up to the strong and rocky \
+            Gabilan Mountains, but on the valley side the water is lined with trees—willows \
+            fresh and green with every spring, carrying in their lower leaf junctures the \
+            debris of the winter’s flooding; and sycamores with mottled, white, recumbent \
+            limbs and branches that arch over the pool"
+    ));
+
     // Multivalued field just need to be repeated.
     index_writer.add_document(doc!(
     title => "Frankenstein",
@@ -133,8 +150,8 @@ fn main() -> tantivy::Result<()> {
     // At this point our documents are not searchable.
     //
     //
-    // We need to call `.commit()` explicitly to force the
-    // `index_writer` to finish processing the documents in the queue,
+    // We need to call .commit() explicitly to force the
+    // index_writer to finish processing the documents in the queue,
     // flush the current index to the disk, and advertise
     // the existence of new documents.
     //
@@ -146,40 +163,31 @@ fn main() -> tantivy::Result<()> {
     // persistently indexed.
     //
     // In the scenario of a crash or a power failure,
-    // tantivy behaves as if it has rolled back to its last
+    // tantivy behaves as if has rolled back to its last
     // commit.
 
     // # Searching
     //
     // ### Searcher
     //
-    // A reader is required first in order to search an index.
-    // It acts as a `Searcher` pool that reloads itself,
-    // depending on a `ReloadPolicy`.
-    //
-    // For a search server you will typically create one reader for the entire lifetime of your
-    // program, and acquire a new searcher for every single request.
-    //
-    // In the code below, we rely on the 'ON_COMMIT' policy: the reader
-    // will reload the index automatically after each commit.
-    let reader = index
-        .reader_builder()
-        .reload_policy(ReloadPolicy::OnCommit)
-        .try_into()?;
+    // Let's search our index. Start by reloading
+    // searchers in the index. This should be done
+    // after every `commit()`.
+    index.load_searchers()?;
 
     // We now need to acquire a searcher.
-    //
-    // A searcher points to a snapshotted, immutable version of the index.
-    //
     // Some search experience might require more than
-    // one query. Using the same searcher ensures that all of these queries will run on the
-    // same version of the index.
+    // one query.
+    //
+    // The searcher ensure that we get to work
+    // with a consistent version of the index.
     //
     // Acquiring a `searcher` is very cheap.
     //
-    // You should acquire a searcher every time you start processing a request and
+    // You should acquire a searcher every time you
+    // start processing a request and
     // and release it right after your query is finished.
-    let searcher = reader.searcher();
+    let searcher = index.searcher();
 
     // ### Query
 
@@ -189,7 +197,7 @@ fn main() -> tantivy::Result<()> {
     // in both title and body.
     let query_parser = QueryParser::for_index(&index, vec![title, body]);
 
-    // `QueryParser` may fail if the query is not in the right
+    // QueryParser may fail if the query is not in the right
     // format. For user facing applications, this can be a problem.
     // A ticket has been opened regarding this problem.
     let query = query_parser.parse_query("sea whale")?;
@@ -205,7 +213,7 @@ fn main() -> tantivy::Result<()> {
     //
     // We are not interested in all of the documents but
     // only in the top 10. Keeping track of our top 10 best documents
-    // is the role of the `TopDocs` collector.
+    // is the role of the TopDocs.
 
     // We can now perform our query.
     let top_docs = searcher.search(&query, &TopDocs::with_limit(10))?;
@@ -216,6 +224,7 @@ fn main() -> tantivy::Result<()> {
     // Since the body field was not configured as stored,
     // the document returned will only contain
     // a title.
+
     for (_score, doc_address) in top_docs {
         let retrieved_doc = searcher.doc(doc_address)?;
         println!("{}", schema.to_json(&retrieved_doc));

examples/custom_collector.rs

@@ -7,14 +7,19 @@
 // Of course, you can have a look at the tantivy's built-in collectors
 // such as the `CountCollector` for more examples.
 
+extern crate tempdir;
+
 // ---
 // Importing tantivy...
+#[macro_use]
+extern crate tantivy;
 use tantivy::collector::{Collector, SegmentCollector};
 use tantivy::fastfield::FastFieldReader;
 use tantivy::query::QueryParser;
 use tantivy::schema::Field;
-use tantivy::schema::{Schema, FAST, INDEXED, TEXT};
-use tantivy::{doc, Index, Score, SegmentReader, TantivyError};
+use tantivy::schema::{Schema, FAST, INT_INDEXED, TEXT};
+use tantivy::Index;
+use tantivy::SegmentReader;
 
 #[derive(Default)]
 struct Stats {
@@ -65,23 +70,16 @@ impl Collector for StatsCollector {
     // Our standard deviation will be a float.
     type Fruit = Option<Stats>;
 
+    type SegmentFruit = Self::Fruit;
+
     type Child = StatsSegmentCollector;
 
     fn for_segment(
         &self,
         _segment_local_id: u32,
-        segment_reader: &SegmentReader,
+        segment: &SegmentReader,
     ) -> tantivy::Result<StatsSegmentCollector> {
-        let fast_field_reader = segment_reader
-            .fast_fields()
-            .u64(self.field)
-            .ok_or_else(|| {
-                let field_name = segment_reader.schema().get_field_name(self.field);
-                TantivyError::SchemaError(format!(
-                    "Field {:?} is not a u64 fast field.",
-                    field_name
-                ))
-            })?;
+        let fast_field_reader = segment.fast_field_reader(self.field)?;
         Ok(StatsSegmentCollector {
             fast_field_reader,
             stats: Stats::default(),
@@ -114,7 +112,7 @@ struct StatsSegmentCollector {
 impl SegmentCollector for StatsSegmentCollector {
     type Fruit = Option<Stats>;
 
-    fn collect(&mut self, doc: u32, _score: Score) {
+    fn collect(&mut self, doc: u32, _score: f32) {
         let value = self.fast_field_reader.get(doc) as f64;
         self.stats.count += 1;
         self.stats.sum += value;
@@ -141,7 +139,7 @@ fn main() -> tantivy::Result<()> {
     // products, and with a name, a description, and a price.
     let product_name = schema_builder.add_text_field("name", TEXT);
     let product_description = schema_builder.add_text_field("description", TEXT);
-    let price = schema_builder.add_u64_field("price", INDEXED | FAST);
+    let price = schema_builder.add_u64_field("price", INT_INDEXED | FAST);
     let schema = schema_builder.build();
 
     // # Indexing documents
@@ -174,9 +172,9 @@ fn main() -> tantivy::Result<()> {
         price => 5_200u64
     ));
     index_writer.commit()?;
+    index.load_searchers()?;
 
-    let reader = index.reader()?;
-    let searcher = reader.searcher();
+    let searcher = index.searcher();
     let query_parser = QueryParser::for_index(&index, vec![product_name, product_description]);
 
     // here we want to get a hit on the 'ken' in Frankenstein

examples/custom_tokenizer.rs

@@ -2,11 +2,14 @@
 //
 // In this example, we'll see how to define a tokenizer pipeline
 // by aligning a bunch of `TokenFilter`.
+
+#[macro_use]
+extern crate tantivy;
 use tantivy::collector::TopDocs;
 use tantivy::query::QueryParser;
 use tantivy::schema::*;
 use tantivy::tokenizer::NgramTokenizer;
-use tantivy::{doc, Index};
+use tantivy::Index;
 
 fn main() -> tantivy::Result<()> {
     // # Defining the schema
@@ -88,9 +91,9 @@ fn main() -> tantivy::Result<()> {
                 increasing confidence in the success of my undertaking."#
     ));
     index_writer.commit()?;
+    index.load_searchers()?;
 
-    let reader = index.reader()?;
-    let searcher = reader.searcher();
+    let searcher = index.searcher();
 
     // The query parser can interpret human queries.
     // Here, if the user does not specify which

examples/deleting_updating_documents.rs

@@ -8,19 +8,18 @@
 //
 // ---
 // Importing tantivy...
+#[macro_use]
+extern crate tantivy;
 use tantivy::collector::TopDocs;
 use tantivy::query::TermQuery;
 use tantivy::schema::*;
-use tantivy::{doc, Index, IndexReader};
+use tantivy::Index;
 
 // A simple helper function to fetch a single document
 // given its id from our index.
 // It will be helpful to check our work.
-fn extract_doc_given_isbn(
-    reader: &IndexReader,
-    isbn_term: &Term,
-) -> tantivy::Result<Option<Document>> {
-    let searcher = reader.searcher();
+fn extract_doc_given_isbn(index: &Index, isbn_term: &Term) -> tantivy::Result<Option<Document>> {
+    let searcher = index.searcher();
 
     // This is the simplest query you can think of.
     // It matches all of the documents containing a specific term.
@@ -86,12 +85,12 @@ fn main() -> tantivy::Result<()> {
        isbn => "978-9176370711",
     ));
     index_writer.commit()?;
-    let reader = index.reader()?;
+    index.load_searchers()?;
 
     let frankenstein_isbn = Term::from_field_text(isbn, "978-9176370711");
 
     // Oops our frankenstein doc seems mispelled
-    let frankenstein_doc_misspelled = extract_doc_given_isbn(&reader, &frankenstein_isbn)?.unwrap();
+    let frankenstein_doc_misspelled = extract_doc_given_isbn(&index, &frankenstein_isbn)?.unwrap();
     assert_eq!(
         schema.to_json(&frankenstein_doc_misspelled),
         r#"{"isbn":["978-9176370711"],"title":["Frankentein"]}"#,
@@ -130,10 +129,10 @@ fn main() -> tantivy::Result<()> {
     // Everything happened as if the document was updated.
     index_writer.commit()?;
     // We reload our searcher to make our change available to clients.
-    reader.reload()?;
+    index.load_searchers()?;
 
     // No more typo!
-    let frankenstein_new_doc = extract_doc_given_isbn(&reader, &frankenstein_isbn)?.unwrap();
+    let frankenstein_new_doc = extract_doc_given_isbn(&index, &frankenstein_isbn)?.unwrap();
     assert_eq!(
         schema.to_json(&frankenstein_new_doc),
         r#"{"isbn":["978-9176370711"],"title":["Frankenstein"]}"#,

examples/faceted_search.rs

@@ -10,103 +10,71 @@
 // - search for the best document matchings "sea whale"
 // - retrieve the best document original content.
 
+extern crate tempdir;
+
 // ---
 // Importing tantivy...
+#[macro_use]
+extern crate tantivy;
 use tantivy::collector::FacetCollector;
-use tantivy::query::{AllQuery, TermQuery};
+use tantivy::query::AllQuery;
 use tantivy::schema::*;
-use tantivy::{doc, Index};
+use tantivy::Index;
 
 fn main() -> tantivy::Result<()> {
-    // Let's create a temporary directory for the sake of this example
+    // Let's create a temporary directory for the
+    // sake of this example
+    let index_path = TempDir::new("tantivy_facet_example_dir")?;
     let mut schema_builder = Schema::builder();
 
-    let name = schema_builder.add_text_field("felin_name", TEXT | STORED);
-    // this is our faceted field: its scientific classification
-    let classification = schema_builder.add_facet_field("classification");
+    schema_builder.add_text_field("name", TEXT | STORED);
+
+    // this is our faceted field
+    schema_builder.add_facet_field("tags");
 
     let schema = schema_builder.build();
-    let index = Index::create_in_ram(schema);
 
-    let mut index_writer = index.writer(30_000_000)?;
+    let index = Index::create_in_dir(&index_path, schema.clone())?;
+
+    let mut index_writer = index.writer(50_000_000)?;
+
+    let name = schema.get_field("name").unwrap();
+    let tags = schema.get_field("tags").unwrap();
 
     // For convenience, tantivy also comes with a macro to
     // reduce the boilerplate above.
     index_writer.add_document(doc!(
-        name => "Cat",
-        classification => Facet::from("/Felidae/Felinae/Felis")
+        name => "the ditch",
+        tags => Facet::from("/pools/north")
     ));
+
     index_writer.add_document(doc!(
-        name => "Canada lynx",
-        classification => Facet::from("/Felidae/Felinae/Lynx")
-    ));
-    index_writer.add_document(doc!(
-        name => "Cheetah",
-        classification => Facet::from("/Felidae/Felinae/Acinonyx")
-    ));
-    index_writer.add_document(doc!(
-        name => "Tiger",
-        classification => Facet::from("/Felidae/Pantherinae/Panthera")
-    ));
-    index_writer.add_document(doc!(
-        name => "Lion",
-        classification => Facet::from("/Felidae/Pantherinae/Panthera")
-    ));
-    index_writer.add_document(doc!(
-        name => "Jaguar",
-        classification => Facet::from("/Felidae/Pantherinae/Panthera")
-    ));
-    index_writer.add_document(doc!(
-        name => "Sunda clouded leopard",
-        classification => Facet::from("/Felidae/Pantherinae/Neofelis")
-    ));
-    index_writer.add_document(doc!(
-        name => "Fossa",
-        classification => Facet::from("/Eupleridae/Cryptoprocta")
+        name => "little stacey",
+        tags => Facet::from("/pools/south")
     ));
+
     index_writer.commit()?;
 
-    let reader = index.reader()?;
-    let searcher = reader.searcher();
-    {
-        let mut facet_collector = FacetCollector::for_field(classification);
-        facet_collector.add_facet("/Felidae");
-        let facet_counts = searcher.search(&AllQuery, &facet_collector)?;
-        // This lists all of the facet counts, right below "/Felidae".
-        let facets: Vec<(&Facet, u64)> = facet_counts.get("/Felidae").collect();
-        assert_eq!(
-            facets,
-            vec![
-                (&Facet::from("/Felidae/Felinae"), 3),
-                (&Facet::from("/Felidae/Pantherinae"), 4),
-            ]
-        );
-    }
+    index.load_searchers()?;
 
-    // Facets are also searchable.
-    //
-    // For instance a common UI pattern is to allow the user someone to click on a facet link
-    // (e.g: `Pantherinae`) to drill down and filter the current result set with this subfacet.
-    //
-    // The search would then look as follows.
+    let searcher = index.searcher();
 
-    // Check the reference doc for different ways to create a `Facet` object.
-    {
-        let facet = Facet::from_text("/Felidae/Pantherinae");
-        let facet_term = Term::from_facet(classification, &facet);
-        let facet_term_query = TermQuery::new(facet_term, IndexRecordOption::Basic);
-        let mut facet_collector = FacetCollector::for_field(classification);
-        facet_collector.add_facet("/Felidae/Pantherinae");
-        let facet_counts = searcher.search(&facet_term_query, &facet_collector)?;
-        let facets: Vec<(&Facet, u64)> = facet_counts.get("/Felidae/Pantherinae").collect();
-        assert_eq!(
-            facets,
-            vec![
-                (&Facet::from("/Felidae/Pantherinae/Neofelis"), 1),
-                (&Facet::from("/Felidae/Pantherinae/Panthera"), 3),
-            ]
-        );
-    }
+    let mut facet_collector = FacetCollector::for_field(tags);
+    facet_collector.add_facet("/pools");
+
+    let facet_counts = searcher.search(&AllQuery, &facet_collector).unwrap();
+
+    // This lists all of the facet counts
+    let facets: Vec<(&Facet, u64)> = facet_counts.get("/pools").collect();
+    assert_eq!(
+        facets,
+        vec![
+            (&Facet::from("/pools/north"), 1),
+            (&Facet::from("/pools/south"), 1),
+        ]
+    );
 
     Ok(())
 }
+
+use tempdir::TempDir;

examples/faceted_search_with_tweaked_score.rs

@@ -1,98 +0,0 @@
-use std::collections::HashSet;
-use tantivy::collector::TopDocs;
-use tantivy::doc;
-use tantivy::query::BooleanQuery;
-use tantivy::schema::*;
-use tantivy::{DocId, Index, Score, SegmentReader};
-
-fn main() -> tantivy::Result<()> {
-    let mut schema_builder = Schema::builder();
-
-    let title = schema_builder.add_text_field("title", STORED);
-    let ingredient = schema_builder.add_facet_field("ingredient");
-
-    let schema = schema_builder.build();
-    let index = Index::create_in_ram(schema.clone());
-
-    let mut index_writer = index.writer(30_000_000)?;
-
-    index_writer.add_document(doc!(
-        title => "Fried egg",
-        ingredient => Facet::from("/ingredient/egg"),
-        ingredient => Facet::from("/ingredient/oil"),
-    ));
-    index_writer.add_document(doc!(
-        title => "Scrambled egg",
-        ingredient => Facet::from("/ingredient/egg"),
-        ingredient => Facet::from("/ingredient/butter"),
-        ingredient => Facet::from("/ingredient/milk"),
-        ingredient => Facet::from("/ingredient/salt"),
-    ));
-    index_writer.add_document(doc!(
-        title => "Egg rolls",
-        ingredient => Facet::from("/ingredient/egg"),
-        ingredient => Facet::from("/ingredient/garlic"),
-        ingredient => Facet::from("/ingredient/salt"),
-        ingredient => Facet::from("/ingredient/oil"),
-        ingredient => Facet::from("/ingredient/tortilla-wrap"),
-        ingredient => Facet::from("/ingredient/mushroom"),
-    ));
-    index_writer.commit()?;
-
-    let reader = index.reader()?;
-    let searcher = reader.searcher();
-    {
-        let facets = vec![
-            Facet::from("/ingredient/egg"),
-            Facet::from("/ingredient/oil"),
-            Facet::from("/ingredient/garlic"),
-            Facet::from("/ingredient/mushroom"),
-        ];
-        let query = BooleanQuery::new_multiterms_query(
-            facets
-                .iter()
-                .map(|key| Term::from_facet(ingredient, &key))
-                .collect(),
-        );
-        let top_docs_by_custom_score =
-            TopDocs::with_limit(2).tweak_score(move |segment_reader: &SegmentReader| {
-                let mut ingredient_reader = segment_reader.facet_reader(ingredient).unwrap();
-                let facet_dict = ingredient_reader.facet_dict();
-
-                let query_ords: HashSet<u64> = facets
-                    .iter()
-                    .filter_map(|key| facet_dict.term_ord(key.encoded_str()))
-                    .collect();
-
-                let mut facet_ords_buffer: Vec<u64> = Vec::with_capacity(20);
-
-                move |doc: DocId, original_score: Score| {
-                    ingredient_reader.facet_ords(doc, &mut facet_ords_buffer);
-                    let missing_ingredients = facet_ords_buffer
-                        .iter()
-                        .filter(|ord| !query_ords.contains(ord))
-                        .count();
-                    let tweak = 1.0 / 4_f32.powi(missing_ingredients as i32);
-
-                    original_score * tweak
-                }
-            });
-        let top_docs = searcher.search(&query, &top_docs_by_custom_score)?;
-
-        let titles: Vec<String> = top_docs
-            .iter()
-            .map(|(_, doc_id)| {
-                searcher
-                    .doc(*doc_id)
-                    .unwrap()
-                    .get_first(title)
-                    .unwrap()
-                    .text()
-                    .unwrap()
-                    .to_owned()
-            })
-            .collect();
-        assert_eq!(titles, vec!["Fried egg", "Egg rolls"]);
-    }
-    Ok(())
-}

examples/integer_range_search.rs

@@ -1,39 +0,0 @@
-// # Searching a range on an indexed int field.
-//
-// Below is an example of creating an indexed integer field in your schema
-// You can use RangeQuery to get a Count of all occurrences in a given range.
-use tantivy::collector::Count;
-use tantivy::query::RangeQuery;
-use tantivy::schema::{Schema, INDEXED};
-use tantivy::{doc, Index, Result};
-
-fn run() -> Result<()> {
-    // For the sake of simplicity, this schema will only have 1 field
-    let mut schema_builder = Schema::builder();
-
-    // `INDEXED` is a short-hand to indicate that our field should be "searchable".
-    let year_field = schema_builder.add_u64_field("year", INDEXED);
-    let schema = schema_builder.build();
-    let index = Index::create_in_ram(schema);
-    let reader = index.reader()?;
-    {
-        let mut index_writer = index.writer_with_num_threads(1, 6_000_000)?;
-        for year in 1950u64..2019u64 {
-            index_writer.add_document(doc!(year_field => year));
-        }
-        index_writer.commit()?;
-        // The index will be a range of years
-    }
-    reader.reload()?;
-    let searcher = reader.searcher();
-    // The end is excluded i.e. here we are searching up to 1969
-    let docs_in_the_sixties = RangeQuery::new_u64(year_field, 1960..1970);
-    // Uses a Count collector to sum the total number of docs in the range
-    let num_60s_books = searcher.search(&docs_in_the_sixties, &Count)?;
-    assert_eq!(num_60s_books, 10);
-    Ok(())
-}
-
-fn main() {
-    run().unwrap()
-}

examples/iterating_docs_and_positions.rs

@@ -9,8 +9,11 @@
 
 // ---
 // Importing tantivy...
+#[macro_use]
+extern crate tantivy;
 use tantivy::schema::*;
-use tantivy::{doc, DocSet, Index, Postings, TERMINATED};
+use tantivy::Index;
+use tantivy::{DocId, DocSet, Postings};
 
 fn main() -> tantivy::Result<()> {
     // We first create a schema for the sake of the
@@ -30,9 +33,9 @@ fn main() -> tantivy::Result<()> {
     index_writer.add_document(doc!(title => "The modern Promotheus"));
     index_writer.commit()?;
 
-    let reader = index.reader()?;
+    index.load_searchers()?;
 
-    let searcher = reader.searcher();
+    let searcher = index.searcher();
 
     // A tantivy index is actually a collection of segments.
     // Similarly, a searcher just wraps a list `segment_reader`.
@@ -62,11 +65,12 @@ fn main() -> tantivy::Result<()> {
         {
             // this buffer will be used to request for positions
             let mut positions: Vec<u32> = Vec::with_capacity(100);
-            let mut doc_id = segment_postings.doc();
-            while doc_id != TERMINATED {
+            while segment_postings.advance() {
+                // the number of time the term appears in the document.
+                let doc_id: DocId = segment_postings.doc(); //< do not try to access this before calling advance once.
+
                 // This MAY contains deleted documents as well.
                 if segment_reader.is_deleted(doc_id) {
-                    doc_id = segment_postings.advance();
                     continue;
                 }
 
@@ -85,7 +89,6 @@ fn main() -> tantivy::Result<()> {
                 // Doc 2: TermFreq 1: [0]
                 // ```
                 println!("Doc {}: TermFreq {}: {:?}", doc_id, term_freq, positions);
-                doc_id = segment_postings.advance();
             }
         }
     }
@@ -117,16 +120,11 @@ fn main() -> tantivy::Result<()> {
         if let Some(mut block_segment_postings) =
             inverted_index.read_block_postings(&term_the, IndexRecordOption::Basic)
         {
-            loop {
-                let docs = block_segment_postings.docs();
-                if docs.is_empty() {
-                    break;
-                }
+            while block_segment_postings.advance() {
                 // Once again these docs MAY contains deleted documents as well.
                 let docs = block_segment_postings.docs();
                 // Prints `Docs [0, 2].`
                 println!("Docs {:?}", docs);
-                block_segment_postings.advance();
             }
         }
     }

examples/multiple_producer.rs

@@ -1,100 +0,0 @@
-// # Indexing from different threads.
-//
-// It is fairly common to have to index from different threads.
-// Tantivy forbids to create more than one `IndexWriter` at a time.
-//
-// This `IndexWriter` itself has its own multithreaded layer, so managing your own
-// indexing threads will not help. However, it can still be useful for some applications.
-//
-// For instance, if preparing documents to send to tantivy before indexing is the bottleneck of
-// your application, it is reasonable to have multiple threads.
-//
-// Another very common reason to want to index from multiple threads, is implementing a webserver
-// with CRUD capabilities. The server framework will most likely handle request from
-// different threads.
-//
-// The recommended way to address both of these use case is to wrap your `IndexWriter` into a
-// `Arc<RwLock<IndexWriter>>`.
-//
-// While this is counterintuitive, adding and deleting documents do not require mutability
-// over the `IndexWriter`, so several threads will be able to do this operation concurrently.
-//
-// The example below does not represent an actual real-life use case (who would spawn thread to
-// index a single document?), but aims at demonstrating the mechanism that makes indexing
-// from several threads possible.
-
-// ---
-// Importing tantivy...
-use std::sync::{Arc, RwLock};
-use std::thread;
-use std::time::Duration;
-use tantivy::schema::{Schema, STORED, TEXT};
-use tantivy::{doc, Index, IndexWriter, Opstamp};
-
-fn main() -> tantivy::Result<()> {
-    // # Defining the schema
-    let mut schema_builder = Schema::builder();
-    let title = schema_builder.add_text_field("title", TEXT | STORED);
-    let body = schema_builder.add_text_field("body", TEXT);
-    let schema = schema_builder.build();
-
-    let index = Index::create_in_ram(schema);
-    let index_writer: Arc<RwLock<IndexWriter>> = Arc::new(RwLock::new(index.writer(50_000_000)?));
-
-    // # First indexing thread.
-    let index_writer_clone_1 = index_writer.clone();
-    thread::spawn(move || {
-        // we index 100 times the document... for the sake of the example.
-        for i in 0..100 {
-            let opstamp = index_writer_clone_1
-                .read().unwrap() //< A read lock is sufficient here.
-                .add_document(
-                    doc!(
-                        title => "Of Mice and Men",
-                        body => "A few miles south of Soledad, the Salinas River drops in close to the hillside \
-                        bank and runs deep and green. The water is warm too, for it has slipped twinkling \
-                        over the yellow sands in the sunlight before reaching the narrow pool. On one \
-                        side of the river the golden foothill slopes curve up to the strong and rocky \
-                        Gabilan Mountains, but on the valley side the water is lined with trees—willows \
-                        fresh and green with every spring, carrying in their lower leaf junctures the \
-                        debris of the winter’s flooding; and sycamores with mottled, white, recumbent \
-                        limbs and branches that arch over the pool"
-                    ));
-            println!("add doc {} from thread 1 - opstamp {}", i, opstamp);
-            thread::sleep(Duration::from_millis(20));
-        }
-    });
-
-    // # Second indexing thread.
-    let index_writer_clone_2 = index_writer.clone();
-    // For convenience, tantivy also comes with a macro to
-    // reduce the boilerplate above.
-    thread::spawn(move || {
-        // we index 100 times the document... for the sake of the example.
-        for i in 0..100 {
-            // A read lock is sufficient here.
-            let opstamp = {
-                let index_writer_rlock = index_writer_clone_2.read().unwrap();
-                index_writer_rlock.add_document(doc!(
-                    title => "Manufacturing consent",
-                    body => "Some great book description..."
-                ))
-            };
-            println!("add doc {} from thread 2 - opstamp {}", i, opstamp);
-            thread::sleep(Duration::from_millis(10));
-        }
-    });
-
-    // # In the main thread, we commit 10 times, once every 500ms.
-    for _ in 0..10 {
-        let opstamp: Opstamp = {
-            // Committing or rollbacking on the other hand requires write lock. This will block other threads.
-            let mut index_writer_wlock = index_writer.write().unwrap();
-            index_writer_wlock.commit().unwrap()
-        };
-        println!("committed with opstamp {}", opstamp);
-        thread::sleep(Duration::from_millis(500));
-    }
-
-    Ok(())
-}

examples/pre_tokenized_text.rs

@@ -1,139 +0,0 @@
-// # Pre-tokenized text example
-//
-// This example shows how to use pre-tokenized text. Sometimes yout might
-// want to index and search through text which is already split into
-// tokens by some external tool.
-//
-// In this example we will:
-// - use tantivy tokenizer to create tokens and load them directly into tantivy,
-// - import tokenized text straight from json,
-// - perform a search on documents with pre-tokenized text
-
-use tantivy::collector::{Count, TopDocs};
-use tantivy::query::TermQuery;
-use tantivy::schema::*;
-use tantivy::tokenizer::{PreTokenizedString, SimpleTokenizer, Token, Tokenizer};
-use tantivy::{doc, Index, ReloadPolicy};
-use tempfile::TempDir;
-
-fn pre_tokenize_text(text: &str) -> Vec<Token> {
-    let mut token_stream = SimpleTokenizer.token_stream(text);
-    let mut tokens = vec![];
-    while token_stream.advance() {
-        tokens.push(token_stream.token().clone());
-    }
-    tokens
-}
-
-fn main() -> tantivy::Result<()> {
-    let index_path = TempDir::new()?;
-
-    let mut schema_builder = Schema::builder();
-
-    schema_builder.add_text_field("title", TEXT | STORED);
-    schema_builder.add_text_field("body", TEXT);
-
-    let schema = schema_builder.build();
-
-    let index = Index::create_in_dir(&index_path, schema.clone())?;
-
-    let mut index_writer = index.writer(50_000_000)?;
-
-    // We can create a document manually, by setting the fields
-    // one by one in a Document object.
-    let title = schema.get_field("title").unwrap();
-    let body = schema.get_field("body").unwrap();
-
-    let title_text = "The Old Man and the Sea";
-    let body_text = "He was an old man who fished alone in a skiff in the Gulf Stream";
-
-    // Content of our first document
-    // We create `PreTokenizedString` which contains original text and vector of tokens
-    let title_tok = PreTokenizedString {
-        text: String::from(title_text),
-        tokens: pre_tokenize_text(title_text),
-    };
-
-    println!(
-        "Original text: \"{}\" and tokens: {:?}",
-        title_tok.text, title_tok.tokens
-    );
-
-    let body_tok = PreTokenizedString {
-        text: String::from(body_text),
-        tokens: pre_tokenize_text(body_text),
-    };
-
-    // Now lets create a document and add our `PreTokenizedString`
-    let old_man_doc = doc!(title => title_tok, body => body_tok);
-
-    // ... now let's just add it to the IndexWriter
-    index_writer.add_document(old_man_doc);
-
-    // Pretokenized text can also be fed as JSON
-    let short_man_json = r#"{
-        "title":[{
-            "text":"The Old Man",
-            "tokens":[
-                {"offset_from":0,"offset_to":3,"position":0,"text":"The","position_length":1},
-                {"offset_from":4,"offset_to":7,"position":1,"text":"Old","position_length":1},
-                {"offset_from":8,"offset_to":11,"position":2,"text":"Man","position_length":1}
-            ]
-        }]
-    }"#;
-
-    let short_man_doc = schema.parse_document(&short_man_json)?;
-
-    index_writer.add_document(short_man_doc);
-
-    // Let's commit changes
-    index_writer.commit()?;
-
-    // ... and now is the time to query our index
-
-    let reader = index
-        .reader_builder()
-        .reload_policy(ReloadPolicy::OnCommit)
-        .try_into()?;
-
-    let searcher = reader.searcher();
-
-    // We want to get documents with token "Man", we will use TermQuery to do it
-    // Using PreTokenizedString means the tokens are stored as is avoiding stemming
-    // and lowercasing, which preserves full words in their original form
-    let query = TermQuery::new(
-        Term::from_field_text(title, "Man"),
-        IndexRecordOption::Basic,
-    );
-
-    let (top_docs, count) = searcher
-        .search(&query, &(TopDocs::with_limit(2), Count))
-        .unwrap();
-
-    assert_eq!(count, 2);
-
-    // Now let's print out the results.
-    // Note that the tokens are not stored along with the original text
-    // in the document store
-    for (_score, doc_address) in top_docs {
-        let retrieved_doc = searcher.doc(doc_address)?;
-        println!("Document: {}", schema.to_json(&retrieved_doc));
-    }
-
-    // In contrary to the previous query, when we search for the "man" term we
-    // should get no results, as it's not one of the indexed tokens. SimpleTokenizer
-    // only splits text on whitespace / punctuation.
-
-    let query = TermQuery::new(
-        Term::from_field_text(title, "man"),
-        IndexRecordOption::Basic,
-    );
-
-    let (_top_docs, count) = searcher
-        .search(&query, &(TopDocs::with_limit(2), Count))
-        .unwrap();
-
-    assert_eq!(count, 0);
-
-    Ok(())
-}

examples/snippet.rs

@@ -4,19 +4,23 @@
 // your hit result.
 // Snippet are an extracted of a target document, and returned in HTML format.
 // The keyword searched by the user are highlighted with a `<b>` tag.
+extern crate tempdir;
 
 // ---
 // Importing tantivy...
+#[macro_use]
+extern crate tantivy;
 use tantivy::collector::TopDocs;
 use tantivy::query::QueryParser;
 use tantivy::schema::*;
-use tantivy::{doc, Index, Snippet, SnippetGenerator};
-use tempfile::TempDir;
+use tantivy::Index;
+use tantivy::{Snippet, SnippetGenerator};
+use tempdir::TempDir;
 
 fn main() -> tantivy::Result<()> {
     // Let's create a temporary directory for the
     // sake of this example
-    let index_path = TempDir::new()?;
+    let index_path = TempDir::new("tantivy_example_dir")?;
 
     // # Defining the schema
     let mut schema_builder = Schema::builder();
@@ -44,8 +48,9 @@ fn main() -> tantivy::Result<()> {
     // ...
     index_writer.commit()?;
 
-    let reader = index.reader()?;
-    let searcher = reader.searcher();
+    index.load_searchers()?;
+
+    let searcher = index.searcher();
     let query_parser = QueryParser::for_index(&index, vec![title, body]);
     let query = query_parser.parse_query("sycamore spring")?;
 

examples/stop_words.rs

@@ -9,13 +9,17 @@
 // - add a few stop words
 // - index few documents in our index
 
+extern crate tempdir;
+
 // ---
 // Importing tantivy...
+#[macro_use]
+extern crate tantivy;
 use tantivy::collector::TopDocs;
 use tantivy::query::QueryParser;
 use tantivy::schema::*;
 use tantivy::tokenizer::*;
-use tantivy::{doc, Index};
+use tantivy::Index;
 
 fn main() -> tantivy::Result<()> {
     // this example assumes you understand the content in `basic_search`
@@ -50,7 +54,7 @@ fn main() -> tantivy::Result<()> {
 
     // This tokenizer lowers all of the text (to help with stop word matching)
     // then removes all instances of `the` and `and` from the corpus
-    let tokenizer = TextAnalyzer::from(SimpleTokenizer)
+    let tokenizer = SimpleTokenizer
         .filter(LowerCaser)
         .filter(StopWordFilter::remove(vec![
             "the".to_string(),
@@ -92,9 +96,9 @@ fn main() -> tantivy::Result<()> {
 
     index_writer.commit()?;
 
-    let reader = index.reader()?;
+    index.load_searchers()?;
 
-    let searcher = reader.searcher();
+    let searcher = index.searcher();
 
     let query_parser = QueryParser::for_index(&index, vec![title, body]);
 

examples/working_with_json.rs

@@ -1,4 +1,4 @@
-use tantivy;
+extern crate tantivy;
 use tantivy::schema::*;
 
 // # Document from json
@@ -12,7 +12,7 @@ fn main() -> tantivy::Result<()> {
     let mut schema_builder = Schema::builder();
     schema_builder.add_text_field("title", TEXT | STORED);
     schema_builder.add_text_field("body", TEXT);
-    schema_builder.add_u64_field("year", INDEXED);
+    schema_builder.add_u64_field("year", INT_INDEXED);
     let schema = schema_builder.build();
 
     // Let's assume we have a json-serialized document.

query-grammar/Cargo.toml

@@ -1,16 +0,0 @@
-[package]
-name = "tantivy-query-grammar"
-version = "0.14.0-dev"
-authors = ["Paul Masurel <paul.masurel@gmail.com>"]
-license = "MIT"
-categories = ["database-implementations", "data-structures"]
-description = """Search engine library"""
-documentation = "https://tantivy-search.github.io/tantivy/tantivy/index.html"
-homepage = "https://github.com/tantivy-search/tantivy"
-repository = "https://github.com/tantivy-search/tantivy"
-readme = "README.md"
-keywords = ["search", "information", "retrieval"]
-edition = "2018"
-
-[dependencies]
-combine = {version="4", default-features=false, features=[] }

query-grammar/README.md

@@ -1,3 +0,0 @@
-# Tantivy Query Grammar
-
-This crate is used by tantivy to parse queries.

query-grammar/src/lib.rs

@@ -1,15 +0,0 @@
-mod occur;
-mod query_grammar;
-mod user_input_ast;
-use combine::parser::Parser;
-
-pub use crate::occur::Occur;
-use crate::query_grammar::parse_to_ast;
-pub use crate::user_input_ast::{UserInputAST, UserInputBound, UserInputLeaf, UserInputLiteral};
-
-pub struct Error;
-
-pub fn parse_query(query: &str) -> Result<UserInputAST, Error> {
-    let (user_input_ast, _remaining) = parse_to_ast().parse(query).map_err(|_| Error)?;
-    Ok(user_input_ast)
-}

query-grammar/src/occur.rs

@@ -1,72 +0,0 @@
-use std::fmt;
-use std::fmt::Write;
-
-/// Defines whether a term in a query must be present,
-/// should be present or must be not present.
-#[derive(Debug, Clone, Hash, Copy, Eq, PartialEq)]
-pub enum Occur {
-    /// For a given document to be considered for scoring,
-    /// at least one of the document with the Should or the Must
-    /// Occur constraint must be within the document.
-    Should,
-    /// Document without the term are excluded from the search.
-    Must,
-    /// Document that contain the term are excluded from the
-    /// search.
-    MustNot,
-}
-
-impl Occur {
-    /// Returns the one-char prefix symbol for this `Occur`.
-    /// - `Should` => '?',
-    /// - `Must` => '+'
-    /// - `Not` => '-'
-    fn to_char(self) -> char {
-        match self {
-            Occur::Should => '?',
-            Occur::Must => '+',
-            Occur::MustNot => '-',
-        }
-    }
-
-    /// Compose two occur values.
-    pub fn compose(left: Occur, right: Occur) -> Occur {
-        match (left, right) {
-            (Occur::Should, _) => right,
-            (Occur::Must, Occur::MustNot) => Occur::MustNot,
-            (Occur::Must, _) => Occur::Must,
-            (Occur::MustNot, Occur::MustNot) => Occur::Must,
-            (Occur::MustNot, _) => Occur::MustNot,
-        }
-    }
-}
-
-impl fmt::Display for Occur {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        f.write_char(self.to_char())
-    }
-}
-
-#[cfg(test)]
-mod test {
-    use crate::Occur;
-
-    #[test]
-    fn test_occur_compose() {
-        assert_eq!(Occur::compose(Occur::Should, Occur::Should), Occur::Should);
-        assert_eq!(Occur::compose(Occur::Should, Occur::Must), Occur::Must);
-        assert_eq!(
-            Occur::compose(Occur::Should, Occur::MustNot),
-            Occur::MustNot
-        );
-        assert_eq!(Occur::compose(Occur::Must, Occur::Should), Occur::Must);
-        assert_eq!(Occur::compose(Occur::Must, Occur::Must), Occur::Must);
-        assert_eq!(Occur::compose(Occur::Must, Occur::MustNot), Occur::MustNot);
-        assert_eq!(
-            Occur::compose(Occur::MustNot, Occur::Should),
-            Occur::MustNot
-        );
-        assert_eq!(Occur::compose(Occur::MustNot, Occur::Must), Occur::MustNot);
-        assert_eq!(Occur::compose(Occur::MustNot, Occur::MustNot), Occur::Must);
-    }
-}

query-grammar/src/query_grammar.rs

@@ -1,534 +0,0 @@
-use super::user_input_ast::{UserInputAST, UserInputBound, UserInputLeaf, UserInputLiteral};
-use crate::Occur;
-use combine::error::StringStreamError;
-use combine::parser::char::{char, digit, letter, space, spaces, string};
-use combine::parser::Parser;
-use combine::{
-    attempt, choice, eof, many, many1, one_of, optional, parser, satisfy, skip_many1, value,
-};
-
-fn field<'a>() -> impl Parser<&'a str, Output = String> {
-    (
-        (letter().or(char('_'))),
-        many(satisfy(|c: char| {
-            c.is_alphanumeric() || c == '_' || c == '-'
-        })),
-    )
-        .skip(char(':'))
-        .map(|(s1, s2): (char, String)| format!("{}{}", s1, s2))
-}
-
-fn word<'a>() -> impl Parser<&'a str, Output = String> {
-    (
-        satisfy(|c: char| {
-            !c.is_whitespace()
-                && !['-', '^', '`', ':', '{', '}', '"', '[', ']', '(', ')'].contains(&c)
-        }),
-        many(satisfy(|c: char| {
-            !c.is_whitespace() && ![':', '^', '{', '}', '"', '[', ']', '(', ')'].contains(&c)
-        })),
-    )
-        .map(|(s1, s2): (char, String)| format!("{}{}", s1, s2))
-        .and_then(|s: String| match s.as_str() {
-            "OR" | "AND " | "NOT" => Err(StringStreamError::UnexpectedParse),
-            _ => Ok(s),
-        })
-}
-
-fn term_val<'a>() -> impl Parser<&'a str, Output = String> {
-    let phrase = char('"').with(many1(satisfy(|c| c != '"'))).skip(char('"'));
-    phrase.or(word())
-}
-
-fn term_query<'a>() -> impl Parser<&'a str, Output = UserInputLiteral> {
-    let term_val_with_field = negative_number().or(term_val());
-    (field(), term_val_with_field).map(|(field_name, phrase)| UserInputLiteral {
-        field_name: Some(field_name),
-        phrase,
-    })
-}
-
-fn literal<'a>() -> impl Parser<&'a str, Output = UserInputLeaf> {
-    let term_default_field = term_val().map(|phrase| UserInputLiteral {
-        field_name: None,
-        phrase,
-    });
-    attempt(term_query())
-        .or(term_default_field)
-        .map(UserInputLeaf::from)
-}
-
-fn negative_number<'a>() -> impl Parser<&'a str, Output = String> {
-    (
-        char('-'),
-        many1(digit()),
-        optional((char('.'), many1(digit()))),
-    )
-        .map(|(s1, s2, s3): (char, String, Option<(char, String)>)| {
-            if let Some(('.', s3)) = s3 {
-                format!("{}{}.{}", s1, s2, s3)
-            } else {
-                format!("{}{}", s1, s2)
-            }
-        })
-}
-
-fn spaces1<'a>() -> impl Parser<&'a str, Output = ()> {
-    skip_many1(space())
-}
-
-/// Function that parses a range out of a Stream
-/// Supports ranges like:
-/// [5 TO 10], {5 TO 10}, [* TO 10], [10 TO *], {10 TO *], >5, <=10
-/// [a TO *], [a TO c], [abc TO bcd}
-fn range<'a>() -> impl Parser<&'a str, Output = UserInputLeaf> {
-    let range_term_val = || {
-        word()
-            .or(negative_number())
-            .or(char('*').with(value("*".to_string())))
-    };
-
-    // check for unbounded range in the form of <5, <=10, >5, >=5
-    let elastic_unbounded_range = (
-        choice([
-            attempt(string(">=")),
-            attempt(string("<=")),
-            attempt(string("<")),
-            attempt(string(">")),
-        ])
-        .skip(spaces()),
-        range_term_val(),
-    )
-        .map(
-            |(comparison_sign, bound): (&str, String)| match comparison_sign {
-                ">=" => (UserInputBound::Inclusive(bound), UserInputBound::Unbounded),
-                "<=" => (UserInputBound::Unbounded, UserInputBound::Inclusive(bound)),
-                "<" => (UserInputBound::Unbounded, UserInputBound::Exclusive(bound)),
-                ">" => (UserInputBound::Exclusive(bound), UserInputBound::Unbounded),
-                // default case
-                _ => (UserInputBound::Unbounded, UserInputBound::Unbounded),
-            },
-        );
-    let lower_bound = (one_of("{[".chars()), range_term_val()).map(
-        |(boundary_char, lower_bound): (char, String)| {
-            if lower_bound == "*" {
-                UserInputBound::Unbounded
-            } else if boundary_char == '{' {
-                UserInputBound::Exclusive(lower_bound)
-            } else {
-                UserInputBound::Inclusive(lower_bound)
-            }
-        },
-    );
-    let upper_bound = (range_term_val(), one_of("}]".chars())).map(
-        |(higher_bound, boundary_char): (String, char)| {
-            if higher_bound == "*" {
-                UserInputBound::Unbounded
-            } else if boundary_char == '}' {
-                UserInputBound::Exclusive(higher_bound)
-            } else {
-                UserInputBound::Inclusive(higher_bound)
-            }
-        },
-    );
-    // return only lower and upper
-    let lower_to_upper = (
-        lower_bound.skip((spaces(), string("TO"), spaces())),
-        upper_bound,
-    );
-
-    (
-        optional(field()).skip(spaces()),
-        // try elastic first, if it matches, the range is unbounded
-        attempt(elastic_unbounded_range).or(lower_to_upper),
-    )
-        .map(|(field, (lower, upper))|
-             // Construct the leaf from extracted field (optional)
-             // and bounds
-             UserInputLeaf::Range {
-                 field,
-                 lower,
-                 upper
-    })
-}
-
-fn negate(expr: UserInputAST) -> UserInputAST {
-    expr.unary(Occur::MustNot)
-}
-
-fn leaf<'a>() -> impl Parser<&'a str, Output = UserInputAST> {
-    parser(|input| {
-        char('(')
-            .with(ast())
-            .skip(char(')'))
-            .or(char('*').map(|_| UserInputAST::from(UserInputLeaf::All)))
-            .or(attempt(
-                string("NOT").skip(spaces1()).with(leaf()).map(negate),
-            ))
-            .or(attempt(range().map(UserInputAST::from)))
-            .or(literal().map(UserInputAST::from))
-            .parse_stream(input)
-            .into_result()
-    })
-}
-
-fn occur_symbol<'a>() -> impl Parser<&'a str, Output = Occur> {
-    char('-')
-        .map(|_| Occur::MustNot)
-        .or(char('+').map(|_| Occur::Must))
-}
-
-fn occur_leaf<'a>() -> impl Parser<&'a str, Output = (Option<Occur>, UserInputAST)> {
-    (optional(occur_symbol()), boosted_leaf())
-}
-
-fn positive_float_number<'a>() -> impl Parser<&'a str, Output = f64> {
-    (many1(digit()), optional((char('.'), many1(digit())))).map(
-        |(int_part, decimal_part_opt): (String, Option<(char, String)>)| {
-            let mut float_str = int_part;
-            if let Some((chr, decimal_str)) = decimal_part_opt {
-                float_str.push(chr);
-                float_str.push_str(&decimal_str);
-            }
-            float_str.parse::<f64>().unwrap()
-        },
-    )
-}
-
-fn boost<'a>() -> impl Parser<&'a str, Output = f64> {
-    (char('^'), positive_float_number()).map(|(_, boost)| boost)
-}
-
-fn boosted_leaf<'a>() -> impl Parser<&'a str, Output = UserInputAST> {
-    (leaf(), optional(boost())).map(|(leaf, boost_opt)| match boost_opt {
-        Some(boost) if (boost - 1.0).abs() > std::f64::EPSILON => {
-            UserInputAST::Boost(Box::new(leaf), boost)
-        }
-        _ => leaf,
-    })
-}
-
-#[derive(Clone, Copy)]
-enum BinaryOperand {
-    Or,
-    And,
-}
-
-fn binary_operand<'a>() -> impl Parser<&'a str, Output = BinaryOperand> {
-    string("AND")
-        .with(value(BinaryOperand::And))
-        .or(string("OR").with(value(BinaryOperand::Or)))
-}
-
-fn aggregate_binary_expressions(
-    left: UserInputAST,
-    others: Vec<(BinaryOperand, UserInputAST)>,
-) -> UserInputAST {
-    let mut dnf: Vec<Vec<UserInputAST>> = vec![vec![left]];
-    for (operator, operand_ast) in others {
-        match operator {
-            BinaryOperand::And => {
-                if let Some(last) = dnf.last_mut() {
-                    last.push(operand_ast);
-                }
-            }
-            BinaryOperand::Or => {
-                dnf.push(vec![operand_ast]);
-            }
-        }
-    }
-    if dnf.len() == 1 {
-        UserInputAST::and(dnf.into_iter().next().unwrap()) //< safe
-    } else {
-        let conjunctions = dnf.into_iter().map(UserInputAST::and).collect();
-        UserInputAST::or(conjunctions)
-    }
-}
-
-fn operand_leaf<'a>() -> impl Parser<&'a str, Output = (BinaryOperand, UserInputAST)> {
-    (
-        binary_operand().skip(spaces()),
-        boosted_leaf().skip(spaces()),
-    )
-}
-
-pub fn ast<'a>() -> impl Parser<&'a str, Output = UserInputAST> {
-    let boolean_expr = (boosted_leaf().skip(spaces()), many1(operand_leaf()))
-        .map(|(left, right)| aggregate_binary_expressions(left, right));
-    let whitespace_separated_leaves = many1(occur_leaf().skip(spaces().silent())).map(
-        |subqueries: Vec<(Option<Occur>, UserInputAST)>| {
-            if subqueries.len() == 1 {
-                let (occur_opt, ast) = subqueries.into_iter().next().unwrap();
-                match occur_opt.unwrap_or(Occur::Should) {
-                    Occur::Must | Occur::Should => ast,
-                    Occur::MustNot => UserInputAST::Clause(vec![(Some(Occur::MustNot), ast)]),
-                }
-            } else {
-                UserInputAST::Clause(subqueries.into_iter().collect())
-            }
-        },
-    );
-    let expr = attempt(boolean_expr).or(whitespace_separated_leaves);
-    spaces().with(expr).skip(spaces())
-}
-
-pub fn parse_to_ast<'a>() -> impl Parser<&'a str, Output = UserInputAST> {
-    spaces()
-        .with(optional(ast()).skip(eof()))
-        .map(|opt_ast| opt_ast.unwrap_or_else(UserInputAST::empty_query))
-}
-
-#[cfg(test)]
-mod test {
-
-    type TestParseResult = Result<(), StringStreamError>;
-
-    use super::*;
-    use combine::parser::Parser;
-
-    pub fn nearly_equals(a: f64, b: f64) -> bool {
-        (a - b).abs() < 0.0005 * (a + b).abs()
-    }
-
-    fn assert_nearly_equals(expected: f64, val: f64) {
-        assert!(
-            nearly_equals(val, expected),
-            "Got {}, expected {}.",
-            val,
-            expected
-        );
-    }
-
-    #[test]
-    fn test_occur_symbol() -> TestParseResult {
-        assert_eq!(super::occur_symbol().parse("-")?, (Occur::MustNot, ""));
-        assert_eq!(super::occur_symbol().parse("+")?, (Occur::Must, ""));
-        Ok(())
-    }
-
-    #[test]
-    fn test_positive_float_number() {
-        fn valid_parse(float_str: &str, expected_val: f64, expected_remaining: &str) {
-            let (val, remaining) = positive_float_number().parse(float_str).unwrap();
-            assert_eq!(remaining, expected_remaining);
-            assert_nearly_equals(val, expected_val);
-        }
-        fn error_parse(float_str: &str) {
-            assert!(positive_float_number().parse(float_str).is_err());
-        }
-        valid_parse("1.0", 1.0, "");
-        valid_parse("1", 1.0, "");
-        valid_parse("0.234234 aaa", 0.234234f64, " aaa");
-        error_parse(".3332");
-        error_parse("1.");
-        error_parse("-1.");
-    }
-
-    fn test_parse_query_to_ast_helper(query: &str, expected: &str) {
-        let query = parse_to_ast().parse(query).unwrap().0;
-        let query_str = format!("{:?}", query);
-        assert_eq!(query_str, expected);
-    }
-
-    fn test_is_parse_err(query: &str) {
-        assert!(parse_to_ast().parse(query).is_err());
-    }
-
-    #[test]
-    fn test_parse_empty_to_ast() {
-        test_parse_query_to_ast_helper("", "<emptyclause>");
-    }
-
-    #[test]
-    fn test_parse_query_to_ast_hyphen() {
-        test_parse_query_to_ast_helper("\"www-form-encoded\"", "\"www-form-encoded\"");
-        test_parse_query_to_ast_helper("www-form-encoded", "\"www-form-encoded\"");
-        test_parse_query_to_ast_helper("www-form-encoded", "\"www-form-encoded\"");
-    }
-
-    #[test]
-    fn test_parse_query_to_ast_not_op() {
-        assert_eq!(
-            format!("{:?}", parse_to_ast().parse("NOT")),
-            "Err(UnexpectedParse)"
-        );
-        test_parse_query_to_ast_helper("NOTa", "\"NOTa\"");
-        test_parse_query_to_ast_helper("NOT a", "(-\"a\")");
-    }
-
-    #[test]
-    fn test_boosting() {
-        assert!(parse_to_ast().parse("a^2^3").is_err());
-        assert!(parse_to_ast().parse("a^2^").is_err());
-        test_parse_query_to_ast_helper("a^3", "(\"a\")^3");
-        test_parse_query_to_ast_helper("a^3 b^2", "(*(\"a\")^3 *(\"b\")^2)");
-        test_parse_query_to_ast_helper("a^1", "\"a\"");
-    }
-
-    #[test]
-    fn test_parse_query_to_ast_binary_op() {
-        test_parse_query_to_ast_helper("a AND b", "(+\"a\" +\"b\")");
-        test_parse_query_to_ast_helper("a OR b", "(?\"a\" ?\"b\")");
-        test_parse_query_to_ast_helper("a OR b AND c", "(?\"a\" ?(+\"b\" +\"c\"))");
-        test_parse_query_to_ast_helper("a AND b         AND c", "(+\"a\" +\"b\" +\"c\")");
-        assert_eq!(
-            format!("{:?}", parse_to_ast().parse("a OR b aaa")),
-            "Err(UnexpectedParse)"
-        );
-        assert_eq!(
-            format!("{:?}", parse_to_ast().parse("a AND b aaa")),
-            "Err(UnexpectedParse)"
-        );
-        assert_eq!(
-            format!("{:?}", parse_to_ast().parse("aaa a OR b ")),
-            "Err(UnexpectedParse)"
-        );
-        assert_eq!(
-            format!("{:?}", parse_to_ast().parse("aaa ccc a OR b ")),
-            "Err(UnexpectedParse)"
-        );
-    }
-
-    #[test]
-    fn test_parse_elastic_query_ranges() {
-        test_parse_query_to_ast_helper("title: >a", "title:{\"a\" TO \"*\"}");
-        test_parse_query_to_ast_helper("title:>=a", "title:[\"a\" TO \"*\"}");
-        test_parse_query_to_ast_helper("title: <a", "title:{\"*\" TO \"a\"}");
-        test_parse_query_to_ast_helper("title:<=a", "title:{\"*\" TO \"a\"]");
-        test_parse_query_to_ast_helper("title:<=bsd", "title:{\"*\" TO \"bsd\"]");
-
-        test_parse_query_to_ast_helper("weight: >70", "weight:{\"70\" TO \"*\"}");
-        test_parse_query_to_ast_helper("weight:>=70", "weight:[\"70\" TO \"*\"}");
-        test_parse_query_to_ast_helper("weight: <70", "weight:{\"*\" TO \"70\"}");
-        test_parse_query_to_ast_helper("weight:<=70", "weight:{\"*\" TO \"70\"]");
-        test_parse_query_to_ast_helper("weight: >60.7", "weight:{\"60.7\" TO \"*\"}");
-
-        test_parse_query_to_ast_helper("weight: <= 70", "weight:{\"*\" TO \"70\"]");
-
-        test_parse_query_to_ast_helper("weight: <= 70.5", "weight:{\"*\" TO \"70.5\"]");
-    }
-
-    #[test]
-    fn test_occur_leaf() {
-        let ((occur, ast), _) = super::occur_leaf().parse("+abc").unwrap();
-        assert_eq!(occur, Some(Occur::Must));
-        assert_eq!(format!("{:?}", ast), "\"abc\"");
-    }
-
-    #[test]
-    fn test_field_name() -> TestParseResult {
-        assert_eq!(
-            super::field().parse("my-field-name:a")?,
-            ("my-field-name".to_string(), "a")
-        );
-        assert_eq!(
-            super::field().parse("my_field_name:a")?,
-            ("my_field_name".to_string(), "a")
-        );
-        assert!(super::field().parse(":a").is_err());
-        assert!(super::field().parse("-my_field:a").is_err());
-        assert_eq!(
-            super::field().parse("_my_field:a")?,
-            ("_my_field".to_string(), "a")
-        );
-        Ok(())
-    }
-
-    #[test]
-    fn test_range_parser() {
-        // testing the range() parser separately
-        let res = range().parse("title: <hello").unwrap().0;
-        let expected = UserInputLeaf::Range {
-            field: Some("title".to_string()),
-            lower: UserInputBound::Unbounded,
-            upper: UserInputBound::Exclusive("hello".to_string()),
-        };
-        let res2 = range().parse("title:{* TO hello}").unwrap().0;
-        assert_eq!(res, expected);
-        assert_eq!(res2, expected);
-        let expected_weight = UserInputLeaf::Range {
-            field: Some("weight".to_string()),
-            lower: UserInputBound::Inclusive("71.2".to_string()),
-            upper: UserInputBound::Unbounded,
-        };
-
-        let res3 = range().parse("weight: >=71.2").unwrap().0;
-        let res4 = range().parse("weight:[71.2 TO *}").unwrap().0;
-        assert_eq!(res3, expected_weight);
-        assert_eq!(res4, expected_weight);
-    }
-
-    #[test]
-    fn test_parse_query_to_triming_spaces() {
-        test_parse_query_to_ast_helper("   abc", "\"abc\"");
-        test_parse_query_to_ast_helper("abc ", "\"abc\"");
-        test_parse_query_to_ast_helper("(  a OR abc)", "(?\"a\" ?\"abc\")");
-        test_parse_query_to_ast_helper("(a  OR abc)", "(?\"a\" ?\"abc\")");
-        test_parse_query_to_ast_helper("(a OR  abc)", "(?\"a\" ?\"abc\")");
-        test_parse_query_to_ast_helper("a OR abc ", "(?\"a\" ?\"abc\")");
-        test_parse_query_to_ast_helper("(a OR abc )", "(?\"a\" ?\"abc\")");
-        test_parse_query_to_ast_helper("(a OR  abc) ", "(?\"a\" ?\"abc\")");
-    }
-
-    #[test]
-    fn test_parse_query_single_term() {
-        test_parse_query_to_ast_helper("abc", "\"abc\"");
-    }
-
-    #[test]
-    fn test_parse_query_default_clause() {
-        test_parse_query_to_ast_helper("a b", "(*\"a\" *\"b\")");
-    }
-
-    #[test]
-    fn test_parse_query_must_default_clause() {
-        test_parse_query_to_ast_helper("+(a b)", "(*\"a\" *\"b\")");
-    }
-
-    #[test]
-    fn test_parse_query_must_single_term() {
-        test_parse_query_to_ast_helper("+d", "\"d\"");
-    }
-
-    #[test]
-    fn test_single_term_with_field() {
-        test_parse_query_to_ast_helper("abc:toto", "abc:\"toto\"");
-    }
-
-    #[test]
-    fn test_single_term_with_float() {
-        test_parse_query_to_ast_helper("abc:1.1", "abc:\"1.1\"");
-    }
-
-    #[test]
-    fn test_must_clause() {
-        test_parse_query_to_ast_helper("(+a +b)", "(+\"a\" +\"b\")");
-    }
-
-    #[test]
-    fn test_parse_test_query_plus_a_b_plus_d() {
-        test_parse_query_to_ast_helper("+(a b) +d", "(+(*\"a\" *\"b\") +\"d\")");
-    }
-
-    #[test]
-    fn test_parse_test_query_other() {
-        test_parse_query_to_ast_helper("(+a +b) d", "(*(+\"a\" +\"b\") *\"d\")");
-        test_parse_query_to_ast_helper("+abc:toto", "abc:\"toto\"");
-        test_parse_query_to_ast_helper("(+abc:toto -titi)", "(+abc:\"toto\" -\"titi\")");
-        test_parse_query_to_ast_helper("-abc:toto", "(-abc:\"toto\")");
-        test_parse_query_to_ast_helper("abc:a b", "(*abc:\"a\" *\"b\")");
-        test_parse_query_to_ast_helper("abc:\"a b\"", "abc:\"a b\"");
-        test_parse_query_to_ast_helper("foo:[1 TO 5]", "foo:[\"1\" TO \"5\"]");
-    }
-
-    #[test]
-    fn test_parse_query_with_range() {
-        test_parse_query_to_ast_helper("[1 TO 5]", "[\"1\" TO \"5\"]");
-        test_parse_query_to_ast_helper("foo:{a TO z}", "foo:{\"a\" TO \"z\"}");
-        test_parse_query_to_ast_helper("foo:[1 TO toto}", "foo:[\"1\" TO \"toto\"}");
-        test_parse_query_to_ast_helper("foo:[* TO toto}", "foo:{\"*\" TO \"toto\"}");
-        test_parse_query_to_ast_helper("foo:[1 TO *}", "foo:[\"1\" TO \"*\"}");
-        test_parse_query_to_ast_helper("foo:[1.1 TO *}", "foo:[\"1.1\" TO \"*\"}");
-        test_is_parse_err("abc +    ");
-    }
-}

run-tests.sh

@@ -1,2 +1,2 @@
 #!/bin/bash
-cargo test
+cargo test --no-default-features --features mmap -- --test-threads 1

src/collector/count_collector.rs

@@ -1,53 +1,68 @@
 use super::Collector;
-use crate::collector::SegmentCollector;
-use crate::DocId;
-use crate::Score;
-use crate::SegmentLocalId;
-use crate::SegmentReader;
+use collector::SegmentCollector;
+use DocId;
+use Result;
+use Score;
+use SegmentLocalId;
+use SegmentReader;
 
 /// `CountCollector` collector only counts how many
 /// documents match the query.
 ///
 /// ```rust
+/// #[macro_use]
+/// extern crate tantivy;
+/// use tantivy::schema::{Schema, TEXT};
+/// use tantivy::{Index, Result};
 /// use tantivy::collector::Count;
 /// use tantivy::query::QueryParser;
-/// use tantivy::schema::{Schema, TEXT};
-/// use tantivy::{doc, Index};
 ///
-/// let mut schema_builder = Schema::builder();
-/// let title = schema_builder.add_text_field("title", TEXT);
-/// let schema = schema_builder.build();
-/// let index = Index::create_in_ram(schema);
+/// # fn main() { example().unwrap(); }
+/// fn example() -> Result<()> {
+///     let mut schema_builder = Schema::builder();
+///     let title = schema_builder.add_text_field("title", TEXT);
+///     let schema = schema_builder.build();
+///     let index = Index::create_in_ram(schema);
+///     {
+///         let mut index_writer = index.writer(3_000_000)?;
+///         index_writer.add_document(doc!(
+///             title => "The Name of the Wind",
+///         ));
+///         index_writer.add_document(doc!(
+///             title => "The Diary of Muadib",
+///         ));
+///         index_writer.add_document(doc!(
+///             title => "A Dairy Cow",
+///         ));
+///         index_writer.add_document(doc!(
+///             title => "The Diary of a Young Girl",
+///         ));
+///         index_writer.commit().unwrap();
+///     }
 ///
-/// let mut index_writer = index.writer(3_000_000).unwrap();
-/// index_writer.add_document(doc!(title => "The Name of the Wind"));
-/// index_writer.add_document(doc!(title => "The Diary of Muadib"));
-/// index_writer.add_document(doc!(title => "A Dairy Cow"));
-/// index_writer.add_document(doc!(title => "The Diary of a Young Girl"));
-/// assert!(index_writer.commit().is_ok());
+///     index.load_searchers()?;
+///     let searcher = index.searcher();
 ///
-/// let reader = index.reader().unwrap();
-/// let searcher = reader.searcher();
+///     {
+///         let query_parser = QueryParser::for_index(&index, vec![title]);
+///         let query = query_parser.parse_query("diary")?;
+///         let count = searcher.search(&query, &Count).unwrap();
 ///
-/// // Here comes the important part
-/// let query_parser = QueryParser::for_index(&index, vec![title]);
-/// let query = query_parser.parse_query("diary").unwrap();
-/// let count = searcher.search(&query, &Count).unwrap();
+///         assert_eq!(count, 2);
+///     }
 ///
-/// assert_eq!(count, 2);
+///     Ok(())
+/// }
 /// ```
 pub struct Count;
 
 impl Collector for Count {
     type Fruit = usize;
+    type SegmentFruit = usize;
 
     type Child = SegmentCountCollector;
 
-    fn for_segment(
-        &self,
-        _: SegmentLocalId,
-        _: &SegmentReader,
-    ) -> crate::Result<SegmentCountCollector> {
+    fn for_segment(&self, _: SegmentLocalId, _: &SegmentReader) -> Result<SegmentCountCollector> {
         Ok(SegmentCountCollector::default())
     }
 
@@ -55,7 +70,7 @@ impl Collector for Count {
         false
     }
 
-    fn merge_fruits(&self, segment_counts: Vec<usize>) -> crate::Result<usize> {
+    fn merge_fruits(&self, segment_counts: Vec<usize>) -> Result<usize> {
         Ok(segment_counts.into_iter().sum())
     }
 }
@@ -80,8 +95,8 @@ impl SegmentCollector for SegmentCountCollector {
 #[cfg(test)]
 mod tests {
     use super::{Count, SegmentCountCollector};
-    use crate::collector::Collector;
-    use crate::collector::SegmentCollector;
+    use collector::Collector;
+    use collector::SegmentCollector;
 
     #[test]
     fn test_count_collect_does_not_requires_scoring() {
@@ -96,19 +111,20 @@ mod tests {
         }
         {
             let mut count_collector = SegmentCountCollector::default();
-            count_collector.collect(0u32, 1.0);
+            count_collector.collect(0u32, 1f32);
             assert_eq!(count_collector.harvest(), 1);
         }
         {
             let mut count_collector = SegmentCountCollector::default();
-            count_collector.collect(0u32, 1.0);
+            count_collector.collect(0u32, 1f32);
             assert_eq!(count_collector.harvest(), 1);
         }
         {
             let mut count_collector = SegmentCountCollector::default();
-            count_collector.collect(0u32, 1.0);
-            count_collector.collect(1u32, 1.0);
+            count_collector.collect(0u32, 1f32);
+            count_collector.collect(1u32, 1f32);
             assert_eq!(count_collector.harvest(), 2);
         }
     }
+
 }

src/collector/custom_score_top_collector.rs

@@ -1,125 +0,0 @@
-use crate::collector::top_collector::{TopCollector, TopSegmentCollector};
-use crate::collector::{Collector, SegmentCollector};
-use crate::{DocAddress, DocId, Score, SegmentReader};
-
-pub(crate) struct CustomScoreTopCollector<TCustomScorer, TScore = Score> {
-    custom_scorer: TCustomScorer,
-    collector: TopCollector<TScore>,
-}
-
-impl<TCustomScorer, TScore> CustomScoreTopCollector<TCustomScorer, TScore>
-where
-    TScore: Clone + PartialOrd,
-{
-    pub(crate) fn new(
-        custom_scorer: TCustomScorer,
-        collector: TopCollector<TScore>,
-    ) -> CustomScoreTopCollector<TCustomScorer, TScore> {
-        CustomScoreTopCollector {
-            custom_scorer,
-            collector,
-        }
-    }
-}
-
-/// A custom segment scorer makes it possible to define any kind of score
-/// for a given document belonging to a specific segment.
-///
-/// It is the segment local version of the [`CustomScorer`](./trait.CustomScorer.html).
-pub trait CustomSegmentScorer<TScore>: 'static {
-    /// Computes the score of a specific `doc`.
-    fn score(&mut self, doc: DocId) -> TScore;
-}
-
-/// `CustomScorer` makes it possible to define any kind of score.
-///
-/// The `CustomerScorer` itself does not make much of the computation itself.
-/// Instead, it helps constructing `Self::Child` instances that will compute
-/// the score at a segment scale.
-pub trait CustomScorer<TScore>: Sync {
-    /// Type of the associated [`CustomSegmentScorer`](./trait.CustomSegmentScorer.html).
-    type Child: CustomSegmentScorer<TScore>;
-    /// Builds a child scorer for a specific segment. The child scorer is associated to
-    /// a specific segment.
-    fn segment_scorer(&self, segment_reader: &SegmentReader) -> crate::Result<Self::Child>;
-}
-
-impl<TCustomScorer, TScore> Collector for CustomScoreTopCollector<TCustomScorer, TScore>
-where
-    TCustomScorer: CustomScorer<TScore> + Send + Sync,
-    TScore: 'static + PartialOrd + Clone + Send + Sync,
-{
-    type Fruit = Vec<(TScore, DocAddress)>;
-
-    type Child = CustomScoreTopSegmentCollector<TCustomScorer::Child, TScore>;
-
-    fn for_segment(
-        &self,
-        segment_local_id: u32,
-        segment_reader: &SegmentReader,
-    ) -> crate::Result<Self::Child> {
-        let segment_scorer = self.custom_scorer.segment_scorer(segment_reader)?;
-        let segment_collector = self
-            .collector
-            .for_segment(segment_local_id, segment_reader)?;
-        Ok(CustomScoreTopSegmentCollector {
-            segment_collector,
-            segment_scorer,
-        })
-    }
-
-    fn requires_scoring(&self) -> bool {
-        false
-    }
-
-    fn merge_fruits(&self, segment_fruits: Vec<Self::Fruit>) -> crate::Result<Self::Fruit> {
-        self.collector.merge_fruits(segment_fruits)
-    }
-}
-
-pub struct CustomScoreTopSegmentCollector<T, TScore>
-where
-    TScore: 'static + PartialOrd + Clone + Send + Sync + Sized,
-    T: CustomSegmentScorer<TScore>,
-{
-    segment_collector: TopSegmentCollector<TScore>,
-    segment_scorer: T,
-}
-
-impl<T, TScore> SegmentCollector for CustomScoreTopSegmentCollector<T, TScore>
-where
-    TScore: 'static + PartialOrd + Clone + Send + Sync,
-    T: 'static + CustomSegmentScorer<TScore>,
-{
-    type Fruit = Vec<(TScore, DocAddress)>;
-
-    fn collect(&mut self, doc: DocId, _score: Score) {
-        let score = self.segment_scorer.score(doc);
-        self.segment_collector.collect(doc, score);
-    }
-
-    fn harvest(self) -> Vec<(TScore, DocAddress)> {
-        self.segment_collector.harvest()
-    }
-}
-
-impl<F, TScore, T> CustomScorer<TScore> for F
-where
-    F: 'static + Send + Sync + Fn(&SegmentReader) -> T,
-    T: CustomSegmentScorer<TScore>,
-{
-    type Child = T;
-
-    fn segment_scorer(&self, segment_reader: &SegmentReader) -> crate::Result<Self::Child> {
-        Ok((self)(segment_reader))
-    }
-}
-
-impl<F, TScore> CustomSegmentScorer<TScore> for F
-where
-    F: 'static + FnMut(DocId) -> TScore,
-{
-    fn score(&mut self, doc: DocId) -> TScore {
-        (self)(doc)
-    }
-}

src/collector/facet_collector.rs

@@ -1,13 +1,9 @@
-use crate::collector::Collector;
-use crate::collector::SegmentCollector;
-use crate::fastfield::FacetReader;
-use crate::schema::Facet;
-use crate::schema::Field;
-use crate::DocId;
-use crate::Score;
-use crate::SegmentLocalId;
-use crate::SegmentReader;
-use crate::TantivyError;
+use collector::Collector;
+use collector::SegmentCollector;
+use docset::SkipResult;
+use fastfield::FacetReader;
+use schema::Facet;
+use schema::Field;
 use std::cmp::Ordering;
 use std::collections::btree_map;
 use std::collections::BTreeMap;
@@ -16,6 +12,11 @@ use std::collections::BinaryHeap;
 use std::collections::Bound;
 use std::iter::Peekable;
 use std::{u64, usize};
+use DocId;
+use Result;
+use Score;
+use SegmentLocalId;
+use SegmentReader;
 
 struct Hit<'a> {
     count: u64,
@@ -25,13 +26,13 @@ struct Hit<'a> {
 impl<'a> Eq for Hit<'a> {}
 
 impl<'a> PartialEq<Hit<'a>> for Hit<'a> {
-    fn eq(&self, other: &Hit<'_>) -> bool {
+    fn eq(&self, other: &Hit) -> bool {
         self.count == other.count
     }
 }
 
 impl<'a> PartialOrd<Hit<'a>> for Hit<'a> {
-    fn partial_cmp(&self, other: &Hit<'_>) -> Option<Ordering> {
+    fn partial_cmp(&self, other: &Hit) -> Option<Ordering> {
         Some(self.cmp(other))
     }
 }
@@ -79,12 +80,15 @@ fn facet_depth(facet_bytes: &[u8]) -> usize {
 ///
 ///
 /// ```rust
+/// #[macro_use]
+/// extern crate tantivy;
+/// use tantivy::schema::{Facet, Schema, TEXT};
+/// use tantivy::{Index, Result};
 /// use tantivy::collector::FacetCollector;
 /// use tantivy::query::AllQuery;
-/// use tantivy::schema::{Facet, Schema, TEXT};
-/// use tantivy::{doc, Index};
 ///
-/// fn example() -> tantivy::Result<()> {
+/// # fn main() { example().unwrap(); }
+/// fn example() -> Result<()> {
 ///     let mut schema_builder = Schema::builder();
 ///
 ///     // Facet have their own specific type.
@@ -118,16 +122,17 @@ fn facet_depth(facet_bytes: &[u8]) -> usize {
 ///             facet => Facet::from("/lang/en"),
 ///             facet => Facet::from("/category/biography")
 ///         ));
-///         index_writer.commit()?;
+///         index_writer.commit().unwrap();
 ///     }
-///     let reader = index.reader()?;
-///     let searcher = reader.searcher();
+///
+///     index.load_searchers()?;
+///     let searcher = index.searcher();
 ///
 ///     {
-///         let mut facet_collector = FacetCollector::for_field(facet);
+///			let mut facet_collector = FacetCollector::for_field(facet);
 ///         facet_collector.add_facet("/lang");
 ///         facet_collector.add_facet("/category");
-///         let facet_counts = searcher.search(&AllQuery, &facet_collector)?;
+///         let facet_counts = searcher.search(&AllQuery, &facet_collector).unwrap();
 ///
 ///         // This lists all of the facet counts
 ///         let facets: Vec<(&Facet, u64)> = facet_counts
@@ -140,9 +145,9 @@ fn facet_depth(facet_bytes: &[u8]) -> usize {
 ///     }
 ///
 ///     {
-///         let mut facet_collector = FacetCollector::for_field(facet);
+///			let mut facet_collector = FacetCollector::for_field(facet);
 ///         facet_collector.add_facet("/category/fiction");
-///         let facet_counts = searcher.search(&AllQuery, &facet_collector)?;
+///         let facet_counts = searcher.search(&AllQuery, &facet_collector).unwrap();
 ///
 ///         // This lists all of the facet counts
 ///         let facets: Vec<(&Facet, u64)> = facet_counts
@@ -155,10 +160,10 @@ fn facet_depth(facet_bytes: &[u8]) -> usize {
 ///         ]);
 ///     }
 ///
-///     {
-///         let mut facet_collector = FacetCollector::for_field(facet);
+///    {
+///			let mut facet_collector = FacetCollector::for_field(facet);
 ///         facet_collector.add_facet("/category/fiction");
-///         let facet_counts = searcher.search(&AllQuery, &facet_collector)?;
+///         let facet_counts = searcher.search(&AllQuery, &facet_collector).unwrap();
 ///
 ///         // This lists all of the facet counts
 ///         let facets: Vec<(&Facet, u64)> = facet_counts.top_k("/category/fiction", 1);
@@ -169,7 +174,6 @@ fn facet_depth(facet_bytes: &[u8]) -> usize {
 ///
 ///     Ok(())
 /// }
-/// # assert!(example().is_ok());
 /// ```
 pub struct FacetCollector {
     field: Field,
@@ -187,28 +191,23 @@ pub struct FacetSegmentCollector {
     collapse_facet_ords: Vec<u64>,
 }
 
-enum SkipResult {
-    Found,
-    NotFound,
-}
-
 fn skip<'a, I: Iterator<Item = &'a Facet>>(
     target: &[u8],
     collapse_it: &mut Peekable<I>,
 ) -> SkipResult {
     loop {
         match collapse_it.peek() {
-            Some(facet_bytes) => match facet_bytes.encoded_str().as_bytes().cmp(target) {
+            Some(facet_bytes) => match facet_bytes.encoded_bytes().cmp(target) {
                 Ordering::Less => {}
                 Ordering::Greater => {
-                    return SkipResult::NotFound;
+                    return SkipResult::OverStep;
                 }
                 Ordering::Equal => {
-                    return SkipResult::Found;
+                    return SkipResult::Reached;
                 }
             },
             None => {
-                return SkipResult::NotFound;
+                return SkipResult::End;
             }
         }
         collapse_it.next();
@@ -259,17 +258,16 @@ impl FacetCollector {
 impl Collector for FacetCollector {
     type Fruit = FacetCounts;
 
+    type SegmentFruit = FacetCounts;
+
     type Child = FacetSegmentCollector;
 
     fn for_segment(
         &self,
         _: SegmentLocalId,
         reader: &SegmentReader,
-    ) -> crate::Result<FacetSegmentCollector> {
-        let field_name = reader.schema().get_field_name(self.field);
-        let facet_reader = reader.facet_reader(self.field).ok_or_else(|| {
-            TantivyError::SchemaError(format!("Field {:?} is not a facet field.", field_name))
-        })?;
+    ) -> Result<FacetSegmentCollector> {
+        let facet_reader = reader.facet_reader(self.field)?;
 
         let mut collapse_mapping = Vec::new();
         let mut counts = Vec::new();
@@ -285,7 +283,7 @@ impl Collector for FacetCollector {
                     // is positionned on a term that has not been processed yet.
                     let skip_result = skip(facet_streamer.key(), &mut collapse_facet_it);
                     match skip_result {
-                        SkipResult::Found => {
+                        SkipResult::Reached => {
                             // we reach a facet we decided to collapse.
                             let collapse_depth = facet_depth(facet_streamer.key());
                             let mut collapsed_id = 0;
@@ -305,7 +303,7 @@ impl Collector for FacetCollector {
                             }
                             break;
                         }
-                        SkipResult::NotFound => {
+                        SkipResult::End | SkipResult::OverStep => {
                             collapse_mapping.push(0);
                             if !facet_streamer.advance() {
                                 break;
@@ -331,7 +329,7 @@ impl Collector for FacetCollector {
         false
     }
 
-    fn merge_fruits(&self, segments_facet_counts: Vec<FacetCounts>) -> crate::Result<FacetCounts> {
+    fn merge_fruits(&self, segments_facet_counts: Vec<FacetCounts>) -> Result<FacetCounts> {
         let mut facet_counts: BTreeMap<Facet, u64> = BTreeMap::new();
         for segment_facet_counts in segments_facet_counts {
             for (facet, count) in segment_facet_counts.facet_counts {
@@ -373,8 +371,7 @@ impl SegmentCollector for FacetSegmentCollector {
             let mut facet = vec![];
             let facet_ord = self.collapse_facet_ords[collapsed_facet_ord];
             facet_dict.ord_to_term(facet_ord as u64, &mut facet);
-            // TODO
-            facet_counts.insert(Facet::from_encoded(facet).unwrap(), count);
+            facet_counts.insert(unsafe { Facet::from_encoded(facet) }, count);
         }
         FacetCounts { facet_counts }
     }
@@ -399,7 +396,7 @@ impl<'a> Iterator for FacetChildIterator<'a> {
 }
 
 impl FacetCounts {
-    pub fn get<T>(&self, facet_from: T) -> FacetChildIterator<'_>
+    pub fn get<T>(&self, facet_from: T) -> FacetChildIterator
     where
         Facet: From<T>,
     {
@@ -408,13 +405,12 @@ impl FacetCounts {
         let right_bound = if facet.is_root() {
             Bound::Unbounded
         } else {
-            let mut facet_after_bytes: String = facet.encoded_str().to_owned();
-            facet_after_bytes.push('\u{1}');
-            let facet_after = Facet::from_encoded_string(facet_after_bytes);
+            let mut facet_after_bytes: Vec<u8> = facet.encoded_bytes().to_owned();
+            facet_after_bytes.push(1u8);
+            let facet_after = unsafe { Facet::from_encoded(facet_after_bytes) }; // ok logic
             Bound::Excluded(facet_after)
         };
-        let underlying: btree_map::Range<'_, _, _> =
-            self.facet_counts.range((left_bound, right_bound));
+        let underlying: btree_map::Range<_, _> = self.facet_counts.range((left_bound, right_bound));
         FacetChildIterator { underlying }
     }
 
@@ -455,14 +451,12 @@ impl FacetCounts {
 #[cfg(test)]
 mod tests {
     use super::{FacetCollector, FacetCounts};
-    use crate::collector::Count;
-    use crate::core::Index;
-    use crate::query::{AllQuery, QueryParser, TermQuery};
-    use crate::schema::{Document, Facet, Field, IndexRecordOption, Schema};
-    use crate::Term;
+    use core::Index;
+    use query::AllQuery;
     use rand::distributions::Uniform;
     use rand::prelude::SliceRandom;
     use rand::{thread_rng, Rng};
+    use schema::{Document, Facet, Field, Schema};
     use std::iter;
 
     #[test]
@@ -472,7 +466,7 @@ mod tests {
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
 
-        let mut index_writer = index.writer_for_tests().unwrap();
+        let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
         let num_facets: usize = 3 * 4 * 5;
         let facets: Vec<Facet> = (0..num_facets)
             .map(|mut n| {
@@ -490,8 +484,8 @@ mod tests {
             index_writer.add_document(doc);
         }
         index_writer.commit().unwrap();
-        let reader = index.reader().unwrap();
-        let searcher = reader.searcher();
+        index.load_searchers().unwrap();
+        let searcher = index.searcher();
         let mut facet_collector = FacetCollector::for_field(facet_field);
         facet_collector.add_facet(Facet::from("/top1"));
         let counts = searcher.search(&AllQuery, &facet_collector).unwrap();
@@ -520,7 +514,7 @@ mod tests {
     #[should_panic(expected = "Tried to add a facet which is a descendant of \
                                an already added facet.")]
     fn test_misused_facet_collector() {
-        let mut facet_collector = FacetCollector::for_field(Field::from_field_id(0));
+        let mut facet_collector = FacetCollector::for_field(Field(0));
         facet_collector.add_facet(Facet::from("/country"));
         facet_collector.add_facet(Facet::from("/country/europe"));
     }
@@ -531,7 +525,7 @@ mod tests {
         let facet_field = schema_builder.add_facet_field("facets");
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_for_tests().unwrap();
+        let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
         index_writer.add_document(doc!(
             facet_field => Facet::from_text(&"/subjects/A/a"),
             facet_field => Facet::from_text(&"/subjects/B/a"),
@@ -539,8 +533,8 @@ mod tests {
             facet_field => Facet::from_text(&"/subjects/B/b"),
         ));
         index_writer.commit().unwrap();
-        let reader = index.reader().unwrap();
-        let searcher = reader.searcher();
+        index.load_searchers().unwrap();
+        let searcher = index.searcher();
         assert_eq!(searcher.num_docs(), 1);
         let mut facet_collector = FacetCollector::for_field(facet_field);
         facet_collector.add_facet("/subjects");
@@ -549,59 +543,9 @@ mod tests {
         assert_eq!(facets[0].1, 1);
     }
 
-    #[test]
-    fn test_doc_search_by_facet() -> crate::Result<()> {
-        let mut schema_builder = Schema::builder();
-        let facet_field = schema_builder.add_facet_field("facet");
-        let schema = schema_builder.build();
-        let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_for_tests()?;
-        index_writer.add_document(doc!(
-            facet_field => Facet::from_text(&"/A/A"),
-        ));
-        index_writer.add_document(doc!(
-            facet_field => Facet::from_text(&"/A/B"),
-        ));
-        index_writer.add_document(doc!(
-            facet_field => Facet::from_text(&"/A/C/A"),
-        ));
-        index_writer.add_document(doc!(
-            facet_field => Facet::from_text(&"/D/C/A"),
-        ));
-        index_writer.commit()?;
-        let reader = index.reader()?;
-        let searcher = reader.searcher();
-        assert_eq!(searcher.num_docs(), 4);
-
-        let count_facet = |facet_str: &str| {
-            let term = Term::from_facet(facet_field, &Facet::from_text(facet_str));
-            searcher
-                .search(&TermQuery::new(term, IndexRecordOption::Basic), &Count)
-                .unwrap()
-        };
-
-        assert_eq!(count_facet("/"), 4);
-        assert_eq!(count_facet("/A"), 3);
-        assert_eq!(count_facet("/A/B"), 1);
-        assert_eq!(count_facet("/A/C"), 1);
-        assert_eq!(count_facet("/A/C/A"), 1);
-        assert_eq!(count_facet("/C/A"), 0);
-
-        let query_parser = QueryParser::for_index(&index, vec![]);
-        {
-            let query = query_parser.parse_query("facet:/A/B")?;
-            assert_eq!(1, searcher.search(&query, &Count).unwrap());
-        }
-        {
-            let query = query_parser.parse_query("facet:/A")?;
-            assert_eq!(3, searcher.search(&query, &Count)?);
-        }
-        Ok(())
-    }
-
     #[test]
     fn test_non_used_facet_collector() {
-        let mut facet_collector = FacetCollector::for_field(Field::from_field_id(0));
+        let mut facet_collector = FacetCollector::for_field(Field(0));
         facet_collector.add_facet(Facet::from("/country"));
         facet_collector.add_facet(Facet::from("/countryeurope"));
     }
@@ -631,12 +575,14 @@ mod tests {
             .collect();
         docs[..].shuffle(&mut thread_rng());
 
-        let mut index_writer = index.writer_for_tests().unwrap();
+        let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
         for doc in docs {
             index_writer.add_document(doc);
         }
         index_writer.commit().unwrap();
-        let searcher = index.reader().unwrap().searcher();
+        index.load_searchers().unwrap();
+
+        let searcher = index.searcher();
 
         let mut facet_collector = FacetCollector::for_field(facet_field);
         facet_collector.add_facet("/facet");
@@ -654,18 +600,19 @@ mod tests {
             );
         }
     }
+
 }
 
 #[cfg(all(test, feature = "unstable"))]
 mod bench {
 
-    use crate::collector::FacetCollector;
-    use crate::query::AllQuery;
-    use crate::schema::{Facet, Schema};
-    use crate::Index;
-    use rand::seq::SliceRandom;
-    use rand::thread_rng;
+    use collector::FacetCollector;
+    use query::AllQuery;
+    use rand::{thread_rng, Rng};
+    use schema::Facet;
+    use schema::Schema;
     use test::Bencher;
+    use Index;
 
     #[bench]
     fn bench_facet_collector(b: &mut Bencher) {
@@ -682,16 +629,17 @@ mod bench {
             }
         }
         // 40425 docs
-        docs[..].shuffle(&mut thread_rng());
+        thread_rng().shuffle(&mut docs[..]);
 
-        let mut index_writer = index.writer_for_tests().unwrap();
+        let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
         for doc in docs {
             index_writer.add_document(doc);
         }
         index_writer.commit().unwrap();
-        let reader = index.reader().unwrap();
+        index.load_searchers().unwrap();
+
         b.iter(|| {
-            let searcher = reader.searcher();
+            let searcher = index.searcher();
             let facet_collector = FacetCollector::for_field(facet_field);
             searcher.search(&AllQuery, &facet_collector).unwrap();
         });

src/collector/int_facet_collector.rs

@@ -82,20 +82,18 @@ mod tests {
         let mut schema_builder = schema::Schema::builder();
         let num_field_i64 = schema_builder.add_i64_field("num_i64", FAST);
         let num_field_u64 = schema_builder.add_u64_field("num_u64", FAST);
-        let num_field_f64 = schema_builder.add_f64_field("num_f64", FAST);
         let text_field = schema_builder.add_text_field("text", STRING);
         let schema = schema_builder.build();
 
         let index = Index::create_in_ram(schema.clone());
 
         {
-            let mut index_writer = index.writer_for_tests().unwrap();
+            let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
             {
                 for i in 0u64..10u64 {
                     index_writer.add_document(doc!(
                         num_field_i64 => ((i as i64) % 3i64) as i64,
                         num_field_u64 => (i % 2u64) as u64,
-                        num_field_f64 => (i % 4u64) as f64,
                         text_field => "text"
                     ));
                 }
@@ -103,14 +101,14 @@ mod tests {
             assert_eq!(index_writer.commit().unwrap(), 10u64);
         }
 
-        let searcher = index.reader().searcher();
+        index.load_searchers().unwrap();
+        let searcher = index.searcher();
         let mut ffvf_i64: IntFacetCollector<I64FastFieldReader> = IntFacetCollector::new(num_field_i64);
         let mut ffvf_u64: IntFacetCollector<U64FastFieldReader> = IntFacetCollector::new(num_field_u64);
-        let mut ffvf_f64: IntFacetCollector<F64FastFieldReader> = IntFacetCollector::new(num_field_f64);
 
         {
             // perform the query
-            let mut facet_collectors = chain().push(&mut ffvf_i64).push(&mut ffvf_u64).push(&mut ffvf_f64);
+            let mut facet_collectors = chain().push(&mut ffvf_i64).push(&mut ffvf_u64);
             let mut query_parser = QueryParser::for_index(index, vec![text_field]);
             let query = query_parser.parse_query("text:text").unwrap();
             query.search(&searcher, &mut facet_collectors).unwrap();
@@ -120,8 +118,6 @@ mod tests {
         assert_eq!(ffvf_u64.counters[&1], 5);
         assert_eq!(ffvf_i64.counters[&0], 4);
         assert_eq!(ffvf_i64.counters[&1], 3);
-        assert_eq!(ffvf_f64.counters[&0.0], 3);
-        assert_eq!(ffvf_f64.counters[&2.0], 2);
 
     }
 }

src/collector/mod.rs

@@ -35,6 +35,7 @@ The resulting `Fruit` will then be a typed tuple with each collector's original
 in their respective position.
 
 ```rust
+# extern crate tantivy;
 # use tantivy::schema::*;
 # use tantivy::*;
 # use tantivy::query::*;
@@ -52,9 +53,9 @@ use tantivy::collector::{Count, TopDocs};
 #     index_writer.add_document(doc!(
 #        title => "The Diary of Muadib",
 #     ));
-#     index_writer.commit()?;
-#     let reader = index.reader()?;
-#     let searcher = reader.searcher();
+#     index_writer.commit().unwrap();
+#     index.load_searchers()?;
+#     let searcher = index.searcher();
 #     let query_parser = QueryParser::for_index(&index, vec![title]);
 #     let query = query_parser.parse_query("diary")?;
 let (doc_count, top_docs): (usize, Vec<(Score, DocAddress)>) =
@@ -65,7 +66,7 @@ let (doc_count, top_docs): (usize, Vec<(Score, DocAddress)>) =
 
 The `Collector` trait is implemented for up to 4 collectors.
 If you have more than 4 collectors, you can either group them into
-tuples of tuples `(a,(b,(c,d)))`, or rely on [`MultiCollector`](./struct.MultiCollector.html).
+tuples of tuples `(a,(b,(c,d)))`, or rely on `MultiCollector`'s.
 
 # Combining several collectors dynamically
 
@@ -84,11 +85,12 @@ See the `custom_collector` example.
 
 */
 
-use crate::DocId;
-use crate::Score;
-use crate::SegmentLocalId;
-use crate::SegmentReader;
-use downcast_rs::impl_downcast;
+use downcast;
+use DocId;
+use Result;
+use Score;
+use SegmentLocalId;
+use SegmentReader;
 
 mod count_collector;
 pub use self::count_collector::Count;
@@ -101,21 +103,17 @@ mod top_collector;
 mod top_score_collector;
 pub use self::top_score_collector::TopDocs;
 
-mod custom_score_top_collector;
-pub use self::custom_score_top_collector::{CustomScorer, CustomSegmentScorer};
-
-mod tweak_score_top_collector;
-pub use self::tweak_score_top_collector::{ScoreSegmentTweaker, ScoreTweaker};
+mod top_field_collector;
+pub use self::top_field_collector::TopDocsByField;
 
 mod facet_collector;
 pub use self::facet_collector::FacetCollector;
-use crate::query::Weight;
 
 /// `Fruit` is the type for the result of our collection.
 /// e.g. `usize` for the `Count` collector.
-pub trait Fruit: Send + downcast_rs::Downcast {}
+pub trait Fruit: Send + downcast::Any {}
 
-impl<T> Fruit for T where T: Send + downcast_rs::Downcast {}
+impl<T> Fruit for T where T: Send + downcast::Any {}
 
 /// Collectors are in charge of collecting and retaining relevant
 /// information from the document found and scored by the query.
@@ -133,13 +131,15 @@ impl<T> Fruit for T where T: Send + downcast_rs::Downcast {}
 /// The collection logic itself is in the `SegmentCollector`.
 ///
 /// Segments are not guaranteed to be visited in any specific order.
-pub trait Collector: Sync + Send {
+pub trait Collector: Sync {
     /// `Fruit` is the type for the result of our collection.
     /// e.g. `usize` for the `Count` collector.
     type Fruit: Fruit;
 
+    type SegmentFruit: Fruit;
+
     /// Type of the `SegmentCollector` associated to this collector.
-    type Child: SegmentCollector<Fruit = Self::Fruit>;
+    type Child: SegmentCollector<Fruit = Self::SegmentFruit>;
 
     /// `set_segment` is called before beginning to enumerate
     /// on this segment.
@@ -147,37 +147,14 @@ pub trait Collector: Sync + Send {
         &self,
         segment_local_id: SegmentLocalId,
         segment: &SegmentReader,
-    ) -> crate::Result<Self::Child>;
+    ) -> Result<Self::Child>;
 
     /// Returns true iff the collector requires to compute scores for documents.
     fn requires_scoring(&self) -> bool;
 
     /// Combines the fruit associated to the collection of each segments
     /// into one fruit.
-    fn merge_fruits(&self, segment_fruits: Vec<Self::Fruit>) -> crate::Result<Self::Fruit>;
-
-    /// Created a segment collector and
-    fn collect_segment(
-        &self,
-        weight: &dyn Weight,
-        segment_ord: u32,
-        reader: &SegmentReader,
-    ) -> crate::Result<<Self::Child as SegmentCollector>::Fruit> {
-        let mut segment_collector = self.for_segment(segment_ord as u32, reader)?;
-
-        if let Some(delete_bitset) = reader.delete_bitset() {
-            weight.for_each(reader, &mut |doc, score| {
-                if delete_bitset.is_alive(doc) {
-                    segment_collector.collect(doc, score);
-                }
-            })?;
-        } else {
-            weight.for_each(reader, &mut |doc, score| {
-                segment_collector.collect(doc, score);
-            })?;
-        }
-        Ok(segment_collector.harvest())
-    }
+    fn merge_fruits(&self, segment_fruits: Vec<Self::SegmentFruit>) -> Result<Self::Fruit>;
 }
 
 /// The `SegmentCollector` is the trait in charge of defining the
@@ -206,13 +183,12 @@ where
     Right: Collector,
 {
     type Fruit = (Left::Fruit, Right::Fruit);
+
+    type SegmentFruit = (Left::SegmentFruit, Right::SegmentFruit);
+
     type Child = (Left::Child, Right::Child);
 
-    fn for_segment(
-        &self,
-        segment_local_id: u32,
-        segment: &SegmentReader,
-    ) -> crate::Result<Self::Child> {
+    fn for_segment(&self, segment_local_id: u32, segment: &SegmentReader) -> Result<Self::Child> {
         let left = self.0.for_segment(segment_local_id, segment)?;
         let right = self.1.for_segment(segment_local_id, segment)?;
         Ok((left, right))
@@ -224,8 +200,8 @@ where
 
     fn merge_fruits(
         &self,
-        children: Vec<(Left::Fruit, Right::Fruit)>,
-    ) -> crate::Result<(Left::Fruit, Right::Fruit)> {
+        children: Vec<(Left::SegmentFruit, Right::SegmentFruit)>,
+    ) -> Result<(Left::Fruit, Right::Fruit)> {
         let mut left_fruits = vec![];
         let mut right_fruits = vec![];
         for (left_fruit, right_fruit) in children {
@@ -265,13 +241,10 @@ where
     Three: Collector,
 {
     type Fruit = (One::Fruit, Two::Fruit, Three::Fruit);
+    type SegmentFruit = (One::SegmentFruit, Two::SegmentFruit, Three::SegmentFruit);
     type Child = (One::Child, Two::Child, Three::Child);
 
-    fn for_segment(
-        &self,
-        segment_local_id: u32,
-        segment: &SegmentReader,
-    ) -> crate::Result<Self::Child> {
+    fn for_segment(&self, segment_local_id: u32, segment: &SegmentReader) -> Result<Self::Child> {
         let one = self.0.for_segment(segment_local_id, segment)?;
         let two = self.1.for_segment(segment_local_id, segment)?;
         let three = self.2.for_segment(segment_local_id, segment)?;
@@ -282,7 +255,7 @@ where
         self.0.requires_scoring() || self.1.requires_scoring() || self.2.requires_scoring()
     }
 
-    fn merge_fruits(&self, children: Vec<Self::Fruit>) -> crate::Result<Self::Fruit> {
+    fn merge_fruits(&self, children: Vec<Self::SegmentFruit>) -> Result<Self::Fruit> {
         let mut one_fruits = vec![];
         let mut two_fruits = vec![];
         let mut three_fruits = vec![];
@@ -328,13 +301,10 @@ where
     Four: Collector,
 {
     type Fruit = (One::Fruit, Two::Fruit, Three::Fruit, Four::Fruit);
+    type SegmentFruit = (One::SegmentFruit, Two::SegmentFruit, Three::SegmentFruit, Four::SegmentFruit);
     type Child = (One::Child, Two::Child, Three::Child, Four::Child);
 
-    fn for_segment(
-        &self,
-        segment_local_id: u32,
-        segment: &SegmentReader,
-    ) -> crate::Result<Self::Child> {
+    fn for_segment(&self, segment_local_id: u32, segment: &SegmentReader) -> Result<Self::Child> {
         let one = self.0.for_segment(segment_local_id, segment)?;
         let two = self.1.for_segment(segment_local_id, segment)?;
         let three = self.2.for_segment(segment_local_id, segment)?;
@@ -349,7 +319,7 @@ where
             || self.3.requires_scoring()
     }
 
-    fn merge_fruits(&self, children: Vec<Self::Fruit>) -> crate::Result<Self::Fruit> {
+    fn merge_fruits(&self, children: Vec<Self::SegmentFruit>) -> Result<Self::Fruit> {
         let mut one_fruits = vec![];
         let mut two_fruits = vec![];
         let mut three_fruits = vec![];
@@ -395,7 +365,10 @@ where
     }
 }
 
-impl_downcast!(Fruit);
+#[allow(missing_docs)]
+mod downcast_impl {
+    downcast!(super::Fruit);
+}
 
 #[cfg(test)]
 pub mod tests;

src/collector/multi_collector.rs

@@ -1,29 +1,31 @@
 use super::Collector;
 use super::SegmentCollector;
-use crate::collector::Fruit;
-use crate::DocId;
-use crate::Score;
-use crate::SegmentLocalId;
-use crate::SegmentReader;
-use crate::TantivyError;
+use collector::Fruit;
+use downcast::Downcast;
 use std::marker::PhantomData;
-use std::ops::Deref;
+use DocId;
+use Result;
+use Score;
+use SegmentLocalId;
+use SegmentReader;
+use TantivyError;
 
 pub struct MultiFruit {
-    sub_fruits: Vec<Option<Box<dyn Fruit>>>,
+    sub_fruits: Vec<Option<Box<Fruit>>>,
 }
 
 pub struct CollectorWrapper<TCollector: Collector>(TCollector);
 
 impl<TCollector: Collector> Collector for CollectorWrapper<TCollector> {
-    type Fruit = Box<dyn Fruit>;
-    type Child = Box<dyn BoxableSegmentCollector>;
+    type Fruit = Box<Fruit>;
+    type SegmentFruit = Box<Fruit>;
+    type Child = Box<BoxableSegmentCollector>;
 
     fn for_segment(
         &self,
         segment_local_id: u32,
         reader: &SegmentReader,
-    ) -> crate::Result<Box<dyn BoxableSegmentCollector>> {
+    ) -> Result<Box<BoxableSegmentCollector>> {
         let child = self.0.for_segment(segment_local_id, reader)?;
         Ok(Box::new(SegmentCollectorWrapper(child)))
     }
@@ -32,41 +34,38 @@ impl<TCollector: Collector> Collector for CollectorWrapper<TCollector> {
         self.0.requires_scoring()
     }
 
-    fn merge_fruits(
-        &self,
-        children: Vec<<Self as Collector>::Fruit>,
-    ) -> crate::Result<Box<dyn Fruit>> {
-        let typed_fruit: Vec<TCollector::Fruit> = children
+    fn merge_fruits(&self, children: Vec<<Self as Collector>::Fruit>) -> Result<Box<Fruit>> {
+        let typed_fruit: Vec<TCollector::SegmentFruit> = children
             .into_iter()
             .map(|untyped_fruit| {
-                untyped_fruit
-                    .downcast::<TCollector::Fruit>()
+                Downcast::<TCollector::SegmentFruit>::downcast(untyped_fruit)
                     .map(|boxed_but_typed| *boxed_but_typed)
-                    .map_err(|_| {
-                        TantivyError::InvalidArgument("Failed to cast child fruit.".to_string())
+                    .map_err(|e| {
+                        let err_msg = format!("Failed to cast child collector fruit. {:?}", e);
+                        TantivyError::InvalidArgument(err_msg)
                     })
             })
-            .collect::<crate::Result<_>>()?;
+            .collect::<Result<_>>()?;
         let merged_fruit = self.0.merge_fruits(typed_fruit)?;
         Ok(Box::new(merged_fruit))
     }
 }
 
-impl SegmentCollector for Box<dyn BoxableSegmentCollector> {
-    type Fruit = Box<dyn Fruit>;
+impl SegmentCollector for Box<BoxableSegmentCollector> {
+    type Fruit = Box<Fruit>;
 
-    fn collect(&mut self, doc: u32, score: Score) {
+    fn collect(&mut self, doc: u32, score: f32) {
         self.as_mut().collect(doc, score);
     }
 
-    fn harvest(self) -> Box<dyn Fruit> {
+    fn harvest(self) -> Box<Fruit> {
         BoxableSegmentCollector::harvest_from_box(self)
     }
 }
 
 pub trait BoxableSegmentCollector {
-    fn collect(&mut self, doc: u32, score: Score);
-    fn harvest_from_box(self: Box<Self>) -> Box<dyn Fruit>;
+    fn collect(&mut self, doc: u32, score: f32);
+    fn harvest_from_box(self: Box<Self>) -> Box<Fruit>;
 }
 
 pub struct SegmentCollectorWrapper<TSegmentCollector: SegmentCollector>(TSegmentCollector);
@@ -74,11 +73,11 @@ pub struct SegmentCollectorWrapper<TSegmentCollector: SegmentCollector>(TSegment
 impl<TSegmentCollector: SegmentCollector> BoxableSegmentCollector
     for SegmentCollectorWrapper<TSegmentCollector>
 {
-    fn collect(&mut self, doc: u32, score: Score) {
+    fn collect(&mut self, doc: u32, score: f32) {
         self.0.collect(doc, score);
     }
 
-    fn harvest_from_box(self: Box<Self>) -> Box<dyn Fruit> {
+    fn harvest_from_box(self: Box<Self>) -> Box<Fruit> {
         Box::new(self.0.harvest())
     }
 }
@@ -91,61 +90,70 @@ pub struct FruitHandle<TFruit: Fruit> {
 impl<TFruit: Fruit> FruitHandle<TFruit> {
     pub fn extract(self, fruits: &mut MultiFruit) -> TFruit {
         let boxed_fruit = fruits.sub_fruits[self.pos].take().expect("");
-        *boxed_fruit
-            .downcast::<TFruit>()
-            .map_err(|_| ())
-            .expect("Failed to downcast collector fruit.")
+        *Downcast::<TFruit>::downcast(boxed_fruit).expect("Failed")
     }
 }
 
 /// Multicollector makes it possible to collect on more than one collector.
 /// It should only be used for use cases where the Collector types is unknown
 /// at compile time.
-///
-/// If the type of the collectors is known, you can just group yours collectors
-/// in a tuple. See the
-/// [Combining several collectors section of the collector documentation](./index.html#combining-several-collectors).
+/// If the type of the collectors is known, you should prefer to use `ChainedCollector`.
 ///
 /// ```rust
+/// #[macro_use]
+/// extern crate tantivy;
+/// use tantivy::schema::{Schema, TEXT};
+/// use tantivy::{Index, Result};
 /// use tantivy::collector::{Count, TopDocs, MultiCollector};
 /// use tantivy::query::QueryParser;
-/// use tantivy::schema::{Schema, TEXT};
-/// use tantivy::{doc, Index};
 ///
-/// let mut schema_builder = Schema::builder();
-/// let title = schema_builder.add_text_field("title", TEXT);
-/// let schema = schema_builder.build();
-/// let index = Index::create_in_ram(schema);
+/// # fn main() { example().unwrap(); }
+/// fn example() -> Result<()> {
+///     let mut schema_builder = Schema::builder();
+///     let title = schema_builder.add_text_field("title", TEXT);
+///     let schema = schema_builder.build();
+///     let index = Index::create_in_ram(schema);
+///     {
+///         let mut index_writer = index.writer(3_000_000)?;
+///         index_writer.add_document(doc!(
+///             title => "The Name of the Wind",
+///         ));
+///         index_writer.add_document(doc!(
+///             title => "The Diary of Muadib",
+///         ));
+///         index_writer.add_document(doc!(
+///             title => "A Dairy Cow",
+///         ));
+///         index_writer.add_document(doc!(
+///             title => "The Diary of a Young Girl",
+///         ));
+///         index_writer.commit().unwrap();
+///     }
 ///
-/// let mut index_writer = index.writer(3_000_000).unwrap();
-/// index_writer.add_document(doc!(title => "The Name of the Wind"));
-/// index_writer.add_document(doc!(title => "The Diary of Muadib"));
-/// index_writer.add_document(doc!(title => "A Dairy Cow"));
-/// index_writer.add_document(doc!(title => "The Diary of a Young Girl"));
-/// assert!(index_writer.commit().is_ok());
+///     index.load_searchers()?;
+///     let searcher = index.searcher();
 ///
-/// let reader = index.reader().unwrap();
-/// let searcher = reader.searcher();
+///     let mut collectors = MultiCollector::new();
+///     let top_docs_handle = collectors.add_collector(TopDocs::with_limit(2));
+///     let count_handle = collectors.add_collector(Count);
+///     let query_parser = QueryParser::for_index(&index, vec![title]);
+///     let query = query_parser.parse_query("diary")?;
+///     let mut multi_fruit = searcher.search(&query, &collectors)?;
 ///
-/// let mut collectors = MultiCollector::new();
-/// let top_docs_handle = collectors.add_collector(TopDocs::with_limit(2));
-/// let count_handle = collectors.add_collector(Count);
-/// let query_parser = QueryParser::for_index(&index, vec![title]);
-/// let query = query_parser.parse_query("diary").unwrap();
-/// let mut multi_fruit = searcher.search(&query, &collectors).unwrap();
+///     let count = count_handle.extract(&mut multi_fruit);
+///     let top_docs = top_docs_handle.extract(&mut multi_fruit);
 ///
-/// let count = count_handle.extract(&mut multi_fruit);
-/// let top_docs = top_docs_handle.extract(&mut multi_fruit);
+///     # assert_eq!(count, 2);
+///     # assert_eq!(top_docs.len(), 2);
 ///
-/// assert_eq!(count, 2);
-/// assert_eq!(top_docs.len(), 2);
+///     Ok(())
+/// }
 /// ```
 #[allow(clippy::type_complexity)]
 #[derive(Default)]
 pub struct MultiCollector<'a> {
-    collector_wrappers: Vec<
-        Box<dyn Collector<Child = Box<dyn BoxableSegmentCollector>, Fruit = Box<dyn Fruit>> + 'a>,
-    >,
+    collector_wrappers:
+        Vec<Box<Collector<Child = Box<BoxableSegmentCollector>, Fruit = Box<Fruit>, SegmentFruit = Box<Fruit>> + 'a>>,
 }
 
 impl<'a> MultiCollector<'a> {
@@ -170,31 +178,30 @@ impl<'a> MultiCollector<'a> {
 }
 
 impl<'a> Collector for MultiCollector<'a> {
+
     type Fruit = MultiFruit;
+    type SegmentFruit = MultiFruit;
     type Child = MultiCollectorChild;
 
     fn for_segment(
         &self,
         segment_local_id: SegmentLocalId,
         segment: &SegmentReader,
-    ) -> crate::Result<MultiCollectorChild> {
+    ) -> Result<MultiCollectorChild> {
         let children = self
             .collector_wrappers
             .iter()
             .map(|collector_wrapper| collector_wrapper.for_segment(segment_local_id, segment))
-            .collect::<crate::Result<Vec<_>>>()?;
+            .collect::<Result<Vec<_>>>()?;
         Ok(MultiCollectorChild { children })
     }
 
     fn requires_scoring(&self) -> bool {
-        self.collector_wrappers
-            .iter()
-            .map(Deref::deref)
-            .any(Collector::requires_scoring)
+        self.collector_wrappers.iter().any(|c| c.requires_scoring())
     }
 
-    fn merge_fruits(&self, segments_multifruits: Vec<MultiFruit>) -> crate::Result<MultiFruit> {
-        let mut segment_fruits_list: Vec<Vec<Box<dyn Fruit>>> = (0..self.collector_wrappers.len())
+    fn merge_fruits(&self, segments_multifruits: Vec<MultiFruit>) -> Result<MultiFruit> {
+        let mut segment_fruits_list: Vec<Vec<Box<Fruit>>> = (0..self.collector_wrappers.len())
             .map(|_| Vec::with_capacity(segments_multifruits.len()))
             .collect::<Vec<_>>();
         for segment_multifruit in segments_multifruits {
@@ -211,13 +218,13 @@ impl<'a> Collector for MultiCollector<'a> {
             .map(|(child_collector, segment_fruits)| {
                 Ok(Some(child_collector.merge_fruits(segment_fruits)?))
             })
-            .collect::<crate::Result<_>>()?;
+            .collect::<Result<_>>()?;
         Ok(MultiFruit { sub_fruits })
     }
 }
 
 pub struct MultiCollectorChild {
-    children: Vec<Box<dyn BoxableSegmentCollector>>,
+    children: Vec<Box<BoxableSegmentCollector>>,
 }
 
 impl SegmentCollector for MultiCollectorChild {
@@ -244,12 +251,12 @@ impl SegmentCollector for MultiCollectorChild {
 mod tests {
 
     use super::*;
-    use crate::collector::{Count, TopDocs};
-    use crate::query::TermQuery;
-    use crate::schema::IndexRecordOption;
-    use crate::schema::{Schema, TEXT};
-    use crate::Index;
-    use crate::Term;
+    use collector::{Count, TopDocs};
+    use query::TermQuery;
+    use schema::IndexRecordOption;
+    use schema::{Schema, TEXT};
+    use Index;
+    use Term;
 
     #[test]
     fn test_multi_collector() {
@@ -259,7 +266,7 @@ mod tests {
 
         let index = Index::create_in_ram(schema);
         {
-            let mut index_writer = index.writer_for_tests().unwrap();
+            let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
             index_writer.add_document(doc!(text=>"abc"));
             index_writer.add_document(doc!(text=>"abc abc abc"));
             index_writer.add_document(doc!(text=>"abc abc"));
@@ -269,7 +276,8 @@ mod tests {
             index_writer.add_document(doc!(text=>"abc"));
             index_writer.commit().unwrap();
         }
-        let searcher = index.reader().unwrap().searcher();
+        index.load_searchers().unwrap();
+        let searcher = index.searcher();
         let term = Term::from_field_text(text, "abc");
         let query = TermQuery::new(term, IndexRecordOption::Basic);
 

src/collector/tests.rs

@@ -1,20 +1,12 @@
 use super::*;
-use crate::core::SegmentReader;
-use crate::fastfield::BytesFastFieldReader;
-use crate::fastfield::FastFieldReader;
-use crate::schema::Field;
-use crate::DocAddress;
-use crate::DocId;
-use crate::Score;
-use crate::SegmentLocalId;
-
-pub const TEST_COLLECTOR_WITH_SCORE: TestCollector = TestCollector {
-    compute_score: true,
-};
-
-pub const TEST_COLLECTOR_WITHOUT_SCORE: TestCollector = TestCollector {
-    compute_score: true,
-};
+use core::SegmentReader;
+use fastfield::BytesFastFieldReader;
+use fastfield::FastFieldReader;
+use schema::Field;
+use DocAddress;
+use DocId;
+use Score;
+use SegmentLocalId;
 
 /// Stores all of the doc ids.
 /// This collector is only used for tests.
@@ -22,9 +14,7 @@ pub const TEST_COLLECTOR_WITHOUT_SCORE: TestCollector = TestCollector {
 ///
 /// actise, as it does not store
 /// the segment ordinals
-pub struct TestCollector {
-    pub compute_score: bool,
-}
+pub struct TestCollector;
 
 pub struct TestSegmentCollector {
     segment_id: SegmentLocalId,
@@ -42,6 +32,7 @@ impl TestFruit {
     pub fn docs(&self) -> &[DocAddress] {
         &self.docs[..]
     }
+
     pub fn scores(&self) -> &[Score] {
         &self.scores[..]
     }
@@ -49,13 +40,14 @@ impl TestFruit {
 
 impl Collector for TestCollector {
     type Fruit = TestFruit;
+    type SegmentFruit = Self::Fruit;
     type Child = TestSegmentCollector;
 
     fn for_segment(
         &self,
         segment_id: SegmentLocalId,
         _reader: &SegmentReader,
-    ) -> crate::Result<TestSegmentCollector> {
+    ) -> Result<TestSegmentCollector> {
         Ok(TestSegmentCollector {
             segment_id,
             fruit: TestFruit::default(),
@@ -63,10 +55,10 @@ impl Collector for TestCollector {
     }
 
     fn requires_scoring(&self) -> bool {
-        self.compute_score
+        true
     }
 
-    fn merge_fruits(&self, mut children: Vec<TestFruit>) -> crate::Result<TestFruit> {
+    fn merge_fruits(&self, mut children: Vec<TestFruit>) -> Result<TestFruit> {
         children.sort_by_key(|fruit| {
             if fruit.docs().is_empty() {
                 0
@@ -118,20 +110,18 @@ impl FastFieldTestCollector {
 
 impl Collector for FastFieldTestCollector {
     type Fruit = Vec<u64>;
+    type SegmentFruit = Self::Fruit;
+
     type Child = FastFieldSegmentCollector;
 
     fn for_segment(
         &self,
         _: SegmentLocalId,
-        segment_reader: &SegmentReader,
-    ) -> crate::Result<FastFieldSegmentCollector> {
-        let reader = segment_reader
-            .fast_fields()
-            .u64(self.field)
-            .expect("Requested field is not a fast field.");
+        reader: &SegmentReader,
+    ) -> Result<FastFieldSegmentCollector> {
         Ok(FastFieldSegmentCollector {
             vals: Vec::new(),
-            reader,
+            reader: reader.fast_field_reader(self.field)?,
         })
     }
 
@@ -139,7 +129,7 @@ impl Collector for FastFieldTestCollector {
         false
     }
 
-    fn merge_fruits(&self, children: Vec<Vec<u64>>) -> crate::Result<Vec<u64>> {
+    fn merge_fruits(&self, children: Vec<Vec<u64>>) -> Result<Vec<u64>> {
         Ok(children.into_iter().flat_map(|v| v.into_iter()).collect())
     }
 }
@@ -178,19 +168,17 @@ impl BytesFastFieldTestCollector {
 
 impl Collector for BytesFastFieldTestCollector {
     type Fruit = Vec<u8>;
+    type SegmentFruit = Self::Fruit;
     type Child = BytesFastFieldSegmentCollector;
 
     fn for_segment(
         &self,
         _segment_local_id: u32,
-        segment_reader: &SegmentReader,
-    ) -> crate::Result<BytesFastFieldSegmentCollector> {
+        segment: &SegmentReader,
+    ) -> Result<BytesFastFieldSegmentCollector> {
         Ok(BytesFastFieldSegmentCollector {
             vals: Vec::new(),
-            reader: segment_reader
-                .fast_fields()
-                .bytes(self.field)
-                .expect("Field is not a bytes fast field."),
+            reader: segment.bytes_fast_field_reader(self.field)?,
         })
     }
 
@@ -198,7 +186,7 @@ impl Collector for BytesFastFieldTestCollector {
         false
     }
 
-    fn merge_fruits(&self, children: Vec<Vec<u8>>) -> crate::Result<Vec<u8>> {
+    fn merge_fruits(&self, children: Vec<Vec<u8>>) -> Result<Vec<u8>> {
         Ok(children.into_iter().flat_map(|c| c.into_iter()).collect())
     }
 }
@@ -206,8 +194,8 @@ impl Collector for BytesFastFieldTestCollector {
 impl SegmentCollector for BytesFastFieldSegmentCollector {
     type Fruit = Vec<u8>;
 
-    fn collect(&mut self, doc: u32, _score: Score) {
-        let data = self.reader.get_bytes(doc);
+    fn collect(&mut self, doc: u32, _score: f32) {
+        let data = self.reader.get_val(doc);
         self.vals.extend(data);
     }
 

src/collector/top_collector.rs

@@ -1,63 +1,52 @@
-use crate::DocAddress;
-use crate::DocId;
-use crate::SegmentLocalId;
-use crate::SegmentReader;
 use serde::export::PhantomData;
 use std::cmp::Ordering;
 use std::collections::BinaryHeap;
+use DocAddress;
+use DocId;
+use Result;
+use SegmentLocalId;
+use SegmentReader;
 
 /// Contains a feature (field, score, etc.) of a document along with the document address.
 ///
 /// It has a custom implementation of `PartialOrd` that reverses the order. This is because the
 /// default Rust heap is a max heap, whereas a min heap is needed.
 ///
-/// Additionally, it guarantees stable sorting: in case of a tie on the feature, the document
-/// address is used.
-///
 /// WARNING: equality is not what you would expect here.
 /// Two elements are equal if their feature is equal, and regardless of whether `doc`
 /// is equal. This should be perfectly fine for this usage, but let's make sure this
 /// struct is never public.
-pub(crate) struct ComparableDoc<T, D> {
-    pub feature: T,
-    pub doc: D,
+struct ComparableDoc<T, D> {
+    feature: T,
+    doc: D,
 }
 
-impl<T: PartialOrd, D: PartialOrd> PartialOrd for ComparableDoc<T, D> {
+impl<T: PartialOrd, D> PartialOrd for ComparableDoc<T, D> {
     fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
         Some(self.cmp(other))
     }
 }
 
-impl<T: PartialOrd, D: PartialOrd> Ord for ComparableDoc<T, D> {
+impl<T: PartialOrd, D> Ord for ComparableDoc<T, D> {
     #[inline]
     fn cmp(&self, other: &Self) -> Ordering {
-        // Reversed to make BinaryHeap work as a min-heap
-        let by_feature = other
+        other
             .feature
             .partial_cmp(&self.feature)
-            .unwrap_or(Ordering::Equal);
-
-        let lazy_by_doc_address = || self.doc.partial_cmp(&other.doc).unwrap_or(Ordering::Equal);
-
-        // In case of a tie on the feature, we sort by ascending
-        // `DocAddress` in order to ensure a stable sorting of the
-        // documents.
-        by_feature.then_with(lazy_by_doc_address)
+            .unwrap_or_else(|| Ordering::Equal)
     }
 }
 
-impl<T: PartialOrd, D: PartialOrd> PartialEq for ComparableDoc<T, D> {
+impl<T: PartialOrd, D> PartialEq for ComparableDoc<T, D> {
     fn eq(&self, other: &Self) -> bool {
         self.cmp(other) == Ordering::Equal
     }
 }
 
-impl<T: PartialOrd, D: PartialOrd> Eq for ComparableDoc<T, D> {}
+impl<T: PartialOrd, D> Eq for ComparableDoc<T, D> {}
 
 pub(crate) struct TopCollector<T> {
-    pub limit: usize,
-    pub offset: usize,
+    limit: usize,
     _marker: PhantomData<T>,
 }
 
@@ -73,33 +62,27 @@ where
         if limit < 1 {
             panic!("Limit must be strictly greater than 0.");
         }
-        Self {
+        TopCollector {
             limit,
-            offset: 0,
             _marker: PhantomData,
         }
     }
 
-    /// Skip the first "offset" documents when collecting.
-    ///
-    /// This is equivalent to `OFFSET` in MySQL or PostgreSQL and `start` in
-    /// Lucene's TopDocsCollector.
-    pub fn and_offset(mut self, offset: usize) -> TopCollector<T> {
-        self.offset = offset;
-        self
+    pub fn limit(&self) -> usize {
+        self.limit
     }
 
     pub fn merge_fruits(
         &self,
         children: Vec<Vec<(T, DocAddress)>>,
-    ) -> crate::Result<Vec<(T, DocAddress)>> {
+    ) -> Result<Vec<(T, DocAddress)>> {
         if self.limit == 0 {
             return Ok(Vec::new());
         }
         let mut top_collector = BinaryHeap::new();
         for child_fruit in children {
             for (feature, doc) in child_fruit {
-                if top_collector.len() < (self.limit + self.offset) {
+                if top_collector.len() < self.limit {
                     top_collector.push(ComparableDoc { feature, doc });
                 } else if let Some(mut head) = top_collector.peek_mut() {
                     if head.feature < feature {
@@ -111,33 +94,16 @@ where
         Ok(top_collector
             .into_sorted_vec()
             .into_iter()
-            .skip(self.offset)
             .map(|cdoc| (cdoc.feature, cdoc.doc))
             .collect())
     }
 
-    pub(crate) fn for_segment<F: PartialOrd>(
+    pub(crate) fn for_segment(
         &self,
         segment_id: SegmentLocalId,
         _: &SegmentReader,
-    ) -> crate::Result<TopSegmentCollector<F>> {
-        Ok(TopSegmentCollector::new(
-            segment_id,
-            self.limit + self.offset,
-        ))
-    }
-
-    /// Create a new TopCollector with the same limit and offset.
-    ///
-    /// Ideally we would use Into but the blanket implementation seems to cause the Scorer traits
-    /// to fail.
-    #[doc(hidden)]
-    pub(crate) fn into_tscore<TScore: PartialOrd + Clone>(self) -> TopCollector<TScore> {
-        TopCollector {
-            limit: self.limit,
-            offset: self.offset,
-            _marker: PhantomData,
-        }
+    ) -> Result<TopSegmentCollector<T>> {
+        Ok(TopSegmentCollector::new(segment_id, self.limit))
     }
 }
 
@@ -212,7 +178,8 @@ impl<T: PartialOrd + Clone> TopSegmentCollector<T> {
 #[cfg(test)]
 mod tests {
     use super::{TopCollector, TopSegmentCollector};
-    use crate::DocAddress;
+    use DocAddress;
+    use Score;
 
     #[test]
     fn test_top_collector_not_at_capacity() {
@@ -250,134 +217,8 @@ mod tests {
     }
 
     #[test]
-    fn test_top_segment_collector_stable_ordering_for_equal_feature() {
-        // given that the documents are collected in ascending doc id order,
-        // when harvesting we have to guarantee stable sorting in case of a tie
-        // on the score
-        let doc_ids_collection = [4, 5, 6];
-        let score = 3.14;
-
-        let mut top_collector_limit_2 = TopSegmentCollector::new(0, 2);
-        for id in &doc_ids_collection {
-            top_collector_limit_2.collect(*id, score);
-        }
-
-        let mut top_collector_limit_3 = TopSegmentCollector::new(0, 3);
-        for id in &doc_ids_collection {
-            top_collector_limit_3.collect(*id, score);
-        }
-
-        assert_eq!(
-            top_collector_limit_2.harvest(),
-            top_collector_limit_3.harvest()[..2].to_vec(),
-        );
-    }
-
-    #[test]
-    fn test_top_collector_with_limit_and_offset() {
-        let collector = TopCollector::with_limit(2).and_offset(1);
-
-        let results = collector
-            .merge_fruits(vec![vec![
-                (0.9, DocAddress(0, 1)),
-                (0.8, DocAddress(0, 2)),
-                (0.7, DocAddress(0, 3)),
-                (0.6, DocAddress(0, 4)),
-                (0.5, DocAddress(0, 5)),
-            ]])
-            .unwrap();
-
-        assert_eq!(
-            results,
-            vec![(0.8, DocAddress(0, 2)), (0.7, DocAddress(0, 3)),]
-        );
-    }
-
-    #[test]
-    fn test_top_collector_with_limit_larger_than_set_and_offset() {
-        let collector = TopCollector::with_limit(2).and_offset(1);
-
-        let results = collector
-            .merge_fruits(vec![vec![(0.9, DocAddress(0, 1)), (0.8, DocAddress(0, 2))]])
-            .unwrap();
-
-        assert_eq!(results, vec![(0.8, DocAddress(0, 2)),]);
-    }
-
-    #[test]
-    fn test_top_collector_with_limit_and_offset_larger_than_set() {
-        let collector = TopCollector::with_limit(2).and_offset(20);
-
-        let results = collector
-            .merge_fruits(vec![vec![(0.9, DocAddress(0, 1)), (0.8, DocAddress(0, 2))]])
-            .unwrap();
-
-        assert_eq!(results, vec![]);
-    }
-}
-
-#[cfg(all(test, feature = "unstable"))]
-mod bench {
-    use super::TopSegmentCollector;
-    use test::Bencher;
-
-    #[bench]
-    fn bench_top_segment_collector_collect_not_at_capacity(b: &mut Bencher) {
-        let mut top_collector = TopSegmentCollector::new(0, 400);
-
-        b.iter(|| {
-            for i in 0..100 {
-                top_collector.collect(i, 0.8);
-            }
-        });
-    }
-
-    #[bench]
-    fn bench_top_segment_collector_collect_at_capacity(b: &mut Bencher) {
-        let mut top_collector = TopSegmentCollector::new(0, 100);
-
-        for i in 0..100 {
-            top_collector.collect(i, 0.8);
-        }
-
-        b.iter(|| {
-            for i in 0..100 {
-                top_collector.collect(i, 0.8);
-            }
-        });
-    }
-
-    #[bench]
-    fn bench_top_segment_collector_collect_and_harvest_many_ties(b: &mut Bencher) {
-        b.iter(|| {
-            let mut top_collector = TopSegmentCollector::new(0, 100);
-
-            for i in 0..100 {
-                top_collector.collect(i, 0.8);
-            }
-
-            // it would be nice to be able to do the setup N times but still
-            // measure only harvest(). We can't since harvest() consumes
-            // the top_collector.
-            top_collector.harvest()
-        });
-    }
-
-    #[bench]
-    fn bench_top_segment_collector_collect_and_harvest_no_tie(b: &mut Bencher) {
-        b.iter(|| {
-            let mut top_collector = TopSegmentCollector::new(0, 100);
-            let mut score = 1.0;
-
-            for i in 0..100 {
-                score += 1.0;
-                top_collector.collect(i, score);
-            }
-
-            // it would be nice to be able to do the setup N times but still
-            // measure only harvest(). We can't since harvest() consumes
-            // the top_collector.
-            top_collector.harvest()
-        });
+    #[should_panic]
+    fn test_top_0() {
+        let _collector: TopCollector<Score> = TopCollector::with_limit(0);
     }
 }

src/collector/top_field_collector.rs

@@ -0,0 +1,251 @@
+use super::Collector;
+use collector::top_collector::TopCollector;
+use collector::top_collector::TopSegmentCollector;
+use collector::SegmentCollector;
+use fastfield::FastFieldReader;
+use fastfield::FastValue;
+use schema::Field;
+use DocAddress;
+use Result;
+use SegmentLocalId;
+use SegmentReader;
+
+/// The Top Field Collector keeps track of the K documents
+/// sorted by a fast field in the index
+///
+/// The implementation is based on a `BinaryHeap`.
+/// The theorical complexity for collecting the top `K` out of `n` documents
+/// is `O(n log K)`.
+///
+/// ```rust
+/// #[macro_use]
+/// extern crate tantivy;
+/// # use tantivy::schema::{Schema, Field, FAST, TEXT};
+/// # use tantivy::{Index, Result, DocAddress};
+/// # use tantivy::query::{Query, QueryParser};
+/// use tantivy::collector::TopDocs;
+///
+/// # fn main() {
+/// #   let mut schema_builder = Schema::builder();
+/// #   let title = schema_builder.add_text_field("title", TEXT);
+/// #   let rating = schema_builder.add_u64_field("rating", FAST);
+/// #   let schema = schema_builder.build();
+/// #   let index = Index::create_in_ram(schema);
+/// #   let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
+/// #   index_writer.add_document(doc!(
+/// #       title => "The Name of the Wind",
+/// #       rating => 92u64,
+/// #   ));
+/// #   index_writer.add_document(doc!(title => "The Diary of Muadib", rating => 97u64));
+/// #   index_writer.add_document(doc!(title => "A Dairy Cow", rating => 63u64));
+/// #   index_writer.add_document(doc!(title => "The Diary of a Young Girl", rating => 80u64));
+/// #   index_writer.commit().unwrap();
+/// #   index.load_searchers().unwrap();
+///	#   let query = QueryParser::for_index(&index, vec![title]).parse_query("diary").unwrap();
+/// #   let top_docs = docs_sorted_by_rating(&index, &query, rating).unwrap();
+/// #   assert_eq!(top_docs,
+/// #            vec![(97u64, DocAddress(0u32, 1)),
+/// #                 (80u64, DocAddress(0u32, 3))]);
+/// # }
+/// #
+/// /// Searches the document matching the given query, and
+/// /// collects the top 10 documents, order by the `field`
+/// /// given in argument.
+/// ///
+/// /// `field` is required to be a FAST field.
+/// fn docs_sorted_by_rating(index: &Index, query: &Query, sort_by_field: Field)
+///     -> Result<Vec<(u64, DocAddress)>> {
+///
+///     // This is where we build our collector!
+///     let top_docs_by_rating = TopDocs::with_limit(2).order_by_field(sort_by_field);
+///
+///     // ... and here is our documents. Not this is a simple vec.
+///     // The `u64` in the pair is the value of our fast field for each documents.
+///     index.searcher()
+///          .search(query, &top_docs_by_rating)
+/// }
+/// ```
+pub struct TopDocsByField<T> {
+    collector: TopCollector<T>,
+    field: Field,
+}
+
+impl<T: FastValue + PartialOrd + Clone> TopDocsByField<T> {
+    /// Creates a top field collector, with a number of documents equal to "limit".
+    ///
+    /// The given field name must be a fast field, otherwise the collector have an error while
+    /// collecting results.
+    ///
+    /// # Panics
+    /// The method panics if limit is 0
+    pub(crate) fn new(field: Field, limit: usize) -> TopDocsByField<T> {
+        TopDocsByField {
+            collector: TopCollector::with_limit(limit),
+            field,
+        }
+    }
+}
+
+impl<T: FastValue + PartialOrd + Send + Sync + 'static> Collector for TopDocsByField<T> {
+    type Fruit = Vec<(T, DocAddress)>;
+    type SegmentFruit = Vec<(T, DocAddress)>;
+
+    type Child = TopFieldSegmentCollector<T>;
+
+    fn for_segment(
+        &self,
+        segment_local_id: SegmentLocalId,
+        reader: &SegmentReader,
+    ) -> Result<TopFieldSegmentCollector<T>> {
+        let collector = self.collector.for_segment(segment_local_id, reader)?;
+        let reader = reader.fast_field_reader(self.field)?;
+        Ok(TopFieldSegmentCollector { collector, reader })
+    }
+
+    fn requires_scoring(&self) -> bool {
+        false
+    }
+
+    fn merge_fruits(
+        &self,
+        segment_fruits: Vec<Vec<(T, DocAddress)>>,
+    ) -> Result<Vec<(T, DocAddress)>> {
+        self.collector.merge_fruits(segment_fruits)
+    }
+}
+
+pub struct TopFieldSegmentCollector<T: FastValue + PartialOrd> {
+    collector: TopSegmentCollector<T>,
+    reader: FastFieldReader<T>,
+}
+
+impl<T: FastValue + PartialOrd + Send + Sync + 'static> SegmentCollector
+    for TopFieldSegmentCollector<T>
+{
+    type Fruit = Vec<(T, DocAddress)>;
+
+    fn collect(&mut self, doc: u32, _score: f32) {
+        let field_value = self.reader.get(doc);
+        self.collector.collect(doc, field_value);
+    }
+
+    fn harvest(self) -> Vec<(T, DocAddress)> {
+        self.collector.harvest()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::TopDocsByField;
+    use collector::Collector;
+    use collector::TopDocs;
+    use query::Query;
+    use query::QueryParser;
+    use schema::Field;
+    use schema::IntOptions;
+    use schema::{Schema, FAST, TEXT};
+    use DocAddress;
+    use Index;
+    use IndexWriter;
+    use TantivyError;
+
+    const TITLE: &str = "title";
+    const SIZE: &str = "size";
+
+    #[test]
+    fn test_top_collector_not_at_capacity() {
+        let mut schema_builder = Schema::builder();
+        let title = schema_builder.add_text_field(TITLE, TEXT);
+        let size = schema_builder.add_u64_field(SIZE, FAST);
+        let schema = schema_builder.build();
+        let (index, query) = index("beer", title, schema, |index_writer| {
+            index_writer.add_document(doc!(
+                title => "bottle of beer",
+                size => 12u64,
+            ));
+            index_writer.add_document(doc!(
+                title => "growler of beer",
+                size => 64u64,
+            ));
+            index_writer.add_document(doc!(
+                title => "pint of beer",
+                size => 16u64,
+            ));
+        });
+        let searcher = index.searcher();
+
+        let top_collector = TopDocs::with_limit(4).order_by_field(size);
+        let top_docs: Vec<(u64, DocAddress)> = searcher.search(&query, &top_collector).unwrap();
+        assert_eq!(
+            top_docs,
+            vec![
+                (64, DocAddress(0, 1)),
+                (16, DocAddress(0, 2)),
+                (12, DocAddress(0, 0))
+            ]
+        );
+    }
+
+    #[test]
+    #[should_panic]
+    fn test_field_does_not_exist() {
+        let mut schema_builder = Schema::builder();
+        let title = schema_builder.add_text_field(TITLE, TEXT);
+        let size = schema_builder.add_u64_field(SIZE, FAST);
+        let schema = schema_builder.build();
+        let (index, _) = index("beer", title, schema, |index_writer| {
+            index_writer.add_document(doc!(
+                title => "bottle of beer",
+                size => 12u64,
+            ));
+        });
+        let searcher = index.searcher();
+        let top_collector: TopDocsByField<u64> = TopDocs::with_limit(4).order_by_field(Field(2));
+        let segment_reader = searcher.segment_reader(0u32);
+        top_collector
+            .for_segment(0, segment_reader)
+            .expect("should panic");
+    }
+
+    #[test]
+    fn test_field_not_fast_field() {
+        let mut schema_builder = Schema::builder();
+        let title = schema_builder.add_text_field(TITLE, TEXT);
+        let size = schema_builder.add_u64_field(SIZE, IntOptions::default());
+        let schema = schema_builder.build();
+        let (index, _) = index("beer", title, schema, |index_writer| {
+            index_writer.add_document(doc!(
+                title => "bottle of beer",
+                size => 12u64,
+            ));
+        });
+        let searcher = index.searcher();
+        let segment = searcher.segment_reader(0);
+        let top_collector: TopDocsByField<u64> = TopDocs::with_limit(4).order_by_field(size);
+        assert_matches!(
+            top_collector
+                .for_segment(0, segment)
+                .map(|_| ())
+                .unwrap_err(),
+            TantivyError::FastFieldError(_)
+        );
+    }
+
+    fn index(
+        query: &str,
+        query_field: Field,
+        schema: Schema,
+        mut doc_adder: impl FnMut(&mut IndexWriter) -> (),
+    ) -> (Index, Box<Query>) {
+        let index = Index::create_in_ram(schema);
+
+        let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
+        doc_adder(&mut index_writer);
+        index_writer.commit().unwrap();
+        index.load_searchers().unwrap();
+
+        let query_parser = QueryParser::for_index(&index, vec![query_field]);
+        let query = query_parser.parse_query(query).unwrap();
+        (index, query)
+    }
+}

src/collector/top_score_collector.rs

@@ -1,103 +1,71 @@
 use super::Collector;
-use crate::collector::custom_score_top_collector::CustomScoreTopCollector;
-use crate::collector::top_collector::TopSegmentCollector;
-use crate::collector::top_collector::{ComparableDoc, TopCollector};
-use crate::collector::tweak_score_top_collector::TweakedScoreTopCollector;
-use crate::collector::{
-    CustomScorer, CustomSegmentScorer, ScoreSegmentTweaker, ScoreTweaker, SegmentCollector,
-};
-use crate::fastfield::FastFieldReader;
-use crate::query::Weight;
-use crate::schema::Field;
-use crate::DocAddress;
-use crate::DocId;
-use crate::Score;
-use crate::SegmentLocalId;
-use crate::SegmentReader;
-use std::collections::BinaryHeap;
-use std::fmt;
+use collector::top_collector::TopCollector;
+use collector::top_collector::TopSegmentCollector;
+use collector::SegmentCollector;
+use collector::TopDocsByField;
+use fastfield::FastValue;
+use schema::Field;
+use DocAddress;
+use DocId;
+use Result;
+use Score;
+use SegmentLocalId;
+use SegmentReader;
 
-/// The `TopDocs` collector keeps track of the top `K` documents
+/// The Top Score Collector keeps track of the K documents
 /// sorted by their score.
 ///
 /// The implementation is based on a `BinaryHeap`.
 /// The theorical complexity for collecting the top `K` out of `n` documents
 /// is `O(n log K)`.
 ///
-/// This collector guarantees a stable sorting in case of a tie on the
-/// document score. As such, it is suitable to implement pagination.
-///
 /// ```rust
+/// #[macro_use]
+/// extern crate tantivy;
+/// use tantivy::DocAddress;
+/// use tantivy::schema::{Schema, TEXT};
+/// use tantivy::{Index, Result};
 /// use tantivy::collector::TopDocs;
 /// use tantivy::query::QueryParser;
-/// use tantivy::schema::{Schema, TEXT};
-/// use tantivy::{doc, DocAddress, Index};
 ///
-/// let mut schema_builder = Schema::builder();
-/// let title = schema_builder.add_text_field("title", TEXT);
-/// let schema = schema_builder.build();
-/// let index = Index::create_in_ram(schema);
+/// # fn main() { example().unwrap(); }
+/// fn example() -> Result<()> {
+///     let mut schema_builder = Schema::builder();
+///     let title = schema_builder.add_text_field("title", TEXT);
+///     let schema = schema_builder.build();
+///     let index = Index::create_in_ram(schema);
+///     {
+///         let mut index_writer = index.writer_with_num_threads(1, 3_000_000)?;
+///         index_writer.add_document(doc!(
+///             title => "The Name of the Wind",
+///         ));
+///         index_writer.add_document(doc!(
+///             title => "The Diary of Muadib",
+///         ));
+///         index_writer.add_document(doc!(
+///             title => "A Dairy Cow",
+///         ));
+///         index_writer.add_document(doc!(
+///             title => "The Diary of a Young Girl",
+///         ));
+///         index_writer.commit().unwrap();
+///     }
 ///
-/// let mut index_writer = index.writer_with_num_threads(1, 10_000_000).unwrap();
-/// index_writer.add_document(doc!(title => "The Name of the Wind"));
-/// index_writer.add_document(doc!(title => "The Diary of Muadib"));
-/// index_writer.add_document(doc!(title => "A Dairy Cow"));
-/// index_writer.add_document(doc!(title => "The Diary of a Young Girl"));
-/// assert!(index_writer.commit().is_ok());
+///     index.load_searchers()?;
+///     let searcher = index.searcher();
 ///
-/// let reader = index.reader().unwrap();
-/// let searcher = reader.searcher();
+///     let query_parser = QueryParser::for_index(&index, vec![title]);
+///     let query = query_parser.parse_query("diary")?;
+///     let top_docs = searcher.search(&query, &TopDocs::with_limit(2))?;
 ///
-/// let query_parser = QueryParser::for_index(&index, vec![title]);
-/// let query = query_parser.parse_query("diary").unwrap();
-/// let top_docs = searcher.search(&query, &TopDocs::with_limit(2)).unwrap();
+///     assert_eq!(&top_docs[0], &(0.7261542, DocAddress(0, 1)));
+///     assert_eq!(&top_docs[1], &(0.6099695, DocAddress(0, 3)));
 ///
-/// assert_eq!(top_docs[0].1, DocAddress(0, 1));
-/// assert_eq!(top_docs[1].1, DocAddress(0, 3));
+///     Ok(())
+/// }
 /// ```
 pub struct TopDocs(TopCollector<Score>);
 
-impl fmt::Debug for TopDocs {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(
-            f,
-            "TopDocs(limit={}, offset={})",
-            self.0.limit, self.0.offset
-        )
-    }
-}
-
-struct ScorerByFastFieldReader {
-    ff_reader: FastFieldReader<u64>,
-}
-
-impl CustomSegmentScorer<u64> for ScorerByFastFieldReader {
-    fn score(&mut self, doc: DocId) -> u64 {
-        self.ff_reader.get_u64(u64::from(doc))
-    }
-}
-
-struct ScorerByField {
-    field: Field,
-}
-
-impl CustomScorer<u64> for ScorerByField {
-    type Child = ScorerByFastFieldReader;
-
-    fn segment_scorer(&self, segment_reader: &SegmentReader) -> crate::Result<Self::Child> {
-        let ff_reader = segment_reader
-            .fast_fields()
-            .u64(self.field)
-            .ok_or_else(|| {
-                crate::TantivyError::SchemaError(format!(
-                    "Field requested ({:?}) is not a i64/u64 fast field.",
-                    self.field
-                ))
-            })?;
-        Ok(ScorerByFastFieldReader { ff_reader })
-    }
-}
-
 impl TopDocs {
     /// Creates a top score collector, with a number of documents equal to "limit".
     ///
@@ -107,345 +75,21 @@ impl TopDocs {
         TopDocs(TopCollector::with_limit(limit))
     }
 
-    /// Skip the first "offset" documents when collecting.
-    ///
-    /// This is equivalent to `OFFSET` in MySQL or PostgreSQL and `start` in
-    /// Lucene's TopDocsCollector.
-    ///
-    /// ```rust
-    /// use tantivy::collector::TopDocs;
-    /// use tantivy::query::QueryParser;
-    /// use tantivy::schema::{Schema, TEXT};
-    /// use tantivy::{doc, DocAddress, Index};
-    ///
-    /// let mut schema_builder = Schema::builder();
-    /// let title = schema_builder.add_text_field("title", TEXT);
-    /// let schema = schema_builder.build();
-    /// let index = Index::create_in_ram(schema);
-    ///
-    /// let mut index_writer = index.writer_with_num_threads(1, 10_000_000).unwrap();
-    /// index_writer.add_document(doc!(title => "The Name of the Wind"));
-    /// index_writer.add_document(doc!(title => "The Diary of Muadib"));
-    /// index_writer.add_document(doc!(title => "A Dairy Cow"));
-    /// index_writer.add_document(doc!(title => "The Diary of a Young Girl"));
-    /// index_writer.add_document(doc!(title => "The Diary of Lena Mukhina"));
-    /// assert!(index_writer.commit().is_ok());
-    ///
-    /// let reader = index.reader().unwrap();
-    /// let searcher = reader.searcher();
-    ///
-    /// let query_parser = QueryParser::for_index(&index, vec![title]);
-    /// let query = query_parser.parse_query("diary").unwrap();
-    /// let top_docs = searcher.search(&query, &TopDocs::with_limit(2).and_offset(1)).unwrap();
-    ///
-    /// assert_eq!(top_docs.len(), 2);
-    /// assert_eq!(top_docs[0].1, DocAddress(0, 4));
-    /// assert_eq!(top_docs[1].1, DocAddress(0, 3));
-    /// ```
-    pub fn and_offset(self, offset: usize) -> TopDocs {
-        TopDocs(self.0.and_offset(offset))
-    }
-
     /// Set top-K to rank documents by a given fast field.
     ///
-    /// ```rust
-    /// # use tantivy::schema::{Schema, FAST, TEXT};
-    /// # use tantivy::{doc, Index, DocAddress};
-    /// # use tantivy::query::{Query, QueryParser};
-    /// use tantivy::Searcher;
-    /// use tantivy::collector::TopDocs;
-    /// use tantivy::schema::Field;
-    ///
-    /// # fn main() -> tantivy::Result<()> {
-    /// #   let mut schema_builder = Schema::builder();
-    /// #   let title = schema_builder.add_text_field("title", TEXT);
-    /// #   let rating = schema_builder.add_u64_field("rating", FAST);
-    /// #   let schema = schema_builder.build();
-    /// #  
-    /// #   let index = Index::create_in_ram(schema);
-    /// #   let mut index_writer = index.writer_with_num_threads(1, 10_000_000)?;
-    /// #   index_writer.add_document(doc!(title => "The Name of the Wind", rating => 92u64));
-    /// #   index_writer.add_document(doc!(title => "The Diary of Muadib", rating => 97u64));
-    /// #   index_writer.add_document(doc!(title => "A Dairy Cow", rating => 63u64));
-    /// #   index_writer.add_document(doc!(title => "The Diary of a Young Girl", rating => 80u64));
-    /// #   assert!(index_writer.commit().is_ok());
-    /// #   let reader = index.reader().unwrap();
-    /// #   let query = QueryParser::for_index(&index, vec![title]).parse_query("diary")?;
-    /// #   let top_docs = docs_sorted_by_rating(&reader.searcher(), &query, rating)?;
-    /// #   assert_eq!(top_docs,
-    /// #            vec![(97u64, DocAddress(0u32, 1)),
-    /// #                 (80u64, DocAddress(0u32, 3))]);
-    /// #   Ok(())
-    /// # }
-    ///
-    ///
-    /// /// Searches the document matching the given query, and
-    /// /// collects the top 10 documents, order by the u64-`field`
-    /// /// given in argument.
-    /// ///
-    /// /// `field` is required to be a FAST field.
-    /// fn docs_sorted_by_rating(searcher: &Searcher,
-    ///                          query: &dyn Query,
-    ///                          sort_by_field: Field)
-    ///     -> tantivy::Result<Vec<(u64, DocAddress)>> {
-    ///
-    ///     // This is where we build our topdocs collector
-    ///     //
-    ///     // Note the generics parameter that needs to match the
-    ///     // type `sort_by_field`.
-    ///     let top_docs_by_rating = TopDocs
-    ///                 ::with_limit(10)
-    ///                  .order_by_u64_field(sort_by_field);
-    ///     
-    ///     // ... and here are our documents. Note this is a simple vec.
-    ///     // The `u64` in the pair is the value of our fast field for
-    ///     // each documents.
-    ///     //
-    ///     // The vec is sorted decreasingly by `sort_by_field`, and has a
-    ///     // length of 10, or less if not enough documents matched the
-    ///     // query.
-    ///     let resulting_docs: Vec<(u64, DocAddress)> =
-    ///          searcher.search(query, &top_docs_by_rating)?;
-    ///     
-    ///     Ok(resulting_docs)
-    /// }
-    /// ```
-    ///
-    /// # Panics
-    ///
-    /// May panic if the field requested is not a fast field.
-    ///
-    pub fn order_by_u64_field(
+    /// (By default, `TopDocs` collects the top-K documents sorted by
+    /// the similarity score.)
+    pub fn order_by_field<T: PartialOrd + FastValue + Clone>(
         self,
         field: Field,
-    ) -> impl Collector<Fruit = Vec<(u64, DocAddress)>> {
-        self.custom_score(ScorerByField { field })
-    }
-
-    /// Ranks the documents using a custom score.
-    ///
-    /// This method offers a convenient way to tweak or replace
-    /// the documents score. As suggested by the prototype you can
-    /// manually define your own [`ScoreTweaker`](./trait.ScoreTweaker.html)
-    /// and pass it as an argument, but there is a much simpler way to
-    /// tweak your score: you can use a closure as in the following
-    /// example.
-    ///
-    /// # Example
-    ///
-    /// Typically, you will want to rely on one or more fast fields,
-    /// to alter the original relevance `Score`.
-    ///
-    /// For instance, in the following, we assume that we are implementing
-    /// an e-commerce website that has a fast field called `popularity`
-    /// that rates whether a product is typically often bought by users.
-    ///
-    /// In the following example will will tweak our ranking a bit by
-    /// boosting popular products a notch.
-    ///  
-    /// In more serious application, this tweaking could involved running a
-    /// learning-to-rank model over various features
-    ///
-    /// ```rust
-    /// # use tantivy::schema::{Schema, FAST, TEXT};
-    /// # use tantivy::{doc, Index, DocAddress, DocId, Score};
-    /// # use tantivy::query::QueryParser;
-    /// use tantivy::SegmentReader;
-    /// use tantivy::collector::TopDocs;
-    /// use tantivy::schema::Field;
-    ///
-    /// fn create_schema() -> Schema {
-    ///    let mut schema_builder = Schema::builder();
-    ///    schema_builder.add_text_field("product_name", TEXT);
-    ///    schema_builder.add_u64_field("popularity", FAST);
-    ///    schema_builder.build()
-    /// }
-    ///
-    /// fn create_index() -> tantivy::Result<Index> {
-    ///   let schema = create_schema();
-    ///   let index = Index::create_in_ram(schema);
-    ///   let mut index_writer = index.writer_with_num_threads(1, 10_000_000)?;
-    ///   let product_name = index.schema().get_field("product_name").unwrap();
-    ///   let popularity: Field = index.schema().get_field("popularity").unwrap();
-    ///   index_writer.add_document(doc!(product_name => "The Diary of Muadib", popularity => 1u64));
-    ///   index_writer.add_document(doc!(product_name => "A Dairy Cow", popularity => 10u64));
-    ///   index_writer.add_document(doc!(product_name => "The Diary of a Young Girl", popularity => 15u64));
-    ///   index_writer.commit()?;
-    ///   Ok(index)
-    /// }
-    ///
-    /// let index = create_index().unwrap();
-    /// let product_name = index.schema().get_field("product_name").unwrap();
-    /// let popularity: Field = index.schema().get_field("popularity").unwrap();
-    ///
-    /// let user_query_str = "diary";
-    /// let query_parser = QueryParser::for_index(&index, vec![product_name]);
-    /// let query = query_parser.parse_query(user_query_str).unwrap();
-    ///
-    /// // This is where we build our collector with our custom score.
-    /// let top_docs_by_custom_score = TopDocs
-    ///         ::with_limit(10)
-    ///          .tweak_score(move |segment_reader: &SegmentReader| {
-    ///             // The argument is a function that returns our scoring
-    ///             // function.
-    ///             //
-    ///             // The point of this "mother" function is to gather all
-    ///             // of the segment level information we need for scoring.
-    ///             // Typically, fast_fields.
-    ///             //
-    ///             // In our case, we will get a reader for the popularity
-    ///             // fast field.
-    ///             let popularity_reader =
-    ///                 segment_reader.fast_fields().u64(popularity).unwrap();
-    ///
-    ///             // We can now define our actual scoring function
-    ///             move |doc: DocId, original_score: Score| {
-    ///                 let popularity: u64 = popularity_reader.get(doc);
-    ///                 // Well.. For the sake of the example we use a simple logarithm
-    ///                 // function.
-    ///                 let popularity_boost_score = ((2u64 + popularity) as Score).log2();
-    ///                 popularity_boost_score * original_score
-    ///             }
-    ///           });
-    /// let reader = index.reader().unwrap();
-    /// let searcher = reader.searcher();
-    /// // ... and here are our documents. Note this is a simple vec.
-    /// // The `Score` in the pair is our tweaked score.
-    /// let resulting_docs: Vec<(Score, DocAddress)> =
-    ///      searcher.search(&query, &top_docs_by_custom_score).unwrap();
-    /// ```
-    ///
-    /// # See also
-    /// [custom_score(...)](#method.custom_score).
-    pub fn tweak_score<TScore, TScoreSegmentTweaker, TScoreTweaker>(
-        self,
-        score_tweaker: TScoreTweaker,
-    ) -> impl Collector<Fruit = Vec<(TScore, DocAddress)>>
-    where
-        TScore: 'static + Send + Sync + Clone + PartialOrd,
-        TScoreSegmentTweaker: ScoreSegmentTweaker<TScore> + 'static,
-        TScoreTweaker: ScoreTweaker<TScore, Child = TScoreSegmentTweaker> + Send + Sync,
-    {
-        TweakedScoreTopCollector::new(score_tweaker, self.0.into_tscore())
-    }
-
-    /// Ranks the documents using a custom score.
-    ///
-    /// This method offers a convenient way to use a different score.
-    ///
-    /// As suggested by the prototype you can manually define your
-    /// own [`CustomScorer`](./trait.CustomScorer.html)
-    /// and pass it as an argument, but there is a much simpler way to
-    /// tweak your score: you can use a closure as in the following
-    /// example.
-    ///
-    /// # Limitation
-    ///
-    /// This method only makes it possible to compute the score from a given
-    /// `DocId`, fastfield values for the doc and any information you could
-    /// have precomputed beforehands. It does not make it possible for instance
-    /// to compute something like TfIdf as it does not have access to the list of query
-    /// terms present in the document, nor the term frequencies for the different terms.
-    ///
-    /// It can be used if your search engine relies on a learning-to-rank model for instance,
-    /// which does not rely on the term frequencies or positions as features.
-    ///
-    /// # Example
-    ///
-    /// ```rust
-    /// # use tantivy::schema::{Schema, FAST, TEXT};
-    /// # use tantivy::{doc, Index, DocAddress, DocId};
-    /// # use tantivy::query::QueryParser;
-    /// use tantivy::SegmentReader;
-    /// use tantivy::collector::TopDocs;
-    /// use tantivy::schema::Field;
-    ///
-    /// # fn create_schema() -> Schema {
-    /// #    let mut schema_builder = Schema::builder();
-    /// #    schema_builder.add_text_field("product_name", TEXT);
-    /// #    schema_builder.add_u64_field("popularity", FAST);
-    /// #    schema_builder.add_u64_field("boosted", FAST);
-    /// #    schema_builder.build()
-    /// # }
-    /// #
-    /// # fn main() -> tantivy::Result<()> {
-    /// #   let schema = create_schema();
-    /// #   let index = Index::create_in_ram(schema);
-    /// #   let mut index_writer = index.writer_with_num_threads(1, 10_000_000)?;
-    /// #   let product_name = index.schema().get_field("product_name").unwrap();
-    /// #   
-    /// let popularity: Field = index.schema().get_field("popularity").unwrap();
-    /// let boosted: Field = index.schema().get_field("boosted").unwrap();
-    /// #   index_writer.add_document(doc!(boosted=>1u64, product_name => "The Diary of Muadib", popularity => 1u64));
-    /// #   index_writer.add_document(doc!(boosted=>0u64, product_name => "A Dairy Cow", popularity => 10u64));
-    /// #   index_writer.add_document(doc!(boosted=>0u64, product_name => "The Diary of a Young Girl", popularity => 15u64));
-    /// #   index_writer.commit()?;
-    /// // ...
-    /// # let user_query = "diary";
-    /// # let query = QueryParser::for_index(&index, vec![product_name]).parse_query(user_query)?;
-    ///
-    /// // This is where we build our collector with our custom score.
-    /// let top_docs_by_custom_score = TopDocs
-    ///         ::with_limit(10)
-    ///          .custom_score(move |segment_reader: &SegmentReader| {
-    ///             // The argument is a function that returns our scoring
-    ///             // function.
-    ///             //
-    ///             // The point of this "mother" function is to gather all
-    ///             // of the segment level information we need for scoring.
-    ///             // Typically, fast_fields.
-    ///             //
-    ///             // In our case, we will get a reader for the popularity
-    ///             // fast field and a boosted field.
-    ///             //
-    ///             // We want to get boosted items score, and when we get
-    ///             // a tie, return the item with the highest popularity.
-    ///             //
-    ///             // Note that this is implemented by using a `(u64, u64)`
-    ///             // as a score.
-    ///             let popularity_reader =
-    ///                 segment_reader.fast_fields().u64(popularity).unwrap();
-    ///             let boosted_reader =
-    ///                 segment_reader.fast_fields().u64(boosted).unwrap();
-    ///    
-    ///             // We can now define our actual scoring function
-    ///             move |doc: DocId| {
-    ///                 let popularity: u64 = popularity_reader.get(doc);
-    ///                 let boosted: u64 = boosted_reader.get(doc);
-    ///                 // Score do not have to be `f64` in tantivy.
-    ///                 // Here we return a couple to get lexicographical order
-    ///                 // for free.
-    ///                 (boosted, popularity)
-    ///             }
-    ///           });
-    /// # let reader = index.reader()?;
-    /// # let searcher = reader.searcher();
-    /// // ... and here are our documents. Note this is a simple vec.
-    /// // The `Score` in the pair is our tweaked score.
-    /// let resulting_docs: Vec<((u64, u64), DocAddress)> =
-    ///      searcher.search(&*query, &top_docs_by_custom_score)?;
-    ///
-    /// # Ok(())
-    /// # }
-    /// ```
-    ///
-    /// # See also
-    /// [tweak_score(...)](#method.tweak_score).
-    pub fn custom_score<TScore, TCustomSegmentScorer, TCustomScorer>(
-        self,
-        custom_score: TCustomScorer,
-    ) -> impl Collector<Fruit = Vec<(TScore, DocAddress)>>
-    where
-        TScore: 'static + Send + Sync + Clone + PartialOrd,
-        TCustomSegmentScorer: CustomSegmentScorer<TScore> + 'static,
-        TCustomScorer: CustomScorer<TScore, Child = TCustomSegmentScorer> + Send + Sync,
-    {
-        CustomScoreTopCollector::new(custom_score, self.0.into_tscore())
+    ) -> TopDocsByField<T> {
+        TopDocsByField::new(field, self.0.limit())
     }
 }
 
 impl Collector for TopDocs {
     type Fruit = Vec<(Score, DocAddress)>;
+    type SegmentFruit = Vec<(Score, DocAddress)>;
 
     type Child = TopScoreSegmentCollector;
 
@@ -453,7 +97,7 @@ impl Collector for TopDocs {
         &self,
         segment_local_id: SegmentLocalId,
         reader: &SegmentReader,
-    ) -> crate::Result<Self::Child> {
+    ) -> Result<Self::Child> {
         let collector = self.0.for_segment(segment_local_id, reader)?;
         Ok(TopScoreSegmentCollector(collector))
     }
@@ -462,70 +106,9 @@ impl Collector for TopDocs {
         true
     }
 
-    fn merge_fruits(
-        &self,
-        child_fruits: Vec<Vec<(Score, DocAddress)>>,
-    ) -> crate::Result<Self::Fruit> {
+    fn merge_fruits(&self, child_fruits: Vec<Vec<(Score, DocAddress)>>) -> Result<Self::Fruit> {
         self.0.merge_fruits(child_fruits)
     }
-
-    fn collect_segment(
-        &self,
-        weight: &dyn Weight,
-        segment_ord: u32,
-        reader: &SegmentReader,
-    ) -> crate::Result<<Self::Child as SegmentCollector>::Fruit> {
-        let heap_len = self.0.limit + self.0.offset;
-        let mut heap: BinaryHeap<ComparableDoc<Score, DocId>> = BinaryHeap::with_capacity(heap_len);
-
-        if let Some(delete_bitset) = reader.delete_bitset() {
-            let mut threshold = Score::MIN;
-            weight.for_each_pruning(threshold, reader, &mut |doc, score| {
-                if delete_bitset.is_deleted(doc) {
-                    return threshold;
-                }
-                let heap_item = ComparableDoc {
-                    feature: score,
-                    doc,
-                };
-                if heap.len() < heap_len {
-                    heap.push(heap_item);
-                    if heap.len() == heap_len {
-                        threshold = heap.peek().map(|el| el.feature).unwrap_or(Score::MIN);
-                    }
-                    return threshold;
-                }
-                *heap.peek_mut().unwrap() = heap_item;
-                threshold = heap.peek().map(|el| el.feature).unwrap_or(Score::MIN);
-                threshold
-            })?;
-        } else {
-            weight.for_each_pruning(Score::MIN, reader, &mut |doc, score| {
-                let heap_item = ComparableDoc {
-                    feature: score,
-                    doc,
-                };
-                if heap.len() < heap_len {
-                    heap.push(heap_item);
-                    // TODO the threshold is suboptimal for heap.len == heap_len
-                    if heap.len() == heap_len {
-                        return heap.peek().map(|el| el.feature).unwrap_or(Score::MIN);
-                    } else {
-                        return Score::MIN;
-                    }
-                }
-                *heap.peek_mut().unwrap() = heap_item;
-                heap.peek().map(|el| el.feature).unwrap_or(Score::MIN)
-            })?;
-        }
-
-        let fruit = heap
-            .into_sorted_vec()
-            .into_iter()
-            .map(|cid| (cid.feature, DocAddress(segment_ord, cid.doc)))
-            .collect();
-        Ok(fruit)
-    }
 }
 
 /// Segment Collector associated to `TopDocs`.
@@ -535,7 +118,7 @@ impl SegmentCollector for TopScoreSegmentCollector {
     type Fruit = Vec<(Score, DocAddress)>;
 
     fn collect(&mut self, doc: DocId, score: Score) {
-        self.0.collect(doc, score);
+        self.0.collect(doc, score)
     }
 
     fn harvest(self) -> Vec<(Score, DocAddress)> {
@@ -546,13 +129,12 @@ impl SegmentCollector for TopScoreSegmentCollector {
 #[cfg(test)]
 mod tests {
     use super::TopDocs;
-    use crate::collector::Collector;
-    use crate::query::{AllQuery, Query, QueryParser};
-    use crate::schema::{Field, Schema, FAST, STORED, TEXT};
-    use crate::Index;
-    use crate::IndexWriter;
-    use crate::Score;
-    use crate::{DocAddress, DocId, SegmentReader};
+    use query::QueryParser;
+    use schema::Schema;
+    use schema::TEXT;
+    use DocAddress;
+    use Index;
+    use Score;
 
     fn make_index() -> Index {
         let mut schema_builder = Schema::builder();
@@ -561,22 +143,16 @@ mod tests {
         let index = Index::create_in_ram(schema);
         {
             // writing the segment
-            let mut index_writer = index.writer_with_num_threads(1, 10_000_000).unwrap();
+            let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
             index_writer.add_document(doc!(text_field=>"Hello happy tax payer."));
             index_writer.add_document(doc!(text_field=>"Droopy says hello happy tax payer"));
             index_writer.add_document(doc!(text_field=>"I like Droopy"));
             assert!(index_writer.commit().is_ok());
         }
+        index.load_searchers().unwrap();
         index
     }
 
-    fn assert_results_equals(results: &[(Score, DocAddress)], expected: &[(Score, DocAddress)]) {
-        for (result, expected) in results.iter().zip(expected.iter()) {
-            assert_eq!(result.1, expected.1);
-            crate::assert_nearly_equals!(result.0, expected.0);
-        }
-    }
-
     #[test]
     fn test_top_collector_not_at_capacity() {
         let index = make_index();
@@ -584,36 +160,19 @@ mod tests {
         let query_parser = QueryParser::for_index(&index, vec![field]);
         let text_query = query_parser.parse_query("droopy tax").unwrap();
         let score_docs: Vec<(Score, DocAddress)> = index
-            .reader()
-            .unwrap()
             .searcher()
             .search(&text_query, &TopDocs::with_limit(4))
             .unwrap();
-        assert_results_equals(
-            &score_docs,
-            &[
+        assert_eq!(
+            score_docs,
+            vec![
                 (0.81221175, DocAddress(0u32, 1)),
                 (0.5376842, DocAddress(0u32, 2)),
-                (0.48527452, DocAddress(0, 0)),
-            ],
+                (0.48527452, DocAddress(0, 0))
+            ]
         );
     }
 
-    #[test]
-    fn test_top_collector_not_at_capacity_with_offset() {
-        let index = make_index();
-        let field = index.schema().get_field("text").unwrap();
-        let query_parser = QueryParser::for_index(&index, vec![field]);
-        let text_query = query_parser.parse_query("droopy tax").unwrap();
-        let score_docs: Vec<(Score, DocAddress)> = index
-            .reader()
-            .unwrap()
-            .searcher()
-            .search(&text_query, &TopDocs::with_limit(4).and_offset(2))
-            .unwrap();
-        assert_results_equals(&score_docs[..], &[(0.48527452, DocAddress(0, 0))]);
-    }
-
     #[test]
     fn test_top_collector_at_capacity() {
         let index = make_index();
@@ -621,211 +180,22 @@ mod tests {
         let query_parser = QueryParser::for_index(&index, vec![field]);
         let text_query = query_parser.parse_query("droopy tax").unwrap();
         let score_docs: Vec<(Score, DocAddress)> = index
-            .reader()
-            .unwrap()
             .searcher()
             .search(&text_query, &TopDocs::with_limit(2))
             .unwrap();
-        assert_results_equals(
-            &score_docs,
-            &[
+        assert_eq!(
+            score_docs,
+            vec![
                 (0.81221175, DocAddress(0u32, 1)),
                 (0.5376842, DocAddress(0u32, 2)),
-            ],
+            ]
         );
     }
 
-    #[test]
-    fn test_top_collector_at_capacity_with_offset() {
-        let index = make_index();
-        let field = index.schema().get_field("text").unwrap();
-        let query_parser = QueryParser::for_index(&index, vec![field]);
-        let text_query = query_parser.parse_query("droopy tax").unwrap();
-        let score_docs: Vec<(Score, DocAddress)> = index
-            .reader()
-            .unwrap()
-            .searcher()
-            .search(&text_query, &TopDocs::with_limit(2).and_offset(1))
-            .unwrap();
-        assert_results_equals(
-            &score_docs[..],
-            &[
-                (0.5376842, DocAddress(0u32, 2)),
-                (0.48527452, DocAddress(0, 0)),
-            ],
-        );
-    }
-
-    #[test]
-    fn test_top_collector_stable_sorting() {
-        let index = make_index();
-
-        // using AllQuery to get a constant score
-        let searcher = index.reader().unwrap().searcher();
-
-        let page_1 = searcher.search(&AllQuery, &TopDocs::with_limit(2)).unwrap();
-
-        let page_2 = searcher.search(&AllQuery, &TopDocs::with_limit(3)).unwrap();
-
-        // precondition for the test to be meaningful: we did get documents
-        // with the same score
-        assert!(page_1.iter().all(|result| result.0 == page_1[0].0));
-        assert!(page_2.iter().all(|result| result.0 == page_2[0].0));
-
-        // sanity check since we're relying on make_index()
-        assert_eq!(page_1.len(), 2);
-        assert_eq!(page_2.len(), 3);
-
-        assert_eq!(page_1, &page_2[..page_1.len()]);
-    }
-
     #[test]
     #[should_panic]
     fn test_top_0() {
         TopDocs::with_limit(0);
     }
 
-    const TITLE: &str = "title";
-    const SIZE: &str = "size";
-
-    #[test]
-    fn test_top_field_collector_not_at_capacity() {
-        let mut schema_builder = Schema::builder();
-        let title = schema_builder.add_text_field(TITLE, TEXT);
-        let size = schema_builder.add_u64_field(SIZE, FAST);
-        let schema = schema_builder.build();
-        let (index, query) = index("beer", title, schema, |index_writer| {
-            index_writer.add_document(doc!(
-                title => "bottle of beer",
-                size => 12u64,
-            ));
-            index_writer.add_document(doc!(
-                title => "growler of beer",
-                size => 64u64,
-            ));
-            index_writer.add_document(doc!(
-                title => "pint of beer",
-                size => 16u64,
-            ));
-        });
-        let searcher = index.reader().unwrap().searcher();
-
-        let top_collector = TopDocs::with_limit(4).order_by_u64_field(size);
-        let top_docs: Vec<(u64, DocAddress)> = searcher.search(&query, &top_collector).unwrap();
-        assert_eq!(
-            &top_docs[..],
-            &[
-                (64, DocAddress(0, 1)),
-                (16, DocAddress(0, 2)),
-                (12, DocAddress(0, 0))
-            ]
-        );
-    }
-
-    #[test]
-    #[should_panic]
-    fn test_field_does_not_exist() {
-        let mut schema_builder = Schema::builder();
-        let title = schema_builder.add_text_field(TITLE, TEXT);
-        let size = schema_builder.add_u64_field(SIZE, FAST);
-        let schema = schema_builder.build();
-        let (index, _) = index("beer", title, schema, |index_writer| {
-            index_writer.add_document(doc!(
-                title => "bottle of beer",
-                size => 12u64,
-            ));
-        });
-        let searcher = index.reader().unwrap().searcher();
-        let top_collector = TopDocs::with_limit(4).order_by_u64_field(Field::from_field_id(2));
-        let segment_reader = searcher.segment_reader(0u32);
-        top_collector
-            .for_segment(0, segment_reader)
-            .expect("should panic");
-    }
-
-    #[test]
-    fn test_field_not_fast_field() {
-        let mut schema_builder = Schema::builder();
-        let title = schema_builder.add_text_field(TITLE, TEXT);
-        let size = schema_builder.add_u64_field(SIZE, STORED);
-        let schema = schema_builder.build();
-        let (index, _) = index("beer", title, schema, |index_writer| {
-            index_writer.add_document(doc!(
-                title => "bottle of beer",
-                size => 12u64,
-            ));
-        });
-        let searcher = index.reader().unwrap().searcher();
-        let segment = searcher.segment_reader(0);
-        let top_collector = TopDocs::with_limit(4).order_by_u64_field(size);
-        let err = top_collector.for_segment(0, segment);
-        if let Err(crate::TantivyError::SchemaError(msg)) = err {
-            assert_eq!(
-                msg,
-                "Field requested (Field(1)) is not a i64/u64 fast field."
-            );
-        } else {
-            assert!(false);
-        }
-    }
-
-    #[test]
-    fn test_tweak_score_top_collector_with_offset() {
-        let index = make_index();
-        let field = index.schema().get_field("text").unwrap();
-        let query_parser = QueryParser::for_index(&index, vec![field]);
-        let text_query = query_parser.parse_query("droopy tax").unwrap();
-        let collector = TopDocs::with_limit(2).and_offset(1).tweak_score(
-            move |_segment_reader: &SegmentReader| move |doc: DocId, _original_score: Score| doc,
-        );
-        let score_docs: Vec<(u32, DocAddress)> = index
-            .reader()
-            .unwrap()
-            .searcher()
-            .search(&text_query, &collector)
-            .unwrap();
-
-        assert_eq!(
-            score_docs,
-            vec![(1, DocAddress(0, 1)), (0, DocAddress(0, 0)),]
-        );
-    }
-
-    #[test]
-    fn test_custom_score_top_collector_with_offset() {
-        let index = make_index();
-        let field = index.schema().get_field("text").unwrap();
-        let query_parser = QueryParser::for_index(&index, vec![field]);
-        let text_query = query_parser.parse_query("droopy tax").unwrap();
-        let collector = TopDocs::with_limit(2)
-            .and_offset(1)
-            .custom_score(move |_segment_reader: &SegmentReader| move |doc: DocId| doc);
-        let score_docs: Vec<(u32, DocAddress)> = index
-            .reader()
-            .unwrap()
-            .searcher()
-            .search(&text_query, &collector)
-            .unwrap();
-
-        assert_eq!(
-            score_docs,
-            vec![(1, DocAddress(0, 1)), (0, DocAddress(0, 0)),]
-        );
-    }
-
-    fn index(
-        query: &str,
-        query_field: Field,
-        schema: Schema,
-        mut doc_adder: impl FnMut(&mut IndexWriter) -> (),
-    ) -> (Index, Box<dyn Query>) {
-        let index = Index::create_in_ram(schema);
-
-        let mut index_writer = index.writer_with_num_threads(1, 10_000_000).unwrap();
-        doc_adder(&mut index_writer);
-        index_writer.commit().unwrap();
-        let query_parser = QueryParser::for_index(&index, vec![query_field]);
-        let query = query_parser.parse_query(query).unwrap();
-        (index, query)
-    }
 }

src/collector/tweak_score_top_collector.rs

@@ -1,129 +0,0 @@
-use crate::collector::top_collector::{TopCollector, TopSegmentCollector};
-use crate::collector::{Collector, SegmentCollector};
-use crate::DocAddress;
-use crate::{DocId, Result, Score, SegmentReader};
-
-pub(crate) struct TweakedScoreTopCollector<TScoreTweaker, TScore = Score> {
-    score_tweaker: TScoreTweaker,
-    collector: TopCollector<TScore>,
-}
-
-impl<TScoreTweaker, TScore> TweakedScoreTopCollector<TScoreTweaker, TScore>
-where
-    TScore: Clone + PartialOrd,
-{
-    pub fn new(
-        score_tweaker: TScoreTweaker,
-        collector: TopCollector<TScore>,
-    ) -> TweakedScoreTopCollector<TScoreTweaker, TScore> {
-        TweakedScoreTopCollector {
-            score_tweaker,
-            collector,
-        }
-    }
-}
-
-/// A `ScoreSegmentTweaker` makes it possible to modify the default score
-/// for a given document belonging to a specific segment.
-///
-/// It is the segment local version of the [`ScoreTweaker`](./trait.ScoreTweaker.html).
-pub trait ScoreSegmentTweaker<TScore>: 'static {
-    /// Tweak the given `score` for the document `doc`.
-    fn score(&mut self, doc: DocId, score: Score) -> TScore;
-}
-
-/// `ScoreTweaker` makes it possible to tweak the score
-/// emitted  by the scorer into another one.
-///
-/// The `ScoreTweaker` itself does not make much of the computation itself.
-/// Instead, it helps constructing `Self::Child` instances that will compute
-/// the score at a segment scale.
-pub trait ScoreTweaker<TScore>: Sync {
-    /// Type of the associated [`ScoreSegmentTweaker`](./trait.ScoreSegmentTweaker.html).
-    type Child: ScoreSegmentTweaker<TScore>;
-
-    /// Builds a child tweaker for a specific segment. The child scorer is associated to
-    /// a specific segment.
-    fn segment_tweaker(&self, segment_reader: &SegmentReader) -> Result<Self::Child>;
-}
-
-impl<TScoreTweaker, TScore> Collector for TweakedScoreTopCollector<TScoreTweaker, TScore>
-where
-    TScoreTweaker: ScoreTweaker<TScore> + Send + Sync,
-    TScore: 'static + PartialOrd + Clone + Send + Sync,
-{
-    type Fruit = Vec<(TScore, DocAddress)>;
-
-    type Child = TopTweakedScoreSegmentCollector<TScoreTweaker::Child, TScore>;
-
-    fn for_segment(
-        &self,
-        segment_local_id: u32,
-        segment_reader: &SegmentReader,
-    ) -> Result<Self::Child> {
-        let segment_scorer = self.score_tweaker.segment_tweaker(segment_reader)?;
-        let segment_collector = self
-            .collector
-            .for_segment(segment_local_id, segment_reader)?;
-        Ok(TopTweakedScoreSegmentCollector {
-            segment_collector,
-            segment_scorer,
-        })
-    }
-
-    fn requires_scoring(&self) -> bool {
-        true
-    }
-
-    fn merge_fruits(&self, segment_fruits: Vec<Self::Fruit>) -> Result<Self::Fruit> {
-        self.collector.merge_fruits(segment_fruits)
-    }
-}
-
-pub struct TopTweakedScoreSegmentCollector<TSegmentScoreTweaker, TScore>
-where
-    TScore: 'static + PartialOrd + Clone + Send + Sync + Sized,
-    TSegmentScoreTweaker: ScoreSegmentTweaker<TScore>,
-{
-    segment_collector: TopSegmentCollector<TScore>,
-    segment_scorer: TSegmentScoreTweaker,
-}
-
-impl<TSegmentScoreTweaker, TScore> SegmentCollector
-    for TopTweakedScoreSegmentCollector<TSegmentScoreTweaker, TScore>
-where
-    TScore: 'static + PartialOrd + Clone + Send + Sync,
-    TSegmentScoreTweaker: 'static + ScoreSegmentTweaker<TScore>,
-{
-    type Fruit = Vec<(TScore, DocAddress)>;
-
-    fn collect(&mut self, doc: DocId, score: Score) {
-        let score = self.segment_scorer.score(doc, score);
-        self.segment_collector.collect(doc, score);
-    }
-
-    fn harvest(self) -> Vec<(TScore, DocAddress)> {
-        self.segment_collector.harvest()
-    }
-}
-
-impl<F, TScore, TSegmentScoreTweaker> ScoreTweaker<TScore> for F
-where
-    F: 'static + Send + Sync + Fn(&SegmentReader) -> TSegmentScoreTweaker,
-    TSegmentScoreTweaker: ScoreSegmentTweaker<TScore>,
-{
-    type Child = TSegmentScoreTweaker;
-
-    fn segment_tweaker(&self, segment_reader: &SegmentReader) -> Result<Self::Child> {
-        Ok((self)(segment_reader))
-    }
-}
-
-impl<F, TScore> ScoreSegmentTweaker<TScore> for F
-where
-    F: 'static + FnMut(DocId, Score) -> TScore,
-{
-    fn score(&mut self, doc: DocId, score: Score) -> TScore {
-        (self)(doc, score)
-    }
-}

src/common/bitpacker.rs

@@ -1,6 +1,9 @@
-use byteorder::{ByteOrder, LittleEndian, WriteBytesExt};
+use common::serialize::BinarySerializable;
 use std::io;
+use std::io::Write;
+use std::mem;
 use std::ops::Deref;
+use std::ptr;
 
 pub(crate) struct BitPacker {
     mini_buffer: u64,
@@ -15,7 +18,7 @@ impl BitPacker {
         }
     }
 
-    pub fn write<TWrite: io::Write>(
+    pub fn write<TWrite: Write>(
         &mut self,
         val: u64,
         num_bits: u8,
@@ -25,14 +28,14 @@ impl BitPacker {
         let num_bits = num_bits as usize;
         if self.mini_buffer_written + num_bits > 64 {
             self.mini_buffer |= val_u64.wrapping_shl(self.mini_buffer_written as u32);
-            output.write_u64::<LittleEndian>(self.mini_buffer)?;
+            self.mini_buffer.serialize(output)?;
             self.mini_buffer = val_u64.wrapping_shr((64 - self.mini_buffer_written) as u32);
             self.mini_buffer_written = self.mini_buffer_written + num_bits - 64;
         } else {
             self.mini_buffer |= val_u64 << self.mini_buffer_written;
             self.mini_buffer_written += num_bits;
             if self.mini_buffer_written == 64 {
-                output.write_u64::<LittleEndian>(self.mini_buffer)?;
+                self.mini_buffer.serialize(output)?;
                 self.mini_buffer_written = 0;
                 self.mini_buffer = 0u64;
             }
@@ -40,18 +43,17 @@ impl BitPacker {
         Ok(())
     }
 
-    pub fn flush<TWrite: io::Write>(&mut self, output: &mut TWrite) -> io::Result<()> {
+    pub fn flush<TWrite: Write>(&mut self, output: &mut TWrite) -> io::Result<()> {
         if self.mini_buffer_written > 0 {
             let num_bytes = (self.mini_buffer_written + 7) / 8;
-            let mut arr: [u8; 8] = [0u8; 8];
-            LittleEndian::write_u64(&mut arr, self.mini_buffer);
+            let arr: [u8; 8] = unsafe { mem::transmute::<u64, [u8; 8]>(self.mini_buffer.to_le()) };
             output.write_all(&arr[..num_bytes])?;
             self.mini_buffer_written = 0;
         }
         Ok(())
     }
 
-    pub fn close<TWrite: io::Write>(&mut self, output: &mut TWrite) -> io::Result<()> {
+    pub fn close<TWrite: Write>(&mut self, output: &mut TWrite) -> io::Result<()> {
         self.flush(output)?;
         // Padding the write file to simplify reads.
         output.write_all(&[0u8; 7])?;
@@ -64,7 +66,7 @@ pub struct BitUnpacker<Data>
 where
     Data: Deref<Target = [u8]>,
 {
-    num_bits: u64,
+    num_bits: usize,
     mask: u64,
     data: Data,
 }
@@ -80,13 +82,13 @@ where
             (1u64 << num_bits) - 1u64
         };
         BitUnpacker {
-            num_bits: u64::from(num_bits),
+            num_bits: num_bits as usize,
             mask,
             data,
         }
     }
 
-    pub fn get(&self, idx: u64) -> u64 {
+    pub fn get(&self, idx: usize) -> u64 {
         if self.num_bits == 0 {
             return 0u64;
         }
@@ -97,13 +99,42 @@ where
         let addr = addr_in_bits >> 3;
         let bit_shift = addr_in_bits & 7;
         debug_assert!(
-            addr + 8 <= data.len() as u64,
+            addr + 8 <= data.len(),
             "The fast field field should have been padded with 7 bytes."
         );
-        let val_unshifted_unmasked: u64 = LittleEndian::read_u64(&data[(addr as usize)..]);
+        #[cfg_attr(feature = "cargo-clippy", allow(clippy::cast_ptr_alignment))]
+        let val_unshifted_unmasked: u64 =
+            u64::from_le(unsafe { ptr::read_unaligned(data[addr..].as_ptr() as *const u64) });
         let val_shifted = (val_unshifted_unmasked >> bit_shift) as u64;
         val_shifted & mask
     }
+
+    /// Reads a range of values from the fast field.
+    ///
+    /// The range of values read is from
+    /// `[start..start + output.len()[`
+    pub fn get_range(&self, start: u32, output: &mut [u64]) {
+        if self.num_bits == 0 {
+            for val in output.iter_mut() {
+                *val = 0u64;
+            }
+        } else {
+            let data: &[u8] = &*self.data;
+            let num_bits = self.num_bits;
+            let mask = self.mask;
+            let mut addr_in_bits = (start as usize) * num_bits;
+            for output_val in output.iter_mut() {
+                let addr = addr_in_bits >> 3;
+                let bit_shift = addr_in_bits & 7;
+                #[cfg_attr(feature = "cargo-clippy", allow(clippy::cast_ptr_alignment))]
+                let val_unshifted_unmasked: u64 =
+                    unsafe { ptr::read_unaligned(data[addr..].as_ptr() as *const u64) };
+                let val_shifted = (val_unshifted_unmasked >> bit_shift) as u64;
+                *output_val = val_shifted & mask;
+                addr_in_bits += num_bits;
+            }
+        }
+    }
 }
 
 #[cfg(test)]
@@ -129,7 +160,7 @@ mod test {
     fn test_bitpacker_util(len: usize, num_bits: u8) {
         let (bitunpacker, vals) = create_fastfield_bitpacker(len, num_bits);
         for (i, val) in vals.iter().enumerate() {
-            assert_eq!(bitunpacker.get(i as u64), *val);
+            assert_eq!(bitunpacker.get(i), *val);
         }
     }
 
@@ -141,4 +172,17 @@ mod test {
         test_bitpacker_util(6, 14);
         test_bitpacker_util(1000, 14);
     }
+
+    #[test]
+    fn test_bitpacker_range() {
+        let (bitunpacker, vals) = create_fastfield_bitpacker(100_000, 12);
+        let buffer_len = 100;
+        let mut buffer = vec![0u64; buffer_len];
+        for start in vec![0, 10, 20, 100, 1_000] {
+            bitunpacker.get_range(start as u32, &mut buffer[..]);
+            for i in 0..buffer_len {
+                assert_eq!(buffer[i], vals[start + i]);
+            }
+        }
+    }
 }

src/common/bitset.rs

@@ -5,7 +5,7 @@ use std::u64;
 pub(crate) struct TinySet(u64);
 
 impl fmt::Debug for TinySet {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         self.into_iter().collect::<Vec<u32>>().fmt(f)
     }
 }
@@ -33,10 +33,6 @@ impl TinySet {
         TinySet(0u64)
     }
 
-    pub fn clear(&mut self) {
-        self.0 = 0u64;
-    }
-
     /// Returns the complement of the set in `[0, 64[`.
     fn complement(self) -> TinySet {
         TinySet(!self.0)
@@ -47,11 +43,6 @@ impl TinySet {
         !self.intersect(TinySet::singleton(el)).is_empty()
     }
 
-    /// Returns the number of elements in the TinySet.
-    pub fn len(self) -> u32 {
-        self.0.count_ones()
-    }
-
     /// Returns the intersection of `self` and `other`
     pub fn intersect(self, other: TinySet) -> TinySet {
         TinySet(self.0 & other.0)
@@ -118,12 +109,22 @@ impl TinySet {
     pub fn range_greater_or_equal(from_included: u32) -> TinySet {
         TinySet::range_lower(from_included).complement()
     }
+
+    pub fn clear(&mut self) {
+        self.0 = 0u64;
+    }
+
+    pub fn len(self) -> u32 {
+        self.0.count_ones()
+    }
 }
 
 #[derive(Clone)]
 pub struct BitSet {
     tinysets: Box<[TinySet]>,
-    len: usize,
+    len: usize, //< Technically it should be u32, but we
+    // count multiple inserts.
+    // `usize` guards us from overflow.
     max_value: u32,
 }
 
@@ -203,12 +204,12 @@ mod tests {
 
     use super::BitSet;
     use super::TinySet;
-    use crate::docset::{DocSet, TERMINATED};
-    use crate::query::BitSetDocSet;
-    use crate::tests;
-    use crate::tests::generate_nonunique_unsorted;
+    use docset::DocSet;
+    use query::BitSetDocSet;
     use std::collections::BTreeSet;
     use std::collections::HashSet;
+    use tests;
+    use tests::generate_nonunique_unsorted;
 
     #[test]
     fn test_tiny_set() {
@@ -277,13 +278,11 @@ mod tests {
         }
         assert_eq!(btreeset.len(), bitset.len());
         let mut bitset_docset = BitSetDocSet::from(bitset);
-        let mut remaining = true;
         for el in btreeset.into_iter() {
-            assert!(remaining);
+            bitset_docset.advance();
             assert_eq!(bitset_docset.doc(), el);
-            remaining = bitset_docset.advance() != TERMINATED;
         }
-        assert!(!remaining);
+        assert!(!bitset_docset.advance());
     }
 
     #[test]

src/common/composite_file.rs

@@ -1,11 +1,11 @@
-use crate::common::BinarySerializable;
-use crate::common::CountingWriter;
-use crate::common::VInt;
-use crate::directory::ReadOnlySource;
-use crate::directory::{TerminatingWrite, WritePtr};
-use crate::schema::Field;
-use crate::space_usage::FieldUsage;
-use crate::space_usage::PerFieldSpaceUsage;
+use common::BinarySerializable;
+use common::CountingWriter;
+use common::VInt;
+use directory::ReadOnlySource;
+use directory::WritePtr;
+use schema::Field;
+use space_usage::FieldUsage;
+use space_usage::PerFieldSpaceUsage;
 use std::collections::HashMap;
 use std::io::Write;
 use std::io::{self, Read};
@@ -39,10 +39,10 @@ impl BinarySerializable for FileAddr {
 /// A `CompositeWrite` is used to write a `CompositeFile`.
 pub struct CompositeWrite<W = WritePtr> {
     write: CountingWriter<W>,
-    offsets: HashMap<FileAddr, u64>,
+    offsets: HashMap<FileAddr, usize>,
 }
 
-impl<W: TerminatingWrite + Write> CompositeWrite<W> {
+impl<W: Write> CompositeWrite<W> {
     /// Crate a new API writer that writes a composite file
     /// in a given write.
     pub fn wrap(w: W) -> CompositeWrite<W> {
@@ -91,7 +91,8 @@ impl<W: TerminatingWrite + Write> CompositeWrite<W> {
 
         let footer_len = (self.write.written_bytes() - footer_offset) as u32;
         footer_len.serialize(&mut self.write)?;
-        self.write.terminate()
+        self.write.flush()?;
+        Ok(())
     }
 }
 
@@ -184,10 +185,10 @@ impl CompositeFile {
 mod test {
 
     use super::{CompositeFile, CompositeWrite};
-    use crate::common::BinarySerializable;
-    use crate::common::VInt;
-    use crate::directory::{Directory, RAMDirectory};
-    use crate::schema::Field;
+    use common::BinarySerializable;
+    use common::VInt;
+    use directory::{Directory, RAMDirectory};
+    use schema::Field;
     use std::io::Write;
     use std::path::Path;
 
@@ -199,13 +200,13 @@ mod test {
             let w = directory.open_write(path).unwrap();
             let mut composite_write = CompositeWrite::wrap(w);
             {
-                let mut write_0 = composite_write.for_field(Field::from_field_id(0u32));
+                let mut write_0 = composite_write.for_field(Field(0u32));
                 VInt(32431123u64).serialize(&mut write_0).unwrap();
                 write_0.flush().unwrap();
             }
 
             {
-                let mut write_4 = composite_write.for_field(Field::from_field_id(4u32));
+                let mut write_4 = composite_write.for_field(Field(4u32));
                 VInt(2).serialize(&mut write_4).unwrap();
                 write_4.flush().unwrap();
             }
@@ -215,18 +216,14 @@ mod test {
             let r = directory.open_read(path).unwrap();
             let composite_file = CompositeFile::open(&r).unwrap();
             {
-                let file0 = composite_file
-                    .open_read(Field::from_field_id(0u32))
-                    .unwrap();
+                let file0 = composite_file.open_read(Field(0u32)).unwrap();
                 let mut file0_buf = file0.as_slice();
                 let payload_0 = VInt::deserialize(&mut file0_buf).unwrap().0;
                 assert_eq!(file0_buf.len(), 0);
                 assert_eq!(payload_0, 32431123u64);
             }
             {
-                let file4 = composite_file
-                    .open_read(Field::from_field_id(4u32))
-                    .unwrap();
+                let file4 = composite_file.open_read(Field(4u32)).unwrap();
                 let mut file4_buf = file4.as_slice();
                 let payload_4 = VInt::deserialize(&mut file4_buf).unwrap().0;
                 assert_eq!(file4_buf.len(), 0);
@@ -234,4 +231,5 @@ mod test {
             }
         }
     }
+
 }

src/common/counting_writer.rs

@@ -1,11 +1,9 @@
-use crate::directory::AntiCallToken;
-use crate::directory::TerminatingWrite;
 use std::io;
 use std::io::Write;
 
 pub struct CountingWriter<W> {
     underlying: W,
-    written_bytes: u64,
+    written_bytes: usize,
 }
 
 impl<W: Write> CountingWriter<W> {
@@ -16,11 +14,11 @@ impl<W: Write> CountingWriter<W> {
         }
     }
 
-    pub fn written_bytes(&self) -> u64 {
+    pub fn written_bytes(&self) -> usize {
         self.written_bytes
     }
 
-    pub fn finish(mut self) -> io::Result<(W, u64)> {
+    pub fn finish(mut self) -> io::Result<(W, usize)> {
         self.flush()?;
         Ok((self.underlying, self.written_bytes))
     }
@@ -29,28 +27,15 @@ impl<W: Write> CountingWriter<W> {
 impl<W: Write> Write for CountingWriter<W> {
     fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
         let written_size = self.underlying.write(buf)?;
-        self.written_bytes += written_size as u64;
+        self.written_bytes += written_size;
         Ok(written_size)
     }
 
-    fn write_all(&mut self, buf: &[u8]) -> io::Result<()> {
-        self.underlying.write_all(buf)?;
-        self.written_bytes += buf.len() as u64;
-        Ok(())
-    }
-
     fn flush(&mut self) -> io::Result<()> {
         self.underlying.flush()
     }
 }
 
-impl<W: TerminatingWrite> TerminatingWrite for CountingWriter<W> {
-    fn terminate_ref(&mut self, token: AntiCallToken) -> io::Result<()> {
-        self.flush()?;
-        self.underlying.terminate_ref(token)
-    }
-}
-
 #[cfg(test)]
 mod test {
 
@@ -63,8 +48,8 @@ mod test {
         let mut counting_writer = CountingWriter::wrap(buffer);
         let bytes = (0u8..10u8).collect::<Vec<u8>>();
         counting_writer.write_all(&bytes).unwrap();
-        let (w, len): (Vec<u8>, u64) = counting_writer.finish().unwrap();
-        assert_eq!(len, 10u64);
+        let (w, len): (Vec<u8>, usize) = counting_writer.finish().unwrap();
+        assert_eq!(len, 10);
         assert_eq!(w.len(), 10);
     }
 }

src/common/mod.rs

@@ -10,28 +10,10 @@ pub(crate) use self::bitset::TinySet;
 pub(crate) use self::composite_file::{CompositeFile, CompositeWrite};
 pub use self::counting_writer::CountingWriter;
 pub use self::serialize::{BinarySerializable, FixedSize};
-pub use self::vint::{
-    read_u32_vint, read_u32_vint_no_advance, serialize_vint_u32, write_u32_vint, VInt,
-};
+pub use self::vint::VInt;
 pub use byteorder::LittleEndian as Endianness;
 
-/// Segment's max doc must be `< MAX_DOC_LIMIT`.
-///
-/// We do not allow segments with more than
-pub const MAX_DOC_LIMIT: u32 = 1 << 31;
-
-pub fn minmax<I, T>(mut vals: I) -> Option<(T, T)>
-where
-    I: Iterator<Item = T>,
-    T: Copy + Ord,
-{
-    if let Some(first_el) = vals.next() {
-        return Some(vals.fold((first_el, first_el), |(min_val, max_val), el| {
-            (min_val.min(el), max_val.max(el))
-        }));
-    }
-    None
-}
+use std::io;
 
 /// Computes the number of bits that will be used for bitpacking.
 ///
@@ -70,6 +52,11 @@ pub(crate) fn is_power_of_2(n: usize) -> bool {
     (n > 0) && (n & (n - 1) == 0)
 }
 
+/// Create a default io error given a string.
+pub(crate) fn make_io_err(msg: String) -> io::Error {
+    io::Error::new(io::ErrorKind::Other, msg)
+}
+
 /// Has length trait
 pub trait HasLen {
     /// Return length
@@ -114,54 +101,16 @@ pub fn u64_to_i64(val: u64) -> i64 {
     (val ^ HIGHEST_BIT) as i64
 }
 
-/// Maps a `f64` to `u64`
-///
-/// For simplicity, tantivy internally handles `f64` as `u64`.
-/// The mapping is defined by this function.
-///
-/// Maps `f64` to `u64` so that lexical order is preserved.
-///
-/// This is more suited than simply casting (`val as u64`)
-/// which would truncate the result
-///
-/// # See also
-/// The [reverse mapping is `u64_to_f64`](./fn.u64_to_f64.html).
-#[inline(always)]
-pub fn f64_to_u64(val: f64) -> u64 {
-    let bits = val.to_bits();
-    if val.is_sign_positive() {
-        bits ^ HIGHEST_BIT
-    } else {
-        !bits
-    }
-}
-
-/// Reverse the mapping given by [`i64_to_u64`](./fn.i64_to_u64.html).
-#[inline(always)]
-pub fn u64_to_f64(val: u64) -> f64 {
-    f64::from_bits(if val & HIGHEST_BIT != 0 {
-        val ^ HIGHEST_BIT
-    } else {
-        !val
-    })
-}
-
 #[cfg(test)]
 pub(crate) mod test {
 
-    pub use super::minmax;
     pub use super::serialize::test::fixed_size_test;
-    use super::{compute_num_bits, f64_to_u64, i64_to_u64, u64_to_f64, u64_to_i64};
-    use std::f64;
+    use super::{compute_num_bits, i64_to_u64, u64_to_i64};
 
     fn test_i64_converter_helper(val: i64) {
         assert_eq!(u64_to_i64(i64_to_u64(val)), val);
     }
 
-    fn test_f64_converter_helper(val: f64) {
-        assert_eq!(u64_to_f64(f64_to_u64(val)), val);
-    }
-
     #[test]
     fn test_i64_converter() {
         assert_eq!(i64_to_u64(i64::min_value()), u64::min_value());
@@ -174,29 +123,6 @@ pub(crate) mod test {
         }
     }
 
-    #[test]
-    fn test_f64_converter() {
-        test_f64_converter_helper(f64::INFINITY);
-        test_f64_converter_helper(f64::NEG_INFINITY);
-        test_f64_converter_helper(0.0);
-        test_f64_converter_helper(-0.0);
-        test_f64_converter_helper(1.0);
-        test_f64_converter_helper(-1.0);
-    }
-
-    #[test]
-    fn test_f64_order() {
-        assert!(!(f64_to_u64(f64::NEG_INFINITY)..f64_to_u64(f64::INFINITY))
-            .contains(&f64_to_u64(f64::NAN))); //nan is not a number
-        assert!(f64_to_u64(1.5) > f64_to_u64(1.0)); //same exponent, different mantissa
-        assert!(f64_to_u64(2.0) > f64_to_u64(1.0)); //same mantissa, different exponent
-        assert!(f64_to_u64(2.0) > f64_to_u64(1.5)); //different exponent and mantissa
-        assert!(f64_to_u64(1.0) > f64_to_u64(-1.0)); // pos > neg
-        assert!(f64_to_u64(-1.5) < f64_to_u64(-1.0));
-        assert!(f64_to_u64(-2.0) < f64_to_u64(1.0));
-        assert!(f64_to_u64(-2.0) < f64_to_u64(-1.5));
-    }
-
     #[test]
     fn test_compute_num_bits() {
         assert_eq!(compute_num_bits(1), 1u8);
@@ -208,28 +134,4 @@ pub(crate) mod test {
         assert_eq!(compute_num_bits(256), 9u8);
         assert_eq!(compute_num_bits(5_000_000_000), 33u8);
     }
-
-    #[test]
-    fn test_max_doc() {
-        // this is the first time I write a unit test for a constant.
-        assert!(((super::MAX_DOC_LIMIT - 1) as i32) >= 0);
-        assert!((super::MAX_DOC_LIMIT as i32) < 0);
-    }
-
-    #[test]
-    fn test_minmax_empty() {
-        let vals: Vec<u32> = vec![];
-        assert_eq!(minmax(vals.into_iter()), None);
-    }
-
-    #[test]
-    fn test_minmax_one() {
-        assert_eq!(minmax(vec![1].into_iter()), Some((1, 1)));
-    }
-
-    #[test]
-    fn test_minmax_two() {
-        assert_eq!(minmax(vec![1, 2].into_iter()), Some((1, 2)));
-        assert_eq!(minmax(vec![2, 1].into_iter()), Some((1, 2)));
-    }
 }

src/common/serialize.rs

@@ -1,6 +1,6 @@
-use crate::common::Endianness;
-use crate::common::VInt;
 use byteorder::{ReadBytesExt, WriteBytesExt};
+use common::Endianness;
+use common::VInt;
 use std::fmt;
 use std::io;
 use std::io::Read;
@@ -89,19 +89,6 @@ impl FixedSize for u64 {
     const SIZE_IN_BYTES: usize = 8;
 }
 
-impl BinarySerializable for f32 {
-    fn serialize<W: Write>(&self, writer: &mut W) -> io::Result<()> {
-        writer.write_f32::<Endianness>(*self)
-    }
-    fn deserialize<R: Read>(reader: &mut R) -> io::Result<Self> {
-        reader.read_f32::<Endianness>()
-    }
-}
-
-impl FixedSize for f32 {
-    const SIZE_IN_BYTES: usize = 4;
-}
-
 impl BinarySerializable for i64 {
     fn serialize<W: Write>(&self, writer: &mut W) -> io::Result<()> {
         writer.write_i64::<Endianness>(*self)
@@ -115,19 +102,6 @@ impl FixedSize for i64 {
     const SIZE_IN_BYTES: usize = 8;
 }
 
-impl BinarySerializable for f64 {
-    fn serialize<W: Write>(&self, writer: &mut W) -> io::Result<()> {
-        writer.write_f64::<Endianness>(*self)
-    }
-    fn deserialize<R: Read>(reader: &mut R) -> io::Result<Self> {
-        reader.read_f64::<Endianness>()
-    }
-}
-
-impl FixedSize for f64 {
-    const SIZE_IN_BYTES: usize = 8;
-}
-
 impl BinarySerializable for u8 {
     fn serialize<W: Write>(&self, writer: &mut W) -> io::Result<()> {
         writer.write_u8(*self)
@@ -162,7 +136,7 @@ impl BinarySerializable for String {
 pub mod test {
 
     use super::*;
-    use crate::common::VInt;
+    use common::VInt;
 
     pub fn fixed_size_test<O: BinarySerializable + FixedSize + Default>() {
         let mut buffer = Vec::new();
@@ -198,11 +172,6 @@ pub mod test {
         fixed_size_test::<i64>();
     }
 
-    #[test]
-    fn test_serialize_f64() {
-        fixed_size_test::<f64>();
-    }
-
     #[test]
     fn test_serialize_u64() {
         fixed_size_test::<u64>();
@@ -212,7 +181,10 @@ pub mod test {
     fn test_serialize_string() {
         assert_eq!(serialize_test(String::from("")), 1);
         assert_eq!(serialize_test(String::from("ぽよぽよ")), 1 + 3 * 4);
-        assert_eq!(serialize_test(String::from("富士さん見える。")), 1 + 3 * 8);
+        assert_eq!(
+            serialize_test(String::from("富士さん見える。")),
+            1 + 3 * 8
+        );
     }
 
     #[test]

src/common/vint.rs

@@ -1,114 +1,14 @@
 use super::BinarySerializable;
-use byteorder::{ByteOrder, LittleEndian};
 use std::io;
 use std::io::Read;
 use std::io::Write;
 
 ///   Wrapper over a `u64` that serializes as a variable int.
-#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+#[derive(Debug, Eq, PartialEq)]
 pub struct VInt(pub u64);
 
 const STOP_BIT: u8 = 128;
 
-pub fn serialize_vint_u32(val: u32, buf: &mut [u8; 8]) -> &[u8] {
-    const START_2: u64 = 1 << 7;
-    const START_3: u64 = 1 << 14;
-    const START_4: u64 = 1 << 21;
-    const START_5: u64 = 1 << 28;
-
-    const STOP_1: u64 = START_2 - 1;
-    const STOP_2: u64 = START_3 - 1;
-    const STOP_3: u64 = START_4 - 1;
-    const STOP_4: u64 = START_5 - 1;
-
-    const MASK_1: u64 = 127;
-    const MASK_2: u64 = MASK_1 << 7;
-    const MASK_3: u64 = MASK_2 << 7;
-    const MASK_4: u64 = MASK_3 << 7;
-    const MASK_5: u64 = MASK_4 << 7;
-
-    let val = u64::from(val);
-    const STOP_BIT: u64 = 128u64;
-    let (res, num_bytes) = match val {
-        0..=STOP_1 => (val | STOP_BIT, 1),
-        START_2..=STOP_2 => (
-            (val & MASK_1) | ((val & MASK_2) << 1) | (STOP_BIT << (8)),
-            2,
-        ),
-        START_3..=STOP_3 => (
-            (val & MASK_1) | ((val & MASK_2) << 1) | ((val & MASK_3) << 2) | (STOP_BIT << (8 * 2)),
-            3,
-        ),
-        START_4..=STOP_4 => (
-            (val & MASK_1)
-                | ((val & MASK_2) << 1)
-                | ((val & MASK_3) << 2)
-                | ((val & MASK_4) << 3)
-                | (STOP_BIT << (8 * 3)),
-            4,
-        ),
-        _ => (
-            (val & MASK_1)
-                | ((val & MASK_2) << 1)
-                | ((val & MASK_3) << 2)
-                | ((val & MASK_4) << 3)
-                | ((val & MASK_5) << 4)
-                | (STOP_BIT << (8 * 4)),
-            5,
-        ),
-    };
-    LittleEndian::write_u64(&mut buf[..], res);
-    &buf[0..num_bytes]
-}
-
-/// Returns the number of bytes covered by a
-/// serialized vint `u32`.
-///
-/// Expects a buffer data that starts
-/// by the serialized `vint`, scans at most 5 bytes ahead until
-/// it finds the vint final byte.
-///
-/// # May Panic
-/// If the payload does not start by a valid `vint`
-fn vint_len(data: &[u8]) -> usize {
-    for (i, &val) in data.iter().enumerate().take(5) {
-        if val >= STOP_BIT {
-            return i + 1;
-        }
-    }
-    panic!("Corrupted data. Invalid VInt 32");
-}
-
-/// Reads a vint `u32` from a buffer, and
-/// consumes its payload data.
-///
-/// # Panics
-///
-/// If the buffer does not start by a valid
-/// vint payload
-pub fn read_u32_vint(data: &mut &[u8]) -> u32 {
-    let (result, vlen) = read_u32_vint_no_advance(*data);
-    *data = &data[vlen..];
-    result
-}
-
-pub fn read_u32_vint_no_advance(data: &[u8]) -> (u32, usize) {
-    let vlen = vint_len(data);
-    let mut result = 0u32;
-    let mut shift = 0u64;
-    for &b in &data[..vlen] {
-        result |= u32::from(b & 127u8) << shift;
-        shift += 7;
-    }
-    (result, vlen)
-}
-/// Write a `u32` as a vint payload.
-pub fn write_u32_vint<W: io::Write>(val: u32, writer: &mut W) -> io::Result<()> {
-    let mut buf = [0u8; 8];
-    let data = serialize_vint_u32(val, &mut buf);
-    writer.write_all(&data)
-}
-
 impl VInt {
     pub fn val(&self) -> u64 {
         self.0
@@ -124,7 +24,7 @@ impl VInt {
         output.extend(&buffer[0..num_bytes]);
     }
 
-    pub fn serialize_into(&self, buffer: &mut [u8; 10]) -> usize {
+    fn serialize_into(&self, buffer: &mut [u8; 10]) -> usize {
         let mut remaining = self.0;
         for (i, b) in buffer.iter_mut().enumerate() {
             let next_byte: u8 = (remaining % 128u64) as u8;
@@ -164,7 +64,7 @@ impl BinarySerializable for VInt {
                     return Err(io::Error::new(
                         io::ErrorKind::InvalidData,
                         "Reach end of buffer while reading VInt",
-                    ));
+                    ))
                 }
             }
         }
@@ -174,9 +74,8 @@ impl BinarySerializable for VInt {
 #[cfg(test)]
 mod tests {
 
-    use super::serialize_vint_u32;
     use super::VInt;
-    use crate::common::BinarySerializable;
+    use common::BinarySerializable;
 
     fn aux_test_vint(val: u64) {
         let mut v = [14u8; 10];
@@ -209,26 +108,4 @@ mod tests {
         }
         aux_test_vint(10);
     }
-
-    fn aux_test_serialize_vint_u32(val: u32) {
-        let mut buffer = [0u8; 10];
-        let mut buffer2 = [0u8; 8];
-        let len_vint = VInt(val as u64).serialize_into(&mut buffer);
-        let res2 = serialize_vint_u32(val, &mut buffer2);
-        assert_eq!(&buffer[..len_vint], res2, "array wrong for {}", val);
-    }
-
-    #[test]
-    fn test_vint_u32() {
-        aux_test_serialize_vint_u32(0);
-        aux_test_serialize_vint_u32(1);
-        aux_test_serialize_vint_u32(5);
-        for i in 1..3 {
-            let power_of_128 = 1u32 << (7 * i);
-            aux_test_serialize_vint_u32(power_of_128 - 1u32);
-            aux_test_serialize_vint_u32(power_of_128);
-            aux_test_serialize_vint_u32(power_of_128 + 1u32);
-        }
-        aux_test_serialize_vint_u32(u32::max_value());
-    }
 }

src/core/executor.rs

@@ -1,6 +1,6 @@
 use crossbeam::channel;
-use rayon::{ThreadPool, ThreadPoolBuilder};
-use slog::{error, Logger};
+use scoped_pool::{Pool, ThreadConfig};
+use Result;
 
 /// Search executor whether search request are single thread or multithread.
 ///
@@ -10,10 +10,8 @@ use slog::{error, Logger};
 /// API of a dependency, knowing it might conflict with a different version
 /// used by the client. Second, we may stop using rayon in the future.
 pub enum Executor {
-    /// Single thread variant of an Executor
     SingleThread,
-    /// Thread pool variant of an Executor
-    ThreadPool(ThreadPool),
+    ThreadPool(Pool),
 }
 
 impl Executor {
@@ -22,44 +20,41 @@ impl Executor {
         Executor::SingleThread
     }
 
-    /// Creates an Executor that dispatches the tasks in a thread pool.
-    pub fn multi_thread(num_threads: usize, prefix: &'static str) -> crate::Result<Executor> {
-        let pool = ThreadPoolBuilder::new()
-            .num_threads(num_threads)
-            .thread_name(move |num| format!("{}{}", prefix, num))
-            .build()?;
-        Ok(Executor::ThreadPool(pool))
+    // Creates an Executor that dispatches the tasks in a thread pool.
+    pub fn multi_thread(num_threads: usize, prefix: &'static str) -> Executor {
+        let thread_config = ThreadConfig::new().prefix(prefix);
+        let pool = Pool::with_thread_config(num_threads, thread_config);
+        Executor::ThreadPool(pool)
     }
 
-    /// Perform a map in the thread pool.
-    ///
-    /// Regardless of the executor (`SingleThread` or `ThreadPool`), panics in the task
-    /// will propagate to the caller.
+    // Perform a map in the thread pool.
+    //
+    // Regardless of the executor (`SingleThread` or `ThreadPool`), panics in the task
+    // will propagate to the caller.
     pub fn map<
         A: Send,
         R: Send,
         AIterator: Iterator<Item = A>,
-        F: Sized + Sync + Fn(A) -> crate::Result<R>,
+        F: Sized + Sync + Fn(A) -> Result<R>,
     >(
         &self,
         f: F,
         args: AIterator,
-        logger: Logger,
-    ) -> crate::Result<Vec<R>> {
+    ) -> Result<Vec<R>> {
         match self {
-            Executor::SingleThread => args.map(f).collect::<crate::Result<_>>(),
+            Executor::SingleThread => args.map(f).collect::<Result<_>>(),
             Executor::ThreadPool(pool) => {
                 let args_with_indices: Vec<(usize, A)> = args.enumerate().collect();
                 let num_fruits = args_with_indices.len();
                 let fruit_receiver = {
                     let (fruit_sender, fruit_receiver) = channel::unbounded();
-                    pool.scope(|scope| {
+                    pool.scoped(|scope| {
                         for arg_with_idx in args_with_indices {
-                            scope.spawn(|_| {
+                            scope.execute(|| {
                                 let (idx, arg) = arg_with_idx;
                                 let fruit = f(arg);
                                 if let Err(err) = fruit_sender.send((idx, fruit)) {
-                                    error!(logger, "Failed to send search task. It probably means all search threads have panicked. {:?}", err);
+                                    error!("Failed to send search task. It probably means all search threads have panicked. {:?}", err);
                                 }
                             });
                         }
@@ -69,18 +64,17 @@ impl Executor {
                     // This is important as it makes it possible for the fruit_receiver iteration to
                     // terminate.
                 };
-                // This is lame, but safe.
-                let mut results_with_position = Vec::with_capacity(num_fruits);
+                let mut results = Vec::with_capacity(num_fruits);
+                unsafe { results.set_len(num_fruits) };
+                let mut num_items = 0;
                 for (pos, fruit_res) in fruit_receiver {
-                    let fruit = fruit_res?;
-                    results_with_position.push((pos, fruit));
+                    results[pos] = fruit_res?;
+                    num_items += 1;
                 }
-                results_with_position.sort_by_key(|(pos, _)| *pos);
-                assert_eq!(results_with_position.len(), num_fruits);
-                Ok(results_with_position
-                    .into_iter()
-                    .map(|(_, fruit)| fruit)
-                    .collect::<Vec<_>>())
+                // this checks ensures that we filled of this
+                // uninitialized memory.
+                assert_eq!(num_items, results.len());
+                Ok(results)
             }
         }
     }
@@ -89,21 +83,17 @@ impl Executor {
 #[cfg(test)]
 mod tests {
 
-    use slog::{o, Discard, Logger};
-
     use super::Executor;
 
     #[test]
     #[should_panic(expected = "panic should propagate")]
     fn test_panic_propagates_single_thread() {
-        let logger = Logger::root(Discard, o!());
         let _result: Vec<usize> = Executor::single_thread()
             .map(
                 |_| {
                     panic!("panic should propagate");
                 },
                 vec![0].into_iter(),
-                logger,
             )
             .unwrap();
     }
@@ -111,24 +101,20 @@ mod tests {
     #[test]
     #[should_panic] //< unfortunately the panic message is not propagated
     fn test_panic_propagates_multi_thread() {
-        let logger = Logger::root(Discard, o!());
         let _result: Vec<usize> = Executor::multi_thread(1, "search-test")
-            .unwrap()
             .map(
                 |_| {
                     panic!("panic should propagate");
                 },
                 vec![0].into_iter(),
-                logger,
             )
             .unwrap();
     }
 
     #[test]
     fn test_map_singlethread() {
-        let logger = Logger::root(Discard, o!());
         let result: Vec<usize> = Executor::single_thread()
-            .map(|i| Ok(i * 2), 0..1_000, logger)
+            .map(|i| Ok(i * 2), 0..1_000)
             .unwrap();
         assert_eq!(result.len(), 1_000);
         for i in 0..1_000 {
@@ -136,16 +122,15 @@ mod tests {
         }
     }
 
-    #[test]
-    fn test_map_multithread() {
-        let logger = Logger::root(Discard, o!());
-        let result: Vec<usize> = Executor::multi_thread(3, "search-test")
-            .unwrap()
-            .map(|i| Ok(i * 2), 0..10, logger)
-            .unwrap();
-        assert_eq!(result.len(), 10);
-        for i in 0..10 {
-            assert_eq!(result[i], i * 2);
-        }
+}
+
+#[test]
+fn test_map_multithread() {
+    let result: Vec<usize> = Executor::multi_thread(3, "search-test")
+        .map(|i| Ok(i * 2), 0..10)
+        .unwrap();
+    assert_eq!(result.len(), 10);
+    for i in 0..10 {
+        assert_eq!(result[i], i * 2);
     }
 }

src/core/index.rs

@@ -1,71 +1,57 @@
+use super::pool::LeasedItem;
+use super::pool::Pool;
+use super::segment::create_segment;
 use super::segment::Segment;
-use crate::core::Executor;
-use crate::core::IndexMeta;
-use crate::core::SegmentId;
-use crate::core::SegmentMeta;
-use crate::core::SegmentMetaInventory;
-use crate::core::META_FILEPATH;
-use crate::directory::ManagedDirectory;
+use core::searcher::Searcher;
+use core::Executor;
+use core::IndexMeta;
+use core::SegmentId;
+use core::SegmentMeta;
+use core::SegmentReader;
+use core::META_FILEPATH;
+use directory::ManagedDirectory;
 #[cfg(feature = "mmap")]
-use crate::directory::MmapDirectory;
-use crate::directory::INDEX_WRITER_LOCK;
-use crate::directory::{Directory, RAMDirectory};
-use crate::error::DataCorruption;
-use crate::error::TantivyError;
-use crate::indexer::index_writer::HEAP_SIZE_MIN;
-use crate::indexer::segment_updater::save_new_metas;
-use crate::reader::IndexReader;
-use crate::reader::IndexReaderBuilder;
-use crate::schema::Field;
-use crate::schema::FieldType;
-use crate::schema::Schema;
-use crate::tokenizer::{TextAnalyzer, TokenizerManager};
-use crate::IndexWriter;
-use slog::Logger;
+use directory::MmapDirectory;
+use directory::{Directory, RAMDirectory};
+use error::TantivyError;
+use indexer::index_writer::open_index_writer;
+use indexer::index_writer::HEAP_SIZE_MIN;
+use indexer::segment_updater::save_new_metas;
+use indexer::LockType;
+use num_cpus;
+use schema::Field;
+use schema::FieldType;
+use schema::Schema;
+use serde_json;
 use std::borrow::BorrowMut;
-use std::collections::HashSet;
 use std::fmt;
-
-#[cfg(feature = "mmap")]
 use std::path::Path;
-use std::path::PathBuf;
+use std::sync::atomic::{AtomicUsize, Ordering};
 use std::sync::Arc;
+use tokenizer::BoxedTokenizer;
+use tokenizer::TokenizerManager;
+use IndexWriter;
+use Result;
 
-fn load_metas(
-    directory: &dyn Directory,
-    inventory: &SegmentMetaInventory,
-) -> crate::Result<IndexMeta> {
+fn load_metas(directory: &Directory) -> Result<IndexMeta> {
     let meta_data = directory.atomic_read(&META_FILEPATH)?;
     let meta_string = String::from_utf8_lossy(&meta_data);
-    IndexMeta::deserialize(&meta_string, &inventory)
-        .map_err(|e| {
-            DataCorruption::new(
-                META_FILEPATH.to_path_buf(),
-                format!("Meta file cannot be deserialized. {:?}.", e),
-            )
-        })
-        .map_err(From::from)
+    serde_json::from_str(&meta_string)
+        .map_err(|_| TantivyError::CorruptedFile(META_FILEPATH.clone()))
 }
 
 /// Search Index
-#[derive(Clone)]
 pub struct Index {
     directory: ManagedDirectory,
     schema: Schema,
+    num_searchers: Arc<AtomicUsize>,
+    searcher_pool: Arc<Pool<Searcher>>,
     executor: Arc<Executor>,
     tokenizers: TokenizerManager,
-    inventory: SegmentMetaInventory,
 }
 
 impl Index {
-
-    pub(crate) fn logger(&self) -> &Logger {
-        self.directory.logger()
-    }
-
-    /// Examines the directory to see if it contains an index.
-    ///
-    /// Effectively, it only checks for the presence of the `meta.json` file.
+    /// Examines the director to see if it contains an index
     pub fn exists<Dir: Directory>(dir: &Dir) -> bool {
         dir.exists(&META_FILEPATH)
     }
@@ -82,16 +68,15 @@ impl Index {
 
     /// Replace the default single thread search executor pool
     /// by a thread pool with a given number of threads.
-    pub fn set_multithread_executor(&mut self, num_threads: usize) -> crate::Result<()> {
-        self.executor = Arc::new(Executor::multi_thread(num_threads, "thrd-tantivy-search-")?);
-        Ok(())
+    pub fn set_multithread_executor(&mut self, num_threads: usize) {
+        self.executor = Arc::new(Executor::multi_thread(num_threads, "thrd-tantivy-search-"));
     }
 
     /// Replace the default single thread search executor pool
     /// by a thread pool with a given number of threads.
-    pub fn set_default_multithread_executor(&mut self) -> crate::Result<()> {
+    pub fn set_default_multithread_executor(&mut self) {
         let default_num_threads = num_cpus::get();
-        self.set_multithread_executor(default_num_threads)
+        self.set_multithread_executor(default_num_threads);
     }
 
     /// Creates a new index using the `RAMDirectory`.
@@ -108,29 +93,29 @@ impl Index {
     ///
     /// If a previous index was in this directory, then its meta file will be destroyed.
     #[cfg(feature = "mmap")]
-    pub fn create_in_dir<P: AsRef<Path>>(
-        directory_path: P,
-        schema: Schema,
-    ) -> crate::Result<Index> {
+    pub fn create_in_dir<P: AsRef<Path>>(directory_path: P, schema: Schema) -> Result<Index> {
         let mmap_directory = MmapDirectory::open(directory_path)?;
         if Index::exists(&mmap_directory) {
             return Err(TantivyError::IndexAlreadyExists);
         }
+
         Index::create(mmap_directory, schema)
     }
 
     /// Opens or creates a new index in the provided directory
-    pub fn open_or_create<Dir: Directory>(dir: Dir, schema: Schema) -> crate::Result<Index> {
-        if !Index::exists(&dir) {
-            return Index::create(dir, schema);
-        }
-        let index = Index::open(dir)?;
-        if index.schema() == schema {
-            Ok(index)
+    #[cfg(feature = "mmap")]
+    pub fn open_or_create<Dir: Directory>(dir: Dir, schema: Schema) -> Result<Index> {
+        if Index::exists(&dir) {
+            let index = Index::open(dir)?;
+            if index.schema() == schema {
+                Ok(index)
+            } else {
+                Err(TantivyError::SchemaError(
+                    "An index exists but the schema does not match.".to_string(),
+                ))
+            }
         } else {
-            Err(TantivyError::SchemaError(
-                "An index exists but the schema does not match.".to_string(),
-            ))
+            Index::create(dir, schema)
         }
     }
 
@@ -143,42 +128,40 @@ impl Index {
     /// The temp directory is only used for testing the `MmapDirectory`.
     /// For other unit tests, prefer the `RAMDirectory`, see: `create_in_ram`.
     #[cfg(feature = "mmap")]
-    pub fn create_from_tempdir(schema: Schema) -> crate::Result<Index> {
+    pub fn create_from_tempdir(schema: Schema) -> Result<Index> {
         let mmap_directory = MmapDirectory::create_from_tempdir()?;
         Index::create(mmap_directory, schema)
     }
 
-    /// Creates a new index given an implementation of the trait `Directory`.
-    ///
-    /// If a directory previously existed, it will be erased.
-    pub fn create<Dir: Directory>(dir: Dir, schema: Schema) -> crate::Result<Index> {
+    /// Creates a new index given an implementation of the trait `Directory`
+    pub fn create<Dir: Directory>(dir: Dir, schema: Schema) -> Result<Index> {
         let directory = ManagedDirectory::wrap(dir)?;
-        Index::new_from_directory(directory, schema)
+        Index::from_directory(directory, schema)
     }
 
     /// Create a new index from a directory.
     ///
     /// This will overwrite existing meta.json
-    fn new_from_directory(mut directory: ManagedDirectory, schema: Schema) -> crate::Result<Index> {
-        save_new_metas(schema.clone(), directory.borrow_mut())?;
+    fn from_directory(mut directory: ManagedDirectory, schema: Schema) -> Result<Index> {
+        save_new_metas(schema.clone(), 0, directory.borrow_mut())?;
         let metas = IndexMeta::with_schema(schema);
-        Index::create_from_metas(directory, &metas, SegmentMetaInventory::default())
+        Index::create_from_metas(directory, &metas)
     }
 
     /// Creates a new index given a directory and an `IndexMeta`.
-    fn create_from_metas(
-        directory: ManagedDirectory,
-        metas: &IndexMeta,
-        inventory: SegmentMetaInventory,
-    ) -> crate::Result<Index> {
+    fn create_from_metas(directory: ManagedDirectory, metas: &IndexMeta) -> Result<Index> {
         let schema = metas.schema.clone();
-        Ok(Index {
+        let n_cpus = num_cpus::get();
+        let index = Index {
             directory,
             schema,
+            num_searchers: Arc::new(AtomicUsize::new(n_cpus)),
+            searcher_pool: Arc::new(Pool::new()),
             tokenizers: TokenizerManager::default(),
             executor: Arc::new(Executor::single_thread()),
-            inventory,
-        })
+        };
+        index.load_searchers()?;
+        Ok(index)
     }
 
     /// Accessor for the tokenizer manager.
@@ -187,11 +170,11 @@ impl Index {
     }
 
     /// Helper to access the tokenizer associated to a specific field.
-    pub fn tokenizer_for_field(&self, field: Field) -> crate::Result<TextAnalyzer> {
+    pub fn tokenizer_for_field(&self, field: Field) -> Result<Box<BoxedTokenizer>> {
         let field_entry = self.schema.get_field_entry(field);
         let field_type = field_entry.field_type();
         let tokenizer_manager: &TokenizerManager = self.tokenizers();
-        let tokenizer_name_opt: Option<TextAnalyzer> = match field_type {
+        let tokenizer_name_opt: Option<Box<BoxedTokenizer>> = match field_type {
             FieldType::Str(text_options) => text_options
                 .get_indexing_options()
                 .map(|text_indexing_options| text_indexing_options.tokenizer().to_string())
@@ -207,60 +190,23 @@ impl Index {
         }
     }
 
-    /// Create a default `IndexReader` for the given index.
-    ///
-    /// See [`Index.reader_builder()`](#method.reader_builder).
-    pub fn reader(&self) -> crate::Result<IndexReader> {
-        self.reader_builder().try_into()
-    }
-
-    /// Create a `IndexReader` for the given index.
-    ///
-    /// Most project should create at most one reader for a given index.
-    /// This method is typically called only once per `Index` instance,
-    /// over the lifetime of most problem.
-    pub fn reader_builder(&self) -> IndexReaderBuilder {
-        IndexReaderBuilder::new(self.clone())
-    }
-
     /// Opens a new directory from an index path.
     #[cfg(feature = "mmap")]
-    pub fn open_in_dir<P: AsRef<Path>>(directory_path: P) -> crate::Result<Index> {
+    pub fn open_in_dir<P: AsRef<Path>>(directory_path: P) -> Result<Index> {
         let mmap_directory = MmapDirectory::open(directory_path)?;
         Index::open(mmap_directory)
     }
 
-    /// Returns the list of the segment metas tracked by the index.
-    ///
-    /// Such segments can of course be part of the index,
-    /// but also they could be segments being currently built or in the middle of a merge
-    /// operation.
-    pub fn list_all_segment_metas(&self) -> Vec<SegmentMeta> {
-        self.inventory.all()
-    }
-
-    /// Creates a new segment_meta (Advanced user only).
-    ///
-    /// As long as the `SegmentMeta` lives, the files associated with the
-    /// `SegmentMeta` are guaranteed to not be garbage collected, regardless of
-    /// whether the segment is recorded as part of the index or not.
-    pub fn new_segment_meta(&self, segment_id: SegmentId, max_doc: u32) -> SegmentMeta {
-        self.inventory.new_segment_meta(segment_id, max_doc)
-    }
-
     /// Open the index using the provided directory
-    pub fn open<D: Directory>(directory: D) -> crate::Result<Index> {
-        let logger: &Logger = directory.logger();
-        slog::info!(logger, "index-open"; "directory" => format!("{:?}", directory));
+    pub fn open<D: Directory>(directory: D) -> Result<Index> {
         let directory = ManagedDirectory::wrap(directory)?;
-        let inventory = SegmentMetaInventory::default();
-        let metas = load_metas(&directory, &inventory)?;
-        Index::create_from_metas(directory, &metas, inventory)
+        let metas = load_metas(&directory)?;
+        Index::create_from_metas(directory, &metas)
     }
 
     /// Reads the index meta file from the directory.
-    pub fn load_metas(&self) -> crate::Result<IndexMeta> {
-        load_metas(self.directory(), &self.inventory)
+    pub fn load_metas(&self) -> Result<IndexMeta> {
+        load_metas(self.directory())
     }
 
     /// Open a new index writer. Attempts to acquire a lockfile.
@@ -279,32 +225,17 @@ impl Index {
     /// Each thread will receive a budget of  `overall_heap_size_in_bytes / num_threads`.
     ///
     /// # Errors
-    /// If the lockfile already exists, returns `Error::DirectoryLockBusy` or an `Error::IOError`.
-    ///
+    /// If the lockfile already exists, returns `Error::FileAlreadyExists`.
     /// # Panics
     /// If the heap size per thread is too small, panics.
     pub fn writer_with_num_threads(
         &self,
         num_threads: usize,
         overall_heap_size_in_bytes: usize,
-    ) -> crate::Result<IndexWriter> {
-        let directory_lock = self
-            .directory
-            .acquire_lock(&INDEX_WRITER_LOCK)
-            .map_err(|err| {
-                TantivyError::LockFailure(
-                    err,
-                    Some(
-                        "Failed to acquire index lock. If you are using \
-                         a regular directory, this means there is already an \
-                         `IndexWriter` working on this `Directory`, in this process \
-                         or in a different process."
-                            .to_string(),
-                    ),
-                )
-            })?;
+    ) -> Result<IndexWriter> {
+        let directory_lock = LockType::IndexWriterLock.acquire_lock(&self.directory)?;
         let heap_size_in_bytes_per_thread = overall_heap_size_in_bytes / num_threads;
-        IndexWriter::new(
+        open_index_writer(
             self,
             num_threads,
             heap_size_in_bytes_per_thread,
@@ -312,15 +243,6 @@ impl Index {
         )
     }
 
-    /// Helper to create an index writer for tests.
-    ///
-    /// That index writer only simply has a single thread and a heap of 5 MB.
-    /// Using a single thread gives us a deterministic allocation of DocId.
-    #[cfg(test)]
-    pub fn writer_for_tests(&self) -> crate::Result<IndexWriter> {
-        self.writer_with_num_threads(1, 10_000_000)
-    }
-
     /// Creates a multithreaded writer
     ///
     /// Tantivy will automatically define the number of threads to use.
@@ -331,7 +253,7 @@ impl Index {
     /// If the lockfile already exists, returns `Error::FileAlreadyExists`.
     /// # Panics
     /// If the heap size per thread is too small, panics.
-    pub fn writer(&self, overall_heap_size_in_bytes: usize) -> crate::Result<IndexWriter> {
+    pub fn writer(&self, overall_heap_size_in_bytes: usize) -> Result<IndexWriter> {
         let mut num_threads = num_cpus::get();
         let heap_size_in_bytes_per_thread = overall_heap_size_in_bytes / num_threads;
         if heap_size_in_bytes_per_thread < HEAP_SIZE_MIN {
@@ -348,7 +270,7 @@ impl Index {
     }
 
     /// Returns the list of segments that are searchable
-    pub fn searchable_segments(&self) -> crate::Result<Vec<Segment>> {
+    pub fn searchable_segments(&self) -> Result<Vec<Segment>> {
         Ok(self
             .searchable_segment_metas()?
             .into_iter()
@@ -358,14 +280,12 @@ impl Index {
 
     #[doc(hidden)]
     pub fn segment(&self, segment_meta: SegmentMeta) -> Segment {
-        Segment::for_index(self.clone(), segment_meta)
+        create_segment(self.clone(), segment_meta)
     }
 
     /// Creates a new segment.
     pub fn new_segment(&self) -> Segment {
-        let segment_meta = self
-            .inventory
-            .new_segment_meta(SegmentId::generate_random(), 0);
+        let segment_meta = SegmentMeta::new(SegmentId::generate_random(), 0);
         self.segment(segment_meta)
     }
 
@@ -381,44 +301,96 @@ impl Index {
 
     /// Reads the meta.json and returns the list of
     /// `SegmentMeta` from the last commit.
-    pub fn searchable_segment_metas(&self) -> crate::Result<Vec<SegmentMeta>> {
+    pub fn searchable_segment_metas(&self) -> Result<Vec<SegmentMeta>> {
         Ok(self.load_metas()?.segments)
     }
 
     /// Returns the list of segment ids that are searchable.
-    pub fn searchable_segment_ids(&self) -> crate::Result<Vec<SegmentId>> {
+    pub fn searchable_segment_ids(&self) -> Result<Vec<SegmentId>> {
         Ok(self
             .searchable_segment_metas()?
             .iter()
-            .map(SegmentMeta::id)
+            .map(|segment_meta| segment_meta.id())
             .collect())
     }
 
-    /// Returns the set of corrupted files
-    pub fn validate_checksum(&self) -> crate::Result<HashSet<PathBuf>> {
-        self.directory.list_damaged().map_err(Into::into)
+    /// Sets the number of searchers to use
+    ///
+    /// Only works after the next call to `load_searchers`
+    pub fn set_num_searchers(&mut self, num_searchers: usize) {
+        self.num_searchers.store(num_searchers, Ordering::Release);
+    }
+
+    /// Update searchers so that they reflect the state of the last
+    /// `.commit()`.
+    ///
+    /// If indexing happens in the same process as searching,
+    /// you most likely want to call `.load_searchers()` right after each
+    /// successful call to `.commit()`.
+    ///
+    /// If indexing and searching happen in different processes, the way to
+    /// get the freshest `index` at all time, is to watch `meta.json` and
+    /// call `load_searchers` whenever a changes happen.
+    pub fn load_searchers(&self) -> Result<()> {
+        let _meta_lock = LockType::MetaLock.acquire_lock(self.directory())?;
+        let searchable_segments = self.searchable_segments()?;
+        let segment_readers: Vec<SegmentReader> = searchable_segments
+            .iter()
+            .map(SegmentReader::open)
+            .collect::<Result<_>>()?;
+        let schema = self.schema();
+        let num_searchers: usize = self.num_searchers.load(Ordering::Acquire);
+        let searchers = (0..num_searchers)
+            .map(|_| Searcher::new(schema.clone(), self.clone(), segment_readers.clone()))
+            .collect();
+        self.searcher_pool.publish_new_generation(searchers);
+        Ok(())
+    }
+
+    /// Returns a searcher
+    ///
+    /// This method should be called every single time a search
+    /// query is performed.
+    /// The searchers are taken from a pool of `num_searchers` searchers.
+    /// If no searcher is available
+    /// this may block.
+    ///
+    /// The same searcher must be used for a given query, as it ensures
+    /// the use of a consistent segment set.
+    pub fn searcher(&self) -> LeasedItem<Searcher> {
+        self.searcher_pool.acquire()
     }
 }
 
 impl fmt::Debug for Index {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         write!(f, "Index({:?})", self.directory)
     }
 }
 
+impl Clone for Index {
+    fn clone(&self) -> Index {
+        Index {
+            directory: self.directory.clone(),
+            schema: self.schema.clone(),
+            num_searchers: Arc::clone(&self.num_searchers),
+            searcher_pool: Arc::clone(&self.searcher_pool),
+            tokenizers: self.tokenizers.clone(),
+            executor: self.executor.clone(),
+        }
+    }
+}
+
 #[cfg(test)]
 mod tests {
-    use crate::directory::RAMDirectory;
-    use crate::schema::Field;
-    use crate::schema::{Schema, INDEXED, TEXT};
-    use crate::IndexReader;
-    use crate::ReloadPolicy;
-    use crate::{Directory, Index};
+    use directory::RAMDirectory;
+    use schema::{Schema, INT_INDEXED, TEXT};
+    use Index;
 
     #[test]
     fn test_indexer_for_field() {
         let mut schema_builder = Schema::builder();
-        let num_likes_field = schema_builder.add_u64_field("num_likes", INDEXED);
+        let num_likes_field = schema_builder.add_u64_field("num_likes", INT_INDEXED);
         let body_field = schema_builder.add_text_field("body", TEXT);
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
@@ -476,158 +448,7 @@ mod tests {
 
     fn throw_away_schema() -> Schema {
         let mut schema_builder = Schema::builder();
-        let _ = schema_builder.add_u64_field("num_likes", INDEXED);
+        let _ = schema_builder.add_u64_field("num_likes", INT_INDEXED);
         schema_builder.build()
     }
-
-    #[test]
-    fn test_index_on_commit_reload_policy() {
-        let schema = throw_away_schema();
-        let field = schema.get_field("num_likes").unwrap();
-        let index = Index::create_in_ram(schema);
-        let reader = index
-            .reader_builder()
-            .reload_policy(ReloadPolicy::OnCommit)
-            .try_into()
-            .unwrap();
-        assert_eq!(reader.searcher().num_docs(), 0);
-        test_index_on_commit_reload_policy_aux(field, &index, &reader);
-    }
-
-    #[cfg(feature = "mmap")]
-    mod mmap_specific {
-
-        use super::*;
-        use crate::Directory;
-        use std::path::PathBuf;
-        use tempfile::TempDir;
-
-        #[test]
-        fn test_index_on_commit_reload_policy_mmap() {
-            let schema = throw_away_schema();
-            let field = schema.get_field("num_likes").unwrap();
-            let tempdir = TempDir::new().unwrap();
-            let tempdir_path = PathBuf::from(tempdir.path());
-            let index = Index::create_in_dir(&tempdir_path, schema).unwrap();
-            let reader = index
-                .reader_builder()
-                .reload_policy(ReloadPolicy::OnCommit)
-                .try_into()
-                .unwrap();
-            assert_eq!(reader.searcher().num_docs(), 0);
-            test_index_on_commit_reload_policy_aux(field, &index, &reader);
-        }
-
-        #[test]
-        fn test_index_manual_policy_mmap() {
-            let schema = throw_away_schema();
-            let field = schema.get_field("num_likes").unwrap();
-            let mut index = Index::create_from_tempdir(schema).unwrap();
-            let mut writer = index.writer_for_tests().unwrap();
-            writer.commit().unwrap();
-            let reader = index
-                .reader_builder()
-                .reload_policy(ReloadPolicy::Manual)
-                .try_into()
-                .unwrap();
-            assert_eq!(reader.searcher().num_docs(), 0);
-            writer.add_document(doc!(field=>1u64));
-            let (sender, receiver) = crossbeam::channel::unbounded();
-            let _handle = index.directory_mut().watch(Box::new(move || {
-                let _ = sender.send(());
-            }));
-            writer.commit().unwrap();
-            assert!(receiver.recv().is_ok());
-            assert_eq!(reader.searcher().num_docs(), 0);
-            reader.reload().unwrap();
-            assert_eq!(reader.searcher().num_docs(), 1);
-        }
-
-        #[test]
-        fn test_index_on_commit_reload_policy_different_directories() {
-            let schema = throw_away_schema();
-            let field = schema.get_field("num_likes").unwrap();
-            let tempdir = TempDir::new().unwrap();
-            let tempdir_path = PathBuf::from(tempdir.path());
-            let write_index = Index::create_in_dir(&tempdir_path, schema).unwrap();
-            let read_index = Index::open_in_dir(&tempdir_path).unwrap();
-            let reader = read_index
-                .reader_builder()
-                .reload_policy(ReloadPolicy::OnCommit)
-                .try_into()
-                .unwrap();
-            assert_eq!(reader.searcher().num_docs(), 0);
-            test_index_on_commit_reload_policy_aux(field, &write_index, &reader);
-        }
-    }
-    fn test_index_on_commit_reload_policy_aux(field: Field, index: &Index, reader: &IndexReader) {
-        let mut reader_index = reader.index();
-        let (sender, receiver) = crossbeam::channel::unbounded();
-        let _watch_handle = reader_index.directory_mut().watch(Box::new(move || {
-            let _ = sender.send(());
-        }));
-        let mut writer = index.writer_for_tests().unwrap();
-        assert_eq!(reader.searcher().num_docs(), 0);
-        writer.add_document(doc!(field=>1u64));
-        writer.commit().unwrap();
-        // We need a loop here because it is possible for notify to send more than
-        // one modify event. It was observed on CI on MacOS.
-        loop {
-            assert!(receiver.recv().is_ok());
-            if reader.searcher().num_docs() == 1 {
-                break;
-            }
-        }
-        writer.add_document(doc!(field=>2u64));
-        writer.commit().unwrap();
-        // ... Same as above
-        loop {
-            assert!(receiver.recv().is_ok());
-            if reader.searcher().num_docs() == 2 {
-                break;
-            }
-        }
-    }
-
-    // This test will not pass on windows, because windows
-    // prevent deleting files that are MMapped.
-    #[cfg(not(target_os = "windows"))]
-    #[test]
-    fn garbage_collect_works_as_intended() {
-        let directory = RAMDirectory::create();
-        let schema = throw_away_schema();
-        let field = schema.get_field("num_likes").unwrap();
-        let index = Index::create(directory.clone(), schema).unwrap();
-
-        let mut writer = index.writer_with_num_threads(8, 24_000_000).unwrap();
-        for i in 0u64..8_000u64 {
-            writer.add_document(doc!(field => i));
-        }
-        let (sender, receiver) = crossbeam::channel::unbounded();
-        let _handle = directory.watch(Box::new(move || {
-            let _ = sender.send(());
-        }));
-        writer.commit().unwrap();
-        let mem_right_after_commit = directory.total_mem_usage();
-        assert!(receiver.recv().is_ok());
-        let reader = index
-            .reader_builder()
-            .reload_policy(ReloadPolicy::Manual)
-            .try_into()
-            .unwrap();
-
-        assert_eq!(reader.searcher().num_docs(), 8_000);
-        writer.wait_merging_threads().unwrap();
-        let mem_right_after_merge_finished = directory.total_mem_usage();
-
-        reader.reload().unwrap();
-        let searcher = reader.searcher();
-        assert_eq!(searcher.num_docs(), 8_000);
-        assert!(
-            mem_right_after_merge_finished < mem_right_after_commit,
-            "(mem after merge){} is expected < (mem before merge){}",
-            mem_right_after_merge_finished,
-            mem_right_after_commit
-        );
-    }
 }

src/core/index_meta.rs

@@ -1,198 +1,7 @@
-use super::SegmentComponent;
-use crate::core::SegmentId;
-use crate::schema::Schema;
-use crate::Opstamp;
-use census::{Inventory, TrackedObject};
-use serde::{Deserialize, Serialize};
-use std::collections::HashSet;
+use core::SegmentMeta;
+use schema::Schema;
+use serde_json;
 use std::fmt;
-use std::path::PathBuf;
-
-#[derive(Clone, Debug, Serialize, Deserialize)]
-struct DeleteMeta {
-    num_deleted_docs: u32,
-    opstamp: Opstamp,
-}
-
-#[derive(Clone, Default)]
-pub struct SegmentMetaInventory {
-    inventory: Inventory<InnerSegmentMeta>,
-}
-
-impl SegmentMetaInventory {
-    /// Lists all living `SegmentMeta` object at the time of the call.
-    pub fn all(&self) -> Vec<SegmentMeta> {
-        self.inventory
-            .list()
-            .into_iter()
-            .map(SegmentMeta::from)
-            .collect::<Vec<_>>()
-    }
-
-    pub fn new_segment_meta(&self, segment_id: SegmentId, max_doc: u32) -> SegmentMeta {
-        let inner = InnerSegmentMeta {
-            segment_id,
-            max_doc,
-            deletes: None,
-        };
-        SegmentMeta::from(self.inventory.track(inner))
-    }
-}
-
-/// `SegmentMeta` contains simple meta information about a segment.
-///
-/// For instance the number of docs it contains,
-/// how many are deleted, etc.
-#[derive(Clone)]
-pub struct SegmentMeta {
-    tracked: TrackedObject<InnerSegmentMeta>,
-}
-
-impl fmt::Debug for SegmentMeta {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> {
-        self.tracked.fmt(f)
-    }
-}
-
-impl serde::Serialize for SegmentMeta {
-    fn serialize<S>(
-        &self,
-        serializer: S,
-    ) -> Result<<S as serde::Serializer>::Ok, <S as serde::Serializer>::Error>
-    where
-        S: serde::Serializer,
-    {
-        self.tracked.serialize(serializer)
-    }
-}
-
-impl From<TrackedObject<InnerSegmentMeta>> for SegmentMeta {
-    fn from(tracked: TrackedObject<InnerSegmentMeta>) -> SegmentMeta {
-        SegmentMeta { tracked }
-    }
-}
-
-impl SegmentMeta {
-    // Creates a new `SegmentMeta` object.
-
-    /// Returns the segment id.
-    pub fn id(&self) -> SegmentId {
-        self.tracked.segment_id
-    }
-
-    /// Returns the number of deleted documents.
-    pub fn num_deleted_docs(&self) -> u32 {
-        self.tracked
-            .deletes
-            .as_ref()
-            .map(|delete_meta| delete_meta.num_deleted_docs)
-            .unwrap_or(0u32)
-    }
-
-    /// Returns the list of files that
-    /// are required for the segment meta.
-    ///
-    /// This is useful as the way tantivy removes files
-    /// is by removing all files that have been created by tantivy
-    /// and are not used by any segment anymore.
-    pub fn list_files(&self) -> HashSet<PathBuf> {
-        SegmentComponent::iterator()
-            .map(|component| self.relative_path(*component))
-            .collect::<HashSet<PathBuf>>()
-    }
-
-    /// Returns the relative path of a component of our segment.
-    ///
-    /// It just joins the segment id with the extension
-    /// associated to a segment component.
-    pub fn relative_path(&self, component: SegmentComponent) -> PathBuf {
-        let mut path = self.id().uuid_string();
-        path.push_str(&*match component {
-            SegmentComponent::POSTINGS => ".idx".to_string(),
-            SegmentComponent::POSITIONS => ".pos".to_string(),
-            SegmentComponent::POSITIONSSKIP => ".posidx".to_string(),
-            SegmentComponent::TERMS => ".term".to_string(),
-            SegmentComponent::STORE => ".store".to_string(),
-            SegmentComponent::FASTFIELDS => ".fast".to_string(),
-            SegmentComponent::FIELDNORMS => ".fieldnorm".to_string(),
-            SegmentComponent::DELETE => format!(".{}.del", self.delete_opstamp().unwrap_or(0)),
-        });
-        PathBuf::from(path)
-    }
-
-    /// Return the highest doc id + 1
-    ///
-    /// If there are no deletes, then num_docs = max_docs
-    /// and all the doc ids contains in this segment
-    /// are exactly (0..max_doc).
-    pub fn max_doc(&self) -> u32 {
-        self.tracked.max_doc
-    }
-
-    /// Return the number of documents in the segment.
-    pub fn num_docs(&self) -> u32 {
-        self.max_doc() - self.num_deleted_docs()
-    }
-
-    /// Returns the `Opstamp` of the last delete operation
-    /// taken in account in this segment.
-    pub fn delete_opstamp(&self) -> Option<Opstamp> {
-        self.tracked
-            .deletes
-            .as_ref()
-            .map(|delete_meta| delete_meta.opstamp)
-    }
-
-    /// Returns true iff the segment meta contains
-    /// delete information.
-    pub fn has_deletes(&self) -> bool {
-        self.num_deleted_docs() > 0
-    }
-
-    /// Updates the max_doc value from the `SegmentMeta`.
-    ///
-    /// This method is only used when updating `max_doc` from 0
-    /// as we finalize a fresh new segment.
-    pub(crate) fn with_max_doc(self, max_doc: u32) -> SegmentMeta {
-        assert_eq!(self.tracked.max_doc, 0);
-        assert!(self.tracked.deletes.is_none());
-        let tracked = self.tracked.map(move |inner_meta| InnerSegmentMeta {
-            segment_id: inner_meta.segment_id,
-            max_doc,
-            deletes: None,
-        });
-        SegmentMeta { tracked }
-    }
-
-    #[doc(hidden)]
-    pub fn with_delete_meta(self, num_deleted_docs: u32, opstamp: Opstamp) -> SegmentMeta {
-        let delete_meta = DeleteMeta {
-            num_deleted_docs,
-            opstamp,
-        };
-        let tracked = self.tracked.map(move |inner_meta| InnerSegmentMeta {
-            segment_id: inner_meta.segment_id,
-            max_doc: inner_meta.max_doc,
-            deletes: Some(delete_meta),
-        });
-        SegmentMeta { tracked }
-    }
-}
-
-#[derive(Debug, Clone, Serialize, Deserialize)]
-struct InnerSegmentMeta {
-    segment_id: SegmentId,
-    max_doc: u32,
-    deletes: Option<DeleteMeta>,
-}
-
-impl InnerSegmentMeta {
-    pub fn track(self, inventory: &SegmentMetaInventory) -> SegmentMeta {
-        SegmentMeta {
-            tracked: inventory.inventory.track(self),
-        }
-    }
-}
 
 /// Meta information about the `Index`.
 ///
@@ -202,53 +11,16 @@ impl InnerSegmentMeta {
 /// * the index `docstamp`
 /// * the schema
 ///
-#[derive(Clone, Serialize)]
+#[derive(Clone, Serialize, Deserialize)]
 pub struct IndexMeta {
-    /// List of `SegmentMeta` informations associated to each finalized segment of the index.
     pub segments: Vec<SegmentMeta>,
-    /// Index `Schema`
     pub schema: Schema,
-    /// Opstamp associated to the last `commit` operation.
-    pub opstamp: Opstamp,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    /// Payload associated to the last commit.
-    ///
-    /// Upon commit, clients can optionally add a small `String` payload to their commit
-    /// to help identify this commit.
-    /// This payload is entirely unused by tantivy.
-    pub payload: Option<String>,
-}
-
-#[derive(Deserialize)]
-struct UntrackedIndexMeta {
-    pub segments: Vec<InnerSegmentMeta>,
-    pub schema: Schema,
-    pub opstamp: Opstamp,
+    pub opstamp: u64,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub payload: Option<String>,
 }
 
-impl UntrackedIndexMeta {
-    pub fn track(self, inventory: &SegmentMetaInventory) -> IndexMeta {
-        IndexMeta {
-            segments: self
-                .segments
-                .into_iter()
-                .map(|inner_seg_meta| inner_seg_meta.track(inventory))
-                .collect::<Vec<SegmentMeta>>(),
-            schema: self.schema,
-            opstamp: self.opstamp,
-            payload: self.payload,
-        }
-    }
-}
-
 impl IndexMeta {
-    /// Create an `IndexMeta` object representing a brand new `Index`
-    /// with the given index.
-    ///
-    /// This new index does not contains any segments.
-    /// Opstamp will the value `0u64`.
     pub fn with_schema(schema: Schema) -> IndexMeta {
         IndexMeta {
             segments: vec![],
@@ -257,18 +29,10 @@ impl IndexMeta {
             payload: None,
         }
     }
-
-    pub(crate) fn deserialize(
-        meta_json: &str,
-        inventory: &SegmentMetaInventory,
-    ) -> serde_json::Result<IndexMeta> {
-        let untracked_meta_json: UntrackedIndexMeta = serde_json::from_str(meta_json)?;
-        Ok(untracked_meta_json.track(inventory))
-    }
 }
 
 impl fmt::Debug for IndexMeta {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         write!(
             f,
             "{}",
@@ -282,7 +46,7 @@ impl fmt::Debug for IndexMeta {
 mod tests {
 
     use super::IndexMeta;
-    use crate::schema::{Schema, TEXT};
+    use schema::{Schema, TEXT};
     use serde_json;
 
     #[test]
@@ -299,9 +63,6 @@ mod tests {
             payload: None,
         };
         let json = serde_json::ser::to_string(&index_metas).expect("serialization failed");
-        assert_eq!(
-            json,
-            r#"{"segments":[],"schema":[{"name":"text","type":"text","options":{"indexing":{"record":"position","tokenizer":"default"},"stored":false}}],"opstamp":0}"#
-        );
+        assert_eq!(json, r#"{"segments":[],"schema":[{"name":"text","type":"text","options":{"indexing":{"record":"position","tokenizer":"default"},"stored":false}}],"opstamp":0}"#);
     }
 }

src/core/inverted_index_reader.rs

@@ -1,11 +1,13 @@
-use crate::common::BinarySerializable;
-use crate::directory::ReadOnlySource;
-use crate::positions::PositionReader;
-use crate::postings::TermInfo;
-use crate::postings::{BlockSegmentPostings, SegmentPostings};
-use crate::schema::IndexRecordOption;
-use crate::schema::Term;
-use crate::termdict::TermDictionary;
+use common::BinarySerializable;
+use directory::ReadOnlySource;
+use owned_read::OwnedRead;
+use positions::PositionReader;
+use postings::TermInfo;
+use postings::{BlockSegmentPostings, SegmentPostings};
+use schema::FieldType;
+use schema::IndexRecordOption;
+use schema::Term;
+use termdict::TermDictionary;
 
 /// The inverted index reader is in charge of accessing
 /// the inverted index associated to a specific field.
@@ -30,7 +32,7 @@ pub struct InvertedIndexReader {
 }
 
 impl InvertedIndexReader {
-    #[cfg_attr(feature = "cargo-clippy", allow(clippy::needless_pass_by_value))] // for symmetry
+    #[cfg_attr(feature = "cargo-clippy", allow(clippy::needless_pass_by_value))] // for symetry
     pub(crate) fn new(
         termdict: TermDictionary,
         postings_source: ReadOnlySource,
@@ -53,9 +55,12 @@ impl InvertedIndexReader {
 
     /// Creates an empty `InvertedIndexReader` object, which
     /// contains no terms at all.
-    pub fn empty(record_option: IndexRecordOption) -> InvertedIndexReader {
+    pub fn empty(field_type: &FieldType) -> InvertedIndexReader {
+        let record_option = field_type
+            .get_index_record_option()
+            .unwrap_or(IndexRecordOption::Basic);
         InvertedIndexReader {
-            termdict: TermDictionary::empty(),
+            termdict: TermDictionary::empty(&field_type),
             postings_source: ReadOnlySource::empty(),
             positions_source: ReadOnlySource::empty(),
             positions_idx_source: ReadOnlySource::empty(),
@@ -92,7 +97,8 @@ impl InvertedIndexReader {
         let offset = term_info.postings_offset as usize;
         let end_source = self.postings_source.len();
         let postings_slice = self.postings_source.slice(offset, end_source);
-        block_postings.reset(term_info.doc_freq, postings_slice);
+        let postings_reader = OwnedRead::new(postings_slice);
+        block_postings.reset(term_info.doc_freq, postings_reader);
     }
 
     /// Returns a block postings given a `Term`.
@@ -121,7 +127,7 @@ impl InvertedIndexReader {
         let postings_data = self.postings_source.slice_from(offset);
         BlockSegmentPostings::from_data(
             term_info.doc_freq,
-            postings_data,
+            OwnedRead::new(postings_data),
             self.record_option,
             requested_option,
         )

src/core/mod.rs

@@ -2,33 +2,37 @@ mod executor;
 pub mod index;
 mod index_meta;
 mod inverted_index_reader;
+mod pool;
 pub mod searcher;
 mod segment;
 mod segment_component;
 mod segment_id;
+mod segment_meta;
 mod segment_reader;
 
 pub use self::executor::Executor;
 pub use self::index::Index;
-pub use self::index_meta::{IndexMeta, SegmentMeta, SegmentMetaInventory};
+pub use self::index_meta::IndexMeta;
 pub use self::inverted_index_reader::InvertedIndexReader;
 pub use self::searcher::Searcher;
 pub use self::segment::Segment;
 pub use self::segment::SerializableSegment;
 pub use self::segment_component::SegmentComponent;
 pub use self::segment_id::SegmentId;
+pub use self::segment_meta::SegmentMeta;
 pub use self::segment_reader::SegmentReader;
 
-use once_cell::sync::Lazy;
-use std::path::Path;
+use std::path::PathBuf;
 
-/// The meta file contains all the information about the list of segments and the schema
-/// of the index.
-pub static META_FILEPATH: Lazy<&'static Path> = Lazy::new(|| Path::new("meta.json"));
+lazy_static! {
+    /// The meta file contains all the information about the list of segments and the schema
+    /// of the index.
+    pub static ref META_FILEPATH: PathBuf = PathBuf::from("meta.json");
 
-/// The managed file contains a list of files that were created by the tantivy
-/// and will therefore be garbage collected when they are deemed useless by tantivy.
-///
-/// Removing this file is safe, but will prevent the garbage collection of all of the file that
-/// are currently in the directory
-pub static MANAGED_FILEPATH: Lazy<&'static Path> = Lazy::new(|| Path::new(".managed.json"));
+    /// The managed file contains a list of files that were created by the tantivy
+    /// and will therefore be garbage collected when they are deemed useless by tantivy.
+    ///
+    /// Removing this file is safe, but will prevent the garbage collection of all of the file that
+    /// are currently in the directory
+    pub static ref MANAGED_FILEPATH: PathBuf = PathBuf::from(".managed.json");
+}

src/core/pool.rs

@@ -0,0 +1,136 @@
+use crossbeam::queue::MsQueue;
+use std::mem;
+use std::ops::{Deref, DerefMut};
+use std::sync::atomic::AtomicUsize;
+use std::sync::atomic::Ordering;
+use std::sync::Arc;
+
+pub struct GenerationItem<T> {
+    generation: usize,
+    item: T,
+}
+
+pub struct Pool<T> {
+    queue: Arc<MsQueue<GenerationItem<T>>>,
+    freshest_generation: AtomicUsize,
+    next_generation: AtomicUsize,
+}
+
+impl<T> Pool<T> {
+    pub fn new() -> Pool<T> {
+        let queue = Arc::new(MsQueue::new());
+        Pool {
+            queue,
+            freshest_generation: AtomicUsize::default(),
+            next_generation: AtomicUsize::default(),
+        }
+    }
+
+    pub fn publish_new_generation(&self, items: Vec<T>) {
+        let next_generation = self.next_generation.fetch_add(1, Ordering::SeqCst) + 1;
+        for item in items {
+            let gen_item = GenerationItem {
+                item,
+                generation: next_generation,
+            };
+            self.queue.push(gen_item);
+        }
+        self.advertise_generation(next_generation);
+    }
+
+    /// At the exit of this method,
+    /// - freshest_generation has a value greater or equal than generation
+    /// - freshest_generation has a value that has been advertised
+    /// - freshest_generation has)
+    fn advertise_generation(&self, generation: usize) {
+        // not optimal at all but the easiest to read proof.
+        loop {
+            let former_generation = self.freshest_generation.load(Ordering::Acquire);
+            if former_generation >= generation {
+                break;
+            }
+            self.freshest_generation.compare_and_swap(
+                former_generation,
+                generation,
+                Ordering::SeqCst,
+            );
+        }
+    }
+
+    fn generation(&self) -> usize {
+        self.freshest_generation.load(Ordering::Acquire)
+    }
+
+    pub fn acquire(&self) -> LeasedItem<T> {
+        let generation = self.generation();
+        loop {
+            let gen_item = self.queue.pop();
+            if gen_item.generation >= generation {
+                return LeasedItem {
+                    gen_item: Some(gen_item),
+                    recycle_queue: Arc::clone(&self.queue),
+                };
+            } else {
+                // this searcher is obsolete,
+                // removing it from the pool.
+            }
+        }
+    }
+}
+
+pub struct LeasedItem<T> {
+    gen_item: Option<GenerationItem<T>>,
+    recycle_queue: Arc<MsQueue<GenerationItem<T>>>,
+}
+
+impl<T> Deref for LeasedItem<T> {
+    type Target = T;
+
+    fn deref(&self) -> &T {
+        &self
+            .gen_item
+            .as_ref()
+            .expect("Unwrapping a leased item should never fail")
+            .item // unwrap is safe here
+    }
+}
+
+impl<T> DerefMut for LeasedItem<T> {
+    fn deref_mut(&mut self) -> &mut T {
+        &mut self
+            .gen_item
+            .as_mut()
+            .expect("Unwrapping a mut leased item should never fail")
+            .item // unwrap is safe here
+    }
+}
+
+impl<T> Drop for LeasedItem<T> {
+    fn drop(&mut self) {
+        let gen_item: GenerationItem<T> = mem::replace(&mut self.gen_item, None)
+            .expect("Unwrapping a leased item should never fail");
+        self.recycle_queue.push(gen_item);
+    }
+}
+
+#[cfg(test)]
+mod tests {
+
+    use super::Pool;
+    use std::iter;
+
+    #[test]
+    fn test_pool() {
+        let items10: Vec<usize> = iter::repeat(10).take(10).collect();
+        let pool = Pool::new();
+        pool.publish_new_generation(items10);
+        for _ in 0..20 {
+            assert_eq!(*pool.acquire(), 10);
+        }
+        let items11: Vec<usize> = iter::repeat(11).take(10).collect();
+        pool.publish_new_generation(items11);
+        for _ in 0..20 {
+            assert_eq!(*pool.acquire(), 11);
+        }
+    }
+}

src/core/searcher.rs

@@ -1,18 +1,42 @@
-use crate::collector::Collector;
-use crate::core::Executor;
-use crate::core::InvertedIndexReader;
-use crate::core::SegmentReader;
-use crate::query::Query;
-use crate::schema::Document;
-use crate::schema::Schema;
-use crate::schema::{Field, Term};
-use crate::space_usage::SearcherSpaceUsage;
-use crate::store::StoreReader;
-use crate::termdict::TermMerger;
-use crate::DocAddress;
-use crate::Index;
+use collector::Collector;
+use collector::SegmentCollector;
+use core::Executor;
+use core::InvertedIndexReader;
+use core::SegmentReader;
+use query::Query;
+use query::Scorer;
+use query::Weight;
+use schema::Document;
+use schema::Schema;
+use schema::{Field, Term};
+use space_usage::SearcherSpaceUsage;
 use std::fmt;
 use std::sync::Arc;
+use store::StoreReader;
+use termdict::TermMerger;
+use DocAddress;
+use Index;
+use Result;
+
+fn collect_segment<C: Collector>(
+    collector: &C,
+    weight: &Weight,
+    segment_ord: u32,
+    segment_reader: &SegmentReader,
+) -> Result<C::SegmentFruit> {
+    let mut scorer = weight.scorer(segment_reader)?;
+    let mut segment_collector = collector.for_segment(segment_ord as u32, segment_reader)?;
+    if let Some(delete_bitset) = segment_reader.delete_bitset() {
+        scorer.for_each(&mut |doc, score| {
+            if !delete_bitset.is_deleted(doc) {
+                segment_collector.collect(doc, score);
+            }
+        });
+    } else {
+        scorer.for_each(&mut |doc, score| segment_collector.collect(doc, score));
+    }
+    Ok(segment_collector.harvest())
+}
 
 /// Holds a list of `SegmentReader`s ready for search.
 ///
@@ -35,7 +59,7 @@ impl Searcher {
     ) -> Searcher {
         let store_readers = segment_readers
             .iter()
-            .map(SegmentReader::get_store_reader)
+            .map(|segment_reader| segment_reader.get_store_reader())
             .collect();
         Searcher {
             schema,
@@ -54,7 +78,7 @@ impl Searcher {
     ///
     /// The searcher uses the segment ordinal to route the
     /// the request to the right `Segment`.
-    pub fn doc(&self, doc_address: DocAddress) -> crate::Result<Document> {
+    pub fn doc(&self, doc_address: DocAddress) -> Result<Document> {
         let DocAddress(segment_local_id, doc_id) = doc_address;
         let store_reader = &self.store_readers[segment_local_id as usize];
         store_reader.get(doc_id)
@@ -108,11 +132,7 @@ impl Searcher {
     ///
     ///  Finally, the Collector merges each of the child collectors into itself for result usability
     ///  by the caller.
-    pub fn search<C: Collector>(
-        &self,
-        query: &dyn Query,
-        collector: &C,
-    ) -> crate::Result<C::Fruit> {
+    pub fn search<C: Collector>(&self, query: &Query, collector: &C) -> Result<C::Fruit> {
         let executor = self.index.search_executor();
         self.search_with_executor(query, collector, executor)
     }
@@ -131,19 +151,23 @@ impl Searcher {
     /// hurt it. It will however, decrease the average response time.
     pub fn search_with_executor<C: Collector>(
         &self,
-        query: &dyn Query,
+        query: &Query,
         collector: &C,
         executor: &Executor,
-    ) -> crate::Result<C::Fruit> {
+    ) -> Result<C::Fruit> {
         let scoring_enabled = collector.requires_scoring();
         let weight = query.weight(self, scoring_enabled)?;
         let segment_readers = self.segment_readers();
         let fruits = executor.map(
             |(segment_ord, segment_reader)| {
-                collector.collect_segment(weight.as_ref(), segment_ord as u32, segment_reader)
+                collect_segment(
+                    collector,
+                    weight.as_ref(),
+                    segment_ord as u32,
+                    segment_reader,
+                )
             },
             segment_readers.iter().enumerate(),
-            self.index.logger().clone(),
         )?;
         collector.merge_fruits(fruits)
     }
@@ -179,7 +203,7 @@ impl FieldSearcher {
 
     /// Returns a Stream over all of the sorted unique terms of
     /// for the given field.
-    pub fn terms(&self) -> TermMerger<'_> {
+    pub fn terms(&self) -> TermMerger {
         let term_streamers: Vec<_> = self
             .inv_index_readers
             .iter()
@@ -190,11 +214,11 @@ impl FieldSearcher {
 }
 
 impl fmt::Debug for Searcher {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         let segment_ids = self
             .segment_readers
             .iter()
-            .map(SegmentReader::segment_id)
+            .map(|segment_reader| segment_reader.segment_id())
             .collect::<Vec<_>>();
         write!(f, "Searcher({:?})", segment_ids)
     }

src/core/segment.rs

@@ -1,15 +1,16 @@
 use super::SegmentComponent;
-use crate::core::Index;
-use crate::core::SegmentId;
-use crate::core::SegmentMeta;
-use crate::directory::error::{OpenReadError, OpenWriteError};
-use crate::directory::Directory;
-use crate::directory::{ReadOnlySource, WritePtr};
-use crate::indexer::segment_serializer::SegmentSerializer;
-use crate::schema::Schema;
-use crate::Opstamp;
+use core::Index;
+use core::SegmentId;
+use core::SegmentMeta;
+use directory::error::{OpenReadError, OpenWriteError};
+use directory::Directory;
+use directory::{ReadOnlySource, WritePtr};
+use indexer::segment_serializer::SegmentSerializer;
+use schema::Schema;
 use std::fmt;
 use std::path::PathBuf;
+use std::result;
+use Result;
 
 /// A segment is a piece of the index.
 #[derive(Clone)]
@@ -19,17 +20,20 @@ pub struct Segment {
 }
 
 impl fmt::Debug for Segment {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         write!(f, "Segment({:?})", self.id().uuid_string())
     }
 }
 
-impl Segment {
-    /// Creates a new segment given an `Index` and a `SegmentId`
-    pub(crate) fn for_index(index: Index, meta: SegmentMeta) -> Segment {
-        Segment { index, meta }
-    }
+/// Creates a new segment given an `Index` and a `SegmentId`
+///
+/// The function is here to make it private outside `tantivy`.
+/// #[doc(hidden)]
+pub fn create_segment(index: Index, meta: SegmentMeta) -> Segment {
+    Segment { index, meta }
+}
 
+impl Segment {
     /// Returns the index the segment belongs to.
     pub fn index(&self) -> &Index {
         &self.index
@@ -45,19 +49,8 @@ impl Segment {
         &self.meta
     }
 
-    /// Updates the max_doc value from the `SegmentMeta`.
-    ///
-    /// This method is only used when updating `max_doc` from 0
-    /// as we finalize a fresh new segment.
-    pub(crate) fn with_max_doc(self, max_doc: u32) -> Segment {
-        Segment {
-            index: self.index,
-            meta: self.meta.with_max_doc(max_doc),
-        }
-    }
-
     #[doc(hidden)]
-    pub fn with_delete_meta(self, num_deleted_docs: u32, opstamp: Opstamp) -> Segment {
+    pub fn with_delete_meta(self, num_deleted_docs: u32, opstamp: u64) -> Segment {
         Segment {
             index: self.index,
             meta: self.meta.with_delete_meta(num_deleted_docs, opstamp),
@@ -78,14 +71,20 @@ impl Segment {
     }
 
     /// Open one of the component file for a *regular* read.
-    pub fn open_read(&self, component: SegmentComponent) -> Result<ReadOnlySource, OpenReadError> {
+    pub fn open_read(
+        &self,
+        component: SegmentComponent,
+    ) -> result::Result<ReadOnlySource, OpenReadError> {
         let path = self.relative_path(component);
         let source = self.index.directory().open_read(&path)?;
         Ok(source)
     }
 
     /// Open one of the component file for *regular* write.
-    pub fn open_write(&mut self, component: SegmentComponent) -> Result<WritePtr, OpenWriteError> {
+    pub fn open_write(
+        &mut self,
+        component: SegmentComponent,
+    ) -> result::Result<WritePtr, OpenWriteError> {
         let path = self.relative_path(component);
         let write = self.index.directory_mut().open_write(&path)?;
         Ok(write)
@@ -98,5 +97,5 @@ pub trait SerializableSegment {
     ///
     /// # Returns
     /// The number of documents in the segment.
-    fn write(&self, serializer: SegmentSerializer) -> crate::Result<u32>;
+    fn write(&self, serializer: SegmentSerializer) -> Result<u32>;
 }

src/core/segment_component.rs

@@ -41,6 +41,6 @@ impl SegmentComponent {
             SegmentComponent::STORE,
             SegmentComponent::DELETE,
         ];
-        SEGMENT_COMPONENTS.iter()
+        SEGMENT_COMPONENTS.into_iter()
     }
 }

src/core/segment_id.rs

@@ -2,11 +2,6 @@ use std::cmp::{Ord, Ordering};
 use std::fmt;
 use uuid::Uuid;
 
-#[cfg(test)]
-use once_cell::sync::Lazy;
-use serde::{Deserialize, Serialize};
-use std::error::Error;
-use std::str::FromStr;
 #[cfg(test)]
 use std::sync::atomic;
 
@@ -21,18 +16,12 @@ use std::sync::atomic;
 #[derive(Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
 pub struct SegmentId(Uuid);
 
-impl ToString for SegmentId {
-    fn to_string(&self) -> String {
-        self.short_uuid_string()
-    }
+#[cfg(test)]
+lazy_static! {
+    static ref AUTO_INC_COUNTER: atomic::AtomicUsize = atomic::AtomicUsize::default();
+    static ref EMPTY_ARR: [u8; 8] = [0u8; 8];
 }
 
-#[cfg(test)]
-static AUTO_INC_COUNTER: Lazy<atomic::AtomicUsize> = Lazy::new(|| atomic::AtomicUsize::default());
-
-#[cfg(test)]
-const ZERO_ARRAY: [u8; 8] = [0u8; 8];
-
 // During tests, we generate the segment id in a autoincrement manner
 // for consistency of segment id between run.
 //
@@ -41,7 +30,7 @@ const ZERO_ARRAY: [u8; 8] = [0u8; 8];
 #[cfg(test)]
 fn create_uuid() -> Uuid {
     let new_auto_inc_id = (*AUTO_INC_COUNTER).fetch_add(1, atomic::Ordering::SeqCst);
-    Uuid::from_fields(new_auto_inc_id as u32, 0, 0, &ZERO_ARRAY).unwrap()
+    Uuid::from_fields(new_auto_inc_id as u32, 0, 0, &*EMPTY_ARR).unwrap()
 }
 
 #[cfg(not(test))]
@@ -61,55 +50,19 @@ impl SegmentId {
     /// and the rest is random.
     ///
     /// Picking the first 8 chars is ok to identify
-    /// segments in a display message (e.g. a5c4dfcb).
+    /// segments in a display message.
     pub fn short_uuid_string(&self) -> String {
         (&self.0.to_simple_ref().to_string()[..8]).to_string()
     }
 
     /// Returns a segment uuid string.
-    ///
-    /// It consists in 32 lowercase hexadecimal chars
-    /// (e.g. a5c4dfcbdfe645089129e308e26d5523)
     pub fn uuid_string(&self) -> String {
         self.0.to_simple_ref().to_string()
     }
-
-    /// Build a `SegmentId` string from the full uuid string.
-    ///
-    /// E.g. "a5c4dfcbdfe645089129e308e26d5523"
-    pub fn from_uuid_string(uuid_string: &str) -> Result<SegmentId, SegmentIdParseError> {
-        FromStr::from_str(uuid_string)
-    }
-}
-
-/// Error type used when parsing a `SegmentId` from a string fails.
-pub struct SegmentIdParseError(uuid::Error);
-
-impl Error for SegmentIdParseError {}
-
-impl fmt::Debug for SegmentIdParseError {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        self.0.fmt(f)
-    }
-}
-
-impl fmt::Display for SegmentIdParseError {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        self.0.fmt(f)
-    }
-}
-
-impl FromStr for SegmentId {
-    type Err = SegmentIdParseError;
-
-    fn from_str(uuid_string: &str) -> Result<Self, SegmentIdParseError> {
-        let uuid = Uuid::parse_str(uuid_string).map_err(SegmentIdParseError)?;
-        Ok(SegmentId(uuid))
-    }
 }
 
 impl fmt::Debug for SegmentId {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         write!(f, "Seg({:?})", self.short_uuid_string())
     }
 }
@@ -125,18 +78,3 @@ impl Ord for SegmentId {
         self.0.as_bytes().cmp(other.0.as_bytes())
     }
 }
-
-#[cfg(test)]
-mod tests {
-    use super::SegmentId;
-
-    #[test]
-    fn test_to_uuid_string() {
-        let full_uuid = "a5c4dfcbdfe645089129e308e26d5523";
-        let segment_id = SegmentId::from_uuid_string(full_uuid).unwrap();
-        assert_eq!(segment_id.uuid_string(), full_uuid);
-        assert_eq!(segment_id.short_uuid_string(), "a5c4dfcb");
-        // one extra char
-        assert!(SegmentId::from_uuid_string("a5c4dfcbdfe645089129e308e26d5523b").is_err());
-    }
-}

src/core/segment_meta.rs

@@ -0,0 +1,174 @@
+use super::SegmentComponent;
+use census::{Inventory, TrackedObject};
+use core::SegmentId;
+use serde;
+use std::collections::HashSet;
+use std::fmt;
+use std::path::PathBuf;
+
+lazy_static! {
+    static ref INVENTORY: Inventory<InnerSegmentMeta> = { Inventory::new() };
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+struct DeleteMeta {
+    num_deleted_docs: u32,
+    opstamp: u64,
+}
+
+/// `SegmentMeta` contains simple meta information about a segment.
+///
+/// For instance the number of docs it contains,
+/// how many are deleted, etc.
+#[derive(Clone)]
+pub struct SegmentMeta {
+    tracked: TrackedObject<InnerSegmentMeta>,
+}
+
+impl fmt::Debug for SegmentMeta {
+    fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
+        self.tracked.fmt(f)
+    }
+}
+
+impl serde::Serialize for SegmentMeta {
+    fn serialize<S>(
+        &self,
+        serializer: S,
+    ) -> Result<<S as serde::Serializer>::Ok, <S as serde::Serializer>::Error>
+    where
+        S: serde::Serializer,
+    {
+        self.tracked.serialize(serializer)
+    }
+}
+
+impl<'a> serde::Deserialize<'a> for SegmentMeta {
+    fn deserialize<D>(deserializer: D) -> Result<Self, <D as serde::Deserializer<'a>>::Error>
+    where
+        D: serde::Deserializer<'a>,
+    {
+        let inner = InnerSegmentMeta::deserialize(deserializer)?;
+        let tracked = INVENTORY.track(inner);
+        Ok(SegmentMeta { tracked })
+    }
+}
+
+impl SegmentMeta {
+    /// Lists all living `SegmentMeta` object at the time of the call.
+    pub fn all() -> Vec<SegmentMeta> {
+        INVENTORY
+            .list()
+            .into_iter()
+            .map(|inner| SegmentMeta { tracked: inner })
+            .collect::<Vec<_>>()
+    }
+
+    /// Creates a new `SegmentMeta` object.
+    #[doc(hidden)]
+    pub fn new(segment_id: SegmentId, max_doc: u32) -> SegmentMeta {
+        let inner = InnerSegmentMeta {
+            segment_id,
+            max_doc,
+            deletes: None,
+        };
+        SegmentMeta {
+            tracked: INVENTORY.track(inner),
+        }
+    }
+
+    /// Returns the segment id.
+    pub fn id(&self) -> SegmentId {
+        self.tracked.segment_id
+    }
+
+    /// Returns the number of deleted documents.
+    pub fn num_deleted_docs(&self) -> u32 {
+        self.tracked
+            .deletes
+            .as_ref()
+            .map(|delete_meta| delete_meta.num_deleted_docs)
+            .unwrap_or(0u32)
+    }
+
+    /// Returns the list of files that
+    /// are required for the segment meta.
+    ///
+    /// This is useful as the way tantivy removes files
+    /// is by removing all files that have been created by tantivy
+    /// and are not used by any segment anymore.
+    pub fn list_files(&self) -> HashSet<PathBuf> {
+        SegmentComponent::iterator()
+            .map(|component| self.relative_path(*component))
+            .collect::<HashSet<PathBuf>>()
+    }
+
+    /// Returns the relative path of a component of our segment.
+    ///
+    /// It just joins the segment id with the extension
+    /// associated to a segment component.
+    pub fn relative_path(&self, component: SegmentComponent) -> PathBuf {
+        let mut path = self.id().uuid_string();
+        path.push_str(&*match component {
+            SegmentComponent::POSTINGS => ".idx".to_string(),
+            SegmentComponent::POSITIONS => ".pos".to_string(),
+            SegmentComponent::POSITIONSSKIP => ".posidx".to_string(),
+            SegmentComponent::TERMS => ".term".to_string(),
+            SegmentComponent::STORE => ".store".to_string(),
+            SegmentComponent::FASTFIELDS => ".fast".to_string(),
+            SegmentComponent::FIELDNORMS => ".fieldnorm".to_string(),
+            SegmentComponent::DELETE => format!(".{}.del", self.delete_opstamp().unwrap_or(0)),
+        });
+        PathBuf::from(path)
+    }
+
+    /// Return the highest doc id + 1
+    ///
+    /// If there are no deletes, then num_docs = max_docs
+    /// and all the doc ids contains in this segment
+    /// are exactly (0..max_doc).
+    pub fn max_doc(&self) -> u32 {
+        self.tracked.max_doc
+    }
+
+    /// Return the number of documents in the segment.
+    pub fn num_docs(&self) -> u32 {
+        self.max_doc() - self.num_deleted_docs()
+    }
+
+    /// Returns the opstamp of the last delete operation
+    /// taken in account in this segment.
+    pub fn delete_opstamp(&self) -> Option<u64> {
+        self.tracked
+            .deletes
+            .as_ref()
+            .map(|delete_meta| delete_meta.opstamp)
+    }
+
+    /// Returns true iff the segment meta contains
+    /// delete information.
+    pub fn has_deletes(&self) -> bool {
+        self.num_deleted_docs() > 0
+    }
+
+    #[doc(hidden)]
+    pub fn with_delete_meta(self, num_deleted_docs: u32, opstamp: u64) -> SegmentMeta {
+        let delete_meta = DeleteMeta {
+            num_deleted_docs,
+            opstamp,
+        };
+        let tracked = self.tracked.map(move |inner_meta| InnerSegmentMeta {
+            segment_id: inner_meta.segment_id,
+            max_doc: inner_meta.max_doc,
+            deletes: Some(delete_meta),
+        });
+        SegmentMeta { tracked }
+    }
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+struct InnerSegmentMeta {
+    segment_id: SegmentId,
+    max_doc: u32,
+    deletes: Option<DeleteMeta>,
+}

src/core/segment_reader.rs

@@ -1,27 +1,30 @@
-use crate::common::CompositeFile;
-use crate::common::HasLen;
-use crate::core::InvertedIndexReader;
-use crate::core::Segment;
-use crate::core::SegmentComponent;
-use crate::core::SegmentId;
-use crate::directory::ReadOnlySource;
-use crate::fastfield::DeleteBitSet;
-use crate::fastfield::FacetReader;
-use crate::fastfield::FastFieldReaders;
-use crate::fieldnorm::{FieldNormReader, FieldNormReaders};
-use crate::schema::FieldType;
-use crate::schema::Schema;
-use crate::schema::{Field, IndexRecordOption};
-use crate::space_usage::SegmentSpaceUsage;
-use crate::store::StoreReader;
-use crate::termdict::TermDictionary;
-use crate::DocId;
-use fail::fail_point;
-use slog::{warn, Logger};
+use common::CompositeFile;
+use common::HasLen;
+use core::InvertedIndexReader;
+use core::Segment;
+use core::SegmentComponent;
+use core::SegmentId;
+use directory::ReadOnlySource;
+use error::TantivyError;
+use fastfield::DeleteBitSet;
+use fastfield::FacetReader;
+use fastfield::FastFieldReader;
+use fastfield::{self, FastFieldNotAvailableError};
+use fastfield::{BytesFastFieldReader, FastValue, MultiValueIntFastFieldReader};
+use fieldnorm::FieldNormReader;
+use schema::Cardinality;
+use schema::Field;
+use schema::FieldType;
+use schema::Schema;
+use space_usage::SegmentSpaceUsage;
 use std::collections::HashMap;
 use std::fmt;
 use std::sync::Arc;
 use std::sync::RwLock;
+use store::StoreReader;
+use termdict::TermDictionary;
+use DocId;
+use Result;
 
 /// Entry point to access all of the datastructures of the `Segment`
 ///
@@ -48,13 +51,12 @@ pub struct SegmentReader {
     postings_composite: CompositeFile,
     positions_composite: CompositeFile,
     positions_idx_composite: CompositeFile,
-    fast_fields_readers: Arc<FastFieldReaders>,
-    fieldnorm_readers: FieldNormReaders,
+    fast_fields_composite: CompositeFile,
+    fieldnorms_composite: CompositeFile,
 
     store_source: ReadOnlySource,
     delete_bitset_opt: Option<DeleteBitSet>,
     schema: Schema,
-    logger: Logger,
 }
 
 impl SegmentReader {
@@ -103,21 +105,93 @@ impl SegmentReader {
     ///
     /// # Panics
     /// May panic if the index is corrupted.
-    pub fn fast_fields(&self) -> &FastFieldReaders {
-        &self.fast_fields_readers
+    pub fn fast_field_reader<Item: FastValue>(
+        &self,
+        field: Field,
+    ) -> fastfield::Result<FastFieldReader<Item>> {
+        let field_entry = self.schema.get_field_entry(field);
+        if Item::fast_field_cardinality(field_entry.field_type()) == Some(Cardinality::SingleValue)
+        {
+            self.fast_fields_composite
+                .open_read(field)
+                .ok_or_else(|| FastFieldNotAvailableError::new(field_entry))
+                .map(FastFieldReader::open)
+        } else {
+            Err(FastFieldNotAvailableError::new(field_entry))
+        }
+    }
+
+    pub(crate) fn fast_field_reader_with_idx<Item: FastValue>(
+        &self,
+        field: Field,
+        idx: usize,
+    ) -> fastfield::Result<FastFieldReader<Item>> {
+        if let Some(ff_source) = self.fast_fields_composite.open_read_with_idx(field, idx) {
+            Ok(FastFieldReader::open(ff_source))
+        } else {
+            let field_entry = self.schema.get_field_entry(field);
+            Err(FastFieldNotAvailableError::new(field_entry))
+        }
+    }
+
+    /// Accessor to the `MultiValueIntFastFieldReader` associated to a given `Field`.
+    /// May panick if the field is not a multivalued fastfield of the type `Item`.
+    pub fn multi_fast_field_reader<Item: FastValue>(
+        &self,
+        field: Field,
+    ) -> fastfield::Result<MultiValueIntFastFieldReader<Item>> {
+        let field_entry = self.schema.get_field_entry(field);
+        if Item::fast_field_cardinality(field_entry.field_type()) == Some(Cardinality::MultiValues)
+        {
+            let idx_reader = self.fast_field_reader_with_idx(field, 0)?;
+            let vals_reader = self.fast_field_reader_with_idx(field, 1)?;
+            Ok(MultiValueIntFastFieldReader::open(idx_reader, vals_reader))
+        } else {
+            Err(FastFieldNotAvailableError::new(field_entry))
+        }
+    }
+
+    /// Accessor to the `BytesFastFieldReader` associated to a given `Field`.
+    pub fn bytes_fast_field_reader(&self, field: Field) -> fastfield::Result<BytesFastFieldReader> {
+        let field_entry = self.schema.get_field_entry(field);
+        match *field_entry.field_type() {
+            FieldType::Bytes => {}
+            _ => return Err(FastFieldNotAvailableError::new(field_entry)),
+        }
+        let idx_reader = self
+            .fast_fields_composite
+            .open_read_with_idx(field, 0)
+            .ok_or_else(|| FastFieldNotAvailableError::new(field_entry))
+            .map(FastFieldReader::open)?;
+        let values = self
+            .fast_fields_composite
+            .open_read_with_idx(field, 1)
+            .ok_or_else(|| FastFieldNotAvailableError::new(field_entry))?;
+        Ok(BytesFastFieldReader::open(idx_reader, values))
     }
 
     /// Accessor to the `FacetReader` associated to a given `Field`.
-    pub fn facet_reader(&self, field: Field) -> Option<FacetReader> {
+    pub fn facet_reader(&self, field: Field) -> Result<FacetReader> {
         let field_entry = self.schema.get_field_entry(field);
         if field_entry.field_type() != &FieldType::HierarchicalFacet {
-            return None;
+            return Err(TantivyError::InvalidArgument(format!(
+                "The field {:?} is not a \
+                 hierarchical facet.",
+                field_entry
+            )));
         }
-        let term_ords_reader = self.fast_fields().u64s(field)?;
-        let termdict_source = self.termdict_composite.open_read(field)?;
+        let term_ords_reader = self.multi_fast_field_reader(field)?;
+        let termdict_source = self.termdict_composite.open_read(field).ok_or_else(|| {
+            TantivyError::InvalidArgument(format!(
+                "The field \"{}\" is a hierarchical \
+                 but this segment does not seem to have the field term \
+                 dictionary.",
+                field_entry.name()
+            ))
+        })?;
         let termdict = TermDictionary::from_source(&termdict_source);
         let facet_reader = FacetReader::new(term_ords_reader, termdict);
-        Some(facet_reader)
+        Ok(facet_reader)
     }
 
     /// Accessor to the segment's `Field norms`'s reader.
@@ -127,15 +201,17 @@ impl SegmentReader {
     ///
     /// They are simply stored as a fast field, serialized in
     /// the `.fieldnorm` file of the segment.
-    pub fn get_fieldnorms_reader(&self, field: Field) -> crate::Result<FieldNormReader> {
-        self.fieldnorm_readers.get_field(field).ok_or_else(|| {
+    pub fn get_fieldnorms_reader(&self, field: Field) -> FieldNormReader {
+        if let Some(fieldnorm_source) = self.fieldnorms_composite.open_read(field) {
+            FieldNormReader::open(fieldnorm_source)
+        } else {
             let field_name = self.schema.get_field_name(field);
             let err_msg = format!(
                 "Field norm not found for field {:?}. Was it market as indexed during indexing.",
                 field_name
             );
-            crate::TantivyError::SchemaError(err_msg)
-        })
+            panic!(err_msg);
+        }
     }
 
     /// Accessor to the segment's `StoreReader`.
@@ -144,7 +220,7 @@ impl SegmentReader {
     }
 
     /// Open a new segment for reading.
-    pub fn open(segment: &Segment) -> crate::Result<SegmentReader> {
+    pub fn open(segment: &Segment) -> Result<SegmentReader> {
         let termdict_source = segment.open_read(SegmentComponent::TERMS)?;
         let termdict_composite = CompositeFile::open(&termdict_source)?;
 
@@ -171,15 +247,11 @@ impl SegmentReader {
             }
         };
 
-        let schema = segment.schema();
-
         let fast_fields_data = segment.open_read(SegmentComponent::FASTFIELDS)?;
         let fast_fields_composite = CompositeFile::open(&fast_fields_data)?;
-        let fast_field_readers =
-            Arc::new(FastFieldReaders::load_all(&schema, &fast_fields_composite)?);
 
-        let fieldnorm_data = segment.open_read(SegmentComponent::FIELDNORMS)?;
-        let fieldnorm_readers = FieldNormReaders::open(fieldnorm_data)?;
+        let fieldnorms_data = segment.open_read(SegmentComponent::FIELDNORMS)?;
+        let fieldnorms_composite = CompositeFile::open(&fieldnorms_data)?;
 
         let delete_bitset_opt = if segment.meta().has_deletes() {
             let delete_data = segment.open_read(SegmentComponent::DELETE)?;
@@ -188,21 +260,21 @@ impl SegmentReader {
             None
         };
 
+        let schema = segment.schema();
         Ok(SegmentReader {
             inv_idx_reader_cache: Arc::new(RwLock::new(HashMap::new())),
             max_doc: segment.meta().max_doc(),
             num_docs: segment.meta().num_docs(),
             termdict_composite,
             postings_composite,
-            fast_fields_readers: fast_field_readers,
-            fieldnorm_readers,
+            fast_fields_composite,
+            fieldnorms_composite,
             segment_id: segment.id(),
             store_source,
             delete_bitset_opt,
             positions_composite,
             positions_idx_composite,
             schema,
-            logger: segment.index().logger().clone(),
         })
     }
 
@@ -213,11 +285,6 @@ impl SegmentReader {
     /// The field reader is in charge of iterating through the
     /// term dictionary associated to a specific field,
     /// and opening the posting list associated to any term.
-    ///
-    /// If the field is marked as index, a warn is logged and an empty `InvertedIndexReader`
-    /// is returned.
-    /// Similarly if the field is marked as indexed but no term has been indexed for the given
-    /// index. an empty `InvertedIndexReader` is returned (but no warning is logged).
     pub fn inverted_index(&self, field: Field) -> Arc<InvertedIndexReader> {
         if let Some(inv_idx_reader) = self
             .inv_idx_reader_cache
@@ -232,30 +299,27 @@ impl SegmentReader {
         let record_option_opt = field_type.get_index_record_option();
 
         if record_option_opt.is_none() {
-            warn!(
-                self.logger,
-                "Field {:?} does not seem indexed.",
-                field_entry.name()
-            );
+            panic!("Field {:?} does not seem indexed.", field_entry.name());
         }
 
+        let record_option = record_option_opt.unwrap();
+
         let postings_source_opt = self.postings_composite.open_read(field);
 
-        if postings_source_opt.is_none() || record_option_opt.is_none() {
+        if postings_source_opt.is_none() {
             // no documents in the segment contained this field.
             // As a result, no data is associated to the inverted index.
             //
             // Returns an empty inverted index.
-            let record_option = record_option_opt.unwrap_or(IndexRecordOption::Basic);
-            return Arc::new(InvertedIndexReader::empty(record_option));
+            return Arc::new(InvertedIndexReader::empty(field_type));
         }
 
-        let record_option = record_option_opt.unwrap();
         let postings_source = postings_source_opt.unwrap();
 
-        let termdict_source = self.termdict_composite.open_read(field).expect(
-            "Failed to open field term dictionary in composite file. Is the field indexed?",
-        );
+        let termdict_source = self
+            .termdict_composite
+            .open_read(field)
+            .expect("Failed to open field term dictionary in composite file. Is the field indexed");
 
         let positions_source = self
             .positions_composite
@@ -305,8 +369,8 @@ impl SegmentReader {
     }
 
     /// Returns an iterator that will iterate over the alive document ids
-    pub fn doc_ids_alive<'a>(&'a self) -> impl Iterator<Item = DocId> + 'a {
-        (0u32..self.max_doc).filter(move |doc| !self.is_deleted(*doc))
+    pub fn doc_ids_alive(&self) -> SegmentReaderAliveDocsIterator {
+        SegmentReaderAliveDocsIterator::new(&self)
     }
 
     /// Summarize total space usage of this segment.
@@ -317,28 +381,74 @@ impl SegmentReader {
             self.postings_composite.space_usage(),
             self.positions_composite.space_usage(),
             self.positions_idx_composite.space_usage(),
-            self.fast_fields_readers.space_usage(),
-            self.fieldnorm_readers.space_usage(),
+            self.fast_fields_composite.space_usage(),
+            self.fieldnorms_composite.space_usage(),
             self.get_store_reader().space_usage(),
             self.delete_bitset_opt
                 .as_ref()
-                .map(DeleteBitSet::space_usage)
+                .map(|x| x.space_usage())
                 .unwrap_or(0),
         )
     }
 }
 
 impl fmt::Debug for SegmentReader {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         write!(f, "SegmentReader({:?})", self.segment_id)
     }
 }
 
+/// Implements the iterator trait to allow easy iteration
+/// over non-deleted ("alive") DocIds in a SegmentReader
+pub struct SegmentReaderAliveDocsIterator<'a> {
+    reader: &'a SegmentReader,
+    max_doc: DocId,
+    current: DocId,
+}
+
+impl<'a> SegmentReaderAliveDocsIterator<'a> {
+    pub fn new(reader: &'a SegmentReader) -> SegmentReaderAliveDocsIterator<'a> {
+        SegmentReaderAliveDocsIterator {
+            reader,
+            max_doc: reader.max_doc(),
+            current: 0,
+        }
+    }
+}
+
+impl<'a> Iterator for SegmentReaderAliveDocsIterator<'a> {
+    type Item = DocId;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        // TODO: Use TinySet (like in BitSetDocSet) to speed this process up
+        if self.current >= self.max_doc {
+            return None;
+        }
+
+        // find the next alive doc id
+        while self.reader.is_deleted(self.current) {
+            self.current += 1;
+
+            if self.current >= self.max_doc {
+                return None;
+            }
+        }
+
+        // capture the current alive DocId
+        let result = Some(self.current);
+
+        // move down the chain
+        self.current += 1;
+
+        result
+    }
+}
+
 #[cfg(test)]
 mod test {
-    use crate::core::Index;
-    use crate::schema::{Schema, Term, STORED, TEXT};
-    use crate::DocId;
+    use core::Index;
+    use schema::{Schema, Term, STORED, TEXT};
+    use DocId;
 
     #[test]
     fn test_alive_docs_iterator() {
@@ -349,7 +459,7 @@ mod test {
         let name = schema.get_field("name").unwrap();
 
         {
-            let mut index_writer = index.writer_for_tests().unwrap();
+            let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
             index_writer.add_document(doc!(name => "tantivy"));
             index_writer.add_document(doc!(name => "horse"));
             index_writer.add_document(doc!(name => "jockey"));
@@ -367,7 +477,9 @@ mod test {
             // ok, now we should have a deleted doc
             index_writer2.commit().unwrap();
         }
-        let searcher = index.reader().unwrap().searcher();
+
+        index.load_searchers().unwrap();
+        let searcher = index.searcher();
         let docs: Vec<DocId> = searcher.segment_reader(0).doc_ids_alive().collect();
         assert_eq!(vec![0u32, 2u32], docs);
     }

src/directory/directory.rs

@@ -1,109 +1,11 @@
-use slog::{error, Logger};
-
-use crate::directory::directory_lock::Lock;
-use crate::directory::error::LockError;
-use crate::directory::error::{DeleteError, OpenReadError, OpenWriteError};
-use crate::directory::WatchCallback;
-use crate::directory::WatchHandle;
-use crate::directory::{ReadOnlySource, WritePtr};
+use directory::error::{DeleteError, OpenReadError, OpenWriteError};
+use directory::{ReadOnlySource, WritePtr};
 use std::fmt;
 use std::io;
-use std::io::Write;
 use std::marker::Send;
 use std::marker::Sync;
 use std::path::Path;
-use std::path::PathBuf;
 use std::result;
-use std::thread;
-use std::time::Duration;
-
-/// Retry the logic of acquiring locks is pretty simple.
-/// We just retry `n` times after a given `duratio`, both
-/// depending on the type of lock.
-struct RetryPolicy {
-    num_retries: usize,
-    wait_in_ms: u64,
-}
-
-impl RetryPolicy {
-    fn no_retry() -> RetryPolicy {
-        RetryPolicy {
-            num_retries: 0,
-            wait_in_ms: 0,
-        }
-    }
-
-    fn wait_and_retry(&mut self) -> bool {
-        if self.num_retries == 0 {
-            false
-        } else {
-            self.num_retries -= 1;
-            let wait_duration = Duration::from_millis(self.wait_in_ms);
-            thread::sleep(wait_duration);
-            true
-        }
-    }
-}
-
-/// The `DirectoryLock` is an object that represents a file lock.
-/// See  [`LockType`](struct.LockType.html)
-///
-/// It is transparently associated to a lock file, that gets deleted
-/// on `Drop.` The lock is released automatically on `Drop`.
-pub struct DirectoryLock(Box<dyn Send + Sync + 'static>);
-
-struct DirectoryLockGuard {
-    directory: Box<dyn Directory>,
-    path: PathBuf,
-}
-
-impl<T: Send + Sync + 'static> From<Box<T>> for DirectoryLock {
-    fn from(underlying: Box<T>) -> Self {
-        DirectoryLock(underlying)
-    }
-}
-
-impl Drop for DirectoryLockGuard {
-    fn drop(&mut self) {
-        if let Err(e) = self.directory.delete(&*self.path) {
-            error!(
-                self.directory.logger(),
-                "Failed to remove the lock file. {:?}", e
-            );
-        }
-    }
-}
-
-enum TryAcquireLockError {
-    FileExists,
-    IOError(io::Error),
-}
-
-fn try_acquire_lock(
-    filepath: &Path,
-    directory: &mut dyn Directory,
-) -> Result<DirectoryLock, TryAcquireLockError> {
-    let mut write = directory.open_write(filepath).map_err(|e| match e {
-        OpenWriteError::FileAlreadyExists(_) => TryAcquireLockError::FileExists,
-        OpenWriteError::IOError { io_error, .. } => TryAcquireLockError::IOError(io_error),
-    })?;
-    write.flush().map_err(TryAcquireLockError::IOError)?;
-    Ok(DirectoryLock::from(Box::new(DirectoryLockGuard {
-        directory: directory.box_clone(),
-        path: filepath.to_owned(),
-    })))
-}
-
-fn retry_policy(is_blocking: bool) -> RetryPolicy {
-    if is_blocking {
-        RetryPolicy {
-            num_retries: 100,
-            wait_in_ms: 100,
-        }
-    } else {
-        RetryPolicy::no_retry()
-    }
-}
 
 /// Write-once read many (WORM) abstraction for where
 /// tantivy's data should be stored.
@@ -123,8 +25,6 @@ pub trait Directory: DirectoryClone + fmt::Debug + Send + Sync + 'static {
     ///
     /// Specifically, subsequent writes or flushes should
     /// have no effect on the returned `ReadOnlySource` object.
-    ///
-    /// You should only use this to read files create with [Directory::open_write].
     fn open_read(&self, path: &Path) -> result::Result<ReadOnlySource, OpenReadError>;
 
     /// Removes a file
@@ -164,8 +64,6 @@ pub trait Directory: DirectoryClone + fmt::Debug + Send + Sync + 'static {
     /// atomic_write.
     ///
     /// This should only be used for small files.
-    ///
-    /// You should only use this to read files create with [Directory::atomic_write].
     fn atomic_read(&self, path: &Path) -> Result<Vec<u8>, OpenReadError>;
 
     /// Atomically replace the content of a file with data.
@@ -175,61 +73,19 @@ pub trait Directory: DirectoryClone + fmt::Debug + Send + Sync + 'static {
     ///
     /// The file may or may not previously exist.
     fn atomic_write(&mut self, path: &Path, data: &[u8]) -> io::Result<()>;
-
-    /// Acquire a lock in the given directory.
-    ///
-    /// The method is blocking or not depending on the `Lock` object.
-    fn acquire_lock(&self, lock: &Lock) -> Result<DirectoryLock, LockError> {
-        let mut box_directory = self.box_clone();
-        let mut retry_policy = retry_policy(lock.is_blocking);
-        loop {
-            match try_acquire_lock(&lock.filepath, &mut *box_directory) {
-                Ok(result) => {
-                    return Ok(result);
-                }
-                Err(TryAcquireLockError::FileExists) => {
-                    if !retry_policy.wait_and_retry() {
-                        return Err(LockError::LockBusy);
-                    }
-                }
-                Err(TryAcquireLockError::IOError(io_error)) => {
-                    return Err(LockError::IOError(io_error));
-                }
-            }
-        }
-    }
-
-    /// Registers a callback that will be called whenever a change on the `meta.json`
-    /// using the `atomic_write` API is detected.
-    ///
-    /// The behavior when using `.watch()` on a file using [Directory::open_write] is, on the other
-    /// hand, undefined.
-    ///
-    /// The file will be watched for the lifetime of the returned `WatchHandle`. The caller is
-    /// required to keep it.
-    /// It does not override previous callbacks. When the file is modified, all callback that are
-    /// registered (and whose `WatchHandle` is still alive) are triggered.
-    ///
-    /// Internally, tantivy only uses this API to detect new commits to implement the
-    /// `OnCommit` `ReloadPolicy`. Not implementing watch in a `Directory` only prevents the
-    /// `OnCommit` `ReloadPolicy` to work properly.
-    fn watch(&self, watch_callback: WatchCallback) -> crate::Result<WatchHandle>;
-
-    /// Returns the `slog::Logger` configured for the `Directory`.
-    fn logger(&self) -> &Logger;
 }
 
 /// DirectoryClone
 pub trait DirectoryClone {
     /// Clones the directory and boxes the clone
-    fn box_clone(&self) -> Box<dyn Directory>;
+    fn box_clone(&self) -> Box<Directory>;
 }
 
 impl<T> DirectoryClone for T
 where
     T: 'static + Directory + Clone,
 {
-    fn box_clone(&self) -> Box<dyn Directory> {
+    fn box_clone(&self) -> Box<Directory> {
         Box::new(self.clone())
     }
 }

src/directory/directory_lock.rs

@@ -1,55 +0,0 @@
-use once_cell::sync::Lazy;
-use std::path::PathBuf;
-
-/// A directory lock.
-///
-/// A lock is associated to a specific path and some
-/// [`LockParams`](./enum.LockParams.html).
-/// Tantivy itself uses only two locks but client application
-/// can use the directory facility to define their own locks.
-/// - [INDEX_WRITER_LOCK](./struct.INDEX_WRITER_LOCK.html)
-/// - [META_LOCK](./struct.META_LOCK.html)
-///
-/// Check out these locks documentation for more information.
-///
-#[derive(Debug)]
-pub struct Lock {
-    /// The lock needs to be associated with its own file `path`.
-    /// Depending on the platform, the lock might rely on the creation
-    /// and deletion of this filepath.
-    pub filepath: PathBuf,
-    /// `lock_params` describes whether acquiring the lock is meant
-    /// to be a blocking operation or a non-blocking.
-    ///
-    /// Acquiring a blocking lock blocks until the lock is
-    /// available.
-    /// Acquiring a blocking lock returns rapidly, either successfully
-    /// or with an error signifying that someone is already holding
-    /// the lock.
-    pub is_blocking: bool,
-}
-
-/// Only one process should be able to write tantivy's index at a time.
-/// This lock file, when present, is in charge of preventing other processes to open an IndexWriter.
-///
-/// If the process is killed and this file remains, it is safe to remove it manually.
-///
-/// Failing to acquire this lock usually means a misuse of tantivy's API,
-/// (creating more than one instance of the `IndexWriter`), are a spurious
-/// lock file remaining after a crash. In the latter case, removing the file after
-/// checking no process running tantivy is running is safe.
-pub static INDEX_WRITER_LOCK: Lazy<Lock> = Lazy::new(|| Lock {
-    filepath: PathBuf::from(".tantivy-writer.lock"),
-    is_blocking: false,
-});
-/// The meta lock file is here to protect the segment files being opened by
-/// `IndexReader::reload()` from being garbage collected.
-/// It makes it possible for another process to safely consume
-/// our index in-writing. Ideally, we may have prefered `RWLock` semantics
-/// here, but it is difficult to achieve on Windows.
-///
-/// Opening segment readers is a very fast process.
-pub static META_LOCK: Lazy<Lock> = Lazy::new(|| Lock {
-    filepath: PathBuf::from(".tantivy-meta.lock"),
-    is_blocking: true,
-});

src/directory/error.rs

@@ -1,161 +1,208 @@
-use crate::Version;
+use std::error::Error as StdError;
 use std::fmt;
 use std::io;
 use std::path::PathBuf;
 
-/// Error while trying to acquire a directory lock.
-#[derive(Debug, Error)]
-pub enum LockError {
-    /// Failed to acquired a lock as it is already held by another
-    /// client.
-    /// - In the context of a blocking lock, this means the lock was not released within some `timeout` period.
-    /// - In the context of a non-blocking lock, this means the lock was busy at the moment of the call.
-    #[error("Could not acquire lock as it is already held, possibly by a different process.")]
-    LockBusy,
-    /// Trying to acquire a lock failed with an `IOError`
-    #[error("Failed to acquire the lock due to an io:Error.")]
-    IOError(io::Error),
+/// General IO error with an optional path to the offending file.
+#[derive(Debug)]
+pub struct IOError {
+    path: Option<PathBuf>,
+    err: io::Error,
+}
+
+impl fmt::Display for IOError {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match self.path {
+            Some(ref path) => write!(f, "io error occurred on path '{:?}': '{}'", path, self.err),
+            None => write!(f, "io error occurred: '{}'", self.err),
+        }
+    }
+}
+
+impl StdError for IOError {
+    fn description(&self) -> &str {
+        "io error occurred"
+    }
+
+    fn cause(&self) -> Option<&StdError> {
+        Some(&self.err)
+    }
+}
+
+impl IOError {
+    pub(crate) fn with_path(path: PathBuf, err: io::Error) -> Self {
+        IOError {
+            path: Some(path),
+            err,
+        }
+    }
+}
+
+impl From<io::Error> for IOError {
+    fn from(err: io::Error) -> IOError {
+        IOError { path: None, err }
+    }
 }
 
 /// Error that may occur when opening a directory
-#[derive(Debug, Error)]
+#[derive(Debug)]
 pub enum OpenDirectoryError {
     /// The underlying directory does not exists.
-    #[error("Directory does not exist: '{0}'.")]
     DoesNotExist(PathBuf),
     /// The path exists but is not a directory.
-    #[error("Path exists but is not a directory: '{0}'.")]
     NotADirectory(PathBuf),
-    /// Failed to create a temp directory.
-    #[error("Failed to create a temporary directory: '{0}'.")]
-    FailedToCreateTempDir(io::Error),
-    /// IoError
-    #[error("IOError '{io_error:?}' while create directory in: '{directory_path:?}'.")]
-    IoError {
-        /// underlying io Error.
-        io_error: io::Error,
-        /// directory we tried to open.
-        directory_path: PathBuf,
-    },
+}
+
+impl fmt::Display for OpenDirectoryError {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match *self {
+            OpenDirectoryError::DoesNotExist(ref path) => {
+                write!(f, "the underlying directory '{:?}' does not exist", path)
+            }
+            OpenDirectoryError::NotADirectory(ref path) => {
+                write!(f, "the path '{:?}' exists but is not a directory", path)
+            }
+        }
+    }
+}
+
+impl StdError for OpenDirectoryError {
+    fn description(&self) -> &str {
+        "error occurred while opening a directory"
+    }
+
+    fn cause(&self) -> Option<&StdError> {
+        None
+    }
 }
 
 /// Error that may occur when starting to write in a file
-#[derive(Debug, Error)]
+#[derive(Debug)]
 pub enum OpenWriteError {
     /// Our directory is WORM, writing an existing file is forbidden.
     /// Checkout the `Directory` documentation.
-    #[error("File already exists: '{0}'")]
     FileAlreadyExists(PathBuf),
     /// Any kind of IO error that happens when
     /// writing in the underlying IO device.
-    #[error("IOError '{io_error:?}' while opening file for write: '{filepath}'.")]
-    IOError {
-        /// The underlying `io::Error`.
-        io_error: io::Error,
-        /// File path of the file that tantivy failed to open for write.
-        filepath: PathBuf,
-    },
+    IOError(IOError),
 }
 
-/// Type of index incompatibility between the library and the index found on disk
-/// Used to catch and provide a hint to solve this incompatibility issue
-pub enum Incompatibility {
-    /// This library cannot decompress the index found on disk
-    CompressionMismatch {
-        /// Compression algorithm used by the current version of tantivy
-        library_compression_format: String,
-        /// Compression algorithm that was used to serialise the index
-        index_compression_format: String,
-    },
-    /// The index format found on disk isn't supported by this version of the library
-    IndexMismatch {
-        /// Version used by the library
-        library_version: Version,
-        /// Version the index was built with
-        index_version: Version,
-    },
+impl From<IOError> for OpenWriteError {
+    fn from(err: IOError) -> OpenWriteError {
+        OpenWriteError::IOError(err)
+    }
 }
 
-impl fmt::Debug for Incompatibility {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> {
-        match self {
-            Incompatibility::CompressionMismatch {
-                library_compression_format,
-                index_compression_format,
-            } => {
-                let err = format!(
-                    "Library was compiled with {:?} compression, index was compressed with {:?}",
-                    library_compression_format, index_compression_format
-                );
-                let advice = format!(
-                    "Change the feature flag to {:?} and rebuild the library",
-                    index_compression_format
-                );
-                write!(f, "{}. {}", err, advice)?;
-            }
-            Incompatibility::IndexMismatch {
-                library_version,
-                index_version,
-            } => {
-                let err = format!(
-                    "Library version: {}, index version: {}",
-                    library_version.index_format_version, index_version.index_format_version
-                );
-                // TODO make a more useful error message
-                // include the version range that supports this index_format_version
-                let advice = format!(
-                    "Change tantivy to a version compatible with index format {} (e.g. {}.{}.x) \
-                     and rebuild your project.",
-                    index_version.index_format_version, index_version.major, index_version.minor
-                );
-                write!(f, "{}. {}", err, advice)?;
+impl fmt::Display for OpenWriteError {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match *self {
+            OpenWriteError::FileAlreadyExists(ref path) => {
+                write!(f, "the file '{:?}' already exists", path)
             }
+            OpenWriteError::IOError(ref err) => write!(
+                f,
+                "an io error occurred while opening a file for writing: '{}'",
+                err
+            ),
         }
+    }
+}
 
-        Ok(())
+impl StdError for OpenWriteError {
+    fn description(&self) -> &str {
+        "error occurred while opening a file for writing"
+    }
+
+    fn cause(&self) -> Option<&StdError> {
+        match *self {
+            OpenWriteError::FileAlreadyExists(_) => None,
+            OpenWriteError::IOError(ref err) => Some(err),
+        }
     }
 }
 
 /// Error that may occur when accessing a file read
-#[derive(Debug, Error)]
+#[derive(Debug)]
 pub enum OpenReadError {
     /// The file does not exists.
-    #[error("Files does not exists: {0:?}")]
-    FileDoesNotExist(PathBuf),
-    /// Any kind of io::Error.
-    #[error(
-        "IOError: '{io_error:?}' happened while opening the following file for Read: {filepath}."
-    )]
-    IOError {
-        /// The underlying `io::Error`.
-        io_error: io::Error,
-        /// File path of the file that tantivy failed to open for read.
-        filepath: PathBuf,
-    },
-    /// This library does not support the index version found in file footer.
-    #[error("Index version unsupported: {0:?}")]
-    IncompatibleIndex(Incompatibility),
-}
-
-/// Error that may occur when trying to delete a file
-#[derive(Debug, Error)]
-pub enum DeleteError {
-    /// The file does not exists.
-    #[error("File does not exists: '{0}'.")]
     FileDoesNotExist(PathBuf),
     /// Any kind of IO error that happens when
     /// interacting with the underlying IO device.
-    #[error("The following IO error happened while deleting file '{filepath}': '{io_error:?}'.")]
-    IOError {
-        /// The underlying `io::Error`.
-        io_error: io::Error,
-        /// File path of the file that tantivy failed to delete.
-        filepath: PathBuf,
-    },
+    IOError(IOError),
 }
 
-impl From<Incompatibility> for OpenReadError {
-    fn from(incompatibility: Incompatibility) -> Self {
-        OpenReadError::IncompatibleIndex(incompatibility)
+impl From<IOError> for OpenReadError {
+    fn from(err: IOError) -> OpenReadError {
+        OpenReadError::IOError(err)
+    }
+}
+
+impl fmt::Display for OpenReadError {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match *self {
+            OpenReadError::FileDoesNotExist(ref path) => {
+                write!(f, "the file '{:?}' does not exist", path)
+            }
+            OpenReadError::IOError(ref err) => write!(
+                f,
+                "an io error occurred while opening a file for reading: '{}'",
+                err
+            ),
+        }
+    }
+}
+
+impl StdError for OpenReadError {
+    fn description(&self) -> &str {
+        "error occurred while opening a file for reading"
+    }
+
+    fn cause(&self) -> Option<&StdError> {
+        match *self {
+            OpenReadError::FileDoesNotExist(_) => None,
+            OpenReadError::IOError(ref err) => Some(err),
+        }
+    }
+}
+
+/// Error that may occur when trying to delete a file
+#[derive(Debug)]
+pub enum DeleteError {
+    /// The file does not exists.
+    FileDoesNotExist(PathBuf),
+    /// Any kind of IO error that happens when
+    /// interacting with the underlying IO device.
+    IOError(IOError),
+}
+
+impl From<IOError> for DeleteError {
+    fn from(err: IOError) -> DeleteError {
+        DeleteError::IOError(err)
+    }
+}
+
+impl fmt::Display for DeleteError {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match *self {
+            DeleteError::FileDoesNotExist(ref path) => {
+                write!(f, "the file '{:?}' does not exist", path)
+            }
+            DeleteError::IOError(ref err) => {
+                write!(f, "an io error occurred while deleting a file: '{}'", err)
+            }
+        }
+    }
+}
+
+impl StdError for DeleteError {
+    fn description(&self) -> &str {
+        "error occurred while deleting a file"
+    }
+
+    fn cause(&self) -> Option<&StdError> {
+        match *self {
+            DeleteError::FileDoesNotExist(_) => None,
+            DeleteError::IOError(ref err) => Some(err),
+        }
     }
 }

src/directory/footer.rs

@@ -1,394 +0,0 @@
-use crate::common::{BinarySerializable, CountingWriter, FixedSize, VInt};
-use crate::directory::error::Incompatibility;
-use crate::directory::read_only_source::ReadOnlySource;
-use crate::directory::{AntiCallToken, TerminatingWrite};
-use crate::Version;
-use byteorder::{ByteOrder, LittleEndian, WriteBytesExt};
-use crc32fast::Hasher;
-use std::io;
-use std::io::Write;
-
-const FOOTER_MAX_LEN: usize = 10_000;
-
-type CrcHashU32 = u32;
-
-#[derive(Debug, Clone, PartialEq)]
-pub struct Footer {
-    pub version: Version,
-    pub meta: String,
-    pub versioned_footer: VersionedFooter,
-}
-
-/// Serialises the footer to a byte-array
-/// - versioned_footer_len : 4 bytes
-///-  versioned_footer: variable bytes
-/// - meta_len: 4 bytes
-/// - meta: variable bytes
-/// - version_len: 4 bytes
-/// - version json: variable bytes
-impl BinarySerializable for Footer {
-    fn serialize<W: io::Write>(&self, writer: &mut W) -> io::Result<()> {
-        BinarySerializable::serialize(&self.versioned_footer, writer)?;
-        BinarySerializable::serialize(&self.meta, writer)?;
-        let version_string =
-            serde_json::to_string(&self.version).map_err(|_err| io::ErrorKind::InvalidInput)?;
-        BinarySerializable::serialize(&version_string, writer)?;
-        Ok(())
-    }
-
-    fn deserialize<R: io::Read>(reader: &mut R) -> io::Result<Self> {
-        let versioned_footer = VersionedFooter::deserialize(reader)?;
-        let meta = String::deserialize(reader)?;
-        let version_json = String::deserialize(reader)?;
-        let version = serde_json::from_str(&version_json)?;
-        Ok(Footer {
-            version,
-            meta,
-            versioned_footer,
-        })
-    }
-}
-
-impl Footer {
-    pub fn new(versioned_footer: VersionedFooter) -> Self {
-        let version = crate::VERSION.clone();
-        let meta = version.to_string();
-        Footer {
-            version,
-            meta,
-            versioned_footer,
-        }
-    }
-
-    pub fn append_footer<W: io::Write>(&self, mut write: &mut W) -> io::Result<()> {
-        let mut counting_write = CountingWriter::wrap(&mut write);
-        self.serialize(&mut counting_write)?;
-        let written_len = counting_write.written_bytes();
-        write.write_u32::<LittleEndian>(written_len as u32)?;
-        Ok(())
-    }
-
-    pub fn extract_footer(source: ReadOnlySource) -> Result<(Footer, ReadOnlySource), io::Error> {
-        if source.len() < 4 {
-            return Err(io::Error::new(
-                io::ErrorKind::UnexpectedEof,
-                format!(
-                    "File corrupted. The file is smaller than 4 bytes (len={}).",
-                    source.len()
-                ),
-            ));
-        }
-        let (body_footer, footer_len_bytes) = source.split_from_end(u32::SIZE_IN_BYTES);
-        let footer_len = LittleEndian::read_u32(footer_len_bytes.as_slice()) as usize;
-        let body_len = body_footer.len() - footer_len;
-        let (body, footer_data) = body_footer.split(body_len);
-        let mut cursor = footer_data.as_slice();
-        let footer = Footer::deserialize(&mut cursor)?;
-        Ok((footer, body))
-    }
-
-    /// Confirms that the index will be read correctly by this version of tantivy
-    /// Has to be called after `extract_footer` to make sure it's not accessing uninitialised memory
-    pub fn is_compatible(&self) -> Result<(), Incompatibility> {
-        let library_version = crate::version();
-        match &self.versioned_footer {
-            VersionedFooter::V1 {
-                crc32: _crc,
-                store_compression,
-            } => {
-                if &library_version.store_compression != store_compression {
-                    return Err(Incompatibility::CompressionMismatch {
-                        library_compression_format: library_version.store_compression.to_string(),
-                        index_compression_format: store_compression.to_string(),
-                    });
-                }
-                Ok(())
-            }
-            VersionedFooter::V2 {
-                crc32: _crc,
-                store_compression,
-            } => {
-                if &library_version.store_compression != store_compression {
-                    return Err(Incompatibility::CompressionMismatch {
-                        library_compression_format: library_version.store_compression.to_string(),
-                        index_compression_format: store_compression.to_string(),
-                    });
-                }
-                Ok(())
-            }
-            VersionedFooter::UnknownVersion => Err(Incompatibility::IndexMismatch {
-                library_version: library_version.clone(),
-                index_version: self.version.clone(),
-            }),
-        }
-    }
-}
-
-/// Footer that includes a crc32 hash that enables us to checksum files in the index
-#[derive(Debug, Clone, PartialEq)]
-pub enum VersionedFooter {
-    UnknownVersion,
-    V1 {
-        crc32: CrcHashU32,
-        store_compression: String,
-    },
-    // Introduction of the Block WAND information.
-    V2 {
-        crc32: CrcHashU32,
-        store_compression: String,
-    },
-}
-
-impl BinarySerializable for VersionedFooter {
-    fn serialize<W: io::Write>(&self, writer: &mut W) -> io::Result<()> {
-        let mut buf = Vec::new();
-        match self {
-            VersionedFooter::V2 {
-                crc32,
-                store_compression: compression,
-            } => {
-                // Serializes a valid `VersionedFooter` or panics if the version is unknown
-                // [   version    |   crc_hash  | compression_mode ]
-                // [    0..4      |     4..8    |     variable     ]
-                BinarySerializable::serialize(&2u32, &mut buf)?;
-                BinarySerializable::serialize(crc32, &mut buf)?;
-                BinarySerializable::serialize(compression, &mut buf)?;
-            }
-            VersionedFooter::V1 { .. } | VersionedFooter::UnknownVersion => {
-                return Err(io::Error::new(
-                    io::ErrorKind::InvalidInput,
-                    "Cannot serialize an unknown versioned footer ",
-                ));
-            }
-        }
-        BinarySerializable::serialize(&VInt(buf.len() as u64), writer)?;
-        assert!(buf.len() <= FOOTER_MAX_LEN);
-        writer.write_all(&buf[..])?;
-        Ok(())
-    }
-
-    fn deserialize<R: io::Read>(reader: &mut R) -> io::Result<Self> {
-        let len = VInt::deserialize(reader)?.0 as usize;
-        if len > FOOTER_MAX_LEN {
-            return Err(io::Error::new(
-                io::ErrorKind::InvalidData,
-                format!(
-                    "Footer seems invalid as it suggests a footer len of {}. File is corrupted, \
-            or the index was created with a different & old version of tantivy.",
-                    len
-                ),
-            ));
-        }
-        let mut buf = vec![0u8; len];
-        reader.read_exact(&mut buf[..])?;
-        let mut cursor = &buf[..];
-        let version = u32::deserialize(&mut cursor)?;
-        if version != 1 && version != 2 {
-            return Ok(VersionedFooter::UnknownVersion);
-        }
-        let crc32 = u32::deserialize(&mut cursor)?;
-        let store_compression = String::deserialize(&mut cursor)?;
-        Ok(if version == 1 {
-            VersionedFooter::V1 {
-                crc32,
-                store_compression,
-            }
-        } else {
-            assert_eq!(version, 2);
-            VersionedFooter::V2 {
-                crc32,
-                store_compression,
-            }
-        })
-    }
-}
-
-impl VersionedFooter {
-    pub fn crc(&self) -> Option<CrcHashU32> {
-        match self {
-            VersionedFooter::V2 { crc32, .. } => Some(*crc32),
-            VersionedFooter::V1 { crc32, .. } => Some(*crc32),
-            VersionedFooter::UnknownVersion { .. } => None,
-        }
-    }
-}
-
-pub(crate) struct FooterProxy<W: TerminatingWrite> {
-    /// always Some except after terminate call
-    hasher: Option<Hasher>,
-    /// always Some except after terminate call
-    writer: Option<W>,
-}
-
-impl<W: TerminatingWrite> FooterProxy<W> {
-    pub fn new(writer: W) -> Self {
-        FooterProxy {
-            hasher: Some(Hasher::new()),
-            writer: Some(writer),
-        }
-    }
-}
-
-impl<W: TerminatingWrite> Write for FooterProxy<W> {
-    fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
-        let count = self.writer.as_mut().unwrap().write(buf)?;
-        self.hasher.as_mut().unwrap().update(&buf[..count]);
-        Ok(count)
-    }
-
-    fn flush(&mut self) -> io::Result<()> {
-        self.writer.as_mut().unwrap().flush()
-    }
-}
-
-impl<W: TerminatingWrite> TerminatingWrite for FooterProxy<W> {
-    fn terminate_ref(&mut self, _: AntiCallToken) -> io::Result<()> {
-        let crc32 = self.hasher.take().unwrap().finalize();
-        let footer = Footer::new(VersionedFooter::V2 {
-            crc32,
-            store_compression: crate::store::COMPRESSION.to_string(),
-        });
-        let mut writer = self.writer.take().unwrap();
-        footer.append_footer(&mut writer)?;
-        writer.terminate()
-    }
-}
-
-#[cfg(test)]
-mod tests {
-
-    use super::CrcHashU32;
-    use super::FooterProxy;
-    use crate::common::{BinarySerializable, VInt};
-    use crate::directory::footer::{Footer, VersionedFooter};
-    use crate::directory::TerminatingWrite;
-    use byteorder::{ByteOrder, LittleEndian};
-    use regex::Regex;
-    use std::io;
-
-    #[test]
-    fn test_versioned_footer() {
-        let mut vec = Vec::new();
-        let footer_proxy = FooterProxy::new(&mut vec);
-        assert!(footer_proxy.terminate().is_ok());
-        if crate::store::COMPRESSION == "lz4" {
-            assert_eq!(vec.len(), 158);
-        } else {
-            assert_eq!(vec.len(), 167);
-        }
-        let footer = Footer::deserialize(&mut &vec[..]).unwrap();
-        assert!(matches!(
-           footer.versioned_footer,
-           VersionedFooter::V2 { store_compression, .. }
-           if store_compression == crate::store::COMPRESSION
-        ));
-        assert_eq!(&footer.version, crate::version());
-    }
-
-    #[test]
-    fn test_serialize_deserialize_footer() {
-        let mut buffer = Vec::new();
-        let crc32 = 123456u32;
-        let footer: Footer = Footer::new(VersionedFooter::V2 {
-            crc32,
-            store_compression: "lz4".to_string(),
-        });
-        footer.serialize(&mut buffer).unwrap();
-        let footer_deser = Footer::deserialize(&mut &buffer[..]).unwrap();
-        assert_eq!(footer_deser, footer);
-    }
-
-    #[test]
-    fn footer_length() {
-        let crc32 = 1111111u32;
-        let versioned_footer = VersionedFooter::V2 {
-            crc32,
-            store_compression: "lz4".to_string(),
-        };
-        let mut buf = Vec::new();
-        versioned_footer.serialize(&mut buf).unwrap();
-        assert_eq!(buf.len(), 13);
-        let footer = Footer::new(versioned_footer);
-        let regex_ptn = Regex::new(
-            "tantivy v[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.{0,10}, index_format v[0-9]{1,5}",
-        )
-        .unwrap();
-        assert!(regex_ptn.is_match(&footer.meta));
-    }
-
-    #[test]
-    fn versioned_footer_from_bytes() {
-        let v_footer_bytes = vec![
-            // versionned footer length
-            12 | 128,
-            // index format version
-            2,
-            0,
-            0,
-            0,
-            // crc 32
-            12,
-            35,
-            89,
-            18,
-            // compression format
-            3 | 128,
-            b'l',
-            b'z',
-            b'4',
-        ];
-        let mut cursor = &v_footer_bytes[..];
-        let versioned_footer = VersionedFooter::deserialize(&mut cursor).unwrap();
-        assert!(cursor.is_empty());
-        let expected_crc: u32 = LittleEndian::read_u32(&v_footer_bytes[5..9]) as CrcHashU32;
-        let expected_versioned_footer: VersionedFooter = VersionedFooter::V2 {
-            crc32: expected_crc,
-            store_compression: "lz4".to_string(),
-        };
-        assert_eq!(versioned_footer, expected_versioned_footer);
-        let mut buffer = Vec::new();
-        assert!(versioned_footer.serialize(&mut buffer).is_ok());
-        assert_eq!(&v_footer_bytes[..], &buffer[..]);
-    }
-
-    #[test]
-    fn versioned_footer_panic() {
-        let v_footer_bytes = vec![6u8 | 128u8, 3u8, 0u8, 0u8, 1u8, 0u8, 0u8];
-        let mut b = &v_footer_bytes[..];
-        let versioned_footer = VersionedFooter::deserialize(&mut b).unwrap();
-        assert!(b.is_empty());
-        let expected_versioned_footer = VersionedFooter::UnknownVersion;
-        assert_eq!(versioned_footer, expected_versioned_footer);
-        let mut buf = Vec::new();
-        assert!(versioned_footer.serialize(&mut buf).is_err());
-    }
-
-    #[test]
-    #[cfg(not(feature = "lz4"))]
-    fn compression_mismatch() {
-        let crc32 = 1111111u32;
-        let versioned_footer = VersionedFooter::V1 {
-            crc32,
-            store_compression: "lz4".to_string(),
-        };
-        let footer = Footer::new(versioned_footer);
-        let res = footer.is_compatible();
-        assert!(res.is_err());
-    }
-
-    #[test]
-    fn test_deserialize_too_large_footer() {
-        let mut buf = vec![];
-        assert!(FooterProxy::new(&mut buf).terminate().is_ok());
-        let mut long_len_buf = [0u8; 10];
-        let num_bytes = VInt(super::FOOTER_MAX_LEN as u64 + 1u64).serialize_into(&mut long_len_buf);
-        buf[0..num_bytes].copy_from_slice(&long_len_buf[..num_bytes]);
-        let err = Footer::deserialize(&mut &buf[..]).unwrap_err();
-        assert_eq!(err.kind(), io::ErrorKind::InvalidData);
-        assert_eq!(
-            err.to_string(),
-            "Footer seems invalid as it suggests a footer len of 10001. File is corrupted, \
-            or the index was created with a different & old version of tantivy."
-        );
-    }
-}

src/directory/managed_directory.rs

@@ -1,23 +1,18 @@
-use crate::core::{MANAGED_FILEPATH, META_FILEPATH};
-use crate::directory::error::{DeleteError, LockError, OpenReadError, OpenWriteError};
-use crate::directory::footer::{Footer, FooterProxy};
-use crate::directory::DirectoryLock;
-use crate::directory::GarbageCollectionResult;
-use crate::directory::Lock;
-use crate::directory::META_LOCK;
-use crate::directory::{ReadOnlySource, WritePtr};
-use crate::directory::{WatchCallback, WatchHandle};
-use crate::error::DataCorruption;
-use crate::Directory;
-
-use crc32fast::Hasher;
-use slog::{debug, error, info};
+use core::MANAGED_FILEPATH;
+use directory::error::{DeleteError, IOError, OpenReadError, OpenWriteError};
+use directory::{ReadOnlySource, WritePtr};
+use error::TantivyError;
+use indexer::LockType;
+use serde_json;
 use std::collections::HashSet;
 use std::io;
+use std::io::Write;
 use std::path::{Path, PathBuf};
 use std::result;
 use std::sync::RwLockWriteGuard;
 use std::sync::{Arc, RwLock};
+use Directory;
+use Result;
 
 /// Returns true iff the file is "managed".
 /// Non-managed file are not subject to garbage collection.
@@ -41,7 +36,7 @@ fn is_managed(path: &Path) -> bool {
 /// useful anymore.
 #[derive(Debug)]
 pub struct ManagedDirectory {
-    directory: Box<dyn Directory>,
+    directory: Box<Directory>,
     meta_informations: Arc<RwLock<MetaInformation>>,
 }
 
@@ -53,28 +48,23 @@ struct MetaInformation {
 /// Saves the file containing the list of existing files
 /// that were created by tantivy.
 fn save_managed_paths(
-    directory: &mut dyn Directory,
-    wlock: &RwLockWriteGuard<'_, MetaInformation>,
+    directory: &mut Directory,
+    wlock: &RwLockWriteGuard<MetaInformation>,
 ) -> io::Result<()> {
-    let mut managed_json = serde_json::to_string_pretty(&wlock.managed_paths)?;
-    managed_json.push_str("\n");
-    directory.atomic_write(&MANAGED_FILEPATH, managed_json.as_bytes())?;
+    let mut w = serde_json::to_vec(&wlock.managed_paths)?;
+    writeln!(&mut w)?;
+    directory.atomic_write(&MANAGED_FILEPATH, &w[..])?;
     Ok(())
 }
 
 impl ManagedDirectory {
     /// Wraps a directory as managed directory.
-    pub fn wrap<Dir: Directory>(directory: Dir) -> crate::Result<ManagedDirectory> {
+    pub fn wrap<Dir: Directory>(directory: Dir) -> Result<ManagedDirectory> {
         match directory.atomic_read(&MANAGED_FILEPATH) {
             Ok(data) => {
                 let managed_files_json = String::from_utf8_lossy(&data);
                 let managed_files: HashSet<PathBuf> = serde_json::from_str(&managed_files_json)
-                    .map_err(|e| {
-                        DataCorruption::new(
-                            MANAGED_FILEPATH.to_path_buf(),
-                            format!("Managed file cannot be deserialized: {:?}. ", e),
-                        )
-                    })?;
+                    .map_err(|_| TantivyError::CorruptedFile(MANAGED_FILEPATH.clone()))?;
                 Ok(ManagedDirectory {
                     directory: Box::new(directory),
                     meta_informations: Arc::new(RwLock::new(MetaInformation {
@@ -86,17 +76,7 @@ impl ManagedDirectory {
                 directory: Box::new(directory),
                 meta_informations: Arc::default(),
             }),
-            Err(OpenReadError::IOError { io_error, filepath }) => {
-                Err(crate::TantivyError::OpenReadError(OpenReadError::IOError {
-                    io_error,
-                    filepath,
-                }))
-            }
-            Err(OpenReadError::IncompatibleIndex(incompatibility)) => {
-                // For the moment, this should never happen  `meta.json`
-                // do not have any footer and cannot detect incompatibility.
-                Err(crate::TantivyError::IncompatibleIndex(incompatibility))
-            }
+            Err(OpenReadError::IOError(e)) => Err(From::from(e)),
         }
     }
 
@@ -107,18 +87,12 @@ impl ManagedDirectory {
     ///
     /// * `living_files` - List of files that are still used by the index.
     ///
-    /// The use a callback ensures that the list of living_files is computed
-    /// while we hold the lock on meta.
-    ///
     /// This method does not panick nor returns errors.
     /// If a file cannot be deleted (for permission reasons for instance)
     /// an error is simply logged, and the file remains in the list of managed
     /// files.
-    pub fn garbage_collect<L: FnOnce() -> HashSet<PathBuf>>(
-        &mut self,
-        get_living_files: L,
-    ) -> crate::Result<GarbageCollectionResult> {
-        info!(self.directory.logger(), "gc"; "stage"=>"start");
+    pub fn garbage_collect<L: FnOnce() -> HashSet<PathBuf>>(&mut self, get_living_files: L) {
+        info!("Garbage collect");
         let mut files_to_delete = vec![];
 
         // It is crucial to get the living files after acquiring the
@@ -143,42 +117,35 @@ impl ManagedDirectory {
             // 2) writer change meta.json (for instance after a merge or a commit)
             // 3) gc kicks in.
             // 4) gc removes a file that was useful for process B, before process B opened it.
-            match self.acquire_lock(&META_LOCK) {
-                Ok(_meta_lock) => {
-                    let living_files = get_living_files();
-                    for managed_path in &meta_informations_rlock.managed_paths {
-                        if !living_files.contains(managed_path) {
-                            files_to_delete.push(managed_path.clone());
-                        }
+            if let Ok(_meta_lock) = LockType::MetaLock.acquire_lock(self) {
+                let living_files = get_living_files();
+                for managed_path in &meta_informations_rlock.managed_paths {
+                    if !living_files.contains(managed_path) {
+                        files_to_delete.push(managed_path.clone());
                     }
                 }
-                Err(err) => {
-                    error!(self.logger(), "Failed to acquire lock for GC");
-                    return Err(crate::TantivyError::from(err));
-                }
             }
         }
 
-        let mut failed_to_delete_files = vec![];
         let mut deleted_files = vec![];
-
-        for file_to_delete in files_to_delete {
-            match self.delete(&file_to_delete) {
-                Ok(_) => {
-                    debug!(self.logger(), "deleted-success"; "file"=>format!("{:?}", file_to_delete));
-                    deleted_files.push(file_to_delete);
-                }
-                Err(file_error) => {
-                    match file_error {
-                        DeleteError::FileDoesNotExist(_) => {
-                            deleted_files.push(file_to_delete.clone());
-                        }
-                        DeleteError::IOError { .. } => {
-                            failed_to_delete_files.push(file_to_delete.clone());
-                            if !cfg!(target_os = "windows") {
-                                // On windows, delete is expected to fail if the file
-                                // is mmapped.
-                                error!(self.logger(), "delete-file-fail"; "path"=>file_to_delete.to_str().unwrap_or("<invalid-utf8>"));
+        {
+            for file_to_delete in files_to_delete {
+                match self.delete(&file_to_delete) {
+                    Ok(_) => {
+                        info!("Deleted {:?}", file_to_delete);
+                        deleted_files.push(file_to_delete);
+                    }
+                    Err(file_error) => {
+                        match file_error {
+                            DeleteError::FileDoesNotExist(_) => {
+                                deleted_files.push(file_to_delete);
+                            }
+                            DeleteError::IOError(_) => {
+                                if !cfg!(target_os = "windows") {
+                                    // On windows, delete is expected to fail if the file
+                                    // is mmapped.
+                                    error!("Failed to delete {:?}", file_to_delete);
+                                }
                             }
                         }
                     }
@@ -193,21 +160,16 @@ impl ManagedDirectory {
                 .meta_informations
                 .write()
                 .expect("Managed directory wlock poisoned (2).");
-            let managed_paths_write = &mut meta_informations_wlock.managed_paths;
-            for delete_file in &deleted_files {
-                managed_paths_write.remove(delete_file);
+            {
+                let managed_paths_write = &mut meta_informations_wlock.managed_paths;
+                for delete_file in &deleted_files {
+                    managed_paths_write.remove(delete_file);
+                }
+            }
+            if save_managed_paths(self.directory.as_mut(), &meta_informations_wlock).is_err() {
+                error!("Failed to save the list of managed files.");
             }
-            save_managed_paths(self.directory.as_mut(), &meta_informations_wlock)?;
         }
-
-        info!(self.directory.logger(), "gc"; "stage"=>"end", 
-                "num-sucess-file-deletes"=>deleted_files.len(), 
-                "num-failed-file-deletes"=>failed_to_delete_files.len());
-
-        Ok(GarbageCollectionResult {
-            deleted_files,
-            failed_to_delete_files,
-        })
     }
 
     /// Registers a file as managed
@@ -236,81 +198,22 @@ impl ManagedDirectory {
         }
         Ok(())
     }
-
-    /// Verify checksum of a managed file
-    pub fn validate_checksum(&self, path: &Path) -> result::Result<bool, OpenReadError> {
-        let reader = self.directory.open_read(path)?;
-        let (footer, data) =
-            Footer::extract_footer(reader).map_err(|io_error| OpenReadError::IOError {
-                io_error,
-                filepath: path.to_path_buf(),
-            })?;
-        let mut hasher = Hasher::new();
-        hasher.update(data.as_slice());
-        let crc = hasher.finalize();
-        Ok(footer
-            .versioned_footer
-            .crc()
-            .map(|v| v == crc)
-            .unwrap_or(false))
-    }
-
-    /// List files for which checksum does not match content
-    pub fn list_damaged(&self) -> result::Result<HashSet<PathBuf>, OpenReadError> {
-        let mut managed_paths = self
-            .meta_informations
-            .read()
-            .expect("Managed directory rlock poisoned in list damaged.")
-            .managed_paths
-            .clone();
-
-        managed_paths.remove(*META_FILEPATH);
-
-        let mut damaged_files = HashSet::new();
-        for path in managed_paths {
-            if !self.validate_checksum(&path)? {
-                damaged_files.insert(path);
-            }
-        }
-        Ok(damaged_files)
-    }
 }
 
 impl Directory for ManagedDirectory {
     fn open_read(&self, path: &Path) -> result::Result<ReadOnlySource, OpenReadError> {
-        slog::debug!(self.logger(), "open-read"; "path" => path.to_str().unwrap_or("<invalid-utf8>"));
-        let read_only_source = self.directory.open_read(path)?;
-        let (footer, reader) = Footer::extract_footer(read_only_source).map_err(|io_error| {
-            OpenReadError::IOError {
-                io_error,
-                filepath: path.to_path_buf(),
-            }
-        })?;
-        footer.is_compatible()?;
-        Ok(reader)
+        self.directory.open_read(path)
     }
 
     fn open_write(&mut self, path: &Path) -> result::Result<WritePtr, OpenWriteError> {
-        slog::debug!(self.logger(), "open-write"; "path" => path.to_str().unwrap_or("<invalid-utf8>"));
         self.register_file_as_managed(path)
-            .map_err(|io_error| OpenWriteError::IOError {
-                io_error,
-                filepath: path.to_path_buf(),
-            })?;
-        Ok(io::BufWriter::new(Box::new(FooterProxy::new(
-            self.directory
-                .open_write(path)?
-                .into_inner()
-                .map_err(|_| ())
-                .expect("buffer should be empty"),
-        ))))
+            .map_err(|e| IOError::with_path(path.to_owned(), e))?;
+        self.directory.open_write(path)
     }
 
-    fn atomic_write(&mut self, path: &Path, content: &[u8]) -> io::Result<()> {
-        let content_str = std::str::from_utf8(content).unwrap_or("<content-not-utf-8>");
-        slog::debug!(self.logger(), "Atomic write"; "path" => format!("{:?}", path), "content_length"=>content_str);
+    fn atomic_write(&mut self, path: &Path, data: &[u8]) -> io::Result<()> {
         self.register_file_as_managed(path)?;
-        self.directory.atomic_write(path, content)
+        self.directory.atomic_write(path, data)
     }
 
     fn atomic_read(&self, path: &Path) -> result::Result<Vec<u8>, OpenReadError> {
@@ -324,18 +227,6 @@ impl Directory for ManagedDirectory {
     fn exists(&self, path: &Path) -> bool {
         self.directory.exists(path)
     }
-
-    fn acquire_lock(&self, lock: &Lock) -> result::Result<DirectoryLock, LockError> {
-        self.directory.acquire_lock(lock)
-    }
-
-    fn watch(&self, watch_callback: WatchCallback) -> crate::Result<WatchHandle> {
-        self.directory.watch(watch_callback)
-    }
-
-    fn logger(&self) -> &slog::Logger {
-        self.directory.logger()
-    }
 }
 
 impl Clone for ManagedDirectory {
@@ -347,118 +238,98 @@ impl Clone for ManagedDirectory {
     }
 }
 
-#[cfg(feature = "mmap")]
 #[cfg(test)]
-mod tests_mmap_specific {
+mod tests {
 
-    use crate::directory::{Directory, ManagedDirectory, MmapDirectory, TerminatingWrite};
-    use std::collections::HashSet;
-    use std::fs::OpenOptions;
+    use super::*;
+    #[cfg(feature = "mmap")]
+    use directory::MmapDirectory;
     use std::io::Write;
-    use std::path::{Path, PathBuf};
-    use tempfile::TempDir;
+    use std::path::Path;
+    use tempdir::TempDir;
+
+    lazy_static! {
+        static ref TEST_PATH1: &'static Path = Path::new("some_path_for_test");
+        static ref TEST_PATH2: &'static Path = Path::new("some_path_for_test2");
+    }
 
     #[test]
+    #[cfg(feature = "mmap")]
     fn test_managed_directory() {
-        let tempdir = TempDir::new().unwrap();
+        let tempdir = TempDir::new("index").unwrap();
         let tempdir_path = PathBuf::from(tempdir.path());
-
-        let test_path1: &'static Path = Path::new("some_path_for_test");
-        let test_path2: &'static Path = Path::new("some_path_for_test_2");
         {
             let mmap_directory = MmapDirectory::open(&tempdir_path).unwrap();
             let mut managed_directory = ManagedDirectory::wrap(mmap_directory).unwrap();
-            let write_file = managed_directory.open_write(test_path1).unwrap();
-            write_file.terminate().unwrap();
-            managed_directory
-                .atomic_write(test_path2, &[0u8, 1u8])
-                .unwrap();
-            assert!(managed_directory.exists(test_path1));
-            assert!(managed_directory.exists(test_path2));
-            let living_files: HashSet<PathBuf> = [test_path1.to_owned()].iter().cloned().collect();
-            assert!(managed_directory.garbage_collect(|| living_files).is_ok());
-            assert!(managed_directory.exists(test_path1));
-            assert!(!managed_directory.exists(test_path2));
+            {
+                let mut write_file = managed_directory.open_write(*TEST_PATH1).unwrap();
+                write_file.flush().unwrap();
+            }
+            {
+                managed_directory
+                    .atomic_write(*TEST_PATH2, &vec![0u8, 1u8])
+                    .unwrap();
+            }
+            {
+                assert!(managed_directory.exists(*TEST_PATH1));
+                assert!(managed_directory.exists(*TEST_PATH2));
+            }
+            {
+                let living_files: HashSet<PathBuf> =
+                    [TEST_PATH1.to_owned()].into_iter().cloned().collect();
+                managed_directory.garbage_collect(|| living_files);
+            }
+            {
+                assert!(managed_directory.exists(*TEST_PATH1));
+                assert!(!managed_directory.exists(*TEST_PATH2));
+            }
         }
         {
             let mmap_directory = MmapDirectory::open(&tempdir_path).unwrap();
             let mut managed_directory = ManagedDirectory::wrap(mmap_directory).unwrap();
-            assert!(managed_directory.exists(test_path1));
-            assert!(!managed_directory.exists(test_path2));
-            let living_files: HashSet<PathBuf> = HashSet::new();
-            assert!(managed_directory.garbage_collect(|| living_files).is_ok());
-            assert!(!managed_directory.exists(test_path1));
-            assert!(!managed_directory.exists(test_path2));
+            {
+                assert!(managed_directory.exists(*TEST_PATH1));
+                assert!(!managed_directory.exists(*TEST_PATH2));
+            }
+            {
+                let living_files: HashSet<PathBuf> = HashSet::new();
+                managed_directory.garbage_collect(|| living_files);
+            }
+            {
+                assert!(!managed_directory.exists(*TEST_PATH1));
+                assert!(!managed_directory.exists(*TEST_PATH2));
+            }
         }
     }
 
     #[test]
+    #[cfg(feature = "mmap ")]
     fn test_managed_directory_gc_while_mmapped() {
-        let test_path1: &'static Path = Path::new("some_path_for_test");
-
-        let tempdir = TempDir::new().unwrap();
+        let tempdir = TempDir::new("index").unwrap();
         let tempdir_path = PathBuf::from(tempdir.path());
         let living_files = HashSet::new();
 
         let mmap_directory = MmapDirectory::open(&tempdir_path).unwrap();
         let mut managed_directory = ManagedDirectory::wrap(mmap_directory).unwrap();
-        let mut write = managed_directory.open_write(test_path1).unwrap();
-        write.write_all(&[0u8, 1u8]).unwrap();
-        write.terminate().unwrap();
-        assert!(managed_directory.exists(test_path1));
+        managed_directory
+            .atomic_write(*TEST_PATH1, &vec![0u8, 1u8])
+            .unwrap();
+        assert!(managed_directory.exists(*TEST_PATH1));
 
-        let _mmap_read = managed_directory.open_read(test_path1).unwrap();
-        assert!(managed_directory
-            .garbage_collect(|| living_files.clone())
-            .is_ok());
+        let _mmap_read = managed_directory.open_read(*TEST_PATH1).unwrap();
+        managed_directory.garbage_collect(|| living_files.clone());
         if cfg!(target_os = "windows") {
             // On Windows, gc should try and fail the file as it is mmapped.
-            assert!(managed_directory.exists(test_path1));
+            assert!(managed_directory.exists(*TEST_PATH1));
             // unmap should happen here.
             drop(_mmap_read);
             // The file should still be in the list of managed file and
             // eventually be deleted once mmap is released.
-            assert!(managed_directory.garbage_collect(|| living_files).is_ok());
-            assert!(!managed_directory.exists(test_path1));
+            managed_directory.garbage_collect(|| living_files);
+            assert!(!managed_directory.exists(*TEST_PATH1));
         } else {
-            assert!(!managed_directory.exists(test_path1));
+            assert!(!managed_directory.exists(*TEST_PATH1));
         }
     }
 
-    #[test]
-    fn test_checksum() {
-        let test_path1: &'static Path = Path::new("some_path_for_test");
-        let test_path2: &'static Path = Path::new("other_test_path");
-
-        let tempdir = TempDir::new().unwrap();
-        let tempdir_path = PathBuf::from(tempdir.path());
-
-        let mmap_directory = MmapDirectory::open(&tempdir_path).unwrap();
-        let mut managed_directory = ManagedDirectory::wrap(mmap_directory).unwrap();
-        let mut write = managed_directory.open_write(test_path1).unwrap();
-        write.write_all(&[0u8, 1u8]).unwrap();
-        write.terminate().unwrap();
-
-        let mut write = managed_directory.open_write(test_path2).unwrap();
-        write.write_all(&[3u8, 4u8, 5u8]).unwrap();
-        write.terminate().unwrap();
-
-        let read_source = managed_directory.open_read(test_path2).unwrap();
-        assert_eq!(read_source.as_slice(), &[3u8, 4u8, 5u8]);
-        assert!(managed_directory.list_damaged().unwrap().is_empty());
-
-        let mut corrupted_path = tempdir_path.clone();
-        corrupted_path.push(test_path2);
-        let mut file = OpenOptions::new()
-            .write(true)
-            .open(&corrupted_path)
-            .unwrap();
-        file.write_all(&[255u8]).unwrap();
-        file.flush().unwrap();
-        drop(file);
-
-        let damaged = managed_directory.list_damaged().unwrap();
-        assert_eq!(damaged.len(), 1);
-        assert!(damaged.contains(test_path2));
-    }
 }

src/directory/mmap_directory.rs

@@ -1,24 +1,12 @@
-use crate::core::META_FILEPATH;
-use crate::directory::error::LockError;
-use crate::directory::error::{DeleteError, OpenDirectoryError, OpenReadError, OpenWriteError};
-use crate::directory::read_only_source::BoxedData;
-use crate::directory::AntiCallToken;
-use crate::directory::Directory;
-use crate::directory::DirectoryLock;
-use crate::directory::Lock;
-use crate::directory::ReadOnlySource;
-use crate::directory::WatchCallback;
-use crate::directory::WatchCallbackList;
-use crate::directory::WatchHandle;
-use crate::directory::{TerminatingWrite, WritePtr};
-use fs2::FileExt;
-use memmap::Mmap;
-use notify::RawEvent;
-use notify::RecursiveMode;
-use notify::Watcher;
-use serde::{Deserialize, Serialize};
-use slog::{debug, o, Drain, Logger};
-use slog_stdlog::StdLog;
+use atomicwrites;
+use common::make_io_err;
+use directory::error::{DeleteError, IOError, OpenDirectoryError, OpenReadError, OpenWriteError};
+use directory::shared_vec_slice::SharedVecSlice;
+use directory::Directory;
+use directory::ReadOnlySource;
+use directory::WritePtr;
+use fst::raw::MmapReadOnly;
+use std::collections::hash_map::Entry as HashMapEntry;
 use std::collections::HashMap;
 use std::convert::From;
 use std::fmt;
@@ -28,32 +16,25 @@ use std::io::{self, Seek, SeekFrom};
 use std::io::{BufWriter, Read, Write};
 use std::path::{Path, PathBuf};
 use std::result;
-use std::sync::mpsc::{channel, Receiver, Sender};
 use std::sync::Arc;
-use std::sync::Mutex;
 use std::sync::RwLock;
-use std::sync::Weak;
-use std::thread;
-use tempfile::TempDir;
+use tempdir::TempDir;
 
 /// Returns None iff the file exists, can be read, but is empty (and hence
-/// cannot be mmapped)
-fn open_mmap(full_path: &Path) -> result::Result<Option<Mmap>, OpenReadError> {
+/// cannot be mmapped).
+///
+fn open_mmap(full_path: &Path) -> result::Result<Option<MmapReadOnly>, OpenReadError> {
     let file = File::open(full_path).map_err(|e| {
         if e.kind() == io::ErrorKind::NotFound {
             OpenReadError::FileDoesNotExist(full_path.to_owned())
         } else {
-            OpenReadError::IOError {
-                io_error: e,
-                filepath: full_path.to_owned(),
-            }
+            OpenReadError::IOError(IOError::with_path(full_path.to_owned(), e))
         }
     })?;
 
-    let meta_data = file.metadata().map_err(|e| OpenReadError::IOError {
-        io_error: e,
-        filepath: full_path.to_owned(),
-    })?;
+    let meta_data = file
+        .metadata()
+        .map_err(|e| IOError::with_path(full_path.to_owned(), e))?;
     if meta_data.len() == 0 {
         // if the file size is 0, it will not be possible
         // to mmap the file, so we return None
@@ -61,12 +42,9 @@ fn open_mmap(full_path: &Path) -> result::Result<Option<Mmap>, OpenReadError> {
         return Ok(None);
     }
     unsafe {
-        memmap::Mmap::map(&file)
+        MmapReadOnly::open(&file)
             .map(Some)
-            .map_err(|e| OpenReadError::IOError {
-                io_error: e,
-                filepath: full_path.to_owned(),
-            })
+            .map_err(|e| From::from(IOError::with_path(full_path.to_owned(), e)))
     }
 }
 
@@ -87,7 +65,7 @@ pub struct CacheInfo {
 
 struct MmapCache {
     counters: CacheCounters,
-    cache: HashMap<PathBuf, Weak<BoxedData>>,
+    cache: HashMap<PathBuf, MmapReadOnly>,
 }
 
 impl Default for MmapCache {
@@ -100,7 +78,12 @@ impl Default for MmapCache {
 }
 
 impl MmapCache {
-    fn get_info(&self) -> CacheInfo {
+    /// Removes a `MmapReadOnly` entry from the mmap cache.
+    fn discard_from_cache(&mut self, full_path: &Path) -> bool {
+        self.cache.remove(full_path).is_some()
+    }
+
+    fn get_info(&mut self) -> CacheInfo {
         let paths: Vec<PathBuf> = self.cache.keys().cloned().collect();
         CacheInfo {
             counters: self.counters.clone(),
@@ -108,196 +91,64 @@ impl MmapCache {
         }
     }
 
-    fn remove_weak_ref(&mut self) {
-        let keys_to_remove: Vec<PathBuf> = self
-            .cache
-            .iter()
-            .filter(|(_, mmap_weakref)| mmap_weakref.upgrade().is_none())
-            .map(|(key, _)| key.clone())
-            .collect();
-        for key in keys_to_remove {
-            self.cache.remove(&key);
-        }
-    }
-
-    // Returns None if the file exists but as a len of 0 (and hence is not mmappable).
-    fn get_mmap(&mut self, full_path: &Path) -> Result<Option<Arc<BoxedData>>, OpenReadError> {
-        if let Some(mmap_weak) = self.cache.get(full_path) {
-            if let Some(mmap_arc) = mmap_weak.upgrade() {
+    fn get_mmap(&mut self, full_path: &Path) -> Result<Option<MmapReadOnly>, OpenReadError> {
+        Ok(match self.cache.entry(full_path.to_owned()) {
+            HashMapEntry::Occupied(occupied_entry) => {
+                let mmap = occupied_entry.get();
                 self.counters.hit += 1;
-                return Ok(Some(mmap_arc));
+                Some(mmap.clone())
             }
-        }
-        self.cache.remove(full_path);
-        self.counters.miss += 1;
-        let mmap_opt = open_mmap(full_path)?;
-        Ok(mmap_opt.map(|mmap| {
-            let mmap_arc: Arc<BoxedData> = Arc::new(Box::new(mmap));
-            let mmap_weak = Arc::downgrade(&mmap_arc);
-            self.cache.insert(full_path.to_owned(), mmap_weak);
-            mmap_arc
-        }))
-    }
-}
-
-struct WatcherWrapper {
-    _watcher: Mutex<notify::RecommendedWatcher>,
-    watcher_router: Arc<WatchCallbackList>,
-}
-
-impl WatcherWrapper {
-    pub(crate) fn new(path: &Path, logger: Logger) -> Result<Self, OpenDirectoryError> {
-        let (tx, watcher_recv): (Sender<RawEvent>, Receiver<RawEvent>) = channel();
-        // We need to initialize the
-        let watcher = notify::raw_watcher(tx)
-            .and_then(|mut watcher| {
-                watcher.watch(path, RecursiveMode::Recursive)?;
-                Ok(watcher)
-            })
-            .map_err(|err| match err {
-                notify::Error::PathNotFound => OpenDirectoryError::DoesNotExist(path.to_owned()),
-                _ => {
-                    panic!("Unknown error while starting watching directory {:?}", path);
+            HashMapEntry::Vacant(vacant_entry) => {
+                self.counters.miss += 1;
+                if let Some(mmap) = open_mmap(full_path)? {
+                    vacant_entry.insert(mmap.clone());
+                    Some(mmap)
+                } else {
+                    None
                 }
-            })?;
-        let watcher_router: Arc<WatchCallbackList> =
-            Arc::new(WatchCallbackList::with_logger(logger));
-        let watcher_router_clone = watcher_router.clone();
-        thread::Builder::new()
-            .name("meta-file-watch-thread".to_string())
-            .spawn(move || {
-                loop {
-                    match watcher_recv.recv().map(|evt| evt.path) {
-                        Ok(Some(changed_path)) => {
-                            // ... Actually subject to false positive.
-                            // We might want to be more accurate than this at one point.
-                            if let Some(filename) = changed_path.file_name() {
-                                if filename == *META_FILEPATH {
-                                    let _ = watcher_router_clone.broadcast();
-                                }
-                            }
-                        }
-                        Ok(None) => {
-                            // not an event we are interested in.
-                        }
-                        Err(_e) => {
-                            // the watch send channel was dropped
-                            break;
-                        }
-                    }
-                }
-            })
-            .map_err(|io_error| OpenDirectoryError::IoError {
-                io_error,
-                directory_path: path.to_path_buf(),
-            })?;
-        Ok(WatcherWrapper {
-            _watcher: Mutex::new(watcher),
-            watcher_router,
+            }
         })
     }
-
-    pub fn watch(&mut self, watch_callback: WatchCallback) -> WatchHandle {
-        self.watcher_router.subscribe(watch_callback)
-    }
 }
 
 /// Directory storing data in files, read via mmap.
 ///
 /// The Mmap object are cached to limit the
 /// system calls.
-///
-/// In the `MmapDirectory`, locks are implemented using the `fs2` crate definition of locks.
-///
-/// On MacOS & linux, it relies on `flock` (aka `BSD Lock`). These locks solve most of the
-/// problems related to POSIX Locks, but may their contract may not be respected on `NFS`
-/// depending on the implementation.
-///
-/// On Windows the semantics are again different.
 #[derive(Clone)]
 pub struct MmapDirectory {
-    inner: Arc<MmapDirectoryInner>,
-}
-
-struct MmapDirectoryInner {
     root_path: PathBuf,
-    mmap_cache: RwLock<MmapCache>,
-    _temp_directory: Option<TempDir>,
-    watcher: RwLock<Option<WatcherWrapper>>,
-    logger: Logger,
-}
-
-impl MmapDirectoryInner {
-    fn new(
-        root_path: PathBuf,
-        temp_directory: Option<TempDir>,
-        logger: Logger,
-    ) -> MmapDirectoryInner {
-        MmapDirectoryInner {
-            root_path,
-            mmap_cache: Default::default(),
-            _temp_directory: temp_directory,
-            watcher: RwLock::new(None),
-            logger,
-        }
-    }
-
-    fn watch(&self, watch_callback: WatchCallback) -> crate::Result<WatchHandle> {
-        // a lot of juggling here, to ensure we don't do anything that panics
-        // while the rwlock is held. That way we ensure that the rwlock cannot
-        // be poisoned.
-        //
-        // The downside is that we might create a watch wrapper that is not useful.
-        let need_initialization = self.watcher.read().unwrap().is_none();
-        if need_initialization {
-            let watch_wrapper = WatcherWrapper::new(&self.root_path, self.logger.clone())?;
-            let mut watch_wlock = self.watcher.write().unwrap();
-            // the watcher could have been initialized when we released the lock, and
-            // we do not want to lose the watched files that were set.
-            if watch_wlock.is_none() {
-                *watch_wlock = Some(watch_wrapper);
-            }
-        }
-        if let Some(watch_wrapper) = self.watcher.write().unwrap().as_mut() {
-            Ok(watch_wrapper.watch(watch_callback))
-        } else {
-            unreachable!("At this point, watch wrapper is supposed to be initialized");
-        }
-    }
+    mmap_cache: Arc<RwLock<MmapCache>>,
+    _temp_directory: Arc<Option<TempDir>>,
 }
 
 impl fmt::Debug for MmapDirectory {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(f, "MmapDirectory({:?})", self.inner.root_path)
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "MmapDirectory({:?})", self.root_path)
     }
 }
 
 impl MmapDirectory {
-    fn new(root_path: PathBuf, temp_directory: Option<TempDir>, logger: Logger) -> MmapDirectory {
-        let inner = MmapDirectoryInner::new(root_path, temp_directory, logger);
-        MmapDirectory {
-            inner: Arc::new(inner),
-        }
-    }
-
     /// Creates a new MmapDirectory in a temporary directory.
     ///
     /// This is mostly useful to test the MmapDirectory itself.
     /// For your unit tests, prefer the RAMDirectory.
-    pub fn create_from_tempdir() -> Result<MmapDirectory, OpenDirectoryError> {
-        let tempdir = TempDir::new().map_err(OpenDirectoryError::FailedToCreateTempDir)?;
-        let logger = Logger::root(StdLog.fuse(), o!());
-        Ok(MmapDirectory::new(tempdir.path().to_owned(), Some(tempdir), logger))
+    pub fn create_from_tempdir() -> io::Result<MmapDirectory> {
+        let tempdir = TempDir::new("index")?;
+        let tempdir_path = PathBuf::from(tempdir.path());
+        let directory = MmapDirectory {
+            root_path: tempdir_path,
+            mmap_cache: Arc::new(RwLock::new(MmapCache::default())),
+            _temp_directory: Arc::new(Some(tempdir)),
+        };
+        Ok(directory)
     }
 
     /// Opens a MmapDirectory in a directory.
     ///
     /// Returns an error if the `directory_path` does not
     /// exist or if it is not a directory.
-    pub fn open_with_logger<P: AsRef<Path>>(
-        directory_path: P,
-        logger: Logger,
-    ) -> Result<MmapDirectory, OpenDirectoryError> {
+    pub fn open<P: AsRef<Path>>(directory_path: P) -> Result<MmapDirectory, OpenDirectoryError> {
         let directory_path: &Path = directory_path.as_ref();
         if !directory_path.exists() {
             Err(OpenDirectoryError::DoesNotExist(PathBuf::from(
@@ -308,24 +159,18 @@ impl MmapDirectory {
                 directory_path,
             )))
         } else {
-            Ok(MmapDirectory::new(
-                PathBuf::from(directory_path),
-                None,
-                logger,
-            ))
+            Ok(MmapDirectory {
+                root_path: PathBuf::from(directory_path),
+                mmap_cache: Arc::new(RwLock::new(MmapCache::default())),
+                _temp_directory: Arc::new(None),
+            })
         }
     }
 
-    /// Creates an `MmapDirectory` at the given path.
-    pub fn open<P: AsRef<Path>>(directory_path: P) -> Result<MmapDirectory, OpenDirectoryError> {
-        let logger = Logger::root(StdLog.fuse(), o!());
-        Self::open_with_logger(directory_path, logger)
-    }
-
     /// Joins a relative_path to the directory `root_path`
     /// to create a proper complete `filepath`.
     fn resolve_path(&self, relative_path: &Path) -> PathBuf {
-        self.inner.root_path.join(relative_path)
+        self.root_path.join(relative_path)
     }
 
     /// Sync the root directory.
@@ -343,14 +188,14 @@ impl MmapDirectory {
         #[cfg(windows)]
         {
             use std::os::windows::fs::OpenOptionsExt;
-            use winapi::um::winbase;
+            use winapi::winbase;
 
             open_opts
                 .write(true)
                 .custom_flags(winbase::FILE_FLAG_BACKUP_SEMANTICS);
         }
 
-        let fd = open_opts.open(&self.inner.root_path)?;
+        let fd = open_opts.open(&self.root_path)?;
         fd.sync_all()?;
         Ok(())
     }
@@ -360,36 +205,14 @@ impl MmapDirectory {
     ///
     /// The `MmapDirectory` embeds a `MmapDirectory`
     /// to avoid multiplying the `mmap` system calls.
-    pub fn get_cache_info(&self) -> CacheInfo {
-        self.inner
-            .mmap_cache
+    pub fn get_cache_info(&mut self) -> CacheInfo {
+        self.mmap_cache
             .write()
-            .expect("mmap cache lock is poisoned")
-            .remove_weak_ref();
-        self.inner
-            .mmap_cache
-            .read()
             .expect("Mmap cache lock is poisoned.")
             .get_info()
     }
 }
 
-/// We rely on fs2 for file locking. On Windows & MacOS this
-/// uses BSD locks (`flock`). The lock is actually released when
-/// the `File` object is dropped and its associated file descriptor
-/// is closed.
-struct ReleaseLockFile {
-    _file: File,
-    path: PathBuf,
-    logger: Logger,
-}
-
-impl Drop for ReleaseLockFile {
-    fn drop(&mut self) {
-        debug!(self.logger, "Releasing lock {:?}", self.path);
-    }
-}
-
 /// This Write wraps a File, but has the specificity of
 /// call `sync_all` on flush.
 struct SafeFileWriter(File);
@@ -417,50 +240,84 @@ impl Seek for SafeFileWriter {
     }
 }
 
-impl TerminatingWrite for SafeFileWriter {
-    fn terminate_ref(&mut self, _: AntiCallToken) -> io::Result<()> {
-        self.flush()
-    }
-}
-
 impl Directory for MmapDirectory {
     fn open_read(&self, path: &Path) -> result::Result<ReadOnlySource, OpenReadError> {
+        debug!("Open Read {:?}", path);
         let full_path = self.resolve_path(path);
-        let mut mmap_cache = self.inner.mmap_cache.write().map_err(|_| {
+
+        let mut mmap_cache = self.mmap_cache.write().map_err(|_| {
             let msg = format!(
                 "Failed to acquired write lock \
                  on mmap cache while reading {:?}",
                 path
             );
-            let io_error = io::Error::new(io::ErrorKind::Other, msg);
-            OpenReadError::IOError {
-                io_error,
-                filepath: path.to_owned(),
-            }
+            IOError::with_path(path.to_owned(), make_io_err(msg))
         })?;
+
         Ok(mmap_cache
             .get_mmap(&full_path)?
-            .map(ReadOnlySource::from)
-            .unwrap_or_else(ReadOnlySource::empty))
+            .map(ReadOnlySource::Mmap)
+            .unwrap_or_else(|| ReadOnlySource::Anonymous(SharedVecSlice::empty())))
+    }
+
+    fn open_write(&mut self, path: &Path) -> Result<WritePtr, OpenWriteError> {
+        debug!("Open Write {:?}", path);
+        let full_path = self.resolve_path(path);
+
+        let open_res = OpenOptions::new()
+            .write(true)
+            .create_new(true)
+            .open(full_path);
+
+        let mut file = open_res.map_err(|err| {
+            if err.kind() == io::ErrorKind::AlreadyExists {
+                OpenWriteError::FileAlreadyExists(path.to_owned())
+            } else {
+                IOError::with_path(path.to_owned(), err).into()
+            }
+        })?;
+
+        // making sure the file is created.
+        file.flush()
+            .map_err(|e| IOError::with_path(path.to_owned(), e))?;
+
+        // Apparetntly, on some filesystem syncing the parent
+        // directory is required.
+        self.sync_directory()
+            .map_err(|e| IOError::with_path(path.to_owned(), e))?;
+
+        let writer = SafeFileWriter::new(file);
+        Ok(BufWriter::new(Box::new(writer)))
     }
 
     /// Any entry associated to the path in the mmap will be
     /// removed before the file is deleted.
     fn delete(&self, path: &Path) -> result::Result<(), DeleteError> {
+        debug!("Deleting file {:?}", path);
         let full_path = self.resolve_path(path);
+        let mut mmap_cache = self.mmap_cache.write().map_err(|_| {
+            let msg = format!(
+                "Failed to acquired write lock \
+                 on mmap cache while deleting {:?}",
+                path
+            );
+            IOError::with_path(path.to_owned(), make_io_err(msg))
+        })?;
+        mmap_cache.discard_from_cache(path);
+
+        // Removing the entry in the MMap cache.
+        // The munmap will appear on Drop,
+        // when the last reference is gone.
+        mmap_cache.cache.remove(&full_path);
         match fs::remove_file(&full_path) {
-            Ok(_) => self.sync_directory().map_err(|e| DeleteError::IOError {
-                io_error: e,
-                filepath: path.to_path_buf(),
-            }),
+            Ok(_) => self
+                .sync_directory()
+                .map_err(|e| IOError::with_path(path.to_owned(), e).into()),
             Err(e) => {
                 if e.kind() == io::ErrorKind::NotFound {
                     Err(DeleteError::FileDoesNotExist(path.to_owned()))
                 } else {
-                    Err(DeleteError::IOError {
-                        io_error: e,
-                        filepath: path.to_path_buf(),
-                    })
+                    Err(IOError::with_path(path.to_owned(), e).into())
                 }
             }
         }
@@ -471,104 +328,32 @@ impl Directory for MmapDirectory {
         full_path.exists()
     }
 
-    fn open_write(&mut self, path: &Path) -> Result<WritePtr, OpenWriteError> {
-        let full_path = self.resolve_path(path);
-        let open_res = OpenOptions::new()
-            .write(true)
-            .create_new(true)
-            .open(full_path);
-
-        let mut file = open_res.map_err(|err| {
-            if err.kind() == io::ErrorKind::AlreadyExists {
-                OpenWriteError::FileAlreadyExists(path.to_owned())
-            } else {
-                OpenWriteError::IOError {
-                    io_error: err,
-                    filepath: path.to_owned(),
-                }
-            }
-        })?;
-
-        // making sure the file is created.
-        file.flush().map_err(|io_error| OpenWriteError::IOError {
-            io_error,
-            filepath: path.to_owned(),
-        })?;
-
-        // Apparetntly, on some filesystem syncing the parent
-        // directory is required.
-        self.sync_directory().map_err(|e| OpenWriteError::IOError {
-            io_error: e,
-            filepath: path.to_owned(),
-        })?;
-
-        let writer = SafeFileWriter::new(file);
-        Ok(BufWriter::new(Box::new(writer)))
-    }
-
     fn atomic_read(&self, path: &Path) -> Result<Vec<u8>, OpenReadError> {
         let full_path = self.resolve_path(path);
         let mut buffer = Vec::new();
         match File::open(&full_path) {
             Ok(mut file) => {
                 file.read_to_end(&mut buffer)
-                    .map_err(|io_error| OpenReadError::IOError {
-                        io_error,
-                        filepath: path.to_owned(),
-                    })?;
+                    .map_err(|e| IOError::with_path(path.to_owned(), e))?;
                 Ok(buffer)
             }
-            Err(io_error) => {
-                if io_error.kind() == io::ErrorKind::NotFound {
+            Err(e) => {
+                if e.kind() == io::ErrorKind::NotFound {
                     Err(OpenReadError::FileDoesNotExist(path.to_owned()))
                 } else {
-                    Err(OpenReadError::IOError {
-                        io_error,
-                        filepath: path.to_owned(),
-                    })
+                    Err(IOError::with_path(path.to_owned(), e).into())
                 }
             }
         }
     }
 
-    fn atomic_write(&mut self, path: &Path, content: &[u8]) -> io::Result<()> {
-        let mut tempfile = tempfile::Builder::new().tempfile_in(&self.inner.root_path)?;
-        tempfile.write_all(content)?;
-        tempfile.flush()?;
+    fn atomic_write(&mut self, path: &Path, data: &[u8]) -> io::Result<()> {
+        debug!("Atomic Write {:?}", path);
         let full_path = self.resolve_path(path);
-        tempfile.into_temp_path().persist(full_path)?;
+        let meta_file = atomicwrites::AtomicFile::new(full_path, atomicwrites::AllowOverwrite);
+        meta_file.write(|f| f.write_all(data))?;
         Ok(())
     }
-
-    fn acquire_lock(&self, lock: &Lock) -> Result<DirectoryLock, LockError> {
-        let full_path = self.resolve_path(&lock.filepath);
-        // We make sure that the file exists.
-        let file: File = OpenOptions::new()
-            .write(true)
-            .create(true) //< if the file does not exist yet, create it.
-            .open(&full_path)
-            .map_err(LockError::IOError)?;
-        if lock.is_blocking {
-            file.lock_exclusive().map_err(LockError::IOError)?;
-        } else {
-            file.try_lock_exclusive().map_err(|_| LockError::LockBusy)?
-        }
-        let logger = self.inner.logger.clone();
-        // dropping the file handle will release the lock.
-        Ok(DirectoryLock::from(Box::new(ReleaseLockFile {
-            path: lock.filepath.clone(),
-            _file: file,
-            logger,
-        })))
-    }
-
-    fn watch(&self, watch_callback: WatchCallback) -> crate::Result<WatchHandle> {
-        self.inner.watch(watch_callback)
-    }
-
-    fn logger(&self) -> &Logger {
-        &self.inner.logger
-    }
 }
 
 #[cfg(test)]
@@ -578,15 +363,9 @@ mod tests {
     // The following tests are specific to the MmapDirectory
 
     use super::*;
-    use crate::indexer::LogMergePolicy;
-    use crate::schema::{Schema, SchemaBuilder, TEXT};
-    use crate::Index;
-    use crate::ReloadPolicy;
-    use std::fs;
-    use std::sync::atomic::{AtomicUsize, Ordering};
 
     #[test]
-    fn test_open_non_existent_path() {
+    fn test_open_non_existant_path() {
         assert!(MmapDirectory::open(PathBuf::from("./nowhere")).is_err());
     }
 
@@ -608,7 +387,7 @@ mod tests {
 
     #[test]
     fn test_cache() {
-        let content = b"abc";
+        let content = "abc".as_bytes();
 
         // here we test if the cache releases
         // mmaps correctly.
@@ -624,118 +403,26 @@ mod tests {
                 w.flush().unwrap();
             }
         }
-
-        let mut keep = vec![];
-        for (i, path) in paths.iter().enumerate() {
-            keep.push(mmap_directory.open_read(path).unwrap());
-            assert_eq!(mmap_directory.get_cache_info().mmapped.len(), i + 1);
-        }
-        assert_eq!(mmap_directory.get_cache_info().counters.hit, 0);
-        assert_eq!(mmap_directory.get_cache_info().counters.miss, 10);
-        assert_eq!(mmap_directory.get_cache_info().mmapped.len(), 10);
-        for path in paths.iter() {
-            let _r = mmap_directory.open_read(path).unwrap();
-            assert_eq!(mmap_directory.get_cache_info().mmapped.len(), num_paths);
+        {
+            for (i, path) in paths.iter().enumerate() {
+                let _r = mmap_directory.open_read(path).unwrap();
+                assert_eq!(mmap_directory.get_cache_info().mmapped.len(), i + 1);
+            }
+            for path in paths.iter() {
+                let _r = mmap_directory.open_read(path).unwrap();
+                assert_eq!(mmap_directory.get_cache_info().mmapped.len(), num_paths);
+            }
+            for (i, path) in paths.iter().enumerate() {
+                mmap_directory.delete(path).unwrap();
+                assert_eq!(
+                    mmap_directory.get_cache_info().mmapped.len(),
+                    num_paths - i - 1
+                );
+            }
         }
         assert_eq!(mmap_directory.get_cache_info().counters.hit, 10);
         assert_eq!(mmap_directory.get_cache_info().counters.miss, 10);
-        assert_eq!(mmap_directory.get_cache_info().mmapped.len(), 10);
-
-        for path in paths.iter() {
-            let _r = mmap_directory.open_read(path).unwrap();
-            assert_eq!(mmap_directory.get_cache_info().mmapped.len(), 10);
-        }
-
-        assert_eq!(mmap_directory.get_cache_info().counters.hit, 20);
-        assert_eq!(mmap_directory.get_cache_info().counters.miss, 10);
-        assert_eq!(mmap_directory.get_cache_info().mmapped.len(), 10);
-        drop(keep);
-        for path in paths.iter() {
-            let _r = mmap_directory.open_read(path).unwrap();
-            assert_eq!(mmap_directory.get_cache_info().mmapped.len(), 1);
-        }
-        assert_eq!(mmap_directory.get_cache_info().counters.hit, 20);
-        assert_eq!(mmap_directory.get_cache_info().counters.miss, 20);
-        assert_eq!(mmap_directory.get_cache_info().mmapped.len(), 0);
-
-        for path in &paths {
-            mmap_directory.delete(path).unwrap();
-        }
-        assert_eq!(mmap_directory.get_cache_info().counters.hit, 20);
-        assert_eq!(mmap_directory.get_cache_info().counters.miss, 20);
-        assert_eq!(mmap_directory.get_cache_info().mmapped.len(), 0);
-        for path in paths.iter() {
-            assert!(mmap_directory.open_read(path).is_err());
-        }
-        assert_eq!(mmap_directory.get_cache_info().counters.hit, 20);
-        assert_eq!(mmap_directory.get_cache_info().counters.miss, 30);
         assert_eq!(mmap_directory.get_cache_info().mmapped.len(), 0);
     }
 
-    #[test]
-    fn test_watch_wrapper() {
-        let counter: Arc<AtomicUsize> = Default::default();
-        let counter_clone = counter.clone();
-        let tmp_dir = tempfile::TempDir::new().unwrap();
-        let tmp_dirpath = tmp_dir.path().to_owned();
-        let logger = Logger::root(slog::Discard, o!());
-        let mut watch_wrapper = WatcherWrapper::new(&tmp_dirpath, logger).unwrap();
-        let tmp_file = tmp_dirpath.join(*META_FILEPATH);
-        let _handle = watch_wrapper.watch(Box::new(move || {
-            counter_clone.fetch_add(1, Ordering::SeqCst);
-        }));
-        let (sender, receiver) = crossbeam::channel::unbounded();
-        let _handle2 = watch_wrapper.watch(Box::new(move || {
-            let _ = sender.send(());
-        }));
-        assert_eq!(counter.load(Ordering::SeqCst), 0);
-        fs::write(&tmp_file, b"whateverwilldo").unwrap();
-        assert!(receiver.recv().is_ok());
-        assert!(counter.load(Ordering::SeqCst) >= 1);
-    }
-
-    #[test]
-    fn test_mmap_released() {
-        let mmap_directory = MmapDirectory::create_from_tempdir().unwrap();
-        let mut schema_builder: SchemaBuilder = Schema::builder();
-        let text_field = schema_builder.add_text_field("text", TEXT);
-        let schema = schema_builder.build();
-
-        {
-            let index = Index::create(mmap_directory.clone(), schema).unwrap();
-
-            let mut index_writer = index.writer_for_tests().unwrap();
-            let mut log_merge_policy = LogMergePolicy::default();
-            log_merge_policy.set_min_merge_size(3);
-            index_writer.set_merge_policy(Box::new(log_merge_policy));
-            for _num_commits in 0..10 {
-                for _ in 0..10 {
-                    index_writer.add_document(doc!(text_field=>"abc"));
-                }
-                index_writer.commit().unwrap();
-            }
-
-            let reader = index
-                .reader_builder()
-                .reload_policy(ReloadPolicy::Manual)
-                .try_into()
-                .unwrap();
-
-            for _ in 0..4 {
-                index_writer.add_document(doc!(text_field=>"abc"));
-                index_writer.commit().unwrap();
-                reader.reload().unwrap();
-            }
-            index_writer.wait_merging_threads().unwrap();
-
-            reader.reload().unwrap();
-            let num_segments = reader.searcher().segment_readers().len();
-            assert!(num_segments <= 4);
-            assert_eq!(
-                num_segments * 7,
-                mmap_directory.get_cache_info().mmapped.len()
-            );
-        }
-        assert!(mmap_directory.get_cache_info().mmapped.is_empty());
-    }
 }

src/directory/mod.rs

@@ -8,90 +8,158 @@ WORM directory abstraction.
 mod mmap_directory;
 
 mod directory;
-mod directory_lock;
-mod footer;
 mod managed_directory;
 mod ram_directory;
 mod read_only_source;
-mod watch_event_router;
+mod shared_vec_slice;
 
 /// Errors specific to the directory module.
 pub mod error;
 
-pub use self::directory::DirectoryLock;
+use std::io::{BufWriter, Seek, Write};
+
 pub use self::directory::{Directory, DirectoryClone};
-pub use self::directory_lock::{Lock, INDEX_WRITER_LOCK, META_LOCK};
 pub use self::ram_directory::RAMDirectory;
 pub use self::read_only_source::ReadOnlySource;
-pub(crate) use self::watch_event_router::WatchCallbackList;
-pub use self::watch_event_router::{WatchCallback, WatchHandle};
-use std::io::{self, BufWriter, Write};
-use std::path::PathBuf;
-/// Outcome of the Garbage collection
-pub struct GarbageCollectionResult {
-    /// List of files that were deleted in this cycle
-    pub deleted_files: Vec<PathBuf>,
-    /// List of files that were schedule to be deleted in this cycle,
-    /// but deletion did not work. This typically happens on windows,
-    /// as deleting a memory mapped file is forbidden.
-    ///
-    /// If a searcher is still held, a file cannot be deleted.
-    /// This is not considered a bug, the file will simply be deleted
-    /// in the next GC.
-    pub failed_to_delete_files: Vec<PathBuf>,
-}
 
 #[cfg(feature = "mmap")]
 pub use self::mmap_directory::MmapDirectory;
 
-pub use self::managed_directory::ManagedDirectory;
+pub(crate) use self::managed_directory::ManagedDirectory;
 
-/// Struct used to prevent from calling [`terminate_ref`](trait.TerminatingWrite#method.terminate_ref) directly
-///
-/// The point is that while the type is public, it cannot be built by anyone
-/// outside of this module.
-pub struct AntiCallToken(());
-
-/// Trait used to indicate when no more write need to be done on a writer
-pub trait TerminatingWrite: Write {
-    /// Indicate that the writer will no longer be used. Internally call terminate_ref.
-    fn terminate(mut self) -> io::Result<()>
-    where
-        Self: Sized,
-    {
-        self.terminate_ref(AntiCallToken(()))
-    }
-
-    /// You should implement this function to define custom behavior.
-    /// This function should flush any buffer it may hold.
-    fn terminate_ref(&mut self, _: AntiCallToken) -> io::Result<()>;
-}
-
-impl<W: TerminatingWrite + ?Sized> TerminatingWrite for Box<W> {
-    fn terminate_ref(&mut self, token: AntiCallToken) -> io::Result<()> {
-        self.as_mut().terminate_ref(token)
-    }
-}
-
-impl<W: TerminatingWrite> TerminatingWrite for BufWriter<W> {
-    fn terminate_ref(&mut self, a: AntiCallToken) -> io::Result<()> {
-        self.flush()?;
-        self.get_mut().terminate_ref(a)
-    }
-}
-
-#[cfg(test)]
-impl<'a> TerminatingWrite for &'a mut Vec<u8> {
-    fn terminate_ref(&mut self, _a: AntiCallToken) -> io::Result<()> {
-        self.flush()
-    }
-}
+/// Synonym of Seek + Write
+pub trait SeekableWrite: Seek + Write {}
+impl<T: Seek + Write> SeekableWrite for T {}
 
 /// Write object for Directory.
 ///
 /// `WritePtr` are required to implement both Write
 /// and Seek.
-pub type WritePtr = BufWriter<Box<dyn TerminatingWrite>>;
+pub type WritePtr = BufWriter<Box<SeekableWrite>>;
 
 #[cfg(test)]
-mod tests;
+mod tests {
+
+    use super::*;
+    use std::io::{Seek, SeekFrom, Write};
+    use std::path::Path;
+
+    lazy_static! {
+        static ref TEST_PATH: &'static Path = Path::new("some_path_for_test");
+    }
+
+    #[test]
+    fn test_ram_directory() {
+        let mut ram_directory = RAMDirectory::create();
+        test_directory(&mut ram_directory);
+    }
+
+    #[test]
+    #[cfg(feature = "mmap")]
+    fn test_mmap_directory() {
+        let mut mmap_directory = MmapDirectory::create_from_tempdir().unwrap();
+        test_directory(&mut mmap_directory);
+    }
+
+    #[test]
+    #[should_panic]
+    fn ram_directory_panics_if_flush_forgotten() {
+        let mut ram_directory = RAMDirectory::create();
+        let mut write_file = ram_directory.open_write(*TEST_PATH).unwrap();
+        assert!(write_file.write_all(&[4]).is_ok());
+    }
+
+    fn test_simple(directory: &mut Directory) {
+        {
+            {
+                let mut write_file = directory.open_write(*TEST_PATH).unwrap();
+                assert!(directory.exists(*TEST_PATH));
+                write_file.write_all(&[4]).unwrap();
+                write_file.write_all(&[3]).unwrap();
+                write_file.write_all(&[7, 3, 5]).unwrap();
+                write_file.flush().unwrap();
+            }
+            let read_file = directory.open_read(*TEST_PATH).unwrap();
+            let data: &[u8] = &*read_file;
+            assert_eq!(data, &[4u8, 3u8, 7u8, 3u8, 5u8]);
+        }
+
+        assert!(directory.delete(*TEST_PATH).is_ok());
+        assert!(!directory.exists(*TEST_PATH));
+    }
+
+    fn test_seek(directory: &mut Directory) {
+        {
+            {
+                let mut write_file = directory.open_write(*TEST_PATH).unwrap();
+                write_file.write_all(&[4, 3, 7, 3, 5]).unwrap();
+                write_file.seek(SeekFrom::Start(0)).unwrap();
+                write_file.write_all(&[3, 1]).unwrap();
+                write_file.flush().unwrap();
+            }
+            let read_file = directory.open_read(*TEST_PATH).unwrap();
+            let data: &[u8] = &*read_file;
+            assert_eq!(data, &[3u8, 1u8, 7u8, 3u8, 5u8]);
+        }
+
+        assert!(directory.delete(*TEST_PATH).is_ok());
+    }
+
+    fn test_rewrite_forbidden(directory: &mut Directory) {
+        {
+            directory.open_write(*TEST_PATH).unwrap();
+            assert!(directory.exists(*TEST_PATH));
+        }
+        {
+            assert!(directory.open_write(*TEST_PATH).is_err());
+        }
+        assert!(directory.delete(*TEST_PATH).is_ok());
+    }
+
+    fn test_write_create_the_file(directory: &mut Directory) {
+        {
+            assert!(directory.open_read(*TEST_PATH).is_err());
+            let _w = directory.open_write(*TEST_PATH).unwrap();
+            assert!(directory.exists(*TEST_PATH));
+            assert!(directory.open_read(*TEST_PATH).is_ok());
+            assert!(directory.delete(*TEST_PATH).is_ok());
+        }
+    }
+
+    fn test_directory_delete(directory: &mut Directory) {
+        assert!(directory.open_read(*TEST_PATH).is_err());
+        let mut write_file = directory.open_write(*TEST_PATH).unwrap();
+        write_file.write_all(&[1, 2, 3, 4]).unwrap();
+        write_file.flush().unwrap();
+        {
+            let read_handle = directory.open_read(*TEST_PATH).unwrap();
+            {
+                assert_eq!(&*read_handle, &[1u8, 2u8, 3u8, 4u8]);
+
+                // Mapped files can't be deleted on Windows
+                if !cfg!(windows) {
+                    assert!(directory.delete(*TEST_PATH).is_ok());
+                    assert_eq!(&*read_handle, &[1u8, 2u8, 3u8, 4u8]);
+                }
+
+                assert!(directory.delete(Path::new("SomeOtherPath")).is_err());
+            }
+        }
+
+        if cfg!(windows) {
+            assert!(directory.delete(*TEST_PATH).is_ok());
+        }
+
+        assert!(directory.open_read(*TEST_PATH).is_err());
+        assert!(directory.delete(*TEST_PATH).is_err());
+    }
+
+    fn test_directory(directory: &mut Directory) {
+        test_simple(directory);
+        test_seek(directory);
+        test_rewrite_forbidden(directory);
+        test_write_create_the_file(directory);
+        test_directory_delete(directory);
+    }
+
+}

src/directory/ram_directory.rs

@@ -1,12 +1,8 @@
-use crate::core::META_FILEPATH;
-use crate::directory::error::{DeleteError, OpenReadError, OpenWriteError};
-use crate::directory::AntiCallToken;
-use crate::directory::WatchCallbackList;
-use crate::directory::{Directory, ReadOnlySource, WatchCallback, WatchHandle};
-use crate::directory::{TerminatingWrite, WritePtr};
-use fail::fail_point;
-use slog::{o, Drain, Logger};
-use slog_stdlog::StdLog;
+use super::shared_vec_slice::SharedVecSlice;
+use common::make_io_err;
+use directory::error::{DeleteError, IOError, OpenReadError, OpenWriteError};
+use directory::WritePtr;
+use directory::{Directory, ReadOnlySource};
 use std::collections::HashMap;
 use std::fmt;
 use std::io::{self, BufWriter, Cursor, Seek, SeekFrom, Write};
@@ -26,13 +22,13 @@ use std::sync::{Arc, RwLock};
 ///
 struct VecWriter {
     path: PathBuf,
-    shared_directory: RAMDirectory,
+    shared_directory: InnerDirectory,
     data: Cursor<Vec<u8>>,
     is_flushed: bool,
 }
 
 impl VecWriter {
-    fn new(path_buf: PathBuf, shared_directory: RAMDirectory) -> VecWriter {
+    fn new(path_buf: PathBuf, shared_directory: InnerDirectory) -> VecWriter {
         VecWriter {
             path: path_buf,
             data: Cursor::new(Vec::new()),
@@ -68,81 +64,84 @@ impl Write for VecWriter {
 
     fn flush(&mut self) -> io::Result<()> {
         self.is_flushed = true;
-        let mut fs = self.shared_directory.fs.inner_directory.write().unwrap();
-        fs.write(self.path.clone(), self.data.get_ref());
+        self.shared_directory
+            .write(self.path.clone(), self.data.get_ref())?;
         Ok(())
     }
 }
 
-impl TerminatingWrite for VecWriter {
-    fn terminate_ref(&mut self, _: AntiCallToken) -> io::Result<()> {
-        self.flush()
-    }
-}
-
-struct InnerDirectory {
-    fs: HashMap<PathBuf, ReadOnlySource>,
-    watch_router: WatchCallbackList,
-}
+#[derive(Clone)]
+struct InnerDirectory(Arc<RwLock<HashMap<PathBuf, Arc<Vec<u8>>>>>);
 
 impl InnerDirectory {
-    fn with_logger(logger: Logger) -> Self {
-        InnerDirectory {
-            fs: Default::default(),
-            watch_router: WatchCallbackList::with_logger(logger.clone()),
-        }
+    fn new() -> InnerDirectory {
+        InnerDirectory(Arc::new(RwLock::new(HashMap::new())))
     }
 
-    fn write(&mut self, path: PathBuf, data: &[u8]) -> bool {
-        let data = ReadOnlySource::new(Vec::from(data));
-        self.fs.insert(path, data).is_some()
+    fn write(&self, path: PathBuf, data: &[u8]) -> io::Result<bool> {
+        let mut map = self.0.write().map_err(|_| {
+            make_io_err(format!(
+                "Failed to lock the directory, when trying to write {:?}",
+                path
+            ))
+        })?;
+        let prev_value = map.insert(path, Arc::new(Vec::from(data)));
+        Ok(prev_value.is_some())
     }
 
     fn open_read(&self, path: &Path) -> Result<ReadOnlySource, OpenReadError> {
-        self.fs
-            .get(path)
-            .ok_or_else(|| OpenReadError::FileDoesNotExist(PathBuf::from(path)))
-            .map(Clone::clone)
+        self.0
+            .read()
+            .map_err(|_| {
+                let msg = format!(
+                    "Failed to acquire read lock for the \
+                     directory when trying to read {:?}",
+                    path
+                );
+                let io_err = make_io_err(msg);
+                OpenReadError::IOError(IOError::with_path(path.to_owned(), io_err))
+            })
+            .and_then(|readable_map| {
+                readable_map
+                    .get(path)
+                    .ok_or_else(|| OpenReadError::FileDoesNotExist(PathBuf::from(path)))
+                    .map(Arc::clone)
+                    .map(|data| ReadOnlySource::Anonymous(SharedVecSlice::new(data)))
+            })
     }
 
-    fn delete(&mut self, path: &Path) -> result::Result<(), DeleteError> {
-        match self.fs.remove(path) {
-            Some(_) => Ok(()),
-            None => Err(DeleteError::FileDoesNotExist(PathBuf::from(path))),
-        }
+    fn delete(&self, path: &Path) -> result::Result<(), DeleteError> {
+        self.0
+            .write()
+            .map_err(|_| {
+                let msg = format!(
+                    "Failed to acquire write lock for the \
+                     directory when trying to delete {:?}",
+                    path
+                );
+                let io_err = make_io_err(msg);
+                DeleteError::IOError(IOError::with_path(path.to_owned(), io_err))
+            })
+            .and_then(|mut writable_map| match writable_map.remove(path) {
+                Some(_) => Ok(()),
+                None => Err(DeleteError::FileDoesNotExist(PathBuf::from(path))),
+            })
     }
 
     fn exists(&self, path: &Path) -> bool {
-        self.fs.contains_key(path)
-    }
-
-    fn watch(&mut self, watch_handle: WatchCallback) -> WatchHandle {
-        self.watch_router.subscribe(watch_handle)
-    }
-
-    fn total_mem_usage(&self) -> usize {
-        self.fs.values().map(|f| f.len()).sum()
-    }
-}
-
-impl Default for RAMDirectory {
-    fn default() -> RAMDirectory {
-        let logger = Logger::root(StdLog.fuse(), o!());
-        Self::with_logger(logger)
+        self.0
+            .read()
+            .expect("Failed to get read lock directory.")
+            .contains_key(path)
     }
 }
 
 impl fmt::Debug for RAMDirectory {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         write!(f, "RAMDirectory")
     }
 }
 
-struct Inner {
-    inner_directory: RwLock<InnerDirectory>,
-    logger: Logger,
-}
-
 /// A Directory storing everything in anonymous memory.
 ///
 /// It is mainly meant for unit testing.
@@ -150,73 +149,31 @@ struct Inner {
 ///
 #[derive(Clone)]
 pub struct RAMDirectory {
-    fs: Arc<Inner>,
+    fs: InnerDirectory,
 }
 
 impl RAMDirectory {
     /// Constructor
     pub fn create() -> RAMDirectory {
-        Self::default()
-    }
-
-    /// Create a `RAMDirectory` with a custom logger.
-    pub fn with_logger(logger: Logger) -> RAMDirectory {
-        let inner_directory = InnerDirectory::with_logger(logger.clone()).into();
         RAMDirectory {
-            fs: Arc::new(Inner {
-                inner_directory,
-                logger,
-            }),
+            fs: InnerDirectory::new(),
         }
     }
-
-    /// Returns the sum of the size of the different files
-    /// in the RAMDirectory.
-    pub fn total_mem_usage(&self) -> usize {
-        self.fs.inner_directory.read().unwrap().total_mem_usage()
-    }
-
-    /// Write a copy of all of the files saved in the RAMDirectory in the target `Directory`.
-    ///
-    /// Files are all written using the `Directory::write` meaning, even if they were
-    /// written using the `atomic_write` api.
-    ///
-    /// If an error is encounterred, files may be persisted partially.
-    pub fn persist(&self, dest: &mut dyn Directory) -> crate::Result<()> {
-        let wlock = self.fs.inner_directory.write().unwrap();
-        for (path, source) in wlock.fs.iter() {
-            let mut dest_wrt = dest.open_write(path)?;
-            dest_wrt.write_all(source.as_slice())?;
-            dest_wrt.terminate()?;
-        }
-        Ok(())
-    }
 }
 
 impl Directory for RAMDirectory {
     fn open_read(&self, path: &Path) -> result::Result<ReadOnlySource, OpenReadError> {
-        self.fs.inner_directory.read().unwrap().open_read(path)
-    }
-
-    fn delete(&self, path: &Path) -> result::Result<(), DeleteError> {
-        fail_point!("RAMDirectory::delete", |_| {
-            Err(DeleteError::IOError {
-                io_error: io::Error::from(io::ErrorKind::Other),
-                filepath: path.to_path_buf(),
-            })
-        });
-        self.fs.inner_directory.write().unwrap().delete(path)
-    }
-
-    fn exists(&self, path: &Path) -> bool {
-        self.fs.inner_directory.read().unwrap().exists(path)
+        self.fs.open_read(path)
     }
 
     fn open_write(&mut self, path: &Path) -> Result<WritePtr, OpenWriteError> {
-        let mut fs = self.fs.inner_directory.write().unwrap();
         let path_buf = PathBuf::from(path);
-        let vec_writer = VecWriter::new(path_buf.clone(), self.clone());
-        let exists = fs.write(path_buf.clone(), &[]);
+        let vec_writer = VecWriter::new(path_buf.clone(), self.fs.clone());
+
+        let exists = self
+            .fs
+            .write(path_buf.clone(), &Vec::new())
+            .map_err(|err| IOError::with_path(path.to_owned(), err))?;
         // force the creation of the file to mimic the MMap directory.
         if exists {
             Err(OpenWriteError::FileAlreadyExists(path_buf))
@@ -225,74 +182,29 @@ impl Directory for RAMDirectory {
         }
     }
 
+    fn delete(&self, path: &Path) -> result::Result<(), DeleteError> {
+        self.fs.delete(path)
+    }
+
+    fn exists(&self, path: &Path) -> bool {
+        self.fs.exists(path)
+    }
+
     fn atomic_read(&self, path: &Path) -> Result<Vec<u8>, OpenReadError> {
-        Ok(self.open_read(path)?.as_slice().to_owned())
+        let read = self.open_read(path)?;
+        Ok(read.as_slice().to_owned())
     }
 
     fn atomic_write(&mut self, path: &Path, data: &[u8]) -> io::Result<()> {
         fail_point!("RAMDirectory::atomic_write", |msg| Err(io::Error::new(
             io::ErrorKind::Other,
-            msg.unwrap_or_else(|| "Undefined".to_string())
+            msg.unwrap_or("Undefined".to_string())
         )));
         let path_buf = PathBuf::from(path);
-
-        // Reserve the path to prevent calls to .write() to succeed.
-        self.fs
-            .inner_directory
-            .write()
-            .unwrap()
-            .write(path_buf.clone(), &[]);
-
-        let mut vec_writer = VecWriter::new(path_buf, self.clone());
+        let mut vec_writer = VecWriter::new(path_buf.clone(), self.fs.clone());
+        self.fs.write(path_buf, &Vec::new())?;
         vec_writer.write_all(data)?;
         vec_writer.flush()?;
-        if path == Path::new(&*META_FILEPATH) {
-            let _ = self
-                .fs
-                .inner_directory
-                .write()
-                .unwrap()
-                .watch_router
-                .broadcast();
-        }
         Ok(())
     }
-
-    fn watch(&self, watch_callback: WatchCallback) -> crate::Result<WatchHandle> {
-        Ok(self
-            .fs
-            .inner_directory
-            .write()
-            .unwrap()
-            .watch(watch_callback))
-    }
-
-    fn logger(&self) -> &Logger {
-        &self.fs.logger
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::RAMDirectory;
-    use crate::Directory;
-    use std::io::Write;
-    use std::path::Path;
-
-    #[test]
-    fn test_persist() {
-        let msg_atomic: &'static [u8] = b"atomic is the way";
-        let msg_seq: &'static [u8] = b"sequential is the way";
-        let path_atomic: &'static Path = Path::new("atomic");
-        let path_seq: &'static Path = Path::new("seq");
-        let mut directory = RAMDirectory::create();
-        assert!(directory.atomic_write(path_atomic, msg_atomic).is_ok());
-        let mut wrt = directory.open_write(path_seq).unwrap();
-        assert!(wrt.write_all(msg_seq).is_ok());
-        assert!(wrt.flush().is_ok());
-        let mut directory_copy = RAMDirectory::create();
-        assert!(directory.persist(&mut directory_copy).is_ok());
-        assert_eq!(directory_copy.atomic_read(path_atomic).unwrap(), msg_atomic);
-        assert_eq!(directory_copy.atomic_read(path_seq).unwrap(), msg_seq);
-    }
 }

src/directory/read_only_source.rs

@@ -1,9 +1,9 @@
-use crate::common::HasLen;
+use super::shared_vec_slice::SharedVecSlice;
+use common::HasLen;
+#[cfg(feature = "mmap")]
+use fst::raw::MmapReadOnly;
 use stable_deref_trait::{CloneStableDeref, StableDeref};
 use std::ops::Deref;
-use std::sync::Arc;
-
-pub type BoxedData = Box<dyn Deref<Target = [u8]> + Send + Sync + 'static>;
 
 /// Read object that represents files in tantivy.
 ///
@@ -11,10 +11,12 @@ pub type BoxedData = Box<dyn Deref<Target = [u8]> + Send + Sync + 'static>;
 /// the data in the form of a constant read-only `&[u8]`.
 /// Whatever happens to the directory file, the data
 /// hold by this object should never be altered or destroyed.
-pub struct ReadOnlySource {
-    data: Arc<BoxedData>,
-    start: usize,
-    stop: usize,
+pub enum ReadOnlySource {
+    /// Mmap source of data
+    #[cfg(feature = "mmap")]
+    Mmap(MmapReadOnly),
+    /// Wrapping a `Vec<u8>`
+    Anonymous(SharedVecSlice),
 }
 
 unsafe impl StableDeref for ReadOnlySource {}
@@ -28,38 +30,19 @@ impl Deref for ReadOnlySource {
     }
 }
 
-impl From<Arc<BoxedData>> for ReadOnlySource {
-    fn from(data: Arc<BoxedData>) -> Self {
-        let len = data.len();
-        ReadOnlySource {
-            data,
-            start: 0,
-            stop: len,
-        }
-    }
-}
-
 impl ReadOnlySource {
-    pub(crate) fn new<D>(data: D) -> ReadOnlySource
-    where
-        D: Deref<Target = [u8]> + Send + Sync + 'static,
-    {
-        let len = data.len();
-        ReadOnlySource {
-            data: Arc::new(Box::new(data)),
-            start: 0,
-            stop: len,
-        }
-    }
-
     /// Creates an empty ReadOnlySource
     pub fn empty() -> ReadOnlySource {
-        ReadOnlySource::new(&[][..])
+        ReadOnlySource::Anonymous(SharedVecSlice::empty())
     }
 
     /// Returns the data underlying the ReadOnlySource object.
     pub fn as_slice(&self) -> &[u8] {
-        &self.data[self.start..self.stop]
+        match *self {
+            #[cfg(feature = "mmap")]
+            ReadOnlySource::Mmap(ref mmap_read_only) => mmap_read_only.as_slice(),
+            ReadOnlySource::Anonymous(ref shared_vec) => shared_vec.as_slice(),
+        }
     }
 
     /// Splits into 2 `ReadOnlySource`, at the offset given
@@ -70,12 +53,6 @@ impl ReadOnlySource {
         (left, right)
     }
 
-    /// Splits into 2 `ReadOnlySource`, at the offset `end - right_len`.
-    pub fn split_from_end(self, right_len: usize) -> (ReadOnlySource, ReadOnlySource) {
-        let left_len = self.len() - right_len;
-        self.split(left_len)
-    }
-
     /// Creates a ReadOnlySource that is just a
     /// view over a slice of the data.
     ///
@@ -86,18 +63,22 @@ impl ReadOnlySource {
     /// worth of data in anonymous memory, and only a
     /// 1KB slice is remaining, the whole `500MBs`
     /// are retained in memory.
-    pub fn slice(&self, start: usize, stop: usize) -> ReadOnlySource {
+    pub fn slice(&self, from_offset: usize, to_offset: usize) -> ReadOnlySource {
         assert!(
-            start <= stop,
+            from_offset <= to_offset,
             "Requested negative slice [{}..{}]",
-            start,
-            stop
+            from_offset,
+            to_offset
         );
-        assert!(stop <= self.len());
-        ReadOnlySource {
-            data: self.data.clone(),
-            start: self.start + start,
-            stop: self.start + stop,
+        match *self {
+            #[cfg(feature = "mmap")]
+            ReadOnlySource::Mmap(ref mmap_read_only) => {
+                let sliced_mmap = mmap_read_only.range(from_offset, to_offset - from_offset);
+                ReadOnlySource::Mmap(sliced_mmap)
+            }
+            ReadOnlySource::Anonymous(ref shared_vec) => {
+                ReadOnlySource::Anonymous(shared_vec.slice(from_offset, to_offset))
+            }
         }
     }
 
@@ -106,7 +87,8 @@ impl ReadOnlySource {
     ///
     /// Equivalent to `.slice(from_offset, self.len())`
     pub fn slice_from(&self, from_offset: usize) -> ReadOnlySource {
-        self.slice(from_offset, self.len())
+        let len = self.len();
+        self.slice(from_offset, len)
     }
 
     /// Like `.slice(...)` but enforcing only the `to`
@@ -120,18 +102,19 @@ impl ReadOnlySource {
 
 impl HasLen for ReadOnlySource {
     fn len(&self) -> usize {
-        self.stop - self.start
+        self.as_slice().len()
     }
 }
 
 impl Clone for ReadOnlySource {
     fn clone(&self) -> Self {
-        self.slice_from(0)
+        self.slice(0, self.len())
     }
 }
 
 impl From<Vec<u8>> for ReadOnlySource {
     fn from(data: Vec<u8>) -> ReadOnlySource {
-        ReadOnlySource::new(data)
+        let shared_data = SharedVecSlice::from(data);
+        ReadOnlySource::Anonymous(shared_data)
     }
 }

src/directory/shared_vec_slice.rs

@@ -0,0 +1,41 @@
+use std::sync::Arc;
+
+#[derive(Clone)]
+pub struct SharedVecSlice {
+    pub data: Arc<Vec<u8>>,
+    pub start: usize,
+    pub len: usize,
+}
+
+impl SharedVecSlice {
+    pub fn empty() -> SharedVecSlice {
+        SharedVecSlice::new(Arc::new(Vec::new()))
+    }
+
+    pub fn new(data: Arc<Vec<u8>>) -> SharedVecSlice {
+        let data_len = data.len();
+        SharedVecSlice {
+            data,
+            start: 0,
+            len: data_len,
+        }
+    }
+
+    pub fn as_slice(&self) -> &[u8] {
+        &self.data[self.start..self.start + self.len]
+    }
+
+    pub fn slice(&self, from_offset: usize, to_offset: usize) -> SharedVecSlice {
+        SharedVecSlice {
+            data: Arc::clone(&self.data),
+            start: self.start + from_offset,
+            len: to_offset - from_offset,
+        }
+    }
+}
+
+impl From<Vec<u8>> for SharedVecSlice {
+    fn from(data: Vec<u8>) -> SharedVecSlice {
+        SharedVecSlice::new(Arc::new(data))
+    }
+}

src/directory/tests.rs

@@ -1,293 +0,0 @@
-use super::*;
-use futures::channel::oneshot;
-use futures::executor::block_on;
-use std::io::Write;
-use std::mem;
-use std::path::{Path, PathBuf};
-use std::sync::atomic::Ordering::SeqCst;
-use std::sync::atomic::{AtomicBool, AtomicUsize};
-use std::sync::Arc;
-use std::time::Duration;
-
-#[cfg(feature = "mmap")]
-mod mmap_directory_tests {
-    use crate::directory::MmapDirectory;
-
-    type DirectoryImpl = MmapDirectory;
-
-    fn make_directory() -> DirectoryImpl {
-        MmapDirectory::create_from_tempdir().unwrap()
-    }
-
-    #[test]
-    fn test_simple() {
-        let mut directory = make_directory();
-        super::test_simple(&mut directory);
-    }
-
-    #[test]
-    fn test_write_create_the_file() {
-        let mut directory = make_directory();
-        super::test_write_create_the_file(&mut directory);
-    }
-
-    #[test]
-    fn test_rewrite_forbidden() {
-        let mut directory = make_directory();
-        super::test_rewrite_forbidden(&mut directory);
-    }
-
-    #[test]
-    fn test_directory_delete() {
-        let mut directory = make_directory();
-        super::test_directory_delete(&mut directory);
-    }
-
-    #[test]
-    fn test_lock_non_blocking() {
-        let mut directory = make_directory();
-        super::test_lock_non_blocking(&mut directory);
-    }
-
-    #[test]
-    fn test_lock_blocking() {
-        let mut directory = make_directory();
-        super::test_lock_blocking(&mut directory);
-    }
-
-    #[test]
-    fn test_watch() {
-        let mut directory = make_directory();
-        super::test_watch(&mut directory);
-    }
-}
-
-mod ram_directory_tests {
-    use crate::directory::RAMDirectory;
-
-    type DirectoryImpl = RAMDirectory;
-
-    fn make_directory() -> DirectoryImpl {
-        RAMDirectory::default()
-    }
-
-    #[test]
-    fn test_simple() {
-        let mut directory = make_directory();
-        super::test_simple(&mut directory);
-    }
-
-    #[test]
-    fn test_write_create_the_file() {
-        let mut directory = make_directory();
-        super::test_write_create_the_file(&mut directory);
-    }
-
-    #[test]
-    fn test_rewrite_forbidden() {
-        let mut directory = make_directory();
-        super::test_rewrite_forbidden(&mut directory);
-    }
-
-    #[test]
-    fn test_directory_delete() {
-        let mut directory = make_directory();
-        super::test_directory_delete(&mut directory);
-    }
-
-    #[test]
-    fn test_lock_non_blocking() {
-        let mut directory = make_directory();
-        super::test_lock_non_blocking(&mut directory);
-    }
-
-    #[test]
-    fn test_lock_blocking() {
-        let mut directory = make_directory();
-        super::test_lock_blocking(&mut directory);
-    }
-
-    #[test]
-    fn test_watch() {
-        let mut directory = make_directory();
-        super::test_watch(&mut directory);
-    }
-}
-
-#[test]
-#[should_panic]
-fn ram_directory_panics_if_flush_forgotten() {
-    let test_path: &'static Path = Path::new("some_path_for_test");
-    let mut ram_directory = RAMDirectory::create();
-    let mut write_file = ram_directory.open_write(test_path).unwrap();
-    assert!(write_file.write_all(&[4]).is_ok());
-}
-
-fn test_simple(directory: &mut dyn Directory) {
-    let test_path: &'static Path = Path::new("some_path_for_test");
-    {
-        let mut write_file = directory.open_write(test_path).unwrap();
-        assert!(directory.exists(test_path));
-        write_file.write_all(&[4]).unwrap();
-        write_file.write_all(&[3]).unwrap();
-        write_file.write_all(&[7, 3, 5]).unwrap();
-        write_file.flush().unwrap();
-    }
-    {
-        let read_file = directory.open_read(test_path).unwrap();
-        let data: &[u8] = &*read_file;
-        assert_eq!(data, &[4u8, 3u8, 7u8, 3u8, 5u8]);
-    }
-    assert!(directory.delete(test_path).is_ok());
-    assert!(!directory.exists(test_path));
-}
-
-fn test_rewrite_forbidden(directory: &mut dyn Directory) {
-    let test_path: &'static Path = Path::new("some_path_for_test");
-    {
-        directory.open_write(test_path).unwrap();
-        assert!(directory.exists(test_path));
-    }
-    {
-        assert!(directory.open_write(test_path).is_err());
-    }
-    assert!(directory.delete(test_path).is_ok());
-}
-
-fn test_write_create_the_file(directory: &mut dyn Directory) {
-    let test_path: &'static Path = Path::new("some_path_for_test");
-    {
-        assert!(directory.open_read(test_path).is_err());
-        let _w = directory.open_write(test_path).unwrap();
-        assert!(directory.exists(test_path));
-        assert!(directory.open_read(test_path).is_ok());
-        assert!(directory.delete(test_path).is_ok());
-    }
-}
-
-fn test_directory_delete(directory: &mut dyn Directory) {
-    let test_path: &'static Path = Path::new("some_path_for_test");
-    assert!(directory.open_read(test_path).is_err());
-    let mut write_file = directory.open_write(&test_path).unwrap();
-    write_file.write_all(&[1, 2, 3, 4]).unwrap();
-    write_file.flush().unwrap();
-    {
-        let read_handle = directory.open_read(&test_path).unwrap();
-        assert_eq!(&*read_handle, &[1u8, 2u8, 3u8, 4u8]);
-        // Mapped files can't be deleted on Windows
-        if !cfg!(windows) {
-            assert!(directory.delete(&test_path).is_ok());
-            assert_eq!(&*read_handle, &[1u8, 2u8, 3u8, 4u8]);
-        }
-
-        assert!(directory.delete(Path::new("SomeOtherPath")).is_err());
-    }
-
-    if cfg!(windows) {
-        assert!(directory.delete(&test_path).is_ok());
-    }
-
-    assert!(directory.open_read(&test_path).is_err());
-    assert!(directory.delete(&test_path).is_err());
-}
-
-fn test_watch(directory: &mut dyn Directory) {
-    let num_progress: Arc<AtomicUsize> = Default::default();
-    let counter: Arc<AtomicUsize> = Default::default();
-    let counter_clone = counter.clone();
-    let (sender, receiver) = crossbeam::channel::unbounded();
-    let watch_callback = Box::new(move || {
-        counter_clone.fetch_add(1, SeqCst);
-    });
-    // This callback is used to synchronize watching in our unit test.
-    // We bind it to a variable because the callback is removed when that
-    // handle is dropped.
-    let watch_handle = directory.watch(watch_callback).unwrap();
-    let _progress_listener = directory
-        .watch(Box::new(move || {
-            let val = num_progress.fetch_add(1, SeqCst);
-            let _ = sender.send(val);
-        }))
-        .unwrap();
-
-    for i in 0..10 {
-        assert!(i <= counter.load(SeqCst));
-        assert!(directory
-            .atomic_write(Path::new("meta.json"), b"random_test_data_2")
-            .is_ok());
-        assert_eq!(receiver.recv_timeout(Duration::from_millis(500)), Ok(i));
-        assert!(i + 1 <= counter.load(SeqCst)); // notify can trigger more than once.
-    }
-    mem::drop(watch_handle);
-    assert!(directory
-        .atomic_write(Path::new("meta.json"), b"random_test_data")
-        .is_ok());
-    assert!(receiver.recv_timeout(Duration::from_millis(500)).is_ok());
-    assert!(10 <= counter.load(SeqCst));
-}
-
-fn test_lock_non_blocking(directory: &mut dyn Directory) {
-    {
-        let lock_a_res = directory.acquire_lock(&Lock {
-            filepath: PathBuf::from("a.lock"),
-            is_blocking: false,
-        });
-        assert!(lock_a_res.is_ok());
-        let lock_b_res = directory.acquire_lock(&Lock {
-            filepath: PathBuf::from("b.lock"),
-            is_blocking: false,
-        });
-        assert!(lock_b_res.is_ok());
-        let lock_a_res2 = directory.acquire_lock(&Lock {
-            filepath: PathBuf::from("a.lock"),
-            is_blocking: false,
-        });
-        assert!(lock_a_res2.is_err());
-    }
-    let lock_a_res = directory.acquire_lock(&Lock {
-        filepath: PathBuf::from("a.lock"),
-        is_blocking: false,
-    });
-    assert!(lock_a_res.is_ok());
-}
-
-fn test_lock_blocking(directory: &mut dyn Directory) {
-    let lock_a_res = directory.acquire_lock(&Lock {
-        filepath: PathBuf::from("a.lock"),
-        is_blocking: true,
-    });
-    assert!(lock_a_res.is_ok());
-    let in_thread = Arc::new(AtomicBool::default());
-    let in_thread_clone = in_thread.clone();
-    let (sender, receiver) = oneshot::channel();
-    std::thread::spawn(move || {
-        //< lock_a_res is sent to the thread.
-        in_thread_clone.store(true, SeqCst);
-        let _just_sync = block_on(receiver);
-        // explicitely droping lock_a_res. It would have been sufficient to just force it
-        // to be part of the move, but the intent seems clearer that way.
-        drop(lock_a_res);
-    });
-    {
-        // A non-blocking call should fail, as the thread is running and holding the lock.
-        let lock_a_res = directory.acquire_lock(&Lock {
-            filepath: PathBuf::from("a.lock"),
-            is_blocking: false,
-        });
-        assert!(lock_a_res.is_err());
-    }
-    let directory_clone = directory.box_clone();
-    let (sender2, receiver2) = oneshot::channel();
-    let join_handle = std::thread::spawn(move || {
-        assert!(sender2.send(()).is_ok());
-        let lock_a_res = directory_clone.acquire_lock(&Lock {
-            filepath: PathBuf::from("a.lock"),
-            is_blocking: true,
-        });
-        assert!(in_thread.load(SeqCst));
-        assert!(lock_a_res.is_ok());
-    });
-    assert!(block_on(receiver2).is_ok());
-    assert!(sender.send(()).is_ok());
-    assert!(join_handle.join().is_ok());
-}

src/directory/watch_event_router.rs

@@ -1,183 +0,0 @@
-use futures::channel::oneshot;
-use futures::{Future, TryFutureExt};
-use slog::{error, Logger};
-use std::sync::Arc;
-use std::sync::RwLock;
-use std::sync::Weak;
-
-/// Type alias for callbacks registered when watching files of a `Directory`.
-pub type WatchCallback = Box<dyn Fn() + Sync + Send>;
-
-/// Helper struct to implement the watch method in `Directory` implementations.
-///
-/// It registers callbacks (See `.subscribe(...)`) and
-/// calls them upon calls to `.broadcast(...)`.
-pub(crate) struct WatchCallbackList {
-    router: RwLock<Vec<Weak<WatchCallback>>>,
-    logger: Logger,
-}
-
-/// Controls how long a directory should watch for a file change.
-///
-/// After all the clones of `WatchHandle` are dropped, the associated will not be called when a
-/// file change is detected.
-#[must_use = "This `WatchHandle` controls the lifetime of the watch and should therefore be used."]
-#[derive(Clone)]
-pub struct WatchHandle(Arc<WatchCallback>);
-
-impl WatchHandle {
-    /// Create a WatchHandle handle.
-    pub fn new(watch_callback: Arc<WatchCallback>) -> WatchHandle {
-        WatchHandle(watch_callback)
-    }
-}
-
-impl WatchCallbackList {
-    pub fn with_logger(logger: Logger) -> Self {
-        WatchCallbackList {
-            logger,
-            router: Default::default(),
-        }
-    }
-
-    /// Subscribes a new callback and returns a handle that controls the lifetime of the callback.
-    pub fn subscribe(&self, watch_callback: WatchCallback) -> WatchHandle {
-        let watch_callback_arc = Arc::new(watch_callback);
-        let watch_callback_weak = Arc::downgrade(&watch_callback_arc);
-        self.router.write().unwrap().push(watch_callback_weak);
-        WatchHandle::new(watch_callback_arc)
-    }
-
-    fn list_callback(&self) -> Vec<Arc<WatchCallback>> {
-        let mut callbacks = vec![];
-        let mut router_wlock = self.router.write().unwrap();
-        let mut i = 0;
-        while i < router_wlock.len() {
-            if let Some(watch) = router_wlock[i].upgrade() {
-                callbacks.push(watch);
-                i += 1;
-            } else {
-                router_wlock.swap_remove(i);
-            }
-        }
-        callbacks
-    }
-
-    /// Triggers all callbacks
-    pub fn broadcast(&self) -> impl Future<Output = ()> {
-        let callbacks = self.list_callback();
-        let (sender, receiver) = oneshot::channel();
-        let result = receiver.unwrap_or_else(|_| ());
-        if callbacks.is_empty() {
-            let _ = sender.send(());
-            return result;
-        }
-        let spawn_res = std::thread::Builder::new()
-            .name("watch-callbacks".to_string())
-            .spawn(move || {
-                for callback in callbacks {
-                    callback();
-                }
-                let _ = sender.send(());
-            });
-        if let Err(err) = spawn_res {
-            error!(
-                self.logger,
-                "Failed to spawn thread to call watch callbacks. Cause: {:?}", err
-            );
-        }
-        result
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use crate::directory::WatchCallbackList;
-    use futures::executor::block_on;
-    use slog::{o, Discard, Logger};
-    use std::mem;
-    use std::sync::atomic::{AtomicUsize, Ordering};
-    use std::sync::Arc;
-
-    fn default_watch_callback_list() -> WatchCallbackList {
-        WatchCallbackList::with_logger(Logger::root(Discard, o!()))
-    }
-
-    #[test]
-    fn test_watch_event_router_simple() {
-        let watch_event_router = default_watch_callback_list();
-        let counter: Arc<AtomicUsize> = Default::default();
-        let counter_clone = counter.clone();
-        let inc_callback = Box::new(move || {
-            counter_clone.fetch_add(1, Ordering::SeqCst);
-        });
-        block_on(watch_event_router.broadcast());
-        assert_eq!(0, counter.load(Ordering::SeqCst));
-        let handle_a = watch_event_router.subscribe(inc_callback);
-        assert_eq!(0, counter.load(Ordering::SeqCst));
-        block_on(watch_event_router.broadcast());
-        assert_eq!(1, counter.load(Ordering::SeqCst));
-        block_on(async {
-            (
-                watch_event_router.broadcast().await,
-                watch_event_router.broadcast().await,
-                watch_event_router.broadcast().await,
-            )
-        });
-        assert_eq!(4, counter.load(Ordering::SeqCst));
-        mem::drop(handle_a);
-        block_on(watch_event_router.broadcast());
-        assert_eq!(4, counter.load(Ordering::SeqCst));
-    }
-
-    #[test]
-    fn test_watch_event_router_multiple_callback_same_key() {
-        let watch_event_router = default_watch_callback_list();
-        let counter: Arc<AtomicUsize> = Default::default();
-        let inc_callback = |inc: usize| {
-            let counter_clone = counter.clone();
-            Box::new(move || {
-                counter_clone.fetch_add(inc, Ordering::SeqCst);
-            })
-        };
-        let handle_a = watch_event_router.subscribe(inc_callback(1));
-        let handle_a2 = watch_event_router.subscribe(inc_callback(10));
-        assert_eq!(0, counter.load(Ordering::SeqCst));
-        block_on(async {
-            futures::join!(
-                watch_event_router.broadcast(),
-                watch_event_router.broadcast()
-            )
-        });
-        assert_eq!(22, counter.load(Ordering::SeqCst));
-        mem::drop(handle_a);
-        block_on(watch_event_router.broadcast());
-        assert_eq!(32, counter.load(Ordering::SeqCst));
-        mem::drop(handle_a2);
-        block_on(watch_event_router.broadcast());
-        block_on(watch_event_router.broadcast());
-        assert_eq!(32, counter.load(Ordering::SeqCst));
-    }
-
-    #[test]
-    fn test_watch_event_router_multiple_callback_different_key() {
-        let watch_event_router = default_watch_callback_list();
-        let counter: Arc<AtomicUsize> = Default::default();
-        let counter_clone = counter.clone();
-        let inc_callback = Box::new(move || {
-            counter_clone.fetch_add(1, Ordering::SeqCst);
-        });
-        let handle_a = watch_event_router.subscribe(inc_callback);
-        assert_eq!(0, counter.load(Ordering::SeqCst));
-        block_on(async {
-            let future1 = watch_event_router.broadcast();
-            let future2 = watch_event_router.broadcast();
-            futures::join!(future1, future2)
-        });
-        assert_eq!(2, counter.load(Ordering::SeqCst));
-        mem::drop(handle_a);
-        let _ = watch_event_router.broadcast();
-        block_on(watch_event_router.broadcast());
-        assert_eq!(2, counter.load(Ordering::SeqCst));
-    }
-}

src/docset.rs

@@ -1,48 +1,57 @@
-use crate::fastfield::DeleteBitSet;
-use crate::DocId;
+use common::BitSet;
 use std::borrow::Borrow;
 use std::borrow::BorrowMut;
+use std::cmp::Ordering;
+use DocId;
 
-/// Sentinel value returned when a DocSet has been entirely consumed.
-///
-/// This is not u32::MAX as one would have expected, due to the lack of SSE2 instructions
-/// to compare [u32; 4].
-pub const TERMINATED: DocId = std::i32::MAX as u32;
+/// Expresses the outcome of a call to `DocSet`'s `.skip_next(...)`.
+#[derive(PartialEq, Eq, Debug)]
+pub enum SkipResult {
+    /// target was in the docset
+    Reached,
+    /// target was not in the docset, skipping stopped as a greater element was found
+    OverStep,
+    /// the docset was entirely consumed without finding the target, nor any
+    /// element greater than the target.
+    End,
+}
 
 /// Represents an iterable set of sorted doc ids.
 pub trait DocSet {
     /// Goes to the next element.
-    ///
-    /// The DocId of the next element is returned.
-    /// In other words we should always have :
-    /// ```ignore
-    /// let doc = docset.advance();
-    /// assert_eq!(doc, docset.doc());
-    /// ```
-    ///
-    /// If we reached the end of the DocSet, TERMINATED should be returned.
-    ///
-    /// Calling `.advance()` on a terminated DocSet should be supported, and TERMINATED should
-    /// be returned.
-    /// TODO Test existing docsets.
-    fn advance(&mut self) -> DocId;
+    /// `.advance(...)` needs to be called a first time to point to the correct
+    /// element.
+    fn advance(&mut self) -> bool;
 
-    /// Advances the DocSet forward until reaching the target, or going to the
-    /// lowest DocId greater than the target.
+    /// After skipping, position the iterator in such a way that `.doc()`
+    /// will return a value greater than or equal to target.
     ///
-    /// If the end of the DocSet is reached, TERMINATED is returned.
+    /// SkipResult expresses whether the `target value` was reached, overstepped,
+    /// or if the `DocSet` was entirely consumed without finding any value
+    /// greater or equal to the `target`.
     ///
-    /// Calling `.seek(target)` on a terminated DocSet is legal. Implementation
-    /// of DocSet should support it.
+    /// WARNING: Calling skip always advances the docset.
+    /// More specifically, if the docset is already positionned on the target
+    /// skipping will advance to the next position and return SkipResult::Overstep.
     ///
-    /// Calling `seek(TERMINATED)` is also legal and is the normal way to consume a DocSet.
-    fn seek(&mut self, target: DocId) -> DocId {
-        let mut doc = self.doc();
-        debug_assert!(doc <= target);
-        while doc < target {
-            doc = self.advance();
+    /// If `.skip_next()` oversteps, then the docset must be positionned correctly
+    /// on an existing document. In other words, `.doc()` should return the first document
+    /// greater than `DocId`.
+    fn skip_next(&mut self, target: DocId) -> SkipResult {
+        if !self.advance() {
+            return SkipResult::End;
+        }
+        loop {
+            match self.doc().cmp(&target) {
+                Ordering::Less => {
+                    if !self.advance() {
+                        return SkipResult::End;
+                    }
+                }
+                Ordering::Equal => return SkipResult::Reached,
+                Ordering::Greater => return SkipResult::OverStep,
+            }
         }
-        doc
     }
 
     /// Fills a given mutable buffer with the next doc ids from the
@@ -61,85 +70,51 @@ pub trait DocSet {
     /// use case where batching. The normal way to
     /// go through the `DocId`'s is to call `.advance()`.
     fn fill_buffer(&mut self, buffer: &mut [DocId]) -> usize {
-        if self.doc() == TERMINATED {
-            return 0;
-        }
         for (i, buffer_val) in buffer.iter_mut().enumerate() {
-            *buffer_val = self.doc();
-            if self.advance() == TERMINATED {
-                return i + 1;
+            if self.advance() {
+                *buffer_val = self.doc();
+            } else {
+                return i;
             }
         }
         buffer.len()
     }
 
     /// Returns the current document
-    /// Right after creating a new DocSet, the docset points to the first document.
-    ///
-    /// If the DocSet is empty, .doc() should return `TERMINATED`.
     fn doc(&self) -> DocId;
 
     /// Returns a best-effort hint of the
     /// length of the docset.
     fn size_hint(&self) -> u32;
 
+    /// Appends all docs to a `bitset`.
+    fn append_to_bitset(&mut self, bitset: &mut BitSet) {
+        while self.advance() {
+            bitset.insert(self.doc());
+        }
+    }
+
     /// Returns the number documents matching.
-    /// Calling this method consumes the `DocSet`.
-    fn count(&mut self, delete_bitset: &DeleteBitSet) -> u32 {
-        let mut count = 0u32;
-        let mut doc = self.doc();
-        while doc != TERMINATED {
-            if !delete_bitset.is_deleted(doc) {
-                count += 1u32;
-            }
-            doc = self.advance();
-        }
-        count
-    }
-
-    /// Returns the count of documents, deleted or not.
-    /// Calling this method consumes the `DocSet`.
     ///
-    /// Of course, the result is an upper bound of the result
-    /// given by `count()`.
-    fn count_including_deleted(&mut self) -> u32 {
+    /// Calling this method consumes the `DocSet`.
+    fn count(&mut self) -> u32 {
         let mut count = 0u32;
-        let mut doc = self.doc();
-        while doc != TERMINATED {
+        while self.advance() {
             count += 1u32;
-            doc = self.advance();
         }
         count
     }
 }
 
-impl<'a> DocSet for &'a mut dyn DocSet {
-    fn advance(&mut self) -> u32 {
-        (**self).advance()
-    }
-
-    fn seek(&mut self, target: DocId) -> DocId {
-        (**self).seek(target)
-    }
-
-    fn doc(&self) -> u32 {
-        (**self).doc()
-    }
-
-    fn size_hint(&self) -> u32 {
-        (**self).size_hint()
-    }
-}
-
 impl<TDocSet: DocSet + ?Sized> DocSet for Box<TDocSet> {
-    fn advance(&mut self) -> DocId {
+    fn advance(&mut self) -> bool {
         let unboxed: &mut TDocSet = self.borrow_mut();
         unboxed.advance()
     }
 
-    fn seek(&mut self, target: DocId) -> DocId {
+    fn skip_next(&mut self, target: DocId) -> SkipResult {
         let unboxed: &mut TDocSet = self.borrow_mut();
-        unboxed.seek(target)
+        unboxed.skip_next(target)
     }
 
     fn doc(&self) -> DocId {
@@ -152,13 +127,13 @@ impl<TDocSet: DocSet + ?Sized> DocSet for Box<TDocSet> {
         unboxed.size_hint()
     }
 
-    fn count(&mut self, delete_bitset: &DeleteBitSet) -> u32 {
+    fn count(&mut self) -> u32 {
         let unboxed: &mut TDocSet = self.borrow_mut();
-        unboxed.count(delete_bitset)
+        unboxed.count()
     }
 
-    fn count_including_deleted(&mut self) -> u32 {
+    fn append_to_bitset(&mut self, bitset: &mut BitSet) {
         let unboxed: &mut TDocSet = self.borrow_mut();
-        unboxed.count_including_deleted()
+        unboxed.append_to_bitset(bitset);
     }
 }

src/error.rs

@@ -2,106 +2,74 @@
 
 use std::io;
 
-use crate::directory::error::{Incompatibility, LockError};
-use crate::fastfield::FastFieldNotAvailableError;
-use crate::query;
-use crate::{
-    directory::error::{OpenDirectoryError, OpenReadError, OpenWriteError},
-    schema,
-};
-use std::fmt;
+use directory::error::{IOError, OpenDirectoryError, OpenReadError, OpenWriteError};
+use fastfield::FastFieldNotAvailableError;
+use indexer::LockType;
+use query;
+use schema;
+use serde_json;
 use std::path::PathBuf;
 use std::sync::PoisonError;
 
-pub struct DataCorruption {
-    filepath: Option<PathBuf>,
-    comment: String,
-}
-
-impl DataCorruption {
-    pub fn new(filepath: PathBuf, comment: String) -> DataCorruption {
-        DataCorruption {
-            filepath: Some(filepath),
-            comment,
-        }
-    }
-
-    pub fn comment_only(comment: String) -> DataCorruption {
-        DataCorruption {
-            filepath: None,
-            comment,
-        }
-    }
-}
-
-impl fmt::Debug for DataCorruption {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> {
-        write!(f, "Data corruption: ")?;
-        if let Some(ref filepath) = &self.filepath {
-            write!(f, "(in file `{:?}`)", filepath)?;
-        }
-        write!(f, ": {}.", self.comment)?;
-        Ok(())
-    }
-}
-
-/// The library's error enum
-#[derive(Debug, Error)]
+/// The library's failure based error enum
+#[derive(Debug, Fail)]
 pub enum TantivyError {
-    /// Failed to open the directory.
-    #[error("Failed to open the directory: '{0:?}'")]
-    OpenDirectoryError(#[from] OpenDirectoryError),
-    /// Failed to open a file for read.
-    #[error("Failed to open file for read: '{0:?}'")]
-    OpenReadError(#[from] OpenReadError),
-    /// Failed to open a file for write.
-    #[error("Failed to open file for write: '{0:?}'")]
-    OpenWriteError(#[from] OpenWriteError),
+    /// Path does not exist.
+    #[fail(display = "Path does not exist: '{:?}'", _0)]
+    PathDoesNotExist(PathBuf),
+    /// File already exists, this is a problem when we try to write into a new file.
+    #[fail(display = "File already exists: '{:?}'", _0)]
+    FileAlreadyExists(PathBuf),
     /// Index already exists in this directory
-    #[error("Index already exists")]
+    #[fail(display = "Index already exists")]
     IndexAlreadyExists,
     /// Failed to acquire file lock
-    #[error("Failed to acquire Lockfile: {0:?}. {1:?}")]
-    LockFailure(LockError, Option<String>),
+    #[fail(
+        display = "Failed to acquire Lockfile: {:?}. Possible causes: another IndexWriter instance or panic during previous lock drop.",
+        _0
+    )]
+    LockFailure(LockType),
     /// IO Error.
-    #[error("An IO error occurred: '{0}'")]
-    IOError(#[from] io::Error),
+    #[fail(display = "An IO error occurred: '{}'", _0)]
+    IOError(#[cause] IOError),
     /// Data corruption.
-    #[error("Data corrupted: '{0:?}'")]
-    DataCorruption(DataCorruption),
+    #[fail(display = "File contains corrupted data: '{:?}'", _0)]
+    CorruptedFile(PathBuf),
     /// A thread holding the locked panicked and poisoned the lock.
-    #[error("A thread holding the locked panicked and poisoned the lock")]
+    #[fail(display = "A thread holding the locked panicked and poisoned the lock")]
     Poisoned,
     /// Invalid argument was passed by the user.
-    #[error("An invalid argument was passed: '{0}'")]
+    #[fail(display = "An invalid argument was passed: '{}'", _0)]
     InvalidArgument(String),
     /// An Error happened in one of the thread.
-    #[error("An error occurred in a thread: '{0}'")]
+    #[fail(display = "An error occurred in a thread: '{}'", _0)]
     ErrorInThread(String),
     /// An Error appeared related to the schema.
-    #[error("Schema error: '{0}'")]
+    #[fail(display = "Schema error: '{}'", _0)]
     SchemaError(String),
+    /// Tried to access a fastfield reader for a field not configured accordingly.
+    #[fail(display = "Fast field not available: '{:?}'", _0)]
+    FastFieldError(#[cause] FastFieldNotAvailableError),
     /// System error. (e.g.: We failed spawning a new thread)
-    #[error("System error.'{0}'")]
+    #[fail(display = "System error.'{}'", _0)]
     SystemError(String),
-    /// Index incompatible with current version of tantivy
-    #[error("{0:?}")]
-    IncompatibleIndex(Incompatibility),
 }
 
-impl From<DataCorruption> for TantivyError {
-    fn from(data_corruption: DataCorruption) -> TantivyError {
-        TantivyError::DataCorruption(data_corruption)
-    }
-}
 impl From<FastFieldNotAvailableError> for TantivyError {
     fn from(fastfield_error: FastFieldNotAvailableError) -> TantivyError {
-        TantivyError::SchemaError(format!("{}", fastfield_error))
+        TantivyError::FastFieldError(fastfield_error)
     }
 }
-impl From<LockError> for TantivyError {
-    fn from(lock_error: LockError) -> TantivyError {
-        TantivyError::LockFailure(lock_error, None)
+
+impl From<IOError> for TantivyError {
+    fn from(io_error: IOError) -> TantivyError {
+        TantivyError::IOError(io_error)
+    }
+}
+
+impl From<io::Error> for TantivyError {
+    fn from(io_error: io::Error) -> TantivyError {
+        TantivyError::IOError(io_error.into())
     }
 }
 
@@ -117,21 +85,48 @@ impl<Guard> From<PoisonError<Guard>> for TantivyError {
     }
 }
 
+impl From<OpenReadError> for TantivyError {
+    fn from(error: OpenReadError) -> TantivyError {
+        match error {
+            OpenReadError::FileDoesNotExist(filepath) => TantivyError::PathDoesNotExist(filepath),
+            OpenReadError::IOError(io_error) => TantivyError::IOError(io_error),
+        }
+    }
+}
+
 impl From<schema::DocParsingError> for TantivyError {
     fn from(error: schema::DocParsingError) -> TantivyError {
         TantivyError::InvalidArgument(format!("Failed to parse document {:?}", error))
     }
 }
 
+impl From<OpenWriteError> for TantivyError {
+    fn from(error: OpenWriteError) -> TantivyError {
+        match error {
+            OpenWriteError::FileAlreadyExists(filepath) => {
+                TantivyError::FileAlreadyExists(filepath)
+            }
+            OpenWriteError::IOError(io_error) => TantivyError::IOError(io_error),
+        }
+    }
+}
+
+impl From<OpenDirectoryError> for TantivyError {
+    fn from(error: OpenDirectoryError) -> TantivyError {
+        match error {
+            OpenDirectoryError::DoesNotExist(directory_path) => {
+                TantivyError::PathDoesNotExist(directory_path)
+            }
+            OpenDirectoryError::NotADirectory(directory_path) => {
+                TantivyError::InvalidArgument(format!("{:?} is not a directory", directory_path))
+            }
+        }
+    }
+}
+
 impl From<serde_json::Error> for TantivyError {
     fn from(error: serde_json::Error) -> TantivyError {
         let io_err = io::Error::from(error);
         TantivyError::IOError(io_err.into())
     }
 }
-
-impl From<rayon::ThreadPoolBuildError> for TantivyError {
-    fn from(error: rayon::ThreadPoolBuildError) -> TantivyError {
-        TantivyError::SystemError(error.to_string())
-    }
-}

src/fastfield/bytes/mod.rs

@@ -6,8 +6,8 @@ pub use self::writer::BytesFastFieldWriter;
 
 #[cfg(test)]
 mod tests {
-    use crate::schema::Schema;
-    use crate::Index;
+    use schema::Schema;
+    use Index;
 
     #[test]
     fn test_bytes() {
@@ -15,22 +15,24 @@ mod tests {
         let field = schema_builder.add_bytes_field("bytesfield");
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_for_tests().unwrap();
+        let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
         index_writer.add_document(doc!(field=>vec![0u8, 1, 2, 3]));
         index_writer.add_document(doc!(field=>vec![]));
         index_writer.add_document(doc!(field=>vec![255u8]));
         index_writer.add_document(doc!(field=>vec![1u8, 3, 5, 7, 9]));
         index_writer.add_document(doc!(field=>vec![0u8; 1000]));
         assert!(index_writer.commit().is_ok());
-        let searcher = index.reader().unwrap().searcher();
-        let segment_reader = searcher.segment_reader(0);
-        let bytes_reader = segment_reader.fast_fields().bytes(field).unwrap();
 
-        assert_eq!(bytes_reader.get_bytes(0), &[0u8, 1, 2, 3]);
-        assert!(bytes_reader.get_bytes(1).is_empty());
-        assert_eq!(bytes_reader.get_bytes(2), &[255u8]);
-        assert_eq!(bytes_reader.get_bytes(3), &[1u8, 3, 5, 7, 9]);
+        index.load_searchers().unwrap();
+        let searcher = index.searcher();
+        let reader = searcher.segment_reader(0);
+        let bytes_reader = reader.bytes_fast_field_reader(field).unwrap();
+
+        assert_eq!(bytes_reader.get_val(0), &[0u8, 1, 2, 3]);
+        assert!(bytes_reader.get_val(1).is_empty());
+        assert_eq!(bytes_reader.get_val(2), &[255u8]);
+        assert_eq!(bytes_reader.get_val(3), &[1u8, 3, 5, 7, 9]);
         let long = vec![0u8; 1000];
-        assert_eq!(bytes_reader.get_bytes(4), long.as_slice());
+        assert_eq!(bytes_reader.get_val(4), long.as_slice());
     }
 }

src/fastfield/bytes/reader.rs

@@ -1,8 +1,8 @@
 use owning_ref::OwningRef;
 
-use crate::directory::ReadOnlySource;
-use crate::fastfield::FastFieldReader;
-use crate::DocId;
+use directory::ReadOnlySource;
+use fastfield::FastFieldReader;
+use DocId;
 
 /// Reader for byte array fast fields
 ///
@@ -14,7 +14,6 @@ use crate::DocId;
 ///
 /// Reading the value for a document is done by reading the start index for it,
 /// and the start index for the next document, and keeping the bytes in between.
-#[derive(Clone)]
 pub struct BytesFastFieldReader {
     idx_reader: FastFieldReader<u64>,
     values: OwningRef<ReadOnlySource, [u8]>,
@@ -29,20 +28,10 @@ impl BytesFastFieldReader {
         BytesFastFieldReader { idx_reader, values }
     }
 
-    fn range(&self, doc: DocId) -> (usize, usize) {
+    /// Returns the bytes associated to the given `doc`
+    pub fn get_val(&self, doc: DocId) -> &[u8] {
         let start = self.idx_reader.get(doc) as usize;
         let stop = self.idx_reader.get(doc + 1) as usize;
-        (start, stop)
-    }
-
-    /// Returns the bytes associated to the given `doc`
-    pub fn get_bytes(&self, doc: DocId) -> &[u8] {
-        let (start, stop) = self.range(doc);
         &self.values[start..stop]
     }
-
-    /// Returns the overall number of bytes in this bytes fast field.
-    pub fn total_num_bytes(&self) -> usize {
-        self.values.len()
-    }
 }

src/fastfield/bytes/writer.rs

@@ -1,8 +1,8 @@
 use std::io;
 
-use crate::fastfield::serializer::FastFieldSerializer;
-use crate::schema::{Document, Field, Value};
-use crate::DocId;
+use fastfield::serializer::FastFieldSerializer;
+use schema::{Document, Field, Value};
+use DocId;
 
 /// Writer for byte array (as in, any number of bytes per document) fast fields
 ///

src/fastfield/delete.rs

@@ -1,21 +1,17 @@
-use crate::common::{BitSet, HasLen};
-use crate::directory::ReadOnlySource;
-use crate::directory::WritePtr;
-use crate::space_usage::ByteCount;
-use crate::DocId;
+use bit_set::BitSet;
+use common::HasLen;
+use directory::ReadOnlySource;
+use directory::WritePtr;
+use space_usage::ByteCount;
 use std::io;
 use std::io::Write;
+use DocId;
 
 /// Write a delete `BitSet`
 ///
 /// where `delete_bitset` is the set of deleted `DocId`.
-/// Warning: this function does not call terminate. The caller is in charge of
-/// closing the writer properly.
-pub fn write_delete_bitset(
-    delete_bitset: &BitSet,
-    max_doc: u32,
-    writer: &mut WritePtr,
-) -> io::Result<()> {
+pub fn write_delete_bitset(delete_bitset: &BitSet, writer: &mut WritePtr) -> io::Result<()> {
+    let max_doc = delete_bitset.capacity();
     let mut byte = 0u8;
     let mut shift = 0u8;
     for doc in 0..max_doc {
@@ -33,7 +29,7 @@ pub fn write_delete_bitset(
     if max_doc % 8 > 0 {
         writer.write_all(&[byte])?;
     }
-    Ok(())
+    writer.flush()
 }
 
 /// Set of deleted `DocId`s.
@@ -44,24 +40,6 @@ pub struct DeleteBitSet {
 }
 
 impl DeleteBitSet {
-    #[cfg(test)]
-    pub(crate) fn for_test(docs: &[DocId], max_doc: u32) -> DeleteBitSet {
-        use crate::directory::{Directory, RAMDirectory, TerminatingWrite};
-        use std::path::Path;
-        assert!(docs.iter().all(|&doc| doc < max_doc));
-        let mut bitset = BitSet::with_max_value(max_doc);
-        for &doc in docs {
-            bitset.insert(doc);
-        }
-        let mut directory = RAMDirectory::create();
-        let path = Path::new("dummydeletebitset");
-        let mut wrt = directory.open_write(path).unwrap();
-        write_delete_bitset(&bitset, max_doc, &mut wrt).unwrap();
-        wrt.terminate().unwrap();
-        let source = directory.open_read(path).unwrap();
-        Self::open(source)
-    }
-
     /// Opens a delete bitset given its data source.
     pub fn open(data: ReadOnlySource) -> DeleteBitSet {
         let num_deleted: usize = data
@@ -75,18 +53,16 @@ impl DeleteBitSet {
         }
     }
 
-    /// Returns true iff the document is still "alive". In other words, if it has not been deleted.
-    pub fn is_alive(&self, doc: DocId) -> bool {
-        !self.is_deleted(doc)
-    }
-
-    /// Returns true iff the document has been marked as deleted.
-    #[inline(always)]
+    /// Returns whether the document has been marked as deleted.
     pub fn is_deleted(&self, doc: DocId) -> bool {
-        let byte_offset = doc / 8u32;
-        let b: u8 = (*self.data)[byte_offset as usize];
-        let shift = (doc & 7u32) as u8;
-        b & (1u8 << shift) != 0
+        if self.len == 0 {
+            false
+        } else {
+            let byte_offset = doc / 8u32;
+            let b: u8 = (*self.data)[byte_offset as usize];
+            let shift = (doc & 7u32) as u8;
+            b & (1u8 << shift) != 0
+        }
     }
 
     /// Summarize total space usage of this bitset.
@@ -103,35 +79,45 @@ impl HasLen for DeleteBitSet {
 
 #[cfg(test)]
 mod tests {
-    use super::DeleteBitSet;
-    use crate::common::HasLen;
+    use super::*;
+    use bit_set::BitSet;
+    use directory::*;
+    use std::path::PathBuf;
 
-    #[test]
-    fn test_delete_bitset_empty() {
-        let delete_bitset = DeleteBitSet::for_test(&[], 10);
-        for doc in 0..10 {
-            assert_eq!(delete_bitset.is_deleted(doc), !delete_bitset.is_alive(doc));
+    fn test_delete_bitset_helper(bitset: &BitSet) {
+        let test_path = PathBuf::from("test");
+        let mut directory = RAMDirectory::create();
+        {
+            let mut writer = directory.open_write(&*test_path).unwrap();
+            write_delete_bitset(bitset, &mut writer).unwrap();
+        }
+        {
+            let source = directory.open_read(&test_path).unwrap();
+            let delete_bitset = DeleteBitSet::open(source);
+            let n = bitset.capacity();
+            for doc in 0..n {
+                assert_eq!(bitset.contains(doc), delete_bitset.is_deleted(doc as DocId));
+            }
+            assert_eq!(delete_bitset.len(), bitset.len());
         }
-        assert_eq!(delete_bitset.len(), 0);
     }
 
     #[test]
     fn test_delete_bitset() {
-        let delete_bitset = DeleteBitSet::for_test(&[1, 9], 10);
-        assert!(delete_bitset.is_alive(0));
-        assert!(delete_bitset.is_deleted(1));
-        assert!(delete_bitset.is_alive(2));
-        assert!(delete_bitset.is_alive(3));
-        assert!(delete_bitset.is_alive(4));
-        assert!(delete_bitset.is_alive(5));
-        assert!(delete_bitset.is_alive(6));
-        assert!(delete_bitset.is_alive(6));
-        assert!(delete_bitset.is_alive(7));
-        assert!(delete_bitset.is_alive(8));
-        assert!(delete_bitset.is_deleted(9));
-        for doc in 0..10 {
-            assert_eq!(delete_bitset.is_deleted(doc), !delete_bitset.is_alive(doc));
+        {
+            let mut bitset = BitSet::with_capacity(10);
+            bitset.insert(1);
+            bitset.insert(9);
+            test_delete_bitset_helper(&bitset);
+        }
+        {
+            let mut bitset = BitSet::with_capacity(8);
+            bitset.insert(1);
+            bitset.insert(2);
+            bitset.insert(3);
+            bitset.insert(5);
+            bitset.insert(7);
+            test_delete_bitset_helper(&bitset);
         }
-        assert_eq!(delete_bitset.len(), 2);
     }
 }

src/fastfield/error.rs

@@ -1,11 +1,11 @@
-use crate::schema::FieldEntry;
+use schema::FieldEntry;
 use std::result;
 
 /// `FastFieldNotAvailableError` is returned when the
 /// user requested for a fast field reader, and the field was not
 /// defined in the schema as a fast field.
-#[derive(Debug, Error)]
-#[error("Fast field not available: '{field_name:?}'")]
+#[derive(Debug, Fail)]
+#[fail(display = "field not available: '{:?}'", field_name)]
 pub struct FastFieldNotAvailableError {
     field_name: String,
 }

src/fastfield/facet_reader.rs

@@ -1,9 +1,8 @@
 use super::MultiValueIntFastFieldReader;
-use crate::schema::Facet;
-use crate::termdict::TermDictionary;
-use crate::termdict::TermOrdinal;
-use crate::DocId;
-use std::str;
+use schema::Facet;
+use termdict::TermDictionary;
+use termdict::TermOrdinal;
+use DocId;
 
 /// The facet reader makes it possible to access the list of
 /// facets associated to a given document in a specific
@@ -21,7 +20,6 @@ use std::str;
 pub struct FacetReader {
     term_ords: MultiValueIntFastFieldReader<u64>,
     term_dict: TermDictionary,
-    buffer: Vec<u8>,
 }
 
 impl FacetReader {
@@ -39,7 +37,6 @@ impl FacetReader {
         FacetReader {
             term_ords,
             term_dict,
-            buffer: vec![],
         }
     }
 
@@ -58,18 +55,11 @@ impl FacetReader {
     }
 
     /// Given a term ordinal returns the term associated to it.
-    pub fn facet_from_ord(
-        &mut self,
-        facet_ord: TermOrdinal,
-        output: &mut Facet,
-    ) -> Result<(), str::Utf8Error> {
+    pub fn facet_from_ord(&self, facet_ord: TermOrdinal, output: &mut Facet) {
         let found_term = self
             .term_dict
-            .ord_to_term(facet_ord as u64, &mut self.buffer);
+            .ord_to_term(facet_ord as u64, output.inner_buffer_mut());
         assert!(found_term, "Term ordinal {} no found.", facet_ord);
-        let facet_str = str::from_utf8(&self.buffer[..])?;
-        output.set_facet_str(facet_str);
-        Ok(())
     }
 
     /// Return the list of facet ordinals associated to a document.

src/fastfield/mod.rs

@@ -30,14 +30,12 @@ pub use self::error::{FastFieldNotAvailableError, Result};
 pub use self::facet_reader::FacetReader;
 pub use self::multivalued::{MultiValueIntFastFieldReader, MultiValueIntFastFieldWriter};
 pub use self::reader::FastFieldReader;
-pub use self::readers::FastFieldReaders;
 pub use self::serializer::FastFieldSerializer;
 pub use self::writer::{FastFieldsWriter, IntFastFieldWriter};
-use crate::chrono::{NaiveDateTime, Utc};
-use crate::common;
-use crate::schema::Cardinality;
-use crate::schema::FieldType;
-use crate::schema::Value;
+use common;
+use schema::Cardinality;
+use schema::FieldType;
+use schema::Value;
 
 mod bytes;
 mod delete;
@@ -45,12 +43,11 @@ mod error;
 mod facet_reader;
 mod multivalued;
 mod reader;
-mod readers;
 mod serializer;
 mod writer;
 
-/// Trait for types that are allowed for fast fields: (u64, i64 and f64).
-pub trait FastValue: Clone + Copy + Send + Sync + PartialOrd {
+/// Trait for types that are allowed for fast fields: (u64 or i64).
+pub trait FastValue: Default + Clone + Copy {
     /// Converts a value from u64
     ///
     /// Internally all fast field values are encoded as u64.
@@ -70,12 +67,6 @@ pub trait FastValue: Clone + Copy + Send + Sync + PartialOrd {
     /// Cast value to `u64`.
     /// The value is just reinterpreted in memory.
     fn as_u64(&self) -> u64;
-
-    /// Build a default value. This default value is never used, so the value does not
-    /// really matter.
-    fn make_zero() -> Self {
-        Self::from_u64(0i64.to_u64())
-    }
 }
 
 impl FastValue for u64 {
@@ -87,6 +78,10 @@ impl FastValue for u64 {
         *self
     }
 
+    fn as_u64(&self) -> u64 {
+        *self
+    }
+
     fn fast_field_cardinality(field_type: &FieldType) -> Option<Cardinality> {
         match *field_type {
             FieldType::U64(ref integer_options) => integer_options.get_fastfield_cardinality(),
@@ -94,10 +89,6 @@ impl FastValue for u64 {
             _ => None,
         }
     }
-
-    fn as_u64(&self) -> u64 {
-        *self
-    }
 }
 
 impl FastValue for i64 {
@@ -121,56 +112,11 @@ impl FastValue for i64 {
     }
 }
 
-impl FastValue for f64 {
-    fn from_u64(val: u64) -> Self {
-        common::u64_to_f64(val)
-    }
-
-    fn to_u64(&self) -> u64 {
-        common::f64_to_u64(*self)
-    }
-
-    fn fast_field_cardinality(field_type: &FieldType) -> Option<Cardinality> {
-        match *field_type {
-            FieldType::F64(ref integer_options) => integer_options.get_fastfield_cardinality(),
-            _ => None,
-        }
-    }
-
-    fn as_u64(&self) -> u64 {
-        self.to_bits()
-    }
-}
-
-impl FastValue for crate::DateTime {
-    fn from_u64(timestamp_u64: u64) -> Self {
-        let timestamp_i64 = i64::from_u64(timestamp_u64);
-        crate::DateTime::from_utc(NaiveDateTime::from_timestamp(timestamp_i64, 0), Utc)
-    }
-
-    fn to_u64(&self) -> u64 {
-        self.timestamp().to_u64()
-    }
-
-    fn fast_field_cardinality(field_type: &FieldType) -> Option<Cardinality> {
-        match *field_type {
-            FieldType::Date(ref integer_options) => integer_options.get_fastfield_cardinality(),
-            _ => None,
-        }
-    }
-
-    fn as_u64(&self) -> u64 {
-        self.timestamp().as_u64()
-    }
-}
-
 fn value_to_u64(value: &Value) -> u64 {
     match *value {
         Value::U64(ref val) => *val,
         Value::I64(ref val) => common::i64_to_u64(*val),
-        Value::F64(ref val) => common::f64_to_u64(*val),
-        Value::Date(ref datetime) => common::i64_to_u64(datetime.timestamp()),
-        _ => panic!("Expected a u64/i64/f64 field, got {:?} ", value),
+        _ => panic!("Expected a u64/i64 field, got {:?} ", value),
     }
 }
 
@@ -178,29 +124,27 @@ fn value_to_u64(value: &Value) -> u64 {
 mod tests {
 
     use super::*;
-    use crate::common::CompositeFile;
-    use crate::directory::{Directory, RAMDirectory, WritePtr};
-    use crate::fastfield::FastFieldReader;
-    use crate::merge_policy::NoMergePolicy;
-    use crate::schema::Field;
-    use crate::schema::Schema;
-    use crate::schema::FAST;
-    use crate::schema::{Document, IntOptions};
-    use crate::{Index, SegmentId, SegmentReader};
-    use once_cell::sync::Lazy;
+    use common::CompositeFile;
+    use directory::{Directory, RAMDirectory, WritePtr};
+    use fastfield::FastFieldReader;
     use rand::prelude::SliceRandom;
     use rand::rngs::StdRng;
     use rand::SeedableRng;
+    use schema::Document;
+    use schema::Field;
+    use schema::Schema;
+    use schema::FAST;
     use std::collections::HashMap;
     use std::path::Path;
 
-    pub static SCHEMA: Lazy<Schema> = Lazy::new(|| {
-        let mut schema_builder = Schema::builder();
-        schema_builder.add_u64_field("field", FAST);
-        schema_builder.build()
-    });
-
-    pub static FIELD: Lazy<Field> = Lazy::new(|| SCHEMA.get_field("field").unwrap());
+    lazy_static! {
+        pub static ref SCHEMA: Schema = {
+            let mut schema_builder = Schema::builder();
+            schema_builder.add_u64_field("field", FAST);
+            schema_builder.build()
+        };
+        pub static ref FIELD: Field = { SCHEMA.get_field("field").unwrap() };
+    }
 
     #[test]
     pub fn test_fastfield() {
@@ -210,12 +154,6 @@ mod tests {
         assert_eq!(test_fastfield.get(2), 300);
     }
 
-    #[test]
-    pub fn test_fastfield_i64_u64() {
-        let datetime = crate::DateTime::from_utc(NaiveDateTime::from_timestamp(0i64, 0), Utc);
-        assert_eq!(i64::from_u64(datetime.to_u64()), 0i64);
-    }
-
     #[test]
     fn test_intfastfield_small() {
         let path = Path::new("test");
@@ -468,92 +406,6 @@ mod tests {
         }
     }
 
-    #[test]
-    fn test_merge_missing_date_fast_field() {
-        let mut schema_builder = Schema::builder();
-        let date_field = schema_builder.add_date_field("date", FAST);
-        let schema = schema_builder.build();
-        let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_for_tests().unwrap();
-        index_writer.set_merge_policy(Box::new(NoMergePolicy));
-        index_writer.add_document(doc!(date_field =>crate::chrono::prelude::Utc::now()));
-        index_writer.commit().unwrap();
-        index_writer.add_document(doc!());
-        index_writer.commit().unwrap();
-        let reader = index.reader().unwrap();
-        let segment_ids: Vec<SegmentId> = reader
-            .searcher()
-            .segment_readers()
-            .iter()
-            .map(SegmentReader::segment_id)
-            .collect();
-        assert_eq!(segment_ids.len(), 2);
-        let merge_future = index_writer.merge(&segment_ids[..]);
-        let merge_res = futures::executor::block_on(merge_future);
-        assert!(merge_res.is_ok());
-        assert!(reader.reload().is_ok());
-        assert_eq!(reader.searcher().segment_readers().len(), 1);
-    }
-
-    #[test]
-    fn test_default_datetime() {
-        assert_eq!(crate::DateTime::make_zero().timestamp(), 0i64);
-    }
-
-    #[test]
-    fn test_datefastfield() {
-        use crate::fastfield::FastValue;
-        let mut schema_builder = Schema::builder();
-        let date_field = schema_builder.add_date_field("date", FAST);
-        let multi_date_field = schema_builder.add_date_field(
-            "multi_date",
-            IntOptions::default().set_fast(Cardinality::MultiValues),
-        );
-        let schema = schema_builder.build();
-        let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_for_tests().unwrap();
-        index_writer.set_merge_policy(Box::new(NoMergePolicy));
-        index_writer.add_document(doc!(
-            date_field => crate::DateTime::from_u64(1i64.to_u64()),
-            multi_date_field => crate::DateTime::from_u64(2i64.to_u64()),
-            multi_date_field => crate::DateTime::from_u64(3i64.to_u64())
-        ));
-        index_writer.add_document(doc!(
-            date_field => crate::DateTime::from_u64(4i64.to_u64())
-        ));
-        index_writer.add_document(doc!(
-            multi_date_field => crate::DateTime::from_u64(5i64.to_u64()),
-            multi_date_field => crate::DateTime::from_u64(6i64.to_u64())
-        ));
-        index_writer.commit().unwrap();
-        let reader = index.reader().unwrap();
-        let searcher = reader.searcher();
-        assert_eq!(searcher.segment_readers().len(), 1);
-        let segment_reader = searcher.segment_reader(0);
-        let fast_fields = segment_reader.fast_fields();
-        let date_fast_field = fast_fields.date(date_field).unwrap();
-        let dates_fast_field = fast_fields.dates(multi_date_field).unwrap();
-        let mut dates = vec![];
-        {
-            assert_eq!(date_fast_field.get(0u32).timestamp(), 1i64);
-            dates_fast_field.get_vals(0u32, &mut dates);
-            assert_eq!(dates.len(), 2);
-            assert_eq!(dates[0].timestamp(), 2i64);
-            assert_eq!(dates[1].timestamp(), 3i64);
-        }
-        {
-            assert_eq!(date_fast_field.get(1u32).timestamp(), 4i64);
-            dates_fast_field.get_vals(1u32, &mut dates);
-            assert!(dates.is_empty());
-        }
-        {
-            assert_eq!(date_fast_field.get(2u32).timestamp(), 0i64);
-            dates_fast_field.get_vals(2u32, &mut dates);
-            assert_eq!(dates.len(), 2);
-            assert_eq!(dates[0].timestamp(), 5i64);
-            assert_eq!(dates[1].timestamp(), 6i64);
-        }
-    }
 }
 
 #[cfg(all(test, feature = "unstable"))]
@@ -561,9 +413,9 @@ mod bench {
     use super::tests::FIELD;
     use super::tests::{generate_permutation, SCHEMA};
     use super::*;
-    use crate::common::CompositeFile;
-    use crate::directory::{Directory, RAMDirectory, WritePtr};
-    use crate::fastfield::FastFieldReader;
+    use common::CompositeFile;
+    use directory::{Directory, RAMDirectory, WritePtr};
+    use fastfield::FastFieldReader;
     use std::collections::HashMap;
     use std::path::Path;
     use test::{self, Bencher};
@@ -661,4 +513,5 @@ mod bench {
             });
         }
     }
+
 }

src/fastfield/multivalued/mod.rs

@@ -7,14 +7,10 @@ pub use self::writer::MultiValueIntFastFieldWriter;
 #[cfg(test)]
 mod tests {
 
-    use crate::collector::TopDocs;
-    use crate::query::QueryParser;
-    use crate::schema::Cardinality;
-    use crate::schema::Facet;
-    use crate::schema::IntOptions;
-    use crate::schema::Schema;
-    use crate::Index;
-    use chrono::Duration;
+    use schema::Cardinality;
+    use schema::IntOptions;
+    use schema::Schema;
+    use Index;
 
     #[test]
     fn test_multivalued_u64() {
@@ -25,17 +21,18 @@ mod tests {
         );
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_for_tests().unwrap();
+        let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
         index_writer.add_document(doc!(field=>1u64, field=>3u64));
         index_writer.add_document(doc!());
         index_writer.add_document(doc!(field=>4u64));
         index_writer.add_document(doc!(field=>5u64, field=>20u64,field=>1u64));
         assert!(index_writer.commit().is_ok());
 
-        let searcher = index.reader().unwrap().searcher();
-        let segment_reader = searcher.segment_reader(0);
+        index.load_searchers().unwrap();
+        let searcher = index.searcher();
+        let reader = searcher.segment_reader(0);
         let mut vals = Vec::new();
-        let multi_value_reader = segment_reader.fast_fields().u64s(field).unwrap();
+        let multi_value_reader = reader.multi_fast_field_reader::<u64>(field).unwrap();
         {
             multi_value_reader.get_vals(2, &mut vals);
             assert_eq!(&vals, &[4u64]);
@@ -50,133 +47,6 @@ mod tests {
         }
     }
 
-    #[test]
-    fn test_multivalued_date() {
-        let mut schema_builder = Schema::builder();
-        let date_field = schema_builder.add_date_field(
-            "multi_date_field",
-            IntOptions::default()
-                .set_fast(Cardinality::MultiValues)
-                .set_indexed()
-                .set_stored(),
-        );
-        let time_i =
-            schema_builder.add_i64_field("time_stamp_i", IntOptions::default().set_stored());
-        let schema = schema_builder.build();
-        let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_for_tests().unwrap();
-        let first_time_stamp = chrono::Utc::now();
-        index_writer.add_document(
-            doc!(date_field=>first_time_stamp, date_field=>first_time_stamp, time_i=>1i64),
-        );
-        index_writer.add_document(doc!(time_i=>0i64));
-        // add one second
-        index_writer
-            .add_document(doc!(date_field=>first_time_stamp + Duration::seconds(1), time_i=>2i64));
-        // add another second
-        let two_secs_ahead = first_time_stamp + Duration::seconds(2);
-        index_writer.add_document(doc!(date_field=>two_secs_ahead, date_field=>two_secs_ahead,date_field=>two_secs_ahead, time_i=>3i64));
-        assert!(index_writer.commit().is_ok());
-
-        let reader = index.reader().unwrap();
-        let searcher = reader.searcher();
-        let reader = searcher.segment_reader(0);
-        assert_eq!(reader.num_docs(), 4);
-
-        {
-            let parser = QueryParser::for_index(&index, vec![date_field]);
-            let query = parser
-                .parse_query(&format!("\"{}\"", first_time_stamp.to_rfc3339()).to_string())
-                .expect("could not parse query");
-            let results = searcher
-                .search(&query, &TopDocs::with_limit(5))
-                .expect("could not query index");
-
-            assert_eq!(results.len(), 1);
-            for (_score, doc_address) in results {
-                let retrieved_doc = searcher.doc(doc_address).expect("cannot fetch doc");
-                assert_eq!(
-                    retrieved_doc
-                        .get_first(date_field)
-                        .expect("cannot find value")
-                        .date_value()
-                        .timestamp(),
-                    first_time_stamp.timestamp()
-                );
-                assert_eq!(
-                    retrieved_doc
-                        .get_first(time_i)
-                        .expect("cannot find value")
-                        .i64_value(),
-                    1i64
-                );
-            }
-        }
-
-        {
-            let parser = QueryParser::for_index(&index, vec![date_field]);
-            let query = parser
-                .parse_query(&format!("\"{}\"", two_secs_ahead.to_rfc3339()).to_string())
-                .expect("could not parse query");
-            let results = searcher
-                .search(&query, &TopDocs::with_limit(5))
-                .expect("could not query index");
-
-            assert_eq!(results.len(), 1);
-
-            for (_score, doc_address) in results {
-                let retrieved_doc = searcher.doc(doc_address).expect("cannot fetch doc");
-                assert_eq!(
-                    retrieved_doc
-                        .get_first(date_field)
-                        .expect("cannot find value")
-                        .date_value()
-                        .timestamp(),
-                    two_secs_ahead.timestamp()
-                );
-                assert_eq!(
-                    retrieved_doc
-                        .get_first(time_i)
-                        .expect("cannot find value")
-                        .i64_value(),
-                    3i64
-                );
-            }
-        }
-
-        // TODO: support Date range queries
-        //        {
-        //            let parser = QueryParser::for_index(&index, vec![date_field]);
-        //            let range_q = format!("\"{}\"..\"{}\"",
-        //                                  (first_time_stamp + Duration::seconds(1)).to_rfc3339(),
-        //                                  (first_time_stamp + Duration::seconds(3)).to_rfc3339()
-        //            );
-        //            let query = parser.parse_query(&range_q)
-        //                .expect("could not parse query");
-        //            let results = searcher.search(&query, &TopDocs::with_limit(5))
-        //                .expect("could not query index");
-        //
-        //
-        //            assert_eq!(results.len(), 2);
-        //            for (i, doc_pair) in results.iter().enumerate() {
-        //                let retrieved_doc = searcher.doc(doc_pair.1).expect("cannot fetch doc");
-        //                let offset_sec = match i {
-        //                    0 => 1,
-        //                    1 => 3,
-        //                    _ => panic!("should not have more than 2 docs")
-        //                };
-        //                let time_i_val = match i {
-        //                    0 => 2,
-        //                    1 => 3,
-        //                    _ => panic!("should not have more than 2 docs")
-        //                };
-        //                assert_eq!(retrieved_doc.get_first(date_field).expect("cannot find value").date_value().timestamp(),
-        //                           (first_time_stamp + Duration::seconds(offset_sec)).timestamp());
-        //                assert_eq!(retrieved_doc.get_first(time_i).expect("cannot find value").i64_value(), time_i_val);
-        //            }
-        //        }
-    }
-
     #[test]
     fn test_multivalued_i64() {
         let mut schema_builder = Schema::builder();
@@ -186,17 +56,18 @@ mod tests {
         );
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_for_tests().unwrap();
+        let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
         index_writer.add_document(doc!(field=> 1i64, field => 3i64));
         index_writer.add_document(doc!());
         index_writer.add_document(doc!(field=> -4i64));
         index_writer.add_document(doc!(field=> -5i64, field => -20i64, field=>1i64));
         assert!(index_writer.commit().is_ok());
 
-        let searcher = index.reader().unwrap().searcher();
-        let segment_reader = searcher.segment_reader(0);
+        index.load_searchers().unwrap();
+        let searcher = index.searcher();
+        let reader = searcher.segment_reader(0);
         let mut vals = Vec::new();
-        let multi_value_reader = segment_reader.fast_fields().i64s(field).unwrap();
+        let multi_value_reader = reader.multi_fast_field_reader::<i64>(field).unwrap();
         {
             multi_value_reader.get_vals(2, &mut vals);
             assert_eq!(&vals, &[-4i64]);
@@ -214,17 +85,4 @@ mod tests {
             assert_eq!(&vals, &[-5i64, -20i64, 1i64]);
         }
     }
-    #[test]
-    #[ignore]
-    fn test_many_facets() {
-        let mut schema_builder = Schema::builder();
-        let field = schema_builder.add_facet_field("facetfield");
-        let schema = schema_builder.build();
-        let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_for_tests().unwrap();
-        for i in 0..100_000 {
-            index_writer.add_document(doc!(field=> Facet::from(format!("/lang/{}", i).as_str())));
-        }
-        assert!(index_writer.commit().is_ok());
-    }
 }

src/fastfield/multivalued/reader.rs

@@ -1,5 +1,5 @@
-use crate::fastfield::{FastFieldReader, FastValue};
-use crate::DocId;
+use fastfield::{FastFieldReader, FastValue};
+use DocId;
 
 /// Reader for a multivalued `u64` fast field.
 ///
@@ -26,13 +26,6 @@ impl<Item: FastValue> MultiValueIntFastFieldReader<Item> {
         }
     }
 
-    pub(crate) fn into_u64s_reader(self) -> MultiValueIntFastFieldReader<u64> {
-        MultiValueIntFastFieldReader {
-            idx_reader: self.idx_reader,
-            vals_reader: self.vals_reader.into_u64_reader(),
-        }
-    }
-
     /// Returns `(start, stop)`, such that the values associated
     /// to the given document are `start..stop`.
     fn range(&self, doc: DocId) -> (u64, u64) {
@@ -45,27 +38,16 @@ impl<Item: FastValue> MultiValueIntFastFieldReader<Item> {
     pub fn get_vals(&self, doc: DocId, vals: &mut Vec<Item>) {
         let (start, stop) = self.range(doc);
         let len = (stop - start) as usize;
-        vals.resize(len, Item::make_zero());
-        self.vals_reader.get_range_u64(start, &mut vals[..]);
-    }
-
-    /// Returns the number of values associated with the document `DocId`.
-    pub fn num_vals(&self, doc: DocId) -> usize {
-        let (start, stop) = self.range(doc);
-        (stop - start) as usize
-    }
-
-    /// Returns the overall number of values in this field  .
-    pub fn total_num_vals(&self) -> u64 {
-        self.idx_reader.max_value()
+        vals.resize(len, Item::default());
+        self.vals_reader.get_range(start as u32, &mut vals[..]);
     }
 }
 
 #[cfg(test)]
 mod tests {
 
-    use crate::core::Index;
-    use crate::schema::{Facet, Schema};
+    use core::Index;
+    use schema::{Document, Facet, Schema};
 
     #[test]
     fn test_multifastfield_reader() {
@@ -74,35 +56,46 @@ mod tests {
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
         let mut index_writer = index
-            .writer_for_tests()
+            .writer_with_num_threads(1, 30_000_000)
             .expect("Failed to create index writer.");
-        index_writer.add_document(doc!(
-            facet_field => Facet::from("/category/cat2"),
-            facet_field => Facet::from("/category/cat1"),
-        ));
-        index_writer.add_document(doc!(facet_field => Facet::from("/category/cat2")));
-        index_writer.add_document(doc!(facet_field => Facet::from("/category/cat3")));
+        {
+            let mut doc = Document::new();
+            doc.add_facet(facet_field, "/category/cat2");
+            doc.add_facet(facet_field, "/category/cat1");
+            index_writer.add_document(doc);
+        }
+        {
+            let mut doc = Document::new();
+            doc.add_facet(facet_field, "/category/cat2");
+            index_writer.add_document(doc);
+        }
+        {
+            let mut doc = Document::new();
+            doc.add_facet(facet_field, "/category/cat3");
+            index_writer.add_document(doc);
+        }
         index_writer.commit().expect("Commit failed");
-        let searcher = index.reader().unwrap().searcher();
+        index.load_searchers().expect("Reloading searchers");
+        let searcher = index.searcher();
         let segment_reader = searcher.segment_reader(0);
         let mut facet_reader = segment_reader.facet_reader(facet_field).unwrap();
 
         let mut facet = Facet::root();
         {
-            facet_reader.facet_from_ord(1, &mut facet).unwrap();
+            facet_reader.facet_from_ord(1, &mut facet);
             assert_eq!(facet, Facet::from("/category"));
         }
         {
-            facet_reader.facet_from_ord(2, &mut facet).unwrap();
+            facet_reader.facet_from_ord(2, &mut facet);
             assert_eq!(facet, Facet::from("/category/cat1"));
         }
         {
-            facet_reader.facet_from_ord(3, &mut facet).unwrap();
+            facet_reader.facet_from_ord(3, &mut facet);
             assert_eq!(format!("{}", facet), "/category/cat2");
             assert_eq!(facet, Facet::from("/category/cat2"));
         }
         {
-            facet_reader.facet_from_ord(4, &mut facet).unwrap();
+            facet_reader.facet_from_ord(4, &mut facet);
             assert_eq!(facet, Facet::from("/category/cat3"));
         }
 

src/fastfield/multivalued/writer.rs

@@ -1,12 +1,13 @@
-use crate::fastfield::serializer::FastSingleFieldSerializer;
-use crate::fastfield::value_to_u64;
-use crate::fastfield::FastFieldSerializer;
-use crate::postings::UnorderedTermId;
-use crate::schema::{Document, Field};
-use crate::termdict::TermOrdinal;
-use crate::DocId;
-use fnv::FnvHashMap;
+use fastfield::serializer::FastSingleFieldSerializer;
+use fastfield::value_to_u64;
+use fastfield::FastFieldSerializer;
+use itertools::Itertools;
+use postings::UnorderedTermId;
+use schema::{Document, Field};
+use std::collections::HashMap;
 use std::io;
+use termdict::TermOrdinal;
+use DocId;
 
 /// Writer for multi-valued (as in, more than one value per document)
 /// int fast field.
@@ -31,7 +32,7 @@ use std::io;
 /// term ids when the segment is getting serialized.
 pub struct MultiValueIntFastFieldWriter {
     field: Field,
-    vals: Vec<UnorderedTermId>,
+    vals: Vec<u64>,
     doc_index: Vec<u64>,
     is_facet: bool,
 }
@@ -101,7 +102,7 @@ impl MultiValueIntFastFieldWriter {
     pub fn serialize(
         &self,
         serializer: &mut FastFieldSerializer,
-        mapping_opt: Option<&FnvHashMap<UnorderedTermId, TermOrdinal>>,
+        mapping_opt: Option<&HashMap<UnorderedTermId, TermOrdinal>>,
     ) -> io::Result<()> {
         {
             // writing the offset index
@@ -115,7 +116,7 @@ impl MultiValueIntFastFieldWriter {
         }
         {
             // writing the values themselves.
-            let mut value_serializer: FastSingleFieldSerializer<'_, _>;
+            let mut value_serializer: FastSingleFieldSerializer<_>;
             match mapping_opt {
                 Some(mapping) => {
                     value_serializer = serializer.new_u64_fast_field_with_idx(
@@ -150,8 +151,8 @@ impl MultiValueIntFastFieldWriter {
                     }
                 }
                 None => {
-                    let val_min_max = crate::common::minmax(self.vals.iter().cloned());
-                    let (val_min, val_max) = val_min_max.unwrap_or((0u64, 0u64));
+                    let val_min_max = self.vals.iter().cloned().minmax();
+                    let (val_min, val_max) = val_min_max.into_option().unwrap_or((0u64, 0u64));
                     value_serializer =
                         serializer.new_u64_fast_field_with_idx(self.field, val_min, val_max, 1)?;
                     for &val in &self.vals {

src/fastfield/reader.rs

@@ -1,18 +1,18 @@
 use super::FastValue;
-use crate::common::bitpacker::BitUnpacker;
-use crate::common::compute_num_bits;
-use crate::common::BinarySerializable;
-use crate::common::CompositeFile;
-use crate::directory::ReadOnlySource;
-use crate::directory::{Directory, RAMDirectory, WritePtr};
-use crate::fastfield::{FastFieldSerializer, FastFieldsWriter};
-use crate::schema::Schema;
-use crate::schema::FAST;
-use crate::DocId;
+use common::bitpacker::BitUnpacker;
+use common::compute_num_bits;
+use common::BinarySerializable;
+use common::CompositeFile;
+use directory::ReadOnlySource;
+use directory::{Directory, RAMDirectory, WritePtr};
+use fastfield::{FastFieldSerializer, FastFieldsWriter};
 use owning_ref::OwningRef;
+use schema::Schema;
+use schema::FAST;
 use std::collections::HashMap;
 use std::marker::PhantomData;
 use std::path::Path;
+use DocId;
 
 /// Trait for accessing a fastfield.
 ///
@@ -50,15 +50,6 @@ impl<Item: FastValue> FastFieldReader<Item> {
         }
     }
 
-    pub(crate) fn into_u64_reader(self) -> FastFieldReader<u64> {
-        FastFieldReader {
-            bit_unpacker: self.bit_unpacker,
-            min_value_u64: self.min_value_u64,
-            max_value_u64: self.max_value_u64,
-            _phantom: PhantomData,
-        }
-    }
-
     /// Return the value associated to the given document.
     ///
     /// This accessor should return as fast as possible.
@@ -68,29 +59,7 @@ impl<Item: FastValue> FastFieldReader<Item> {
     /// May panic if `doc` is greater than the segment
     // `maxdoc`.
     pub fn get(&self, doc: DocId) -> Item {
-        self.get_u64(u64::from(doc))
-    }
-
-    pub(crate) fn get_u64(&self, doc: u64) -> Item {
-        Item::from_u64(self.min_value_u64 + self.bit_unpacker.get(doc))
-    }
-
-    /// Internally `multivalued` also use SingleValue Fast fields.
-    /// It works as follows... A first column contains the list of start index
-    /// for each document, a second column contains the actual values.
-    ///
-    /// The values associated to a given doc, are then
-    ///  `second_column[first_column.get(doc)..first_column.get(doc+1)]`.
-    ///
-    /// Which means single value fast field reader can be indexed internally with
-    /// something different from a `DocId`. For this use case, we want to use `u64`
-    /// values.
-    ///
-    /// See `get_range` for an actual documentation about this method.
-    pub(crate) fn get_range_u64(&self, start: u64, output: &mut [Item]) {
-        for (i, out) in output.iter_mut().enumerate() {
-            *out = self.get_u64(start + (i as u64));
-        }
+        Item::from_u64(self.min_value_u64 + self.bit_unpacker.get(doc as usize))
     }
 
     /// Fills an output buffer with the fast field values
@@ -106,8 +75,16 @@ impl<Item: FastValue> FastFieldReader<Item> {
     ///
     /// May panic if `start + output.len()` is greater than
     /// the segment's `maxdoc`.
-    pub fn get_range(&self, start: DocId, output: &mut [Item]) {
-        self.get_range_u64(u64::from(start), output);
+    ///
+    // TODO change start to `u64`.
+    // For multifastfield, start is an index in a second fastfield, not a `DocId`
+    pub fn get_range(&self, start: u32, output: &mut [Item]) {
+        // ok: Item is either `u64` or `i64`
+        let output_u64: &mut [u64] = unsafe { &mut *(output as *mut [Item] as *mut [u64]) };
+        self.bit_unpacker.get_range(start, output_u64);
+        for out in output_u64.iter_mut() {
+            *out = Item::from_u64(*out + self.min_value_u64).as_u64();
+        }
     }
 
     /// Returns the minimum value for this fast field.

src/fastfield/readers.rs

@@ -1,270 +0,0 @@
-use crate::common::CompositeFile;
-use crate::fastfield::BytesFastFieldReader;
-use crate::fastfield::MultiValueIntFastFieldReader;
-use crate::fastfield::{FastFieldNotAvailableError, FastFieldReader};
-use crate::schema::{Cardinality, Field, FieldType, Schema};
-use crate::space_usage::PerFieldSpaceUsage;
-use std::collections::HashMap;
-
-/// Provides access to all of the FastFieldReader.
-///
-/// Internally, `FastFieldReaders` have preloaded fast field readers,
-/// and just wraps several `HashMap`.
-pub struct FastFieldReaders {
-    fast_field_i64: HashMap<Field, FastFieldReader<i64>>,
-    fast_field_u64: HashMap<Field, FastFieldReader<u64>>,
-    fast_field_f64: HashMap<Field, FastFieldReader<f64>>,
-    fast_field_date: HashMap<Field, FastFieldReader<crate::DateTime>>,
-    fast_field_i64s: HashMap<Field, MultiValueIntFastFieldReader<i64>>,
-    fast_field_u64s: HashMap<Field, MultiValueIntFastFieldReader<u64>>,
-    fast_field_f64s: HashMap<Field, MultiValueIntFastFieldReader<f64>>,
-    fast_field_dates: HashMap<Field, MultiValueIntFastFieldReader<crate::DateTime>>,
-    fast_bytes: HashMap<Field, BytesFastFieldReader>,
-    fast_fields_composite: CompositeFile,
-}
-
-enum FastType {
-    I64,
-    U64,
-    F64,
-    Date,
-}
-
-fn type_and_cardinality(field_type: &FieldType) -> Option<(FastType, Cardinality)> {
-    match field_type {
-        FieldType::U64(options) => options
-            .get_fastfield_cardinality()
-            .map(|cardinality| (FastType::U64, cardinality)),
-        FieldType::I64(options) => options
-            .get_fastfield_cardinality()
-            .map(|cardinality| (FastType::I64, cardinality)),
-        FieldType::F64(options) => options
-            .get_fastfield_cardinality()
-            .map(|cardinality| (FastType::F64, cardinality)),
-        FieldType::Date(options) => options
-            .get_fastfield_cardinality()
-            .map(|cardinality| (FastType::Date, cardinality)),
-        FieldType::HierarchicalFacet => Some((FastType::U64, Cardinality::MultiValues)),
-        _ => None,
-    }
-}
-
-impl FastFieldReaders {
-    pub(crate) fn load_all(
-        schema: &Schema,
-        fast_fields_composite: &CompositeFile,
-    ) -> crate::Result<FastFieldReaders> {
-        let mut fast_field_readers = FastFieldReaders {
-            fast_field_i64: Default::default(),
-            fast_field_u64: Default::default(),
-            fast_field_f64: Default::default(),
-            fast_field_date: Default::default(),
-            fast_field_i64s: Default::default(),
-            fast_field_u64s: Default::default(),
-            fast_field_f64s: Default::default(),
-            fast_field_dates: Default::default(),
-            fast_bytes: Default::default(),
-            fast_fields_composite: fast_fields_composite.clone(),
-        };
-        for (field, field_entry) in schema.fields() {
-            let field_type = field_entry.field_type();
-            if field_type == &FieldType::Bytes {
-                let idx_reader = fast_fields_composite
-                    .open_read_with_idx(field, 0)
-                    .ok_or_else(|| FastFieldNotAvailableError::new(field_entry))
-                    .map(FastFieldReader::open)?;
-                let data = fast_fields_composite
-                    .open_read_with_idx(field, 1)
-                    .ok_or_else(|| FastFieldNotAvailableError::new(field_entry))?;
-                fast_field_readers
-                    .fast_bytes
-                    .insert(field, BytesFastFieldReader::open(idx_reader, data));
-            } else if let Some((fast_type, cardinality)) = type_and_cardinality(field_type) {
-                match cardinality {
-                    Cardinality::SingleValue => {
-                        if let Some(fast_field_data) = fast_fields_composite.open_read(field) {
-                            match fast_type {
-                                FastType::U64 => {
-                                    let fast_field_reader = FastFieldReader::open(fast_field_data);
-                                    fast_field_readers
-                                        .fast_field_u64
-                                        .insert(field, fast_field_reader);
-                                }
-                                FastType::I64 => {
-                                    fast_field_readers.fast_field_i64.insert(
-                                        field,
-                                        FastFieldReader::open(fast_field_data.clone()),
-                                    );
-                                }
-                                FastType::F64 => {
-                                    fast_field_readers.fast_field_f64.insert(
-                                        field,
-                                        FastFieldReader::open(fast_field_data.clone()),
-                                    );
-                                }
-                                FastType::Date => {
-                                    fast_field_readers.fast_field_date.insert(
-                                        field,
-                                        FastFieldReader::open(fast_field_data.clone()),
-                                    );
-                                }
-                            }
-                        } else {
-                            return Err(From::from(FastFieldNotAvailableError::new(field_entry)));
-                        }
-                    }
-                    Cardinality::MultiValues => {
-                        let idx_opt = fast_fields_composite.open_read_with_idx(field, 0);
-                        let data_opt = fast_fields_composite.open_read_with_idx(field, 1);
-                        if let (Some(fast_field_idx), Some(fast_field_data)) = (idx_opt, data_opt) {
-                            let idx_reader = FastFieldReader::open(fast_field_idx);
-                            match fast_type {
-                                FastType::I64 => {
-                                    let vals_reader = FastFieldReader::open(fast_field_data);
-                                    let multivalued_int_fast_field =
-                                        MultiValueIntFastFieldReader::open(idx_reader, vals_reader);
-                                    fast_field_readers
-                                        .fast_field_i64s
-                                        .insert(field, multivalued_int_fast_field);
-                                }
-                                FastType::U64 => {
-                                    let vals_reader = FastFieldReader::open(fast_field_data);
-                                    let multivalued_int_fast_field =
-                                        MultiValueIntFastFieldReader::open(idx_reader, vals_reader);
-                                    fast_field_readers
-                                        .fast_field_u64s
-                                        .insert(field, multivalued_int_fast_field);
-                                }
-                                FastType::F64 => {
-                                    let vals_reader = FastFieldReader::open(fast_field_data);
-                                    let multivalued_int_fast_field =
-                                        MultiValueIntFastFieldReader::open(idx_reader, vals_reader);
-                                    fast_field_readers
-                                        .fast_field_f64s
-                                        .insert(field, multivalued_int_fast_field);
-                                }
-                                FastType::Date => {
-                                    let vals_reader = FastFieldReader::open(fast_field_data);
-                                    let multivalued_int_fast_field =
-                                        MultiValueIntFastFieldReader::open(idx_reader, vals_reader);
-                                    fast_field_readers
-                                        .fast_field_dates
-                                        .insert(field, multivalued_int_fast_field);
-                                }
-                            }
-                        } else {
-                            return Err(From::from(FastFieldNotAvailableError::new(field_entry)));
-                        }
-                    }
-                }
-            }
-        }
-        Ok(fast_field_readers)
-    }
-
-    pub(crate) fn space_usage(&self) -> PerFieldSpaceUsage {
-        self.fast_fields_composite.space_usage()
-    }
-
-    /// Returns the `u64` fast field reader reader associated to `field`.
-    ///
-    /// If `field` is not a u64 fast field, this method returns `None`.
-    pub fn u64(&self, field: Field) -> Option<FastFieldReader<u64>> {
-        self.fast_field_u64.get(&field).cloned()
-    }
-
-    /// If the field is a u64-fast field return the associated reader.
-    /// If the field is a i64-fast field, return the associated u64 reader. Values are
-    /// mapped from i64 to u64 using a (well the, it is unique) monotonic mapping.    ///
-    ///
-    /// This method is useful when merging segment reader.
-    pub(crate) fn u64_lenient(&self, field: Field) -> Option<FastFieldReader<u64>> {
-        if let Some(u64_ff_reader) = self.u64(field) {
-            return Some(u64_ff_reader);
-        }
-        if let Some(i64_ff_reader) = self.i64(field) {
-            return Some(i64_ff_reader.into_u64_reader());
-        }
-        if let Some(f64_ff_reader) = self.f64(field) {
-            return Some(f64_ff_reader.into_u64_reader());
-        }
-        if let Some(date_ff_reader) = self.date(field) {
-            return Some(date_ff_reader.into_u64_reader());
-        }
-        None
-    }
-
-    /// Returns the `i64` fast field reader reader associated to `field`.
-    ///
-    /// If `field` is not a i64 fast field, this method returns `None`.
-    pub fn i64(&self, field: Field) -> Option<FastFieldReader<i64>> {
-        self.fast_field_i64.get(&field).cloned()
-    }
-
-    /// Returns the `i64` fast field reader reader associated to `field`.
-    ///
-    /// If `field` is not a i64 fast field, this method returns `None`.
-    pub fn date(&self, field: Field) -> Option<FastFieldReader<crate::DateTime>> {
-        self.fast_field_date.get(&field).cloned()
-    }
-
-    /// Returns the `f64` fast field reader reader associated to `field`.
-    ///
-    /// If `field` is not a f64 fast field, this method returns `None`.
-    pub fn f64(&self, field: Field) -> Option<FastFieldReader<f64>> {
-        self.fast_field_f64.get(&field).cloned()
-    }
-
-    /// Returns a `u64s` multi-valued fast field reader reader associated to `field`.
-    ///
-    /// If `field` is not a u64 multi-valued fast field, this method returns `None`.
-    pub fn u64s(&self, field: Field) -> Option<MultiValueIntFastFieldReader<u64>> {
-        self.fast_field_u64s.get(&field).cloned()
-    }
-
-    /// If the field is a u64s-fast field return the associated reader.
-    /// If the field is a i64s-fast field, return the associated u64s reader. Values are
-    /// mapped from i64 to u64 using a (well the, it is unique) monotonic mapping.
-    ///
-    /// This method is useful when merging segment reader.
-    pub(crate) fn u64s_lenient(&self, field: Field) -> Option<MultiValueIntFastFieldReader<u64>> {
-        if let Some(u64s_ff_reader) = self.u64s(field) {
-            return Some(u64s_ff_reader);
-        }
-        if let Some(i64s_ff_reader) = self.i64s(field) {
-            return Some(i64s_ff_reader.into_u64s_reader());
-        }
-        if let Some(f64s_ff_reader) = self.f64s(field) {
-            return Some(f64s_ff_reader.into_u64s_reader());
-        }
-        None
-    }
-
-    /// Returns a `i64s` multi-valued fast field reader reader associated to `field`.
-    ///
-    /// If `field` is not a i64 multi-valued fast field, this method returns `None`.
-    pub fn i64s(&self, field: Field) -> Option<MultiValueIntFastFieldReader<i64>> {
-        self.fast_field_i64s.get(&field).cloned()
-    }
-
-    /// Returns a `f64s` multi-valued fast field reader reader associated to `field`.
-    ///
-    /// If `field` is not a f64 multi-valued fast field, this method returns `None`.
-    pub fn f64s(&self, field: Field) -> Option<MultiValueIntFastFieldReader<f64>> {
-        self.fast_field_f64s.get(&field).cloned()
-    }
-
-    /// Returns a `crate::DateTime` multi-valued fast field reader reader associated to `field`.
-    ///
-    /// If `field` is not a `crate::DateTime` multi-valued fast field, this method returns `None`.
-    pub fn dates(&self, field: Field) -> Option<MultiValueIntFastFieldReader<crate::DateTime>> {
-        self.fast_field_dates.get(&field).cloned()
-    }
-
-    /// Returns the `bytes` fast field reader associated to `field`.
-    ///
-    /// If `field` is not a bytes fast field, returns `None`.
-    pub fn bytes(&self, field: Field) -> Option<BytesFastFieldReader> {
-        self.fast_bytes.get(&field).cloned()
-    }
-}

src/fastfield/serializer.rs

@@ -1,10 +1,10 @@
-use crate::common::bitpacker::BitPacker;
-use crate::common::compute_num_bits;
-use crate::common::BinarySerializable;
-use crate::common::CompositeWrite;
-use crate::common::CountingWriter;
-use crate::directory::WritePtr;
-use crate::schema::Field;
+use common::bitpacker::BitPacker;
+use common::compute_num_bits;
+use common::BinarySerializable;
+use common::CompositeWrite;
+use common::CountingWriter;
+use directory::WritePtr;
+use schema::Field;
 use std::io::{self, Write};
 
 /// `FastFieldSerializer` is in charge of serializing
@@ -45,7 +45,7 @@ impl FastFieldSerializer {
         field: Field,
         min_value: u64,
         max_value: u64,
-    ) -> io::Result<FastSingleFieldSerializer<'_, CountingWriter<WritePtr>>> {
+    ) -> io::Result<FastSingleFieldSerializer<CountingWriter<WritePtr>>> {
         self.new_u64_fast_field_with_idx(field, min_value, max_value, 0)
     }
 
@@ -56,7 +56,7 @@ impl FastFieldSerializer {
         min_value: u64,
         max_value: u64,
         idx: usize,
-    ) -> io::Result<FastSingleFieldSerializer<'_, CountingWriter<WritePtr>>> {
+    ) -> io::Result<FastSingleFieldSerializer<CountingWriter<WritePtr>>> {
         let field_write = self.composite_write.for_field_with_idx(field, idx);
         FastSingleFieldSerializer::open(field_write, min_value, max_value)
     }
@@ -66,7 +66,7 @@ impl FastFieldSerializer {
         &mut self,
         field: Field,
         idx: usize,
-    ) -> io::Result<FastBytesFieldSerializer<'_, CountingWriter<WritePtr>>> {
+    ) -> io::Result<FastBytesFieldSerializer<CountingWriter<WritePtr>>> {
         let field_write = self.composite_write.for_field_with_idx(field, idx);
         FastBytesFieldSerializer::open(field_write)
     }
@@ -79,7 +79,7 @@ impl FastFieldSerializer {
     }
 }
 
-pub struct FastSingleFieldSerializer<'a, W: Write> {
+pub struct FastSingleFieldSerializer<'a, W: Write + 'a> {
     bit_packer: BitPacker,
     write: &'a mut W,
     min_value: u64,
@@ -127,7 +127,7 @@ impl<'a, W: Write> FastSingleFieldSerializer<'a, W> {
     }
 }
 
-pub struct FastBytesFieldSerializer<'a, W: Write> {
+pub struct FastBytesFieldSerializer<'a, W: Write + 'a> {
     write: &'a mut W,
 }
 

src/fastfield/writer.rs

@@ -1,14 +1,13 @@
 use super::multivalued::MultiValueIntFastFieldWriter;
-use crate::common;
-use crate::common::BinarySerializable;
-use crate::common::VInt;
-use crate::fastfield::{BytesFastFieldWriter, FastFieldSerializer};
-use crate::postings::UnorderedTermId;
-use crate::schema::{Cardinality, Document, Field, FieldEntry, FieldType, Schema};
-use crate::termdict::TermOrdinal;
-use fnv::FnvHashMap;
+use common;
+use common::BinarySerializable;
+use common::VInt;
+use fastfield::{BytesFastFieldWriter, FastFieldSerializer};
+use postings::UnorderedTermId;
+use schema::{Cardinality, Document, Field, FieldType, Schema};
 use std::collections::HashMap;
 use std::io;
+use termdict::TermOrdinal;
 
 /// The fastfieldswriter regroup all of the fast field writers.
 pub struct FastFieldsWriter {
@@ -17,14 +16,6 @@ pub struct FastFieldsWriter {
     bytes_value_writers: Vec<BytesFastFieldWriter>,
 }
 
-fn fast_field_default_value(field_entry: &FieldEntry) -> u64 {
-    match *field_entry.field_type() {
-        FieldType::I64(_) | FieldType::Date(_) => common::i64_to_u64(0i64),
-        FieldType::F64(_) => common::f64_to_u64(0.0f64),
-        _ => 0u64,
-    }
-}
-
 impl FastFieldsWriter {
     /// Create all `FastFieldWriter` required by the schema.
     pub fn from_schema(schema: &Schema) -> FastFieldsWriter {
@@ -32,16 +23,18 @@ impl FastFieldsWriter {
         let mut multi_values_writers = Vec::new();
         let mut bytes_value_writers = Vec::new();
 
-        for (field, field_entry) in schema.fields() {
+        for (field_id, field_entry) in schema.fields().iter().enumerate() {
+            let field = Field(field_id as u32);
+            let default_value = if let FieldType::I64(_) = *field_entry.field_type() {
+                common::i64_to_u64(0i64)
+            } else {
+                0u64
+            };
             match *field_entry.field_type() {
-                FieldType::I64(ref int_options)
-                | FieldType::U64(ref int_options)
-                | FieldType::F64(ref int_options)
-                | FieldType::Date(ref int_options) => {
+                FieldType::I64(ref int_options) | FieldType::U64(ref int_options) => {
                     match int_options.get_fastfield_cardinality() {
                         Some(Cardinality::SingleValue) => {
                             let mut fast_field_writer = IntFastFieldWriter::new(field);
-                            let default_value = fast_field_default_value(field_entry);
                             fast_field_writer.set_val_if_missing(default_value);
                             single_value_writers.push(fast_field_writer);
                         }
@@ -121,7 +114,7 @@ impl FastFieldsWriter {
     pub fn serialize(
         &self,
         serializer: &mut FastFieldSerializer,
-        mapping: &HashMap<Field, FnvHashMap<UnorderedTermId, TermOrdinal>>,
+        mapping: &HashMap<Field, HashMap<UnorderedTermId, TermOrdinal>>,
     ) -> io::Result<()> {
         for field_writer in &self.single_value_writers {
             field_writer.serialize(serializer)?;
@@ -149,9 +142,9 @@ impl FastFieldsWriter {
 /// bitpacked and the number of bits required for bitpacking
 /// can only been known once we have seen all of the values.
 ///
-/// Both u64, i64 and f64 use the same writer.
-/// i64 and f64 are just remapped to the `0..2^64 - 1`
-/// using `common::i64_to_u64` and `common::f64_to_u64`.
+/// Both u64, and i64 use the same writer.
+/// i64 are just remapped to the `0..2^64 - 1`
+/// using `common::i64_to_u64`.
 pub struct IntFastFieldWriter {
     field: Field,
     vals: Vec<u8>,
@@ -210,8 +203,8 @@ impl IntFastFieldWriter {
     /// Extract the value associated to the fast field for
     /// this document.
     ///
-    /// i64 and f64 are remapped to u64 using the logic
-    /// in `common::i64_to_u64` and `common::f64_to_u64`.
+    /// i64 are remapped to u64 using the logic
+    /// in `common::i64_to_u64`.
     ///
     /// If the value is missing, then the default value is used
     /// instead.

src/fieldnorm/code.rs

@@ -10,263 +10,28 @@ pub fn fieldnorm_to_id(fieldnorm: u32) -> u8 {
         .unwrap_or_else(|idx| idx - 1) as u8
 }
 
+#[cfg_attr(feature = "cargo-clippy", allow(clippy::unreadable_literal))]
 pub const FIELD_NORMS_TABLE: [u32; 256] = [
-    0,
-    1,
-    2,
-    3,
-    4,
-    5,
-    6,
-    7,
-    8,
-    9,
-    10,
-    11,
-    12,
-    13,
-    14,
-    15,
-    16,
-    17,
-    18,
-    19,
-    20,
-    21,
-    22,
-    23,
-    24,
-    25,
-    26,
-    27,
-    28,
-    29,
-    30,
-    31,
-    32,
-    33,
-    34,
-    35,
-    36,
-    37,
-    38,
-    39,
-    40,
-    42,
-    44,
-    46,
-    48,
-    50,
-    52,
-    54,
-    56,
-    60,
-    64,
-    68,
-    72,
-    76,
-    80,
-    84,
-    88,
-    96,
-    104,
-    112,
-    120,
-    128,
-    136,
-    144,
-    152,
-    168,
-    184,
-    200,
-    216,
-    232,
-    248,
-    264,
-    280,
-    312,
-    344,
-    376,
-    408,
-    440,
-    472,
-    504,
-    536,
-    600,
-    664,
-    728,
-    792,
-    856,
-    920,
-    984,
-    1_048,
-    1_176,
-    1_304,
-    1_432,
-    1_560,
-    1_688,
-    1_816,
-    1_944,
-    2_072,
-    2_328,
-    2_584,
-    2_840,
-    3_096,
-    3_352,
-    3_608,
-    3_864,
-    4_120,
-    4_632,
-    5_144,
-    5_656,
-    6_168,
-    6_680,
-    7_192,
-    7_704,
-    8_216,
-    9_240,
-    10_264,
-    11_288,
-    12_312,
-    13_336,
-    14_360,
-    15_384,
-    16_408,
-    18_456,
-    20_504,
-    22_552,
-    24_600,
-    26_648,
-    28_696,
-    30_744,
-    32_792,
-    36_888,
-    40_984,
-    45_080,
-    49_176,
-    53_272,
-    57_368,
-    61_464,
-    65_560,
-    73_752,
-    81_944,
-    90_136,
-    98_328,
-    106_520,
-    114_712,
-    122_904,
-    131_096,
-    147_480,
-    163_864,
-    180_248,
-    196_632,
-    213_016,
-    229_400,
-    245_784,
-    262_168,
-    294_936,
-    327_704,
-    360_472,
-    393_240,
-    426_008,
-    458_776,
-    491_544,
-    524_312,
-    589_848,
-    655_384,
-    720_920,
-    786_456,
-    851_992,
-    917_528,
-    983_064,
-    1_048_600,
-    1_179_672,
-    1_310_744,
-    1_441_816,
-    1_572_888,
-    1_703_960,
-    1_835_032,
-    1_966_104,
-    2_097_176,
-    2_359_320,
-    2_621_464,
-    2_883_608,
-    3_145_752,
-    3_407_896,
-    3_670_040,
-    3_932_184,
-    4_194_328,
-    4_718_616,
-    5_242_904,
-    5_767_192,
-    6_291_480,
-    6_815_768,
-    7_340_056,
-    7_864_344,
-    8_388_632,
-    9_437_208,
-    10_485_784,
-    11_534_360,
-    12_582_936,
-    13_631_512,
-    14_680_088,
-    15_728_664,
-    16_777_240,
-    18_874_392,
-    20_971_544,
-    23_068_696,
-    25_165_848,
-    27_263_000,
-    29_360_152,
-    31_457_304,
-    33_554_456,
-    37_748_760,
-    41_943_064,
-    46_137_368,
-    50_331_672,
-    54_525_976,
-    58_720_280,
-    62_914_584,
-    67_108_888,
-    75_497_496,
-    83_886_104,
-    92_274_712,
-    100_663_320,
-    109_051_928,
-    117_440_536,
-    125_829_144,
-    134_217_752,
-    150_994_968,
-    167_772_184,
-    184_549_400,
-    201_326_616,
-    218_103_832,
-    234_881_048,
-    251_658_264,
-    268_435_480,
-    301_989_912,
-    335_544_344,
-    369_098_776,
-    402_653_208,
-    436_207_640,
-    469_762_072,
-    503_316_504,
-    536_870_936,
-    603_979_800,
-    671_088_664,
-    738_197_528,
-    805_306_392,
-    872_415_256,
-    939_524_120,
-    1_006_632_984,
-    1_073_741_848,
-    1_207_959_576,
-    1_342_177_304,
-    1_476_395_032,
-    1_610_612_760,
-    1_744_830_488,
-    1_879_048_216,
-    2_013_265_944,
+    0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25,
+    26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 42, 44, 46, 48, 50, 52, 54, 56, 60,
+    64, 68, 72, 76, 80, 84, 88, 96, 104, 112, 120, 128, 136, 144, 152, 168, 184, 200, 216, 232,
+    248, 264, 280, 312, 344, 376, 408, 440, 472, 504, 536, 600, 664, 728, 792, 856, 920, 984,
+    1_048, 1176, 1304, 1432, 1560, 1688, 1816, 1944, 2072, 2328, 2584, 2840, 3096, 3352, 3608,
+    3864, 4120, 4632, 5144, 5656, 6168, 6680, 7192, 7704, 8216, 9240, 10264, 11288, 12312, 13336,
+    14360, 15384, 16408, 18456, 20504, 22552, 24600, 26648, 28696, 30744, 32792, 36888, 40984,
+    45080, 49176, 53272, 57368, 61464, 65560, 73752, 81944, 90136, 98328, 106520, 114712, 122904,
+    131096, 147480, 163864, 180248, 196632, 213016, 229400, 245784, 262168, 294936, 327704, 360472,
+    393240, 426008, 458776, 491544, 524312, 589848, 655384, 720920, 786456, 851992, 917528, 983064,
+    1048600, 1179672, 1310744, 1441816, 1572888, 1703960, 1835032, 1966104, 2097176, 2359320,
+    2621464, 2883608, 3145752, 3407896, 3670040, 3932184, 4194328, 4718616, 5242904, 5767192,
+    6291480, 6815768, 7340056, 7864344, 8388632, 9437208, 10485784, 11534360, 12582936, 13631512,
+    14680088, 15728664, 16777240, 18874392, 20971544, 23068696, 25165848, 27263000, 29360152,
+    31457304, 33554456, 37748760, 41943064, 46137368, 50331672, 54525976, 58720280, 62914584,
+    67108888, 75497496, 83886104, 92274712, 100663320, 109051928, 117440536, 125829144, 134217752,
+    150994968, 167772184, 184549400, 201326616, 218103832, 234881048, 251658264, 268435480,
+    301989912, 335544344, 369098776, 402653208, 436207640, 469762072, 503316504, 536870936,
+    603979800, 671088664, 738197528, 805306392, 872415256, 939524120, 1006632984, 1073741848,
+    1207959576, 1342177304, 1476395032, 1610612760, 1744830488, 1879048216, 2013265944,
 ];
 
 #[cfg(test)]

src/fieldnorm/mod.rs

@@ -21,7 +21,7 @@ mod reader;
 mod serializer;
 mod writer;
 
-pub use self::reader::{FieldNormReader, FieldNormReaders};
+pub use self::reader::FieldNormReader;
 pub use self::serializer::FieldNormsSerializer;
 pub use self::writer::FieldNormsWriter;
 

src/fieldnorm/reader.rs

@@ -1,41 +1,6 @@
 use super::{fieldnorm_to_id, id_to_fieldnorm};
-use crate::common::CompositeFile;
-use crate::directory::ReadOnlySource;
-use crate::schema::Field;
-use crate::space_usage::PerFieldSpaceUsage;
-use crate::DocId;
-use std::sync::Arc;
-
-/// Reader for the fieldnorm (for each document, the number of tokens indexed in the
-/// field) of all indexed fields in the index.
-///
-/// Each fieldnorm is approximately compressed over one byte. We refer to this byte as
-/// `fieldnorm_id`.
-/// The mapping from `fieldnorm` to `fieldnorm_id` is given by monotonic.
-#[derive(Clone)]
-pub struct FieldNormReaders {
-    data: Arc<CompositeFile>,
-}
-
-impl FieldNormReaders {
-    /// Creates a field norm reader.
-    pub fn open(source: ReadOnlySource) -> crate::Result<FieldNormReaders> {
-        let data = CompositeFile::open(&source)?;
-        Ok(FieldNormReaders {
-            data: Arc::new(data),
-        })
-    }
-
-    /// Returns the FieldNormReader for a specific field.
-    pub fn get_field(&self, field: Field) -> Option<FieldNormReader> {
-        self.data.open_read(field).map(FieldNormReader::open)
-    }
-
-    /// Return a break down of the space usage per field.
-    pub fn space_usage(&self) -> PerFieldSpaceUsage {
-        self.data.space_usage()
-    }
-}
+use directory::ReadOnlySource;
+use DocId;
 
 /// Reads the fieldnorm associated to a document.
 /// The fieldnorm represents the length associated to
@@ -54,7 +19,6 @@ impl FieldNormReaders {
 /// Apart from compression, this scale also makes it possible to
 /// precompute computationally expensive functions of the fieldnorm
 /// in a very short array.
-#[derive(Clone)]
 pub struct FieldNormReader {
     data: ReadOnlySource,
 }
@@ -65,11 +29,6 @@ impl FieldNormReader {
         FieldNormReader { data }
     }
 
-    /// Returns the number of documents in this segment.
-    pub fn num_docs(&self) -> u32 {
-        self.data.len() as u32
-    }
-
     /// Returns the `fieldnorm` associated to a doc id.
     /// The fieldnorm is a value approximating the number
     /// of tokens in a given field of the `doc_id`.
@@ -103,12 +62,13 @@ impl FieldNormReader {
     pub fn fieldnorm_to_id(fieldnorm: u32) -> u8 {
         fieldnorm_to_id(fieldnorm)
     }
+}
 
-    #[cfg(test)]
-    pub fn for_test(field_norms: &[u32]) -> FieldNormReader {
+#[cfg(test)]
+impl From<Vec<u32>> for FieldNormReader {
+    fn from(field_norms: Vec<u32>) -> FieldNormReader {
         let field_norms_id = field_norms
-            .iter()
-            .cloned()
+            .into_iter()
             .map(FieldNormReader::fieldnorm_to_id)
             .collect::<Vec<u8>>();
         let field_norms_data = ReadOnlySource::from(field_norms_id);
@@ -117,20 +77,3 @@ impl FieldNormReader {
         }
     }
 }
-
-#[cfg(test)]
-mod tests {
-    use crate::fieldnorm::FieldNormReader;
-
-    #[test]
-    fn test_from_fieldnorms_array() {
-        let fieldnorms = &[1, 2, 3, 4, 1_000_000];
-        let fieldnorm_reader = FieldNormReader::for_test(fieldnorms);
-        assert_eq!(fieldnorm_reader.num_docs(), 5);
-        assert_eq!(fieldnorm_reader.fieldnorm(0), 1);
-        assert_eq!(fieldnorm_reader.fieldnorm(1), 2);
-        assert_eq!(fieldnorm_reader.fieldnorm(2), 3);
-        assert_eq!(fieldnorm_reader.fieldnorm(3), 4);
-        assert_eq!(fieldnorm_reader.fieldnorm(4), 983_064);
-    }
-}

src/fieldnorm/serializer.rs

@@ -1,6 +1,6 @@
-use crate::common::CompositeWrite;
-use crate::directory::WritePtr;
-use crate::schema::Field;
+use common::CompositeWrite;
+use directory::WritePtr;
+use schema::Field;
 use std::io;
 use std::io::Write;
 

src/fieldnorm/writer.rs

@@ -1,9 +1,9 @@
-use crate::DocId;
+use DocId;
 
 use super::fieldnorm_to_id;
 use super::FieldNormsSerializer;
-use crate::schema::Field;
-use crate::schema::Schema;
+use schema::Field;
+use schema::Schema;
 use std::io;
 
 /// The `FieldNormsWriter` is in charge of tracking the fieldnorm byte
@@ -22,14 +22,11 @@ impl FieldNormsWriter {
     pub(crate) fn fields_with_fieldnorm(schema: &Schema) -> Vec<Field> {
         schema
             .fields()
-            .filter_map(|(field, field_entry)| {
-                if field_entry.is_indexed() {
-                    Some(field)
-                } else {
-                    None
-                }
-            })
-            .collect::<Vec<_>>()
+            .iter()
+            .enumerate()
+            .filter(|&(_, field_entry)| field_entry.is_indexed())
+            .map(|(field, _)| Field(field as u32))
+            .collect::<Vec<Field>>()
     }
 
     /// Initialize with state for tracking the field norm fields
@@ -38,7 +35,7 @@ impl FieldNormsWriter {
         let fields = FieldNormsWriter::fields_with_fieldnorm(schema);
         let max_field = fields
             .iter()
-            .map(Field::field_id)
+            .map(|field| field.0)
             .max()
             .map(|max_field_id| max_field_id as usize + 1)
             .unwrap_or(0);
@@ -53,8 +50,8 @@ impl FieldNormsWriter {
     ///
     /// Will extend with 0-bytes for documents that have not been seen.
     pub fn fill_up_to_max_doc(&mut self, max_doc: DocId) {
-        for field in self.fields.iter() {
-            self.fieldnorms_buffer[field.field_id() as usize].resize(max_doc as usize, 0u8);
+        for &field in self.fields.iter() {
+            self.fieldnorms_buffer[field.0 as usize].resize(max_doc as usize, 0u8);
         }
     }
 
@@ -67,7 +64,7 @@ impl FieldNormsWriter {
     /// * field     - the field being set
     /// * fieldnorm - the number of terms present in document `doc` in field `field`
     pub fn record(&mut self, doc: DocId, field: Field, fieldnorm: u32) {
-        let fieldnorm_buffer: &mut Vec<u8> = &mut self.fieldnorms_buffer[field.field_id() as usize];
+        let fieldnorm_buffer: &mut Vec<u8> = &mut self.fieldnorms_buffer[field.0 as usize];
         assert!(
             fieldnorm_buffer.len() <= doc as usize,
             "Cannot register a given fieldnorm twice"
@@ -78,12 +75,11 @@ impl FieldNormsWriter {
     }
 
     /// Serialize the seen fieldnorm values to the serializer for all fields.
-    pub fn serialize(&self, mut fieldnorms_serializer: FieldNormsSerializer) -> io::Result<()> {
+    pub fn serialize(&self, fieldnorms_serializer: &mut FieldNormsSerializer) -> io::Result<()> {
         for &field in self.fields.iter() {
-            let fieldnorm_values: &[u8] = &self.fieldnorms_buffer[field.field_id() as usize][..];
+            let fieldnorm_values: &[u8] = &self.fieldnorms_buffer[field.0 as usize][..];
             fieldnorms_serializer.serialize_field(field, fieldnorm_values)?;
         }
-        fieldnorms_serializer.close()?;
         Ok(())
     }
 }

src/functional_test.rs

@@ -1,10 +1,10 @@
 use rand::thread_rng;
 use std::collections::HashSet;
 
-use crate::schema::*;
-use crate::Index;
-use crate::Searcher;
 use rand::Rng;
+use schema::*;
+use Index;
+use Searcher;
 
 fn check_index_content(searcher: &Searcher, vals: &HashSet<u64>) {
     assert!(searcher.segment_readers().len() < 20);
@@ -13,15 +13,15 @@ fn check_index_content(searcher: &Searcher, vals: &HashSet<u64>) {
 
 #[test]
 #[ignore]
+#[cfg(feature = "mmap")]
 fn test_indexing() {
     let mut schema_builder = Schema::builder();
 
-    let id_field = schema_builder.add_u64_field("id", INDEXED);
-    let multiples_field = schema_builder.add_u64_field("multiples", INDEXED);
+    let id_field = schema_builder.add_u64_field("id", INT_INDEXED);
+    let multiples_field = schema_builder.add_u64_field("multiples", INT_INDEXED);
     let schema = schema_builder.build();
 
     let index = Index::create_from_tempdir(schema).unwrap();
-    let reader = index.reader().unwrap();
 
     let mut rng = thread_rng();
 
@@ -36,8 +36,8 @@ fn test_indexing() {
             index_writer.commit().expect("Commit failed");
             committed_docs.extend(&uncommitted_docs);
             uncommitted_docs.clear();
-            reader.reload().unwrap();
-            let searcher = reader.searcher();
+            index.load_searchers().unwrap();
+            let searcher = index.searcher();
             // check that everything is correct.
             check_index_content(&searcher, &committed_docs);
         } else {

src/indexer/delete_queue.rs

@@ -1,8 +1,7 @@
 use super::operation::DeleteOperation;
-use crate::Opstamp;
 use std::mem;
 use std::ops::DerefMut;
-use std::sync::{Arc, RwLock, Weak};
+use std::sync::{Arc, RwLock};
 
 // The DeleteQueue is similar in conceptually to a multiple
 // consumer single producer broadcast channel.
@@ -14,18 +13,17 @@ use std::sync::{Arc, RwLock, Weak};
 //
 // New consumer can be created in two ways
 // - calling `delete_queue.cursor()` returns a cursor, that
-//   will include all future delete operation (and some or none
-//   of the past operations... The client is in charge of checking the opstamps.).
+//   will include all future delete operation (and no past operations).
 // - cloning an existing cursor returns a new cursor, that
 //   is at the exact same position, and can now advance independently
 //   from the original cursor.
 #[derive(Default)]
 struct InnerDeleteQueue {
     writer: Vec<DeleteOperation>,
-    last_block: Weak<Block>,
+    last_block: Option<Arc<Block>>,
 }
 
-#[derive(Clone)]
+#[derive(Clone, Default)]
 pub struct DeleteQueue {
     inner: Arc<RwLock<InnerDeleteQueue>>,
 }
@@ -33,31 +31,20 @@ pub struct DeleteQueue {
 impl DeleteQueue {
     // Creates a new delete queue.
     pub fn new() -> DeleteQueue {
-        DeleteQueue {
+        let delete_queue = DeleteQueue {
             inner: Arc::default(),
-        }
-    }
+        };
 
-    fn get_last_block(&self) -> Arc<Block> {
+        let next_block = NextBlock::from(delete_queue.clone());
         {
-            // try get the last block with simply acquiring the read lock.
-            let rlock = self.inner.read().unwrap();
-            if let Some(block) = rlock.last_block.upgrade() {
-                return block;
-            }
+            let mut delete_queue_wlock = delete_queue.inner.write().unwrap();
+            delete_queue_wlock.last_block = Some(Arc::new(Block {
+                operations: Arc::default(),
+                next: next_block,
+            }));
         }
-        // It failed. Let's double check after acquiring the write, as someone could have called
-        // `get_last_block` right after we released the rlock.
-        let mut wlock = self.inner.write().unwrap();
-        if let Some(block) = wlock.last_block.upgrade() {
-            return block;
-        }
-        let block = Arc::new(Block {
-            operations: Arc::default(),
-            next: NextBlock::from(self.clone()),
-        });
-        wlock.last_block = Arc::downgrade(&block);
-        block
+
+        delete_queue
     }
 
     // Creates a new cursor that makes it possible to
@@ -65,7 +52,17 @@ impl DeleteQueue {
     //
     // Past delete operations are not accessible.
     pub fn cursor(&self) -> DeleteCursor {
-        let last_block = self.get_last_block();
+        let last_block = self
+            .inner
+            .read()
+            .expect("Read lock poisoned when opening delete queue cursor")
+            .last_block
+            .clone()
+            .expect(
+                "Failed to unwrap last_block. This should never happen
+                as the Option<> is only here to make
+                initialization possible",
+            );
         let operations_len = last_block.operations.len();
         DeleteCursor {
             block: last_block,
@@ -101,19 +98,23 @@ impl DeleteQueue {
             .write()
             .expect("Failed to acquire write lock on delete queue writer");
 
-        if self_wlock.writer.is_empty() {
-            return None;
+        let delete_operations;
+        {
+            let writer: &mut Vec<DeleteOperation> = &mut self_wlock.writer;
+            if writer.is_empty() {
+                return None;
+            }
+            delete_operations = mem::replace(writer, vec![]);
         }
 
-        let delete_operations = mem::replace(&mut self_wlock.writer, vec![]);
-
-        let new_block = Arc::new(Block {
-            operations: Arc::new(delete_operations.into_boxed_slice()),
-            next: NextBlock::from(self.clone()),
-        });
-
-        self_wlock.last_block = Arc::downgrade(&new_block);
-        Some(new_block)
+        let next_block = NextBlock::from(self.clone());
+        {
+            self_wlock.last_block = Some(Arc::new(Block {
+                operations: Arc::new(delete_operations),
+                next: next_block,
+            }));
+        }
+        self_wlock.last_block.clone()
     }
 }
 
@@ -167,7 +168,7 @@ impl NextBlock {
 }
 
 struct Block {
-    operations: Arc<Box<[DeleteOperation]>>,
+    operations: Arc<Vec<DeleteOperation>>,
     next: NextBlock,
 }
 
@@ -183,7 +184,7 @@ impl DeleteCursor {
     ///   queue are consume and the next get will return None.
     /// - the next get will return the first operation with an
     /// `opstamp >= target_opstamp`.
-    pub fn skip_to(&mut self, target_opstamp: Opstamp) {
+    pub fn skip_to(&mut self, target_opstamp: u64) {
         // TODO Can be optimize as we work with block.
         while self.is_behind_opstamp(target_opstamp) {
             self.advance();
@@ -191,7 +192,7 @@ impl DeleteCursor {
     }
 
     #[cfg_attr(feature = "cargo-clippy", allow(clippy::wrong_self_convention))]
-    fn is_behind_opstamp(&mut self, target_opstamp: Opstamp) -> bool {
+    fn is_behind_opstamp(&mut self, target_opstamp: u64) -> bool {
         self.get()
             .map(|operation| operation.opstamp < target_opstamp)
             .unwrap_or(false)
@@ -248,14 +249,14 @@ impl DeleteCursor {
 mod tests {
 
     use super::{DeleteOperation, DeleteQueue};
-    use crate::schema::{Field, Term};
+    use schema::{Field, Term};
 
     #[test]
     fn test_deletequeue() {
         let delete_queue = DeleteQueue::new();
 
         let make_op = |i: usize| {
-            let field = Field::from_field_id(1u32);
+            let field = Field(1u32);
             DeleteOperation {
                 opstamp: i as u64,
                 term: Term::from_field_u64(field, i as u64),

src/indexer/directory_lock.rs

@@ -0,0 +1,131 @@
+use directory::error::OpenWriteError;
+use std::io::Write;
+use std::path::{Path, PathBuf};
+use std::thread;
+use std::time::Duration;
+use Directory;
+use TantivyError;
+
+#[derive(Debug, Clone, Copy)]
+pub enum LockType {
+    /// Only one process should be able to write tantivy's index at a time.
+    /// This lock file, when present, is in charge of preventing other processes to open an IndexWriter.
+    ///
+    /// If the process is killed and this file remains, it is safe to remove it manually.
+    ///
+    /// Failing to acquire this lock usually means a misuse of tantivy's API,
+    /// (creating more than one instance of the `IndexWriter`), are a spurious
+    /// lock file remaining after a crash. In the latter case, removing the file after
+    /// checking no process running tantivy is running is safe.
+    IndexWriterLock,
+    /// The meta lock file is here to protect the segment files being opened by
+    /// `.load_searchers()` from being garbage collected.
+    /// It makes it possible for another process to safely consume
+    /// our index in-writing. Ideally, we may have prefered `RWLock` semantics
+    /// here, but it is difficult to achieve on Windows.
+    ///
+    /// Opening segment readers is a very fast process.
+    /// Right now if the lock cannot be acquire on the first attempt, the logic
+    /// is very simplistic. We retry after `100ms` until we effectively
+    /// acquire the lock.
+    /// This lock should not have much contention in normal usage.
+    MetaLock,
+}
+
+/// Retry the logic of acquiring locks is pretty simple.
+/// We just retry `n` times after a given `duratio`, both
+/// depending on the type of lock.
+struct RetryPolicy {
+    num_retries: usize,
+    wait_in_ms: u64,
+}
+
+impl RetryPolicy {
+    fn no_retry() -> RetryPolicy {
+        RetryPolicy {
+            num_retries: 0,
+            wait_in_ms: 0,
+        }
+    }
+
+    fn wait_and_retry(&mut self) -> bool {
+        if self.num_retries == 0 {
+            false
+        } else {
+            self.num_retries -= 1;
+            let wait_duration = Duration::from_millis(self.wait_in_ms);
+            thread::sleep(wait_duration);
+            true
+        }
+    }
+}
+
+impl LockType {
+    fn retry_policy(self) -> RetryPolicy {
+        match self {
+            LockType::IndexWriterLock => RetryPolicy::no_retry(),
+            LockType::MetaLock => RetryPolicy {
+                num_retries: 100,
+                wait_in_ms: 100,
+            },
+        }
+    }
+
+    fn try_acquire_lock(self, directory: &mut Directory) -> Result<DirectoryLock, TantivyError> {
+        let path = self.filename();
+        let mut write = directory.open_write(path).map_err(|e| match e {
+            OpenWriteError::FileAlreadyExists(_) => TantivyError::LockFailure(self),
+            OpenWriteError::IOError(io_error) => TantivyError::IOError(io_error),
+        })?;
+        write.flush()?;
+        Ok(DirectoryLock {
+            directory: directory.box_clone(),
+            path: path.to_owned(),
+        })
+    }
+
+    /// Acquire a lock in the given directory.
+    pub fn acquire_lock(self, directory: &Directory) -> Result<DirectoryLock, TantivyError> {
+        let mut box_directory = directory.box_clone();
+        let mut retry_policy = self.retry_policy();
+        loop {
+            let lock_result = self.try_acquire_lock(&mut *box_directory);
+            match lock_result {
+                Ok(result) => {
+                    return Ok(result);
+                }
+                Err(TantivyError::LockFailure(ref filepath)) => {
+                    if !retry_policy.wait_and_retry() {
+                        return Err(TantivyError::LockFailure(filepath.to_owned()));
+                    }
+                }
+                Err(_) => {}
+            }
+        }
+    }
+
+    fn filename(&self) -> &Path {
+        match *self {
+            LockType::MetaLock => Path::new(".tantivy-meta.lock"),
+            LockType::IndexWriterLock => Path::new(".tantivy-indexer.lock"),
+        }
+    }
+}
+
+/// The `DirectoryLock` is an object that represents a file lock.
+/// See  [`LockType`](struct.LockType.html)
+///
+/// It is transparently associated to a lock file, that gets deleted
+/// on `Drop.` The lock is release automatically on `Drop`.
+pub struct DirectoryLock {
+    directory: Box<Directory>,
+    path: PathBuf,
+}
+
+impl Drop for DirectoryLock {
+    fn drop(&mut self) {
+        if let Err(e) = self.directory.delete(&*self.path) {
+            error!("Failed to remove the lock file. {:?}", e);
+        }
+    }
+}

src/indexer/doc_opstamp_mapping.rs

@@ -1,5 +1,5 @@
-use crate::DocId;
-use crate::Opstamp;
+use std::sync::Arc;
+use DocId;
 
 // Doc to opstamp is used to identify which
 // document should be deleted.
@@ -17,25 +17,25 @@ use crate::Opstamp;
 // This mapping is (for the moment) stricly increasing
 // because of the way document id are allocated.
 #[derive(Clone)]
-pub enum DocToOpstampMapping<'a> {
-    WithMap(&'a [Opstamp]),
+pub enum DocToOpstampMapping {
+    WithMap(Arc<Vec<u64>>),
     None,
 }
 
-impl<'a> From<&'a [u64]> for DocToOpstampMapping<'a> {
-    fn from(opstamps: &[Opstamp]) -> DocToOpstampMapping {
-        DocToOpstampMapping::WithMap(opstamps)
+impl From<Vec<u64>> for DocToOpstampMapping {
+    fn from(opstamps: Vec<u64>) -> DocToOpstampMapping {
+        DocToOpstampMapping::WithMap(Arc::new(opstamps))
     }
 }
 
-impl<'a> DocToOpstampMapping<'a> {
+impl DocToOpstampMapping {
     /// Given an opstamp return the limit doc id L
     /// such that all doc id D such that
     // D >= L iff opstamp(D) >= than `target_opstamp`.
     //
     // The edge case opstamp = some doc opstamp is in practise
     // never called.
-    pub fn compute_doc_limit(&self, target_opstamp: Opstamp) -> DocId {
+    pub fn compute_doc_limit(&self, target_opstamp: u64) -> DocId {
         match *self {
             DocToOpstampMapping::WithMap(ref doc_opstamps) => {
                 match doc_opstamps.binary_search(&target_opstamp) {
@@ -64,18 +64,17 @@ mod tests {
     #[test]
     fn test_doc_to_opstamp_mapping_complex() {
         {
-            let doc_to_opstamp_mapping = DocToOpstampMapping::from(&[][..]);
+            let doc_to_opstamp_mapping = DocToOpstampMapping::from(vec![]);
             assert_eq!(doc_to_opstamp_mapping.compute_doc_limit(0u64), 0);
             assert_eq!(doc_to_opstamp_mapping.compute_doc_limit(2u64), 0);
         }
         {
-            let doc_to_opstamp_mapping = DocToOpstampMapping::from(&[1u64][..]);
+            let doc_to_opstamp_mapping = DocToOpstampMapping::from(vec![1u64]);
             assert_eq!(doc_to_opstamp_mapping.compute_doc_limit(0u64), 0);
             assert_eq!(doc_to_opstamp_mapping.compute_doc_limit(2u64), 1);
         }
         {
-            let doc_to_opstamp_mapping =
-                DocToOpstampMapping::from(&[1u64, 12u64, 17u64, 23u64][..]);
+            let doc_to_opstamp_mapping = DocToOpstampMapping::from(vec![1u64, 12u64, 17u64, 23u64]);
             assert_eq!(doc_to_opstamp_mapping.compute_doc_limit(0u64), 0);
             for i in 2u64..13u64 {
                 assert_eq!(doc_to_opstamp_mapping.compute_doc_limit(i), 1);

src/indexer/log_merge_policy.rs

@@ -1,19 +1,17 @@
 use super::merge_policy::{MergeCandidate, MergePolicy};
-use crate::core::SegmentMeta;
+use core::SegmentMeta;
 use std::cmp;
 use std::f64;
 
 const DEFAULT_LEVEL_LOG_SIZE: f64 = 0.75;
 const DEFAULT_MIN_LAYER_SIZE: u32 = 10_000;
 const DEFAULT_MIN_MERGE_SIZE: usize = 8;
-const DEFAULT_MAX_MERGE_SIZE: usize = 10_000_000;
 
 /// `LogMergePolicy` tries tries to merge segments that have a similar number of
 /// documents.
 #[derive(Debug, Clone)]
 pub struct LogMergePolicy {
     min_merge_size: usize,
-    max_merge_size: usize,
     min_layer_size: u32,
     level_log_size: f64,
 }
@@ -28,12 +26,6 @@ impl LogMergePolicy {
         self.min_merge_size = min_merge_size;
     }
 
-    /// Set the maximum number docs in a segment for it to be considered for
-    /// merging.
-    pub fn set_max_merge_size(&mut self, max_merge_size: usize) {
-        self.max_merge_size = max_merge_size;
-    }
-
     /// Set the minimum segment size under which all segment belong
     /// to the same level.
     pub fn set_min_layer_size(&mut self, min_layer_size: u32) {
@@ -54,44 +46,39 @@ impl LogMergePolicy {
 
 impl MergePolicy for LogMergePolicy {
     fn compute_merge_candidates(&self, segments: &[SegmentMeta]) -> Vec<MergeCandidate> {
+        if segments.is_empty() {
+            return Vec::new();
+        }
+
         let mut size_sorted_tuples = segments
             .iter()
-            .map(SegmentMeta::num_docs)
-            .filter(|s| s <= &(self.max_merge_size as u32))
+            .map(|x| x.num_docs())
             .enumerate()
             .collect::<Vec<(usize, u32)>>();
 
         size_sorted_tuples.sort_by(|x, y| y.1.cmp(&(x.1)));
 
-        if size_sorted_tuples.len() <= 1 {
-            return Vec::new();
-        }
-
         let size_sorted_log_tuples: Vec<_> = size_sorted_tuples
             .into_iter()
             .map(|(ind, num_docs)| (ind, f64::from(self.clip_min_size(num_docs)).log2()))
             .collect();
 
-        if let Some(&(first_ind, first_score)) = size_sorted_log_tuples.first() {
-            let mut current_max_log_size = first_score;
-            let mut levels = vec![vec![first_ind]];
-            for &(ind, score) in (&size_sorted_log_tuples).iter().skip(1) {
-                if score < (current_max_log_size - self.level_log_size) {
-                    current_max_log_size = score;
-                    levels.push(Vec::new());
-                }
-                levels.last_mut().unwrap().push(ind);
+        let (first_ind, first_score) = size_sorted_log_tuples[0];
+        let mut current_max_log_size = first_score;
+        let mut levels = vec![vec![first_ind]];
+        for &(ind, score) in (&size_sorted_log_tuples).iter().skip(1) {
+            if score < (current_max_log_size - self.level_log_size) {
+                current_max_log_size = score;
+                levels.push(Vec::new());
             }
-            levels
-                .iter()
-                .filter(|level| level.len() >= self.min_merge_size)
-                .map(|ind_vec| {
-                    MergeCandidate(ind_vec.iter().map(|&ind| segments[ind].id()).collect())
-                })
-                .collect()
-        } else {
-            return vec![];
+            levels.last_mut().unwrap().push(ind);
         }
+
+        levels
+            .iter()
+            .filter(|level| level.len() >= self.min_merge_size)
+            .map(|ind_vec| MergeCandidate(ind_vec.iter().map(|&ind| segments[ind].id()).collect()))
+            .collect()
     }
 }
 
@@ -99,7 +86,6 @@ impl Default for LogMergePolicy {
     fn default() -> LogMergePolicy {
         LogMergePolicy {
             min_merge_size: DEFAULT_MIN_MERGE_SIZE,
-            max_merge_size: DEFAULT_MAX_MERGE_SIZE,
             min_layer_size: DEFAULT_MIN_LAYER_SIZE,
             level_log_size: DEFAULT_LEVEL_LOG_SIZE,
         }
@@ -109,16 +95,12 @@ impl Default for LogMergePolicy {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::core::{SegmentId, SegmentMeta, SegmentMetaInventory};
-    use crate::indexer::merge_policy::MergePolicy;
-    use once_cell::sync::Lazy;
-
-    static INVENTORY: Lazy<SegmentMetaInventory> = Lazy::new(SegmentMetaInventory::default);
+    use core::{SegmentId, SegmentMeta};
+    use indexer::merge_policy::MergePolicy;
 
     fn test_merge_policy() -> LogMergePolicy {
         let mut log_merge_policy = LogMergePolicy::default();
         log_merge_policy.set_min_merge_size(3);
-        log_merge_policy.set_max_merge_size(100_000);
         log_merge_policy.set_min_layer_size(2);
         log_merge_policy
     }
@@ -131,7 +113,7 @@ mod tests {
     }
 
     fn create_random_segment_meta(num_docs: u32) -> SegmentMeta {
-        INVENTORY.new_segment_meta(SegmentId::generate_random(), num_docs)
+        SegmentMeta::new(SegmentId::generate_random(), num_docs)
     }
 
     #[test]
@@ -156,11 +138,11 @@ mod tests {
             create_random_segment_meta(10),
             create_random_segment_meta(10),
             create_random_segment_meta(10),
-            create_random_segment_meta(1_000),
-            create_random_segment_meta(1_000),
-            create_random_segment_meta(1_000),
-            create_random_segment_meta(10_000),
-            create_random_segment_meta(10_000),
+            create_random_segment_meta(1000),
+            create_random_segment_meta(1000),
+            create_random_segment_meta(1000),
+            create_random_segment_meta(10000),
+            create_random_segment_meta(10000),
             create_random_segment_meta(10),
             create_random_segment_meta(10),
             create_random_segment_meta(10),
@@ -183,7 +165,6 @@ mod tests {
         let result_list = test_merge_policy().compute_merge_candidates(&test_input);
         assert_eq!(result_list.len(), 2);
     }
-
     #[test]
     fn test_log_merge_policy_small_segments() {
         // segments under min_layer_size are merged together
@@ -198,30 +179,4 @@ mod tests {
         let result_list = test_merge_policy().compute_merge_candidates(&test_input);
         assert_eq!(result_list.len(), 1);
     }
-
-    #[test]
-    fn test_log_merge_policy_all_segments_too_large_to_merge() {
-        let eight_large_segments: Vec<SegmentMeta> =
-            std::iter::repeat_with(|| create_random_segment_meta(100_001))
-                .take(8)
-                .collect();
-        assert!(test_merge_policy()
-            .compute_merge_candidates(&eight_large_segments)
-            .is_empty());
-    }
-
-    #[test]
-    fn test_large_merge_segments() {
-        let test_input = vec![
-            create_random_segment_meta(1_000_000),
-            create_random_segment_meta(100_001),
-            create_random_segment_meta(100_000),
-            create_random_segment_meta(100_000),
-            create_random_segment_meta(100_000),
-        ];
-        let result_list = test_merge_policy().compute_merge_candidates(&test_input);
-        // Do not include large segments
-        assert_eq!(result_list.len(), 1);
-        assert_eq!(result_list[0].0.len(), 3)
-    }
 }

src/indexer/merge_operation.rs

@@ -1,74 +0,0 @@
-use crate::Opstamp;
-use crate::SegmentId;
-use census::{Inventory, TrackedObject};
-use std::collections::HashSet;
-use std::ops::Deref;
-
-#[derive(Default)]
-pub(crate) struct MergeOperationInventory(Inventory<InnerMergeOperation>);
-
-impl Deref for MergeOperationInventory {
-    type Target = Inventory<InnerMergeOperation>;
-
-    fn deref(&self) -> &Self::Target {
-        &self.0
-    }
-}
-
-impl MergeOperationInventory {
-    pub fn segment_in_merge(&self) -> HashSet<SegmentId> {
-        let mut segment_in_merge = HashSet::default();
-        for merge_op in self.list() {
-            for &segment_id in &merge_op.segment_ids {
-                segment_in_merge.insert(segment_id);
-            }
-        }
-        segment_in_merge
-    }
-}
-
-/// A `MergeOperation` has two roles.
-/// It carries all of the information required to describe a merge:
-/// - `target_opstamp` is the opstamp up to which we want to consume the
-/// delete queue and reflect their deletes.
-/// - `segment_ids` is the list of segment to be merged.
-///
-/// The second role is to ensure keep track of the fact that these
-/// segments are in merge and avoid starting a merge operation that
-/// may conflict with this one.
-///
-/// This works by tracking merge operations. When considering computing
-/// merge candidates, we simply list tracked merge operations and remove
-/// their segments from possible merge candidates.
-pub struct MergeOperation {
-    inner: TrackedObject<InnerMergeOperation>,
-}
-
-pub(crate) struct InnerMergeOperation {
-    target_opstamp: Opstamp,
-    segment_ids: Vec<SegmentId>,
-}
-
-impl MergeOperation {
-    pub(crate) fn new(
-        inventory: &MergeOperationInventory,
-        target_opstamp: Opstamp,
-        segment_ids: Vec<SegmentId>,
-    ) -> MergeOperation {
-        let inner_merge_operation = InnerMergeOperation {
-            target_opstamp,
-            segment_ids,
-        };
-        MergeOperation {
-            inner: inventory.track(inner_merge_operation),
-        }
-    }
-
-    pub fn target_opstamp(&self) -> Opstamp {
-        self.inner.target_opstamp
-    }
-
-    pub fn segment_ids(&self) -> &[SegmentId] {
-        &self.inner.segment_ids[..]
-    }
-}

src/indexer/merge_policy.rs

@@ -1,5 +1,5 @@
-use crate::core::SegmentId;
-use crate::core::SegmentMeta;
+use core::SegmentId;
+use core::SegmentMeta;
 use std::fmt::Debug;
 use std::marker;
 
@@ -11,7 +11,7 @@ pub struct MergeCandidate(pub Vec<SegmentId>);
 ///
 /// Every time a the list of segments changes, the segment updater
 /// asks the merge policy if some segments should be merged.
-pub trait MergePolicy: marker::Send + marker::Sync + Debug {
+pub trait MergePolicy: MergePolicyClone + marker::Send + marker::Sync + Debug {
     /// Given the list of segment metas, returns the list of merge candidates.
     ///
     /// This call happens on the segment updater thread, and will block
@@ -19,6 +19,21 @@ pub trait MergePolicy: marker::Send + marker::Sync + Debug {
     fn compute_merge_candidates(&self, segments: &[SegmentMeta]) -> Vec<MergeCandidate>;
 }
 
+/// MergePolicyClone
+pub trait MergePolicyClone {
+    /// Returns a boxed clone of the MergePolicy.
+    fn box_clone(&self) -> Box<MergePolicy>;
+}
+
+impl<T> MergePolicyClone for T
+where
+    T: 'static + MergePolicy + Clone,
+{
+    fn box_clone(&self) -> Box<MergePolicy> {
+        Box::new(self.clone())
+    }
+}
+
 /// Never merge segments.
 #[derive(Debug, Clone)]
 pub struct NoMergePolicy;
@@ -39,8 +54,8 @@ impl MergePolicy for NoMergePolicy {
 pub mod tests {
 
     use super::*;
-    use crate::core::SegmentId;
-    use crate::core::SegmentMeta;
+    use core::SegmentId;
+    use core::SegmentMeta;
 
     /// `MergePolicy` useful for test purposes.
     ///