rebasing, fixing some test

Updates the requirements on [once_cell](https://github.com/matklad/once_cell) to permit the latest version.
- [Release notes](https://github.com/matklad/once_cell/releases)
- [Changelog](https://github.com/matklad/once_cell/blob/master/CHANGELOG.md)
- [Commits](https://github.com/matklad/once_cell/compare/v0.2.0...v1.0.2)

Signed-off-by: dependabot-preview[bot] <support@dependabot.com>

.travis.yml

@@ -47,6 +47,7 @@ matrix:
 before_install:
   - set -e
   - rustup self update
+  - rustup component add rustfmt
 
 install:
   - sh ci/install.sh
@@ -60,6 +61,7 @@ before_script:
 
 script:
   - bash ci/script.sh
+  - cargo fmt --all -- --check
 
 before_deploy:
   - sh ci/before_deploy.sh

CHANGELOG.md

@@ -5,7 +5,15 @@ Tantivy 0.11.0
 - Various bugfixes in the query parser.
     - Better handling of hyphens in query parser. (#609)
     - Better handling of whitespaces.
+- Closes #498 - add support for Elastic-style unbounded range queries for alphanumeric types eg. "title:>hello", "weight:>=70.5", "height:<200" (@petr-tik)
+- API change around `Box<BoxableTokenizer>`. See detail in #629
+- Avoid rebuilding Regex automaton whenever a regex query is reused. #630 (@brainlock)
+ 
+## How to update?
 
+- `Box<dyn BoxableTokenizer>` has been replaced by a `BoxedTokenizer` struct.
+- Regex are now compiled when the `RegexQuery` instance is built. As a result, it can now return
+an error and handling the `Result` is required.
 
 Tantivy 0.10.1
 =====================

Cargo.toml

@@ -15,7 +15,7 @@ edition = "2018"
 [dependencies]
 base64 = "0.10.0"
 byteorder = "1.0"
-once_cell = "0.2"
+once_cell = "1.0"
 regex = "1.0"
 tantivy-fst = "0.1"
 memmap = {version = "0.7", optional=true}
@@ -54,6 +54,10 @@ murmurhash32 = "0.2"
 chrono = "0.4"
 smallvec = "0.6"
 
+tantivy-schema = {path= "./tantivy-schema"}
+tantivy-tokenizer = {path= "./tantivy-tokenizer"}
+tantivy-common = {path="./tantivy-common"}
+
 [target.'cfg(windows)'.dependencies]
 winapi = "0.3"
 
@@ -62,7 +66,6 @@ rand = "0.7"
 maplit = "1"
 matches = "0.1.8"
 time = "0.1.42"
-criterion = "0.2"
 
 [profile.release]
 opt-level = 3
@@ -75,7 +78,6 @@ overflow-checks = true
 
 [features]
 default = ["mmap"]
-forbench = []
 mmap = ["atomicwrites", "fs2", "memmap", "notify"]
 lz4-compression = ["lz4"]
 failpoints = ["fail/failpoints"]
@@ -100,14 +102,6 @@ name = "failpoints"
 path = "tests/failpoints/mod.rs"
 required-features = ["fail/failpoints"]
 
-[profile.bench]
-lto = true
 
-[[bench]]
-name = "vint"
-harness = false
-
-
-[[bench]]
-name = "fastfield"
-harness = false
+[workspace]
+members = ["tantivy-schema", "tantivy-common", "tantivy-tokenizer"]

benches/bitset.rs

@@ -1,73 +0,0 @@
-use criterion::{criterion_group, criterion_main, Criterion};
-use rand::distributions::{Bernoulli, Uniform};
-use rand::rngs::StdRng;
-use rand::{Rng, SeedableRng};
-use tantivy::forbench::bitset::{BitSet, TinySet};
-use tantivy::query::BitSetDocSet;
-use tantivy::DocSet;
-
-fn sample_with_seed(n: u32, ratio: f64, seed_val: u8) -> Vec<u32> {
-    StdRng::from_seed([seed_val; 32])
-        .sample_iter(&Bernoulli::new(ratio).unwrap())
-        .take(n as usize)
-        .enumerate()
-        .filter_map(|(val, keep)| if keep { Some(val as u32) } else { None })
-        .collect()
-}
-
-fn generate_nonunique_unsorted(max_value: u32, n_elems: usize) -> Vec<u32> {
-    let seed: [u8; 32] = [1; 32];
-    StdRng::from_seed(seed)
-        .sample_iter(&Uniform::new(0u32, max_value))
-        .take(n_elems)
-        .collect::<Vec<u32>>()
-}
-
-fn bench_tinyset_pop(criterion: &mut Criterion) {
-    criterion.bench_function("pop_lowest", |b| {
-        b.iter(|| {
-            let mut tinyset = TinySet::singleton(criterion::black_box(31u32));
-            tinyset.pop_lowest();
-            tinyset.pop_lowest();
-            tinyset.pop_lowest();
-            tinyset.pop_lowest();
-            tinyset.pop_lowest();
-            tinyset.pop_lowest();
-        })
-    });
-}
-
-fn bench_bitset_insert(criterion: &mut Criterion) {
-    criterion.bench_function_over_inputs(
-        "bitset_insert",
-        |bench, (max_value, n_elems)| {
-            let els = generate_nonunique_unsorted(*max_value, *n_elems);
-            bench.iter(move || {
-                let mut bitset = BitSet::with_max_value(1_000_000);
-                for el in els.iter().cloned() {
-                    bitset.insert(el);
-                }
-            });
-        },
-        vec![(1_000_000u32, 10_000)],
-    );
-}
-
-fn bench_bitsetdocset_iterate(b: &mut test::Bencher) {
-    let mut bitset = BitSet::with_max_value(1_000_000);
-    for el in sample_with_seed(1_000_000u32, 0.01, 0u8) {
-        bitset.insert(el);
-    }
-    b.iter(|| {
-        let mut docset = BitSetDocSet::from(bitset.clone());
-        while docset.advance() {}
-    });
-}
-
-criterion_group!(
-    benches,
-    bench_tinyset_pop,
-    bench_bitset_insert,
-    bench_bitsetdocset_iterate
-);
-criterion_main!(benches);

benches/fastfield.rs

@@ -1,107 +0,0 @@
-use criterion::criterion_group;
-use criterion::criterion_main;
-use criterion::Criterion;
-use criterion::ParameterizedBenchmark;
-use rand::rngs::StdRng;
-use rand::seq::SliceRandom;
-use rand::SeedableRng;
-use tantivy::schema::{Schema, FAST};
-use tantivy::{doc, DocId, Index};
-
-const NUM_LOOKUPS: usize = 1_000;
-
-fn generate_permutation(stride: usize, bit_width: u8) -> Vec<u64> {
-    let mut permutation: Vec<u64> = (0u64..(NUM_LOOKUPS * stride) as u64).collect();
-    permutation.shuffle(&mut StdRng::from_seed([1u8; 32]));
-    permutation.push(1u64 << (bit_width as u64)); //< just to force the bit_width
-    permutation
-}
-
-fn bench_linear_lookup(c: &mut Criterion) {
-    c.bench(
-        "lookup_stride",
-        ParameterizedBenchmark::new(
-            "baseline_vec",
-            |bench, (stride, num_bits)| {
-                let arr = generate_permutation(*stride, *num_bits);
-                bench.iter(move || {
-                    let mut a = 0u64;
-                    for i in (0..NUM_LOOKUPS / stride).map(|v| v * 7) {
-                        a ^= arr[i as usize];
-                    }
-                    a
-                })
-            },
-            vec![(7, 1), (7, 5), (7, 20)],
-        )
-        .with_function("fastfield", |bench, (stride, num_bits)| {
-            let mut schema_builder = Schema::builder();
-            let val_field = schema_builder.add_u64_field("val", FAST);
-            let schema = schema_builder.build();
-
-            let index = Index::create_in_ram(schema);
-            let mut index_writer = index.writer_with_num_threads(1, 80_000_000).unwrap();
-            for el in generate_permutation(*stride, *num_bits) {
-                index_writer.add_document(doc!(val_field=>el));
-            }
-            index_writer.commit().unwrap();
-            let reader = index.reader().unwrap();
-            let searcher = reader.searcher();
-            let segment_reader = searcher.segment_reader(0u32);
-            let fast_field_reader = segment_reader.fast_fields().u64(val_field).unwrap();
-            bench.iter(move || {
-                let mut a = 0u64;
-                for i in (0..NUM_LOOKUPS / stride).map(|v| v * 7) {
-                    a ^= fast_field_reader.get(i as DocId);
-                }
-                a
-            })
-        }),
-    );
-}
-
-fn bench_jumpy_lookup(c: &mut Criterion) {
-    c.bench(
-        "lookup_jumpy",
-        ParameterizedBenchmark::new(
-            "baseline_vec",
-            |bench, (stride, num_bits)| {
-                let arr = generate_permutation(*stride, *num_bits);
-                bench.iter(move || {
-                    let mut a = 0u64;
-                    for _ in 0..NUM_LOOKUPS {
-                        a = arr[a as usize];
-                    }
-                    a
-                })
-            },
-            vec![(7, 1), (7, 5), (7, 20)],
-        )
-        .with_function("fastfield", |bench, (stride, num_bits)| {
-            let mut schema_builder = Schema::builder();
-            let val_field = schema_builder.add_u64_field("val", FAST);
-            let schema = schema_builder.build();
-
-            let index = Index::create_in_ram(schema);
-            let mut index_writer = index.writer_with_num_threads(1, 80_000_000).unwrap();
-            for el in generate_permutation(*stride, *num_bits) {
-                index_writer.add_document(doc!(val_field=>el));
-            }
-            index_writer.commit().unwrap();
-            let reader = index.reader().unwrap();
-            let searcher = reader.searcher();
-            let segment_reader = searcher.segment_reader(0u32);
-            let fast_field_reader = segment_reader.fast_fields().u64(val_field).unwrap();
-            bench.iter(move || {
-                let mut a = 0u64;
-                for _ in 0..NUM_LOOKUPS {
-                    a = fast_field_reader.get(a as DocId);
-                }
-                a
-            })
-        }),
-    );
-}
-
-criterion_group!(benches, bench_linear_lookup, bench_jumpy_lookup);
-criterion_main!(benches);

benches/union.rs

@@ -1,50 +0,0 @@
-use criterion::{criterion_group, criterion_main, Criterion};
-use rand::rngs::StdRng;
-use rand::{Rng, SeedableRng};
-use tantivy::query::QueryParser;
-use tantivy::schema::{Schema, STRING};
-use tantivy::{Document, Index};
-
-fn bench_union(criterion: &mut Criterion) {
-    criterion.bench_function_over_inputs(
-        "union_docset_fulladvance",
-        |bench, (ratio_left, ratio_right)| {
-            let mut schema_builder = Schema::builder();
-            let field = schema_builder.add_text_field("val", STRING);
-            let schema = schema_builder.build();
-            let index = Index::create_in_ram(schema);
-            let mut index_writer = index.writer_with_num_threads(1, 80_000_000).unwrap();
-            let mut stdrng = StdRng::from_seed([0u8; 32]);
-            for _ in 0u32..100_000u32 {
-                let mut doc = Document::default();
-                if stdrng.gen_bool(*ratio_left) {
-                    doc.add_text(field, "left");
-                }
-                if stdrng.gen_bool(*ratio_right) {
-                    doc.add_text(field, "right");
-                }
-                index_writer.add_document(doc);
-            }
-            index_writer.commit().unwrap();
-            let reader = index.reader().unwrap();
-            let searcher = reader.searcher();
-
-            let query = QueryParser::for_index(&index, vec![field])
-                .parse_query("left right")
-                .unwrap();
-
-            bench.iter(move || {
-                let weight = query.weight(&searcher, false).unwrap();
-                let mut scorer = weight.scorer(searcher.segment_reader(0u32)).unwrap();
-                let mut sum_docs = 0u64;
-                scorer.for_each(&mut |doc_id, _score| {
-                    sum_docs += doc_id as u64;
-                });
-            });
-        },
-        vec![(0.2, 0.1), (0.2, 0.02)],
-    );
-}
-
-criterion_group!(benches, bench_union);
-criterion_main!(benches);

benches/vint.rs

@@ -1,72 +0,0 @@
-use criterion::{criterion_group, criterion_main, Criterion, ParameterizedBenchmark};
-use rand::rngs::StdRng;
-use rand::Rng;
-use rand::SeedableRng;
-use tantivy::forbench::compression::{compressed_block_size, BlockDecoder};
-use tantivy::forbench::compression::{BlockEncoder, VIntEncoder};
-use tantivy::forbench::compression::{VIntDecoder, COMPRESSION_BLOCK_SIZE};
-
-fn generate_array_with_seed(n: usize, ratio: f64, seed_val: u8) -> Vec<u32> {
-    let seed: [u8; 32] = [seed_val; 32];
-    let mut rng = StdRng::from_seed(seed);
-    (0u32..).filter(|_| rng.gen_bool(ratio)).take(n).collect()
-}
-
-pub fn generate_array(n: usize, ratio: f64) -> Vec<u32> {
-    generate_array_with_seed(n, ratio, 4)
-}
-
-fn bench_compress(criterion: &mut Criterion) {
-    criterion.bench(
-        "compress_sorted",
-        ParameterizedBenchmark::new(
-            "bitpack",
-            |bench, ratio| {
-                let mut encoder = BlockEncoder::new();
-                let data = generate_array(COMPRESSION_BLOCK_SIZE, *ratio);
-                bench.iter(|| {
-                    encoder.compress_block_sorted(&data, 0u32);
-                });
-            },
-            vec![0.1],
-        )
-        .with_function("vint", |bench, ratio| {
-            let mut encoder = BlockEncoder::new();
-            let data = generate_array(COMPRESSION_BLOCK_SIZE, *ratio);
-            bench.iter(|| {
-                encoder.compress_vint_sorted(&data, 0u32);
-            });
-        }),
-    );
-}
-
-fn bench_uncompress(criterion: &mut Criterion) {
-    criterion.bench(
-        "uncompress_sorted",
-        ParameterizedBenchmark::new(
-            "bitpack",
-            |bench, ratio| {
-                let mut encoder = BlockEncoder::new();
-                let data = generate_array(COMPRESSION_BLOCK_SIZE, *ratio);
-                let (num_bits, compressed) = encoder.compress_block_sorted(&data, 0u32);
-                let mut decoder = BlockDecoder::new();
-                bench.iter(|| {
-                    decoder.uncompress_block_sorted(compressed, 0u32, num_bits);
-                });
-            },
-            vec![0.1],
-        )
-        .with_function("vint", |bench, ratio| {
-            let mut encoder = BlockEncoder::new();
-            let data = generate_array(COMPRESSION_BLOCK_SIZE, *ratio);
-            let compressed = encoder.compress_vint_sorted(&data, 0u32);
-            let mut decoder = BlockDecoder::new();
-            bench.iter(move || {
-                decoder.uncompress_vint_sorted(compressed, 0u32, COMPRESSION_BLOCK_SIZE);
-            });
-        }),
-    );
-}
-
-criterion_group!(benches, bench_compress, bench_uncompress);
-criterion_main!(benches);

examples/basic_search.rs

@@ -5,20 +5,17 @@
 //
 // We will :
 // - define our schema
-// = create an index in a directory
-// - index few documents in our index
-// - search for the best document matchings "sea whale"
-// - retrieve the best document original content.
+// - create an index in a directory
+// - index a few documents into our index
+// - search for the best document matching a basic query
+// - retrieve the best document's original content.
 
 // ---
 // Importing tantivy...
-#[macro_use]
-extern crate tantivy;
 use tantivy::collector::TopDocs;
 use tantivy::query::QueryParser;
 use tantivy::schema::*;
-use tantivy::Index;
-use tantivy::ReloadPolicy;
+use tantivy::{doc, Index, ReloadPolicy};
 use tempfile::TempDir;
 
 fn main() -> tantivy::Result<()> {
@@ -33,7 +30,7 @@ fn main() -> tantivy::Result<()> {
     // and for each field, its type and "the way it should
     // be indexed".
 
-    // first we need to define a schema ...
+    // First we need to define a schema ...
     let mut schema_builder = Schema::builder();
 
     // Our first field is title.
@@ -48,7 +45,7 @@ fn main() -> tantivy::Result<()> {
     //
     // `STORED` means that the field will also be saved
     // in a compressed, row-oriented key-value store.
-    // This store is useful to reconstruct the
+    // This store is useful for reconstructing the
     // documents that were selected during the search phase.
     schema_builder.add_text_field("title", TEXT | STORED);
 
@@ -57,8 +54,7 @@ fn main() -> tantivy::Result<()> {
     // need to be able to be able to retrieve it
     // for our application.
     //
-    // We can make our index lighter and
-    // by omitting `STORED` flag.
+    // We can make our index lighter by omitting the `STORED` flag.
     schema_builder.add_text_field("body", TEXT);
 
     let schema = schema_builder.build();
@@ -71,7 +67,7 @@ fn main() -> tantivy::Result<()> {
     // with our schema in the directory.
     let index = Index::create_in_dir(&index_path, schema.clone())?;
 
-    // To insert document we need an index writer.
+    // To insert a document we will need an index writer.
     // There must be only one writer at a time.
     // This single `IndexWriter` is already
     // multithreaded.
@@ -149,8 +145,8 @@ fn main() -> tantivy::Result<()> {
     // At this point our documents are not searchable.
     //
     //
-    // We need to call .commit() explicitly to force the
-    // index_writer to finish processing the documents in the queue,
+    // We need to call `.commit()` explicitly to force the
+    // `index_writer` to finish processing the documents in the queue,
     // flush the current index to the disk, and advertise
     // the existence of new documents.
     //
@@ -162,14 +158,14 @@ fn main() -> tantivy::Result<()> {
     // persistently indexed.
     //
     // In the scenario of a crash or a power failure,
-    // tantivy behaves as if has rolled back to its last
+    // tantivy behaves as if it has rolled back to its last
     // commit.
 
     // # Searching
     //
     // ### Searcher
     //
-    // A reader is required to get search the index.
+    // A reader is required first in order to search an index.
     // It acts as a `Searcher` pool that reloads itself,
     // depending on a `ReloadPolicy`.
     //
@@ -185,7 +181,7 @@ fn main() -> tantivy::Result<()> {
 
     // We now need to acquire a searcher.
     //
-    // A searcher points to snapshotted, immutable version of the index.
+    // A searcher points to a snapshotted, immutable version of the index.
     //
     // Some search experience might require more than
     // one query. Using the same searcher ensures that all of these queries will run on the
@@ -205,7 +201,7 @@ fn main() -> tantivy::Result<()> {
     // in both title and body.
     let query_parser = QueryParser::for_index(&index, vec![title, body]);
 
-    // QueryParser may fail if the query is not in the right
+    // `QueryParser` may fail if the query is not in the right
     // format. For user facing applications, this can be a problem.
     // A ticket has been opened regarding this problem.
     let query = query_parser.parse_query("sea whale")?;
@@ -221,7 +217,7 @@ fn main() -> tantivy::Result<()> {
     //
     // We are not interested in all of the documents but
     // only in the top 10. Keeping track of our top 10 best documents
-    // is the role of the TopDocs.
+    // is the role of the `TopDocs` collector.
 
     // We can now perform our query.
     let top_docs = searcher.search(&query, &TopDocs::with_limit(10))?;

examples/custom_collector.rs

@@ -9,15 +9,12 @@
 
 // ---
 // Importing tantivy...
-#[macro_use]
-extern crate tantivy;
 use tantivy::collector::{Collector, SegmentCollector};
 use tantivy::fastfield::FastFieldReader;
 use tantivy::query::QueryParser;
 use tantivy::schema::Field;
 use tantivy::schema::{Schema, FAST, INDEXED, TEXT};
-use tantivy::SegmentReader;
-use tantivy::{Index, TantivyError};
+use tantivy::{doc, Index, SegmentReader, TantivyError};
 
 #[derive(Default)]
 struct Stats {

examples/custom_tokenizer.rs

@@ -2,14 +2,11 @@
 //
 // In this example, we'll see how to define a tokenizer pipeline
 // by aligning a bunch of `TokenFilter`.
-
-#[macro_use]
-extern crate tantivy;
 use tantivy::collector::TopDocs;
 use tantivy::query::QueryParser;
 use tantivy::schema::*;
 use tantivy::tokenizer::NgramTokenizer;
-use tantivy::Index;
+use tantivy::{doc, Index};
 
 fn main() -> tantivy::Result<()> {
     // # Defining the schema

examples/deleting_updating_documents.rs

@@ -8,13 +8,10 @@
 //
 // ---
 // Importing tantivy...
-#[macro_use]
-extern crate tantivy;
 use tantivy::collector::TopDocs;
 use tantivy::query::TermQuery;
 use tantivy::schema::*;
-use tantivy::Index;
-use tantivy::IndexReader;
+use tantivy::{doc, Index, IndexReader};
 
 // A simple helper function to fetch a single document
 // given its id from our index.

examples/faceted_search.rs

@@ -12,12 +12,10 @@
 
 // ---
 // Importing tantivy...
-#[macro_use]
-extern crate tantivy;
 use tantivy::collector::FacetCollector;
 use tantivy::query::AllQuery;
 use tantivy::schema::*;
-use tantivy::Index;
+use tantivy::{doc, Index};
 use tempfile::TempDir;
 
 fn main() -> tantivy::Result<()> {

examples/integer_range_search.rs

@@ -2,14 +2,10 @@
 //
 // Below is an example of creating an indexed integer field in your schema
 // You can use RangeQuery to get a Count of all occurrences in a given range.
-
-#[macro_use]
-extern crate tantivy;
 use tantivy::collector::Count;
 use tantivy::query::RangeQuery;
 use tantivy::schema::{Schema, INDEXED};
-use tantivy::Index;
-use tantivy::Result;
+use tantivy::{doc, Index, Result};
 
 fn run() -> Result<()> {
     // For the sake of simplicity, this schema will only have 1 field

examples/iterating_docs_and_positions.rs

@@ -9,11 +9,8 @@
 
 // ---
 // Importing tantivy...
-#[macro_use]
-extern crate tantivy;
 use tantivy::schema::*;
-use tantivy::Index;
-use tantivy::{DocId, DocSet, Postings};
+use tantivy::{doc, DocId, DocSet, Index, Postings};
 
 fn main() -> tantivy::Result<()> {
     // We first create a schema for the sake of the

examples/multiple_producer.rs

@@ -25,14 +25,11 @@
 
 // ---
 // Importing tantivy...
-#[macro_use]
-extern crate tantivy;
 use std::sync::{Arc, RwLock};
 use std::thread;
 use std::time::Duration;
 use tantivy::schema::{Schema, STORED, TEXT};
-use tantivy::Opstamp;
-use tantivy::{Index, IndexWriter};
+use tantivy::{doc, Index, IndexWriter, Opstamp};
 
 fn main() -> tantivy::Result<()> {
     // # Defining the schema

examples/snippet.rs

@@ -7,13 +7,10 @@
 
 // ---
 // Importing tantivy...
-#[macro_use]
-extern crate tantivy;
 use tantivy::collector::TopDocs;
 use tantivy::query::QueryParser;
 use tantivy::schema::*;
-use tantivy::Index;
-use tantivy::{Snippet, SnippetGenerator};
+use tantivy::{doc, Index, Snippet, SnippetGenerator};
 use tempfile::TempDir;
 
 fn main() -> tantivy::Result<()> {

examples/stop_words.rs

@@ -11,13 +11,11 @@
 
 // ---
 // Importing tantivy...
-#[macro_use]
-extern crate tantivy;
 use tantivy::collector::TopDocs;
 use tantivy::query::QueryParser;
 use tantivy::schema::*;
 use tantivy::tokenizer::*;
-use tantivy::Index;
+use tantivy::{doc, Index};
 
 fn main() -> tantivy::Result<()> {
     // this example assumes you understand the content in `basic_search`

run-bench.rs

@@ -1,2 +0,0 @@
-#!/usr/bin/env bash
-cargo bench --features forbench 

src/collector/count_collector.rs

@@ -10,12 +10,10 @@ use crate::SegmentReader;
 /// documents match the query.
 ///
 /// ```rust
-/// #[macro_use]
-/// extern crate tantivy;
-/// use tantivy::schema::{Schema, TEXT};
-/// use tantivy::{Index, Result};
 /// use tantivy::collector::Count;
 /// use tantivy::query::QueryParser;
+/// use tantivy::schema::{Schema, TEXT};
+/// use tantivy::{doc, Index, Result};
 ///
 /// # fn main() { example().unwrap(); }
 /// fn example() -> Result<()> {

src/collector/facet_collector.rs

@@ -81,12 +81,10 @@ fn facet_depth(facet_bytes: &[u8]) -> usize {
 ///
 ///
 /// ```rust
-/// #[macro_use]
-/// extern crate tantivy;
-/// use tantivy::schema::{Facet, Schema, TEXT};
-/// use tantivy::{Index, Result};
 /// use tantivy::collector::FacetCollector;
 /// use tantivy::query::AllQuery;
+/// use tantivy::schema::{Facet, Schema, TEXT};
+/// use tantivy::{doc, Index, Result};
 ///
 /// # fn main() { example().unwrap(); }
 /// fn example() -> Result<()> {

src/collector/mod.rs

@@ -35,7 +35,6 @@ The resulting `Fruit` will then be a typed tuple with each collector's original
 in their respective position.
 
 ```rust
-# extern crate tantivy;
 # use tantivy::schema::*;
 # use tantivy::*;
 # use tantivy::query::*;

src/collector/multi_collector.rs

@@ -105,12 +105,10 @@ impl<TFruit: Fruit> FruitHandle<TFruit> {
 /// [Combining several collectors section of the collector documentation](./index.html#combining-several-collectors).
 ///
 /// ```rust
-/// #[macro_use]
-/// extern crate tantivy;
-/// use tantivy::schema::{Schema, TEXT};
-/// use tantivy::{Index, Result};
 /// use tantivy::collector::{Count, TopDocs, MultiCollector};
 /// use tantivy::query::QueryParser;
+/// use tantivy::schema::{Schema, TEXT};
+/// use tantivy::{doc, Index, Result};
 ///
 /// # fn main() { example().unwrap(); }
 /// fn example() -> Result<()> {

src/collector/top_score_collector.rs

@@ -23,13 +23,10 @@ use std::fmt;
 /// is `O(n log K)`.
 ///
 /// ```rust
-/// #[macro_use]
-/// extern crate tantivy;
-/// use tantivy::DocAddress;
-/// use tantivy::schema::{Schema, TEXT};
-/// use tantivy::{Index, Result};
 /// use tantivy::collector::TopDocs;
 /// use tantivy::query::QueryParser;
+/// use tantivy::schema::{Schema, TEXT};
+/// use tantivy::{doc, DocAddress, Index, Result};
 ///
 /// # fn main() { example().unwrap(); }
 /// fn example() -> Result<()> {
@@ -87,10 +84,8 @@ impl TopDocs {
     /// Set top-K to rank documents by a given fast field.
     ///
     /// ```rust
-    /// #[macro_use]
-    /// extern crate tantivy;
     /// # use tantivy::schema::{Schema, FAST, TEXT};
-    /// # use tantivy::{Index, Result, DocAddress};
+    /// # use tantivy::{doc, Index, Result, DocAddress};
     /// # use tantivy::query::{Query, QueryParser};
     /// use tantivy::Searcher;
     /// use tantivy::collector::TopDocs;
@@ -128,7 +123,7 @@ impl TopDocs {
     /// ///
     /// /// `field` is required to be a FAST field.
     /// fn docs_sorted_by_rating(searcher: &Searcher,
-    ///                          query: &Query,
+    ///                          query: &dyn Query,
     ///                          sort_by_field: Field)
     ///     -> Result<Vec<(u64, DocAddress)>> {
     ///
@@ -197,10 +192,8 @@ impl TopDocs {
     /// learning-to-rank model over various features
     ///
     /// ```rust
-    /// #[macro_use]
-    /// extern crate tantivy;
     /// # use tantivy::schema::{Schema, FAST, TEXT};
-    /// # use tantivy::{Index, DocAddress, DocId, Score};
+    /// # use tantivy::{doc, Index, DocAddress, DocId, Score};
     /// # use tantivy::query::QueryParser;
     /// use tantivy::SegmentReader;
     /// use tantivy::collector::TopDocs;
@@ -302,10 +295,8 @@ impl TopDocs {
     /// # Example
     ///
     /// ```rust
-    /// # #[macro_use]
-    /// # extern crate tantivy;
     /// # use tantivy::schema::{Schema, FAST, TEXT};
-    /// # use tantivy::{Index, DocAddress, DocId};
+    /// # use tantivy::{doc, Index, DocAddress, DocId};
     /// # use tantivy::query::QueryParser;
     /// use tantivy::SegmentReader;
     /// use tantivy::collector::TopDocs;

src/core/index.rs

@@ -173,11 +173,11 @@ impl Index {
     }
 
     /// Helper to access the tokenizer associated to a specific field.
-    pub fn tokenizer_for_field(&self, field: Field) -> Result<Box<dyn BoxedTokenizer>> {
+    pub fn tokenizer_for_field(&self, field: Field) -> Result<BoxedTokenizer> {
         let field_entry = self.schema.get_field_entry(field);
         let field_type = field_entry.field_type();
         let tokenizer_manager: &TokenizerManager = self.tokenizers();
-        let tokenizer_name_opt: Option<Box<dyn BoxedTokenizer>> = match field_type {
+        let tokenizer_name_opt: Option<BoxedTokenizer> = match field_type {
             FieldType::Str(text_options) => text_options
                 .get_indexing_options()
                 .map(|text_indexing_options| text_indexing_options.tokenizer().to_string())
@@ -216,8 +216,22 @@ impl Index {
         Index::open(mmap_directory)
     }
 
-    pub(crate) fn inventory(&self) -> &SegmentMetaInventory {
-        &self.inventory
+    /// Returns the list of the segment metas tracked by the index.
+    ///
+    /// Such segments can of course be part of the index,
+    /// but also they could be segments being currently built or in the middle of a merge
+    /// operation.
+    pub fn list_all_segment_metas(&self) -> Vec<SegmentMeta> {
+        self.inventory.all()
+    }
+
+    /// Creates a new segment_meta (Advanced user only).
+    ///
+    /// As long as the `SegmentMeta` lives, the files associated with the
+    /// `SegmentMeta` are guaranteed to not be garbage collected, regardless of
+    /// whether the segment is recorded as part of the index or not.
+    pub fn new_segment_meta(&self, segment_id: SegmentId, max_doc: u32) -> SegmentMeta {
+        self.inventory.new_segment_meta(segment_id, max_doc)
     }
 
     /// Open the index using the provided directory

src/core/index_meta.rs

@@ -30,7 +30,6 @@ impl SegmentMetaInventory {
             .collect::<Vec<_>>()
     }
 
-    #[doc(hidden)]
     pub fn new_segment_meta(&self, segment_id: SegmentId, max_doc: u32) -> SegmentMeta {
         let inner = InnerSegmentMeta {
             segment_id,

src/core/segment_id.rs

@@ -4,6 +4,8 @@ use uuid::Uuid;
 
 #[cfg(test)]
 use once_cell::sync::Lazy;
+use std::error::Error;
+use std::str::FromStr;
 #[cfg(test)]
 use std::sync::atomic;
 
@@ -52,15 +54,51 @@ impl SegmentId {
     /// and the rest is random.
     ///
     /// Picking the first 8 chars is ok to identify
-    /// segments in a display message.
+    /// segments in a display message (e.g. a5c4dfcb).
     pub fn short_uuid_string(&self) -> String {
         (&self.0.to_simple_ref().to_string()[..8]).to_string()
     }
 
     /// Returns a segment uuid string.
+    ///
+    /// It consists in 32 lowercase hexadecimal chars
+    /// (e.g. a5c4dfcbdfe645089129e308e26d5523)
     pub fn uuid_string(&self) -> String {
         self.0.to_simple_ref().to_string()
     }
+
+    /// Build a `SegmentId` string from the full uuid string.
+    ///
+    /// E.g. "a5c4dfcbdfe645089129e308e26d5523"
+    pub fn from_uuid_string(uuid_string: &str) -> Result<SegmentId, SegmentIdParseError> {
+        FromStr::from_str(uuid_string)
+    }
+}
+
+/// Error type used when parsing a `SegmentId` from a string fails.
+pub struct SegmentIdParseError(uuid::parser::ParseError);
+
+impl Error for SegmentIdParseError {}
+
+impl fmt::Debug for SegmentIdParseError {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        self.0.fmt(f)
+    }
+}
+
+impl fmt::Display for SegmentIdParseError {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        self.0.fmt(f)
+    }
+}
+
+impl FromStr for SegmentId {
+    type Err = SegmentIdParseError;
+
+    fn from_str(uuid_string: &str) -> Result<Self, SegmentIdParseError> {
+        let uuid = Uuid::parse_str(uuid_string).map_err(SegmentIdParseError)?;
+        Ok(SegmentId(uuid))
+    }
 }
 
 impl fmt::Debug for SegmentId {
@@ -80,3 +118,18 @@ impl Ord for SegmentId {
         self.0.as_bytes().cmp(other.0.as_bytes())
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::SegmentId;
+
+    #[test]
+    fn test_to_uuid_string() {
+        let full_uuid = "a5c4dfcbdfe645089129e308e26d5523";
+        let segment_id = SegmentId::from_uuid_string(full_uuid).unwrap();
+        assert_eq!(segment_id.uuid_string(), full_uuid);
+        assert_eq!(segment_id.short_uuid_string(), "a5c4dfcb");
+        // one extra char
+        assert!(SegmentId::from_uuid_string("a5c4dfcbdfe645089129e308e26d5523b").is_err());
+    }
+}

src/core/segment_reader.rs

@@ -1,4 +1,3 @@
-use crate::common::CompositeFile;
 use crate::common::HasLen;
 use crate::core::InvertedIndexReader;
 use crate::core::Segment;
@@ -15,6 +14,7 @@ use crate::schema::Schema;
 use crate::space_usage::SegmentSpaceUsage;
 use crate::store::StoreReader;
 use crate::termdict::TermDictionary;
+use crate::CompositeFile;
 use crate::DocId;
 use crate::Result;
 use fail::fail_point;

src/directory/mmap_directory.rs

@@ -539,7 +539,7 @@ impl Directory for MmapDirectory {
 #[cfg(test)]
 mod tests {
 
-    // There are more tests in directory/mod.rs
+    // There are more tests in directory/lib.rs
     // The following tests are specific to the MmapDirectory
 
     use super::*;

src/fastfield/mod.rs

@@ -148,13 +148,13 @@ fn value_to_u64(value: &Value) -> u64 {
 mod tests {
 
     use super::*;
-    use crate::common::CompositeFile;
     use crate::directory::{Directory, RAMDirectory, WritePtr};
     use crate::fastfield::FastFieldReader;
     use crate::schema::Document;
     use crate::schema::Field;
     use crate::schema::Schema;
     use crate::schema::FAST;
+    use crate::CompositeFile;
     use once_cell::sync::Lazy;
     use rand::prelude::SliceRandom;
     use rand::rngs::StdRng;
@@ -431,3 +431,111 @@ mod tests {
     }
 
 }
+
+#[cfg(all(test, feature = "unstable"))]
+mod bench {
+    use super::tests::FIELD;
+    use super::tests::{generate_permutation, SCHEMA};
+    use super::*;
+    use common::CompositeFile;
+    use directory::{Directory, RAMDirectory, WritePtr};
+    use fastfield::FastFieldReader;
+    use std::collections::HashMap;
+    use std::path::Path;
+    use test::{self, Bencher};
+
+    #[bench]
+    fn bench_intfastfield_linear_veclookup(b: &mut Bencher) {
+        let permutation = generate_permutation();
+        b.iter(|| {
+            let n = test::black_box(7000u32);
+            let mut a = 0u64;
+            for i in (0u32..n / 7).map(|v| v * 7) {
+                a ^= permutation[i as usize];
+            }
+            a
+        });
+    }
+
+    #[bench]
+    fn bench_intfastfield_veclookup(b: &mut Bencher) {
+        let permutation = generate_permutation();
+        b.iter(|| {
+            let n = test::black_box(1000u32);
+            let mut a = 0u64;
+            for _ in 0u32..n {
+                a = permutation[a as usize];
+            }
+            a
+        });
+    }
+
+    #[bench]
+    fn bench_intfastfield_linear_fflookup(b: &mut Bencher) {
+        let path = Path::new("test");
+        let permutation = generate_permutation();
+        let mut directory: RAMDirectory = RAMDirectory::create();
+        {
+            let write: WritePtr = directory.open_write(Path::new("test")).unwrap();
+            let mut serializer = FastFieldSerializer::from_write(write).unwrap();
+            let mut fast_field_writers = FastFieldsWriter::from_schema(&SCHEMA);
+            for &x in &permutation {
+                fast_field_writers.add_document(&doc!(*FIELD=>x));
+            }
+            fast_field_writers
+                .serialize(&mut serializer, &HashMap::new())
+                .unwrap();
+            serializer.close().unwrap();
+        }
+        let source = directory.open_read(&path).unwrap();
+        {
+            let fast_fields_composite = CompositeFile::open(&source).unwrap();
+            let data = fast_fields_composite.open_read(*FIELD).unwrap();
+            let fast_field_reader = FastFieldReader::<u64>::open(data);
+
+            b.iter(|| {
+                let n = test::black_box(7000u32);
+                let mut a = 0u64;
+                for i in (0u32..n / 7).map(|val| val * 7) {
+                    a ^= fast_field_reader.get(i);
+                }
+                a
+            });
+        }
+    }
+
+    #[bench]
+    fn bench_intfastfield_fflookup(b: &mut Bencher) {
+        let path = Path::new("test");
+        let permutation = generate_permutation();
+        let mut directory: RAMDirectory = RAMDirectory::create();
+        {
+            let write: WritePtr = directory.open_write(Path::new("test")).unwrap();
+            let mut serializer = FastFieldSerializer::from_write(write).unwrap();
+            let mut fast_field_writers = FastFieldsWriter::from_schema(&SCHEMA);
+            for &x in &permutation {
+                fast_field_writers.add_document(&doc!(*FIELD=>x));
+            }
+            fast_field_writers
+                .serialize(&mut serializer, &HashMap::new())
+                .unwrap();
+            serializer.close().unwrap();
+        }
+        let source = directory.open_read(&path).unwrap();
+        {
+            let fast_fields_composite = CompositeFile::open(&source).unwrap();
+            let data = fast_fields_composite.open_read(*FIELD).unwrap();
+            let fast_field_reader = FastFieldReader::<u64>::open(data);
+
+            b.iter(|| {
+                let n = test::black_box(1000u32);
+                let mut a = 0u32;
+                for _ in 0u32..n {
+                    a = fast_field_reader.get(a) as u32;
+                }
+                a
+            });
+        }
+    }
+
+}

src/fastfield/reader.rs

@@ -2,12 +2,12 @@ use super::FastValue;
 use crate::common::bitpacker::BitUnpacker;
 use crate::common::compute_num_bits;
 use crate::common::BinarySerializable;
-use crate::common::CompositeFile;
 use crate::directory::ReadOnlySource;
 use crate::directory::{Directory, RAMDirectory, WritePtr};
 use crate::fastfield::{FastFieldSerializer, FastFieldsWriter};
 use crate::schema::Schema;
 use crate::schema::FAST;
+use crate::CompositeFile;
 use crate::DocId;
 use owning_ref::OwningRef;
 use std::collections::HashMap;
@@ -67,12 +67,10 @@ impl<Item: FastValue> FastFieldReader<Item> {
     ///
     /// May panic if `doc` is greater than the segment
     // `maxdoc`.
-    #[inline(always)]
     pub fn get(&self, doc: DocId) -> Item {
         self.get_u64(u64::from(doc))
     }
 
-    #[inline(always)]
     pub(crate) fn get_u64(&self, doc: u64) -> Item {
         Item::from_u64(self.min_value_u64 + self.bit_unpacker.get(doc))
     }

src/fastfield/readers.rs

@@ -1,9 +1,9 @@
-use crate::common::CompositeFile;
 use crate::fastfield::BytesFastFieldReader;
 use crate::fastfield::MultiValueIntFastFieldReader;
 use crate::fastfield::{FastFieldNotAvailableError, FastFieldReader};
 use crate::schema::{Cardinality, Field, FieldType, Schema};
 use crate::space_usage::PerFieldSpaceUsage;
+use crate::CompositeFile;
 use crate::Result;
 use std::collections::HashMap;
 

src/fastfield/serializer.rs

@@ -1,10 +1,10 @@
 use crate::common::bitpacker::BitPacker;
 use crate::common::compute_num_bits;
 use crate::common::BinarySerializable;
-use crate::common::CompositeWrite;
 use crate::common::CountingWriter;
 use crate::directory::WritePtr;
 use crate::schema::Field;
+use crate::CompositeWrite;
 use std::io::{self, Write};
 
 /// `FastFieldSerializer` is in charge of serializing

src/fieldnorm/serializer.rs

@@ -1,6 +1,6 @@
-use crate::common::CompositeWrite;
 use crate::directory::WritePtr;
 use crate::schema::Field;
+use crate::CompositeWrite;
 use std::io;
 use std::io::Write;
 

src/indexer/index_writer.rs

@@ -209,10 +209,7 @@ fn index_documents(
     assert!(num_docs > 0);
 
     let doc_opstamps: Vec<Opstamp> = segment_writer.finalize()?;
-    let segment_meta = segment
-        .index()
-        .inventory()
-        .new_segment_meta(segment_id, num_docs);
+    let segment_meta = segment.index().new_segment_meta(segment_id, num_docs);
 
     let last_docstamp: Opstamp = *(doc_opstamps.last().unwrap());
 
@@ -450,12 +447,10 @@ impl IndexWriter {
     /// by clearing and resubmitting necessary documents
     ///
     /// ```rust
-    /// #[macro_use]
-    /// extern crate tantivy;
-    /// use tantivy::query::QueryParser;
     /// use tantivy::collector::TopDocs;
+    /// use tantivy::query::QueryParser;
     /// use tantivy::schema::*;
-    /// use tantivy::Index;
+    /// use tantivy::{doc, Index};
     ///
     /// fn main() -> tantivy::Result<()> {
     ///     let mut schema_builder = Schema::builder();

src/indexer/segment_updater.rs

@@ -126,9 +126,7 @@ fn perform_merge(
 
     let num_docs = merger.write(segment_serializer)?;
 
-    let segment_meta = index
-        .inventory()
-        .new_segment_meta(merged_segment.id(), num_docs);
+    let segment_meta = index.new_segment_meta(merged_segment.id(), num_docs);
 
     let after_merge_segment_entry = SegmentEntry::new(segment_meta.clone(), delete_cursor, None);
     Ok(after_merge_segment_entry)
@@ -282,7 +280,7 @@ impl SegmentUpdater {
     fn list_files(&self) -> HashSet<PathBuf> {
         let mut files = HashSet::new();
         files.insert(META_FILEPATH.to_path_buf());
-        for segment_meta in self.0.index.inventory().all() {
+        for segment_meta in self.0.index.list_all_segment_metas() {
             files.extend(segment_meta.list_files());
         }
         files

src/indexer/segment_writer.rs

@@ -49,7 +49,7 @@ pub struct SegmentWriter {
     fast_field_writers: FastFieldsWriter,
     fieldnorms_writer: FieldNormsWriter,
     doc_opstamps: Vec<Opstamp>,
-    tokenizers: Vec<Option<Box<dyn BoxedTokenizer>>>,
+    tokenizers: Vec<Option<BoxedTokenizer>>,
 }
 
 impl SegmentWriter {

src/lib.rs

@@ -11,21 +11,12 @@
 //! Think `Lucene`, but in Rust.
 //!
 //! ```rust
-
-//! # extern crate tempfile;
-//! #
-//! #[macro_use]
-//! extern crate tantivy;
-//!
-//! // ...
-//!
 //! # use std::path::Path;
 //! # use tempfile::TempDir;
-//! # use tantivy::Index;
-//! # use tantivy::schema::*;
-//! # use tantivy::{Score, DocAddress};
 //! # use tantivy::collector::TopDocs;
 //! # use tantivy::query::QueryParser;
+//! # use tantivy::schema::*;
+//! # use tantivy::{doc, DocAddress, Index, Score};
 //! #
 //! # fn main() {
 //! #     // Let's create a temporary directory for the
@@ -111,9 +102,6 @@
 #[macro_use]
 extern crate serde_derive;
 
-#[cfg_attr(test, macro_use)]
-extern crate serde_json;
-
 #[macro_use]
 extern crate log;
 
@@ -130,6 +118,9 @@ mod functional_test;
 #[macro_use]
 mod macros;
 
+mod composite_file;
+pub(crate) use composite_file::{CompositeFile, CompositeWrite};
+
 pub use crate::error::TantivyError;
 
 #[deprecated(since = "0.7.0", note = "please use `tantivy::TantivyError` instead")]
@@ -142,22 +133,22 @@ pub type Result<T> = std::result::Result<T, error::TantivyError>;
 /// Tantivy DateTime
 pub type DateTime = chrono::DateTime<chrono::Utc>;
 
-mod common;
+pub use tantivy_common as common;
+pub use tantivy_schema as schema;
+pub use tantivy_tokenizer as tokenizer;
+
 mod core;
 mod indexer;
 
-#[allow(unused_doc_comments)]
-mod error;
-pub mod tokenizer;
-
 pub mod collector;
 pub mod directory;
+#[allow(unused_doc_comments)]
+mod error;
 pub mod fastfield;
 pub mod fieldnorm;
 pub(crate) mod positions;
 pub mod postings;
 pub mod query;
-pub mod schema;
 pub mod space_usage;
 pub mod store;
 pub mod termdict;
@@ -249,6 +240,7 @@ pub struct DocAddress(pub SegmentLocalId, pub DocId);
 
 #[cfg(test)]
 mod tests {
+
     use crate::collector::tests::TEST_COLLECTOR_WITH_SCORE;
     use crate::core::SegmentReader;
     use crate::docset::DocSet;
@@ -260,7 +252,6 @@ mod tests {
     use crate::Postings;
     use crate::ReloadPolicy;
     use rand::distributions::Bernoulli;
-    use rand::distributions::Uniform;
     use rand::rngs::StdRng;
     use rand::{Rng, SeedableRng};
 
@@ -277,14 +268,6 @@ mod tests {
         (a - b).abs() < 0.0005 * (a + b).abs()
     }
 
-    pub fn generate_nonunique_unsorted(max_value: u32, n_elems: usize) -> Vec<u32> {
-        let seed: [u8; 32] = [1; 32];
-        StdRng::from_seed(seed)
-            .sample_iter(&Uniform::new(0u32, max_value))
-            .take(n_elems)
-            .collect::<Vec<u32>>()
-    }
-
     pub fn sample_with_seed(n: u32, ratio: f64, seed_val: u8) -> Vec<u32> {
         StdRng::from_seed([seed_val; 32])
             .sample_iter(&Bernoulli::new(ratio).unwrap())
@@ -294,10 +277,6 @@ mod tests {
             .collect()
     }
 
-    pub fn sample(n: u32, ratio: f64) -> Vec<u32> {
-        sample_with_seed(n, ratio, 4)
-    }
-
     #[test]
     #[cfg(feature = "mmap")]
     fn test_indexing() {
@@ -894,15 +873,3 @@ mod tests {
         }
     }
 }
-
-#[cfg(feature = "forbench")]
-pub mod forbench {
-    pub mod compression {
-        pub use crate::postings::compression::*;
-    }
-
-    pub mod bitset {
-        pub use crate::common::BitSet;
-        pub use crate::common::TinySet;
-    }
-}

src/macros.rs

@@ -22,11 +22,9 @@
 ///
 /// # Example
 ///
-/// ```
-/// #[macro_use]
-/// extern crate tantivy;
-///
+/// ```rust
 /// use tantivy::schema::{Schema, TEXT, FAST};
+/// use tantivy::doc;
 ///
 /// //...
 ///

src/postings/compression/mod.rs

@@ -160,9 +160,9 @@ impl VIntEncoder for BlockEncoder {
 }
 
 impl VIntDecoder for BlockDecoder {
-    fn uncompress_vint_sorted(
+    fn uncompress_vint_sorted<'a>(
         &mut self,
-        compressed_data: &[u8],
+        compressed_data: &'a [u8],
         offset: u32,
         num_els: usize,
     ) -> usize {
@@ -170,7 +170,7 @@ impl VIntDecoder for BlockDecoder {
         vint::uncompress_sorted(compressed_data, &mut self.output.0[..num_els], offset)
     }
 
-    fn uncompress_vint_unsorted(&mut self, compressed_data: &[u8], num_els: usize) -> usize {
+    fn uncompress_vint_unsorted<'a>(&mut self, compressed_data: &'a [u8], num_els: usize) -> usize {
         self.output_len = num_els;
         vint::uncompress_unsorted(compressed_data, &mut self.output.0[..num_els])
     }
@@ -268,17 +268,78 @@ pub mod tests {
             }
         }
     }
+}
+
+#[cfg(all(test, feature = "unstable"))]
+mod bench {
+
+    use super::*;
+    use rand::SeedableRng;
+    use rand::{Rng, XorShiftRng};
+    use test::Bencher;
+
+    fn generate_array_with_seed(n: usize, ratio: f64, seed_val: u8) -> Vec<u32> {
+        let seed: &[u8; 16] = &[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, seed_val];
+        let mut rng: XorShiftRng = XorShiftRng::from_seed(*seed);
+        (0u32..).filter(|_| rng.gen_bool(ratio)).take(n).collect()
+    }
+
+    pub fn generate_array(n: usize, ratio: f64) -> Vec<u32> {
+        generate_array_with_seed(n, ratio, 4)
+    }
+
+    #[bench]
+    fn bench_compress(b: &mut Bencher) {
+        let mut encoder = BlockEncoder::new();
+        let data = generate_array(COMPRESSION_BLOCK_SIZE, 0.1);
+        b.iter(|| {
+            encoder.compress_block_sorted(&data, 0u32);
+        });
+    }
+
+    #[bench]
+    fn bench_uncompress(b: &mut Bencher) {
+        let mut encoder = BlockEncoder::new();
+        let data = generate_array(COMPRESSION_BLOCK_SIZE, 0.1);
+        let (num_bits, compressed) = encoder.compress_block_sorted(&data, 0u32);
+        let mut decoder = BlockDecoder::new();
+        b.iter(|| {
+            decoder.uncompress_block_sorted(compressed, 0u32, num_bits);
+        });
+    }
 
     #[test]
     fn test_all_docs_compression_numbits() {
-        for expected_num_bits in 0u8..33u8 {
+        for expected_num_bits in 0u8.. {
             let mut data = [0u32; 128];
             if expected_num_bits > 0 {
-                data[0] = (1u64 << (expected_num_bits as u64) - 1u64) as u32;
+                data[0] = (1u64 << (expected_num_bits as usize) - 1) as u32;
             }
             let mut encoder = BlockEncoder::new();
             let (num_bits, compressed) = encoder.compress_block_unsorted(&data);
             assert_eq!(compressed.len(), compressed_block_size(num_bits));
         }
     }
+
+    const NUM_INTS_BENCH_VINT: usize = 10;
+
+    #[bench]
+    fn bench_compress_vint(b: &mut Bencher) {
+        let mut encoder = BlockEncoder::new();
+        let data = generate_array(NUM_INTS_BENCH_VINT, 0.001);
+        b.iter(|| {
+            encoder.compress_vint_sorted(&data, 0u32);
+        });
+    }
+
+    #[bench]
+    fn bench_uncompress_vint(b: &mut Bencher) {
+        let mut encoder = BlockEncoder::new();
+        let data = generate_array(NUM_INTS_BENCH_VINT, 0.001);
+        let compressed = encoder.compress_vint_sorted(&data, 0u32);
+        let mut decoder = BlockDecoder::new();
+        b.iter(|| {
+            decoder.uncompress_vint_sorted(compressed, 0u32, NUM_INTS_BENCH_VINT);
+        });
+    }
 }

src/postings/mod.rs

@@ -3,7 +3,6 @@ Postings module (also called inverted index)
 */
 
 mod block_search;
-
 pub(crate) mod compression;
 /// Postings module
 ///

src/postings/serializer.rs

@@ -1,6 +1,6 @@
 use super::TermInfo;
+use crate::common::CountingWriter;
 use crate::common::{BinarySerializable, VInt};
-use crate::common::{CompositeWrite, CountingWriter};
 use crate::core::Segment;
 use crate::directory::WritePtr;
 use crate::positions::PositionSerializer;
@@ -10,6 +10,7 @@ use crate::postings::USE_SKIP_INFO_LIMIT;
 use crate::schema::Schema;
 use crate::schema::{Field, FieldEntry, FieldType};
 use crate::termdict::{TermDictionaryBuilder, TermOrdinal};
+use crate::CompositeWrite;
 use crate::DocId;
 use crate::Result;
 use std::io::{self, Write};

src/postings/term_info.rs

@@ -45,7 +45,7 @@ impl BinarySerializable for TermInfo {
 mod tests {
 
     use super::TermInfo;
-    use crate::common::test::fixed_size_test;
+    use crate::common::fixed_size_test;
 
     #[test]
     fn test_fixed_size() {

src/query/automaton_weight.rs

@@ -8,15 +8,13 @@ use crate::termdict::{TermDictionary, TermStreamer};
 use crate::DocId;
 use crate::TantivyError;
 use crate::{Result, SkipResult};
+use std::sync::Arc;
 use tantivy_fst::Automaton;
 
 /// A weight struct for Fuzzy Term and Regex Queries
-pub struct AutomatonWeight<A>
-where
-    A: Automaton + Send + Sync + 'static,
-{
+pub struct AutomatonWeight<A> {
     field: Field,
-    automaton: A,
+    automaton: Arc<A>,
 }
 
 impl<A> AutomatonWeight<A>
@@ -24,12 +22,16 @@ where
     A: Automaton + Send + Sync + 'static,
 {
     /// Create a new AutomationWeight
-    pub fn new(field: Field, automaton: A) -> AutomatonWeight<A> {
-        AutomatonWeight { field, automaton }
+    pub fn new<IntoArcA: Into<Arc<A>>>(field: Field, automaton: IntoArcA) -> AutomatonWeight<A> {
+        AutomatonWeight {
+            field,
+            automaton: automaton.into(),
+        }
     }
 
     fn automaton_stream<'a>(&'a self, term_dict: &'a TermDictionary) -> TermStreamer<'a, &'a A> {
-        let term_stream_builder = term_dict.search(&self.automaton);
+        let automaton: &A = &*self.automaton;
+        let term_stream_builder = term_dict.search(automaton);
         term_stream_builder.into_stream()
     }
 }

src/query/bitset/mod.rs

@@ -216,5 +216,50 @@ mod tests {
             assert!(!docset.advance());
         }
     }
-
+}
+
+#[cfg(all(test, feature = "unstable"))]
+mod bench {
+
+    use super::BitSet;
+    use super::BitSetDocSet;
+    use test;
+    use tests;
+    use DocSet;
+
+    #[bench]
+    fn bench_bitset_1pct_insert(b: &mut test::Bencher) {
+        use tests;
+        let els = tests::generate_nonunique_unsorted(1_000_000u32, 10_000);
+        b.iter(|| {
+            let mut bitset = BitSet::with_max_value(1_000_000);
+            for el in els.iter().cloned() {
+                bitset.insert(el);
+            }
+        });
+    }
+
+    #[bench]
+    fn bench_bitset_1pct_clone(b: &mut test::Bencher) {
+        use tests;
+        let els = tests::generate_nonunique_unsorted(1_000_000u32, 10_000);
+        let mut bitset = BitSet::with_max_value(1_000_000);
+        for el in els {
+            bitset.insert(el);
+        }
+        b.iter(|| bitset.clone());
+    }
+
+    #[bench]
+    fn bench_bitset_1pct_clone_iterate(b: &mut test::Bencher) {
+        let els = tests::sample(1_000_000u32, 0.01);
+        let mut bitset = BitSet::with_max_value(1_000_000);
+        for el in els {
+            bitset.insert(el);
+        }
+        b.iter(|| {
+            let mut docset = BitSetDocSet::from(bitset.clone());
+            while docset.advance() {}
+        });
+    }
 }

src/query/fuzzy_query.rs

@@ -28,12 +28,10 @@ static LEV_BUILDER: Lazy<HashMap<(u8, bool), LevenshteinAutomatonBuilder>> = Laz
 /// containing a specific term that is within
 /// Levenshtein distance
 /// ```rust
-/// #[macro_use]
-/// extern crate tantivy;
-/// use tantivy::schema::{Schema, TEXT};
-/// use tantivy::{Index, Result, Term};
 /// use tantivy::collector::{Count, TopDocs};
 /// use tantivy::query::FuzzyTermQuery;
+/// use tantivy::schema::{Schema, TEXT};
+/// use tantivy::{doc, Index, Result, Term};
 ///
 /// # fn main() { example().unwrap(); }
 /// fn example() -> Result<()> {

src/query/query_parser/query_grammar.rs

@@ -83,28 +83,67 @@ parser! {
 }
 
 parser! {
+    /// Function that parses a range out of a Stream
+    /// Supports ranges like:
+    /// [5 TO 10], {5 TO 10}, [* TO 10], [10 TO *], {10 TO *], >5, <=10
+    /// [a TO *], [a TO c], [abc TO bcd}
     fn range[I]()(I) -> UserInputLeaf
     where [I: Stream<Item = char>] {
         let range_term_val = || {
             word().or(negative_number()).or(char('*').with(value("*".to_string())))
         };
+
+        // check for unbounded range in the form of <5, <=10, >5, >=5
+        let elastic_unbounded_range = (choice([attempt(string(">=")),
+                                               attempt(string("<=")),
+                                               attempt(string("<")),
+                                               attempt(string(">"))])
+                                       .skip(spaces()),
+                                       range_term_val()).
+            map(|(comparison_sign, bound): (&str, String)|
+                match comparison_sign {
+                    ">=" => (UserInputBound::Inclusive(bound), UserInputBound::Unbounded),
+                    "<=" => (UserInputBound::Unbounded, UserInputBound::Inclusive(bound)),
+                    "<" => (UserInputBound::Unbounded, UserInputBound::Exclusive(bound)),
+                    ">" => (UserInputBound::Exclusive(bound), UserInputBound::Unbounded),
+                    // default case
+                    _ => (UserInputBound::Unbounded, UserInputBound::Unbounded)
+                });
         let lower_bound = (one_of("{[".chars()), range_term_val())
             .map(|(boundary_char, lower_bound): (char, String)|
-                if boundary_char == '{' { UserInputBound::Exclusive(lower_bound) }
-                else { UserInputBound::Inclusive(lower_bound) });
+                 if lower_bound == "*" {
+                     UserInputBound::Unbounded
+                 } else if boundary_char == '{' {
+                         UserInputBound::Exclusive(lower_bound)
+                 } else {
+                     UserInputBound::Inclusive(lower_bound)
+                 });
         let upper_bound = (range_term_val(), one_of("}]".chars()))
             .map(|(higher_bound, boundary_char): (String, char)|
-                if boundary_char == '}' { UserInputBound::Exclusive(higher_bound) }
-                else { UserInputBound::Inclusive(higher_bound) });
-        (
-            optional(field()),
-            lower_bound
-            .skip((spaces(), string("TO"), spaces())),
-            upper_bound,
-        ).map(|(field, lower, upper)| UserInputLeaf::Range {
-                field,
-                lower,
-                upper
+                 if higher_bound == "*" {
+                     UserInputBound::Unbounded
+                 } else if boundary_char == '}' {
+                     UserInputBound::Exclusive(higher_bound)
+                 } else {
+                     UserInputBound::Inclusive(higher_bound)
+                 });
+         // return only lower and upper
+        let lower_to_upper = (lower_bound.
+                                    skip((spaces(),
+                                          string("TO"),
+                                          spaces())),
+                                    upper_bound);
+
+        (optional(field()).skip(spaces()),
+         // try elastic first, if it matches, the range is unbounded
+         attempt(elastic_unbounded_range).or(lower_to_upper))
+            .map(|(field, (lower, upper))|
+                 // Construct the leaf from extracted field (optional)
+                 // and bounds
+                 UserInputLeaf::Range {
+                     field,
+                     lower,
+                     upper
         })
     }
 }
@@ -258,6 +297,49 @@ mod test {
         );
     }
 
+    #[test]
+    fn test_parse_elastic_query_ranges() {
+        test_parse_query_to_ast_helper("title: >a", "title:{\"a\" TO \"*\"}");
+        test_parse_query_to_ast_helper("title:>=a", "title:[\"a\" TO \"*\"}");
+        test_parse_query_to_ast_helper("title: <a", "title:{\"*\" TO \"a\"}");
+        test_parse_query_to_ast_helper("title:<=a", "title:{\"*\" TO \"a\"]");
+        test_parse_query_to_ast_helper("title:<=bsd", "title:{\"*\" TO \"bsd\"]");
+
+        test_parse_query_to_ast_helper("weight: >70", "weight:{\"70\" TO \"*\"}");
+        test_parse_query_to_ast_helper("weight:>=70", "weight:[\"70\" TO \"*\"}");
+        test_parse_query_to_ast_helper("weight: <70", "weight:{\"*\" TO \"70\"}");
+        test_parse_query_to_ast_helper("weight:<=70", "weight:{\"*\" TO \"70\"]");
+        test_parse_query_to_ast_helper("weight: >60.7", "weight:{\"60.7\" TO \"*\"}");
+
+        test_parse_query_to_ast_helper("weight: <= 70", "weight:{\"*\" TO \"70\"]");
+
+        test_parse_query_to_ast_helper("weight: <= 70.5", "weight:{\"*\" TO \"70.5\"]");
+    }
+
+    #[test]
+    fn test_range_parser() {
+        // testing the range() parser separately
+        let res = range().parse("title: <hello").unwrap().0;
+        let expected = UserInputLeaf::Range {
+            field: Some("title".to_string()),
+            lower: UserInputBound::Unbounded,
+            upper: UserInputBound::Exclusive("hello".to_string()),
+        };
+        let res2 = range().parse("title:{* TO hello}").unwrap().0;
+        assert_eq!(res, expected);
+        assert_eq!(res2, expected);
+        let expected_weight = UserInputLeaf::Range {
+            field: Some("weight".to_string()),
+            lower: UserInputBound::Inclusive("71.2".to_string()),
+            upper: UserInputBound::Unbounded,
+        };
+
+        let res3 = range().parse("weight: >=71.2").unwrap().0;
+        let res4 = range().parse("weight:[71.2 TO *}").unwrap().0;
+        assert_eq!(res3, expected_weight);
+        assert_eq!(res4, expected_weight);
+    }
+
     #[test]
     fn test_parse_query_to_triming_spaces() {
         test_parse_query_to_ast_helper("   abc", "\"abc\"");
@@ -291,7 +373,7 @@ mod test {
         test_parse_query_to_ast_helper("[1 TO 5]", "[\"1\" TO \"5\"]");
         test_parse_query_to_ast_helper("foo:{a TO z}", "foo:{\"a\" TO \"z\"}");
         test_parse_query_to_ast_helper("foo:[1 TO toto}", "foo:[\"1\" TO \"toto\"}");
-        test_parse_query_to_ast_helper("foo:[* TO toto}", "foo:[\"*\" TO \"toto\"}");
+        test_parse_query_to_ast_helper("foo:[* TO toto}", "foo:{\"*\" TO \"toto\"}");
         test_parse_query_to_ast_helper("foo:[1 TO *}", "foo:[\"1\" TO \"*\"}");
         test_parse_query_to_ast_helper("foo:[1.1 TO *}", "foo:[\"1.1\" TO \"*\"}");
         test_is_parse_err("abc +    ");

src/query/query_parser/query_parser.rs

@@ -369,6 +369,7 @@ impl QueryParser {
         match *bound {
             UserInputBound::Inclusive(_) => Ok(Bound::Included(term)),
             UserInputBound::Exclusive(_) => Ok(Bound::Excluded(term)),
+            UserInputBound::Unbounded => Ok(Bound::Unbounded),
         }
     }
 
@@ -628,7 +629,7 @@ mod test {
     pub fn test_parse_query_untokenized() {
         test_parse_query_to_logical_ast_helper(
             "nottokenized:\"wordone wordtwo\"",
-            "Term([0, 0, 0, 7, 119, 111, 114, 100, 111, 110, \
+            "Term(field=7,bytes=[119, 111, 114, 100, 111, 110, \
              101, 32, 119, 111, 114, 100, 116, 119, 111])",
             false,
         );
@@ -672,7 +673,7 @@ mod test {
             .is_ok());
         test_parse_query_to_logical_ast_helper(
             "unsigned:2324",
-            "Term([0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 9, 20])",
+            "Term(field=3,bytes=[0, 0, 0, 0, 0, 0, 9, 20])",
             false,
         );
 
@@ -693,19 +694,19 @@ mod test {
     pub fn test_parse_query_to_ast_single_term() {
         test_parse_query_to_logical_ast_helper(
             "title:toto",
-            "Term([0, 0, 0, 0, 116, 111, 116, 111])",
+            "Term(field=0,bytes=[116, 111, 116, 111])",
             false,
         );
         test_parse_query_to_logical_ast_helper(
             "+title:toto",
-            "Term([0, 0, 0, 0, 116, 111, 116, 111])",
+            "Term(field=0,bytes=[116, 111, 116, 111])",
             false,
         );
         test_parse_query_to_logical_ast_helper(
             "+title:toto -titi",
-            "(+Term([0, 0, 0, 0, 116, 111, 116, 111]) \
-             -(Term([0, 0, 0, 0, 116, 105, 116, 105]) \
-             Term([0, 0, 0, 1, 116, 105, 116, 105])))",
+            "(+Term(field=0,bytes=[116, 111, 116, 111]) \
+             -(Term(field=0,bytes=[116, 105, 116, 105]) \
+             Term(field=1,bytes=[116, 105, 116, 105])))",
             false,
         );
         assert_eq!(
@@ -720,14 +721,13 @@ mod test {
     pub fn test_parse_query_to_ast_two_terms() {
         test_parse_query_to_logical_ast_helper(
             "title:a b",
-            "(Term([0, 0, 0, 0, 97]) (Term([0, 0, 0, 0, 98]) \
-             Term([0, 0, 0, 1, 98])))",
+            "(Term(field=0,bytes=[97]) (Term(field=0,bytes=[98]) Term(field=1,bytes=[98])))",
             false,
         );
         test_parse_query_to_logical_ast_helper(
             "title:\"a b\"",
-            "\"[(0, Term([0, 0, 0, 0, 97])), \
-             (1, Term([0, 0, 0, 0, 98]))]\"",
+            "\"[(0, Term(field=0,bytes=[97])), \
+             (1, Term(field=0,bytes=[98]))]\"",
             false,
         );
     }
@@ -736,45 +736,43 @@ mod test {
     pub fn test_parse_query_to_ast_ranges() {
         test_parse_query_to_logical_ast_helper(
             "title:[a TO b]",
-            "(Included(Term([0, 0, 0, 0, 97])) TO \
-             Included(Term([0, 0, 0, 0, 98])))",
+            "(Included(Term(field=0,bytes=[97])) TO Included(Term(field=0,bytes=[98])))",
             false,
         );
         test_parse_query_to_logical_ast_helper(
             "[a TO b]",
-            "((Included(Term([0, 0, 0, 0, 97])) TO \
-             Included(Term([0, 0, 0, 0, 98]))) \
-             (Included(Term([0, 0, 0, 1, 97])) TO \
-             Included(Term([0, 0, 0, 1, 98]))))",
+            "((Included(Term(field=0,bytes=[97])) TO \
+             Included(Term(field=0,bytes=[98]))) \
+             (Included(Term(field=1,bytes=[97])) TO \
+             Included(Term(field=1,bytes=[98]))))",
             false,
         );
         test_parse_query_to_logical_ast_helper(
             "title:{titi TO toto}",
-            "(Excluded(Term([0, 0, 0, 0, 116, 105, 116, 105])) TO \
-             Excluded(Term([0, 0, 0, 0, 116, 111, 116, 111])))",
+            "(Excluded(Term(field=0,bytes=[116, 105, 116, 105])) TO \
+             Excluded(Term(field=0,bytes=[116, 111, 116, 111])))",
             false,
         );
         test_parse_query_to_logical_ast_helper(
             "title:{* TO toto}",
-            "(Unbounded TO \
-             Excluded(Term([0, 0, 0, 0, 116, 111, 116, 111])))",
+            "(Unbounded TO Excluded(Term(field=0,bytes=[116, 111, 116, 111])))",
             false,
         );
         test_parse_query_to_logical_ast_helper(
             "title:{titi TO *}",
-            "(Excluded(Term([0, 0, 0, 0, 116, 105, 116, 105])) TO Unbounded)",
+            "(Excluded(Term(field=0,bytes=[116, 105, 116, 105])) TO Unbounded)",
             false,
         );
         test_parse_query_to_logical_ast_helper(
             "signed:{-5 TO 3}",
-            "(Excluded(Term([0, 0, 0, 2, 127, 255, 255, 255, 255, 255, 255, 251])) TO \
-            Excluded(Term([0, 0, 0, 2, 128, 0, 0, 0, 0, 0, 0, 3])))",
+            "(Excluded(Term(field=2,bytes=[127, 255, 255, 255, 255, 255, 255, 251])) TO \
+             Excluded(Term(field=2,bytes=[128, 0, 0, 0, 0, 0, 0, 3])))",
             false,
         );
         test_parse_query_to_logical_ast_helper(
             "float:{-1.5 TO 1.5}",
-            "(Excluded(Term([0, 0, 0, 10, 64, 7, 255, 255, 255, 255, 255, 255])) TO \
-            Excluded(Term([0, 0, 0, 10, 191, 248, 0, 0, 0, 0, 0, 0])))",
+            "(Excluded(Term(field=10,bytes=[64, 7, 255, 255, 255, 255, 255, 255])) TO \
+             Excluded(Term(field=10,bytes=[191, 248, 0, 0, 0, 0, 0, 0])))",
             false,
         );
 
@@ -879,19 +877,19 @@ mod test {
     pub fn test_parse_query_to_ast_conjunction() {
         test_parse_query_to_logical_ast_helper(
             "title:toto",
-            "Term([0, 0, 0, 0, 116, 111, 116, 111])",
+            "Term(field=0,bytes=[116, 111, 116, 111])",
             true,
         );
         test_parse_query_to_logical_ast_helper(
             "+title:toto",
-            "Term([0, 0, 0, 0, 116, 111, 116, 111])",
+            "Term(field=0,bytes=[116, 111, 116, 111])",
             true,
         );
         test_parse_query_to_logical_ast_helper(
             "+title:toto -titi",
-            "(+Term([0, 0, 0, 0, 116, 111, 116, 111]) \
-             -(Term([0, 0, 0, 0, 116, 105, 116, 105]) \
-             Term([0, 0, 0, 1, 116, 105, 116, 105])))",
+            "(+Term(field=0,bytes=[116, 111, 116, 111]) \
+             -(Term(field=0,bytes=[116, 105, 116, 105]) \
+             Term(field=1,bytes=[116, 105, 116, 105])))",
             true,
         );
         assert_eq!(
@@ -902,15 +900,15 @@ mod test {
         );
         test_parse_query_to_logical_ast_helper(
             "title:a b",
-            "(+Term([0, 0, 0, 0, 97]) \
-             +(Term([0, 0, 0, 0, 98]) \
-             Term([0, 0, 0, 1, 98])))",
+            "(+Term(field=0,bytes=[97]) \
+             +(Term(field=0,bytes=[98]) \
+             Term(field=1,bytes=[98])))",
             true,
         );
         test_parse_query_to_logical_ast_helper(
             "title:\"a b\"",
-            "\"[(0, Term([0, 0, 0, 0, 97])), \
-             (1, Term([0, 0, 0, 0, 98]))]\"",
+            "\"[(0, Term(field=0,bytes=[97])), \
+             (1, Term(field=0,bytes=[98]))]\"",
             true,
         );
     }
@@ -919,10 +917,8 @@ mod test {
     pub fn test_query_parser_hyphen() {
         test_parse_query_to_logical_ast_helper(
             "title:www-form-encoded",
-            "\"[(0, Term([0, 0, 0, 0, 119, 119, 119])), \
-             (1, Term([0, 0, 0, 0, 102, 111, 114, 109])), \
-             (2, Term([0, 0, 0, 0, 101, 110, 99, 111, 100, 101, 100]))]\"",
-            false,
+            "\"[(0, Term(field=0,bytes=[119, 119, 119])), (1, Term(field=0,bytes=[102, 111, 114, 109])), (2, Term(field=0,bytes=[101, 110, 99, 111, 100, 101, 100]))]\"",
+            false
         );
     }
 }

src/query/query_parser/user_input_ast.rs

@@ -3,6 +3,7 @@ use std::fmt::{Debug, Formatter};
 
 use crate::query::Occur;
 
+#[derive(PartialEq)]
 pub enum UserInputLeaf {
     Literal(UserInputLiteral),
     All,
@@ -35,6 +36,7 @@ impl Debug for UserInputLeaf {
     }
 }
 
+#[derive(PartialEq)]
 pub struct UserInputLiteral {
     pub field_name: Option<String>,
     pub phrase: String,
@@ -49,9 +51,11 @@ impl fmt::Debug for UserInputLiteral {
     }
 }
 
+#[derive(PartialEq)]
 pub enum UserInputBound {
     Inclusive(String),
     Exclusive(String),
+    Unbounded,
 }
 
 impl UserInputBound {
@@ -59,6 +63,7 @@ impl UserInputBound {
         match *self {
             UserInputBound::Inclusive(ref word) => write!(formatter, "[\"{}\"", word),
             UserInputBound::Exclusive(ref word) => write!(formatter, "{{\"{}\"", word),
+            UserInputBound::Unbounded => write!(formatter, "{{\"*\""),
         }
     }
 
@@ -66,6 +71,7 @@ impl UserInputBound {
         match *self {
             UserInputBound::Inclusive(ref word) => write!(formatter, "\"{}\"]", word),
             UserInputBound::Exclusive(ref word) => write!(formatter, "\"{}\"}}", word),
+            UserInputBound::Unbounded => write!(formatter, "\"*\"}}"),
         }
     }
 
@@ -73,6 +79,7 @@ impl UserInputBound {
         match *self {
             UserInputBound::Inclusive(ref contents) => contents,
             UserInputBound::Exclusive(ref contents) => contents,
+            UserInputBound::Unbounded => &"*",
         }
     }
 }

src/query/range_query.rs

@@ -38,14 +38,10 @@ fn map_bound<TFrom, TTo, Transform: Fn(&TFrom) -> TTo>(
 /// # Example
 ///
 /// ```rust
-///
-/// # #[macro_use]
-/// # extern crate tantivy;
-/// # use tantivy::Index;
-/// # use tantivy::schema::{Schema, INDEXED};
 /// # use tantivy::collector::Count;
-/// # use tantivy::Result;
 /// # use tantivy::query::RangeQuery;
+/// # use tantivy::schema::{Schema, INDEXED};
+/// # use tantivy::{doc, Index, Result};
 /// #
 /// # fn run() -> Result<()> {
 /// #     let mut schema_builder = Schema::builder();
@@ -338,39 +334,33 @@ mod tests {
     use crate::collector::Count;
     use crate::schema::{Document, Field, Schema, INDEXED};
     use crate::Index;
-    use crate::Result;
     use std::collections::Bound;
 
     #[test]
     fn test_range_query_simple() {
-        fn run() -> Result<()> {
-            let mut schema_builder = Schema::builder();
-            let year_field = schema_builder.add_u64_field("year", INDEXED);
-            let schema = schema_builder.build();
+        let mut schema_builder = Schema::builder();
+        let year_field = schema_builder.add_u64_field("year", INDEXED);
+        let schema = schema_builder.build();
 
-            let index = Index::create_in_ram(schema);
-            {
-                let mut index_writer = index.writer_with_num_threads(1, 6_000_000).unwrap();
-                for year in 1950u64..2017u64 {
-                    let num_docs_within_year = 10 + (year - 1950) * (year - 1950);
-                    for _ in 0..num_docs_within_year {
-                        index_writer.add_document(doc!(year_field => year));
-                    }
+        let index = Index::create_in_ram(schema);
+        {
+            let mut index_writer = index.writer_with_num_threads(1, 6_000_000).unwrap();
+            for year in 1950u64..2017u64 {
+                let num_docs_within_year = 10 + (year - 1950) * (year - 1950);
+                for _ in 0..num_docs_within_year {
+                    index_writer.add_document(doc!(year_field => year));
                 }
-                index_writer.commit().unwrap();
             }
-            let reader = index.reader().unwrap();
-            let searcher = reader.searcher();
-
-            let docs_in_the_sixties = RangeQuery::new_u64(year_field, 1960u64..1970u64);
-
-            // ... or `1960..=1969` if inclusive range is enabled.
-            let count = searcher.search(&docs_in_the_sixties, &Count)?;
-            assert_eq!(count, 2285);
-            Ok(())
+            index_writer.commit().unwrap();
         }
+        let reader = index.reader().unwrap();
+        let searcher = reader.searcher();
 
-        run().unwrap();
+        let docs_in_the_sixties = RangeQuery::new_u64(year_field, 1960u64..1970u64);
+
+        // ... or `1960..=1969` if inclusive range is enabled.
+        let count = searcher.search(&docs_in_the_sixties, &Count).unwrap();
+        assert_eq!(count, 2285);
     }
 
     #[test]

src/query/regex_query.rs

@@ -4,22 +4,18 @@ use crate::schema::Field;
 use crate::Result;
 use crate::Searcher;
 use std::clone::Clone;
+use std::sync::Arc;
 use tantivy_fst::Regex;
 
-// A Regex Query matches all of the documents
+/// A Regex Query matches all of the documents
 /// containing a specific term that matches
-/// a regex pattern
-/// A Fuzzy Query matches all of the documents
-/// containing a specific term that is within
-/// Levenshtein distance
+/// a regex pattern.
 ///
 /// ```rust
-/// #[macro_use]
-/// extern crate tantivy;
-/// use tantivy::schema::{Schema, TEXT};
-/// use tantivy::{Index, Result, Term};
 /// use tantivy::collector::Count;
 /// use tantivy::query::RegexQuery;
+/// use tantivy::schema::{Schema, TEXT};
+/// use tantivy::{doc, Index, Result, Term};
 ///
 /// # fn main() { example().unwrap(); }
 /// fn example() -> Result<()> {
@@ -48,7 +44,7 @@ use tantivy_fst::Regex;
 ///     let searcher = reader.searcher();
 ///
 ///     let term = Term::from_field_text(title, "Diary");
-///     let query = RegexQuery::new("d[ai]{2}ry".to_string(), title);
+///     let query = RegexQuery::from_pattern("d[ai]{2}ry", title)?;
 ///     let count = searcher.search(&query, &Count)?;
 ///     assert_eq!(count, 3);
 ///     Ok(())
@@ -56,30 +52,34 @@ use tantivy_fst::Regex;
 /// ```
 #[derive(Debug, Clone)]
 pub struct RegexQuery {
-    regex_pattern: String,
+    regex: Arc<Regex>,
     field: Field,
 }
 
 impl RegexQuery {
-    /// Creates a new Fuzzy Query
-    pub fn new(regex_pattern: String, field: Field) -> RegexQuery {
+    /// Creates a new RegexQuery from a given pattern
+    pub fn from_pattern(regex_pattern: &str, field: Field) -> Result<Self> {
+        let regex = Regex::new(&regex_pattern)
+            .map_err(|_| TantivyError::InvalidArgument(regex_pattern.to_string()))?;
+        Ok(RegexQuery::from_regex(regex, field))
+    }
+
+    /// Creates a new RegexQuery from a fully built Regex
+    pub fn from_regex<T: Into<Arc<Regex>>>(regex: T, field: Field) -> Self {
         RegexQuery {
-            regex_pattern,
+            regex: regex.into(),
             field,
         }
     }
 
-    fn specialized_weight(&self) -> Result<AutomatonWeight<Regex>> {
-        let automaton = Regex::new(&self.regex_pattern)
-            .map_err(|_| TantivyError::InvalidArgument(self.regex_pattern.clone()))?;
-
-        Ok(AutomatonWeight::new(self.field, automaton))
+    fn specialized_weight(&self) -> AutomatonWeight<Regex> {
+        AutomatonWeight::new(self.field, self.regex.clone())
     }
 }
 
 impl Query for RegexQuery {
     fn weight(&self, _searcher: &Searcher, _scoring_enabled: bool) -> Result<Box<dyn Weight>> {
-        Ok(Box::new(self.specialized_weight()?))
+        Ok(Box::new(self.specialized_weight()))
     }
 }
 
@@ -87,13 +87,14 @@ impl Query for RegexQuery {
 mod test {
     use super::RegexQuery;
     use crate::collector::TopDocs;
-    use crate::schema::Schema;
     use crate::schema::TEXT;
+    use crate::schema::{Field, Schema};
     use crate::tests::assert_nearly_equals;
-    use crate::Index;
+    use crate::{Index, IndexReader};
+    use std::sync::Arc;
+    use tantivy_fst::Regex;
 
-    #[test]
-    pub fn test_regex_query() {
+    fn build_test_index() -> (IndexReader, Field) {
         let mut schema_builder = Schema::builder();
         let country_field = schema_builder.add_text_field("country", TEXT);
         let schema = schema_builder.build();
@@ -109,20 +110,65 @@ mod test {
             index_writer.commit().unwrap();
         }
         let reader = index.reader().unwrap();
+
+        (reader, country_field)
+    }
+
+    fn verify_regex_query(
+        query_matching_one: RegexQuery,
+        query_matching_zero: RegexQuery,
+        reader: IndexReader,
+    ) {
         let searcher = reader.searcher();
         {
-            let regex_query = RegexQuery::new("jap[ao]n".to_string(), country_field);
             let scored_docs = searcher
-                .search(&regex_query, &TopDocs::with_limit(2))
+                .search(&query_matching_one, &TopDocs::with_limit(2))
                 .unwrap();
             assert_eq!(scored_docs.len(), 1, "Expected only 1 document");
             let (score, _) = scored_docs[0];
             assert_nearly_equals(1f32, score);
         }
-        let regex_query = RegexQuery::new("jap[A-Z]n".to_string(), country_field);
         let top_docs = searcher
-            .search(&regex_query, &TopDocs::with_limit(2))
+            .search(&query_matching_zero, &TopDocs::with_limit(2))
             .unwrap();
         assert!(top_docs.is_empty(), "Expected ZERO document");
     }
+
+    #[test]
+    pub fn test_regex_query() {
+        let (reader, field) = build_test_index();
+
+        let matching_one = RegexQuery::from_pattern("jap[ao]n", field).unwrap();
+        let matching_zero = RegexQuery::from_pattern("jap[A-Z]n", field).unwrap();
+
+        verify_regex_query(matching_one, matching_zero, reader);
+    }
+
+    #[test]
+    pub fn test_construct_from_regex() {
+        let (reader, field) = build_test_index();
+
+        let matching_one = RegexQuery::from_regex(Regex::new("jap[ao]n").unwrap(), field);
+        let matching_zero = RegexQuery::from_regex(Regex::new("jap[A-Z]n").unwrap(), field);
+
+        verify_regex_query(matching_one, matching_zero, reader);
+    }
+
+    #[test]
+    pub fn test_construct_from_reused_regex() {
+        let r1 = Arc::new(Regex::new("jap[ao]n").unwrap());
+        let r2 = Arc::new(Regex::new("jap[A-Z]n").unwrap());
+
+        let (reader, field) = build_test_index();
+
+        let matching_one = RegexQuery::from_regex(r1.clone(), field);
+        let matching_zero = RegexQuery::from_regex(r2.clone(), field);
+
+        verify_regex_query(matching_one, matching_zero, reader.clone());
+
+        let matching_one = RegexQuery::from_regex(r1.clone(), field);
+        let matching_zero = RegexQuery::from_regex(r2.clone(), field);
+
+        verify_regex_query(matching_one, matching_zero, reader.clone());
+    }
 }

src/query/term_query/mod.rs

@@ -12,7 +12,7 @@ mod tests {
     use crate::collector::TopDocs;
     use crate::docset::DocSet;
     use crate::query::{Query, QueryParser, Scorer, TermQuery};
-    use crate::schema::{IndexRecordOption, Schema, STRING, TEXT};
+    use crate::schema::{Field, IndexRecordOption, Schema, STRING, TEXT};
     use crate::tests::assert_nearly_equals;
     use crate::Index;
     use crate::Term;
@@ -114,4 +114,16 @@ mod tests {
         let reader = index.reader().unwrap();
         assert_eq!(term_query.count(&*reader.searcher()).unwrap(), 1);
     }
+
+    #[test]
+    fn test_term_query_debug() {
+        let term_query = TermQuery::new(
+            Term::from_field_text(Field(1), "hello"),
+            IndexRecordOption::WithFreqs,
+        );
+        assert_eq!(
+            format!("{:?}", term_query),
+            "TermQuery(Term(field=1,bytes=[104, 101, 108, 108, 111]))"
+        );
+    }
 }

src/query/term_query/term_query.rs

@@ -7,6 +7,7 @@ use crate::Result;
 use crate::Searcher;
 use crate::Term;
 use std::collections::BTreeSet;
+use std::fmt;
 
 /// A Term query matches all of the documents
 /// containing a specific term.
@@ -19,12 +20,10 @@ use std::collections::BTreeSet;
 /// * `field norm` - number of tokens in the field.
 ///
 /// ```rust
-/// #[macro_use]
-/// extern crate tantivy;
-/// use tantivy::schema::{Schema, TEXT, IndexRecordOption};
-/// use tantivy::{Index, Result, Term};
 /// use tantivy::collector::{Count, TopDocs};
 /// use tantivy::query::TermQuery;
+/// use tantivy::schema::{Schema, TEXT, IndexRecordOption};
+/// use tantivy::{doc, Index, Result, Term};
 ///
 /// # fn main() { example().unwrap(); }
 /// fn example() -> Result<()> {
@@ -61,12 +60,18 @@ use std::collections::BTreeSet;
 ///     Ok(())
 /// }
 /// ```
-#[derive(Clone, Debug)]
+#[derive(Clone)]
 pub struct TermQuery {
     term: Term,
     index_record_option: IndexRecordOption,
 }
 
+impl fmt::Debug for TermQuery {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "TermQuery({:?})", self.term)
+    }
+}
+
 impl TermQuery {
     /// Creates a new term query.
     pub fn new(term: Term, segment_postings_options: IndexRecordOption) -> TermQuery {

src/query/union.rs

@@ -411,3 +411,52 @@ mod tests {
     }
 
 }
+
+#[cfg(all(test, feature = "unstable"))]
+mod bench {
+
+    use query::score_combiner::DoNothingCombiner;
+    use query::ConstScorer;
+    use query::Union;
+    use query::VecDocSet;
+    use test::Bencher;
+    use tests;
+    use DocId;
+    use DocSet;
+
+    #[bench]
+    fn bench_union_3_high(bench: &mut Bencher) {
+        let union_docset: Vec<Vec<DocId>> = vec![
+            tests::sample_with_seed(100_000, 0.1, 0),
+            tests::sample_with_seed(100_000, 0.2, 1),
+        ];
+        bench.iter(|| {
+            let mut v = Union::<_, DoNothingCombiner>::from(
+                union_docset
+                    .iter()
+                    .map(|doc_ids| VecDocSet::from(doc_ids.clone()))
+                    .map(ConstScorer::new)
+                    .collect::<Vec<_>>(),
+            );
+            while v.advance() {}
+        });
+    }
+    #[bench]
+    fn bench_union_3_low(bench: &mut Bencher) {
+        let union_docset: Vec<Vec<DocId>> = vec![
+            tests::sample_with_seed(100_000, 0.01, 0),
+            tests::sample_with_seed(100_000, 0.05, 1),
+            tests::sample_with_seed(100_000, 0.001, 2),
+        ];
+        bench.iter(|| {
+            let mut v = Union::<_, DoNothingCombiner>::from(
+                union_docset
+                    .iter()
+                    .map(|doc_ids| VecDocSet::from(doc_ids.clone()))
+                    .map(ConstScorer::new)
+                    .collect::<Vec<_>>(),
+            );
+            while v.advance() {}
+        });
+    }
+}

src/snippet/mod.rs

@@ -63,7 +63,7 @@ impl FragmentCandidate {
     fn try_add_token(&mut self, token: &Token, terms: &BTreeMap<String, f32>) {
         self.stop_offset = token.offset_to;
 
-        if let Some(score) = terms.get(&token.text.to_lowercase()) {
+        if let Some(&score) = terms.get(&token.text.to_lowercase()) {
             self.score += score;
             self.highlighted
                 .push(HighlightSection::new(token.offset_from, token.offset_to));
@@ -142,7 +142,7 @@ impl Snippet {
 /// Fragments must be valid in the sense that `&text[fragment.start..fragment.stop]`\
 /// has to be a valid string.
 fn search_fragments<'a>(
-    tokenizer: &dyn BoxedTokenizer,
+    tokenizer: &BoxedTokenizer,
     text: &'a str,
     terms: &BTreeMap<String, f32>,
     max_num_chars: usize,
@@ -150,7 +150,6 @@ fn search_fragments<'a>(
     let mut token_stream = tokenizer.token_stream(text);
     let mut fragment = FragmentCandidate::new(0);
     let mut fragments: Vec<FragmentCandidate> = vec![];
-
     while let Some(next) = token_stream.next() {
         if (next.offset_to - fragment.start_offset) > max_num_chars {
             if fragment.score > 0.0 {
@@ -214,11 +213,9 @@ fn select_best_fragment_combination(fragments: &[FragmentCandidate], text: &str)
 /// # Example
 ///
 /// ```rust
-/// # #[macro_use]
-/// # extern crate tantivy;
-/// # use tantivy::Index;
-/// # use tantivy::schema::{Schema, TEXT};
 /// # use tantivy::query::QueryParser;
+/// # use tantivy::schema::{Schema, TEXT};
+/// # use tantivy::{doc, Index};
 /// use tantivy::SnippetGenerator;
 ///
 /// # fn main() -> tantivy::Result<()> {
@@ -254,7 +251,7 @@ fn select_best_fragment_combination(fragments: &[FragmentCandidate], text: &str)
 /// ```
 pub struct SnippetGenerator {
     terms_text: BTreeMap<String, f32>,
-    tokenizer: Box<dyn BoxedTokenizer>,
+    tokenizer: BoxedTokenizer,
     field: Field,
     max_num_chars: usize,
 }
@@ -316,12 +313,8 @@ impl SnippetGenerator {
 
     /// Generates a snippet for the given text.
     pub fn snippet(&self, text: &str) -> Snippet {
-        let fragment_candidates = search_fragments(
-            &*self.tokenizer,
-            &text,
-            &self.terms_text,
-            self.max_num_chars,
-        );
+        let fragment_candidates =
+            search_fragments(&self.tokenizer, &text, &self.terms_text, self.max_num_chars);
         select_best_fragment_combination(&fragment_candidates[..], &text)
     }
 }
@@ -331,7 +324,7 @@ mod tests {
     use super::{search_fragments, select_best_fragment_combination};
     use crate::query::QueryParser;
     use crate::schema::{IndexRecordOption, Schema, TextFieldIndexing, TextOptions, TEXT};
-    use crate::tokenizer::{box_tokenizer, SimpleTokenizer};
+    use crate::tokenizer::SimpleTokenizer;
     use crate::Index;
     use crate::SnippetGenerator;
     use maplit::btreemap;
@@ -355,12 +348,12 @@ Survey in 2016, 2017, and 2018."#;
 
     #[test]
     fn test_snippet() {
-        let boxed_tokenizer = box_tokenizer(SimpleTokenizer);
+        let boxed_tokenizer = SimpleTokenizer.into();
         let terms = btreemap! {
             String::from("rust") => 1.0,
             String::from("language") => 0.9
         };
-        let fragments = search_fragments(&*boxed_tokenizer, TEST_TEXT, &terms, 100);
+        let fragments = search_fragments(&boxed_tokenizer, TEST_TEXT, &terms, 100);
         assert_eq!(fragments.len(), 7);
         {
             let first = &fragments[0];
@@ -382,13 +375,13 @@ Survey in 2016, 2017, and 2018."#;
 
     #[test]
     fn test_snippet_scored_fragment() {
-        let boxed_tokenizer = box_tokenizer(SimpleTokenizer);
+        let boxed_tokenizer = SimpleTokenizer.into();
         {
             let terms = btreemap! {
                 String::from("rust") =>1.0f32,
                 String::from("language") => 0.9f32
             };
-            let fragments = search_fragments(&*boxed_tokenizer, TEST_TEXT, &terms, 20);
+            let fragments = search_fragments(&boxed_tokenizer, TEST_TEXT, &terms, 20);
             {
                 let first = &fragments[0];
                 assert_eq!(first.score, 1.0);
@@ -397,13 +390,13 @@ Survey in 2016, 2017, and 2018."#;
             let snippet = select_best_fragment_combination(&fragments[..], &TEST_TEXT);
             assert_eq!(snippet.to_html(), "<b>Rust</b> is a systems")
         }
-        let boxed_tokenizer = box_tokenizer(SimpleTokenizer);
+        let boxed_tokenizer = SimpleTokenizer.into();
         {
             let terms = btreemap! {
                 String::from("rust") =>0.9f32,
                 String::from("language") => 1.0f32
             };
-            let fragments = search_fragments(&*boxed_tokenizer, TEST_TEXT, &terms, 20);
+            let fragments = search_fragments(&boxed_tokenizer, TEST_TEXT, &terms, 20);
             //assert_eq!(fragments.len(), 7);
             {
                 let first = &fragments[0];
@@ -417,14 +410,14 @@ Survey in 2016, 2017, and 2018."#;
 
     #[test]
     fn test_snippet_in_second_fragment() {
-        let boxed_tokenizer = box_tokenizer(SimpleTokenizer);
+        let boxed_tokenizer = SimpleTokenizer.into();
 
         let text = "a b c d e f g";
 
         let mut terms = BTreeMap::new();
         terms.insert(String::from("c"), 1.0);
 
-        let fragments = search_fragments(&*boxed_tokenizer, &text, &terms, 3);
+        let fragments = search_fragments(&boxed_tokenizer, &text, &terms, 3);
 
         assert_eq!(fragments.len(), 1);
         {
@@ -441,14 +434,14 @@ Survey in 2016, 2017, and 2018."#;
 
     #[test]
     fn test_snippet_with_term_at_the_end_of_fragment() {
-        let boxed_tokenizer = box_tokenizer(SimpleTokenizer);
+        let boxed_tokenizer = SimpleTokenizer.into();
 
         let text = "a b c d e f f g";
 
         let mut terms = BTreeMap::new();
         terms.insert(String::from("f"), 1.0);
 
-        let fragments = search_fragments(&*boxed_tokenizer, &text, &terms, 3);
+        let fragments = search_fragments(&boxed_tokenizer, &text, &terms, 3);
 
         assert_eq!(fragments.len(), 2);
         {
@@ -465,7 +458,7 @@ Survey in 2016, 2017, and 2018."#;
 
     #[test]
     fn test_snippet_with_second_fragment_has_the_highest_score() {
-        let boxed_tokenizer = box_tokenizer(SimpleTokenizer);
+        let boxed_tokenizer = SimpleTokenizer.into();
 
         let text = "a b c d e f g";
 
@@ -473,7 +466,7 @@ Survey in 2016, 2017, and 2018."#;
         terms.insert(String::from("f"), 1.0);
         terms.insert(String::from("a"), 0.9);
 
-        let fragments = search_fragments(&*boxed_tokenizer, &text, &terms, 7);
+        let fragments = search_fragments(&boxed_tokenizer, &text, &terms, 7);
 
         assert_eq!(fragments.len(), 2);
         {
@@ -490,14 +483,14 @@ Survey in 2016, 2017, and 2018."#;
 
     #[test]
     fn test_snippet_with_term_not_in_text() {
-        let boxed_tokenizer = box_tokenizer(SimpleTokenizer);
+        let boxed_tokenizer = SimpleTokenizer.into();
 
         let text = "a b c d";
 
         let mut terms = BTreeMap::new();
         terms.insert(String::from("z"), 1.0);
 
-        let fragments = search_fragments(&*boxed_tokenizer, &text, &terms, 3);
+        let fragments = search_fragments(&boxed_tokenizer, &text, &terms, 3);
 
         assert_eq!(fragments.len(), 0);
 
@@ -508,12 +501,12 @@ Survey in 2016, 2017, and 2018."#;
 
     #[test]
     fn test_snippet_with_no_terms() {
-        let boxed_tokenizer = box_tokenizer(SimpleTokenizer);
+        let boxed_tokenizer = SimpleTokenizer.into();
 
         let text = "a b c d";
 
         let terms = BTreeMap::new();
-        let fragments = search_fragments(&*boxed_tokenizer, &text, &terms, 3);
+        let fragments = search_fragments(&boxed_tokenizer, &text, &terms, 3);
         assert_eq!(fragments.len(), 0);
 
         let snippet = select_best_fragment_combination(&fragments[..], &text);

src/termdict/term_info_store.rs

@@ -268,7 +268,7 @@ mod tests {
 
     #[test]
     fn test_term_info_block() {
-        common::test::fixed_size_test::<TermInfoBlockMeta>();
+        common::fixed_size_test::<TermInfoBlockMeta>();
     }
 
     #[test]

tantivy-common/Cargo.toml

@@ -0,0 +1,10 @@
+[package]
+name = "tantivy-common"
+version = "0.1.0"
+authors = ["Paul Masurel <paul.masurel@gmail.com>"]
+edition = "2018"
+workspace = ".."
+
+[dependencies]
+byteorder = "*"
+chrono = "*"

tantivy-common/src/bitpacker.rs

@@ -2,7 +2,7 @@ use byteorder::{ByteOrder, LittleEndian, WriteBytesExt};
 use std::io;
 use std::ops::Deref;
 
-pub(crate) struct BitPacker {
+pub struct BitPacker {
     mini_buffer: u64,
     mini_buffer_written: usize,
 }
@@ -86,7 +86,6 @@ where
         }
     }
 
-    #[inline(always)]
     pub fn get(&self, idx: u64) -> u64 {
         if self.num_bits == 0 {
             return 0u64;

tantivy-common/src/bitset.rs

@@ -179,7 +179,7 @@ impl BitSet {
     ///
     /// Reminder: the tiny set with the bucket `bucket`, represents the
     /// elements from `bucket * 64` to `(bucket+1) * 64`.
-    pub(crate) fn first_non_empty_bucket(&self, bucket: u32) -> Option<u32> {
+    pub fn first_non_empty_bucket(&self, bucket: u32) -> Option<u32> {
         self.tinysets[bucket as usize..]
             .iter()
             .cloned()
@@ -194,7 +194,7 @@ impl BitSet {
     /// Returns the tiny bitset representing the
     /// the set restricted to the number range from
     /// `bucket * 64` to `(bucket + 1) * 64`.
-    pub(crate) fn tinyset(&self, bucket: u32) -> TinySet {
+    pub fn tinyset(&self, bucket: u32) -> TinySet {
         self.tinysets[bucket as usize]
     }
 }
@@ -204,12 +204,7 @@ mod tests {
 
     use super::BitSet;
     use super::TinySet;
-    use crate::docset::DocSet;
-    use crate::query::BitSetDocSet;
-    use crate::tests;
-    use crate::tests::generate_nonunique_unsorted;
-    use std::collections::BTreeSet;
-    use std::collections::HashSet;
+    use std::collections::{BTreeSet, HashSet};
 
     #[test]
     fn test_tiny_set() {
@@ -264,26 +259,19 @@ mod tests {
         test_against_hashset(&[62u32, 63u32], 64);
     }
 
-    #[test]
-    fn test_bitset_large() {
-        let arr = generate_nonunique_unsorted(100_000, 5_000);
-        let mut btreeset: BTreeSet<u32> = BTreeSet::new();
-        let mut bitset = BitSet::with_max_value(100_000);
-        for el in arr {
-            btreeset.insert(el);
-            bitset.insert(el);
-        }
-        for i in 0..100_000 {
-            assert_eq!(btreeset.contains(&i), bitset.contains(i));
-        }
-        assert_eq!(btreeset.len(), bitset.len());
-        let mut bitset_docset = BitSetDocSet::from(bitset);
-        for el in btreeset.into_iter() {
-            bitset_docset.advance();
-            assert_eq!(bitset_docset.doc(), el);
-        }
-        assert!(!bitset_docset.advance());
-    }
+    //    #[test]
+    //    fn test_bitset_clear() {
+    //        let mut bitset = BitSet::with_max_value(1_000);
+    //        let els = tests::sample(1_000, 0.01f64);
+    //        for &el in &els {
+    //            bitset.insert(el);
+    //        }
+    //        assert!(els.iter().all(|el| bitset.contains(*el)));
+    //        bitset.clear();
+    //        for el in 0u32..1000u32 {
+    //            assert!(!bitset.contains(el));
+    //        }
+    //    }
 
     #[test]
     fn test_bitset_num_buckets() {
@@ -339,17 +327,44 @@ mod tests {
         assert_eq!(bitset.len(), 3);
     }
 
-    #[test]
-    fn test_bitset_clear() {
-        let mut bitset = BitSet::with_max_value(1_000);
-        let els = tests::sample(1_000, 0.01f64);
-        for &el in &els {
-            bitset.insert(el);
-        }
-        assert!(els.iter().all(|el| bitset.contains(*el)));
-        bitset.clear();
-        for el in 0u32..1000u32 {
-            assert!(!bitset.contains(el));
-        }
+}
+
+#[cfg(all(test, feature = "unstable"))]
+mod bench {
+
+    use super::BitSet;
+    use super::TinySet;
+    use test;
+
+    #[bench]
+    fn bench_tinyset_pop(b: &mut test::Bencher) {
+        b.iter(|| {
+            let mut tinyset = TinySet::singleton(test::black_box(31u32));
+            tinyset.pop_lowest();
+            tinyset.pop_lowest();
+            tinyset.pop_lowest();
+            tinyset.pop_lowest();
+            tinyset.pop_lowest();
+            tinyset.pop_lowest();
+        });
+    }
+
+    #[bench]
+    fn bench_tinyset_sum(b: &mut test::Bencher) {
+        let tiny_set = TinySet::empty().insert(10u32).insert(14u32).insert(21u32);
+        b.iter(|| {
+            assert_eq!(test::black_box(tiny_set).into_iter().sum::<u32>(), 45u32);
+        });
+    }
+
+    #[bench]
+    fn bench_tinyarr_sum(b: &mut test::Bencher) {
+        let v = [10u32, 14u32, 21u32];
+        b.iter(|| test::black_box(v).iter().cloned().sum::<u32>());
+    }
+
+    #[bench]
+    fn bench_bitset_initialize(b: &mut test::Bencher) {
+        b.iter(|| BitSet::with_max_value(1_000_000));
     }
 }

tantivy-common/src/composite_file.rs

@@ -0,0 +1,235 @@
+use crate::common::BinarySerializable;
+use crate::common::CountingWriter;
+use crate::common::VInt;
+use crate::directory::ReadOnlySource;
+use crate::directory::WritePtr;
+use crate::schema::Field;
+use crate::space_usage::FieldUsage;
+use crate::space_usage::PerFieldSpaceUsage;
+use std::collections::HashMap;
+use std::io::Write;
+use std::io::{self, Read};
+
+#[derive(Eq, PartialEq, Hash, Copy, Ord, PartialOrd, Clone, Debug)]
+pub struct FileAddr {
+    field: Field,
+    idx: usize,
+}
+
+impl FileAddr {
+    fn new(field: Field, idx: usize) -> FileAddr {
+        FileAddr { field, idx }
+    }
+}
+
+impl BinarySerializable for FileAddr {
+    fn serialize<W: Write>(&self, writer: &mut W) -> io::Result<()> {
+        self.field.serialize(writer)?;
+        VInt(self.idx as u64).serialize(writer)?;
+        Ok(())
+    }
+
+    fn deserialize<R: Read>(reader: &mut R) -> io::Result<Self> {
+        let field = Field::deserialize(reader)?;
+        let idx = VInt::deserialize(reader)?.0 as usize;
+        Ok(FileAddr { field, idx })
+    }
+}
+
+/// A `CompositeWrite` is used to write a `CompositeFile`.
+pub struct CompositeWrite<W = WritePtr> {
+    write: CountingWriter<W>,
+    offsets: HashMap<FileAddr, u64>,
+}
+
+impl<W: Write> CompositeWrite<W> {
+    /// Crate a new API writer that writes a composite file
+    /// in a given write.
+    pub fn wrap(w: W) -> CompositeWrite<W> {
+        CompositeWrite {
+            write: CountingWriter::wrap(w),
+            offsets: HashMap::new(),
+        }
+    }
+
+    /// Start writing a new field.
+    pub fn for_field(&mut self, field: Field) -> &mut CountingWriter<W> {
+        self.for_field_with_idx(field, 0)
+    }
+
+    /// Start writing a new field.
+    pub fn for_field_with_idx(&mut self, field: Field, idx: usize) -> &mut CountingWriter<W> {
+        let offset = self.write.written_bytes();
+        let file_addr = FileAddr::new(field, idx);
+        assert!(!self.offsets.contains_key(&file_addr));
+        self.offsets.insert(file_addr, offset);
+        &mut self.write
+    }
+
+    /// Close the composite file
+    ///
+    /// An index of the different field offsets
+    /// will be written as a footer.
+    pub fn close(mut self) -> io::Result<()> {
+        let footer_offset = self.write.written_bytes();
+        VInt(self.offsets.len() as u64).serialize(&mut self.write)?;
+
+        let mut offset_fields: Vec<_> = self
+            .offsets
+            .iter()
+            .map(|(file_addr, offset)| (*offset, *file_addr))
+            .collect();
+
+        offset_fields.sort();
+
+        let mut prev_offset = 0;
+        for (offset, file_addr) in offset_fields {
+            VInt((offset - prev_offset) as u64).serialize(&mut self.write)?;
+            file_addr.serialize(&mut self.write)?;
+            prev_offset = offset;
+        }
+
+        let footer_len = (self.write.written_bytes() - footer_offset) as u32;
+        footer_len.serialize(&mut self.write)?;
+        self.write.flush()?;
+        Ok(())
+    }
+}
+
+/// A composite file is an abstraction to store a
+/// file partitioned by field.
+///
+/// The file needs to be written field by field.
+/// A footer describes the start and stop offsets
+/// for each field.
+#[derive(Clone)]
+pub struct CompositeFile {
+    data: ReadOnlySource,
+    offsets_index: HashMap<FileAddr, (usize, usize)>,
+}
+
+impl CompositeFile {
+    /// Opens a composite file stored in a given
+    /// `ReadOnlySource`.
+    pub fn open(data: &ReadOnlySource) -> io::Result<CompositeFile> {
+        let end = data.len();
+        let footer_len_data = data.slice_from(end - 4);
+        let footer_len = u32::deserialize(&mut footer_len_data.as_slice())? as usize;
+        let footer_start = end - 4 - footer_len;
+        let footer_data = data.slice(footer_start, footer_start + footer_len);
+        let mut footer_buffer = footer_data.as_slice();
+        let num_fields = VInt::deserialize(&mut footer_buffer)?.0 as usize;
+
+        let mut file_addrs = vec![];
+        let mut offsets = vec![];
+
+        let mut field_index = HashMap::new();
+
+        let mut offset = 0;
+        for _ in 0..num_fields {
+            offset += VInt::deserialize(&mut footer_buffer)?.0 as usize;
+            let file_addr = FileAddr::deserialize(&mut footer_buffer)?;
+            offsets.push(offset);
+            file_addrs.push(file_addr);
+        }
+        offsets.push(footer_start);
+        for i in 0..num_fields {
+            let file_addr = file_addrs[i];
+            let start_offset = offsets[i];
+            let end_offset = offsets[i + 1];
+            field_index.insert(file_addr, (start_offset, end_offset));
+        }
+
+        Ok(CompositeFile {
+            data: data.slice_to(footer_start),
+            offsets_index: field_index,
+        })
+    }
+
+    /// Returns a composite file that stores
+    /// no fields.
+    pub fn empty() -> CompositeFile {
+        CompositeFile {
+            offsets_index: HashMap::new(),
+            data: ReadOnlySource::empty(),
+        }
+    }
+
+    /// Returns the `ReadOnlySource` associated
+    /// to a given `Field` and stored in a `CompositeFile`.
+    pub fn open_read(&self, field: Field) -> Option<ReadOnlySource> {
+        self.open_read_with_idx(field, 0)
+    }
+
+    /// Returns the `ReadOnlySource` associated
+    /// to a given `Field` and stored in a `CompositeFile`.
+    pub fn open_read_with_idx(&self, field: Field, idx: usize) -> Option<ReadOnlySource> {
+        self.offsets_index
+            .get(&FileAddr { field, idx })
+            .map(|&(from, to)| self.data.slice(from, to))
+    }
+
+    pub fn space_usage(&self) -> PerFieldSpaceUsage {
+        let mut fields = HashMap::new();
+        for (&field_addr, &(start, end)) in self.offsets_index.iter() {
+            fields
+                .entry(field_addr.field)
+                .or_insert_with(|| FieldUsage::empty(field_addr.field))
+                .add_field_idx(field_addr.idx, end - start);
+        }
+        PerFieldSpaceUsage::new(fields)
+    }
+}
+
+#[cfg(test)]
+mod test {
+
+    use super::{CompositeFile, CompositeWrite};
+    use crate::common::BinarySerializable;
+    use crate::common::VInt;
+    use crate::directory::{Directory, RAMDirectory};
+    use crate::schema::Field;
+    use std::io::Write;
+    use std::path::Path;
+
+    #[test]
+    fn test_composite_file() {
+        let path = Path::new("test_path");
+        let mut directory = RAMDirectory::create();
+        {
+            let w = directory.open_write(path).unwrap();
+            let mut composite_write = CompositeWrite::wrap(w);
+            {
+                let mut write_0 = composite_write.for_field(Field(0u32));
+                VInt(32431123u64).serialize(&mut write_0).unwrap();
+                write_0.flush().unwrap();
+            }
+
+            {
+                let mut write_4 = composite_write.for_field(Field(4u32));
+                VInt(2).serialize(&mut write_4).unwrap();
+                write_4.flush().unwrap();
+            }
+            composite_write.close().unwrap();
+        }
+        {
+            let r = directory.open_read(path).unwrap();
+            let composite_file = CompositeFile::open(&r).unwrap();
+            {
+                let file0 = composite_file.open_read(Field(0u32)).unwrap();
+                let mut file0_buf = file0.as_slice();
+                let payload_0 = VInt::deserialize(&mut file0_buf).unwrap().0;
+                assert_eq!(file0_buf.len(), 0);
+                assert_eq!(payload_0, 32431123u64);
+            }
+            {
+                let file4 = composite_file.open_read(Field(4u32)).unwrap();
+                let mut file4_buf = file4.as_slice();
+                let payload_4 = VInt::deserialize(&mut file4_buf).unwrap().0;
+                assert_eq!(file4_buf.len(), 0);
+                assert_eq!(payload_4, 2u64);
+            }
+        }
+    }
+
+}

tantivy-common/src/lib.rs

@@ -1,18 +1,18 @@
 pub mod bitpacker;
 mod bitset;
-mod composite_file;
 mod counting_writer;
 mod serialize;
 mod vint;
 
 pub use self::bitset::BitSet;
 pub use self::bitset::TinySet;
-pub(crate) use self::composite_file::{CompositeFile, CompositeWrite};
 pub use self::counting_writer::CountingWriter;
 pub use self::serialize::{BinarySerializable, FixedSize};
 pub use self::vint::{read_u32_vint, serialize_vint_u32, write_u32_vint, VInt};
 pub use byteorder::LittleEndian as Endianness;
 
+pub type DateTime = chrono::DateTime<chrono::Utc>;
+
 /// Segment's max doc must be `< MAX_DOC_LIMIT`.
 ///
 /// We do not allow segments with more than
@@ -42,7 +42,7 @@ pub const MAX_DOC_LIMIT: u32 = 1 << 31;
 /// a very large range of values. Even in this case, it results
 /// in an extra cost of at most 12% compared to the optimal
 /// number of bits.
-pub(crate) fn compute_num_bits(n: u64) -> u8 {
+pub fn compute_num_bits(n: u64) -> u8 {
     let amplitude = (64u32 - n.leading_zeros()) as u8;
     if amplitude <= 64 - 8 {
         amplitude
@@ -51,7 +51,7 @@ pub(crate) fn compute_num_bits(n: u64) -> u8 {
     }
 }
 
-pub(crate) fn is_power_of_2(n: usize) -> bool {
+pub fn is_power_of_2(n: usize) -> bool {
     (n > 0) && (n & (n - 1) == 0)
 }
 
@@ -131,10 +131,12 @@ pub fn u64_to_f64(val: u64) -> f64 {
     })
 }
 
+pub use self::serialize::fixed_size_test;
+
 #[cfg(test)]
 pub(crate) mod test {
 
-    pub use super::serialize::test::fixed_size_test;
+    use super::fixed_size_test;
     use super::{compute_num_bits, f64_to_u64, i64_to_u64, u64_to_f64, u64_to_i64};
     use std::f64;
 

tantivy-common/src/serialize.rs

@@ -1,5 +1,5 @@
-use crate::common::Endianness;
-use crate::common::VInt;
+use crate::Endianness;
+use crate::VInt;
 use byteorder::{ReadBytesExt, WriteBytesExt};
 use std::fmt;
 use std::io;
@@ -145,17 +145,17 @@ impl BinarySerializable for String {
     }
 }
 
+pub fn fixed_size_test<O: BinarySerializable + FixedSize + Default>() {
+    let mut buffer = Vec::new();
+    O::default().serialize(&mut buffer).unwrap();
+    assert_eq!(buffer.len(), O::SIZE_IN_BYTES);
+}
+
 #[cfg(test)]
-pub mod test {
+mod test {
 
     use super::*;
-    use crate::common::VInt;
-
-    pub fn fixed_size_test<O: BinarySerializable + FixedSize + Default>() {
-        let mut buffer = Vec::new();
-        O::default().serialize(&mut buffer).unwrap();
-        assert_eq!(buffer.len(), O::SIZE_IN_BYTES);
-    }
+    use crate::VInt;
 
     fn serialize_test<T: BinarySerializable + Eq>(v: T) -> usize {
         let mut buffer: Vec<u8> = Vec::new();

tantivy-common/src/vint.rs

@@ -171,7 +171,7 @@ mod tests {
 
     use super::serialize_vint_u32;
     use super::VInt;
-    use crate::common::BinarySerializable;
+    use crate::BinarySerializable;
     use byteorder::{ByteOrder, LittleEndian};
 
     fn aux_test_vint(val: u64) {

tantivy-schema/Cargo.toml

@@ -0,0 +1,33 @@
+[package]
+name = "tantivy-schema"
+version = "0.1.0"
+authors = ["Paul Masurel <paul.masurel@gmail.com>"]
+edition = "2018"
+workspace = ".."
+
+
+[dependencies]
+base64 = "0.10.0"
+byteorder = "1.0"
+once_cell = "0.2"
+regex = "1.0"
+serde = "1.0"
+serde_derive = "1.0"
+serde_json = "1.0"
+num_cpus = "1.2"
+itertools = "0.8"
+notify = {version="4", optional=true}
+crossbeam = "0.7"
+owning_ref = "0.4"
+stable_deref_trait = "1.0.0"
+downcast-rs = { version="1.0" }
+census = "0.2"
+failure = "0.1"
+fail = "0.3"
+scoped-pool = "1.0"
+tantivy-common = {path="../tantivy-common"}
+chrono = "*"
+
+
+[dev-dependencies]
+matches = "0.1.8"

tantivy-schema/src/document.rs

@@ -1,9 +1,10 @@
 use super::*;
-use crate::common::BinarySerializable;
-use crate::common::VInt;
-use crate::DateTime;
 use itertools::Itertools;
+use serde_derive::{Deserialize, Serialize};
 use std::io::{self, Read, Write};
+use tantivy_common::BinarySerializable;
+use tantivy_common::DateTime;
+use tantivy_common::VInt;
 
 /// Tantivy's Document is the object that can
 /// be indexed and then searched for.
@@ -168,7 +169,7 @@ impl BinarySerializable for Document {
 #[cfg(test)]
 mod tests {
 
-    use crate::schema::*;
+    use crate::*;
 
     #[test]
     fn test_doc() {

tantivy-schema/src/facet.rs

@@ -1,4 +1,3 @@
-use crate::common::BinarySerializable;
 use once_cell::sync::Lazy;
 use regex::Regex;
 use serde::{Deserialize, Deserializer, Serialize, Serializer};
@@ -8,6 +7,7 @@ use std::fmt::{self, Debug, Display, Formatter};
 use std::io::{self, Read, Write};
 use std::str;
 use std::string::FromUtf8Error;
+use tantivy_common::BinarySerializable;
 
 const SLASH_BYTE: u8 = b'/';
 const ESCAPE_BYTE: u8 = b'\\';
@@ -59,7 +59,7 @@ impl Facet {
         &self.0
     }
 
-    pub(crate) fn from_encoded_string(facet_string: String) -> Facet {
+    pub fn from_encoded_string(facet_string: String) -> Facet {
         Facet(facet_string)
     }
 
@@ -104,7 +104,7 @@ impl Facet {
     }
 
     /// Accessor for the inner buffer of the `Facet`.
-    pub(crate) fn set_facet_str(&mut self, facet_str: &str) {
+    pub fn set_facet_str(&mut self, facet_str: &str) {
         self.0.clear();
         self.0.push_str(facet_str);
     }

tantivy-schema/src/field.rs

@@ -1,7 +1,8 @@
-use crate::common::BinarySerializable;
+use serde_derive::{Deserialize, Serialize};
 use std::io;
 use std::io::Read;
 use std::io::Write;
+use tantivy_common::BinarySerializable;
 
 /// `Field` is actually a `u8` identifying a `Field`
 /// The schema is in charge of holding mapping between field names

tantivy-schema/src/field_entry.rs

@@ -1,7 +1,8 @@
-use crate::schema::IntOptions;
-use crate::schema::TextOptions;
+use serde_derive::*;
 
-use crate::schema::FieldType;
+use crate::FieldType;
+use crate::IntOptions;
+use crate::TextOptions;
 use serde::de::{self, MapAccess, Visitor};
 use serde::ser::SerializeStruct;
 use serde::{Deserialize, Deserializer, Serialize, Serializer};
@@ -265,7 +266,7 @@ impl<'de> Deserialize<'de> for FieldEntry {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::schema::TEXT;
+    use crate::TEXT;
     use serde_json;
 
     #[test]

tantivy-schema/src/field_type.rs

@@ -1,16 +1,15 @@
 use base64::decode;
 
-use crate::schema::{IntOptions, TextOptions};
-
-use crate::schema::Facet;
-use crate::schema::IndexRecordOption;
-use crate::schema::TextFieldIndexing;
-use crate::schema::Value;
+use crate::Facet;
+use crate::IndexRecordOption;
+use crate::TextFieldIndexing;
+use crate::Value;
+use crate::{IntOptions, TextOptions};
 use serde_json::Value as JsonValue;
 
 /// Possible error that may occur while parsing a field value
 /// At this point the JSON is known to be valid.
-#[derive(Debug)]
+#[derive(Debug, PartialEq)]
 pub enum ValueParsingError {
     /// Encountered a numerical value that overflows or underflow its integer type.
     OverflowError(String),
@@ -183,8 +182,9 @@ impl FieldType {
 #[cfg(test)]
 mod tests {
     use super::FieldType;
-    use crate::schema::field_type::ValueParsingError;
-    use crate::schema::Value;
+    use crate::field_type::ValueParsingError;
+    use crate::Value;
+    use serde_json::json;
 
     #[test]
     fn test_bytes_value_from_json() {

tantivy-schema/src/field_value.rs

@@ -1,9 +1,11 @@
-use crate::common::BinarySerializable;
-use crate::schema::Field;
-use crate::schema::Value;
+use crate::Field;
+use crate::Value;
+//use serde::Deserialize;
+use serde_derive::{Deserialize, Serialize};
 use std::io;
 use std::io::Read;
 use std::io::Write;
+use tantivy_common::BinarySerializable;
 
 /// `FieldValue` holds together a `Field` and its `Value`.
 #[derive(Debug, Clone, Ord, PartialEq, Eq, PartialOrd, Serialize, Deserialize)]

tantivy-schema/src/flags.rs

@@ -1,5 +1,5 @@
-use crate::schema::IntOptions;
-use crate::schema::TextOptions;
+use crate::IntOptions;
+use crate::TextOptions;
 use std::ops::BitOr;
 
 #[derive(Clone)]

tantivy-schema/src/index_record_option.rs

@@ -1,3 +1,5 @@
+use serde_derive::{Deserialize, Serialize};
+
 /// `IndexRecordOption` describes an amount information associated
 /// to a given indexed field.
 ///

tantivy-schema/src/int_options.rs

@@ -1,4 +1,5 @@
-use crate::schema::flags::{FastFlag, IndexedFlag, SchemaFlagList, StoredFlag};
+use crate::flags::{FastFlag, IndexedFlag, SchemaFlagList, StoredFlag};
+use serde_derive::{Deserialize, Serialize};
 use std::ops::BitOr;
 
 /// Express whether a field is single-value or multi-valued.

tantivy-schema/src/lib.rs

@@ -26,7 +26,7 @@ directory.
 ### Example
 
 ```
-use tantivy::schema::*;
+use tantivy_schema::*;
 let mut schema_builder = Schema::builder();
 let title_options = TextOptions::default()
     .set_stored()
@@ -59,7 +59,7 @@ when [`searcher.doc(doc_address)`](../struct.Searcher.html#method.doc) is called
 ### Example
 
 ```
-use tantivy::schema::*;
+use tantivy_schema::*;
 let mut schema_builder = Schema::builder();
 let num_stars_options = IntOptions::default()
     .set_stored()
@@ -93,7 +93,7 @@ using the  `|` operator.
 For instance, a schema containing the two fields defined in the example above could be rewritten :
 
 ```
-use tantivy::schema::*;
+use tantivy_schema::*;
 let mut schema_builder = Schema::builder();
 schema_builder.add_u64_field("num_stars", INDEXED | STORED);
 schema_builder.add_text_field("title", TEXT | STORED);
@@ -126,7 +126,6 @@ pub use self::schema::{Schema, SchemaBuilder};
 pub use self::value::Value;
 
 pub use self::facet::Facet;
-pub(crate) use self::facet::FACET_SEP_BYTE;
 
 pub use self::document::Document;
 pub use self::field::Field;

tantivy-schema/src/named_field_document.rs

@@ -1,4 +1,5 @@
-use crate::schema::Value;
+use crate::Value;
+use serde_derive::Serialize;
 use std::collections::BTreeMap;
 
 /// Internal representation of a document used for JSON

tantivy-schema/src/schema.rs

@@ -1,14 +1,14 @@
-use crate::schema::field_type::ValueParsingError;
-use std::collections::BTreeMap;
-use std::collections::HashMap;
-use std::sync::Arc;
-
 use super::*;
+use crate::schema::field_type::ValueParsingError;
+use failure::Fail;
 use serde::de::{SeqAccess, Visitor};
 use serde::ser::SerializeSeq;
 use serde::{Deserialize, Deserializer, Serialize, Serializer};
 use serde_json::{self, Map as JsonObject, Value as JsonValue};
+use std::collections::BTreeMap;
+use std::collections::HashMap;
 use std::fmt;
+use std::sync::Arc;
 
 /// Tantivy has a very strict schema.
 /// You need to specify in advance whether a field is indexed or not,
@@ -21,7 +21,7 @@ use std::fmt;
 /// # Examples
 ///
 /// ```
-/// use tantivy::schema::*;
+/// use tantivy_schema::*;
 ///
 /// let mut schema_builder = Schema::builder();
 /// let id_field = schema_builder.add_text_field("id", STRING);
@@ -208,7 +208,7 @@ impl Eq for InnerSchema {}
 /// # Examples
 ///
 /// ```
-/// use tantivy::schema::*;
+/// use tantivy_schema::*;
 ///
 /// let mut schema_builder = Schema::builder();
 /// let id_field = schema_builder.add_text_field("id", STRING);
@@ -246,6 +246,25 @@ impl Schema {
         self.0.fields_map.get(field_name).cloned()
     }
 
+    /// Create a named document off the doc.
+    pub fn convert_named_doc(
+        &self,
+        named_doc: NamedFieldDocument,
+    ) -> Result<Document, DocParsingError> {
+        let mut document = Document::new();
+        for (field_name, values) in named_doc.0 {
+            if let Some(field) = self.get_field(&field_name) {
+                for value in values {
+                    let field_value = FieldValue::new(field, value);
+                    document.add(field_value);
+                }
+            } else {
+                return Err(DocParsingError::NoSuchFieldInSchema(field_name));
+            }
+        }
+        Ok(document)
+    }
+
     /// Create a named document off the doc.
     pub fn to_named_doc(&self, doc: &Document) -> NamedFieldDocument {
         let mut field_map = BTreeMap::new();
@@ -282,28 +301,26 @@ impl Schema {
 
         let mut doc = Document::default();
         for (field_name, json_value) in json_obj.iter() {
-            match self.get_field(field_name) {
-                Some(field) => {
-                    let field_entry = self.get_field_entry(field);
-                    let field_type = field_entry.field_type();
-                    match *json_value {
-                        JsonValue::Array(ref json_items) => {
-                            for json_item in json_items {
-                                let value = field_type.value_from_json(json_item).map_err(|e| {
-                                    DocParsingError::ValueError(field_name.clone(), e)
-                                })?;
-                                doc.add(FieldValue::new(field, value));
-                            }
-                        }
-                        _ => {
-                            let value = field_type
-                                .value_from_json(json_value)
-                                .map_err(|e| DocParsingError::ValueError(field_name.clone(), e))?;
-                            doc.add(FieldValue::new(field, value));
-                        }
+            let field = self
+                .get_field(field_name)
+                .ok_or_else(|| DocParsingError::NoSuchFieldInSchema(field_name.clone()))?;
+            let field_entry = self.get_field_entry(field);
+            let field_type = field_entry.field_type();
+            match *json_value {
+                JsonValue::Array(ref json_items) => {
+                    for json_item in json_items {
+                        let value = field_type
+                            .value_from_json(json_item)
+                            .map_err(|e| DocParsingError::ValueError(field_name.clone(), e))?;
+                        doc.add(FieldValue::new(field, value));
                     }
                 }
-                None => return Err(DocParsingError::NoSuchFieldInSchema(field_name.clone())),
+                _ => {
+                    let value = field_type
+                        .value_from_json(json_value)
+                        .map_err(|e| DocParsingError::ValueError(field_name.clone(), e))?;
+                    doc.add(FieldValue::new(field, value));
+                }
             }
         }
         Ok(doc)
@@ -360,7 +377,7 @@ impl<'de> Deserialize<'de> for Schema {
 
 /// Error that may happen when deserializing
 /// a document from JSON.
-#[derive(Debug, Fail)]
+#[derive(Debug, Fail, PartialEq)]
 pub enum DocParsingError {
     /// The payload given is not valid JSON.
     #[fail(display = "The provided string is not valid JSON")]
@@ -369,7 +386,10 @@ pub enum DocParsingError {
     #[fail(display = "The field '{:?}' could not be parsed: {:?}", _0, _1)]
     ValueError(String, ValueParsingError),
     /// The json-document contains a field that is not declared in the schema.
-    #[fail(display = "The json-document contains an unknown field: {:?}", _0)]
+    #[fail(
+        display = "The document contains a field that is not declared in the schema: {:?}",
+        _0
+    )]
     NoSuchFieldInSchema(String),
 }
 
@@ -381,6 +401,7 @@ mod tests {
     use crate::schema::*;
     use matches::{assert_matches, matches};
     use serde_json;
+    use std::collections::BTreeMap;
 
     #[test]
     pub fn is_indexed_test() {
@@ -495,6 +516,54 @@ mod tests {
         assert_eq!(doc, doc_serdeser);
     }
 
+    #[test]
+    pub fn test_document_from_nameddoc() {
+        let mut schema_builder = Schema::builder();
+        let title = schema_builder.add_text_field("title", TEXT);
+        let val = schema_builder.add_i64_field("val", INDEXED);
+        let schema = schema_builder.build();
+        let mut named_doc_map = BTreeMap::default();
+        named_doc_map.insert(
+            "title".to_string(),
+            vec![Value::from("title1"), Value::from("title2")],
+        );
+        named_doc_map.insert(
+            "val".to_string(),
+            vec![Value::from(14u64), Value::from(-1i64)],
+        );
+        let doc = schema
+            .convert_named_doc(NamedFieldDocument(named_doc_map))
+            .unwrap();
+        assert_eq!(
+            doc.get_all(title),
+            vec![
+                &Value::from("title1".to_string()),
+                &Value::from("title2".to_string())
+            ]
+        );
+        assert_eq!(
+            doc.get_all(val),
+            vec![&Value::from(14u64), &Value::from(-1i64)]
+        );
+    }
+
+    #[test]
+    pub fn test_document_from_nameddoc_error() {
+        let schema = Schema::builder().build();
+        let mut named_doc_map = BTreeMap::default();
+        named_doc_map.insert(
+            "title".to_string(),
+            vec![Value::from("title1"), Value::from("title2")],
+        );
+        let err = schema
+            .convert_named_doc(NamedFieldDocument(named_doc_map))
+            .unwrap_err();
+        assert_eq!(
+            err,
+            DocParsingError::NoSuchFieldInSchema("title".to_string())
+        );
+    }
+
     #[test]
     pub fn test_parse_document() {
         let mut schema_builder = Schema::builder();

tantivy-schema/src/term.rs

@@ -1,11 +1,11 @@
 use std::fmt;
 
-use super::Field;
-use crate::common;
-use crate::schema::Facet;
-use crate::DateTime;
+use crate::Facet;
+use crate::Field;
 use byteorder::{BigEndian, ByteOrder};
 use std::str;
+use tantivy_common as common;
+use tantivy_common::DateTime;
 
 /// Size (in bytes) of the buffer of a int field.
 const INT_TERM_LEN: usize = 4 + 8;
@@ -94,7 +94,7 @@ impl Term {
     }
 
     /// Creates a new Term for a given field.
-    pub(crate) fn for_field(field: Field) -> Term {
+    pub fn for_field(field: Field) -> Term {
         let mut term = Term(Vec::with_capacity(100));
         term.set_field(field);
         term
@@ -134,7 +134,7 @@ impl Term {
         self.0.extend(bytes);
     }
 
-    pub(crate) fn from_field_bytes(field: Field, bytes: &[u8]) -> Term {
+    pub fn from_field_bytes(field: Field, bytes: &[u8]) -> Term {
         let mut term = Term::for_field(field);
         term.set_bytes(bytes);
         term
@@ -224,14 +224,19 @@ where
 
 impl fmt::Debug for Term {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(f, "Term({:?})", &self.0[..])
+        write!(
+            f,
+            "Term(field={},bytes={:?})",
+            self.field().0,
+            self.value_bytes()
+        )
     }
 }
 
 #[cfg(test)]
 mod tests {
 
-    use crate::schema::*;
+    use crate::{Schema, Term, STRING};
 
     #[test]
     pub fn test_term() {

tantivy-schema/src/text_options.rs

@@ -1,6 +1,7 @@
-use crate::schema::flags::SchemaFlagList;
-use crate::schema::flags::StoredFlag;
-use crate::schema::IndexRecordOption;
+use crate::flags::SchemaFlagList;
+use crate::flags::StoredFlag;
+use crate::IndexRecordOption;
+use serde_derive::{Deserialize, Serialize};
 use std::borrow::Cow;
 use std::ops::BitOr;
 
@@ -151,7 +152,7 @@ where
 
 #[cfg(test)]
 mod tests {
-    use crate::schema::*;
+    use crate::{FieldType, IndexRecordOption, Schema, STORED, TEXT};
 
     #[test]
     fn test_field_options() {

tantivy-schema/src/value.rs

@@ -1,9 +1,11 @@
-use crate::schema::Facet;
-use crate::DateTime;
+use crate::Facet;
+use chrono;
 use serde::de::Visitor;
 use serde::{Deserialize, Deserializer, Serialize, Serializer};
 use std::{cmp::Ordering, fmt};
 
+pub(crate) type DateTime = chrono::DateTime<chrono::Utc>;
+
 /// Value represents the value of a any field.
 /// It is an enum over all over all of the possible field type.
 #[derive(Debug, Clone, PartialEq, PartialOrd)]
@@ -218,11 +220,11 @@ impl From<Vec<u8>> for Value {
 }
 
 mod binary_serialize {
-    use super::Value;
-    use crate::common::{f64_to_u64, u64_to_f64, BinarySerializable};
-    use crate::schema::Facet;
+    use crate::Facet;
+    use crate::Value;
     use chrono::{TimeZone, Utc};
     use std::io::{self, Read, Write};
+    use tantivy_common::{f64_to_u64, u64_to_f64, BinarySerializable};
 
     const TEXT_CODE: u8 = 0;
     const U64_CODE: u8 = 1;

tantivy-tokenizer/Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+name = "tantivy-tokenizer"
+version = "0.1.0"
+authors = ["Paul Masurel <paul.masurel@gmail.com>"]
+edition = "2018"
+workspace = ".."
+
+[dependencies]
+fnv = "*"
+rust-stemmers = "*"
+serde = "*"
+serde_derive = "*"
+tantivy-schema = {path="../tantivy-schema"}

tantivy-tokenizer/src/alphanum_only.rs

@@ -1,7 +1,6 @@
 //! # Example
-//! ```
-//! extern crate tantivy;
-//! use tantivy::tokenizer::*;
+//! ```rust
+//! use tantivy_tokenizer::*;
 //!
 //! # fn main() {
 //!
@@ -65,14 +64,6 @@ impl<TailTokenStream> TokenStream for AlphaNumOnlyFilterStream<TailTokenStream>
 where
     TailTokenStream: TokenStream,
 {
-    fn token(&self) -> &Token {
-        self.tail.token()
-    }
-
-    fn token_mut(&mut self) -> &mut Token {
-        self.tail.token_mut()
-    }
-
     fn advance(&mut self) -> bool {
         while self.tail.advance() {
             if self.predicate(self.tail.token()) {
@@ -82,4 +73,12 @@ where
 
         false
     }
+
+    fn token(&self) -> &Token {
+        self.tail.token()
+    }
+
+    fn token_mut(&mut self) -> &mut Token {
+        self.tail.token_mut()
+    }
 }

tantivy-tokenizer/src/ascii_folding_filter.rs

@@ -1558,11 +1558,11 @@ fn to_ascii(text: &mut String, output: &mut String) {
 #[cfg(test)]
 mod tests {
     use super::to_ascii;
-    use crate::tokenizer::AsciiFoldingFilter;
-    use crate::tokenizer::RawTokenizer;
-    use crate::tokenizer::SimpleTokenizer;
-    use crate::tokenizer::TokenStream;
-    use crate::tokenizer::Tokenizer;
+    use crate::AsciiFoldingFilter;
+    use crate::RawTokenizer;
+    use crate::SimpleTokenizer;
+    use crate::TokenStream;
+    use crate::Tokenizer;
     use std::iter;
 
     #[test]

tantivy-tokenizer/src/facet_tokenizer.rs

@@ -1,5 +1,5 @@
 use super::{Token, TokenStream, Tokenizer};
-use crate::schema::FACET_SEP_BYTE;
+use crate::FACET_SEP_BYTE;
 
 /// The `FacetTokenizer` process a `Facet` binary representation
 /// and emits a token for all of its parent.
@@ -83,8 +83,8 @@ impl<'a> TokenStream for FacetTokenStream<'a> {
 mod tests {
 
     use super::FacetTokenizer;
-    use crate::schema::Facet;
     use crate::tokenizer::{Token, TokenStream, Tokenizer};
+    use tantivy_schema::Facet;
 
     #[test]
     fn test_facet_tokenizer() {

tantivy-tokenizer/src/lib.rs

@@ -4,9 +4,8 @@
 //! You must define in your schema which tokenizer should be used for
 //! each of your fields :
 //!
-//! ```
-//! extern crate tantivy;
-//! use tantivy::schema::*;
+//! ```rust
+//! use tantivy_schema::*;
 //!
 //! # fn main() {
 //! let mut schema_builder = Schema::builder();
@@ -65,9 +64,7 @@
 //! For instance, the `en_stem` is defined as follows.
 //!
 //! ```rust
-//! # extern crate tantivy;
-//!
-//! use tantivy::tokenizer::*;
+//! use tantivy_tokenizer::*;
 //!
 //! # fn main() {
 //! let en_stem = SimpleTokenizer
@@ -80,10 +77,9 @@
 //! Once your tokenizer is defined, you need to
 //! register it with a name in your index's [`TokenizerManager`](./struct.TokenizerManager.html).
 //!
-//! ```
-//! # extern crate tantivy;
-//! # use tantivy::schema::Schema;
-//! # use tantivy::tokenizer::*;
+//! ```rust
+//! # use tantivy_schema::Schema;
+//! # use tantivy_tokenizer::*;
 //! # use tantivy::Index;
 //! # fn main() {
 //! # let custom_en_tokenizer = SimpleTokenizer;
@@ -101,10 +97,9 @@
 //!
 //! # Example
 //!
-//! ```
-//! extern crate tantivy;
-//! use tantivy::schema::{Schema, IndexRecordOption, TextOptions, TextFieldIndexing};
-//! use tantivy::tokenizer::*;
+//! ```rust
+//! use tantivy_schema::{Schema, IndexRecordOption, TextOptions, TextFieldIndexing};
+//! use tantivy_tokenizer::*;
 //! use tantivy::Index;
 //!
 //! # fn main() {
@@ -155,9 +150,10 @@ pub use self::simple_tokenizer::SimpleTokenizer;
 pub use self::stemmer::{Language, Stemmer};
 pub use self::stop_word_filter::StopWordFilter;
 pub(crate) use self::token_stream_chain::TokenStreamChain;
-pub(crate) use self::tokenizer::box_tokenizer;
+
 pub use self::tokenizer::BoxedTokenizer;
 
+pub(crate) const FACET_SEP_BYTE: u8 = 0u8;
 pub use self::tokenizer::{Token, TokenFilter, TokenStream, Tokenizer};
 pub use self::tokenizer_manager::TokenizerManager;
 

tantivy-tokenizer/src/lower_caser.rs

@@ -72,10 +72,10 @@ where
 
 #[cfg(test)]
 mod tests {
-    use crate::tokenizer::LowerCaser;
-    use crate::tokenizer::SimpleTokenizer;
-    use crate::tokenizer::TokenStream;
-    use crate::tokenizer::Tokenizer;
+    use crate::LowerCaser;
+    use crate::SimpleTokenizer;
+    use crate::TokenStream;
+    use crate::Tokenizer;
 
     #[test]
     fn test_to_lower_case() {

tantivy-tokenizer/src/ngram_tokenizer.rs

@@ -29,9 +29,8 @@ use super::{Token, TokenStream, Tokenizer};
 ///
 /// # Example
 ///
-/// ```
-/// # extern crate tantivy;
-/// use tantivy::tokenizer::*;
+/// ```rust
+/// use tantivy_tokenizer::*;
 /// # fn main() {
 /// let tokenizer = NgramTokenizer::new(2, 3, false);
 /// let mut stream = tokenizer.token_stream("hello");
@@ -309,9 +308,9 @@ mod tests {
     use super::CodepointFrontiers;
     use super::NgramTokenizer;
     use super::StutteringIterator;
-    use crate::tokenizer::tests::assert_token;
-    use crate::tokenizer::tokenizer::{TokenStream, Tokenizer};
-    use crate::tokenizer::Token;
+    use crate::tests::assert_token;
+    use crate::tokenizer::{TokenStream, Tokenizer};
+    use crate::Token;
 
     fn test_helper<T: TokenStream>(mut tokenizer: T) -> Vec<Token> {
         let mut tokens: Vec<Token> = vec![];

tantivy-tokenizer/src/remove_long.rs

@@ -1,7 +1,6 @@
 //! # Example
-//! ```
-//! extern crate tantivy;
-//! use tantivy::tokenizer::*;
+//! ```rust
+//! use tantivy_tokenizer::*;
 //!
 //! # fn main() {
 //!

tantivy-tokenizer/src/stemmer.rs

@@ -1,5 +1,6 @@
 use super::{Token, TokenFilter, TokenStream};
 use rust_stemmers::{self, Algorithm};
+use serde_derive::{Deserialize, Serialize};
 
 /// Available stemmer languages.
 #[derive(Debug, Serialize, Deserialize, Eq, PartialEq, Copy, Clone)]

tantivy-tokenizer/src/stop_word_filter.rs

@@ -1,7 +1,6 @@
 //! # Example
-//! ```
-//! extern crate tantivy;
-//! use tantivy::tokenizer::*;
+//! ```rust
+//! use tantivy_tokenizer::*;
 //!
 //! # fn main() {
 //! let tokenizer = SimpleTokenizer

tantivy-tokenizer/src/token_stream_chain.rs

@@ -1,4 +1,4 @@
-use crate::tokenizer::{Token, TokenStream};
+use crate::{Token, TokenStream};
 
 const POSITION_GAP: usize = 2;
 

tantivy-tokenizer/src/tokenizer.rs

@@ -1,4 +1,4 @@
-use crate::tokenizer::TokenStreamChain;
+use crate::TokenStreamChain;
 /// The tokenizer module contains all of the tools used to process
 /// text in `tantivy`.
 use std::borrow::{Borrow, BorrowMut};
@@ -56,9 +56,7 @@ pub trait Tokenizer<'a>: Sized + Clone {
     /// # Example
     ///
     /// ```rust
-    /// # extern crate tantivy;
-    ///
-    /// use tantivy::tokenizer::*;
+    /// use tantivy_tokenizer::*;
     ///
     /// # fn main() {
     /// let en_stem = SimpleTokenizer
@@ -80,7 +78,7 @@ pub trait Tokenizer<'a>: Sized + Clone {
 }
 
 /// A boxed tokenizer
-pub trait BoxedTokenizer: Send + Sync {
+trait BoxedTokenizerTrait: Send + Sync {
     /// Tokenize a `&str`
     fn token_stream<'a>(&self, text: &'a str) -> Box<dyn TokenStream + 'a>;
 
@@ -92,7 +90,41 @@ pub trait BoxedTokenizer: Send + Sync {
     fn token_stream_texts<'b>(&self, texts: &'b [&'b str]) -> Box<dyn TokenStream + 'b>;
 
     /// Return a boxed clone of the tokenizer
-    fn boxed_clone(&self) -> Box<dyn BoxedTokenizer>;
+    fn boxed_clone(&self) -> BoxedTokenizer;
+}
+
+/// A boxed tokenizer
+pub struct BoxedTokenizer(Box<dyn BoxedTokenizerTrait>);
+
+impl<T> From<T> for BoxedTokenizer
+where
+    T: 'static + Send + Sync + for<'a> Tokenizer<'a>,
+{
+    fn from(tokenizer: T) -> BoxedTokenizer {
+        BoxedTokenizer(Box::new(BoxableTokenizer(tokenizer)))
+    }
+}
+
+impl BoxedTokenizer {
+    /// Tokenize a `&str`
+    pub fn token_stream<'a>(&self, text: &'a str) -> Box<dyn TokenStream + 'a> {
+        self.0.token_stream(text)
+    }
+
+    /// Tokenize an array`&str`
+    ///
+    /// The resulting `TokenStream` is equivalent to what would be obtained if the &str were
+    /// one concatenated `&str`, with an artificial position gap of `2` between the different fields
+    /// to prevent accidental `PhraseQuery` to match accross two terms.
+    pub fn token_stream_texts<'b>(&self, texts: &'b [&'b str]) -> Box<dyn TokenStream + 'b> {
+        self.0.token_stream_texts(texts)
+    }
+}
+
+impl Clone for BoxedTokenizer {
+    fn clone(&self) -> BoxedTokenizer {
+        self.0.boxed_clone()
+    }
 }
 
 #[derive(Clone)]
@@ -100,7 +132,7 @@ struct BoxableTokenizer<A>(A)
 where
     A: for<'a> Tokenizer<'a> + Send + Sync;
 
-impl<A> BoxedTokenizer for BoxableTokenizer<A>
+impl<A> BoxedTokenizerTrait for BoxableTokenizer<A>
 where
     A: 'static + Send + Sync + for<'a> Tokenizer<'a>,
 {
@@ -125,18 +157,11 @@ where
         }
     }
 
-    fn boxed_clone(&self) -> Box<dyn BoxedTokenizer> {
-        Box::new(self.clone())
+    fn boxed_clone(&self) -> BoxedTokenizer {
+        self.0.clone().into()
     }
 }
 
-pub(crate) fn box_tokenizer<A>(a: A) -> Box<dyn BoxedTokenizer>
-where
-    A: 'static + Send + Sync + for<'a> Tokenizer<'a>,
-{
-    Box::new(BoxableTokenizer(a))
-}
-
 impl<'b> TokenStream for Box<dyn TokenStream + 'b> {
     fn advance(&mut self) -> bool {
         let token_stream: &mut dyn TokenStream = self.borrow_mut();
@@ -161,8 +186,7 @@ impl<'b> TokenStream for Box<dyn TokenStream + 'b> {
 /// # Example
 ///
 /// ```
-/// extern crate tantivy;
-/// use tantivy::tokenizer::*;
+/// use tantivy_tokenizer::*;
 ///
 /// # fn main() {
 /// let tokenizer = SimpleTokenizer
@@ -203,8 +227,7 @@ pub trait TokenStream {
     /// and `.token()`.
     ///
     /// ```
-    /// # extern crate tantivy;
-    /// # use tantivy::tokenizer::*;
+    /// # use tantivy_tokenizer::*;
     /// #
     /// # fn main() {
     /// # let tokenizer = SimpleTokenizer

tantivy-tokenizer/src/tokenizer_manager.rs

@@ -1,14 +1,12 @@
-use crate::tokenizer::box_tokenizer;
-use crate::tokenizer::stemmer::Language;
-use crate::tokenizer::BoxedTokenizer;
-use crate::tokenizer::LowerCaser;
-use crate::tokenizer::RawTokenizer;
-use crate::tokenizer::RemoveLongFilter;
-use crate::tokenizer::SimpleTokenizer;
-use crate::tokenizer::Stemmer;
-use crate::tokenizer::Tokenizer;
+use crate::stemmer::Language;
+use crate::BoxedTokenizer;
+use crate::LowerCaser;
+use crate::RawTokenizer;
+use crate::RemoveLongFilter;
+use crate::SimpleTokenizer;
+use crate::Stemmer;
+use crate::Tokenizer;
 use std::collections::HashMap;
-use std::ops::Deref;
 use std::sync::{Arc, RwLock};
 
 /// The tokenizer manager serves as a store for
@@ -25,30 +23,28 @@ use std::sync::{Arc, RwLock};
 ///  search engine.
 #[derive(Clone)]
 pub struct TokenizerManager {
-    tokenizers: Arc<RwLock<HashMap<String, Box<dyn BoxedTokenizer>>>>,
+    tokenizers: Arc<RwLock<HashMap<String, BoxedTokenizer>>>,
 }
 
 impl TokenizerManager {
     /// Registers a new tokenizer associated with a given name.
     pub fn register<A>(&self, tokenizer_name: &str, tokenizer: A)
     where
-        A: 'static + Send + Sync + for<'a> Tokenizer<'a>,
+        A: Into<BoxedTokenizer>,
     {
-        let boxed_tokenizer = box_tokenizer(tokenizer);
         self.tokenizers
             .write()
             .expect("Acquiring the lock should never fail")
-            .insert(tokenizer_name.to_string(), boxed_tokenizer);
+            .insert(tokenizer_name.to_string(), tokenizer.into());
     }
 
     /// Accessing a tokenizer given its name.
-    pub fn get(&self, tokenizer_name: &str) -> Option<Box<dyn BoxedTokenizer>> {
+    pub fn get(&self, tokenizer_name: &str) -> Option<BoxedTokenizer> {
         self.tokenizers
             .read()
             .expect("Acquiring the lock should never fail")
             .get(tokenizer_name)
-            .map(Deref::deref)
-            .map(BoxedTokenizer::boxed_clone)
+            .cloned()
     }
 }