doc

Cargo.toml

@@ -72,6 +72,7 @@ default = ["mmap", "no_fail"]
 mmap = ["fst/mmap", "atomicwrites"]
 lz4-compression = ["lz4"]
 no_fail = ["fail/no_fail"]
+unstable = [] # useful for benches.
 
 [badges]
 travis-ci = { repository = "tantivy-search/tantivy" }

doc/src/SUMMARY.md

@@ -4,8 +4,9 @@
 
 [Avant Propos](./avant-propos.md)
 
+- [Schema](./schema.md)
+- [Indexing](./indexing.md)
 - [Segments](./basis.md)
-- [Defining your schema](./schema.md)
 - [Facetting](./facetting.md)
 - [Innerworkings](./innerworkings.md)
   - [Inverted index](./inverted_index.md)

doc/src/avant-propos.md

@@ -31,4 +31,3 @@ relevancy, collapsing, highlighting, spatial search.
   index from a different format.
 
   Tantivy exposes a lot of low level API to do all of these things.
-  

doc/src/schema.md

@@ -1 +1,50 @@
-# Defining your schema
+# Schema
+
+When starting a new project using tantivy, your first step will be to your schema. Be aware that changing it will probably require you to reindex all of your data.
+It is strongly recommended you keep the means to iterate through your original data when this happens.
+
+If not specified otherwise, tantivy does not keep a raw version of your data,
+so the good practise is to rely on a distinct storage to store your
+raw documents.
+
+The schema defines both the type of the fields you are indexing, but also the type of indexing you want to apply to them. The set of search operations that you will be able to perform depends on the way you set up your schema.
+
+Here is what defining your schema could look like.
+
+```Rust
+use tantivy::schema::{Schema, TEXT, STORED, INT_INDEXED};
+
+let mut schema_builder = SchemaBuilder::default();
+let text_field = schema_builder.add_text_field("name", TEXT | STORED);
+let tag_field = schema_builder.add_facet_field("tags");
+let timestamp_field = schema_buider.add_u64_field("timestamp", INT_INDEXED)
+let schema = schema_builder.build();
+```
+
+Notice how adding a new field to your schema builder
+follows the following pattern :
+
+```verbatim
+  schema_builder.add_<fieldtype>_field("<fieldname>", <field_configuration>);
+```
+
+This method returns a `Field` handle that will be used for all kind of 
+
+# Field types
+
+Tantivy currently supports only 4 types.
+
+- `text` (understand `&str`)
+- `u64` and `i64`
+- `HierarchicalFacet`
+
+Let's go into their specificities.
+
+# Text
+
+Full-text search is the bread and butter of search engine.
+The key idea is fairly simple. Your text is broken apart into tokens (that's
+what we call tokenization). Tantivy then keeps track of the list of the documents containing each token.
+
+In order to increase recall you might want to normalize tokens. For instance,
+you most likely want to lowercase your tokens so that documents match the query `cat` regardless of whether your they contain the token `cat` or `Cat`.

src/postings/compression/mod.rs

@@ -266,21 +266,20 @@ pub mod tests {
 mod bench {
 
     use super::*;
-    use rand::Rng;
     use rand::SeedableRng;
-    use rand::XorShiftRng;
+    use rand::{Rng, XorShiftRng};
     use test::Bencher;
 
-    fn generate_array_with_seed(n: usize, ratio: f32, seed_val: u32) -> Vec<u32> {
-        let seed: &[u32; 4] = &[1, 2, 3, seed_val];
+    fn generate_array_with_seed(n: usize, ratio: f64, seed_val: u8) -> Vec<u32> {
+        let seed: &[u8; 16] = &[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,seed_val];
         let mut rng: XorShiftRng = XorShiftRng::from_seed(*seed);
-        (0..u32::max_value())
-            .filter(|_| rng.next_f32() < ratio)
+        (0u32..)
+            .filter(|_| rng.gen_bool(ratio))
             .take(n)
             .collect()
     }
 
-    pub fn generate_array(n: usize, ratio: f32) -> Vec<u32> {
+    pub fn generate_array(n: usize, ratio: f64) -> Vec<u32> {
         generate_array_with_seed(n, ratio, 4)
     }
 
@@ -297,24 +296,23 @@ mod bench {
     fn bench_uncompress(b: &mut Bencher) {
         let mut encoder = BlockEncoder::new();
         let data = generate_array(COMPRESSION_BLOCK_SIZE, 0.1);
-        let (_, compressed) = encoder.compress_block_sorted(&data, 0u32);
+        let (num_bits, compressed) = encoder.compress_block_sorted(&data, 0u32);
         let mut decoder = BlockDecoder::new();
         b.iter(|| {
-            decoder.uncompress_block_sorted(compressed, 0u32);
+            decoder.uncompress_block_sorted(compressed, 0u32, num_bits);
         });
     }
 
     #[test]
     fn test_all_docs_compression_numbits() {
-        for num_bits in 0..33 {
+        for expected_num_bits in 0u8.. {
             let mut data = [0u32; 128];
-            if num_bits > 0 {
-                data[0] = 1 << (num_bits - 1);
+            if expected_num_bits > 0 {
+                data[0] = (1u64 << (expected_num_bits as usize) - 1) as u32;
             }
             let mut encoder = BlockEncoder::new();
-            let compressed = encoder.compress_block_unsorted(&data);
-            assert_eq!(compressed[0] as usize, num_bits);
-            assert_eq!(compressed.len(), compressed_block_size(compressed[0]));
+            let (num_bits, compressed) = encoder.compress_block_unsorted(&data);
+            assert_eq!(compressed.len(), compressed_block_size(num_bits));
         }
     }
 

src/postings/mod.rs

@@ -654,7 +654,7 @@ mod bench {
         });
     }
 
-    fn bench_skip_next(p: f32, b: &mut Bencher) {
+    fn bench_skip_next(p: f64, b: &mut Bencher) {
         let searcher = INDEX.searcher();
         let segment_reader = searcher.segment_reader(0);
         let docs = tests::sample(segment_reader.num_docs(), p);

src/postings/stacker/expull.rs

@@ -175,7 +175,7 @@ mod tests {
 #[cfg(all(test, feature = "unstable"))]
 mod bench {
     use super::ExpUnrolledLinkedList;
-    use tantivy_memory_arena::MemoryArena;
+    use super::super::MemoryArena;
     use test::Bencher;
 
     const NUM_STACK: usize = 10_000;
@@ -199,20 +199,19 @@ mod bench {
 
     #[bench]
     fn bench_push_stack(bench: &mut Bencher) {
-        let heap = MemoryArena::new();
         bench.iter(|| {
+            let mut heap = MemoryArena::new();
             let mut stacks = Vec::with_capacity(100);
             for _ in 0..NUM_STACK {
-                let (_, stack) = heap.allocate_object::<ExpUnrolledLinkedList>();
+                let mut stack = ExpUnrolledLinkedList::new(&mut heap);
                 stacks.push(stack);
             }
             for s in 0..NUM_STACK {
                 for i in 0u32..STACK_SIZE {
                     let t = s * 392017 % NUM_STACK;
-                    stacks[t].push(i, &heap);
+                    stacks[t].push(i, &mut heap);
                 }
             }
-            heap.clear();
         });
     }
 }