added a simple bench for the default analyzer

Cargo.toml

@@ -57,6 +57,7 @@ rand = "0.8"
 maplit = "1"
 matches = "0.1.8"
 proptest = "0.10"
+criterion = "0.3"
 
 [dev-dependencies.fail]
 version = "0.4"
@@ -97,3 +98,7 @@ travis-ci = { repository = "tantivy-search/tantivy" }
 name = "failpoints"
 path = "tests/failpoints/mod.rs"
 required-features = ["fail/failpoints"]
+
+[[bench]]
+name = "analyzer"
+harness = false

benches/analyzer.rs

@@ -0,0 +1,22 @@
+use criterion::{criterion_group, criterion_main, Criterion};
+use tantivy::tokenizer::TokenizerManager;
+
+const ALICE_TXT: &'static str = include_str!("alice.txt");
+
+pub fn criterion_benchmark(c: &mut Criterion) {
+    let tokenizer_manager = TokenizerManager::default();
+    let tokenizer = tokenizer_manager.get("default").unwrap();
+    c.bench_function("default-tokenize-alice", |b| {
+        b.iter(|| {
+            let mut word_count = 0;
+            let mut token_stream = tokenizer.token_stream(ALICE_TXT);
+            while token_stream.advance() {
+                word_count += 1;
+            }
+            assert_eq!(word_count, 30_731);
+        })
+    });
+}
+
+criterion_group!(benches, criterion_benchmark);
+criterion_main!(benches);