Test.

CHANGELOG.md

@@ -1,14 +1,5 @@
 
-Tantivy 0.20.2
-================================
-- Align numerical type priority order on the search side.  [#2088](https://github.com/quickwit-oss/tantivy/issues/2088) (@fmassot)
-- Fix is_child_of function not considering the root facet. [#2086](https://github.com/quickwit-oss/tantivy/issues/2086) (@adamreichhold)
-
-Tantivy 0.20.1
-================================
-- Fix building on windows with mmap [#2070](https://github.com/quickwit-oss/tantivy/issues/2070) (@ChillFish8)
-
-Tantivy 0.20
+Tantivy 0.20 [Unreleased]
 ================================
 #### Bugfixes
 - Fix phrase queries with slop (slop supports now transpositions, algorithm that carries slop so far for num terms > 2) [#2031](https://github.com/quickwit-oss/tantivy/issues/2031)[#2020](https://github.com/quickwit-oss/tantivy/issues/2020)(@PSeitz)
@@ -47,14 +38,12 @@ Tantivy 0.20
     - Add aggregation support for JSON type [#1888](https://github.com/quickwit-oss/tantivy/issues/1888) (@PSeitz)
     - Mixed types support on JSON fields in aggs [#1971](https://github.com/quickwit-oss/tantivy/issues/1971) (@PSeitz)
   - Perf: Fetch blocks of vals in aggregation for all cardinality [#1950](https://github.com/quickwit-oss/tantivy/issues/1950) (@PSeitz)
-  - Allow histogram bounds to be passed as Rfc3339 [#2076](https://github.com/quickwit-oss/tantivy/issues/2076) (@PSeitz)
 - `Searcher` with disabled scoring via `EnableScoring::Disabled` [#1780](https://github.com/quickwit-oss/tantivy/issues/1780) (@shikhar)
 - Enable tokenizer on json fields [#2053](https://github.com/quickwit-oss/tantivy/issues/2053) (@PSeitz)
 - Enforcing "NOT" and "-" queries consistency in UserInputAst [#1609](https://github.com/quickwit-oss/tantivy/issues/1609) (@bazhenov)
 - Faster indexing
   - Refactor tokenization pipeline to use GATs [#1924](https://github.com/quickwit-oss/tantivy/issues/1924) (@trinity-1686a)
   - Faster term hash map [#2058](https://github.com/quickwit-oss/tantivy/issues/2058)[#1940](https://github.com/quickwit-oss/tantivy/issues/1940) (@PSeitz)
-  - tokenizer-api: reduce Tokenizer allocation overhead [#2062](https://github.com/quickwit-oss/tantivy/issues/2062) (@PSeitz)
   - Refactor vint [#2010](https://github.com/quickwit-oss/tantivy/issues/2010) (@PSeitz)
 - Faster search
   - Work in batches of docs on the SegmentCollector (Only for cases without score for now) [#1937](https://github.com/quickwit-oss/tantivy/issues/1937) (@PSeitz)

Cargo.toml

@@ -49,9 +49,9 @@ murmurhash32 = "0.3.0"
 time = { version = "0.3.10", features = ["serde-well-known"] }
 smallvec = "1.8.0"
 rayon = "1.5.2"
-lru = "0.11.0"
+lru = "0.10.0"
 fastdivide = "0.4.0"
-itertools = "0.11.0"
+itertools = "0.10.3"
 measure_time = "0.8.2"
 async-trait = "0.1.53"
 arc-swap = "1.5.0"

columnar/Cargo.toml

@@ -9,7 +9,7 @@ description = "column oriented storage for tantivy"
 categories = ["database-implementations", "data-structures", "compression"]
 
 [dependencies]
-itertools = "0.11.0"
+itertools = "0.10.5"
 fnv = "1.0.7"
 fastdivide = "0.4.0"
 

src/collector/top_score_collector.rs

@@ -14,7 +14,7 @@ use crate::collector::{
 };
 use crate::fastfield::{FastFieldNotAvailableError, FastValue};
 use crate::query::Weight;
-use crate::{DocAddress, DocId, Order, Score, SegmentOrdinal, SegmentReader, TantivyError};
+use crate::{DocAddress, DocId, Score, SegmentOrdinal, SegmentReader, TantivyError};
 
 struct FastFieldConvertCollector<
     TCollector: Collector<Fruit = Vec<(u64, DocAddress)>>,
@@ -23,7 +23,6 @@ struct FastFieldConvertCollector<
     pub collector: TCollector,
     pub field: String,
     pub fast_value: std::marker::PhantomData<TFastValue>,
-    order: Order,
 }
 
 impl<TCollector, TFastValue> Collector for FastFieldConvertCollector<TCollector, TFastValue>
@@ -71,13 +70,7 @@ where
         let raw_result = self.collector.merge_fruits(segment_fruits)?;
         let transformed_result = raw_result
             .into_iter()
-            .map(|(score, doc_address)| {
-                if self.order.is_desc() {
-                    (TFastValue::from_u64(score), doc_address)
-                } else {
-                    (TFastValue::from_u64(u64::MAX - score), doc_address)
-                }
-            })
+            .map(|(score, doc_address)| (TFastValue::from_u64(score), doc_address))
             .collect::<Vec<_>>();
         Ok(transformed_result)
     }
@@ -138,23 +131,16 @@ impl fmt::Debug for TopDocs {
 
 struct ScorerByFastFieldReader {
     sort_column: Arc<dyn ColumnValues<u64>>,
-    order: Order,
 }
 
 impl CustomSegmentScorer<u64> for ScorerByFastFieldReader {
     fn score(&mut self, doc: DocId) -> u64 {
-        let value = self.sort_column.get_val(doc);
-        if self.order.is_desc() {
-            value
-        } else {
-            u64::MAX - value
-        }
+        self.sort_column.get_val(doc)
     }
 }
 
 struct ScorerByField {
     field: String,
-    order: Order,
 }
 
 impl CustomScorer<u64> for ScorerByField {
@@ -171,13 +157,8 @@ impl CustomScorer<u64> for ScorerByField {
             sort_column_opt.ok_or_else(|| FastFieldNotAvailableError {
                 field_name: self.field.clone(),
             })?;
-        let mut default_value = 0u64;
-        if self.order.is_asc() {
-            default_value = u64::MAX;
-        }
         Ok(ScorerByFastFieldReader {
-            sort_column: sort_column.first_or_default_col(default_value),
-            order: self.order.clone(),
+            sort_column: sort_column.first_or_default_col(0u64),
         })
     }
 }
@@ -249,7 +230,7 @@ impl TopDocs {
     ///
     /// ```rust
     /// # use tantivy::schema::{Schema, FAST, TEXT};
-    /// # use tantivy::{doc, Index, DocAddress, Order};
+    /// # use tantivy::{doc, Index, DocAddress};
     /// # use tantivy::query::{Query, QueryParser};
     /// use tantivy::Searcher;
     /// use tantivy::collector::TopDocs;
@@ -287,7 +268,7 @@ impl TopDocs {
     ///     // Note the `rating_field` needs to be a FAST field here.
     ///     let top_books_by_rating = TopDocs
     ///                 ::with_limit(10)
-    ///                  .order_by_fast_field("rating", Order::Desc);
+    ///                  .order_by_u64_field("rating");
     ///
     ///     // ... and here are our documents. Note this is a simple vec.
     ///     // The `u64` in the pair is the value of our fast field for
@@ -307,15 +288,13 @@ impl TopDocs {
     ///
     /// To comfortably work with `u64`s, `i64`s, `f64`s, or `date`s, please refer to
     /// the [.order_by_fast_field(...)](TopDocs::order_by_fast_field) method.
-    fn order_by_u64_field(
+    pub fn order_by_u64_field(
         self,
         field: impl ToString,
-        order: Order,
     ) -> impl Collector<Fruit = Vec<(u64, DocAddress)>> {
         CustomScoreTopCollector::new(
             ScorerByField {
                 field: field.to_string(),
-                order,
             },
             self.0.into_tscore(),
         )
@@ -337,7 +316,7 @@ impl TopDocs {
     ///
     /// ```rust
     /// # use tantivy::schema::{Schema, FAST, TEXT};
-    /// # use tantivy::{doc, Index, DocAddress,Order};
+    /// # use tantivy::{doc, Index, DocAddress};
     /// # use tantivy::query::{Query, AllQuery};
     /// use tantivy::Searcher;
     /// use tantivy::collector::TopDocs;
@@ -375,7 +354,7 @@ impl TopDocs {
     ///     // type `sort_by_field`. revenue_field here is a FAST i64 field.
     ///     let top_company_by_revenue = TopDocs
     ///                 ::with_limit(2)
-    ///                  .order_by_fast_field("revenue", Order::Desc);
+    ///                  .order_by_fast_field("revenue");
     ///
     ///     // ... and here are our documents. Note this is a simple vec.
     ///     // The `i64` in the pair is the value of our fast field for
@@ -393,17 +372,15 @@ impl TopDocs {
     pub fn order_by_fast_field<TFastValue>(
         self,
         fast_field: impl ToString,
-        order: Order,
     ) -> impl Collector<Fruit = Vec<(TFastValue, DocAddress)>>
     where
         TFastValue: FastValue,
     {
-        let u64_collector = self.order_by_u64_field(fast_field.to_string(), order.clone());
+        let u64_collector = self.order_by_u64_field(fast_field.to_string());
         FastFieldConvertCollector {
             collector: u64_collector,
             field: fast_field.to_string(),
             fast_value: PhantomData,
-            order,
         }
     }
 
@@ -744,7 +721,7 @@ mod tests {
     use crate::schema::{Field, Schema, FAST, STORED, TEXT};
     use crate::time::format_description::well_known::Rfc3339;
     use crate::time::OffsetDateTime;
-    use crate::{DateTime, DocAddress, DocId, Index, IndexWriter, Order, Score, SegmentReader};
+    use crate::{DateTime, DocAddress, DocId, Index, IndexWriter, Score, SegmentReader};
 
     fn make_index() -> crate::Result<Index> {
         let mut schema_builder = Schema::builder();
@@ -905,7 +882,7 @@ mod tests {
         });
         let searcher = index.reader()?.searcher();
 
-        let top_collector = TopDocs::with_limit(4).order_by_u64_field(SIZE, Order::Desc);
+        let top_collector = TopDocs::with_limit(4).order_by_u64_field(SIZE);
         let top_docs: Vec<(u64, DocAddress)> = searcher.search(&query, &top_collector)?;
         assert_eq!(
             &top_docs[..],
@@ -944,7 +921,7 @@ mod tests {
         ))?;
         index_writer.commit()?;
         let searcher = index.reader()?.searcher();
-        let top_collector = TopDocs::with_limit(3).order_by_fast_field("birthday", Order::Desc);
+        let top_collector = TopDocs::with_limit(3).order_by_fast_field("birthday");
         let top_docs: Vec<(DateTime, DocAddress)> = searcher.search(&AllQuery, &top_collector)?;
         assert_eq!(
             &top_docs[..],
@@ -974,7 +951,7 @@ mod tests {
         ))?;
         index_writer.commit()?;
         let searcher = index.reader()?.searcher();
-        let top_collector = TopDocs::with_limit(3).order_by_fast_field("altitude", Order::Desc);
+        let top_collector = TopDocs::with_limit(3).order_by_fast_field("altitude");
         let top_docs: Vec<(i64, DocAddress)> = searcher.search(&AllQuery, &top_collector)?;
         assert_eq!(
             &top_docs[..],
@@ -1004,7 +981,7 @@ mod tests {
         ))?;
         index_writer.commit()?;
         let searcher = index.reader()?.searcher();
-        let top_collector = TopDocs::with_limit(3).order_by_fast_field("altitude", Order::Desc);
+        let top_collector = TopDocs::with_limit(3).order_by_fast_field("altitude");
         let top_docs: Vec<(f64, DocAddress)> = searcher.search(&AllQuery, &top_collector)?;
         assert_eq!(
             &top_docs[..],
@@ -1032,7 +1009,7 @@ mod tests {
                 .unwrap();
         });
         let searcher = index.reader().unwrap().searcher();
-        let top_collector = TopDocs::with_limit(4).order_by_u64_field("missing_field", Order::Desc);
+        let top_collector = TopDocs::with_limit(4).order_by_u64_field("missing_field");
         let segment_reader = searcher.segment_reader(0u32);
         top_collector
             .for_segment(0, segment_reader)
@@ -1050,7 +1027,7 @@ mod tests {
         index_writer.commit()?;
         let searcher = index.reader()?.searcher();
         let segment = searcher.segment_reader(0);
-        let top_collector = TopDocs::with_limit(4).order_by_u64_field(SIZE, Order::Desc);
+        let top_collector = TopDocs::with_limit(4).order_by_u64_field(SIZE);
         let err = top_collector.for_segment(0, segment).err().unwrap();
         assert!(matches!(err, crate::TantivyError::InvalidArgument(_)));
         Ok(())
@@ -1067,7 +1044,7 @@ mod tests {
         index_writer.commit()?;
         let searcher = index.reader()?.searcher();
         let segment = searcher.segment_reader(0);
-        let top_collector = TopDocs::with_limit(4).order_by_fast_field::<i64>(SIZE, Order::Desc);
+        let top_collector = TopDocs::with_limit(4).order_by_fast_field::<i64>(SIZE);
         let err = top_collector.for_segment(0, segment).err().unwrap();
         assert!(
             matches!(err, crate::TantivyError::SchemaError(msg) if msg == "Field \"size\" is not a fast field.")
@@ -1129,50 +1106,4 @@ mod tests {
         let query = query_parser.parse_query(query).unwrap();
         (index, query)
     }
-    #[test]
-    fn test_fast_field_ascending_order() -> crate::Result<()> {
-        let mut schema_builder = Schema::builder();
-        let title = schema_builder.add_text_field(TITLE, TEXT);
-        let size = schema_builder.add_u64_field(SIZE, FAST);
-        let schema = schema_builder.build();
-        let (index, query) = index("beer", title, schema, |index_writer| {
-            index_writer
-                .add_document(doc!(
-                    title => "bottle of beer",
-                    size => 12u64,
-                ))
-                .unwrap();
-            index_writer
-                .add_document(doc!(
-                    title => "growler of beer",
-                    size => 64u64,
-                ))
-                .unwrap();
-            index_writer
-                .add_document(doc!(
-                    title => "pint of beer",
-                    size => 16u64,
-                ))
-                .unwrap();
-            index_writer
-                .add_document(doc!(
-                    title => "empty beer",
-                ))
-                .unwrap();
-        });
-        let searcher = index.reader()?.searcher();
-
-        let top_collector = TopDocs::with_limit(4).order_by_fast_field(SIZE, Order::Asc);
-        let top_docs: Vec<(u64, DocAddress)> = searcher.search(&query, &top_collector)?;
-        assert_eq!(
-            &top_docs[..],
-            &[
-                (12, DocAddress::new(0, 0)),
-                (16, DocAddress::new(0, 2)),
-                (64, DocAddress::new(0, 1)),
-                (18446744073709551615, DocAddress::new(0, 3)),
-            ]
-        );
-        Ok(())
-    }
 }

src/fastfield/mod.rs

@@ -1291,28 +1291,4 @@ mod tests {
         let vals: Vec<i64> = column.values_for_doc(0u32).collect();
         assert_eq!(&vals, &[33]);
     }
-
-    #[test]
-    fn check_num_columnar_fields() -> crate::Result<()> {
-        let mut schema_builder = Schema::builder();
-        let id_field = schema_builder.add_text_field("id", FAST);
-        let index = Index::create_in_ram(schema_builder.build());
-        {
-            let mut index_writer = index.writer_with_num_threads(1, 20_000_000)?;
-            index_writer.set_merge_policy(Box::new(NoMergePolicy));
-            index_writer.add_document(doc!(
-                id_field => 1u64,
-            ))?;
-            index_writer.commit()?;
-        }
-
-        let reader = index.reader().unwrap();
-
-        let searcher = reader.searcher();
-        let ff_reader = searcher.segment_reader(0).fast_fields();
-        let fields = ff_reader.u64_lenient_for_type_all(None, "id").unwrap();
-        assert_eq!(fields.len(), 1);
-
-        Ok(())
-    }
 }

src/tokenizer/regex_tokenizer.rs

@@ -49,6 +49,7 @@ use crate::TantivyError;
 pub struct RegexTokenizer {
     regex: Regex,
     token: Token,
+    group: usize,
 }
 
 impl RegexTokenizer {
@@ -59,6 +60,7 @@ impl RegexTokenizer {
             .map(|regex| Self {
                 regex,
                 token: Token::default(),
+                group: 0,
             })
     }
 }

src/tokenizer/split_compound_words.rs

@@ -86,8 +86,6 @@ impl TokenFilter for SplitCompoundWords {
         SplitCompoundWordsFilter {
             dict: self.dict,
             inner: tokenizer,
-            cuts: Vec::new(),
-            parts: Vec::new(),
         }
     }
 }
@@ -96,33 +94,29 @@ impl TokenFilter for SplitCompoundWords {
 pub struct SplitCompoundWordsFilter<T> {
     dict: AhoCorasick,
     inner: T,
-    cuts: Vec<usize>,
-    parts: Vec<Token>,
 }
 
 impl<T: Tokenizer> Tokenizer for SplitCompoundWordsFilter<T> {
-    type TokenStream<'a> = SplitCompoundWordsTokenStream<'a, T::TokenStream<'a>>;
+    type TokenStream<'a> = SplitCompoundWordsTokenStream<T::TokenStream<'a>>;
 
     fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> {
-        self.cuts.clear();
-        self.parts.clear();
         SplitCompoundWordsTokenStream {
             dict: self.dict.clone(),
             tail: self.inner.token_stream(text),
-            cuts: &mut self.cuts,
-            parts: &mut self.parts,
+            cuts: Vec::new(),
+            parts: Vec::new(),
         }
     }
 }
 
-pub struct SplitCompoundWordsTokenStream<'a, T> {
+pub struct SplitCompoundWordsTokenStream<T> {
     dict: AhoCorasick,
     tail: T,
-    cuts: &'a mut Vec<usize>,
-    parts: &'a mut Vec<Token>,
+    cuts: Vec<usize>,
+    parts: Vec<Token>,
 }
 
-impl<'a, T: TokenStream> SplitCompoundWordsTokenStream<'a, T> {
+impl<T: TokenStream> SplitCompoundWordsTokenStream<T> {
     // Will use `self.cuts` to fill `self.parts` if `self.tail.token()`
     // can fully be split into consecutive matches against `self.dict`.
     fn split(&mut self) {
@@ -158,7 +152,7 @@ impl<'a, T: TokenStream> SplitCompoundWordsTokenStream<'a, T> {
     }
 }
 
-impl<'a, T: TokenStream> TokenStream for SplitCompoundWordsTokenStream<'a, T> {
+impl<T: TokenStream> TokenStream for SplitCompoundWordsTokenStream<T> {
     fn advance(&mut self) -> bool {
         self.parts.pop();