accept * as field name

CHANGELOG.md

@@ -1,11 +1,12 @@
 Tantivy 0.23 - Unreleased
 ================================
-Tantivy 0.23 will be backwards compatible with indices created with v0.22 and v0.21.
+Tantivy 0.23 will be backwards compatible with indices created with v0.22 and v0.21. The new minimum rust version will be 1.75.
 
 #### Bugfixes
 - fix potential endless loop in merge [#2457](https://github.com/quickwit-oss/tantivy/pull/2457)(@PSeitz)
 - fix bug that causes out-of-order sstable key. [#2445](https://github.com/quickwit-oss/tantivy/pull/2445)(@fulmicoton)
 - fix ReferenceValue API flaw [#2372](https://github.com/quickwit-oss/tantivy/pull/2372)(@PSeitz)
+- fix `OwnedBytes` debug panic [#2512](https://github.com/quickwit-oss/tantivy/pull/2512)(@b41sh)
 
 #### Breaking API Changes
 - remove index sorting [#2434](https://github.com/quickwit-oss/tantivy/pull/2434)(@PSeitz)
@@ -35,7 +36,15 @@ Tantivy 0.23 will be backwards compatible with indices created with v0.22 and v0
 - make find_field_with_default return json fields without path [#2476](https://github.com/quickwit-oss/tantivy/pull/2476)(@trinity-1686a)
 - feat(query): Make `BooleanQuery` support `minimum_number_should_match` [#2405](https://github.com/quickwit-oss/tantivy/pull/2405)(@LebranceBW)
 
-- **Optional Index in Multivalue Columnar Index** For mostly empty multivalued indices there was a large overhead during creation when iterating all docids (merge case). This is alleviated by placing an optional index in the multivalued index to mark documents that have values. This will slightly increase space and access time. [#2439](https://github.com/quickwit-oss/tantivy/pull/2439)(@PSeitz)
+- **RegexPhraseQuery** 
+`RegexPhraseQuery` supports phrase queries with regex. E.g. query "b.* b.* wolf" matches "big bad wolf". Slop is supported as well: "b.* wolf"~2 matches "big bad wolf" [#2516](https://github.com/quickwit-oss/tantivy/pull/2516)(@PSeitz)
+
+- **Optional Index in Multivalue Columnar Index** 
+For mostly empty multivalued indices there was a large overhead during creation when iterating all docids (merge case). 
+This is alleviated by placing an optional index in the multivalued index to mark documents that have values. 
+This will slightly increase space and access time. [#2439](https://github.com/quickwit-oss/tantivy/pull/2439)(@PSeitz)
+
+- **Store DateTime as nanoseconds in doc store** DateTime in the doc store was truncated to microseconds previously. This removes this truncation, while still keeping backwards compatibility. [#2486](https://github.com/quickwit-oss/tantivy/pull/2486)(@PSeitz)
 
 - **Performace/Memory**
     - lift clauses in LogicalAst for optimized ast during execution [#2449](https://github.com/quickwit-oss/tantivy/pull/2449)(@PSeitz)
@@ -57,12 +66,13 @@ Tantivy 0.23 will be backwards compatible with indices created with v0.22 and v0
 - add bench & test for columnar merging [#2428](https://github.com/quickwit-oss/tantivy/pull/2428)(@PSeitz)
 - Change in Executor API [#2391](https://github.com/quickwit-oss/tantivy/pull/2391)(@fulmicoton)
 - Removed usage of num_cpus [#2387](https://github.com/quickwit-oss/tantivy/pull/2387)(@fulmicoton)
-- use bingang for agg benchmark [#2378](https://github.com/quickwit-oss/tantivy/pull/2378)(@PSeitz)
+- use bingang for agg and stacker benchmark [#2378](https://github.com/quickwit-oss/tantivy/pull/2378)[#2492](https://github.com/quickwit-oss/tantivy/pull/2492)(@PSeitz) 
 - cleanup top level exports [#2382](https://github.com/quickwit-oss/tantivy/pull/2382)(@PSeitz)
 - make convert_to_fast_value_and_append_to_json_term pub [#2370](https://github.com/quickwit-oss/tantivy/pull/2370)(@PSeitz)
 - remove JsonTermWriter [#2238](https://github.com/quickwit-oss/tantivy/pull/2238)(@PSeitz)
 - validate sort by field type [#2336](https://github.com/quickwit-oss/tantivy/pull/2336)(@PSeitz)
 - Fix trait bound of StoreReader::iter [#2360](https://github.com/quickwit-oss/tantivy/pull/2360)(@adamreichold)
+- remove read_postings_no_deletes [#2526](https://github.com/quickwit-oss/tantivy/pull/2526)(@PSeitz)
 
 Tantivy 0.22
 ================================
@@ -717,7 +727,7 @@ Tantivy 0.4.0
 - Raise the limit of number of fields (previously 256 fields) (@fulmicoton)
 - Removed u32 fields. They are replaced by u64 and i64 fields (#65) (@fulmicoton)
 - Optimized skip in SegmentPostings (#130) (@lnicola)
-- Replacing rustc_serialize by serde. Kudos to @KodrAus and @lnicola
+- Replacing rustc_serialize by serde. Kudos to  benchmark@KodrAus and @lnicola
 - Using error-chain (@KodrAus)
 - QueryParser: (@fulmicoton)
   - Explicit error returned when searched for a term that is not indexed

columnar/src/column_index/merge/shuffled.rs

@@ -58,7 +58,7 @@ struct ShuffledIndex<'a> {
     merge_order: &'a ShuffleMergeOrder,
 }
 
-impl<'a> Iterable<u32> for ShuffledIndex<'a> {
+impl Iterable<u32> for ShuffledIndex<'_> {
     fn boxed_iter(&self) -> Box<dyn Iterator<Item = u32> + '_> {
         Box::new(
             self.merge_order
@@ -127,7 +127,7 @@ fn integrate_num_vals(num_vals: impl Iterator<Item = u32>) -> impl Iterator<Item
     )
 }
 
-impl<'a> Iterable<u32> for ShuffledMultivaluedIndex<'a> {
+impl Iterable<u32> for ShuffledMultivaluedIndex<'_> {
     fn boxed_iter(&self) -> Box<dyn Iterator<Item = u32> + '_> {
         let num_vals_per_row = iter_num_values(self.column_indexes, self.merge_order);
         Box::new(integrate_num_vals(num_vals_per_row))

columnar/src/column_index/merge/stacked.rs

@@ -123,7 +123,7 @@ fn get_num_values_iterator<'a>(
     }
 }
 
-impl<'a> Iterable<u32> for StackedStartOffsets<'a> {
+impl Iterable<u32> for StackedStartOffsets<'_> {
     fn boxed_iter(&self) -> Box<dyn Iterator<Item = u32> + '_> {
         let num_values_it = (0..self.column_indexes.len()).flat_map(|columnar_id| {
             let num_docs = self.stack_merge_order.columnar_range(columnar_id).len() as u32;

columnar/src/column_index/optional_index/mod.rs

@@ -86,7 +86,7 @@ pub struct OptionalIndex {
     block_metas: Arc<[BlockMeta]>,
 }
 
-impl<'a> Iterable<u32> for &'a OptionalIndex {
+impl Iterable<u32> for &OptionalIndex {
     fn boxed_iter(&self) -> Box<dyn Iterator<Item = u32> + '_> {
         Box::new(self.iter_rows())
     }
@@ -123,7 +123,7 @@ enum BlockSelectCursor<'a> {
     Sparse(<SparseBlock<'a> as Set<u16>>::SelectCursor<'a>),
 }
 
-impl<'a> BlockSelectCursor<'a> {
+impl BlockSelectCursor<'_> {
     fn select(&mut self, rank: u16) -> u16 {
         match self {
             BlockSelectCursor::Dense(dense_select_cursor) => dense_select_cursor.select(rank),
@@ -141,7 +141,7 @@ pub struct OptionalIndexSelectCursor<'a> {
     num_null_rows_before_block: RowId,
 }
 
-impl<'a> OptionalIndexSelectCursor<'a> {
+impl OptionalIndexSelectCursor<'_> {
     fn search_and_load_block(&mut self, rank: RowId) {
         if rank < self.current_block_end_rank {
             // we are already in the right block
@@ -165,7 +165,7 @@ impl<'a> OptionalIndexSelectCursor<'a> {
     }
 }
 
-impl<'a> SelectCursor<RowId> for OptionalIndexSelectCursor<'a> {
+impl SelectCursor<RowId> for OptionalIndexSelectCursor<'_> {
     fn select(&mut self, rank: RowId) -> RowId {
         self.search_and_load_block(rank);
         let index_in_block = (rank - self.num_null_rows_before_block) as u16;
@@ -505,7 +505,7 @@ fn deserialize_optional_index_block_metadatas(
         non_null_rows_before_block += num_non_null_rows;
     }
     block_metas.resize(
-        ((num_rows + ELEMENTS_PER_BLOCK - 1) / ELEMENTS_PER_BLOCK) as usize,
+        num_rows.div_ceil(ELEMENTS_PER_BLOCK) as usize,
         BlockMeta {
             non_null_rows_before_block,
             start_byte_offset,

columnar/src/column_index/optional_index/set_block/dense.rs

@@ -23,7 +23,6 @@ fn set_bit_at(input: &mut u64, n: u16) {
 ///
 /// When translating a dense index to the original index, we can use the offset to find the correct
 /// block. Direct computation is not possible, but we can employ a linear or binary search.
-
 const ELEMENTS_PER_MINI_BLOCK: u16 = 64;
 const MINI_BLOCK_BITVEC_NUM_BYTES: usize = 8;
 const MINI_BLOCK_OFFSET_NUM_BYTES: usize = 2;
@@ -109,7 +108,7 @@ pub struct DenseBlockSelectCursor<'a> {
     dense_block: DenseBlock<'a>,
 }
 
-impl<'a> SelectCursor<u16> for DenseBlockSelectCursor<'a> {
+impl SelectCursor<u16> for DenseBlockSelectCursor<'_> {
     #[inline]
     fn select(&mut self, rank: u16) -> u16 {
         self.block_id = self
@@ -175,7 +174,7 @@ impl<'a> Set<u16> for DenseBlock<'a> {
     }
 }
 
-impl<'a> DenseBlock<'a> {
+impl DenseBlock<'_> {
     #[inline]
     fn mini_block(&self, mini_block_id: u16) -> DenseMiniBlock {
         let data_start_pos = mini_block_id as usize * MINI_BLOCK_NUM_BYTES;

columnar/src/column_index/optional_index/set_block/sparse.rs

@@ -31,7 +31,7 @@ impl<'a> SelectCursor<u16> for SparseBlock<'a> {
     }
 }
 
-impl<'a> Set<u16> for SparseBlock<'a> {
+impl Set<u16> for SparseBlock<'_> {
     type SelectCursor<'b>
         = Self
     where Self: 'b;
@@ -69,7 +69,7 @@ fn get_u16(data: &[u8], byte_position: usize) -> u16 {
     u16::from_le_bytes(bytes)
 }
 
-impl<'a> SparseBlock<'a> {
+impl SparseBlock<'_> {
     #[inline(always)]
     fn value_at_idx(&self, data: &[u8], idx: u16) -> u16 {
         let start_offset: usize = idx as usize * 2;

columnar/src/column_index/serialize.rs

@@ -31,7 +31,7 @@ pub enum SerializableColumnIndex<'a> {
     Multivalued(SerializableMultivalueIndex<'a>),
 }
 
-impl<'a> SerializableColumnIndex<'a> {
+impl SerializableColumnIndex<'_> {
     pub fn get_cardinality(&self) -> Cardinality {
         match self {
             SerializableColumnIndex::Full => Cardinality::Full,

columnar/src/column_values/merge.rs

@@ -10,7 +10,7 @@ pub(crate) struct MergedColumnValues<'a, T> {
     pub(crate) merge_row_order: &'a MergeRowOrder,
 }
 
-impl<'a, T: Copy + PartialOrd + Debug + 'static> Iterable<T> for MergedColumnValues<'a, T> {
+impl<T: Copy + PartialOrd + Debug + 'static> Iterable<T> for MergedColumnValues<'_, T> {
     fn boxed_iter(&self) -> Box<dyn Iterator<Item = T> + '_> {
         match self.merge_row_order {
             MergeRowOrder::Stack(_) => Box::new(

columnar/src/column_values/u64_based/blockwise_linear.rs

@@ -39,7 +39,7 @@ impl BinarySerializable for Block {
 }
 
 fn compute_num_blocks(num_vals: u32) -> u32 {
-    (num_vals + BLOCK_SIZE - 1) / BLOCK_SIZE
+    num_vals.div_ceil(BLOCK_SIZE)
 }
 
 pub struct BlockwiseLinearEstimator {

columnar/src/columnar/merge/merge_dict_column.rs

@@ -39,7 +39,7 @@ struct RemappedTermOrdinalsValues<'a> {
     merge_row_order: &'a MergeRowOrder,
 }
 
-impl<'a> Iterable for RemappedTermOrdinalsValues<'a> {
+impl Iterable for RemappedTermOrdinalsValues<'_> {
     fn boxed_iter(&self) -> Box<dyn Iterator<Item = u64> + '_> {
         match self.merge_row_order {
             MergeRowOrder::Stack(_) => self.boxed_iter_stacked(),
@@ -50,7 +50,7 @@ impl<'a> Iterable for RemappedTermOrdinalsValues<'a> {
     }
 }
 
-impl<'a> RemappedTermOrdinalsValues<'a> {
+impl RemappedTermOrdinalsValues<'_> {
     fn boxed_iter_stacked(&self) -> Box<dyn Iterator<Item = u64> + '_> {
         let iter = self
             .bytes_columns

columnar/src/columnar/merge/term_merger.rs

@@ -10,13 +10,13 @@ pub struct HeapItem<'a> {
     pub segment_ord: usize,
 }
 
-impl<'a> PartialEq for HeapItem<'a> {
+impl PartialEq for HeapItem<'_> {
     fn eq(&self, other: &Self) -> bool {
         self.segment_ord == other.segment_ord
     }
 }
 
-impl<'a> Eq for HeapItem<'a> {}
+impl Eq for HeapItem<'_> {}
 
 impl<'a> PartialOrd for HeapItem<'a> {
     fn partial_cmp(&self, other: &HeapItem<'a>) -> Option<Ordering> {

columnar/src/columnar/writer/serializer.rs

@@ -67,7 +67,7 @@ pub struct ColumnSerializer<'a, W: io::Write> {
     start_offset: u64,
 }
 
-impl<'a, W: io::Write> ColumnSerializer<'a, W> {
+impl<W: io::Write> ColumnSerializer<'_, W> {
     pub fn finalize(self) -> io::Result<()> {
         let end_offset: u64 = self.columnar_serializer.wrt.written_bytes();
         let byte_range = self.start_offset..end_offset;
@@ -80,7 +80,7 @@ impl<'a, W: io::Write> ColumnSerializer<'a, W> {
     }
 }
 
-impl<'a, W: io::Write> io::Write for ColumnSerializer<'a, W> {
+impl<W: io::Write> io::Write for ColumnSerializer<'_, W> {
     fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
         self.columnar_serializer.wrt.write(buf)
     }

columnar/src/iterable.rs

@@ -7,7 +7,7 @@ pub trait Iterable<T = u64> {
     fn boxed_iter(&self) -> Box<dyn Iterator<Item = T> + '_>;
 }
 
-impl<'a, T: Copy> Iterable<T> for &'a [T] {
+impl<T: Copy> Iterable<T> for &[T] {
     fn boxed_iter(&self) -> Box<dyn Iterator<Item = T> + '_> {
         Box::new(self.iter().copied())
     }

common/src/writer.rs

@@ -87,7 +87,7 @@ impl<W: TerminatingWrite> TerminatingWrite for BufWriter<W> {
     }
 }
 
-impl<'a> TerminatingWrite for &'a mut Vec<u8> {
+impl TerminatingWrite for &mut Vec<u8> {
     fn terminate_ref(&mut self, _a: AntiCallToken) -> io::Result<()> {
         self.flush()
     }

query-grammar/src/query_grammar.rs

@@ -6,7 +6,7 @@ use nom::bytes::complete::tag;
 use nom::character::complete::{
     anychar, char, digit1, multispace0, multispace1, none_of, one_of, satisfy, u32,
 };
-use nom::combinator::{eof, map, map_res, opt, peek, recognize, value, verify};
+use nom::combinator::{eof, map, map_res, not, opt, peek, recognize, value, verify};
 use nom::error::{Error, ErrorKind};
 use nom::multi::{many0, many1, separated_list0};
 use nom::sequence::{delimited, preceded, separated_pair, terminated, tuple};
@@ -20,7 +20,7 @@ use crate::Occur;
 // Note: '-' char is only forbidden at the beginning of a field name, would be clearer to add it to
 // special characters.
 const SPECIAL_CHARS: &[char] = &[
-    '+', '^', '`', ':', '{', '}', '"', '\'', '[', ']', '(', ')', '!', '\\', '*', ' ',
+    '+', '^', '`', ':', '{', '}', '"', '\'', '[', ']', '(', ')', '!', '\\', ' ',
 ];
 
 /// consume a field name followed by colon. Return the field name with escape sequence
@@ -679,7 +679,10 @@ fn negate(expr: UserInputAst) -> UserInputAst {
 fn leaf(inp: &str) -> IResult<&str, UserInputAst> {
     alt((
         delimited(char('('), ast, char(')')),
-        map(char('*'), |_| UserInputAst::from(UserInputLeaf::All)),
+        preceded(
+            peek(not(tag("*:"))),
+            map(char('*'), |_| UserInputAst::from(UserInputLeaf::All)),
+        ),
         map(preceded(tuple((tag("NOT"), multispace1)), leaf), negate),
         literal,
     ))(inp)
@@ -700,7 +703,13 @@ fn leaf_infallible(inp: &str) -> JResult<&str, Option<UserInputAst>> {
                 ),
             ),
             (
-                value((), char('*')),
+                value(
+                    (),
+                    preceded(
+                        peek(not(tag("*:"))), // Fail if `*:` is detected
+                        char('*'),            // Match standalone `*`
+                    ),
+                ),
                 map(nothing, |_| {
                     (Some(UserInputAst::from(UserInputLeaf::All)), Vec::new())
                 }),
@@ -1222,6 +1231,7 @@ mod test {
 
     #[test]
     fn test_field_name() {
+        assert_eq!(super::field_name("*:a"), Ok(("a", "*".to_string())));
         assert_eq!(
             super::field_name(".my.field.name:a"),
             Ok(("a", ".my.field.name".to_string()))
@@ -1497,6 +1507,11 @@ mod test {
         test_is_parse_err(r#"field:(+a -"b c""#, r#"(+"field":a -"field":"b c")"#);
     }
 
+    #[test]
+    fn field_re_specification() {
+        test_parse_query_to_ast_helper(r#"field:(abc AND b:cde)"#, r#"(+"field":abc +"b":cde)"#);
+    }
+
     #[test]
     fn test_parse_query_single_term() {
         test_parse_query_to_ast_helper("abc", "abc");
@@ -1522,6 +1537,11 @@ mod test {
         test_parse_query_to_ast_helper("abc:toto", "\"abc\":toto");
     }
 
+    #[test]
+    fn all_field_star() {
+        test_parse_query_to_ast_helper("*:toto", "\"*\":toto");
+    }
+
     #[test]
     fn test_phrase_with_field() {
         test_parse_query_to_ast_helper("abc:\"happy tax payer\"", "\"abc\":\"happy tax payer\"");

sstable/src/streamer.rs

@@ -161,7 +161,7 @@ where
     _lifetime: std::marker::PhantomData<&'a ()>,
 }
 
-impl<'a, TSSTable> Streamer<'a, TSSTable, AlwaysMatch>
+impl<TSSTable> Streamer<'_, TSSTable, AlwaysMatch>
 where TSSTable: SSTable
 {
     pub fn empty() -> Self {
@@ -178,7 +178,7 @@ where TSSTable: SSTable
     }
 }
 
-impl<'a, TSSTable, A> Streamer<'a, TSSTable, A>
+impl<TSSTable, A> Streamer<'_, TSSTable, A>
 where
     A: Automaton,
     A::State: Clone,

stacker/src/expull.rs

@@ -74,7 +74,7 @@ fn ensure_capacity<'a>(
     eull.remaining_cap = allocate as u16;
 }
 
-impl<'a> ExpUnrolledLinkedListWriter<'a> {
+impl ExpUnrolledLinkedListWriter<'_> {
     #[inline]
     pub fn write_u32_vint(&mut self, val: u32) {
         let mut buf = [0u8; 8];

tokenizer-api/src/lib.rs

@@ -63,7 +63,7 @@ pub trait Tokenizer: 'static + Clone + Send + Sync {
 /// Simple wrapper of `Box<dyn TokenStream + 'a>`.
 pub struct BoxTokenStream<'a>(Box<dyn TokenStream + 'a>);
 
-impl<'a> TokenStream for BoxTokenStream<'a> {
+impl TokenStream for BoxTokenStream<'_> {
     fn advance(&mut self) -> bool {
         self.0.advance()
     }
@@ -90,7 +90,7 @@ impl<'a> Deref for BoxTokenStream<'a> {
         &*self.0
     }
 }
-impl<'a> DerefMut for BoxTokenStream<'a> {
+impl DerefMut for BoxTokenStream<'_> {
     fn deref_mut(&mut self) -> &mut Self::Target {
         &mut *self.0
     }