Removing the need for column on Multivalued blabla

.github/workflows/test.yml

@@ -48,7 +48,7 @@ jobs:
     strategy:
       matrix:
         features: [
-            { label: "all", flags: "mmap,stopwords,brotli-compression,lz4-compression,snappy-compression,zstd-compression,failpoints" },
+            { label: "all", flags: "mmap,brotli-compression,lz4-compression,snappy-compression,zstd-compression,failpoints" },
             { label: "quickwit", flags: "mmap,quickwit,failpoints" }
         ]
 

CHANGELOG.md

@@ -24,11 +24,10 @@ Tantivy 0.19
   - Add support for keyed parameter in range and histgram aggregations [#1424](https://github.com/quickwit-oss/tantivy/pull/1424) (@k-yomo)
   - Add aggregation bucket limit [#1363](https://github.com/quickwit-oss/tantivy/pull/1363) (@PSeitz)
 - Faster indexing
-  - [#1610](https://github.com/quickwit-oss/tantivy/pull/1610) (@PSeitz)
-  - [#1594](https://github.com/quickwit-oss/tantivy/pull/1594) (@PSeitz)
-  - [#1582](https://github.com/quickwit-oss/tantivy/pull/1582) (@PSeitz)
-  - [#1611](https://github.com/quickwit-oss/tantivy/pull/1611) (@PSeitz)
-  - Added a pre-configured stop word filter for various language [#1666](https://github.com/quickwit-oss/tantivy/pull/1666) (@adamreichold)
+  - [#1610](https://github.com/quickwit-oss/tantivy/pull/1610 (@PSeitz)
+  - [#1594](https://github.com/quickwit-oss/tantivy/pull/1594 (@PSeitz)
+  - [#1582](https://github.com/quickwit-oss/tantivy/pull/1582 (@PSeitz)
+  - [#1611](https://github.com/quickwit-oss/tantivy/pull/1611 (@PSeitz)
 
 Tantivy 0.18
 ================================

Cargo.toml

@@ -91,9 +91,8 @@ debug-assertions = true
 overflow-checks = true
 
 [features]
-default = ["mmap", "stopwords", "lz4-compression"]
+default = ["mmap", "lz4-compression" ]
 mmap = ["fs2", "tempfile", "memmap2"]
-stopwords = []
 
 brotli-compression = ["brotli"]
 lz4-compression = ["lz4_flex"]

examples/custom_collector.rs

@@ -9,7 +9,7 @@
 
 use std::sync::Arc;
 
-use fastfield_codecs::OptionalColumn;
+use fastfield_codecs::Column;
 // ---
 // Importing tantivy...
 use tantivy::collector::{Collector, SegmentCollector};
@@ -97,7 +97,7 @@ impl Collector for StatsCollector {
 }
 
 struct StatsSegmentCollector {
-    fast_field_reader: Arc<dyn OptionalColumn<u64>>,
+    fast_field_reader: Arc<dyn Column<u64>>,
     stats: Stats,
 }
 
@@ -105,12 +105,10 @@ impl SegmentCollector for StatsSegmentCollector {
     type Fruit = Option<Stats>;
 
     fn collect(&mut self, doc: u32, _score: Score) {
-        if let Some(value) = self.fast_field_reader.get_val(doc) {
-            let value = value as f64;
-            self.stats.count += 1;
-            self.stats.sum += value;
-            self.stats.squared_sum += value * value;
-        }
+        let value = self.fast_field_reader.get_val(doc) as f64;
+        self.stats.count += 1;
+        self.stats.sum += value;
+        self.stats.squared_sum += value * value;
     }
 
     fn harvest(self) -> <Self as SegmentCollector>::Fruit {

examples/warmer.rs

@@ -51,7 +51,7 @@ impl Warmer for DynamicPriceColumn {
             let product_id_reader = segment.fast_fields().u64(self.field)?;
             let product_ids: Vec<ProductId> = segment
                 .doc_ids_alive()
-                .flat_map(|doc| product_id_reader.get_val(doc))
+                .map(|doc| product_id_reader.get_val(doc))
                 .collect();
             let mut prices_it = self.price_fetcher.fetch_prices(&product_ids).into_iter();
             let mut price_vals: Vec<Price> = Vec::new();

fastfield_codecs/benches/bench.rs

@@ -41,7 +41,7 @@ mod tests {
     ) -> Arc<dyn Column<T>> {
         let mut buffer = Vec::new();
         serialize(VecColumn::from(&column), &mut buffer, &ALL_CODEC_TYPES).unwrap();
-        open(OwnedBytes::new(buffer)).unwrap().to_full().unwrap()
+        open(OwnedBytes::new(buffer)).unwrap()
     }
 
     #[bench]
@@ -103,7 +103,7 @@ mod tests {
         let iter_gen = || data.iter().cloned();
         serialize_u128(iter_gen, data.len() as u32, &mut out).unwrap();
         let out = OwnedBytes::new(out);
-        open_u128::<u128>(out).unwrap().to_full().unwrap()
+        open_u128::<u128>(out).unwrap()
     }
 
     #[bench]

fastfield_codecs/src/column.rs

@@ -70,6 +70,11 @@ pub trait Column<T: PartialOrd = u64>: Send + Sync {
     /// The number of values in the column.
     fn num_vals(&self) -> u32;
 
+    /// The number of docs in the column. For single value columns this equals num_vals.
+    fn num_docs(&self) -> u32 {
+        self.num_vals()
+    }
+
     /// Returns a iterator over the data
     fn iter<'a>(&'a self) -> Box<dyn Iterator<Item = T> + 'a> {
         Box::new((0..self.num_vals()).map(|idx| self.get_val(idx)))

fastfield_codecs/src/compact_space/mod.rs

@@ -456,7 +456,6 @@ impl CompactSpaceDecompressor {
 mod tests {
 
     use super::*;
-    use crate::serialize::U128Header;
     use crate::{open_u128, serialize_u128};
 
     #[test]
@@ -502,8 +501,7 @@ mod tests {
         assert_eq!(amplitude, 2);
     }
 
-    fn test_all(mut data: OwnedBytes, expected: &[u128]) {
-        let _header = U128Header::deserialize(&mut data);
+    fn test_all(data: OwnedBytes, expected: &[u128]) {
         let decompressor = CompactSpaceDecompressor::open(data).unwrap();
         for (idx, expected_val) in expected.iter().cloned().enumerate() {
             let val = decompressor.get(idx as u32);
@@ -558,8 +556,7 @@ mod tests {
             4_000_211_222u128,
             333u128,
         ];
-        let mut data = test_aux_vals(vals);
-        let _header = U128Header::deserialize(&mut data);
+        let data = test_aux_vals(vals);
         let decomp = CompactSpaceDecompressor::open(data).unwrap();
         let complete_range = 0..vals.len() as u32;
         for (pos, val) in vals.iter().enumerate() {
@@ -684,8 +681,7 @@ mod tests {
             4_000_211_222u128,
             333u128,
         ];
-        let mut data = test_aux_vals(vals);
-        let _header = U128Header::deserialize(&mut data);
+        let data = test_aux_vals(vals);
         let decomp = CompactSpaceDecompressor::open(data).unwrap();
         let complete_range = 0..vals.len() as u32;
         assert_eq!(
@@ -731,10 +727,7 @@ mod tests {
         ];
         let mut out = Vec::new();
         serialize_u128(|| vals.iter().cloned(), vals.len() as u32, &mut out).unwrap();
-        let decomp = open_u128::<u128>(OwnedBytes::new(out))
-            .unwrap()
-            .to_full()
-            .unwrap();
+        let decomp = open_u128::<u128>(OwnedBytes::new(out)).unwrap();
         let complete_range = 0..vals.len() as u32;
 
         assert_eq!(

fastfield_codecs/src/gcd.rs

@@ -59,11 +59,11 @@ mod tests {
         crate::serialize(VecColumn::from(&vals), &mut buffer, &[codec_type])?;
         let buffer = OwnedBytes::new(buffer);
         let column = crate::open::<i64>(buffer.clone())?;
-        assert_eq!(column.get_val(0), Some(-4000i64));
-        assert_eq!(column.get_val(1), Some(-3000i64));
-        assert_eq!(column.get_val(2), Some(-2000i64));
-        assert_eq!(column.max_value(), Some((num_vals as i64 - 5) * 1000));
-        assert_eq!(column.min_value(), Some(-4000i64));
+        assert_eq!(column.get_val(0), -4000i64);
+        assert_eq!(column.get_val(1), -3000i64);
+        assert_eq!(column.get_val(2), -2000i64);
+        assert_eq!(column.max_value(), (num_vals as i64 - 5) * 1000);
+        assert_eq!(column.min_value(), -4000i64);
 
         // Can't apply gcd
         let mut buffer_without_gcd = Vec::new();
@@ -101,11 +101,11 @@ mod tests {
         crate::serialize(VecColumn::from(&vals), &mut buffer, &[codec_type])?;
         let buffer = OwnedBytes::new(buffer);
         let column = crate::open::<u64>(buffer.clone())?;
-        assert_eq!(column.get_val(0), Some(1000u64));
-        assert_eq!(column.get_val(1), Some(2000u64));
-        assert_eq!(column.get_val(2), Some(3000u64));
-        assert_eq!(column.max_value(), Some(num_vals as u64 * 1000));
-        assert_eq!(column.min_value(), Some(1000u64));
+        assert_eq!(column.get_val(0), 1000u64);
+        assert_eq!(column.get_val(1), 2000u64);
+        assert_eq!(column.get_val(2), 3000u64);
+        assert_eq!(column.max_value(), num_vals as u64 * 1000);
+        assert_eq!(column.min_value(), 1000u64);
 
         // Can't apply gcd
         let mut buffer_without_gcd = Vec::new();

fastfield_codecs/src/lib.rs

@@ -24,10 +24,8 @@ use monotonic_mapping::{
     StrictlyMonotonicMappingInverter, StrictlyMonotonicMappingToInternal,
     StrictlyMonotonicMappingToInternalBaseval, StrictlyMonotonicMappingToInternalGCDBaseval,
 };
-pub use optional_column::OptionalColumn;
-use optional_column::ToOptionalColumn;
 use ownedbytes::OwnedBytes;
-use serialize::{Header, U128Header};
+use serialize::Header;
 
 mod bitpacked;
 mod blockwise_linear;
@@ -36,7 +34,6 @@ mod line;
 mod linear;
 mod monotonic_mapping;
 mod monotonic_mapping_u128;
-mod optional_column;
 
 mod column;
 mod gcd;
@@ -95,60 +92,20 @@ impl FastFieldCodecType {
     }
 }
 
-#[derive(PartialEq, Eq, PartialOrd, Ord, Debug, Clone, Copy)]
-#[repr(u8)]
-/// Available codecs to use to encode the u128 (via [`MonotonicallyMappableToU128`]) converted data.
-pub enum U128FastFieldCodecType {
-    /// This codec takes a large number space (u128) and reduces it to a compact number space, by
-    /// removing the holes.
-    CompactSpace = 1,
-}
-
-impl BinarySerializable for U128FastFieldCodecType {
-    fn serialize<W: Write>(&self, wrt: &mut W) -> io::Result<()> {
-        self.to_code().serialize(wrt)
-    }
-
-    fn deserialize<R: io::Read>(reader: &mut R) -> io::Result<Self> {
-        let code = u8::deserialize(reader)?;
-        let codec_type: Self = Self::from_code(code)
-            .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "Unknown code `{code}.`"))?;
-        Ok(codec_type)
-    }
-}
-
-impl U128FastFieldCodecType {
-    pub(crate) fn to_code(self) -> u8 {
-        self as u8
-    }
-
-    pub(crate) fn from_code(code: u8) -> Option<Self> {
-        match code {
-            1 => Some(Self::CompactSpace),
-            _ => None,
-        }
-    }
-}
-
 /// Returns the correct codec reader wrapped in the `Arc` for the data.
 pub fn open_u128<Item: MonotonicallyMappableToU128>(
-    mut bytes: OwnedBytes,
-) -> io::Result<Arc<dyn OptionalColumn<Item>>> {
-    let header = U128Header::deserialize(&mut bytes)?;
-    assert_eq!(header.codec_type, U128FastFieldCodecType::CompactSpace);
+    bytes: OwnedBytes,
+) -> io::Result<Arc<dyn Column<Item>>> {
     let reader = CompactSpaceDecompressor::open(bytes)?;
     let inverted: StrictlyMonotonicMappingInverter<StrictlyMonotonicMappingToInternal<Item>> =
         StrictlyMonotonicMappingToInternal::<Item>::new().into();
-
-    Ok(Arc::new(ToOptionalColumn::new(Arc::new(
-        monotonic_map_column(reader, inverted),
-    ))))
+    Ok(Arc::new(monotonic_map_column(reader, inverted)))
 }
 
 /// Returns the correct codec reader wrapped in the `Arc` for the data.
 pub fn open<T: MonotonicallyMappableToU64>(
     mut bytes: OwnedBytes,
-) -> io::Result<Arc<dyn OptionalColumn<T>>> {
+) -> io::Result<Arc<dyn Column<T>>> {
     let header = Header::deserialize(&mut bytes)?;
     match header.codec_type {
         FastFieldCodecType::Bitpacked => open_specific_codec::<BitpackedCodec, _>(bytes, &header),
@@ -162,7 +119,7 @@ pub fn open<T: MonotonicallyMappableToU64>(
 fn open_specific_codec<C: FastFieldCodec, Item: MonotonicallyMappableToU64>(
     bytes: OwnedBytes,
     header: &Header,
-) -> io::Result<Arc<dyn OptionalColumn<Item>>> {
+) -> io::Result<Arc<dyn Column<Item>>> {
     let normalized_header = header.normalized();
     let reader = C::open_from_bytes(bytes, normalized_header)?;
     let min_value = header.min_value;
@@ -170,16 +127,12 @@ fn open_specific_codec<C: FastFieldCodec, Item: MonotonicallyMappableToU64>(
         let mapping = StrictlyMonotonicMappingInverter::from(
             StrictlyMonotonicMappingToInternalGCDBaseval::new(gcd.get(), min_value),
         );
-        Ok(Arc::new(ToOptionalColumn::new(Arc::new(
-            monotonic_map_column(reader, mapping),
-        ))))
+        Ok(Arc::new(monotonic_map_column(reader, mapping)))
     } else {
         let mapping = StrictlyMonotonicMappingInverter::from(
             StrictlyMonotonicMappingToInternalBaseval::new(min_value),
         );
-        Ok(Arc::new(ToOptionalColumn::new(Arc::new(
-            monotonic_map_column(reader, mapping),
-        ))))
+        Ok(Arc::new(monotonic_map_column(reader, mapping)))
     }
 }
 
@@ -250,9 +203,8 @@ mod tests {
         for (doc, orig_val) in data.iter().copied().enumerate() {
             let val = reader.get_val(doc as u32);
             assert_eq!(
-                val,
-                Some(orig_val),
-                "val `{val:?}` does not match orig_val {orig_val:?}, in data set {name}, data \
+                val, orig_val,
+                "val `{val}` does not match orig_val {orig_val:?}, in data set {name}, data \
                  `{data:?}`",
             );
         }

fastfield_codecs/src/main.rs

@@ -113,10 +113,7 @@ fn bench_ip() {
         (data.len() * 8) as f32 / dataset.len() as f32
     );
 
-    let decompressor = open_u128::<u128>(OwnedBytes::new(data))
-        .unwrap()
-        .to_full()
-        .unwrap();
+    let decompressor = open_u128::<u128>(OwnedBytes::new(data)).unwrap();
     // Sample some ranges
     let mut doc_values = Vec::new();
     for value in dataset.iter().take(1110).skip(1100).cloned() {

fastfield_codecs/src/optional_column.rs

@@ -1,119 +0,0 @@
-use std::ops::{Range, RangeInclusive};
-use std::sync::Arc;
-
-use crate::Column;
-
-/// `OptionalColumn` provides columnar access on a field.
-pub trait OptionalColumn<T: PartialOrd = u64>: Send + Sync {
-    /// Return the value associated with the given idx.
-    ///
-    /// This accessor should return as fast as possible.
-    ///
-    /// # Panics
-    ///
-    /// May panic if `idx` is greater than the column length.
-    fn get_val(&self, idx: u32) -> Option<T>;
-
-    /// Fills an output buffer with the fast field values
-    /// associated with the `DocId` going from
-    /// `start` to `start + output.len()`.
-    ///
-    /// # Panics
-    ///
-    /// Must panic if `start + output.len()` is greater than
-    /// the segment's `maxdoc`.
-    fn get_range(&self, start: u64, output: &mut [Option<T>]) {
-        for (out, idx) in output.iter_mut().zip(start..) {
-            *out = self.get_val(idx as u32);
-        }
-    }
-
-    /// Return the positions of values which are in the provided range.
-    fn get_docids_for_value_range(
-        &self,
-        value_range: RangeInclusive<T>,
-        doc_id_range: Range<u32>,
-        positions: &mut Vec<u32>,
-    ) {
-        let doc_id_range = doc_id_range.start..doc_id_range.end.min(self.num_vals());
-
-        for idx in doc_id_range {
-            let val = self.get_val(idx);
-            if let Some(val) = val {
-                if value_range.contains(&val) {
-                    positions.push(idx);
-                }
-            }
-        }
-    }
-
-    /// Returns the minimum value for this fast field.
-    ///
-    /// This min_value may not be exact.
-    /// For instance, the min value does not take in account of possible
-    /// deleted document. All values are however guaranteed to be higher than
-    /// `.min_value()`.
-    fn min_value(&self) -> Option<T>;
-
-    /// Returns the maximum value for this fast field.
-    ///
-    /// This max_value may not be exact.
-    /// For instance, the max value does not take in account of possible
-    /// deleted document. All values are however guaranteed to be higher than
-    /// `.max_value()`.
-    fn max_value(&self) -> Option<T>;
-
-    /// The number of values including `None` in the column.
-    fn num_vals(&self) -> u32;
-
-    /// Returns a iterator over the data
-    fn iter<'a>(&'a self) -> Box<dyn Iterator<Item = Option<T>> + 'a> {
-        Box::new((0..self.num_vals()).map(|idx| self.get_val(idx)))
-    }
-
-    /// return full column if all values are set and is not empty
-    fn to_full(&self) -> Option<Arc<dyn Column<T>>> {
-        None
-    }
-}
-
-/// Temporary wrapper to migrate to optional column
-pub(crate) struct ToOptionalColumn<T> {
-    column: Arc<dyn Column<T>>,
-}
-
-impl<T: PartialOrd> ToOptionalColumn<T> {
-    pub(crate) fn new(column: Arc<dyn Column<T>>) -> Self {
-        Self { column }
-    }
-}
-
-impl<T: PartialOrd> OptionalColumn<T> for ToOptionalColumn<T> {
-    #[inline]
-    fn get_val(&self, idx: u32) -> Option<T> {
-        let val = self.column.get_val(idx);
-        Some(val)
-    }
-
-    fn min_value(&self) -> Option<T> {
-        let min_value = self.column.min_value();
-        Some(min_value)
-    }
-
-    fn max_value(&self) -> Option<T> {
-        let max_value = self.column.max_value();
-        Some(max_value)
-    }
-
-    fn num_vals(&self) -> u32 {
-        self.column.num_vals()
-    }
-
-    fn iter(&self) -> Box<dyn Iterator<Item = Option<T>> + '_> {
-        Box::new(self.column.iter().map(|el| Some(el)))
-    }
-    /// return full column if all values are set and is not empty
-    fn to_full(&self) -> Option<Arc<dyn Column<T>>> {
-        Some(self.column.clone())
-    }
-}

fastfield_codecs/src/serialize.rs

@@ -35,7 +35,7 @@ use crate::monotonic_mapping::{
 };
 use crate::{
     monotonic_map_column, Column, FastFieldCodec, FastFieldCodecType, MonotonicallyMappableToU64,
-    U128FastFieldCodecType, VecColumn, ALL_CODEC_TYPES,
+    VecColumn, ALL_CODEC_TYPES,
 };
 
 /// The normalized header gives some parameters after applying the following
@@ -98,29 +98,6 @@ impl Header {
     }
 }
 
-#[derive(Debug, Copy, Clone, PartialEq, Eq)]
-pub(crate) struct U128Header {
-    pub num_vals: u32,
-    pub codec_type: U128FastFieldCodecType,
-}
-
-impl BinarySerializable for U128Header {
-    fn serialize<W: io::Write>(&self, writer: &mut W) -> io::Result<()> {
-        VInt(self.num_vals as u64).serialize(writer)?;
-        self.codec_type.serialize(writer)?;
-        Ok(())
-    }
-
-    fn deserialize<R: io::Read>(reader: &mut R) -> io::Result<Self> {
-        let num_vals = VInt::deserialize(reader)?.0 as u32;
-        let codec_type = U128FastFieldCodecType::deserialize(reader)?;
-        Ok(U128Header {
-            num_vals,
-            codec_type,
-        })
-    }
-}
-
 pub fn normalize_column<C: Column>(
     from_column: C,
     min_value: u64,
@@ -190,11 +167,7 @@ pub fn serialize_u128<F: Fn() -> I, I: Iterator<Item = u128>>(
     num_vals: u32,
     output: &mut impl io::Write,
 ) -> io::Result<()> {
-    let header = U128Header {
-        num_vals,
-        codec_type: U128FastFieldCodecType::CompactSpace,
-    };
-    header.serialize(output)?;
+    // TODO write header, to later support more codecs
     let compressor = CompactSpaceCompressor::train_from(iter_gen(), num_vals);
     compressor.compress_into(iter_gen(), output).unwrap();
 
@@ -278,28 +251,13 @@ pub fn serialize_and_load<T: MonotonicallyMappableToU64 + Ord + Default>(
 ) -> Arc<dyn Column<T>> {
     let mut buffer = Vec::new();
     super::serialize(VecColumn::from(&column), &mut buffer, &ALL_CODEC_TYPES).unwrap();
-    super::open(OwnedBytes::new(buffer))
-        .unwrap()
-        .to_full()
-        .unwrap()
+    super::open(OwnedBytes::new(buffer)).unwrap()
 }
 
 #[cfg(test)]
 mod tests {
     use super::*;
 
-    #[test]
-    fn test_serialize_deserialize_u128_header() {
-        let original = U128Header {
-            num_vals: 11,
-            codec_type: U128FastFieldCodecType::CompactSpace,
-        };
-        let mut out = Vec::new();
-        original.serialize(&mut out).unwrap();
-        let restored = U128Header::deserialize(&mut &out[..]).unwrap();
-        assert_eq!(restored, original);
-    }
-
     #[test]
     fn test_serialize_deserialize() {
         let original = [1u64, 5u64, 10u64];

query-grammar/src/query_grammar.rs

@@ -5,8 +5,7 @@ use combine::parser::range::{take_while, take_while1};
 use combine::parser::repeat::escaped;
 use combine::parser::Parser;
 use combine::{
-    attempt, between, choice, eof, many, many1, one_of, optional, parser, satisfy, sep_by,
-    skip_many1, value,
+    attempt, choice, eof, many, many1, one_of, optional, parser, satisfy, skip_many1, value,
 };
 use once_cell::sync::Lazy;
 use regex::Regex;
@@ -265,17 +264,6 @@ fn range<'a>() -> impl Parser<&'a str, Output = UserInputLeaf> {
     })
 }
 
-/// Function that parses a set out of a Stream
-/// Supports ranges like: `IN [val1 val2 val3]`
-fn set<'a>() -> impl Parser<&'a str, Output = UserInputLeaf> {
-    let term_list = between(char('['), char(']'), sep_by(term_val(), spaces()));
-
-    let set_content = ((string("IN"), spaces()), term_list).map(|(_, elements)| elements);
-
-    (optional(attempt(field_name().skip(spaces()))), set_content)
-        .map(|(field, elements)| UserInputLeaf::Set { field, elements })
-}
-
 fn negate(expr: UserInputAst) -> UserInputAst {
     expr.unary(Occur::MustNot)
 }
@@ -290,7 +278,6 @@ fn leaf<'a>() -> impl Parser<&'a str, Output = UserInputAst> {
                 string("NOT").skip(spaces1()).with(leaf()).map(negate),
             ))
             .or(attempt(range().map(UserInputAst::from)))
-            .or(attempt(set().map(UserInputAst::from)))
             .or(literal().map(UserInputAst::from))
             .parse_stream(input)
             .into_result()
@@ -760,14 +747,6 @@ mod test {
         test_parse_query_to_ast_helper("+(a b) +d", "(+(*\"a\" *\"b\") +\"d\")");
     }
 
-    #[test]
-    fn test_parse_test_query_set() {
-        test_parse_query_to_ast_helper("abc: IN [a b c]", r#""abc": IN ["a" "b" "c"]"#);
-        test_parse_query_to_ast_helper("abc: IN [1]", r#""abc": IN ["1"]"#);
-        test_parse_query_to_ast_helper("abc: IN []", r#""abc": IN []"#);
-        test_parse_query_to_ast_helper("IN [1 2]", r#"IN ["1" "2"]"#);
-    }
-
     #[test]
     fn test_parse_test_query_other() {
         test_parse_query_to_ast_helper("(+a +b) d", "(*(+\"a\" +\"b\") *\"d\")");

query-grammar/src/user_input_ast.rs

@@ -12,10 +12,6 @@ pub enum UserInputLeaf {
         lower: UserInputBound,
         upper: UserInputBound,
     },
-    Set {
-        field: Option<String>,
-        elements: Vec<String>,
-    },
 }
 
 impl Debug for UserInputLeaf {
@@ -35,19 +31,6 @@ impl Debug for UserInputLeaf {
                 upper.display_upper(formatter)?;
                 Ok(())
             }
-            UserInputLeaf::Set { field, elements } => {
-                if let Some(ref field) = field {
-                    write!(formatter, "\"{}\": ", field)?;
-                }
-                write!(formatter, "IN [")?;
-                for (i, element) in elements.iter().enumerate() {
-                    if i != 0 {
-                        write!(formatter, " ")?;
-                    }
-                    write!(formatter, "\"{}\"", element)?;
-                }
-                write!(formatter, "]")
-            }
             UserInputLeaf::All => write!(formatter, "*"),
         }
     }

src/aggregation/agg_req_with_accessor.rs

@@ -4,7 +4,7 @@ use std::rc::Rc;
 use std::sync::atomic::AtomicU32;
 use std::sync::Arc;
 
-use fastfield_codecs::OptionalColumn;
+use fastfield_codecs::Column;
 
 use super::agg_req::{Aggregation, Aggregations, BucketAggregationType, MetricAggregation};
 use super::bucket::{HistogramAggregation, RangeAggregation, TermsAggregation};
@@ -37,16 +37,16 @@ impl AggregationsWithAccessor {
 #[derive(Clone)]
 pub(crate) enum FastFieldAccessor {
     Multi(MultiValuedFastFieldReader<u64>),
-    Single(Arc<dyn OptionalColumn<u64>>),
+    Single(Arc<dyn Column<u64>>),
 }
 impl FastFieldAccessor {
-    pub fn as_single(&self) -> Option<&dyn OptionalColumn<u64>> {
+    pub fn as_single(&self) -> Option<&dyn Column<u64>> {
         match self {
             FastFieldAccessor::Multi(_) => None,
             FastFieldAccessor::Single(reader) => Some(&**reader),
         }
     }
-    pub fn into_single(self) -> Option<Arc<dyn OptionalColumn<u64>>> {
+    pub fn into_single(self) -> Option<Arc<dyn Column<u64>>> {
         match self {
             FastFieldAccessor::Multi(_) => None,
             FastFieldAccessor::Single(reader) => Some(reader),
@@ -124,7 +124,7 @@ impl BucketAggregationWithAccessor {
 pub struct MetricAggregationWithAccessor {
     pub metric: MetricAggregation,
     pub field_type: Type,
-    pub accessor: Arc<dyn OptionalColumn>,
+    pub accessor: Arc<dyn Column>,
 }
 
 impl MetricAggregationWithAccessor {

src/aggregation/bucket/histogram/histogram.rs

@@ -1,7 +1,7 @@
 use std::cmp::Ordering;
 use std::fmt::Display;
 
-use fastfield_codecs::OptionalColumn;
+use fastfield_codecs::Column;
 use itertools::Itertools;
 use serde::{Deserialize, Serialize};
 
@@ -263,17 +263,13 @@ impl SegmentHistogramCollector {
         req: &HistogramAggregation,
         sub_aggregation: &AggregationsWithAccessor,
         field_type: Type,
-        accessor: &dyn OptionalColumn<u64>,
+        accessor: &dyn Column<u64>,
     ) -> crate::Result<Self> {
         req.validate()?;
-        let min_max_u64 = accessor.min_value().zip(accessor.max_value());
-        let min_max_f64 = min_max_u64.map(|(min, max)| {
-            let min = f64_from_fastfield_u64(min, &field_type);
-            let max = f64_from_fastfield_u64(max, &field_type);
-            (min, max)
-        });
+        let min = f64_from_fastfield_u64(accessor.min_value(), &field_type);
+        let max = f64_from_fastfield_u64(accessor.max_value(), &field_type);
 
-        let (min, max) = get_req_min_max(req, min_max_f64);
+        let (min, max) = get_req_min_max(req, Some((min, max)));
 
         // We compute and generate the buckets range (min, max) based on the request and the min
         // max in the fast field, but this is likely not ideal when this is a subbucket, where many
@@ -335,58 +331,47 @@ impl SegmentHistogramCollector {
             .expect("unexpected fast field cardinatility");
         let mut iter = doc.chunks_exact(4);
         for docs in iter.by_ref() {
-            if let Some(val) = accessor.get_val(docs[0]) {
-                let val = self.f64_from_fastfield_u64(val);
-                let bucket_pos = get_bucket_num(val);
-                self.increment_bucket_if_in_bounds(
-                    val,
-                    &bounds,
-                    bucket_pos,
-                    docs[0],
-                    &bucket_with_accessor.sub_aggregation,
-                )?;
-            }
+            let val0 = self.f64_from_fastfield_u64(accessor.get_val(docs[0]));
+            let val1 = self.f64_from_fastfield_u64(accessor.get_val(docs[1]));
+            let val2 = self.f64_from_fastfield_u64(accessor.get_val(docs[2]));
+            let val3 = self.f64_from_fastfield_u64(accessor.get_val(docs[3]));
 
-            if let Some(val) = accessor.get_val(docs[1]) {
-                let val = self.f64_from_fastfield_u64(val);
-                let bucket_pos = get_bucket_num(val);
-                self.increment_bucket_if_in_bounds(
-                    val,
-                    &bounds,
-                    bucket_pos,
-                    docs[1],
-                    &bucket_with_accessor.sub_aggregation,
-                )?;
-            }
+            let bucket_pos0 = get_bucket_num(val0);
+            let bucket_pos1 = get_bucket_num(val1);
+            let bucket_pos2 = get_bucket_num(val2);
+            let bucket_pos3 = get_bucket_num(val3);
 
-            if let Some(val) = accessor.get_val(docs[2]) {
-                let val = self.f64_from_fastfield_u64(val);
-                let bucket_pos = get_bucket_num(val);
-                self.increment_bucket_if_in_bounds(
-                    val,
-                    &bounds,
-                    bucket_pos,
-                    docs[2],
-                    &bucket_with_accessor.sub_aggregation,
-                )?;
-            }
-
-            if let Some(val) = accessor.get_val(docs[3]) {
-                let val = self.f64_from_fastfield_u64(val);
-                let bucket_pos = get_bucket_num(val);
-                self.increment_bucket_if_in_bounds(
-                    val,
-                    &bounds,
-                    bucket_pos,
-                    docs[3],
-                    &bucket_with_accessor.sub_aggregation,
-                )?;
-            }
+            self.increment_bucket_if_in_bounds(
+                val0,
+                &bounds,
+                bucket_pos0,
+                docs[0],
+                &bucket_with_accessor.sub_aggregation,
+            )?;
+            self.increment_bucket_if_in_bounds(
+                val1,
+                &bounds,
+                bucket_pos1,
+                docs[1],
+                &bucket_with_accessor.sub_aggregation,
+            )?;
+            self.increment_bucket_if_in_bounds(
+                val2,
+                &bounds,
+                bucket_pos2,
+                docs[2],
+                &bucket_with_accessor.sub_aggregation,
+            )?;
+            self.increment_bucket_if_in_bounds(
+                val3,
+                &bounds,
+                bucket_pos3,
+                docs[3],
+                &bucket_with_accessor.sub_aggregation,
+            )?;
         }
         for &doc in iter.remainder() {
-            let Some(val) = accessor.get_val(doc).map(|val|f64_from_fastfield_u64(val, &self.field_type)) else{
-                continue;
-            };
+            let val = f64_from_fastfield_u64(accessor.get_val(doc), &self.field_type);
             if !bounds.contains(val) {
                 continue;
             }

src/aggregation/bucket/range.rs

@@ -267,29 +267,20 @@ impl SegmentRangeCollector {
             let val2 = accessor.get_val(docs[1]);
             let val3 = accessor.get_val(docs[2]);
             let val4 = accessor.get_val(docs[3]);
-            if let Some(val) = val1 {
-                let bucket_pos = self.get_bucket_pos(val);
-                self.increment_bucket(bucket_pos, docs[0], &bucket_with_accessor.sub_aggregation)?;
-            }
-            if let Some(val) = val2 {
-                let bucket_pos = self.get_bucket_pos(val);
-                self.increment_bucket(bucket_pos, docs[1], &bucket_with_accessor.sub_aggregation)?;
-            }
-            if let Some(val) = val3 {
-                let bucket_pos = self.get_bucket_pos(val);
-                self.increment_bucket(bucket_pos, docs[2], &bucket_with_accessor.sub_aggregation)?;
-            }
-            if let Some(val) = val4 {
-                let bucket_pos = self.get_bucket_pos(val);
-                self.increment_bucket(bucket_pos, docs[3], &bucket_with_accessor.sub_aggregation)?;
-            }
+            let bucket_pos1 = self.get_bucket_pos(val1);
+            let bucket_pos2 = self.get_bucket_pos(val2);
+            let bucket_pos3 = self.get_bucket_pos(val3);
+            let bucket_pos4 = self.get_bucket_pos(val4);
+
+            self.increment_bucket(bucket_pos1, docs[0], &bucket_with_accessor.sub_aggregation)?;
+            self.increment_bucket(bucket_pos2, docs[1], &bucket_with_accessor.sub_aggregation)?;
+            self.increment_bucket(bucket_pos3, docs[2], &bucket_with_accessor.sub_aggregation)?;
+            self.increment_bucket(bucket_pos4, docs[3], &bucket_with_accessor.sub_aggregation)?;
         }
         for &doc in iter.remainder() {
             let val = accessor.get_val(doc);
-            if let Some(val) = val {
-                let bucket_pos = self.get_bucket_pos(val);
-                self.increment_bucket(bucket_pos, doc, &bucket_with_accessor.sub_aggregation)?;
-            }
+            let bucket_pos = self.get_bucket_pos(val);
+            self.increment_bucket(bucket_pos, doc, &bucket_with_accessor.sub_aggregation)?;
         }
         if force_flush {
             for bucket in &mut self.buckets {

src/aggregation/metric/average.rs

@@ -1,6 +1,6 @@
 use std::fmt::Debug;
 
-use fastfield_codecs::OptionalColumn;
+use fastfield_codecs::Column;
 use serde::{Deserialize, Serialize};
 
 use crate::aggregation::f64_from_fastfield_u64;
@@ -57,33 +57,26 @@ impl SegmentAverageCollector {
             data: Default::default(),
         }
     }
-    pub(crate) fn collect_block(&mut self, doc: &[DocId], field: &dyn OptionalColumn<u64>) {
+    pub(crate) fn collect_block(&mut self, doc: &[DocId], field: &dyn Column<u64>) {
         let mut iter = doc.chunks_exact(4);
         for docs in iter.by_ref() {
-            if let Some(val) = field.get_val(docs[0]) {
-                let val = f64_from_fastfield_u64(val, &self.field_type);
-                self.data.collect(val);
-            }
-            if let Some(val) = field.get_val(docs[1]) {
-                let val = f64_from_fastfield_u64(val, &self.field_type);
-                self.data.collect(val);
-            }
-
-            if let Some(val) = field.get_val(docs[2]) {
-                let val = f64_from_fastfield_u64(val, &self.field_type);
-                self.data.collect(val);
-            }
-
-            if let Some(val) = field.get_val(docs[3]) {
-                let val = f64_from_fastfield_u64(val, &self.field_type);
-                self.data.collect(val);
-            }
+            let val1 = field.get_val(docs[0]);
+            let val2 = field.get_val(docs[1]);
+            let val3 = field.get_val(docs[2]);
+            let val4 = field.get_val(docs[3]);
+            let val1 = f64_from_fastfield_u64(val1, &self.field_type);
+            let val2 = f64_from_fastfield_u64(val2, &self.field_type);
+            let val3 = f64_from_fastfield_u64(val3, &self.field_type);
+            let val4 = f64_from_fastfield_u64(val4, &self.field_type);
+            self.data.collect(val1);
+            self.data.collect(val2);
+            self.data.collect(val3);
+            self.data.collect(val4);
         }
         for &doc in iter.remainder() {
-            if let Some(val) = field.get_val(doc) {
-                let val = f64_from_fastfield_u64(val, &self.field_type);
-                self.data.collect(val);
-            }
+            let val = field.get_val(doc);
+            let val = f64_from_fastfield_u64(val, &self.field_type);
+            self.data.collect(val);
         }
     }
 }

src/aggregation/metric/stats.rs

@@ -1,4 +1,4 @@
-use fastfield_codecs::OptionalColumn;
+use fastfield_codecs::Column;
 use serde::{Deserialize, Serialize};
 
 use crate::aggregation::f64_from_fastfield_u64;
@@ -163,31 +163,26 @@ impl SegmentStatsCollector {
             stats: IntermediateStats::default(),
         }
     }
-    pub(crate) fn collect_block(&mut self, doc: &[DocId], field: &dyn OptionalColumn<u64>) {
+    pub(crate) fn collect_block(&mut self, doc: &[DocId], field: &dyn Column<u64>) {
         let mut iter = doc.chunks_exact(4);
         for docs in iter.by_ref() {
-            if let Some(val) = field.get_val(docs[0]) {
-                let val = f64_from_fastfield_u64(val, &self.field_type);
-                self.stats.collect(val);
-            }
-            if let Some(val) = field.get_val(docs[1]) {
-                let val = f64_from_fastfield_u64(val, &self.field_type);
-                self.stats.collect(val);
-            }
-            if let Some(val) = field.get_val(docs[2]) {
-                let val = f64_from_fastfield_u64(val, &self.field_type);
-                self.stats.collect(val);
-            }
-            if let Some(val) = field.get_val(docs[3]) {
-                let val = f64_from_fastfield_u64(val, &self.field_type);
-                self.stats.collect(val);
-            }
+            let val1 = field.get_val(docs[0]);
+            let val2 = field.get_val(docs[1]);
+            let val3 = field.get_val(docs[2]);
+            let val4 = field.get_val(docs[3]);
+            let val1 = f64_from_fastfield_u64(val1, &self.field_type);
+            let val2 = f64_from_fastfield_u64(val2, &self.field_type);
+            let val3 = f64_from_fastfield_u64(val3, &self.field_type);
+            let val4 = f64_from_fastfield_u64(val4, &self.field_type);
+            self.stats.collect(val1);
+            self.stats.collect(val2);
+            self.stats.collect(val3);
+            self.stats.collect(val4);
         }
         for &doc in iter.remainder() {
-            if let Some(val) = field.get_val(doc) {
-                let val = f64_from_fastfield_u64(val, &self.field_type);
-                self.stats.collect(val);
-            }
+            let val = field.get_val(doc);
+            let val = f64_from_fastfield_u64(val, &self.field_type);
+            self.stats.collect(val);
         }
     }
 }

src/collector/filter_collector_wrapper.rs

@@ -130,9 +130,7 @@ where
 
         let fast_field_reader = segment_reader
             .fast_fields()
-            .typed_fast_field_reader(self.field)?
-            .to_full()
-            .expect("temp migration solution");
+            .typed_fast_field_reader(self.field)?;
 
         let segment_collector = self
             .collector

src/collector/histogram_collector.rs

@@ -112,11 +112,7 @@ impl Collector for HistogramCollector {
         _segment_local_id: crate::SegmentOrdinal,
         segment: &crate::SegmentReader,
     ) -> crate::Result<Self::Child> {
-        let ff_reader = segment
-            .fast_fields()
-            .u64_lenient(self.field)?
-            .to_full()
-            .expect("temp migration solution");
+        let ff_reader = segment.fast_fields().u64_lenient(self.field)?;
         Ok(SegmentHistogramCollector {
             histogram_computer: HistogramComputer {
                 counts: vec![0; self.num_buckets],

src/collector/tests.rs

@@ -1,6 +1,6 @@
 use std::sync::Arc;
 
-use fastfield_codecs::OptionalColumn;
+use fastfield_codecs::Column;
 
 use super::*;
 use crate::collector::{Count, FilterCollector, TopDocs};
@@ -160,7 +160,7 @@ pub struct FastFieldTestCollector {
 
 pub struct FastFieldSegmentCollector {
     vals: Vec<u64>,
-    reader: Arc<dyn OptionalColumn<u64>>,
+    reader: Arc<dyn Column<u64>>,
 }
 
 impl FastFieldTestCollector {
@@ -202,9 +202,7 @@ impl SegmentCollector for FastFieldSegmentCollector {
 
     fn collect(&mut self, doc: DocId, _score: Score) {
         let val = self.reader.get_val(doc);
-        if let Some(val) = val {
-            self.vals.push(val);
-        }
+        self.vals.push(val);
     }
 
     fn harvest(self) -> Vec<u64> {

src/collector/top_score_collector.rs

@@ -156,9 +156,7 @@ impl CustomScorer<u64> for ScorerByField {
         // The conversion will then happen only on the top-K docs.
         let ff_reader = segment_reader
             .fast_fields()
-            .typed_fast_field_reader(self.field)?
-            .to_full()
-            .expect("temp migration solution");
+            .typed_fast_field_reader(self.field)?;
         Ok(ScorerByFastFieldReader { ff_reader })
     }
 }
@@ -460,7 +458,7 @@ impl TopDocs {
     ///
     ///             // We can now define our actual scoring function
     ///             move |doc: DocId, original_score: Score| {
-    ///                 let popularity: u64 = popularity_reader.get_val(doc).unwrap();
+    ///                 let popularity: u64 = popularity_reader.get_val(doc);
     ///                 // Well.. For the sake of the example we use a simple logarithm
     ///                 // function.
     ///                 let popularity_boost_score = ((2u64 + popularity) as Score).log2();
@@ -569,8 +567,8 @@ impl TopDocs {
     ///
     ///             // We can now define our actual scoring function
     ///             move |doc: DocId| {
-    ///                 let popularity: u64 = popularity_reader.get_val(doc).unwrap();
-    ///                 let boosted: u64 = boosted_reader.get_val(doc).unwrap();
+    ///                 let popularity: u64 = popularity_reader.get_val(doc);
+    ///                 let boosted: u64 = boosted_reader.get_val(doc);
     ///                 // Score do not have to be `f64` in tantivy.
     ///                 // Here we return a couple to get lexicographical order
     ///                 // for free.

src/core/inverted_index_reader.rs

@@ -230,18 +230,6 @@ impl InvertedIndexReader {
         Ok(())
     }
 
-    /// Read the block postings for all terms.
-    /// This method is for an advanced usage only.
-    ///
-    /// If you know which terms to pre-load, prefer using [`Self::warm_postings`] instead.
-    pub async fn warm_postings_full(&self, with_positions: bool) -> crate::AsyncIoResult<()> {
-        self.postings_file_slice.read_bytes_async().await?;
-        if with_positions {
-            self.positions_file_slice.read_bytes_async().await?;
-        }
-        Ok(())
-    }
-
     /// Returns the number of documents containing the term asynchronously.
     pub async fn doc_freq_async(&self, term: &Term) -> crate::AsyncIoResult<u32> {
         Ok(self

src/core/searcher.rs

@@ -4,7 +4,7 @@ use std::{fmt, io};
 
 use crate::collector::Collector;
 use crate::core::{Executor, SegmentReader};
-use crate::query::{EnableScoring, Query};
+use crate::query::Query;
 use crate::schema::{Document, Schema, Term};
 use crate::space_usage::SearcherSpaceUsage;
 use crate::store::{CacheStats, StoreReader};
@@ -199,12 +199,7 @@ impl Searcher {
         executor: &Executor,
     ) -> crate::Result<C::Fruit> {
         let scoring_enabled = collector.requires_scoring();
-        let enabled_scoring = if scoring_enabled {
-            EnableScoring::Enabled(self)
-        } else {
-            EnableScoring::Disabled(self.schema())
-        };
-        let weight = query.weight(enabled_scoring)?;
+        let weight = query.weight(self, scoring_enabled)?;
         let segment_readers = self.segment_readers();
         let fruits = executor.map(
             |(segment_ord, segment_reader)| {

src/fastfield/bytes/mod.rs

@@ -6,7 +6,7 @@ pub use self::writer::BytesFastFieldWriter;
 
 #[cfg(test)]
 mod tests {
-    use crate::query::{EnableScoring, TermQuery};
+    use crate::query::TermQuery;
     use crate::schema::{BytesOptions, IndexRecordOption, Schema, Value, FAST, INDEXED, STORED};
     use crate::{DocAddress, DocSet, Index, Searcher, Term};
 
@@ -82,7 +82,7 @@ mod tests {
         let field = searcher.schema().get_field("string_bytes").unwrap();
         let term = Term::from_field_bytes(field, b"lucene".as_ref());
         let term_query = TermQuery::new(term, IndexRecordOption::Basic);
-        let term_weight = term_query.specialized_weight(EnableScoring::Enabled(&searcher))?;
+        let term_weight = term_query.specialized_weight(&searcher, true)?;
         let term_scorer = term_weight.specialized_scorer(searcher.segment_reader(0), 1.0)?;
         assert_eq!(term_scorer.doc(), 0u32);
         Ok(())
@@ -95,8 +95,7 @@ mod tests {
         let field = searcher.schema().get_field("string_bytes").unwrap();
         let term = Term::from_field_bytes(field, b"lucene".as_ref());
         let term_query = TermQuery::new(term, IndexRecordOption::Basic);
-        let term_weight_err =
-            term_query.specialized_weight(EnableScoring::Disabled(searcher.schema()));
+        let term_weight_err = term_query.specialized_weight(&searcher, false);
         assert!(matches!(
             term_weight_err,
             Err(crate::TantivyError::SchemaError(_))

src/fastfield/bytes/reader.rs

@@ -52,7 +52,7 @@ impl BytesFastFieldReader {
     }
 
     /// Returns the overall number of bytes in this bytes fast field.
-    pub fn total_num_bytes(&self) -> u32 {
-        self.values.len() as u32
+    pub fn total_num_bytes(&self) -> u64 {
+        self.values.len() as u64
     }
 }

src/fastfield/mod.rs

@@ -210,7 +210,7 @@ mod tests {
         assert_eq!(file.len(), 25);
         let composite_file = CompositeFile::open(&file)?;
         let fast_field_bytes = composite_file.open_read(*FIELD).unwrap().read_bytes()?;
-        let fast_field_reader = open::<u64>(fast_field_bytes)?.to_full().unwrap();
+        let fast_field_reader = open::<u64>(fast_field_bytes)?;
         assert_eq!(fast_field_reader.get_val(0), 13u64);
         assert_eq!(fast_field_reader.get_val(1), 14u64);
         assert_eq!(fast_field_reader.get_val(2), 2u64);
@@ -263,7 +263,7 @@ mod tests {
                 .open_read(*FIELD)
                 .unwrap()
                 .read_bytes()?;
-            let fast_field_reader = open::<u64>(data)?.to_full().unwrap();
+            let fast_field_reader = open::<u64>(data)?;
             assert_eq!(fast_field_reader.get_val(0), 4u64);
             assert_eq!(fast_field_reader.get_val(1), 14_082_001u64);
             assert_eq!(fast_field_reader.get_val(2), 3_052u64);
@@ -304,7 +304,7 @@ mod tests {
                 .open_read(*FIELD)
                 .unwrap()
                 .read_bytes()?;
-            let fast_field_reader = open::<u64>(data)?.to_full().unwrap();
+            let fast_field_reader = open::<u64>(data)?;
             for doc in 0..10_000 {
                 assert_eq!(fast_field_reader.get_val(doc), 100_000u64);
             }
@@ -343,7 +343,7 @@ mod tests {
                 .open_read(*FIELD)
                 .unwrap()
                 .read_bytes()?;
-            let fast_field_reader = open::<u64>(data)?.to_full().unwrap();
+            let fast_field_reader = open::<u64>(data)?;
             assert_eq!(fast_field_reader.get_val(0), 0u64);
             for doc in 1..10_001 {
                 assert_eq!(
@@ -386,7 +386,7 @@ mod tests {
                 .open_read(i64_field)
                 .unwrap()
                 .read_bytes()?;
-            let fast_field_reader = open::<i64>(data)?.to_full().unwrap();
+            let fast_field_reader = open::<i64>(data)?;
 
             assert_eq!(fast_field_reader.min_value(), -100i64);
             assert_eq!(fast_field_reader.max_value(), 9_999i64);
@@ -429,7 +429,7 @@ mod tests {
                 .open_read(i64_field)
                 .unwrap()
                 .read_bytes()?;
-            let fast_field_reader = open::<i64>(data)?.to_full().unwrap();
+            let fast_field_reader = open::<i64>(data)?;
             assert_eq!(fast_field_reader.get_val(0), 0i64);
         }
         Ok(())
@@ -470,7 +470,7 @@ mod tests {
                 .open_read(*FIELD)
                 .unwrap()
                 .read_bytes()?;
-            let fast_field_reader = open::<u64>(data)?.to_full().unwrap();
+            let fast_field_reader = open::<u64>(data)?;
 
             for a in 0..n {
                 assert_eq!(fast_field_reader.get_val(a as u32), permutation[a as usize]);
@@ -763,28 +763,19 @@ mod tests {
         let dates_fast_field = fast_fields.dates(multi_date_field).unwrap();
         let mut dates = vec![];
         {
-            assert_eq!(
-                date_fast_field.get_val(0).unwrap().into_timestamp_micros(),
-                1i64
-            );
+            assert_eq!(date_fast_field.get_val(0).into_timestamp_micros(), 1i64);
             dates_fast_field.get_vals(0u32, &mut dates);
             assert_eq!(dates.len(), 2);
             assert_eq!(dates[0].into_timestamp_micros(), 2i64);
             assert_eq!(dates[1].into_timestamp_micros(), 3i64);
         }
         {
-            assert_eq!(
-                date_fast_field.get_val(1).unwrap().into_timestamp_micros(),
-                4i64
-            );
+            assert_eq!(date_fast_field.get_val(1).into_timestamp_micros(), 4i64);
             dates_fast_field.get_vals(1u32, &mut dates);
             assert!(dates.is_empty());
         }
         {
-            assert_eq!(
-                date_fast_field.get_val(2).unwrap().into_timestamp_micros(),
-                0i64
-            );
+            assert_eq!(date_fast_field.get_val(2).into_timestamp_micros(), 0i64);
             dates_fast_field.get_vals(2u32, &mut dates);
             assert_eq!(dates.len(), 2);
             assert_eq!(dates[0].into_timestamp_micros(), 5i64);
@@ -834,7 +825,7 @@ mod tests {
         assert_eq!(file.len(), 24);
         let composite_file = CompositeFile::open(&file)?;
         let data = composite_file.open_read(field).unwrap().read_bytes()?;
-        let fast_field_reader = open::<bool>(data)?.to_full().unwrap();
+        let fast_field_reader = open::<bool>(data)?;
         assert_eq!(fast_field_reader.get_val(0), true);
         assert_eq!(fast_field_reader.get_val(1), false);
         assert_eq!(fast_field_reader.get_val(2), true);
@@ -872,7 +863,7 @@ mod tests {
         assert_eq!(file.len(), 36);
         let composite_file = CompositeFile::open(&file)?;
         let data = composite_file.open_read(field).unwrap().read_bytes()?;
-        let fast_field_reader = open::<bool>(data)?.to_full().unwrap();
+        let fast_field_reader = open::<bool>(data)?;
         for i in 0..25 {
             assert_eq!(fast_field_reader.get_val(i * 2), true);
             assert_eq!(fast_field_reader.get_val(i * 2 + 1), false);
@@ -903,7 +894,7 @@ mod tests {
         let composite_file = CompositeFile::open(&file)?;
         assert_eq!(file.len(), 23);
         let data = composite_file.open_read(field).unwrap().read_bytes()?;
-        let fast_field_reader = open::<bool>(data)?.to_full().unwrap();
+        let fast_field_reader = open::<bool>(data)?;
         assert_eq!(fast_field_reader.get_val(0), false);
 
         Ok(())
@@ -971,9 +962,7 @@ mod tests {
         let composite_file = CompositeFile::open(&file)?;
         let file = composite_file.open_read(*FIELD).unwrap();
         let len = file.len();
-        let test_fastfield = open::<DateTime>(file.read_bytes()?)?
-            .to_full()
-            .expect("temp migration solution");
+        let test_fastfield = open::<DateTime>(file.read_bytes()?)?;
 
         for (i, time) in times.iter().enumerate() {
             assert_eq!(test_fastfield.get_val(i as u32), time.truncate(precision));

src/fastfield/multivalued/index.rs

@@ -26,18 +26,10 @@ impl MultiValueIndex {
         start..end
     }
 
-    /// Given a range of documents, returns the Range of value offsets fo
-    /// these documents.
+    /// Returns `[start, end)`, such that the values associated with
+    /// the given documents are `start..end`.
     ///
-    /// For instance, `given start_doc..end_doc`,
-    /// if we assume Document #start_doc end #end_doc both
-    /// have values, this function returns `start..end`
-    /// such that `value_column.get(start_doc)` is the first value of
-    /// `start_doc` (well, if there is one), and `value_column.get(end_doc - 1)`
-    /// is the last value of `end_doc`.
-    ///
-    /// The passed end range is allowed to be out of bounds, in which case
-    /// it will be clipped to make it valid.
+    /// The passed end range is allowed to be out of bounds.
     #[inline]
     pub(crate) fn docid_range_to_position_range(&self, range: Range<DocId>) -> Range<u32> {
         let end_docid = range.end.min(self.num_docs() - 1) + 1;
@@ -58,8 +50,8 @@ impl MultiValueIndex {
 
     /// Returns the overall number of values in this field.
     #[inline]
-    pub fn total_num_vals(&self) -> u32 {
-        self.idx.max_value() as u32
+    pub fn total_num_vals(&self) -> u64 {
+        self.idx.max_value()
     }
 
     /// Returns the number of documents in the index.

src/fastfield/multivalued/mod.rs

@@ -533,17 +533,14 @@ mod bench {
                 .unwrap()
                 .read_bytes()
                 .unwrap();
-            let idx_reader = fastfield_codecs::open(data_idx).unwrap().to_full().unwrap();
+            let idx_reader = fastfield_codecs::open(data_idx).unwrap();
 
             let data_vals = fast_fields_composite
                 .open_read_with_idx(field, 1)
                 .unwrap()
                 .read_bytes()
                 .unwrap();
-            let vals_reader = fastfield_codecs::open(data_vals)
-                .unwrap()
-                .to_full()
-                .unwrap();
+            let vals_reader = fastfield_codecs::open(data_vals).unwrap();
             let fast_field_reader = MultiValuedFastFieldReader::open(idx_reader, vals_reader);
             b.iter(|| {
                 let mut sum = 0u64;

src/fastfield/multivalued/reader.rs

@@ -79,7 +79,7 @@ impl<Item: FastValue> MultiValuedFastFieldReader<Item> {
 
     /// Returns the overall number of values in this field.
     #[inline]
-    pub fn total_num_vals(&self) -> u32 {
+    pub fn total_num_vals(&self) -> u64 {
         self.idx_reader.total_num_vals()
     }
 }
@@ -108,6 +108,22 @@ impl<T: MonotonicallyMappableToU128> MultiValuedU128FastFieldReader<T> {
         }
     }
 
+    #[inline]
+    fn get_docids_for_value_range(
+        &self,
+        value_range: RangeInclusive<T>,
+        doc_id_range: Range<u32>,
+        positions: &mut Vec<u32>,
+    ) {
+        let position_range = self
+            .get_index_reader()
+            .docid_range_to_position_range(doc_id_range.clone());
+        self.vals_reader
+            .get_docids_for_value_range(value_range, position_range, positions);
+
+        self.idx_reader.positions_to_docids(doc_id_range, positions);
+    }
+
     /// Returns the array of values associated to the given `doc`.
     #[inline]
     pub fn get_first_val(&self, doc: DocId) -> Option<T> {
@@ -170,30 +186,13 @@ impl<T: MonotonicallyMappableToU128> MultiValuedU128FastFieldReader<T> {
 
     /// Returns the overall number of values in this field. It does not include deletes.
     #[inline]
-    pub fn total_num_vals(&self) -> u32 {
+    pub fn total_num_vals(&self) -> u64 {
         assert_eq!(
-            self.vals_reader.num_vals(),
+            self.vals_reader.num_vals() as u64,
             self.get_index_reader().total_num_vals()
         );
         self.idx_reader.total_num_vals()
     }
-
-    /// Returns the docids matching given doc_id_range and value_range.
-    #[inline]
-    pub fn get_docids_for_value_range(
-        &self,
-        value_range: RangeInclusive<T>,
-        doc_id_range: Range<u32>,
-        positions: &mut Vec<u32>,
-    ) {
-        let position_range = self
-            .get_index_reader()
-            .docid_range_to_position_range(doc_id_range.clone());
-        self.vals_reader
-            .get_docids_for_value_range(value_range, position_range, positions);
-
-        self.idx_reader.positions_to_docids(doc_id_range, positions);
-    }
 }
 
 #[cfg(test)]

src/fastfield/readers.rs

@@ -1,7 +1,7 @@
 use std::net::Ipv6Addr;
 use std::sync::Arc;
 
-use fastfield_codecs::{open, open_u128, Column, OptionalColumn};
+use fastfield_codecs::{open, open_u128, Column};
 
 use super::multivalued::MultiValuedU128FastFieldReader;
 use crate::directory::{CompositeFile, FileSlice};
@@ -118,7 +118,7 @@ impl FastFieldReaders {
         &self,
         field: Field,
         index: usize,
-    ) -> crate::Result<Arc<dyn OptionalColumn<TFastValue>>> {
+    ) -> crate::Result<Arc<dyn Column<TFastValue>>> {
         let fast_field_slice = self.fast_field_data(field, index)?;
         let bytes = fast_field_slice.read_bytes()?;
         let column = fastfield_codecs::open(bytes)?;
@@ -128,7 +128,7 @@ impl FastFieldReaders {
     pub(crate) fn typed_fast_field_reader<TFastValue: FastValue>(
         &self,
         field: Field,
-    ) -> crate::Result<Arc<dyn OptionalColumn<TFastValue>>> {
+    ) -> crate::Result<Arc<dyn Column<TFastValue>>> {
         self.typed_fast_field_reader_with_idx(field, 0)
     }
 
@@ -138,20 +138,13 @@ impl FastFieldReaders {
     ) -> crate::Result<MultiValuedFastFieldReader<TFastValue>> {
         let idx_reader = self.typed_fast_field_reader(field)?;
         let vals_reader = self.typed_fast_field_reader_with_idx(field, 1)?;
-        Ok(MultiValuedFastFieldReader::open(
-            idx_reader
-                .to_full()
-                .expect("multivalue fast field are always full"),
-            vals_reader
-                .to_full()
-                .expect("multivalue fast field are always full"),
-        ))
+        Ok(MultiValuedFastFieldReader::open(idx_reader, vals_reader))
     }
 
     /// Returns the `u64` fast field reader reader associated with `field`.
     ///
     /// If `field` is not a u64 fast field, this method returns an Error.
-    pub fn u64(&self, field: Field) -> crate::Result<Arc<dyn OptionalColumn<u64>>> {
+    pub fn u64(&self, field: Field) -> crate::Result<Arc<dyn Column<u64>>> {
         self.check_type(field, FastType::U64, Cardinality::SingleValue)?;
         self.typed_fast_field_reader(field)
     }
@@ -159,7 +152,7 @@ impl FastFieldReaders {
     /// Returns the `ip` fast field reader reader associated to `field`.
     ///
     /// If `field` is not a u128 fast field, this method returns an Error.
-    pub fn ip_addr(&self, field: Field) -> crate::Result<Arc<dyn OptionalColumn<Ipv6Addr>>> {
+    pub fn ip_addr(&self, field: Field) -> crate::Result<Arc<dyn Column<Ipv6Addr>>> {
         self.check_type(field, FastType::U128, Cardinality::SingleValue)?;
         let bytes = self.fast_field_data(field, 0)?.read_bytes()?;
         Ok(open_u128::<Ipv6Addr>(bytes)?)
@@ -173,15 +166,10 @@ impl FastFieldReaders {
         field: Field,
     ) -> crate::Result<MultiValuedU128FastFieldReader<Ipv6Addr>> {
         self.check_type(field, FastType::U128, Cardinality::MultiValues)?;
-        let idx_reader: Arc<dyn Column<u64>> = self
-            .typed_fast_field_reader(field)?
-            .to_full()
-            .expect("multivalue fast fields are always full");
+        let idx_reader: Arc<dyn Column<u64>> = self.typed_fast_field_reader(field)?;
 
         let bytes = self.fast_field_data(field, 1)?.read_bytes()?;
-        let vals_reader = open_u128::<Ipv6Addr>(bytes)?
-            .to_full()
-            .expect("multivalue fields are always full");
+        let vals_reader = open_u128::<Ipv6Addr>(bytes)?;
 
         Ok(MultiValuedU128FastFieldReader::open(
             idx_reader,
@@ -191,9 +179,8 @@ impl FastFieldReaders {
 
     /// Returns the `u128` fast field reader reader associated to `field`.
     ///
-    /// If `field` is not a u128 base type fast field, this method returns an Error.
-    /// Ip addresses use u128 as base type.
-    pub(crate) fn u128(&self, field: Field) -> crate::Result<Arc<dyn OptionalColumn<u128>>> {
+    /// If `field` is not a u128 fast field, this method returns an Error.
+    pub(crate) fn u128(&self, field: Field) -> crate::Result<Arc<dyn Column<u128>>> {
         self.check_type(field, FastType::U128, Cardinality::SingleValue)?;
         let bytes = self.fast_field_data(field, 0)?.read_bytes()?;
         Ok(open_u128::<u128>(bytes)?)
@@ -204,15 +191,10 @@ impl FastFieldReaders {
     /// If `field` is not a u128 multi-valued fast field, this method returns an Error.
     pub fn u128s(&self, field: Field) -> crate::Result<MultiValuedU128FastFieldReader<u128>> {
         self.check_type(field, FastType::U128, Cardinality::MultiValues)?;
-        let idx_reader: Arc<dyn Column<u64>> = self
-            .typed_fast_field_reader(field)?
-            .to_full()
-            .expect("multivalue fast fields are always full");
+        let idx_reader: Arc<dyn Column<u64>> = self.typed_fast_field_reader(field)?;
 
         let bytes = self.fast_field_data(field, 1)?.read_bytes()?;
-        let vals_reader = open_u128::<u128>(bytes)?
-            .to_full()
-            .expect("multivalue fast fields are always full");
+        let vals_reader = open_u128::<u128>(bytes)?;
 
         Ok(MultiValuedU128FastFieldReader::open(
             idx_reader,
@@ -225,14 +207,14 @@ impl FastFieldReaders {
     ///
     /// If not, the fastfield reader will returns the u64-value associated with the original
     /// FastValue.
-    pub fn u64_lenient(&self, field: Field) -> crate::Result<Arc<dyn OptionalColumn<u64>>> {
+    pub fn u64_lenient(&self, field: Field) -> crate::Result<Arc<dyn Column<u64>>> {
         self.typed_fast_field_reader(field)
     }
 
     /// Returns the `i64` fast field reader reader associated with `field`.
     ///
     /// If `field` is not a i64 fast field, this method returns an Error.
-    pub fn i64(&self, field: Field) -> crate::Result<Arc<dyn OptionalColumn<i64>>> {
+    pub fn i64(&self, field: Field) -> crate::Result<Arc<dyn Column<i64>>> {
         self.check_type(field, FastType::I64, Cardinality::SingleValue)?;
         self.typed_fast_field_reader(field)
     }
@@ -240,7 +222,7 @@ impl FastFieldReaders {
     /// Returns the `date` fast field reader reader associated with `field`.
     ///
     /// If `field` is not a date fast field, this method returns an Error.
-    pub fn date(&self, field: Field) -> crate::Result<Arc<dyn OptionalColumn<DateTime>>> {
+    pub fn date(&self, field: Field) -> crate::Result<Arc<dyn Column<DateTime>>> {
         self.check_type(field, FastType::Date, Cardinality::SingleValue)?;
         self.typed_fast_field_reader(field)
     }
@@ -248,7 +230,7 @@ impl FastFieldReaders {
     /// Returns the `f64` fast field reader reader associated with `field`.
     ///
     /// If `field` is not a f64 fast field, this method returns an Error.
-    pub fn f64(&self, field: Field) -> crate::Result<Arc<dyn OptionalColumn<f64>>> {
+    pub fn f64(&self, field: Field) -> crate::Result<Arc<dyn Column<f64>>> {
         self.check_type(field, FastType::F64, Cardinality::SingleValue)?;
         self.typed_fast_field_reader(field)
     }
@@ -256,7 +238,7 @@ impl FastFieldReaders {
     /// Returns the `bool` fast field reader reader associated with `field`.
     ///
     /// If `field` is not a bool fast field, this method returns an Error.
-    pub fn bool(&self, field: Field) -> crate::Result<Arc<dyn OptionalColumn<bool>>> {
+    pub fn bool(&self, field: Field) -> crate::Result<Arc<dyn Column<bool>>> {
         self.check_type(field, FastType::Bool, Cardinality::SingleValue)?;
         self.typed_fast_field_reader(field)
     }
@@ -327,12 +309,7 @@ impl FastFieldReaders {
             let fast_field_idx_bytes = fast_field_idx_file.read_bytes()?;
             let idx_reader = open(fast_field_idx_bytes)?;
             let data = self.fast_field_data(field, 1)?;
-            BytesFastFieldReader::open(
-                idx_reader
-                    .to_full()
-                    .expect("multivalue fields are always full"),
-                data,
-            )
+            BytesFastFieldReader::open(idx_reader, data)
         } else {
             Err(FastFieldNotAvailableError::new(field_entry).into())
         }

src/fieldnorm/mod.rs

@@ -34,7 +34,7 @@ mod tests {
 
     use crate::directory::{CompositeFile, Directory, RamDirectory, WritePtr};
     use crate::fieldnorm::{FieldNormReader, FieldNormsSerializer, FieldNormsWriter};
-    use crate::query::{EnableScoring, Query, TermQuery};
+    use crate::query::{Query, TermQuery};
     use crate::schema::{
         Field, IndexRecordOption, Schema, TextFieldIndexing, TextOptions, STORED, TEXT,
     };
@@ -112,7 +112,7 @@ mod tests {
             Term::from_field_text(text, "hello"),
             IndexRecordOption::WithFreqs,
         );
-        let weight = query.weight(EnableScoring::Enabled(&searcher))?;
+        let weight = query.weight(&searcher, true)?;
         let mut scorer = weight.scorer(searcher.segment_reader(0), 1.0f32)?;
         assert_eq!(scorer.doc(), 0);
         assert!((scorer.score() - 0.22920431).abs() < 0.001f32);
@@ -141,7 +141,7 @@ mod tests {
             Term::from_field_text(text, "hello"),
             IndexRecordOption::WithFreqs,
         );
-        let weight = query.weight(EnableScoring::Enabled(&searcher))?;
+        let weight = query.weight(&searcher, true)?;
         let mut scorer = weight.scorer(searcher.segment_reader(0), 1.0f32)?;
         assert_eq!(scorer.doc(), 0);
         assert!((scorer.score() - 0.22920431).abs() < 0.001f32);

src/fieldnorm/writer.rs

@@ -9,7 +9,7 @@ use crate::DocId;
 /// The `FieldNormsWriter` is in charge of tracking the fieldnorm byte
 /// of each document for each field with field norms.
 ///
-/// `FieldNormsWriter` stores a `Vec<u8>` for each tracked field, using a
+/// `FieldNormsWriter` stores a Vec<u8> for each tracked field, using a
 /// byte per document per field.
 pub struct FieldNormsWriter {
     fieldnorms_buffers: Vec<Option<Vec<u8>>>,

src/indexer/doc_id_mapping.rs

@@ -465,9 +465,9 @@ mod tests_indexsorting {
         let my_number = index.schema().get_field("my_number").unwrap();
 
         let fast_field = fast_fields.u64(my_number).unwrap();
-        assert_eq!(fast_field.get_val(0), Some(10u64));
-        assert_eq!(fast_field.get_val(1), Some(20u64));
-        assert_eq!(fast_field.get_val(2), Some(30u64));
+        assert_eq!(fast_field.get_val(0), 10u64);
+        assert_eq!(fast_field.get_val(1), 20u64);
+        assert_eq!(fast_field.get_val(2), 30u64);
 
         let multi_numbers = index.schema().get_field("multi_numbers").unwrap();
         let multifield = fast_fields.u64s(multi_numbers).unwrap();

src/indexer/index_writer.rs

@@ -19,9 +19,9 @@ use crate::indexer::index_writer_status::IndexWriterStatus;
 use crate::indexer::operation::DeleteOperation;
 use crate::indexer::stamper::Stamper;
 use crate::indexer::{MergePolicy, SegmentEntry, SegmentWriter};
-use crate::query::{EnableScoring, Query, TermQuery};
+use crate::query::{Query, TermQuery};
 use crate::schema::{Document, IndexRecordOption, Term};
-use crate::{FutureResult, Opstamp};
+use crate::{FutureResult, IndexReader, Opstamp};
 
 // Size of the margin for the `memory_arena`. A segment is closed when the remaining memory
 // in the `memory_arena` goes below MARGIN_IN_BYTES.
@@ -57,6 +57,7 @@ pub struct IndexWriter {
     _directory_lock: Option<DirectoryLock>,
 
     index: Index,
+    index_reader: IndexReader,
 
     memory_arena_in_bytes_per_thread: usize,
 
@@ -297,6 +298,8 @@ impl IndexWriter {
 
             memory_arena_in_bytes_per_thread,
             index: index.clone(),
+            index_reader: index.reader()?,
+
             index_writer_status: IndexWriterStatus::from(document_receiver),
             operation_sender: document_sender,
 
@@ -678,7 +681,8 @@ impl IndexWriter {
     /// only after calling `commit()`.
     #[doc(hidden)]
     pub fn delete_query(&self, query: Box<dyn Query>) -> crate::Result<Opstamp> {
-        let weight = query.weight(EnableScoring::Disabled(&self.index.schema()))?;
+        let weight = query.weight(&self.index_reader.searcher(), false)?;
+
         let opstamp = self.stamper.stamp();
         let delete_operation = DeleteOperation {
             opstamp,
@@ -759,7 +763,8 @@ impl IndexWriter {
             match user_op {
                 UserOperation::Delete(term) => {
                     let query = TermQuery::new(term, IndexRecordOption::Basic);
-                    let weight = query.weight(EnableScoring::Disabled(&self.index.schema()))?;
+                    let weight = query.weight(&self.index_reader.searcher(), false)?;
+
                     let delete_operation = DeleteOperation {
                         opstamp,
                         target: weight,
@@ -800,7 +805,7 @@ mod tests {
     use std::collections::{HashMap, HashSet};
     use std::net::Ipv6Addr;
 
-    use fastfield_codecs::MonotonicallyMappableToU128;
+    use fastfield_codecs::{Column, MonotonicallyMappableToU128};
     use proptest::prelude::*;
     use proptest::prop_oneof;
     use proptest::strategy::Strategy;
@@ -1467,7 +1472,7 @@ mod tests {
         let fast_field_reader = segment_reader.fast_fields().u64(id_field)?;
         let in_order_alive_ids: Vec<u64> = segment_reader
             .doc_ids_alive()
-            .map(|doc| fast_field_reader.get_val(doc).unwrap())
+            .map(|doc| fast_field_reader.get_val(doc))
             .collect();
         assert_eq!(&in_order_alive_ids[..], &[9, 8, 7, 6, 5, 4, 1, 0]);
         Ok(())
@@ -1528,7 +1533,7 @@ mod tests {
         let fast_field_reader = segment_reader.fast_fields().u64(id_field)?;
         let in_order_alive_ids: Vec<u64> = segment_reader
             .doc_ids_alive()
-            .map(|doc| fast_field_reader.get_val(doc).unwrap())
+            .map(|doc| fast_field_reader.get_val(doc))
             .collect();
         assert_eq!(&in_order_alive_ids[..], &[9, 8, 7, 6, 5, 4, 2, 0]);
         Ok(())
@@ -1777,12 +1782,7 @@ mod tests {
             .segment_readers()
             .iter()
             .flat_map(|segment_reader| {
-                let ff_reader = segment_reader
-                    .fast_fields()
-                    .u64(id_field)
-                    .unwrap()
-                    .to_full()
-                    .unwrap();
+                let ff_reader = segment_reader.fast_fields().u64(id_field).unwrap();
                 segment_reader
                     .doc_ids_alive()
                     .map(move |doc| ff_reader.get_val(doc))
@@ -1793,12 +1793,7 @@ mod tests {
             .segment_readers()
             .iter()
             .flat_map(|segment_reader| {
-                let ff_reader = segment_reader
-                    .fast_fields()
-                    .u64(id_field)
-                    .unwrap()
-                    .to_full()
-                    .unwrap();
+                let ff_reader = segment_reader.fast_fields().u64(id_field).unwrap();
                 segment_reader
                     .doc_ids_alive()
                     .map(move |doc| ff_reader.get_val(doc))
@@ -1849,7 +1844,7 @@ mod tests {
                 .filter(|id| ip_exists(**id))
                 .flat_map(|id| vec![ip_from_id(*id), ip_from_id(*id)])
                 .collect();
-            assert_eq!(num_ips, expected_multi_ips.len() as u32);
+            assert_eq!(num_ips, expected_multi_ips.len() as u64);
 
             expected_multi_ips.sort();
             all_ips.sort();
@@ -1861,7 +1856,7 @@ mod tests {
                 .iter()
                 .map(|segment_reader| {
                     let ff_reader = segment_reader.fast_fields().ip_addrs(ips_field).unwrap();
-                    ff_reader.get_index_reader().num_docs() as usize
+                    ff_reader.num_docs() as usize
                 })
                 .sum();
             assert_eq!(num_docs, num_docs_expected);
@@ -1874,7 +1869,7 @@ mod tests {
             .flat_map(|segment_reader| {
                 let ff_reader = segment_reader.fast_fields().ip_addr(ip_field).unwrap();
                 segment_reader.doc_ids_alive().flat_map(move |doc| {
-                    let val = ff_reader.get_val(doc).unwrap(); // TODO handle null
+                    let val = ff_reader.get_val(doc);
                     if val == Ipv6Addr::from_u128(0) {
                         // TODO Fix null handling
                         None
@@ -1931,7 +1926,7 @@ mod tests {
                 ff_reader.get_vals(doc, &mut vals);
                 assert_eq!(vals.len(), 2);
                 assert_eq!(vals[0], vals[1]);
-                assert_eq!(id_reader.get_val(doc).unwrap(), vals[0]);
+                assert_eq!(id_reader.get_val(doc), vals[0]);
 
                 let mut bool_vals = vec![];
                 bool_ff_reader.get_vals(doc, &mut bool_vals);
@@ -2127,7 +2122,7 @@ mod tests {
                 facet_reader
                     .facet_from_ord(facet_ords[0], &mut facet)
                     .unwrap();
-                let id = ff_reader.get_val(doc_id).unwrap();
+                let id = ff_reader.get_val(doc_id);
                 let facet_expected = Facet::from(&("/cola/".to_string() + &id.to_string()));
 
                 assert_eq!(facet, facet_expected);

src/indexer/json_term_writer.rs

@@ -261,39 +261,6 @@ pub struct JsonTermWriter<'a> {
     path_stack: Vec<usize>,
 }
 
-/// Splits a json path supplied to the query parser in such a way that
-/// `.` can be escaped.
-///
-/// In other words,
-/// - `k8s.node` ends up as `["k8s", "node"]`.
-/// - `k8s\.node` ends up as `["k8s.node"]`.
-fn split_json_path(json_path: &str) -> Vec<String> {
-    let mut escaped_state: bool = false;
-    let mut json_path_segments = Vec::new();
-    let mut buffer = String::new();
-    for ch in json_path.chars() {
-        if escaped_state {
-            buffer.push(ch);
-            escaped_state = false;
-            continue;
-        }
-        match ch {
-            '\\' => {
-                escaped_state = true;
-            }
-            '.' => {
-                let new_segment = std::mem::take(&mut buffer);
-                json_path_segments.push(new_segment);
-            }
-            _ => {
-                buffer.push(ch);
-            }
-        }
-    }
-    json_path_segments.push(buffer);
-    json_path_segments
-}
-
 impl<'a> JsonTermWriter<'a> {
     pub fn from_field_and_json_path(
         field: Field,
@@ -302,8 +269,8 @@ impl<'a> JsonTermWriter<'a> {
     ) -> Self {
         term_buffer.set_field_and_type(field, Type::Json);
         let mut json_term_writer = Self::wrap(term_buffer);
-        for segment in split_json_path(json_path) {
-            json_term_writer.push_path_segment(&segment);
+        for segment in json_path.split('.') {
+            json_term_writer.push_path_segment(segment);
         }
         json_term_writer
     }
@@ -383,7 +350,7 @@ impl<'a> JsonTermWriter<'a> {
 
 #[cfg(test)]
 mod tests {
-    use super::{split_json_path, JsonTermWriter};
+    use super::JsonTermWriter;
     use crate::schema::{Field, Type};
     use crate::Term;
 
@@ -528,48 +495,4 @@ mod tests {
         json_writer.set_str("pink");
         assert_eq!(json_writer.path(), b"color\x01hue");
     }
-
-    #[test]
-    fn test_split_json_path_simple() {
-        let json_path = split_json_path("titi.toto");
-        assert_eq!(&json_path, &["titi", "toto"]);
-    }
-
-    #[test]
-    fn test_split_json_path_single_segment() {
-        let json_path = split_json_path("toto");
-        assert_eq!(&json_path, &["toto"]);
-    }
-
-    #[test]
-    fn test_split_json_path_trailing_dot() {
-        let json_path = split_json_path("toto.");
-        assert_eq!(&json_path, &["toto", ""]);
-    }
-
-    #[test]
-    fn test_split_json_path_heading_dot() {
-        let json_path = split_json_path(".toto");
-        assert_eq!(&json_path, &["", "toto"]);
-    }
-
-    #[test]
-    fn test_split_json_path_escaped_dot() {
-        let json_path = split_json_path(r#"toto\.titi"#);
-        assert_eq!(&json_path, &["toto.titi"]);
-        let json_path_2 = split_json_path(r#"k8s\.container\.name"#);
-        assert_eq!(&json_path_2, &["k8s.container.name"]);
-    }
-
-    #[test]
-    fn test_split_json_path_escaped_backslash() {
-        let json_path = split_json_path(r#"toto\\titi"#);
-        assert_eq!(&json_path, &[r#"toto\titi"#]);
-    }
-
-    #[test]
-    fn test_split_json_path_escaped_normal_letter() {
-        let json_path = split_json_path(r#"toto\titi"#);
-        assert_eq!(&json_path, &[r#"tototiti"#]);
-    }
 }

src/indexer/merger.rs

@@ -401,15 +401,10 @@ impl IndexMerger {
             .readers
             .iter()
             .map(|reader| {
-                let u128_reader: Arc<dyn Column<u128>> = reader
-                    .fast_fields()
-                    .u128(field)
-                    .expect(
-                        "Failed to find a reader for single fast field. This is a tantivy bug and \
-                         it should never happen.",
-                    )
-                    .to_full()
-                    .expect("temp migration solution");
+                let u128_reader: Arc<dyn Column<u128>> = reader.fast_fields().u128(field).expect(
+                    "Failed to find a reader for single fast field. This is a tantivy bug and it \
+                     should never happen.",
+                );
                 u128_reader
             })
             .collect::<Vec<_>>();
@@ -470,11 +465,7 @@ impl IndexMerger {
         sort_by_field: &IndexSortByField,
     ) -> crate::Result<Arc<dyn Column>> {
         let field_id = expect_field_id_for_sort_field(reader.schema(), sort_by_field)?; // for now expect fastfield, but not strictly required
-        let value_accessor = reader
-            .fast_fields()
-            .u64_lenient(field_id)?
-            .to_full()
-            .expect("temp migration solution");
+        let value_accessor = reader.fast_fields().u64_lenient(field_id)?;
         Ok(value_accessor)
     }
     /// Collecting value_accessors into a vec to bind the lifetime.
@@ -1073,7 +1064,7 @@ mod tests {
     };
     use crate::collector::{Count, FacetCollector};
     use crate::core::Index;
-    use crate::query::{AllQuery, BooleanQuery, EnableScoring, Scorer, TermQuery};
+    use crate::query::{AllQuery, BooleanQuery, Scorer, TermQuery};
     use crate::schema::{
         Cardinality, Document, Facet, FacetOptions, IndexRecordOption, NumericOptions, Term,
         TextFieldIndexing, INDEXED, TEXT,
@@ -1377,16 +1368,16 @@ mod tests {
                 .fast_fields()
                 .u64(score_field)
                 .unwrap();
-            assert_eq!(score_field_reader.min_value(), Some(4000));
-            assert_eq!(score_field_reader.max_value(), Some(7000));
+            assert_eq!(score_field_reader.min_value(), 4000);
+            assert_eq!(score_field_reader.max_value(), 7000);
 
             let score_field_reader = searcher
                 .segment_reader(1)
                 .fast_fields()
                 .u64(score_field)
                 .unwrap();
-            assert_eq!(score_field_reader.min_value(), Some(1));
-            assert_eq!(score_field_reader.max_value(), Some(3));
+            assert_eq!(score_field_reader.min_value(), 1);
+            assert_eq!(score_field_reader.max_value(), 3);
         }
         {
             // merging the segments
@@ -1431,8 +1422,8 @@ mod tests {
                 .fast_fields()
                 .u64(score_field)
                 .unwrap();
-            assert_eq!(score_field_reader.min_value(), Some(3));
-            assert_eq!(score_field_reader.max_value(), Some(7000));
+            assert_eq!(score_field_reader.min_value(), 3);
+            assert_eq!(score_field_reader.max_value(), 7000);
         }
         {
             // test a commit with only deletes
@@ -1478,8 +1469,8 @@ mod tests {
                 .fast_fields()
                 .u64(score_field)
                 .unwrap();
-            assert_eq!(score_field_reader.min_value(), Some(3));
-            assert_eq!(score_field_reader.max_value(), Some(7000));
+            assert_eq!(score_field_reader.min_value(), 3);
+            assert_eq!(score_field_reader.max_value(), 7000);
         }
         {
             // Test merging a single segment in order to remove deletes.
@@ -1525,8 +1516,8 @@ mod tests {
                 .fast_fields()
                 .u64(score_field)
                 .unwrap();
-            assert_eq!(score_field_reader.min_value(), Some(6000));
-            assert_eq!(score_field_reader.max_value(), Some(7000));
+            assert_eq!(score_field_reader.min_value(), 6000);
+            assert_eq!(score_field_reader.max_value(), 7000);
         }
 
         {
@@ -1986,7 +1977,7 @@ mod tests {
         let reader = index.reader()?;
         let searcher = reader.searcher();
         let mut term_scorer = term_query
-            .specialized_weight(EnableScoring::Enabled(&searcher))?
+            .specialized_weight(&searcher, true)?
             .specialized_scorer(searcher.segment_reader(0u32), 1.0)?;
         assert_eq!(term_scorer.doc(), 0);
         assert_nearly_equals!(term_scorer.block_max_score(), 0.0079681855);
@@ -2001,7 +1992,7 @@ mod tests {
         assert_eq!(searcher.segment_readers().len(), 2);
         for segment_reader in searcher.segment_readers() {
             let mut term_scorer = term_query
-                .specialized_weight(EnableScoring::Enabled(&searcher))?
+                .specialized_weight(&searcher, true)?
                 .specialized_scorer(segment_reader, 1.0)?;
             // the difference compared to before is intrinsic to the bm25 formula. no worries
             // there.
@@ -2026,7 +2017,7 @@ mod tests {
 
         let segment_reader = searcher.segment_reader(0u32);
         let mut term_scorer = term_query
-            .specialized_weight(EnableScoring::Enabled(&searcher))?
+            .specialized_weight(&searcher, true)?
             .specialized_scorer(segment_reader, 1.0)?;
         // the difference compared to before is intrinsic to the bm25 formula. no worries there.
         for doc in segment_reader.doc_ids_alive() {

src/indexer/merger_sorted_index_test.rs

@@ -186,17 +186,17 @@ mod tests {
 
         let fast_fields = segment_reader.fast_fields();
         let fast_field = fast_fields.u64(int_field).unwrap();
-        assert_eq!(fast_field.get_val(5), Some(1u64));
-        assert_eq!(fast_field.get_val(4), Some(2u64));
-        assert_eq!(fast_field.get_val(3), Some(3u64));
+        assert_eq!(fast_field.get_val(5), 1u64);
+        assert_eq!(fast_field.get_val(4), 2u64);
+        assert_eq!(fast_field.get_val(3), 3u64);
         if force_disjunct_segment_sort_values {
-            assert_eq!(fast_field.get_val(2), Some(20u64));
-            assert_eq!(fast_field.get_val(1), Some(100u64));
+            assert_eq!(fast_field.get_val(2), 20u64);
+            assert_eq!(fast_field.get_val(1), 100u64);
         } else {
-            assert_eq!(fast_field.get_val(2), Some(10u64));
-            assert_eq!(fast_field.get_val(1), Some(20u64));
+            assert_eq!(fast_field.get_val(2), 10u64);
+            assert_eq!(fast_field.get_val(1), 20u64);
         }
-        assert_eq!(fast_field.get_val(0), Some(1_000u64));
+        assert_eq!(fast_field.get_val(0), 1_000u64);
 
         // test new field norm mapping
         {
@@ -373,12 +373,12 @@ mod tests {
 
         let fast_fields = segment_reader.fast_fields();
         let fast_field = fast_fields.u64(int_field).unwrap();
-        assert_eq!(fast_field.get_val(0), Some(1u64));
-        assert_eq!(fast_field.get_val(1), Some(2u64));
-        assert_eq!(fast_field.get_val(2), Some(3u64));
-        assert_eq!(fast_field.get_val(3), Some(10u64));
-        assert_eq!(fast_field.get_val(4), Some(20u64));
-        assert_eq!(fast_field.get_val(5), Some(1_000u64));
+        assert_eq!(fast_field.get_val(0), 1u64);
+        assert_eq!(fast_field.get_val(1), 2u64);
+        assert_eq!(fast_field.get_val(2), 3u64);
+        assert_eq!(fast_field.get_val(3), 10u64);
+        assert_eq!(fast_field.get_val(4), 20u64);
+        assert_eq!(fast_field.get_val(5), 1_000u64);
 
         let get_vals = |fast_field: &MultiValuedFastFieldReader<u64>, doc_id: u32| -> Vec<u64> {
             let mut vals = vec![];
@@ -535,15 +535,11 @@ mod bench_sorted_index_merge {
         b.iter(|| {
             let sorted_doc_ids = doc_id_mapping.iter_old_doc_addrs().map(|doc_addr| {
                 let reader = &merger.readers[doc_addr.segment_ord as usize];
-                let u64_reader: Arc<dyn Column<u64>> = reader
-                    .fast_fields()
-                    .typed_fast_field_reader(field)
-                    .expect(
+                let u64_reader: Arc<dyn Column<u64>> =
+                    reader.fast_fields().typed_fast_field_reader(field).expect(
                         "Failed to find a reader for single fast field. This is a tantivy bug and \
                          it should never happen.",
-                    )
-                    .to_full()
-                    .unwrap();
+                    );
                 (doc_addr.doc_id, reader, u64_reader)
             });
             // add values in order of the new doc_ids

src/indexer/mod.rs

@@ -58,15 +58,13 @@ type AddBatchReceiver = channel::Receiver<AddBatch>;
 #[cfg(feature = "mmap")]
 #[cfg(test)]
 mod tests_mmap {
-    use crate::collector::Count;
-    use crate::query::QueryParser;
-    use crate::schema::{Schema, STORED, TEXT};
+    use crate::schema::{self, Schema};
     use crate::{Index, Term};
 
     #[test]
     fn test_advance_delete_bug() -> crate::Result<()> {
         let mut schema_builder = Schema::builder();
-        let text_field = schema_builder.add_text_field("text", TEXT);
+        let text_field = schema_builder.add_text_field("text", schema::TEXT);
         let index = Index::create_from_tempdir(schema_builder.build())?;
         let mut index_writer = index.writer_for_tests()?;
         // there must be one deleted document in the segment
@@ -77,26 +75,7 @@ mod tests_mmap {
             index_writer.add_document(doc!(text_field=>"c"))?;
         }
         index_writer.commit()?;
+        index_writer.commit()?;
         Ok(())
     }
-
-    #[test]
-    fn test_json_field_espace() {
-        let mut schema_builder = Schema::builder();
-        let json_field = schema_builder.add_json_field("json", TEXT | STORED);
-        let index = Index::create_in_ram(schema_builder.build());
-        let mut index_writer = index.writer_for_tests().unwrap();
-        let json = serde_json::json!({"k8s.container.name": "prometheus", "val": "hello"});
-        index_writer.add_document(doc!(json_field=>json)).unwrap();
-        index_writer.commit().unwrap();
-        let reader = index.reader().unwrap();
-        let searcher = reader.searcher();
-        assert_eq!(searcher.num_docs(), 1);
-        let parse_query = QueryParser::for_index(&index, Vec::new());
-        let query = parse_query
-            .parse_query(r#"json.k8s\.container\.name:prometheus"#)
-            .unwrap();
-        let num_docs = searcher.search(&query, &Count).unwrap();
-        assert_eq!(num_docs, 1);
-    }
 }

src/indexer/sorted_doc_id_column.rs

@@ -46,15 +46,11 @@ impl<'a> RemappedDocIdColumn<'a> {
         let (min_value, max_value) = readers
             .iter()
             .filter_map(|reader| {
-                let u64_reader: Arc<dyn Column<u64>> = reader
-                    .fast_fields()
-                    .typed_fast_field_reader(field)
-                    .expect(
+                let u64_reader: Arc<dyn Column<u64>> =
+                    reader.fast_fields().typed_fast_field_reader(field).expect(
                         "Failed to find a reader for single fast field. This is a tantivy bug and \
                          it should never happen.",
-                    )
-                    .to_full()
-                    .expect("temp migration solution");
+                    );
                 compute_min_max_val(&*u64_reader, reader)
             })
             .reduce(|a, b| (a.0.min(b.0), a.1.max(b.1)))
@@ -63,15 +59,11 @@ impl<'a> RemappedDocIdColumn<'a> {
         let fast_field_readers = readers
             .iter()
             .map(|reader| {
-                let u64_reader: Arc<dyn Column<u64>> = reader
-                    .fast_fields()
-                    .typed_fast_field_reader(field)
-                    .expect(
+                let u64_reader: Arc<dyn Column<u64>> =
+                    reader.fast_fields().typed_fast_field_reader(field).expect(
                         "Failed to find a reader for single fast field. This is a tantivy bug and \
                          it should never happen.",
-                    )
-                    .to_full()
-                    .expect("temp migration solution");
+                    );
                 u64_reader
             })
             .collect::<Vec<_>>();

src/indexer/sorted_doc_id_multivalue_column.rs

@@ -117,7 +117,7 @@ impl<'a> RemappedDocIdMultiValueIndexColumn<'a> {
             let segment_reader = segment_and_ff_reader.0;
             let multi_value_length_reader = segment_and_ff_reader.1;
             if !segment_reader.has_deletes() {
-                max_value += multi_value_length_reader.total_num_vals() as u64;
+                max_value += multi_value_length_reader.total_num_vals();
             } else {
                 for doc in segment_reader.doc_ids_alive() {
                     max_value += multi_value_length_reader.num_vals_for_doc(doc) as u64;

src/lib.rs

@@ -277,8 +277,6 @@ pub mod fastfield;
 pub mod fieldnorm;
 pub mod positions;
 pub mod postings;
-
-/// Module containing the different query implementations.
 pub mod query;
 pub mod schema;
 pub mod space_usage;
@@ -1037,21 +1035,21 @@ pub mod tests {
             let fast_field_reader_opt = segment_reader.fast_fields().u64(fast_field_unsigned);
             assert!(fast_field_reader_opt.is_ok());
             let fast_field_reader = fast_field_reader_opt.unwrap();
-            assert_eq!(fast_field_reader.get_val(0), Some(4u64))
+            assert_eq!(fast_field_reader.get_val(0), 4u64)
         }
 
         {
             let fast_field_reader_res = segment_reader.fast_fields().i64(fast_field_signed);
             assert!(fast_field_reader_res.is_ok());
             let fast_field_reader = fast_field_reader_res.unwrap();
-            assert_eq!(fast_field_reader.get_val(0), Some(4i64))
+            assert_eq!(fast_field_reader.get_val(0), 4i64)
         }
 
         {
             let fast_field_reader_res = segment_reader.fast_fields().f64(fast_field_float);
             assert!(fast_field_reader_res.is_ok());
             let fast_field_reader = fast_field_reader_res.unwrap();
-            assert_eq!(fast_field_reader.get_val(0), Some(4f64))
+            assert_eq!(fast_field_reader.get_val(0), 4f64)
         }
         Ok(())
     }

src/query/all_query.rs

@@ -1,8 +1,8 @@
-use crate::core::SegmentReader;
+use crate::core::{Searcher, SegmentReader};
 use crate::docset::{DocSet, TERMINATED};
 use crate::query::boost_query::BoostScorer;
 use crate::query::explanation::does_not_match;
-use crate::query::{EnableScoring, Explanation, Query, Scorer, Weight};
+use crate::query::{Explanation, Query, Scorer, Weight};
 use crate::{DocId, Score};
 
 /// Query that matches all of the documents.
@@ -12,7 +12,7 @@ use crate::{DocId, Score};
 pub struct AllQuery;
 
 impl Query for AllQuery {
-    fn weight(&self, _: EnableScoring<'_>) -> crate::Result<Box<dyn Weight>> {
+    fn weight(&self, _: &Searcher, _: bool) -> crate::Result<Box<dyn Weight>> {
         Ok(Box::new(AllWeight))
     }
 }
@@ -72,7 +72,7 @@ impl Scorer for AllScorer {
 mod tests {
     use super::AllQuery;
     use crate::docset::TERMINATED;
-    use crate::query::{EnableScoring, Query};
+    use crate::query::Query;
     use crate::schema::{Schema, TEXT};
     use crate::Index;
 
@@ -95,7 +95,7 @@ mod tests {
         let index = create_test_index()?;
         let reader = index.reader()?;
         let searcher = reader.searcher();
-        let weight = AllQuery.weight(EnableScoring::Disabled(&index.schema()))?;
+        let weight = AllQuery.weight(&searcher, false)?;
         {
             let reader = searcher.segment_reader(0);
             let mut scorer = weight.scorer(reader, 1.0)?;
@@ -118,7 +118,7 @@ mod tests {
         let index = create_test_index()?;
         let reader = index.reader()?;
         let searcher = reader.searcher();
-        let weight = AllQuery.weight(EnableScoring::Disabled(searcher.schema()))?;
+        let weight = AllQuery.weight(&searcher, false)?;
         let reader = searcher.segment_reader(0);
         {
             let mut scorer = weight.scorer(reader, 2.0)?;

src/query/boolean_query/boolean_query.rs

@@ -1,6 +1,7 @@
 use super::boolean_weight::BooleanWeight;
-use crate::query::{EnableScoring, Occur, Query, SumWithCoordsCombiner, TermQuery, Weight};
+use crate::query::{Occur, Query, SumWithCoordsCombiner, TermQuery, Weight};
 use crate::schema::{IndexRecordOption, Term};
+use crate::Searcher;
 
 /// The boolean query returns a set of documents
 /// that matches the Boolean combination of constituent subqueries.
@@ -142,15 +143,17 @@ impl From<Vec<(Occur, Box<dyn Query>)>> for BooleanQuery {
 }
 
 impl Query for BooleanQuery {
-    fn weight(&self, enable_scoring: EnableScoring<'_>) -> crate::Result<Box<dyn Weight>> {
+    fn weight(&self, searcher: &Searcher, scoring_enabled: bool) -> crate::Result<Box<dyn Weight>> {
         let sub_weights = self
             .subqueries
             .iter()
-            .map(|&(ref occur, ref subquery)| Ok((*occur, subquery.weight(enable_scoring)?)))
+            .map(|&(ref occur, ref subquery)| {
+                Ok((*occur, subquery.weight(searcher, scoring_enabled)?))
+            })
             .collect::<crate::Result<_>>()?;
         Ok(Box::new(BooleanWeight::new(
             sub_weights,
-            enable_scoring.is_scoring_enabled(),
+            scoring_enabled,
             Box::new(SumWithCoordsCombiner::default),
         )))
     }

src/query/boolean_query/mod.rs

@@ -15,8 +15,7 @@ mod tests {
     use crate::query::score_combiner::SumWithCoordsCombiner;
     use crate::query::term_query::TermScorer;
     use crate::query::{
-        EnableScoring, Intersection, Occur, Query, QueryParser, RequiredOptionalScorer, Scorer,
-        TermQuery,
+        Intersection, Occur, Query, QueryParser, RequiredOptionalScorer, Scorer, TermQuery,
     };
     use crate::schema::*;
     use crate::{assert_nearly_equals, DocAddress, DocId, Index, Score};
@@ -55,7 +54,7 @@ mod tests {
         let query_parser = QueryParser::for_index(&index, vec![text_field]);
         let query = query_parser.parse_query("+a")?;
         let searcher = index.reader()?.searcher();
-        let weight = query.weight(EnableScoring::Enabled(&searcher))?;
+        let weight = query.weight(&searcher, true)?;
         let scorer = weight.scorer(searcher.segment_reader(0u32), 1.0)?;
         assert!(scorer.is::<TermScorer>());
         Ok(())
@@ -68,13 +67,13 @@ mod tests {
         let searcher = index.reader()?.searcher();
         {
             let query = query_parser.parse_query("+a +b +c")?;
-            let weight = query.weight(EnableScoring::Enabled(&searcher))?;
+            let weight = query.weight(&searcher, true)?;
             let scorer = weight.scorer(searcher.segment_reader(0u32), 1.0)?;
             assert!(scorer.is::<Intersection<TermScorer>>());
         }
         {
             let query = query_parser.parse_query("+a +(b c)")?;
-            let weight = query.weight(EnableScoring::Enabled(&searcher))?;
+            let weight = query.weight(&searcher, true)?;
             let scorer = weight.scorer(searcher.segment_reader(0u32), 1.0)?;
             assert!(scorer.is::<Intersection<Box<dyn Scorer>>>());
         }
@@ -88,7 +87,7 @@ mod tests {
         let searcher = index.reader()?.searcher();
         {
             let query = query_parser.parse_query("+a b")?;
-            let weight = query.weight(EnableScoring::Enabled(&searcher))?;
+            let weight = query.weight(&searcher, true)?;
             let scorer = weight.scorer(searcher.segment_reader(0u32), 1.0)?;
             assert!(scorer.is::<RequiredOptionalScorer<
                 Box<dyn Scorer>,
@@ -98,7 +97,7 @@ mod tests {
         }
         {
             let query = query_parser.parse_query("+a b")?;
-            let weight = query.weight(EnableScoring::Disabled(searcher.schema()))?;
+            let weight = query.weight(&searcher, false)?;
             let scorer = weight.scorer(searcher.segment_reader(0u32), 1.0)?;
             assert!(scorer.is::<TermScorer>());
         }
@@ -242,9 +241,7 @@ mod tests {
         let searcher = reader.searcher();
         let boolean_query =
             BooleanQuery::new(vec![(Occur::Should, term_a), (Occur::Should, term_b)]);
-        let boolean_weight = boolean_query
-            .weight(EnableScoring::Enabled(&searcher))
-            .unwrap();
+        let boolean_weight = boolean_query.weight(&searcher, true).unwrap();
         {
             let mut boolean_scorer = boolean_weight.scorer(searcher.segment_reader(0u32), 1.0)?;
             assert_eq!(boolean_scorer.doc(), 0u32);

src/query/boost_query.rs

@@ -2,8 +2,8 @@ use std::fmt;
 
 use crate::fastfield::AliveBitSet;
 use crate::query::explanation::does_not_match;
-use crate::query::{EnableScoring, Explanation, Query, Scorer, Weight};
-use crate::{DocId, DocSet, Score, SegmentReader, Term};
+use crate::query::{Explanation, Query, Scorer, Weight};
+use crate::{DocId, DocSet, Score, Searcher, SegmentReader, Term};
 
 /// `BoostQuery` is a wrapper over a query used to boost its score.
 ///
@@ -38,9 +38,9 @@ impl fmt::Debug for BoostQuery {
 }
 
 impl Query for BoostQuery {
-    fn weight(&self, enable_scoring: EnableScoring<'_>) -> crate::Result<Box<dyn Weight>> {
-        let weight_without_boost = self.query.weight(enable_scoring)?;
-        let boosted_weight = if enable_scoring.is_scoring_enabled() {
+    fn weight(&self, searcher: &Searcher, scoring_enabled: bool) -> crate::Result<Box<dyn Weight>> {
+        let weight_without_boost = self.query.weight(searcher, scoring_enabled)?;
+        let boosted_weight = if scoring_enabled {
             Box::new(BoostWeight::new(weight_without_boost, self.boost))
         } else {
             weight_without_boost

src/query/const_score_query.rs

@@ -1,7 +1,7 @@
 use std::fmt;
 
-use crate::query::{EnableScoring, Explanation, Query, Scorer, Weight};
-use crate::{DocId, DocSet, Score, SegmentReader, TantivyError, Term};
+use crate::query::{Explanation, Query, Scorer, Weight};
+use crate::{DocId, DocSet, Score, Searcher, SegmentReader, TantivyError, Term};
 
 /// `ConstScoreQuery` is a wrapper over a query to provide a constant score.
 /// It can avoid unnecessary score computation on the wrapped query.
@@ -36,9 +36,9 @@ impl fmt::Debug for ConstScoreQuery {
 }
 
 impl Query for ConstScoreQuery {
-    fn weight(&self, enable_scoring: EnableScoring<'_>) -> crate::Result<Box<dyn Weight>> {
-        let inner_weight = self.query.weight(enable_scoring)?;
-        Ok(if enable_scoring.is_scoring_enabled() {
+    fn weight(&self, searcher: &Searcher, scoring_enabled: bool) -> crate::Result<Box<dyn Weight>> {
+        let inner_weight = self.query.weight(searcher, scoring_enabled)?;
+        Ok(if scoring_enabled {
             Box::new(ConstWeight::new(inner_weight, self.score))
         } else {
             inner_weight

src/query/disjunction_max_query.rs

@@ -1,7 +1,7 @@
 use tantivy_query_grammar::Occur;
 
-use crate::query::{BooleanWeight, DisjunctionMaxCombiner, EnableScoring, Query, Weight};
-use crate::{Score, Term};
+use crate::query::{BooleanWeight, DisjunctionMaxCombiner, Query, Weight};
+use crate::{Score, Searcher, Term};
 
 /// The disjunction max query кeturns documents matching one or more wrapped queries,
 /// called query clauses or clauses.
@@ -91,16 +91,16 @@ impl Clone for DisjunctionMaxQuery {
 }
 
 impl Query for DisjunctionMaxQuery {
-    fn weight(&self, enable_scoring: EnableScoring<'_>) -> crate::Result<Box<dyn Weight>> {
+    fn weight(&self, searcher: &Searcher, scoring_enabled: bool) -> crate::Result<Box<dyn Weight>> {
         let disjuncts = self
             .disjuncts
             .iter()
-            .map(|disjunct| Ok((Occur::Should, disjunct.weight(enable_scoring)?)))
+            .map(|disjunct| Ok((Occur::Should, disjunct.weight(searcher, scoring_enabled)?)))
             .collect::<crate::Result<_>>()?;
         let tie_breaker = self.tie_breaker;
         Ok(Box::new(BooleanWeight::new(
             disjuncts,
-            enable_scoring.is_scoring_enabled(),
+            scoring_enabled,
             Box::new(move || DisjunctionMaxCombiner::with_tie_breaker(tie_breaker)),
         )))
     }

src/query/empty_query.rs

@@ -1,7 +1,7 @@
 use super::Scorer;
 use crate::docset::TERMINATED;
 use crate::query::explanation::does_not_match;
-use crate::query::{EnableScoring, Explanation, Query, Weight};
+use crate::query::{Explanation, Query, Weight};
 use crate::{DocId, DocSet, Score, Searcher, SegmentReader};
 
 /// `EmptyQuery` is a dummy `Query` in which no document matches.
@@ -11,7 +11,11 @@ use crate::{DocId, DocSet, Score, Searcher, SegmentReader};
 pub struct EmptyQuery;
 
 impl Query for EmptyQuery {
-    fn weight(&self, _enable_scoring: EnableScoring<'_>) -> crate::Result<Box<dyn Weight>> {
+    fn weight(
+        &self,
+        _searcher: &Searcher,
+        _scoring_enabled: bool,
+    ) -> crate::Result<Box<dyn Weight>> {
         Ok(Box::new(EmptyWeight))
     }
 

src/query/fuzzy_query.rs

@@ -5,8 +5,9 @@ use levenshtein_automata::{Distance, LevenshteinAutomatonBuilder, DFA};
 use once_cell::sync::Lazy;
 use tantivy_fst::Automaton;
 
-use crate::query::{AutomatonWeight, EnableScoring, Query, Weight};
+use crate::query::{AutomatonWeight, Query, Weight};
 use crate::schema::Term;
+use crate::Searcher;
 use crate::TantivyError::InvalidArgument;
 
 pub(crate) struct DfaWrapper(pub DFA);
@@ -157,7 +158,11 @@ impl FuzzyTermQuery {
 }
 
 impl Query for FuzzyTermQuery {
-    fn weight(&self, _enable_scoring: EnableScoring<'_>) -> crate::Result<Box<dyn Weight>> {
+    fn weight(
+        &self,
+        _searcher: &Searcher,
+        _scoring_enabled: bool,
+    ) -> crate::Result<Box<dyn Weight>> {
         Ok(Box::new(self.specialized_weight()?))
     }
 }

src/query/mod.rs

@@ -1,3 +1,5 @@
+//! Query Module
+
 mod all_query;
 mod automaton_weight;
 mod bitset;
@@ -49,7 +51,7 @@ pub use self::fuzzy_query::FuzzyTermQuery;
 pub use self::intersection::{intersect_scorers, Intersection};
 pub use self::more_like_this::{MoreLikeThisQuery, MoreLikeThisQueryBuilder};
 pub use self::phrase_query::PhraseQuery;
-pub use self::query::{EnableScoring, Query, QueryClone};
+pub use self::query::{Query, QueryClone};
 pub use self::query_parser::{QueryParser, QueryParserError};
 pub use self::range_query::RangeQuery;
 pub use self::regex_query::RegexQuery;

src/query/more_like_this/mod.rs

@@ -1,6 +1,4 @@
 mod more_like_this;
-
-/// Module containing the different query implementations.
 mod query;
 
 pub use self::more_like_this::MoreLikeThis;

src/query/more_like_this/query.rs

@@ -1,7 +1,7 @@
 use super::MoreLikeThis;
-use crate::query::{EnableScoring, Query, Weight};
+use crate::query::{Query, Weight};
 use crate::schema::{Field, Value};
-use crate::DocAddress;
+use crate::{DocAddress, Result, Searcher};
 
 /// A query that matches all of the documents similar to a document
 /// or a set of field values provided.
@@ -42,23 +42,16 @@ impl MoreLikeThisQuery {
 }
 
 impl Query for MoreLikeThisQuery {
-    fn weight(&self, enable_scoring: EnableScoring<'_>) -> crate::Result<Box<dyn Weight>> {
-        let searcher = match enable_scoring {
-            EnableScoring::Enabled(searcher) => searcher,
-            EnableScoring::Disabled(_) => {
-                let err = "MoreLikeThisQuery requires to enable scoring.".to_string();
-                return Err(crate::TantivyError::InvalidArgument(err));
-            }
-        };
+    fn weight(&self, searcher: &Searcher, scoring_enabled: bool) -> Result<Box<dyn Weight>> {
         match &self.target {
             TargetDocument::DocumentAdress(doc_address) => self
                 .mlt
                 .query_with_document(searcher, *doc_address)?
-                .weight(enable_scoring),
+                .weight(searcher, scoring_enabled),
             TargetDocument::DocumentFields(doc_fields) => self
                 .mlt
                 .query_with_document_fields(searcher, doc_fields)?
-                .weight(enable_scoring),
+                .weight(searcher, scoring_enabled),
         }
     }
 }

src/query/phrase_query/mod.rs

@@ -14,7 +14,7 @@ pub mod tests {
     use super::*;
     use crate::collector::tests::{TEST_COLLECTOR_WITHOUT_SCORE, TEST_COLLECTOR_WITH_SCORE};
     use crate::core::Index;
-    use crate::query::{EnableScoring, QueryParser, Weight};
+    use crate::query::{QueryParser, Weight};
     use crate::schema::{Schema, Term, TEXT};
     use crate::{assert_nearly_equals, DocAddress, DocId, TERMINATED};
 
@@ -79,8 +79,7 @@ pub mod tests {
             .map(|text| Term::from_field_text(text_field, text))
             .collect();
         let phrase_query = PhraseQuery::new(terms);
-        let phrase_weight =
-            phrase_query.phrase_weight(EnableScoring::Disabled(searcher.schema()))?;
+        let phrase_weight = phrase_query.phrase_weight(&searcher, false)?;
         let mut phrase_scorer = phrase_weight.scorer(searcher.segment_reader(0), 1.0)?;
         assert_eq!(phrase_scorer.doc(), 1);
         assert_eq!(phrase_scorer.advance(), TERMINATED);
@@ -360,9 +359,7 @@ pub mod tests {
         let matching_docs = |query: &str| {
             let query_parser = QueryParser::for_index(&index, vec![json_field]);
             let phrase_query = query_parser.parse_query(query).unwrap();
-            let phrase_weight = phrase_query
-                .weight(EnableScoring::Disabled(searcher.schema()))
-                .unwrap();
+            let phrase_weight = phrase_query.weight(&searcher, false).unwrap();
             let mut phrase_scorer = phrase_weight
                 .scorer(searcher.segment_reader(0), 1.0f32)
                 .unwrap();

src/query/phrase_query/phrase_query.rs

@@ -1,6 +1,7 @@
 use super::PhraseWeight;
+use crate::core::searcher::Searcher;
 use crate::query::bm25::Bm25Weight;
-use crate::query::{EnableScoring, Query, Weight};
+use crate::query::{Query, Weight};
 use crate::schema::{Field, IndexRecordOption, Term};
 
 /// `PhraseQuery` matches a specific sequence of words.
@@ -66,7 +67,7 @@ impl PhraseQuery {
     /// Slop allowed for the phrase.
     ///
     /// The query will match if its terms are separated by `slop` terms at most.
-    /// By default the slop is 0 meaning query terms need to be adjacent.
+    /// By default the slop is 0 meaning query terms need to be adjacent.  
     pub fn set_slop(&mut self, value: u32) {
         self.slop = value;
     }
@@ -90,9 +91,10 @@ impl PhraseQuery {
     /// a specialized type [`PhraseWeight`] instead of a Boxed trait.
     pub(crate) fn phrase_weight(
         &self,
-        enable_scoring: EnableScoring<'_>,
+        searcher: &Searcher,
+        scoring_enabled: bool,
     ) -> crate::Result<PhraseWeight> {
-        let schema = enable_scoring.schema();
+        let schema = searcher.schema();
         let field_entry = schema.get_field_entry(self.field);
         let has_positions = field_entry
             .field_type()
@@ -107,11 +109,8 @@ impl PhraseQuery {
             )));
         }
         let terms = self.phrase_terms();
-        let bm25_weight_opt = match enable_scoring {
-            EnableScoring::Enabled(searcher) => Some(Bm25Weight::for_terms(searcher, &terms)?),
-            EnableScoring::Disabled(_) => None,
-        };
-        let mut weight = PhraseWeight::new(self.phrase_terms.clone(), bm25_weight_opt);
+        let bm25_weight = Bm25Weight::for_terms(searcher, &terms)?;
+        let mut weight = PhraseWeight::new(self.phrase_terms.clone(), bm25_weight, scoring_enabled);
         if self.slop > 0 {
             weight.slop(self.slop);
         }
@@ -123,8 +122,8 @@ impl Query for PhraseQuery {
     /// Create the weight associated with a query.
     ///
     /// See [`Weight`].
-    fn weight(&self, enable_scoring: EnableScoring<'_>) -> crate::Result<Box<dyn Weight>> {
-        let phrase_weight = self.phrase_weight(enable_scoring)?;
+    fn weight(&self, searcher: &Searcher, scoring_enabled: bool) -> crate::Result<Box<dyn Weight>> {
+        let phrase_weight = self.phrase_weight(searcher, scoring_enabled)?;
         Ok(Box::new(phrase_weight))
     }
 

src/query/phrase_query/phrase_scorer.rs

@@ -50,7 +50,8 @@ pub struct PhraseScorer<TPostings: Postings> {
     right: Vec<u32>,
     phrase_count: u32,
     fieldnorm_reader: FieldNormReader,
-    similarity_weight_opt: Option<Bm25Weight>,
+    similarity_weight: Bm25Weight,
+    scoring_enabled: bool,
     slop: u32,
 }
 
@@ -244,11 +245,11 @@ fn intersection_exists_with_slop(left: &[u32], right: &[u32], slop: u32) -> bool
 }
 
 impl<TPostings: Postings> PhraseScorer<TPostings> {
-    // If similarity_weight is None, then scoring is disabled.
     pub fn new(
         term_postings: Vec<(usize, TPostings)>,
-        similarity_weight_opt: Option<Bm25Weight>,
+        similarity_weight: Bm25Weight,
         fieldnorm_reader: FieldNormReader,
+        scoring_enabled: bool,
         slop: u32,
     ) -> PhraseScorer<TPostings> {
         let max_offset = term_postings
@@ -269,8 +270,9 @@ impl<TPostings: Postings> PhraseScorer<TPostings> {
             left: Vec::with_capacity(100),
             right: Vec::with_capacity(100),
             phrase_count: 0u32,
-            similarity_weight_opt,
+            similarity_weight,
             fieldnorm_reader,
+            scoring_enabled,
             slop,
         };
         if scorer.doc() != TERMINATED && !scorer.phrase_match() {
@@ -284,7 +286,7 @@ impl<TPostings: Postings> PhraseScorer<TPostings> {
     }
 
     fn phrase_match(&mut self) -> bool {
-        if self.similarity_weight_opt.is_some() {
+        if self.scoring_enabled {
             let count = self.compute_phrase_count();
             self.phrase_count = count;
             count > 0u32
@@ -386,11 +388,8 @@ impl<TPostings: Postings> Scorer for PhraseScorer<TPostings> {
     fn score(&mut self) -> Score {
         let doc = self.doc();
         let fieldnorm_id = self.fieldnorm_reader.fieldnorm_id(doc);
-        if let Some(similarity_weight) = self.similarity_weight_opt.as_ref() {
-            similarity_weight.score(fieldnorm_id, self.phrase_count)
-        } else {
-            1.0f32
-        }
+        self.similarity_weight
+            .score(fieldnorm_id, self.phrase_count)
     }
 }
 

src/query/phrase_query/phrase_weight.rs

@@ -10,28 +10,30 @@ use crate::{DocId, DocSet, Score};
 
 pub struct PhraseWeight {
     phrase_terms: Vec<(usize, Term)>,
-    similarity_weight_opt: Option<Bm25Weight>,
+    similarity_weight: Bm25Weight,
+    scoring_enabled: bool,
     slop: u32,
 }
 
 impl PhraseWeight {
     /// Creates a new phrase weight.
-    /// If `similarity_weight_opt` is None, then scoring is disabled
     pub fn new(
         phrase_terms: Vec<(usize, Term)>,
-        similarity_weight_opt: Option<Bm25Weight>,
+        similarity_weight: Bm25Weight,
+        scoring_enabled: bool,
     ) -> PhraseWeight {
         let slop = 0;
         PhraseWeight {
             phrase_terms,
-            similarity_weight_opt,
+            similarity_weight,
+            scoring_enabled,
             slop,
         }
     }
 
     fn fieldnorm_reader(&self, reader: &SegmentReader) -> crate::Result<FieldNormReader> {
         let field = self.phrase_terms[0].1.field();
-        if self.similarity_weight_opt.is_some() {
+        if self.scoring_enabled {
             if let Some(fieldnorm_reader) = reader.fieldnorms_readers().get_field(field)? {
                 return Ok(fieldnorm_reader);
             }
@@ -44,10 +46,7 @@ impl PhraseWeight {
         reader: &SegmentReader,
         boost: Score,
     ) -> crate::Result<Option<PhraseScorer<SegmentPostings>>> {
-        let similarity_weight_opt = self
-            .similarity_weight_opt
-            .as_ref()
-            .map(|similarity_weight| similarity_weight.boost_by(boost));
+        let similarity_weight = self.similarity_weight.boost_by(boost);
         let fieldnorm_reader = self.fieldnorm_reader(reader)?;
         let mut term_postings_list = Vec::new();
         if reader.has_deletes() {
@@ -75,8 +74,9 @@ impl PhraseWeight {
         }
         Ok(Some(PhraseScorer::new(
             term_postings_list,
-            similarity_weight_opt,
+            similarity_weight,
             fieldnorm_reader,
+            self.scoring_enabled,
             self.slop,
         )))
     }
@@ -108,9 +108,7 @@ impl Weight for PhraseWeight {
         let fieldnorm_id = fieldnorm_reader.fieldnorm_id(doc);
         let phrase_count = scorer.phrase_count();
         let mut explanation = Explanation::new("Phrase Scorer", scorer.score());
-        if let Some(similarity_weight) = self.similarity_weight_opt.as_ref() {
-            explanation.add_detail(similarity_weight.explain(fieldnorm_id, phrase_count));
-        }
+        explanation.add_detail(self.similarity_weight.explain(fieldnorm_id, phrase_count));
         Ok(explanation)
     }
 }
@@ -119,7 +117,7 @@ impl Weight for PhraseWeight {
 mod tests {
     use super::super::tests::create_index;
     use crate::docset::TERMINATED;
-    use crate::query::{EnableScoring, PhraseQuery};
+    use crate::query::PhraseQuery;
     use crate::{DocSet, Term};
 
     #[test]
@@ -132,8 +130,7 @@ mod tests {
             Term::from_field_text(text_field, "a"),
             Term::from_field_text(text_field, "b"),
         ]);
-        let enable_scoring = EnableScoring::Enabled(&searcher);
-        let phrase_weight = phrase_query.phrase_weight(enable_scoring).unwrap();
+        let phrase_weight = phrase_query.phrase_weight(&searcher, true).unwrap();
         let mut phrase_scorer = phrase_weight
             .phrase_scorer(searcher.segment_reader(0u32), 1.0)?
             .unwrap();

src/query/query.rs

@@ -5,37 +5,8 @@ use downcast_rs::impl_downcast;
 use super::Weight;
 use crate::core::searcher::Searcher;
 use crate::query::Explanation;
-use crate::schema::Schema;
 use crate::{DocAddress, Term};
 
-/// Argument used in `Query::weight(..)`
-#[derive(Copy, Clone)]
-pub enum EnableScoring<'a> {
-    /// Pass this to enable scoring.
-    Enabled(&'a Searcher),
-    /// Pass this to disable scoring.
-    /// This can improve performance.
-    Disabled(&'a Schema),
-}
-
-impl<'a> EnableScoring<'a> {
-    /// Returns the schema.
-    pub fn schema(&self) -> &Schema {
-        match self {
-            EnableScoring::Enabled(searcher) => searcher.schema(),
-            EnableScoring::Disabled(schema) => schema,
-        }
-    }
-
-    /// Returns true if the scoring is enabled.
-    pub fn is_scoring_enabled(&self) -> bool {
-        match self {
-            EnableScoring::Enabled(_) => true,
-            EnableScoring::Disabled(_) => false,
-        }
-    }
-}
-
 /// The `Query` trait defines a set of documents and a scoring method
 /// for those documents.
 ///
@@ -77,18 +48,18 @@ pub trait Query: QueryClone + Send + Sync + downcast_rs::Downcast + fmt::Debug {
     /// can increase performances.
     ///
     /// See [`Weight`].
-    fn weight(&self, enable_scoring: EnableScoring<'_>) -> crate::Result<Box<dyn Weight>>;
+    fn weight(&self, searcher: &Searcher, scoring_enabled: bool) -> crate::Result<Box<dyn Weight>>;
 
     /// Returns an `Explanation` for the score of the document.
     fn explain(&self, searcher: &Searcher, doc_address: DocAddress) -> crate::Result<Explanation> {
-        let weight = self.weight(EnableScoring::Enabled(searcher))?;
         let reader = searcher.segment_reader(doc_address.segment_ord);
+        let weight = self.weight(searcher, true)?;
         weight.explain(reader, doc_address.doc_id)
     }
 
     /// Returns the number of documents matching the query.
     fn count(&self, searcher: &Searcher) -> crate::Result<usize> {
-        let weight = self.weight(EnableScoring::Disabled(searcher.schema()))?;
+        let weight = self.weight(searcher, false)?;
         let mut result = 0;
         for reader in searcher.segment_readers() {
             result += weight.count(reader)? as usize;
@@ -122,8 +93,8 @@ where T: 'static + Query + Clone
 }
 
 impl Query for Box<dyn Query> {
-    fn weight(&self, enabled_scoring: EnableScoring) -> crate::Result<Box<dyn Weight>> {
-        self.as_ref().weight(enabled_scoring)
+    fn weight(&self, searcher: &Searcher, scoring_enabled: bool) -> crate::Result<Box<dyn Weight>> {
+        self.as_ref().weight(searcher, scoring_enabled)
     }
 
     fn count(&self, searcher: &Searcher) -> crate::Result<usize> {

src/query/query_parser/logical_ast.rs

@@ -15,11 +15,6 @@ pub enum LogicalLiteral {
         lower: Bound<Term>,
         upper: Bound<Term>,
     },
-    Set {
-        field: Field,
-        value_type: Type,
-        elements: Vec<Term>,
-    },
     All,
 }
 
@@ -92,27 +87,6 @@ impl fmt::Debug for LogicalLiteral {
                 ref upper,
                 ..
             } => write!(formatter, "({:?} TO {:?})", lower, upper),
-            LogicalLiteral::Set { ref elements, .. } => {
-                const MAX_DISPLAYED: usize = 10;
-
-                write!(formatter, "IN [")?;
-                for (i, element) in elements.iter().enumerate() {
-                    if i == 0 {
-                        write!(formatter, "{:?}", element)?;
-                    } else if i == MAX_DISPLAYED - 1 {
-                        write!(
-                            formatter,
-                            ", {:?}, ... ({} more)",
-                            element,
-                            elements.len() - i - 1
-                        )?;
-                        break;
-                    } else {
-                        write!(formatter, ", {:?}", element)?;
-                    }
-                }
-                write!(formatter, "]")
-            }
             LogicalLiteral::All => write!(formatter, "*"),
         }
     }

src/query/query_parser/query_parser.rs

@@ -13,7 +13,7 @@ use crate::indexer::{
 };
 use crate::query::{
     AllQuery, BooleanQuery, BoostQuery, EmptyQuery, Occur, PhraseQuery, Query, RangeQuery,
-    TermQuery, TermSetQuery,
+    TermQuery,
 };
 use crate::schema::{
     Facet, FacetParseError, Field, FieldType, IndexRecordOption, IntoIpv6Addr, Schema, Term, Type,
@@ -333,9 +333,7 @@ impl QueryParser {
     ) -> Result<Term, QueryParserError> {
         let field_entry = self.schema.get_field_entry(field);
         let field_type = field_entry.field_type();
-
-        let is_ip_and_fast = field_type.is_ip_addr() && field_type.is_fast();
-        if !field_type.is_indexed() && !is_ip_and_fast {
+        if !field_type.is_indexed() {
             return Err(QueryParserError::FieldNotIndexed(
                 field_entry.name().to_string(),
             ));
@@ -685,31 +683,6 @@ impl QueryParser {
                 }));
                 Ok(logical_ast)
             }
-            UserInputLeaf::Set {
-                field: full_field_opt,
-                elements,
-            } => {
-                let full_path = full_field_opt.ok_or_else(|| {
-                    QueryParserError::UnsupportedQuery(
-                        "Set query need to target a specific field.".to_string(),
-                    )
-                })?;
-                let (field, json_path) = self
-                    .split_full_path(&full_path)
-                    .ok_or_else(|| QueryParserError::FieldDoesNotExist(full_path.clone()))?;
-                let field_entry = self.schema.get_field_entry(field);
-                let value_type = field_entry.field_type().value_type();
-                let logical_ast = LogicalAst::Leaf(Box::new(LogicalLiteral::Set {
-                    elements: elements
-                        .into_iter()
-                        .map(|element| self.compute_boundary_term(field, json_path, &element))
-                        .collect::<Result<Vec<_>, _>>()?,
-
-                    field,
-                    value_type,
-                }));
-                Ok(logical_ast)
-            }
         }
     }
 }
@@ -728,7 +701,6 @@ fn convert_literal_to_query(logical_literal: LogicalLiteral) -> Box<dyn Query> {
         } => Box::new(RangeQuery::new_term_bounds(
             field, value_type, &lower, &upper,
         )),
-        LogicalLiteral::Set { elements, .. } => Box::new(TermSetQuery::new(elements)),
         LogicalLiteral::All => Box::new(AllQuery),
     }
 }
@@ -1088,28 +1060,6 @@ mod test {
         );
     }
 
-    fn extract_query_term_json_path(query: &str) -> String {
-        let LogicalAst::Leaf(literal) = parse_query_to_logical_ast(query, false).unwrap() else {
-            panic!();
-        };
-        let LogicalLiteral::Term(term) = *literal else {
-            panic!();
-        };
-        std::str::from_utf8(term.value_bytes()).unwrap().to_string()
-    }
-
-    #[test]
-    fn test_json_field_query_with_espaced_dot() {
-        assert_eq!(
-            extract_query_term_json_path(r#"json.k8s.node.name:hello"#),
-            "k8s\u{1}node\u{1}name\0shello"
-        );
-        assert_eq!(
-            extract_query_term_json_path(r#"json.k8s\.node\.name:hello"#),
-            "k8s.node.name\0shello"
-        );
-    }
-
     #[test]
     fn test_json_field_possibly_a_number() {
         test_parse_query_to_logical_ast_helper(
@@ -1589,29 +1539,4 @@ mod test {
             false,
         );
     }
-
-    #[test]
-    pub fn test_term_set_query() {
-        test_parse_query_to_logical_ast_helper(
-            "title: IN [a b cd]",
-            r#"IN [Term(type=Str, field=0, "a"), Term(type=Str, field=0, "b"), Term(type=Str, field=0, "cd")]"#,
-            false,
-        );
-        test_parse_query_to_logical_ast_helper(
-            "bytes: IN [AA== ABA= ABCD]",
-            r#"IN [Term(type=Bytes, field=12, [0]), Term(type=Bytes, field=12, [0, 16]), Term(type=Bytes, field=12, [0, 16, 131])]"#,
-            false,
-        );
-        test_parse_query_to_logical_ast_helper(
-            "signed: IN [1 2 -3]",
-            r#"IN [Term(type=I64, field=2, 1), Term(type=I64, field=2, 2), Term(type=I64, field=2, -3)]"#,
-            false,
-        );
-
-        test_parse_query_to_logical_ast_helper(
-            "float: IN [1.1 2.2 -3.3]",
-            r#"IN [Term(type=F64, field=10, 1.1), Term(type=F64, field=10, 2.2), Term(type=F64, field=10, -3.3)]"#,
-            false,
-        );
-    }
 }

src/query/range_query.rs

@@ -3,11 +3,11 @@ use std::ops::{Bound, Range};
 
 use common::BitSet;
 
-use crate::core::SegmentReader;
+use crate::core::{Searcher, SegmentReader};
 use crate::error::TantivyError;
 use crate::query::explanation::does_not_match;
 use crate::query::range_query_ip_fastfield::IPFastFieldRangeWeight;
-use crate::query::{BitSetDocSet, ConstScorer, EnableScoring, Explanation, Query, Scorer, Weight};
+use crate::query::{BitSetDocSet, ConstScorer, Explanation, Query, Scorer, Weight};
 use crate::schema::{Field, IndexRecordOption, Term, Type};
 use crate::termdict::{TermDictionary, TermStreamer};
 use crate::{DocId, Score};
@@ -253,8 +253,12 @@ impl RangeQuery {
 }
 
 impl Query for RangeQuery {
-    fn weight(&self, enable_scoring: EnableScoring<'_>) -> crate::Result<Box<dyn Weight>> {
-        let schema = enable_scoring.schema();
+    fn weight(
+        &self,
+        searcher: &Searcher,
+        _scoring_enabled: bool,
+    ) -> crate::Result<Box<dyn Weight>> {
+        let schema = searcher.schema();
         let field_type = schema.get_field_entry(self.field).field_type();
         let value_type = field_type.value_type();
         if value_type != self.value_type {

src/query/range_query_ip_fastfield.rs

@@ -7,7 +7,7 @@ use std::ops::{Bound, RangeInclusive};
 use std::sync::Arc;
 
 use common::BinarySerializable;
-use fastfield_codecs::{MonotonicallyMappableToU128, OptionalColumn};
+use fastfield_codecs::{Column, MonotonicallyMappableToU128};
 
 use super::range_query::map_bound;
 use super::{ConstScorer, Explanation, Scorer, Weight};
@@ -44,31 +44,27 @@ impl Weight for IPFastFieldRangeWeight {
         let field_type = reader.schema().get_field_entry(self.field).field_type();
         match field_type.fastfield_cardinality().unwrap() {
             Cardinality::SingleValue => {
-                let ip_addr_fast_field = reader.fast_fields().ip_addr(self.field)?;
-                let minmax = ip_addr_fast_field
-                    .min_value()
-                    .zip(ip_addr_fast_field.max_value());
-                let value_range = bound_to_value_range(&self.left_bound, &self.right_bound, minmax);
-                let docset = IpRangeDocSet::new(
-                    value_range,
-                    IpFastFieldCardinality::SingleValue(ip_addr_fast_field),
-                );
-                Ok(Box::new(ConstScorer::new(docset, boost)))
-            }
-            Cardinality::MultiValues => {
-                let ip_addr_fast_field = reader.fast_fields().ip_addrs(self.field)?;
+                let ip_addr_fast_field: Arc<dyn Column<Ipv6Addr>> =
+                    reader.fast_fields().ip_addr(self.field)?;
                 let value_range = bound_to_value_range(
                     &self.left_bound,
                     &self.right_bound,
-                    Some((
-                        ip_addr_fast_field.min_value(),
-                        ip_addr_fast_field.max_value(),
-                    )),
+                    ip_addr_fast_field.min_value(),
+                    ip_addr_fast_field.max_value(),
                 );
-                let docset = IpRangeDocSet::new(
-                    value_range,
-                    IpFastFieldCardinality::MultiValue(ip_addr_fast_field),
+                let docset = IpRangeDocSet::new(value_range, ip_addr_fast_field);
+                Ok(Box::new(ConstScorer::new(docset, boost)))
+            }
+            Cardinality::MultiValues => {
+                let ip_addr_fast_field: MultiValuedU128FastFieldReader<Ipv6Addr> =
+                    reader.fast_fields().ip_addrs(self.field)?;
+                let value_range = bound_to_value_range(
+                    &self.left_bound,
+                    &self.right_bound,
+                    ip_addr_fast_field.min_value(),
+                    ip_addr_fast_field.max_value(),
                 );
+                let docset = IpRangeDocSet::new(value_range, Arc::new(ip_addr_fast_field));
                 Ok(Box::new(ConstScorer::new(docset, boost)))
             }
         }
@@ -91,10 +87,9 @@ impl Weight for IPFastFieldRangeWeight {
 fn bound_to_value_range(
     left_bound: &Bound<Ipv6Addr>,
     right_bound: &Bound<Ipv6Addr>,
-    min_max: Option<(Ipv6Addr, Ipv6Addr)>,
+    min_value: Ipv6Addr,
+    max_value: Ipv6Addr,
 ) -> RangeInclusive<Ipv6Addr> {
-    let (min_value, max_value) =
-        min_max.unwrap_or((Ipv6Addr::from(u128::MIN), Ipv6Addr::from(u128::MAX)));
     let start_value = match left_bound {
         Bound::Included(ip_addr) => *ip_addr,
         Bound::Excluded(ip_addr) => Ipv6Addr::from(ip_addr.to_u128() + 1),
@@ -142,26 +137,10 @@ impl VecCursor {
     }
 }
 
-pub(crate) enum IpFastFieldCardinality {
-    SingleValue(Arc<dyn OptionalColumn<Ipv6Addr>>),
-    MultiValue(MultiValuedU128FastFieldReader<Ipv6Addr>),
-}
-
-impl IpFastFieldCardinality {
-    fn num_docs(&self) -> u32 {
-        match self {
-            IpFastFieldCardinality::SingleValue(single_value) => single_value.num_vals(),
-            IpFastFieldCardinality::MultiValue(multi_value) => {
-                multi_value.get_index_reader().num_docs()
-            }
-        }
-    }
-}
-
-struct IpRangeDocSet {
+struct IpRangeDocSet<T> {
     /// The range filter on the values.
     value_range: RangeInclusive<Ipv6Addr>,
-    ip_addr_fast_field: IpFastFieldCardinality,
+    ip_addrs: T,
     /// The next docid start range to fetch (inclusive).
     next_fetch_start: u32,
     /// Number of docs range checked in a batch.
@@ -179,14 +158,13 @@ struct IpRangeDocSet {
 }
 
 const DEFAULT_FETCH_HORIZON: u32 = 128;
-impl IpRangeDocSet {
-    fn new(
-        value_range: RangeInclusive<Ipv6Addr>,
-        ip_addr_fast_field: IpFastFieldCardinality,
-    ) -> Self {
+impl<T> IpRangeDocSet<T>
+where Self: SingleOrMultivalued
+{
+    fn new(value_range: RangeInclusive<Ipv6Addr>, ip_addrs: T) -> Self {
         let mut ip_range_docset = Self {
             value_range,
-            ip_addr_fast_field,
+            ip_addrs,
             loaded_docs: VecCursor::new(),
             next_fetch_start: 0,
             fetch_horizon: DEFAULT_FETCH_HORIZON,
@@ -222,56 +200,72 @@ impl IpRangeDocSet {
             true
         }
     }
+}
 
-    /// Fetches a block for docid range [next_fetch_start .. next_fetch_start + HORIZON]
+trait SingleOrMultivalued {
+    fn num_docs(&self) -> u32;
     fn fetch_horizon(&mut self, horizon: u32) -> bool {
-        let mut finished_to_end = false;
+        // Have different implem for single value and multivalue
+        todo!();
+        // let mut finished_to_end = false;
 
-        let limit = self.ip_addr_fast_field.num_docs();
-        let mut end = self.next_fetch_start + horizon;
-        if end >= limit {
-            end = limit;
-            finished_to_end = true;
-        }
+        // let limit = self.num_docs();
+        // let mut end = self.next_fetch_start + horizon;
+        // if end >= limit {
+        //     end = limit;
+        //     finished_to_end = true;
+        // }
 
-        match &self.ip_addr_fast_field {
-            IpFastFieldCardinality::MultiValue(multi) => {
-                let last_value = self.loaded_docs.last_value();
+        // let last_loaded_docs_val = self
+        //     .is_multivalue
+        //     .then(|| self.loaded_docs.last_value())
+        //     .flatten();
 
-                multi.get_docids_for_value_range(
-                    self.value_range.clone(),
-                    self.next_fetch_start..end,
-                    self.loaded_docs.get_cleared_data(),
-                );
-                // In case of multivalues, we may have an overlap of the same docid between fetching
-                // blocks
-                if let Some(last_value) = last_value {
-                    while self.loaded_docs.current() == Some(last_value) {
-                        self.loaded_docs.next();
-                    }
-                }
-            }
-            IpFastFieldCardinality::SingleValue(single) => {
-                single.get_docids_for_value_range(
-                    self.value_range.clone(),
-                    self.next_fetch_start..end,
-                    self.loaded_docs.get_cleared_data(),
-                );
-            }
-        }
-        self.next_fetch_start = end;
+        // let last_loaded_docs_val =
+        //     if self.is_multivalue {
+        //         self.loaded_docs.last_value()
+        //     } else {
+        //         None
+        //     };
 
-        finished_to_end
+        // let loaded_docs_data = self.loaded_docs.get_cleared_data();
+        // self.ip_addr_fast_field.get_docids_for_value_range(
+        //     self.value_range.clone(),
+        //     self.next_fetch_start..end,
+        //     loaded_docs_data,
+        // );
+        // // In case of multivalues, we may have an overlap of the same docid between fetching
+        // blocks if let Some(last_value) = last_loaded_docs_val {
+        //     while self.loaded_docs.current() == Some(last_value) {
+        //         self.loaded_docs.next();
+        //     }
+        // }
+        // self.next_fetch_start = end;
+        // finished_to_end
     }
 }
 
-impl DocSet for IpRangeDocSet {
+impl SingleOrMultivalued for IpRangeDocSet<Arc<dyn Column<Ipv6Addr>>> {
+    fn num_docs(&self) -> u32 {
+        self.ip_addrs.num_docs()
+    }
+}
+
+impl SingleOrMultivalued for IpRangeDocSet<Arc<MultiValuedU128FastFieldReader<Ipv6Addr>>> {
+    fn num_docs(&self) -> u32 {
+        self.ip_addrs.get_index_reader().num_docs()
+    }
+}
+
+impl<T: Send> DocSet for IpRangeDocSet<T>
+where Self: SingleOrMultivalued
+{
     #[inline]
     fn advance(&mut self) -> DocId {
         if let Some(docid) = self.loaded_docs.next() {
             docid as u32
         } else {
-            if self.next_fetch_start >= self.ip_addr_fast_field.num_docs() as u32 {
+            if self.next_fetch_start >= self.num_docs() as u32 {
                 return TERMINATED;
             }
             self.fetch_block();
@@ -326,7 +320,7 @@ mod tests {
     use super::*;
     use crate::collector::Count;
     use crate::query::QueryParser;
-    use crate::schema::{IpAddrOptions, Schema, FAST, STORED, STRING};
+    use crate::schema::{IpAddrOptions, Schema, FAST, INDEXED, STORED, STRING};
     use crate::Index;
 
     #[derive(Clone, Debug)]
@@ -393,7 +387,7 @@ mod tests {
 
     pub fn create_index_from_docs(docs: &[Doc]) -> Index {
         let mut schema_builder = Schema::builder();
-        let ip_field = schema_builder.add_ip_addr_field("ip", STORED | FAST);
+        let ip_field = schema_builder.add_ip_addr_field("ip", INDEXED | STORED | FAST);
         let ips_field = schema_builder.add_ip_addr_field(
             "ips",
             IpAddrOptions::default()

src/query/regex_query.rs

@@ -4,8 +4,9 @@ use std::sync::Arc;
 use tantivy_fst::Regex;
 
 use crate::error::TantivyError;
-use crate::query::{AutomatonWeight, EnableScoring, Query, Weight};
+use crate::query::{AutomatonWeight, Query, Weight};
 use crate::schema::Field;
+use crate::Searcher;
 
 /// A Regex Query matches all of the documents
 /// containing a specific term that matches
@@ -81,7 +82,11 @@ impl RegexQuery {
 }
 
 impl Query for RegexQuery {
-    fn weight(&self, _enabled_scoring: EnableScoring<'_>) -> crate::Result<Box<dyn Weight>> {
+    fn weight(
+        &self,
+        _searcher: &Searcher,
+        _scoring_enabled: bool,
+    ) -> crate::Result<Box<dyn Weight>> {
         Ok(Box::new(self.specialized_weight()))
     }
 }

src/query/set_query.rs

@@ -4,9 +4,9 @@ use tantivy_fst::raw::CompiledAddr;
 use tantivy_fst::{Automaton, Map};
 
 use crate::query::score_combiner::DoNothingCombiner;
-use crate::query::{AutomatonWeight, BooleanWeight, EnableScoring, Occur, Query, Weight};
-use crate::schema::{Field, Schema};
-use crate::Term;
+use crate::query::{AutomatonWeight, BooleanWeight, Occur, Query, Weight};
+use crate::schema::Field;
+use crate::{Searcher, Term};
 
 /// A Term Set Query matches all of the documents containing any of the Term provided
 #[derive(Debug, Clone)]
@@ -32,12 +32,12 @@ impl TermSetQuery {
 
     fn specialized_weight(
         &self,
-        schema: &Schema,
+        searcher: &Searcher,
     ) -> crate::Result<BooleanWeight<DoNothingCombiner>> {
         let mut sub_queries: Vec<(_, Box<dyn Weight>)> = Vec::with_capacity(self.terms_map.len());
 
         for (&field, sorted_terms) in self.terms_map.iter() {
-            let field_entry = schema.get_field_entry(field);
+            let field_entry = searcher.schema().get_field_entry(field);
             let field_type = field_entry.field_type();
             if !field_type.is_indexed() {
                 let error_msg = format!("Field {:?} is not indexed.", field_entry.name());
@@ -65,8 +65,12 @@ impl TermSetQuery {
 }
 
 impl Query for TermSetQuery {
-    fn weight(&self, enable_scoring: EnableScoring<'_>) -> crate::Result<Box<dyn Weight>> {
-        Ok(Box::new(self.specialized_weight(enable_scoring.schema())?))
+    fn weight(
+        &self,
+        searcher: &Searcher,
+        _scoring_enabled: bool,
+    ) -> crate::Result<Box<dyn Weight>> {
+        Ok(Box::new(self.specialized_weight(searcher)?))
     }
 }
 
@@ -101,8 +105,9 @@ impl Automaton for SetDfaWrapper {
 
 #[cfg(test)]
 mod tests {
+
     use crate::collector::TopDocs;
-    use crate::query::{QueryParser, TermSetQuery};
+    use crate::query::TermSetQuery;
     use crate::schema::{Schema, TEXT};
     use crate::{assert_nearly_equals, Index, Term};
 
@@ -214,31 +219,4 @@ mod tests {
 
         Ok(())
     }
-
-    #[test]
-    fn test_term_set_query_parser() -> crate::Result<()> {
-        let mut schema_builder = Schema::builder();
-        schema_builder.add_text_field("field", TEXT);
-        let schema = schema_builder.build();
-        let index = Index::create_in_ram(schema.clone());
-        let mut index_writer = index.writer_for_tests()?;
-        let field = schema.get_field("field").unwrap();
-        index_writer.add_document(doc!(
-          field => "val1",
-        ))?;
-        index_writer.add_document(doc!(
-          field => "val2",
-        ))?;
-        index_writer.add_document(doc!(
-          field => "val3",
-        ))?;
-        index_writer.commit()?;
-        let reader = index.reader()?;
-        let searcher = reader.searcher();
-        let query_parser = QueryParser::for_index(&index, vec![]);
-        let query = query_parser.parse_query("field: IN [val1 val2]")?;
-        let top_docs = searcher.search(&query, &TopDocs::with_limit(3))?;
-        assert_eq!(top_docs.len(), 2);
-        Ok(())
-    }
 }

src/query/term_query/mod.rs

@@ -12,7 +12,7 @@ mod tests {
     use crate::collector::TopDocs;
     use crate::docset::DocSet;
     use crate::postings::compression::COMPRESSION_BLOCK_SIZE;
-    use crate::query::{EnableScoring, Query, QueryParser, Scorer, TermQuery};
+    use crate::query::{Query, QueryParser, Scorer, TermQuery};
     use crate::schema::{Field, IndexRecordOption, Schema, STRING, TEXT};
     use crate::{assert_nearly_equals, DocAddress, Index, Term, TERMINATED};
 
@@ -34,7 +34,7 @@ mod tests {
             Term::from_field_text(text_field, "a"),
             IndexRecordOption::Basic,
         );
-        let term_weight = term_query.weight(EnableScoring::Enabled(&searcher))?;
+        let term_weight = term_query.weight(&searcher, true)?;
         let segment_reader = searcher.segment_reader(0);
         let mut term_scorer = term_weight.scorer(segment_reader, 1.0)?;
         assert_eq!(term_scorer.doc(), 0);
@@ -62,7 +62,7 @@ mod tests {
             Term::from_field_text(text_field, "a"),
             IndexRecordOption::Basic,
         );
-        let term_weight = term_query.weight(EnableScoring::Enabled(&searcher))?;
+        let term_weight = term_query.weight(&searcher, true)?;
         let segment_reader = searcher.segment_reader(0);
         let mut term_scorer = term_weight.scorer(segment_reader, 1.0)?;
         for i in 0u32..COMPRESSION_BLOCK_SIZE as u32 {
@@ -158,7 +158,7 @@ mod tests {
         let term_a = Term::from_field_text(text_field, "a");
         let term_query = TermQuery::new(term_a, IndexRecordOption::Basic);
         let searcher = index.reader()?.searcher();
-        let term_weight = term_query.weight(EnableScoring::Disabled(searcher.schema()))?;
+        let term_weight = term_query.weight(&searcher, false)?;
         let mut term_scorer = term_weight.scorer(searcher.segment_reader(0u32), 1.0)?;
         assert_eq!(term_scorer.doc(), 0u32);
         term_scorer.seek(1u32);

src/query/term_query/term_query.rs

@@ -2,9 +2,9 @@ use std::fmt;
 
 use super::term_weight::TermWeight;
 use crate::query::bm25::Bm25Weight;
-use crate::query::{EnableScoring, Explanation, Query, Weight};
+use crate::query::{Explanation, Query, Weight};
 use crate::schema::IndexRecordOption;
-use crate::Term;
+use crate::{Searcher, Term};
 
 /// A Term query matches all of the documents
 /// containing a specific term.
@@ -87,23 +87,19 @@ impl TermQuery {
     /// This is useful for optimization purpose.
     pub fn specialized_weight(
         &self,
-        enable_scoring: EnableScoring<'_>,
+        searcher: &Searcher,
+        scoring_enabled: bool,
     ) -> crate::Result<TermWeight> {
-        let schema = enable_scoring.schema();
-        let field_entry = schema.get_field_entry(self.term.field());
+        let field_entry = searcher.schema().get_field_entry(self.term.field());
         if !field_entry.is_indexed() {
             let error_msg = format!("Field {:?} is not indexed.", field_entry.name());
             return Err(crate::TantivyError::SchemaError(error_msg));
         }
-        let bm25_weight = match enable_scoring {
-            EnableScoring::Enabled(searcher) => {
-                Bm25Weight::for_terms(searcher, &[self.term.clone()])?
-            }
-            EnableScoring::Disabled(_schema) => {
-                Bm25Weight::new(Explanation::new("<no score>".to_string(), 1.0f32), 1.0f32)
-            }
+        let bm25_weight = if scoring_enabled {
+            Bm25Weight::for_terms(searcher, &[self.term.clone()])?
+        } else {
+            Bm25Weight::new(Explanation::new("<no score>".to_string(), 1.0f32), 1.0f32)
         };
-        let scoring_enabled = enable_scoring.is_scoring_enabled();
         let index_record_option = if scoring_enabled {
             self.index_record_option
         } else {
@@ -119,8 +115,10 @@ impl TermQuery {
 }
 
 impl Query for TermQuery {
-    fn weight(&self, enable_scoring: EnableScoring<'_>) -> crate::Result<Box<dyn Weight>> {
-        Ok(Box::new(self.specialized_weight(enable_scoring)?))
+    fn weight(&self, searcher: &Searcher, scoring_enabled: bool) -> crate::Result<Box<dyn Weight>> {
+        Ok(Box::new(
+            self.specialized_weight(searcher, scoring_enabled)?,
+        ))
     }
     fn query_terms<'a>(&'a self, visitor: &mut dyn FnMut(&'a Term, bool)) {
         visitor(&self.term, false);

src/query/term_query/term_scorer.rs

@@ -130,7 +130,7 @@ mod tests {
     use crate::merge_policy::NoMergePolicy;
     use crate::postings::compression::COMPRESSION_BLOCK_SIZE;
     use crate::query::term_query::TermScorer;
-    use crate::query::{Bm25Weight, EnableScoring, Scorer, TermQuery};
+    use crate::query::{Bm25Weight, Scorer, TermQuery};
     use crate::schema::{IndexRecordOption, Schema, TEXT};
     use crate::{
         assert_nearly_equals, DocId, DocSet, Index, Score, Searcher, SegmentId, Term, TERMINATED,
@@ -250,7 +250,7 @@ mod tests {
     }
 
     fn test_block_wand_aux(term_query: &TermQuery, searcher: &Searcher) -> crate::Result<()> {
-        let term_weight = term_query.specialized_weight(EnableScoring::Enabled(searcher))?;
+        let term_weight = term_query.specialized_weight(searcher, true)?;
         for reader in searcher.segment_readers() {
             let mut block_max_scores = vec![];
             let mut block_max_scores_b = vec![];

src/tokenizer/stop_word_filter.rs

@@ -10,15 +10,12 @@
 //! assert_eq!(stream.next().unwrap().text, "crafty");
 //! assert!(stream.next().is_none());
 //! ```
-#[cfg(feature = "stopwords")]
-#[rustfmt::skip]
-mod stopwords;
-
 use std::sync::Arc;
 
 use rustc_hash::FxHashSet;
 
-use super::{BoxTokenStream, Language, Token, TokenFilter, TokenStream};
+use super::{Token, TokenFilter, TokenStream};
+use crate::tokenizer::BoxTokenStream;
 
 /// `TokenFilter` that removes stop words from a token stream
 #[derive(Clone)]
@@ -27,44 +24,22 @@ pub struct StopWordFilter {
 }
 
 impl StopWordFilter {
-    /// Creates a new [`StopWordFilter`] for the given [`Language`]
-    ///
-    /// Returns `Some` if a list of stop words is available and `None` otherwise.
-    #[cfg(feature = "stopwords")]
-    pub fn new(language: Language) -> Option<Self> {
-        let words = match language {
-            Language::Danish => stopwords::DANISH,
-            Language::Dutch => stopwords::DUTCH,
-            Language::English => {
-                // This is the same list of words used by the Apache-licensed Lucene project,
-                // c.f. https://github.com/apache/lucene/blob/d5d6dc079395c47cd6d12dcce3bcfdd2c7d9dc63/lucene/analysis/common/src/java/org/apache/lucene/analysis/en/EnglishAnalyzer.java#L46
-                &[
-                    "a", "an", "and", "are", "as", "at", "be", "but", "by", "for", "if", "in",
-                    "into", "is", "it", "no", "not", "of", "on", "or", "such", "that", "the",
-                    "their", "then", "there", "these", "they", "this", "to", "was", "will", "with",
-                ]
-            }
-            Language::Finnish => stopwords::FINNISH,
-            Language::French => stopwords::FRENCH,
-            Language::German => stopwords::GERMAN,
-            Language::Italian => stopwords::ITALIAN,
-            Language::Norwegian => stopwords::NORWEGIAN,
-            Language::Portuguese => stopwords::PORTUGUESE,
-            Language::Russian => stopwords::RUSSIAN,
-            Language::Spanish => stopwords::SPANISH,
-            Language::Swedish => stopwords::SWEDISH,
-            _ => return None,
-        };
-
-        Some(Self::remove(words.iter().map(|&word| word.to_owned())))
-    }
-
     /// Creates a `StopWordFilter` given a list of words to remove
     pub fn remove<W: IntoIterator<Item = String>>(words: W) -> StopWordFilter {
         StopWordFilter {
             words: Arc::new(words.into_iter().collect()),
         }
     }
+
+    fn english() -> StopWordFilter {
+        let words: [&'static str; 33] = [
+            "a", "an", "and", "are", "as", "at", "be", "but", "by", "for", "if", "in", "into",
+            "is", "it", "no", "not", "of", "on", "or", "such", "that", "the", "their", "then",
+            "there", "these", "they", "this", "to", "was", "will", "with",
+        ];
+
+        StopWordFilter::remove(words.iter().map(|&s| s.to_string()))
+    }
 }
 
 pub struct StopWordFilterStream<'a> {
@@ -106,6 +81,12 @@ impl<'a> TokenStream for StopWordFilterStream<'a> {
     }
 }
 
+impl Default for StopWordFilter {
+    fn default() -> StopWordFilter {
+        StopWordFilter::english()
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use crate::tokenizer::tests::assert_token;

src/tokenizer/stop_word_filter/gen_stopwords.py

@@ -1,42 +0,0 @@
-import requests
-
-LANGUAGES = [
-    "danish",
-    "dutch",
-    "finnish",
-    "french",
-    "german",
-    "italian",
-    "norwegian",
-    "portuguese",
-    "russian",
-    "spanish",
-    "swedish",
-]
-
-with requests.Session() as sess, open("stopwords.rs", "w") as mod:
-    mod.write("/*\n")
-    mod.write(
-        "These stop word lists are from the Snowball project (https://snowballstem.org/)\nwhich carries the following copyright and license:\n\n"
-    )
-
-    resp = sess.get(
-        "https://raw.githubusercontent.com/snowballstem/snowball/master/COPYING"
-    )
-    resp.raise_for_status()
-    mod.write(resp.text)
-    mod.write("*/\n\n")
-
-    for lang in LANGUAGES:
-        resp = sess.get(f"https://snowballstem.org/algorithms/{lang}/stop.txt")
-        resp.raise_for_status()
-
-        mod.write(f"pub const {lang.upper()}: &[&str] = &[\n")
-
-        for line in resp.text.splitlines():
-            line, _, _ = line.partition("|")
-
-            for word in line.split():
-                mod.write(f'    "{word}",\n')
-
-        mod.write("];\n\n")