remove superfluous impls

Co-authored-by: Paul Masurel <paul@quickwit.io>

examples/custom_collector.rs

@@ -9,7 +9,7 @@
 
 use std::sync::Arc;
 
-use fastfield_codecs::Column;
+use fastfield_codecs::OptionalColumn;
 // ---
 // Importing tantivy...
 use tantivy::collector::{Collector, SegmentCollector};
@@ -97,7 +97,7 @@ impl Collector for StatsCollector {
 }
 
 struct StatsSegmentCollector {
-    fast_field_reader: Arc<dyn Column<u64>>,
+    fast_field_reader: Arc<dyn OptionalColumn<u64>>,
     stats: Stats,
 }
 
@@ -105,10 +105,12 @@ impl SegmentCollector for StatsSegmentCollector {
     type Fruit = Option<Stats>;
 
     fn collect(&mut self, doc: u32, _score: Score) {
-        let value = self.fast_field_reader.get_val(doc) as f64;
-        self.stats.count += 1;
-        self.stats.sum += value;
-        self.stats.squared_sum += value * value;
+        if let Some(value) = self.fast_field_reader.get_val(doc) {
+            let value = value as f64;
+            self.stats.count += 1;
+            self.stats.sum += value;
+            self.stats.squared_sum += value * value;
+        }
     }
 
     fn harvest(self) -> <Self as SegmentCollector>::Fruit {

examples/warmer.rs

@@ -51,7 +51,7 @@ impl Warmer for DynamicPriceColumn {
             let product_id_reader = segment.fast_fields().u64(self.field)?;
             let product_ids: Vec<ProductId> = segment
                 .doc_ids_alive()
-                .map(|doc| product_id_reader.get_val(doc))
+                .flat_map(|doc| product_id_reader.get_val(doc))
                 .collect();
             let mut prices_it = self.price_fetcher.fetch_prices(&product_ids).into_iter();
             let mut price_vals: Vec<Price> = Vec::new();

fastfield_codecs/benches/bench.rs

@@ -41,7 +41,7 @@ mod tests {
     ) -> Arc<dyn Column<T>> {
         let mut buffer = Vec::new();
         serialize(VecColumn::from(&column), &mut buffer, &ALL_CODEC_TYPES).unwrap();
-        open(OwnedBytes::new(buffer)).unwrap()
+        open(OwnedBytes::new(buffer)).unwrap().to_full().unwrap()
     }
 
     #[bench]
@@ -103,7 +103,7 @@ mod tests {
         let iter_gen = || data.iter().cloned();
         serialize_u128(iter_gen, data.len() as u32, &mut out).unwrap();
         let out = OwnedBytes::new(out);
-        open_u128::<u128>(out).unwrap()
+        open_u128::<u128>(out).unwrap().to_full().unwrap()
     }
 
     #[bench]

fastfield_codecs/src/compact_space/mod.rs

@@ -731,7 +731,10 @@ mod tests {
         ];
         let mut out = Vec::new();
         serialize_u128(|| vals.iter().cloned(), vals.len() as u32, &mut out).unwrap();
-        let decomp = open_u128::<u128>(OwnedBytes::new(out)).unwrap();
+        let decomp = open_u128::<u128>(OwnedBytes::new(out))
+            .unwrap()
+            .to_full()
+            .unwrap();
         let complete_range = 0..vals.len() as u32;
 
         assert_eq!(

fastfield_codecs/src/gcd.rs

@@ -59,11 +59,11 @@ mod tests {
         crate::serialize(VecColumn::from(&vals), &mut buffer, &[codec_type])?;
         let buffer = OwnedBytes::new(buffer);
         let column = crate::open::<i64>(buffer.clone())?;
-        assert_eq!(column.get_val(0), -4000i64);
-        assert_eq!(column.get_val(1), -3000i64);
-        assert_eq!(column.get_val(2), -2000i64);
-        assert_eq!(column.max_value(), (num_vals as i64 - 5) * 1000);
-        assert_eq!(column.min_value(), -4000i64);
+        assert_eq!(column.get_val(0), Some(-4000i64));
+        assert_eq!(column.get_val(1), Some(-3000i64));
+        assert_eq!(column.get_val(2), Some(-2000i64));
+        assert_eq!(column.max_value(), Some((num_vals as i64 - 5) * 1000));
+        assert_eq!(column.min_value(), Some(-4000i64));
 
         // Can't apply gcd
         let mut buffer_without_gcd = Vec::new();
@@ -101,11 +101,11 @@ mod tests {
         crate::serialize(VecColumn::from(&vals), &mut buffer, &[codec_type])?;
         let buffer = OwnedBytes::new(buffer);
         let column = crate::open::<u64>(buffer.clone())?;
-        assert_eq!(column.get_val(0), 1000u64);
-        assert_eq!(column.get_val(1), 2000u64);
-        assert_eq!(column.get_val(2), 3000u64);
-        assert_eq!(column.max_value(), num_vals as u64 * 1000);
-        assert_eq!(column.min_value(), 1000u64);
+        assert_eq!(column.get_val(0), Some(1000u64));
+        assert_eq!(column.get_val(1), Some(2000u64));
+        assert_eq!(column.get_val(2), Some(3000u64));
+        assert_eq!(column.max_value(), Some(num_vals as u64 * 1000));
+        assert_eq!(column.min_value(), Some(1000u64));
 
         // Can't apply gcd
         let mut buffer_without_gcd = Vec::new();

fastfield_codecs/src/lib.rs

@@ -24,6 +24,8 @@ use monotonic_mapping::{
     StrictlyMonotonicMappingInverter, StrictlyMonotonicMappingToInternal,
     StrictlyMonotonicMappingToInternalBaseval, StrictlyMonotonicMappingToInternalGCDBaseval,
 };
+pub use optional_column::OptionalColumn;
+use optional_column::ToOptionalColumn;
 use ownedbytes::OwnedBytes;
 use serialize::{Header, U128Header};
 
@@ -34,6 +36,7 @@ mod line;
 mod linear;
 mod monotonic_mapping;
 mod monotonic_mapping_u128;
+mod optional_column;
 
 mod column;
 mod gcd;
@@ -130,19 +133,22 @@ impl U128FastFieldCodecType {
 /// Returns the correct codec reader wrapped in the `Arc` for the data.
 pub fn open_u128<Item: MonotonicallyMappableToU128>(
     mut bytes: OwnedBytes,
-) -> io::Result<Arc<dyn Column<Item>>> {
+) -> io::Result<Arc<dyn OptionalColumn<Item>>> {
     let header = U128Header::deserialize(&mut bytes)?;
     assert_eq!(header.codec_type, U128FastFieldCodecType::CompactSpace);
     let reader = CompactSpaceDecompressor::open(bytes)?;
     let inverted: StrictlyMonotonicMappingInverter<StrictlyMonotonicMappingToInternal<Item>> =
         StrictlyMonotonicMappingToInternal::<Item>::new().into();
-    Ok(Arc::new(monotonic_map_column(reader, inverted)))
+
+    Ok(Arc::new(ToOptionalColumn::new(Arc::new(
+        monotonic_map_column(reader, inverted),
+    ))))
 }
 
 /// Returns the correct codec reader wrapped in the `Arc` for the data.
 pub fn open<T: MonotonicallyMappableToU64>(
     mut bytes: OwnedBytes,
-) -> io::Result<Arc<dyn Column<T>>> {
+) -> io::Result<Arc<dyn OptionalColumn<T>>> {
     let header = Header::deserialize(&mut bytes)?;
     match header.codec_type {
         FastFieldCodecType::Bitpacked => open_specific_codec::<BitpackedCodec, _>(bytes, &header),
@@ -156,7 +162,7 @@ pub fn open<T: MonotonicallyMappableToU64>(
 fn open_specific_codec<C: FastFieldCodec, Item: MonotonicallyMappableToU64>(
     bytes: OwnedBytes,
     header: &Header,
-) -> io::Result<Arc<dyn Column<Item>>> {
+) -> io::Result<Arc<dyn OptionalColumn<Item>>> {
     let normalized_header = header.normalized();
     let reader = C::open_from_bytes(bytes, normalized_header)?;
     let min_value = header.min_value;
@@ -164,12 +170,16 @@ fn open_specific_codec<C: FastFieldCodec, Item: MonotonicallyMappableToU64>(
         let mapping = StrictlyMonotonicMappingInverter::from(
             StrictlyMonotonicMappingToInternalGCDBaseval::new(gcd.get(), min_value),
         );
-        Ok(Arc::new(monotonic_map_column(reader, mapping)))
+        Ok(Arc::new(ToOptionalColumn::new(Arc::new(
+            monotonic_map_column(reader, mapping),
+        ))))
     } else {
         let mapping = StrictlyMonotonicMappingInverter::from(
             StrictlyMonotonicMappingToInternalBaseval::new(min_value),
         );
-        Ok(Arc::new(monotonic_map_column(reader, mapping)))
+        Ok(Arc::new(ToOptionalColumn::new(Arc::new(
+            monotonic_map_column(reader, mapping),
+        ))))
     }
 }
 
@@ -240,8 +250,9 @@ mod tests {
         for (doc, orig_val) in data.iter().copied().enumerate() {
             let val = reader.get_val(doc as u32);
             assert_eq!(
-                val, orig_val,
-                "val `{val}` does not match orig_val {orig_val:?}, in data set {name}, data \
+                val,
+                Some(orig_val),
+                "val `{val:?}` does not match orig_val {orig_val:?}, in data set {name}, data \
                  `{data:?}`",
             );
         }

fastfield_codecs/src/main.rs

@@ -113,7 +113,10 @@ fn bench_ip() {
         (data.len() * 8) as f32 / dataset.len() as f32
     );
 
-    let decompressor = open_u128::<u128>(OwnedBytes::new(data)).unwrap();
+    let decompressor = open_u128::<u128>(OwnedBytes::new(data))
+        .unwrap()
+        .to_full()
+        .unwrap();
     // Sample some ranges
     let mut doc_values = Vec::new();
     for value in dataset.iter().take(1110).skip(1100).cloned() {

fastfield_codecs/src/optional_column.rs

@@ -0,0 +1,119 @@
+use std::ops::{Range, RangeInclusive};
+use std::sync::Arc;
+
+use crate::Column;
+
+/// `OptionalColumn` provides columnar access on a field.
+pub trait OptionalColumn<T: PartialOrd = u64>: Send + Sync {
+    /// Return the value associated with the given idx.
+    ///
+    /// This accessor should return as fast as possible.
+    ///
+    /// # Panics
+    ///
+    /// May panic if `idx` is greater than the column length.
+    fn get_val(&self, idx: u32) -> Option<T>;
+
+    /// Fills an output buffer with the fast field values
+    /// associated with the `DocId` going from
+    /// `start` to `start + output.len()`.
+    ///
+    /// # Panics
+    ///
+    /// Must panic if `start + output.len()` is greater than
+    /// the segment's `maxdoc`.
+    fn get_range(&self, start: u64, output: &mut [Option<T>]) {
+        for (out, idx) in output.iter_mut().zip(start..) {
+            *out = self.get_val(idx as u32);
+        }
+    }
+
+    /// Return the positions of values which are in the provided range.
+    fn get_docids_for_value_range(
+        &self,
+        value_range: RangeInclusive<T>,
+        doc_id_range: Range<u32>,
+        positions: &mut Vec<u32>,
+    ) {
+        let doc_id_range = doc_id_range.start..doc_id_range.end.min(self.num_vals());
+
+        for idx in doc_id_range {
+            let val = self.get_val(idx);
+            if let Some(val) = val {
+                if value_range.contains(&val) {
+                    positions.push(idx);
+                }
+            }
+        }
+    }
+
+    /// Returns the minimum value for this fast field.
+    ///
+    /// This min_value may not be exact.
+    /// For instance, the min value does not take in account of possible
+    /// deleted document. All values are however guaranteed to be higher than
+    /// `.min_value()`.
+    fn min_value(&self) -> Option<T>;
+
+    /// Returns the maximum value for this fast field.
+    ///
+    /// This max_value may not be exact.
+    /// For instance, the max value does not take in account of possible
+    /// deleted document. All values are however guaranteed to be higher than
+    /// `.max_value()`.
+    fn max_value(&self) -> Option<T>;
+
+    /// The number of values including `None` in the column.
+    fn num_vals(&self) -> u32;
+
+    /// Returns a iterator over the data
+    fn iter<'a>(&'a self) -> Box<dyn Iterator<Item = Option<T>> + 'a> {
+        Box::new((0..self.num_vals()).map(|idx| self.get_val(idx)))
+    }
+
+    /// return full column if all values are set and is not empty
+    fn to_full(&self) -> Option<Arc<dyn Column<T>>> {
+        None
+    }
+}
+
+/// Temporary wrapper to migrate to optional column
+pub(crate) struct ToOptionalColumn<T> {
+    column: Arc<dyn Column<T>>,
+}
+
+impl<T: PartialOrd> ToOptionalColumn<T> {
+    pub(crate) fn new(column: Arc<dyn Column<T>>) -> Self {
+        Self { column }
+    }
+}
+
+impl<T: PartialOrd> OptionalColumn<T> for ToOptionalColumn<T> {
+    #[inline]
+    fn get_val(&self, idx: u32) -> Option<T> {
+        let val = self.column.get_val(idx);
+        Some(val)
+    }
+
+    fn min_value(&self) -> Option<T> {
+        let min_value = self.column.min_value();
+        Some(min_value)
+    }
+
+    fn max_value(&self) -> Option<T> {
+        let max_value = self.column.max_value();
+        Some(max_value)
+    }
+
+    fn num_vals(&self) -> u32 {
+        self.column.num_vals()
+    }
+
+    fn iter(&self) -> Box<dyn Iterator<Item = Option<T>> + '_> {
+        Box::new(self.column.iter().map(|el| Some(el)))
+    }
+    /// return full column if all values are set and is not empty
+    fn to_full(&self) -> Option<Arc<dyn Column<T>>> {
+        Some(self.column.clone())
+    }
+}

fastfield_codecs/src/serialize.rs

@@ -278,7 +278,10 @@ pub fn serialize_and_load<T: MonotonicallyMappableToU64 + Ord + Default>(
 ) -> Arc<dyn Column<T>> {
     let mut buffer = Vec::new();
     super::serialize(VecColumn::from(&column), &mut buffer, &ALL_CODEC_TYPES).unwrap();
-    super::open(OwnedBytes::new(buffer)).unwrap()
+    super::open(OwnedBytes::new(buffer))
+        .unwrap()
+        .to_full()
+        .unwrap()
 }
 
 #[cfg(test)]

src/aggregation/agg_req_with_accessor.rs

@@ -4,7 +4,7 @@ use std::rc::Rc;
 use std::sync::atomic::AtomicU32;
 use std::sync::Arc;
 
-use fastfield_codecs::Column;
+use fastfield_codecs::OptionalColumn;
 
 use super::agg_req::{Aggregation, Aggregations, BucketAggregationType, MetricAggregation};
 use super::bucket::{HistogramAggregation, RangeAggregation, TermsAggregation};
@@ -37,16 +37,16 @@ impl AggregationsWithAccessor {
 #[derive(Clone)]
 pub(crate) enum FastFieldAccessor {
     Multi(MultiValuedFastFieldReader<u64>),
-    Single(Arc<dyn Column<u64>>),
+    Single(Arc<dyn OptionalColumn<u64>>),
 }
 impl FastFieldAccessor {
-    pub fn as_single(&self) -> Option<&dyn Column<u64>> {
+    pub fn as_single(&self) -> Option<&dyn OptionalColumn<u64>> {
         match self {
             FastFieldAccessor::Multi(_) => None,
             FastFieldAccessor::Single(reader) => Some(&**reader),
         }
     }
-    pub fn into_single(self) -> Option<Arc<dyn Column<u64>>> {
+    pub fn into_single(self) -> Option<Arc<dyn OptionalColumn<u64>>> {
         match self {
             FastFieldAccessor::Multi(_) => None,
             FastFieldAccessor::Single(reader) => Some(reader),
@@ -124,7 +124,7 @@ impl BucketAggregationWithAccessor {
 pub struct MetricAggregationWithAccessor {
     pub metric: MetricAggregation,
     pub field_type: Type,
-    pub accessor: Arc<dyn Column>,
+    pub accessor: Arc<dyn OptionalColumn>,
 }
 
 impl MetricAggregationWithAccessor {

src/aggregation/bucket/histogram/histogram.rs

@@ -1,7 +1,7 @@
 use std::cmp::Ordering;
 use std::fmt::Display;
 
-use fastfield_codecs::Column;
+use fastfield_codecs::OptionalColumn;
 use itertools::Itertools;
 use serde::{Deserialize, Serialize};
 
@@ -263,13 +263,17 @@ impl SegmentHistogramCollector {
         req: &HistogramAggregation,
         sub_aggregation: &AggregationsWithAccessor,
         field_type: Type,
-        accessor: &dyn Column<u64>,
+        accessor: &dyn OptionalColumn<u64>,
     ) -> crate::Result<Self> {
         req.validate()?;
-        let min = f64_from_fastfield_u64(accessor.min_value(), &field_type);
-        let max = f64_from_fastfield_u64(accessor.max_value(), &field_type);
+        let min_max_u64 = accessor.min_value().zip(accessor.max_value());
+        let min_max_f64 = min_max_u64.map(|(min, max)| {
+            let min = f64_from_fastfield_u64(min, &field_type);
+            let max = f64_from_fastfield_u64(max, &field_type);
+            (min, max)
+        });
 
-        let (min, max) = get_req_min_max(req, Some((min, max)));
+        let (min, max) = get_req_min_max(req, min_max_f64);
 
         // We compute and generate the buckets range (min, max) based on the request and the min
         // max in the fast field, but this is likely not ideal when this is a subbucket, where many
@@ -331,47 +335,58 @@ impl SegmentHistogramCollector {
             .expect("unexpected fast field cardinatility");
         let mut iter = doc.chunks_exact(4);
         for docs in iter.by_ref() {
-            let val0 = self.f64_from_fastfield_u64(accessor.get_val(docs[0]));
-            let val1 = self.f64_from_fastfield_u64(accessor.get_val(docs[1]));
-            let val2 = self.f64_from_fastfield_u64(accessor.get_val(docs[2]));
-            let val3 = self.f64_from_fastfield_u64(accessor.get_val(docs[3]));
+            if let Some(val) = accessor.get_val(docs[0]) {
+                let val = self.f64_from_fastfield_u64(val);
+                let bucket_pos = get_bucket_num(val);
+                self.increment_bucket_if_in_bounds(
+                    val,
+                    &bounds,
+                    bucket_pos,
+                    docs[0],
+                    &bucket_with_accessor.sub_aggregation,
+                )?;
+            }
 
-            let bucket_pos0 = get_bucket_num(val0);
-            let bucket_pos1 = get_bucket_num(val1);
-            let bucket_pos2 = get_bucket_num(val2);
-            let bucket_pos3 = get_bucket_num(val3);
+            if let Some(val) = accessor.get_val(docs[1]) {
+                let val = self.f64_from_fastfield_u64(val);
+                let bucket_pos = get_bucket_num(val);
+                self.increment_bucket_if_in_bounds(
+                    val,
+                    &bounds,
+                    bucket_pos,
+                    docs[1],
+                    &bucket_with_accessor.sub_aggregation,
+                )?;
+            }
 
-            self.increment_bucket_if_in_bounds(
-                val0,
-                &bounds,
-                bucket_pos0,
-                docs[0],
-                &bucket_with_accessor.sub_aggregation,
-            )?;
-            self.increment_bucket_if_in_bounds(
-                val1,
-                &bounds,
-                bucket_pos1,
-                docs[1],
-                &bucket_with_accessor.sub_aggregation,
-            )?;
-            self.increment_bucket_if_in_bounds(
-                val2,
-                &bounds,
-                bucket_pos2,
-                docs[2],
-                &bucket_with_accessor.sub_aggregation,
-            )?;
-            self.increment_bucket_if_in_bounds(
-                val3,
-                &bounds,
-                bucket_pos3,
-                docs[3],
-                &bucket_with_accessor.sub_aggregation,
-            )?;
+            if let Some(val) = accessor.get_val(docs[2]) {
+                let val = self.f64_from_fastfield_u64(val);
+                let bucket_pos = get_bucket_num(val);
+                self.increment_bucket_if_in_bounds(
+                    val,
+                    &bounds,
+                    bucket_pos,
+                    docs[2],
+                    &bucket_with_accessor.sub_aggregation,
+                )?;
+            }
+
+            if let Some(val) = accessor.get_val(docs[3]) {
+                let val = self.f64_from_fastfield_u64(val);
+                let bucket_pos = get_bucket_num(val);
+                self.increment_bucket_if_in_bounds(
+                    val,
+                    &bounds,
+                    bucket_pos,
+                    docs[3],
+                    &bucket_with_accessor.sub_aggregation,
+                )?;
+            }
         }
         for &doc in iter.remainder() {
-            let val = f64_from_fastfield_u64(accessor.get_val(doc), &self.field_type);
+            let Some(val) = accessor.get_val(doc).map(|val|f64_from_fastfield_u64(val, &self.field_type)) else{
+                continue;
+            };
             if !bounds.contains(val) {
                 continue;
             }

src/aggregation/bucket/range.rs

@@ -267,20 +267,29 @@ impl SegmentRangeCollector {
             let val2 = accessor.get_val(docs[1]);
             let val3 = accessor.get_val(docs[2]);
             let val4 = accessor.get_val(docs[3]);
-            let bucket_pos1 = self.get_bucket_pos(val1);
-            let bucket_pos2 = self.get_bucket_pos(val2);
-            let bucket_pos3 = self.get_bucket_pos(val3);
-            let bucket_pos4 = self.get_bucket_pos(val4);
-
-            self.increment_bucket(bucket_pos1, docs[0], &bucket_with_accessor.sub_aggregation)?;
-            self.increment_bucket(bucket_pos2, docs[1], &bucket_with_accessor.sub_aggregation)?;
-            self.increment_bucket(bucket_pos3, docs[2], &bucket_with_accessor.sub_aggregation)?;
-            self.increment_bucket(bucket_pos4, docs[3], &bucket_with_accessor.sub_aggregation)?;
+            if let Some(val) = val1 {
+                let bucket_pos = self.get_bucket_pos(val);
+                self.increment_bucket(bucket_pos, docs[0], &bucket_with_accessor.sub_aggregation)?;
+            }
+            if let Some(val) = val2 {
+                let bucket_pos = self.get_bucket_pos(val);
+                self.increment_bucket(bucket_pos, docs[1], &bucket_with_accessor.sub_aggregation)?;
+            }
+            if let Some(val) = val3 {
+                let bucket_pos = self.get_bucket_pos(val);
+                self.increment_bucket(bucket_pos, docs[2], &bucket_with_accessor.sub_aggregation)?;
+            }
+            if let Some(val) = val4 {
+                let bucket_pos = self.get_bucket_pos(val);
+                self.increment_bucket(bucket_pos, docs[3], &bucket_with_accessor.sub_aggregation)?;
+            }
         }
         for &doc in iter.remainder() {
             let val = accessor.get_val(doc);
-            let bucket_pos = self.get_bucket_pos(val);
-            self.increment_bucket(bucket_pos, doc, &bucket_with_accessor.sub_aggregation)?;
+            if let Some(val) = val {
+                let bucket_pos = self.get_bucket_pos(val);
+                self.increment_bucket(bucket_pos, doc, &bucket_with_accessor.sub_aggregation)?;
+            }
         }
         if force_flush {
             for bucket in &mut self.buckets {

src/aggregation/metric/average.rs

@@ -1,6 +1,6 @@
 use std::fmt::Debug;
 
-use fastfield_codecs::Column;
+use fastfield_codecs::OptionalColumn;
 use serde::{Deserialize, Serialize};
 
 use crate::aggregation::f64_from_fastfield_u64;
@@ -57,26 +57,33 @@ impl SegmentAverageCollector {
             data: Default::default(),
         }
     }
-    pub(crate) fn collect_block(&mut self, doc: &[DocId], field: &dyn Column<u64>) {
+    pub(crate) fn collect_block(&mut self, doc: &[DocId], field: &dyn OptionalColumn<u64>) {
         let mut iter = doc.chunks_exact(4);
         for docs in iter.by_ref() {
-            let val1 = field.get_val(docs[0]);
-            let val2 = field.get_val(docs[1]);
-            let val3 = field.get_val(docs[2]);
-            let val4 = field.get_val(docs[3]);
-            let val1 = f64_from_fastfield_u64(val1, &self.field_type);
-            let val2 = f64_from_fastfield_u64(val2, &self.field_type);
-            let val3 = f64_from_fastfield_u64(val3, &self.field_type);
-            let val4 = f64_from_fastfield_u64(val4, &self.field_type);
-            self.data.collect(val1);
-            self.data.collect(val2);
-            self.data.collect(val3);
-            self.data.collect(val4);
+            if let Some(val) = field.get_val(docs[0]) {
+                let val = f64_from_fastfield_u64(val, &self.field_type);
+                self.data.collect(val);
+            }
+            if let Some(val) = field.get_val(docs[1]) {
+                let val = f64_from_fastfield_u64(val, &self.field_type);
+                self.data.collect(val);
+            }
+
+            if let Some(val) = field.get_val(docs[2]) {
+                let val = f64_from_fastfield_u64(val, &self.field_type);
+                self.data.collect(val);
+            }
+
+            if let Some(val) = field.get_val(docs[3]) {
+                let val = f64_from_fastfield_u64(val, &self.field_type);
+                self.data.collect(val);
+            }
         }
         for &doc in iter.remainder() {
-            let val = field.get_val(doc);
-            let val = f64_from_fastfield_u64(val, &self.field_type);
-            self.data.collect(val);
+            if let Some(val) = field.get_val(doc) {
+                let val = f64_from_fastfield_u64(val, &self.field_type);
+                self.data.collect(val);
+            }
         }
     }
 }

src/aggregation/metric/stats.rs

@@ -1,4 +1,4 @@
-use fastfield_codecs::Column;
+use fastfield_codecs::OptionalColumn;
 use serde::{Deserialize, Serialize};
 
 use crate::aggregation::f64_from_fastfield_u64;
@@ -163,26 +163,31 @@ impl SegmentStatsCollector {
             stats: IntermediateStats::default(),
         }
     }
-    pub(crate) fn collect_block(&mut self, doc: &[DocId], field: &dyn Column<u64>) {
+    pub(crate) fn collect_block(&mut self, doc: &[DocId], field: &dyn OptionalColumn<u64>) {
         let mut iter = doc.chunks_exact(4);
         for docs in iter.by_ref() {
-            let val1 = field.get_val(docs[0]);
-            let val2 = field.get_val(docs[1]);
-            let val3 = field.get_val(docs[2]);
-            let val4 = field.get_val(docs[3]);
-            let val1 = f64_from_fastfield_u64(val1, &self.field_type);
-            let val2 = f64_from_fastfield_u64(val2, &self.field_type);
-            let val3 = f64_from_fastfield_u64(val3, &self.field_type);
-            let val4 = f64_from_fastfield_u64(val4, &self.field_type);
-            self.stats.collect(val1);
-            self.stats.collect(val2);
-            self.stats.collect(val3);
-            self.stats.collect(val4);
+            if let Some(val) = field.get_val(docs[0]) {
+                let val = f64_from_fastfield_u64(val, &self.field_type);
+                self.stats.collect(val);
+            }
+            if let Some(val) = field.get_val(docs[1]) {
+                let val = f64_from_fastfield_u64(val, &self.field_type);
+                self.stats.collect(val);
+            }
+            if let Some(val) = field.get_val(docs[2]) {
+                let val = f64_from_fastfield_u64(val, &self.field_type);
+                self.stats.collect(val);
+            }
+            if let Some(val) = field.get_val(docs[3]) {
+                let val = f64_from_fastfield_u64(val, &self.field_type);
+                self.stats.collect(val);
+            }
         }
         for &doc in iter.remainder() {
-            let val = field.get_val(doc);
-            let val = f64_from_fastfield_u64(val, &self.field_type);
-            self.stats.collect(val);
+            if let Some(val) = field.get_val(doc) {
+                let val = f64_from_fastfield_u64(val, &self.field_type);
+                self.stats.collect(val);
+            }
         }
     }
 }

src/collector/filter_collector_wrapper.rs

@@ -130,7 +130,9 @@ where
 
         let fast_field_reader = segment_reader
             .fast_fields()
-            .typed_fast_field_reader(self.field)?;
+            .typed_fast_field_reader(self.field)?
+            .to_full()
+            .expect("temp migration solution");
 
         let segment_collector = self
             .collector

src/collector/histogram_collector.rs

@@ -112,7 +112,11 @@ impl Collector for HistogramCollector {
         _segment_local_id: crate::SegmentOrdinal,
         segment: &crate::SegmentReader,
     ) -> crate::Result<Self::Child> {
-        let ff_reader = segment.fast_fields().u64_lenient(self.field)?;
+        let ff_reader = segment
+            .fast_fields()
+            .u64_lenient(self.field)?
+            .to_full()
+            .expect("temp migration solution");
         Ok(SegmentHistogramCollector {
             histogram_computer: HistogramComputer {
                 counts: vec![0; self.num_buckets],

src/collector/tests.rs

@@ -1,6 +1,6 @@
 use std::sync::Arc;
 
-use fastfield_codecs::Column;
+use fastfield_codecs::OptionalColumn;
 
 use super::*;
 use crate::collector::{Count, FilterCollector, TopDocs};
@@ -160,7 +160,7 @@ pub struct FastFieldTestCollector {
 
 pub struct FastFieldSegmentCollector {
     vals: Vec<u64>,
-    reader: Arc<dyn Column<u64>>,
+    reader: Arc<dyn OptionalColumn<u64>>,
 }
 
 impl FastFieldTestCollector {
@@ -202,7 +202,9 @@ impl SegmentCollector for FastFieldSegmentCollector {
 
     fn collect(&mut self, doc: DocId, _score: Score) {
         let val = self.reader.get_val(doc);
-        self.vals.push(val);
+        if let Some(val) = val {
+            self.vals.push(val);
+        }
     }
 
     fn harvest(self) -> Vec<u64> {

src/collector/top_score_collector.rs

@@ -156,7 +156,9 @@ impl CustomScorer<u64> for ScorerByField {
         // The conversion will then happen only on the top-K docs.
         let ff_reader = segment_reader
             .fast_fields()
-            .typed_fast_field_reader(self.field)?;
+            .typed_fast_field_reader(self.field)?
+            .to_full()
+            .expect("temp migration solution");
         Ok(ScorerByFastFieldReader { ff_reader })
     }
 }
@@ -458,7 +460,7 @@ impl TopDocs {
     ///
     ///             // We can now define our actual scoring function
     ///             move |doc: DocId, original_score: Score| {
-    ///                 let popularity: u64 = popularity_reader.get_val(doc);
+    ///                 let popularity: u64 = popularity_reader.get_val(doc).unwrap();
     ///                 // Well.. For the sake of the example we use a simple logarithm
     ///                 // function.
     ///                 let popularity_boost_score = ((2u64 + popularity) as Score).log2();
@@ -567,8 +569,8 @@ impl TopDocs {
     ///
     ///             // We can now define our actual scoring function
     ///             move |doc: DocId| {
-    ///                 let popularity: u64 = popularity_reader.get_val(doc);
-    ///                 let boosted: u64 = boosted_reader.get_val(doc);
+    ///                 let popularity: u64 = popularity_reader.get_val(doc).unwrap();
+    ///                 let boosted: u64 = boosted_reader.get_val(doc).unwrap();
     ///                 // Score do not have to be `f64` in tantivy.
     ///                 // Here we return a couple to get lexicographical order
     ///                 // for free.

src/fastfield/mod.rs

@@ -210,7 +210,7 @@ mod tests {
         assert_eq!(file.len(), 25);
         let composite_file = CompositeFile::open(&file)?;
         let fast_field_bytes = composite_file.open_read(*FIELD).unwrap().read_bytes()?;
-        let fast_field_reader = open::<u64>(fast_field_bytes)?;
+        let fast_field_reader = open::<u64>(fast_field_bytes)?.to_full().unwrap();
         assert_eq!(fast_field_reader.get_val(0), 13u64);
         assert_eq!(fast_field_reader.get_val(1), 14u64);
         assert_eq!(fast_field_reader.get_val(2), 2u64);
@@ -263,7 +263,7 @@ mod tests {
                 .open_read(*FIELD)
                 .unwrap()
                 .read_bytes()?;
-            let fast_field_reader = open::<u64>(data)?;
+            let fast_field_reader = open::<u64>(data)?.to_full().unwrap();
             assert_eq!(fast_field_reader.get_val(0), 4u64);
             assert_eq!(fast_field_reader.get_val(1), 14_082_001u64);
             assert_eq!(fast_field_reader.get_val(2), 3_052u64);
@@ -304,7 +304,7 @@ mod tests {
                 .open_read(*FIELD)
                 .unwrap()
                 .read_bytes()?;
-            let fast_field_reader = open::<u64>(data)?;
+            let fast_field_reader = open::<u64>(data)?.to_full().unwrap();
             for doc in 0..10_000 {
                 assert_eq!(fast_field_reader.get_val(doc), 100_000u64);
             }
@@ -343,7 +343,7 @@ mod tests {
                 .open_read(*FIELD)
                 .unwrap()
                 .read_bytes()?;
-            let fast_field_reader = open::<u64>(data)?;
+            let fast_field_reader = open::<u64>(data)?.to_full().unwrap();
             assert_eq!(fast_field_reader.get_val(0), 0u64);
             for doc in 1..10_001 {
                 assert_eq!(
@@ -386,7 +386,7 @@ mod tests {
                 .open_read(i64_field)
                 .unwrap()
                 .read_bytes()?;
-            let fast_field_reader = open::<i64>(data)?;
+            let fast_field_reader = open::<i64>(data)?.to_full().unwrap();
 
             assert_eq!(fast_field_reader.min_value(), -100i64);
             assert_eq!(fast_field_reader.max_value(), 9_999i64);
@@ -429,7 +429,7 @@ mod tests {
                 .open_read(i64_field)
                 .unwrap()
                 .read_bytes()?;
-            let fast_field_reader = open::<i64>(data)?;
+            let fast_field_reader = open::<i64>(data)?.to_full().unwrap();
             assert_eq!(fast_field_reader.get_val(0), 0i64);
         }
         Ok(())
@@ -470,7 +470,7 @@ mod tests {
                 .open_read(*FIELD)
                 .unwrap()
                 .read_bytes()?;
-            let fast_field_reader = open::<u64>(data)?;
+            let fast_field_reader = open::<u64>(data)?.to_full().unwrap();
 
             for a in 0..n {
                 assert_eq!(fast_field_reader.get_val(a as u32), permutation[a as usize]);
@@ -763,19 +763,28 @@ mod tests {
         let dates_fast_field = fast_fields.dates(multi_date_field).unwrap();
         let mut dates = vec![];
         {
-            assert_eq!(date_fast_field.get_val(0).into_timestamp_micros(), 1i64);
+            assert_eq!(
+                date_fast_field.get_val(0).unwrap().into_timestamp_micros(),
+                1i64
+            );
             dates_fast_field.get_vals(0u32, &mut dates);
             assert_eq!(dates.len(), 2);
             assert_eq!(dates[0].into_timestamp_micros(), 2i64);
             assert_eq!(dates[1].into_timestamp_micros(), 3i64);
         }
         {
-            assert_eq!(date_fast_field.get_val(1).into_timestamp_micros(), 4i64);
+            assert_eq!(
+                date_fast_field.get_val(1).unwrap().into_timestamp_micros(),
+                4i64
+            );
             dates_fast_field.get_vals(1u32, &mut dates);
             assert!(dates.is_empty());
         }
         {
-            assert_eq!(date_fast_field.get_val(2).into_timestamp_micros(), 0i64);
+            assert_eq!(
+                date_fast_field.get_val(2).unwrap().into_timestamp_micros(),
+                0i64
+            );
             dates_fast_field.get_vals(2u32, &mut dates);
             assert_eq!(dates.len(), 2);
             assert_eq!(dates[0].into_timestamp_micros(), 5i64);
@@ -825,7 +834,7 @@ mod tests {
         assert_eq!(file.len(), 24);
         let composite_file = CompositeFile::open(&file)?;
         let data = composite_file.open_read(field).unwrap().read_bytes()?;
-        let fast_field_reader = open::<bool>(data)?;
+        let fast_field_reader = open::<bool>(data)?.to_full().unwrap();
         assert_eq!(fast_field_reader.get_val(0), true);
         assert_eq!(fast_field_reader.get_val(1), false);
         assert_eq!(fast_field_reader.get_val(2), true);
@@ -863,7 +872,7 @@ mod tests {
         assert_eq!(file.len(), 36);
         let composite_file = CompositeFile::open(&file)?;
         let data = composite_file.open_read(field).unwrap().read_bytes()?;
-        let fast_field_reader = open::<bool>(data)?;
+        let fast_field_reader = open::<bool>(data)?.to_full().unwrap();
         for i in 0..25 {
             assert_eq!(fast_field_reader.get_val(i * 2), true);
             assert_eq!(fast_field_reader.get_val(i * 2 + 1), false);
@@ -894,7 +903,7 @@ mod tests {
         let composite_file = CompositeFile::open(&file)?;
         assert_eq!(file.len(), 23);
         let data = composite_file.open_read(field).unwrap().read_bytes()?;
-        let fast_field_reader = open::<bool>(data)?;
+        let fast_field_reader = open::<bool>(data)?.to_full().unwrap();
         assert_eq!(fast_field_reader.get_val(0), false);
 
         Ok(())
@@ -962,7 +971,9 @@ mod tests {
         let composite_file = CompositeFile::open(&file)?;
         let file = composite_file.open_read(*FIELD).unwrap();
         let len = file.len();
-        let test_fastfield = open::<DateTime>(file.read_bytes()?)?;
+        let test_fastfield = open::<DateTime>(file.read_bytes()?)?
+            .to_full()
+            .expect("temp migration solution");
 
         for (i, time) in times.iter().enumerate() {
             assert_eq!(test_fastfield.get_val(i as u32), time.truncate(precision));

src/fastfield/multivalued/mod.rs

@@ -533,14 +533,17 @@ mod bench {
                 .unwrap()
                 .read_bytes()
                 .unwrap();
-            let idx_reader = fastfield_codecs::open(data_idx).unwrap();
+            let idx_reader = fastfield_codecs::open(data_idx).unwrap().to_full().unwrap();
 
             let data_vals = fast_fields_composite
                 .open_read_with_idx(field, 1)
                 .unwrap()
                 .read_bytes()
                 .unwrap();
-            let vals_reader = fastfield_codecs::open(data_vals).unwrap();
+            let vals_reader = fastfield_codecs::open(data_vals)
+                .unwrap()
+                .to_full()
+                .unwrap();
             let fast_field_reader = MultiValuedFastFieldReader::open(idx_reader, vals_reader);
             b.iter(|| {
                 let mut sum = 0u64;

src/fastfield/readers.rs

@@ -1,7 +1,7 @@
 use std::net::Ipv6Addr;
 use std::sync::Arc;
 
-use fastfield_codecs::{open, open_u128, Column};
+use fastfield_codecs::{open, open_u128, Column, OptionalColumn};
 
 use super::multivalued::MultiValuedU128FastFieldReader;
 use crate::directory::{CompositeFile, FileSlice};
@@ -118,7 +118,7 @@ impl FastFieldReaders {
         &self,
         field: Field,
         index: usize,
-    ) -> crate::Result<Arc<dyn Column<TFastValue>>> {
+    ) -> crate::Result<Arc<dyn OptionalColumn<TFastValue>>> {
         let fast_field_slice = self.fast_field_data(field, index)?;
         let bytes = fast_field_slice.read_bytes()?;
         let column = fastfield_codecs::open(bytes)?;
@@ -128,7 +128,7 @@ impl FastFieldReaders {
     pub(crate) fn typed_fast_field_reader<TFastValue: FastValue>(
         &self,
         field: Field,
-    ) -> crate::Result<Arc<dyn Column<TFastValue>>> {
+    ) -> crate::Result<Arc<dyn OptionalColumn<TFastValue>>> {
         self.typed_fast_field_reader_with_idx(field, 0)
     }
 
@@ -138,13 +138,20 @@ impl FastFieldReaders {
     ) -> crate::Result<MultiValuedFastFieldReader<TFastValue>> {
         let idx_reader = self.typed_fast_field_reader(field)?;
         let vals_reader = self.typed_fast_field_reader_with_idx(field, 1)?;
-        Ok(MultiValuedFastFieldReader::open(idx_reader, vals_reader))
+        Ok(MultiValuedFastFieldReader::open(
+            idx_reader
+                .to_full()
+                .expect("multivalue fast field are always full"),
+            vals_reader
+                .to_full()
+                .expect("multivalue fast field are always full"),
+        ))
     }
 
     /// Returns the `u64` fast field reader reader associated with `field`.
     ///
     /// If `field` is not a u64 fast field, this method returns an Error.
-    pub fn u64(&self, field: Field) -> crate::Result<Arc<dyn Column<u64>>> {
+    pub fn u64(&self, field: Field) -> crate::Result<Arc<dyn OptionalColumn<u64>>> {
         self.check_type(field, FastType::U64, Cardinality::SingleValue)?;
         self.typed_fast_field_reader(field)
     }
@@ -152,7 +159,7 @@ impl FastFieldReaders {
     /// Returns the `ip` fast field reader reader associated to `field`.
     ///
     /// If `field` is not a u128 fast field, this method returns an Error.
-    pub fn ip_addr(&self, field: Field) -> crate::Result<Arc<dyn Column<Ipv6Addr>>> {
+    pub fn ip_addr(&self, field: Field) -> crate::Result<Arc<dyn OptionalColumn<Ipv6Addr>>> {
         self.check_type(field, FastType::U128, Cardinality::SingleValue)?;
         let bytes = self.fast_field_data(field, 0)?.read_bytes()?;
         Ok(open_u128::<Ipv6Addr>(bytes)?)
@@ -166,10 +173,15 @@ impl FastFieldReaders {
         field: Field,
     ) -> crate::Result<MultiValuedU128FastFieldReader<Ipv6Addr>> {
         self.check_type(field, FastType::U128, Cardinality::MultiValues)?;
-        let idx_reader: Arc<dyn Column<u64>> = self.typed_fast_field_reader(field)?;
+        let idx_reader: Arc<dyn Column<u64>> = self
+            .typed_fast_field_reader(field)?
+            .to_full()
+            .expect("multivalue fast fields are always full");
 
         let bytes = self.fast_field_data(field, 1)?.read_bytes()?;
-        let vals_reader = open_u128::<Ipv6Addr>(bytes)?;
+        let vals_reader = open_u128::<Ipv6Addr>(bytes)?
+            .to_full()
+            .expect("multivalue fields are always full");
 
         Ok(MultiValuedU128FastFieldReader::open(
             idx_reader,
@@ -179,8 +191,9 @@ impl FastFieldReaders {
 
     /// Returns the `u128` fast field reader reader associated to `field`.
     ///
-    /// If `field` is not a u128 fast field, this method returns an Error.
-    pub(crate) fn u128(&self, field: Field) -> crate::Result<Arc<dyn Column<u128>>> {
+    /// If `field` is not a u128 base type fast field, this method returns an Error.
+    /// Ip addresses use u128 as base type.
+    pub(crate) fn u128(&self, field: Field) -> crate::Result<Arc<dyn OptionalColumn<u128>>> {
         self.check_type(field, FastType::U128, Cardinality::SingleValue)?;
         let bytes = self.fast_field_data(field, 0)?.read_bytes()?;
         Ok(open_u128::<u128>(bytes)?)
@@ -191,10 +204,15 @@ impl FastFieldReaders {
     /// If `field` is not a u128 multi-valued fast field, this method returns an Error.
     pub fn u128s(&self, field: Field) -> crate::Result<MultiValuedU128FastFieldReader<u128>> {
         self.check_type(field, FastType::U128, Cardinality::MultiValues)?;
-        let idx_reader: Arc<dyn Column<u64>> = self.typed_fast_field_reader(field)?;
+        let idx_reader: Arc<dyn Column<u64>> = self
+            .typed_fast_field_reader(field)?
+            .to_full()
+            .expect("multivalue fast fields are always full");
 
         let bytes = self.fast_field_data(field, 1)?.read_bytes()?;
-        let vals_reader = open_u128::<u128>(bytes)?;
+        let vals_reader = open_u128::<u128>(bytes)?
+            .to_full()
+            .expect("multivalue fast fields are always full");
 
         Ok(MultiValuedU128FastFieldReader::open(
             idx_reader,
@@ -207,14 +225,14 @@ impl FastFieldReaders {
     ///
     /// If not, the fastfield reader will returns the u64-value associated with the original
     /// FastValue.
-    pub fn u64_lenient(&self, field: Field) -> crate::Result<Arc<dyn Column<u64>>> {
+    pub fn u64_lenient(&self, field: Field) -> crate::Result<Arc<dyn OptionalColumn<u64>>> {
         self.typed_fast_field_reader(field)
     }
 
     /// Returns the `i64` fast field reader reader associated with `field`.
     ///
     /// If `field` is not a i64 fast field, this method returns an Error.
-    pub fn i64(&self, field: Field) -> crate::Result<Arc<dyn Column<i64>>> {
+    pub fn i64(&self, field: Field) -> crate::Result<Arc<dyn OptionalColumn<i64>>> {
         self.check_type(field, FastType::I64, Cardinality::SingleValue)?;
         self.typed_fast_field_reader(field)
     }
@@ -222,7 +240,7 @@ impl FastFieldReaders {
     /// Returns the `date` fast field reader reader associated with `field`.
     ///
     /// If `field` is not a date fast field, this method returns an Error.
-    pub fn date(&self, field: Field) -> crate::Result<Arc<dyn Column<DateTime>>> {
+    pub fn date(&self, field: Field) -> crate::Result<Arc<dyn OptionalColumn<DateTime>>> {
         self.check_type(field, FastType::Date, Cardinality::SingleValue)?;
         self.typed_fast_field_reader(field)
     }
@@ -230,7 +248,7 @@ impl FastFieldReaders {
     /// Returns the `f64` fast field reader reader associated with `field`.
     ///
     /// If `field` is not a f64 fast field, this method returns an Error.
-    pub fn f64(&self, field: Field) -> crate::Result<Arc<dyn Column<f64>>> {
+    pub fn f64(&self, field: Field) -> crate::Result<Arc<dyn OptionalColumn<f64>>> {
         self.check_type(field, FastType::F64, Cardinality::SingleValue)?;
         self.typed_fast_field_reader(field)
     }
@@ -238,7 +256,7 @@ impl FastFieldReaders {
     /// Returns the `bool` fast field reader reader associated with `field`.
     ///
     /// If `field` is not a bool fast field, this method returns an Error.
-    pub fn bool(&self, field: Field) -> crate::Result<Arc<dyn Column<bool>>> {
+    pub fn bool(&self, field: Field) -> crate::Result<Arc<dyn OptionalColumn<bool>>> {
         self.check_type(field, FastType::Bool, Cardinality::SingleValue)?;
         self.typed_fast_field_reader(field)
     }
@@ -309,7 +327,12 @@ impl FastFieldReaders {
             let fast_field_idx_bytes = fast_field_idx_file.read_bytes()?;
             let idx_reader = open(fast_field_idx_bytes)?;
             let data = self.fast_field_data(field, 1)?;
-            BytesFastFieldReader::open(idx_reader, data)
+            BytesFastFieldReader::open(
+                idx_reader
+                    .to_full()
+                    .expect("multivalue fields are always full"),
+                data,
+            )
         } else {
             Err(FastFieldNotAvailableError::new(field_entry).into())
         }

src/indexer/doc_id_mapping.rs

@@ -465,9 +465,9 @@ mod tests_indexsorting {
         let my_number = index.schema().get_field("my_number").unwrap();
 
         let fast_field = fast_fields.u64(my_number).unwrap();
-        assert_eq!(fast_field.get_val(0), 10u64);
-        assert_eq!(fast_field.get_val(1), 20u64);
-        assert_eq!(fast_field.get_val(2), 30u64);
+        assert_eq!(fast_field.get_val(0), Some(10u64));
+        assert_eq!(fast_field.get_val(1), Some(20u64));
+        assert_eq!(fast_field.get_val(2), Some(30u64));
 
         let multi_numbers = index.schema().get_field("multi_numbers").unwrap();
         let multifield = fast_fields.u64s(multi_numbers).unwrap();

src/indexer/index_writer.rs

@@ -1467,7 +1467,7 @@ mod tests {
         let fast_field_reader = segment_reader.fast_fields().u64(id_field)?;
         let in_order_alive_ids: Vec<u64> = segment_reader
             .doc_ids_alive()
-            .map(|doc| fast_field_reader.get_val(doc))
+            .map(|doc| fast_field_reader.get_val(doc).unwrap())
             .collect();
         assert_eq!(&in_order_alive_ids[..], &[9, 8, 7, 6, 5, 4, 1, 0]);
         Ok(())
@@ -1528,7 +1528,7 @@ mod tests {
         let fast_field_reader = segment_reader.fast_fields().u64(id_field)?;
         let in_order_alive_ids: Vec<u64> = segment_reader
             .doc_ids_alive()
-            .map(|doc| fast_field_reader.get_val(doc))
+            .map(|doc| fast_field_reader.get_val(doc).unwrap())
             .collect();
         assert_eq!(&in_order_alive_ids[..], &[9, 8, 7, 6, 5, 4, 2, 0]);
         Ok(())
@@ -1777,7 +1777,12 @@ mod tests {
             .segment_readers()
             .iter()
             .flat_map(|segment_reader| {
-                let ff_reader = segment_reader.fast_fields().u64(id_field).unwrap();
+                let ff_reader = segment_reader
+                    .fast_fields()
+                    .u64(id_field)
+                    .unwrap()
+                    .to_full()
+                    .unwrap();
                 segment_reader
                     .doc_ids_alive()
                     .map(move |doc| ff_reader.get_val(doc))
@@ -1788,7 +1793,12 @@ mod tests {
             .segment_readers()
             .iter()
             .flat_map(|segment_reader| {
-                let ff_reader = segment_reader.fast_fields().u64(id_field).unwrap();
+                let ff_reader = segment_reader
+                    .fast_fields()
+                    .u64(id_field)
+                    .unwrap()
+                    .to_full()
+                    .unwrap();
                 segment_reader
                     .doc_ids_alive()
                     .map(move |doc| ff_reader.get_val(doc))
@@ -1864,7 +1874,7 @@ mod tests {
             .flat_map(|segment_reader| {
                 let ff_reader = segment_reader.fast_fields().ip_addr(ip_field).unwrap();
                 segment_reader.doc_ids_alive().flat_map(move |doc| {
-                    let val = ff_reader.get_val(doc);
+                    let val = ff_reader.get_val(doc).unwrap(); // TODO handle null
                     if val == Ipv6Addr::from_u128(0) {
                         // TODO Fix null handling
                         None
@@ -1921,7 +1931,7 @@ mod tests {
                 ff_reader.get_vals(doc, &mut vals);
                 assert_eq!(vals.len(), 2);
                 assert_eq!(vals[0], vals[1]);
-                assert_eq!(id_reader.get_val(doc), vals[0]);
+                assert_eq!(id_reader.get_val(doc).unwrap(), vals[0]);
 
                 let mut bool_vals = vec![];
                 bool_ff_reader.get_vals(doc, &mut bool_vals);
@@ -2117,7 +2127,7 @@ mod tests {
                 facet_reader
                     .facet_from_ord(facet_ords[0], &mut facet)
                     .unwrap();
-                let id = ff_reader.get_val(doc_id);
+                let id = ff_reader.get_val(doc_id).unwrap();
                 let facet_expected = Facet::from(&("/cola/".to_string() + &id.to_string()));
 
                 assert_eq!(facet, facet_expected);

src/indexer/merger.rs

@@ -401,10 +401,15 @@ impl IndexMerger {
             .readers
             .iter()
             .map(|reader| {
-                let u128_reader: Arc<dyn Column<u128>> = reader.fast_fields().u128(field).expect(
-                    "Failed to find a reader for single fast field. This is a tantivy bug and it \
-                     should never happen.",
-                );
+                let u128_reader: Arc<dyn Column<u128>> = reader
+                    .fast_fields()
+                    .u128(field)
+                    .expect(
+                        "Failed to find a reader for single fast field. This is a tantivy bug and \
+                         it should never happen.",
+                    )
+                    .to_full()
+                    .expect("temp migration solution");
                 u128_reader
             })
             .collect::<Vec<_>>();
@@ -465,7 +470,11 @@ impl IndexMerger {
         sort_by_field: &IndexSortByField,
     ) -> crate::Result<Arc<dyn Column>> {
         let field_id = expect_field_id_for_sort_field(reader.schema(), sort_by_field)?; // for now expect fastfield, but not strictly required
-        let value_accessor = reader.fast_fields().u64_lenient(field_id)?;
+        let value_accessor = reader
+            .fast_fields()
+            .u64_lenient(field_id)?
+            .to_full()
+            .expect("temp migration solution");
         Ok(value_accessor)
     }
     /// Collecting value_accessors into a vec to bind the lifetime.
@@ -1368,16 +1377,16 @@ mod tests {
                 .fast_fields()
                 .u64(score_field)
                 .unwrap();
-            assert_eq!(score_field_reader.min_value(), 4000);
-            assert_eq!(score_field_reader.max_value(), 7000);
+            assert_eq!(score_field_reader.min_value(), Some(4000));
+            assert_eq!(score_field_reader.max_value(), Some(7000));
 
             let score_field_reader = searcher
                 .segment_reader(1)
                 .fast_fields()
                 .u64(score_field)
                 .unwrap();
-            assert_eq!(score_field_reader.min_value(), 1);
-            assert_eq!(score_field_reader.max_value(), 3);
+            assert_eq!(score_field_reader.min_value(), Some(1));
+            assert_eq!(score_field_reader.max_value(), Some(3));
         }
         {
             // merging the segments
@@ -1422,8 +1431,8 @@ mod tests {
                 .fast_fields()
                 .u64(score_field)
                 .unwrap();
-            assert_eq!(score_field_reader.min_value(), 3);
-            assert_eq!(score_field_reader.max_value(), 7000);
+            assert_eq!(score_field_reader.min_value(), Some(3));
+            assert_eq!(score_field_reader.max_value(), Some(7000));
         }
         {
             // test a commit with only deletes
@@ -1469,8 +1478,8 @@ mod tests {
                 .fast_fields()
                 .u64(score_field)
                 .unwrap();
-            assert_eq!(score_field_reader.min_value(), 3);
-            assert_eq!(score_field_reader.max_value(), 7000);
+            assert_eq!(score_field_reader.min_value(), Some(3));
+            assert_eq!(score_field_reader.max_value(), Some(7000));
         }
         {
             // Test merging a single segment in order to remove deletes.
@@ -1516,8 +1525,8 @@ mod tests {
                 .fast_fields()
                 .u64(score_field)
                 .unwrap();
-            assert_eq!(score_field_reader.min_value(), 6000);
-            assert_eq!(score_field_reader.max_value(), 7000);
+            assert_eq!(score_field_reader.min_value(), Some(6000));
+            assert_eq!(score_field_reader.max_value(), Some(7000));
         }
 
         {

src/indexer/merger_sorted_index_test.rs

@@ -186,17 +186,17 @@ mod tests {
 
         let fast_fields = segment_reader.fast_fields();
         let fast_field = fast_fields.u64(int_field).unwrap();
-        assert_eq!(fast_field.get_val(5), 1u64);
-        assert_eq!(fast_field.get_val(4), 2u64);
-        assert_eq!(fast_field.get_val(3), 3u64);
+        assert_eq!(fast_field.get_val(5), Some(1u64));
+        assert_eq!(fast_field.get_val(4), Some(2u64));
+        assert_eq!(fast_field.get_val(3), Some(3u64));
         if force_disjunct_segment_sort_values {
-            assert_eq!(fast_field.get_val(2), 20u64);
-            assert_eq!(fast_field.get_val(1), 100u64);
+            assert_eq!(fast_field.get_val(2), Some(20u64));
+            assert_eq!(fast_field.get_val(1), Some(100u64));
         } else {
-            assert_eq!(fast_field.get_val(2), 10u64);
-            assert_eq!(fast_field.get_val(1), 20u64);
+            assert_eq!(fast_field.get_val(2), Some(10u64));
+            assert_eq!(fast_field.get_val(1), Some(20u64));
         }
-        assert_eq!(fast_field.get_val(0), 1_000u64);
+        assert_eq!(fast_field.get_val(0), Some(1_000u64));
 
         // test new field norm mapping
         {
@@ -373,12 +373,12 @@ mod tests {
 
         let fast_fields = segment_reader.fast_fields();
         let fast_field = fast_fields.u64(int_field).unwrap();
-        assert_eq!(fast_field.get_val(0), 1u64);
-        assert_eq!(fast_field.get_val(1), 2u64);
-        assert_eq!(fast_field.get_val(2), 3u64);
-        assert_eq!(fast_field.get_val(3), 10u64);
-        assert_eq!(fast_field.get_val(4), 20u64);
-        assert_eq!(fast_field.get_val(5), 1_000u64);
+        assert_eq!(fast_field.get_val(0), Some(1u64));
+        assert_eq!(fast_field.get_val(1), Some(2u64));
+        assert_eq!(fast_field.get_val(2), Some(3u64));
+        assert_eq!(fast_field.get_val(3), Some(10u64));
+        assert_eq!(fast_field.get_val(4), Some(20u64));
+        assert_eq!(fast_field.get_val(5), Some(1_000u64));
 
         let get_vals = |fast_field: &MultiValuedFastFieldReader<u64>, doc_id: u32| -> Vec<u64> {
             let mut vals = vec![];
@@ -535,11 +535,15 @@ mod bench_sorted_index_merge {
         b.iter(|| {
             let sorted_doc_ids = doc_id_mapping.iter_old_doc_addrs().map(|doc_addr| {
                 let reader = &merger.readers[doc_addr.segment_ord as usize];
-                let u64_reader: Arc<dyn Column<u64>> =
-                    reader.fast_fields().typed_fast_field_reader(field).expect(
+                let u64_reader: Arc<dyn Column<u64>> = reader
+                    .fast_fields()
+                    .typed_fast_field_reader(field)
+                    .expect(
                         "Failed to find a reader for single fast field. This is a tantivy bug and \
                          it should never happen.",
-                    );
+                    )
+                    .to_full()
+                    .unwrap();
                 (doc_addr.doc_id, reader, u64_reader)
             });
             // add values in order of the new doc_ids

src/indexer/sorted_doc_id_column.rs

@@ -46,11 +46,15 @@ impl<'a> RemappedDocIdColumn<'a> {
         let (min_value, max_value) = readers
             .iter()
             .filter_map(|reader| {
-                let u64_reader: Arc<dyn Column<u64>> =
-                    reader.fast_fields().typed_fast_field_reader(field).expect(
+                let u64_reader: Arc<dyn Column<u64>> = reader
+                    .fast_fields()
+                    .typed_fast_field_reader(field)
+                    .expect(
                         "Failed to find a reader for single fast field. This is a tantivy bug and \
                          it should never happen.",
-                    );
+                    )
+                    .to_full()
+                    .expect("temp migration solution");
                 compute_min_max_val(&*u64_reader, reader)
             })
             .reduce(|a, b| (a.0.min(b.0), a.1.max(b.1)))
@@ -59,11 +63,15 @@ impl<'a> RemappedDocIdColumn<'a> {
         let fast_field_readers = readers
             .iter()
             .map(|reader| {
-                let u64_reader: Arc<dyn Column<u64>> =
-                    reader.fast_fields().typed_fast_field_reader(field).expect(
+                let u64_reader: Arc<dyn Column<u64>> = reader
+                    .fast_fields()
+                    .typed_fast_field_reader(field)
+                    .expect(
                         "Failed to find a reader for single fast field. This is a tantivy bug and \
                          it should never happen.",
-                    );
+                    )
+                    .to_full()
+                    .expect("temp migration solution");
                 u64_reader
             })
             .collect::<Vec<_>>();

src/lib.rs

@@ -1037,21 +1037,21 @@ pub mod tests {
             let fast_field_reader_opt = segment_reader.fast_fields().u64(fast_field_unsigned);
             assert!(fast_field_reader_opt.is_ok());
             let fast_field_reader = fast_field_reader_opt.unwrap();
-            assert_eq!(fast_field_reader.get_val(0), 4u64)
+            assert_eq!(fast_field_reader.get_val(0), Some(4u64))
         }
 
         {
             let fast_field_reader_res = segment_reader.fast_fields().i64(fast_field_signed);
             assert!(fast_field_reader_res.is_ok());
             let fast_field_reader = fast_field_reader_res.unwrap();
-            assert_eq!(fast_field_reader.get_val(0), 4i64)
+            assert_eq!(fast_field_reader.get_val(0), Some(4i64))
         }
 
         {
             let fast_field_reader_res = segment_reader.fast_fields().f64(fast_field_float);
             assert!(fast_field_reader_res.is_ok());
             let fast_field_reader = fast_field_reader_res.unwrap();
-            assert_eq!(fast_field_reader.get_val(0), 4f64)
+            assert_eq!(fast_field_reader.get_val(0), Some(4f64))
         }
         Ok(())
     }

src/query/range_query_ip_fastfield.rs

@@ -7,7 +7,7 @@ use std::ops::{Bound, RangeInclusive};
 use std::sync::Arc;
 
 use common::BinarySerializable;
-use fastfield_codecs::{Column, MonotonicallyMappableToU128};
+use fastfield_codecs::{MonotonicallyMappableToU128, OptionalColumn};
 
 use super::range_query::map_bound;
 use super::{ConstScorer, Explanation, Scorer, Weight};
@@ -45,12 +45,10 @@ impl Weight for IPFastFieldRangeWeight {
         match field_type.fastfield_cardinality().unwrap() {
             Cardinality::SingleValue => {
                 let ip_addr_fast_field = reader.fast_fields().ip_addr(self.field)?;
-                let value_range = bound_to_value_range(
-                    &self.left_bound,
-                    &self.right_bound,
-                    ip_addr_fast_field.min_value(),
-                    ip_addr_fast_field.max_value(),
-                );
+                let minmax = ip_addr_fast_field
+                    .min_value()
+                    .zip(ip_addr_fast_field.max_value());
+                let value_range = bound_to_value_range(&self.left_bound, &self.right_bound, minmax);
                 let docset = IpRangeDocSet::new(
                     value_range,
                     IpFastFieldCardinality::SingleValue(ip_addr_fast_field),
@@ -62,8 +60,10 @@ impl Weight for IPFastFieldRangeWeight {
                 let value_range = bound_to_value_range(
                     &self.left_bound,
                     &self.right_bound,
-                    ip_addr_fast_field.min_value(),
-                    ip_addr_fast_field.max_value(),
+                    Some((
+                        ip_addr_fast_field.min_value(),
+                        ip_addr_fast_field.max_value(),
+                    )),
                 );
                 let docset = IpRangeDocSet::new(
                     value_range,
@@ -91,9 +91,10 @@ impl Weight for IPFastFieldRangeWeight {
 fn bound_to_value_range(
     left_bound: &Bound<Ipv6Addr>,
     right_bound: &Bound<Ipv6Addr>,
-    min_value: Ipv6Addr,
-    max_value: Ipv6Addr,
+    min_max: Option<(Ipv6Addr, Ipv6Addr)>,
 ) -> RangeInclusive<Ipv6Addr> {
+    let (min_value, max_value) =
+        min_max.unwrap_or((Ipv6Addr::from(u128::MIN), Ipv6Addr::from(u128::MAX)));
     let start_value = match left_bound {
         Bound::Included(ip_addr) => *ip_addr,
         Bound::Excluded(ip_addr) => Ipv6Addr::from(ip_addr.to_u128() + 1),
@@ -142,7 +143,7 @@ impl VecCursor {
 }
 
 pub(crate) enum IpFastFieldCardinality {
-    SingleValue(Arc<dyn Column<Ipv6Addr>>),
+    SingleValue(Arc<dyn OptionalColumn<Ipv6Addr>>),
     MultiValue(MultiValuedU128FastFieldReader<Ipv6Addr>),
 }