From 11d3409286232a20d04cee810c4ab941e8efc5c3 Mon Sep 17 00:00:00 2001 From: PSeitz Date: Tue, 11 Oct 2022 17:54:24 +0800 Subject: [PATCH] add missing docs for fastfield_codecs crate (#1613) closes #1603 --- fastfield_codecs/src/column.rs | 3 +++ fastfield_codecs/src/lib.rs | 19 +++++++++++++++++-- fastfield_codecs/src/monotonic_mapping.rs | 2 ++ .../src/monotonic_mapping_u128.rs | 2 ++ fastfield_codecs/src/serialize.rs | 7 +++++++ 5 files changed, 31 insertions(+), 2 deletions(-) diff --git a/fastfield_codecs/src/column.rs b/fastfield_codecs/src/column.rs index afbb7785f..96621996e 100644 --- a/fastfield_codecs/src/column.rs +++ b/fastfield_codecs/src/column.rs @@ -5,6 +5,7 @@ use tantivy_bitpacker::minmax; use crate::monotonic_mapping::StrictlyMonotonicFn; +/// `Column` provides columnar access on a field. pub trait Column: Send + Sync { /// Return the value associated with the given idx. /// @@ -59,6 +60,7 @@ pub trait Column: Send + Sync { /// `.max_value()`. fn max_value(&self) -> T; + /// The number of values in the column. fn num_vals(&self) -> u64; /// Returns a iterator over the data @@ -67,6 +69,7 @@ pub trait Column: Send + Sync { } } +/// VecColumn provides `Column` over a slice. pub struct VecColumn<'a, T = u64> { values: &'a [T], min_value: T, diff --git a/fastfield_codecs/src/lib.rs b/fastfield_codecs/src/lib.rs index 07a86cc76..91b6ff534 100644 --- a/fastfield_codecs/src/lib.rs +++ b/fastfield_codecs/src/lib.rs @@ -1,5 +1,12 @@ +#![warn(missing_docs)] #![cfg_attr(all(feature = "unstable", test), feature(test))] +//! # `fastfield_codecs` +//! +//! - Columnar storage of data for tantivy [`Column`]. +//! - Encode data in different codecs. +//! - Monotonically map values to u64/u128 + #[cfg(test)] #[macro_use] extern crate more_asserts; @@ -44,9 +51,16 @@ pub use self::serialize::{ #[derive(PartialEq, Eq, PartialOrd, Ord, Debug, Clone, Copy)] #[repr(u8)] +/// Available codecs to use to encode the u64 (via [`MonotonicallyMappableToU64`]) converted data. pub enum FastFieldCodecType { + /// Bitpack all values in the value range. The number of bits is defined by the amplitude + /// column.max_value()-column.min_value() Bitpacked = 1, + /// Linear interpolation puts a line between the first and last value and then bitpacks the + /// values by the offset from the line. The number of bits is defined by the max deviation from + /// the line. Linear = 2, + /// Same as [`FastFieldCodecType::Linear`], but encodes in blocks of 512 elements. BlockwiseLinear = 3, } @@ -64,11 +78,11 @@ impl BinarySerializable for FastFieldCodecType { } impl FastFieldCodecType { - pub fn to_code(self) -> u8 { + pub(crate) fn to_code(self) -> u8 { self as u8 } - pub fn from_code(code: u8) -> Option { + pub(crate) fn from_code(code: u8) -> Option { match code { 1 => Some(Self::Bitpacked), 2 => Some(Self::Linear), @@ -150,6 +164,7 @@ trait FastFieldCodec: 'static { fn estimate(column: &dyn Column) -> Option; } +/// The list of all available codecs for u64 convertible data. pub const ALL_CODEC_TYPES: [FastFieldCodecType; 3] = [ FastFieldCodecType::Bitpacked, FastFieldCodecType::BlockwiseLinear, diff --git a/fastfield_codecs/src/monotonic_mapping.rs b/fastfield_codecs/src/monotonic_mapping.rs index ebd34e2fa..d43ace93f 100644 --- a/fastfield_codecs/src/monotonic_mapping.rs +++ b/fastfield_codecs/src/monotonic_mapping.rs @@ -4,6 +4,8 @@ use fastdivide::DividerU64; use crate::MonotonicallyMappableToU128; +/// Monotonic maps a value to u64 value space. +/// Monotonic mapping enables `PartialOrd` on u64 space without conversion to original space. pub trait MonotonicallyMappableToU64: 'static + PartialOrd + Copy + Send + Sync { /// Converts a value to u64. /// diff --git a/fastfield_codecs/src/monotonic_mapping_u128.rs b/fastfield_codecs/src/monotonic_mapping_u128.rs index 979d6c8c3..83758d2b1 100644 --- a/fastfield_codecs/src/monotonic_mapping_u128.rs +++ b/fastfield_codecs/src/monotonic_mapping_u128.rs @@ -1,5 +1,7 @@ use std::net::Ipv6Addr; +/// Montonic maps a value to u128 value space +/// Monotonic mapping enables `PartialOrd` on u128 space without conversion to original space. pub trait MonotonicallyMappableToU128: 'static + PartialOrd + Copy + Send + Sync { /// Converts a value to u128. /// diff --git a/fastfield_codecs/src/serialize.rs b/fastfield_codecs/src/serialize.rs index c916c758e..5a6c790d8 100644 --- a/fastfield_codecs/src/serialize.rs +++ b/fastfield_codecs/src/serialize.rs @@ -45,7 +45,9 @@ use crate::{ /// By design, after normalization, `min_value = 0` and `gcd = 1`. #[derive(Debug, Copy, Clone)] pub struct NormalizedHeader { + /// The number of values in the underlying column. pub num_vals: u64, + /// The max value of the underlying column. pub max_value: u64, } @@ -137,6 +139,8 @@ impl BinarySerializable for Header { } } +/// Return estimated compression for given codec in the value range [0.0..1.0], where 1.0 means no +/// compression. pub fn estimate( typed_column: impl Column, codec_type: FastFieldCodecType, @@ -157,6 +161,7 @@ pub fn estimate( } } +/// Serializes u128 values with the compact space codec. pub fn serialize_u128 I, I: Iterator>( iter_gen: F, num_vals: u64, @@ -169,6 +174,7 @@ pub fn serialize_u128 I, I: Iterator>( Ok(()) } +/// Serializes the column with the codec with the best estimate on the data. pub fn serialize( typed_column: impl Column, output: &mut impl io::Write, @@ -239,6 +245,7 @@ fn serialize_given_codec( Ok(()) } +/// Helper function to serialize a column (autodetect from all codecs) and then open it pub fn serialize_and_load( column: &[T], ) -> Arc> {