rust/tantivy
1
0
Fork 0
mirror of https://github.com/quickwit-oss/tantivy.git synced 2026-01-09 02:22:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

Homogeneous codec names. (#1481)

Browse Source
This commit is contained in:
Paul Masurel
2022-08-24 22:51:37 +02:00
committed by GitHub
parent 298b5dd726
commit 3673a5df9b
9 changed files with 116 additions and 135 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
fastfield_codecs/benches/bench.rs
View File

@@ -6,11 +6,9 @@ extern crate test;
mod tests {
use fastfield_codecs::bitpacked::{BitpackedFastFieldReader, BitpackedFastFieldSerializer};
use fastfield_codecs::blockwise_linear::{
BlockwiseLinearInterpolFastFieldSerializer, MultiLinearInterpolFastFieldReader,
};
use fastfield_codecs::linear::{
LinearInterpolFastFieldReader, LinearInterpolFastFieldSerializer,
BlockwiseLinearFastFieldReader, BlockwiseLinearFastFieldSerializer,
};
use fastfield_codecs::linear::{LinearFastFieldReader, LinearFastFieldSerializer};
use fastfield_codecs::*;
fn get_data() -> Vec<u64> {
@@ -59,12 +57,12 @@ mod tests {
#[bench]
fn bench_fastfield_linearinterpol_create(b: &mut Bencher) {
let data: Vec<_> = get_data();
bench_create::<LinearInterpolFastFieldSerializer>(b, &data);
bench_create::<LinearFastFieldSerializer>(b, &data);
}
#[bench]
fn bench_fastfield_multilinearinterpol_create(b: &mut Bencher) {
let data: Vec<_> = get_data();
bench_create::<BlockwiseLinearInterpolFastFieldSerializer>(b, &data);
bench_create::<BlockwiseLinearFastFieldSerializer>(b, &data);
}
#[bench]
fn bench_fastfield_bitpack_get(b: &mut Bencher) {
@@ -74,14 +72,12 @@ mod tests {
#[bench]
fn bench_fastfield_linearinterpol_get(b: &mut Bencher) {
let data: Vec<_> = get_data();
bench_get::<LinearInterpolFastFieldSerializer, LinearInterpolFastFieldReader>(b, &data);
bench_get::<LinearFastFieldSerializer, LinearFastFieldReader>(b, &data);
}
#[bench]
fn bench_fastfield_multilinearinterpol_get(b: &mut Bencher) {
let data: Vec<_> = get_data();
bench_get::<BlockwiseLinearInterpolFastFieldSerializer, MultiLinearInterpolFastFieldReader>(
b, &data,
);
bench_get::<BlockwiseLinearFastFieldSerializer, BlockwiseLinearFastFieldReader>(b, &data);
}
pub fn stats_from_vec(data: &[u64]) -> FastFieldStats {
let min_value = data.iter().cloned().min().unwrap_or(0);

32
fastfield_codecs/src/blockwise_linear.rs
View File

@@ -1,4 +1,4 @@
//! MultiLinearInterpol compressor uses linear interpolation to guess a values and stores the
//! The BlockwiseLinear codec uses linear interpolation to guess a values and stores the
//! offset, but in blocks of 512.
//!
//! With a CHUNK_SIZE of 512 and 29 byte metadata per block, we get a overhead for metadata of 232 /
@@ -27,9 +27,9 @@ const CHUNK_SIZE: u64 = 512;
/// Depending on the field type, a different
/// fast field is required.
#[derive(Clone)]
pub struct MultiLinearInterpolFastFieldReader {
pub struct BlockwiseLinearFastFieldReader {
data: OwnedBytes,
pub footer: MultiLinearInterpolFooter,
pub footer: BlockwiseLinearFooter,
}
#[derive(Clone, Debug, Default)]
@@ -104,14 +104,14 @@ impl BinarySerializable for Function {
}
#[derive(Clone, Debug)]
pub struct MultiLinearInterpolFooter {
pub struct BlockwiseLinearFooter {
pub num_vals: u64,
pub min_value: u64,
pub max_value: u64,
interpolations: Vec<Function>,
}
impl BinarySerializable for MultiLinearInterpolFooter {
impl BinarySerializable for BlockwiseLinearFooter {
fn serialize<W: Write>(&self, write: &mut W) -> io::Result<()> {
let mut out = vec![];
self.num_vals.serialize(&mut out)?;
@@ -123,8 +123,8 @@ impl BinarySerializable for MultiLinearInterpolFooter {
Ok(())
}
fn deserialize<R: Read>(reader: &mut R) -> io::Result<MultiLinearInterpolFooter> {
let mut footer = MultiLinearInterpolFooter {
fn deserialize<R: Read>(reader: &mut R) -> io::Result<BlockwiseLinearFooter> {
let mut footer = BlockwiseLinearFooter {
num_vals: u64::deserialize(reader)?,
min_value: u64::deserialize(reader)?,
max_value: u64::deserialize(reader)?,
@@ -148,14 +148,14 @@ fn get_interpolation_function(doc: u64, interpolations: &[Function]) -> &Functio
&interpolations[get_interpolation_position(doc)]
}
impl FastFieldCodecReader for MultiLinearInterpolFastFieldReader {
impl FastFieldCodecReader for BlockwiseLinearFastFieldReader {
/// Opens a fast field given a file.
fn open_from_bytes(bytes: OwnedBytes) -> io::Result<Self> {
let footer_len: u32 = (&bytes[bytes.len() - 4..]).deserialize()?;
let footer_offset = bytes.len() - 4 - footer_len as usize;
let (data, mut footer) = bytes.split(footer_offset);
let footer = MultiLinearInterpolFooter::deserialize(&mut footer)?;
Ok(MultiLinearInterpolFastFieldReader { data, footer })
let footer = BlockwiseLinearFooter::deserialize(&mut footer)?;
Ok(BlockwiseLinearFastFieldReader { data, footer })
}
#[inline]
@@ -181,10 +181,10 @@ impl FastFieldCodecReader for MultiLinearInterpolFastFieldReader {
}
/// Same as LinearInterpolFastFieldSerializer, but working on chunks of CHUNK_SIZE elements.
pub struct BlockwiseLinearInterpolFastFieldSerializer {}
pub struct BlockwiseLinearFastFieldSerializer {}
impl FastFieldCodecSerializer for BlockwiseLinearInterpolFastFieldSerializer {
const CODEC_TYPE: FastFieldCodecType = FastFieldCodecType::BlockwiseLinearInterpol;
impl FastFieldCodecSerializer for BlockwiseLinearFastFieldSerializer {
const CODEC_TYPE: FastFieldCodecType = FastFieldCodecType::BlockwiseLinear;
/// Creates a new fast field serializer.
fn serialize(
write: &mut impl Write,
@@ -270,7 +270,7 @@ impl FastFieldCodecSerializer for BlockwiseLinearInterpolFastFieldSerializer {
}
bit_packer.close(write)?;
let footer = MultiLinearInterpolFooter {
let footer = BlockwiseLinearFooter {
num_vals: fastfield_accessor.num_vals(),
min_value: fastfield_accessor.min_value(),
max_value: fastfield_accessor.max_value(),
@@ -360,8 +360,8 @@ mod tests {
fn create_and_validate(data: &[u64], name: &str) -> (f32, f32) {
crate::tests::create_and_validate::<
BlockwiseLinearInterpolFastFieldSerializer,
MultiLinearInterpolFastFieldReader,
BlockwiseLinearFastFieldSerializer,
BlockwiseLinearFastFieldReader,
>(data, name)
}

34
fastfield_codecs/src/lib.rs
View File

@@ -24,8 +24,8 @@ pub trait FastFieldCodecReader: Sized {
#[repr(u8)]
pub enum FastFieldCodecType {
Bitpacked = 1,
LinearInterpol = 2,
BlockwiseLinearInterpol = 3,
Linear = 2,
BlockwiseLinear = 3,
Gcd = 4,
}
@@ -50,8 +50,8 @@ impl FastFieldCodecType {
pub fn from_code(code: u8) -> Option<Self> {
match code {
1 => Some(Self::Bitpacked),
2 => Some(Self::LinearInterpol),
3 => Some(Self::BlockwiseLinearInterpol),
2 => Some(Self::Linear),
3 => Some(Self::BlockwiseLinear),
4 => Some(Self::Gcd),
_ => None,
}
@@ -167,9 +167,9 @@ mod tests {
use crate::bitpacked::{BitpackedFastFieldReader, BitpackedFastFieldSerializer};
use crate::blockwise_linear::{
BlockwiseLinearInterpolFastFieldSerializer, MultiLinearInterpolFastFieldReader,
BlockwiseLinearFastFieldReader, BlockwiseLinearFastFieldSerializer,
};
use crate::linear::{LinearInterpolFastFieldReader, LinearInterpolFastFieldSerializer};
use crate::linear::{LinearFastFieldReader, LinearFastFieldSerializer};
pub fn create_and_validate<S: FastFieldCodecSerializer, R: FastFieldCodecReader>(
data: &[u64],
@@ -200,15 +200,15 @@ mod tests {
proptest! {
#[test]
fn test_proptest_small(data in proptest::collection::vec(any::<u64>(), 1..10)) {
create_and_validate::<LinearInterpolFastFieldSerializer, LinearInterpolFastFieldReader>(&data, "proptest linearinterpol");
create_and_validate::<BlockwiseLinearInterpolFastFieldSerializer, MultiLinearInterpolFastFieldReader>(&data, "proptest multilinearinterpol");
create_and_validate::<LinearFastFieldSerializer, LinearFastFieldReader>(&data, "proptest linearinterpol");
create_and_validate::<BlockwiseLinearFastFieldSerializer, BlockwiseLinearFastFieldReader>(&data, "proptest multilinearinterpol");
create_and_validate::<BitpackedFastFieldSerializer, BitpackedFastFieldReader>(&data, "proptest bitpacked");
}
#[test]
fn test_proptest_large(data in proptest::collection::vec(any::<u64>(), 1..6000)) {
create_and_validate::<LinearInterpolFastFieldSerializer, LinearInterpolFastFieldReader>(&data, "proptest linearinterpol");
create_and_validate::<BlockwiseLinearInterpolFastFieldSerializer, MultiLinearInterpolFastFieldReader>(&data, "proptest multilinearinterpol");
create_and_validate::<LinearFastFieldSerializer, LinearFastFieldReader>(&data, "proptest linearinterpol");
create_and_validate::<BlockwiseLinearFastFieldSerializer, BlockwiseLinearFastFieldReader>(&data, "proptest multilinearinterpol");
create_and_validate::<BitpackedFastFieldSerializer, BitpackedFastFieldReader>(&data, "proptest bitpacked");
}
@@ -248,12 +248,11 @@ mod tests {
}
#[test]
fn test_codec_interpolation() {
test_codec::<LinearInterpolFastFieldSerializer, LinearInterpolFastFieldReader>();
test_codec::<LinearFastFieldSerializer, LinearFastFieldReader>();
}
#[test]
fn test_codec_multi_interpolation() {
test_codec::<BlockwiseLinearInterpolFastFieldSerializer, MultiLinearInterpolFastFieldReader>(
);
test_codec::<BlockwiseLinearFastFieldSerializer, BlockwiseLinearFastFieldReader>();
}
use super::*;
@@ -262,11 +261,10 @@ mod tests {
fn estimation_good_interpolation_case() {
let data = (10..=20000_u64).collect::<Vec<_>>();
let linear_interpol_estimation = LinearInterpolFastFieldSerializer::estimate(&data);
let linear_interpol_estimation = LinearFastFieldSerializer::estimate(&data);
assert_le!(linear_interpol_estimation, 0.01);
let multi_linear_interpol_estimation =
BlockwiseLinearInterpolFastFieldSerializer::estimate(&data);
let multi_linear_interpol_estimation = BlockwiseLinearFastFieldSerializer::estimate(&data);
assert_le!(multi_linear_interpol_estimation, 0.2);
assert_le!(linear_interpol_estimation, multi_linear_interpol_estimation);
@@ -277,7 +275,7 @@ mod tests {
fn estimation_test_bad_interpolation_case() {
let data = vec![200, 10, 10, 10, 10, 1000, 20];
let linear_interpol_estimation = LinearInterpolFastFieldSerializer::estimate(&data);
let linear_interpol_estimation = LinearFastFieldSerializer::estimate(&data);
assert_le!(linear_interpol_estimation, 0.32);
let bitpacked_estimation = BitpackedFastFieldSerializer::estimate(&data);
@@ -290,7 +288,7 @@ mod tests {
// in this case the linear interpolation can't in fact not be worse than bitpacking,
// but the estimator adds some threshold, which leads to estimated worse behavior
let linear_interpol_estimation = LinearInterpolFastFieldSerializer::estimate(&data);
let linear_interpol_estimation = LinearFastFieldSerializer::estimate(&data);
assert_le!(linear_interpol_estimation, 0.35);
let bitpacked_estimation = BitpackedFastFieldSerializer::estimate(&data);

39
fastfield_codecs/src/linear.rs
View File

@@ -12,15 +12,15 @@ use crate::{
/// Depending on the field type, a different
/// fast field is required.
#[derive(Clone)]
pub struct LinearInterpolFastFieldReader {
pub struct LinearFastFieldReader {
data: OwnedBytes,
bit_unpacker: BitUnpacker,
pub footer: LinearInterpolFooter,
pub footer: LinearFooter,
pub slope: f32,
}
#[derive(Clone, Debug)]
pub struct LinearInterpolFooter {
pub struct LinearFooter {
pub relative_max_value: u64,
pub offset: u64,
pub first_val: u64,
@@ -30,7 +30,7 @@ pub struct LinearInterpolFooter {
pub max_value: u64,
}
impl BinarySerializable for LinearInterpolFooter {
impl BinarySerializable for LinearFooter {
fn serialize<W: Write>(&self, write: &mut W) -> io::Result<()> {
self.relative_max_value.serialize(write)?;
self.offset.serialize(write)?;
@@ -42,8 +42,8 @@ impl BinarySerializable for LinearInterpolFooter {
Ok(())
}
fn deserialize<R: Read>(reader: &mut R) -> io::Result<LinearInterpolFooter> {
Ok(LinearInterpolFooter {
fn deserialize<R: Read>(reader: &mut R) -> io::Result<LinearFooter> {
Ok(LinearFooter {
relative_max_value: u64::deserialize(reader)?,
offset: u64::deserialize(reader)?,
first_val: u64::deserialize(reader)?,
@@ -55,20 +55,20 @@ impl BinarySerializable for LinearInterpolFooter {
}
}
impl FixedSize for LinearInterpolFooter {
impl FixedSize for LinearFooter {
const SIZE_IN_BYTES: usize = 56;
}
impl FastFieldCodecReader for LinearInterpolFastFieldReader {
impl FastFieldCodecReader for LinearFastFieldReader {
/// Opens a fast field given a file.
fn open_from_bytes(bytes: OwnedBytes) -> io::Result<Self> {
let footer_offset = bytes.len() - LinearInterpolFooter::SIZE_IN_BYTES;
let footer_offset = bytes.len() - LinearFooter::SIZE_IN_BYTES;
let (data, mut footer) = bytes.split(footer_offset);
let footer = LinearInterpolFooter::deserialize(&mut footer)?;
let footer = LinearFooter::deserialize(&mut footer)?;
let slope = get_slope(footer.first_val, footer.last_val, footer.num_vals);
let num_bits = compute_num_bits(footer.relative_max_value);
let bit_unpacker = BitUnpacker::new(num_bits);
Ok(LinearInterpolFastFieldReader {
Ok(LinearFastFieldReader {
data,
bit_unpacker,
footer,
@@ -93,7 +93,7 @@ impl FastFieldCodecReader for LinearInterpolFastFieldReader {
/// Fastfield serializer, which tries to guess values by linear interpolation
/// and stores the difference bitpacked.
pub struct LinearInterpolFastFieldSerializer {}
pub struct LinearFastFieldSerializer {}
#[inline]
pub(crate) fn get_slope(first_val: u64, last_val: u64, num_vals: u64) -> f32 {
@@ -134,8 +134,8 @@ pub fn get_calculated_value(first_val: u64, pos: u64, slope: f32) -> u64 {
}
}
impl FastFieldCodecSerializer for LinearInterpolFastFieldSerializer {
const CODEC_TYPE: FastFieldCodecType = FastFieldCodecType::LinearInterpol;
impl FastFieldCodecSerializer for LinearFastFieldSerializer {
const CODEC_TYPE: FastFieldCodecType = FastFieldCodecType::Linear;
/// Creates a new fast field serializer.
fn serialize(
@@ -175,7 +175,7 @@ impl FastFieldCodecSerializer for LinearInterpolFastFieldSerializer {
}
bit_packer.close(write)?;
let footer = LinearInterpolFooter {
let footer = LinearFooter {
relative_max_value,
offset,
first_val,
@@ -239,7 +239,7 @@ impl FastFieldCodecSerializer for LinearInterpolFastFieldSerializer {
let num_bits = compute_num_bits(relative_max_value as u64) as u64
* fastfield_accessor.num_vals()
+ LinearInterpolFooter::SIZE_IN_BYTES as u64;
+ LinearFooter::SIZE_IN_BYTES as u64;
let num_bits_uncompressed = 64 * fastfield_accessor.num_vals();
num_bits as f32 / num_bits_uncompressed as f32
}
@@ -260,10 +260,9 @@ mod tests {
use crate::tests::get_codec_test_data_sets;
fn create_and_validate(data: &[u64], name: &str) -> (f32, f32) {
crate::tests::create_and_validate::<
LinearInterpolFastFieldSerializer,
LinearInterpolFastFieldReader,
>(data, name)
crate::tests::create_and_validate::<LinearFastFieldSerializer, LinearFastFieldReader>(
data, name,
)
}
#[test]

8
fastfield_codecs/src/main.rs
View File

@@ -1,7 +1,7 @@
#[macro_use]
extern crate prettytable;
use fastfield_codecs::blockwise_linear::BlockwiseLinearInterpolFastFieldSerializer;
use fastfield_codecs::linear::LinearInterpolFastFieldSerializer;
use fastfield_codecs::blockwise_linear::BlockwiseLinearFastFieldSerializer;
use fastfield_codecs::linear::LinearFastFieldSerializer;
use fastfield_codecs::{FastFieldCodecSerializer, FastFieldCodecType, FastFieldStats};
use prettytable::{Cell, Row, Table};
@@ -13,9 +13,9 @@ fn main() {
for (data, data_set_name) in get_codec_test_data_sets() {
let mut results = vec![];
let res = serialize_with_codec::<LinearInterpolFastFieldSerializer>(&data);
let res = serialize_with_codec::<LinearFastFieldSerializer>(&data);
results.push(res);
let res = serialize_with_codec::<BlockwiseLinearInterpolFastFieldSerializer>(&data);
let res = serialize_with_codec::<BlockwiseLinearFastFieldSerializer>(&data);
results.push(res);
let res = serialize_with_codec::<fastfield_codecs::bitpacked::BitpackedFastFieldSerializer>(
&data,

8
src/fastfield/gcd.rs
View File

@@ -237,11 +237,9 @@ mod tests {
assert!(size_prec_sec < size_prec_micro);
let size_prec_sec =
test_gcd_date_with_codec(FastFieldCodecType::LinearInterpol, DatePrecision::Seconds)?;
let size_prec_micro = test_gcd_date_with_codec(
FastFieldCodecType::LinearInterpol,
DatePrecision::Microseconds,
)?;
test_gcd_date_with_codec(FastFieldCodecType::Linear, DatePrecision::Seconds)?;
let size_prec_micro =
test_gcd_date_with_codec(FastFieldCodecType::Linear, DatePrecision::Microseconds)?;
assert!(size_prec_sec < size_prec_micro);
Ok(())

4
src/fastfield/mod.rs
View File

@@ -49,8 +49,8 @@ mod writer;
pub(crate) const ALL_CODECS: &[FastFieldCodecType; 3] = &[
FastFieldCodecType::Bitpacked,
FastFieldCodecType::LinearInterpol,
FastFieldCodecType::BlockwiseLinearInterpol,
FastFieldCodecType::Linear,
FastFieldCodecType::BlockwiseLinear,
];
/// Trait for `BytesFastFieldReader` and `MultiValuedFastFieldReader` to return the length of data

84
src/fastfield/reader.rs
View File

@@ -4,8 +4,8 @@ use std::path::Path;
use common::BinarySerializable;
use fastfield_codecs::bitpacked::BitpackedFastFieldReader as BitpackedReader;
use fastfield_codecs::blockwise_linear::MultiLinearInterpolFastFieldReader;
use fastfield_codecs::linear::LinearInterpolFastFieldReader;
use fastfield_codecs::blockwise_linear::BlockwiseLinearFastFieldReader;
use fastfield_codecs::linear::LinearFastFieldReader;
use fastfield_codecs::{FastFieldCodecReader, FastFieldCodecType};
use super::{FastValue, GCDFastFieldCodec};
@@ -63,19 +63,17 @@ pub enum DynamicFastFieldReader<Item: FastValue> {
/// Bitpacked compressed fastfield data.
Bitpacked(FastFieldReaderCodecWrapper<Item, BitpackedReader>),
/// Linear interpolated values + bitpacked
LinearInterpol(FastFieldReaderCodecWrapper<Item, LinearInterpolFastFieldReader>),
Linear(FastFieldReaderCodecWrapper<Item, LinearFastFieldReader>),
/// Blockwise linear interpolated values + bitpacked
MultiLinearInterpol(FastFieldReaderCodecWrapper<Item, MultiLinearInterpolFastFieldReader>),
BlockwiseLinear(FastFieldReaderCodecWrapper<Item, BlockwiseLinearFastFieldReader>),
/// GCD and Bitpacked compressed fastfield data.
BitpackedGCD(FastFieldReaderCodecWrapper<Item, GCDFastFieldCodec<BitpackedReader>>),
/// GCD and Linear interpolated values + bitpacked
LinearInterpolGCD(
FastFieldReaderCodecWrapper<Item, GCDFastFieldCodec<LinearInterpolFastFieldReader>>,
),
LinearGCD(FastFieldReaderCodecWrapper<Item, GCDFastFieldCodec<LinearFastFieldReader>>),
/// GCD and Blockwise linear interpolated values + bitpacked
MultiLinearInterpolGCD(
FastFieldReaderCodecWrapper<Item, GCDFastFieldCodec<MultiLinearInterpolFastFieldReader>>,
BlockwiseLinearGCD(
FastFieldReaderCodecWrapper<Item, GCDFastFieldCodec<BlockwiseLinearFastFieldReader>>,
),
}
@@ -92,19 +90,17 @@ impl<Item: FastValue> DynamicFastFieldReader<Item> {
BitpackedReader,
>::open_from_bytes(bytes)?)
}
FastFieldCodecType::LinearInterpol => {
DynamicFastFieldReader::LinearInterpol(FastFieldReaderCodecWrapper::<
FastFieldCodecType::Linear => {
DynamicFastFieldReader::Linear(FastFieldReaderCodecWrapper::<
Item,
LinearInterpolFastFieldReader,
LinearFastFieldReader,
>::open_from_bytes(bytes)?)
}
FastFieldCodecType::BlockwiseLinearInterpol => {
DynamicFastFieldReader::MultiLinearInterpol(FastFieldReaderCodecWrapper::<
FastFieldCodecType::BlockwiseLinear => {
DynamicFastFieldReader::BlockwiseLinear(FastFieldReaderCodecWrapper::<
Item,
MultiLinearInterpolFastFieldReader,
>::open_from_bytes(
bytes
)?)
BlockwiseLinearFastFieldReader,
>::open_from_bytes(bytes)?)
}
FastFieldCodecType::Gcd => {
let codec_type = FastFieldCodecType::deserialize(&mut bytes)?;
@@ -117,21 +113,21 @@ impl<Item: FastValue> DynamicFastFieldReader<Item> {
bytes
)?)
}
FastFieldCodecType::LinearInterpol => {
DynamicFastFieldReader::LinearInterpolGCD(FastFieldReaderCodecWrapper::<
FastFieldCodecType::Linear => {
DynamicFastFieldReader::LinearGCD(FastFieldReaderCodecWrapper::<
Item,
GCDFastFieldCodec<LinearInterpolFastFieldReader>,
GCDFastFieldCodec<LinearFastFieldReader>,
>::open_from_bytes(
bytes
)?)
}
FastFieldCodecType::BlockwiseLinearInterpol => {
DynamicFastFieldReader::MultiLinearInterpolGCD(
FastFieldReaderCodecWrapper::<
Item,
GCDFastFieldCodec<MultiLinearInterpolFastFieldReader>,
>::open_from_bytes(bytes)?,
)
FastFieldCodecType::BlockwiseLinear => {
DynamicFastFieldReader::BlockwiseLinearGCD(FastFieldReaderCodecWrapper::<
Item,
GCDFastFieldCodec<BlockwiseLinearFastFieldReader>,
>::open_from_bytes(
bytes
)?)
}
FastFieldCodecType::Gcd => {
return Err(DataCorruption::comment_only(
@@ -159,42 +155,42 @@ impl<Item: FastValue> FastFieldReader<Item> for DynamicFastFieldReader<Item> {
fn get(&self, doc: DocId) -> Item {
match self {
Self::Bitpacked(reader) => reader.get(doc),
Self::LinearInterpol(reader) => reader.get(doc),
Self::MultiLinearInterpol(reader) => reader.get(doc),
Self::Linear(reader) => reader.get(doc),
Self::BlockwiseLinear(reader) => reader.get(doc),
Self::BitpackedGCD(reader) => reader.get(doc),
Self::LinearInterpolGCD(reader) => reader.get(doc),
Self::MultiLinearInterpolGCD(reader) => reader.get(doc),
Self::LinearGCD(reader) => reader.get(doc),
Self::BlockwiseLinearGCD(reader) => reader.get(doc),
}
}
#[inline]
fn get_range(&self, start: u64, output: &mut [Item]) {
match self {
Self::Bitpacked(reader) => reader.get_range(start, output),
Self::LinearInterpol(reader) => reader.get_range(start, output),
Self::MultiLinearInterpol(reader) => reader.get_range(start, output),
Self::Linear(reader) => reader.get_range(start, output),
Self::BlockwiseLinear(reader) => reader.get_range(start, output),
Self::BitpackedGCD(reader) => reader.get_range(start, output),
Self::LinearInterpolGCD(reader) => reader.get_range(start, output),
Self::MultiLinearInterpolGCD(reader) => reader.get_range(start, output),
Self::LinearGCD(reader) => reader.get_range(start, output),
Self::BlockwiseLinearGCD(reader) => reader.get_range(start, output),
}
}
fn min_value(&self) -> Item {
match self {
Self::Bitpacked(reader) => reader.min_value(),
Self::LinearInterpol(reader) => reader.min_value(),
Self::MultiLinearInterpol(reader) => reader.min_value(),
Self::Linear(reader) => reader.min_value(),
Self::BlockwiseLinear(reader) => reader.min_value(),
Self::BitpackedGCD(reader) => reader.min_value(),
Self::LinearInterpolGCD(reader) => reader.min_value(),
Self::MultiLinearInterpolGCD(reader) => reader.min_value(),
Self::LinearGCD(reader) => reader.min_value(),
Self::BlockwiseLinearGCD(reader) => reader.min_value(),
}
}
fn max_value(&self) -> Item {
match self {
Self::Bitpacked(reader) => reader.max_value(),
Self::LinearInterpol(reader) => reader.max_value(),
Self::MultiLinearInterpol(reader) => reader.max_value(),
Self::Linear(reader) => reader.max_value(),
Self::BlockwiseLinear(reader) => reader.max_value(),
Self::BitpackedGCD(reader) => reader.max_value(),
Self::LinearInterpolGCD(reader) => reader.max_value(),
Self::MultiLinearInterpolGCD(reader) => reader.max_value(),
Self::LinearGCD(reader) => reader.max_value(),
Self::BlockwiseLinearGCD(reader) => reader.max_value(),
}
}
}

26
src/fastfield/serializer/mod.rs
View File

@@ -6,8 +6,8 @@ use fastdivide::DividerU64;
pub use fastfield_codecs::bitpacked::{
BitpackedFastFieldSerializer, BitpackedFastFieldSerializerLegacy,
};
use fastfield_codecs::blockwise_linear::BlockwiseLinearInterpolFastFieldSerializer;
use fastfield_codecs::linear::LinearInterpolFastFieldSerializer;
use fastfield_codecs::blockwise_linear::BlockwiseLinearFastFieldSerializer;
use fastfield_codecs::linear::LinearFastFieldSerializer;
use fastfield_codecs::FastFieldCodecType;
pub use fastfield_codecs::{FastFieldCodecSerializer, FastFieldDataAccess, FastFieldStats};
@@ -211,14 +211,11 @@ impl CompositeFastFieldSerializer {
&mut estimations,
);
}
if codec_enable_checker.is_enabled(FastFieldCodecType::LinearInterpol) {
codec_estimation::<LinearInterpolFastFieldSerializer, _>(
&fastfield_accessor,
&mut estimations,
);
if codec_enable_checker.is_enabled(FastFieldCodecType::Linear) {
codec_estimation::<LinearFastFieldSerializer, _>(&fastfield_accessor, &mut estimations);
}
if codec_enable_checker.is_enabled(FastFieldCodecType::BlockwiseLinearInterpol) {
codec_estimation::<BlockwiseLinearInterpolFastFieldSerializer, _>(
if codec_enable_checker.is_enabled(FastFieldCodecType::BlockwiseLinear) {
codec_estimation::<BlockwiseLinearFastFieldSerializer, _>(
&fastfield_accessor,
&mut estimations,
);
@@ -242,14 +239,11 @@ impl CompositeFastFieldSerializer {
FastFieldCodecType::Bitpacked => {
BitpackedFastFieldSerializer::serialize(field_write, &fastfield_accessor)?;
}
FastFieldCodecType::LinearInterpol => {
LinearInterpolFastFieldSerializer::serialize(field_write, &fastfield_accessor)?;
FastFieldCodecType::Linear => {
LinearFastFieldSerializer::serialize(field_write, &fastfield_accessor)?;
}
FastFieldCodecType::BlockwiseLinearInterpol => {
BlockwiseLinearInterpolFastFieldSerializer::serialize(
field_write,
&fastfield_accessor,
)?;
FastFieldCodecType::BlockwiseLinear => {
BlockwiseLinearFastFieldSerializer::serialize(field_write, &fastfield_accessor)?;
}
FastFieldCodecType::Gcd => {
return Err(io::Error::new(