chore: replace bitvec impl (#214)

* chore: replace bitvec impl

* chore: reduce one copy of nullmask

* chore: move bitvec to common_base
This commit is contained in:
fys
2022-08-31 14:13:36 +08:00
committed by GitHub
parent 38d5febafe
commit ba93aa83f2
13 changed files with 46 additions and 268 deletions

View File

@@ -4,7 +4,7 @@ use std::{
};
use api::v1::{codec::InsertBatch, column::Values, Column, InsertExpr};
use common_base::bitset::BitSet;
use common_base::BitVec;
use datatypes::{data_type::ConcreteDataType, value::Value, vectors::VectorBuilder};
use snafu::{ensure, OptionExt, ResultExt};
use table::{requests::InsertRequest, Table};
@@ -50,7 +50,6 @@ pub fn insertion_expr_to_request(
))
}
};
add_values_to_builder(vector_builder, values, row_count as usize, null_mask)?;
}
}
@@ -78,10 +77,9 @@ fn add_values_to_builder(
builder: &mut VectorBuilder,
values: Values,
row_count: usize,
null_mask: impl Into<BitSet>,
null_mask: Vec<u8>,
) -> Result<()> {
let data_type = builder.data_type();
let null_mask: BitSet = null_mask.into();
let values = convert_values(&data_type, values);
if null_mask.is_empty() {
@@ -91,8 +89,9 @@ fn add_values_to_builder(
builder.push(value);
});
} else {
let null_mask = BitVec::from_vec(null_mask);
ensure!(
null_mask.ones_count() + values.len() == row_count,
null_mask.count_ones() + values.len() == row_count,
IllegalInsertDataSnafu
);
@@ -107,7 +106,6 @@ fn add_values_to_builder(
}
}
}
Ok(())
}
@@ -175,8 +173,8 @@ fn convert_values(data_type: &ConcreteDataType, values: Values) -> Vec<Value> {
}
}
fn is_null(null_mask: &BitSet, idx: usize) -> Option<bool> {
null_mask.get(idx)
fn is_null(null_mask: &BitVec, idx: usize) -> Option<bool> {
null_mask.get(idx).as_deref().copied()
}
#[cfg(test)]
@@ -188,7 +186,7 @@ mod tests {
column::{self, Values},
Column, InsertExpr,
};
use common_base::bitset::BitSet;
use common_base::BitVec;
use common_query::prelude::Expr;
use common_recordbatch::SendableRecordBatchStream;
use datatypes::{
@@ -252,7 +250,7 @@ mod tests {
#[test]
fn test_is_null() {
let null_mask: BitSet = vec![0b0000_0001, 0b0000_1000].into();
let null_mask = BitVec::from_slice(&[0b0000_0001, 0b0000_1000]);
assert_eq!(Some(true), is_null(&null_mask, 0));
assert_eq!(Some(false), is_null(&null_mask, 1));

View File

@@ -2,7 +2,7 @@ use std::sync::Arc;
use api::v1::{codec::SelectResult, column::Values, Column, ObjectResult};
use arrow::array::{Array, BooleanArray, PrimitiveArray};
use common_base::bitset::BitSet;
use common_base::BitVec;
use common_error::prelude::ErrorExt;
use common_error::status_code::StatusCode;
use common_recordbatch::{util, RecordBatch, SendableRecordBatchStream};
@@ -87,19 +87,16 @@ fn null_mask(arrays: &Vec<Arc<dyn Array>>, row_count: usize) -> Vec<u8> {
return Vec::default();
}
let mut nulls_set = BitSet::with_capacity(row_count);
let mut null_mask = BitVec::with_capacity(row_count);
for array in arrays {
let validity = array.validity();
// TODO(fys): Improve in the future, better way: repeat(false, len).
if let Some(v) = validity {
let nulls: Vec<bool> = v.iter().map(|x| !x).collect();
nulls_set.append(&nulls);
v.iter().for_each(|x| null_mask.push(!x));
} else {
nulls_set.append(&vec![false; array.len()]);
null_mask.extend_from_bitslice(&BitVec::repeat(false, array.len()));
}
}
nulls_set.buffer()
null_mask.into_vec()
}
macro_rules! convert_arrow_array_to_grpc_vals {