mirror of
https://github.com/GreptimeTeam/greptimedb.git
synced 2026-11-17 08:14:56 +00:00
fix: handle dictionary query edge cases
Signed-off-by: Ruihang Xia <waynestxia@gmail.com>
This commit is contained in:
@@ -18,8 +18,8 @@ use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
use api::v1::{AffectedRows, FlightMetadata, Metrics};
|
||||
use arrow_flight::FlightData;
|
||||
use arrow_flight::utils::flight_data_to_arrow_batch;
|
||||
use arrow_flight::{FlightData, SchemaAsIpc};
|
||||
use common_base::bytes::Bytes;
|
||||
use common_recordbatch::DfRecordBatch;
|
||||
use datatypes::arrow;
|
||||
@@ -88,8 +88,14 @@ impl FlightEncoder {
|
||||
}
|
||||
|
||||
/// Encode the Arrow schema to [FlightData].
|
||||
pub fn encode_schema(&self, schema: &ArrowSchema) -> FlightData {
|
||||
SchemaAsIpc::new(schema, &self.write_options).into()
|
||||
pub fn encode_schema(&mut self, schema: &ArrowSchema) -> FlightData {
|
||||
self.data_gen
|
||||
.schema_to_bytes_with_dictionary_tracker(
|
||||
schema,
|
||||
&mut self.dictionary_tracker,
|
||||
&self.write_options,
|
||||
)
|
||||
.into()
|
||||
}
|
||||
|
||||
/// Encode the [FlightMessage] to a list (at least one element) of [FlightData]s.
|
||||
@@ -99,15 +105,7 @@ impl FlightEncoder {
|
||||
/// be encoded to exactly one [FlightData].
|
||||
pub fn encode(&mut self, flight_message: FlightMessage) -> Vec1<FlightData> {
|
||||
match flight_message {
|
||||
FlightMessage::Schema(schema) => vec1![
|
||||
self.data_gen
|
||||
.schema_to_bytes_with_dictionary_tracker(
|
||||
schema.as_ref(),
|
||||
&mut self.dictionary_tracker,
|
||||
&self.write_options,
|
||||
)
|
||||
.into(),
|
||||
],
|
||||
FlightMessage::Schema(schema) => vec1![self.encode_schema(schema.as_ref())],
|
||||
FlightMessage::RecordBatch(record_batch) => {
|
||||
let (encoded_dictionaries, encoded_batch) = self
|
||||
.data_gen
|
||||
@@ -607,7 +605,7 @@ mod test {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_schema_with_nested_dictionary_array() -> Result<()> {
|
||||
fn test_encode_schema_with_nested_dictionary_array() -> Result<()> {
|
||||
let item = Arc::new(Field::new_dictionary(
|
||||
"item",
|
||||
DataType::UInt32,
|
||||
@@ -632,12 +630,12 @@ mod test {
|
||||
let batch = DfRecordBatch::try_new(schema.clone(), vec![Arc::new(list)]).unwrap();
|
||||
|
||||
let mut encoder = FlightEncoder::default();
|
||||
let encoded_schema = encoder.encode(FlightMessage::Schema(schema.clone()));
|
||||
let encoded_schema = encoder.encode_schema(schema.as_ref());
|
||||
let encoded_batch = encoder.encode(FlightMessage::RecordBatch(batch.clone()));
|
||||
|
||||
let mut decoder = FlightDecoder::default();
|
||||
assert!(matches!(
|
||||
decoder.try_decode(&encoded_schema[0])?,
|
||||
decoder.try_decode(&encoded_schema)?,
|
||||
Some(FlightMessage::Schema(actual)) if actual == schema
|
||||
));
|
||||
for data in encoded_batch.iter().take(encoded_batch.len() - 1) {
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use arrow::array::{ArrayRef, AsArray, DictionaryArray};
|
||||
use arrow::array::{ArrayRef, AsArray};
|
||||
use arrow::datatypes::{
|
||||
DataType, DurationMicrosecondType, DurationMillisecondType, DurationNanosecondType,
|
||||
DurationSecondType, Int8Type, Int16Type, Int32Type, Int64Type, Time32MillisecondType,
|
||||
@@ -156,12 +156,12 @@ pub fn string_array_value_at_index(array: &ArrayRef, i: usize) -> Option<&str> {
|
||||
array.is_valid(i).then(|| array.value(i))
|
||||
}
|
||||
DataType::Dictionary(key_type, value_type)
|
||||
if key_type.as_ref() == &DataType::UInt32 && value_type.as_ref() == &DataType::Utf8 =>
|
||||
if key_type.is_integer() && value_type.is_string() =>
|
||||
{
|
||||
let array = array
|
||||
.as_any()
|
||||
.downcast_ref::<DictionaryArray<UInt32Type>>()?;
|
||||
string_array_value_at_index(array.values(), array.key(i)?)
|
||||
arrow::downcast_dictionary_array! {
|
||||
array => string_array_value_at_index(array.values(), array.key(i)?),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
@@ -265,12 +265,12 @@ mod tests {
|
||||
use std::sync::Arc;
|
||||
|
||||
use arrow::array::StringDictionaryBuilder;
|
||||
use arrow::datatypes::ArrowDictionaryKeyType;
|
||||
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn reads_dictionary_encoded_strings() {
|
||||
let mut builder = StringDictionaryBuilder::<UInt32Type>::new();
|
||||
fn assert_dictionary_key_type<K: ArrowDictionaryKeyType>() {
|
||||
let mut builder = StringDictionaryBuilder::<K>::new();
|
||||
builder.append("foo").unwrap();
|
||||
builder.append("bar").unwrap();
|
||||
builder.append("foo").unwrap();
|
||||
@@ -282,4 +282,16 @@ mod tests {
|
||||
assert_eq!(Some("foo"), string_array_value_at_index(&array, 2));
|
||||
assert_eq!(None, string_array_value_at_index(&array, 3));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reads_dictionary_encoded_strings_with_all_key_types() {
|
||||
assert_dictionary_key_type::<Int8Type>();
|
||||
assert_dictionary_key_type::<Int16Type>();
|
||||
assert_dictionary_key_type::<Int32Type>();
|
||||
assert_dictionary_key_type::<Int64Type>();
|
||||
assert_dictionary_key_type::<UInt8Type>();
|
||||
assert_dictionary_key_type::<UInt16Type>();
|
||||
assert_dictionary_key_type::<UInt32Type>();
|
||||
assert_dictionary_key_type::<UInt64Type>();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -193,7 +193,10 @@ impl<K: ArrowDictionaryKeyType> Vector for DictionaryVector<K> {
|
||||
}
|
||||
|
||||
fn validity(&self) -> Validity {
|
||||
vectors::impl_validity_for_vector!(self.array)
|
||||
self.array
|
||||
.logical_nulls()
|
||||
.map(Validity::from_null_buffer)
|
||||
.unwrap_or_else(|| Validity::all_valid(self.len()))
|
||||
}
|
||||
|
||||
fn memory_size(&self) -> usize {
|
||||
@@ -201,11 +204,13 @@ impl<K: ArrowDictionaryKeyType> Vector for DictionaryVector<K> {
|
||||
}
|
||||
|
||||
fn null_count(&self) -> usize {
|
||||
self.array.null_count()
|
||||
self.array.logical_null_count()
|
||||
}
|
||||
|
||||
fn is_null(&self, row: usize) -> bool {
|
||||
self.array.is_null(row)
|
||||
self.array
|
||||
.key(row)
|
||||
.is_none_or(|key| self.item_vector.is_null(key))
|
||||
}
|
||||
|
||||
fn slice(&self, offset: usize, length: usize) -> VectorRef {
|
||||
@@ -448,6 +453,27 @@ mod tests {
|
||||
assert_eq!(dict_vec.get(4), Value::String("b".to_string().into()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_dictionary_vector_logical_nulls() {
|
||||
let values = StringArray::from(vec![Some("a"), None]);
|
||||
let keys = Int64Array::from(vec![Some(0), Some(1), None, Some(1)]);
|
||||
let dict_array = DictionaryArray::new(keys, Arc::new(values));
|
||||
let dict_vec = DictionaryVector::<Int64Type>::try_from(dict_array).unwrap();
|
||||
|
||||
assert_eq!(3, dict_vec.null_count());
|
||||
assert!(!dict_vec.is_null(0));
|
||||
assert!(dict_vec.is_null(1));
|
||||
assert!(dict_vec.is_null(2));
|
||||
assert!(dict_vec.is_null(3));
|
||||
|
||||
let validity = dict_vec.validity();
|
||||
assert_eq!(3, validity.null_count());
|
||||
assert!(validity.is_set(0));
|
||||
assert!(!validity.is_set(1));
|
||||
assert!(!validity.is_set(2));
|
||||
assert!(!validity.is_set(3));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_slice() {
|
||||
let dict_vec = create_test_dictionary();
|
||||
|
||||
@@ -50,6 +50,19 @@ impl Validity {
|
||||
}
|
||||
}
|
||||
|
||||
/// Creates a `Validity` from a logical null buffer.
|
||||
pub(crate) fn from_null_buffer(bitmap: NullBuffer) -> Validity {
|
||||
let len = bitmap.len();
|
||||
let null_count = bitmap.null_count();
|
||||
Validity {
|
||||
kind: ValidityKind::Slots {
|
||||
bitmap,
|
||||
len,
|
||||
null_count,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns `Validity` that all elements are valid.
|
||||
pub fn all_valid(len: usize) -> Validity {
|
||||
Validity {
|
||||
|
||||
@@ -128,9 +128,7 @@ impl<'a> RawTagColumn<'a> {
|
||||
fn try_new(array: &'a ArrayRef) -> DataFusionResult<Self> {
|
||||
match array.data_type() {
|
||||
DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View => Ok(Self(array)),
|
||||
DataType::Dictionary(key, value)
|
||||
if key.as_ref() == &DataType::UInt32 && value.as_ref() == &DataType::Utf8 =>
|
||||
{
|
||||
DataType::Dictionary(key, value) if key.is_integer() && value.is_string() => {
|
||||
Ok(Self(array))
|
||||
}
|
||||
other => Err(datafusion::error::DataFusionError::Internal(format!(
|
||||
@@ -626,9 +624,9 @@ impl SeriesDivideStream {
|
||||
#[cfg(test)]
|
||||
mod test {
|
||||
use datafusion::arrow::array::{
|
||||
DictionaryArray, LargeStringArray, StringArray, StringViewArray, UInt32Array,
|
||||
DictionaryArray, Int32Array, LargeStringArray, StringArray, StringViewArray, UInt32Array,
|
||||
};
|
||||
use datafusion::arrow::datatypes::{DataType, Field, Schema, UInt32Type};
|
||||
use datafusion::arrow::datatypes::{DataType, Field, Int32Type, Schema, UInt32Type};
|
||||
use datafusion::common::ToDFSchema;
|
||||
use datafusion::datasource::memory::MemorySourceConfig;
|
||||
use datafusion::datasource::source::DataSourceExec;
|
||||
@@ -650,6 +648,19 @@ mod test {
|
||||
assert!(!tags.equal_at(1, &tags, 2));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_dictionary_tag_with_non_uint32_keys() {
|
||||
let dictionary: ArrayRef = Arc::new(DictionaryArray::<Int32Type>::new(
|
||||
Int32Array::from(vec![0, 1]),
|
||||
Arc::new(StringArray::from(vec!["host-a", "host-b"])),
|
||||
));
|
||||
|
||||
let tags = RawTagColumn::try_new(&dictionary).unwrap();
|
||||
|
||||
assert!(tags.equal_at(0, &tags, 0));
|
||||
assert!(!tags.equal_at(0, &tags, 1));
|
||||
}
|
||||
|
||||
fn prepare_test_data() -> DataSourceExec {
|
||||
let schema = Arc::new(Schema::new(vec![
|
||||
Field::new("host", DataType::Utf8, true),
|
||||
|
||||
@@ -610,6 +610,11 @@ mod tests {
|
||||
|
||||
use api::prom_store::remote::LabelMatcher;
|
||||
use api::v1::{ColumnDataType, Row, SemanticType};
|
||||
use arrow::array::{
|
||||
DictionaryArray, Float64Array, StringArray, TimestampMillisecondArray, UInt32Array,
|
||||
};
|
||||
use arrow::datatypes::{DataType as ArrowDataType, Field, Schema as ArrowSchema, UInt32Type};
|
||||
use common_recordbatch::DfRecordBatch;
|
||||
use datafusion::prelude::SessionContext;
|
||||
use datatypes::data_type::ConcreteDataType;
|
||||
use datatypes::schema::{ColumnSchema, Schema};
|
||||
@@ -954,4 +959,54 @@ mod tests {
|
||||
}]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_recordbatches_to_timeseries_omits_dictionary_value_null_label() {
|
||||
let arrow_schema = Arc::new(ArrowSchema::new(vec![
|
||||
Field::new(
|
||||
greptime_timestamp(),
|
||||
ArrowDataType::Timestamp(arrow::datatypes::TimeUnit::Millisecond, None),
|
||||
false,
|
||||
),
|
||||
Field::new(greptime_value(), ArrowDataType::Float64, false),
|
||||
Field::new_dictionary("instance", ArrowDataType::UInt32, ArrowDataType::Utf8, true),
|
||||
]));
|
||||
let schema = Arc::new(Schema::try_from(arrow_schema.clone()).unwrap());
|
||||
let instance = DictionaryArray::<UInt32Type>::new(
|
||||
UInt32Array::from(vec![0]),
|
||||
Arc::new(StringArray::from(vec![None::<&str>])),
|
||||
);
|
||||
let batch = DfRecordBatch::try_new(
|
||||
arrow_schema,
|
||||
vec![
|
||||
Arc::new(TimestampMillisecondArray::from(vec![1000])),
|
||||
Arc::new(Float64Array::from(vec![3.0])),
|
||||
Arc::new(instance),
|
||||
],
|
||||
)
|
||||
.unwrap();
|
||||
let recordbatches = RecordBatches::try_new(
|
||||
schema.clone(),
|
||||
vec![RecordBatch::from_df_record_batch(schema, batch)],
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let timeseries = recordbatches_to_timeseries("metric1", recordbatches).unwrap();
|
||||
|
||||
assert_eq!(1, timeseries.len());
|
||||
assert_eq!(
|
||||
vec![Label {
|
||||
name: METRIC_NAME_LABEL.to_string(),
|
||||
value: "metric1".to_string(),
|
||||
}],
|
||||
timeseries[0].labels
|
||||
);
|
||||
assert_eq!(
|
||||
vec![Sample {
|
||||
value: 3.0,
|
||||
timestamp: 1000,
|
||||
}],
|
||||
timeseries[0].samples
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user