From 6c6c4f0dcf16798ddb7a202aafea1bfcbef3ea60 Mon Sep 17 00:00:00 2001 From: Ruihang Xia Date: Thu, 16 Jul 2026 19:22:01 +0800 Subject: [PATCH] fix: handle dictionary query edge cases Signed-off-by: Ruihang Xia --- src/common/grpc/src/flight.rs | 28 +++++----- src/datatypes/src/arrow_array.rs | 30 +++++++--- src/datatypes/src/vectors/dictionary.rs | 32 ++++++++++- src/datatypes/src/vectors/validity.rs | 13 +++++ .../src/extension_plan/series_divide.rs | 21 +++++-- src/servers/src/prom_store.rs | 55 +++++++++++++++++++ 6 files changed, 147 insertions(+), 32 deletions(-) diff --git a/src/common/grpc/src/flight.rs b/src/common/grpc/src/flight.rs index 4234e3c62e3..1245a86efb1 100644 --- a/src/common/grpc/src/flight.rs +++ b/src/common/grpc/src/flight.rs @@ -18,8 +18,8 @@ use std::collections::HashMap; use std::sync::Arc; use api::v1::{AffectedRows, FlightMetadata, Metrics}; +use arrow_flight::FlightData; use arrow_flight::utils::flight_data_to_arrow_batch; -use arrow_flight::{FlightData, SchemaAsIpc}; use common_base::bytes::Bytes; use common_recordbatch::DfRecordBatch; use datatypes::arrow; @@ -88,8 +88,14 @@ impl FlightEncoder { } /// Encode the Arrow schema to [FlightData]. - pub fn encode_schema(&self, schema: &ArrowSchema) -> FlightData { - SchemaAsIpc::new(schema, &self.write_options).into() + pub fn encode_schema(&mut self, schema: &ArrowSchema) -> FlightData { + self.data_gen + .schema_to_bytes_with_dictionary_tracker( + schema, + &mut self.dictionary_tracker, + &self.write_options, + ) + .into() } /// Encode the [FlightMessage] to a list (at least one element) of [FlightData]s. @@ -99,15 +105,7 @@ impl FlightEncoder { /// be encoded to exactly one [FlightData]. pub fn encode(&mut self, flight_message: FlightMessage) -> Vec1 { match flight_message { - FlightMessage::Schema(schema) => vec1![ - self.data_gen - .schema_to_bytes_with_dictionary_tracker( - schema.as_ref(), - &mut self.dictionary_tracker, - &self.write_options, - ) - .into(), - ], + FlightMessage::Schema(schema) => vec1![self.encode_schema(schema.as_ref())], FlightMessage::RecordBatch(record_batch) => { let (encoded_dictionaries, encoded_batch) = self .data_gen @@ -607,7 +605,7 @@ mod test { } #[test] - fn test_schema_with_nested_dictionary_array() -> Result<()> { + fn test_encode_schema_with_nested_dictionary_array() -> Result<()> { let item = Arc::new(Field::new_dictionary( "item", DataType::UInt32, @@ -632,12 +630,12 @@ mod test { let batch = DfRecordBatch::try_new(schema.clone(), vec![Arc::new(list)]).unwrap(); let mut encoder = FlightEncoder::default(); - let encoded_schema = encoder.encode(FlightMessage::Schema(schema.clone())); + let encoded_schema = encoder.encode_schema(schema.as_ref()); let encoded_batch = encoder.encode(FlightMessage::RecordBatch(batch.clone())); let mut decoder = FlightDecoder::default(); assert!(matches!( - decoder.try_decode(&encoded_schema[0])?, + decoder.try_decode(&encoded_schema)?, Some(FlightMessage::Schema(actual)) if actual == schema )); for data in encoded_batch.iter().take(encoded_batch.len() - 1) { diff --git a/src/datatypes/src/arrow_array.rs b/src/datatypes/src/arrow_array.rs index 10c4a5e6922..462ea984de3 100644 --- a/src/datatypes/src/arrow_array.rs +++ b/src/datatypes/src/arrow_array.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use arrow::array::{ArrayRef, AsArray, DictionaryArray}; +use arrow::array::{ArrayRef, AsArray}; use arrow::datatypes::{ DataType, DurationMicrosecondType, DurationMillisecondType, DurationNanosecondType, DurationSecondType, Int8Type, Int16Type, Int32Type, Int64Type, Time32MillisecondType, @@ -156,12 +156,12 @@ pub fn string_array_value_at_index(array: &ArrayRef, i: usize) -> Option<&str> { array.is_valid(i).then(|| array.value(i)) } DataType::Dictionary(key_type, value_type) - if key_type.as_ref() == &DataType::UInt32 && value_type.as_ref() == &DataType::Utf8 => + if key_type.is_integer() && value_type.is_string() => { - let array = array - .as_any() - .downcast_ref::>()?; - string_array_value_at_index(array.values(), array.key(i)?) + arrow::downcast_dictionary_array! { + array => string_array_value_at_index(array.values(), array.key(i)?), + _ => None, + } } _ => None, } @@ -265,12 +265,12 @@ mod tests { use std::sync::Arc; use arrow::array::StringDictionaryBuilder; + use arrow::datatypes::ArrowDictionaryKeyType; use super::*; - #[test] - fn reads_dictionary_encoded_strings() { - let mut builder = StringDictionaryBuilder::::new(); + fn assert_dictionary_key_type() { + let mut builder = StringDictionaryBuilder::::new(); builder.append("foo").unwrap(); builder.append("bar").unwrap(); builder.append("foo").unwrap(); @@ -282,4 +282,16 @@ mod tests { assert_eq!(Some("foo"), string_array_value_at_index(&array, 2)); assert_eq!(None, string_array_value_at_index(&array, 3)); } + + #[test] + fn reads_dictionary_encoded_strings_with_all_key_types() { + assert_dictionary_key_type::(); + assert_dictionary_key_type::(); + assert_dictionary_key_type::(); + assert_dictionary_key_type::(); + assert_dictionary_key_type::(); + assert_dictionary_key_type::(); + assert_dictionary_key_type::(); + assert_dictionary_key_type::(); + } } diff --git a/src/datatypes/src/vectors/dictionary.rs b/src/datatypes/src/vectors/dictionary.rs index 895ef8a1d52..d41daf4638f 100644 --- a/src/datatypes/src/vectors/dictionary.rs +++ b/src/datatypes/src/vectors/dictionary.rs @@ -193,7 +193,10 @@ impl Vector for DictionaryVector { } fn validity(&self) -> Validity { - vectors::impl_validity_for_vector!(self.array) + self.array + .logical_nulls() + .map(Validity::from_null_buffer) + .unwrap_or_else(|| Validity::all_valid(self.len())) } fn memory_size(&self) -> usize { @@ -201,11 +204,13 @@ impl Vector for DictionaryVector { } fn null_count(&self) -> usize { - self.array.null_count() + self.array.logical_null_count() } fn is_null(&self, row: usize) -> bool { - self.array.is_null(row) + self.array + .key(row) + .is_none_or(|key| self.item_vector.is_null(key)) } fn slice(&self, offset: usize, length: usize) -> VectorRef { @@ -448,6 +453,27 @@ mod tests { assert_eq!(dict_vec.get(4), Value::String("b".to_string().into())); } + #[test] + fn test_dictionary_vector_logical_nulls() { + let values = StringArray::from(vec![Some("a"), None]); + let keys = Int64Array::from(vec![Some(0), Some(1), None, Some(1)]); + let dict_array = DictionaryArray::new(keys, Arc::new(values)); + let dict_vec = DictionaryVector::::try_from(dict_array).unwrap(); + + assert_eq!(3, dict_vec.null_count()); + assert!(!dict_vec.is_null(0)); + assert!(dict_vec.is_null(1)); + assert!(dict_vec.is_null(2)); + assert!(dict_vec.is_null(3)); + + let validity = dict_vec.validity(); + assert_eq!(3, validity.null_count()); + assert!(validity.is_set(0)); + assert!(!validity.is_set(1)); + assert!(!validity.is_set(2)); + assert!(!validity.is_set(3)); + } + #[test] fn test_slice() { let dict_vec = create_test_dictionary(); diff --git a/src/datatypes/src/vectors/validity.rs b/src/datatypes/src/vectors/validity.rs index eaa32d8aee5..b62c1b73d5d 100644 --- a/src/datatypes/src/vectors/validity.rs +++ b/src/datatypes/src/vectors/validity.rs @@ -50,6 +50,19 @@ impl Validity { } } + /// Creates a `Validity` from a logical null buffer. + pub(crate) fn from_null_buffer(bitmap: NullBuffer) -> Validity { + let len = bitmap.len(); + let null_count = bitmap.null_count(); + Validity { + kind: ValidityKind::Slots { + bitmap, + len, + null_count, + }, + } + } + /// Returns `Validity` that all elements are valid. pub fn all_valid(len: usize) -> Validity { Validity { diff --git a/src/promql/src/extension_plan/series_divide.rs b/src/promql/src/extension_plan/series_divide.rs index 49f114bfd83..9e6a46d991a 100644 --- a/src/promql/src/extension_plan/series_divide.rs +++ b/src/promql/src/extension_plan/series_divide.rs @@ -128,9 +128,7 @@ impl<'a> RawTagColumn<'a> { fn try_new(array: &'a ArrayRef) -> DataFusionResult { match array.data_type() { DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View => Ok(Self(array)), - DataType::Dictionary(key, value) - if key.as_ref() == &DataType::UInt32 && value.as_ref() == &DataType::Utf8 => - { + DataType::Dictionary(key, value) if key.is_integer() && value.is_string() => { Ok(Self(array)) } other => Err(datafusion::error::DataFusionError::Internal(format!( @@ -626,9 +624,9 @@ impl SeriesDivideStream { #[cfg(test)] mod test { use datafusion::arrow::array::{ - DictionaryArray, LargeStringArray, StringArray, StringViewArray, UInt32Array, + DictionaryArray, Int32Array, LargeStringArray, StringArray, StringViewArray, UInt32Array, }; - use datafusion::arrow::datatypes::{DataType, Field, Schema, UInt32Type}; + use datafusion::arrow::datatypes::{DataType, Field, Int32Type, Schema, UInt32Type}; use datafusion::common::ToDFSchema; use datafusion::datasource::memory::MemorySourceConfig; use datafusion::datasource::source::DataSourceExec; @@ -650,6 +648,19 @@ mod test { assert!(!tags.equal_at(1, &tags, 2)); } + #[test] + fn test_dictionary_tag_with_non_uint32_keys() { + let dictionary: ArrayRef = Arc::new(DictionaryArray::::new( + Int32Array::from(vec![0, 1]), + Arc::new(StringArray::from(vec!["host-a", "host-b"])), + )); + + let tags = RawTagColumn::try_new(&dictionary).unwrap(); + + assert!(tags.equal_at(0, &tags, 0)); + assert!(!tags.equal_at(0, &tags, 1)); + } + fn prepare_test_data() -> DataSourceExec { let schema = Arc::new(Schema::new(vec![ Field::new("host", DataType::Utf8, true), diff --git a/src/servers/src/prom_store.rs b/src/servers/src/prom_store.rs index 487bd59812f..46e396f720c 100644 --- a/src/servers/src/prom_store.rs +++ b/src/servers/src/prom_store.rs @@ -610,6 +610,11 @@ mod tests { use api::prom_store::remote::LabelMatcher; use api::v1::{ColumnDataType, Row, SemanticType}; + use arrow::array::{ + DictionaryArray, Float64Array, StringArray, TimestampMillisecondArray, UInt32Array, + }; + use arrow::datatypes::{DataType as ArrowDataType, Field, Schema as ArrowSchema, UInt32Type}; + use common_recordbatch::DfRecordBatch; use datafusion::prelude::SessionContext; use datatypes::data_type::ConcreteDataType; use datatypes::schema::{ColumnSchema, Schema}; @@ -954,4 +959,54 @@ mod tests { }] ); } + + #[test] + fn test_recordbatches_to_timeseries_omits_dictionary_value_null_label() { + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new( + greptime_timestamp(), + ArrowDataType::Timestamp(arrow::datatypes::TimeUnit::Millisecond, None), + false, + ), + Field::new(greptime_value(), ArrowDataType::Float64, false), + Field::new_dictionary("instance", ArrowDataType::UInt32, ArrowDataType::Utf8, true), + ])); + let schema = Arc::new(Schema::try_from(arrow_schema.clone()).unwrap()); + let instance = DictionaryArray::::new( + UInt32Array::from(vec![0]), + Arc::new(StringArray::from(vec![None::<&str>])), + ); + let batch = DfRecordBatch::try_new( + arrow_schema, + vec![ + Arc::new(TimestampMillisecondArray::from(vec![1000])), + Arc::new(Float64Array::from(vec![3.0])), + Arc::new(instance), + ], + ) + .unwrap(); + let recordbatches = RecordBatches::try_new( + schema.clone(), + vec![RecordBatch::from_df_record_batch(schema, batch)], + ) + .unwrap(); + + let timeseries = recordbatches_to_timeseries("metric1", recordbatches).unwrap(); + + assert_eq!(1, timeseries.len()); + assert_eq!( + vec![Label { + name: METRIC_NAME_LABEL.to_string(), + value: "metric1".to_string(), + }], + timeseries[0].labels + ); + assert_eq!( + vec![Sample { + value: 3.0, + timestamp: 1000, + }], + timeseries[0].samples + ); + } }