fix: handle dictionary query edge cases

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>
This commit is contained in:
Ruihang Xia
2026-07-16 19:22:01 +08:00
parent f57530ccdb
commit 6c6c4f0dcf
6 changed files with 147 additions and 32 deletions
+13 -15
View File
@@ -18,8 +18,8 @@ use std::collections::HashMap;
use std::sync::Arc;
use api::v1::{AffectedRows, FlightMetadata, Metrics};
use arrow_flight::FlightData;
use arrow_flight::utils::flight_data_to_arrow_batch;
use arrow_flight::{FlightData, SchemaAsIpc};
use common_base::bytes::Bytes;
use common_recordbatch::DfRecordBatch;
use datatypes::arrow;
@@ -88,8 +88,14 @@ impl FlightEncoder {
}
/// Encode the Arrow schema to [FlightData].
pub fn encode_schema(&self, schema: &ArrowSchema) -> FlightData {
SchemaAsIpc::new(schema, &self.write_options).into()
pub fn encode_schema(&mut self, schema: &ArrowSchema) -> FlightData {
self.data_gen
.schema_to_bytes_with_dictionary_tracker(
schema,
&mut self.dictionary_tracker,
&self.write_options,
)
.into()
}
/// Encode the [FlightMessage] to a list (at least one element) of [FlightData]s.
@@ -99,15 +105,7 @@ impl FlightEncoder {
/// be encoded to exactly one [FlightData].
pub fn encode(&mut self, flight_message: FlightMessage) -> Vec1<FlightData> {
match flight_message {
FlightMessage::Schema(schema) => vec1![
self.data_gen
.schema_to_bytes_with_dictionary_tracker(
schema.as_ref(),
&mut self.dictionary_tracker,
&self.write_options,
)
.into(),
],
FlightMessage::Schema(schema) => vec1![self.encode_schema(schema.as_ref())],
FlightMessage::RecordBatch(record_batch) => {
let (encoded_dictionaries, encoded_batch) = self
.data_gen
@@ -607,7 +605,7 @@ mod test {
}
#[test]
fn test_schema_with_nested_dictionary_array() -> Result<()> {
fn test_encode_schema_with_nested_dictionary_array() -> Result<()> {
let item = Arc::new(Field::new_dictionary(
"item",
DataType::UInt32,
@@ -632,12 +630,12 @@ mod test {
let batch = DfRecordBatch::try_new(schema.clone(), vec![Arc::new(list)]).unwrap();
let mut encoder = FlightEncoder::default();
let encoded_schema = encoder.encode(FlightMessage::Schema(schema.clone()));
let encoded_schema = encoder.encode_schema(schema.as_ref());
let encoded_batch = encoder.encode(FlightMessage::RecordBatch(batch.clone()));
let mut decoder = FlightDecoder::default();
assert!(matches!(
decoder.try_decode(&encoded_schema[0])?,
decoder.try_decode(&encoded_schema)?,
Some(FlightMessage::Schema(actual)) if actual == schema
));
for data in encoded_batch.iter().take(encoded_batch.len() - 1) {
+21 -9
View File
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use arrow::array::{ArrayRef, AsArray, DictionaryArray};
use arrow::array::{ArrayRef, AsArray};
use arrow::datatypes::{
DataType, DurationMicrosecondType, DurationMillisecondType, DurationNanosecondType,
DurationSecondType, Int8Type, Int16Type, Int32Type, Int64Type, Time32MillisecondType,
@@ -156,12 +156,12 @@ pub fn string_array_value_at_index(array: &ArrayRef, i: usize) -> Option<&str> {
array.is_valid(i).then(|| array.value(i))
}
DataType::Dictionary(key_type, value_type)
if key_type.as_ref() == &DataType::UInt32 && value_type.as_ref() == &DataType::Utf8 =>
if key_type.is_integer() && value_type.is_string() =>
{
let array = array
.as_any()
.downcast_ref::<DictionaryArray<UInt32Type>>()?;
string_array_value_at_index(array.values(), array.key(i)?)
arrow::downcast_dictionary_array! {
array => string_array_value_at_index(array.values(), array.key(i)?),
_ => None,
}
}
_ => None,
}
@@ -265,12 +265,12 @@ mod tests {
use std::sync::Arc;
use arrow::array::StringDictionaryBuilder;
use arrow::datatypes::ArrowDictionaryKeyType;
use super::*;
#[test]
fn reads_dictionary_encoded_strings() {
let mut builder = StringDictionaryBuilder::<UInt32Type>::new();
fn assert_dictionary_key_type<K: ArrowDictionaryKeyType>() {
let mut builder = StringDictionaryBuilder::<K>::new();
builder.append("foo").unwrap();
builder.append("bar").unwrap();
builder.append("foo").unwrap();
@@ -282,4 +282,16 @@ mod tests {
assert_eq!(Some("foo"), string_array_value_at_index(&array, 2));
assert_eq!(None, string_array_value_at_index(&array, 3));
}
#[test]
fn reads_dictionary_encoded_strings_with_all_key_types() {
assert_dictionary_key_type::<Int8Type>();
assert_dictionary_key_type::<Int16Type>();
assert_dictionary_key_type::<Int32Type>();
assert_dictionary_key_type::<Int64Type>();
assert_dictionary_key_type::<UInt8Type>();
assert_dictionary_key_type::<UInt16Type>();
assert_dictionary_key_type::<UInt32Type>();
assert_dictionary_key_type::<UInt64Type>();
}
}
+29 -3
View File
@@ -193,7 +193,10 @@ impl<K: ArrowDictionaryKeyType> Vector for DictionaryVector<K> {
}
fn validity(&self) -> Validity {
vectors::impl_validity_for_vector!(self.array)
self.array
.logical_nulls()
.map(Validity::from_null_buffer)
.unwrap_or_else(|| Validity::all_valid(self.len()))
}
fn memory_size(&self) -> usize {
@@ -201,11 +204,13 @@ impl<K: ArrowDictionaryKeyType> Vector for DictionaryVector<K> {
}
fn null_count(&self) -> usize {
self.array.null_count()
self.array.logical_null_count()
}
fn is_null(&self, row: usize) -> bool {
self.array.is_null(row)
self.array
.key(row)
.is_none_or(|key| self.item_vector.is_null(key))
}
fn slice(&self, offset: usize, length: usize) -> VectorRef {
@@ -448,6 +453,27 @@ mod tests {
assert_eq!(dict_vec.get(4), Value::String("b".to_string().into()));
}
#[test]
fn test_dictionary_vector_logical_nulls() {
let values = StringArray::from(vec![Some("a"), None]);
let keys = Int64Array::from(vec![Some(0), Some(1), None, Some(1)]);
let dict_array = DictionaryArray::new(keys, Arc::new(values));
let dict_vec = DictionaryVector::<Int64Type>::try_from(dict_array).unwrap();
assert_eq!(3, dict_vec.null_count());
assert!(!dict_vec.is_null(0));
assert!(dict_vec.is_null(1));
assert!(dict_vec.is_null(2));
assert!(dict_vec.is_null(3));
let validity = dict_vec.validity();
assert_eq!(3, validity.null_count());
assert!(validity.is_set(0));
assert!(!validity.is_set(1));
assert!(!validity.is_set(2));
assert!(!validity.is_set(3));
}
#[test]
fn test_slice() {
let dict_vec = create_test_dictionary();
+13
View File
@@ -50,6 +50,19 @@ impl Validity {
}
}
/// Creates a `Validity` from a logical null buffer.
pub(crate) fn from_null_buffer(bitmap: NullBuffer) -> Validity {
let len = bitmap.len();
let null_count = bitmap.null_count();
Validity {
kind: ValidityKind::Slots {
bitmap,
len,
null_count,
},
}
}
/// Returns `Validity` that all elements are valid.
pub fn all_valid(len: usize) -> Validity {
Validity {
+16 -5
View File
@@ -128,9 +128,7 @@ impl<'a> RawTagColumn<'a> {
fn try_new(array: &'a ArrayRef) -> DataFusionResult<Self> {
match array.data_type() {
DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View => Ok(Self(array)),
DataType::Dictionary(key, value)
if key.as_ref() == &DataType::UInt32 && value.as_ref() == &DataType::Utf8 =>
{
DataType::Dictionary(key, value) if key.is_integer() && value.is_string() => {
Ok(Self(array))
}
other => Err(datafusion::error::DataFusionError::Internal(format!(
@@ -626,9 +624,9 @@ impl SeriesDivideStream {
#[cfg(test)]
mod test {
use datafusion::arrow::array::{
DictionaryArray, LargeStringArray, StringArray, StringViewArray, UInt32Array,
DictionaryArray, Int32Array, LargeStringArray, StringArray, StringViewArray, UInt32Array,
};
use datafusion::arrow::datatypes::{DataType, Field, Schema, UInt32Type};
use datafusion::arrow::datatypes::{DataType, Field, Int32Type, Schema, UInt32Type};
use datafusion::common::ToDFSchema;
use datafusion::datasource::memory::MemorySourceConfig;
use datafusion::datasource::source::DataSourceExec;
@@ -650,6 +648,19 @@ mod test {
assert!(!tags.equal_at(1, &tags, 2));
}
#[test]
fn test_dictionary_tag_with_non_uint32_keys() {
let dictionary: ArrayRef = Arc::new(DictionaryArray::<Int32Type>::new(
Int32Array::from(vec![0, 1]),
Arc::new(StringArray::from(vec!["host-a", "host-b"])),
));
let tags = RawTagColumn::try_new(&dictionary).unwrap();
assert!(tags.equal_at(0, &tags, 0));
assert!(!tags.equal_at(0, &tags, 1));
}
fn prepare_test_data() -> DataSourceExec {
let schema = Arc::new(Schema::new(vec![
Field::new("host", DataType::Utf8, true),
+55
View File
@@ -610,6 +610,11 @@ mod tests {
use api::prom_store::remote::LabelMatcher;
use api::v1::{ColumnDataType, Row, SemanticType};
use arrow::array::{
DictionaryArray, Float64Array, StringArray, TimestampMillisecondArray, UInt32Array,
};
use arrow::datatypes::{DataType as ArrowDataType, Field, Schema as ArrowSchema, UInt32Type};
use common_recordbatch::DfRecordBatch;
use datafusion::prelude::SessionContext;
use datatypes::data_type::ConcreteDataType;
use datatypes::schema::{ColumnSchema, Schema};
@@ -954,4 +959,54 @@ mod tests {
}]
);
}
#[test]
fn test_recordbatches_to_timeseries_omits_dictionary_value_null_label() {
let arrow_schema = Arc::new(ArrowSchema::new(vec![
Field::new(
greptime_timestamp(),
ArrowDataType::Timestamp(arrow::datatypes::TimeUnit::Millisecond, None),
false,
),
Field::new(greptime_value(), ArrowDataType::Float64, false),
Field::new_dictionary("instance", ArrowDataType::UInt32, ArrowDataType::Utf8, true),
]));
let schema = Arc::new(Schema::try_from(arrow_schema.clone()).unwrap());
let instance = DictionaryArray::<UInt32Type>::new(
UInt32Array::from(vec![0]),
Arc::new(StringArray::from(vec![None::<&str>])),
);
let batch = DfRecordBatch::try_new(
arrow_schema,
vec![
Arc::new(TimestampMillisecondArray::from(vec![1000])),
Arc::new(Float64Array::from(vec![3.0])),
Arc::new(instance),
],
)
.unwrap();
let recordbatches = RecordBatches::try_new(
schema.clone(),
vec![RecordBatch::from_df_record_batch(schema, batch)],
)
.unwrap();
let timeseries = recordbatches_to_timeseries("metric1", recordbatches).unwrap();
assert_eq!(1, timeseries.len());
assert_eq!(
vec![Label {
name: METRIC_NAME_LABEL.to_string(),
value: "metric1".to_string(),
}],
timeseries[0].labels
);
assert_eq!(
vec![Sample {
value: 3.0,
timestamp: 1000,
}],
timeseries[0].samples
);
}
}