fix(query): preserve pushdown for dictionary labels

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>
This commit is contained in:
Ruihang Xia
2026-07-15 00:24:34 +08:00
parent d39a08f0a9
commit 6540c65ee0
4 changed files with 53 additions and 37 deletions
+13 -24
View File
@@ -25,7 +25,7 @@ use common_recordbatch::DfRecordBatch;
use datatypes::arrow;
use datatypes::arrow::array::ArrayRef;
use datatypes::arrow::buffer::Buffer;
use datatypes::arrow::datatypes::{DataType, Schema as ArrowSchema, SchemaRef};
use datatypes::arrow::datatypes::{Schema as ArrowSchema, SchemaRef};
use datatypes::arrow::error::ArrowError;
use datatypes::arrow::ipc::{MessageHeader, convert, reader, root_as_message, writer};
use flatbuffers::FlatBufferBuilder;
@@ -87,17 +87,6 @@ impl FlightEncoder {
}
}
/// Encodes a query-result schema and tracks dictionaries recursively.
pub fn encode_query_schema(&mut self, schema: &ArrowSchema) -> FlightData {
self.data_gen
.schema_to_bytes_with_dictionary_tracker(
schema,
&mut self.dictionary_tracker,
&self.write_options,
)
.into()
}
/// Encode the Arrow schema to [FlightData].
pub fn encode_schema(&self, schema: &ArrowSchema) -> FlightData {
SchemaAsIpc::new(schema, &self.write_options).into()
@@ -110,15 +99,15 @@ impl FlightEncoder {
/// be encoded to exactly one [FlightData].
pub fn encode(&mut self, flight_message: FlightMessage) -> Vec1<FlightData> {
match flight_message {
FlightMessage::Schema(schema) => {
schema.fields().iter().for_each(|x| {
if matches!(x.data_type(), DataType::Dictionary(_, _)) {
self.dictionary_tracker.next_dict_id();
}
});
vec1![self.encode_schema(schema.as_ref())]
}
FlightMessage::Schema(schema) => vec1![
self.data_gen
.schema_to_bytes_with_dictionary_tracker(
schema.as_ref(),
&mut self.dictionary_tracker,
&self.write_options,
)
.into(),
],
FlightMessage::RecordBatch(record_batch) => {
let (encoded_dictionaries, encoded_batch) = self
.data_gen
@@ -618,7 +607,7 @@ mod test {
}
#[test]
fn test_query_schema_with_nested_dictionary_array() -> Result<()> {
fn test_schema_with_nested_dictionary_array() -> Result<()> {
let item = Arc::new(Field::new_dictionary(
"item",
DataType::UInt32,
@@ -643,12 +632,12 @@ mod test {
let batch = DfRecordBatch::try_new(schema.clone(), vec![Arc::new(list)]).unwrap();
let mut encoder = FlightEncoder::default();
let encoded_schema = encoder.encode_query_schema(&schema);
let encoded_schema = encoder.encode(FlightMessage::Schema(schema.clone()));
let encoded_batch = encoder.encode(FlightMessage::RecordBatch(batch.clone()));
let mut decoder = FlightDecoder::default();
assert!(matches!(
decoder.try_decode(&encoded_schema)?,
decoder.try_decode(&encoded_schema[0])?,
Some(FlightMessage::Schema(actual)) if actual == schema
));
for data in encoded_batch.iter().take(encoded_batch.len() - 1) {
+10 -8
View File
@@ -167,6 +167,9 @@ impl AnalyzerRule for DistPlannerAnalyzer {
}
}
};
let plan = plan
.transform_down_with_subqueries(&unwrap_dictionary_literals)?
.data;
let result = match self.try_push_down(plan.clone()) {
Ok(plan) => plan,
@@ -225,6 +228,12 @@ fn pre_merge_scan_optimizer() -> Optimizer {
}
fn unwrap_dictionary_literals(plan: LogicalPlan) -> DfResult<Transformed<LogicalPlan>> {
// A Values plan derives its schema from its expressions. Rewriting only the expressions would
// leave that schema inconsistent, which affects DML planning.
if matches!(&plan, LogicalPlan::Values(_)) {
return Ok(Transformed::no(plan));
}
plan.map_expressions(|expr| {
expr.transform_up(|expr| match expr {
Expr::Literal(ScalarValue::Dictionary(_, value), metadata) => {
@@ -735,16 +744,9 @@ impl PlanRewriter {
self.partition_cols
);
// Substrait does not encode dictionary literals. Only unwrap literals in the remote
// input; rewriting the whole plan also changes DML Values without rebuilding its schema.
let remote_input = on_node
.clone()
.transform_down_with_subqueries(&unwrap_dictionary_literals)?
.data;
// add merge scan as the new root
let mut node = MergeScanLogicalPlan::new(
remote_input,
on_node.clone(),
false,
// at this stage, the partition cols should be set
// treat it as non-partitioned if None
+30 -2
View File
@@ -301,7 +301,7 @@ fn find_merge_scan(plan: &LogicalPlan) -> Option<&MergeScanLogicalPlan> {
}
#[test]
fn dictionary_literals_are_unwrapped_only_in_remote_input() {
fn dictionary_literals_are_unwrapped_before_pushdown() {
let table = Arc::new(Table::new(
TestTable::table_info(0, "t".to_string(), MITO_ENGINE.to_string()),
FilterPushDownType::Inexact,
@@ -330,7 +330,7 @@ fn dictionary_literals_are_unwrapped_only_in_remote_input() {
assert!(
result
.to_string()
.contains("Filter: t.pk1 = Dictionary(UInt32, Utf8(\"host-a\"))")
.contains("Filter: t.pk1 = Utf8(\"host-a\")")
);
let remote_input = find_merge_scan(&result).unwrap().input();
@@ -344,6 +344,34 @@ fn dictionary_literals_are_unwrapped_only_in_remote_input() {
.unwrap();
}
#[test]
fn dictionary_literals_in_values_keep_their_schema() {
let literal = Expr::Literal(
ScalarValue::Dictionary(
Box::new(DataType::UInt32),
Box::new(ScalarValue::Utf8(Some("host-a".to_string()))),
),
None,
);
let plan = LogicalPlanBuilder::values(vec![vec![literal]])
.unwrap()
.build()
.unwrap();
let schema = plan.schema().clone();
let transformed = plan
.transform_down_with_subqueries(&unwrap_dictionary_literals)
.unwrap()
.data;
assert_eq!(&schema, transformed.schema());
assert!(
transformed
.to_string()
.contains("Dictionary(UInt32, Utf8(\"host-a\"))")
);
}
#[cfg(feature = "vector_index")]
mod vector_search_tests {
use std::sync::Arc;
-3
View File
@@ -232,9 +232,6 @@ impl Stream for FlightRecordBatchStream {
Poll::Ready(None)
}
Poll::Ready(Some(result)) => match result {
Ok(FlightMessage::Schema(schema)) => {
Poll::Ready(Some(Ok(this.encoder.encode_query_schema(schema.as_ref()))))
}
Ok(flight_message) => {
let mut iter = this.encoder.encode(flight_message).into_iter();
let Some(first) = iter.next() else {