diff --git a/src/common/grpc/src/flight.rs b/src/common/grpc/src/flight.rs index eba397997e2..4234e3c62e3 100644 --- a/src/common/grpc/src/flight.rs +++ b/src/common/grpc/src/flight.rs @@ -25,7 +25,7 @@ use common_recordbatch::DfRecordBatch; use datatypes::arrow; use datatypes::arrow::array::ArrayRef; use datatypes::arrow::buffer::Buffer; -use datatypes::arrow::datatypes::{DataType, Schema as ArrowSchema, SchemaRef}; +use datatypes::arrow::datatypes::{Schema as ArrowSchema, SchemaRef}; use datatypes::arrow::error::ArrowError; use datatypes::arrow::ipc::{MessageHeader, convert, reader, root_as_message, writer}; use flatbuffers::FlatBufferBuilder; @@ -87,17 +87,6 @@ impl FlightEncoder { } } - /// Encodes a query-result schema and tracks dictionaries recursively. - pub fn encode_query_schema(&mut self, schema: &ArrowSchema) -> FlightData { - self.data_gen - .schema_to_bytes_with_dictionary_tracker( - schema, - &mut self.dictionary_tracker, - &self.write_options, - ) - .into() - } - /// Encode the Arrow schema to [FlightData]. pub fn encode_schema(&self, schema: &ArrowSchema) -> FlightData { SchemaAsIpc::new(schema, &self.write_options).into() @@ -110,15 +99,15 @@ impl FlightEncoder { /// be encoded to exactly one [FlightData]. pub fn encode(&mut self, flight_message: FlightMessage) -> Vec1 { match flight_message { - FlightMessage::Schema(schema) => { - schema.fields().iter().for_each(|x| { - if matches!(x.data_type(), DataType::Dictionary(_, _)) { - self.dictionary_tracker.next_dict_id(); - } - }); - - vec1![self.encode_schema(schema.as_ref())] - } + FlightMessage::Schema(schema) => vec1![ + self.data_gen + .schema_to_bytes_with_dictionary_tracker( + schema.as_ref(), + &mut self.dictionary_tracker, + &self.write_options, + ) + .into(), + ], FlightMessage::RecordBatch(record_batch) => { let (encoded_dictionaries, encoded_batch) = self .data_gen @@ -618,7 +607,7 @@ mod test { } #[test] - fn test_query_schema_with_nested_dictionary_array() -> Result<()> { + fn test_schema_with_nested_dictionary_array() -> Result<()> { let item = Arc::new(Field::new_dictionary( "item", DataType::UInt32, @@ -643,12 +632,12 @@ mod test { let batch = DfRecordBatch::try_new(schema.clone(), vec![Arc::new(list)]).unwrap(); let mut encoder = FlightEncoder::default(); - let encoded_schema = encoder.encode_query_schema(&schema); + let encoded_schema = encoder.encode(FlightMessage::Schema(schema.clone())); let encoded_batch = encoder.encode(FlightMessage::RecordBatch(batch.clone())); let mut decoder = FlightDecoder::default(); assert!(matches!( - decoder.try_decode(&encoded_schema)?, + decoder.try_decode(&encoded_schema[0])?, Some(FlightMessage::Schema(actual)) if actual == schema )); for data in encoded_batch.iter().take(encoded_batch.len() - 1) { diff --git a/src/query/src/dist_plan/analyzer.rs b/src/query/src/dist_plan/analyzer.rs index 5119c7d7f50..433b66519b9 100644 --- a/src/query/src/dist_plan/analyzer.rs +++ b/src/query/src/dist_plan/analyzer.rs @@ -167,6 +167,9 @@ impl AnalyzerRule for DistPlannerAnalyzer { } } }; + let plan = plan + .transform_down_with_subqueries(&unwrap_dictionary_literals)? + .data; let result = match self.try_push_down(plan.clone()) { Ok(plan) => plan, @@ -225,6 +228,12 @@ fn pre_merge_scan_optimizer() -> Optimizer { } fn unwrap_dictionary_literals(plan: LogicalPlan) -> DfResult> { + // A Values plan derives its schema from its expressions. Rewriting only the expressions would + // leave that schema inconsistent, which affects DML planning. + if matches!(&plan, LogicalPlan::Values(_)) { + return Ok(Transformed::no(plan)); + } + plan.map_expressions(|expr| { expr.transform_up(|expr| match expr { Expr::Literal(ScalarValue::Dictionary(_, value), metadata) => { @@ -735,16 +744,9 @@ impl PlanRewriter { self.partition_cols ); - // Substrait does not encode dictionary literals. Only unwrap literals in the remote - // input; rewriting the whole plan also changes DML Values without rebuilding its schema. - let remote_input = on_node - .clone() - .transform_down_with_subqueries(&unwrap_dictionary_literals)? - .data; - // add merge scan as the new root let mut node = MergeScanLogicalPlan::new( - remote_input, + on_node.clone(), false, // at this stage, the partition cols should be set // treat it as non-partitioned if None diff --git a/src/query/src/dist_plan/analyzer/test.rs b/src/query/src/dist_plan/analyzer/test.rs index 4526775180f..9ed77eed6c9 100644 --- a/src/query/src/dist_plan/analyzer/test.rs +++ b/src/query/src/dist_plan/analyzer/test.rs @@ -301,7 +301,7 @@ fn find_merge_scan(plan: &LogicalPlan) -> Option<&MergeScanLogicalPlan> { } #[test] -fn dictionary_literals_are_unwrapped_only_in_remote_input() { +fn dictionary_literals_are_unwrapped_before_pushdown() { let table = Arc::new(Table::new( TestTable::table_info(0, "t".to_string(), MITO_ENGINE.to_string()), FilterPushDownType::Inexact, @@ -330,7 +330,7 @@ fn dictionary_literals_are_unwrapped_only_in_remote_input() { assert!( result .to_string() - .contains("Filter: t.pk1 = Dictionary(UInt32, Utf8(\"host-a\"))") + .contains("Filter: t.pk1 = Utf8(\"host-a\")") ); let remote_input = find_merge_scan(&result).unwrap().input(); @@ -344,6 +344,34 @@ fn dictionary_literals_are_unwrapped_only_in_remote_input() { .unwrap(); } +#[test] +fn dictionary_literals_in_values_keep_their_schema() { + let literal = Expr::Literal( + ScalarValue::Dictionary( + Box::new(DataType::UInt32), + Box::new(ScalarValue::Utf8(Some("host-a".to_string()))), + ), + None, + ); + let plan = LogicalPlanBuilder::values(vec![vec![literal]]) + .unwrap() + .build() + .unwrap(); + let schema = plan.schema().clone(); + + let transformed = plan + .transform_down_with_subqueries(&unwrap_dictionary_literals) + .unwrap() + .data; + + assert_eq!(&schema, transformed.schema()); + assert!( + transformed + .to_string() + .contains("Dictionary(UInt32, Utf8(\"host-a\"))") + ); +} + #[cfg(feature = "vector_index")] mod vector_search_tests { use std::sync::Arc; diff --git a/src/servers/src/grpc/flight/stream.rs b/src/servers/src/grpc/flight/stream.rs index 6c2a832b51d..ee08f8f20d4 100644 --- a/src/servers/src/grpc/flight/stream.rs +++ b/src/servers/src/grpc/flight/stream.rs @@ -232,9 +232,6 @@ impl Stream for FlightRecordBatchStream { Poll::Ready(None) } Poll::Ready(Some(result)) => match result { - Ok(FlightMessage::Schema(schema)) => { - Poll::Ready(Some(Ok(this.encoder.encode_query_schema(schema.as_ref())))) - } Ok(flight_message) => { let mut iter = this.encoder.encode(flight_message).into_iter(); let Some(first) = iter.next() else {