From 1a32a709e563885e9ae43f1310b5503ebbeeaadf Mon Sep 17 00:00:00 2001 From: Ruihang Xia Date: Tue, 14 Jul 2026 18:35:07 +0800 Subject: [PATCH] fix(query): keep dictionary tags within query paths Signed-off-by: Ruihang Xia --- src/common/grpc/src/flight.rs | 66 +++++-- src/common/query/src/lib.rs | 37 +++- src/common/recordbatch/src/lib.rs | 166 +++++++++++++++++- src/common/recordbatch/src/recordbatch.rs | 34 +++- src/flow/src/batching_mode/table_creator.rs | 56 +++++- src/frontend/src/instance.rs | 34 +++- src/frontend/src/instance/grpc.rs | 4 +- src/frontend/src/instance/jaeger.rs | 8 +- src/frontend/src/instance/logs.rs | 7 +- src/query/src/dist_plan/analyzer.rs | 13 +- src/query/src/dist_plan/analyzer/test.rs | 59 ++++++- src/servers/src/http.rs | 49 +----- src/servers/src/http/prometheus.rs | 56 ------ src/servers/src/http/result.rs | 15 -- .../src/http/result/prometheus_resp.rs | 14 +- src/servers/src/mysql/handler.rs | 8 +- src/servers/src/mysql/helper.rs | 32 +++- src/servers/src/mysql/writer.rs | 22 +-- src/servers/src/postgres/handler.rs | 3 +- src/servers/src/postgres/types.rs | 97 +++++++--- src/servers/tests/mod.rs | 24 --- src/servers/tests/mysql/mysql_server_test.rs | 24 +-- src/servers/tests/postgres/mod.rs | 31 +--- .../common/prepare/mysql_prepare.result | 33 ++++ .../common/prepare/mysql_prepare.sql | 15 ++ 25 files changed, 612 insertions(+), 295 deletions(-) diff --git a/src/common/grpc/src/flight.rs b/src/common/grpc/src/flight.rs index 69ff48a5896..9a6bdc4a608 100644 --- a/src/common/grpc/src/flight.rs +++ b/src/common/grpc/src/flight.rs @@ -18,8 +18,8 @@ use std::collections::HashMap; use std::sync::Arc; use api::v1::{AffectedRows, FlightMetadata, Metrics}; +use arrow_flight::FlightData; use arrow_flight::utils::flight_data_to_arrow_batch; -use arrow_flight::{FlightData, SchemaAsIpc}; use common_base::bytes::Bytes; use common_recordbatch::DfRecordBatch; use datatypes::arrow; @@ -190,8 +190,14 @@ impl FlightEncoder { } /// Encode the Arrow schema to [FlightData]. - pub fn encode_schema(&self, schema: &ArrowSchema) -> FlightData { - SchemaAsIpc::new(schema, &self.write_options).into() + pub fn encode_schema(&mut self, schema: &ArrowSchema) -> FlightData { + self.data_gen + .schema_to_bytes_with_dictionary_tracker( + schema, + &mut self.dictionary_tracker, + &self.write_options, + ) + .into() } /// Encode the [FlightMessage] to a list (at least one element) of [FlightData]s. @@ -202,12 +208,6 @@ impl FlightEncoder { pub fn encode(&mut self, flight_message: FlightMessage) -> Vec1 { match flight_message { FlightMessage::Schema(schema) => { - schema.fields().iter().for_each(|x| { - if matches!(x.data_type(), DataType::Dictionary(_, _)) { - self.dictionary_tracker.next_dict_id(); - } - }); - vec1![self.encode_schema(schema.as_ref())] } FlightMessage::RecordBatch(record_batch) => { @@ -474,8 +474,9 @@ fn build_none_flight_msg() -> Bytes { mod test { use arrow_flight::utils::batches_to_flight_data; use datatypes::arrow::array::{ - DictionaryArray, Int32Array, StringArray, UInt8Array, UInt32Array, + DictionaryArray, Int32Array, ListArray, StringArray, UInt8Array, UInt32Array, }; + use datatypes::arrow::buffer::OffsetBuffer; use datatypes::arrow::datatypes::{DataType, Field, Schema}; use super::*; @@ -727,6 +728,51 @@ mod test { Ok(()) } + #[test] + fn test_flight_encode_decode_with_nested_dictionary_array() -> Result<()> { + let item_field = Arc::new(Field::new_dictionary( + "item", + DataType::UInt32, + DataType::Utf8, + true, + )); + let schema = Arc::new(Schema::new(vec![Field::new( + "tags", + DataType::List(item_field.clone()), + true, + )])); + let values = DictionaryArray::new( + UInt32Array::from_iter_values([0, 1, 0]), + Arc::new(StringArray::from_iter_values(["host-a", "host-b"])), + ); + let list = ListArray::new( + item_field, + OffsetBuffer::from_lengths([2, 1]), + Arc::new(values), + None, + ); + let batch = DfRecordBatch::try_new(schema.clone(), vec![Arc::new(list)]).unwrap(); + + let mut encoder = FlightEncoder::default(); + let encoded_schema = encoder.encode(FlightMessage::Schema(schema.clone())); + let encoded_batch = encoder.encode(FlightMessage::RecordBatch(batch.clone())); + + let mut decoder = FlightDecoder::default(); + assert!(matches!( + decoder.try_decode(encoded_schema.first())?, + Some(FlightMessage::Schema(actual)) if actual == schema + )); + + let mut decoded_batch = None; + for data in encoded_batch { + if let Some(FlightMessage::RecordBatch(decoded)) = decoder.try_decode(&data)? { + decoded_batch = Some(decoded); + } + } + assert_eq!(Some(batch), decoded_batch); + Ok(()) + } + #[test] fn test_affected_rows_roundtrip_through_flight_codec() { // Verify the full FlightEncoder → FlightDecoder pipeline handles diff --git a/src/common/query/src/lib.rs b/src/common/query/src/lib.rs index bdba49eaf70..c701ba9f1c0 100644 --- a/src/common/query/src/lib.rs +++ b/src/common/query/src/lib.rs @@ -27,7 +27,10 @@ use std::sync::Arc; use api::greptime_proto::v1::AddColumnLocation as Location; use api::greptime_proto::v1::add_column_location::LocationType; -use common_recordbatch::{RecordBatches, SendableRecordBatchStream}; +use common_recordbatch::{ + RecordBatches, SendableRecordBatchMapper, SendableRecordBatchStream, map_dictionary_to_values, + map_dictionary_to_values_schema, +}; use datafusion::physical_plan::ExecutionPlan; use serde::{Deserialize, Serialize}; use sqlparser_derive::{Visit, VisitMut}; @@ -97,6 +100,38 @@ impl Output { Self { data, meta } } + /// Expands dictionary arrays before exposing a query result to a client. + pub fn map_dictionary_to_values(self) -> common_recordbatch::error::Result { + let Self { data, meta } = self; + let data = match data { + OutputData::AffectedRows(rows) => OutputData::AffectedRows(rows), + OutputData::RecordBatches(record_batches) => { + let original_schema = record_batches.schema(); + let (mapped_schema, apply_mapper) = + map_dictionary_to_values_schema(original_schema.clone()); + if !apply_mapper { + OutputData::RecordBatches(record_batches) + } else { + let batches = record_batches + .into_iter() + .map(|batch| { + map_dictionary_to_values(batch, &original_schema, &mapped_schema) + }) + .collect::>>()?; + OutputData::RecordBatches(RecordBatches::try_new(mapped_schema, batches)?) + } + } + OutputData::Stream(stream) => { + OutputData::Stream(Box::pin(SendableRecordBatchMapper::new( + stream, + map_dictionary_to_values, + map_dictionary_to_values_schema, + ))) + } + }; + Ok(Self { data, meta }) + } + pub fn extract_rows_and_cost(&self) -> (OutputRows, OutputCost) { match self.data { OutputData::AffectedRows(rows) => (rows, self.meta.cost), diff --git a/src/common/recordbatch/src/lib.rs b/src/common/recordbatch/src/lib.rs index 3db3d9c08b6..d6838d67f0b 100644 --- a/src/common/recordbatch/src/lib.rs +++ b/src/common/recordbatch/src/lib.rs @@ -40,16 +40,16 @@ use datatypes::arrow::array::{ArrayRef, AsArray, StringBuilder}; use datatypes::arrow::compute::SortOptions; pub use datatypes::arrow::record_batch::RecordBatch as DfRecordBatch; use datatypes::arrow::util::pretty; -use datatypes::prelude::{ConcreteDataType, VectorRef}; +use datatypes::prelude::{ConcreteDataType, DataType, VectorRef}; use datatypes::schema::{ColumnSchema, Schema, SchemaRef}; -use datatypes::types::{JsonFormat, jsonb_to_string}; +use datatypes::types::{JsonFormat, StructField, StructType, jsonb_to_string}; use error::Result; use futures::task::{Context, Poll}; use futures::{Stream, TryStreamExt}; pub use recordbatch::RecordBatch; use snafu::{IntoError, ResultExt, ensure}; -use crate::error::NewDfRecordBatchSnafu; +use crate::error::{ArrowComputeSnafu, NewDfRecordBatchSnafu}; pub trait RecordBatchStream: Stream> { fn name(&self) -> &str { @@ -163,6 +163,84 @@ pub fn map_json_type_to_string_schema(schema: SchemaRef) -> (SchemaRef, bool) { (Arc::new(Schema::new(new_columns)), apply_mapper) } +/// Replaces dictionary types with their value types, including dictionaries nested in lists and +/// structs. +pub fn map_dictionary_to_values_data_type(data_type: &ConcreteDataType) -> ConcreteDataType { + match data_type { + ConcreteDataType::Dictionary(dictionary) => { + map_dictionary_to_values_data_type(dictionary.value_type()) + } + ConcreteDataType::List(list) => ConcreteDataType::list_datatype(Arc::new( + map_dictionary_to_values_data_type(list.item_type()), + )), + ConcreteDataType::Struct(struct_type) => { + let fields = struct_type + .fields() + .iter() + .map(|field| { + StructField::new( + field.name(), + map_dictionary_to_values_data_type(field.data_type()), + field.is_nullable(), + ) + }) + .collect(); + ConcreteDataType::struct_datatype(StructType::new(Arc::new(fields))) + } + _ => data_type.clone(), + } +} + +/// Maps dictionary columns in a schema to their value types. +pub fn map_dictionary_to_values_schema(schema: SchemaRef) -> (SchemaRef, bool) { + let mut apply_mapper = false; + let columns = schema + .column_schemas() + .iter() + .map(|column| { + let data_type = map_dictionary_to_values_data_type(&column.data_type); + apply_mapper |= data_type != column.data_type; + let mut column = column.clone(); + column.data_type = data_type; + column + }) + .collect(); + + ( + Arc::new(Schema::new_with_version(columns, schema.version())), + apply_mapper, + ) +} + +/// Expands dictionary arrays to their value arrays according to `mapped_schema`. +pub fn map_dictionary_to_values( + batch: RecordBatch, + original_schema: &SchemaRef, + mapped_schema: &SchemaRef, +) -> Result { + let arrays = batch + .columns() + .iter() + .zip(original_schema.column_schemas()) + .zip(mapped_schema.column_schemas()) + .map(|((array, original), mapped)| { + if original.data_type == mapped.data_type { + Ok(array.clone()) + } else { + datatypes::arrow::compute::cast(array, &mapped.data_type.as_arrow_type()) + .context(ArrowComputeSnafu) + } + }) + .collect::>>()?; + + let record_batch = DfRecordBatch::try_new(mapped_schema.arrow_schema().clone(), arrays) + .context(NewDfRecordBatchSnafu)?; + Ok(RecordBatch::from_df_record_batch( + mapped_schema.clone(), + record_batch, + )) +} + impl SendableRecordBatchMapper { /// Creates a new [SendableRecordBatchMapper] with the given inner [RecordBatchStream], mapper function, and schema mapper function. pub fn new( @@ -835,6 +913,11 @@ mod tests { use std::time::Duration; use common_memory_manager::{OnExhaustedPolicy, PermitGranularity}; + use datatypes::arrow::array::{ + DictionaryArray, Int32Array, ListArray, StringArray, UInt32Array, + }; + use datatypes::arrow::buffer::OffsetBuffer; + use datatypes::arrow::datatypes::{DataType as ArrowDataType, Int32Type}; use datatypes::prelude::{ConcreteDataType, VectorRef}; use datatypes::schema::{ColumnSchema, Schema}; use datatypes::vectors::{BooleanVector, Int32Vector, StringVector}; @@ -918,6 +1001,83 @@ mod tests { assert_eq!(vec![batch1], batches.take()); } + #[test] + fn test_map_dictionary_to_values_recursively() { + let string_dictionary = ConcreteDataType::dictionary_datatype( + ConcreteDataType::int32_datatype(), + ConcreteDataType::string_datatype(), + ); + let list_dictionary = + ConcreteDataType::list_datatype(Arc::new(ConcreteDataType::dictionary_datatype( + ConcreteDataType::uint32_datatype(), + ConcreteDataType::string_datatype(), + ))); + let schema = Arc::new(Schema::new(vec![ + ColumnSchema::new("host", string_dictionary, true), + ColumnSchema::new("tags", list_dictionary, true), + ])); + + let host = DictionaryArray::::new( + Int32Array::from(vec![0, 1]), + Arc::new(StringArray::from(vec![Some("host-a"), None])), + ); + let ArrowDataType::List(item_field) = schema.arrow_schema().field(1).data_type().clone() + else { + unreachable!() + }; + let tag_values = DictionaryArray::new( + UInt32Array::from_iter_values([0, 1, 0]), + Arc::new(StringArray::from(vec![Some("a"), None])), + ); + let tags = ListArray::new( + item_field, + OffsetBuffer::from_lengths([2, 1]), + Arc::new(tag_values), + None, + ); + let batch = DfRecordBatch::try_new( + schema.arrow_schema().clone(), + vec![Arc::new(host), Arc::new(tags)], + ) + .unwrap(); + let batch = RecordBatch::from_df_record_batch(schema.clone(), batch); + + let (mapped_schema, apply_mapper) = map_dictionary_to_values_schema(schema.clone()); + assert!(apply_mapper); + assert_eq!( + &ArrowDataType::Utf8, + mapped_schema.arrow_schema().field(0).data_type() + ); + assert_eq!( + &ArrowDataType::List(Arc::new( + datatypes::arrow::datatypes::Field::new_list_field(ArrowDataType::Utf8, true,) + )), + mapped_schema.arrow_schema().field(1).data_type() + ); + + let mapped = map_dictionary_to_values(batch, &schema, &mapped_schema).unwrap(); + let host = mapped + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(vec![Some("host-a"), None], host.iter().collect::>()); + let tags = mapped + .column(1) + .as_any() + .downcast_ref::() + .unwrap(); + let tag_values = tags + .values() + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!( + vec![Some("a"), None, Some("a")], + tag_values.iter().collect::>() + ); + } + #[tokio::test] async fn test_simple_recordbatch_stream() { let column_a = ColumnSchema::new("a", ConcreteDataType::int32_datatype(), false); diff --git a/src/common/recordbatch/src/recordbatch.rs b/src/common/recordbatch/src/recordbatch.rs index 7280772d05d..2f35c21aaff 100644 --- a/src/common/recordbatch/src/recordbatch.rs +++ b/src/common/recordbatch/src/recordbatch.rs @@ -303,10 +303,10 @@ impl RecordBatch { } _ => { if let Ok(column) = Helper::try_into_vector(column) { - Box::new( - (0..column.len()) - .map(move |i| (!column.is_null(i)).then(|| column.get(i).to_string())), - ) + Box::new((0..column.len()).map(move |i| { + let value = column.get_ref(i); + (!value.is_null()).then(|| column.get(i).to_string()) + })) } else { Box::new(std::iter::empty()) } @@ -382,13 +382,13 @@ fn maybe_align_json_array_with_schema( mod tests { use std::sync::Arc; - use datatypes::arrow::array::{AsArray, BinaryArray, UInt32Array}; + use datatypes::arrow::array::{AsArray, BinaryArray, DictionaryArray, UInt32Array}; use datatypes::arrow::datatypes::{DataType, Field, Schema as ArrowSchema, UInt32Type}; use datatypes::arrow_array::StringArray; use datatypes::data_type::ConcreteDataType; use datatypes::extension::json::{JsonExtensionType, JsonMetadata}; use datatypes::schema::{ColumnSchema, Schema}; - use datatypes::vectors::{StringVector, UInt32Vector}; + use datatypes::vectors::{DictionaryVector, StringVector, UInt32Vector}; use super::*; @@ -475,6 +475,28 @@ mod tests { assert!(recordbatch.slice(1, 5).is_err()); } + #[test] + fn test_iter_dictionary_column_uses_logical_nulls() { + let data_type = ConcreteDataType::dictionary_datatype( + ConcreteDataType::uint32_datatype(), + ConcreteDataType::string_datatype(), + ); + let schema = Arc::new(Schema::new(vec![ColumnSchema::new( + "host", data_type, true, + )])); + let dictionary = DictionaryArray::::new( + UInt32Array::from(vec![Some(0), Some(1), None]), + Arc::new(StringArray::from(vec![None, Some("host-b")])), + ); + let vector = Arc::new(DictionaryVector::try_from(dictionary).unwrap()) as VectorRef; + let batch = RecordBatch::new(schema, vec![vector]).unwrap(); + + assert_eq!( + vec![None, Some("host-b".to_string()), None], + batch.iter_column_as_string(0).collect::>() + ); + } + #[test] fn test_merge_record_batch() { let column_schemas = vec![ diff --git a/src/flow/src/batching_mode/table_creator.rs b/src/flow/src/batching_mode/table_creator.rs index 05da055a40d..dfad054834e 100644 --- a/src/flow/src/batching_mode/table_creator.rs +++ b/src/flow/src/batching_mode/table_creator.rs @@ -164,7 +164,7 @@ fn build_by_tql_schema(plan: &LogicalPlan) -> Result { .fields() .iter() .filter_map(|f| { - if ConcreteDataType::from_arrow_type(f.data_type()).is_string() { + if is_tql_label_type(&ConcreteDataType::from_arrow_type(f.data_type())) { Some(f.name().clone()) } else { None @@ -178,6 +178,14 @@ fn build_by_tql_schema(plan: &LogicalPlan) -> Result { }) } +fn is_tql_label_type(data_type: &ConcreteDataType) -> bool { + data_type.is_string() + || matches!( + data_type, + ConcreteDataType::Dictionary(dictionary) if dictionary.value_type().is_string() + ) +} + struct TableDef { ts_col: Option, pks: Vec, @@ -246,7 +254,14 @@ fn build_pk_from_aggr(plan: &LogicalPlan) -> Result, Error> { #[cfg(test)] mod test { + use std::sync::Arc; + use api::v1::column_def::try_as_column_schema; + use datafusion::arrow::datatypes::{ + DataType as ArrowDataType, Field, Schema as ArrowSchema, TimeUnit, + }; + use datafusion_common::DFSchema; + use datafusion_expr::logical_plan::EmptyRelation; use datatypes::prelude::ConcreteDataType; use datatypes::schema::ColumnSchema; use pretty_assertions::assert_eq; @@ -257,6 +272,45 @@ mod test { use crate::batching_mode::utils::sql_to_df_plan; use crate::test_utils::create_test_query_engine; + #[test] + fn test_tql_dictionary_string_is_label() { + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new_dictionary("host", ArrowDataType::UInt32, ArrowDataType::Utf8, true), + Field::new("value", ArrowDataType::Float64, true), + Field::new( + "ts", + ArrowDataType::Timestamp(TimeUnit::Millisecond, None), + false, + ), + ])); + let plan = LogicalPlan::EmptyRelation(EmptyRelation { + produce_one_row: false, + schema: Arc::new(DFSchema::try_from(arrow_schema).unwrap()), + }); + + let expr = create_table_with_expr( + &plan, + &[ + "greptime".to_string(), + "public".to_string(), + "sink".to_string(), + ], + &QueryType::Tql, + ) + .unwrap(); + let columns = expr + .column_defs + .iter() + .map(|column| try_as_column_schema(column).unwrap()) + .collect::>(); + + assert_eq!(vec!["host".to_string()], expr.primary_keys); + assert_eq!("ts", expr.time_index); + assert_eq!(ConcreteDataType::string_datatype(), columns[0].data_type); + assert_eq!(ConcreteDataType::float64_datatype(), columns[1].data_type); + assert!(columns[2].is_time_index()); + } + #[tokio::test] async fn test_gen_create_table_sql() { let query_engine = create_test_query_engine(); diff --git a/src/frontend/src/instance.rs b/src/frontend/src/instance.rs index 7242d601c1b..11ca6e33436 100644 --- a/src/frontend/src/instance.rs +++ b/src/frontend/src/instance.rs @@ -95,9 +95,9 @@ use table::requests::{OTLP_METRIC_COMPAT_KEY, OTLP_METRIC_COMPAT_PROM}; use tracing::Span; use crate::error::{ - self, Error, ExecLogicalPlanSnafu, ExecutePromqlSnafu, ExternalSnafu, InvalidSqlSnafu, - ParseSqlSnafu, PermissionSnafu, PlanStatementSnafu, Result, SqlExecInterceptedSnafu, - StatementTimeoutSnafu, TableOperationSnafu, + self, CollectRecordbatchSnafu, Error, ExecLogicalPlanSnafu, ExecutePromqlSnafu, ExternalSnafu, + InvalidSqlSnafu, ParseSqlSnafu, PermissionSnafu, PlanStatementSnafu, Result, + SqlExecInterceptedSnafu, StatementTimeoutSnafu, TableOperationSnafu, }; use crate::service_config::InfluxdbMergeMode; use crate::stream_wrapper::CancellableStreamWrapper; @@ -885,7 +885,12 @@ impl SqlQueryHandler for Instance { self.do_query_inner(query, query_ctx) .await .into_iter() - .map(|result| result.map_err(BoxedError::new).context(ExecuteQuerySnafu)) + .map(|result| { + result + .and_then(map_query_output) + .map_err(BoxedError::new) + .context(ExecuteQuerySnafu) + }) .collect() } @@ -896,6 +901,7 @@ impl SqlQueryHandler for Instance { ) -> server_error::Result { self.do_analyze_stream_query_inner(query, query_ctx) .await + .and_then(map_query_output) .map_err(BoxedError::new) .context(ExecuteQuerySnafu) } @@ -908,6 +914,7 @@ impl SqlQueryHandler for Instance { ) -> server_error::Result { self.do_exec_plan_inner(plan, stmt, query_ctx) .await + .and_then(map_query_output) .map_err(BoxedError::new) .context(server_error::ExecutePlanSnafu) } @@ -920,7 +927,12 @@ impl SqlQueryHandler for Instance { self.do_promql_query_inner(query, query_ctx) .await .into_iter() - .map(|result| result.map_err(BoxedError::new).context(ExecuteQuerySnafu)) + .map(|result| { + result + .and_then(map_query_output) + .map_err(BoxedError::new) + .context(ExecuteQuerySnafu) + }) .collect() } @@ -943,6 +955,13 @@ impl SqlQueryHandler for Instance { } } +/// Expands scan-time dictionaries only when a query result leaves the frontend. +pub(crate) fn map_query_output(output: Output) -> Result { + output + .map_dictionary_to_values() + .context(CollectRecordbatchSnafu) +} + /// Attaches a timer to the output and observes it once the output is exhausted. pub fn attach_timer(output: Output, timer: HistogramTimer) -> Output { match output.data { @@ -1048,7 +1067,10 @@ impl PrometheusHandler for Instance { .map_err(BoxedError::new) .context(ExecuteQuerySnafu)?; - Ok(interceptor.post_execute(output, query_ctx)?) + let output = interceptor.post_execute(output, query_ctx)?; + map_query_output(output) + .map_err(BoxedError::new) + .context(ExecuteQuerySnafu) } async fn query_metric_names( diff --git a/src/frontend/src/instance/grpc.rs b/src/frontend/src/instance/grpc.rs index 0ca1a2cf209..1324604c827 100644 --- a/src/frontend/src/instance/grpc.rs +++ b/src/frontend/src/instance/grpc.rs @@ -51,7 +51,7 @@ use crate::error::{ NotSupportedSnafu, PermissionSnafu, PlanStatementSnafu, Result, SubstraitDecodeLogicalPlanSnafu, TableNotFoundSnafu, TableOperationSnafu, }; -use crate::instance::{Instance, attach_timer}; +use crate::instance::{Instance, attach_timer, map_query_output}; use crate::metrics::{ GRPC_HANDLE_PLAN_ELAPSED, GRPC_HANDLE_PROMQL_ELAPSED, GRPC_HANDLE_SQL_ELAPSED, }; @@ -251,7 +251,7 @@ impl GrpcQueryHandler for Instance { }; let output = interceptor.post_execute(output, ctx)?; - Ok(output) + map_query_output(output) } .await; diff --git a/src/frontend/src/instance/jaeger.rs b/src/frontend/src/instance/jaeger.rs index e500de7a855..4d14aed0dc6 100644 --- a/src/frontend/src/instance/jaeger.rs +++ b/src/frontend/src/instance/jaeger.rs @@ -408,7 +408,9 @@ async fn query_trace_table( RecordBatchStreamAdapter::try_new(stream).context(CollectRecordbatchSnafu)?, )); - Ok(output) + output + .map_dictionary_to_values() + .context(CollectRecordbatchSnafu) } async fn get_table( @@ -485,7 +487,9 @@ async fn find_traces_rank_3( RecordBatchStreamAdapter::try_new(stream).context(CollectRecordbatchSnafu)?, )); - Ok(output) + output + .map_dictionary_to_values() + .context(CollectRecordbatchSnafu) } // The current implementation registers UDFs during the planning stage, which makes it difficult diff --git a/src/frontend/src/instance/logs.rs b/src/frontend/src/instance/logs.rs index 8aacb86f5d8..5e36ab105b0 100644 --- a/src/frontend/src/instance/logs.rs +++ b/src/frontend/src/instance/logs.rs @@ -26,7 +26,7 @@ use session::context::{QueryContext, QueryContextRef}; use snafu::ResultExt; use tonic::async_trait; -use crate::instance::Instance; +use crate::instance::{Instance, map_query_output}; #[async_trait] impl LogQueryHandler for Instance { @@ -64,7 +64,10 @@ impl LogQueryHandler for Instance { .map_err(BoxedError::new) .context(ExecuteQuerySnafu)?; - Ok(interceptor.as_ref().post_query(output, ctx.clone())?) + let output = interceptor.as_ref().post_query(output, ctx.clone())?; + map_query_output(output) + .map_err(BoxedError::new) + .context(ExecuteQuerySnafu) } fn catalog_manager(&self, _ctx: &QueryContext) -> ServerResult<&dyn catalog::CatalogManager> { diff --git a/src/query/src/dist_plan/analyzer.rs b/src/query/src/dist_plan/analyzer.rs index 3397ebc4395..d85bbb0ec31 100644 --- a/src/query/src/dist_plan/analyzer.rs +++ b/src/query/src/dist_plan/analyzer.rs @@ -167,10 +167,6 @@ impl AnalyzerRule for DistPlannerAnalyzer { } } }; - let plan = plan - .transform_down_with_subqueries(&Self::unwrap_dictionary_literals)? - .data; - let result = match self.try_push_down(plan.clone()) { Ok(plan) => plan, Err(err) => { @@ -738,9 +734,16 @@ impl PlanRewriter { self.partition_cols ); + // Substrait does not encode dictionary literals. Only unwrap literals in the remote + // input; rewriting the whole plan also changes DML Values without rebuilding its schema. + let remote_input = on_node + .clone() + .transform_down_with_subqueries(&DistPlannerAnalyzer::unwrap_dictionary_literals)? + .data; + // add merge scan as the new root let mut node = MergeScanLogicalPlan::new( - on_node.clone(), + remote_input, false, // at this stage, the partition cols should be set // treat it as non-partitioned if None diff --git a/src/query/src/dist_plan/analyzer/test.rs b/src/query/src/dist_plan/analyzer/test.rs index 04602d440a0..4526775180f 100644 --- a/src/query/src/dist_plan/analyzer/test.rs +++ b/src/query/src/dist_plan/analyzer/test.rs @@ -17,7 +17,7 @@ use std::pin::Pin; use std::sync::Arc; use arrow::datatypes::{DataType, IntervalDayTime, TimeUnit}; -use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME}; +use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, MITO_ENGINE}; use common_error::ext::BoxedError; use common_function::aggrs::aggr_wrapper::{StateMergeHelper, StateWrapper}; use common_recordbatch::adapter::RecordBatchMetrics; @@ -287,6 +287,63 @@ impl Stream for EmptyStream { } } +fn find_merge_scan(plan: &LogicalPlan) -> Option<&MergeScanLogicalPlan> { + if let LogicalPlan::Extension(extension) = plan + && let Some(merge_scan) = extension + .node + .as_any() + .downcast_ref::() + { + return Some(merge_scan); + } + + plan.inputs().into_iter().find_map(find_merge_scan) +} + +#[test] +fn dictionary_literals_are_unwrapped_only_in_remote_input() { + let table = Arc::new(Table::new( + TestTable::table_info(0, "t".to_string(), MITO_ENGINE.to_string()), + FilterPushDownType::Inexact, + Arc::new(TestDataSource::new(TestTable::schema())), + )); + let table_source = Arc::new(DefaultTableSource::new(Arc::new( + DfTableProviderAdapter::new(table), + ))); + let literal = Expr::Literal( + ScalarValue::Dictionary( + Box::new(DataType::UInt32), + Box::new(ScalarValue::Utf8(Some("host-a".to_string()))), + ), + None, + ); + let plan = LogicalPlanBuilder::scan_with_filters("t", table_source, None, vec![]) + .unwrap() + .filter(col("pk1").eq(literal)) + .unwrap() + .build() + .unwrap(); + + let result = DistPlannerAnalyzer {} + .analyze(plan, &ConfigOptions::default()) + .unwrap(); + assert!( + result + .to_string() + .contains("Filter: t.pk1 = Dictionary(UInt32, Utf8(\"host-a\"))") + ); + + let remote_input = find_merge_scan(&result).unwrap().input(); + assert!( + remote_input + .to_string() + .contains("partial_filters=[t.pk1 = Utf8(\"host-a\")]") + ); + DFLogicalSubstraitConvertor + .encode(remote_input, DefaultSerializer) + .unwrap(); +} + #[cfg(feature = "vector_index")] mod vector_search_tests { use std::sync::Arc; diff --git a/src/servers/src/http.rs b/src/servers/src/http.rs index 1fdd4225c96..3713e99454d 100644 --- a/src/servers/src/http.rs +++ b/src/servers/src/http.rs @@ -216,22 +216,13 @@ impl OutputSchema { impl From for OutputSchema { fn from(schema: SchemaRef) -> OutputSchema { - fn public_type_name(data_type: &datatypes::prelude::ConcreteDataType) -> String { - match data_type { - datatypes::prelude::ConcreteDataType::Dictionary(dictionary) => { - public_type_name(dictionary.value_type()) - } - data_type => data_type.name(), - } - } - OutputSchema { column_schemas: schema .column_schemas() .iter() .map(|cs| ColumnSchema { name: cs.name.clone(), - data_type: public_type_name(&cs.data_type), + data_type: cs.data_type.name(), }) .collect(), } @@ -1366,8 +1357,6 @@ mod test { use std::io::Cursor; use std::sync::Arc; - use arrow::array::{DictionaryArray, StringArray, UInt32Array}; - use arrow::datatypes::UInt32Type; use arrow_ipc::reader::StreamReader; use arrow_schema::DataType; use axum::handler::Handler; @@ -1378,7 +1367,7 @@ mod test { use datafusion_expr::LogicalPlan; use datatypes::prelude::*; use datatypes::schema::{ColumnSchema, Schema}; - use datatypes::vectors::{DictionaryVector, StringVector, UInt32Vector}; + use datatypes::vectors::{StringVector, UInt32Vector}; use header::constants::GREPTIME_DB_HEADER_TIMEOUT; use query::parser::PromQuery; use query::query_engine::DescribeResult; @@ -1437,40 +1426,6 @@ mod test { DynamicTimeoutLayer::new(Duration::from_millis(10)) } - fn dictionary_record_batch() -> (SchemaRef, RecordBatch) { - let data_type = ConcreteDataType::dictionary_datatype( - ConcreteDataType::uint32_datatype(), - ConcreteDataType::string_datatype(), - ); - let schema = Arc::new(Schema::new(vec![ColumnSchema::new( - "host", data_type, true, - )])); - let dictionary = DictionaryArray::::new( - UInt32Array::from(vec![0, 1]), - Arc::new(StringArray::from(vec![Some("host-a"), None])), - ); - let vector = Arc::new(DictionaryVector::try_from(dictionary).unwrap()) as VectorRef; - let batch = RecordBatch::new(schema.clone(), vec![vector]).unwrap(); - (schema, batch) - } - - #[test] - fn test_http_dictionary_output_uses_logical_string_type() { - let (schema, batch) = dictionary_record_batch(); - let output = HttpRecordsOutput::try_new(schema.clone(), vec![batch]).unwrap(); - - assert_eq!("String", output.schema.column_schemas[0].data_type); - assert_eq!( - serde_json::Value::String("host-a".to_string()), - output.rows[0][0] - ); - assert_eq!(serde_json::Value::Null, output.rows[1][0]); - - let empty = HttpRecordsOutput::try_new(schema, vec![]).unwrap(); - assert_eq!("String", empty.schema.column_schemas[0].data_type); - assert!(empty.rows.is_empty()); - } - async fn forever() { pending().await } diff --git a/src/servers/src/http/prometheus.rs b/src/servers/src/http/prometheus.rs index fea24e1e19c..9d17c59b784 100644 --- a/src/servers/src/http/prometheus.rs +++ b/src/servers/src/http/prometheus.rs @@ -893,21 +893,6 @@ impl RowWriter { let v = datatypes::arrow_array::string_array_value(array, i); self.insert(column, v); } - DataType::Dictionary(key_type, value_type) - if key_type.as_ref() == &DataType::UInt32 - && matches!( - value_type.as_ref(), - DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View - ) => - { - if let Some(v) = - datatypes::arrow_array::string_array_value_at_index(array, i) - { - self.insert(column, v); - } else { - self.insert(column, "Null"); - } - } DataType::Binary | DataType::LargeBinary | DataType::BinaryView => { let v = datatypes::arrow_array::binary_array_value(array, i); let column_schema = &schema.column_schemas()[j]; @@ -1697,14 +1682,10 @@ mod tests { use std::collections::HashSet; use std::sync::Arc; - use arrow::array::{DictionaryArray, StringArray, UInt32Array}; - use arrow::datatypes::UInt32Type; use catalog::memory::MemoryCatalogManager; use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME}; - use common_recordbatch::RecordBatches; use datatypes::prelude::ConcreteDataType; use datatypes::schema::{ColumnSchema, Schema}; - use datatypes::vectors::{DictionaryVector, VectorRef}; use promql_parser::parser::value::ValueType; use table::metadata::{TableInfoBuilder, TableMetaBuilder, TableType, TableVersion}; use table::test_util::EmptyTable; @@ -1719,43 +1700,6 @@ mod tests { should_error: bool, } - #[test] - fn test_record_batches_to_series_with_dictionary_labels() { - let data_type = ConcreteDataType::dictionary_datatype( - ConcreteDataType::uint32_datatype(), - ConcreteDataType::string_datatype(), - ); - let schema = Arc::new(Schema::new(vec![ColumnSchema::new( - "host", data_type, true, - )])); - let dictionary = DictionaryArray::::new( - UInt32Array::from(vec![0, 1]), - Arc::new(StringArray::from(vec![Some("host-a"), None])), - ); - let vector = Arc::new(DictionaryVector::try_from(dictionary).unwrap()) as VectorRef; - let batch = RecordBatch::new(schema.clone(), vec![vector]).unwrap(); - let batches = RecordBatches::try_new(schema, vec![batch]).unwrap(); - let mut series = Vec::new(); - - record_batches_to_series( - batches, - &mut series, - "metric", - &HashSet::from(["host".to_string()]), - ) - .unwrap(); - - assert_eq!(2, series.len()); - assert_eq!( - Some("host-a"), - series[0].get(&Column::from("host")).map(String::as_str) - ); - assert_eq!( - Some("Null"), - series[1].get(&Column::from("host")).map(String::as_str) - ); - } - #[test] fn test_retrieve_metric_name_and_result_type() { let test_cases = &[ diff --git a/src/servers/src/http/result.rs b/src/servers/src/http/result.rs index dedc5a8c1b6..93736fb859d 100644 --- a/src/servers/src/http/result.rs +++ b/src/servers/src/http/result.rs @@ -179,21 +179,6 @@ impl HttpOutputWriter { let v = datatypes::arrow_array::string_array_value(array, i); self.write_value(v)?; } - DataType::Dictionary(key_type, value_type) - if key_type.as_ref() == &DataType::UInt32 - && matches!( - value_type.as_ref(), - DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View - ) => - { - if let Some(v) = - datatypes::arrow_array::string_array_value_at_index(array, i) - { - self.write_value(v)?; - } else { - self.write_value(Value::Null)?; - } - } DataType::Binary | DataType::LargeBinary | DataType::BinaryView => { let v = datatypes::arrow_array::binary_array_value(array, i); self.write_bytes(v, &schema.data_type)?; diff --git a/src/servers/src/http/result/prometheus_resp.rs b/src/servers/src/http/result/prometheus_resp.rs index aee516ec934..6f2b1156862 100644 --- a/src/servers/src/http/result/prometheus_resp.rs +++ b/src/servers/src/http/result/prometheus_resp.rs @@ -228,12 +228,6 @@ impl PrometheusJsonResponse { tag_column_indices.push(i); num_label_columns += 1; } - ConcreteDataType::Dictionary(ref dictionary) - if matches!(dictionary.value_type(), ConcreteDataType::String(_)) => - { - tag_column_indices.push(i); - num_label_columns += 1; - } _ => {} } } @@ -254,7 +248,7 @@ impl PrometheusJsonResponse { // prepare things... let tag_columns = tag_column_indices .iter() - .map(|i| batch.column(*i)) + .map(|i| batch.column(*i).as_string::()) .collect::>(); let tag_names = tag_column_indices .iter() @@ -286,10 +280,8 @@ impl PrometheusJsonResponse { } for (tag_column, tag_name) in tag_columns.iter().zip(tag_names.iter()) { // TODO(ruihang): add test for NULL tag - if let Some(value) = datatypes::arrow_array::string_array_value_at_index( - tag_column, row_index, - ) { - tags.push((tag_name, value)); + if tag_column.is_valid(row_index) { + tags.push((tag_name, tag_column.value(row_index))); } } diff --git a/src/servers/src/mysql/handler.rs b/src/servers/src/mysql/handler.rs index b0eb33f8fd5..7d7cd0b93aa 100644 --- a/src/servers/src/mysql/handler.rs +++ b/src/servers/src/mysql/handler.rs @@ -24,6 +24,7 @@ use chrono::{NaiveDate, NaiveDateTime}; use common_catalog::parse_optional_catalog_and_schema_from_db_string; use common_error::ext::ErrorExt; use common_query::Output; +use common_recordbatch::map_dictionary_to_values_data_type; use common_telemetry::{debug, error, tracing, warn}; use datafusion_common::ParamValues; use datafusion_expr::LogicalPlan; @@ -233,7 +234,10 @@ impl MysqlInstanceShim { .column_schemas() .iter() .map(|column_schema| { - create_mysql_column(&column_schema.data_type, &column_schema.name) + create_mysql_column( + &map_dictionary_to_values_data_type(&column_schema.data_type), + &column_schema.name, + ) }) .collect::>>() }) @@ -906,7 +910,7 @@ fn prepared_params( // Placeholder index starts from 1 for i in 1..param_num { let column = if let Some(Some(t)) = param_types.get(&format_placeholder(i)) { - create_mysql_column(t, "")? + create_mysql_column(&map_dictionary_to_values_data_type(t), "")? } else { create_mysql_column(&ConcreteDataType::null_datatype(), "")? }; diff --git a/src/servers/src/mysql/helper.rs b/src/servers/src/mysql/helper.rs index c4c072b0072..a4f289cdfb1 100644 --- a/src/servers/src/mysql/helper.rs +++ b/src/servers/src/mysql/helper.rs @@ -19,7 +19,7 @@ use chrono::NaiveDate; use common_query::prelude::ScalarValue; use common_sql::convert::sql_value_to_value; use common_time::{Date, Timestamp}; -use datatypes::prelude::ConcreteDataType; +use datatypes::prelude::{ConcreteDataType, DataType}; use datatypes::schema::ColumnSchema; use datatypes::types::TimestampType; use datatypes::value::{self, Value}; @@ -138,6 +138,13 @@ where /// Convert [`ParamValue`] into [`Value`] according to param type. /// It will try it's best to do type conversions if possible pub fn convert_value(param: &ParamValue, t: &ConcreteDataType) -> Result { + if let ConcreteDataType::Dictionary(dictionary) = t { + return Ok(ScalarValue::Dictionary( + Box::new(dictionary.key_type().as_arrow_type()), + Box::new(convert_value(param, dictionary.value_type())?), + )); + } + match param.value.into_inner() { ValueInner::Int(i) => match t { ConcreteDataType::Int8(_) => Ok(ScalarValue::Int8(Some(i as i8))), @@ -258,6 +265,16 @@ pub fn convert_value(param: &ParamValue, t: &ConcreteDataType) -> Result Result { + if let ConcreteDataType::Dictionary(dictionary) = t { + return Ok(ScalarValue::Dictionary( + Box::new(dictionary.key_type().as_arrow_type()), + Box::new(convert_expr_to_scalar_value( + param, + dictionary.value_type(), + )?), + )); + } + let column_schema = ColumnSchema::new("", t.clone(), true); match param { Expr::Value(v) => { @@ -464,6 +481,19 @@ mod tests { let v = convert_expr_to_scalar_value(&expr, &t).unwrap(); assert_eq!(ScalarValue::Utf8(Some("hello".to_string())), v); + let t = ConcreteDataType::dictionary_datatype( + ConcreteDataType::uint32_datatype(), + ConcreteDataType::string_datatype(), + ); + let v = convert_expr_to_scalar_value(&expr, &t).unwrap(); + assert_eq!( + ScalarValue::Dictionary( + Box::new(arrow_schema::DataType::UInt32), + Box::new(ScalarValue::Utf8(Some("hello".to_string()))), + ), + v + ); + let expr = Expr::Value(ValueExpr::Null.into()); let t = ConcreteDataType::time_microsecond_datatype(); let v = convert_expr_to_scalar_value(&expr, &t).unwrap(); diff --git a/src/servers/src/mysql/writer.rs b/src/servers/src/mysql/writer.rs index 5395f2b1f74..2ff2c4481cb 100644 --- a/src/servers/src/mysql/writer.rs +++ b/src/servers/src/mysql/writer.rs @@ -42,7 +42,7 @@ use session::context::QueryContextRef; use snafu::prelude::*; use tokio::io::AsyncWrite; -use crate::error::{ConvertSqlValueSnafu, DataFusionSnafu, NotSupportedSnafu, Result}; +use crate::error::{self, ConvertSqlValueSnafu, DataFusionSnafu, NotSupportedSnafu, Result}; use crate::metrics::*; /// Try to write multiple output to the writer if possible. @@ -249,20 +249,6 @@ impl MysqlResultWriter { let v = datatypes::arrow_array::string_array_value(column, i); row_writer.write_col(v)?; } - DataType::Dictionary(_, value_type) - if matches!( - value_type.as_ref(), - DataType::Utf8 | DataType::Utf8View | DataType::LargeUtf8 - ) => - { - if let Some(v) = - datatypes::arrow_array::string_array_value_at_index(column, i) - { - row_writer.write_col(v)?; - } else { - row_writer.write_col(None::)?; - } - } DataType::Binary | DataType::BinaryView | DataType::LargeBinary => { let v = datatypes::arrow_array::binary_array_value(column, i); if let ConcreteDataType::Json(_) = &schema.column_schemas()[j].data_type { @@ -378,9 +364,11 @@ pub(crate) fn create_mysql_column( ConcreteDataType::Vector(_) => Ok(ColumnType::MYSQL_TYPE_BLOB), ConcreteDataType::List(_) => Ok(ColumnType::MYSQL_TYPE_VARCHAR), ConcreteDataType::Struct(_) => Ok(ColumnType::MYSQL_TYPE_VARCHAR), - ConcreteDataType::Dictionary(dictionary) => { - return create_mysql_column(dictionary.value_type(), column_name); + _ => error::UnsupportedDataTypeSnafu { + data_type, + reason: "not implemented", } + .fail(), }; let mut colflags = ColumnFlags::empty(); match data_type { diff --git a/src/servers/src/postgres/handler.rs b/src/servers/src/postgres/handler.rs index e372a542375..22aa23cd42f 100644 --- a/src/servers/src/postgres/handler.rs +++ b/src/servers/src/postgres/handler.rs @@ -18,8 +18,8 @@ use std::sync::Arc; use async_trait::async_trait; use common_query::{Output, OutputData}; -use common_recordbatch::RecordBatch; use common_recordbatch::error::Result as RecordBatchResult; +use common_recordbatch::{RecordBatch, map_dictionary_to_values_schema}; use common_telemetry::{debug, info, tracing}; use datafusion::sql::sqlparser::ast::{CopyOption, CopyTarget, Statement as SqlParserStatement}; use datafusion_common::ParamValues; @@ -557,6 +557,7 @@ fn describe_fields( // query SqlPlan::Plan(plan, _) if !matches!(plan, LogicalPlan::Dml(_) | LogicalPlan::Ddl(_)) => { let schema: Schema = plan.schema().clone().try_into().map_err(convert_err)?; + let (schema, _) = map_dictionary_to_values_schema(Arc::new(schema)); schema_to_pg(&schema, format, None).map_err(convert_err) } // We can cover only part of show statements diff --git a/src/servers/src/postgres/types.rs b/src/servers/src/postgres/types.rs index 8c3126ce00c..97ded7cb9b2 100644 --- a/src/servers/src/postgres/types.rs +++ b/src/servers/src/postgres/types.rs @@ -24,14 +24,14 @@ use arrow_pg::encoder::{Encoder, encode_value}; use arrow_pg::list_encoder::encode_list; use arrow_schema::{DataType, TimeUnit}; use chrono::{DateTime, FixedOffset, NaiveDate, NaiveDateTime}; -use common_recordbatch::RecordBatch; use common_recordbatch::error::Result as RecordBatchResult; +use common_recordbatch::{RecordBatch, map_dictionary_to_values_data_type}; use common_time::{IntervalDayTime, IntervalMonthDayNano, IntervalYearMonth}; use datafusion_common::ScalarValue; use datafusion_expr::LogicalPlan; use datatypes::arrow::datatypes::DataType as ArrowDataType; use datatypes::json::JsonSettings; -use datatypes::prelude::{ConcreteDataType, Value}; +use datatypes::prelude::{ConcreteDataType, DataType as _, Value}; use datatypes::schema::{Schema, SchemaRef}; use datatypes::types::{Decimal128Type, IntervalType, TimestampType, jsonb_to_string}; use datatypes::value::StructValue; @@ -222,15 +222,6 @@ where DataType::Struct(_) => { encode_struct(query_ctx, Default::default(), encoder, pg_field)?; } - DataType::Dictionary(_, value_type) - if matches!( - value_type.as_ref(), - DataType::Utf8 | DataType::Utf8View | DataType::LargeUtf8 - ) => - { - let value = datatypes::arrow_array::string_array_value_at_index(column, i); - encoder.encode_field(&value, pg_field)?; - } _ => { // Encode value using arrow-pg let arrow_field = arrow_schema.field(j); @@ -289,7 +280,11 @@ pub(super) fn type_gt_to_pg(origin: &ConcreteDataType) -> Result { } .fail(), }, - ConcreteDataType::Dictionary(dictionary) => type_gt_to_pg(dictionary.value_type()), + &ConcreteDataType::Dictionary(_) => server_error::UnsupportedDataTypeSnafu { + data_type: origin, + reason: "not implemented", + } + .fail(), &ConcreteDataType::Duration(_) => Ok(Type::INTERVAL), &ConcreteDataType::Struct(_) => Ok(Type::JSON), } @@ -375,6 +370,29 @@ fn numeric_out_of_range_error(value: impl std::fmt::Display) -> PgWireError { ) } +fn string_parameter_to_scalar_value( + data: Option, + data_type: &ConcreteDataType, +) -> Option { + match data_type { + ConcreteDataType::String(string_type) => { + if string_type.is_large() { + Some(ScalarValue::LargeUtf8(data)) + } else { + Some(ScalarValue::Utf8(data)) + } + } + ConcreteDataType::Dictionary(dictionary) => Some(ScalarValue::Dictionary( + Box::new(dictionary.key_type().as_arrow_type()), + Box::new(string_parameter_to_scalar_value( + data, + dictionary.value_type(), + )?), + )), + _ => None, + } +} + pub(super) fn parameters_to_scalar_values( plan: &LogicalPlan, portal: &Portal, @@ -398,7 +416,8 @@ pub(super) fn parameters_to_scalar_values( let client_type = if let Some(Some(client_given_type)) = client_param_types.get(idx) { client_given_type.clone() } else if let Some(server_provided_type) = &server_type { - type_gt_to_pg(server_provided_type).map_err(convert_err)? + type_gt_to_pg(&map_dictionary_to_values_data_type(server_provided_type)) + .map_err(convert_err)? } else { return Err(invalid_parameter_error( "unknown_parameter_type", @@ -413,21 +432,12 @@ pub(super) fn parameters_to_scalar_values( &Type::VARCHAR | &Type::TEXT | &Type::CHAR => { let data = portal.parameter::(idx, &client_type)?; if let Some(server_type) = &server_type { - match server_type { - ConcreteDataType::String(t) => { - if t.is_large() { - ScalarValue::LargeUtf8(data) - } else { - ScalarValue::Utf8(data) - } - } - _ => { - return Err(invalid_parameter_error( - "invalid_parameter_type", - Some(format!("Expected: {}, found: {}", server_type, client_type)), - )); - } - } + string_parameter_to_scalar_value(data, server_type).ok_or_else(|| { + invalid_parameter_error( + "invalid_parameter_type", + Some(format!("Expected: {}, found: {}", server_type, client_type)), + ) + })? } else { ScalarValue::Utf8(data) } @@ -1223,7 +1233,7 @@ pub(super) fn param_types_to_pg_types( let mut types = Vec::with_capacity(param_count); for i in 0..param_count { if let Some(Some(param_type)) = param_types.get(&format!("${}", i + 1)) { - let pg_type = type_gt_to_pg(param_type)?; + let pg_type = type_gt_to_pg(&map_dictionary_to_values_data_type(param_type))?; types.push(pg_type); } else { types.push(Type::UNKNOWN); @@ -1745,6 +1755,35 @@ mod test { Portal::try_new(&bind, statement).unwrap() } + #[test] + fn test_dictionary_string_parameter() { + let dictionary_type = + DataType::Dictionary(Box::new(DataType::UInt32), Box::new(DataType::Utf8)); + let plan = build_plan_with_params(vec![("$1", dictionary_type)]); + let portal = make_portal(vec![None], vec![s("host-a")]); + + let values = parameters_to_scalar_values(&plan, &portal).unwrap(); + assert_eq!( + vec![ScalarValue::Dictionary( + Box::new(DataType::UInt32), + Box::new(ScalarValue::Utf8(s("host-a"))), + )], + values + ); + + let param_types = HashMap::from([( + "$1".to_string(), + Some(ConcreteDataType::dictionary_datatype( + ConcreteDataType::uint32_datatype(), + ConcreteDataType::string_datatype(), + )), + )]); + assert_eq!( + vec![Type::VARCHAR], + param_types_to_pg_types(¶m_types).unwrap() + ); + } + #[test] fn test_int2_coerce_in_range() { let plan = build_plan_with_params(vec![ diff --git a/src/servers/tests/mod.rs b/src/servers/tests/mod.rs index 20efce7ca0f..770cc7b20b7 100644 --- a/src/servers/tests/mod.rs +++ b/src/servers/tests/mod.rs @@ -16,19 +16,13 @@ use std::sync::Arc; use api::v1::greptime_request::Request; use api::v1::query_request::Query; -use arrow::array::{DictionaryArray, StringArray, UInt32Array}; -use arrow::datatypes::UInt32Type; use async_trait::async_trait; use catalog::memory::MemoryCatalogManager; use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME}; use common_error::ext::BoxedError; use common_grpc::flight::do_put::DoPutResponse; use common_query::Output; -use common_recordbatch::RecordBatch; use datafusion_expr::LogicalPlan; -use datatypes::prelude::{ConcreteDataType, VectorRef}; -use datatypes::schema::{ColumnSchema, Schema}; -use datatypes::vectors::DictionaryVector; use futures_util::TryFutureExt; use query::options::QueryOptions; use query::parser::{PromQuery, QueryStatement}; @@ -42,7 +36,6 @@ use snafu::{ResultExt, ensure}; use sql::parser::{ParseOptions, ParserContext}; use sql::statements::statement::Statement; use table::TableRef; -use table::test_util::MemTable; mod http; mod interceptor; @@ -50,23 +43,6 @@ mod mysql; mod postgres; mod prom_remote_write_v2_test; -fn dictionary_child_null_table() -> TableRef { - let data_type = ConcreteDataType::dictionary_datatype( - ConcreteDataType::uint32_datatype(), - ConcreteDataType::string_datatype(), - ); - let schema = Arc::new(Schema::new(vec![ColumnSchema::new( - "host", data_type, true, - )])); - let dictionary = DictionaryArray::::new( - UInt32Array::from(vec![0, 1]), - Arc::new(StringArray::from(vec![None, Some("host-b")])), - ); - let vector = Arc::new(DictionaryVector::try_from(dictionary).unwrap()) as VectorRef; - let record_batch = RecordBatch::new(schema, vec![vector]).unwrap(); - MemTable::table("dictionary_values", record_batch) -} - pub struct DummyInstance { query_engine: QueryEngineRef, } diff --git a/src/servers/tests/mysql/mysql_server_test.rs b/src/servers/tests/mysql/mysql_server_test.rs index 7da3cbe4142..888ac92fc3b 100644 --- a/src/servers/tests/mysql/mysql_server_test.rs +++ b/src/servers/tests/mysql/mysql_server_test.rs @@ -34,8 +34,8 @@ use servers::tls::{ReloadableTlsServerConfig, TlsOption}; use table::TableRef; use table::test_util::MemTable; +use crate::create_testing_sql_query_handler; use crate::mysql::{MysqlTextRow, TestingData, all_datatype_testing_data}; -use crate::{create_testing_sql_query_handler, dictionary_child_null_table}; #[derive(Default)] struct MysqlOpts<'a> { @@ -77,28 +77,6 @@ fn create_mysql_server(table: TableRef, opts: MysqlOpts<'_>) -> Result Result<()> { - let mut mysql_server = create_mysql_server(dictionary_child_null_table(), Default::default())?; - mysql_server - .start("127.0.0.1:0".parse::().unwrap()) - .await - .unwrap(); - let server_port = mysql_server.bind_addr().unwrap().port(); - let mut connection = create_connection_default_db_name(server_port, false) - .await - .unwrap(); - - let values: Vec> = connection - .query("SELECT host FROM dictionary_values") - .await - .unwrap(); - assert_eq!(vec![None, Some("host-b".to_string())], values); - - mysql_server.shutdown().await.unwrap(); - Ok(()) -} - #[tokio::test] async fn test_start_mysql_server() -> Result<()> { let table = MemTable::default_numbers_table(); diff --git a/src/servers/tests/postgres/mod.rs b/src/servers/tests/postgres/mod.rs index 6abdbe78d7a..5a3e97e3e71 100644 --- a/src/servers/tests/postgres/mod.rs +++ b/src/servers/tests/postgres/mod.rs @@ -35,7 +35,7 @@ use table::TableRef; use table::test_util::MemTable; use tokio_postgres::{Client, Error as PgError, NoTls, SimpleQueryMessage}; -use crate::{create_testing_instance, dictionary_child_null_table}; +use crate::create_testing_instance; fn create_postgres_server( table: TableRef, @@ -75,35 +75,6 @@ fn create_postgres_server( ))) } -#[tokio::test] -async fn test_postgres_dictionary_child_null() -> Result<()> { - let mut postgres_server = create_postgres_server( - dictionary_child_null_table(), - false, - Default::default(), - None, - )?; - postgres_server - .start("127.0.0.1:0".parse::().unwrap()) - .await - .unwrap(); - let server_port = postgres_server.bind_addr().unwrap().port(); - let client = create_plain_connection(server_port, false).await.unwrap(); - - let rows = client - .query("SELECT host FROM dictionary_values", &[]) - .await - .unwrap(); - let values = rows - .iter() - .map(|row| row.get::<_, Option>(0)) - .collect::>(); - assert_eq!(vec![None, Some("host-b".to_string())], values); - - postgres_server.shutdown().await.unwrap(); - Ok(()) -} - #[tokio::test] pub async fn test_start_postgres_server() -> Result<()> { let table = MemTable::default_numbers_table(); diff --git a/tests/cases/standalone/common/prepare/mysql_prepare.result b/tests/cases/standalone/common/prepare/mysql_prepare.result index 5ef242a891b..f3e9bc368b6 100644 --- a/tests/cases/standalone/common/prepare/mysql_prepare.result +++ b/tests/cases/standalone/common/prepare/mysql_prepare.result @@ -87,6 +87,39 @@ DEALLOCATE stmt; affected_rows: 0 -- test with data +-- A Mito scan exposes string primary keys as dictionaries, but that physical read type must not +-- leak into INSERT Values planning. +CREATE TABLE insert_values_dictionary ( + ts TIMESTAMP TIME INDEX, + host STRING PRIMARY KEY +) ENGINE=mito; + +Affected Rows: 0 + +INSERT INTO insert_values_dictionary VALUES(now(), 'a'); + +Affected Rows: 1 + +SELECT host FROM insert_values_dictionary; + ++------+ +| host | ++------+ +| a | ++------+ + +SELECT array_agg(host) FROM insert_values_dictionary; + ++------------------------------------------+ +| array_agg(insert_values_dictionary.host) | ++------------------------------------------+ +| [a] | ++------------------------------------------+ + +DROP TABLE insert_values_dictionary; + +Affected Rows: 0 + CREATE TABLE IF NOT EXISTS "cake" ( `domain` STRING, is_expire BOOLEAN NULL, diff --git a/tests/cases/standalone/common/prepare/mysql_prepare.sql b/tests/cases/standalone/common/prepare/mysql_prepare.sql index e96e945f881..bf6f2118f1b 100644 --- a/tests/cases/standalone/common/prepare/mysql_prepare.sql +++ b/tests/cases/standalone/common/prepare/mysql_prepare.sql @@ -49,6 +49,21 @@ EXECUTE stmt USING 1, 2; DEALLOCATE stmt; -- test with data +-- A Mito scan exposes string primary keys as dictionaries, but that physical read type must not +-- leak into INSERT Values planning. +CREATE TABLE insert_values_dictionary ( + ts TIMESTAMP TIME INDEX, + host STRING PRIMARY KEY +) ENGINE=mito; + +INSERT INTO insert_values_dictionary VALUES(now(), 'a'); + +SELECT host FROM insert_values_dictionary; + +SELECT array_agg(host) FROM insert_values_dictionary; + +DROP TABLE insert_values_dictionary; + CREATE TABLE IF NOT EXISTS "cake" ( `domain` STRING, is_expire BOOLEAN NULL,