mirror of
https://github.com/GreptimeTeam/greptimedb.git
synced 2026-11-17 08:14:56 +00:00
fix(query): keep dictionary tags within query paths
Signed-off-by: Ruihang Xia <waynestxia@gmail.com>
This commit is contained in:
@@ -18,8 +18,8 @@ use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
use api::v1::{AffectedRows, FlightMetadata, Metrics};
|
||||
use arrow_flight::FlightData;
|
||||
use arrow_flight::utils::flight_data_to_arrow_batch;
|
||||
use arrow_flight::{FlightData, SchemaAsIpc};
|
||||
use common_base::bytes::Bytes;
|
||||
use common_recordbatch::DfRecordBatch;
|
||||
use datatypes::arrow;
|
||||
@@ -190,8 +190,14 @@ impl FlightEncoder {
|
||||
}
|
||||
|
||||
/// Encode the Arrow schema to [FlightData].
|
||||
pub fn encode_schema(&self, schema: &ArrowSchema) -> FlightData {
|
||||
SchemaAsIpc::new(schema, &self.write_options).into()
|
||||
pub fn encode_schema(&mut self, schema: &ArrowSchema) -> FlightData {
|
||||
self.data_gen
|
||||
.schema_to_bytes_with_dictionary_tracker(
|
||||
schema,
|
||||
&mut self.dictionary_tracker,
|
||||
&self.write_options,
|
||||
)
|
||||
.into()
|
||||
}
|
||||
|
||||
/// Encode the [FlightMessage] to a list (at least one element) of [FlightData]s.
|
||||
@@ -202,12 +208,6 @@ impl FlightEncoder {
|
||||
pub fn encode(&mut self, flight_message: FlightMessage) -> Vec1<FlightData> {
|
||||
match flight_message {
|
||||
FlightMessage::Schema(schema) => {
|
||||
schema.fields().iter().for_each(|x| {
|
||||
if matches!(x.data_type(), DataType::Dictionary(_, _)) {
|
||||
self.dictionary_tracker.next_dict_id();
|
||||
}
|
||||
});
|
||||
|
||||
vec1![self.encode_schema(schema.as_ref())]
|
||||
}
|
||||
FlightMessage::RecordBatch(record_batch) => {
|
||||
@@ -474,8 +474,9 @@ fn build_none_flight_msg() -> Bytes {
|
||||
mod test {
|
||||
use arrow_flight::utils::batches_to_flight_data;
|
||||
use datatypes::arrow::array::{
|
||||
DictionaryArray, Int32Array, StringArray, UInt8Array, UInt32Array,
|
||||
DictionaryArray, Int32Array, ListArray, StringArray, UInt8Array, UInt32Array,
|
||||
};
|
||||
use datatypes::arrow::buffer::OffsetBuffer;
|
||||
use datatypes::arrow::datatypes::{DataType, Field, Schema};
|
||||
|
||||
use super::*;
|
||||
@@ -727,6 +728,51 @@ mod test {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_flight_encode_decode_with_nested_dictionary_array() -> Result<()> {
|
||||
let item_field = Arc::new(Field::new_dictionary(
|
||||
"item",
|
||||
DataType::UInt32,
|
||||
DataType::Utf8,
|
||||
true,
|
||||
));
|
||||
let schema = Arc::new(Schema::new(vec![Field::new(
|
||||
"tags",
|
||||
DataType::List(item_field.clone()),
|
||||
true,
|
||||
)]));
|
||||
let values = DictionaryArray::new(
|
||||
UInt32Array::from_iter_values([0, 1, 0]),
|
||||
Arc::new(StringArray::from_iter_values(["host-a", "host-b"])),
|
||||
);
|
||||
let list = ListArray::new(
|
||||
item_field,
|
||||
OffsetBuffer::from_lengths([2, 1]),
|
||||
Arc::new(values),
|
||||
None,
|
||||
);
|
||||
let batch = DfRecordBatch::try_new(schema.clone(), vec![Arc::new(list)]).unwrap();
|
||||
|
||||
let mut encoder = FlightEncoder::default();
|
||||
let encoded_schema = encoder.encode(FlightMessage::Schema(schema.clone()));
|
||||
let encoded_batch = encoder.encode(FlightMessage::RecordBatch(batch.clone()));
|
||||
|
||||
let mut decoder = FlightDecoder::default();
|
||||
assert!(matches!(
|
||||
decoder.try_decode(encoded_schema.first())?,
|
||||
Some(FlightMessage::Schema(actual)) if actual == schema
|
||||
));
|
||||
|
||||
let mut decoded_batch = None;
|
||||
for data in encoded_batch {
|
||||
if let Some(FlightMessage::RecordBatch(decoded)) = decoder.try_decode(&data)? {
|
||||
decoded_batch = Some(decoded);
|
||||
}
|
||||
}
|
||||
assert_eq!(Some(batch), decoded_batch);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_affected_rows_roundtrip_through_flight_codec() {
|
||||
// Verify the full FlightEncoder → FlightDecoder pipeline handles
|
||||
|
||||
@@ -27,7 +27,10 @@ use std::sync::Arc;
|
||||
|
||||
use api::greptime_proto::v1::AddColumnLocation as Location;
|
||||
use api::greptime_proto::v1::add_column_location::LocationType;
|
||||
use common_recordbatch::{RecordBatches, SendableRecordBatchStream};
|
||||
use common_recordbatch::{
|
||||
RecordBatches, SendableRecordBatchMapper, SendableRecordBatchStream, map_dictionary_to_values,
|
||||
map_dictionary_to_values_schema,
|
||||
};
|
||||
use datafusion::physical_plan::ExecutionPlan;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sqlparser_derive::{Visit, VisitMut};
|
||||
@@ -97,6 +100,38 @@ impl Output {
|
||||
Self { data, meta }
|
||||
}
|
||||
|
||||
/// Expands dictionary arrays before exposing a query result to a client.
|
||||
pub fn map_dictionary_to_values(self) -> common_recordbatch::error::Result<Self> {
|
||||
let Self { data, meta } = self;
|
||||
let data = match data {
|
||||
OutputData::AffectedRows(rows) => OutputData::AffectedRows(rows),
|
||||
OutputData::RecordBatches(record_batches) => {
|
||||
let original_schema = record_batches.schema();
|
||||
let (mapped_schema, apply_mapper) =
|
||||
map_dictionary_to_values_schema(original_schema.clone());
|
||||
if !apply_mapper {
|
||||
OutputData::RecordBatches(record_batches)
|
||||
} else {
|
||||
let batches = record_batches
|
||||
.into_iter()
|
||||
.map(|batch| {
|
||||
map_dictionary_to_values(batch, &original_schema, &mapped_schema)
|
||||
})
|
||||
.collect::<common_recordbatch::error::Result<Vec<_>>>()?;
|
||||
OutputData::RecordBatches(RecordBatches::try_new(mapped_schema, batches)?)
|
||||
}
|
||||
}
|
||||
OutputData::Stream(stream) => {
|
||||
OutputData::Stream(Box::pin(SendableRecordBatchMapper::new(
|
||||
stream,
|
||||
map_dictionary_to_values,
|
||||
map_dictionary_to_values_schema,
|
||||
)))
|
||||
}
|
||||
};
|
||||
Ok(Self { data, meta })
|
||||
}
|
||||
|
||||
pub fn extract_rows_and_cost(&self) -> (OutputRows, OutputCost) {
|
||||
match self.data {
|
||||
OutputData::AffectedRows(rows) => (rows, self.meta.cost),
|
||||
|
||||
@@ -40,16 +40,16 @@ use datatypes::arrow::array::{ArrayRef, AsArray, StringBuilder};
|
||||
use datatypes::arrow::compute::SortOptions;
|
||||
pub use datatypes::arrow::record_batch::RecordBatch as DfRecordBatch;
|
||||
use datatypes::arrow::util::pretty;
|
||||
use datatypes::prelude::{ConcreteDataType, VectorRef};
|
||||
use datatypes::prelude::{ConcreteDataType, DataType, VectorRef};
|
||||
use datatypes::schema::{ColumnSchema, Schema, SchemaRef};
|
||||
use datatypes::types::{JsonFormat, jsonb_to_string};
|
||||
use datatypes::types::{JsonFormat, StructField, StructType, jsonb_to_string};
|
||||
use error::Result;
|
||||
use futures::task::{Context, Poll};
|
||||
use futures::{Stream, TryStreamExt};
|
||||
pub use recordbatch::RecordBatch;
|
||||
use snafu::{IntoError, ResultExt, ensure};
|
||||
|
||||
use crate::error::NewDfRecordBatchSnafu;
|
||||
use crate::error::{ArrowComputeSnafu, NewDfRecordBatchSnafu};
|
||||
|
||||
pub trait RecordBatchStream: Stream<Item = Result<RecordBatch>> {
|
||||
fn name(&self) -> &str {
|
||||
@@ -163,6 +163,84 @@ pub fn map_json_type_to_string_schema(schema: SchemaRef) -> (SchemaRef, bool) {
|
||||
(Arc::new(Schema::new(new_columns)), apply_mapper)
|
||||
}
|
||||
|
||||
/// Replaces dictionary types with their value types, including dictionaries nested in lists and
|
||||
/// structs.
|
||||
pub fn map_dictionary_to_values_data_type(data_type: &ConcreteDataType) -> ConcreteDataType {
|
||||
match data_type {
|
||||
ConcreteDataType::Dictionary(dictionary) => {
|
||||
map_dictionary_to_values_data_type(dictionary.value_type())
|
||||
}
|
||||
ConcreteDataType::List(list) => ConcreteDataType::list_datatype(Arc::new(
|
||||
map_dictionary_to_values_data_type(list.item_type()),
|
||||
)),
|
||||
ConcreteDataType::Struct(struct_type) => {
|
||||
let fields = struct_type
|
||||
.fields()
|
||||
.iter()
|
||||
.map(|field| {
|
||||
StructField::new(
|
||||
field.name(),
|
||||
map_dictionary_to_values_data_type(field.data_type()),
|
||||
field.is_nullable(),
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
ConcreteDataType::struct_datatype(StructType::new(Arc::new(fields)))
|
||||
}
|
||||
_ => data_type.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Maps dictionary columns in a schema to their value types.
|
||||
pub fn map_dictionary_to_values_schema(schema: SchemaRef) -> (SchemaRef, bool) {
|
||||
let mut apply_mapper = false;
|
||||
let columns = schema
|
||||
.column_schemas()
|
||||
.iter()
|
||||
.map(|column| {
|
||||
let data_type = map_dictionary_to_values_data_type(&column.data_type);
|
||||
apply_mapper |= data_type != column.data_type;
|
||||
let mut column = column.clone();
|
||||
column.data_type = data_type;
|
||||
column
|
||||
})
|
||||
.collect();
|
||||
|
||||
(
|
||||
Arc::new(Schema::new_with_version(columns, schema.version())),
|
||||
apply_mapper,
|
||||
)
|
||||
}
|
||||
|
||||
/// Expands dictionary arrays to their value arrays according to `mapped_schema`.
|
||||
pub fn map_dictionary_to_values(
|
||||
batch: RecordBatch,
|
||||
original_schema: &SchemaRef,
|
||||
mapped_schema: &SchemaRef,
|
||||
) -> Result<RecordBatch> {
|
||||
let arrays = batch
|
||||
.columns()
|
||||
.iter()
|
||||
.zip(original_schema.column_schemas())
|
||||
.zip(mapped_schema.column_schemas())
|
||||
.map(|((array, original), mapped)| {
|
||||
if original.data_type == mapped.data_type {
|
||||
Ok(array.clone())
|
||||
} else {
|
||||
datatypes::arrow::compute::cast(array, &mapped.data_type.as_arrow_type())
|
||||
.context(ArrowComputeSnafu)
|
||||
}
|
||||
})
|
||||
.collect::<Result<Vec<_>>>()?;
|
||||
|
||||
let record_batch = DfRecordBatch::try_new(mapped_schema.arrow_schema().clone(), arrays)
|
||||
.context(NewDfRecordBatchSnafu)?;
|
||||
Ok(RecordBatch::from_df_record_batch(
|
||||
mapped_schema.clone(),
|
||||
record_batch,
|
||||
))
|
||||
}
|
||||
|
||||
impl SendableRecordBatchMapper {
|
||||
/// Creates a new [SendableRecordBatchMapper] with the given inner [RecordBatchStream], mapper function, and schema mapper function.
|
||||
pub fn new(
|
||||
@@ -835,6 +913,11 @@ mod tests {
|
||||
use std::time::Duration;
|
||||
|
||||
use common_memory_manager::{OnExhaustedPolicy, PermitGranularity};
|
||||
use datatypes::arrow::array::{
|
||||
DictionaryArray, Int32Array, ListArray, StringArray, UInt32Array,
|
||||
};
|
||||
use datatypes::arrow::buffer::OffsetBuffer;
|
||||
use datatypes::arrow::datatypes::{DataType as ArrowDataType, Int32Type};
|
||||
use datatypes::prelude::{ConcreteDataType, VectorRef};
|
||||
use datatypes::schema::{ColumnSchema, Schema};
|
||||
use datatypes::vectors::{BooleanVector, Int32Vector, StringVector};
|
||||
@@ -918,6 +1001,83 @@ mod tests {
|
||||
assert_eq!(vec![batch1], batches.take());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_map_dictionary_to_values_recursively() {
|
||||
let string_dictionary = ConcreteDataType::dictionary_datatype(
|
||||
ConcreteDataType::int32_datatype(),
|
||||
ConcreteDataType::string_datatype(),
|
||||
);
|
||||
let list_dictionary =
|
||||
ConcreteDataType::list_datatype(Arc::new(ConcreteDataType::dictionary_datatype(
|
||||
ConcreteDataType::uint32_datatype(),
|
||||
ConcreteDataType::string_datatype(),
|
||||
)));
|
||||
let schema = Arc::new(Schema::new(vec![
|
||||
ColumnSchema::new("host", string_dictionary, true),
|
||||
ColumnSchema::new("tags", list_dictionary, true),
|
||||
]));
|
||||
|
||||
let host = DictionaryArray::<Int32Type>::new(
|
||||
Int32Array::from(vec![0, 1]),
|
||||
Arc::new(StringArray::from(vec![Some("host-a"), None])),
|
||||
);
|
||||
let ArrowDataType::List(item_field) = schema.arrow_schema().field(1).data_type().clone()
|
||||
else {
|
||||
unreachable!()
|
||||
};
|
||||
let tag_values = DictionaryArray::new(
|
||||
UInt32Array::from_iter_values([0, 1, 0]),
|
||||
Arc::new(StringArray::from(vec![Some("a"), None])),
|
||||
);
|
||||
let tags = ListArray::new(
|
||||
item_field,
|
||||
OffsetBuffer::from_lengths([2, 1]),
|
||||
Arc::new(tag_values),
|
||||
None,
|
||||
);
|
||||
let batch = DfRecordBatch::try_new(
|
||||
schema.arrow_schema().clone(),
|
||||
vec![Arc::new(host), Arc::new(tags)],
|
||||
)
|
||||
.unwrap();
|
||||
let batch = RecordBatch::from_df_record_batch(schema.clone(), batch);
|
||||
|
||||
let (mapped_schema, apply_mapper) = map_dictionary_to_values_schema(schema.clone());
|
||||
assert!(apply_mapper);
|
||||
assert_eq!(
|
||||
&ArrowDataType::Utf8,
|
||||
mapped_schema.arrow_schema().field(0).data_type()
|
||||
);
|
||||
assert_eq!(
|
||||
&ArrowDataType::List(Arc::new(
|
||||
datatypes::arrow::datatypes::Field::new_list_field(ArrowDataType::Utf8, true,)
|
||||
)),
|
||||
mapped_schema.arrow_schema().field(1).data_type()
|
||||
);
|
||||
|
||||
let mapped = map_dictionary_to_values(batch, &schema, &mapped_schema).unwrap();
|
||||
let host = mapped
|
||||
.column(0)
|
||||
.as_any()
|
||||
.downcast_ref::<StringArray>()
|
||||
.unwrap();
|
||||
assert_eq!(vec![Some("host-a"), None], host.iter().collect::<Vec<_>>());
|
||||
let tags = mapped
|
||||
.column(1)
|
||||
.as_any()
|
||||
.downcast_ref::<ListArray>()
|
||||
.unwrap();
|
||||
let tag_values = tags
|
||||
.values()
|
||||
.as_any()
|
||||
.downcast_ref::<StringArray>()
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
vec![Some("a"), None, Some("a")],
|
||||
tag_values.iter().collect::<Vec<_>>()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_simple_recordbatch_stream() {
|
||||
let column_a = ColumnSchema::new("a", ConcreteDataType::int32_datatype(), false);
|
||||
|
||||
@@ -303,10 +303,10 @@ impl RecordBatch {
|
||||
}
|
||||
_ => {
|
||||
if let Ok(column) = Helper::try_into_vector(column) {
|
||||
Box::new(
|
||||
(0..column.len())
|
||||
.map(move |i| (!column.is_null(i)).then(|| column.get(i).to_string())),
|
||||
)
|
||||
Box::new((0..column.len()).map(move |i| {
|
||||
let value = column.get_ref(i);
|
||||
(!value.is_null()).then(|| column.get(i).to_string())
|
||||
}))
|
||||
} else {
|
||||
Box::new(std::iter::empty())
|
||||
}
|
||||
@@ -382,13 +382,13 @@ fn maybe_align_json_array_with_schema(
|
||||
mod tests {
|
||||
use std::sync::Arc;
|
||||
|
||||
use datatypes::arrow::array::{AsArray, BinaryArray, UInt32Array};
|
||||
use datatypes::arrow::array::{AsArray, BinaryArray, DictionaryArray, UInt32Array};
|
||||
use datatypes::arrow::datatypes::{DataType, Field, Schema as ArrowSchema, UInt32Type};
|
||||
use datatypes::arrow_array::StringArray;
|
||||
use datatypes::data_type::ConcreteDataType;
|
||||
use datatypes::extension::json::{JsonExtensionType, JsonMetadata};
|
||||
use datatypes::schema::{ColumnSchema, Schema};
|
||||
use datatypes::vectors::{StringVector, UInt32Vector};
|
||||
use datatypes::vectors::{DictionaryVector, StringVector, UInt32Vector};
|
||||
|
||||
use super::*;
|
||||
|
||||
@@ -475,6 +475,28 @@ mod tests {
|
||||
assert!(recordbatch.slice(1, 5).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_iter_dictionary_column_uses_logical_nulls() {
|
||||
let data_type = ConcreteDataType::dictionary_datatype(
|
||||
ConcreteDataType::uint32_datatype(),
|
||||
ConcreteDataType::string_datatype(),
|
||||
);
|
||||
let schema = Arc::new(Schema::new(vec![ColumnSchema::new(
|
||||
"host", data_type, true,
|
||||
)]));
|
||||
let dictionary = DictionaryArray::<UInt32Type>::new(
|
||||
UInt32Array::from(vec![Some(0), Some(1), None]),
|
||||
Arc::new(StringArray::from(vec![None, Some("host-b")])),
|
||||
);
|
||||
let vector = Arc::new(DictionaryVector::try_from(dictionary).unwrap()) as VectorRef;
|
||||
let batch = RecordBatch::new(schema, vec![vector]).unwrap();
|
||||
|
||||
assert_eq!(
|
||||
vec![None, Some("host-b".to_string()), None],
|
||||
batch.iter_column_as_string(0).collect::<Vec<_>>()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_merge_record_batch() {
|
||||
let column_schemas = vec![
|
||||
|
||||
@@ -164,7 +164,7 @@ fn build_by_tql_schema(plan: &LogicalPlan) -> Result<TableDef, Error> {
|
||||
.fields()
|
||||
.iter()
|
||||
.filter_map(|f| {
|
||||
if ConcreteDataType::from_arrow_type(f.data_type()).is_string() {
|
||||
if is_tql_label_type(&ConcreteDataType::from_arrow_type(f.data_type())) {
|
||||
Some(f.name().clone())
|
||||
} else {
|
||||
None
|
||||
@@ -178,6 +178,14 @@ fn build_by_tql_schema(plan: &LogicalPlan) -> Result<TableDef, Error> {
|
||||
})
|
||||
}
|
||||
|
||||
fn is_tql_label_type(data_type: &ConcreteDataType) -> bool {
|
||||
data_type.is_string()
|
||||
|| matches!(
|
||||
data_type,
|
||||
ConcreteDataType::Dictionary(dictionary) if dictionary.value_type().is_string()
|
||||
)
|
||||
}
|
||||
|
||||
struct TableDef {
|
||||
ts_col: Option<String>,
|
||||
pks: Vec<String>,
|
||||
@@ -246,7 +254,14 @@ fn build_pk_from_aggr(plan: &LogicalPlan) -> Result<Option<TableDef>, Error> {
|
||||
|
||||
#[cfg(test)]
|
||||
mod test {
|
||||
use std::sync::Arc;
|
||||
|
||||
use api::v1::column_def::try_as_column_schema;
|
||||
use datafusion::arrow::datatypes::{
|
||||
DataType as ArrowDataType, Field, Schema as ArrowSchema, TimeUnit,
|
||||
};
|
||||
use datafusion_common::DFSchema;
|
||||
use datafusion_expr::logical_plan::EmptyRelation;
|
||||
use datatypes::prelude::ConcreteDataType;
|
||||
use datatypes::schema::ColumnSchema;
|
||||
use pretty_assertions::assert_eq;
|
||||
@@ -257,6 +272,45 @@ mod test {
|
||||
use crate::batching_mode::utils::sql_to_df_plan;
|
||||
use crate::test_utils::create_test_query_engine;
|
||||
|
||||
#[test]
|
||||
fn test_tql_dictionary_string_is_label() {
|
||||
let arrow_schema = Arc::new(ArrowSchema::new(vec![
|
||||
Field::new_dictionary("host", ArrowDataType::UInt32, ArrowDataType::Utf8, true),
|
||||
Field::new("value", ArrowDataType::Float64, true),
|
||||
Field::new(
|
||||
"ts",
|
||||
ArrowDataType::Timestamp(TimeUnit::Millisecond, None),
|
||||
false,
|
||||
),
|
||||
]));
|
||||
let plan = LogicalPlan::EmptyRelation(EmptyRelation {
|
||||
produce_one_row: false,
|
||||
schema: Arc::new(DFSchema::try_from(arrow_schema).unwrap()),
|
||||
});
|
||||
|
||||
let expr = create_table_with_expr(
|
||||
&plan,
|
||||
&[
|
||||
"greptime".to_string(),
|
||||
"public".to_string(),
|
||||
"sink".to_string(),
|
||||
],
|
||||
&QueryType::Tql,
|
||||
)
|
||||
.unwrap();
|
||||
let columns = expr
|
||||
.column_defs
|
||||
.iter()
|
||||
.map(|column| try_as_column_schema(column).unwrap())
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
assert_eq!(vec!["host".to_string()], expr.primary_keys);
|
||||
assert_eq!("ts", expr.time_index);
|
||||
assert_eq!(ConcreteDataType::string_datatype(), columns[0].data_type);
|
||||
assert_eq!(ConcreteDataType::float64_datatype(), columns[1].data_type);
|
||||
assert!(columns[2].is_time_index());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_gen_create_table_sql() {
|
||||
let query_engine = create_test_query_engine();
|
||||
|
||||
@@ -95,9 +95,9 @@ use table::requests::{OTLP_METRIC_COMPAT_KEY, OTLP_METRIC_COMPAT_PROM};
|
||||
use tracing::Span;
|
||||
|
||||
use crate::error::{
|
||||
self, Error, ExecLogicalPlanSnafu, ExecutePromqlSnafu, ExternalSnafu, InvalidSqlSnafu,
|
||||
ParseSqlSnafu, PermissionSnafu, PlanStatementSnafu, Result, SqlExecInterceptedSnafu,
|
||||
StatementTimeoutSnafu, TableOperationSnafu,
|
||||
self, CollectRecordbatchSnafu, Error, ExecLogicalPlanSnafu, ExecutePromqlSnafu, ExternalSnafu,
|
||||
InvalidSqlSnafu, ParseSqlSnafu, PermissionSnafu, PlanStatementSnafu, Result,
|
||||
SqlExecInterceptedSnafu, StatementTimeoutSnafu, TableOperationSnafu,
|
||||
};
|
||||
use crate::service_config::InfluxdbMergeMode;
|
||||
use crate::stream_wrapper::CancellableStreamWrapper;
|
||||
@@ -885,7 +885,12 @@ impl SqlQueryHandler for Instance {
|
||||
self.do_query_inner(query, query_ctx)
|
||||
.await
|
||||
.into_iter()
|
||||
.map(|result| result.map_err(BoxedError::new).context(ExecuteQuerySnafu))
|
||||
.map(|result| {
|
||||
result
|
||||
.and_then(map_query_output)
|
||||
.map_err(BoxedError::new)
|
||||
.context(ExecuteQuerySnafu)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
@@ -896,6 +901,7 @@ impl SqlQueryHandler for Instance {
|
||||
) -> server_error::Result<Output> {
|
||||
self.do_analyze_stream_query_inner(query, query_ctx)
|
||||
.await
|
||||
.and_then(map_query_output)
|
||||
.map_err(BoxedError::new)
|
||||
.context(ExecuteQuerySnafu)
|
||||
}
|
||||
@@ -908,6 +914,7 @@ impl SqlQueryHandler for Instance {
|
||||
) -> server_error::Result<Output> {
|
||||
self.do_exec_plan_inner(plan, stmt, query_ctx)
|
||||
.await
|
||||
.and_then(map_query_output)
|
||||
.map_err(BoxedError::new)
|
||||
.context(server_error::ExecutePlanSnafu)
|
||||
}
|
||||
@@ -920,7 +927,12 @@ impl SqlQueryHandler for Instance {
|
||||
self.do_promql_query_inner(query, query_ctx)
|
||||
.await
|
||||
.into_iter()
|
||||
.map(|result| result.map_err(BoxedError::new).context(ExecuteQuerySnafu))
|
||||
.map(|result| {
|
||||
result
|
||||
.and_then(map_query_output)
|
||||
.map_err(BoxedError::new)
|
||||
.context(ExecuteQuerySnafu)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
@@ -943,6 +955,13 @@ impl SqlQueryHandler for Instance {
|
||||
}
|
||||
}
|
||||
|
||||
/// Expands scan-time dictionaries only when a query result leaves the frontend.
|
||||
pub(crate) fn map_query_output(output: Output) -> Result<Output> {
|
||||
output
|
||||
.map_dictionary_to_values()
|
||||
.context(CollectRecordbatchSnafu)
|
||||
}
|
||||
|
||||
/// Attaches a timer to the output and observes it once the output is exhausted.
|
||||
pub fn attach_timer(output: Output, timer: HistogramTimer) -> Output {
|
||||
match output.data {
|
||||
@@ -1048,7 +1067,10 @@ impl PrometheusHandler for Instance {
|
||||
.map_err(BoxedError::new)
|
||||
.context(ExecuteQuerySnafu)?;
|
||||
|
||||
Ok(interceptor.post_execute(output, query_ctx)?)
|
||||
let output = interceptor.post_execute(output, query_ctx)?;
|
||||
map_query_output(output)
|
||||
.map_err(BoxedError::new)
|
||||
.context(ExecuteQuerySnafu)
|
||||
}
|
||||
|
||||
async fn query_metric_names(
|
||||
|
||||
@@ -51,7 +51,7 @@ use crate::error::{
|
||||
NotSupportedSnafu, PermissionSnafu, PlanStatementSnafu, Result,
|
||||
SubstraitDecodeLogicalPlanSnafu, TableNotFoundSnafu, TableOperationSnafu,
|
||||
};
|
||||
use crate::instance::{Instance, attach_timer};
|
||||
use crate::instance::{Instance, attach_timer, map_query_output};
|
||||
use crate::metrics::{
|
||||
GRPC_HANDLE_PLAN_ELAPSED, GRPC_HANDLE_PROMQL_ELAPSED, GRPC_HANDLE_SQL_ELAPSED,
|
||||
};
|
||||
@@ -251,7 +251,7 @@ impl GrpcQueryHandler for Instance {
|
||||
};
|
||||
|
||||
let output = interceptor.post_execute(output, ctx)?;
|
||||
Ok(output)
|
||||
map_query_output(output)
|
||||
}
|
||||
.await;
|
||||
|
||||
|
||||
@@ -408,7 +408,9 @@ async fn query_trace_table(
|
||||
RecordBatchStreamAdapter::try_new(stream).context(CollectRecordbatchSnafu)?,
|
||||
));
|
||||
|
||||
Ok(output)
|
||||
output
|
||||
.map_dictionary_to_values()
|
||||
.context(CollectRecordbatchSnafu)
|
||||
}
|
||||
|
||||
async fn get_table(
|
||||
@@ -485,7 +487,9 @@ async fn find_traces_rank_3(
|
||||
RecordBatchStreamAdapter::try_new(stream).context(CollectRecordbatchSnafu)?,
|
||||
));
|
||||
|
||||
Ok(output)
|
||||
output
|
||||
.map_dictionary_to_values()
|
||||
.context(CollectRecordbatchSnafu)
|
||||
}
|
||||
|
||||
// The current implementation registers UDFs during the planning stage, which makes it difficult
|
||||
|
||||
@@ -26,7 +26,7 @@ use session::context::{QueryContext, QueryContextRef};
|
||||
use snafu::ResultExt;
|
||||
use tonic::async_trait;
|
||||
|
||||
use crate::instance::Instance;
|
||||
use crate::instance::{Instance, map_query_output};
|
||||
|
||||
#[async_trait]
|
||||
impl LogQueryHandler for Instance {
|
||||
@@ -64,7 +64,10 @@ impl LogQueryHandler for Instance {
|
||||
.map_err(BoxedError::new)
|
||||
.context(ExecuteQuerySnafu)?;
|
||||
|
||||
Ok(interceptor.as_ref().post_query(output, ctx.clone())?)
|
||||
let output = interceptor.as_ref().post_query(output, ctx.clone())?;
|
||||
map_query_output(output)
|
||||
.map_err(BoxedError::new)
|
||||
.context(ExecuteQuerySnafu)
|
||||
}
|
||||
|
||||
fn catalog_manager(&self, _ctx: &QueryContext) -> ServerResult<&dyn catalog::CatalogManager> {
|
||||
|
||||
@@ -167,10 +167,6 @@ impl AnalyzerRule for DistPlannerAnalyzer {
|
||||
}
|
||||
}
|
||||
};
|
||||
let plan = plan
|
||||
.transform_down_with_subqueries(&Self::unwrap_dictionary_literals)?
|
||||
.data;
|
||||
|
||||
let result = match self.try_push_down(plan.clone()) {
|
||||
Ok(plan) => plan,
|
||||
Err(err) => {
|
||||
@@ -738,9 +734,16 @@ impl PlanRewriter {
|
||||
self.partition_cols
|
||||
);
|
||||
|
||||
// Substrait does not encode dictionary literals. Only unwrap literals in the remote
|
||||
// input; rewriting the whole plan also changes DML Values without rebuilding its schema.
|
||||
let remote_input = on_node
|
||||
.clone()
|
||||
.transform_down_with_subqueries(&DistPlannerAnalyzer::unwrap_dictionary_literals)?
|
||||
.data;
|
||||
|
||||
// add merge scan as the new root
|
||||
let mut node = MergeScanLogicalPlan::new(
|
||||
on_node.clone(),
|
||||
remote_input,
|
||||
false,
|
||||
// at this stage, the partition cols should be set
|
||||
// treat it as non-partitioned if None
|
||||
|
||||
@@ -17,7 +17,7 @@ use std::pin::Pin;
|
||||
use std::sync::Arc;
|
||||
|
||||
use arrow::datatypes::{DataType, IntervalDayTime, TimeUnit};
|
||||
use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
|
||||
use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, MITO_ENGINE};
|
||||
use common_error::ext::BoxedError;
|
||||
use common_function::aggrs::aggr_wrapper::{StateMergeHelper, StateWrapper};
|
||||
use common_recordbatch::adapter::RecordBatchMetrics;
|
||||
@@ -287,6 +287,63 @@ impl Stream for EmptyStream {
|
||||
}
|
||||
}
|
||||
|
||||
fn find_merge_scan(plan: &LogicalPlan) -> Option<&MergeScanLogicalPlan> {
|
||||
if let LogicalPlan::Extension(extension) = plan
|
||||
&& let Some(merge_scan) = extension
|
||||
.node
|
||||
.as_any()
|
||||
.downcast_ref::<MergeScanLogicalPlan>()
|
||||
{
|
||||
return Some(merge_scan);
|
||||
}
|
||||
|
||||
plan.inputs().into_iter().find_map(find_merge_scan)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dictionary_literals_are_unwrapped_only_in_remote_input() {
|
||||
let table = Arc::new(Table::new(
|
||||
TestTable::table_info(0, "t".to_string(), MITO_ENGINE.to_string()),
|
||||
FilterPushDownType::Inexact,
|
||||
Arc::new(TestDataSource::new(TestTable::schema())),
|
||||
));
|
||||
let table_source = Arc::new(DefaultTableSource::new(Arc::new(
|
||||
DfTableProviderAdapter::new(table),
|
||||
)));
|
||||
let literal = Expr::Literal(
|
||||
ScalarValue::Dictionary(
|
||||
Box::new(DataType::UInt32),
|
||||
Box::new(ScalarValue::Utf8(Some("host-a".to_string()))),
|
||||
),
|
||||
None,
|
||||
);
|
||||
let plan = LogicalPlanBuilder::scan_with_filters("t", table_source, None, vec![])
|
||||
.unwrap()
|
||||
.filter(col("pk1").eq(literal))
|
||||
.unwrap()
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let result = DistPlannerAnalyzer {}
|
||||
.analyze(plan, &ConfigOptions::default())
|
||||
.unwrap();
|
||||
assert!(
|
||||
result
|
||||
.to_string()
|
||||
.contains("Filter: t.pk1 = Dictionary(UInt32, Utf8(\"host-a\"))")
|
||||
);
|
||||
|
||||
let remote_input = find_merge_scan(&result).unwrap().input();
|
||||
assert!(
|
||||
remote_input
|
||||
.to_string()
|
||||
.contains("partial_filters=[t.pk1 = Utf8(\"host-a\")]")
|
||||
);
|
||||
DFLogicalSubstraitConvertor
|
||||
.encode(remote_input, DefaultSerializer)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[cfg(feature = "vector_index")]
|
||||
mod vector_search_tests {
|
||||
use std::sync::Arc;
|
||||
|
||||
+2
-47
@@ -216,22 +216,13 @@ impl OutputSchema {
|
||||
|
||||
impl From<SchemaRef> for OutputSchema {
|
||||
fn from(schema: SchemaRef) -> OutputSchema {
|
||||
fn public_type_name(data_type: &datatypes::prelude::ConcreteDataType) -> String {
|
||||
match data_type {
|
||||
datatypes::prelude::ConcreteDataType::Dictionary(dictionary) => {
|
||||
public_type_name(dictionary.value_type())
|
||||
}
|
||||
data_type => data_type.name(),
|
||||
}
|
||||
}
|
||||
|
||||
OutputSchema {
|
||||
column_schemas: schema
|
||||
.column_schemas()
|
||||
.iter()
|
||||
.map(|cs| ColumnSchema {
|
||||
name: cs.name.clone(),
|
||||
data_type: public_type_name(&cs.data_type),
|
||||
data_type: cs.data_type.name(),
|
||||
})
|
||||
.collect(),
|
||||
}
|
||||
@@ -1366,8 +1357,6 @@ mod test {
|
||||
use std::io::Cursor;
|
||||
use std::sync::Arc;
|
||||
|
||||
use arrow::array::{DictionaryArray, StringArray, UInt32Array};
|
||||
use arrow::datatypes::UInt32Type;
|
||||
use arrow_ipc::reader::StreamReader;
|
||||
use arrow_schema::DataType;
|
||||
use axum::handler::Handler;
|
||||
@@ -1378,7 +1367,7 @@ mod test {
|
||||
use datafusion_expr::LogicalPlan;
|
||||
use datatypes::prelude::*;
|
||||
use datatypes::schema::{ColumnSchema, Schema};
|
||||
use datatypes::vectors::{DictionaryVector, StringVector, UInt32Vector};
|
||||
use datatypes::vectors::{StringVector, UInt32Vector};
|
||||
use header::constants::GREPTIME_DB_HEADER_TIMEOUT;
|
||||
use query::parser::PromQuery;
|
||||
use query::query_engine::DescribeResult;
|
||||
@@ -1437,40 +1426,6 @@ mod test {
|
||||
DynamicTimeoutLayer::new(Duration::from_millis(10))
|
||||
}
|
||||
|
||||
fn dictionary_record_batch() -> (SchemaRef, RecordBatch) {
|
||||
let data_type = ConcreteDataType::dictionary_datatype(
|
||||
ConcreteDataType::uint32_datatype(),
|
||||
ConcreteDataType::string_datatype(),
|
||||
);
|
||||
let schema = Arc::new(Schema::new(vec![ColumnSchema::new(
|
||||
"host", data_type, true,
|
||||
)]));
|
||||
let dictionary = DictionaryArray::<UInt32Type>::new(
|
||||
UInt32Array::from(vec![0, 1]),
|
||||
Arc::new(StringArray::from(vec![Some("host-a"), None])),
|
||||
);
|
||||
let vector = Arc::new(DictionaryVector::try_from(dictionary).unwrap()) as VectorRef;
|
||||
let batch = RecordBatch::new(schema.clone(), vec![vector]).unwrap();
|
||||
(schema, batch)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_http_dictionary_output_uses_logical_string_type() {
|
||||
let (schema, batch) = dictionary_record_batch();
|
||||
let output = HttpRecordsOutput::try_new(schema.clone(), vec![batch]).unwrap();
|
||||
|
||||
assert_eq!("String", output.schema.column_schemas[0].data_type);
|
||||
assert_eq!(
|
||||
serde_json::Value::String("host-a".to_string()),
|
||||
output.rows[0][0]
|
||||
);
|
||||
assert_eq!(serde_json::Value::Null, output.rows[1][0]);
|
||||
|
||||
let empty = HttpRecordsOutput::try_new(schema, vec![]).unwrap();
|
||||
assert_eq!("String", empty.schema.column_schemas[0].data_type);
|
||||
assert!(empty.rows.is_empty());
|
||||
}
|
||||
|
||||
async fn forever() {
|
||||
pending().await
|
||||
}
|
||||
|
||||
@@ -893,21 +893,6 @@ impl RowWriter {
|
||||
let v = datatypes::arrow_array::string_array_value(array, i);
|
||||
self.insert(column, v);
|
||||
}
|
||||
DataType::Dictionary(key_type, value_type)
|
||||
if key_type.as_ref() == &DataType::UInt32
|
||||
&& matches!(
|
||||
value_type.as_ref(),
|
||||
DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View
|
||||
) =>
|
||||
{
|
||||
if let Some(v) =
|
||||
datatypes::arrow_array::string_array_value_at_index(array, i)
|
||||
{
|
||||
self.insert(column, v);
|
||||
} else {
|
||||
self.insert(column, "Null");
|
||||
}
|
||||
}
|
||||
DataType::Binary | DataType::LargeBinary | DataType::BinaryView => {
|
||||
let v = datatypes::arrow_array::binary_array_value(array, i);
|
||||
let column_schema = &schema.column_schemas()[j];
|
||||
@@ -1697,14 +1682,10 @@ mod tests {
|
||||
use std::collections::HashSet;
|
||||
use std::sync::Arc;
|
||||
|
||||
use arrow::array::{DictionaryArray, StringArray, UInt32Array};
|
||||
use arrow::datatypes::UInt32Type;
|
||||
use catalog::memory::MemoryCatalogManager;
|
||||
use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
|
||||
use common_recordbatch::RecordBatches;
|
||||
use datatypes::prelude::ConcreteDataType;
|
||||
use datatypes::schema::{ColumnSchema, Schema};
|
||||
use datatypes::vectors::{DictionaryVector, VectorRef};
|
||||
use promql_parser::parser::value::ValueType;
|
||||
use table::metadata::{TableInfoBuilder, TableMetaBuilder, TableType, TableVersion};
|
||||
use table::test_util::EmptyTable;
|
||||
@@ -1719,43 +1700,6 @@ mod tests {
|
||||
should_error: bool,
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_record_batches_to_series_with_dictionary_labels() {
|
||||
let data_type = ConcreteDataType::dictionary_datatype(
|
||||
ConcreteDataType::uint32_datatype(),
|
||||
ConcreteDataType::string_datatype(),
|
||||
);
|
||||
let schema = Arc::new(Schema::new(vec![ColumnSchema::new(
|
||||
"host", data_type, true,
|
||||
)]));
|
||||
let dictionary = DictionaryArray::<UInt32Type>::new(
|
||||
UInt32Array::from(vec![0, 1]),
|
||||
Arc::new(StringArray::from(vec![Some("host-a"), None])),
|
||||
);
|
||||
let vector = Arc::new(DictionaryVector::try_from(dictionary).unwrap()) as VectorRef;
|
||||
let batch = RecordBatch::new(schema.clone(), vec![vector]).unwrap();
|
||||
let batches = RecordBatches::try_new(schema, vec![batch]).unwrap();
|
||||
let mut series = Vec::new();
|
||||
|
||||
record_batches_to_series(
|
||||
batches,
|
||||
&mut series,
|
||||
"metric",
|
||||
&HashSet::from(["host".to_string()]),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(2, series.len());
|
||||
assert_eq!(
|
||||
Some("host-a"),
|
||||
series[0].get(&Column::from("host")).map(String::as_str)
|
||||
);
|
||||
assert_eq!(
|
||||
Some("Null"),
|
||||
series[1].get(&Column::from("host")).map(String::as_str)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_retrieve_metric_name_and_result_type() {
|
||||
let test_cases = &[
|
||||
|
||||
@@ -179,21 +179,6 @@ impl HttpOutputWriter {
|
||||
let v = datatypes::arrow_array::string_array_value(array, i);
|
||||
self.write_value(v)?;
|
||||
}
|
||||
DataType::Dictionary(key_type, value_type)
|
||||
if key_type.as_ref() == &DataType::UInt32
|
||||
&& matches!(
|
||||
value_type.as_ref(),
|
||||
DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View
|
||||
) =>
|
||||
{
|
||||
if let Some(v) =
|
||||
datatypes::arrow_array::string_array_value_at_index(array, i)
|
||||
{
|
||||
self.write_value(v)?;
|
||||
} else {
|
||||
self.write_value(Value::Null)?;
|
||||
}
|
||||
}
|
||||
DataType::Binary | DataType::LargeBinary | DataType::BinaryView => {
|
||||
let v = datatypes::arrow_array::binary_array_value(array, i);
|
||||
self.write_bytes(v, &schema.data_type)?;
|
||||
|
||||
@@ -228,12 +228,6 @@ impl PrometheusJsonResponse {
|
||||
tag_column_indices.push(i);
|
||||
num_label_columns += 1;
|
||||
}
|
||||
ConcreteDataType::Dictionary(ref dictionary)
|
||||
if matches!(dictionary.value_type(), ConcreteDataType::String(_)) =>
|
||||
{
|
||||
tag_column_indices.push(i);
|
||||
num_label_columns += 1;
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
@@ -254,7 +248,7 @@ impl PrometheusJsonResponse {
|
||||
// prepare things...
|
||||
let tag_columns = tag_column_indices
|
||||
.iter()
|
||||
.map(|i| batch.column(*i))
|
||||
.map(|i| batch.column(*i).as_string::<i32>())
|
||||
.collect::<Vec<_>>();
|
||||
let tag_names = tag_column_indices
|
||||
.iter()
|
||||
@@ -286,10 +280,8 @@ impl PrometheusJsonResponse {
|
||||
}
|
||||
for (tag_column, tag_name) in tag_columns.iter().zip(tag_names.iter()) {
|
||||
// TODO(ruihang): add test for NULL tag
|
||||
if let Some(value) = datatypes::arrow_array::string_array_value_at_index(
|
||||
tag_column, row_index,
|
||||
) {
|
||||
tags.push((tag_name, value));
|
||||
if tag_column.is_valid(row_index) {
|
||||
tags.push((tag_name, tag_column.value(row_index)));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -24,6 +24,7 @@ use chrono::{NaiveDate, NaiveDateTime};
|
||||
use common_catalog::parse_optional_catalog_and_schema_from_db_string;
|
||||
use common_error::ext::ErrorExt;
|
||||
use common_query::Output;
|
||||
use common_recordbatch::map_dictionary_to_values_data_type;
|
||||
use common_telemetry::{debug, error, tracing, warn};
|
||||
use datafusion_common::ParamValues;
|
||||
use datafusion_expr::LogicalPlan;
|
||||
@@ -233,7 +234,10 @@ impl MysqlInstanceShim {
|
||||
.column_schemas()
|
||||
.iter()
|
||||
.map(|column_schema| {
|
||||
create_mysql_column(&column_schema.data_type, &column_schema.name)
|
||||
create_mysql_column(
|
||||
&map_dictionary_to_values_data_type(&column_schema.data_type),
|
||||
&column_schema.name,
|
||||
)
|
||||
})
|
||||
.collect::<Result<Vec<_>>>()
|
||||
})
|
||||
@@ -906,7 +910,7 @@ fn prepared_params(
|
||||
// Placeholder index starts from 1
|
||||
for i in 1..param_num {
|
||||
let column = if let Some(Some(t)) = param_types.get(&format_placeholder(i)) {
|
||||
create_mysql_column(t, "")?
|
||||
create_mysql_column(&map_dictionary_to_values_data_type(t), "")?
|
||||
} else {
|
||||
create_mysql_column(&ConcreteDataType::null_datatype(), "")?
|
||||
};
|
||||
|
||||
@@ -19,7 +19,7 @@ use chrono::NaiveDate;
|
||||
use common_query::prelude::ScalarValue;
|
||||
use common_sql::convert::sql_value_to_value;
|
||||
use common_time::{Date, Timestamp};
|
||||
use datatypes::prelude::ConcreteDataType;
|
||||
use datatypes::prelude::{ConcreteDataType, DataType};
|
||||
use datatypes::schema::ColumnSchema;
|
||||
use datatypes::types::TimestampType;
|
||||
use datatypes::value::{self, Value};
|
||||
@@ -138,6 +138,13 @@ where
|
||||
/// Convert [`ParamValue`] into [`Value`] according to param type.
|
||||
/// It will try it's best to do type conversions if possible
|
||||
pub fn convert_value(param: &ParamValue, t: &ConcreteDataType) -> Result<ScalarValue> {
|
||||
if let ConcreteDataType::Dictionary(dictionary) = t {
|
||||
return Ok(ScalarValue::Dictionary(
|
||||
Box::new(dictionary.key_type().as_arrow_type()),
|
||||
Box::new(convert_value(param, dictionary.value_type())?),
|
||||
));
|
||||
}
|
||||
|
||||
match param.value.into_inner() {
|
||||
ValueInner::Int(i) => match t {
|
||||
ConcreteDataType::Int8(_) => Ok(ScalarValue::Int8(Some(i as i8))),
|
||||
@@ -258,6 +265,16 @@ pub fn convert_value(param: &ParamValue, t: &ConcreteDataType) -> Result<ScalarV
|
||||
/// Convert an MySQL expression to a scalar value.
|
||||
/// It automatically handles the conversion of strings to numeric values.
|
||||
pub fn convert_expr_to_scalar_value(param: &Expr, t: &ConcreteDataType) -> Result<ScalarValue> {
|
||||
if let ConcreteDataType::Dictionary(dictionary) = t {
|
||||
return Ok(ScalarValue::Dictionary(
|
||||
Box::new(dictionary.key_type().as_arrow_type()),
|
||||
Box::new(convert_expr_to_scalar_value(
|
||||
param,
|
||||
dictionary.value_type(),
|
||||
)?),
|
||||
));
|
||||
}
|
||||
|
||||
let column_schema = ColumnSchema::new("", t.clone(), true);
|
||||
match param {
|
||||
Expr::Value(v) => {
|
||||
@@ -464,6 +481,19 @@ mod tests {
|
||||
let v = convert_expr_to_scalar_value(&expr, &t).unwrap();
|
||||
assert_eq!(ScalarValue::Utf8(Some("hello".to_string())), v);
|
||||
|
||||
let t = ConcreteDataType::dictionary_datatype(
|
||||
ConcreteDataType::uint32_datatype(),
|
||||
ConcreteDataType::string_datatype(),
|
||||
);
|
||||
let v = convert_expr_to_scalar_value(&expr, &t).unwrap();
|
||||
assert_eq!(
|
||||
ScalarValue::Dictionary(
|
||||
Box::new(arrow_schema::DataType::UInt32),
|
||||
Box::new(ScalarValue::Utf8(Some("hello".to_string()))),
|
||||
),
|
||||
v
|
||||
);
|
||||
|
||||
let expr = Expr::Value(ValueExpr::Null.into());
|
||||
let t = ConcreteDataType::time_microsecond_datatype();
|
||||
let v = convert_expr_to_scalar_value(&expr, &t).unwrap();
|
||||
|
||||
@@ -42,7 +42,7 @@ use session::context::QueryContextRef;
|
||||
use snafu::prelude::*;
|
||||
use tokio::io::AsyncWrite;
|
||||
|
||||
use crate::error::{ConvertSqlValueSnafu, DataFusionSnafu, NotSupportedSnafu, Result};
|
||||
use crate::error::{self, ConvertSqlValueSnafu, DataFusionSnafu, NotSupportedSnafu, Result};
|
||||
use crate::metrics::*;
|
||||
|
||||
/// Try to write multiple output to the writer if possible.
|
||||
@@ -249,20 +249,6 @@ impl MysqlResultWriter {
|
||||
let v = datatypes::arrow_array::string_array_value(column, i);
|
||||
row_writer.write_col(v)?;
|
||||
}
|
||||
DataType::Dictionary(_, value_type)
|
||||
if matches!(
|
||||
value_type.as_ref(),
|
||||
DataType::Utf8 | DataType::Utf8View | DataType::LargeUtf8
|
||||
) =>
|
||||
{
|
||||
if let Some(v) =
|
||||
datatypes::arrow_array::string_array_value_at_index(column, i)
|
||||
{
|
||||
row_writer.write_col(v)?;
|
||||
} else {
|
||||
row_writer.write_col(None::<u8>)?;
|
||||
}
|
||||
}
|
||||
DataType::Binary | DataType::BinaryView | DataType::LargeBinary => {
|
||||
let v = datatypes::arrow_array::binary_array_value(column, i);
|
||||
if let ConcreteDataType::Json(_) = &schema.column_schemas()[j].data_type {
|
||||
@@ -378,9 +364,11 @@ pub(crate) fn create_mysql_column(
|
||||
ConcreteDataType::Vector(_) => Ok(ColumnType::MYSQL_TYPE_BLOB),
|
||||
ConcreteDataType::List(_) => Ok(ColumnType::MYSQL_TYPE_VARCHAR),
|
||||
ConcreteDataType::Struct(_) => Ok(ColumnType::MYSQL_TYPE_VARCHAR),
|
||||
ConcreteDataType::Dictionary(dictionary) => {
|
||||
return create_mysql_column(dictionary.value_type(), column_name);
|
||||
_ => error::UnsupportedDataTypeSnafu {
|
||||
data_type,
|
||||
reason: "not implemented",
|
||||
}
|
||||
.fail(),
|
||||
};
|
||||
let mut colflags = ColumnFlags::empty();
|
||||
match data_type {
|
||||
|
||||
@@ -18,8 +18,8 @@ use std::sync::Arc;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use common_query::{Output, OutputData};
|
||||
use common_recordbatch::RecordBatch;
|
||||
use common_recordbatch::error::Result as RecordBatchResult;
|
||||
use common_recordbatch::{RecordBatch, map_dictionary_to_values_schema};
|
||||
use common_telemetry::{debug, info, tracing};
|
||||
use datafusion::sql::sqlparser::ast::{CopyOption, CopyTarget, Statement as SqlParserStatement};
|
||||
use datafusion_common::ParamValues;
|
||||
@@ -557,6 +557,7 @@ fn describe_fields(
|
||||
// query
|
||||
SqlPlan::Plan(plan, _) if !matches!(plan, LogicalPlan::Dml(_) | LogicalPlan::Ddl(_)) => {
|
||||
let schema: Schema = plan.schema().clone().try_into().map_err(convert_err)?;
|
||||
let (schema, _) = map_dictionary_to_values_schema(Arc::new(schema));
|
||||
schema_to_pg(&schema, format, None).map_err(convert_err)
|
||||
}
|
||||
// We can cover only part of show statements
|
||||
|
||||
@@ -24,14 +24,14 @@ use arrow_pg::encoder::{Encoder, encode_value};
|
||||
use arrow_pg::list_encoder::encode_list;
|
||||
use arrow_schema::{DataType, TimeUnit};
|
||||
use chrono::{DateTime, FixedOffset, NaiveDate, NaiveDateTime};
|
||||
use common_recordbatch::RecordBatch;
|
||||
use common_recordbatch::error::Result as RecordBatchResult;
|
||||
use common_recordbatch::{RecordBatch, map_dictionary_to_values_data_type};
|
||||
use common_time::{IntervalDayTime, IntervalMonthDayNano, IntervalYearMonth};
|
||||
use datafusion_common::ScalarValue;
|
||||
use datafusion_expr::LogicalPlan;
|
||||
use datatypes::arrow::datatypes::DataType as ArrowDataType;
|
||||
use datatypes::json::JsonSettings;
|
||||
use datatypes::prelude::{ConcreteDataType, Value};
|
||||
use datatypes::prelude::{ConcreteDataType, DataType as _, Value};
|
||||
use datatypes::schema::{Schema, SchemaRef};
|
||||
use datatypes::types::{Decimal128Type, IntervalType, TimestampType, jsonb_to_string};
|
||||
use datatypes::value::StructValue;
|
||||
@@ -222,15 +222,6 @@ where
|
||||
DataType::Struct(_) => {
|
||||
encode_struct(query_ctx, Default::default(), encoder, pg_field)?;
|
||||
}
|
||||
DataType::Dictionary(_, value_type)
|
||||
if matches!(
|
||||
value_type.as_ref(),
|
||||
DataType::Utf8 | DataType::Utf8View | DataType::LargeUtf8
|
||||
) =>
|
||||
{
|
||||
let value = datatypes::arrow_array::string_array_value_at_index(column, i);
|
||||
encoder.encode_field(&value, pg_field)?;
|
||||
}
|
||||
_ => {
|
||||
// Encode value using arrow-pg
|
||||
let arrow_field = arrow_schema.field(j);
|
||||
@@ -289,7 +280,11 @@ pub(super) fn type_gt_to_pg(origin: &ConcreteDataType) -> Result<Type> {
|
||||
}
|
||||
.fail(),
|
||||
},
|
||||
ConcreteDataType::Dictionary(dictionary) => type_gt_to_pg(dictionary.value_type()),
|
||||
&ConcreteDataType::Dictionary(_) => server_error::UnsupportedDataTypeSnafu {
|
||||
data_type: origin,
|
||||
reason: "not implemented",
|
||||
}
|
||||
.fail(),
|
||||
&ConcreteDataType::Duration(_) => Ok(Type::INTERVAL),
|
||||
&ConcreteDataType::Struct(_) => Ok(Type::JSON),
|
||||
}
|
||||
@@ -375,6 +370,29 @@ fn numeric_out_of_range_error(value: impl std::fmt::Display) -> PgWireError {
|
||||
)
|
||||
}
|
||||
|
||||
fn string_parameter_to_scalar_value(
|
||||
data: Option<String>,
|
||||
data_type: &ConcreteDataType,
|
||||
) -> Option<ScalarValue> {
|
||||
match data_type {
|
||||
ConcreteDataType::String(string_type) => {
|
||||
if string_type.is_large() {
|
||||
Some(ScalarValue::LargeUtf8(data))
|
||||
} else {
|
||||
Some(ScalarValue::Utf8(data))
|
||||
}
|
||||
}
|
||||
ConcreteDataType::Dictionary(dictionary) => Some(ScalarValue::Dictionary(
|
||||
Box::new(dictionary.key_type().as_arrow_type()),
|
||||
Box::new(string_parameter_to_scalar_value(
|
||||
data,
|
||||
dictionary.value_type(),
|
||||
)?),
|
||||
)),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn parameters_to_scalar_values(
|
||||
plan: &LogicalPlan,
|
||||
portal: &Portal<PgSqlPlan>,
|
||||
@@ -398,7 +416,8 @@ pub(super) fn parameters_to_scalar_values(
|
||||
let client_type = if let Some(Some(client_given_type)) = client_param_types.get(idx) {
|
||||
client_given_type.clone()
|
||||
} else if let Some(server_provided_type) = &server_type {
|
||||
type_gt_to_pg(server_provided_type).map_err(convert_err)?
|
||||
type_gt_to_pg(&map_dictionary_to_values_data_type(server_provided_type))
|
||||
.map_err(convert_err)?
|
||||
} else {
|
||||
return Err(invalid_parameter_error(
|
||||
"unknown_parameter_type",
|
||||
@@ -413,21 +432,12 @@ pub(super) fn parameters_to_scalar_values(
|
||||
&Type::VARCHAR | &Type::TEXT | &Type::CHAR => {
|
||||
let data = portal.parameter::<String>(idx, &client_type)?;
|
||||
if let Some(server_type) = &server_type {
|
||||
match server_type {
|
||||
ConcreteDataType::String(t) => {
|
||||
if t.is_large() {
|
||||
ScalarValue::LargeUtf8(data)
|
||||
} else {
|
||||
ScalarValue::Utf8(data)
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
return Err(invalid_parameter_error(
|
||||
"invalid_parameter_type",
|
||||
Some(format!("Expected: {}, found: {}", server_type, client_type)),
|
||||
));
|
||||
}
|
||||
}
|
||||
string_parameter_to_scalar_value(data, server_type).ok_or_else(|| {
|
||||
invalid_parameter_error(
|
||||
"invalid_parameter_type",
|
||||
Some(format!("Expected: {}, found: {}", server_type, client_type)),
|
||||
)
|
||||
})?
|
||||
} else {
|
||||
ScalarValue::Utf8(data)
|
||||
}
|
||||
@@ -1223,7 +1233,7 @@ pub(super) fn param_types_to_pg_types(
|
||||
let mut types = Vec::with_capacity(param_count);
|
||||
for i in 0..param_count {
|
||||
if let Some(Some(param_type)) = param_types.get(&format!("${}", i + 1)) {
|
||||
let pg_type = type_gt_to_pg(param_type)?;
|
||||
let pg_type = type_gt_to_pg(&map_dictionary_to_values_data_type(param_type))?;
|
||||
types.push(pg_type);
|
||||
} else {
|
||||
types.push(Type::UNKNOWN);
|
||||
@@ -1745,6 +1755,35 @@ mod test {
|
||||
Portal::try_new(&bind, statement).unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_dictionary_string_parameter() {
|
||||
let dictionary_type =
|
||||
DataType::Dictionary(Box::new(DataType::UInt32), Box::new(DataType::Utf8));
|
||||
let plan = build_plan_with_params(vec![("$1", dictionary_type)]);
|
||||
let portal = make_portal(vec![None], vec![s("host-a")]);
|
||||
|
||||
let values = parameters_to_scalar_values(&plan, &portal).unwrap();
|
||||
assert_eq!(
|
||||
vec![ScalarValue::Dictionary(
|
||||
Box::new(DataType::UInt32),
|
||||
Box::new(ScalarValue::Utf8(s("host-a"))),
|
||||
)],
|
||||
values
|
||||
);
|
||||
|
||||
let param_types = HashMap::from([(
|
||||
"$1".to_string(),
|
||||
Some(ConcreteDataType::dictionary_datatype(
|
||||
ConcreteDataType::uint32_datatype(),
|
||||
ConcreteDataType::string_datatype(),
|
||||
)),
|
||||
)]);
|
||||
assert_eq!(
|
||||
vec![Type::VARCHAR],
|
||||
param_types_to_pg_types(¶m_types).unwrap()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_int2_coerce_in_range() {
|
||||
let plan = build_plan_with_params(vec![
|
||||
|
||||
@@ -16,19 +16,13 @@ use std::sync::Arc;
|
||||
|
||||
use api::v1::greptime_request::Request;
|
||||
use api::v1::query_request::Query;
|
||||
use arrow::array::{DictionaryArray, StringArray, UInt32Array};
|
||||
use arrow::datatypes::UInt32Type;
|
||||
use async_trait::async_trait;
|
||||
use catalog::memory::MemoryCatalogManager;
|
||||
use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
|
||||
use common_error::ext::BoxedError;
|
||||
use common_grpc::flight::do_put::DoPutResponse;
|
||||
use common_query::Output;
|
||||
use common_recordbatch::RecordBatch;
|
||||
use datafusion_expr::LogicalPlan;
|
||||
use datatypes::prelude::{ConcreteDataType, VectorRef};
|
||||
use datatypes::schema::{ColumnSchema, Schema};
|
||||
use datatypes::vectors::DictionaryVector;
|
||||
use futures_util::TryFutureExt;
|
||||
use query::options::QueryOptions;
|
||||
use query::parser::{PromQuery, QueryStatement};
|
||||
@@ -42,7 +36,6 @@ use snafu::{ResultExt, ensure};
|
||||
use sql::parser::{ParseOptions, ParserContext};
|
||||
use sql::statements::statement::Statement;
|
||||
use table::TableRef;
|
||||
use table::test_util::MemTable;
|
||||
|
||||
mod http;
|
||||
mod interceptor;
|
||||
@@ -50,23 +43,6 @@ mod mysql;
|
||||
mod postgres;
|
||||
mod prom_remote_write_v2_test;
|
||||
|
||||
fn dictionary_child_null_table() -> TableRef {
|
||||
let data_type = ConcreteDataType::dictionary_datatype(
|
||||
ConcreteDataType::uint32_datatype(),
|
||||
ConcreteDataType::string_datatype(),
|
||||
);
|
||||
let schema = Arc::new(Schema::new(vec![ColumnSchema::new(
|
||||
"host", data_type, true,
|
||||
)]));
|
||||
let dictionary = DictionaryArray::<UInt32Type>::new(
|
||||
UInt32Array::from(vec![0, 1]),
|
||||
Arc::new(StringArray::from(vec![None, Some("host-b")])),
|
||||
);
|
||||
let vector = Arc::new(DictionaryVector::try_from(dictionary).unwrap()) as VectorRef;
|
||||
let record_batch = RecordBatch::new(schema, vec![vector]).unwrap();
|
||||
MemTable::table("dictionary_values", record_batch)
|
||||
}
|
||||
|
||||
pub struct DummyInstance {
|
||||
query_engine: QueryEngineRef,
|
||||
}
|
||||
|
||||
@@ -34,8 +34,8 @@ use servers::tls::{ReloadableTlsServerConfig, TlsOption};
|
||||
use table::TableRef;
|
||||
use table::test_util::MemTable;
|
||||
|
||||
use crate::create_testing_sql_query_handler;
|
||||
use crate::mysql::{MysqlTextRow, TestingData, all_datatype_testing_data};
|
||||
use crate::{create_testing_sql_query_handler, dictionary_child_null_table};
|
||||
|
||||
#[derive(Default)]
|
||||
struct MysqlOpts<'a> {
|
||||
@@ -77,28 +77,6 @@ fn create_mysql_server(table: TableRef, opts: MysqlOpts<'_>) -> Result<Box<dyn S
|
||||
))
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_mysql_dictionary_child_null() -> Result<()> {
|
||||
let mut mysql_server = create_mysql_server(dictionary_child_null_table(), Default::default())?;
|
||||
mysql_server
|
||||
.start("127.0.0.1:0".parse::<SocketAddr>().unwrap())
|
||||
.await
|
||||
.unwrap();
|
||||
let server_port = mysql_server.bind_addr().unwrap().port();
|
||||
let mut connection = create_connection_default_db_name(server_port, false)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let values: Vec<Option<String>> = connection
|
||||
.query("SELECT host FROM dictionary_values")
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(vec![None, Some("host-b".to_string())], values);
|
||||
|
||||
mysql_server.shutdown().await.unwrap();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_start_mysql_server() -> Result<()> {
|
||||
let table = MemTable::default_numbers_table();
|
||||
|
||||
@@ -35,7 +35,7 @@ use table::TableRef;
|
||||
use table::test_util::MemTable;
|
||||
use tokio_postgres::{Client, Error as PgError, NoTls, SimpleQueryMessage};
|
||||
|
||||
use crate::{create_testing_instance, dictionary_child_null_table};
|
||||
use crate::create_testing_instance;
|
||||
|
||||
fn create_postgres_server(
|
||||
table: TableRef,
|
||||
@@ -75,35 +75,6 @@ fn create_postgres_server(
|
||||
)))
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_postgres_dictionary_child_null() -> Result<()> {
|
||||
let mut postgres_server = create_postgres_server(
|
||||
dictionary_child_null_table(),
|
||||
false,
|
||||
Default::default(),
|
||||
None,
|
||||
)?;
|
||||
postgres_server
|
||||
.start("127.0.0.1:0".parse::<SocketAddr>().unwrap())
|
||||
.await
|
||||
.unwrap();
|
||||
let server_port = postgres_server.bind_addr().unwrap().port();
|
||||
let client = create_plain_connection(server_port, false).await.unwrap();
|
||||
|
||||
let rows = client
|
||||
.query("SELECT host FROM dictionary_values", &[])
|
||||
.await
|
||||
.unwrap();
|
||||
let values = rows
|
||||
.iter()
|
||||
.map(|row| row.get::<_, Option<String>>(0))
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(vec![None, Some("host-b".to_string())], values);
|
||||
|
||||
postgres_server.shutdown().await.unwrap();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
pub async fn test_start_postgres_server() -> Result<()> {
|
||||
let table = MemTable::default_numbers_table();
|
||||
|
||||
@@ -87,6 +87,39 @@ DEALLOCATE stmt;
|
||||
affected_rows: 0
|
||||
|
||||
-- test with data
|
||||
-- A Mito scan exposes string primary keys as dictionaries, but that physical read type must not
|
||||
-- leak into INSERT Values planning.
|
||||
CREATE TABLE insert_values_dictionary (
|
||||
ts TIMESTAMP TIME INDEX,
|
||||
host STRING PRIMARY KEY
|
||||
) ENGINE=mito;
|
||||
|
||||
Affected Rows: 0
|
||||
|
||||
INSERT INTO insert_values_dictionary VALUES(now(), 'a');
|
||||
|
||||
Affected Rows: 1
|
||||
|
||||
SELECT host FROM insert_values_dictionary;
|
||||
|
||||
+------+
|
||||
| host |
|
||||
+------+
|
||||
| a |
|
||||
+------+
|
||||
|
||||
SELECT array_agg(host) FROM insert_values_dictionary;
|
||||
|
||||
+------------------------------------------+
|
||||
| array_agg(insert_values_dictionary.host) |
|
||||
+------------------------------------------+
|
||||
| [a] |
|
||||
+------------------------------------------+
|
||||
|
||||
DROP TABLE insert_values_dictionary;
|
||||
|
||||
Affected Rows: 0
|
||||
|
||||
CREATE TABLE IF NOT EXISTS "cake" (
|
||||
`domain` STRING,
|
||||
is_expire BOOLEAN NULL,
|
||||
|
||||
@@ -49,6 +49,21 @@ EXECUTE stmt USING 1, 2;
|
||||
DEALLOCATE stmt;
|
||||
|
||||
-- test with data
|
||||
-- A Mito scan exposes string primary keys as dictionaries, but that physical read type must not
|
||||
-- leak into INSERT Values planning.
|
||||
CREATE TABLE insert_values_dictionary (
|
||||
ts TIMESTAMP TIME INDEX,
|
||||
host STRING PRIMARY KEY
|
||||
) ENGINE=mito;
|
||||
|
||||
INSERT INTO insert_values_dictionary VALUES(now(), 'a');
|
||||
|
||||
SELECT host FROM insert_values_dictionary;
|
||||
|
||||
SELECT array_agg(host) FROM insert_values_dictionary;
|
||||
|
||||
DROP TABLE insert_values_dictionary;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS "cake" (
|
||||
`domain` STRING,
|
||||
is_expire BOOLEAN NULL,
|
||||
|
||||
Reference in New Issue
Block a user