fix(query): keep dictionary tags within query paths

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>
This commit is contained in:
Ruihang Xia
2026-07-14 18:35:07 +08:00
parent 0f97088bb6
commit 1a32a709e5
25 changed files with 612 additions and 295 deletions
+56 -10
View File
@@ -18,8 +18,8 @@ use std::collections::HashMap;
use std::sync::Arc;
use api::v1::{AffectedRows, FlightMetadata, Metrics};
use arrow_flight::FlightData;
use arrow_flight::utils::flight_data_to_arrow_batch;
use arrow_flight::{FlightData, SchemaAsIpc};
use common_base::bytes::Bytes;
use common_recordbatch::DfRecordBatch;
use datatypes::arrow;
@@ -190,8 +190,14 @@ impl FlightEncoder {
}
/// Encode the Arrow schema to [FlightData].
pub fn encode_schema(&self, schema: &ArrowSchema) -> FlightData {
SchemaAsIpc::new(schema, &self.write_options).into()
pub fn encode_schema(&mut self, schema: &ArrowSchema) -> FlightData {
self.data_gen
.schema_to_bytes_with_dictionary_tracker(
schema,
&mut self.dictionary_tracker,
&self.write_options,
)
.into()
}
/// Encode the [FlightMessage] to a list (at least one element) of [FlightData]s.
@@ -202,12 +208,6 @@ impl FlightEncoder {
pub fn encode(&mut self, flight_message: FlightMessage) -> Vec1<FlightData> {
match flight_message {
FlightMessage::Schema(schema) => {
schema.fields().iter().for_each(|x| {
if matches!(x.data_type(), DataType::Dictionary(_, _)) {
self.dictionary_tracker.next_dict_id();
}
});
vec1![self.encode_schema(schema.as_ref())]
}
FlightMessage::RecordBatch(record_batch) => {
@@ -474,8 +474,9 @@ fn build_none_flight_msg() -> Bytes {
mod test {
use arrow_flight::utils::batches_to_flight_data;
use datatypes::arrow::array::{
DictionaryArray, Int32Array, StringArray, UInt8Array, UInt32Array,
DictionaryArray, Int32Array, ListArray, StringArray, UInt8Array, UInt32Array,
};
use datatypes::arrow::buffer::OffsetBuffer;
use datatypes::arrow::datatypes::{DataType, Field, Schema};
use super::*;
@@ -727,6 +728,51 @@ mod test {
Ok(())
}
#[test]
fn test_flight_encode_decode_with_nested_dictionary_array() -> Result<()> {
let item_field = Arc::new(Field::new_dictionary(
"item",
DataType::UInt32,
DataType::Utf8,
true,
));
let schema = Arc::new(Schema::new(vec![Field::new(
"tags",
DataType::List(item_field.clone()),
true,
)]));
let values = DictionaryArray::new(
UInt32Array::from_iter_values([0, 1, 0]),
Arc::new(StringArray::from_iter_values(["host-a", "host-b"])),
);
let list = ListArray::new(
item_field,
OffsetBuffer::from_lengths([2, 1]),
Arc::new(values),
None,
);
let batch = DfRecordBatch::try_new(schema.clone(), vec![Arc::new(list)]).unwrap();
let mut encoder = FlightEncoder::default();
let encoded_schema = encoder.encode(FlightMessage::Schema(schema.clone()));
let encoded_batch = encoder.encode(FlightMessage::RecordBatch(batch.clone()));
let mut decoder = FlightDecoder::default();
assert!(matches!(
decoder.try_decode(encoded_schema.first())?,
Some(FlightMessage::Schema(actual)) if actual == schema
));
let mut decoded_batch = None;
for data in encoded_batch {
if let Some(FlightMessage::RecordBatch(decoded)) = decoder.try_decode(&data)? {
decoded_batch = Some(decoded);
}
}
assert_eq!(Some(batch), decoded_batch);
Ok(())
}
#[test]
fn test_affected_rows_roundtrip_through_flight_codec() {
// Verify the full FlightEncoder → FlightDecoder pipeline handles
+36 -1
View File
@@ -27,7 +27,10 @@ use std::sync::Arc;
use api::greptime_proto::v1::AddColumnLocation as Location;
use api::greptime_proto::v1::add_column_location::LocationType;
use common_recordbatch::{RecordBatches, SendableRecordBatchStream};
use common_recordbatch::{
RecordBatches, SendableRecordBatchMapper, SendableRecordBatchStream, map_dictionary_to_values,
map_dictionary_to_values_schema,
};
use datafusion::physical_plan::ExecutionPlan;
use serde::{Deserialize, Serialize};
use sqlparser_derive::{Visit, VisitMut};
@@ -97,6 +100,38 @@ impl Output {
Self { data, meta }
}
/// Expands dictionary arrays before exposing a query result to a client.
pub fn map_dictionary_to_values(self) -> common_recordbatch::error::Result<Self> {
let Self { data, meta } = self;
let data = match data {
OutputData::AffectedRows(rows) => OutputData::AffectedRows(rows),
OutputData::RecordBatches(record_batches) => {
let original_schema = record_batches.schema();
let (mapped_schema, apply_mapper) =
map_dictionary_to_values_schema(original_schema.clone());
if !apply_mapper {
OutputData::RecordBatches(record_batches)
} else {
let batches = record_batches
.into_iter()
.map(|batch| {
map_dictionary_to_values(batch, &original_schema, &mapped_schema)
})
.collect::<common_recordbatch::error::Result<Vec<_>>>()?;
OutputData::RecordBatches(RecordBatches::try_new(mapped_schema, batches)?)
}
}
OutputData::Stream(stream) => {
OutputData::Stream(Box::pin(SendableRecordBatchMapper::new(
stream,
map_dictionary_to_values,
map_dictionary_to_values_schema,
)))
}
};
Ok(Self { data, meta })
}
pub fn extract_rows_and_cost(&self) -> (OutputRows, OutputCost) {
match self.data {
OutputData::AffectedRows(rows) => (rows, self.meta.cost),
+163 -3
View File
@@ -40,16 +40,16 @@ use datatypes::arrow::array::{ArrayRef, AsArray, StringBuilder};
use datatypes::arrow::compute::SortOptions;
pub use datatypes::arrow::record_batch::RecordBatch as DfRecordBatch;
use datatypes::arrow::util::pretty;
use datatypes::prelude::{ConcreteDataType, VectorRef};
use datatypes::prelude::{ConcreteDataType, DataType, VectorRef};
use datatypes::schema::{ColumnSchema, Schema, SchemaRef};
use datatypes::types::{JsonFormat, jsonb_to_string};
use datatypes::types::{JsonFormat, StructField, StructType, jsonb_to_string};
use error::Result;
use futures::task::{Context, Poll};
use futures::{Stream, TryStreamExt};
pub use recordbatch::RecordBatch;
use snafu::{IntoError, ResultExt, ensure};
use crate::error::NewDfRecordBatchSnafu;
use crate::error::{ArrowComputeSnafu, NewDfRecordBatchSnafu};
pub trait RecordBatchStream: Stream<Item = Result<RecordBatch>> {
fn name(&self) -> &str {
@@ -163,6 +163,84 @@ pub fn map_json_type_to_string_schema(schema: SchemaRef) -> (SchemaRef, bool) {
(Arc::new(Schema::new(new_columns)), apply_mapper)
}
/// Replaces dictionary types with their value types, including dictionaries nested in lists and
/// structs.
pub fn map_dictionary_to_values_data_type(data_type: &ConcreteDataType) -> ConcreteDataType {
match data_type {
ConcreteDataType::Dictionary(dictionary) => {
map_dictionary_to_values_data_type(dictionary.value_type())
}
ConcreteDataType::List(list) => ConcreteDataType::list_datatype(Arc::new(
map_dictionary_to_values_data_type(list.item_type()),
)),
ConcreteDataType::Struct(struct_type) => {
let fields = struct_type
.fields()
.iter()
.map(|field| {
StructField::new(
field.name(),
map_dictionary_to_values_data_type(field.data_type()),
field.is_nullable(),
)
})
.collect();
ConcreteDataType::struct_datatype(StructType::new(Arc::new(fields)))
}
_ => data_type.clone(),
}
}
/// Maps dictionary columns in a schema to their value types.
pub fn map_dictionary_to_values_schema(schema: SchemaRef) -> (SchemaRef, bool) {
let mut apply_mapper = false;
let columns = schema
.column_schemas()
.iter()
.map(|column| {
let data_type = map_dictionary_to_values_data_type(&column.data_type);
apply_mapper |= data_type != column.data_type;
let mut column = column.clone();
column.data_type = data_type;
column
})
.collect();
(
Arc::new(Schema::new_with_version(columns, schema.version())),
apply_mapper,
)
}
/// Expands dictionary arrays to their value arrays according to `mapped_schema`.
pub fn map_dictionary_to_values(
batch: RecordBatch,
original_schema: &SchemaRef,
mapped_schema: &SchemaRef,
) -> Result<RecordBatch> {
let arrays = batch
.columns()
.iter()
.zip(original_schema.column_schemas())
.zip(mapped_schema.column_schemas())
.map(|((array, original), mapped)| {
if original.data_type == mapped.data_type {
Ok(array.clone())
} else {
datatypes::arrow::compute::cast(array, &mapped.data_type.as_arrow_type())
.context(ArrowComputeSnafu)
}
})
.collect::<Result<Vec<_>>>()?;
let record_batch = DfRecordBatch::try_new(mapped_schema.arrow_schema().clone(), arrays)
.context(NewDfRecordBatchSnafu)?;
Ok(RecordBatch::from_df_record_batch(
mapped_schema.clone(),
record_batch,
))
}
impl SendableRecordBatchMapper {
/// Creates a new [SendableRecordBatchMapper] with the given inner [RecordBatchStream], mapper function, and schema mapper function.
pub fn new(
@@ -835,6 +913,11 @@ mod tests {
use std::time::Duration;
use common_memory_manager::{OnExhaustedPolicy, PermitGranularity};
use datatypes::arrow::array::{
DictionaryArray, Int32Array, ListArray, StringArray, UInt32Array,
};
use datatypes::arrow::buffer::OffsetBuffer;
use datatypes::arrow::datatypes::{DataType as ArrowDataType, Int32Type};
use datatypes::prelude::{ConcreteDataType, VectorRef};
use datatypes::schema::{ColumnSchema, Schema};
use datatypes::vectors::{BooleanVector, Int32Vector, StringVector};
@@ -918,6 +1001,83 @@ mod tests {
assert_eq!(vec![batch1], batches.take());
}
#[test]
fn test_map_dictionary_to_values_recursively() {
let string_dictionary = ConcreteDataType::dictionary_datatype(
ConcreteDataType::int32_datatype(),
ConcreteDataType::string_datatype(),
);
let list_dictionary =
ConcreteDataType::list_datatype(Arc::new(ConcreteDataType::dictionary_datatype(
ConcreteDataType::uint32_datatype(),
ConcreteDataType::string_datatype(),
)));
let schema = Arc::new(Schema::new(vec![
ColumnSchema::new("host", string_dictionary, true),
ColumnSchema::new("tags", list_dictionary, true),
]));
let host = DictionaryArray::<Int32Type>::new(
Int32Array::from(vec![0, 1]),
Arc::new(StringArray::from(vec![Some("host-a"), None])),
);
let ArrowDataType::List(item_field) = schema.arrow_schema().field(1).data_type().clone()
else {
unreachable!()
};
let tag_values = DictionaryArray::new(
UInt32Array::from_iter_values([0, 1, 0]),
Arc::new(StringArray::from(vec![Some("a"), None])),
);
let tags = ListArray::new(
item_field,
OffsetBuffer::from_lengths([2, 1]),
Arc::new(tag_values),
None,
);
let batch = DfRecordBatch::try_new(
schema.arrow_schema().clone(),
vec![Arc::new(host), Arc::new(tags)],
)
.unwrap();
let batch = RecordBatch::from_df_record_batch(schema.clone(), batch);
let (mapped_schema, apply_mapper) = map_dictionary_to_values_schema(schema.clone());
assert!(apply_mapper);
assert_eq!(
&ArrowDataType::Utf8,
mapped_schema.arrow_schema().field(0).data_type()
);
assert_eq!(
&ArrowDataType::List(Arc::new(
datatypes::arrow::datatypes::Field::new_list_field(ArrowDataType::Utf8, true,)
)),
mapped_schema.arrow_schema().field(1).data_type()
);
let mapped = map_dictionary_to_values(batch, &schema, &mapped_schema).unwrap();
let host = mapped
.column(0)
.as_any()
.downcast_ref::<StringArray>()
.unwrap();
assert_eq!(vec![Some("host-a"), None], host.iter().collect::<Vec<_>>());
let tags = mapped
.column(1)
.as_any()
.downcast_ref::<ListArray>()
.unwrap();
let tag_values = tags
.values()
.as_any()
.downcast_ref::<StringArray>()
.unwrap();
assert_eq!(
vec![Some("a"), None, Some("a")],
tag_values.iter().collect::<Vec<_>>()
);
}
#[tokio::test]
async fn test_simple_recordbatch_stream() {
let column_a = ColumnSchema::new("a", ConcreteDataType::int32_datatype(), false);
+28 -6
View File
@@ -303,10 +303,10 @@ impl RecordBatch {
}
_ => {
if let Ok(column) = Helper::try_into_vector(column) {
Box::new(
(0..column.len())
.map(move |i| (!column.is_null(i)).then(|| column.get(i).to_string())),
)
Box::new((0..column.len()).map(move |i| {
let value = column.get_ref(i);
(!value.is_null()).then(|| column.get(i).to_string())
}))
} else {
Box::new(std::iter::empty())
}
@@ -382,13 +382,13 @@ fn maybe_align_json_array_with_schema(
mod tests {
use std::sync::Arc;
use datatypes::arrow::array::{AsArray, BinaryArray, UInt32Array};
use datatypes::arrow::array::{AsArray, BinaryArray, DictionaryArray, UInt32Array};
use datatypes::arrow::datatypes::{DataType, Field, Schema as ArrowSchema, UInt32Type};
use datatypes::arrow_array::StringArray;
use datatypes::data_type::ConcreteDataType;
use datatypes::extension::json::{JsonExtensionType, JsonMetadata};
use datatypes::schema::{ColumnSchema, Schema};
use datatypes::vectors::{StringVector, UInt32Vector};
use datatypes::vectors::{DictionaryVector, StringVector, UInt32Vector};
use super::*;
@@ -475,6 +475,28 @@ mod tests {
assert!(recordbatch.slice(1, 5).is_err());
}
#[test]
fn test_iter_dictionary_column_uses_logical_nulls() {
let data_type = ConcreteDataType::dictionary_datatype(
ConcreteDataType::uint32_datatype(),
ConcreteDataType::string_datatype(),
);
let schema = Arc::new(Schema::new(vec![ColumnSchema::new(
"host", data_type, true,
)]));
let dictionary = DictionaryArray::<UInt32Type>::new(
UInt32Array::from(vec![Some(0), Some(1), None]),
Arc::new(StringArray::from(vec![None, Some("host-b")])),
);
let vector = Arc::new(DictionaryVector::try_from(dictionary).unwrap()) as VectorRef;
let batch = RecordBatch::new(schema, vec![vector]).unwrap();
assert_eq!(
vec![None, Some("host-b".to_string()), None],
batch.iter_column_as_string(0).collect::<Vec<_>>()
);
}
#[test]
fn test_merge_record_batch() {
let column_schemas = vec![
+55 -1
View File
@@ -164,7 +164,7 @@ fn build_by_tql_schema(plan: &LogicalPlan) -> Result<TableDef, Error> {
.fields()
.iter()
.filter_map(|f| {
if ConcreteDataType::from_arrow_type(f.data_type()).is_string() {
if is_tql_label_type(&ConcreteDataType::from_arrow_type(f.data_type())) {
Some(f.name().clone())
} else {
None
@@ -178,6 +178,14 @@ fn build_by_tql_schema(plan: &LogicalPlan) -> Result<TableDef, Error> {
})
}
fn is_tql_label_type(data_type: &ConcreteDataType) -> bool {
data_type.is_string()
|| matches!(
data_type,
ConcreteDataType::Dictionary(dictionary) if dictionary.value_type().is_string()
)
}
struct TableDef {
ts_col: Option<String>,
pks: Vec<String>,
@@ -246,7 +254,14 @@ fn build_pk_from_aggr(plan: &LogicalPlan) -> Result<Option<TableDef>, Error> {
#[cfg(test)]
mod test {
use std::sync::Arc;
use api::v1::column_def::try_as_column_schema;
use datafusion::arrow::datatypes::{
DataType as ArrowDataType, Field, Schema as ArrowSchema, TimeUnit,
};
use datafusion_common::DFSchema;
use datafusion_expr::logical_plan::EmptyRelation;
use datatypes::prelude::ConcreteDataType;
use datatypes::schema::ColumnSchema;
use pretty_assertions::assert_eq;
@@ -257,6 +272,45 @@ mod test {
use crate::batching_mode::utils::sql_to_df_plan;
use crate::test_utils::create_test_query_engine;
#[test]
fn test_tql_dictionary_string_is_label() {
let arrow_schema = Arc::new(ArrowSchema::new(vec![
Field::new_dictionary("host", ArrowDataType::UInt32, ArrowDataType::Utf8, true),
Field::new("value", ArrowDataType::Float64, true),
Field::new(
"ts",
ArrowDataType::Timestamp(TimeUnit::Millisecond, None),
false,
),
]));
let plan = LogicalPlan::EmptyRelation(EmptyRelation {
produce_one_row: false,
schema: Arc::new(DFSchema::try_from(arrow_schema).unwrap()),
});
let expr = create_table_with_expr(
&plan,
&[
"greptime".to_string(),
"public".to_string(),
"sink".to_string(),
],
&QueryType::Tql,
)
.unwrap();
let columns = expr
.column_defs
.iter()
.map(|column| try_as_column_schema(column).unwrap())
.collect::<Vec<_>>();
assert_eq!(vec!["host".to_string()], expr.primary_keys);
assert_eq!("ts", expr.time_index);
assert_eq!(ConcreteDataType::string_datatype(), columns[0].data_type);
assert_eq!(ConcreteDataType::float64_datatype(), columns[1].data_type);
assert!(columns[2].is_time_index());
}
#[tokio::test]
async fn test_gen_create_table_sql() {
let query_engine = create_test_query_engine();
+28 -6
View File
@@ -95,9 +95,9 @@ use table::requests::{OTLP_METRIC_COMPAT_KEY, OTLP_METRIC_COMPAT_PROM};
use tracing::Span;
use crate::error::{
self, Error, ExecLogicalPlanSnafu, ExecutePromqlSnafu, ExternalSnafu, InvalidSqlSnafu,
ParseSqlSnafu, PermissionSnafu, PlanStatementSnafu, Result, SqlExecInterceptedSnafu,
StatementTimeoutSnafu, TableOperationSnafu,
self, CollectRecordbatchSnafu, Error, ExecLogicalPlanSnafu, ExecutePromqlSnafu, ExternalSnafu,
InvalidSqlSnafu, ParseSqlSnafu, PermissionSnafu, PlanStatementSnafu, Result,
SqlExecInterceptedSnafu, StatementTimeoutSnafu, TableOperationSnafu,
};
use crate::service_config::InfluxdbMergeMode;
use crate::stream_wrapper::CancellableStreamWrapper;
@@ -885,7 +885,12 @@ impl SqlQueryHandler for Instance {
self.do_query_inner(query, query_ctx)
.await
.into_iter()
.map(|result| result.map_err(BoxedError::new).context(ExecuteQuerySnafu))
.map(|result| {
result
.and_then(map_query_output)
.map_err(BoxedError::new)
.context(ExecuteQuerySnafu)
})
.collect()
}
@@ -896,6 +901,7 @@ impl SqlQueryHandler for Instance {
) -> server_error::Result<Output> {
self.do_analyze_stream_query_inner(query, query_ctx)
.await
.and_then(map_query_output)
.map_err(BoxedError::new)
.context(ExecuteQuerySnafu)
}
@@ -908,6 +914,7 @@ impl SqlQueryHandler for Instance {
) -> server_error::Result<Output> {
self.do_exec_plan_inner(plan, stmt, query_ctx)
.await
.and_then(map_query_output)
.map_err(BoxedError::new)
.context(server_error::ExecutePlanSnafu)
}
@@ -920,7 +927,12 @@ impl SqlQueryHandler for Instance {
self.do_promql_query_inner(query, query_ctx)
.await
.into_iter()
.map(|result| result.map_err(BoxedError::new).context(ExecuteQuerySnafu))
.map(|result| {
result
.and_then(map_query_output)
.map_err(BoxedError::new)
.context(ExecuteQuerySnafu)
})
.collect()
}
@@ -943,6 +955,13 @@ impl SqlQueryHandler for Instance {
}
}
/// Expands scan-time dictionaries only when a query result leaves the frontend.
pub(crate) fn map_query_output(output: Output) -> Result<Output> {
output
.map_dictionary_to_values()
.context(CollectRecordbatchSnafu)
}
/// Attaches a timer to the output and observes it once the output is exhausted.
pub fn attach_timer(output: Output, timer: HistogramTimer) -> Output {
match output.data {
@@ -1048,7 +1067,10 @@ impl PrometheusHandler for Instance {
.map_err(BoxedError::new)
.context(ExecuteQuerySnafu)?;
Ok(interceptor.post_execute(output, query_ctx)?)
let output = interceptor.post_execute(output, query_ctx)?;
map_query_output(output)
.map_err(BoxedError::new)
.context(ExecuteQuerySnafu)
}
async fn query_metric_names(
+2 -2
View File
@@ -51,7 +51,7 @@ use crate::error::{
NotSupportedSnafu, PermissionSnafu, PlanStatementSnafu, Result,
SubstraitDecodeLogicalPlanSnafu, TableNotFoundSnafu, TableOperationSnafu,
};
use crate::instance::{Instance, attach_timer};
use crate::instance::{Instance, attach_timer, map_query_output};
use crate::metrics::{
GRPC_HANDLE_PLAN_ELAPSED, GRPC_HANDLE_PROMQL_ELAPSED, GRPC_HANDLE_SQL_ELAPSED,
};
@@ -251,7 +251,7 @@ impl GrpcQueryHandler for Instance {
};
let output = interceptor.post_execute(output, ctx)?;
Ok(output)
map_query_output(output)
}
.await;
+6 -2
View File
@@ -408,7 +408,9 @@ async fn query_trace_table(
RecordBatchStreamAdapter::try_new(stream).context(CollectRecordbatchSnafu)?,
));
Ok(output)
output
.map_dictionary_to_values()
.context(CollectRecordbatchSnafu)
}
async fn get_table(
@@ -485,7 +487,9 @@ async fn find_traces_rank_3(
RecordBatchStreamAdapter::try_new(stream).context(CollectRecordbatchSnafu)?,
));
Ok(output)
output
.map_dictionary_to_values()
.context(CollectRecordbatchSnafu)
}
// The current implementation registers UDFs during the planning stage, which makes it difficult
+5 -2
View File
@@ -26,7 +26,7 @@ use session::context::{QueryContext, QueryContextRef};
use snafu::ResultExt;
use tonic::async_trait;
use crate::instance::Instance;
use crate::instance::{Instance, map_query_output};
#[async_trait]
impl LogQueryHandler for Instance {
@@ -64,7 +64,10 @@ impl LogQueryHandler for Instance {
.map_err(BoxedError::new)
.context(ExecuteQuerySnafu)?;
Ok(interceptor.as_ref().post_query(output, ctx.clone())?)
let output = interceptor.as_ref().post_query(output, ctx.clone())?;
map_query_output(output)
.map_err(BoxedError::new)
.context(ExecuteQuerySnafu)
}
fn catalog_manager(&self, _ctx: &QueryContext) -> ServerResult<&dyn catalog::CatalogManager> {
+8 -5
View File
@@ -167,10 +167,6 @@ impl AnalyzerRule for DistPlannerAnalyzer {
}
}
};
let plan = plan
.transform_down_with_subqueries(&Self::unwrap_dictionary_literals)?
.data;
let result = match self.try_push_down(plan.clone()) {
Ok(plan) => plan,
Err(err) => {
@@ -738,9 +734,16 @@ impl PlanRewriter {
self.partition_cols
);
// Substrait does not encode dictionary literals. Only unwrap literals in the remote
// input; rewriting the whole plan also changes DML Values without rebuilding its schema.
let remote_input = on_node
.clone()
.transform_down_with_subqueries(&DistPlannerAnalyzer::unwrap_dictionary_literals)?
.data;
// add merge scan as the new root
let mut node = MergeScanLogicalPlan::new(
on_node.clone(),
remote_input,
false,
// at this stage, the partition cols should be set
// treat it as non-partitioned if None
+58 -1
View File
@@ -17,7 +17,7 @@ use std::pin::Pin;
use std::sync::Arc;
use arrow::datatypes::{DataType, IntervalDayTime, TimeUnit};
use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, MITO_ENGINE};
use common_error::ext::BoxedError;
use common_function::aggrs::aggr_wrapper::{StateMergeHelper, StateWrapper};
use common_recordbatch::adapter::RecordBatchMetrics;
@@ -287,6 +287,63 @@ impl Stream for EmptyStream {
}
}
fn find_merge_scan(plan: &LogicalPlan) -> Option<&MergeScanLogicalPlan> {
if let LogicalPlan::Extension(extension) = plan
&& let Some(merge_scan) = extension
.node
.as_any()
.downcast_ref::<MergeScanLogicalPlan>()
{
return Some(merge_scan);
}
plan.inputs().into_iter().find_map(find_merge_scan)
}
#[test]
fn dictionary_literals_are_unwrapped_only_in_remote_input() {
let table = Arc::new(Table::new(
TestTable::table_info(0, "t".to_string(), MITO_ENGINE.to_string()),
FilterPushDownType::Inexact,
Arc::new(TestDataSource::new(TestTable::schema())),
));
let table_source = Arc::new(DefaultTableSource::new(Arc::new(
DfTableProviderAdapter::new(table),
)));
let literal = Expr::Literal(
ScalarValue::Dictionary(
Box::new(DataType::UInt32),
Box::new(ScalarValue::Utf8(Some("host-a".to_string()))),
),
None,
);
let plan = LogicalPlanBuilder::scan_with_filters("t", table_source, None, vec![])
.unwrap()
.filter(col("pk1").eq(literal))
.unwrap()
.build()
.unwrap();
let result = DistPlannerAnalyzer {}
.analyze(plan, &ConfigOptions::default())
.unwrap();
assert!(
result
.to_string()
.contains("Filter: t.pk1 = Dictionary(UInt32, Utf8(\"host-a\"))")
);
let remote_input = find_merge_scan(&result).unwrap().input();
assert!(
remote_input
.to_string()
.contains("partial_filters=[t.pk1 = Utf8(\"host-a\")]")
);
DFLogicalSubstraitConvertor
.encode(remote_input, DefaultSerializer)
.unwrap();
}
#[cfg(feature = "vector_index")]
mod vector_search_tests {
use std::sync::Arc;
+2 -47
View File
@@ -216,22 +216,13 @@ impl OutputSchema {
impl From<SchemaRef> for OutputSchema {
fn from(schema: SchemaRef) -> OutputSchema {
fn public_type_name(data_type: &datatypes::prelude::ConcreteDataType) -> String {
match data_type {
datatypes::prelude::ConcreteDataType::Dictionary(dictionary) => {
public_type_name(dictionary.value_type())
}
data_type => data_type.name(),
}
}
OutputSchema {
column_schemas: schema
.column_schemas()
.iter()
.map(|cs| ColumnSchema {
name: cs.name.clone(),
data_type: public_type_name(&cs.data_type),
data_type: cs.data_type.name(),
})
.collect(),
}
@@ -1366,8 +1357,6 @@ mod test {
use std::io::Cursor;
use std::sync::Arc;
use arrow::array::{DictionaryArray, StringArray, UInt32Array};
use arrow::datatypes::UInt32Type;
use arrow_ipc::reader::StreamReader;
use arrow_schema::DataType;
use axum::handler::Handler;
@@ -1378,7 +1367,7 @@ mod test {
use datafusion_expr::LogicalPlan;
use datatypes::prelude::*;
use datatypes::schema::{ColumnSchema, Schema};
use datatypes::vectors::{DictionaryVector, StringVector, UInt32Vector};
use datatypes::vectors::{StringVector, UInt32Vector};
use header::constants::GREPTIME_DB_HEADER_TIMEOUT;
use query::parser::PromQuery;
use query::query_engine::DescribeResult;
@@ -1437,40 +1426,6 @@ mod test {
DynamicTimeoutLayer::new(Duration::from_millis(10))
}
fn dictionary_record_batch() -> (SchemaRef, RecordBatch) {
let data_type = ConcreteDataType::dictionary_datatype(
ConcreteDataType::uint32_datatype(),
ConcreteDataType::string_datatype(),
);
let schema = Arc::new(Schema::new(vec![ColumnSchema::new(
"host", data_type, true,
)]));
let dictionary = DictionaryArray::<UInt32Type>::new(
UInt32Array::from(vec![0, 1]),
Arc::new(StringArray::from(vec![Some("host-a"), None])),
);
let vector = Arc::new(DictionaryVector::try_from(dictionary).unwrap()) as VectorRef;
let batch = RecordBatch::new(schema.clone(), vec![vector]).unwrap();
(schema, batch)
}
#[test]
fn test_http_dictionary_output_uses_logical_string_type() {
let (schema, batch) = dictionary_record_batch();
let output = HttpRecordsOutput::try_new(schema.clone(), vec![batch]).unwrap();
assert_eq!("String", output.schema.column_schemas[0].data_type);
assert_eq!(
serde_json::Value::String("host-a".to_string()),
output.rows[0][0]
);
assert_eq!(serde_json::Value::Null, output.rows[1][0]);
let empty = HttpRecordsOutput::try_new(schema, vec![]).unwrap();
assert_eq!("String", empty.schema.column_schemas[0].data_type);
assert!(empty.rows.is_empty());
}
async fn forever() {
pending().await
}
-56
View File
@@ -893,21 +893,6 @@ impl RowWriter {
let v = datatypes::arrow_array::string_array_value(array, i);
self.insert(column, v);
}
DataType::Dictionary(key_type, value_type)
if key_type.as_ref() == &DataType::UInt32
&& matches!(
value_type.as_ref(),
DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View
) =>
{
if let Some(v) =
datatypes::arrow_array::string_array_value_at_index(array, i)
{
self.insert(column, v);
} else {
self.insert(column, "Null");
}
}
DataType::Binary | DataType::LargeBinary | DataType::BinaryView => {
let v = datatypes::arrow_array::binary_array_value(array, i);
let column_schema = &schema.column_schemas()[j];
@@ -1697,14 +1682,10 @@ mod tests {
use std::collections::HashSet;
use std::sync::Arc;
use arrow::array::{DictionaryArray, StringArray, UInt32Array};
use arrow::datatypes::UInt32Type;
use catalog::memory::MemoryCatalogManager;
use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
use common_recordbatch::RecordBatches;
use datatypes::prelude::ConcreteDataType;
use datatypes::schema::{ColumnSchema, Schema};
use datatypes::vectors::{DictionaryVector, VectorRef};
use promql_parser::parser::value::ValueType;
use table::metadata::{TableInfoBuilder, TableMetaBuilder, TableType, TableVersion};
use table::test_util::EmptyTable;
@@ -1719,43 +1700,6 @@ mod tests {
should_error: bool,
}
#[test]
fn test_record_batches_to_series_with_dictionary_labels() {
let data_type = ConcreteDataType::dictionary_datatype(
ConcreteDataType::uint32_datatype(),
ConcreteDataType::string_datatype(),
);
let schema = Arc::new(Schema::new(vec![ColumnSchema::new(
"host", data_type, true,
)]));
let dictionary = DictionaryArray::<UInt32Type>::new(
UInt32Array::from(vec![0, 1]),
Arc::new(StringArray::from(vec![Some("host-a"), None])),
);
let vector = Arc::new(DictionaryVector::try_from(dictionary).unwrap()) as VectorRef;
let batch = RecordBatch::new(schema.clone(), vec![vector]).unwrap();
let batches = RecordBatches::try_new(schema, vec![batch]).unwrap();
let mut series = Vec::new();
record_batches_to_series(
batches,
&mut series,
"metric",
&HashSet::from(["host".to_string()]),
)
.unwrap();
assert_eq!(2, series.len());
assert_eq!(
Some("host-a"),
series[0].get(&Column::from("host")).map(String::as_str)
);
assert_eq!(
Some("Null"),
series[1].get(&Column::from("host")).map(String::as_str)
);
}
#[test]
fn test_retrieve_metric_name_and_result_type() {
let test_cases = &[
-15
View File
@@ -179,21 +179,6 @@ impl HttpOutputWriter {
let v = datatypes::arrow_array::string_array_value(array, i);
self.write_value(v)?;
}
DataType::Dictionary(key_type, value_type)
if key_type.as_ref() == &DataType::UInt32
&& matches!(
value_type.as_ref(),
DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View
) =>
{
if let Some(v) =
datatypes::arrow_array::string_array_value_at_index(array, i)
{
self.write_value(v)?;
} else {
self.write_value(Value::Null)?;
}
}
DataType::Binary | DataType::LargeBinary | DataType::BinaryView => {
let v = datatypes::arrow_array::binary_array_value(array, i);
self.write_bytes(v, &schema.data_type)?;
+3 -11
View File
@@ -228,12 +228,6 @@ impl PrometheusJsonResponse {
tag_column_indices.push(i);
num_label_columns += 1;
}
ConcreteDataType::Dictionary(ref dictionary)
if matches!(dictionary.value_type(), ConcreteDataType::String(_)) =>
{
tag_column_indices.push(i);
num_label_columns += 1;
}
_ => {}
}
}
@@ -254,7 +248,7 @@ impl PrometheusJsonResponse {
// prepare things...
let tag_columns = tag_column_indices
.iter()
.map(|i| batch.column(*i))
.map(|i| batch.column(*i).as_string::<i32>())
.collect::<Vec<_>>();
let tag_names = tag_column_indices
.iter()
@@ -286,10 +280,8 @@ impl PrometheusJsonResponse {
}
for (tag_column, tag_name) in tag_columns.iter().zip(tag_names.iter()) {
// TODO(ruihang): add test for NULL tag
if let Some(value) = datatypes::arrow_array::string_array_value_at_index(
tag_column, row_index,
) {
tags.push((tag_name, value));
if tag_column.is_valid(row_index) {
tags.push((tag_name, tag_column.value(row_index)));
}
}
+6 -2
View File
@@ -24,6 +24,7 @@ use chrono::{NaiveDate, NaiveDateTime};
use common_catalog::parse_optional_catalog_and_schema_from_db_string;
use common_error::ext::ErrorExt;
use common_query::Output;
use common_recordbatch::map_dictionary_to_values_data_type;
use common_telemetry::{debug, error, tracing, warn};
use datafusion_common::ParamValues;
use datafusion_expr::LogicalPlan;
@@ -233,7 +234,10 @@ impl MysqlInstanceShim {
.column_schemas()
.iter()
.map(|column_schema| {
create_mysql_column(&column_schema.data_type, &column_schema.name)
create_mysql_column(
&map_dictionary_to_values_data_type(&column_schema.data_type),
&column_schema.name,
)
})
.collect::<Result<Vec<_>>>()
})
@@ -906,7 +910,7 @@ fn prepared_params(
// Placeholder index starts from 1
for i in 1..param_num {
let column = if let Some(Some(t)) = param_types.get(&format_placeholder(i)) {
create_mysql_column(t, "")?
create_mysql_column(&map_dictionary_to_values_data_type(t), "")?
} else {
create_mysql_column(&ConcreteDataType::null_datatype(), "")?
};
+31 -1
View File
@@ -19,7 +19,7 @@ use chrono::NaiveDate;
use common_query::prelude::ScalarValue;
use common_sql::convert::sql_value_to_value;
use common_time::{Date, Timestamp};
use datatypes::prelude::ConcreteDataType;
use datatypes::prelude::{ConcreteDataType, DataType};
use datatypes::schema::ColumnSchema;
use datatypes::types::TimestampType;
use datatypes::value::{self, Value};
@@ -138,6 +138,13 @@ where
/// Convert [`ParamValue`] into [`Value`] according to param type.
/// It will try it's best to do type conversions if possible
pub fn convert_value(param: &ParamValue, t: &ConcreteDataType) -> Result<ScalarValue> {
if let ConcreteDataType::Dictionary(dictionary) = t {
return Ok(ScalarValue::Dictionary(
Box::new(dictionary.key_type().as_arrow_type()),
Box::new(convert_value(param, dictionary.value_type())?),
));
}
match param.value.into_inner() {
ValueInner::Int(i) => match t {
ConcreteDataType::Int8(_) => Ok(ScalarValue::Int8(Some(i as i8))),
@@ -258,6 +265,16 @@ pub fn convert_value(param: &ParamValue, t: &ConcreteDataType) -> Result<ScalarV
/// Convert an MySQL expression to a scalar value.
/// It automatically handles the conversion of strings to numeric values.
pub fn convert_expr_to_scalar_value(param: &Expr, t: &ConcreteDataType) -> Result<ScalarValue> {
if let ConcreteDataType::Dictionary(dictionary) = t {
return Ok(ScalarValue::Dictionary(
Box::new(dictionary.key_type().as_arrow_type()),
Box::new(convert_expr_to_scalar_value(
param,
dictionary.value_type(),
)?),
));
}
let column_schema = ColumnSchema::new("", t.clone(), true);
match param {
Expr::Value(v) => {
@@ -464,6 +481,19 @@ mod tests {
let v = convert_expr_to_scalar_value(&expr, &t).unwrap();
assert_eq!(ScalarValue::Utf8(Some("hello".to_string())), v);
let t = ConcreteDataType::dictionary_datatype(
ConcreteDataType::uint32_datatype(),
ConcreteDataType::string_datatype(),
);
let v = convert_expr_to_scalar_value(&expr, &t).unwrap();
assert_eq!(
ScalarValue::Dictionary(
Box::new(arrow_schema::DataType::UInt32),
Box::new(ScalarValue::Utf8(Some("hello".to_string()))),
),
v
);
let expr = Expr::Value(ValueExpr::Null.into());
let t = ConcreteDataType::time_microsecond_datatype();
let v = convert_expr_to_scalar_value(&expr, &t).unwrap();
+5 -17
View File
@@ -42,7 +42,7 @@ use session::context::QueryContextRef;
use snafu::prelude::*;
use tokio::io::AsyncWrite;
use crate::error::{ConvertSqlValueSnafu, DataFusionSnafu, NotSupportedSnafu, Result};
use crate::error::{self, ConvertSqlValueSnafu, DataFusionSnafu, NotSupportedSnafu, Result};
use crate::metrics::*;
/// Try to write multiple output to the writer if possible.
@@ -249,20 +249,6 @@ impl MysqlResultWriter {
let v = datatypes::arrow_array::string_array_value(column, i);
row_writer.write_col(v)?;
}
DataType::Dictionary(_, value_type)
if matches!(
value_type.as_ref(),
DataType::Utf8 | DataType::Utf8View | DataType::LargeUtf8
) =>
{
if let Some(v) =
datatypes::arrow_array::string_array_value_at_index(column, i)
{
row_writer.write_col(v)?;
} else {
row_writer.write_col(None::<u8>)?;
}
}
DataType::Binary | DataType::BinaryView | DataType::LargeBinary => {
let v = datatypes::arrow_array::binary_array_value(column, i);
if let ConcreteDataType::Json(_) = &schema.column_schemas()[j].data_type {
@@ -378,9 +364,11 @@ pub(crate) fn create_mysql_column(
ConcreteDataType::Vector(_) => Ok(ColumnType::MYSQL_TYPE_BLOB),
ConcreteDataType::List(_) => Ok(ColumnType::MYSQL_TYPE_VARCHAR),
ConcreteDataType::Struct(_) => Ok(ColumnType::MYSQL_TYPE_VARCHAR),
ConcreteDataType::Dictionary(dictionary) => {
return create_mysql_column(dictionary.value_type(), column_name);
_ => error::UnsupportedDataTypeSnafu {
data_type,
reason: "not implemented",
}
.fail(),
};
let mut colflags = ColumnFlags::empty();
match data_type {
+2 -1
View File
@@ -18,8 +18,8 @@ use std::sync::Arc;
use async_trait::async_trait;
use common_query::{Output, OutputData};
use common_recordbatch::RecordBatch;
use common_recordbatch::error::Result as RecordBatchResult;
use common_recordbatch::{RecordBatch, map_dictionary_to_values_schema};
use common_telemetry::{debug, info, tracing};
use datafusion::sql::sqlparser::ast::{CopyOption, CopyTarget, Statement as SqlParserStatement};
use datafusion_common::ParamValues;
@@ -557,6 +557,7 @@ fn describe_fields(
// query
SqlPlan::Plan(plan, _) if !matches!(plan, LogicalPlan::Dml(_) | LogicalPlan::Ddl(_)) => {
let schema: Schema = plan.schema().clone().try_into().map_err(convert_err)?;
let (schema, _) = map_dictionary_to_values_schema(Arc::new(schema));
schema_to_pg(&schema, format, None).map_err(convert_err)
}
// We can cover only part of show statements
+68 -29
View File
@@ -24,14 +24,14 @@ use arrow_pg::encoder::{Encoder, encode_value};
use arrow_pg::list_encoder::encode_list;
use arrow_schema::{DataType, TimeUnit};
use chrono::{DateTime, FixedOffset, NaiveDate, NaiveDateTime};
use common_recordbatch::RecordBatch;
use common_recordbatch::error::Result as RecordBatchResult;
use common_recordbatch::{RecordBatch, map_dictionary_to_values_data_type};
use common_time::{IntervalDayTime, IntervalMonthDayNano, IntervalYearMonth};
use datafusion_common::ScalarValue;
use datafusion_expr::LogicalPlan;
use datatypes::arrow::datatypes::DataType as ArrowDataType;
use datatypes::json::JsonSettings;
use datatypes::prelude::{ConcreteDataType, Value};
use datatypes::prelude::{ConcreteDataType, DataType as _, Value};
use datatypes::schema::{Schema, SchemaRef};
use datatypes::types::{Decimal128Type, IntervalType, TimestampType, jsonb_to_string};
use datatypes::value::StructValue;
@@ -222,15 +222,6 @@ where
DataType::Struct(_) => {
encode_struct(query_ctx, Default::default(), encoder, pg_field)?;
}
DataType::Dictionary(_, value_type)
if matches!(
value_type.as_ref(),
DataType::Utf8 | DataType::Utf8View | DataType::LargeUtf8
) =>
{
let value = datatypes::arrow_array::string_array_value_at_index(column, i);
encoder.encode_field(&value, pg_field)?;
}
_ => {
// Encode value using arrow-pg
let arrow_field = arrow_schema.field(j);
@@ -289,7 +280,11 @@ pub(super) fn type_gt_to_pg(origin: &ConcreteDataType) -> Result<Type> {
}
.fail(),
},
ConcreteDataType::Dictionary(dictionary) => type_gt_to_pg(dictionary.value_type()),
&ConcreteDataType::Dictionary(_) => server_error::UnsupportedDataTypeSnafu {
data_type: origin,
reason: "not implemented",
}
.fail(),
&ConcreteDataType::Duration(_) => Ok(Type::INTERVAL),
&ConcreteDataType::Struct(_) => Ok(Type::JSON),
}
@@ -375,6 +370,29 @@ fn numeric_out_of_range_error(value: impl std::fmt::Display) -> PgWireError {
)
}
fn string_parameter_to_scalar_value(
data: Option<String>,
data_type: &ConcreteDataType,
) -> Option<ScalarValue> {
match data_type {
ConcreteDataType::String(string_type) => {
if string_type.is_large() {
Some(ScalarValue::LargeUtf8(data))
} else {
Some(ScalarValue::Utf8(data))
}
}
ConcreteDataType::Dictionary(dictionary) => Some(ScalarValue::Dictionary(
Box::new(dictionary.key_type().as_arrow_type()),
Box::new(string_parameter_to_scalar_value(
data,
dictionary.value_type(),
)?),
)),
_ => None,
}
}
pub(super) fn parameters_to_scalar_values(
plan: &LogicalPlan,
portal: &Portal<PgSqlPlan>,
@@ -398,7 +416,8 @@ pub(super) fn parameters_to_scalar_values(
let client_type = if let Some(Some(client_given_type)) = client_param_types.get(idx) {
client_given_type.clone()
} else if let Some(server_provided_type) = &server_type {
type_gt_to_pg(server_provided_type).map_err(convert_err)?
type_gt_to_pg(&map_dictionary_to_values_data_type(server_provided_type))
.map_err(convert_err)?
} else {
return Err(invalid_parameter_error(
"unknown_parameter_type",
@@ -413,21 +432,12 @@ pub(super) fn parameters_to_scalar_values(
&Type::VARCHAR | &Type::TEXT | &Type::CHAR => {
let data = portal.parameter::<String>(idx, &client_type)?;
if let Some(server_type) = &server_type {
match server_type {
ConcreteDataType::String(t) => {
if t.is_large() {
ScalarValue::LargeUtf8(data)
} else {
ScalarValue::Utf8(data)
}
}
_ => {
return Err(invalid_parameter_error(
"invalid_parameter_type",
Some(format!("Expected: {}, found: {}", server_type, client_type)),
));
}
}
string_parameter_to_scalar_value(data, server_type).ok_or_else(|| {
invalid_parameter_error(
"invalid_parameter_type",
Some(format!("Expected: {}, found: {}", server_type, client_type)),
)
})?
} else {
ScalarValue::Utf8(data)
}
@@ -1223,7 +1233,7 @@ pub(super) fn param_types_to_pg_types(
let mut types = Vec::with_capacity(param_count);
for i in 0..param_count {
if let Some(Some(param_type)) = param_types.get(&format!("${}", i + 1)) {
let pg_type = type_gt_to_pg(param_type)?;
let pg_type = type_gt_to_pg(&map_dictionary_to_values_data_type(param_type))?;
types.push(pg_type);
} else {
types.push(Type::UNKNOWN);
@@ -1745,6 +1755,35 @@ mod test {
Portal::try_new(&bind, statement).unwrap()
}
#[test]
fn test_dictionary_string_parameter() {
let dictionary_type =
DataType::Dictionary(Box::new(DataType::UInt32), Box::new(DataType::Utf8));
let plan = build_plan_with_params(vec![("$1", dictionary_type)]);
let portal = make_portal(vec![None], vec![s("host-a")]);
let values = parameters_to_scalar_values(&plan, &portal).unwrap();
assert_eq!(
vec![ScalarValue::Dictionary(
Box::new(DataType::UInt32),
Box::new(ScalarValue::Utf8(s("host-a"))),
)],
values
);
let param_types = HashMap::from([(
"$1".to_string(),
Some(ConcreteDataType::dictionary_datatype(
ConcreteDataType::uint32_datatype(),
ConcreteDataType::string_datatype(),
)),
)]);
assert_eq!(
vec![Type::VARCHAR],
param_types_to_pg_types(&param_types).unwrap()
);
}
#[test]
fn test_int2_coerce_in_range() {
let plan = build_plan_with_params(vec![
-24
View File
@@ -16,19 +16,13 @@ use std::sync::Arc;
use api::v1::greptime_request::Request;
use api::v1::query_request::Query;
use arrow::array::{DictionaryArray, StringArray, UInt32Array};
use arrow::datatypes::UInt32Type;
use async_trait::async_trait;
use catalog::memory::MemoryCatalogManager;
use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
use common_error::ext::BoxedError;
use common_grpc::flight::do_put::DoPutResponse;
use common_query::Output;
use common_recordbatch::RecordBatch;
use datafusion_expr::LogicalPlan;
use datatypes::prelude::{ConcreteDataType, VectorRef};
use datatypes::schema::{ColumnSchema, Schema};
use datatypes::vectors::DictionaryVector;
use futures_util::TryFutureExt;
use query::options::QueryOptions;
use query::parser::{PromQuery, QueryStatement};
@@ -42,7 +36,6 @@ use snafu::{ResultExt, ensure};
use sql::parser::{ParseOptions, ParserContext};
use sql::statements::statement::Statement;
use table::TableRef;
use table::test_util::MemTable;
mod http;
mod interceptor;
@@ -50,23 +43,6 @@ mod mysql;
mod postgres;
mod prom_remote_write_v2_test;
fn dictionary_child_null_table() -> TableRef {
let data_type = ConcreteDataType::dictionary_datatype(
ConcreteDataType::uint32_datatype(),
ConcreteDataType::string_datatype(),
);
let schema = Arc::new(Schema::new(vec![ColumnSchema::new(
"host", data_type, true,
)]));
let dictionary = DictionaryArray::<UInt32Type>::new(
UInt32Array::from(vec![0, 1]),
Arc::new(StringArray::from(vec![None, Some("host-b")])),
);
let vector = Arc::new(DictionaryVector::try_from(dictionary).unwrap()) as VectorRef;
let record_batch = RecordBatch::new(schema, vec![vector]).unwrap();
MemTable::table("dictionary_values", record_batch)
}
pub struct DummyInstance {
query_engine: QueryEngineRef,
}
+1 -23
View File
@@ -34,8 +34,8 @@ use servers::tls::{ReloadableTlsServerConfig, TlsOption};
use table::TableRef;
use table::test_util::MemTable;
use crate::create_testing_sql_query_handler;
use crate::mysql::{MysqlTextRow, TestingData, all_datatype_testing_data};
use crate::{create_testing_sql_query_handler, dictionary_child_null_table};
#[derive(Default)]
struct MysqlOpts<'a> {
@@ -77,28 +77,6 @@ fn create_mysql_server(table: TableRef, opts: MysqlOpts<'_>) -> Result<Box<dyn S
))
}
#[tokio::test]
async fn test_mysql_dictionary_child_null() -> Result<()> {
let mut mysql_server = create_mysql_server(dictionary_child_null_table(), Default::default())?;
mysql_server
.start("127.0.0.1:0".parse::<SocketAddr>().unwrap())
.await
.unwrap();
let server_port = mysql_server.bind_addr().unwrap().port();
let mut connection = create_connection_default_db_name(server_port, false)
.await
.unwrap();
let values: Vec<Option<String>> = connection
.query("SELECT host FROM dictionary_values")
.await
.unwrap();
assert_eq!(vec![None, Some("host-b".to_string())], values);
mysql_server.shutdown().await.unwrap();
Ok(())
}
#[tokio::test]
async fn test_start_mysql_server() -> Result<()> {
let table = MemTable::default_numbers_table();
+1 -30
View File
@@ -35,7 +35,7 @@ use table::TableRef;
use table::test_util::MemTable;
use tokio_postgres::{Client, Error as PgError, NoTls, SimpleQueryMessage};
use crate::{create_testing_instance, dictionary_child_null_table};
use crate::create_testing_instance;
fn create_postgres_server(
table: TableRef,
@@ -75,35 +75,6 @@ fn create_postgres_server(
)))
}
#[tokio::test]
async fn test_postgres_dictionary_child_null() -> Result<()> {
let mut postgres_server = create_postgres_server(
dictionary_child_null_table(),
false,
Default::default(),
None,
)?;
postgres_server
.start("127.0.0.1:0".parse::<SocketAddr>().unwrap())
.await
.unwrap();
let server_port = postgres_server.bind_addr().unwrap().port();
let client = create_plain_connection(server_port, false).await.unwrap();
let rows = client
.query("SELECT host FROM dictionary_values", &[])
.await
.unwrap();
let values = rows
.iter()
.map(|row| row.get::<_, Option<String>>(0))
.collect::<Vec<_>>();
assert_eq!(vec![None, Some("host-b".to_string())], values);
postgres_server.shutdown().await.unwrap();
Ok(())
}
#[tokio::test]
pub async fn test_start_postgres_server() -> Result<()> {
let table = MemTable::default_numbers_table();
@@ -87,6 +87,39 @@ DEALLOCATE stmt;
affected_rows: 0
-- test with data
-- A Mito scan exposes string primary keys as dictionaries, but that physical read type must not
-- leak into INSERT Values planning.
CREATE TABLE insert_values_dictionary (
ts TIMESTAMP TIME INDEX,
host STRING PRIMARY KEY
) ENGINE=mito;
Affected Rows: 0
INSERT INTO insert_values_dictionary VALUES(now(), 'a');
Affected Rows: 1
SELECT host FROM insert_values_dictionary;
+------+
| host |
+------+
| a |
+------+
SELECT array_agg(host) FROM insert_values_dictionary;
+------------------------------------------+
| array_agg(insert_values_dictionary.host) |
+------------------------------------------+
| [a] |
+------------------------------------------+
DROP TABLE insert_values_dictionary;
Affected Rows: 0
CREATE TABLE IF NOT EXISTS "cake" (
`domain` STRING,
is_expire BOOLEAN NULL,
@@ -49,6 +49,21 @@ EXECUTE stmt USING 1, 2;
DEALLOCATE stmt;
-- test with data
-- A Mito scan exposes string primary keys as dictionaries, but that physical read type must not
-- leak into INSERT Values planning.
CREATE TABLE insert_values_dictionary (
ts TIMESTAMP TIME INDEX,
host STRING PRIMARY KEY
) ENGINE=mito;
INSERT INTO insert_values_dictionary VALUES(now(), 'a');
SELECT host FROM insert_values_dictionary;
SELECT array_agg(host) FROM insert_values_dictionary;
DROP TABLE insert_values_dictionary;
CREATE TABLE IF NOT EXISTS "cake" (
`domain` STRING,
is_expire BOOLEAN NULL,