mirror of
https://github.com/GreptimeTeam/greptimedb.git
synced 2026-11-17 08:14:56 +00:00
fix(query): preserve pushdown for dictionary labels
Signed-off-by: Ruihang Xia <waynestxia@gmail.com>
This commit is contained in:
@@ -25,7 +25,7 @@ use common_recordbatch::DfRecordBatch;
|
||||
use datatypes::arrow;
|
||||
use datatypes::arrow::array::ArrayRef;
|
||||
use datatypes::arrow::buffer::Buffer;
|
||||
use datatypes::arrow::datatypes::{DataType, Schema as ArrowSchema, SchemaRef};
|
||||
use datatypes::arrow::datatypes::{Schema as ArrowSchema, SchemaRef};
|
||||
use datatypes::arrow::error::ArrowError;
|
||||
use datatypes::arrow::ipc::{MessageHeader, convert, reader, root_as_message, writer};
|
||||
use flatbuffers::FlatBufferBuilder;
|
||||
@@ -87,17 +87,6 @@ impl FlightEncoder {
|
||||
}
|
||||
}
|
||||
|
||||
/// Encodes a query-result schema and tracks dictionaries recursively.
|
||||
pub fn encode_query_schema(&mut self, schema: &ArrowSchema) -> FlightData {
|
||||
self.data_gen
|
||||
.schema_to_bytes_with_dictionary_tracker(
|
||||
schema,
|
||||
&mut self.dictionary_tracker,
|
||||
&self.write_options,
|
||||
)
|
||||
.into()
|
||||
}
|
||||
|
||||
/// Encode the Arrow schema to [FlightData].
|
||||
pub fn encode_schema(&self, schema: &ArrowSchema) -> FlightData {
|
||||
SchemaAsIpc::new(schema, &self.write_options).into()
|
||||
@@ -110,15 +99,15 @@ impl FlightEncoder {
|
||||
/// be encoded to exactly one [FlightData].
|
||||
pub fn encode(&mut self, flight_message: FlightMessage) -> Vec1<FlightData> {
|
||||
match flight_message {
|
||||
FlightMessage::Schema(schema) => {
|
||||
schema.fields().iter().for_each(|x| {
|
||||
if matches!(x.data_type(), DataType::Dictionary(_, _)) {
|
||||
self.dictionary_tracker.next_dict_id();
|
||||
}
|
||||
});
|
||||
|
||||
vec1![self.encode_schema(schema.as_ref())]
|
||||
}
|
||||
FlightMessage::Schema(schema) => vec1![
|
||||
self.data_gen
|
||||
.schema_to_bytes_with_dictionary_tracker(
|
||||
schema.as_ref(),
|
||||
&mut self.dictionary_tracker,
|
||||
&self.write_options,
|
||||
)
|
||||
.into(),
|
||||
],
|
||||
FlightMessage::RecordBatch(record_batch) => {
|
||||
let (encoded_dictionaries, encoded_batch) = self
|
||||
.data_gen
|
||||
@@ -618,7 +607,7 @@ mod test {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_query_schema_with_nested_dictionary_array() -> Result<()> {
|
||||
fn test_schema_with_nested_dictionary_array() -> Result<()> {
|
||||
let item = Arc::new(Field::new_dictionary(
|
||||
"item",
|
||||
DataType::UInt32,
|
||||
@@ -643,12 +632,12 @@ mod test {
|
||||
let batch = DfRecordBatch::try_new(schema.clone(), vec![Arc::new(list)]).unwrap();
|
||||
|
||||
let mut encoder = FlightEncoder::default();
|
||||
let encoded_schema = encoder.encode_query_schema(&schema);
|
||||
let encoded_schema = encoder.encode(FlightMessage::Schema(schema.clone()));
|
||||
let encoded_batch = encoder.encode(FlightMessage::RecordBatch(batch.clone()));
|
||||
|
||||
let mut decoder = FlightDecoder::default();
|
||||
assert!(matches!(
|
||||
decoder.try_decode(&encoded_schema)?,
|
||||
decoder.try_decode(&encoded_schema[0])?,
|
||||
Some(FlightMessage::Schema(actual)) if actual == schema
|
||||
));
|
||||
for data in encoded_batch.iter().take(encoded_batch.len() - 1) {
|
||||
|
||||
@@ -167,6 +167,9 @@ impl AnalyzerRule for DistPlannerAnalyzer {
|
||||
}
|
||||
}
|
||||
};
|
||||
let plan = plan
|
||||
.transform_down_with_subqueries(&unwrap_dictionary_literals)?
|
||||
.data;
|
||||
|
||||
let result = match self.try_push_down(plan.clone()) {
|
||||
Ok(plan) => plan,
|
||||
@@ -225,6 +228,12 @@ fn pre_merge_scan_optimizer() -> Optimizer {
|
||||
}
|
||||
|
||||
fn unwrap_dictionary_literals(plan: LogicalPlan) -> DfResult<Transformed<LogicalPlan>> {
|
||||
// A Values plan derives its schema from its expressions. Rewriting only the expressions would
|
||||
// leave that schema inconsistent, which affects DML planning.
|
||||
if matches!(&plan, LogicalPlan::Values(_)) {
|
||||
return Ok(Transformed::no(plan));
|
||||
}
|
||||
|
||||
plan.map_expressions(|expr| {
|
||||
expr.transform_up(|expr| match expr {
|
||||
Expr::Literal(ScalarValue::Dictionary(_, value), metadata) => {
|
||||
@@ -735,16 +744,9 @@ impl PlanRewriter {
|
||||
self.partition_cols
|
||||
);
|
||||
|
||||
// Substrait does not encode dictionary literals. Only unwrap literals in the remote
|
||||
// input; rewriting the whole plan also changes DML Values without rebuilding its schema.
|
||||
let remote_input = on_node
|
||||
.clone()
|
||||
.transform_down_with_subqueries(&unwrap_dictionary_literals)?
|
||||
.data;
|
||||
|
||||
// add merge scan as the new root
|
||||
let mut node = MergeScanLogicalPlan::new(
|
||||
remote_input,
|
||||
on_node.clone(),
|
||||
false,
|
||||
// at this stage, the partition cols should be set
|
||||
// treat it as non-partitioned if None
|
||||
|
||||
@@ -301,7 +301,7 @@ fn find_merge_scan(plan: &LogicalPlan) -> Option<&MergeScanLogicalPlan> {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dictionary_literals_are_unwrapped_only_in_remote_input() {
|
||||
fn dictionary_literals_are_unwrapped_before_pushdown() {
|
||||
let table = Arc::new(Table::new(
|
||||
TestTable::table_info(0, "t".to_string(), MITO_ENGINE.to_string()),
|
||||
FilterPushDownType::Inexact,
|
||||
@@ -330,7 +330,7 @@ fn dictionary_literals_are_unwrapped_only_in_remote_input() {
|
||||
assert!(
|
||||
result
|
||||
.to_string()
|
||||
.contains("Filter: t.pk1 = Dictionary(UInt32, Utf8(\"host-a\"))")
|
||||
.contains("Filter: t.pk1 = Utf8(\"host-a\")")
|
||||
);
|
||||
|
||||
let remote_input = find_merge_scan(&result).unwrap().input();
|
||||
@@ -344,6 +344,34 @@ fn dictionary_literals_are_unwrapped_only_in_remote_input() {
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dictionary_literals_in_values_keep_their_schema() {
|
||||
let literal = Expr::Literal(
|
||||
ScalarValue::Dictionary(
|
||||
Box::new(DataType::UInt32),
|
||||
Box::new(ScalarValue::Utf8(Some("host-a".to_string()))),
|
||||
),
|
||||
None,
|
||||
);
|
||||
let plan = LogicalPlanBuilder::values(vec![vec![literal]])
|
||||
.unwrap()
|
||||
.build()
|
||||
.unwrap();
|
||||
let schema = plan.schema().clone();
|
||||
|
||||
let transformed = plan
|
||||
.transform_down_with_subqueries(&unwrap_dictionary_literals)
|
||||
.unwrap()
|
||||
.data;
|
||||
|
||||
assert_eq!(&schema, transformed.schema());
|
||||
assert!(
|
||||
transformed
|
||||
.to_string()
|
||||
.contains("Dictionary(UInt32, Utf8(\"host-a\"))")
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(feature = "vector_index")]
|
||||
mod vector_search_tests {
|
||||
use std::sync::Arc;
|
||||
|
||||
@@ -232,9 +232,6 @@ impl Stream for FlightRecordBatchStream {
|
||||
Poll::Ready(None)
|
||||
}
|
||||
Poll::Ready(Some(result)) => match result {
|
||||
Ok(FlightMessage::Schema(schema)) => {
|
||||
Poll::Ready(Some(Ok(this.encoder.encode_query_schema(schema.as_ref()))))
|
||||
}
|
||||
Ok(flight_message) => {
|
||||
let mut iter = this.encoder.encode(flight_message).into_iter();
|
||||
let Some(first) = iter.next() else {
|
||||
|
||||
Reference in New Issue
Block a user