1use std::any::Any;
20use std::fmt::Display;
21use std::sync::Arc;
22
23use ahash::HashMap;
24use arrow::array::{StringBuilder, UInt32Builder};
25use arrow_schema::{DataType, Field, Schema, SchemaRef};
26use common_recordbatch::adapter::{MetricCollector, PlanMetrics, RecordBatchMetrics};
27use common_recordbatch::{DfRecordBatch, DfSendableRecordBatchStream};
28use datafusion::error::Result as DfResult;
29use datafusion::execution::TaskContext;
30use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec;
31use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
32use datafusion::physical_plan::{
33 DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties, accept,
34};
35use datafusion_common::tree_node::{TreeNode, TreeNodeRecursion};
36use datafusion_common::{DataFusionError, assert_eq_or_internal_err, internal_err};
37use datafusion_physical_expr::{Distribution, EquivalenceProperties, Partitioning};
38use futures::StreamExt;
39use serde::Serialize;
40use serde_json::{Value, json};
41use sqlparser::ast::AnalyzeFormat;
42
43use crate::dist_plan::MergeScanExec;
44
45const STAGE: &str = "stage";
46const NODE: &str = "node";
47const PLAN: &str = "plan";
48
49pub fn dist_analyze_output_schema() -> SchemaRef {
53 SchemaRef::new(Schema::new(vec![
54 Field::new(STAGE, DataType::UInt32, true),
55 Field::new(NODE, DataType::UInt32, true),
56 Field::new(PLAN, DataType::Utf8, true),
57 ]))
58}
59
60#[derive(Debug)]
61pub struct DistAnalyzeExec {
62 input: Arc<dyn ExecutionPlan>,
63 schema: SchemaRef,
64 properties: Arc<PlanProperties>,
65 verbose: bool,
66 format: AnalyzeFormat,
67}
68
69impl DistAnalyzeExec {
70 pub fn new(input: Arc<dyn ExecutionPlan>, verbose: bool, format: AnalyzeFormat) -> Self {
72 let schema = dist_analyze_output_schema();
73 let properties = Arc::new(Self::compute_properties(&input, schema.clone()));
74 Self {
75 input,
76 schema,
77 properties,
78 verbose,
79 format,
80 }
81 }
82
83 fn compute_properties(input: &Arc<dyn ExecutionPlan>, schema: SchemaRef) -> PlanProperties {
85 let eq_properties = EquivalenceProperties::new(schema);
86 let output_partitioning = Partitioning::UnknownPartitioning(1);
87 let properties = input.properties();
88 PlanProperties::new(
89 eq_properties,
90 output_partitioning,
91 properties.emission_type,
92 properties.boundedness,
93 )
94 }
95
96 pub fn input(&self) -> &Arc<dyn ExecutionPlan> {
97 &self.input
98 }
99}
100
101pub fn analyze_plan_metrics_to_json_value(
109 plan: &Arc<dyn ExecutionPlan>,
110 verbose: bool,
111) -> serde_json::Result<Value> {
112 let input = plan
113 .as_any()
114 .downcast_ref::<DistAnalyzeExec>()
115 .map(|exec| exec.input().clone())
116 .unwrap_or_else(|| plan.clone());
117
118 let mut stages = Vec::new();
119 let mut collector = MetricCollector::new(verbose);
120 accept(input.as_ref(), &mut collector).unwrap();
121 stages.push(json!({
122 "stage": 0,
123 "node": 0,
124 "plan": JsonMetrics::from_record_batch_metrics(collector.record_batch_metrics),
125 }));
126
127 let _ = input.apply(|plan| {
128 if let Some(merge_scan) = plan.as_any().downcast_ref::<MergeScanExec>() {
129 for (node, metric) in merge_scan.sub_stage_metrics().into_iter().enumerate() {
130 stages.push(json!({
131 "stage": 1,
132 "node": node,
133 "plan": JsonMetrics::from_record_batch_metrics(metric),
134 }));
135 }
136 }
137 Ok(TreeNodeRecursion::Continue)
138 });
139
140 Ok(Value::Array(stages))
141}
142
143impl DisplayAs for DistAnalyzeExec {
144 fn fmt_as(&self, t: DisplayFormatType, f: &mut std::fmt::Formatter) -> std::fmt::Result {
145 match t {
146 DisplayFormatType::Default
147 | DisplayFormatType::Verbose
148 | DisplayFormatType::TreeRender => {
149 write!(f, "DistAnalyzeExec",)
150 }
151 }
152 }
153}
154
155impl ExecutionPlan for DistAnalyzeExec {
156 fn name(&self) -> &'static str {
157 "DistAnalyzeExec"
158 }
159
160 fn as_any(&self) -> &dyn Any {
162 self
163 }
164
165 fn properties(&self) -> &Arc<PlanProperties> {
166 &self.properties
167 }
168
169 fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {
170 vec![&self.input]
171 }
172
173 fn required_input_distribution(&self) -> Vec<Distribution> {
175 vec![]
176 }
177
178 fn with_new_children(
179 self: Arc<Self>,
180 mut children: Vec<Arc<dyn ExecutionPlan>>,
181 ) -> DfResult<Arc<dyn ExecutionPlan>> {
182 assert_eq_or_internal_err!(
183 children.len(),
184 1,
185 "DistAnalyzeExec requires exactly one child"
186 );
187 Ok(Arc::new(Self::new(
188 children.swap_remove(0),
189 self.verbose,
190 self.format,
191 )))
192 }
193
194 fn execute(
195 &self,
196 partition: usize,
197 context: Arc<TaskContext>,
198 ) -> DfResult<DfSendableRecordBatchStream> {
199 if 0 != partition {
200 return internal_err!("AnalyzeExec invalid partition. Expected 0, got {partition}");
201 }
202
203 let coalesce_partition_plan = CoalescePartitionsExec::new(self.input.clone());
206
207 let captured_input = self.input.clone();
209 let captured_schema = self.schema.clone();
210
211 let format = self.format;
213 let verbose = self.verbose;
214 let mut input_stream = coalesce_partition_plan.execute(0, context)?;
215 let output = async move {
216 let mut total_rows = 0;
217 while let Some(batch) = input_stream.next().await.transpose()? {
218 total_rows += batch.num_rows();
219 }
220
221 create_output_batch(total_rows, captured_input, captured_schema, format, verbose)
222 };
223
224 Ok(Box::pin(RecordBatchStreamAdapter::new(
225 self.schema.clone(),
226 futures::stream::once(output),
227 )))
228 }
229}
230
231struct AnalyzeOutputBuilder {
233 stage_builder: UInt32Builder,
234 node_builder: UInt32Builder,
235 plan_builder: StringBuilder,
236 schema: SchemaRef,
237}
238
239impl AnalyzeOutputBuilder {
240 fn new(schema: SchemaRef) -> Self {
241 Self {
242 stage_builder: UInt32Builder::with_capacity(4),
243 node_builder: UInt32Builder::with_capacity(4),
244 plan_builder: StringBuilder::with_capacity(1, 1024),
245 schema,
246 }
247 }
248
249 fn append_metric(&mut self, stage: u32, node: u32, content: String) {
250 self.stage_builder.append_value(stage);
251 self.node_builder.append_value(node);
252 self.plan_builder.append_value(content);
253 }
254
255 fn append_total_rows(&mut self, total_rows: usize) {
256 self.stage_builder.append_null();
257 self.node_builder.append_null();
258 self.plan_builder
259 .append_value(format!("Total rows: {}", total_rows));
260 }
261
262 fn finish(mut self) -> DfResult<DfRecordBatch> {
263 DfRecordBatch::try_new(
264 self.schema,
265 vec![
266 Arc::new(self.stage_builder.finish()),
267 Arc::new(self.node_builder.finish()),
268 Arc::new(self.plan_builder.finish()),
269 ],
270 )
271 .map_err(DataFusionError::from)
272 }
273}
274
275fn create_output_batch(
277 total_rows: usize,
278 input: Arc<dyn ExecutionPlan>,
279 schema: SchemaRef,
280 format: AnalyzeFormat,
281 verbose: bool,
282) -> DfResult<DfRecordBatch> {
283 let mut builder = AnalyzeOutputBuilder::new(schema);
284
285 let mut collector = MetricCollector::new(verbose);
287 accept(input.as_ref(), &mut collector).unwrap();
289 let stage_0_metrics = collector.record_batch_metrics;
290
291 builder.append_metric(0, 0, metrics_to_string(stage_0_metrics, format)?);
293
294 input.apply(|plan| {
296 if let Some(merge_scan) = plan.as_any().downcast_ref::<MergeScanExec>() {
297 let sub_stage_metrics = merge_scan.sub_stage_metrics();
298 for (node, metric) in sub_stage_metrics.into_iter().enumerate() {
299 builder.append_metric(1, node as _, metrics_to_string(metric, format)?);
300 }
301 return Ok(TreeNodeRecursion::Continue);
303 }
304 Ok(TreeNodeRecursion::Continue)
305 })?;
306
307 builder.append_total_rows(total_rows);
309
310 builder.finish()
311}
312
313fn metrics_to_string(metrics: RecordBatchMetrics, format: AnalyzeFormat) -> DfResult<String> {
314 match format {
315 AnalyzeFormat::JSON => Ok(JsonMetrics::from_record_batch_metrics(metrics).to_string()),
316 AnalyzeFormat::TEXT => Ok(metrics.to_string()),
317 format => Err(DataFusionError::NotImplemented(format!(
318 "AnalyzeFormat {format}",
319 ))),
320 }
321}
322
323#[derive(Debug, Default, Serialize)]
324struct JsonMetrics {
325 name: String,
326 param: String,
327
328 output_rows: usize,
330 elapsed_compute: usize,
332
333 metrics: HashMap<String, usize>,
335 children: Vec<JsonMetrics>,
336}
337
338impl JsonMetrics {
339 fn from_record_batch_metrics(record_batch_metrics: RecordBatchMetrics) -> Self {
340 let mut layers: HashMap<usize, Vec<Self>> = HashMap::default();
341
342 for plan_metrics in record_batch_metrics.plan_metrics.into_iter().rev() {
343 let (level, mut metrics) = Self::from_plan_metrics(plan_metrics);
344 if let Some(next_layer) = layers.remove(&(level + 1)) {
345 metrics.children = next_layer;
346 }
347 if level == 0 {
348 return metrics;
349 }
350 layers.entry(level).or_default().push(metrics);
351 }
352
353 Self::default()
355 }
356
357 fn from_plan_metrics(plan_metrics: PlanMetrics) -> (usize, Self) {
361 let raw_name = plan_metrics.plan.trim_end();
362 let mut elapsed_compute = 0;
363 let mut output_rows = 0;
364 let mut other_metrics = HashMap::default();
365 let (name, param) = raw_name.split_once(": ").unwrap_or((raw_name, ""));
366
367 for (name, value) in plan_metrics.metrics.into_iter() {
368 if name == "elapsed_compute" {
369 elapsed_compute = value;
370 } else if name == "output_rows" {
371 output_rows = value;
372 } else {
373 other_metrics.insert(name, value);
374 }
375 }
376
377 (
378 plan_metrics.level,
379 Self {
380 name: name.to_string(),
381 param: param.to_string(),
382 output_rows,
383 elapsed_compute,
384 metrics: other_metrics,
385 children: vec![],
386 },
387 )
388 }
389}
390
391impl Display for JsonMetrics {
392 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
393 write!(f, "{}", serde_json::to_string(self).unwrap())
394 }
395}
396
397#[cfg(test)]
398mod tests {
399 use datafusion::physical_plan::empty::EmptyExec;
400
401 use super::*;
402
403 fn empty_plan(name: &str) -> Arc<dyn ExecutionPlan> {
404 Arc::new(EmptyExec::new(Arc::new(Schema::new(vec![Field::new(
405 name,
406 DataType::Utf8,
407 true,
408 )]))))
409 }
410
411 #[test]
412 fn qbs_dist_analyze_rejects_zero_children() {
413 let analyze = Arc::new(DistAnalyzeExec::new(
414 empty_plan("original"),
415 false,
416 AnalyzeFormat::TEXT,
417 ));
418
419 assert!(ExecutionPlan::with_new_children(analyze, vec![]).is_err());
420 }
421
422 #[test]
423 fn qbs_dist_analyze_rejects_multiple_children() {
424 let analyze = Arc::new(DistAnalyzeExec::new(
425 empty_plan("original"),
426 false,
427 AnalyzeFormat::TEXT,
428 ));
429
430 let result = ExecutionPlan::with_new_children(
431 analyze,
432 vec![empty_plan("first"), empty_plan("second")],
433 );
434
435 if let Ok(plan) = result {
436 let retained = plan
437 .as_any()
438 .downcast_ref::<DistAnalyzeExec>()
439 .unwrap()
440 .input()
441 .schema()
442 .field(0)
443 .name()
444 .clone();
445 panic!("expected an arity error for multiple children, but retained `{retained}`");
446 }
447 }
448
449 #[test]
450 fn qbs_dist_analyze_accepts_exactly_one_child() {
451 let analyze = Arc::new(DistAnalyzeExec::new(
452 empty_plan("original"),
453 false,
454 AnalyzeFormat::TEXT,
455 ));
456 let replacement = empty_plan("replacement");
457
458 let rebuilt = ExecutionPlan::with_new_children(analyze, vec![replacement]).unwrap();
459 let rebuilt = rebuilt.as_any().downcast_ref::<DistAnalyzeExec>().unwrap();
460
461 assert_eq!(rebuilt.input().schema().field(0).name(), "replacement");
462 }
463
464 #[test]
465 fn qbs_analyze_json_preserves_plan_name_without_parameters() {
466 let input = empty_plan("input");
467 let mut collector = MetricCollector::new(false);
468 accept(input.as_ref(), &mut collector).unwrap();
469
470 let plan_metrics = &collector.record_batch_metrics.plan_metrics;
471 assert_eq!(plan_metrics.len(), 1);
472 assert_eq!(plan_metrics[0].level, 0);
473 assert_eq!(plan_metrics[0].plan, input.name());
474
475 let analyze: Arc<dyn ExecutionPlan> = Arc::new(DistAnalyzeExec::new(
476 input.clone(),
477 false,
478 AnalyzeFormat::JSON,
479 ));
480 let metrics = analyze_plan_metrics_to_json_value(&analyze, false).unwrap();
481 let plan_name = metrics[0]["plan"]["name"].as_str().unwrap();
482 let plan_param = metrics[0]["plan"]["param"].as_str().unwrap();
483
484 assert!(!plan_name.is_empty());
485 assert_eq!(plan_name, input.name());
486 assert!(plan_param.is_empty());
487 }
488
489 #[test]
490 fn qbs_analyze_json_splits_plan_name_and_parameters() {
491 let (_, metrics) = JsonMetrics::from_plan_metrics(PlanMetrics {
492 plan: "FilterExec: predicate".to_string(),
493 plan_name: "FilterExec".to_string(),
494 level: 0,
495 metrics: vec![],
496 });
497
498 assert_eq!(metrics.name, "FilterExec");
499 assert_eq!(metrics.param, "predicate");
500 }
501}