1use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
16use std::sync::{Arc, RwLock};
17use std::time::{Duration, SystemTime, UNIX_EPOCH};
18
19use api::v1::{CreateTableExpr, TableName};
20use catalog::CatalogManagerRef;
21use common_error::ext::BoxedError;
22use common_query::logical_plan::breakup_insert_plan;
23use common_telemetry::tracing::warn;
24use common_telemetry::{debug, info};
25use common_time::Timestamp;
26use datafusion::datasource::DefaultTableSource;
27use datafusion::sql::unparser::expr_to_sql;
28use datafusion_common::tree_node::{Transformed, TreeNode};
29use datafusion_common::utils::quote_identifier;
30use datafusion_common::{DFSchemaRef, TableReference};
31use datafusion_expr::{DmlStatement, LogicalPlan, WriteOp, col, lit};
32use datatypes::schema::Schema;
33use query::QueryEngineRef;
34use query::options::FLOW_INCREMENTAL_MODE;
35use query::query_engine::DefaultSerializer;
36use session::context::QueryContextRef;
37use snafu::{OptionExt, ResultExt};
38use sql::parsers::utils::is_tql;
39use store_api::mito_engine_options::MERGE_MODE_KEY;
40use substrait::{DFLogicalSubstraitConvertor, SubstraitPlan};
41use table::table::adapter::DfTableProviderAdapter;
42use tokio::sync::oneshot::error::TryRecvError;
43use tokio::sync::{Mutex, oneshot};
44use tokio::time::Instant;
45
46use crate::batching_mode::BatchingModeOptions;
47use crate::batching_mode::checkpoint::checkpoint_mode_label;
48use crate::batching_mode::eval_schedule::{EvalSchedule, select_due_scheduled_times};
49use crate::batching_mode::frontend_client::{FrontendClient, PeerDesc};
50use crate::batching_mode::state::{
51 CheckpointMode, DirtyTimeWindows, FilterExprInfo, TaskState, to_df_literal,
52};
53use crate::batching_mode::table_creator::{QueryType, create_table_with_expr};
54use crate::batching_mode::time_window::TimeWindowExpr;
55use crate::batching_mode::utils::{
56 AddFilterRewriter, ColumnMatcherRewriter, df_plan_to_sql, gen_plan_with_matching_schema,
57 get_table_info_df_schema, sql_to_df_plan,
58};
59use crate::df_optimizer::apply_df_optimizer;
60use crate::error::{
61 DatafusionSnafu, ExternalSnafu, InvalidQuerySnafu, SubstraitEncodeLogicalPlanSnafu,
62 UnexpectedSnafu,
63};
64use crate::metrics::{
65 METRIC_FLOW_BATCHING_ENGINE_ERROR_CNT, METRIC_FLOW_BATCHING_ENGINE_QUERY_TIME,
66 METRIC_FLOW_BATCHING_ENGINE_SLOW_QUERY, METRIC_FLOW_BATCHING_ENGINE_START_QUERY_CNT,
67 METRIC_FLOW_ROWS,
68};
69use crate::{Error, FlowId};
70
71mod ckpt;
72mod inc;
73
74fn wall_clock_unix_secs() -> i64 {
76 SystemTime::now()
77 .duration_since(UNIX_EPOCH)
78 .unwrap_or_default()
79 .as_secs() as i64
80}
81
82fn initial_schedule_cursor(start_secs: i64, interval_secs: i64) -> Result<i64, Error> {
90 let cursor = i128::from(start_secs) - i128::from(interval_secs);
91 i64::try_from(cursor).map_err(|_| {
92 UnexpectedSnafu {
93 reason: format!(
94 "Cannot compute the initial eval schedule cursor one interval before start {start_secs} (interval={interval_secs}): {cursor} does not fit in i64"
95 ),
96 }
97 .build()
98 })
99}
100
101fn sleep_delta_secs(next: i64, wall_now_secs: i64) -> Result<u64, Error> {
109 let delta = i128::from(next) - i128::from(wall_now_secs);
110 u64::try_from(delta).map_err(|_| {
111 UnexpectedSnafu {
112 reason: format!(
113 "Cannot sleep until the next scheduled time {next}: the delta from wall clock {wall_now_secs} is {delta} seconds, which does not fit in u64"
114 ),
115 }
116 .build()
117 })
118}
119
120fn scheduled_time_millis(scheduled_time_secs: i64) -> Result<i64, Error> {
127 scheduled_time_secs.checked_mul(1000).ok_or_else(|| {
128 UnexpectedSnafu {
129 reason: format!(
130 "Cannot convert scheduled time {scheduled_time_secs}s to milliseconds: the product exceeds i64"
131 ),
132 }
133 .build()
134 })
135}
136
137#[derive(Clone)]
139pub struct TaskConfig {
140 pub flow_id: FlowId,
141 pub query: String,
142 pub output_schema: DFSchemaRef,
144 pub time_window_expr: Option<TimeWindowExpr>,
145 pub expire_after: Option<i64>,
147 pub sink_table_name: [String; 3],
148 pub source_table_names: HashSet<[String; 3]>,
149 pub catalog_manager: CatalogManagerRef,
150 pub query_type: QueryType,
151 pub batch_opts: Arc<BatchingModeOptions>,
152 pub exact_sequence_range_required: bool,
153 pub flow_eval_interval: Option<Duration>,
154 pub eval_schedule: Option<EvalSchedule>,
156}
157
158fn determine_query_type(query: &str, query_ctx: &QueryContextRef) -> Result<QueryType, Error> {
159 let is_tql = is_tql(query_ctx.sql_dialect(), query)
160 .map_err(BoxedError::new)
161 .context(ExternalSnafu)?;
162 Ok(if is_tql {
163 QueryType::Tql
164 } else {
165 QueryType::Sql
166 })
167}
168
169fn is_merge_mode_last_non_null(options: &HashMap<String, String>) -> bool {
170 options
171 .get(MERGE_MODE_KEY)
172 .map(|mode| mode.eq_ignore_ascii_case("last_non_null"))
173 .unwrap_or(false)
174}
175
176fn encode_insert_plan_request(
177 insert_to: TableName,
178 insert_input_plan: &LogicalPlan,
179) -> Result<api::v1::QueryRequest, Error> {
180 let message = DFLogicalSubstraitConvertor {}
181 .encode(insert_input_plan, DefaultSerializer)
182 .context(SubstraitEncodeLogicalPlanSnafu)?;
183 Ok(api::v1::QueryRequest {
184 query: Some(api::v1::query_request::Query::InsertIntoPlan(
185 api::v1::InsertIntoPlan {
186 table_name: Some(insert_to),
187 logical_plan: message.to_vec(),
188 },
189 )),
190 })
191}
192
193fn format_insert_target_columns(plan: &LogicalPlan) -> String {
194 plan.schema()
195 .fields()
196 .iter()
197 .map(|field| quote_identifier(field.name()).to_string())
198 .collect::<Vec<_>>()
199 .join(", ")
200}
201
202#[derive(Clone)]
203pub struct BatchingTask {
204 pub config: Arc<TaskConfig>,
205 pub state: Arc<RwLock<TaskState>>,
206 execution_lock: Arc<Mutex<()>>,
210}
211
212pub struct TaskArgs<'a> {
214 pub flow_id: FlowId,
215 pub query: &'a str,
216 pub plan: LogicalPlan,
217 pub time_window_expr: Option<TimeWindowExpr>,
218 pub expire_after: Option<i64>,
219 pub sink_table_name: [String; 3],
220 pub source_table_names: Vec<[String; 3]>,
221 pub query_ctx: QueryContextRef,
222 pub catalog_manager: CatalogManagerRef,
223 pub shutdown_rx: oneshot::Receiver<()>,
224 pub batch_opts: Arc<BatchingModeOptions>,
225 pub flow_eval_interval: Option<Duration>,
226 pub eval_schedule: Option<EvalSchedule>,
228}
229
230pub struct PlanInfo {
231 pub plan: LogicalPlan,
232 pub dirty_restore: DirtyRestore,
233 pub coverage: QueryCoverage,
234}
235
236#[derive(Clone)]
237pub enum QueryCoverage {
238 UnfilteredFull,
243 ScopedBaseRepair,
246 FencedRepairChunk { high: BTreeMap<u64, u64> },
250 IncrementalDelta,
252}
253
254impl QueryCoverage {
255 fn is_incremental_delta(&self) -> bool {
258 matches!(self, Self::IncrementalDelta)
259 }
260
261 fn snapshot_seqs(&self) -> HashMap<u64, u64> {
264 match self {
265 Self::FencedRepairChunk { high } => high.iter().map(|(k, v)| (*k, *v)).collect(),
266 _ => HashMap::new(),
267 }
268 }
269}
270
271pub enum DirtyRestore {
272 Scoped(FilterExprInfo),
275 Unscoped(DirtyTimeWindows),
282}
283
284struct ExecuteOnceOutcome {
285 new_query: Option<PlanInfo>,
286 result: Result<Option<(usize, Duration)>, Error>,
292}
293
294impl BatchingTask {
295 #[allow(clippy::too_many_arguments)]
296 pub fn try_new(args: TaskArgs<'_>) -> Result<Self, Error> {
297 Self::try_new_with_exact_sequence_range_required(args, false)
298 }
299
300 pub fn try_new_with_exact_sequence_range_required(
301 TaskArgs {
302 flow_id,
303 query,
304 plan,
305 time_window_expr,
306 expire_after,
307 sink_table_name,
308 source_table_names,
309 query_ctx,
310 catalog_manager,
311 shutdown_rx,
312 batch_opts,
313 flow_eval_interval,
314 eval_schedule,
315 }: TaskArgs<'_>,
316 exact_sequence_range_required: bool,
317 ) -> Result<Self, Error> {
318 let mut state = TaskState::with_dirty_time_windows(
319 query_ctx.clone(),
320 shutdown_rx,
321 DirtyTimeWindows::new(
322 batch_opts.experimental_max_filter_num_per_query,
323 batch_opts.experimental_time_window_merge_threshold,
324 ),
325 );
326 if !batch_opts.experimental_enable_incremental_read {
327 state.disable_incremental();
328 }
329
330 Ok(Self {
331 config: Arc::new(TaskConfig {
332 flow_id,
333 query: query.to_string(),
334 time_window_expr,
335 expire_after,
336 sink_table_name,
337 source_table_names: source_table_names.into_iter().collect(),
338 catalog_manager,
339 output_schema: plan.schema().clone(),
340 query_type: determine_query_type(query, &query_ctx)?,
341 exact_sequence_range_required,
342 batch_opts,
343 flow_eval_interval,
344 eval_schedule,
345 }),
346 state: Arc::new(RwLock::new(state)),
347 execution_lock: Arc::new(Mutex::new(())),
348 })
349 }
350
351 pub fn last_execution_time_millis(&self) -> Option<i64> {
352 self.state.read().unwrap().last_execution_time_millis()
353 }
354
355 pub fn start_time_millis(&self) -> Option<i64> {
356 self.state.read().unwrap().start_time_millis()
357 }
358
359 fn frontend_extensions(&self) -> HashMap<String, String> {
362 let ctx = self.state.read().unwrap();
363 let all = ctx.query_ctx.extensions();
364 let mut flow_exts = HashMap::new();
365 if let Some(v) = all.get(query::options::FLOW_SCHEDULED_TIME_MILLIS) {
368 flow_exts.insert(
369 query::options::FLOW_SCHEDULED_TIME_MILLIS.to_string(),
370 v.clone(),
371 );
372 }
373 flow_exts
374 }
375
376 pub fn mark_all_windows_as_dirty(&self) -> Result<(), Error> {
380 let now = SystemTime::now();
381 let now = Timestamp::new_second(
382 now.duration_since(UNIX_EPOCH)
383 .expect("Time went backwards")
384 .as_secs() as _,
385 );
386 let lower_bound = self
387 .config
388 .expire_after
389 .map(|e| now.sub_duration(Duration::from_secs(e as _)))
390 .transpose()
391 .map_err(BoxedError::new)
392 .context(ExternalSnafu)?
393 .unwrap_or(Timestamp::new_second(0));
394 debug!(
395 "Flow {} mark range ({:?}, {:?}) as dirty",
396 self.config.flow_id, lower_bound, now
397 );
398 self.state
399 .write()
400 .unwrap()
401 .dirty_time_windows
402 .add_window(lower_bound, Some(now));
403 Ok(())
404 }
405
406 pub async fn check_or_create_sink_table(
408 &self,
409 engine: &QueryEngineRef,
410 frontend_client: &Arc<FrontendClient>,
411 ) -> Result<Option<(usize, Duration)>, Error> {
412 if !self.is_table_exist(&self.config.sink_table_name).await? {
413 let create_table = self.gen_create_table_expr(engine.clone()).await?;
414 info!(
415 "Try creating sink table(if not exists) with expr: {:?}",
416 create_table
417 );
418 self.create_table(frontend_client, create_table).await?;
419 info!(
420 "Sink table {}(if not exists) created",
421 self.config.sink_table_name.join(".")
422 );
423 }
424
425 Ok(None)
426 }
427
428 pub async fn validate_sink_table_schema(&self, engine: &QueryEngineRef) -> Result<(), Error> {
434 let (table, _) = get_table_info_df_schema(
435 self.config.catalog_manager.clone(),
436 self.config.sink_table_name.clone(),
437 )
438 .await?;
439
440 let table_meta = &table.table_info().meta;
441 let merge_mode_last_non_null =
442 is_merge_mode_last_non_null(&table_meta.options.extra_options);
443 let primary_key_indices = table_meta.primary_key_indices.clone();
444 let query_ctx = self.state.read().unwrap().query_ctx.clone();
445
446 gen_plan_with_matching_schema(
447 &self.config.query,
448 query_ctx,
449 engine.clone(),
450 table_meta.schema.clone(),
451 &primary_key_indices,
452 merge_mode_last_non_null,
453 )
454 .await
455 .map(|_| ())
456 }
457
458 async fn is_table_exist(&self, table_name: &[String; 3]) -> Result<bool, Error> {
459 self.config
460 .catalog_manager
461 .table_exists(&table_name[0], &table_name[1], &table_name[2], None)
462 .await
463 .map_err(BoxedError::new)
464 .context(ExternalSnafu)
465 }
466
467 pub(crate) async fn execute_once_serialized(
468 &self,
469 engine: &QueryEngineRef,
470 frontend_client: &Arc<FrontendClient>,
471 max_window_cnt: Option<usize>,
472 ) -> Result<Option<(usize, Duration)>, Error> {
473 let outcome = self
474 .execute_once_serialized_with_outcome(engine, frontend_client, max_window_cnt)
475 .await;
476 outcome.result
477 }
478
479 async fn execute_once_serialized_with_outcome(
482 &self,
483 engine: &QueryEngineRef,
484 frontend_client: &Arc<FrontendClient>,
485 max_window_cnt: Option<usize>,
486 ) -> ExecuteOnceOutcome {
487 let _execution_guard = self.execution_lock.lock().await;
488 self.execute_once_unlocked(engine, frontend_client, max_window_cnt)
489 .await
490 }
491
492 async fn execute_once_unlocked(
494 &self,
495 engine: &QueryEngineRef,
496 frontend_client: &Arc<FrontendClient>,
497 max_window_cnt: Option<usize>,
498 ) -> ExecuteOnceOutcome {
499 let new_query = match self.gen_insert_plan_unlocked(engine, max_window_cnt).await {
500 Ok(new_query) => new_query,
501 Err(err) => {
502 return ExecuteOnceOutcome {
503 new_query: None,
504 result: Err(err),
505 };
506 }
507 };
508
509 if let Some(new_query) = new_query {
510 debug!("Generate new query: {}", new_query.plan);
511 let res = self
512 .execute_logical_plan_unlocked(
513 frontend_client,
514 &new_query.plan,
515 &new_query.dirty_restore,
516 &new_query.coverage,
517 )
518 .await;
519 if res.is_err() {
520 self.handle_executed_query_failure(Some(&new_query));
521 }
522 ExecuteOnceOutcome {
523 new_query: Some(new_query),
524 result: res,
525 }
526 } else {
527 debug!("Generate no query");
528 ExecuteOnceOutcome {
529 new_query: None,
530 result: Ok(None),
531 }
532 }
533 }
534
535 async fn gen_insert_plan_unlocked(
537 &self,
538 engine: &QueryEngineRef,
539 max_window_cnt: Option<usize>,
540 ) -> Result<Option<PlanInfo>, Error> {
541 let (table, df_schema) = get_table_info_df_schema(
542 self.config.catalog_manager.clone(),
543 self.config.sink_table_name.clone(),
544 )
545 .await?;
546
547 let table_meta = &table.table_info().meta;
548 let merge_mode_last_non_null =
549 is_merge_mode_last_non_null(&table_meta.options.extra_options);
550 let primary_key_indices = table_meta.primary_key_indices.clone();
551
552 let new_query = self
553 .gen_query_with_time_window(
554 engine.clone(),
555 &table.table_info().meta.schema,
556 &primary_key_indices,
557 merge_mode_last_non_null,
558 max_window_cnt,
559 )
560 .await?;
561
562 let Some(new_query) = new_query else {
563 return Ok(None);
564 };
565
566 let table_columns = df_schema
569 .columns()
570 .into_iter()
571 .map(|c| c.name)
572 .collect::<BTreeSet<_>>();
573 for column in new_query.plan.schema().columns() {
574 if !table_columns.contains(column.name()) {
575 self.restore_dirty_windows_after_failure(&new_query);
576 return InvalidQuerySnafu {
577 reason: format!(
578 "Column {} not found in sink table with columns {:?}",
579 column, table_columns
580 ),
581 }
582 .fail();
583 }
584 }
585
586 let table_provider = Arc::new(DfTableProviderAdapter::new(table));
587 let table_source = Arc::new(DefaultTableSource::new(table_provider));
588
589 let plan = LogicalPlan::Dml(DmlStatement::new(
591 datafusion_common::TableReference::Full {
592 catalog: self.config.sink_table_name[0].clone().into(),
593 schema: self.config.sink_table_name[1].clone().into(),
594 table: self.config.sink_table_name[2].clone().into(),
595 },
596 table_source,
597 WriteOp::Insert(datafusion_expr::dml::InsertOp::Append),
598 Arc::new(new_query.plan.clone()),
599 ));
600 let insert_into_info = PlanInfo {
601 plan,
602 dirty_restore: new_query.dirty_restore,
603 coverage: new_query.coverage,
604 };
605 let insert_into =
606 match insert_into_info
607 .plan
608 .clone()
609 .recompute_schema()
610 .context(DatafusionSnafu {
611 context: "Failed to recompute schema",
612 }) {
613 Ok(insert_into) => insert_into,
614 Err(err) => {
615 self.restore_dirty_windows_after_failure(&insert_into_info);
616 return Err(err);
617 }
618 };
619
620 Ok(Some(PlanInfo {
621 plan: insert_into,
622 dirty_restore: insert_into_info.dirty_restore,
623 coverage: insert_into_info.coverage,
624 }))
625 }
626
627 pub async fn create_table(
628 &self,
629 frontend_client: &Arc<FrontendClient>,
630 expr: CreateTableExpr,
631 ) -> Result<(), Error> {
632 let catalog = &self.config.sink_table_name[0];
633 let schema = &self.config.sink_table_name[1];
634 frontend_client
635 .create(expr.clone(), catalog, schema)
636 .await?;
637 Ok(())
638 }
639
640 async fn execute_logical_plan_unlocked(
642 &self,
643 frontend_client: &Arc<FrontendClient>,
644 plan: &LogicalPlan,
645 dirty_restore: &DirtyRestore,
646 coverage: &QueryCoverage,
647 ) -> Result<Option<(usize, Duration)>, Error> {
648 let instant = Instant::now();
649 let flow_id = self.config.flow_id;
650
651 debug!(
652 "Executing flow {flow_id}(expire_after={:?} secs) with query {}",
653 self.config.expire_after, &plan
654 );
655
656 let catalog = &self.config.sink_table_name[0];
657 let schema = &self.config.sink_table_name[1];
658
659 let plan = plan
661 .clone()
662 .transform_down_with_subqueries(|p| {
663 if let LogicalPlan::TableScan(mut table_scan) = p {
664 let resolved = table_scan.table_name.resolve(catalog, schema);
665 table_scan.table_name = resolved.into();
666 Ok(Transformed::yes(LogicalPlan::TableScan(table_scan)))
667 } else {
668 Ok(Transformed::no(p))
669 }
670 })
671 .with_context(|_| DatafusionSnafu {
672 context: format!("Failed to fix table ref in logical plan, plan={:?}", plan),
673 })?
674 .data;
675
676 let incremental_plan = if coverage.is_incremental_delta() {
679 self.prepare_plan_for_incremental(&plan).await?
680 } else {
681 None
682 };
683 let incremental_safe = incremental_plan.is_some();
684 if coverage.is_incremental_delta() && !incremental_safe {
685 debug!(
686 "Flow {flow_id} skipped unsafe incremental delta fallback; \
687 restored dirty signal instead of executing an unfiltered full snapshot"
688 );
689 self.restore_dirty_windows(dirty_restore);
690 return Ok(None);
691 }
692 let plan = incremental_plan.unwrap_or_else(|| plan.clone());
693
694 let extensions = self
695 .build_flow_query_extensions(incremental_safe, coverage.is_incremental_delta())
696 .await?;
697 let frontend_extensions = self.frontend_extensions();
698 let extension_refs = extensions
699 .iter()
700 .map(|(key, value)| (*key, value.as_str()))
701 .chain(
702 frontend_extensions
703 .iter()
704 .map(|(key, value)| (key.as_str(), value.as_str())),
705 )
706 .collect::<Vec<_>>();
707 let query_mode = if extensions
708 .iter()
709 .any(|(key, _)| *key == FLOW_INCREMENTAL_MODE)
710 {
711 CheckpointMode::Incremental
712 } else {
713 CheckpointMode::FullSnapshot
714 };
715 Self::record_query_mode(flow_id, query_mode);
716 debug!(
717 "Flow {flow_id} executing batching query with checkpoint_mode={}, extension_count={}",
718 checkpoint_mode_label(query_mode),
719 extensions.len()
720 );
721
722 let mut peer_desc = None;
723 let res = {
724 let _timer = METRIC_FLOW_BATCHING_ENGINE_QUERY_TIME
725 .with_label_values(&[flow_id.to_string().as_str()])
726 .start_timer();
727
728 let req = if let Some((insert_to, insert_input_plan)) =
729 breakup_insert_plan(&plan, catalog, schema)
730 {
731 if query_mode == CheckpointMode::FullSnapshot
732 && matches!(self.config.query_type, QueryType::Sql)
733 && self.config.flow_eval_interval.is_some()
734 && self.config.time_window_expr.is_none()
735 {
736 match df_plan_to_sql(&insert_input_plan) {
746 Ok(select_sql) => {
747 let target_columns = format_insert_target_columns(&insert_input_plan);
748 let sql = format!(
749 "INSERT INTO {} ({}) {}",
750 TableReference::full(
751 insert_to.catalog_name.as_str(),
752 insert_to.schema_name.as_str(),
753 insert_to.table_name.as_str(),
754 )
755 .to_quoted_string(),
756 target_columns,
757 select_sql
758 );
759 api::v1::QueryRequest {
760 query: Some(api::v1::query_request::Query::Sql(sql)),
761 }
762 }
763 Err(err) => {
764 debug!(
765 "Failed to unparse full-snapshot SQL flow {} plan; \
766 falling back to InsertIntoPlan: {:?}",
767 flow_id, err
768 );
769 encode_insert_plan_request(insert_to, &insert_input_plan)?
770 }
771 }
772 } else {
773 encode_insert_plan_request(insert_to, &insert_input_plan)?
774 }
775 } else {
776 let message = DFLogicalSubstraitConvertor {}
777 .encode(&plan, DefaultSerializer)
778 .context(SubstraitEncodeLogicalPlanSnafu)?;
779
780 api::v1::QueryRequest {
781 query: Some(api::v1::query_request::Query::LogicalPlan(message.to_vec())),
782 }
783 };
784
785 let snapshot_seqs = coverage.snapshot_seqs();
786 {
787 let mut state = self.state.write().unwrap();
788 state.record_start_time_if_first();
789 }
790 frontend_client
791 .query_with_terminal_metrics(
792 catalog,
793 schema,
794 req,
795 &extension_refs,
796 &snapshot_seqs,
797 &mut peer_desc,
798 )
799 .await
800 };
801
802 let elapsed = instant.elapsed();
803 let peer_label = peer_desc
804 .as_ref()
805 .map(ToString::to_string)
806 .unwrap_or_else(|| PeerDesc::default().to_string());
807 if let Err(err) = &res {
808 warn!(
809 "Failed to execute Flow {flow_id} on frontend {peer_label}, result: {err:?}, elapsed: {:?} with query: {}",
810 elapsed, &plan
811 );
812 let decision = {
813 let mut state = self.state.write().unwrap();
814 let reason = Self::query_failure_reason(err, coverage);
815 Self::apply_query_failure_to_state(&mut state, elapsed, coverage, reason)
816 };
817 if let Some(decision) = decision {
818 Self::record_checkpoint_decision(flow_id, decision);
819 }
820 }
821
822 if elapsed >= self.config.batch_opts.slow_query_threshold {
824 warn!(
825 "Flow {flow_id} on frontend {peer_label} executed for {:?} before complete, query: {}",
826 elapsed, &plan
827 );
828 let flow_id = flow_id.to_string();
829 METRIC_FLOW_BATCHING_ENGINE_SLOW_QUERY
830 .with_label_values(&[flow_id.as_str(), peer_label.as_str()])
831 .observe(elapsed.as_secs_f64());
832 }
833
834 let res = res?;
835 let (affected_rows, _) = res.output.extract_rows_and_cost();
836 debug!(
837 "Flow {flow_id} executed, affected_rows: {affected_rows:?}, elapsed: {:?}, watermark: {:?}",
838 elapsed,
839 res.region_watermark_map()
840 );
841 METRIC_FLOW_ROWS
842 .with_label_values(&[format!("{}-out-batching", flow_id).as_str()])
843 .inc_by(affected_rows as _);
844 let decision = {
845 let mut state = self.state.write().unwrap();
846 Self::apply_query_result_to_state(&mut state, &res, elapsed, coverage)
847 };
848 Self::record_checkpoint_decision(flow_id, decision);
849
850 Ok(Some((affected_rows, elapsed)))
851 }
852
853 fn restore_dirty_windows(&self, dirty_restore: &DirtyRestore) {
857 match dirty_restore {
858 DirtyRestore::Scoped(filter) => self.restore_scoped_dirty_windows(filter),
859 DirtyRestore::Unscoped(dirty_windows) => self
860 .state
861 .write()
862 .unwrap()
863 .dirty_time_windows
864 .add_dirty_windows(dirty_windows),
865 }
866 }
867
868 fn restore_dirty_windows_after_failure(&self, query: &PlanInfo) {
871 self.restore_dirty_windows(&query.dirty_restore);
872 }
873
874 fn restore_scoped_dirty_windows(&self, filter: &FilterExprInfo) {
877 self.state.write().unwrap().restore_scoped_windows(filter);
878 }
879
880 fn restore_scoped_dirty_windows_on_err<T>(
883 &self,
884 filter: &FilterExprInfo,
885 result: Result<T, Error>,
886 ) -> Result<T, Error> {
887 result.inspect_err(|_| {
888 self.restore_scoped_dirty_windows(filter);
889 })
890 }
891
892 fn restore_unscoped_dirty_windows(&self, dirty_windows: &DirtyTimeWindows) {
895 self.state
896 .write()
897 .unwrap()
898 .dirty_time_windows
899 .add_dirty_windows(dirty_windows);
900 }
901
902 fn restore_unscoped_dirty_windows_on_err<T>(
905 &self,
906 dirty_windows: &DirtyTimeWindows,
907 result: Result<T, Error>,
908 ) -> Result<T, Error> {
909 result.inspect_err(|_| {
910 self.restore_unscoped_dirty_windows(dirty_windows);
911 })
912 }
913
914 fn drain_dirty_windows_signal(&self) -> (bool, DirtyTimeWindows) {
917 let mut state = self.state.write().unwrap();
918 let dirty_windows_to_restore = state.dirty_time_windows.clone();
919 let is_dirty = !dirty_windows_to_restore.is_empty();
920 state.dirty_time_windows.clean();
921 (is_dirty, dirty_windows_to_restore)
922 }
923
924 #[allow(clippy::too_many_arguments)]
925 async fn gen_unfiltered_plan_info(
928 &self,
929 engine: QueryEngineRef,
930 query_ctx: QueryContextRef,
931 sink_table_schema: Arc<Schema>,
932 primary_key_indices: &[usize],
933 allow_partial: bool,
934 dirty_windows_to_restore: DirtyTimeWindows,
935 retention_filter: Option<(&str, Timestamp, &'static str)>,
936 coverage: QueryCoverage,
937 ) -> Result<PlanInfo, Error> {
938 let mut plan = self.restore_unscoped_dirty_windows_on_err(
939 &dirty_windows_to_restore,
940 gen_plan_with_matching_schema(
941 &self.config.query,
942 query_ctx,
943 engine,
944 sink_table_schema,
945 primary_key_indices,
946 allow_partial,
947 )
948 .await,
949 )?;
950
951 if let Some((col_name, lower_bound, context)) = retention_filter {
952 let lower = self.restore_unscoped_dirty_windows_on_err(
953 &dirty_windows_to_restore,
954 to_df_literal(lower_bound),
955 )?;
956 let retention_filter = col(col_name).gt_eq(lit(lower));
957 let mut add_filter = AddFilterRewriter::new(retention_filter);
958 plan = self.restore_unscoped_dirty_windows_on_err(
959 &dirty_windows_to_restore,
960 plan.clone()
961 .rewrite(&mut add_filter)
962 .with_context(|_| DatafusionSnafu {
963 context: format!(
964 "Failed to apply {context} expire_after filter to plan:\n {}\n",
965 plan
966 ),
967 })
968 .map(|rewrite| rewrite.data),
969 )?;
970 }
971
972 Ok(PlanInfo {
973 plan,
974 dirty_restore: DirtyRestore::Unscoped(dirty_windows_to_restore),
975 coverage,
976 })
977 }
978
979 #[allow(clippy::too_many_arguments)]
980 async fn gen_unfiltered_plan_info_if_dirty(
983 &self,
984 engine: QueryEngineRef,
985 query_ctx: QueryContextRef,
986 sink_table_schema: Arc<Schema>,
987 primary_key_indices: &[usize],
988 allow_partial: bool,
989 retention_filter: Option<(&str, Timestamp, &'static str)>,
990 coverage: QueryCoverage,
991 ) -> Result<Option<PlanInfo>, Error> {
992 let (is_dirty, dirty_windows_to_restore) = self.drain_dirty_windows_signal();
993 if !is_dirty {
994 debug!("Flow id={:?}, no new data, not update", self.config.flow_id);
995 return Ok(None);
996 }
997
998 self.gen_unfiltered_plan_info(
999 engine,
1000 query_ctx,
1001 sink_table_schema,
1002 primary_key_indices,
1003 allow_partial,
1004 dirty_windows_to_restore,
1005 retention_filter,
1006 coverage,
1007 )
1008 .await
1009 .map(Some)
1010 }
1011
1012 fn handle_executed_query_failure(&self, query: Option<&PlanInfo>) {
1013 if let Some(query) = query {
1014 self.restore_dirty_windows_after_failure(query);
1015 }
1016 }
1017
1018 pub async fn start_executing_loop(
1026 &self,
1027 engine: QueryEngineRef,
1028 frontend_client: Arc<FrontendClient>,
1029 ) {
1030 if self.config.flow_eval_interval.is_some() {
1031 self.start_scheduled_loop(engine, frontend_client).await;
1032 } else {
1033 self.start_adaptive_loop(engine, frontend_client).await;
1034 }
1035 }
1036
1037 async fn start_scheduled_loop(
1047 &self,
1048 engine: QueryEngineRef,
1049 frontend_client: Arc<FrontendClient>,
1050 ) {
1051 let flow_id_str = self.config.flow_id.to_string();
1052
1053 let schedule = match &self.config.eval_schedule {
1054 Some(s) => s.clone(),
1055 None => {
1056 let eval_interval_secs = self
1057 .config
1058 .flow_eval_interval
1059 .map(|d| d.as_secs() as i64)
1060 .expect("checked by caller");
1061
1062 match EvalSchedule::from_config(Some(eval_interval_secs), None) {
1065 Ok(Some(s)) => s,
1066 Ok(None) => {
1067 warn!(
1068 "Flow {}: EVAL INTERVAL set but no schedule parsed; exiting loop",
1069 flow_id_str
1070 );
1071 return;
1072 }
1073 Err(e) => {
1074 warn!(
1075 "Flow {}: Failed to parse eval schedule: {}; exiting loop",
1076 flow_id_str, e
1077 );
1078 return;
1079 }
1080 }
1081 }
1082 };
1083
1084 let mut cursor_secs =
1089 match initial_schedule_cursor(schedule.start_secs, schedule.interval_secs) {
1090 Ok(cursor) => cursor,
1091 Err(e) => {
1092 warn!(
1093 "Flow {}: invalid eval schedule, exiting loop: {}",
1094 flow_id_str, e
1095 );
1096 return;
1097 }
1098 };
1099
1100 info!(
1101 "Flow {}: entering scheduled loop, interval={}s, start={}, anchor={}, policy={:?}, max_runs={}, max_lag={}s",
1102 flow_id_str,
1103 schedule.interval_secs,
1104 schedule.start_secs,
1105 schedule.anchor_secs,
1106 schedule.missed_tick_policy,
1107 schedule.max_runs,
1108 schedule.max_lag_secs,
1109 );
1110
1111 loop {
1112 if self.is_shutdown_signaled() {
1113 break;
1114 }
1115
1116 let wall_now_secs = wall_clock_unix_secs();
1117
1118 let due = match select_due_scheduled_times(&schedule, cursor_secs, wall_now_secs) {
1119 Ok(d) => d,
1120 Err(e) => {
1121 warn!(
1122 "Flow {}: invalid eval schedule, exiting loop: {}",
1123 flow_id_str, e
1124 );
1125 return;
1126 }
1127 };
1128
1129 if due.scheduled_times_secs.is_empty() {
1130 if due.skipped > 0 {
1131 warn!(
1132 "Flow {}: all {} due scheduled times skipped by max-lag, advancing cursor to wall-clock ({wall_now_secs}) to avoid re-skipping",
1133 flow_id_str, due.skipped
1134 );
1135 cursor_secs = wall_now_secs;
1136 continue;
1137 }
1138
1139 let next = match schedule.next_scheduled_time_after(cursor_secs) {
1141 Ok(next) => next,
1142 Err(e) => {
1143 warn!(
1144 "Flow {}: cannot advance eval schedule past cursor {cursor_secs}: {e}; exiting loop",
1145 flow_id_str
1146 );
1147 return;
1148 }
1149 };
1150 if next <= wall_now_secs {
1151 cursor_secs = wall_now_secs;
1154 continue;
1155 }
1156 let wait_secs = match sleep_delta_secs(next, wall_now_secs) {
1157 Ok(wait_secs) => wait_secs,
1158 Err(e) => {
1159 warn!(
1160 "Flow {}: cannot sleep until next scheduled time {}: {e}; exiting loop",
1161 flow_id_str, next
1162 );
1163 return;
1164 }
1165 };
1166 let wait_dur = Duration::from_secs(wait_secs);
1167 debug!(
1168 "Flow {}: no due scheduled times, sleeping for {}s until next scheduled time at {}",
1169 flow_id_str, wait_secs, next
1170 );
1171 tokio::time::sleep(wait_dur).await;
1172 continue;
1173 }
1174
1175 if due.skipped > 0 {
1176 info!(
1177 "Flow {}: {} due scheduled times, {} skipped (catch-up)",
1178 flow_id_str,
1179 due.scheduled_times_secs.len(),
1180 due.skipped
1181 );
1182 }
1183
1184 for scheduled_time_secs in &due.scheduled_times_secs {
1186 if self.is_shutdown_signaled() {
1187 break;
1188 }
1189
1190 METRIC_FLOW_BATCHING_ENGINE_START_QUERY_CNT
1191 .with_label_values(&[&flow_id_str])
1192 .inc();
1193
1194 let outcome = self
1195 .execute_once_serialized_at_scheduled_time(
1196 &engine,
1197 &frontend_client,
1198 *scheduled_time_secs,
1199 )
1200 .await;
1201
1202 cursor_secs = *scheduled_time_secs;
1204
1205 match outcome.result {
1206 Ok(Some((rows, elapsed))) => {
1207 debug!(
1208 "Flow {}: scheduled time {} completed, rows={}, elapsed={:?}",
1209 flow_id_str, scheduled_time_secs, rows, elapsed
1210 );
1211 }
1212 Ok(None) => {
1213 debug!(
1214 "Flow {}: scheduled time {} produced no query (no dirty signal or no-op)",
1215 flow_id_str, scheduled_time_secs
1216 );
1217 }
1218 Err(err) => {
1219 warn!(
1220 "Flow {}: scheduled time {} failed: {:?}",
1221 flow_id_str, scheduled_time_secs, err
1222 );
1223 METRIC_FLOW_BATCHING_ENGINE_ERROR_CNT
1224 .with_label_values(&[&flow_id_str])
1225 .inc();
1226 }
1230 }
1231 }
1232 }
1233 }
1234
1235 async fn start_adaptive_loop(
1237 &self,
1238 engine: QueryEngineRef,
1239 frontend_client: Arc<FrontendClient>,
1240 ) {
1241 let flow_id_str = self.config.flow_id.to_string();
1242 let mut max_window_cnt = None;
1243 loop {
1244 if self.is_shutdown_signaled() {
1245 break;
1246 }
1247 METRIC_FLOW_BATCHING_ENGINE_START_QUERY_CNT
1248 .with_label_values(&[&flow_id_str])
1249 .inc();
1250
1251 let min_refresh = self.config.batch_opts.experimental_min_refresh_duration;
1252
1253 let outcome = self
1254 .execute_once_serialized_with_outcome(&engine, &frontend_client, max_window_cnt)
1255 .await;
1256
1257 match outcome.result {
1258 Ok(Some(_)) => {
1259 max_window_cnt = max_window_cnt.map(|cnt| {
1260 (cnt + 1).min(self.config.batch_opts.experimental_max_filter_num_per_query)
1261 });
1262
1263 let sleep_until = {
1264 let state = self.state.write().unwrap();
1265
1266 let time_window_size = self
1267 .config
1268 .time_window_expr
1269 .as_ref()
1270 .and_then(|t| *t.time_window_size());
1271
1272 let prefer_short_incremental_cadence = state.checkpoint_mode()
1273 == CheckpointMode::Incremental
1274 && !state.is_incremental_disabled();
1275
1276 state.get_next_start_query_time(
1277 self.config.flow_id,
1278 &time_window_size,
1279 min_refresh,
1280 Some(self.config.batch_opts.query_timeout),
1281 self.config.batch_opts.experimental_max_filter_num_per_query,
1282 prefer_short_incremental_cadence,
1283 )
1284 };
1285
1286 tokio::time::sleep_until(sleep_until).await;
1287 }
1288 Ok(None) => {
1289 debug!(
1290 "Flow id = {:?} found no new data, sleep for {:?} then continue",
1291 self.config.flow_id, min_refresh
1292 );
1293 tokio::time::sleep(min_refresh).await;
1294 continue;
1295 }
1296 Err(err) => {
1297 METRIC_FLOW_BATCHING_ENGINE_ERROR_CNT
1298 .with_label_values(&[&flow_id_str])
1299 .inc();
1300 match outcome.new_query {
1301 Some(query) => {
1302 common_telemetry::error!(err; "Failed to execute query for flow={} with query: {}", self.config.flow_id, query.plan);
1303 max_window_cnt = Some(1);
1304 }
1305 None => {
1306 common_telemetry::error!(err; "Failed to generate query for flow={}", self.config.flow_id)
1307 }
1308 }
1309 tokio::time::sleep(min_refresh).await;
1310 }
1311 }
1312 }
1313 }
1314
1315 fn is_shutdown_signaled(&self) -> bool {
1317 let mut state = self.state.write().unwrap();
1318 match state.shutdown_rx.try_recv() {
1319 Ok(()) | Err(TryRecvError::Closed) => true,
1320 Err(TryRecvError::Empty) => false,
1321 }
1322 }
1323
1324 async fn execute_once_serialized_at_scheduled_time(
1331 &self,
1332 engine: &QueryEngineRef,
1333 frontend_client: &Arc<FrontendClient>,
1334 scheduled_time_secs: i64,
1335 ) -> ExecuteOnceOutcome {
1336 let _execution_guard = self.execution_lock.lock().await;
1337
1338 struct QueryContextRestoreGuard {
1339 state: Arc<RwLock<TaskState>>,
1340 old_ctx: Option<QueryContextRef>,
1341 }
1342
1343 impl Drop for QueryContextRestoreGuard {
1344 fn drop(&mut self) {
1345 let Some(old_ctx) = self.old_ctx.take() else {
1346 return;
1347 };
1348 if let Ok(mut state) = self.state.write() {
1349 state.query_ctx = old_ctx;
1350 }
1351 }
1352 }
1353
1354 let scheduled_time_millis = match scheduled_time_millis(scheduled_time_secs) {
1358 Ok(millis) => millis,
1359 Err(e) => {
1360 return ExecuteOnceOutcome {
1361 new_query: None,
1362 result: Err(e),
1363 };
1364 }
1365 };
1366
1367 let old_ctx = {
1370 let mut state = self.state.write().unwrap();
1371 let old = state.query_ctx.clone();
1372 let mut new_ctx = (*old).clone();
1373 new_ctx.set_extension(
1374 query::options::FLOW_SCHEDULED_TIME_MILLIS,
1375 scheduled_time_millis.to_string(),
1376 );
1377 state.query_ctx = Arc::new(new_ctx);
1378 old
1379 };
1380 let restore_guard = QueryContextRestoreGuard {
1381 state: self.state.clone(),
1382 old_ctx: Some(old_ctx),
1383 };
1384
1385 let outcome = self
1386 .execute_once_unlocked(engine, frontend_client, None)
1387 .await;
1388
1389 drop(restore_guard);
1393
1394 outcome
1395 }
1396
1397 async fn gen_create_table_expr(
1404 &self,
1405 engine: QueryEngineRef,
1406 ) -> Result<CreateTableExpr, Error> {
1407 let query_ctx = self.state.read().unwrap().query_ctx.clone();
1408 let plan =
1409 sql_to_df_plan(query_ctx.clone(), engine.clone(), &self.config.query, true).await?;
1410 create_table_with_expr(&plan, &self.config.sink_table_name, &self.config.query_type)
1411 }
1412
1413 fn should_use_unfiltered_incremental_delta(&self) -> bool {
1416 let state = self.state.read().unwrap();
1417 state.checkpoint_mode() == CheckpointMode::Incremental
1418 && !state.is_incremental_disabled()
1419 && matches!(self.config.query_type, QueryType::Sql)
1420 }
1421
1422 async fn gen_query_with_time_window(
1425 &self,
1426 engine: QueryEngineRef,
1427 sink_table_schema: &Arc<Schema>,
1428 primary_key_indices: &[usize],
1429 allow_partial: bool,
1430 max_window_cnt: Option<usize>,
1431 ) -> Result<Option<PlanInfo>, Error> {
1432 let query_ctx = self.state.read().unwrap().query_ctx.clone();
1433 let start = SystemTime::now();
1434 let since_the_epoch = start
1435 .duration_since(UNIX_EPOCH)
1436 .expect("Time went backwards");
1437 let low_bound = self
1438 .config
1439 .expire_after
1440 .map(|e| since_the_epoch.as_secs() - e as u64)
1441 .unwrap_or(u64::MIN);
1442
1443 let low_bound = Timestamp::new_second(low_bound as i64);
1444
1445 let expire_time_window_bound = self
1446 .config
1447 .time_window_expr
1448 .as_ref()
1449 .map(|expr| expr.eval(low_bound))
1450 .transpose()?;
1451
1452 let (expire_lower_bound, expire_upper_bound) = match (
1453 expire_time_window_bound,
1454 &self.config.query_type,
1455 ) {
1456 (Some((Some(l), Some(u))), QueryType::Sql) => (l, u),
1457 (None, QueryType::Sql) if self.config.flow_eval_interval.is_none() => {
1458 return UnexpectedSnafu {
1459 reason: format!(
1460 "Flow id={} reached execution without a time-window expression or EVAL INTERVAL; create-flow validation should have rejected it",
1461 self.config.flow_id
1462 ),
1463 }
1464 .fail();
1465 }
1466 _ => {
1467 let (_, dirty_windows_to_restore) = self.drain_dirty_windows_signal();
1473
1474 let plan_info = self
1475 .gen_unfiltered_plan_info(
1476 engine,
1477 query_ctx,
1478 sink_table_schema.clone(),
1479 primary_key_indices,
1480 allow_partial,
1481 dirty_windows_to_restore,
1482 None,
1483 QueryCoverage::UnfilteredFull,
1484 )
1485 .await?;
1486
1487 return Ok(Some(plan_info));
1488 }
1489 };
1490
1491 debug!(
1492 "Flow id = {:?}, found time window: precise_lower_bound={:?}, precise_upper_bound={:?} with dirty time windows: {:?}",
1493 self.config.flow_id,
1494 expire_lower_bound,
1495 expire_upper_bound,
1496 self.state.read().unwrap().dirty_time_windows
1497 );
1498 let window_size = expire_upper_bound
1499 .sub(&expire_lower_bound)
1500 .with_context(|| UnexpectedSnafu {
1501 reason: format!(
1502 "Can't get window size from {expire_upper_bound:?} - {expire_lower_bound:?}"
1503 ),
1504 })?;
1505 let col_name = self
1506 .config
1507 .time_window_expr
1508 .as_ref()
1509 .map(|expr| expr.column_name.clone())
1510 .with_context(|| UnexpectedSnafu {
1511 reason: format!(
1512 "Flow id={:?}, Failed to get column name from time window expr",
1513 self.config.flow_id
1514 ),
1515 })?;
1516
1517 if self.should_use_unfiltered_incremental_delta() {
1518 let retention_filter = self
1528 .config
1529 .expire_after
1530 .map(|_| (col_name.as_str(), expire_lower_bound, "incremental"));
1531 return self
1532 .gen_unfiltered_plan_info_if_dirty(
1533 engine,
1534 query_ctx,
1535 sink_table_schema.clone(),
1536 primary_key_indices,
1537 allow_partial,
1538 retention_filter,
1539 QueryCoverage::IncrementalDelta,
1540 )
1541 .await;
1542 }
1543
1544 let (expr, coverage) = {
1545 let mut state = self.state.write().unwrap();
1546 let window_cnt = max_window_cnt
1547 .unwrap_or(self.config.batch_opts.experimental_max_filter_num_per_query);
1548 let expr = state.gen_scoped_filter_exprs(
1549 &col_name,
1550 Some(expire_lower_bound),
1551 window_size,
1552 window_cnt,
1553 self.config.flow_id,
1554 Some(self),
1555 )?;
1556 let repair_high = state
1557 .pending_fenced_repair()
1558 .map(|repair| repair.high().clone());
1559 let coverage = if let Some(high) = repair_high {
1560 QueryCoverage::FencedRepairChunk { high }
1561 } else {
1562 QueryCoverage::ScopedBaseRepair
1563 };
1564 (expr, coverage)
1565 };
1566
1567 let Some(expr) = expr else {
1568 debug!("Flow id={:?}, no new data, not update", self.config.flow_id);
1570 return Ok(None);
1571 };
1572
1573 let filter_sql = expr_to_sql(&expr.expr)
1574 .map(|sql| sql.to_string())
1575 .unwrap_or_else(|err| format!("<failed to format filter expr: {err}>"));
1576
1577 debug!(
1578 "Flow id={:?}, Generated filter expr: {:?}",
1579 self.config.flow_id, filter_sql
1580 );
1581
1582 let mut add_filter = AddFilterRewriter::new(expr.expr.clone());
1583 let mut add_auto_column = ColumnMatcherRewriter::new(
1584 sink_table_schema.clone(),
1585 primary_key_indices.to_vec(),
1586 allow_partial,
1587 );
1588
1589 let plan = self.restore_scoped_dirty_windows_on_err(
1590 &expr,
1591 sql_to_df_plan(query_ctx.clone(), engine.clone(), &self.config.query, false).await,
1592 )?;
1593 let rewrite = self.restore_scoped_dirty_windows_on_err(
1594 &expr,
1595 plan.clone()
1596 .rewrite(&mut add_filter)
1597 .and_then(|p| p.data.rewrite(&mut add_auto_column))
1598 .with_context(|_| DatafusionSnafu {
1599 context: format!("Failed to rewrite plan:\n {}\n", plan),
1600 })
1601 .map(|rewrite| rewrite.data),
1602 )?;
1603 let new_plan = self.restore_scoped_dirty_windows_on_err(
1605 &expr,
1606 apply_df_optimizer(rewrite, &query_ctx).await,
1607 )?;
1608
1609 let info = PlanInfo {
1610 plan: new_plan.clone(),
1611 dirty_restore: DirtyRestore::Scoped(expr),
1612 coverage,
1613 };
1614
1615 Ok(Some(info))
1616 }
1617}
1618
1619#[cfg(test)]
1620mod test;