1use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
16use std::sync::{Arc, RwLock};
17use std::time::{Duration, SystemTime, UNIX_EPOCH};
18
19use api::v1::{CreateTableExpr, TableName};
20use catalog::CatalogManagerRef;
21use client::OutputWithMetrics;
22use common_error::ext::BoxedError;
23use common_query::logical_plan::breakup_insert_plan;
24use common_telemetry::tracing::warn;
25use common_telemetry::{debug, info};
26use common_time::Timestamp;
27use datafusion::datasource::DefaultTableSource;
28use datafusion::sql::unparser::expr_to_sql;
29use datafusion_common::tree_node::{Transformed, TreeNode};
30use datafusion_common::utils::quote_identifier;
31use datafusion_common::{DFSchemaRef, ScalarValue, TableReference};
32use datafusion_expr::{DmlStatement, LogicalPlan, WriteOp, col, lit};
33use datatypes::schema::Schema;
34use query::QueryEngineRef;
35use query::options::FLOW_INCREMENTAL_MODE;
36use query::query_engine::DefaultSerializer;
37use session::context::QueryContextRef;
38use snafu::{OptionExt, ResultExt};
39use sql::parsers::utils::is_tql;
40use store_api::mito_engine_options::MERGE_MODE_KEY;
41use substrait::{DFLogicalSubstraitConvertor, SubstraitPlan};
42use table::TableRef;
43use table::table::adapter::DfTableProviderAdapter;
44use tokio::sync::oneshot::error::TryRecvError;
45use tokio::sync::{Mutex, OwnedMutexGuard, oneshot};
46use tokio::time::Instant;
47
48use crate::batching_mode::BatchingModeOptions;
49use crate::batching_mode::checkpoint::checkpoint_mode_label;
50use crate::batching_mode::eval_schedule::{EvalSchedule, select_due_scheduled_times};
51use crate::batching_mode::frontend_client::{FrontendClient, PeerDesc};
52use crate::batching_mode::state::{
53 CheckpointMode, DirtyTimeWindows, FilterExprInfo, TaskState, to_df_literal,
54};
55use crate::batching_mode::table_creator::{QueryType, create_table_with_expr};
56use crate::batching_mode::time_window::TimeWindowExpr;
57use crate::batching_mode::utils::{
58 AddFilterRewriter, ColumnMatcherRewriter, df_plan_to_sql,
59 gen_plan_with_matching_schema_and_values, get_table_info_df_schema, sql_to_df_plan,
60};
61use crate::df_optimizer::apply_df_optimizer;
62use crate::error::{
63 DatafusionSnafu, ExternalSnafu, InvalidQuerySnafu, SubstraitEncodeLogicalPlanSnafu,
64 UnexpectedSnafu,
65};
66use crate::metrics::{
67 METRIC_FLOW_BATCHING_ENGINE_ERROR_CNT, METRIC_FLOW_BATCHING_ENGINE_QUERY_TIME,
68 METRIC_FLOW_BATCHING_ENGINE_SLOW_QUERY, METRIC_FLOW_BATCHING_ENGINE_START_QUERY_CNT,
69 METRIC_FLOW_ROWS,
70};
71use crate::{Error, FlowId};
72
73mod ckpt;
74mod inc;
75
76fn wall_clock_unix_secs() -> i64 {
78 SystemTime::now()
79 .duration_since(UNIX_EPOCH)
80 .unwrap_or_default()
81 .as_secs() as i64
82}
83
84fn initial_schedule_cursor(start_secs: i64, interval_secs: i64) -> Result<i64, Error> {
92 let cursor = i128::from(start_secs) - i128::from(interval_secs);
93 i64::try_from(cursor).map_err(|_| {
94 UnexpectedSnafu {
95 reason: format!(
96 "Cannot compute the initial eval schedule cursor one interval before start {start_secs} (interval={interval_secs}): {cursor} does not fit in i64"
97 ),
98 }
99 .build()
100 })
101}
102
103fn sleep_delta_secs(next: i64, wall_now_secs: i64) -> Result<u64, Error> {
111 let delta = i128::from(next) - i128::from(wall_now_secs);
112 u64::try_from(delta).map_err(|_| {
113 UnexpectedSnafu {
114 reason: format!(
115 "Cannot sleep until the next scheduled time {next}: the delta from wall clock {wall_now_secs} is {delta} seconds, which does not fit in u64"
116 ),
117 }
118 .build()
119 })
120}
121
122fn scheduled_time_millis(scheduled_time_secs: i64) -> Result<i64, Error> {
129 scheduled_time_secs.checked_mul(1000).ok_or_else(|| {
130 UnexpectedSnafu {
131 reason: format!(
132 "Cannot convert scheduled time {scheduled_time_secs}s to milliseconds: the product exceeds i64"
133 ),
134 }
135 .build()
136 })
137}
138
139#[derive(Clone)]
141pub struct TaskConfig {
142 pub flow_id: FlowId,
143 pub query: String,
144 pub output_schema: DFSchemaRef,
146 pub time_window_expr: Option<TimeWindowExpr>,
147 pub expire_after: Option<i64>,
149 pub sink_table_name: [String; 3],
150 pub source_table_names: HashSet<[String; 3]>,
151 pub catalog_manager: CatalogManagerRef,
152 pub query_type: QueryType,
153 pub batch_opts: Arc<BatchingModeOptions>,
154 pub exact_sequence_range_required: bool,
155 pub flow_eval_interval: Option<Duration>,
156 pub eval_schedule: Option<EvalSchedule>,
158}
159
160fn determine_query_type(query: &str, query_ctx: &QueryContextRef) -> Result<QueryType, Error> {
161 let is_tql = is_tql(query_ctx.sql_dialect(), query)
162 .map_err(BoxedError::new)
163 .context(ExternalSnafu)?;
164 Ok(if is_tql {
165 QueryType::Tql
166 } else {
167 QueryType::Sql
168 })
169}
170
171fn is_merge_mode_last_non_null(options: &HashMap<String, String>) -> bool {
172 options
173 .get(MERGE_MODE_KEY)
174 .map(|mode| mode.eq_ignore_ascii_case("last_non_null"))
175 .unwrap_or(false)
176}
177
178fn encode_insert_plan_request(
179 insert_to: TableName,
180 insert_input_plan: &LogicalPlan,
181) -> Result<api::v1::QueryRequest, Error> {
182 let message = DFLogicalSubstraitConvertor {}
183 .encode(insert_input_plan, DefaultSerializer)
184 .context(SubstraitEncodeLogicalPlanSnafu)?;
185 Ok(api::v1::QueryRequest {
186 query: Some(api::v1::query_request::Query::InsertIntoPlan(
187 api::v1::InsertIntoPlan {
188 table_name: Some(insert_to),
189 logical_plan: message.to_vec(),
190 },
191 )),
192 })
193}
194
195fn format_insert_target_columns(plan: &LogicalPlan) -> String {
196 plan.schema()
197 .fields()
198 .iter()
199 .map(|field| quote_identifier(field.name()).to_string())
200 .collect::<Vec<_>>()
201 .join(", ")
202}
203
204pub struct BatchingExecutionGuard {
207 _lock: OwnedMutexGuard<()>,
208 restore: Option<(Arc<RwLock<TaskState>>, QueryContextRef)>,
209}
210
211impl BatchingExecutionGuard {
212 fn new(lock: OwnedMutexGuard<()>) -> Self {
213 Self {
214 _lock: lock,
215 restore: None,
216 }
217 }
218
219 fn restore_query_context(&mut self, state: Arc<RwLock<TaskState>>, old_ctx: QueryContextRef) {
220 self.restore = Some((state, old_ctx));
221 }
222}
223
224impl Drop for BatchingExecutionGuard {
225 fn drop(&mut self) {
226 if let Some((state, old_ctx)) = self.restore.take()
227 && let Ok(mut state) = state.write()
228 {
229 state.query_ctx = old_ctx;
230 }
231 }
232}
233
234#[derive(Clone)]
235pub struct BatchingTask {
236 pub config: Arc<TaskConfig>,
237 pub state: Arc<RwLock<TaskState>>,
238 execution_lock: Arc<Mutex<()>>,
242 execution: Option<Arc<dyn crate::BatchingExecution>>,
243}
244
245pub struct TaskArgs<'a> {
247 pub flow_id: FlowId,
248 pub query: &'a str,
249 pub plan: LogicalPlan,
250 pub time_window_expr: Option<TimeWindowExpr>,
251 pub expire_after: Option<i64>,
252 pub sink_table_name: [String; 3],
253 pub source_table_names: Vec<[String; 3]>,
254 pub query_ctx: QueryContextRef,
255 pub catalog_manager: CatalogManagerRef,
256 pub shutdown_rx: oneshot::Receiver<()>,
257 pub batch_opts: Arc<BatchingModeOptions>,
258 pub flow_eval_interval: Option<Duration>,
259 pub eval_schedule: Option<EvalSchedule>,
261}
262
263pub struct PlanInfo {
264 pub plan: LogicalPlan,
265 pub dirty_restore: DirtyRestore,
266 pub coverage: QueryCoverage,
267}
268
269#[derive(Clone)]
270pub enum QueryCoverage {
271 UnfilteredFull,
276 ScopedBaseRepair,
279 FencedRepairChunk { high: BTreeMap<u64, u64> },
283 IncrementalDelta,
285}
286
287impl QueryCoverage {
288 fn is_incremental_delta(&self) -> bool {
291 matches!(self, Self::IncrementalDelta)
292 }
293
294 fn snapshot_seqs(&self) -> HashMap<u64, u64> {
297 match self {
298 Self::FencedRepairChunk { high } => high.iter().map(|(k, v)| (*k, *v)).collect(),
299 _ => HashMap::new(),
300 }
301 }
302}
303
304pub enum DirtyRestore {
305 Scoped(FilterExprInfo),
308 Unscoped(DirtyTimeWindows),
315}
316
317pub struct ExecuteOnceOutcome {
318 pub new_query: Option<PlanInfo>,
319 pub result: Result<Option<(usize, Duration)>, Error>,
325}
326
327impl BatchingTask {
328 #[allow(clippy::too_many_arguments)]
329 pub fn try_new(args: TaskArgs<'_>) -> Result<Self, Error> {
330 Self::try_new_with_exact_sequence_range_required(args, false)
331 }
332
333 pub fn try_new_with_exact_sequence_range_required(
334 TaskArgs {
335 flow_id,
336 query,
337 plan,
338 time_window_expr,
339 expire_after,
340 sink_table_name,
341 source_table_names,
342 query_ctx,
343 catalog_manager,
344 shutdown_rx,
345 batch_opts,
346 flow_eval_interval,
347 eval_schedule,
348 }: TaskArgs<'_>,
349 exact_sequence_range_required: bool,
350 ) -> Result<Self, Error> {
351 let mut state = TaskState::with_dirty_time_windows(
352 query_ctx.clone(),
353 shutdown_rx,
354 DirtyTimeWindows::new(
355 batch_opts.experimental_max_filter_num_per_query,
356 batch_opts.experimental_time_window_merge_threshold,
357 ),
358 );
359 if !batch_opts.experimental_enable_incremental_read {
360 state.disable_incremental();
361 }
362
363 Ok(Self {
364 config: Arc::new(TaskConfig {
365 flow_id,
366 query: query.to_string(),
367 time_window_expr,
368 expire_after,
369 sink_table_name,
370 source_table_names: source_table_names.into_iter().collect(),
371 catalog_manager,
372 output_schema: plan.schema().clone(),
373 query_type: determine_query_type(query, &query_ctx)?,
374 exact_sequence_range_required,
375 batch_opts,
376 flow_eval_interval,
377 eval_schedule,
378 }),
379 state: Arc::new(RwLock::new(state)),
380 execution_lock: Arc::new(Mutex::new(())),
381 execution: None,
382 })
383 }
384
385 pub(crate) fn with_execution(
386 mut self,
387 execution: Option<Arc<dyn crate::BatchingExecution>>,
388 ) -> Self {
389 self.execution = execution;
390 self
391 }
392
393 pub(crate) fn stop_execution(&self) {
394 if let Some(execution) = &self.execution {
395 execution.stop();
396 }
397 }
398
399 pub fn last_execution_time_millis(&self) -> Option<i64> {
400 self.state.read().unwrap().last_execution_time_millis()
401 }
402
403 pub fn start_time_millis(&self) -> Option<i64> {
404 self.state.read().unwrap().start_time_millis()
405 }
406
407 fn frontend_extensions(&self) -> HashMap<String, String> {
410 let ctx = self.state.read().unwrap();
411 let all = ctx.query_ctx.extensions();
412 let mut flow_exts = HashMap::new();
413 if let Some(v) = all.get(query::options::FLOW_SCHEDULED_TIME_MILLIS) {
416 flow_exts.insert(
417 query::options::FLOW_SCHEDULED_TIME_MILLIS.to_string(),
418 v.clone(),
419 );
420 }
421 flow_exts
422 }
423
424 pub fn mark_all_windows_as_dirty(&self) -> Result<(), Error> {
428 let now = SystemTime::now();
429 let now = Timestamp::new_second(
430 now.duration_since(UNIX_EPOCH)
431 .expect("Time went backwards")
432 .as_secs() as _,
433 );
434 let lower_bound = self
435 .config
436 .expire_after
437 .map(|e| now.sub_duration(Duration::from_secs(e as _)))
438 .transpose()
439 .map_err(BoxedError::new)
440 .context(ExternalSnafu)?
441 .unwrap_or(Timestamp::new_second(0));
442 debug!(
443 "Flow {} mark range ({:?}, {:?}) as dirty",
444 self.config.flow_id, lower_bound, now
445 );
446 self.state
447 .write()
448 .unwrap()
449 .dirty_time_windows
450 .add_window(lower_bound, Some(now));
451 Ok(())
452 }
453
454 pub async fn check_or_create_sink_table(
456 &self,
457 engine: &QueryEngineRef,
458 frontend_client: &Arc<FrontendClient>,
459 ) -> Result<TableRef, Error> {
460 if !self.is_table_exist(&self.config.sink_table_name).await? {
461 let create_table = self.gen_create_table_expr(engine.clone()).await?;
462 info!(
463 "Try creating sink table(if not exists) with expr: {:?}",
464 create_table
465 );
466 self.create_table(frontend_client, create_table).await?;
467 info!(
468 "Sink table {}(if not exists) created",
469 self.config.sink_table_name.join(".")
470 );
471 }
472 let (table, _) = get_table_info_df_schema(
473 self.config.catalog_manager.clone(),
474 self.config.sink_table_name.clone(),
475 )
476 .await?;
477 Ok(table)
478 }
479
480 pub fn query_context_snapshot(&self) -> QueryContextRef {
482 let query_ctx = self.state.read().unwrap().query_ctx.clone();
483 Arc::new(query_ctx.fork())
484 }
485
486 pub async fn validate_sink_table_schema(&self, engine: &QueryEngineRef) -> Result<(), Error> {
488 self.validate_sink_table_schema_with_values(engine, &BTreeMap::new())
489 .await
490 }
491
492 pub async fn validate_sink_table_schema_with_values(
494 &self,
495 engine: &QueryEngineRef,
496 values: &BTreeMap<String, ScalarValue>,
497 ) -> Result<(), Error> {
498 let (table, _) = get_table_info_df_schema(
499 self.config.catalog_manager.clone(),
500 self.config.sink_table_name.clone(),
501 )
502 .await?;
503
504 let table_meta = &table.table_info().meta;
505 let merge_mode_last_non_null =
506 is_merge_mode_last_non_null(&table_meta.options.extra_options);
507 let primary_key_indices = table_meta.primary_key_indices.clone();
508
509 gen_plan_with_matching_schema_and_values(
510 &self.config.query,
511 self.query_context_snapshot(),
512 engine.clone(),
513 table_meta.schema.clone(),
514 &primary_key_indices,
515 merge_mode_last_non_null,
516 values,
517 )
518 .await
519 .map(|_| ())
520 }
521
522 async fn is_table_exist(&self, table_name: &[String; 3]) -> Result<bool, Error> {
523 self.config
524 .catalog_manager
525 .table_exists(&table_name[0], &table_name[1], &table_name[2], None)
526 .await
527 .map_err(BoxedError::new)
528 .context(ExternalSnafu)
529 }
530
531 pub(crate) async fn execute_once_serialized(
532 &self,
533 engine: &QueryEngineRef,
534 frontend_client: &Arc<FrontendClient>,
535 max_window_cnt: Option<usize>,
536 ) -> Result<Option<(usize, Duration)>, Error> {
537 let outcome = self
538 .execute_once_serialized_with_outcome(engine, frontend_client, max_window_cnt)
539 .await;
540 outcome.result
541 }
542
543 async fn execute_once_serialized_with_outcome(
546 &self,
547 engine: &QueryEngineRef,
548 frontend_client: &Arc<FrontendClient>,
549 max_window_cnt: Option<usize>,
550 ) -> ExecuteOnceOutcome {
551 let guard = BatchingExecutionGuard::new(self.execution_lock.clone().lock_owned().await);
552 self.execute_once_with_guard(guard, engine, frontend_client, max_window_cnt)
553 .await
554 }
555
556 async fn execute_once_with_guard(
557 &self,
558 guard: BatchingExecutionGuard,
559 engine: &QueryEngineRef,
560 frontend_client: &Arc<FrontendClient>,
561 max_window_cnt: Option<usize>,
562 ) -> ExecuteOnceOutcome {
563 if let Some(execution) = &self.execution {
564 return execution
565 .clone()
566 .execute_once(guard, self, engine, frontend_client, max_window_cnt)
567 .await;
568 }
569 self.execute_once_default_unlocked(engine, frontend_client, max_window_cnt)
570 .await
571 }
572
573 async fn execute_once_default_unlocked(
575 &self,
576 engine: &QueryEngineRef,
577 frontend_client: &Arc<FrontendClient>,
578 max_window_cnt: Option<usize>,
579 ) -> ExecuteOnceOutcome {
580 let new_query = match self.gen_insert_plan_unlocked(engine, max_window_cnt).await {
581 Ok(new_query) => new_query,
582 Err(err) => {
583 return ExecuteOnceOutcome {
584 new_query: None,
585 result: Err(err),
586 };
587 }
588 };
589
590 if let Some(new_query) = new_query {
591 debug!("Generate new query: {}", new_query.plan);
592 let res = self
593 .execute_logical_plan_unlocked(
594 engine,
595 frontend_client,
596 &new_query.plan,
597 &new_query.dirty_restore,
598 &new_query.coverage,
599 )
600 .await;
601 if res.is_err() {
602 self.handle_executed_query_failure(Some(&new_query));
603 }
604 ExecuteOnceOutcome {
605 new_query: Some(new_query),
606 result: res,
607 }
608 } else {
609 debug!("Generate no query");
610 ExecuteOnceOutcome {
611 new_query: None,
612 result: Ok(None),
613 }
614 }
615 }
616
617 async fn gen_insert_plan_unlocked(
619 &self,
620 engine: &QueryEngineRef,
621 max_window_cnt: Option<usize>,
622 ) -> Result<Option<PlanInfo>, Error> {
623 self.gen_insert_plan_with_values_unlocked(engine, max_window_cnt, &BTreeMap::new(), false)
624 .await
625 }
626
627 pub async fn gen_insert_plan_with_values_unlocked(
630 &self,
631 engine: &QueryEngineRef,
632 max_window_cnt: Option<usize>,
633 values: &BTreeMap<String, ScalarValue>,
634 force_full_snapshot: bool,
635 ) -> Result<Option<PlanInfo>, Error> {
636 let (table, df_schema) = get_table_info_df_schema(
637 self.config.catalog_manager.clone(),
638 self.config.sink_table_name.clone(),
639 )
640 .await?;
641
642 let table_meta = &table.table_info().meta;
643 let merge_mode_last_non_null =
644 is_merge_mode_last_non_null(&table_meta.options.extra_options);
645 let primary_key_indices = table_meta.primary_key_indices.clone();
646
647 let new_query = self
648 .gen_query_with_time_window_with_values(
649 engine.clone(),
650 &table.table_info().meta.schema,
651 &primary_key_indices,
652 merge_mode_last_non_null,
653 max_window_cnt,
654 values,
655 force_full_snapshot,
656 )
657 .await?;
658
659 let Some(new_query) = new_query else {
660 return Ok(None);
661 };
662
663 let table_columns = df_schema
666 .columns()
667 .into_iter()
668 .map(|c| c.name)
669 .collect::<BTreeSet<_>>();
670 for column in new_query.plan.schema().columns() {
671 if !table_columns.contains(column.name()) {
672 self.restore_dirty_windows_after_failure(&new_query);
673 return InvalidQuerySnafu {
674 reason: format!(
675 "Column {} not found in sink table with columns {:?}",
676 column, table_columns
677 ),
678 }
679 .fail();
680 }
681 }
682
683 let table_provider = Arc::new(DfTableProviderAdapter::new(table));
684 let table_source = Arc::new(DefaultTableSource::new(table_provider));
685
686 let plan = LogicalPlan::Dml(DmlStatement::new(
688 datafusion_common::TableReference::Full {
689 catalog: self.config.sink_table_name[0].clone().into(),
690 schema: self.config.sink_table_name[1].clone().into(),
691 table: self.config.sink_table_name[2].clone().into(),
692 },
693 table_source,
694 WriteOp::Insert(datafusion_expr::dml::InsertOp::Append),
695 Arc::new(new_query.plan.clone()),
696 ));
697 let insert_into_info = PlanInfo {
698 plan,
699 dirty_restore: new_query.dirty_restore,
700 coverage: new_query.coverage,
701 };
702 let insert_into =
703 match insert_into_info
704 .plan
705 .clone()
706 .recompute_schema()
707 .context(DatafusionSnafu {
708 context: "Failed to recompute schema",
709 }) {
710 Ok(insert_into) => insert_into,
711 Err(err) => {
712 self.restore_dirty_windows_after_failure(&insert_into_info);
713 return Err(err);
714 }
715 };
716
717 Ok(Some(PlanInfo {
718 plan: insert_into,
719 dirty_restore: insert_into_info.dirty_restore,
720 coverage: insert_into_info.coverage,
721 }))
722 }
723
724 pub async fn create_table(
725 &self,
726 frontend_client: &Arc<FrontendClient>,
727 expr: CreateTableExpr,
728 ) -> Result<(), Error> {
729 let catalog = &self.config.sink_table_name[0];
730 let schema = &self.config.sink_table_name[1];
731 frontend_client
732 .create(expr.clone(), catalog, schema)
733 .await?;
734 Ok(())
735 }
736
737 async fn execute_logical_plan_unlocked(
739 &self,
740 engine: &QueryEngineRef,
741 frontend_client: &Arc<FrontendClient>,
742 plan: &LogicalPlan,
743 dirty_restore: &DirtyRestore,
744 coverage: &QueryCoverage,
745 ) -> Result<Option<(usize, Duration)>, Error> {
746 let flow_id = self.config.flow_id;
747 let Some((res, elapsed)) = self
748 .execute_plan_unlocked(engine, frontend_client, plan, dirty_restore, coverage)
749 .await?
750 else {
751 return Ok(None);
752 };
753
754 if let Err(err) = &res {
755 let decision = {
756 let mut state = self.state.write().unwrap();
757 let reason = Self::query_failure_reason(err, coverage);
758 Self::apply_query_failure_to_state(&mut state, elapsed, coverage, reason)
759 };
760 if let Some(decision) = decision {
761 Self::record_checkpoint_decision(flow_id, decision);
762 }
763 }
764
765 let res = res?;
766 let (affected_rows, _) = res.output.extract_rows_and_cost();
767 let decision = {
768 let mut state = self.state.write().unwrap();
769 Self::apply_query_result_to_state(&mut state, &res, elapsed, coverage)
770 };
771 Self::record_checkpoint_decision(flow_id, decision);
772 Ok(Some((affected_rows, elapsed)))
773 }
774
775 pub async fn execute_plan_unlocked(
779 &self,
780 engine: &QueryEngineRef,
781 frontend_client: &Arc<FrontendClient>,
782 plan: &LogicalPlan,
783 dirty_restore: &DirtyRestore,
784 coverage: &QueryCoverage,
785 ) -> Result<Option<(Result<OutputWithMetrics, Error>, Duration)>, Error> {
786 let instant = Instant::now();
787 let flow_id = self.config.flow_id;
788
789 debug!(
790 "Executing flow {flow_id}(expire_after={:?} secs) with query {}",
791 self.config.expire_after, &plan
792 );
793
794 let catalog = &self.config.sink_table_name[0];
795 let schema = &self.config.sink_table_name[1];
796
797 let plan = plan
799 .clone()
800 .transform_down_with_subqueries(|p| {
801 if let LogicalPlan::TableScan(mut table_scan) = p {
802 let resolved = table_scan.table_name.resolve(catalog, schema);
803 table_scan.table_name = resolved.into();
804 Ok(Transformed::yes(LogicalPlan::TableScan(table_scan)))
805 } else {
806 Ok(Transformed::no(p))
807 }
808 })
809 .with_context(|_| DatafusionSnafu {
810 context: format!("Failed to fix table ref in logical plan, plan={:?}", plan),
811 })?
812 .data;
813
814 let incremental_plan = if coverage.is_incremental_delta() {
817 self.prepare_plan_for_incremental(engine, &plan).await?
818 } else {
819 None
820 };
821 let incremental_safe = incremental_plan.is_some();
822 if coverage.is_incremental_delta() && !incremental_safe {
823 debug!(
824 "Flow {flow_id} skipped unsafe incremental delta fallback; \
825 restored dirty signal instead of executing an unfiltered full snapshot"
826 );
827 self.restore_dirty_windows(dirty_restore);
828 return Ok(None);
829 }
830 let plan = incremental_plan.unwrap_or_else(|| plan.clone());
831
832 let extensions = self
833 .build_flow_query_extensions(incremental_safe, coverage.is_incremental_delta())
834 .await?;
835 let frontend_extensions = self.frontend_extensions();
836 let extension_refs = extensions
837 .iter()
838 .map(|(key, value)| (*key, value.as_str()))
839 .chain(
840 frontend_extensions
841 .iter()
842 .map(|(key, value)| (key.as_str(), value.as_str())),
843 )
844 .collect::<Vec<_>>();
845 let query_mode = if extensions
846 .iter()
847 .any(|(key, _)| *key == FLOW_INCREMENTAL_MODE)
848 {
849 CheckpointMode::Incremental
850 } else {
851 CheckpointMode::FullSnapshot
852 };
853 Self::record_query_mode(flow_id, query_mode);
854 debug!(
855 "Flow {flow_id} executing batching query with checkpoint_mode={}, extension_count={}",
856 checkpoint_mode_label(query_mode),
857 extensions.len()
858 );
859
860 let mut peer_desc = None;
861 let res = {
862 let _timer = METRIC_FLOW_BATCHING_ENGINE_QUERY_TIME
863 .with_label_values(&[flow_id.to_string().as_str()])
864 .start_timer();
865
866 let req = if let Some((insert_to, insert_input_plan)) =
867 breakup_insert_plan(&plan, catalog, schema)
868 {
869 if query_mode == CheckpointMode::FullSnapshot
870 && matches!(self.config.query_type, QueryType::Sql)
871 && self.config.flow_eval_interval.is_some()
872 && self.config.time_window_expr.is_none()
873 {
874 match df_plan_to_sql(&insert_input_plan) {
884 Ok(select_sql) => {
885 let target_columns = format_insert_target_columns(&insert_input_plan);
886 let sql = format!(
887 "INSERT INTO {} ({}) {}",
888 TableReference::full(
889 insert_to.catalog_name.as_str(),
890 insert_to.schema_name.as_str(),
891 insert_to.table_name.as_str(),
892 )
893 .to_quoted_string(),
894 target_columns,
895 select_sql
896 );
897 api::v1::QueryRequest {
898 query: Some(api::v1::query_request::Query::Sql(sql)),
899 }
900 }
901 Err(err) => {
902 debug!(
903 "Failed to unparse full-snapshot SQL flow {} plan; \
904 falling back to InsertIntoPlan: {:?}",
905 flow_id, err
906 );
907 encode_insert_plan_request(insert_to, &insert_input_plan)?
908 }
909 }
910 } else {
911 encode_insert_plan_request(insert_to, &insert_input_plan)?
912 }
913 } else {
914 let message = DFLogicalSubstraitConvertor {}
915 .encode(&plan, DefaultSerializer)
916 .context(SubstraitEncodeLogicalPlanSnafu)?;
917
918 api::v1::QueryRequest {
919 query: Some(api::v1::query_request::Query::LogicalPlan(message.to_vec())),
920 }
921 };
922
923 let snapshot_seqs = coverage.snapshot_seqs();
924 {
925 let mut state = self.state.write().unwrap();
926 state.record_start_time_if_first();
927 }
928 frontend_client
929 .query_with_terminal_metrics(
930 catalog,
931 schema,
932 req,
933 &extension_refs,
934 &snapshot_seqs,
935 &mut peer_desc,
936 )
937 .await
938 };
939
940 let elapsed = instant.elapsed();
941 let peer_label = peer_desc
942 .as_ref()
943 .map(ToString::to_string)
944 .unwrap_or_else(|| PeerDesc::default().to_string());
945 if let Err(err) = &res {
946 warn!(
950 "Failed to execute Flow {flow_id} on frontend {peer_label}, result: {err:?}, elapsed: {:?} with query: {}",
951 elapsed, &plan
952 );
953 }
954
955 if elapsed >= self.config.batch_opts.slow_query_threshold {
957 warn!(
958 "Flow {flow_id} on frontend {peer_label} executed for {:?} before complete, query: {}",
959 elapsed, &plan
960 );
961 let flow_id = flow_id.to_string();
962 METRIC_FLOW_BATCHING_ENGINE_SLOW_QUERY
963 .with_label_values(&[flow_id.as_str(), peer_label.as_str()])
964 .observe(elapsed.as_secs_f64());
965 }
966
967 match res {
968 Ok(res) => {
969 let (affected_rows, _) = res.output.extract_rows_and_cost();
970 debug!(
971 "Flow {flow_id} executed, affected_rows: {affected_rows:?}, elapsed: {:?}, watermark: {:?}",
972 elapsed,
973 res.region_watermark_map()
974 );
975 METRIC_FLOW_ROWS
976 .with_label_values(&[format!("{}-out-batching", flow_id).as_str()])
977 .inc_by(affected_rows as _);
978 Ok(Some((Ok(res), elapsed)))
979 }
980 Err(err) => Ok(Some((Err(err), elapsed))),
981 }
982 }
983
984 pub fn restore_dirty_windows(&self, dirty_restore: &DirtyRestore) {
988 match dirty_restore {
989 DirtyRestore::Scoped(filter) => self.restore_scoped_dirty_windows(filter),
990 DirtyRestore::Unscoped(dirty_windows) => self
991 .state
992 .write()
993 .unwrap()
994 .dirty_time_windows
995 .add_dirty_windows(dirty_windows),
996 }
997 }
998
999 fn restore_dirty_windows_after_failure(&self, query: &PlanInfo) {
1002 self.restore_dirty_windows(&query.dirty_restore);
1003 }
1004
1005 fn restore_scoped_dirty_windows(&self, filter: &FilterExprInfo) {
1008 self.state.write().unwrap().restore_scoped_windows(filter);
1009 }
1010
1011 fn restore_scoped_dirty_windows_on_err<T>(
1014 &self,
1015 filter: &FilterExprInfo,
1016 result: Result<T, Error>,
1017 ) -> Result<T, Error> {
1018 result.inspect_err(|_| {
1019 self.restore_scoped_dirty_windows(filter);
1020 })
1021 }
1022
1023 fn restore_unscoped_dirty_windows(&self, dirty_windows: &DirtyTimeWindows) {
1026 self.state
1027 .write()
1028 .unwrap()
1029 .dirty_time_windows
1030 .add_dirty_windows(dirty_windows);
1031 }
1032
1033 fn restore_unscoped_dirty_windows_on_err<T>(
1036 &self,
1037 dirty_windows: &DirtyTimeWindows,
1038 result: Result<T, Error>,
1039 ) -> Result<T, Error> {
1040 result.inspect_err(|_| {
1041 self.restore_unscoped_dirty_windows(dirty_windows);
1042 })
1043 }
1044
1045 fn drain_dirty_windows_signal(&self) -> (bool, DirtyTimeWindows) {
1048 let mut state = self.state.write().unwrap();
1049 let dirty_windows_to_restore = state.dirty_time_windows.clone();
1050 let is_dirty = !dirty_windows_to_restore.is_empty();
1051 state.dirty_time_windows.clean();
1052 (is_dirty, dirty_windows_to_restore)
1053 }
1054
1055 #[allow(clippy::too_many_arguments)]
1056 async fn gen_unfiltered_plan_info(
1059 &self,
1060 engine: QueryEngineRef,
1061 query_ctx: QueryContextRef,
1062 sink_table_schema: Arc<Schema>,
1063 primary_key_indices: &[usize],
1064 allow_partial: bool,
1065 values: &BTreeMap<String, ScalarValue>,
1066 dirty_windows_to_restore: DirtyTimeWindows,
1067 retention_filter: Option<(&str, Timestamp, &'static str)>,
1068 coverage: QueryCoverage,
1069 ) -> Result<PlanInfo, Error> {
1070 let mut plan = self.restore_unscoped_dirty_windows_on_err(
1071 &dirty_windows_to_restore,
1072 gen_plan_with_matching_schema_and_values(
1073 &self.config.query,
1074 query_ctx,
1075 engine,
1076 sink_table_schema,
1077 primary_key_indices,
1078 allow_partial,
1079 values,
1080 )
1081 .await,
1082 )?;
1083
1084 if let Some((col_name, lower_bound, context)) = retention_filter {
1085 let lower = self.restore_unscoped_dirty_windows_on_err(
1086 &dirty_windows_to_restore,
1087 to_df_literal(lower_bound),
1088 )?;
1089 let retention_filter = col(col_name).gt_eq(lit(lower));
1090 let mut add_filter = AddFilterRewriter::new(retention_filter);
1091 plan = self.restore_unscoped_dirty_windows_on_err(
1092 &dirty_windows_to_restore,
1093 plan.clone()
1094 .rewrite(&mut add_filter)
1095 .with_context(|_| DatafusionSnafu {
1096 context: format!(
1097 "Failed to apply {context} expire_after filter to plan:\n {}\n",
1098 plan
1099 ),
1100 })
1101 .map(|rewrite| rewrite.data),
1102 )?;
1103 }
1104
1105 Ok(PlanInfo {
1106 plan,
1107 dirty_restore: DirtyRestore::Unscoped(dirty_windows_to_restore),
1108 coverage,
1109 })
1110 }
1111
1112 #[allow(clippy::too_many_arguments)]
1113 async fn gen_unfiltered_plan_info_if_dirty(
1116 &self,
1117 engine: QueryEngineRef,
1118 query_ctx: QueryContextRef,
1119 sink_table_schema: Arc<Schema>,
1120 primary_key_indices: &[usize],
1121 allow_partial: bool,
1122 values: &BTreeMap<String, ScalarValue>,
1123 retention_filter: Option<(&str, Timestamp, &'static str)>,
1124 coverage: QueryCoverage,
1125 ) -> Result<Option<PlanInfo>, Error> {
1126 let (is_dirty, dirty_windows_to_restore) = self.drain_dirty_windows_signal();
1127 if !is_dirty {
1128 debug!("Flow id={:?}, no new data, not update", self.config.flow_id);
1129 return Ok(None);
1130 }
1131
1132 self.gen_unfiltered_plan_info(
1133 engine,
1134 query_ctx,
1135 sink_table_schema,
1136 primary_key_indices,
1137 allow_partial,
1138 values,
1139 dirty_windows_to_restore,
1140 retention_filter,
1141 coverage,
1142 )
1143 .await
1144 .map(Some)
1145 }
1146
1147 fn handle_executed_query_failure(&self, query: Option<&PlanInfo>) {
1148 if let Some(query) = query {
1149 self.restore_dirty_windows_after_failure(query);
1150 }
1151 }
1152
1153 pub async fn start_executing_loop(
1161 &self,
1162 engine: QueryEngineRef,
1163 frontend_client: Arc<FrontendClient>,
1164 ) {
1165 if self.config.flow_eval_interval.is_some() {
1166 self.start_scheduled_loop(engine, frontend_client).await;
1167 } else {
1168 self.start_adaptive_loop(engine, frontend_client).await;
1169 }
1170 }
1171
1172 async fn start_scheduled_loop(
1182 &self,
1183 engine: QueryEngineRef,
1184 frontend_client: Arc<FrontendClient>,
1185 ) {
1186 let flow_id_str = self.config.flow_id.to_string();
1187
1188 let schedule = match &self.config.eval_schedule {
1189 Some(s) => s.clone(),
1190 None => {
1191 let eval_interval_secs = self
1192 .config
1193 .flow_eval_interval
1194 .map(|d| d.as_secs() as i64)
1195 .expect("checked by caller");
1196
1197 match EvalSchedule::from_config(Some(eval_interval_secs), None) {
1200 Ok(Some(s)) => s,
1201 Ok(None) => {
1202 warn!(
1203 "Flow {}: EVAL INTERVAL set but no schedule parsed; exiting loop",
1204 flow_id_str
1205 );
1206 return;
1207 }
1208 Err(e) => {
1209 warn!(
1210 "Flow {}: Failed to parse eval schedule: {}; exiting loop",
1211 flow_id_str, e
1212 );
1213 return;
1214 }
1215 }
1216 }
1217 };
1218
1219 let mut cursor_secs =
1224 match initial_schedule_cursor(schedule.start_secs, schedule.interval_secs) {
1225 Ok(cursor) => cursor,
1226 Err(e) => {
1227 warn!(
1228 "Flow {}: invalid eval schedule, exiting loop: {}",
1229 flow_id_str, e
1230 );
1231 return;
1232 }
1233 };
1234
1235 info!(
1236 "Flow {}: entering scheduled loop, interval={}s, start={}, anchor={}, policy={:?}, max_runs={}, max_lag={}s",
1237 flow_id_str,
1238 schedule.interval_secs,
1239 schedule.start_secs,
1240 schedule.anchor_secs,
1241 schedule.missed_tick_policy,
1242 schedule.max_runs,
1243 schedule.max_lag_secs,
1244 );
1245
1246 loop {
1247 if self.is_shutdown_signaled() {
1248 break;
1249 }
1250
1251 let wall_now_secs = wall_clock_unix_secs();
1252
1253 let due = match select_due_scheduled_times(&schedule, cursor_secs, wall_now_secs) {
1254 Ok(d) => d,
1255 Err(e) => {
1256 warn!(
1257 "Flow {}: invalid eval schedule, exiting loop: {}",
1258 flow_id_str, e
1259 );
1260 return;
1261 }
1262 };
1263
1264 if due.scheduled_times_secs.is_empty() {
1265 if due.skipped > 0 {
1266 warn!(
1267 "Flow {}: all {} due scheduled times skipped by max-lag, advancing cursor to wall-clock ({wall_now_secs}) to avoid re-skipping",
1268 flow_id_str, due.skipped
1269 );
1270 cursor_secs = wall_now_secs;
1271 continue;
1272 }
1273
1274 let next = match schedule.next_scheduled_time_after(cursor_secs) {
1276 Ok(next) => next,
1277 Err(e) => {
1278 warn!(
1279 "Flow {}: cannot advance eval schedule past cursor {cursor_secs}: {e}; exiting loop",
1280 flow_id_str
1281 );
1282 return;
1283 }
1284 };
1285 if next <= wall_now_secs {
1286 cursor_secs = wall_now_secs;
1289 continue;
1290 }
1291 let wait_secs = match sleep_delta_secs(next, wall_now_secs) {
1292 Ok(wait_secs) => wait_secs,
1293 Err(e) => {
1294 warn!(
1295 "Flow {}: cannot sleep until next scheduled time {}: {e}; exiting loop",
1296 flow_id_str, next
1297 );
1298 return;
1299 }
1300 };
1301 let wait_dur = Duration::from_secs(wait_secs);
1302 debug!(
1303 "Flow {}: no due scheduled times, sleeping for {}s until next scheduled time at {}",
1304 flow_id_str, wait_secs, next
1305 );
1306 tokio::time::sleep(wait_dur).await;
1307 continue;
1308 }
1309
1310 if due.skipped > 0 {
1311 info!(
1312 "Flow {}: {} due scheduled times, {} skipped (catch-up)",
1313 flow_id_str,
1314 due.scheduled_times_secs.len(),
1315 due.skipped
1316 );
1317 }
1318
1319 for scheduled_time_secs in &due.scheduled_times_secs {
1321 if self.is_shutdown_signaled() {
1322 break;
1323 }
1324
1325 METRIC_FLOW_BATCHING_ENGINE_START_QUERY_CNT
1326 .with_label_values(&[&flow_id_str])
1327 .inc();
1328
1329 let outcome = self
1330 .execute_once_serialized_at_scheduled_time(
1331 &engine,
1332 &frontend_client,
1333 *scheduled_time_secs,
1334 )
1335 .await;
1336
1337 cursor_secs = *scheduled_time_secs;
1339
1340 match outcome.result {
1341 Ok(Some((rows, elapsed))) => {
1342 debug!(
1343 "Flow {}: scheduled time {} completed, rows={}, elapsed={:?}",
1344 flow_id_str, scheduled_time_secs, rows, elapsed
1345 );
1346 }
1347 Ok(None) => {
1348 debug!(
1349 "Flow {}: scheduled time {} produced no query (no dirty signal or no-op)",
1350 flow_id_str, scheduled_time_secs
1351 );
1352 }
1353 Err(err) => {
1354 warn!(
1355 "Flow {}: scheduled time {} failed: {:?}",
1356 flow_id_str, scheduled_time_secs, err
1357 );
1358 METRIC_FLOW_BATCHING_ENGINE_ERROR_CNT
1359 .with_label_values(&[&flow_id_str])
1360 .inc();
1361 }
1365 }
1366 }
1367 }
1368 }
1369
1370 async fn start_adaptive_loop(
1372 &self,
1373 engine: QueryEngineRef,
1374 frontend_client: Arc<FrontendClient>,
1375 ) {
1376 let flow_id_str = self.config.flow_id.to_string();
1377 let mut max_window_cnt = None;
1378 loop {
1379 if self.is_shutdown_signaled() {
1380 break;
1381 }
1382 METRIC_FLOW_BATCHING_ENGINE_START_QUERY_CNT
1383 .with_label_values(&[&flow_id_str])
1384 .inc();
1385
1386 let min_refresh = self.config.batch_opts.experimental_min_refresh_duration;
1387
1388 let outcome = self
1389 .execute_once_serialized_with_outcome(&engine, &frontend_client, max_window_cnt)
1390 .await;
1391
1392 match outcome.result {
1393 Ok(Some(_)) => {
1394 max_window_cnt = max_window_cnt.map(|cnt| {
1395 (cnt + 1).min(self.config.batch_opts.experimental_max_filter_num_per_query)
1396 });
1397
1398 let sleep_until = {
1399 let state = self.state.write().unwrap();
1400
1401 let time_window_size = self
1402 .config
1403 .time_window_expr
1404 .as_ref()
1405 .and_then(|t| *t.time_window_size());
1406
1407 let prefer_short_incremental_cadence = state.checkpoint_mode()
1408 == CheckpointMode::Incremental
1409 && !state.is_incremental_disabled();
1410
1411 state.get_next_start_query_time(
1412 self.config.flow_id,
1413 &time_window_size,
1414 min_refresh,
1415 Some(self.config.batch_opts.query_timeout),
1416 self.config.batch_opts.experimental_max_filter_num_per_query,
1417 prefer_short_incremental_cadence,
1418 )
1419 };
1420
1421 tokio::time::sleep_until(sleep_until).await;
1422 }
1423 Ok(None) => {
1424 debug!(
1425 "Flow id = {:?} found no new data, sleep for {:?} then continue",
1426 self.config.flow_id, min_refresh
1427 );
1428 tokio::time::sleep(min_refresh).await;
1429 continue;
1430 }
1431 Err(err) => {
1432 METRIC_FLOW_BATCHING_ENGINE_ERROR_CNT
1433 .with_label_values(&[&flow_id_str])
1434 .inc();
1435 match outcome.new_query {
1436 Some(query) => {
1437 common_telemetry::error!(err; "Failed to execute query for flow={} with query: {}", self.config.flow_id, query.plan);
1438 max_window_cnt = Some(1);
1439 }
1440 None => {
1441 common_telemetry::error!(err; "Failed to generate query for flow={}", self.config.flow_id)
1442 }
1443 }
1444 tokio::time::sleep(min_refresh).await;
1445 }
1446 }
1447 }
1448 }
1449
1450 fn is_shutdown_signaled(&self) -> bool {
1452 let mut state = self.state.write().unwrap();
1453 match state.shutdown_rx.try_recv() {
1454 Ok(()) | Err(TryRecvError::Closed) => true,
1455 Err(TryRecvError::Empty) => false,
1456 }
1457 }
1458
1459 async fn execute_once_serialized_at_scheduled_time(
1466 &self,
1467 engine: &QueryEngineRef,
1468 frontend_client: &Arc<FrontendClient>,
1469 scheduled_time_secs: i64,
1470 ) -> ExecuteOnceOutcome {
1471 let mut guard = BatchingExecutionGuard::new(self.execution_lock.clone().lock_owned().await);
1472
1473 let scheduled_time_millis = match scheduled_time_millis(scheduled_time_secs) {
1477 Ok(millis) => millis,
1478 Err(e) => {
1479 return ExecuteOnceOutcome {
1480 new_query: None,
1481 result: Err(e),
1482 };
1483 }
1484 };
1485
1486 let old_ctx = {
1489 let mut state = self.state.write().unwrap();
1490 let old = state.query_ctx.clone();
1491 let mut new_ctx = (*old).clone();
1492 new_ctx.set_extension(
1493 query::options::FLOW_SCHEDULED_TIME_MILLIS,
1494 scheduled_time_millis.to_string(),
1495 );
1496 state.query_ctx = Arc::new(new_ctx);
1497 old
1498 };
1499 guard.restore_query_context(self.state.clone(), old_ctx);
1500
1501 self.execute_once_with_guard(guard, engine, frontend_client, None)
1504 .await
1505 }
1506
1507 async fn gen_create_table_expr(
1514 &self,
1515 engine: QueryEngineRef,
1516 ) -> Result<CreateTableExpr, Error> {
1517 let query_ctx = self.state.read().unwrap().query_ctx.clone();
1518 let plan =
1519 sql_to_df_plan(query_ctx.clone(), engine.clone(), &self.config.query, true).await?;
1520 create_table_with_expr(&plan, &self.config.sink_table_name, &self.config.query_type)
1521 }
1522
1523 fn should_use_unfiltered_incremental_delta(&self) -> bool {
1526 let state = self.state.read().unwrap();
1527 state.checkpoint_mode() == CheckpointMode::Incremental
1528 && !state.is_incremental_disabled()
1529 && matches!(self.config.query_type, QueryType::Sql)
1530 }
1531
1532 async fn gen_query_with_time_window(
1534 &self,
1535 engine: QueryEngineRef,
1536 sink_table_schema: &Arc<Schema>,
1537 primary_key_indices: &[usize],
1538 allow_partial: bool,
1539 max_window_cnt: Option<usize>,
1540 ) -> Result<Option<PlanInfo>, Error> {
1541 self.gen_query_with_time_window_with_values(
1542 engine,
1543 sink_table_schema,
1544 primary_key_indices,
1545 allow_partial,
1546 max_window_cnt,
1547 &BTreeMap::new(),
1548 false,
1549 )
1550 .await
1551 }
1552
1553 #[allow(clippy::too_many_arguments)]
1555 async fn gen_query_with_time_window_with_values(
1556 &self,
1557 engine: QueryEngineRef,
1558 sink_table_schema: &Arc<Schema>,
1559 primary_key_indices: &[usize],
1560 allow_partial: bool,
1561 max_window_cnt: Option<usize>,
1562 values: &BTreeMap<String, ScalarValue>,
1563 force_full_snapshot: bool,
1564 ) -> Result<Option<PlanInfo>, Error> {
1565 let query_ctx = self.state.read().unwrap().query_ctx.clone();
1566 let start = SystemTime::now();
1567 let since_the_epoch = start
1568 .duration_since(UNIX_EPOCH)
1569 .expect("Time went backwards");
1570 let low_bound = self
1571 .config
1572 .expire_after
1573 .map(|e| since_the_epoch.as_secs() - e as u64)
1574 .unwrap_or(u64::MIN);
1575
1576 let low_bound = Timestamp::new_second(low_bound as i64);
1577
1578 let expire_time_window_bound = self
1579 .config
1580 .time_window_expr
1581 .as_ref()
1582 .map(|expr| expr.eval(low_bound))
1583 .transpose()?;
1584
1585 let (expire_lower_bound, expire_upper_bound) = match (
1586 expire_time_window_bound,
1587 &self.config.query_type,
1588 ) {
1589 (Some((Some(l), Some(u))), QueryType::Sql) => (l, u),
1590 (None, QueryType::Sql) if self.config.flow_eval_interval.is_none() => {
1591 return UnexpectedSnafu {
1592 reason: format!(
1593 "Flow id={} reached execution without a time-window expression or EVAL INTERVAL; create-flow validation should have rejected it",
1594 self.config.flow_id
1595 ),
1596 }
1597 .fail();
1598 }
1599 _ => {
1600 let (_, dirty_windows_to_restore) = self.drain_dirty_windows_signal();
1606
1607 let plan_info = self
1608 .gen_unfiltered_plan_info(
1609 engine,
1610 query_ctx,
1611 sink_table_schema.clone(),
1612 primary_key_indices,
1613 allow_partial,
1614 values,
1615 dirty_windows_to_restore,
1616 None,
1617 QueryCoverage::UnfilteredFull,
1618 )
1619 .await?;
1620
1621 return Ok(Some(plan_info));
1622 }
1623 };
1624
1625 debug!(
1626 "Flow id = {:?}, found time window: precise_lower_bound={:?}, precise_upper_bound={:?} with dirty time windows: {:?}",
1627 self.config.flow_id,
1628 expire_lower_bound,
1629 expire_upper_bound,
1630 self.state.read().unwrap().dirty_time_windows
1631 );
1632 let window_size = expire_upper_bound
1633 .sub(&expire_lower_bound)
1634 .with_context(|| UnexpectedSnafu {
1635 reason: format!(
1636 "Can't get window size from {expire_upper_bound:?} - {expire_lower_bound:?}"
1637 ),
1638 })?;
1639 let col_name = self
1640 .config
1641 .time_window_expr
1642 .as_ref()
1643 .map(|expr| expr.column_name.clone())
1644 .with_context(|| UnexpectedSnafu {
1645 reason: format!(
1646 "Flow id={:?}, Failed to get column name from time window expr",
1647 self.config.flow_id
1648 ),
1649 })?;
1650
1651 if force_full_snapshot {
1652 let retention_filter = self.config.expire_after.map(|_| {
1656 (
1657 col_name.as_str(),
1658 expire_lower_bound,
1659 "forced full snapshot",
1660 )
1661 });
1662 let (_, dirty_windows_to_restore) = self.drain_dirty_windows_signal();
1663 return self
1664 .gen_unfiltered_plan_info(
1665 engine,
1666 query_ctx,
1667 sink_table_schema.clone(),
1668 primary_key_indices,
1669 allow_partial,
1670 values,
1671 dirty_windows_to_restore,
1672 retention_filter,
1673 QueryCoverage::UnfilteredFull,
1674 )
1675 .await
1676 .map(Some);
1677 }
1678
1679 if self.should_use_unfiltered_incremental_delta() {
1680 let retention_filter = self
1690 .config
1691 .expire_after
1692 .map(|_| (col_name.as_str(), expire_lower_bound, "incremental"));
1693 return self
1694 .gen_unfiltered_plan_info_if_dirty(
1695 engine,
1696 query_ctx,
1697 sink_table_schema.clone(),
1698 primary_key_indices,
1699 allow_partial,
1700 values,
1701 retention_filter,
1702 QueryCoverage::IncrementalDelta,
1703 )
1704 .await;
1705 }
1706
1707 let (expr, coverage) = {
1708 let mut state = self.state.write().unwrap();
1709 let window_cnt = max_window_cnt
1710 .unwrap_or(self.config.batch_opts.experimental_max_filter_num_per_query);
1711 let expr = state.gen_scoped_filter_exprs(
1712 &col_name,
1713 Some(expire_lower_bound),
1714 window_size,
1715 window_cnt,
1716 self.config.flow_id,
1717 Some(self),
1718 )?;
1719 let repair_high = state
1720 .pending_fenced_repair()
1721 .map(|repair| repair.high().clone());
1722 let coverage = if let Some(high) = repair_high {
1723 QueryCoverage::FencedRepairChunk { high }
1724 } else {
1725 QueryCoverage::ScopedBaseRepair
1726 };
1727 (expr, coverage)
1728 };
1729
1730 let Some(expr) = expr else {
1731 debug!("Flow id={:?}, no new data, not update", self.config.flow_id);
1733 return Ok(None);
1734 };
1735
1736 let filter_sql = expr_to_sql(&expr.expr)
1737 .map(|sql| sql.to_string())
1738 .unwrap_or_else(|err| format!("<failed to format filter expr: {err}>"));
1739
1740 debug!(
1741 "Flow id={:?}, Generated filter expr: {:?}",
1742 self.config.flow_id, filter_sql
1743 );
1744
1745 let mut add_filter = AddFilterRewriter::new(expr.expr.clone());
1746 let mut add_auto_column = ColumnMatcherRewriter::new_with_values(
1747 sink_table_schema.clone(),
1748 primary_key_indices.to_vec(),
1749 allow_partial,
1750 values.clone(),
1751 );
1752
1753 let plan = self.restore_scoped_dirty_windows_on_err(
1754 &expr,
1755 sql_to_df_plan(query_ctx.clone(), engine.clone(), &self.config.query, false).await,
1756 )?;
1757 let rewrite = self.restore_scoped_dirty_windows_on_err(
1758 &expr,
1759 plan.clone()
1760 .rewrite(&mut add_filter)
1761 .and_then(|p| p.data.rewrite(&mut add_auto_column))
1762 .with_context(|_| DatafusionSnafu {
1763 context: format!("Failed to rewrite plan:\n {}\n", plan),
1764 })
1765 .map(|rewrite| rewrite.data),
1766 )?;
1767 let new_plan = self.restore_scoped_dirty_windows_on_err(
1769 &expr,
1770 apply_df_optimizer(rewrite, &query_ctx).await,
1771 )?;
1772
1773 let info = PlanInfo {
1774 plan: new_plan.clone(),
1775 dirty_restore: DirtyRestore::Scoped(expr),
1776 coverage,
1777 };
1778
1779 Ok(Some(info))
1780 }
1781}
1782
1783#[cfg(test)]
1784mod test;