Skip to main content

flow/batching_mode/
task.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
16use std::sync::{Arc, RwLock};
17use std::time::{Duration, SystemTime, UNIX_EPOCH};
18
19use api::v1::{CreateTableExpr, TableName};
20use catalog::CatalogManagerRef;
21use client::OutputWithMetrics;
22use common_error::ext::BoxedError;
23use common_query::logical_plan::breakup_insert_plan;
24use common_telemetry::tracing::warn;
25use common_telemetry::{debug, info};
26use common_time::Timestamp;
27use datafusion::datasource::DefaultTableSource;
28use datafusion::sql::unparser::expr_to_sql;
29use datafusion_common::tree_node::{Transformed, TreeNode};
30use datafusion_common::utils::quote_identifier;
31use datafusion_common::{DFSchemaRef, ScalarValue, TableReference};
32use datafusion_expr::{DmlStatement, LogicalPlan, WriteOp, col, lit};
33use datatypes::schema::Schema;
34use query::QueryEngineRef;
35use query::options::FLOW_INCREMENTAL_MODE;
36use query::query_engine::DefaultSerializer;
37use session::context::QueryContextRef;
38use snafu::{OptionExt, ResultExt};
39use sql::parsers::utils::is_tql;
40use store_api::mito_engine_options::MERGE_MODE_KEY;
41use substrait::{DFLogicalSubstraitConvertor, SubstraitPlan};
42use table::TableRef;
43use table::table::adapter::DfTableProviderAdapter;
44use tokio::sync::oneshot::error::TryRecvError;
45use tokio::sync::{Mutex, OwnedMutexGuard, oneshot};
46use tokio::time::Instant;
47
48use crate::batching_mode::BatchingModeOptions;
49use crate::batching_mode::checkpoint::checkpoint_mode_label;
50use crate::batching_mode::eval_schedule::{EvalSchedule, select_due_scheduled_times};
51use crate::batching_mode::frontend_client::{FrontendClient, PeerDesc};
52use crate::batching_mode::state::{
53    CheckpointMode, DirtyTimeWindows, FilterExprInfo, TaskState, to_df_literal,
54};
55use crate::batching_mode::table_creator::{QueryType, create_table_with_expr};
56use crate::batching_mode::time_window::TimeWindowExpr;
57use crate::batching_mode::utils::{
58    AddFilterRewriter, ColumnMatcherRewriter, df_plan_to_sql,
59    gen_plan_with_matching_schema_and_values, get_table_info_df_schema, sql_to_df_plan,
60};
61use crate::df_optimizer::apply_df_optimizer;
62use crate::error::{
63    DatafusionSnafu, ExternalSnafu, InvalidQuerySnafu, SubstraitEncodeLogicalPlanSnafu,
64    UnexpectedSnafu,
65};
66use crate::metrics::{
67    METRIC_FLOW_BATCHING_ENGINE_ERROR_CNT, METRIC_FLOW_BATCHING_ENGINE_QUERY_TIME,
68    METRIC_FLOW_BATCHING_ENGINE_SLOW_QUERY, METRIC_FLOW_BATCHING_ENGINE_START_QUERY_CNT,
69    METRIC_FLOW_ROWS,
70};
71use crate::{Error, FlowId};
72
73mod ckpt;
74mod inc;
75
76/// Returns the current wall-clock Unix timestamp in seconds.
77fn wall_clock_unix_secs() -> i64 {
78    SystemTime::now()
79        .duration_since(UNIX_EPOCH)
80        .unwrap_or_default()
81        .as_secs() as i64
82}
83
84/// Initial scheduler cursor for `start_scheduled_loop`: exactly one interval
85/// before `start_secs` so the first due scheduled time is `start_secs` itself.
86///
87/// Fallible: a `start_secs - interval_secs` difference that does not fit in
88/// `i64` is an explicit error instead of a saturated cursor that would make
89/// the first due scheduled time `start_secs + interval_secs` and silently skip
90/// the `start_secs` tick.
91fn initial_schedule_cursor(start_secs: i64, interval_secs: i64) -> Result<i64, Error> {
92    let cursor = i128::from(start_secs) - i128::from(interval_secs);
93    i64::try_from(cursor).map_err(|_| {
94        UnexpectedSnafu {
95            reason: format!(
96                "Cannot compute the initial eval schedule cursor one interval before start {start_secs} (interval={interval_secs}): {cursor} does not fit in i64"
97            ),
98        }
99        .build()
100    })
101}
102
103/// Whole seconds to sleep until the next scheduled time `next`, measured from
104/// the current wall clock `wall_now_secs`.
105///
106/// Fallible: `next` must be strictly after `wall_now_secs` and the difference
107/// must fit in `u64`. In practice `i64::MAX - i64::MIN` is exactly `u64::MAX`,
108/// so the difference always fits once `next > wall_now_secs`; the explicit
109/// error keeps the scheduled loop panic-free and wrap-free regardless.
110fn sleep_delta_secs(next: i64, wall_now_secs: i64) -> Result<u64, Error> {
111    let delta = i128::from(next) - i128::from(wall_now_secs);
112    u64::try_from(delta).map_err(|_| {
113        UnexpectedSnafu {
114            reason: format!(
115                "Cannot sleep until the next scheduled time {next}: the delta from wall clock {wall_now_secs} is {delta} seconds, which does not fit in u64"
116            ),
117        }
118        .build()
119    })
120}
121
122/// Scheduled time in seconds converted to milliseconds for the
123/// `FLOW_SCHEDULED_TIME_MILLIS` extension.
124///
125/// Fallible: a seconds value whose millisecond product does not fit in `i64`
126/// is an explicit error instead of a saturated `i64::MAX` that would silently
127/// misrepresent the logical scheduled time.
128fn scheduled_time_millis(scheduled_time_secs: i64) -> Result<i64, Error> {
129    scheduled_time_secs.checked_mul(1000).ok_or_else(|| {
130        UnexpectedSnafu {
131            reason: format!(
132                "Cannot convert scheduled time {scheduled_time_secs}s to milliseconds: the product exceeds i64"
133            ),
134        }
135        .build()
136    })
137}
138
139/// The task's config, immutable once created
140#[derive(Clone)]
141pub struct TaskConfig {
142    pub flow_id: FlowId,
143    pub query: String,
144    /// output schema of the query
145    pub output_schema: DFSchemaRef,
146    pub time_window_expr: Option<TimeWindowExpr>,
147    /// in seconds
148    pub expire_after: Option<i64>,
149    pub sink_table_name: [String; 3],
150    pub source_table_names: HashSet<[String; 3]>,
151    pub catalog_manager: CatalogManagerRef,
152    pub query_type: QueryType,
153    pub batch_opts: Arc<BatchingModeOptions>,
154    pub exact_sequence_range_required: bool,
155    pub flow_eval_interval: Option<Duration>,
156    /// Typed schedule configuration, pre-parsed at task creation time.
157    pub eval_schedule: Option<EvalSchedule>,
158}
159
160fn determine_query_type(query: &str, query_ctx: &QueryContextRef) -> Result<QueryType, Error> {
161    let is_tql = is_tql(query_ctx.sql_dialect(), query)
162        .map_err(BoxedError::new)
163        .context(ExternalSnafu)?;
164    Ok(if is_tql {
165        QueryType::Tql
166    } else {
167        QueryType::Sql
168    })
169}
170
171fn is_merge_mode_last_non_null(options: &HashMap<String, String>) -> bool {
172    options
173        .get(MERGE_MODE_KEY)
174        .map(|mode| mode.eq_ignore_ascii_case("last_non_null"))
175        .unwrap_or(false)
176}
177
178fn encode_insert_plan_request(
179    insert_to: TableName,
180    insert_input_plan: &LogicalPlan,
181) -> Result<api::v1::QueryRequest, Error> {
182    let message = DFLogicalSubstraitConvertor {}
183        .encode(insert_input_plan, DefaultSerializer)
184        .context(SubstraitEncodeLogicalPlanSnafu)?;
185    Ok(api::v1::QueryRequest {
186        query: Some(api::v1::query_request::Query::InsertIntoPlan(
187            api::v1::InsertIntoPlan {
188                table_name: Some(insert_to),
189                logical_plan: message.to_vec(),
190            },
191        )),
192    })
193}
194
195fn format_insert_target_columns(plan: &LogicalPlan) -> String {
196    plan.schema()
197        .fields()
198        .iter()
199        .map(|field| quote_identifier(field.name()).to_string())
200        .collect::<Vec<_>>()
201        .join(", ")
202}
203
204/// Owns a whole serialized execution round. It may be moved to an execution
205/// collaborator, so cancellation of the caller cannot release the round early.
206pub struct BatchingExecutionGuard {
207    _lock: OwnedMutexGuard<()>,
208    restore: Option<(Arc<RwLock<TaskState>>, QueryContextRef)>,
209}
210
211impl BatchingExecutionGuard {
212    fn new(lock: OwnedMutexGuard<()>) -> Self {
213        Self {
214            _lock: lock,
215            restore: None,
216        }
217    }
218
219    fn restore_query_context(&mut self, state: Arc<RwLock<TaskState>>, old_ctx: QueryContextRef) {
220        self.restore = Some((state, old_ctx));
221    }
222}
223
224impl Drop for BatchingExecutionGuard {
225    fn drop(&mut self) {
226        if let Some((state, old_ctx)) = self.restore.take()
227            && let Ok(mut state) = state.write()
228        {
229            state.query_ctx = old_ctx;
230        }
231    }
232}
233
234#[derive(Clone)]
235pub struct BatchingTask {
236    pub config: Arc<TaskConfig>,
237    pub state: Arc<RwLock<TaskState>>,
238    /// Serializes plan generation, execution, checkpoint advancement, and dirty
239    /// window restoration for this flow. Without this, a manual flush and the
240    /// background loop can process the same checkpoint range concurrently.
241    execution_lock: Arc<Mutex<()>>,
242    execution: Option<Arc<dyn crate::BatchingExecution>>,
243}
244
245/// Arguments for creating batching task
246pub struct TaskArgs<'a> {
247    pub flow_id: FlowId,
248    pub query: &'a str,
249    pub plan: LogicalPlan,
250    pub time_window_expr: Option<TimeWindowExpr>,
251    pub expire_after: Option<i64>,
252    pub sink_table_name: [String; 3],
253    pub source_table_names: Vec<[String; 3]>,
254    pub query_ctx: QueryContextRef,
255    pub catalog_manager: CatalogManagerRef,
256    pub shutdown_rx: oneshot::Receiver<()>,
257    pub batch_opts: Arc<BatchingModeOptions>,
258    pub flow_eval_interval: Option<Duration>,
259    /// Typed schedule configuration pre-parsed from `CreateFlowArgs`.
260    pub eval_schedule: Option<EvalSchedule>,
261}
262
263pub struct PlanInfo {
264    pub plan: LogicalPlan,
265    pub dirty_restore: DirtyRestore,
266    pub coverage: QueryCoverage,
267}
268
269#[derive(Clone)]
270pub enum QueryCoverage {
271    /// Explicit full-query snapshot coverage, e.g. TQL or evaluation-interval
272    /// SQL flows whose plan shape cannot be safely dirty-window pruned. This
273    /// must not be used as an implicit recovery path for scoped repair or an
274    /// unsafe incremental rewrite fallback.
275    UnfilteredFull,
276    /// Scoped full-snapshot repair over the current dirty windows. A successful
277    /// result may start a fenced repair if new dirty windows appeared meanwhile.
278    ScopedBaseRepair,
279    /// A chunk of windows being repaired under the frozen high-watermark `H`.
280    /// The `high` map is sent as snapshot read bounds and must be matched by
281    /// the returned terminal watermarks before checkpoints can advance.
282    FencedRepairChunk { high: BTreeMap<u64, u64> },
283    /// Incremental delta query over `(checkpoint, scan-open snapshot]`.
284    IncrementalDelta,
285}
286
287impl QueryCoverage {
288    /// Whether this query should use incremental scan extensions and
289    /// incremental checkpoint advancement rules.
290    fn is_incremental_delta(&self) -> bool {
291        matches!(self, Self::IncrementalDelta)
292    }
293
294    /// Snapshot upper bounds requested from the storage layer. Only fenced
295    /// repair chunks carry bounds; all other coverage relies on normal scans.
296    fn snapshot_seqs(&self) -> HashMap<u64, u64> {
297        match self {
298            Self::FencedRepairChunk { high } => high.iter().map(|(k, v)| (*k, *v)).collect(),
299            _ => HashMap::new(),
300        }
301    }
302}
303
304pub enum DirtyRestore {
305    /// The query was scoped to dirty time ranges; restore those ranges if the
306    /// run fails.
307    Scoped(FilterExprInfo),
308    /// The query could not be scoped to dirty time ranges, so the dirty-window
309    /// state is only a dirty signal. Restore the consumed signal if the full
310    /// run fails.
311    ///
312    /// TODO(discord9): Full-query runs only need a dirty bool flag. Refactor
313    /// the unscoped path to stop reusing `DirtyTimeWindows` for this signal.
314    Unscoped(DirtyTimeWindows),
315}
316
317pub struct ExecuteOnceOutcome {
318    pub new_query: Option<PlanInfo>,
319    /// Execution result of the generated insert plan.
320    ///
321    /// `Ok(Some((affected_rows, elapsed)))` means a query was executed.
322    /// `Ok(None)` means no query was generated because there was no dirty signal.
323    /// `Err(_)` means plan generation or execution failed.
324    pub result: Result<Option<(usize, Duration)>, Error>,
325}
326
327impl BatchingTask {
328    #[allow(clippy::too_many_arguments)]
329    pub fn try_new(args: TaskArgs<'_>) -> Result<Self, Error> {
330        Self::try_new_with_exact_sequence_range_required(args, false)
331    }
332
333    pub fn try_new_with_exact_sequence_range_required(
334        TaskArgs {
335            flow_id,
336            query,
337            plan,
338            time_window_expr,
339            expire_after,
340            sink_table_name,
341            source_table_names,
342            query_ctx,
343            catalog_manager,
344            shutdown_rx,
345            batch_opts,
346            flow_eval_interval,
347            eval_schedule,
348        }: TaskArgs<'_>,
349        exact_sequence_range_required: bool,
350    ) -> Result<Self, Error> {
351        let mut state = TaskState::with_dirty_time_windows(
352            query_ctx.clone(),
353            shutdown_rx,
354            DirtyTimeWindows::new(
355                batch_opts.experimental_max_filter_num_per_query,
356                batch_opts.experimental_time_window_merge_threshold,
357            ),
358        );
359        if !batch_opts.experimental_enable_incremental_read {
360            state.disable_incremental();
361        }
362
363        Ok(Self {
364            config: Arc::new(TaskConfig {
365                flow_id,
366                query: query.to_string(),
367                time_window_expr,
368                expire_after,
369                sink_table_name,
370                source_table_names: source_table_names.into_iter().collect(),
371                catalog_manager,
372                output_schema: plan.schema().clone(),
373                query_type: determine_query_type(query, &query_ctx)?,
374                exact_sequence_range_required,
375                batch_opts,
376                flow_eval_interval,
377                eval_schedule,
378            }),
379            state: Arc::new(RwLock::new(state)),
380            execution_lock: Arc::new(Mutex::new(())),
381            execution: None,
382        })
383    }
384
385    pub(crate) fn with_execution(
386        mut self,
387        execution: Option<Arc<dyn crate::BatchingExecution>>,
388    ) -> Self {
389        self.execution = execution;
390        self
391    }
392
393    pub(crate) fn stop_execution(&self) {
394        if let Some(execution) = &self.execution {
395            execution.stop();
396        }
397    }
398
399    pub fn last_execution_time_millis(&self) -> Option<i64> {
400        self.state.read().unwrap().last_execution_time_millis()
401    }
402
403    pub fn start_time_millis(&self) -> Option<i64> {
404        self.state.read().unwrap().start_time_millis()
405    }
406
407    /// Collect flow-related extensions from the task's query context that should be
408    /// forwarded to the frontend (e.g. scheduled time).
409    fn frontend_extensions(&self) -> HashMap<String, String> {
410        let ctx = self.state.read().unwrap();
411        let all = ctx.query_ctx.extensions();
412        let mut flow_exts = HashMap::new();
413        // Propagate the scheduled time extension if present so that frontend
414        // execution can use the same logical time.
415        if let Some(v) = all.get(query::options::FLOW_SCHEDULED_TIME_MILLIS) {
416            flow_exts.insert(
417                query::options::FLOW_SCHEDULED_TIME_MILLIS.to_string(),
418                v.clone(),
419            );
420        }
421        flow_exts
422    }
423
424    /// mark time window range (now - expire_after, now) as dirty (or (0, now) if expire_after not set)
425    ///
426    /// useful for flush_flow to flush dirty time windows range
427    pub fn mark_all_windows_as_dirty(&self) -> Result<(), Error> {
428        let now = SystemTime::now();
429        let now = Timestamp::new_second(
430            now.duration_since(UNIX_EPOCH)
431                .expect("Time went backwards")
432                .as_secs() as _,
433        );
434        let lower_bound = self
435            .config
436            .expire_after
437            .map(|e| now.sub_duration(Duration::from_secs(e as _)))
438            .transpose()
439            .map_err(BoxedError::new)
440            .context(ExternalSnafu)?
441            .unwrap_or(Timestamp::new_second(0));
442        debug!(
443            "Flow {} mark range ({:?}, {:?}) as dirty",
444            self.config.flow_id, lower_bound, now
445        );
446        self.state
447            .write()
448            .unwrap()
449            .dirty_time_windows
450            .add_window(lower_bound, Some(now));
451        Ok(())
452    }
453
454    /// Create the sink table if needed, then return it.
455    pub async fn check_or_create_sink_table(
456        &self,
457        engine: &QueryEngineRef,
458        frontend_client: &Arc<FrontendClient>,
459    ) -> Result<TableRef, Error> {
460        if !self.is_table_exist(&self.config.sink_table_name).await? {
461            let create_table = self.gen_create_table_expr(engine.clone()).await?;
462            info!(
463                "Try creating sink table(if not exists) with expr: {:?}",
464                create_table
465            );
466            self.create_table(frontend_client, create_table).await?;
467            info!(
468                "Sink table {}(if not exists) created",
469                self.config.sink_table_name.join(".")
470            );
471        }
472        let (table, _) = get_table_info_df_schema(
473            self.config.catalog_manager.clone(),
474            self.config.sink_table_name.clone(),
475        )
476        .await?;
477        Ok(table)
478    }
479
480    /// Returns a forked snapshot of the task query context for extension-owned planning.
481    pub fn query_context_snapshot(&self) -> QueryContextRef {
482        let query_ctx = self.state.read().unwrap().query_ctx.clone();
483        Arc::new(query_ctx.fork())
484    }
485
486    /// Validates that the sink table schema can accept this flow's ordinary output.
487    pub async fn validate_sink_table_schema(&self, engine: &QueryEngineRef) -> Result<(), Error> {
488        self.validate_sink_table_schema_with_values(engine, &BTreeMap::new())
489            .await
490    }
491
492    /// Validates the flow output plus extension-owned typed sink values without consuming dirty work.
493    pub async fn validate_sink_table_schema_with_values(
494        &self,
495        engine: &QueryEngineRef,
496        values: &BTreeMap<String, ScalarValue>,
497    ) -> Result<(), Error> {
498        let (table, _) = get_table_info_df_schema(
499            self.config.catalog_manager.clone(),
500            self.config.sink_table_name.clone(),
501        )
502        .await?;
503
504        let table_meta = &table.table_info().meta;
505        let merge_mode_last_non_null =
506            is_merge_mode_last_non_null(&table_meta.options.extra_options);
507        let primary_key_indices = table_meta.primary_key_indices.clone();
508
509        gen_plan_with_matching_schema_and_values(
510            &self.config.query,
511            self.query_context_snapshot(),
512            engine.clone(),
513            table_meta.schema.clone(),
514            &primary_key_indices,
515            merge_mode_last_non_null,
516            values,
517        )
518        .await
519        .map(|_| ())
520    }
521
522    async fn is_table_exist(&self, table_name: &[String; 3]) -> Result<bool, Error> {
523        self.config
524            .catalog_manager
525            .table_exists(&table_name[0], &table_name[1], &table_name[2], None)
526            .await
527            .map_err(BoxedError::new)
528            .context(ExternalSnafu)
529    }
530
531    pub(crate) async fn execute_once_serialized(
532        &self,
533        engine: &QueryEngineRef,
534        frontend_client: &Arc<FrontendClient>,
535        max_window_cnt: Option<usize>,
536    ) -> Result<Option<(usize, Duration)>, Error> {
537        let outcome = self
538            .execute_once_serialized_with_outcome(engine, frontend_client, max_window_cnt)
539            .await;
540        outcome.result
541    }
542
543    /// Executes one flow evaluation under `execution_lock` and keeps the
544    /// generated query context for the background loop's error logging/backoff.
545    async fn execute_once_serialized_with_outcome(
546        &self,
547        engine: &QueryEngineRef,
548        frontend_client: &Arc<FrontendClient>,
549        max_window_cnt: Option<usize>,
550    ) -> ExecuteOnceOutcome {
551        let guard = BatchingExecutionGuard::new(self.execution_lock.clone().lock_owned().await);
552        self.execute_once_with_guard(guard, engine, frontend_client, max_window_cnt)
553            .await
554    }
555
556    async fn execute_once_with_guard(
557        &self,
558        guard: BatchingExecutionGuard,
559        engine: &QueryEngineRef,
560        frontend_client: &Arc<FrontendClient>,
561        max_window_cnt: Option<usize>,
562    ) -> ExecuteOnceOutcome {
563        if let Some(execution) = &self.execution {
564            return execution
565                .clone()
566                .execute_once(guard, self, engine, frontend_client, max_window_cnt)
567                .await;
568        }
569        self.execute_once_default_unlocked(engine, frontend_client, max_window_cnt)
570            .await
571    }
572
573    /// Executes the default flow evaluation. Caller owns the execution guard.
574    async fn execute_once_default_unlocked(
575        &self,
576        engine: &QueryEngineRef,
577        frontend_client: &Arc<FrontendClient>,
578        max_window_cnt: Option<usize>,
579    ) -> ExecuteOnceOutcome {
580        let new_query = match self.gen_insert_plan_unlocked(engine, max_window_cnt).await {
581            Ok(new_query) => new_query,
582            Err(err) => {
583                return ExecuteOnceOutcome {
584                    new_query: None,
585                    result: Err(err),
586                };
587            }
588        };
589
590        if let Some(new_query) = new_query {
591            debug!("Generate new query: {}", new_query.plan);
592            let res = self
593                .execute_logical_plan_unlocked(
594                    engine,
595                    frontend_client,
596                    &new_query.plan,
597                    &new_query.dirty_restore,
598                    &new_query.coverage,
599                )
600                .await;
601            if res.is_err() {
602                self.handle_executed_query_failure(Some(&new_query));
603            }
604            ExecuteOnceOutcome {
605                new_query: Some(new_query),
606                result: res,
607            }
608        } else {
609            debug!("Generate no query");
610            ExecuteOnceOutcome {
611                new_query: None,
612                result: Ok(None),
613            }
614        }
615    }
616
617    /// Generates the ordinary insert plan. Caller must reach this through the serialized path.
618    async fn gen_insert_plan_unlocked(
619        &self,
620        engine: &QueryEngineRef,
621        max_window_cnt: Option<usize>,
622    ) -> Result<Option<PlanInfo>, Error> {
623        self.gen_insert_plan_with_values_unlocked(engine, max_window_cnt, &BTreeMap::new(), false)
624            .await
625    }
626
627    /// Generates the insert plan with extension-owned typed sink values. Caller must own the
628    /// serialized execution guard.
629    pub async fn gen_insert_plan_with_values_unlocked(
630        &self,
631        engine: &QueryEngineRef,
632        max_window_cnt: Option<usize>,
633        values: &BTreeMap<String, ScalarValue>,
634        force_full_snapshot: bool,
635    ) -> Result<Option<PlanInfo>, Error> {
636        let (table, df_schema) = get_table_info_df_schema(
637            self.config.catalog_manager.clone(),
638            self.config.sink_table_name.clone(),
639        )
640        .await?;
641
642        let table_meta = &table.table_info().meta;
643        let merge_mode_last_non_null =
644            is_merge_mode_last_non_null(&table_meta.options.extra_options);
645        let primary_key_indices = table_meta.primary_key_indices.clone();
646
647        let new_query = self
648            .gen_query_with_time_window_with_values(
649                engine.clone(),
650                &table.table_info().meta.schema,
651                &primary_key_indices,
652                merge_mode_last_non_null,
653                max_window_cnt,
654                values,
655                force_full_snapshot,
656            )
657            .await?;
658
659        let Some(new_query) = new_query else {
660            return Ok(None);
661        };
662
663        // first check if all columns in input query exists in sink table
664        // since insert into ref to names in record batch generate by given query
665        let table_columns = df_schema
666            .columns()
667            .into_iter()
668            .map(|c| c.name)
669            .collect::<BTreeSet<_>>();
670        for column in new_query.plan.schema().columns() {
671            if !table_columns.contains(column.name()) {
672                self.restore_dirty_windows_after_failure(&new_query);
673                return InvalidQuerySnafu {
674                    reason: format!(
675                        "Column {} not found in sink table with columns {:?}",
676                        column, table_columns
677                    ),
678                }
679                .fail();
680            }
681        }
682
683        let table_provider = Arc::new(DfTableProviderAdapter::new(table));
684        let table_source = Arc::new(DefaultTableSource::new(table_provider));
685
686        // update_at& time index placeholder (if exists) should have default value
687        let plan = LogicalPlan::Dml(DmlStatement::new(
688            datafusion_common::TableReference::Full {
689                catalog: self.config.sink_table_name[0].clone().into(),
690                schema: self.config.sink_table_name[1].clone().into(),
691                table: self.config.sink_table_name[2].clone().into(),
692            },
693            table_source,
694            WriteOp::Insert(datafusion_expr::dml::InsertOp::Append),
695            Arc::new(new_query.plan.clone()),
696        ));
697        let insert_into_info = PlanInfo {
698            plan,
699            dirty_restore: new_query.dirty_restore,
700            coverage: new_query.coverage,
701        };
702        let insert_into =
703            match insert_into_info
704                .plan
705                .clone()
706                .recompute_schema()
707                .context(DatafusionSnafu {
708                    context: "Failed to recompute schema",
709                }) {
710                Ok(insert_into) => insert_into,
711                Err(err) => {
712                    self.restore_dirty_windows_after_failure(&insert_into_info);
713                    return Err(err);
714                }
715            };
716
717        Ok(Some(PlanInfo {
718            plan: insert_into,
719            dirty_restore: insert_into_info.dirty_restore,
720            coverage: insert_into_info.coverage,
721        }))
722    }
723
724    pub async fn create_table(
725        &self,
726        frontend_client: &Arc<FrontendClient>,
727        expr: CreateTableExpr,
728    ) -> Result<(), Error> {
729        let catalog = &self.config.sink_table_name[0];
730        let schema = &self.config.sink_table_name[1];
731        frontend_client
732            .create(expr.clone(), catalog, schema)
733            .await?;
734        Ok(())
735    }
736
737    /// Applies checkpoint state updates around the raw insert-plan execution.
738    async fn execute_logical_plan_unlocked(
739        &self,
740        engine: &QueryEngineRef,
741        frontend_client: &Arc<FrontendClient>,
742        plan: &LogicalPlan,
743        dirty_restore: &DirtyRestore,
744        coverage: &QueryCoverage,
745    ) -> Result<Option<(usize, Duration)>, Error> {
746        let flow_id = self.config.flow_id;
747        let Some((res, elapsed)) = self
748            .execute_plan_unlocked(engine, frontend_client, plan, dirty_restore, coverage)
749            .await?
750        else {
751            return Ok(None);
752        };
753
754        if let Err(err) = &res {
755            let decision = {
756                let mut state = self.state.write().unwrap();
757                let reason = Self::query_failure_reason(err, coverage);
758                Self::apply_query_failure_to_state(&mut state, elapsed, coverage, reason)
759            };
760            if let Some(decision) = decision {
761                Self::record_checkpoint_decision(flow_id, decision);
762            }
763        }
764
765        let res = res?;
766        let (affected_rows, _) = res.output.extract_rows_and_cost();
767        let decision = {
768            let mut state = self.state.write().unwrap();
769            Self::apply_query_result_to_state(&mut state, &res, elapsed, coverage)
770        };
771        Self::record_checkpoint_decision(flow_id, decision);
772        Ok(Some((affected_rows, elapsed)))
773    }
774
775    /// Executes a fully constructed insert plan without checkpoint-state mutation.
776    /// Callers must own the serialized execution guard. Its unsafe-incremental safety fallback
777    /// still restores the supplied dirty work before returning `Ok(None)`.
778    pub async fn execute_plan_unlocked(
779        &self,
780        engine: &QueryEngineRef,
781        frontend_client: &Arc<FrontendClient>,
782        plan: &LogicalPlan,
783        dirty_restore: &DirtyRestore,
784        coverage: &QueryCoverage,
785    ) -> Result<Option<(Result<OutputWithMetrics, Error>, Duration)>, Error> {
786        let instant = Instant::now();
787        let flow_id = self.config.flow_id;
788
789        debug!(
790            "Executing flow {flow_id}(expire_after={:?} secs) with query {}",
791            self.config.expire_after, &plan
792        );
793
794        let catalog = &self.config.sink_table_name[0];
795        let schema = &self.config.sink_table_name[1];
796
797        // fix all table ref by make it fully qualified, i.e. "table_name" => "catalog_name.schema_name.table_name"
798        let plan = plan
799            .clone()
800            .transform_down_with_subqueries(|p| {
801                if let LogicalPlan::TableScan(mut table_scan) = p {
802                    let resolved = table_scan.table_name.resolve(catalog, schema);
803                    table_scan.table_name = resolved.into();
804                    Ok(Transformed::yes(LogicalPlan::TableScan(table_scan)))
805                } else {
806                    Ok(Transformed::no(p))
807                }
808            })
809            .with_context(|_| DatafusionSnafu {
810                context: format!("Failed to fix table ref in logical plan, plan={:?}", plan),
811            })?
812            .data;
813
814        // For incremental-mode SQL queries, attempt to rewrite the delta aggregate
815        // plan into a safe delta-LEFT-JOIN-sink form before deciding on extensions.
816        let incremental_plan = if coverage.is_incremental_delta() {
817            self.prepare_plan_for_incremental(engine, &plan).await?
818        } else {
819            None
820        };
821        let incremental_safe = incremental_plan.is_some();
822        if coverage.is_incremental_delta() && !incremental_safe {
823            debug!(
824                "Flow {flow_id} skipped unsafe incremental delta fallback; \
825                 restored dirty signal instead of executing an unfiltered full snapshot"
826            );
827            self.restore_dirty_windows(dirty_restore);
828            return Ok(None);
829        }
830        let plan = incremental_plan.unwrap_or_else(|| plan.clone());
831
832        let extensions = self
833            .build_flow_query_extensions(incremental_safe, coverage.is_incremental_delta())
834            .await?;
835        let frontend_extensions = self.frontend_extensions();
836        let extension_refs = extensions
837            .iter()
838            .map(|(key, value)| (*key, value.as_str()))
839            .chain(
840                frontend_extensions
841                    .iter()
842                    .map(|(key, value)| (key.as_str(), value.as_str())),
843            )
844            .collect::<Vec<_>>();
845        let query_mode = if extensions
846            .iter()
847            .any(|(key, _)| *key == FLOW_INCREMENTAL_MODE)
848        {
849            CheckpointMode::Incremental
850        } else {
851            CheckpointMode::FullSnapshot
852        };
853        Self::record_query_mode(flow_id, query_mode);
854        debug!(
855            "Flow {flow_id} executing batching query with checkpoint_mode={}, extension_count={}",
856            checkpoint_mode_label(query_mode),
857            extensions.len()
858        );
859
860        let mut peer_desc = None;
861        let res = {
862            let _timer = METRIC_FLOW_BATCHING_ENGINE_QUERY_TIME
863                .with_label_values(&[flow_id.to_string().as_str()])
864                .start_timer();
865
866            let req = if let Some((insert_to, insert_input_plan)) =
867                breakup_insert_plan(&plan, catalog, schema)
868            {
869                if query_mode == CheckpointMode::FullSnapshot
870                    && matches!(self.config.query_type, QueryType::Sql)
871                    && self.config.flow_eval_interval.is_some()
872                    && self.config.time_window_expr.is_none()
873                {
874                    // Evaluation-interval SQL flows without a time-window
875                    // expression execute as full-query snapshots. Send these
876                    // as SQL text instead of Substrait to avoid logical-plan
877                    // round-trip issues around complex joins/unions/CTEs and
878                    // duplicate field aliases. Keep ordinary SQL full snapshots
879                    // on the existing InsertIntoPlan path because SQL unparsing
880                    // is not valid for every planned aggregate shape yet.
881                    // If the local SQL unparser does not support this plan,
882                    // keep the previous InsertIntoPlan transport as a fallback.
883                    match df_plan_to_sql(&insert_input_plan) {
884                        Ok(select_sql) => {
885                            let target_columns = format_insert_target_columns(&insert_input_plan);
886                            let sql = format!(
887                                "INSERT INTO {} ({}) {}",
888                                TableReference::full(
889                                    insert_to.catalog_name.as_str(),
890                                    insert_to.schema_name.as_str(),
891                                    insert_to.table_name.as_str(),
892                                )
893                                .to_quoted_string(),
894                                target_columns,
895                                select_sql
896                            );
897                            api::v1::QueryRequest {
898                                query: Some(api::v1::query_request::Query::Sql(sql)),
899                            }
900                        }
901                        Err(err) => {
902                            debug!(
903                                "Failed to unparse full-snapshot SQL flow {} plan; \
904                                 falling back to InsertIntoPlan: {:?}",
905                                flow_id, err
906                            );
907                            encode_insert_plan_request(insert_to, &insert_input_plan)?
908                        }
909                    }
910                } else {
911                    encode_insert_plan_request(insert_to, &insert_input_plan)?
912                }
913            } else {
914                let message = DFLogicalSubstraitConvertor {}
915                    .encode(&plan, DefaultSerializer)
916                    .context(SubstraitEncodeLogicalPlanSnafu)?;
917
918                api::v1::QueryRequest {
919                    query: Some(api::v1::query_request::Query::LogicalPlan(message.to_vec())),
920                }
921            };
922
923            let snapshot_seqs = coverage.snapshot_seqs();
924            {
925                let mut state = self.state.write().unwrap();
926                state.record_start_time_if_first();
927            }
928            frontend_client
929                .query_with_terminal_metrics(
930                    catalog,
931                    schema,
932                    req,
933                    &extension_refs,
934                    &snapshot_seqs,
935                    &mut peer_desc,
936                )
937                .await
938        };
939
940        let elapsed = instant.elapsed();
941        let peer_label = peer_desc
942            .as_ref()
943            .map(ToString::to_string)
944            .unwrap_or_else(|| PeerDesc::default().to_string());
945        if let Err(err) = &res {
946            // This raw helper deliberately leaves checkpoint transitions to its caller. The
947            // default wrapper performs the single OSS transition; extension owners can persist
948            // their durable marker before updating in-memory state.
949            warn!(
950                "Failed to execute Flow {flow_id} on frontend {peer_label}, result: {err:?}, elapsed: {:?} with query: {}",
951                elapsed, &plan
952            );
953        }
954
955        // record slow query
956        if elapsed >= self.config.batch_opts.slow_query_threshold {
957            warn!(
958                "Flow {flow_id} on frontend {peer_label} executed for {:?} before complete, query: {}",
959                elapsed, &plan
960            );
961            let flow_id = flow_id.to_string();
962            METRIC_FLOW_BATCHING_ENGINE_SLOW_QUERY
963                .with_label_values(&[flow_id.as_str(), peer_label.as_str()])
964                .observe(elapsed.as_secs_f64());
965        }
966
967        match res {
968            Ok(res) => {
969                let (affected_rows, _) = res.output.extract_rows_and_cost();
970                debug!(
971                    "Flow {flow_id} executed, affected_rows: {affected_rows:?}, elapsed: {:?}, watermark: {:?}",
972                    elapsed,
973                    res.region_watermark_map()
974                );
975                METRIC_FLOW_ROWS
976                    .with_label_values(&[format!("{}-out-batching", flow_id).as_str()])
977                    .inc_by(affected_rows as _);
978                Ok(Some((Ok(res), elapsed)))
979            }
980            Err(err) => Ok(Some((Err(err), elapsed))),
981        }
982    }
983
984    /// Restore dirty windows consumed by a failed query so they are retried on
985    /// the next execution.
986    ///
987    pub fn restore_dirty_windows(&self, dirty_restore: &DirtyRestore) {
988        match dirty_restore {
989            DirtyRestore::Scoped(filter) => self.restore_scoped_dirty_windows(filter),
990            DirtyRestore::Unscoped(dirty_windows) => self
991                .state
992                .write()
993                .unwrap()
994                .dirty_time_windows
995                .add_dirty_windows(dirty_windows),
996        }
997    }
998
999    /// Restore the dirty signal for a plan that was generated but failed before
1000    /// it could prove any checkpoint advancement.
1001    fn restore_dirty_windows_after_failure(&self, query: &PlanInfo) {
1002        self.restore_dirty_windows(&query.dirty_restore);
1003    }
1004
1005    /// Restore scoped windows through `TaskState` so fenced repair can decide
1006    /// whether they go back to pending repair or live dirty state.
1007    fn restore_scoped_dirty_windows(&self, filter: &FilterExprInfo) {
1008        self.state.write().unwrap().restore_scoped_windows(filter);
1009    }
1010
1011    /// Run a fallible scoped operation and restore its consumed windows if plan
1012    /// generation/rewrite fails before execution.
1013    fn restore_scoped_dirty_windows_on_err<T>(
1014        &self,
1015        filter: &FilterExprInfo,
1016        result: Result<T, Error>,
1017    ) -> Result<T, Error> {
1018        result.inspect_err(|_| {
1019            self.restore_scoped_dirty_windows(filter);
1020        })
1021    }
1022
1023    /// Restore an unscoped dirty signal consumed by an explicit full-query or
1024    /// incremental-delta plan.
1025    fn restore_unscoped_dirty_windows(&self, dirty_windows: &DirtyTimeWindows) {
1026        self.state
1027            .write()
1028            .unwrap()
1029            .dirty_time_windows
1030            .add_dirty_windows(dirty_windows);
1031    }
1032
1033    /// Run a fallible unscoped operation and restore the dirty signal if it
1034    /// fails before a query is executed.
1035    fn restore_unscoped_dirty_windows_on_err<T>(
1036        &self,
1037        dirty_windows: &DirtyTimeWindows,
1038        result: Result<T, Error>,
1039    ) -> Result<T, Error> {
1040        result.inspect_err(|_| {
1041            self.restore_unscoped_dirty_windows(dirty_windows);
1042        })
1043    }
1044
1045    /// Consume the live dirty signal for an unscoped query while keeping a copy
1046    /// that can be restored if planning or execution fails.
1047    fn drain_dirty_windows_signal(&self) -> (bool, DirtyTimeWindows) {
1048        let mut state = self.state.write().unwrap();
1049        let dirty_windows_to_restore = state.dirty_time_windows.clone();
1050        let is_dirty = !dirty_windows_to_restore.is_empty();
1051        state.dirty_time_windows.clean();
1052        (is_dirty, dirty_windows_to_restore)
1053    }
1054
1055    #[allow(clippy::too_many_arguments)]
1056    /// Build an unfiltered plan for explicit full-query or incremental-delta
1057    /// coverage. Callers pass the consumed dirty signal for failure restoration.
1058    async fn gen_unfiltered_plan_info(
1059        &self,
1060        engine: QueryEngineRef,
1061        query_ctx: QueryContextRef,
1062        sink_table_schema: Arc<Schema>,
1063        primary_key_indices: &[usize],
1064        allow_partial: bool,
1065        values: &BTreeMap<String, ScalarValue>,
1066        dirty_windows_to_restore: DirtyTimeWindows,
1067        retention_filter: Option<(&str, Timestamp, &'static str)>,
1068        coverage: QueryCoverage,
1069    ) -> Result<PlanInfo, Error> {
1070        let mut plan = self.restore_unscoped_dirty_windows_on_err(
1071            &dirty_windows_to_restore,
1072            gen_plan_with_matching_schema_and_values(
1073                &self.config.query,
1074                query_ctx,
1075                engine,
1076                sink_table_schema,
1077                primary_key_indices,
1078                allow_partial,
1079                values,
1080            )
1081            .await,
1082        )?;
1083
1084        if let Some((col_name, lower_bound, context)) = retention_filter {
1085            let lower = self.restore_unscoped_dirty_windows_on_err(
1086                &dirty_windows_to_restore,
1087                to_df_literal(lower_bound),
1088            )?;
1089            let retention_filter = col(col_name).gt_eq(lit(lower));
1090            let mut add_filter = AddFilterRewriter::new(retention_filter);
1091            plan = self.restore_unscoped_dirty_windows_on_err(
1092                &dirty_windows_to_restore,
1093                plan.clone()
1094                    .rewrite(&mut add_filter)
1095                    .with_context(|_| DatafusionSnafu {
1096                        context: format!(
1097                            "Failed to apply {context} expire_after filter to plan:\n {}\n",
1098                            plan
1099                        ),
1100                    })
1101                    .map(|rewrite| rewrite.data),
1102            )?;
1103        }
1104
1105        Ok(PlanInfo {
1106            plan,
1107            dirty_restore: DirtyRestore::Unscoped(dirty_windows_to_restore),
1108            coverage,
1109        })
1110    }
1111
1112    #[allow(clippy::too_many_arguments)]
1113    /// Build an unfiltered plan only when the live dirty signal was present;
1114    /// otherwise skip this round without querying.
1115    async fn gen_unfiltered_plan_info_if_dirty(
1116        &self,
1117        engine: QueryEngineRef,
1118        query_ctx: QueryContextRef,
1119        sink_table_schema: Arc<Schema>,
1120        primary_key_indices: &[usize],
1121        allow_partial: bool,
1122        values: &BTreeMap<String, ScalarValue>,
1123        retention_filter: Option<(&str, Timestamp, &'static str)>,
1124        coverage: QueryCoverage,
1125    ) -> Result<Option<PlanInfo>, Error> {
1126        let (is_dirty, dirty_windows_to_restore) = self.drain_dirty_windows_signal();
1127        if !is_dirty {
1128            debug!("Flow id={:?}, no new data, not update", self.config.flow_id);
1129            return Ok(None);
1130        }
1131
1132        self.gen_unfiltered_plan_info(
1133            engine,
1134            query_ctx,
1135            sink_table_schema,
1136            primary_key_indices,
1137            allow_partial,
1138            values,
1139            dirty_windows_to_restore,
1140            retention_filter,
1141            coverage,
1142        )
1143        .await
1144        .map(Some)
1145    }
1146
1147    fn handle_executed_query_failure(&self, query: Option<&PlanInfo>) {
1148        if let Some(query) = query {
1149            self.restore_dirty_windows_after_failure(query);
1150        }
1151    }
1152
1153    /// start executing query in a loop, break when receive shutdown signal
1154    ///
1155    /// any error will be logged when executing query.
1156    ///
1157    /// Dispatches to:
1158    /// - scheduled loop when `flow_eval_interval.is_some()`
1159    /// - adaptive dirty-window loop otherwise
1160    pub async fn start_executing_loop(
1161        &self,
1162        engine: QueryEngineRef,
1163        frontend_client: Arc<FrontendClient>,
1164    ) {
1165        if self.config.flow_eval_interval.is_some() {
1166            self.start_scheduled_loop(engine, frontend_client).await;
1167        } else {
1168            self.start_adaptive_loop(engine, frontend_client).await;
1169        }
1170    }
1171
1172    /// Scheduled batching loop for flows with `EVAL INTERVAL`.
1173    ///
1174    /// Uses the pre-parsed `EvalSchedule` from `TaskConfig` and selects due
1175    /// scheduled times using bounded catch-up semantics. Each scheduled time is the
1176    /// scheduled evaluation time used as logical `now()` for that attempt.
1177    /// Each attempt temporarily sets `flow.scheduled_time_millis` on the
1178    /// task's `QueryContext` and executes under the existing `execution_lock`.
1179    /// After every attempt (success, no-op, or failure) the in-memory
1180    /// cursor advances.
1181    async fn start_scheduled_loop(
1182        &self,
1183        engine: QueryEngineRef,
1184        frontend_client: Arc<FrontendClient>,
1185    ) {
1186        let flow_id_str = self.config.flow_id.to_string();
1187
1188        let schedule = match &self.config.eval_schedule {
1189            Some(s) => s.clone(),
1190            None => {
1191                let eval_interval_secs = self
1192                    .config
1193                    .flow_eval_interval
1194                    .map(|d| d.as_secs() as i64)
1195                    .expect("checked by caller");
1196
1197                // Fallback: no typed config provided. Compute defaults
1198                // anchored at epoch/start=0.
1199                match EvalSchedule::from_config(Some(eval_interval_secs), None) {
1200                    Ok(Some(s)) => s,
1201                    Ok(None) => {
1202                        warn!(
1203                            "Flow {}: EVAL INTERVAL set but no schedule parsed; exiting loop",
1204                            flow_id_str
1205                        );
1206                        return;
1207                    }
1208                    Err(e) => {
1209                        warn!(
1210                            "Flow {}: Failed to parse eval schedule: {}; exiting loop",
1211                            flow_id_str, e
1212                        );
1213                        return;
1214                    }
1215                }
1216            }
1217        };
1218
1219        // Initial cursor is one interval before start so the first due
1220        // scheduled time is `start_secs`. An unrepresentable difference is an
1221        // explicit error, never a saturated cursor that would silently skip
1222        // the first scheduled tick.
1223        let mut cursor_secs =
1224            match initial_schedule_cursor(schedule.start_secs, schedule.interval_secs) {
1225                Ok(cursor) => cursor,
1226                Err(e) => {
1227                    warn!(
1228                        "Flow {}: invalid eval schedule, exiting loop: {}",
1229                        flow_id_str, e
1230                    );
1231                    return;
1232                }
1233            };
1234
1235        info!(
1236            "Flow {}: entering scheduled loop, interval={}s, start={}, anchor={}, policy={:?}, max_runs={}, max_lag={}s",
1237            flow_id_str,
1238            schedule.interval_secs,
1239            schedule.start_secs,
1240            schedule.anchor_secs,
1241            schedule.missed_tick_policy,
1242            schedule.max_runs,
1243            schedule.max_lag_secs,
1244        );
1245
1246        loop {
1247            if self.is_shutdown_signaled() {
1248                break;
1249            }
1250
1251            let wall_now_secs = wall_clock_unix_secs();
1252
1253            let due = match select_due_scheduled_times(&schedule, cursor_secs, wall_now_secs) {
1254                Ok(d) => d,
1255                Err(e) => {
1256                    warn!(
1257                        "Flow {}: invalid eval schedule, exiting loop: {}",
1258                        flow_id_str, e
1259                    );
1260                    return;
1261                }
1262            };
1263
1264            if due.scheduled_times_secs.is_empty() {
1265                if due.skipped > 0 {
1266                    warn!(
1267                        "Flow {}: all {} due scheduled times skipped by max-lag, advancing cursor to wall-clock ({wall_now_secs}) to avoid re-skipping",
1268                        flow_id_str, due.skipped
1269                    );
1270                    cursor_secs = wall_now_secs;
1271                    continue;
1272                }
1273
1274                // No due yet — sleep until the next scheduled time.
1275                let next = match schedule.next_scheduled_time_after(cursor_secs) {
1276                    Ok(next) => next,
1277                    Err(e) => {
1278                        warn!(
1279                            "Flow {}: cannot advance eval schedule past cursor {cursor_secs}: {e}; exiting loop",
1280                            flow_id_str
1281                        );
1282                        return;
1283                    }
1284                };
1285                if next <= wall_now_secs {
1286                    // Shouldn't happen given select_due_scheduled_times returned empty,
1287                    // but guard against clock skew / logic error.
1288                    cursor_secs = wall_now_secs;
1289                    continue;
1290                }
1291                let wait_secs = match sleep_delta_secs(next, wall_now_secs) {
1292                    Ok(wait_secs) => wait_secs,
1293                    Err(e) => {
1294                        warn!(
1295                            "Flow {}: cannot sleep until next scheduled time {}: {e}; exiting loop",
1296                            flow_id_str, next
1297                        );
1298                        return;
1299                    }
1300                };
1301                let wait_dur = Duration::from_secs(wait_secs);
1302                debug!(
1303                    "Flow {}: no due scheduled times, sleeping for {}s until next scheduled time at {}",
1304                    flow_id_str, wait_secs, next
1305                );
1306                tokio::time::sleep(wait_dur).await;
1307                continue;
1308            }
1309
1310            if due.skipped > 0 {
1311                info!(
1312                    "Flow {}: {} due scheduled times, {} skipped (catch-up)",
1313                    flow_id_str,
1314                    due.scheduled_times_secs.len(),
1315                    due.skipped
1316                );
1317            }
1318
1319            // Execute scheduled times oldest → newest.
1320            for scheduled_time_secs in &due.scheduled_times_secs {
1321                if self.is_shutdown_signaled() {
1322                    break;
1323                }
1324
1325                METRIC_FLOW_BATCHING_ENGINE_START_QUERY_CNT
1326                    .with_label_values(&[&flow_id_str])
1327                    .inc();
1328
1329                let outcome = self
1330                    .execute_once_serialized_at_scheduled_time(
1331                        &engine,
1332                        &frontend_client,
1333                        *scheduled_time_secs,
1334                    )
1335                    .await;
1336
1337                // Advance cursor regardless of outcome.
1338                cursor_secs = *scheduled_time_secs;
1339
1340                match outcome.result {
1341                    Ok(Some((rows, elapsed))) => {
1342                        debug!(
1343                            "Flow {}: scheduled time {} completed, rows={}, elapsed={:?}",
1344                            flow_id_str, scheduled_time_secs, rows, elapsed
1345                        );
1346                    }
1347                    Ok(None) => {
1348                        debug!(
1349                            "Flow {}: scheduled time {} produced no query (no dirty signal or no-op)",
1350                            flow_id_str, scheduled_time_secs
1351                        );
1352                    }
1353                    Err(err) => {
1354                        warn!(
1355                            "Flow {}: scheduled time {} failed: {:?}",
1356                            flow_id_str, scheduled_time_secs, err
1357                        );
1358                        METRIC_FLOW_BATCHING_ENGINE_ERROR_CNT
1359                            .with_label_values(&[&flow_id_str])
1360                            .inc();
1361                        // Dirty-window restoration is handled by the
1362                        // existing `handle_executed_query_failure` inside
1363                        // default execution.
1364                    }
1365                }
1366            }
1367        }
1368    }
1369
1370    /// Existing adaptive dirty-window loop for flows without `EVAL INTERVAL`.
1371    async fn start_adaptive_loop(
1372        &self,
1373        engine: QueryEngineRef,
1374        frontend_client: Arc<FrontendClient>,
1375    ) {
1376        let flow_id_str = self.config.flow_id.to_string();
1377        let mut max_window_cnt = None;
1378        loop {
1379            if self.is_shutdown_signaled() {
1380                break;
1381            }
1382            METRIC_FLOW_BATCHING_ENGINE_START_QUERY_CNT
1383                .with_label_values(&[&flow_id_str])
1384                .inc();
1385
1386            let min_refresh = self.config.batch_opts.experimental_min_refresh_duration;
1387
1388            let outcome = self
1389                .execute_once_serialized_with_outcome(&engine, &frontend_client, max_window_cnt)
1390                .await;
1391
1392            match outcome.result {
1393                Ok(Some(_)) => {
1394                    max_window_cnt = max_window_cnt.map(|cnt| {
1395                        (cnt + 1).min(self.config.batch_opts.experimental_max_filter_num_per_query)
1396                    });
1397
1398                    let sleep_until = {
1399                        let state = self.state.write().unwrap();
1400
1401                        let time_window_size = self
1402                            .config
1403                            .time_window_expr
1404                            .as_ref()
1405                            .and_then(|t| *t.time_window_size());
1406
1407                        let prefer_short_incremental_cadence = state.checkpoint_mode()
1408                            == CheckpointMode::Incremental
1409                            && !state.is_incremental_disabled();
1410
1411                        state.get_next_start_query_time(
1412                            self.config.flow_id,
1413                            &time_window_size,
1414                            min_refresh,
1415                            Some(self.config.batch_opts.query_timeout),
1416                            self.config.batch_opts.experimental_max_filter_num_per_query,
1417                            prefer_short_incremental_cadence,
1418                        )
1419                    };
1420
1421                    tokio::time::sleep_until(sleep_until).await;
1422                }
1423                Ok(None) => {
1424                    debug!(
1425                        "Flow id = {:?} found no new data, sleep for {:?} then continue",
1426                        self.config.flow_id, min_refresh
1427                    );
1428                    tokio::time::sleep(min_refresh).await;
1429                    continue;
1430                }
1431                Err(err) => {
1432                    METRIC_FLOW_BATCHING_ENGINE_ERROR_CNT
1433                        .with_label_values(&[&flow_id_str])
1434                        .inc();
1435                    match outcome.new_query {
1436                        Some(query) => {
1437                            common_telemetry::error!(err; "Failed to execute query for flow={} with query: {}", self.config.flow_id, query.plan);
1438                            max_window_cnt = Some(1);
1439                        }
1440                        None => {
1441                            common_telemetry::error!(err; "Failed to generate query for flow={}", self.config.flow_id)
1442                        }
1443                    }
1444                    tokio::time::sleep(min_refresh).await;
1445                }
1446            }
1447        }
1448    }
1449
1450    /// Check whether the shutdown signal has been received.
1451    fn is_shutdown_signaled(&self) -> bool {
1452        let mut state = self.state.write().unwrap();
1453        match state.shutdown_rx.try_recv() {
1454            Ok(()) | Err(TryRecvError::Closed) => true,
1455            Err(TryRecvError::Empty) => false,
1456        }
1457    }
1458
1459    /// Execute one scheduled attempt, temporarily setting
1460    /// `flow.scheduled_time_millis` on the task's QueryContext so
1461    /// SQL/TQL `now()` resolves to the logical scheduled time.
1462    ///
1463    /// The extension is removed after the attempt so a later manual
1464    /// `flush_flow` does not reuse a stale scheduled time.
1465    async fn execute_once_serialized_at_scheduled_time(
1466        &self,
1467        engine: &QueryEngineRef,
1468        frontend_client: &Arc<FrontendClient>,
1469        scheduled_time_secs: i64,
1470    ) -> ExecuteOnceOutcome {
1471        let mut guard = BatchingExecutionGuard::new(self.execution_lock.clone().lock_owned().await);
1472
1473        // Convert to milliseconds before touching the task state so an
1474        // unrepresentable scheduled time fails as an explicit error without
1475        // ever installing a saturated (off-phase) extension value.
1476        let scheduled_time_millis = match scheduled_time_millis(scheduled_time_secs) {
1477            Ok(millis) => millis,
1478            Err(e) => {
1479                return ExecuteOnceOutcome {
1480                    new_query: None,
1481                    result: Err(e),
1482                };
1483            }
1484        };
1485
1486        // Clone the current QueryContext and add the scheduled time
1487        // extension, then swap it into the task state for this attempt.
1488        let old_ctx = {
1489            let mut state = self.state.write().unwrap();
1490            let old = state.query_ctx.clone();
1491            let mut new_ctx = (*old).clone();
1492            new_ctx.set_extension(
1493                query::options::FLOW_SCHEDULED_TIME_MILLIS,
1494                scheduled_time_millis.to_string(),
1495            );
1496            state.query_ctx = Arc::new(new_ctx);
1497            old
1498        };
1499        guard.restore_query_context(self.state.clone(), old_ctx);
1500
1501        // A collaborator may retain the guard in an owned child task. Its Drop
1502        // restores the scheduled context before releasing serialization.
1503        self.execute_once_with_guard(guard, engine, frontend_client, None)
1504            .await
1505    }
1506
1507    /// Generate the create table SQL
1508    ///
1509    /// the auto created table will automatically added a `update_at` Milliseconds DEFAULT now() column in the end
1510    /// (for compatibility with flow streaming mode)
1511    ///
1512    /// and it will use first timestamp column as time index, all other columns will be added as normal columns and nullable
1513    async fn gen_create_table_expr(
1514        &self,
1515        engine: QueryEngineRef,
1516    ) -> Result<CreateTableExpr, Error> {
1517        let query_ctx = self.state.read().unwrap().query_ctx.clone();
1518        let plan =
1519            sql_to_df_plan(query_ctx.clone(), engine.clone(), &self.config.query, true).await?;
1520        create_table_with_expr(&plan, &self.config.sink_table_name, &self.config.query_type)
1521    }
1522
1523    /// Incremental delta scans are unfiltered by dirty windows; the sequence
1524    /// range, not a time predicate, defines source correctness.
1525    fn should_use_unfiltered_incremental_delta(&self) -> bool {
1526        let state = self.state.read().unwrap();
1527        state.checkpoint_mode() == CheckpointMode::Incremental
1528            && !state.is_incremental_disabled()
1529            && matches!(self.config.query_type, QueryType::Sql)
1530    }
1531
1532    /// Generate the next ordinary plan and classify its coverage.
1533    async fn gen_query_with_time_window(
1534        &self,
1535        engine: QueryEngineRef,
1536        sink_table_schema: &Arc<Schema>,
1537        primary_key_indices: &[usize],
1538        allow_partial: bool,
1539        max_window_cnt: Option<usize>,
1540    ) -> Result<Option<PlanInfo>, Error> {
1541        self.gen_query_with_time_window_with_values(
1542            engine,
1543            sink_table_schema,
1544            primary_key_indices,
1545            allow_partial,
1546            max_window_cnt,
1547            &BTreeMap::new(),
1548            false,
1549        )
1550        .await
1551    }
1552
1553    /// Generate the next plan with extension-owned sink values and an optional forced full scan.
1554    #[allow(clippy::too_many_arguments)]
1555    async fn gen_query_with_time_window_with_values(
1556        &self,
1557        engine: QueryEngineRef,
1558        sink_table_schema: &Arc<Schema>,
1559        primary_key_indices: &[usize],
1560        allow_partial: bool,
1561        max_window_cnt: Option<usize>,
1562        values: &BTreeMap<String, ScalarValue>,
1563        force_full_snapshot: bool,
1564    ) -> Result<Option<PlanInfo>, Error> {
1565        let query_ctx = self.state.read().unwrap().query_ctx.clone();
1566        let start = SystemTime::now();
1567        let since_the_epoch = start
1568            .duration_since(UNIX_EPOCH)
1569            .expect("Time went backwards");
1570        let low_bound = self
1571            .config
1572            .expire_after
1573            .map(|e| since_the_epoch.as_secs() - e as u64)
1574            .unwrap_or(u64::MIN);
1575
1576        let low_bound = Timestamp::new_second(low_bound as i64);
1577
1578        let expire_time_window_bound = self
1579            .config
1580            .time_window_expr
1581            .as_ref()
1582            .map(|expr| expr.eval(low_bound))
1583            .transpose()?;
1584
1585        let (expire_lower_bound, expire_upper_bound) = match (
1586            expire_time_window_bound,
1587            &self.config.query_type,
1588        ) {
1589            (Some((Some(l), Some(u))), QueryType::Sql) => (l, u),
1590            (None, QueryType::Sql) if self.config.flow_eval_interval.is_none() => {
1591                return UnexpectedSnafu {
1592                    reason: format!(
1593                        "Flow id={} reached execution without a time-window expression or EVAL INTERVAL; create-flow validation should have rejected it",
1594                        self.config.flow_id
1595                    ),
1596                }
1597                .fail();
1598            }
1599            _ => {
1600                // Explicit full-query flows (TQL and evaluation-interval SQL
1601                // plans whose shape cannot be safely dirty-window pruned) are
1602                // allowed to run as unfiltered full snapshots. This is distinct
1603                // from using unfiltered full as a fallback after scoped repair or
1604                // incremental rewrite failed.
1605                let (_, dirty_windows_to_restore) = self.drain_dirty_windows_signal();
1606
1607                let plan_info = self
1608                    .gen_unfiltered_plan_info(
1609                        engine,
1610                        query_ctx,
1611                        sink_table_schema.clone(),
1612                        primary_key_indices,
1613                        allow_partial,
1614                        values,
1615                        dirty_windows_to_restore,
1616                        None,
1617                        QueryCoverage::UnfilteredFull,
1618                    )
1619                    .await?;
1620
1621                return Ok(Some(plan_info));
1622            }
1623        };
1624
1625        debug!(
1626            "Flow id = {:?}, found time window: precise_lower_bound={:?}, precise_upper_bound={:?} with dirty time windows: {:?}",
1627            self.config.flow_id,
1628            expire_lower_bound,
1629            expire_upper_bound,
1630            self.state.read().unwrap().dirty_time_windows
1631        );
1632        let window_size = expire_upper_bound
1633            .sub(&expire_lower_bound)
1634            .with_context(|| UnexpectedSnafu {
1635                reason: format!(
1636                    "Can't get window size from {expire_upper_bound:?} - {expire_lower_bound:?}"
1637                ),
1638            })?;
1639        let col_name = self
1640            .config
1641            .time_window_expr
1642            .as_ref()
1643            .map(|expr| expr.column_name.clone())
1644            .with_context(|| UnexpectedSnafu {
1645                reason: format!(
1646                    "Flow id={:?}, Failed to get column name from time window expr",
1647                    self.config.flow_id
1648                ),
1649            })?;
1650
1651        if force_full_snapshot {
1652            // Durable execution cannot trust an uncertain prior checkpoint. Re-run an
1653            // unfiltered full snapshot, but retain the normal expiry predicate and the consumed
1654            // dirty signal so every planning/execution failure remains reversible.
1655            let retention_filter = self.config.expire_after.map(|_| {
1656                (
1657                    col_name.as_str(),
1658                    expire_lower_bound,
1659                    "forced full snapshot",
1660                )
1661            });
1662            let (_, dirty_windows_to_restore) = self.drain_dirty_windows_signal();
1663            return self
1664                .gen_unfiltered_plan_info(
1665                    engine,
1666                    query_ctx,
1667                    sink_table_schema.clone(),
1668                    primary_key_indices,
1669                    allow_partial,
1670                    values,
1671                    dirty_windows_to_restore,
1672                    retention_filter,
1673                    QueryCoverage::UnfilteredFull,
1674                )
1675                .await
1676                .map(Some);
1677        }
1678
1679        if self.should_use_unfiltered_incremental_delta() {
1680            // In incremental mode, source correctness is defined by the
1681            // per-region sequence range `(checkpoint, scan-open snapshot]`, not
1682            // by dirty-window predicates. Dirty windows are only a scheduling
1683            // signal here. Applying a stale dirty-window filter to the source can
1684            // exclude rows that are inside the returned watermark and make a
1685            // checkpoint advance skip them forever. The sink side is also left
1686            // unfiltered by dirty windows; the incremental rewrite joins the
1687            // delta groups with the full sink state for correctness. Future
1688            // dynamic filters can prune sink reads as a pure optimization.
1689            let retention_filter = self
1690                .config
1691                .expire_after
1692                .map(|_| (col_name.as_str(), expire_lower_bound, "incremental"));
1693            return self
1694                .gen_unfiltered_plan_info_if_dirty(
1695                    engine,
1696                    query_ctx,
1697                    sink_table_schema.clone(),
1698                    primary_key_indices,
1699                    allow_partial,
1700                    values,
1701                    retention_filter,
1702                    QueryCoverage::IncrementalDelta,
1703                )
1704                .await;
1705        }
1706
1707        let (expr, coverage) = {
1708            let mut state = self.state.write().unwrap();
1709            let window_cnt = max_window_cnt
1710                .unwrap_or(self.config.batch_opts.experimental_max_filter_num_per_query);
1711            let expr = state.gen_scoped_filter_exprs(
1712                &col_name,
1713                Some(expire_lower_bound),
1714                window_size,
1715                window_cnt,
1716                self.config.flow_id,
1717                Some(self),
1718            )?;
1719            let repair_high = state
1720                .pending_fenced_repair()
1721                .map(|repair| repair.high().clone());
1722            let coverage = if let Some(high) = repair_high {
1723                QueryCoverage::FencedRepairChunk { high }
1724            } else {
1725                QueryCoverage::ScopedBaseRepair
1726            };
1727            (expr, coverage)
1728        };
1729
1730        let Some(expr) = expr else {
1731            // no new data, hence no need to update
1732            debug!("Flow id={:?}, no new data, not update", self.config.flow_id);
1733            return Ok(None);
1734        };
1735
1736        let filter_sql = expr_to_sql(&expr.expr)
1737            .map(|sql| sql.to_string())
1738            .unwrap_or_else(|err| format!("<failed to format filter expr: {err}>"));
1739
1740        debug!(
1741            "Flow id={:?}, Generated filter expr: {:?}",
1742            self.config.flow_id, filter_sql
1743        );
1744
1745        let mut add_filter = AddFilterRewriter::new(expr.expr.clone());
1746        let mut add_auto_column = ColumnMatcherRewriter::new_with_values(
1747            sink_table_schema.clone(),
1748            primary_key_indices.to_vec(),
1749            allow_partial,
1750            values.clone(),
1751        );
1752
1753        let plan = self.restore_scoped_dirty_windows_on_err(
1754            &expr,
1755            sql_to_df_plan(query_ctx.clone(), engine.clone(), &self.config.query, false).await,
1756        )?;
1757        let rewrite = self.restore_scoped_dirty_windows_on_err(
1758            &expr,
1759            plan.clone()
1760                .rewrite(&mut add_filter)
1761                .and_then(|p| p.data.rewrite(&mut add_auto_column))
1762                .with_context(|_| DatafusionSnafu {
1763                    context: format!("Failed to rewrite plan:\n {}\n", plan),
1764                })
1765                .map(|rewrite| rewrite.data),
1766        )?;
1767        // only apply optimize after complex rewrite is done
1768        let new_plan = self.restore_scoped_dirty_windows_on_err(
1769            &expr,
1770            apply_df_optimizer(rewrite, &query_ctx).await,
1771        )?;
1772
1773        let info = PlanInfo {
1774            plan: new_plan.clone(),
1775            dirty_restore: DirtyRestore::Scoped(expr),
1776            coverage,
1777        };
1778
1779        Ok(Some(info))
1780    }
1781}
1782
1783#[cfg(test)]
1784mod test;