Skip to main content

flow/batching_mode/
task.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
16use std::sync::{Arc, RwLock};
17use std::time::{Duration, SystemTime, UNIX_EPOCH};
18
19use api::v1::{CreateTableExpr, TableName};
20use catalog::CatalogManagerRef;
21use common_error::ext::BoxedError;
22use common_query::logical_plan::breakup_insert_plan;
23use common_telemetry::tracing::warn;
24use common_telemetry::{debug, info};
25use common_time::Timestamp;
26use datafusion::datasource::DefaultTableSource;
27use datafusion::sql::unparser::expr_to_sql;
28use datafusion_common::tree_node::{Transformed, TreeNode};
29use datafusion_common::utils::quote_identifier;
30use datafusion_common::{DFSchemaRef, TableReference};
31use datafusion_expr::{DmlStatement, LogicalPlan, WriteOp, col, lit};
32use datatypes::schema::Schema;
33use query::QueryEngineRef;
34use query::options::FLOW_INCREMENTAL_MODE;
35use query::query_engine::DefaultSerializer;
36use session::context::QueryContextRef;
37use snafu::{OptionExt, ResultExt};
38use sql::parsers::utils::is_tql;
39use store_api::mito_engine_options::MERGE_MODE_KEY;
40use substrait::{DFLogicalSubstraitConvertor, SubstraitPlan};
41use table::table::adapter::DfTableProviderAdapter;
42use tokio::sync::oneshot::error::TryRecvError;
43use tokio::sync::{Mutex, oneshot};
44use tokio::time::Instant;
45
46use crate::batching_mode::BatchingModeOptions;
47use crate::batching_mode::checkpoint::checkpoint_mode_label;
48use crate::batching_mode::eval_schedule::{EvalSchedule, select_due_scheduled_times};
49use crate::batching_mode::frontend_client::{FrontendClient, PeerDesc};
50use crate::batching_mode::state::{
51    CheckpointMode, DirtyTimeWindows, FilterExprInfo, TaskState, to_df_literal,
52};
53use crate::batching_mode::table_creator::{QueryType, create_table_with_expr};
54use crate::batching_mode::time_window::TimeWindowExpr;
55use crate::batching_mode::utils::{
56    AddFilterRewriter, ColumnMatcherRewriter, df_plan_to_sql, gen_plan_with_matching_schema,
57    get_table_info_df_schema, sql_to_df_plan,
58};
59use crate::df_optimizer::apply_df_optimizer;
60use crate::error::{
61    DatafusionSnafu, ExternalSnafu, InvalidQuerySnafu, SubstraitEncodeLogicalPlanSnafu,
62    UnexpectedSnafu,
63};
64use crate::metrics::{
65    METRIC_FLOW_BATCHING_ENGINE_ERROR_CNT, METRIC_FLOW_BATCHING_ENGINE_QUERY_TIME,
66    METRIC_FLOW_BATCHING_ENGINE_SLOW_QUERY, METRIC_FLOW_BATCHING_ENGINE_START_QUERY_CNT,
67    METRIC_FLOW_ROWS,
68};
69use crate::{Error, FlowId};
70
71mod ckpt;
72mod inc;
73
74/// Returns the current wall-clock Unix timestamp in seconds.
75fn wall_clock_unix_secs() -> i64 {
76    SystemTime::now()
77        .duration_since(UNIX_EPOCH)
78        .unwrap_or_default()
79        .as_secs() as i64
80}
81
82/// Initial scheduler cursor for `start_scheduled_loop`: exactly one interval
83/// before `start_secs` so the first due scheduled time is `start_secs` itself.
84///
85/// Fallible: a `start_secs - interval_secs` difference that does not fit in
86/// `i64` is an explicit error instead of a saturated cursor that would make
87/// the first due scheduled time `start_secs + interval_secs` and silently skip
88/// the `start_secs` tick.
89fn initial_schedule_cursor(start_secs: i64, interval_secs: i64) -> Result<i64, Error> {
90    let cursor = i128::from(start_secs) - i128::from(interval_secs);
91    i64::try_from(cursor).map_err(|_| {
92        UnexpectedSnafu {
93            reason: format!(
94                "Cannot compute the initial eval schedule cursor one interval before start {start_secs} (interval={interval_secs}): {cursor} does not fit in i64"
95            ),
96        }
97        .build()
98    })
99}
100
101/// Whole seconds to sleep until the next scheduled time `next`, measured from
102/// the current wall clock `wall_now_secs`.
103///
104/// Fallible: `next` must be strictly after `wall_now_secs` and the difference
105/// must fit in `u64`. In practice `i64::MAX - i64::MIN` is exactly `u64::MAX`,
106/// so the difference always fits once `next > wall_now_secs`; the explicit
107/// error keeps the scheduled loop panic-free and wrap-free regardless.
108fn sleep_delta_secs(next: i64, wall_now_secs: i64) -> Result<u64, Error> {
109    let delta = i128::from(next) - i128::from(wall_now_secs);
110    u64::try_from(delta).map_err(|_| {
111        UnexpectedSnafu {
112            reason: format!(
113                "Cannot sleep until the next scheduled time {next}: the delta from wall clock {wall_now_secs} is {delta} seconds, which does not fit in u64"
114            ),
115        }
116        .build()
117    })
118}
119
120/// Scheduled time in seconds converted to milliseconds for the
121/// `FLOW_SCHEDULED_TIME_MILLIS` extension.
122///
123/// Fallible: a seconds value whose millisecond product does not fit in `i64`
124/// is an explicit error instead of a saturated `i64::MAX` that would silently
125/// misrepresent the logical scheduled time.
126fn scheduled_time_millis(scheduled_time_secs: i64) -> Result<i64, Error> {
127    scheduled_time_secs.checked_mul(1000).ok_or_else(|| {
128        UnexpectedSnafu {
129            reason: format!(
130                "Cannot convert scheduled time {scheduled_time_secs}s to milliseconds: the product exceeds i64"
131            ),
132        }
133        .build()
134    })
135}
136
137/// The task's config, immutable once created
138#[derive(Clone)]
139pub struct TaskConfig {
140    pub flow_id: FlowId,
141    pub query: String,
142    /// output schema of the query
143    pub output_schema: DFSchemaRef,
144    pub time_window_expr: Option<TimeWindowExpr>,
145    /// in seconds
146    pub expire_after: Option<i64>,
147    pub sink_table_name: [String; 3],
148    pub source_table_names: HashSet<[String; 3]>,
149    pub catalog_manager: CatalogManagerRef,
150    pub query_type: QueryType,
151    pub batch_opts: Arc<BatchingModeOptions>,
152    pub exact_sequence_range_required: bool,
153    pub flow_eval_interval: Option<Duration>,
154    /// Typed schedule configuration, pre-parsed at task creation time.
155    pub eval_schedule: Option<EvalSchedule>,
156}
157
158fn determine_query_type(query: &str, query_ctx: &QueryContextRef) -> Result<QueryType, Error> {
159    let is_tql = is_tql(query_ctx.sql_dialect(), query)
160        .map_err(BoxedError::new)
161        .context(ExternalSnafu)?;
162    Ok(if is_tql {
163        QueryType::Tql
164    } else {
165        QueryType::Sql
166    })
167}
168
169fn is_merge_mode_last_non_null(options: &HashMap<String, String>) -> bool {
170    options
171        .get(MERGE_MODE_KEY)
172        .map(|mode| mode.eq_ignore_ascii_case("last_non_null"))
173        .unwrap_or(false)
174}
175
176fn encode_insert_plan_request(
177    insert_to: TableName,
178    insert_input_plan: &LogicalPlan,
179) -> Result<api::v1::QueryRequest, Error> {
180    let message = DFLogicalSubstraitConvertor {}
181        .encode(insert_input_plan, DefaultSerializer)
182        .context(SubstraitEncodeLogicalPlanSnafu)?;
183    Ok(api::v1::QueryRequest {
184        query: Some(api::v1::query_request::Query::InsertIntoPlan(
185            api::v1::InsertIntoPlan {
186                table_name: Some(insert_to),
187                logical_plan: message.to_vec(),
188            },
189        )),
190    })
191}
192
193fn format_insert_target_columns(plan: &LogicalPlan) -> String {
194    plan.schema()
195        .fields()
196        .iter()
197        .map(|field| quote_identifier(field.name()).to_string())
198        .collect::<Vec<_>>()
199        .join(", ")
200}
201
202#[derive(Clone)]
203pub struct BatchingTask {
204    pub config: Arc<TaskConfig>,
205    pub state: Arc<RwLock<TaskState>>,
206    /// Serializes plan generation, execution, checkpoint advancement, and dirty
207    /// window restoration for this flow. Without this, a manual flush and the
208    /// background loop can process the same checkpoint range concurrently.
209    execution_lock: Arc<Mutex<()>>,
210}
211
212/// Arguments for creating batching task
213pub struct TaskArgs<'a> {
214    pub flow_id: FlowId,
215    pub query: &'a str,
216    pub plan: LogicalPlan,
217    pub time_window_expr: Option<TimeWindowExpr>,
218    pub expire_after: Option<i64>,
219    pub sink_table_name: [String; 3],
220    pub source_table_names: Vec<[String; 3]>,
221    pub query_ctx: QueryContextRef,
222    pub catalog_manager: CatalogManagerRef,
223    pub shutdown_rx: oneshot::Receiver<()>,
224    pub batch_opts: Arc<BatchingModeOptions>,
225    pub flow_eval_interval: Option<Duration>,
226    /// Typed schedule configuration pre-parsed from `CreateFlowArgs`.
227    pub eval_schedule: Option<EvalSchedule>,
228}
229
230pub struct PlanInfo {
231    pub plan: LogicalPlan,
232    pub dirty_restore: DirtyRestore,
233    pub coverage: QueryCoverage,
234}
235
236#[derive(Clone)]
237pub enum QueryCoverage {
238    /// Explicit full-query snapshot coverage, e.g. TQL or evaluation-interval
239    /// SQL flows whose plan shape cannot be safely dirty-window pruned. This
240    /// must not be used as an implicit recovery path for scoped repair or an
241    /// unsafe incremental rewrite fallback.
242    UnfilteredFull,
243    /// Scoped full-snapshot repair over the current dirty windows. A successful
244    /// result may start a fenced repair if new dirty windows appeared meanwhile.
245    ScopedBaseRepair,
246    /// A chunk of windows being repaired under the frozen high-watermark `H`.
247    /// The `high` map is sent as snapshot read bounds and must be matched by
248    /// the returned terminal watermarks before checkpoints can advance.
249    FencedRepairChunk { high: BTreeMap<u64, u64> },
250    /// Incremental delta query over `(checkpoint, scan-open snapshot]`.
251    IncrementalDelta,
252}
253
254impl QueryCoverage {
255    /// Whether this query should use incremental scan extensions and
256    /// incremental checkpoint advancement rules.
257    fn is_incremental_delta(&self) -> bool {
258        matches!(self, Self::IncrementalDelta)
259    }
260
261    /// Snapshot upper bounds requested from the storage layer. Only fenced
262    /// repair chunks carry bounds; all other coverage relies on normal scans.
263    fn snapshot_seqs(&self) -> HashMap<u64, u64> {
264        match self {
265            Self::FencedRepairChunk { high } => high.iter().map(|(k, v)| (*k, *v)).collect(),
266            _ => HashMap::new(),
267        }
268    }
269}
270
271pub enum DirtyRestore {
272    /// The query was scoped to dirty time ranges; restore those ranges if the
273    /// run fails.
274    Scoped(FilterExprInfo),
275    /// The query could not be scoped to dirty time ranges, so the dirty-window
276    /// state is only a dirty signal. Restore the consumed signal if the full
277    /// run fails.
278    ///
279    /// TODO(discord9): Full-query runs only need a dirty bool flag. Refactor
280    /// the unscoped path to stop reusing `DirtyTimeWindows` for this signal.
281    Unscoped(DirtyTimeWindows),
282}
283
284struct ExecuteOnceOutcome {
285    new_query: Option<PlanInfo>,
286    /// Execution result of the generated insert plan.
287    ///
288    /// `Ok(Some((affected_rows, elapsed)))` means a query was executed.
289    /// `Ok(None)` means no query was generated because there was no dirty signal.
290    /// `Err(_)` means plan generation or execution failed.
291    result: Result<Option<(usize, Duration)>, Error>,
292}
293
294impl BatchingTask {
295    #[allow(clippy::too_many_arguments)]
296    pub fn try_new(args: TaskArgs<'_>) -> Result<Self, Error> {
297        Self::try_new_with_exact_sequence_range_required(args, false)
298    }
299
300    pub fn try_new_with_exact_sequence_range_required(
301        TaskArgs {
302            flow_id,
303            query,
304            plan,
305            time_window_expr,
306            expire_after,
307            sink_table_name,
308            source_table_names,
309            query_ctx,
310            catalog_manager,
311            shutdown_rx,
312            batch_opts,
313            flow_eval_interval,
314            eval_schedule,
315        }: TaskArgs<'_>,
316        exact_sequence_range_required: bool,
317    ) -> Result<Self, Error> {
318        let mut state = TaskState::with_dirty_time_windows(
319            query_ctx.clone(),
320            shutdown_rx,
321            DirtyTimeWindows::new(
322                batch_opts.experimental_max_filter_num_per_query,
323                batch_opts.experimental_time_window_merge_threshold,
324            ),
325        );
326        if !batch_opts.experimental_enable_incremental_read {
327            state.disable_incremental();
328        }
329
330        Ok(Self {
331            config: Arc::new(TaskConfig {
332                flow_id,
333                query: query.to_string(),
334                time_window_expr,
335                expire_after,
336                sink_table_name,
337                source_table_names: source_table_names.into_iter().collect(),
338                catalog_manager,
339                output_schema: plan.schema().clone(),
340                query_type: determine_query_type(query, &query_ctx)?,
341                exact_sequence_range_required,
342                batch_opts,
343                flow_eval_interval,
344                eval_schedule,
345            }),
346            state: Arc::new(RwLock::new(state)),
347            execution_lock: Arc::new(Mutex::new(())),
348        })
349    }
350
351    pub fn last_execution_time_millis(&self) -> Option<i64> {
352        self.state.read().unwrap().last_execution_time_millis()
353    }
354
355    pub fn start_time_millis(&self) -> Option<i64> {
356        self.state.read().unwrap().start_time_millis()
357    }
358
359    /// Collect flow-related extensions from the task's query context that should be
360    /// forwarded to the frontend (e.g. scheduled time).
361    fn frontend_extensions(&self) -> HashMap<String, String> {
362        let ctx = self.state.read().unwrap();
363        let all = ctx.query_ctx.extensions();
364        let mut flow_exts = HashMap::new();
365        // Propagate the scheduled time extension if present so that frontend
366        // execution can use the same logical time.
367        if let Some(v) = all.get(query::options::FLOW_SCHEDULED_TIME_MILLIS) {
368            flow_exts.insert(
369                query::options::FLOW_SCHEDULED_TIME_MILLIS.to_string(),
370                v.clone(),
371            );
372        }
373        flow_exts
374    }
375
376    /// mark time window range (now - expire_after, now) as dirty (or (0, now) if expire_after not set)
377    ///
378    /// useful for flush_flow to flush dirty time windows range
379    pub fn mark_all_windows_as_dirty(&self) -> Result<(), Error> {
380        let now = SystemTime::now();
381        let now = Timestamp::new_second(
382            now.duration_since(UNIX_EPOCH)
383                .expect("Time went backwards")
384                .as_secs() as _,
385        );
386        let lower_bound = self
387            .config
388            .expire_after
389            .map(|e| now.sub_duration(Duration::from_secs(e as _)))
390            .transpose()
391            .map_err(BoxedError::new)
392            .context(ExternalSnafu)?
393            .unwrap_or(Timestamp::new_second(0));
394        debug!(
395            "Flow {} mark range ({:?}, {:?}) as dirty",
396            self.config.flow_id, lower_bound, now
397        );
398        self.state
399            .write()
400            .unwrap()
401            .dirty_time_windows
402            .add_window(lower_bound, Some(now));
403        Ok(())
404    }
405
406    /// Create sink table if not exists
407    pub async fn check_or_create_sink_table(
408        &self,
409        engine: &QueryEngineRef,
410        frontend_client: &Arc<FrontendClient>,
411    ) -> Result<Option<(usize, Duration)>, Error> {
412        if !self.is_table_exist(&self.config.sink_table_name).await? {
413            let create_table = self.gen_create_table_expr(engine.clone()).await?;
414            info!(
415                "Try creating sink table(if not exists) with expr: {:?}",
416                create_table
417            );
418            self.create_table(frontend_client, create_table).await?;
419            info!(
420                "Sink table {}(if not exists) created",
421                self.config.sink_table_name.join(".")
422            );
423        }
424
425        Ok(None)
426    }
427
428    /// Validates that the sink table schema can accept this flow's output.
429    ///
430    /// This is a dry-run of the same schema matching logic used by insert-plan
431    /// generation, but without adding dirty-window filters or executing the query. It is used
432    /// during CREATE FLOW to catch existing sink table mismatches early.
433    pub async fn validate_sink_table_schema(&self, engine: &QueryEngineRef) -> Result<(), Error> {
434        let (table, _) = get_table_info_df_schema(
435            self.config.catalog_manager.clone(),
436            self.config.sink_table_name.clone(),
437        )
438        .await?;
439
440        let table_meta = &table.table_info().meta;
441        let merge_mode_last_non_null =
442            is_merge_mode_last_non_null(&table_meta.options.extra_options);
443        let primary_key_indices = table_meta.primary_key_indices.clone();
444        let query_ctx = self.state.read().unwrap().query_ctx.clone();
445
446        gen_plan_with_matching_schema(
447            &self.config.query,
448            query_ctx,
449            engine.clone(),
450            table_meta.schema.clone(),
451            &primary_key_indices,
452            merge_mode_last_non_null,
453        )
454        .await
455        .map(|_| ())
456    }
457
458    async fn is_table_exist(&self, table_name: &[String; 3]) -> Result<bool, Error> {
459        self.config
460            .catalog_manager
461            .table_exists(&table_name[0], &table_name[1], &table_name[2], None)
462            .await
463            .map_err(BoxedError::new)
464            .context(ExternalSnafu)
465    }
466
467    pub(crate) async fn execute_once_serialized(
468        &self,
469        engine: &QueryEngineRef,
470        frontend_client: &Arc<FrontendClient>,
471        max_window_cnt: Option<usize>,
472    ) -> Result<Option<(usize, Duration)>, Error> {
473        let outcome = self
474            .execute_once_serialized_with_outcome(engine, frontend_client, max_window_cnt)
475            .await;
476        outcome.result
477    }
478
479    /// Executes one flow evaluation under `execution_lock` and keeps the
480    /// generated query context for the background loop's error logging/backoff.
481    async fn execute_once_serialized_with_outcome(
482        &self,
483        engine: &QueryEngineRef,
484        frontend_client: &Arc<FrontendClient>,
485        max_window_cnt: Option<usize>,
486    ) -> ExecuteOnceOutcome {
487        let _execution_guard = self.execution_lock.lock().await;
488        self.execute_once_unlocked(engine, frontend_client, max_window_cnt)
489            .await
490    }
491
492    /// Executes one flow evaluation. Caller must hold `execution_lock`.
493    async fn execute_once_unlocked(
494        &self,
495        engine: &QueryEngineRef,
496        frontend_client: &Arc<FrontendClient>,
497        max_window_cnt: Option<usize>,
498    ) -> ExecuteOnceOutcome {
499        let new_query = match self.gen_insert_plan_unlocked(engine, max_window_cnt).await {
500            Ok(new_query) => new_query,
501            Err(err) => {
502                return ExecuteOnceOutcome {
503                    new_query: None,
504                    result: Err(err),
505                };
506            }
507        };
508
509        if let Some(new_query) = new_query {
510            debug!("Generate new query: {}", new_query.plan);
511            let res = self
512                .execute_logical_plan_unlocked(
513                    frontend_client,
514                    &new_query.plan,
515                    &new_query.dirty_restore,
516                    &new_query.coverage,
517                )
518                .await;
519            if res.is_err() {
520                self.handle_executed_query_failure(Some(&new_query));
521            }
522            ExecuteOnceOutcome {
523                new_query: Some(new_query),
524                result: res,
525            }
526        } else {
527            debug!("Generate no query");
528            ExecuteOnceOutcome {
529                new_query: None,
530                result: Ok(None),
531            }
532        }
533    }
534
535    /// Generates the insert plan. Caller must reach this through the serialized path.
536    async fn gen_insert_plan_unlocked(
537        &self,
538        engine: &QueryEngineRef,
539        max_window_cnt: Option<usize>,
540    ) -> Result<Option<PlanInfo>, Error> {
541        let (table, df_schema) = get_table_info_df_schema(
542            self.config.catalog_manager.clone(),
543            self.config.sink_table_name.clone(),
544        )
545        .await?;
546
547        let table_meta = &table.table_info().meta;
548        let merge_mode_last_non_null =
549            is_merge_mode_last_non_null(&table_meta.options.extra_options);
550        let primary_key_indices = table_meta.primary_key_indices.clone();
551
552        let new_query = self
553            .gen_query_with_time_window(
554                engine.clone(),
555                &table.table_info().meta.schema,
556                &primary_key_indices,
557                merge_mode_last_non_null,
558                max_window_cnt,
559            )
560            .await?;
561
562        let Some(new_query) = new_query else {
563            return Ok(None);
564        };
565
566        // first check if all columns in input query exists in sink table
567        // since insert into ref to names in record batch generate by given query
568        let table_columns = df_schema
569            .columns()
570            .into_iter()
571            .map(|c| c.name)
572            .collect::<BTreeSet<_>>();
573        for column in new_query.plan.schema().columns() {
574            if !table_columns.contains(column.name()) {
575                self.restore_dirty_windows_after_failure(&new_query);
576                return InvalidQuerySnafu {
577                    reason: format!(
578                        "Column {} not found in sink table with columns {:?}",
579                        column, table_columns
580                    ),
581                }
582                .fail();
583            }
584        }
585
586        let table_provider = Arc::new(DfTableProviderAdapter::new(table));
587        let table_source = Arc::new(DefaultTableSource::new(table_provider));
588
589        // update_at& time index placeholder (if exists) should have default value
590        let plan = LogicalPlan::Dml(DmlStatement::new(
591            datafusion_common::TableReference::Full {
592                catalog: self.config.sink_table_name[0].clone().into(),
593                schema: self.config.sink_table_name[1].clone().into(),
594                table: self.config.sink_table_name[2].clone().into(),
595            },
596            table_source,
597            WriteOp::Insert(datafusion_expr::dml::InsertOp::Append),
598            Arc::new(new_query.plan.clone()),
599        ));
600        let insert_into_info = PlanInfo {
601            plan,
602            dirty_restore: new_query.dirty_restore,
603            coverage: new_query.coverage,
604        };
605        let insert_into =
606            match insert_into_info
607                .plan
608                .clone()
609                .recompute_schema()
610                .context(DatafusionSnafu {
611                    context: "Failed to recompute schema",
612                }) {
613                Ok(insert_into) => insert_into,
614                Err(err) => {
615                    self.restore_dirty_windows_after_failure(&insert_into_info);
616                    return Err(err);
617                }
618            };
619
620        Ok(Some(PlanInfo {
621            plan: insert_into,
622            dirty_restore: insert_into_info.dirty_restore,
623            coverage: insert_into_info.coverage,
624        }))
625    }
626
627    pub async fn create_table(
628        &self,
629        frontend_client: &Arc<FrontendClient>,
630        expr: CreateTableExpr,
631    ) -> Result<(), Error> {
632        let catalog = &self.config.sink_table_name[0];
633        let schema = &self.config.sink_table_name[1];
634        frontend_client
635            .create(expr.clone(), catalog, schema)
636            .await?;
637        Ok(())
638    }
639
640    /// Executes the insert plan. Caller must reach this through the serialized path.
641    async fn execute_logical_plan_unlocked(
642        &self,
643        frontend_client: &Arc<FrontendClient>,
644        plan: &LogicalPlan,
645        dirty_restore: &DirtyRestore,
646        coverage: &QueryCoverage,
647    ) -> Result<Option<(usize, Duration)>, Error> {
648        let instant = Instant::now();
649        let flow_id = self.config.flow_id;
650
651        debug!(
652            "Executing flow {flow_id}(expire_after={:?} secs) with query {}",
653            self.config.expire_after, &plan
654        );
655
656        let catalog = &self.config.sink_table_name[0];
657        let schema = &self.config.sink_table_name[1];
658
659        // fix all table ref by make it fully qualified, i.e. "table_name" => "catalog_name.schema_name.table_name"
660        let plan = plan
661            .clone()
662            .transform_down_with_subqueries(|p| {
663                if let LogicalPlan::TableScan(mut table_scan) = p {
664                    let resolved = table_scan.table_name.resolve(catalog, schema);
665                    table_scan.table_name = resolved.into();
666                    Ok(Transformed::yes(LogicalPlan::TableScan(table_scan)))
667                } else {
668                    Ok(Transformed::no(p))
669                }
670            })
671            .with_context(|_| DatafusionSnafu {
672                context: format!("Failed to fix table ref in logical plan, plan={:?}", plan),
673            })?
674            .data;
675
676        // For incremental-mode SQL queries, attempt to rewrite the delta aggregate
677        // plan into a safe delta-LEFT-JOIN-sink form before deciding on extensions.
678        let incremental_plan = if coverage.is_incremental_delta() {
679            self.prepare_plan_for_incremental(&plan).await?
680        } else {
681            None
682        };
683        let incremental_safe = incremental_plan.is_some();
684        if coverage.is_incremental_delta() && !incremental_safe {
685            debug!(
686                "Flow {flow_id} skipped unsafe incremental delta fallback; \
687                 restored dirty signal instead of executing an unfiltered full snapshot"
688            );
689            self.restore_dirty_windows(dirty_restore);
690            return Ok(None);
691        }
692        let plan = incremental_plan.unwrap_or_else(|| plan.clone());
693
694        let extensions = self
695            .build_flow_query_extensions(incremental_safe, coverage.is_incremental_delta())
696            .await?;
697        let frontend_extensions = self.frontend_extensions();
698        let extension_refs = extensions
699            .iter()
700            .map(|(key, value)| (*key, value.as_str()))
701            .chain(
702                frontend_extensions
703                    .iter()
704                    .map(|(key, value)| (key.as_str(), value.as_str())),
705            )
706            .collect::<Vec<_>>();
707        let query_mode = if extensions
708            .iter()
709            .any(|(key, _)| *key == FLOW_INCREMENTAL_MODE)
710        {
711            CheckpointMode::Incremental
712        } else {
713            CheckpointMode::FullSnapshot
714        };
715        Self::record_query_mode(flow_id, query_mode);
716        debug!(
717            "Flow {flow_id} executing batching query with checkpoint_mode={}, extension_count={}",
718            checkpoint_mode_label(query_mode),
719            extensions.len()
720        );
721
722        let mut peer_desc = None;
723        let res = {
724            let _timer = METRIC_FLOW_BATCHING_ENGINE_QUERY_TIME
725                .with_label_values(&[flow_id.to_string().as_str()])
726                .start_timer();
727
728            let req = if let Some((insert_to, insert_input_plan)) =
729                breakup_insert_plan(&plan, catalog, schema)
730            {
731                if query_mode == CheckpointMode::FullSnapshot
732                    && matches!(self.config.query_type, QueryType::Sql)
733                    && self.config.flow_eval_interval.is_some()
734                    && self.config.time_window_expr.is_none()
735                {
736                    // Evaluation-interval SQL flows without a time-window
737                    // expression execute as full-query snapshots. Send these
738                    // as SQL text instead of Substrait to avoid logical-plan
739                    // round-trip issues around complex joins/unions/CTEs and
740                    // duplicate field aliases. Keep ordinary SQL full snapshots
741                    // on the existing InsertIntoPlan path because SQL unparsing
742                    // is not valid for every planned aggregate shape yet.
743                    // If the local SQL unparser does not support this plan,
744                    // keep the previous InsertIntoPlan transport as a fallback.
745                    match df_plan_to_sql(&insert_input_plan) {
746                        Ok(select_sql) => {
747                            let target_columns = format_insert_target_columns(&insert_input_plan);
748                            let sql = format!(
749                                "INSERT INTO {} ({}) {}",
750                                TableReference::full(
751                                    insert_to.catalog_name.as_str(),
752                                    insert_to.schema_name.as_str(),
753                                    insert_to.table_name.as_str(),
754                                )
755                                .to_quoted_string(),
756                                target_columns,
757                                select_sql
758                            );
759                            api::v1::QueryRequest {
760                                query: Some(api::v1::query_request::Query::Sql(sql)),
761                            }
762                        }
763                        Err(err) => {
764                            debug!(
765                                "Failed to unparse full-snapshot SQL flow {} plan; \
766                                 falling back to InsertIntoPlan: {:?}",
767                                flow_id, err
768                            );
769                            encode_insert_plan_request(insert_to, &insert_input_plan)?
770                        }
771                    }
772                } else {
773                    encode_insert_plan_request(insert_to, &insert_input_plan)?
774                }
775            } else {
776                let message = DFLogicalSubstraitConvertor {}
777                    .encode(&plan, DefaultSerializer)
778                    .context(SubstraitEncodeLogicalPlanSnafu)?;
779
780                api::v1::QueryRequest {
781                    query: Some(api::v1::query_request::Query::LogicalPlan(message.to_vec())),
782                }
783            };
784
785            let snapshot_seqs = coverage.snapshot_seqs();
786            {
787                let mut state = self.state.write().unwrap();
788                state.record_start_time_if_first();
789            }
790            frontend_client
791                .query_with_terminal_metrics(
792                    catalog,
793                    schema,
794                    req,
795                    &extension_refs,
796                    &snapshot_seqs,
797                    &mut peer_desc,
798                )
799                .await
800        };
801
802        let elapsed = instant.elapsed();
803        let peer_label = peer_desc
804            .as_ref()
805            .map(ToString::to_string)
806            .unwrap_or_else(|| PeerDesc::default().to_string());
807        if let Err(err) = &res {
808            warn!(
809                "Failed to execute Flow {flow_id} on frontend {peer_label}, result: {err:?}, elapsed: {:?} with query: {}",
810                elapsed, &plan
811            );
812            let decision = {
813                let mut state = self.state.write().unwrap();
814                let reason = Self::query_failure_reason(err, coverage);
815                Self::apply_query_failure_to_state(&mut state, elapsed, coverage, reason)
816            };
817            if let Some(decision) = decision {
818                Self::record_checkpoint_decision(flow_id, decision);
819            }
820        }
821
822        // record slow query
823        if elapsed >= self.config.batch_opts.slow_query_threshold {
824            warn!(
825                "Flow {flow_id} on frontend {peer_label} executed for {:?} before complete, query: {}",
826                elapsed, &plan
827            );
828            let flow_id = flow_id.to_string();
829            METRIC_FLOW_BATCHING_ENGINE_SLOW_QUERY
830                .with_label_values(&[flow_id.as_str(), peer_label.as_str()])
831                .observe(elapsed.as_secs_f64());
832        }
833
834        let res = res?;
835        let (affected_rows, _) = res.output.extract_rows_and_cost();
836        debug!(
837            "Flow {flow_id} executed, affected_rows: {affected_rows:?}, elapsed: {:?}, watermark: {:?}",
838            elapsed,
839            res.region_watermark_map()
840        );
841        METRIC_FLOW_ROWS
842            .with_label_values(&[format!("{}-out-batching", flow_id).as_str()])
843            .inc_by(affected_rows as _);
844        let decision = {
845            let mut state = self.state.write().unwrap();
846            Self::apply_query_result_to_state(&mut state, &res, elapsed, coverage)
847        };
848        Self::record_checkpoint_decision(flow_id, decision);
849
850        Ok(Some((affected_rows, elapsed)))
851    }
852
853    /// Restore dirty windows consumed by a failed query so they are retried on
854    /// the next execution.
855    ///
856    fn restore_dirty_windows(&self, dirty_restore: &DirtyRestore) {
857        match dirty_restore {
858            DirtyRestore::Scoped(filter) => self.restore_scoped_dirty_windows(filter),
859            DirtyRestore::Unscoped(dirty_windows) => self
860                .state
861                .write()
862                .unwrap()
863                .dirty_time_windows
864                .add_dirty_windows(dirty_windows),
865        }
866    }
867
868    /// Restore the dirty signal for a plan that was generated but failed before
869    /// it could prove any checkpoint advancement.
870    fn restore_dirty_windows_after_failure(&self, query: &PlanInfo) {
871        self.restore_dirty_windows(&query.dirty_restore);
872    }
873
874    /// Restore scoped windows through `TaskState` so fenced repair can decide
875    /// whether they go back to pending repair or live dirty state.
876    fn restore_scoped_dirty_windows(&self, filter: &FilterExprInfo) {
877        self.state.write().unwrap().restore_scoped_windows(filter);
878    }
879
880    /// Run a fallible scoped operation and restore its consumed windows if plan
881    /// generation/rewrite fails before execution.
882    fn restore_scoped_dirty_windows_on_err<T>(
883        &self,
884        filter: &FilterExprInfo,
885        result: Result<T, Error>,
886    ) -> Result<T, Error> {
887        result.inspect_err(|_| {
888            self.restore_scoped_dirty_windows(filter);
889        })
890    }
891
892    /// Restore an unscoped dirty signal consumed by an explicit full-query or
893    /// incremental-delta plan.
894    fn restore_unscoped_dirty_windows(&self, dirty_windows: &DirtyTimeWindows) {
895        self.state
896            .write()
897            .unwrap()
898            .dirty_time_windows
899            .add_dirty_windows(dirty_windows);
900    }
901
902    /// Run a fallible unscoped operation and restore the dirty signal if it
903    /// fails before a query is executed.
904    fn restore_unscoped_dirty_windows_on_err<T>(
905        &self,
906        dirty_windows: &DirtyTimeWindows,
907        result: Result<T, Error>,
908    ) -> Result<T, Error> {
909        result.inspect_err(|_| {
910            self.restore_unscoped_dirty_windows(dirty_windows);
911        })
912    }
913
914    /// Consume the live dirty signal for an unscoped query while keeping a copy
915    /// that can be restored if planning or execution fails.
916    fn drain_dirty_windows_signal(&self) -> (bool, DirtyTimeWindows) {
917        let mut state = self.state.write().unwrap();
918        let dirty_windows_to_restore = state.dirty_time_windows.clone();
919        let is_dirty = !dirty_windows_to_restore.is_empty();
920        state.dirty_time_windows.clean();
921        (is_dirty, dirty_windows_to_restore)
922    }
923
924    #[allow(clippy::too_many_arguments)]
925    /// Build an unfiltered plan for explicit full-query or incremental-delta
926    /// coverage. Callers pass the consumed dirty signal for failure restoration.
927    async fn gen_unfiltered_plan_info(
928        &self,
929        engine: QueryEngineRef,
930        query_ctx: QueryContextRef,
931        sink_table_schema: Arc<Schema>,
932        primary_key_indices: &[usize],
933        allow_partial: bool,
934        dirty_windows_to_restore: DirtyTimeWindows,
935        retention_filter: Option<(&str, Timestamp, &'static str)>,
936        coverage: QueryCoverage,
937    ) -> Result<PlanInfo, Error> {
938        let mut plan = self.restore_unscoped_dirty_windows_on_err(
939            &dirty_windows_to_restore,
940            gen_plan_with_matching_schema(
941                &self.config.query,
942                query_ctx,
943                engine,
944                sink_table_schema,
945                primary_key_indices,
946                allow_partial,
947            )
948            .await,
949        )?;
950
951        if let Some((col_name, lower_bound, context)) = retention_filter {
952            let lower = self.restore_unscoped_dirty_windows_on_err(
953                &dirty_windows_to_restore,
954                to_df_literal(lower_bound),
955            )?;
956            let retention_filter = col(col_name).gt_eq(lit(lower));
957            let mut add_filter = AddFilterRewriter::new(retention_filter);
958            plan = self.restore_unscoped_dirty_windows_on_err(
959                &dirty_windows_to_restore,
960                plan.clone()
961                    .rewrite(&mut add_filter)
962                    .with_context(|_| DatafusionSnafu {
963                        context: format!(
964                            "Failed to apply {context} expire_after filter to plan:\n {}\n",
965                            plan
966                        ),
967                    })
968                    .map(|rewrite| rewrite.data),
969            )?;
970        }
971
972        Ok(PlanInfo {
973            plan,
974            dirty_restore: DirtyRestore::Unscoped(dirty_windows_to_restore),
975            coverage,
976        })
977    }
978
979    #[allow(clippy::too_many_arguments)]
980    /// Build an unfiltered plan only when the live dirty signal was present;
981    /// otherwise skip this round without querying.
982    async fn gen_unfiltered_plan_info_if_dirty(
983        &self,
984        engine: QueryEngineRef,
985        query_ctx: QueryContextRef,
986        sink_table_schema: Arc<Schema>,
987        primary_key_indices: &[usize],
988        allow_partial: bool,
989        retention_filter: Option<(&str, Timestamp, &'static str)>,
990        coverage: QueryCoverage,
991    ) -> Result<Option<PlanInfo>, Error> {
992        let (is_dirty, dirty_windows_to_restore) = self.drain_dirty_windows_signal();
993        if !is_dirty {
994            debug!("Flow id={:?}, no new data, not update", self.config.flow_id);
995            return Ok(None);
996        }
997
998        self.gen_unfiltered_plan_info(
999            engine,
1000            query_ctx,
1001            sink_table_schema,
1002            primary_key_indices,
1003            allow_partial,
1004            dirty_windows_to_restore,
1005            retention_filter,
1006            coverage,
1007        )
1008        .await
1009        .map(Some)
1010    }
1011
1012    fn handle_executed_query_failure(&self, query: Option<&PlanInfo>) {
1013        if let Some(query) = query {
1014            self.restore_dirty_windows_after_failure(query);
1015        }
1016    }
1017
1018    /// start executing query in a loop, break when receive shutdown signal
1019    ///
1020    /// any error will be logged when executing query.
1021    ///
1022    /// Dispatches to:
1023    /// - scheduled loop when `flow_eval_interval.is_some()`
1024    /// - adaptive dirty-window loop otherwise
1025    pub async fn start_executing_loop(
1026        &self,
1027        engine: QueryEngineRef,
1028        frontend_client: Arc<FrontendClient>,
1029    ) {
1030        if self.config.flow_eval_interval.is_some() {
1031            self.start_scheduled_loop(engine, frontend_client).await;
1032        } else {
1033            self.start_adaptive_loop(engine, frontend_client).await;
1034        }
1035    }
1036
1037    /// Scheduled batching loop for flows with `EVAL INTERVAL`.
1038    ///
1039    /// Uses the pre-parsed `EvalSchedule` from `TaskConfig` and selects due
1040    /// scheduled times using bounded catch-up semantics. Each scheduled time is the
1041    /// scheduled evaluation time used as logical `now()` for that attempt.
1042    /// Each attempt temporarily sets `flow.scheduled_time_millis` on the
1043    /// task's `QueryContext` and executes under the existing `execution_lock`.
1044    /// After every attempt (success, no-op, or failure) the in-memory
1045    /// cursor advances.
1046    async fn start_scheduled_loop(
1047        &self,
1048        engine: QueryEngineRef,
1049        frontend_client: Arc<FrontendClient>,
1050    ) {
1051        let flow_id_str = self.config.flow_id.to_string();
1052
1053        let schedule = match &self.config.eval_schedule {
1054            Some(s) => s.clone(),
1055            None => {
1056                let eval_interval_secs = self
1057                    .config
1058                    .flow_eval_interval
1059                    .map(|d| d.as_secs() as i64)
1060                    .expect("checked by caller");
1061
1062                // Fallback: no typed config provided. Compute defaults
1063                // anchored at epoch/start=0.
1064                match EvalSchedule::from_config(Some(eval_interval_secs), None) {
1065                    Ok(Some(s)) => s,
1066                    Ok(None) => {
1067                        warn!(
1068                            "Flow {}: EVAL INTERVAL set but no schedule parsed; exiting loop",
1069                            flow_id_str
1070                        );
1071                        return;
1072                    }
1073                    Err(e) => {
1074                        warn!(
1075                            "Flow {}: Failed to parse eval schedule: {}; exiting loop",
1076                            flow_id_str, e
1077                        );
1078                        return;
1079                    }
1080                }
1081            }
1082        };
1083
1084        // Initial cursor is one interval before start so the first due
1085        // scheduled time is `start_secs`. An unrepresentable difference is an
1086        // explicit error, never a saturated cursor that would silently skip
1087        // the first scheduled tick.
1088        let mut cursor_secs =
1089            match initial_schedule_cursor(schedule.start_secs, schedule.interval_secs) {
1090                Ok(cursor) => cursor,
1091                Err(e) => {
1092                    warn!(
1093                        "Flow {}: invalid eval schedule, exiting loop: {}",
1094                        flow_id_str, e
1095                    );
1096                    return;
1097                }
1098            };
1099
1100        info!(
1101            "Flow {}: entering scheduled loop, interval={}s, start={}, anchor={}, policy={:?}, max_runs={}, max_lag={}s",
1102            flow_id_str,
1103            schedule.interval_secs,
1104            schedule.start_secs,
1105            schedule.anchor_secs,
1106            schedule.missed_tick_policy,
1107            schedule.max_runs,
1108            schedule.max_lag_secs,
1109        );
1110
1111        loop {
1112            if self.is_shutdown_signaled() {
1113                break;
1114            }
1115
1116            let wall_now_secs = wall_clock_unix_secs();
1117
1118            let due = match select_due_scheduled_times(&schedule, cursor_secs, wall_now_secs) {
1119                Ok(d) => d,
1120                Err(e) => {
1121                    warn!(
1122                        "Flow {}: invalid eval schedule, exiting loop: {}",
1123                        flow_id_str, e
1124                    );
1125                    return;
1126                }
1127            };
1128
1129            if due.scheduled_times_secs.is_empty() {
1130                if due.skipped > 0 {
1131                    warn!(
1132                        "Flow {}: all {} due scheduled times skipped by max-lag, advancing cursor to wall-clock ({wall_now_secs}) to avoid re-skipping",
1133                        flow_id_str, due.skipped
1134                    );
1135                    cursor_secs = wall_now_secs;
1136                    continue;
1137                }
1138
1139                // No due yet — sleep until the next scheduled time.
1140                let next = match schedule.next_scheduled_time_after(cursor_secs) {
1141                    Ok(next) => next,
1142                    Err(e) => {
1143                        warn!(
1144                            "Flow {}: cannot advance eval schedule past cursor {cursor_secs}: {e}; exiting loop",
1145                            flow_id_str
1146                        );
1147                        return;
1148                    }
1149                };
1150                if next <= wall_now_secs {
1151                    // Shouldn't happen given select_due_scheduled_times returned empty,
1152                    // but guard against clock skew / logic error.
1153                    cursor_secs = wall_now_secs;
1154                    continue;
1155                }
1156                let wait_secs = match sleep_delta_secs(next, wall_now_secs) {
1157                    Ok(wait_secs) => wait_secs,
1158                    Err(e) => {
1159                        warn!(
1160                            "Flow {}: cannot sleep until next scheduled time {}: {e}; exiting loop",
1161                            flow_id_str, next
1162                        );
1163                        return;
1164                    }
1165                };
1166                let wait_dur = Duration::from_secs(wait_secs);
1167                debug!(
1168                    "Flow {}: no due scheduled times, sleeping for {}s until next scheduled time at {}",
1169                    flow_id_str, wait_secs, next
1170                );
1171                tokio::time::sleep(wait_dur).await;
1172                continue;
1173            }
1174
1175            if due.skipped > 0 {
1176                info!(
1177                    "Flow {}: {} due scheduled times, {} skipped (catch-up)",
1178                    flow_id_str,
1179                    due.scheduled_times_secs.len(),
1180                    due.skipped
1181                );
1182            }
1183
1184            // Execute scheduled times oldest → newest.
1185            for scheduled_time_secs in &due.scheduled_times_secs {
1186                if self.is_shutdown_signaled() {
1187                    break;
1188                }
1189
1190                METRIC_FLOW_BATCHING_ENGINE_START_QUERY_CNT
1191                    .with_label_values(&[&flow_id_str])
1192                    .inc();
1193
1194                let outcome = self
1195                    .execute_once_serialized_at_scheduled_time(
1196                        &engine,
1197                        &frontend_client,
1198                        *scheduled_time_secs,
1199                    )
1200                    .await;
1201
1202                // Advance cursor regardless of outcome.
1203                cursor_secs = *scheduled_time_secs;
1204
1205                match outcome.result {
1206                    Ok(Some((rows, elapsed))) => {
1207                        debug!(
1208                            "Flow {}: scheduled time {} completed, rows={}, elapsed={:?}",
1209                            flow_id_str, scheduled_time_secs, rows, elapsed
1210                        );
1211                    }
1212                    Ok(None) => {
1213                        debug!(
1214                            "Flow {}: scheduled time {} produced no query (no dirty signal or no-op)",
1215                            flow_id_str, scheduled_time_secs
1216                        );
1217                    }
1218                    Err(err) => {
1219                        warn!(
1220                            "Flow {}: scheduled time {} failed: {:?}",
1221                            flow_id_str, scheduled_time_secs, err
1222                        );
1223                        METRIC_FLOW_BATCHING_ENGINE_ERROR_CNT
1224                            .with_label_values(&[&flow_id_str])
1225                            .inc();
1226                        // Dirty-window restoration is handled by the
1227                        // existing `handle_executed_query_failure` inside
1228                        // `execute_once_unlocked`.
1229                    }
1230                }
1231            }
1232        }
1233    }
1234
1235    /// Existing adaptive dirty-window loop for flows without `EVAL INTERVAL`.
1236    async fn start_adaptive_loop(
1237        &self,
1238        engine: QueryEngineRef,
1239        frontend_client: Arc<FrontendClient>,
1240    ) {
1241        let flow_id_str = self.config.flow_id.to_string();
1242        let mut max_window_cnt = None;
1243        loop {
1244            if self.is_shutdown_signaled() {
1245                break;
1246            }
1247            METRIC_FLOW_BATCHING_ENGINE_START_QUERY_CNT
1248                .with_label_values(&[&flow_id_str])
1249                .inc();
1250
1251            let min_refresh = self.config.batch_opts.experimental_min_refresh_duration;
1252
1253            let outcome = self
1254                .execute_once_serialized_with_outcome(&engine, &frontend_client, max_window_cnt)
1255                .await;
1256
1257            match outcome.result {
1258                Ok(Some(_)) => {
1259                    max_window_cnt = max_window_cnt.map(|cnt| {
1260                        (cnt + 1).min(self.config.batch_opts.experimental_max_filter_num_per_query)
1261                    });
1262
1263                    let sleep_until = {
1264                        let state = self.state.write().unwrap();
1265
1266                        let time_window_size = self
1267                            .config
1268                            .time_window_expr
1269                            .as_ref()
1270                            .and_then(|t| *t.time_window_size());
1271
1272                        let prefer_short_incremental_cadence = state.checkpoint_mode()
1273                            == CheckpointMode::Incremental
1274                            && !state.is_incremental_disabled();
1275
1276                        state.get_next_start_query_time(
1277                            self.config.flow_id,
1278                            &time_window_size,
1279                            min_refresh,
1280                            Some(self.config.batch_opts.query_timeout),
1281                            self.config.batch_opts.experimental_max_filter_num_per_query,
1282                            prefer_short_incremental_cadence,
1283                        )
1284                    };
1285
1286                    tokio::time::sleep_until(sleep_until).await;
1287                }
1288                Ok(None) => {
1289                    debug!(
1290                        "Flow id = {:?} found no new data, sleep for {:?} then continue",
1291                        self.config.flow_id, min_refresh
1292                    );
1293                    tokio::time::sleep(min_refresh).await;
1294                    continue;
1295                }
1296                Err(err) => {
1297                    METRIC_FLOW_BATCHING_ENGINE_ERROR_CNT
1298                        .with_label_values(&[&flow_id_str])
1299                        .inc();
1300                    match outcome.new_query {
1301                        Some(query) => {
1302                            common_telemetry::error!(err; "Failed to execute query for flow={} with query: {}", self.config.flow_id, query.plan);
1303                            max_window_cnt = Some(1);
1304                        }
1305                        None => {
1306                            common_telemetry::error!(err; "Failed to generate query for flow={}", self.config.flow_id)
1307                        }
1308                    }
1309                    tokio::time::sleep(min_refresh).await;
1310                }
1311            }
1312        }
1313    }
1314
1315    /// Check whether the shutdown signal has been received.
1316    fn is_shutdown_signaled(&self) -> bool {
1317        let mut state = self.state.write().unwrap();
1318        match state.shutdown_rx.try_recv() {
1319            Ok(()) | Err(TryRecvError::Closed) => true,
1320            Err(TryRecvError::Empty) => false,
1321        }
1322    }
1323
1324    /// Execute one scheduled attempt, temporarily setting
1325    /// `flow.scheduled_time_millis` on the task's QueryContext so
1326    /// SQL/TQL `now()` resolves to the logical scheduled time.
1327    ///
1328    /// The extension is removed after the attempt so a later manual
1329    /// `flush_flow` does not reuse a stale scheduled time.
1330    async fn execute_once_serialized_at_scheduled_time(
1331        &self,
1332        engine: &QueryEngineRef,
1333        frontend_client: &Arc<FrontendClient>,
1334        scheduled_time_secs: i64,
1335    ) -> ExecuteOnceOutcome {
1336        let _execution_guard = self.execution_lock.lock().await;
1337
1338        struct QueryContextRestoreGuard {
1339            state: Arc<RwLock<TaskState>>,
1340            old_ctx: Option<QueryContextRef>,
1341        }
1342
1343        impl Drop for QueryContextRestoreGuard {
1344            fn drop(&mut self) {
1345                let Some(old_ctx) = self.old_ctx.take() else {
1346                    return;
1347                };
1348                if let Ok(mut state) = self.state.write() {
1349                    state.query_ctx = old_ctx;
1350                }
1351            }
1352        }
1353
1354        // Convert to milliseconds before touching the task state so an
1355        // unrepresentable scheduled time fails as an explicit error without
1356        // ever installing a saturated (off-phase) extension value.
1357        let scheduled_time_millis = match scheduled_time_millis(scheduled_time_secs) {
1358            Ok(millis) => millis,
1359            Err(e) => {
1360                return ExecuteOnceOutcome {
1361                    new_query: None,
1362                    result: Err(e),
1363                };
1364            }
1365        };
1366
1367        // Clone the current QueryContext and add the scheduled time
1368        // extension, then swap it into the task state for this attempt.
1369        let old_ctx = {
1370            let mut state = self.state.write().unwrap();
1371            let old = state.query_ctx.clone();
1372            let mut new_ctx = (*old).clone();
1373            new_ctx.set_extension(
1374                query::options::FLOW_SCHEDULED_TIME_MILLIS,
1375                scheduled_time_millis.to_string(),
1376            );
1377            state.query_ctx = Arc::new(new_ctx);
1378            old
1379        };
1380        let restore_guard = QueryContextRestoreGuard {
1381            state: self.state.clone(),
1382            old_ctx: Some(old_ctx),
1383        };
1384
1385        let outcome = self
1386            .execute_once_unlocked(engine, frontend_client, None)
1387            .await;
1388
1389        // Restore while still holding `execution_lock` so no future manual
1390        // flush can observe the temporary scheduled time. The guard also
1391        // restores during unwind/cancellation.
1392        drop(restore_guard);
1393
1394        outcome
1395    }
1396
1397    /// Generate the create table SQL
1398    ///
1399    /// the auto created table will automatically added a `update_at` Milliseconds DEFAULT now() column in the end
1400    /// (for compatibility with flow streaming mode)
1401    ///
1402    /// and it will use first timestamp column as time index, all other columns will be added as normal columns and nullable
1403    async fn gen_create_table_expr(
1404        &self,
1405        engine: QueryEngineRef,
1406    ) -> Result<CreateTableExpr, Error> {
1407        let query_ctx = self.state.read().unwrap().query_ctx.clone();
1408        let plan =
1409            sql_to_df_plan(query_ctx.clone(), engine.clone(), &self.config.query, true).await?;
1410        create_table_with_expr(&plan, &self.config.sink_table_name, &self.config.query_type)
1411    }
1412
1413    /// Incremental delta scans are unfiltered by dirty windows; the sequence
1414    /// range, not a time predicate, defines source correctness.
1415    fn should_use_unfiltered_incremental_delta(&self) -> bool {
1416        let state = self.state.read().unwrap();
1417        state.checkpoint_mode() == CheckpointMode::Incremental
1418            && !state.is_incremental_disabled()
1419            && matches!(self.config.query_type, QueryType::Sql)
1420    }
1421
1422    /// Generate the next plan and classify its coverage so checkpoint handling
1423    /// knows whether it is full-query, scoped repair, fenced repair, or delta.
1424    async fn gen_query_with_time_window(
1425        &self,
1426        engine: QueryEngineRef,
1427        sink_table_schema: &Arc<Schema>,
1428        primary_key_indices: &[usize],
1429        allow_partial: bool,
1430        max_window_cnt: Option<usize>,
1431    ) -> Result<Option<PlanInfo>, Error> {
1432        let query_ctx = self.state.read().unwrap().query_ctx.clone();
1433        let start = SystemTime::now();
1434        let since_the_epoch = start
1435            .duration_since(UNIX_EPOCH)
1436            .expect("Time went backwards");
1437        let low_bound = self
1438            .config
1439            .expire_after
1440            .map(|e| since_the_epoch.as_secs() - e as u64)
1441            .unwrap_or(u64::MIN);
1442
1443        let low_bound = Timestamp::new_second(low_bound as i64);
1444
1445        let expire_time_window_bound = self
1446            .config
1447            .time_window_expr
1448            .as_ref()
1449            .map(|expr| expr.eval(low_bound))
1450            .transpose()?;
1451
1452        let (expire_lower_bound, expire_upper_bound) = match (
1453            expire_time_window_bound,
1454            &self.config.query_type,
1455        ) {
1456            (Some((Some(l), Some(u))), QueryType::Sql) => (l, u),
1457            (None, QueryType::Sql) if self.config.flow_eval_interval.is_none() => {
1458                return UnexpectedSnafu {
1459                    reason: format!(
1460                        "Flow id={} reached execution without a time-window expression or EVAL INTERVAL; create-flow validation should have rejected it",
1461                        self.config.flow_id
1462                    ),
1463                }
1464                .fail();
1465            }
1466            _ => {
1467                // Explicit full-query flows (TQL and evaluation-interval SQL
1468                // plans whose shape cannot be safely dirty-window pruned) are
1469                // allowed to run as unfiltered full snapshots. This is distinct
1470                // from using unfiltered full as a fallback after scoped repair or
1471                // incremental rewrite failed.
1472                let (_, dirty_windows_to_restore) = self.drain_dirty_windows_signal();
1473
1474                let plan_info = self
1475                    .gen_unfiltered_plan_info(
1476                        engine,
1477                        query_ctx,
1478                        sink_table_schema.clone(),
1479                        primary_key_indices,
1480                        allow_partial,
1481                        dirty_windows_to_restore,
1482                        None,
1483                        QueryCoverage::UnfilteredFull,
1484                    )
1485                    .await?;
1486
1487                return Ok(Some(plan_info));
1488            }
1489        };
1490
1491        debug!(
1492            "Flow id = {:?}, found time window: precise_lower_bound={:?}, precise_upper_bound={:?} with dirty time windows: {:?}",
1493            self.config.flow_id,
1494            expire_lower_bound,
1495            expire_upper_bound,
1496            self.state.read().unwrap().dirty_time_windows
1497        );
1498        let window_size = expire_upper_bound
1499            .sub(&expire_lower_bound)
1500            .with_context(|| UnexpectedSnafu {
1501                reason: format!(
1502                    "Can't get window size from {expire_upper_bound:?} - {expire_lower_bound:?}"
1503                ),
1504            })?;
1505        let col_name = self
1506            .config
1507            .time_window_expr
1508            .as_ref()
1509            .map(|expr| expr.column_name.clone())
1510            .with_context(|| UnexpectedSnafu {
1511                reason: format!(
1512                    "Flow id={:?}, Failed to get column name from time window expr",
1513                    self.config.flow_id
1514                ),
1515            })?;
1516
1517        if self.should_use_unfiltered_incremental_delta() {
1518            // In incremental mode, source correctness is defined by the
1519            // per-region sequence range `(checkpoint, scan-open snapshot]`, not
1520            // by dirty-window predicates. Dirty windows are only a scheduling
1521            // signal here. Applying a stale dirty-window filter to the source can
1522            // exclude rows that are inside the returned watermark and make a
1523            // checkpoint advance skip them forever. The sink side is also left
1524            // unfiltered by dirty windows; the incremental rewrite joins the
1525            // delta groups with the full sink state for correctness. Future
1526            // dynamic filters can prune sink reads as a pure optimization.
1527            let retention_filter = self
1528                .config
1529                .expire_after
1530                .map(|_| (col_name.as_str(), expire_lower_bound, "incremental"));
1531            return self
1532                .gen_unfiltered_plan_info_if_dirty(
1533                    engine,
1534                    query_ctx,
1535                    sink_table_schema.clone(),
1536                    primary_key_indices,
1537                    allow_partial,
1538                    retention_filter,
1539                    QueryCoverage::IncrementalDelta,
1540                )
1541                .await;
1542        }
1543
1544        let (expr, coverage) = {
1545            let mut state = self.state.write().unwrap();
1546            let window_cnt = max_window_cnt
1547                .unwrap_or(self.config.batch_opts.experimental_max_filter_num_per_query);
1548            let expr = state.gen_scoped_filter_exprs(
1549                &col_name,
1550                Some(expire_lower_bound),
1551                window_size,
1552                window_cnt,
1553                self.config.flow_id,
1554                Some(self),
1555            )?;
1556            let repair_high = state
1557                .pending_fenced_repair()
1558                .map(|repair| repair.high().clone());
1559            let coverage = if let Some(high) = repair_high {
1560                QueryCoverage::FencedRepairChunk { high }
1561            } else {
1562                QueryCoverage::ScopedBaseRepair
1563            };
1564            (expr, coverage)
1565        };
1566
1567        let Some(expr) = expr else {
1568            // no new data, hence no need to update
1569            debug!("Flow id={:?}, no new data, not update", self.config.flow_id);
1570            return Ok(None);
1571        };
1572
1573        let filter_sql = expr_to_sql(&expr.expr)
1574            .map(|sql| sql.to_string())
1575            .unwrap_or_else(|err| format!("<failed to format filter expr: {err}>"));
1576
1577        debug!(
1578            "Flow id={:?}, Generated filter expr: {:?}",
1579            self.config.flow_id, filter_sql
1580        );
1581
1582        let mut add_filter = AddFilterRewriter::new(expr.expr.clone());
1583        let mut add_auto_column = ColumnMatcherRewriter::new(
1584            sink_table_schema.clone(),
1585            primary_key_indices.to_vec(),
1586            allow_partial,
1587        );
1588
1589        let plan = self.restore_scoped_dirty_windows_on_err(
1590            &expr,
1591            sql_to_df_plan(query_ctx.clone(), engine.clone(), &self.config.query, false).await,
1592        )?;
1593        let rewrite = self.restore_scoped_dirty_windows_on_err(
1594            &expr,
1595            plan.clone()
1596                .rewrite(&mut add_filter)
1597                .and_then(|p| p.data.rewrite(&mut add_auto_column))
1598                .with_context(|_| DatafusionSnafu {
1599                    context: format!("Failed to rewrite plan:\n {}\n", plan),
1600                })
1601                .map(|rewrite| rewrite.data),
1602        )?;
1603        // only apply optimize after complex rewrite is done
1604        let new_plan = self.restore_scoped_dirty_windows_on_err(
1605            &expr,
1606            apply_df_optimizer(rewrite, &query_ctx).await,
1607        )?;
1608
1609        let info = PlanInfo {
1610            plan: new_plan.clone(),
1611            dirty_restore: DirtyRestore::Scoped(expr),
1612            coverage,
1613        };
1614
1615        Ok(Some(info))
1616    }
1617}
1618
1619#[cfg(test)]
1620mod test;