Struct BatchingTask 

pub struct BatchingTask {
    pub config: Arc<TaskConfig>,
    pub state: Arc<RwLock<TaskState>>,
}

Fields

config: Arc<TaskConfig>

state: Arc<RwLock<TaskState>>

Implementations

impl BatchingTask

pub(super) fn query_failure_reason(err: &Error) -> FlowQueryFallbackReason

pub(super) fn apply_query_failure_to_state( state: &mut TaskState, elapsed: Duration, reason: FlowQueryFallbackReason, ) -> Option<FlowCheckpointDecision>

pub(super) fn apply_query_result_to_state( state: &mut TaskState, res: &OutputWithMetrics, elapsed: Duration, can_advance_checkpoints: bool, ) -> FlowCheckpointDecision

pub(super) fn record_checkpoint_decision( flow_id: u64, decision: FlowCheckpointDecision, )

pub(super) fn record_query_mode(flow_id: u64, mode: CheckpointMode)

impl BatchingTask

async fn sink_table_id(&self) -> Result<TableId, Error>

pub(super) async fn prepare_plan_for_incremental( &self, plan: &LogicalPlan, dirty_filter: Option<&FilterExprInfo>, ) -> Result<Option<LogicalPlan>, Error>

For incremental-mode SQL queries, attempt to prepare an executable plan that is safe for incremental scan extensions.

Returns Some(plan) when incremental extensions are safe, and None when the caller should execute the original plan without incremental extensions. The returned plan may be either a rewritten delta-LEFT-JOIN-sink merge plan or the original plan. In particular, plain GROUP BY queries with no aggregate merge columns are incremental safe without a rewrite, so they return Some(original_plan).

pub(super) async fn build_flow_query_extensions( &self, incremental_safe: bool, can_advance_checkpoints: bool, ) -> Result<Vec<(&'static str, String)>, Error>

impl BatchingTask

pub fn try_new(_: TaskArgs<'_>) -> Result<Self, Error>

pub fn last_execution_time_millis(&self) -> Option<i64>

pub fn mark_all_windows_as_dirty(&self) -> Result<(), Error>

mark time window range (now - expire_after, now) as dirty (or (0, now) if expire_after not set)

useful for flush_flow to flush dirty time windows range

pub async fn check_or_create_sink_table( &self, engine: &QueryEngineRef, frontend_client: &Arc<FrontendClient>, ) -> Result<Option<(usize, Duration)>, Error>

Create sink table if not exists

async fn is_table_exist(&self, table_name: &[String; 3]) -> Result<bool, Error>

pub async fn gen_exec_once( &self, engine: &QueryEngineRef, frontend_client: &Arc<FrontendClient>, max_window_cnt: Option<usize>, ) -> Result<Option<(usize, Duration)>, Error>

pub async fn gen_insert_plan( &self, engine: &QueryEngineRef, max_window_cnt: Option<usize>, ) -> Result<Option<PlanInfo>, Error>

pub async fn create_table( &self, frontend_client: &Arc<FrontendClient>, expr: CreateTableExpr, ) -> Result<(), Error>

pub async fn execute_logical_plan( &self, frontend_client: &Arc<FrontendClient>, plan: &LogicalPlan, dirty_filter: Option<&FilterExprInfo>, can_advance_checkpoints: bool, ) -> Result<Option<(usize, Duration)>, Error>

fn restore_dirty_windows_after_failure(&self, query: &PlanInfo)

Restore dirty windows consumed by a failed query so they are retried on the next execution.

fn restore_scoped_dirty_windows(&self, filter: &FilterExprInfo)

fn restore_scoped_dirty_windows_on_err<T>( &self, filter: &FilterExprInfo, result: Result<T, Error>, ) -> Result<T, Error>

fn handle_executed_query_failure(&self, query: Option<&PlanInfo>)

pub async fn start_executing_loop( &self, engine: QueryEngineRef, frontend_client: Arc<FrontendClient>, )

start executing query in a loop, break when receive shutdown signal

any error will be logged when executing query

async fn gen_create_table_expr( &self, engine: QueryEngineRef, ) -> Result<CreateTableExpr, Error>

Generate the create table SQL

the auto created table will automatically added a update_at Milliseconds DEFAULT now() column in the end (for compatibility with flow streaming mode)

and it will use first timestamp column as time index, all other columns will be added as normal columns and nullable

async fn gen_query_with_time_window( &self, engine: QueryEngineRef, sink_table_schema: &Arc<Schema>, primary_key_indices: &[usize], allow_partial: bool, max_window_cnt: Option<usize>, ) -> Result<Option<PlanInfo>, Error>

will merge and use the first ten time window in query

Trait Implementations

impl Clone for BatchingTask

fn clone(&self) -> BatchingTask

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.