feat(flow): support eval schedule offsets (#8878)

* feat(flow): support eval schedule offsets

Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com>

* fix(flow): remove redundant schedule assertion

Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com>

* refactor(flow): trim eval offset compatibility scope

Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com>

* test(flow): trim eval offset edge coverage

Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com>

* docs(flow): trim eval offset comment noise

Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com>

* fix(flow): address eval offset review feedback

Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com>

* test(compat): cover Flow eval offset persistence

Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com>

---------

Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com>
This commit is contained in:
discord9
2026-09-04 02:39:59 +00:00
committed by GitHub
parent 84bd993131
commit ad7b0ace64
22 changed files with 1702 additions and 151 deletions
@@ -16,7 +16,7 @@ use std::sync::{Arc, Weak};
use common_catalog::consts::INFORMATION_SCHEMA_FLOW_TABLE_ID;
use common_error::ext::BoxedError;
use common_meta::ddl::create_flow::FlowType;
use common_meta::ddl::create_flow::{FlowType, effective_eval_schedule_from_flow_info};
use common_meta::key::FlowId;
use common_meta::key::flow::FlowMetadataManager;
use common_meta::key::flow::flow_info::FlowInfoValue;
@@ -168,6 +168,11 @@ impl InformationSchemaFlows {
if_not_exists: true,
expire_after: flow_info.expire_after(),
eval_interval: flow_info.eval_interval(),
eval_offset: effective_eval_schedule_from_flow_info(flow_info)
.map_err(BoxedError::new)
.context(InternalSnafu)?
.map(|schedule| schedule.anchor_secs)
.filter(|anchor_secs| *anchor_secs != 0),
comment,
flow_options: sql::statements::OptionMap::from_filtered_string_map(
flow_info.options(),
@@ -464,3 +469,97 @@ impl DfPartitionStream for InformationSchemaFlows {
))
}
}
#[cfg(test)]
mod tests {
use std::collections::{BTreeMap, HashMap};
use common_meta::key::flow::flow_info::{FlowMissedTickPolicy, FlowScheduleConfig, FlowStatus};
use sql::parser::ParseOptions;
use table::table_name::TableName;
use super::*;
fn flow_info_for_show_create(
raw_sql: &str,
eval_interval_secs: Option<i64>,
anchor_secs: i64,
options: HashMap<String, String>,
) -> FlowInfoValue {
let created_time = chrono::DateTime::from_timestamp(1_700_000_000, 0).unwrap();
FlowInfoValue {
source_table_ids: vec![],
all_source_table_names: vec![],
unresolved_source_table_names: vec![],
sink_table_name: TableName::new("greptime", "public", "sink"),
flownode_ids: BTreeMap::new(),
catalog_name: "greptime".to_string(),
query_context: None,
flow_name: "my_flow".to_string(),
raw_sql: raw_sql.to_string(),
expire_after: None,
eval_interval_secs,
comment: String::new(),
options,
status: FlowStatus::Active,
created_time,
updated_time: created_time,
eval_schedule: eval_interval_secs.map(|interval| FlowScheduleConfig {
anchor_secs,
start_secs: anchor_secs + interval,
missed_tick_policy: FlowMissedTickPolicy::BoundedCatchUp,
catchup_max_runs: 3,
catchup_max_lag_secs: 300,
}),
}
}
#[test]
fn test_generate_show_create_flow_with_eval_offset() {
// `raw_sql` stores only the query part (the `AS` clause).
let raw_sql = "SELECT max(c1) FROM public.src";
let flow_info = flow_info_for_show_create(raw_sql, Some(3600), 120, HashMap::new());
let sql = InformationSchemaFlows::generate_show_create_flow(&flow_info).unwrap();
assert!(
sql.contains("EVAL OFFSET '120 s'"),
"EVAL OFFSET must be emitted, got:\n{sql}"
);
assert!(
!sql.contains("__greptime_internal_eval_offset_secs"),
"internal key must be absent, got:\n{sql}"
);
let stmts = ParserContext::create_with_dialect(
&sql,
&GreptimeDbDialect {},
ParseOptions::default(),
)
.unwrap();
let Statement::CreateFlow(reparsed) = &stmts[0] else {
panic!("unexpected stmt: {:?}", stmts[0]);
};
assert_eq!(reparsed.eval_interval, Some(3600));
assert_eq!(reparsed.eval_offset, Some(120));
}
#[test]
fn test_generate_show_create_flow_omits_zero_offset() {
let raw_sql = "SELECT max(c1) FROM public.src";
let flow_info = flow_info_for_show_create(raw_sql, Some(3600), 0, HashMap::new());
let sql = InformationSchemaFlows::generate_show_create_flow(&flow_info).unwrap();
assert!(
!sql.contains("EVAL OFFSET"),
"zero offset must be omitted, got:\n{sql}"
);
let stmts = ParserContext::create_with_dialect(
&sql,
&GreptimeDbDialect {},
ParseOptions::default(),
)
.unwrap();
let Statement::CreateFlow(reparsed) = &stmts[0] else {
panic!("unexpected stmt: {:?}", stmts[0]);
};
assert_eq!(reparsed.eval_offset, None);
}
}
+137 -36
View File
@@ -21,6 +21,7 @@ use api::v1::ExpireAfter;
use api::v1::flow::flow_request::Body as PbFlowRequest;
use api::v1::flow::{CreateRequest, FlowRequest, FlowRequestHeader};
use async_trait::async_trait;
use chrono::{DateTime, Utc};
use common_catalog::format_full_flow_name;
use common_procedure::error::{FromJsonSnafu, ToJsonSnafu};
use common_procedure::{
@@ -32,7 +33,7 @@ use common_telemetry::tracing_context::TracingContext;
use futures::future::join_all;
use itertools::Itertools;
use serde::{Deserialize, Serialize};
use snafu::{ResultExt, ensure};
use snafu::{OptionExt, ResultExt, ensure};
use strum::AsRefStr;
use table::metadata::TableId;
use table::table_name::TableName;
@@ -222,7 +223,7 @@ impl CreateFlowProcedure {
.prev_flow_info_value
.as_ref()
.map(|v| v.get_inner_ref()),
);
)?;
self.data.state = if self.data.is_pending() {
self.data.peers.clear();
@@ -465,6 +466,14 @@ pub fn get_flow_type_from_options(flow_task: &CreateFlowTask) -> Result<FlowType
/// The flow option key for creating pending flow metadata when source tables do not exist.
pub const DEFER_ON_MISSING_SOURCE_KEY: &str = "defer_on_missing_source";
/// Internal transient key used to pass the typed `EVAL OFFSET` (whole seconds)
/// from the operator to meta through `CreateFlowExpr.flow_options`. Inserted by
/// the operator only after user option validation; parsed and stripped by
/// `CreateFlowTask::try_from`. Must never be accepted as a user-provided option
/// and must never be persisted into `FlowInfoValue.options` or be visible in
/// user runtime options / SHOW CREATE.
pub const INTERNAL_EVAL_OFFSET_KEY: &str = "__greptime_internal_eval_offset_secs";
/// Internal transient key used to pass the serialized `FlowScheduleConfig` from
/// meta to flownode through `CreateRequest.flow_options`. This key must never
/// be accepted as a user-provided option and must never be persisted into
@@ -515,6 +524,34 @@ pub fn validate_flow_options(flow_task: &CreateFlowTask) -> Result<()> {
.fail();
}
for key in [INTERNAL_EVAL_OFFSET_KEY, INTERNAL_EVAL_SCHEDULE_KEY] {
if flow_task.flow_options.contains_key(key) {
return UnexpectedSnafu {
err_msg: format!("flow option '{key}' is reserved for internal use"),
}
.fail();
}
}
// EVAL OFFSET semantics: only legal with EVAL INTERVAL and must be in
// `[0, eval_interval_secs)`. Never modulo-normalized.
if let Some(offset_secs) = flow_task.eval_offset_secs {
let Some(eval_interval_secs) = flow_task.eval_interval_secs else {
return UnexpectedSnafu {
err_msg: "EVAL OFFSET requires EVAL INTERVAL to be specified".to_string(),
}
.fail();
};
if !(0..eval_interval_secs).contains(&offset_secs) {
return UnexpectedSnafu {
err_msg: format!(
"EVAL OFFSET must be in range [0, EVAL INTERVAL), got {offset_secs} seconds with EVAL INTERVAL {eval_interval_secs} seconds"
),
}
.fail();
}
}
for key in flow_task.flow_options.keys() {
match key.as_str() {
DEFER_ON_MISSING_SOURCE_KEY
@@ -538,12 +575,16 @@ pub fn validate_flow_options(flow_task: &CreateFlowTask) -> Result<()> {
/// Computes the ceiling of `time` to the next schedule boundary aligned with `anchor + k * interval`.
/// All values are Unix timestamps in seconds.
fn ceil_to_boundary(time: i64, anchor: i64, interval: i64) -> i64 {
///
/// Fallible: if the next boundary after `time` does not fit in `i64`, an
/// explicit error is returned instead of clamping to a non-phase value such as
/// `i64::MAX`.
pub(crate) fn ceil_to_boundary(time: i64, anchor: i64, interval: i64) -> Result<i64> {
if interval <= 0 {
return time;
return Ok(time);
}
if time <= anchor {
return anchor;
return Ok(anchor);
}
let diff = i128::from(time) - i128::from(anchor);
@@ -551,7 +592,41 @@ fn ceil_to_boundary(time: i64, anchor: i64, interval: i64) -> i64 {
let k = (diff + interval - 1) / interval;
let boundary = i128::from(anchor) + k * interval;
boundary.clamp(i128::from(i64::MIN), i128::from(i64::MAX)) as i64
i64::try_from(boundary).map_err(|_| {
UnexpectedSnafu {
err_msg: format!(
"Cannot align time {time} to the next `anchor + k * interval` boundary (anchor={anchor}, interval={interval}): result {boundary} does not fit in i64"
),
}
.build()
})
}
/// Rounds a `Utc` instant up to the next whole second (Unix seconds), with
/// nanosecond precision.
///
/// `timestamp()` / `timestamp_millis()` floor the sub-second fraction; using
/// them unchanged could produce a `start_secs` in the past relative to the
/// exact prepare instant, which would make the very first evaluation due before
/// the flow finished being prepared. Uses checked arithmetic so an instant at
/// the very end of the `i64` second range yields an explicit error instead of
/// wrapping.
pub(crate) fn ceil_to_whole_sec(now: DateTime<Utc>) -> Result<i64> {
ceil_whole_sec_from_parts(now.timestamp(), now.timestamp_subsec_nanos() != 0)
}
/// Pure ceiling computation factored out of [`ceil_to_whole_sec`] so the
/// overflow path is testable: `chrono::DateTime<Utc>` cannot represent
/// instants near `i64::MAX` seconds, but the arithmetic below guards it anyway.
pub(crate) fn ceil_whole_sec_from_parts(secs: i64, has_fraction: bool) -> Result<i64> {
if !has_fraction {
return Ok(secs);
}
secs.checked_add(1).context(error::UnexpectedSnafu {
err_msg: format!(
"Cannot round instant at second {secs} up to the next whole second: timestamp overflow"
),
})
}
/// Returns the effective typed schedule config for flow metadata.
@@ -561,26 +636,31 @@ fn ceil_to_boundary(time: i64, anchor: i64, interval: i64) -> i64 {
/// and defaults. This avoids recovery-time wall-clock drift.
pub fn effective_eval_schedule_from_flow_info(
flow_info: &FlowInfoValue,
) -> Option<FlowScheduleConfig> {
) -> Result<Option<FlowScheduleConfig>> {
if let Some(schedule) = &flow_info.eval_schedule {
return Some(schedule.clone());
return Ok(Some(schedule.clone()));
}
let eval_interval_secs = flow_info.eval_interval_secs?;
let Some(eval_interval_secs) = flow_info.eval_interval_secs else {
return Ok(None);
};
if eval_interval_secs <= 0 {
return None;
return Ok(None);
}
// Round the created_time up to the next whole second (same helper as new
// flow resolution) before aligning to the epoch-anchored boundary.
let created_ceil = ceil_to_whole_sec(flow_info.created_time)?;
let start_secs = ceil_to_boundary(
flow_info.created_time.timestamp(),
created_ceil,
FlowScheduleConfig::DEFAULT_ANCHOR_SECS,
eval_interval_secs,
);
)?;
Some(FlowScheduleConfig::default_with_start(
Ok(Some(FlowScheduleConfig::default_with_start(
start_secs,
eval_interval_secs,
))
)))
}
/// Resolve `FlowScheduleConfig` into `task.eval_schedule`.
@@ -591,58 +671,77 @@ pub fn effective_eval_schedule_from_flow_info(
/// The function is idempotent: if `task.eval_schedule` is already `Some`,
/// it returns immediately (important for procedure retry / dump-restore).
///
/// Schedule configuration is NOT read from `task.flow_options` (those keys are
/// no longer user-facing options). For new flows, pure defaults are computed.
/// For OR REPLACE, the previous typed config is preserved when interval+anchor
/// are unchanged; otherwise defaults are recomputed.
/// The schedule phase (anchor) is the `EVAL OFFSET` value: boundaries are
/// `offset + k * interval` (Unix epoch seconds), independent of timezone/DST.
/// An omitted offset means zero. It is NOT anchored to create time and does
/// not drift. Schedule configuration is NOT read from `task.flow_options`
/// (those keys are no longer user-facing options).
///
/// For OR REPLACE, the previous typed config is preserved when interval+offset
/// are unchanged; otherwise the schedule is recomputed.
///
/// Fallible: if the next phase boundary cannot fit in `i64` (extremely far
/// future), an explicit error is returned and flow creation fails instead of
/// silently clamping to a non-phase timestamp.
pub(crate) fn resolve_schedule_defaults_into_task(
task: &mut CreateFlowTask,
prev_flow_info: Option<&FlowInfoValue>,
) {
) -> Result<()> {
// Idempotent: if already computed, skip recomputation.
if task.eval_schedule.is_some() {
return;
return Ok(());
}
let Some(eval_interval_secs) = task.eval_interval_secs else {
return;
return Ok(());
};
if eval_interval_secs <= 0 {
return;
return Ok(());
}
let anchor_secs = FlowScheduleConfig::DEFAULT_ANCHOR_SECS;
let anchor_secs = task.eval_offset_secs.unwrap_or(0);
// Defense: `validate_flow_options` (called before this in `on_prepare`)
// already rejects out-of-range offsets; guard here to never schedule on a
// garbage anchor.
if !(0..eval_interval_secs).contains(&anchor_secs) {
return Ok(());
}
// For OR REPLACE: if interval+anchor unchanged, preserve the entire
// existing typed config so start / policy / limits are stable.
if task.or_replace
&& let Some(prev) = prev_flow_info
&& let Some(old_sched) = effective_eval_schedule_from_flow_info(prev)
&& let Some(old_sched) = effective_eval_schedule_from_flow_info(prev)?
{
let old_interval = prev.eval_interval_secs.unwrap_or(0);
if old_interval == eval_interval_secs && old_sched.anchor_secs == anchor_secs {
task.eval_schedule = Some(old_sched);
return;
return Ok(());
}
}
// --- Compute start ---
let start_secs =
// New flow, or OR REPLACE with changed interval/anchor: start at the next
// aligned boundary after prepare time. The value is written into
// task.eval_schedule once so procedure retry does not recompute it.
ceil_to_boundary(chrono::Utc::now().timestamp(), anchor_secs, eval_interval_secs);
// New flow, or OR REPLACE with changed interval/offset: start at the next
// aligned boundary strictly after the exact prepare instant, rounded up to
// the next whole second so the first boundary is never in the past. The
// value is written into task.eval_schedule once so procedure retry does not
// recompute it.
let prepare_secs = ceil_to_whole_sec(chrono::Utc::now())?;
let start_secs = ceil_to_boundary(prepare_secs, anchor_secs, eval_interval_secs)?;
task.eval_schedule = Some(FlowScheduleConfig::default_with_start(
task.eval_schedule = Some(FlowScheduleConfig::with_anchor(
anchor_secs,
start_secs,
eval_interval_secs,
));
Ok(())
}
fn user_runtime_flow_options(options: &HashMap<String, String>) -> HashMap<String, String> {
let mut options = options.clone();
options.remove(DEFER_ON_MISSING_SOURCE_KEY);
options.remove(INTERNAL_EVAL_SCHEDULE_KEY);
options.remove(INTERNAL_EVAL_OFFSET_KEY);
options
}
@@ -772,14 +871,16 @@ impl From<&CreateFlowData> for (FlowInfoValue, Vec<(FlowPartitionId, FlowRouteVa
let sql = value.task.sql.clone();
let eval_schedule = value.task.eval_schedule.clone();
// Start with a clean options map. The transient schedule payload is
// only for the meta→flownode CreateRequest boundary and must not be
// persisted in FlowInfoValue.options.
// Start with a clean options map. The transient schedule/offset payloads
// are only for the meta→flownode / frontend→meta boundaries and must not
// be persisted in FlowInfoValue.options.
let mut options: HashMap<String, String> = value
.task
.flow_options
.iter()
.filter(|(k, _)| k.as_str() != INTERNAL_EVAL_SCHEDULE_KEY)
.filter(|(k, _)| {
k.as_str() != INTERNAL_EVAL_SCHEDULE_KEY && k.as_str() != INTERNAL_EVAL_OFFSET_KEY
})
.map(|(k, v)| (k.clone(), v.clone()))
.collect();
+314 -12
View File
@@ -25,9 +25,9 @@ use table::table_name::TableName;
use crate::ddl::DdlContext;
use crate::ddl::create_flow::{
CreateFlowData, CreateFlowProcedure, CreateFlowState, DEFER_ON_MISSING_SOURCE_KEY,
FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY, FlowType, defer_on_missing_source,
effective_eval_schedule_from_flow_info, resolve_schedule_defaults_into_task,
validate_flow_options,
FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY, FlowType, ceil_to_whole_sec,
defer_on_missing_source, effective_eval_schedule_from_flow_info,
resolve_schedule_defaults_into_task, validate_flow_options,
};
use crate::ddl::test_util::create_table::test_create_table_task;
use crate::ddl::test_util::flownode_handler::NaiveFlownodeHandler;
@@ -67,6 +67,7 @@ pub(crate) fn test_create_flow_task(
create_if_not_exists,
expire_after: Some(300),
eval_interval_secs: None,
eval_offset_secs: None,
comment: "".to_string(),
sql: "select 1".to_string(),
flow_options: Default::default(),
@@ -333,7 +334,6 @@ fn test_validate_flow_options_rejects_schedule_and_internal_keys_as_unknown() {
"eval_interval_missed_tick_policy",
"eval_interval_catchup_max_runs",
"eval_interval_catchup_max_lag",
"__greptime_internal_eval_schedule",
] {
let mut task = test_create_flow_task(
"my_flow",
@@ -354,6 +354,78 @@ fn test_validate_flow_options_rejects_schedule_and_internal_keys_as_unknown() {
}
}
#[test]
fn test_validate_flow_options_rejects_internal_transport_keys() {
for key in [
"__greptime_internal_eval_schedule",
"__greptime_internal_eval_offset_secs",
] {
let mut task = test_create_flow_task(
"my_flow",
vec![],
TableName::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, "my_sink_table"),
false,
);
task.eval_interval_secs = Some(300);
task.flow_options
.insert(key.to_string(), "value".to_string());
let err = validate_flow_options(&task).unwrap_err();
assert!(
err.to_string()
.contains(&format!("flow option '{key}' is reserved for internal use")),
"unexpected error for {key}: {err}"
);
}
}
#[test]
fn test_validate_flow_options_rejects_invalid_eval_offset() {
let mut task = test_create_flow_task(
"my_flow",
vec![],
TableName::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, "my_sink_table"),
false,
);
task.eval_interval_secs = None;
task.eval_offset_secs = Some(60);
let err = validate_flow_options(&task).unwrap_err();
assert!(
err.to_string()
.contains("EVAL OFFSET requires EVAL INTERVAL"),
"unexpected error: {err}"
);
for offset_secs in [-1, 300, 301] {
let mut task = test_create_flow_task(
"my_flow",
vec![],
TableName::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, "my_sink_table"),
false,
);
task.eval_interval_secs = Some(300);
task.eval_offset_secs = Some(offset_secs);
let err = validate_flow_options(&task).unwrap_err();
assert!(
err.to_string()
.contains("EVAL OFFSET must be in range [0, EVAL INTERVAL)"),
"unexpected error for offset {offset_secs}: {err}"
);
}
for offset_secs in [0, 1, 299] {
let mut task = test_create_flow_task(
"my_flow",
vec![],
TableName::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, "my_sink_table"),
false,
);
task.eval_interval_secs = Some(300);
task.eval_offset_secs = Some(offset_secs);
validate_flow_options(&task).unwrap();
}
}
#[test]
fn test_validate_flow_options_rejects_non_positive_eval_interval() {
for secs in [0, -1] {
@@ -381,7 +453,7 @@ fn test_resolved_schedule_defaults_in_create_request() {
task.eval_interval_secs = Some(300);
// Simulate on_prepare: resolve defaults into task.
resolve_schedule_defaults_into_task(&mut task, None);
resolve_schedule_defaults_into_task(&mut task, None).unwrap();
// Verify typed schedule config is populated.
let sched = task.eval_schedule.as_ref().unwrap();
@@ -410,7 +482,7 @@ fn test_resolved_schedule_defaults_in_create_request() {
// Idempotent: second call does not overwrite.
let start_before = sched.start_secs;
resolve_schedule_defaults_into_task(&mut task, None);
resolve_schedule_defaults_into_task(&mut task, None).unwrap();
assert_eq!(
task.eval_schedule.as_ref().unwrap().start_secs,
start_before
@@ -483,6 +555,221 @@ fn test_resolved_schedule_defaults_in_create_request() {
}
}
#[test]
fn test_resolve_schedule_with_eval_offset_uses_epoch_phase() {
// 1h interval + 120s offset must yield the `:02` phase: anchor 120 and a
// start on `120 + k * 3600`, and no earlier boundary than the prepare ceil.
let prepare_ceil = ceil_to_whole_sec(chrono::Utc::now()).unwrap();
let mut task = test_create_flow_task(
"my_flow",
vec![],
TableName::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, "my_sink_table"),
false,
);
task.eval_interval_secs = Some(3600);
task.eval_offset_secs = Some(120);
resolve_schedule_defaults_into_task(&mut task, None).unwrap();
let sched = task.eval_schedule.as_ref().unwrap();
assert_eq!(sched.anchor_secs, 120);
assert!(sched.start_secs >= prepare_ceil);
assert_eq!((sched.start_secs - 120) % 3600, 0);
// The offset key must not leak into user runtime options / persisted info.
let flow_context = FlowQueryContext {
catalog: DEFAULT_CATALOG_NAME.to_string(),
schema: DEFAULT_SCHEMA_NAME.to_string(),
timezone: "UTC".to_string(),
extensions: HashMap::new(),
channel: 0,
snapshot_seqs: HashMap::new(),
sst_min_sequences: HashMap::new(),
};
let data = CreateFlowData {
state: CreateFlowState::CreateFlows,
task,
flow_id: Some(1024),
peers: vec![],
source_table_ids: vec![],
unresolved_source_table_names: vec![],
flow_context: flow_context.clone(),
prev_flow_info_value: None,
did_replace: false,
flow_type: Some(FlowType::Batching),
};
let request: CreateRequest = (&data).into();
assert!(
!request
.flow_options
.contains_key("__greptime_internal_eval_offset_secs"),
"CreateRequest must not contain internal eval offset key"
);
let (flow_info, _) = <(FlowInfoValue, Vec<(_, _)>)>::from(&data);
assert_eq!(flow_info.eval_schedule.as_ref().unwrap().anchor_secs, 120);
assert!(
!flow_info
.options
.contains_key("__greptime_internal_eval_offset_secs"),
"FlowInfoValue.options must not contain internal eval offset key"
);
}
#[test]
fn test_resolve_schedule_or_replace_preserves_and_recomputes() {
let sink = TableName::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, "my_sink_table");
let mut task = test_create_flow_task("my_flow", vec![], sink.clone(), false);
task.or_replace = false;
task.eval_interval_secs = Some(3600);
task.eval_offset_secs = Some(120);
resolve_schedule_defaults_into_task(&mut task, None).unwrap();
let original_sched = task.eval_schedule.clone().unwrap();
assert_eq!(original_sched.anchor_secs, 120);
let prev_info = flow_info_for_schedule_test(
Some(original_sched.clone()),
Some(3600),
HashMap::new(),
1_700_000_000,
);
let mut replace_same = test_create_flow_task("my_flow", vec![], sink.clone(), false);
replace_same.or_replace = true;
replace_same.eval_interval_secs = Some(3600);
replace_same.eval_offset_secs = Some(120);
resolve_schedule_defaults_into_task(&mut replace_same, Some(&prev_info)).unwrap();
assert_eq!(replace_same.eval_schedule, Some(original_sched.clone()));
let mut replace_changed = test_create_flow_task("my_flow", vec![], sink.clone(), false);
replace_changed.or_replace = true;
replace_changed.eval_interval_secs = Some(3600);
replace_changed.eval_offset_secs = Some(180);
resolve_schedule_defaults_into_task(&mut replace_changed, Some(&prev_info)).unwrap();
let recomputed = replace_changed.eval_schedule.as_ref().unwrap();
assert_eq!(recomputed.anchor_secs, 180);
assert_eq!((recomputed.start_secs - 180) % 3600, 0);
assert_ne!(recomputed.start_secs, original_sched.start_secs);
let mut replace_interval = test_create_flow_task("my_flow", vec![], sink.clone(), false);
replace_interval.or_replace = true;
replace_interval.eval_interval_secs = Some(1800);
replace_interval.eval_offset_secs = Some(120);
resolve_schedule_defaults_into_task(&mut replace_interval, Some(&prev_info)).unwrap();
assert_eq!(
replace_interval.eval_schedule.as_ref().unwrap().anchor_secs,
120
);
let mut replace_omitted = test_create_flow_task("my_flow", vec![], sink, false);
replace_omitted.or_replace = true;
replace_omitted.eval_interval_secs = Some(3600);
replace_omitted.eval_offset_secs = None;
resolve_schedule_defaults_into_task(&mut replace_omitted, Some(&prev_info)).unwrap();
let reset = replace_omitted.eval_schedule.as_ref().unwrap();
assert_eq!(reset.anchor_secs, 0);
assert_eq!(reset.start_secs % 3600, 0);
}
#[test]
fn test_create_flow_task_serde_defaults_for_old_data() {
// Old serialized CreateFlowTask data without `eval_offset_secs` must
// deserialize with `eval_offset_secs: None`.
let json = r#"{
"catalog_name": "catalog",
"flow_name": "flow",
"source_table_names": [],
"sink_table_name": {"catalog_name": "catalog", "schema_name": "schema", "table_name": "sink"},
"or_replace": false,
"create_if_not_exists": false,
"expire_after": null,
"eval_interval_secs": 300,
"comment": "",
"sql": "select 1",
"flow_options": {}
}"#;
let task: CreateFlowTask = serde_json::from_str(json).unwrap();
assert_eq!(task.eval_offset_secs, None);
assert_eq!(task.eval_interval_secs, Some(300));
let encoded = serde_json::to_string(&task).unwrap();
let decoded: CreateFlowTask = serde_json::from_str(&encoded).unwrap();
assert_eq!(decoded.eval_offset_secs, None);
// A task with an offset round-trips through JSON (procedure dump/restore).
let mut task = test_create_flow_task(
"my_flow",
vec![],
TableName::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, "my_sink_table"),
false,
);
task.eval_interval_secs = Some(3600);
task.eval_offset_secs = Some(120);
let encoded = serde_json::to_string(&task).unwrap();
let decoded: CreateFlowTask = serde_json::from_str(&encoded).unwrap();
assert_eq!(decoded.eval_offset_secs, Some(120));
assert_eq!(decoded.eval_interval_secs, Some(3600));
}
#[test]
fn test_create_flow_task_proto_roundtrip_preserves_offset_key() {
// Path A: the operator inserts the trusted offset key into flow_options
// (as produced by `to_create_flow_task_expr`); the proto round-trip
// (frontend -> metasrv DDL task) must preserve it.
let mut task = test_create_flow_task(
"my_flow",
vec![],
TableName::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, "my_sink_table"),
false,
);
task.eval_interval_secs = Some(3600);
task.flow_options.insert(
"__greptime_internal_eval_offset_secs".to_string(),
"120".to_string(),
);
let pb: api::v1::meta::CreateFlowTask = task.clone().into();
assert_eq!(
pb.create_flow
.as_ref()
.unwrap()
.flow_options
.get("__greptime_internal_eval_offset_secs"),
Some(&"120".to_string())
);
let parsed = CreateFlowTask::try_from(pb).unwrap();
assert_eq!(parsed.eval_offset_secs, Some(120));
assert!(
!parsed
.flow_options
.contains_key("__greptime_internal_eval_offset_secs")
);
assert_eq!(parsed.eval_interval_secs, Some(3600));
// Path B: a typed task carrying `eval_offset_secs` (e.g. after procedure
// dump/restore) re-inserts the key when converted back to proto.
let mut typed = test_create_flow_task(
"my_flow",
vec![],
TableName::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, "my_sink_table"),
false,
);
typed.eval_interval_secs = Some(3600);
typed.eval_offset_secs = Some(120);
let pb: api::v1::meta::CreateFlowTask = typed.clone().into();
assert_eq!(
pb.create_flow
.as_ref()
.unwrap()
.flow_options
.get("__greptime_internal_eval_offset_secs"),
Some(&"120".to_string())
);
let reparsed = CreateFlowTask::try_from(pb).unwrap();
assert_eq!(reparsed.eval_offset_secs, Some(120));
assert_eq!(reparsed.flow_options, typed.flow_options);
}
#[test]
fn test_effective_eval_schedule_prefers_typed_config() {
let typed = FlowScheduleConfig {
@@ -496,7 +783,9 @@ fn test_effective_eval_schedule_prefers_typed_config() {
let flow_info =
flow_info_for_schedule_test(Some(typed.clone()), Some(60), unrelated_options, 1);
let effective = effective_eval_schedule_from_flow_info(&flow_info).unwrap();
let effective = effective_eval_schedule_from_flow_info(&flow_info)
.unwrap()
.unwrap();
assert_eq!(effective, typed);
}
@@ -504,7 +793,9 @@ fn test_effective_eval_schedule_prefers_typed_config() {
fn test_effective_eval_schedule_uses_created_time_default() {
let flow_info = flow_info_for_schedule_test(None, Some(60), HashMap::new(), 61);
let effective = effective_eval_schedule_from_flow_info(&flow_info).unwrap();
let effective = effective_eval_schedule_from_flow_info(&flow_info)
.unwrap()
.unwrap();
assert_eq!(effective.anchor_secs, 0);
assert_eq!(effective.start_secs, 120);
assert_eq!(
@@ -518,7 +809,11 @@ fn test_effective_eval_schedule_uses_created_time_default() {
#[test]
fn test_effective_eval_schedule_none_without_eval_interval() {
let flow_info = flow_info_for_schedule_test(None, None, HashMap::new(), 61);
assert!(effective_eval_schedule_from_flow_info(&flow_info).is_none());
assert!(
effective_eval_schedule_from_flow_info(&flow_info)
.unwrap()
.is_none()
);
}
#[test]
@@ -526,8 +821,12 @@ fn test_effective_eval_schedule_deterministic_on_old_metadata() {
// Old metadata: no typed eval_schedule, but has eval_interval_secs and
// a fixed created_time. Repeated calls must produce the same config.
let flow_info = flow_info_for_schedule_test(None, Some(60), HashMap::new(), 61);
let first = effective_eval_schedule_from_flow_info(&flow_info).unwrap();
let second = effective_eval_schedule_from_flow_info(&flow_info).unwrap();
let first = effective_eval_schedule_from_flow_info(&flow_info)
.unwrap()
.unwrap();
let second = effective_eval_schedule_from_flow_info(&flow_info)
.unwrap()
.unwrap();
assert_eq!(first, second);
// Sanity: the derived values are based on created_time=61 + interval=60.
assert_eq!(first.anchor_secs, 0);
@@ -556,7 +855,9 @@ fn test_old_flow_info_json_without_eval_schedule_deserializes() {
let decoded: FlowInfoValue = serde_json::from_value(json).unwrap();
assert!(decoded.eval_schedule.is_none());
let effective = effective_eval_schedule_from_flow_info(&decoded).unwrap();
let effective = effective_eval_schedule_from_flow_info(&decoded)
.unwrap()
.unwrap();
assert_eq!(effective.anchor_secs, 0);
assert_eq!(effective.start_secs, 120);
assert_eq!(
@@ -974,6 +1275,7 @@ fn create_test_flow_task_for_serialization() -> CreateFlowTask {
create_if_not_exists: false,
expire_after: None,
eval_interval_secs: None,
eval_offset_secs: None,
comment: "test comment".to_string(),
sql: "SELECT * FROM source_table".to_string(),
flow_options: HashMap::new(),
+9 -1
View File
@@ -172,8 +172,16 @@ impl FlowScheduleConfig {
}
pub fn default_with_start(start_secs: i64, eval_interval_secs: i64) -> Self {
Self::with_anchor(Self::DEFAULT_ANCHOR_SECS, start_secs, eval_interval_secs)
}
/// Builds a schedule anchored at `anchor_secs` (the `EVAL OFFSET` phase,
/// in Unix epoch seconds) with `start_secs` as the first scheduled time.
/// Callers must ensure `0 <= anchor_secs < eval_interval_secs` and that
/// `start_secs` lies on an `anchor_secs + k * eval_interval_secs` boundary.
pub fn with_anchor(anchor_secs: i64, start_secs: i64, eval_interval_secs: i64) -> Self {
Self {
anchor_secs: Self::DEFAULT_ANCHOR_SECS,
anchor_secs,
start_secs,
missed_tick_policy: FlowMissedTickPolicy::BoundedCatchUp,
catchup_max_runs: Self::DEFAULT_CATCHUP_MAX_RUNS,
+35 -2
View File
@@ -1296,6 +1296,13 @@ pub struct CreateFlowTask {
/// Duration in seconds. Data older than this duration will not be used.
pub expire_after: Option<i64>,
pub eval_interval_secs: Option<i64>,
/// Phase offset of the evaluation schedule within `eval_interval_secs`,
/// in seconds. Must be in `[0, eval_interval_secs)`. `None` means a zero
/// offset (epoch-anchored schedule).
/// Transported through the transient option map (no proto field), see
/// `INTERNAL_EVAL_OFFSET_KEY`.
#[serde(default)]
pub eval_offset_secs: Option<i64>,
pub comment: String,
pub sql: String,
pub flow_options: HashMap<String, String>,
@@ -1321,11 +1328,27 @@ impl TryFrom<PbCreateFlowTask> for CreateFlowTask {
eval_interval,
comment,
sql,
flow_options,
mut flow_options,
} = pb.create_flow.context(error::InvalidProtoMsgSnafu {
err_msg: "expected create_flow",
})?;
// Parse and strip the trusted transient offset key inserted by the
// operator after user option validation. It must never persist in
// user-visible options.
let eval_offset_secs =
match flow_options.remove(crate::ddl::create_flow::INTERNAL_EVAL_OFFSET_KEY) {
Some(value) => Some(value.parse::<i64>().map_err(|_| {
error::UnexpectedSnafu {
err_msg: format!(
"Invalid internal eval offset payload '{value}': expected whole seconds"
),
}
.build()
})?),
None => None,
};
Ok(CreateFlowTask {
catalog_name,
flow_name,
@@ -1339,6 +1362,7 @@ impl TryFrom<PbCreateFlowTask> for CreateFlowTask {
create_if_not_exists,
expire_after: expire_after.map(|e| e.value),
eval_interval_secs: eval_interval.map(|e| e.seconds),
eval_offset_secs,
comment,
sql,
flow_options,
@@ -1358,12 +1382,21 @@ impl From<CreateFlowTask> for PbCreateFlowTask {
create_if_not_exists,
expire_after,
eval_interval_secs: eval_interval,
eval_offset_secs,
comment,
sql,
flow_options,
mut flow_options,
..
}: CreateFlowTask,
) -> Self {
// Re-insert the transient offset key so the proto round-trip (e.g. DDL
// task submission between frontend and metasrv) preserves the offset.
if let Some(offset_secs) = eval_offset_secs {
flow_options.insert(
crate::ddl::create_flow::INTERNAL_EVAL_OFFSET_KEY.to_string(),
offset_secs.to_string(),
);
}
PbCreateFlowTask {
create_flow: Some(CreateFlowExpr {
catalog_name,
+3 -1
View File
@@ -387,7 +387,9 @@ impl FlowDualEngine {
comment: Some(info.comment().clone()),
sql: info.raw_sql().clone(),
flow_options: info.options().clone(),
eval_schedule: effective_eval_schedule_from_flow_info(&info),
eval_schedule: effective_eval_schedule_from_flow_info(&info)
.map_err(BoxedError::new)
.context(ExternalSnafu)?,
query_ctx: info
.query_context()
.clone()
+228 -57
View File
@@ -17,7 +17,7 @@
pub use common_meta::key::flow::flow_info::{FlowMissedTickPolicy, FlowScheduleConfig};
use snafu::ensure;
use crate::error::{InvalidQuerySnafu, Result};
use crate::error::{InvalidQuerySnafu, Result, UnexpectedSnafu};
/// Schedule for an `EVAL INTERVAL` flow.
#[derive(Debug, Clone, PartialEq)]
@@ -72,6 +72,29 @@ impl EvalSchedule {
)
}
);
// The anchor defines the epoch phase `anchor + k * interval`; it
// must be a valid offset within one interval.
ensure!(
c.anchor_secs >= 0 && c.anchor_secs < interval_secs,
InvalidQuerySnafu {
reason: format!(
"Invalid FlowScheduleConfig.anchor_secs: must be in [0, {interval_secs}), got {}",
c.anchor_secs
)
}
);
// The start must be phase-consistent with the anchor (on an
// `anchor + k * interval` boundary) and not before the anchor.
ensure!(
c.start_secs >= c.anchor_secs
&& (c.start_secs - c.anchor_secs) % interval_secs == 0,
InvalidQuerySnafu {
reason: format!(
"Invalid FlowScheduleConfig.start_secs: must be on an anchor + k * interval boundary and >= anchor, got start={}, anchor={}, interval={}",
c.start_secs, c.anchor_secs, interval_secs
)
}
);
Self {
interval_secs,
@@ -96,30 +119,53 @@ impl EvalSchedule {
}))
}
/// Returns the next scheduled time strictly after `cursor_secs`.
pub fn next_scheduled_time_after(&self, cursor_secs: i64) -> i64 {
/// Returns the next scheduled time strictly after `cursor_secs`, on the
/// `anchor + k * interval` lattice.
///
/// Fallible: a non-positive interval or a next boundary that does not fit
/// in `i64` yields an explicit error instead of a saturated non-phase
/// value such as `i64::MAX`.
pub fn next_scheduled_time_after(&self, cursor_secs: i64) -> Result<i64> {
next_in_sequence(cursor_secs, self.start_secs, self.interval_secs)
}
}
fn next_in_sequence(cursor: i64, start: i64, interval: i64) -> i64 {
if interval <= 0 {
return cursor.saturating_add(1).max(start);
}
if cursor < start {
return start;
}
/// The smallest `start + k * interval` value that is strictly after `cursor`
/// (`start` itself lies on the `anchor + k * interval` lattice, so every
/// result is phase-consistent with the anchor). All arithmetic happens in
/// `i128`: `cursor - start` cannot overflow and the result is either exactly
/// on the lattice or an explicit error.
fn next_in_sequence(cursor: i64, start: i64, interval: i64) -> Result<i64> {
ensure!(
interval > 0,
InvalidQuerySnafu {
reason: format!("Invalid eval interval: must be positive, got {interval}")
}
);
let interval = i128::from(interval);
let start = i128::from(start);
let cursor = i128::from(cursor);
let k = (cursor - start) / interval;
start.saturating_add((k + 1).saturating_mul(interval))
let next = if cursor < start {
start
} else {
let k = (cursor - start) / interval;
start + (k + 1) * interval
};
i64::try_from(next).map_err(|_| {
UnexpectedSnafu {
reason: format!(
"Cannot advance the eval schedule past cursor {cursor}: the next scheduled time {next} does not fit in i64 (start={start}, interval={interval})"
),
}
.build()
})
}
fn first_due_in_sequence(cursor: i64, start: i64, interval: i64) -> i64 {
if interval <= 0 {
return cursor.saturating_add(1).max(start);
}
fn first_due_in_sequence(cursor: i64, start: i64, interval: i64) -> Result<i64> {
if cursor < start {
start
Ok(start)
} else {
next_in_sequence(cursor, start, interval)
}
@@ -139,59 +185,112 @@ pub struct DueScheduledTimes {
}
/// Select due scheduled times `<= wall_now_secs` without materializing all missed ticks.
///
/// Fallible: a non-positive interval or a scheduled time that does not fit in
/// `i64` yields an explicit error instead of silently producing saturated
/// non-phase timestamps.
pub fn select_due_scheduled_times(
schedule: &EvalSchedule,
cursor_secs: i64,
wall_now_secs: i64,
) -> Option<DueScheduledTimes> {
if schedule.interval_secs <= 0 {
return None;
}
) -> Result<DueScheduledTimes> {
let interval = schedule.interval_secs;
ensure!(
interval > 0,
InvalidQuerySnafu {
reason: format!("Invalid eval interval: must be positive, got {interval}")
}
);
let first_due = first_due_in_sequence(cursor_secs, schedule.start_secs, schedule.interval_secs);
let first_due = first_due_in_sequence(cursor_secs, schedule.start_secs, interval)?;
if first_due > wall_now_secs {
return Some(DueScheduledTimes {
return Ok(DueScheduledTimes {
scheduled_times_secs: vec![],
skipped: 0,
});
}
let total_count = ((wall_now_secs - first_due) / schedule.interval_secs) as u64 + 1;
// Count and select due scheduled times in i128 so every value stays
// exactly on the `anchor + k * interval` lattice; a value beyond `i64` is
// an explicit error, never a saturated non-phase timestamp.
let first_due = i128::from(first_due);
let wall_now = i128::from(wall_now_secs);
let interval = i128::from(interval);
let total_count = (wall_now - first_due) / interval + 1;
// `first_due >= 0` and `wall_now <= i64::MAX`, so this always fits in u64.
let total_count = u64::try_from(total_count).map_err(|_| {
UnexpectedSnafu {
reason: format!(
"Cannot count due eval scheduled times up to {wall_now}: {total_count} does not fit in u64"
),
}
.build()
})?;
match schedule.missed_tick_policy {
FlowMissedTickPolicy::Skip => {
let last = first_due + (total_count as i64 - 1) * schedule.interval_secs;
Some(DueScheduledTimes {
// Keep only the latest due scheduled time; it is still on-lattice
// and `<= wall_now`.
let last = i64::try_from(first_due + i128::from(total_count - 1) * interval)
.map_err(|_| {
UnexpectedSnafu {
reason: format!(
"Cannot compute the latest due eval scheduled time (first_due={first_due}, interval={interval}, count={total_count}): result does not fit in i64"
),
}
.build()
})?;
Ok(DueScheduledTimes {
scheduled_times_secs: vec![last],
skipped: total_count.saturating_sub(1),
skipped: total_count - 1,
})
}
FlowMissedTickPolicy::BoundedCatchUp => {
let cutoff = wall_now_secs.saturating_sub(schedule.max_lag_secs);
// The cutoff is computed in i128: `wall_now - max_lag` may
// legitimately underflow i64 (a cutoff before the Unix epoch) and
// must not saturate to a wrong value.
let cutoff = wall_now - i128::from(schedule.max_lag_secs);
let skipped_by_cutoff = if first_due >= cutoff {
0
} else {
((cutoff - first_due + schedule.interval_secs - 1) / schedule.interval_secs) as u64
// ceil((cutoff - first_due) / interval), capped at u64::MAX
// before the `.min(total_count)` below.
let skipped = (cutoff - first_due + interval - 1) / interval;
u64::try_from(skipped).unwrap_or(u64::MAX)
}
.min(total_count);
let remaining = total_count.saturating_sub(skipped_by_cutoff);
let remaining = total_count - skipped_by_cutoff;
if remaining == 0 {
return Some(DueScheduledTimes {
return Ok(DueScheduledTimes {
scheduled_times_secs: vec![],
skipped: total_count,
});
}
// max_lag_secs decides which missed scheduled times are recent enough to
// run; max_runs caps how many of those times we execute
// back-to-back in one scheduler pass.
let keep_count = remaining.min(schedule.max_runs as u64);
let keep_start = skipped_by_cutoff + remaining.saturating_sub(keep_count);
let scheduled_times_secs = (0..keep_count)
.map(|i| first_due + (keep_start as i64 + i as i64) * schedule.interval_secs)
.collect::<Vec<_>>();
// max_lag decides which missed scheduled times are recent enough to
// run; max_runs caps how many of those times execute back-to-back
// in one scheduler pass.
let keep_count = remaining.min(u64::from(schedule.max_runs));
let keep_start = skipped_by_cutoff + remaining - keep_count;
let mut scheduled_times_secs = Vec::with_capacity(keep_count as usize);
for i in 0..keep_count {
let t = i64::try_from(
first_due + (i128::from(keep_start) + i128::from(i)) * interval,
)
.map_err(|_| {
UnexpectedSnafu {
reason: format!(
"Cannot compute a due eval scheduled time (first_due={first_due}, interval={interval}, index={i}): result does not fit in i64"
),
}
.build()
})?;
scheduled_times_secs.push(t);
}
Some(DueScheduledTimes {
Ok(DueScheduledTimes {
scheduled_times_secs,
skipped: total_count - keep_count,
})
@@ -200,12 +299,15 @@ pub fn select_due_scheduled_times(
}
/// Ceils `time` to the next `anchor + k * interval` boundary.
pub fn ceil_to_boundary(time: i64, anchor: i64, interval: i64) -> i64 {
///
/// Fallible: if the next boundary does not fit in `i64`, an explicit error is
/// returned instead of clamping to a non-phase value such as `i64::MAX`.
pub fn ceil_to_boundary(time: i64, anchor: i64, interval: i64) -> Result<i64> {
if interval <= 0 {
return time;
return Ok(time);
}
if time <= anchor {
return anchor;
return Ok(anchor);
}
let diff = i128::from(time) - i128::from(anchor);
@@ -213,7 +315,14 @@ pub fn ceil_to_boundary(time: i64, anchor: i64, interval: i64) -> i64 {
let k = (diff + interval - 1) / interval;
let boundary = i128::from(anchor) + k * interval;
boundary.clamp(i128::from(i64::MIN), i128::from(i64::MAX)) as i64
i64::try_from(boundary).map_err(|_| {
crate::error::UnexpectedSnafu {
reason: format!(
"Cannot align time {time} to the next `anchor + k * interval` boundary (anchor={anchor}, interval={interval}): result {boundary} does not fit in i64"
),
}
.build()
})
}
#[cfg(test)]
@@ -239,7 +348,8 @@ mod test {
fn config(policy: FlowMissedTickPolicy) -> FlowScheduleConfig {
FlowScheduleConfig {
anchor_secs: 10,
start_secs: 70,
// phase-consistent: 310 = anchor(10) + 1 * interval(300)
start_secs: 310,
missed_tick_policy: policy,
catchup_max_runs: 4,
catchup_max_lag_secs: 600,
@@ -248,14 +358,15 @@ mod test {
#[test]
fn ceil_to_boundary_handles_anchor_and_interval_edges() {
assert_eq!(ceil_to_boundary(-10, 0, 60), 0);
assert_eq!(ceil_to_boundary(0, 0, 60), 0);
assert_eq!(ceil_to_boundary(1, 0, 60), 60);
assert_eq!(ceil_to_boundary(60, 0, 60), 60);
assert_eq!(ceil_to_boundary(101, 100, 60), 160);
assert_eq!(ceil_to_boundary(50, 0, 0), 50);
assert_eq!(ceil_to_boundary(i64::MAX, 0, 60), i64::MAX);
assert_eq!(ceil_to_boundary(i64::MAX - 1, i64::MIN, 60), i64::MAX);
assert_eq!(ceil_to_boundary(-10, 0, 60).unwrap(), 0);
assert_eq!(ceil_to_boundary(0, 0, 60).unwrap(), 0);
assert_eq!(ceil_to_boundary(1, 0, 60).unwrap(), 60);
assert_eq!(ceil_to_boundary(60, 0, 60).unwrap(), 60);
assert_eq!(ceil_to_boundary(101, 100, 60).unwrap(), 160);
assert_eq!(ceil_to_boundary(50, 0, 0).unwrap(), 50);
// Never clamp to the non-phase i64::MAX: the next boundary does not fit.
assert!(ceil_to_boundary(i64::MAX, 0, 60).is_err());
assert!(ceil_to_boundary(i64::MAX - 1, i64::MIN, 60).is_err());
}
#[test]
@@ -269,7 +380,7 @@ mod test {
.unwrap();
assert_eq!(from_typed.interval_secs, 300);
assert_eq!(from_typed.anchor_secs, 10);
assert_eq!(from_typed.start_secs, 70);
assert_eq!(from_typed.start_secs, 310);
assert_eq!(from_typed.missed_tick_policy, FlowMissedTickPolicy::Skip);
assert_eq!(from_typed.max_runs, 4);
assert_eq!(from_typed.max_lag_secs, 600);
@@ -291,12 +402,72 @@ mod test {
assert!(EvalSchedule::from_config(Some(300), Some(&c)).is_err());
}
#[test]
fn nonzero_anchor_due_selection_follows_phase() {
// anchor=120 (i.e. `EVAL OFFSET '2 minutes'`), interval=3600:
// boundaries at :02 every hour. start=3720 (120 + 3600).
let s = EvalSchedule {
interval_secs: 3600,
anchor_secs: 120,
start_secs: 3720,
missed_tick_policy: FlowMissedTickPolicy::BoundedCatchUp,
max_runs: 3,
max_lag_secs: 3600,
};
assert_eq!(
select_due_scheduled_times(&s, 0, 100)
.unwrap()
.scheduled_times_secs,
Vec::<i64>::new()
);
// From 3720 on, every selected time must be on the :02 phase.
let due = select_due_scheduled_times(&s, 0, 3720).unwrap();
assert_eq!(due.scheduled_times_secs, vec![3720]);
let due = select_due_scheduled_times(&s, 3720, 7320).unwrap();
assert_eq!(due.scheduled_times_secs, vec![7320]);
for t in &due.scheduled_times_secs {
assert_eq!((t - 120) % 3600, 0);
}
assert_eq!(s.next_scheduled_time_after(3720).unwrap(), 7320);
assert_eq!(s.next_scheduled_time_after(7300).unwrap(), 7320);
}
#[test]
fn next_scheduled_time_after_respects_start_sequence() {
let s = schedule(50, FlowMissedTickPolicy::BoundedCatchUp, 3, 300);
assert_eq!(s.next_scheduled_time_after(0), 50);
assert_eq!(s.next_scheduled_time_after(50), 110);
assert_eq!(s.next_scheduled_time_after(100), 110);
assert_eq!(s.next_scheduled_time_after(0).unwrap(), 50);
assert_eq!(s.next_scheduled_time_after(50).unwrap(), 110);
assert_eq!(s.next_scheduled_time_after(100).unwrap(), 110);
}
#[test]
fn near_i64_max_advancement_is_exact_or_explicit_error() {
// anchor=0, interval=60: the next boundary after i64::MAX - 60 is
// 9223372036854775800, still in range and exactly on the lattice.
let s = schedule(0, FlowMissedTickPolicy::Skip, 5, 3600);
let cursor = i64::MAX - 60;
let next = s.next_scheduled_time_after(cursor).unwrap();
assert_eq!(next, 9223372036854775800);
assert_eq!(next % 60, 0);
// Advancing past the last representable boundary is an explicit error,
// never a saturated non-phase value like i64::MAX.
let err = s
.next_scheduled_time_after(9223372036854775800)
.unwrap_err();
assert!(err.to_string().contains("does not fit in i64"));
// A non-positive interval is an explicit error, not a saturating
// `cursor + 1` result.
let invalid = EvalSchedule {
interval_secs: 0,
anchor_secs: 0,
start_secs: 0,
missed_tick_policy: FlowMissedTickPolicy::Skip,
max_runs: 3,
max_lag_secs: 900,
};
assert!(invalid.next_scheduled_time_after(0).is_err());
}
#[test]
+107 -9
View File
@@ -79,6 +79,61 @@ fn wall_clock_unix_secs() -> i64 {
.as_secs() as i64
}
/// Initial scheduler cursor for `start_scheduled_loop`: exactly one interval
/// before `start_secs` so the first due scheduled time is `start_secs` itself.
///
/// Fallible: a `start_secs - interval_secs` difference that does not fit in
/// `i64` is an explicit error instead of a saturated cursor that would make
/// the first due scheduled time `start_secs + interval_secs` and silently skip
/// the `start_secs` tick.
fn initial_schedule_cursor(start_secs: i64, interval_secs: i64) -> Result<i64, Error> {
let cursor = i128::from(start_secs) - i128::from(interval_secs);
i64::try_from(cursor).map_err(|_| {
UnexpectedSnafu {
reason: format!(
"Cannot compute the initial eval schedule cursor one interval before start {start_secs} (interval={interval_secs}): {cursor} does not fit in i64"
),
}
.build()
})
}
/// Whole seconds to sleep until the next scheduled time `next`, measured from
/// the current wall clock `wall_now_secs`.
///
/// Fallible: `next` must be strictly after `wall_now_secs` and the difference
/// must fit in `u64`. In practice `i64::MAX - i64::MIN` is exactly `u64::MAX`,
/// so the difference always fits once `next > wall_now_secs`; the explicit
/// error keeps the scheduled loop panic-free and wrap-free regardless.
fn sleep_delta_secs(next: i64, wall_now_secs: i64) -> Result<u64, Error> {
let delta = i128::from(next) - i128::from(wall_now_secs);
u64::try_from(delta).map_err(|_| {
UnexpectedSnafu {
reason: format!(
"Cannot sleep until the next scheduled time {next}: the delta from wall clock {wall_now_secs} is {delta} seconds, which does not fit in u64"
),
}
.build()
})
}
/// Scheduled time in seconds converted to milliseconds for the
/// `FLOW_SCHEDULED_TIME_MILLIS` extension.
///
/// Fallible: a seconds value whose millisecond product does not fit in `i64`
/// is an explicit error instead of a saturated `i64::MAX` that would silently
/// misrepresent the logical scheduled time.
fn scheduled_time_millis(scheduled_time_secs: i64) -> Result<i64, Error> {
scheduled_time_secs.checked_mul(1000).ok_or_else(|| {
UnexpectedSnafu {
reason: format!(
"Cannot convert scheduled time {scheduled_time_secs}s to milliseconds: the product exceeds i64"
),
}
.build()
})
}
/// The task's config, immutable once created
#[derive(Clone)]
pub struct TaskConfig {
@@ -1020,8 +1075,20 @@ impl BatchingTask {
};
// Initial cursor is one interval before start so the first due
// scheduled time is `start_secs`.
let mut cursor_secs = schedule.start_secs.saturating_sub(schedule.interval_secs);
// scheduled time is `start_secs`. An unrepresentable difference is an
// explicit error, never a saturated cursor that would silently skip
// the first scheduled tick.
let mut cursor_secs =
match initial_schedule_cursor(schedule.start_secs, schedule.interval_secs) {
Ok(cursor) => cursor,
Err(e) => {
warn!(
"Flow {}: invalid eval schedule, exiting loop: {}",
flow_id_str, e
);
return;
}
};
info!(
"Flow {}: entering scheduled loop, interval={}s, start={}, anchor={}, policy={:?}, max_runs={}, max_lag={}s",
@@ -1042,11 +1109,11 @@ impl BatchingTask {
let wall_now_secs = wall_clock_unix_secs();
let due = match select_due_scheduled_times(&schedule, cursor_secs, wall_now_secs) {
Some(d) => d,
None => {
Ok(d) => d,
Err(e) => {
warn!(
"Flow {}: Invalid schedule (interval <= 0), exiting loop",
flow_id_str
"Flow {}: invalid eval schedule, exiting loop: {}",
flow_id_str, e
);
return;
}
@@ -1063,14 +1130,32 @@ impl BatchingTask {
}
// No due yet — sleep until the next scheduled time.
let next = schedule.next_scheduled_time_after(cursor_secs);
let next = match schedule.next_scheduled_time_after(cursor_secs) {
Ok(next) => next,
Err(e) => {
warn!(
"Flow {}: cannot advance eval schedule past cursor {cursor_secs}: {e}; exiting loop",
flow_id_str
);
return;
}
};
if next <= wall_now_secs {
// Shouldn't happen given select_due_scheduled_times returned empty,
// but guard against clock skew / logic error.
cursor_secs = wall_now_secs;
continue;
}
let wait_secs = (next - wall_now_secs) as u64;
let wait_secs = match sleep_delta_secs(next, wall_now_secs) {
Ok(wait_secs) => wait_secs,
Err(e) => {
warn!(
"Flow {}: cannot sleep until next scheduled time {}: {e}; exiting loop",
flow_id_str, next
);
return;
}
};
let wait_dur = Duration::from_secs(wait_secs);
debug!(
"Flow {}: no due scheduled times, sleeping for {}s until next scheduled time at {}",
@@ -1259,6 +1344,19 @@ impl BatchingTask {
}
}
// Convert to milliseconds before touching the task state so an
// unrepresentable scheduled time fails as an explicit error without
// ever installing a saturated (off-phase) extension value.
let scheduled_time_millis = match scheduled_time_millis(scheduled_time_secs) {
Ok(millis) => millis,
Err(e) => {
return ExecuteOnceOutcome {
new_query: None,
result: Err(e),
};
}
};
// Clone the current QueryContext and add the scheduled time
// extension, then swap it into the task state for this attempt.
let old_ctx = {
@@ -1267,7 +1365,7 @@ impl BatchingTask {
let mut new_ctx = (*old).clone();
new_ctx.set_extension(
query::options::FLOW_SCHEDULED_TIME_MILLIS,
(scheduled_time_secs.saturating_mul(1000)).to_string(),
scheduled_time_millis.to_string(),
);
state.query_ctx = Arc::new(new_ctx);
old
+117
View File
@@ -43,6 +43,7 @@ use crate::batching_mode::checkpoint::{
CHECKPOINT_DECISION_ADVANCE, CHECKPOINT_DECISION_FALLBACK, CHECKPOINT_REASON_NONE,
FlowCheckpointDecision, FlowQueryFallbackReason,
};
use crate::batching_mode::eval_schedule::{FlowMissedTickPolicy, FlowScheduleConfig};
use crate::batching_mode::state::CheckpointMode;
use crate::batching_mode::time_window::find_time_window_expr;
use crate::test_utils::create_test_query_engine;
@@ -142,6 +143,77 @@ async fn test_incremental_read_is_disabled_by_default() {
assert!(task.state.read().unwrap().is_incremental_disabled());
}
#[tokio::test]
async fn test_non_aggregate_scheduled_sql_honors_eval_offset_phase() {
// A non-aggregate SQL flow with `EVAL INTERVAL` runs as an explicit
// full-query flow on the batching scheduler. The typed schedule must reach
// the task config unchanged and the offset must not be silently ignored:
// due scheduled times follow the `anchor + k * interval` phase.
let query = "SELECT number, ts FROM numbers_with_ts";
let query_engine = create_test_query_engine();
let ctx = QueryContext::arc();
let plan = sql_to_df_plan(ctx.clone(), query_engine.clone(), query, true)
.await
.unwrap();
let (_tx, rx) = tokio::sync::oneshot::channel();
let schedule = EvalSchedule::from_config(
Some(3600),
Some(&FlowScheduleConfig {
anchor_secs: 120, // `EVAL OFFSET '2 minutes'`
start_secs: 3720, // 120 + 1 * 3600
missed_tick_policy: FlowMissedTickPolicy::BoundedCatchUp,
catchup_max_runs: 3,
catchup_max_lag_secs: 3600,
}),
)
.unwrap()
.unwrap();
let task = BatchingTask::try_new(TaskArgs {
flow_id: 1,
query,
plan,
time_window_expr: None,
expire_after: None,
sink_table_name: [
"greptime".to_string(),
"public".to_string(),
"scheduled_non_aggr_sink".to_string(),
],
source_table_names: vec![[
"greptime".to_string(),
"public".to_string(),
"numbers_with_ts".to_string(),
]],
query_ctx: ctx,
catalog_manager: query_engine.engine_state().catalog_manager().clone(),
shutdown_rx: rx,
batch_opts: Arc::new(BatchingModeOptions::default()),
flow_eval_interval: Some(Duration::from_secs(3600)),
eval_schedule: Some(schedule),
})
.unwrap();
let stored = task.config.eval_schedule.as_ref().unwrap();
assert_eq!(stored.anchor_secs, 120);
assert_eq!(stored.start_secs, 3720);
assert_eq!(stored.interval_secs, 3600);
// Due-time selection proves the offset is honored: the first due time is
// on the `:02` phase (120 + k * 3600), never 0 or 3600.
let due = select_due_scheduled_times(stored, 0, 3720).unwrap();
assert_eq!(due.scheduled_times_secs, vec![3720]);
for t in &due.scheduled_times_secs {
assert_eq!((t - 120) % 3600, 0);
}
let due = select_due_scheduled_times(stored, 3720, 7320).unwrap();
assert_eq!(due.scheduled_times_secs, vec![7320]);
for t in &due.scheduled_times_secs {
assert_eq!((t - 120) % 3600, 0);
}
}
#[tokio::test]
async fn test_dirty_time_windows_uses_batch_opts() {
let task = new_test_task_engine_and_plan_with_query_and_opts(
@@ -764,6 +836,51 @@ async fn test_scheduled_time_now_is_bound_to_selected_attempt() {
assert!(!sent_sql.is_empty());
}
/// The scheduled-loop logical-time arithmetic must never clamp, wrap, or
/// panic near the `i64` boundaries: unrepresentable values are explicit
/// errors, representable values are exact.
#[test]
fn test_scheduled_loop_arithmetic_near_i64_boundary() {
assert_eq!(initial_schedule_cursor(3720, 3600).unwrap(), 120);
assert_eq!(
initial_schedule_cursor(i64::MAX, 3600).unwrap(),
i64::MAX - 3600
);
// start == i64::MIN cannot go one interval earlier: explicit error, never
// a saturated cursor equal to start that would silently skip the first
// scheduled tick.
assert!(initial_schedule_cursor(i64::MIN, 3600).is_err());
assert_eq!(
initial_schedule_cursor(i64::MIN + 3600, 3600).unwrap(),
i64::MIN
);
assert_eq!(
scheduled_time_millis(1_700_000_000).unwrap(),
1_700_000_000_000
);
// Largest seconds value whose millisecond product still fits in i64.
let max_secs = i64::MAX / 1000;
assert_eq!(scheduled_time_millis(max_secs).unwrap(), max_secs * 1000);
// One more second overflows: explicit error, never a saturated i64::MAX.
let err = scheduled_time_millis(max_secs + 1).unwrap_err();
assert!(err.to_string().contains("milliseconds"), "{err}");
assert!(scheduled_time_millis(i64::MAX).is_err());
// The widest representable gap (i64::MIN..=i64::MAX) is exactly u64::MAX;
// subtracting in i64 would panic in debug and wrap in release, so the
// i128 path must return the exact u64 value.
assert_eq!(
sleep_delta_secs(i64::MAX, i64::MIN).unwrap(),
u64::MAX,
"i64::MAX - i64::MIN must be exactly u64::MAX, not a wrapped value"
);
assert_eq!(sleep_delta_secs(7320, 3720).unwrap(), 3600);
// A negative delta (next <= wall_now, violating the caller's guard) is an
// explicit error instead of a wrapped huge `as u64` sleep.
assert!(sleep_delta_secs(100, 200).is_err());
}
fn output_with_region_watermarks(
watermarks: impl IntoIterator<Item = (u64, Option<u64>)>,
) -> OutputWithMetrics {
+2 -1
View File
@@ -1098,6 +1098,7 @@ pub fn to_create_flow_task_expr(
let eval_interval = create_flow.eval_interval;
let flow_options = stringify_flow_options(create_flow.flow_options)?;
Ok(CreateFlowExpr {
catalog_name: query_ctx.current_catalog().to_string(),
flow_name: sanitize_flow_name(create_flow.flow_name)?,
@@ -1109,7 +1110,7 @@ pub fn to_create_flow_task_expr(
eval_interval: eval_interval.map(|seconds| api::v1::EvalInterval { seconds }),
comment: create_flow.comment.unwrap_or_default(),
sql: create_flow.query.to_string(),
flow_options: stringify_flow_options(create_flow.flow_options)?,
flow_options,
})
}
+134 -14
View File
@@ -42,6 +42,7 @@ use common_meta::cache_invalidator::CacheInvalidatorRef;
use common_meta::cache_invalidator::Context;
use common_meta::ddl::create_flow::{
DEFER_ON_MISSING_SOURCE_KEY, FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY, FlowType,
INTERNAL_EVAL_OFFSET_KEY, INTERNAL_EVAL_SCHEDULE_KEY,
};
use common_meta::instruction::CacheIdent;
#[cfg(feature = "enterprise")]
@@ -205,7 +206,10 @@ fn validate_and_normalize_flow_options(
options
.into_iter()
.map(|(key, value)| {
if key == FlowType::FLOW_TYPE_KEY {
if matches!(
key.as_str(),
FlowType::FLOW_TYPE_KEY | INTERNAL_EVAL_OFFSET_KEY | INTERNAL_EVAL_SCHEDULE_KEY
) {
return InvalidSqlSnafu {
err_msg: format!("flow option '{key}' is reserved for internal use"),
}
@@ -232,12 +236,51 @@ fn validate_and_normalize_flow_options(
.collect()
}
/// Validates `EVAL OFFSET` semantics at the operator boundary: an offset is
/// only legal together with `EVAL INTERVAL` and must lie in
/// `[0, eval_interval)`. Never modulo-normalized.
fn validate_eval_offset(
eval_offset_secs: Option<i64>,
eval_interval_secs: Option<i64>,
) -> Result<()> {
if let Some(offset_secs) = eval_offset_secs {
let Some(eval_interval_secs) = eval_interval_secs else {
return InvalidSqlSnafu {
err_msg: "EVAL OFFSET requires EVAL INTERVAL to be specified".to_string(),
}
.fail();
};
if !(0..eval_interval_secs).contains(&offset_secs) {
return InvalidSqlSnafu {
err_msg: format!(
"EVAL OFFSET must be in range [0, EVAL INTERVAL), got {offset_secs} seconds with EVAL INTERVAL {eval_interval_secs} seconds"
),
}
.fail();
}
}
Ok(())
}
fn determine_flow_type_for_source_state(
flow_name: &str,
flow_options: &HashMap<String, String>,
has_missing_source_table: bool,
has_instant_ttl_source_table: bool,
force_batching: bool,
) -> Result<Option<FlowType>> {
if has_instant_ttl_source_table && force_batching {
// The batching scheduler cannot read instant-TTL source tables, so
// reject this combination even when another source table is missing.
return InvalidSqlSnafu {
err_msg: format!(
"flow '{}' with EVAL INTERVAL requires the batching scheduler, but source tables with ttl=instant are not supported under batching mode; use a TTL longer than the flush interval",
flow_name
),
}
.fail();
}
if has_missing_source_table {
let defer_on_missing_source = flow_options
.get(DEFER_ON_MISSING_SOURCE_KEY)
@@ -741,23 +784,31 @@ impl StatementExecutor {
query_context: QueryContextRef,
) -> Result<Output> {
// TODO(ruihang): do some verification
let eval_offset_secs = stmt.eval_offset;
let expr = expr_helper::to_create_flow_task_expr(stmt, &query_context)?;
self.create_flow_inner(expr, query_context).await
// The typed `EVAL OFFSET` comes straight from the parsed SQL AST; it is
// the only trusted source of the offset at this boundary.
self.create_flow_procedure(expr, eval_offset_secs, query_context)
.await?;
Ok(Output::new_with_affected_rows(0))
}
/// Direct gRPC entry point for `CREATE FLOW`.
pub async fn create_flow_inner(
&self,
expr: CreateFlowExpr,
query_context: QueryContextRef,
) -> Result<Output> {
self.create_flow_procedure(expr, query_context).await?;
self.create_flow_procedure(expr, None, query_context)
.await?;
Ok(Output::new_with_affected_rows(0))
}
async fn create_flow_procedure(
&self,
mut expr: CreateFlowExpr,
eval_offset_secs: Option<i64>,
query_context: QueryContextRef,
) -> Result<SubmitDdlTaskResponse> {
let eval_interval_secs = expr.eval_interval.as_ref().map(|e| e.seconds);
@@ -772,6 +823,14 @@ impl StatementExecutor {
.fail();
}
// Validate EVAL OFFSET semantics at the operator boundary (defense in
// depth; the parser already enforces them deterministically). The
// offset arrives as a typed parameter, never from `flow_options`.
validate_eval_offset(eval_offset_secs, eval_interval_secs)?;
// Validate user options. `validate_and_normalize_flow_options` also
// rejects the internal transport keys, so a spoofed key smuggled into
// the expr can never be honored here.
expr.flow_options =
validate_and_normalize_flow_options(expr.flow_options, eval_interval_secs)?;
@@ -783,10 +842,11 @@ impl StatementExecutor {
expr.flow_options
.insert(FlowType::FLOW_TYPE_KEY.to_string(), flow_type.to_string());
let task = CreateFlowTask::try_from(PbCreateFlowTask {
let mut task = CreateFlowTask::try_from(PbCreateFlowTask {
create_flow: Some(expr),
})
.context(error::InvalidExprSnafu)?;
task.eval_offset_secs = eval_offset_secs;
let executor_context = to_executor_context(query_context, TriggerReason::Manual);
let request = SubmitDdlTaskRequest::new(DdlTask::new_create_flow(task));
@@ -796,14 +856,15 @@ impl StatementExecutor {
.context(error::ExecuteDdlSnafu)
}
/// Determine the flow type based on the SQL query
///
/// If it contains aggregation or distinct, then it is a batch flow, otherwise it is a streaming flow
/// Determines the flow type from source-table state, schedule requirements,
/// and SQL shape.
async fn determine_flow_type(
&self,
expr: &CreateFlowExpr,
query_ctx: QueryContextRef,
) -> Result<FlowType> {
let has_eval_interval = expr.eval_interval.is_some();
let mut has_missing_source_table = false;
let mut has_instant_ttl_source_table = false;
@@ -827,13 +888,18 @@ impl StatementExecutor {
if table.table_info().meta.options.ttl == Some(common_time::TimeToLive::Instant) {
warn!(
"Source table `{}` for flow `{}`'s ttl=instant, fallback to streaming mode",
"Source table `{}` for flow `{}`'s ttl=instant, {}",
format_full_table_name(
&src_table_name.catalog_name,
&src_table_name.schema_name,
&src_table_name.table_name
),
expr.flow_name
expr.flow_name,
if has_eval_interval {
"rejecting flow because EVAL INTERVAL requires the batching scheduler which does not support instant TTL"
} else {
"fallback to streaming mode"
}
);
has_instant_ttl_source_table = true;
}
@@ -844,10 +910,20 @@ impl StatementExecutor {
&expr.flow_options,
has_missing_source_table,
has_instant_ttl_source_table,
has_eval_interval,
)? {
return Ok(flow_type);
}
// A flow with `EVAL INTERVAL` (and therefore possibly `EVAL OFFSET`)
// always uses the batching scheduler: the fixed epoch-phase schedule is
// only honored by the batching scheduler, regardless of the SQL shape.
// Non-aggregate SQL is supported as an explicit full-query flow (the
// batching engine requires a time-window expression or EVAL INTERVAL).
if has_eval_interval {
return Ok(FlowType::Batching);
}
let engine = &self.query_engine;
let stmts = ParserContext::create_with_dialect(
&expr.sql,
@@ -3229,7 +3305,6 @@ SELECT max(c1), min(c2) FROM schema_2.table_2;";
"eval_interval_missed_tick_policy",
"eval_interval_catchup_max_runs",
"eval_interval_catchup_max_lag",
"__greptime_internal_eval_schedule",
] {
let err = validate_and_normalize_flow_options(
HashMap::from([(key.to_string(), "value".to_string())]),
@@ -3246,10 +3321,31 @@ SELECT max(c1), min(c2) FROM schema_2.table_2;";
}
#[test]
fn test_determine_flow_type_for_source_state_missing_sources_require_opt_in() {
let err = determine_flow_type_for_source_state("my_flow", &HashMap::new(), true, false)
fn test_internal_transport_keys_rejected_as_reserved() {
for key in [
"__greptime_internal_eval_schedule",
"__greptime_internal_eval_offset_secs",
] {
let err = validate_and_normalize_flow_options(
HashMap::from([(key.to_string(), "120".to_string())]),
Some(300),
)
.unwrap_err();
assert!(
err.to_string()
.contains(&format!("flow option '{key}' is reserved for internal use")),
"unexpected error for {key}: {err}"
);
}
}
#[test]
fn test_determine_flow_type_for_source_state_missing_sources_require_opt_in() {
let err =
determine_flow_type_for_source_state("my_flow", &HashMap::new(), true, false, false)
.unwrap_err();
assert!(err.to_string().contains(
"missing source tables for flow 'my_flow'; use WITH (defer_on_missing_source = true) to create a pending flow"
));
@@ -3261,19 +3357,43 @@ SELECT max(c1), min(c2) FROM schema_2.table_2;";
HashMap::from([(DEFER_ON_MISSING_SOURCE_KEY.to_string(), "true".to_string())]);
assert_eq!(
determine_flow_type_for_source_state("my_flow", &flow_options, true, true).unwrap(),
determine_flow_type_for_source_state("my_flow", &flow_options, true, true, false)
.unwrap(),
Some(FlowType::Batching)
);
let err = determine_flow_type_for_source_state("my_flow", &flow_options, true, true, true)
.unwrap_err();
assert!(err.to_string().contains(
"flow 'my_flow' with EVAL INTERVAL requires the batching scheduler, but source tables with ttl=instant are not supported under batching mode"
));
}
#[test]
fn test_determine_flow_type_for_source_state_instant_ttl_without_missing_sources() {
assert_eq!(
determine_flow_type_for_source_state("my_flow", &HashMap::new(), false, true).unwrap(),
determine_flow_type_for_source_state("my_flow", &HashMap::new(), false, true, false)
.unwrap(),
Some(FlowType::Streaming)
);
}
#[test]
fn test_determine_flow_type_for_source_state_instant_ttl_with_eval_interval_is_rejected() {
// A flow with `EVAL INTERVAL` is forced onto the batching scheduler,
// which cannot read instant-TTL source tables: reject instead of
// silently falling back to streaming (where the schedule/offset would
// be ignored).
let err =
determine_flow_type_for_source_state("my_flow", &HashMap::new(), false, true, true)
.unwrap_err();
assert!(
err.to_string().contains(
"flow 'my_flow' with EVAL INTERVAL requires the batching scheduler, but source tables with ttl=instant are not supported under batching mode"
),
"unexpected error: {err}"
);
}
#[test]
fn test_name_is_match() {
assert!(!NAME_PATTERN_REG.is_match("/adaf"));
+7 -1
View File
@@ -32,8 +32,9 @@ use common_catalog::format_full_table_name;
use common_datasource::file_format::{FileFormat, Format, infer_schemas};
use common_datasource::lister::{Lister, Source};
use common_datasource::object_store::{LocalFileAccess, build_backend_with_path};
use common_error::ext::BoxedError;
use common_meta::SchemaOptions;
use common_meta::ddl::create_flow::FlowType;
use common_meta::ddl::create_flow::{FlowType, effective_eval_schedule_from_flow_info};
use common_meta::key::flow::flow_info::FlowInfoValue;
use common_query::Output;
use common_query::prelude::greptime_timestamp;
@@ -1227,6 +1228,11 @@ pub fn show_create_flow(
if_not_exists: true,
expire_after: flow_val.expire_after(),
eval_interval: flow_val.eval_interval(),
eval_offset: effective_eval_schedule_from_flow_info(&flow_val)
.map_err(BoxedError::new)
.context(error::QueryExecutionSnafu)?
.map(|schedule| schedule.anchor_secs)
.filter(|anchor_secs| *anchor_secs != 0),
comment,
flow_options: OptionMap::from_filtered_string_map(
flow_val.options(),
+230 -12
View File
@@ -325,11 +325,55 @@ impl<'a> ParserContext<'a> {
None
};
let eval_interval = if self
.parser
.consume_tokens(&[Token::make_keyword("EVAL"), Token::make_keyword("INTERVAL")])
{
Some(self.parse_interval_no_month("EVAL INTERVAL")?)
let (eval_interval, eval_interval_has_fractional_secs) =
if self.consume_eval_pair("INTERVAL") {
let (secs, has_fractional) =
self.parse_interval_no_month_whole_secs("EVAL INTERVAL")?;
(Some(secs), has_fractional)
} else {
(None, false)
};
// `EVAL OFFSET` is only legal together with `EVAL INTERVAL`. The phase
// offset must be a whole number of seconds; when an offset is present
// the interval must also be whole seconds (never silently truncated).
let eval_offset = if self.consume_eval_pair("OFFSET") {
let Some(eval_interval) = eval_interval else {
return InvalidIntervalSnafu {
reason: "EVAL OFFSET requires EVAL INTERVAL to be specified".to_string(),
}
.fail();
};
let (offset_secs, has_fractional) =
self.parse_interval_no_month_whole_secs("EVAL OFFSET")?;
if has_fractional {
return InvalidIntervalSnafu {
reason: "EVAL OFFSET must be a whole number of seconds".to_string(),
}
.fail();
}
if eval_interval_has_fractional_secs {
return InvalidIntervalSnafu {
reason: "EVAL INTERVAL must be a whole number of seconds when EVAL OFFSET is specified"
.to_string(),
}
.fail();
}
if !(0..eval_interval).contains(&offset_secs) {
return InvalidIntervalSnafu {
reason: format!(
"EVAL OFFSET must be in range [0, EVAL INTERVAL), got {offset_secs} seconds with EVAL INTERVAL {eval_interval} seconds"
),
}
.fail();
}
// Canonicalize a zero offset to `None` (the default epoch-anchored
// schedule) so that parse/display/reparse round-trips are stable.
if offset_secs == 0 {
None
} else {
Some(offset_secs)
}
} else {
None
};
@@ -372,6 +416,7 @@ impl<'a> ParserContext<'a> {
if_not_exists,
expire_after,
eval_interval,
eval_offset,
comment,
flow_options: flow_option_map(flow_options),
query,
@@ -437,6 +482,15 @@ impl<'a> ParserContext<'a> {
/// Parse the interval expr to duration in seconds.
fn parse_interval_no_month(&mut self, context: &str) -> Result<i64> {
Ok(self.parse_interval_no_month_whole_secs(context)?.0)
}
/// Parses an interval that must not contain months and returns the total
/// whole seconds together with whether the interval contains a sub-second
/// fraction. Whole seconds are computed by truncating the nanosecond part;
/// callers that require exact whole-second precision (e.g. `EVAL OFFSET`)
/// must reject `has_fractional_secs`.
fn parse_interval_no_month_whole_secs(&mut self, context: &str) -> Result<(i64, bool)> {
let interval = self.parse_interval_month_day_nano()?.0;
if interval.months != 0 {
return InvalidIntervalSnafu {
@@ -444,13 +498,30 @@ impl<'a> ParserContext<'a> {
}
.fail();
}
Ok(
interval.nanoseconds / 1_000_000_000
+ interval.days as i64 * 60 * 60 * 24
+ interval.months as i64 * 60 * 60 * 24 * 3044 / 1000, // 1 month=365.25/12=30.44 days
// this is to keep the same as https://docs.rs/humantime/latest/humantime/fn.parse_duration.html
// which we use in database to parse i.e. ttl interval and many other intervals
)
let has_fractional_secs = interval.nanoseconds % 1_000_000_000 != 0;
let whole_secs = interval.nanoseconds / 1_000_000_000 + interval.days as i64 * 60 * 60 * 24;
Ok((whole_secs, has_fractional_secs))
}
/// Consumes an `EVAL <keyword>` token pair case-insensitively.
///
/// `EVAL` is not a sqlparser keyword, so a plain
/// `consume_tokens([Token::make_keyword("EVAL"), ...])` comparison would be
/// case-sensitive on the word value and reject lower-case `eval interval`.
fn consume_eval_pair(&mut self, second: &str) -> bool {
let matches = matches!(
(
&self.parser.peek_token().token,
&self.parser.peek_nth_token(1).token,
),
(Token::Word(w1), Token::Word(w2))
if w1.value.eq_ignore_ascii_case("EVAL") && w2.value.eq_ignore_ascii_case(second)
);
if matches {
self.parser.next_token();
self.parser.next_token();
}
matches
}
/// Parse interval expr to [`IntervalMonthDayNano`].
@@ -1721,6 +1792,7 @@ SELECT max(c1), min(c2) FROM schema_2.table_2;",
if_not_exists: expected.if_not_exists,
expire_after: expected.expire_after,
eval_interval: None,
eval_offset: None,
comment: expected.comment,
flow_options: expected.flow_options,
// ignore query parse result
@@ -1766,6 +1838,9 @@ SELECT max(c1), min(c2) FROM schema_2.table_2;",
/// Duration in seconds as `i64`
/// If not set, flow will be evaluated based on time window size and other args.
pub eval_interval: Option<i64>,
/// Phase offset of the flow evaluation schedule within `eval_interval`.
/// Duration in seconds as `i64`.
pub eval_offset: Option<i64>,
/// Comment string
pub comment: Option<String>,
/// Flow creation options
@@ -1792,6 +1867,7 @@ SELECT max(c1), min(c2) FROM schema_2.table_2;",
if_not_exists: true,
expire_after: Some(300),
eval_interval: None,
eval_offset: None,
comment: Some("test comment".to_string()),
flow_options: OptionMap::default(),
},
@@ -1814,6 +1890,7 @@ SELECT max(c1), min(c2) FROM schema_2.table_2;",
if_not_exists: true,
expire_after: Some(300),
eval_interval: None,
eval_offset: None,
comment: Some("test comment".to_string()),
flow_options: OptionMap::default(),
},
@@ -1837,6 +1914,7 @@ SELECT max(c1), min(c2) FROM schema_2.table_2;",
if_not_exists: true,
expire_after: Some(300),
eval_interval: Some(10),
eval_offset: None,
comment: Some("test comment".to_string()),
flow_options: OptionMap::default(),
},
@@ -1860,6 +1938,7 @@ SELECT max(c1), min(c2) FROM schema_2.table_2;",
if_not_exists: true,
expire_after: Some(300),
eval_interval: Some(10),
eval_offset: None,
comment: Some("test comment".to_string()),
flow_options: OptionMap::default(),
},
@@ -1883,6 +1962,7 @@ SELECT max(c1), min(c2) FROM schema_2.table_2;",
if_not_exists: false,
expire_after: Some(2 * 86400 + 3600 + 2 * 60),
eval_interval: None,
eval_offset: None,
comment: None,
flow_options: OptionMap::default(),
},
@@ -1905,6 +1985,7 @@ SELECT max(c1), min(c2) FROM schema_2.table_2;",
if_not_exists: false,
expire_after: None,
eval_interval: Some(10),
eval_offset: None,
comment: None,
flow_options: string_option_map([
("defer_on_missing_source", "true"),
@@ -1924,6 +2005,7 @@ SELECT max(c1), min(c2) FROM schema_2.table_2;",
if_not_exists: expected.if_not_exists,
expire_after: expected.expire_after,
eval_interval: expected.eval_interval,
eval_offset: expected.eval_offset,
comment: expected.comment,
flow_options: expected.flow_options,
// ignore query parse result
@@ -1937,6 +2019,142 @@ SELECT max(c1), min(c2) FROM schema_2.table_2;",
}
}
#[test]
fn test_parse_create_flow_with_eval_offset() {
use pretty_assertions::assert_eq;
fn parse_create_flow(sql: &str) -> CreateFlow {
let stmts = ParserContext::create_with_dialect(
sql,
&GreptimeDbDialect {},
ParseOptions::default(),
)
.unwrap();
assert_eq!(1, stmts.len());
match &stmts[0] {
Statement::CreateFlow(c) => c.clone(),
_ => panic!("{:?}", stmts[0]),
}
}
let sql = r#"
CREATE FLOW task_1
SINK TO schema_1.table_1
EVAL INTERVAL '1 hour'
EVAL OFFSET '2 minutes'
AS
SELECT max(c1), min(c2) FROM schema_2.table_2;"#;
let create_task = parse_create_flow(sql);
assert_eq!(create_task.eval_interval, Some(3600));
assert_eq!(create_task.eval_offset, Some(120));
let show_create = create_task.to_string();
assert!(
show_create.contains("EVAL OFFSET '120 s'"),
"unexpected display:\n{show_create}"
);
let recreated = parse_create_flow(&show_create);
assert_eq!(recreated, create_task, "input sql is:\n{show_create}");
let sql = r#"
create flow task_2
sink to schema_1.table_1
eval interval '1h'
eval offset '2m'
as
select max(c1), min(c2) from schema_2.table_2;"#;
let create_task = parse_create_flow(sql);
assert_eq!(create_task.eval_interval, Some(3600));
assert_eq!(create_task.eval_offset, Some(120));
// zero offset is canonicalized to `None` and omitted on display
let sql = r#"
CREATE FLOW task_3
SINK TO schema_1.table_1
EVAL INTERVAL '1 hour'
EVAL OFFSET '0 seconds'
AS
SELECT max(c1), min(c2) FROM schema_2.table_2;"#;
let create_task = parse_create_flow(sql);
assert_eq!(create_task.eval_interval, Some(3600));
assert_eq!(create_task.eval_offset, None);
assert!(
!create_task.to_string().contains("EVAL OFFSET"),
"zero offset should be omitted on display"
);
let sql = r#"
CREATE FLOW task_4
SINK TO schema_1.table_1
EVAL OFFSET '2 minutes'
AS
SELECT max(c1), min(c2) FROM schema_2.table_2;"#;
let err =
ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
.unwrap_err()
.to_string();
assert!(
err.contains("EVAL OFFSET requires EVAL INTERVAL"),
"unexpected error: {err}"
);
for (offset, interval) in [
("-1 seconds", "1 hour"),
("1 hour", "1 hour"),
("2 hours", "1 hour"),
] {
let sql = format!(
r#"
CREATE FLOW task_invalid
SINK TO schema_1.table_1
EVAL INTERVAL '{interval}'
EVAL OFFSET '{offset}'
AS
SELECT max(c1), min(c2) FROM schema_2.table_2;"#
);
let err = ParserContext::create_with_dialect(
&sql,
&GreptimeDbDialect {},
ParseOptions::default(),
)
.unwrap_err()
.to_string();
assert!(
err.contains("EVAL OFFSET must be in range"),
"unexpected error for offset {offset}: {err}"
);
}
let sql = r#"
CREATE FLOW task_fractional_offset
SINK TO schema_1.table_1
EVAL INTERVAL '1 hour'
EVAL OFFSET '1.5 seconds'
AS
SELECT max(c1), min(c2) FROM schema_2.table_2;"#;
let err =
ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
.unwrap_err()
.to_string();
assert!(
err.contains("EVAL OFFSET must be a whole number of seconds"),
"unexpected error: {err}"
);
let sql = r#"
CREATE FLOW task_fractional_interval
SINK TO schema_1.table_1
EVAL INTERVAL '1.5 seconds'
EVAL OFFSET '1 second'
AS
SELECT max(c1), min(c2) FROM schema_2.table_2;"#;
let err =
ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
.unwrap_err()
.to_string();
assert!(
err.contains("EVAL INTERVAL must be a whole number of seconds"),
"unexpected error: {err}"
);
}
#[test]
fn test_parse_create_flow_with_tql_cte_query() {
let sql = r#"
+13
View File
@@ -695,6 +695,12 @@ pub struct CreateFlow {
/// Duration in seconds as `i64`
/// If not set, flow will be evaluated based on time window size and other args.
pub eval_interval: Option<i64>,
/// Phase offset of the flow evaluation schedule within `eval_interval`.
/// Duration in seconds as `i64`.
/// Must be in range `[0, eval_interval)`. Only legal together with
/// `eval_interval`. A value of zero (the default) means the schedule is
/// anchored to the Unix epoch, i.e. phases at `k * eval_interval`.
pub eval_offset: Option<i64>,
/// Comment string
pub comment: Option<String>,
/// Flow creation options from `WITH (...)`
@@ -753,6 +759,13 @@ impl Display for CreateFlow {
if let Some(eval_interval) = &self.eval_interval {
writeln!(f, "EVAL INTERVAL '{} s'", eval_interval)?;
}
// Canonical display: omit a zero offset (equivalent to the default
// epoch-anchored schedule). Non-zero offsets are always emitted.
if let Some(eval_offset) = &self.eval_offset
&& *eval_offset != 0
{
writeln!(f, "EVAL OFFSET '{} s'", eval_offset)?;
}
if let Some(comment) = &self.comment {
writeln!(f, "COMMENT '{}'", comment)?;
}
@@ -8,9 +8,9 @@ CREATE TABLE distinct_basic (
Affected Rows: 0
-- should fallback to streaming mode
-- should fallback to streaming mode when there is no EVAL INTERVAL
-- SQLNESS REPLACE id=\d+ id=REDACTED
CREATE FLOW test_distinct_basic SINK TO out_distinct_basic EVAL INTERVAL '1m' AS
CREATE FLOW test_distinct_basic SINK TO out_distinct_basic AS
SELECT
DISTINCT number as dis
FROM
@@ -157,6 +157,39 @@ DROP TABLE out_distinct_basic;
Affected Rows: 0
-- test ttl = instant with EVAL INTERVAL must be rejected
-- since the batching scheduler cannot read instant-TTL source tables and the
-- streaming engine cannot honor the schedule
CREATE TABLE distinct_basic (
"number" INT,
ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY(number),
TIME INDEX(ts)
)WITH ('ttl' = 'instant');
Affected Rows: 0
-- SQLNESS REPLACE id=\d+ id=REDACTED
CREATE FLOW test_distinct_basic SINK TO out_distinct_basic EVAL INTERVAL '1m' AS
SELECT
DISTINCT number as dis
FROM
distinct_basic;
Error: 1004(InvalidArguments), Invalid SQL, error: flow 'test_distinct_basic' with EVAL INTERVAL requires the batching scheduler, but source tables with ttl=instant are not supported under batching mode; use a TTL longer than the flush interval
SELECT count(*) FROM INFORMATION_SCHEMA.FLOWS WHERE flow_name = 'test_distinct_basic';
+----------+
| count(*) |
+----------+
| 0 |
+----------+
DROP TABLE distinct_basic;
Affected Rows: 0
-- test ttl = 5s
CREATE TABLE distinct_basic (
"number" INT,
@@ -6,9 +6,9 @@ CREATE TABLE distinct_basic (
TIME INDEX(ts)
)WITH ('ttl' = 'instant');
-- should fallback to streaming mode
-- should fallback to streaming mode when there is no EVAL INTERVAL
-- SQLNESS REPLACE id=\d+ id=REDACTED
CREATE FLOW test_distinct_basic SINK TO out_distinct_basic EVAL INTERVAL '1m' AS
CREATE FLOW test_distinct_basic SINK TO out_distinct_basic AS
SELECT
DISTINCT number as dis
FROM
@@ -63,6 +63,27 @@ DROP FLOW test_distinct_basic;
DROP TABLE distinct_basic;
DROP TABLE out_distinct_basic;
-- test ttl = instant with EVAL INTERVAL must be rejected
-- since the batching scheduler cannot read instant-TTL source tables and the
-- streaming engine cannot honor the schedule
CREATE TABLE distinct_basic (
"number" INT,
ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY(number),
TIME INDEX(ts)
)WITH ('ttl' = 'instant');
-- SQLNESS REPLACE id=\d+ id=REDACTED
CREATE FLOW test_distinct_basic SINK TO out_distinct_basic EVAL INTERVAL '1m' AS
SELECT
DISTINCT number as dis
FROM
distinct_basic;
SELECT count(*) FROM INFORMATION_SCHEMA.FLOWS WHERE flow_name = 'test_distinct_basic';
DROP TABLE distinct_basic;
-- test ttl = 5s
CREATE TABLE distinct_basic (
"number" INT,
@@ -0,0 +1,110 @@
-- EVAL OFFSET: fixed UTC epoch phase (offset + k * interval), deterministic.
CREATE TABLE eval_offset_input (
ts TIMESTAMP(3) TIME INDEX,
series STRING,
v DOUBLE,
PRIMARY KEY(series)
);
Affected Rows: 0
-- Scheduled at odd seconds (1 + 2k): phase anchored to the Unix epoch.
CREATE FLOW eval_offset_phase_flow
SINK TO eval_offset_phase_sink
EVAL INTERVAL '2s'
EVAL OFFSET '1s'
AS
SELECT date_trunc('second', now()) AS ts, count(v) AS c
FROM eval_offset_input
GROUP BY date_trunc('second', now());
Affected Rows: 0
SHOW CREATE FLOW eval_offset_phase_flow;
+------------------------+------------------------------------------------------------------------------------------------------------------------+
| Flow | Create Flow |
+------------------------+------------------------------------------------------------------------------------------------------------------------+
| eval_offset_phase_flow | CREATE FLOW IF NOT EXISTS eval_offset_phase_flow |
| | SINK TO public.eval_offset_phase_sink |
| | EVAL INTERVAL '2 s' |
| | EVAL OFFSET '1 s' |
| | AS SELECT date_trunc('second', now()) AS ts, count(v) AS c FROM eval_offset_input GROUP BY date_trunc('second', now()) |
+------------------------+------------------------------------------------------------------------------------------------------------------------+
SELECT flow_definition FROM information_schema.flows WHERE flow_name = 'eval_offset_phase_flow';
+------------------------------------------------------------------------------------------------------------------------+
| flow_definition |
+------------------------------------------------------------------------------------------------------------------------+
| CREATE FLOW IF NOT EXISTS eval_offset_phase_flow |
| SINK TO public.eval_offset_phase_sink |
| EVAL INTERVAL '2 s' |
| EVAL OFFSET '1 s' |
| AS SELECT date_trunc('second', now()) AS ts, count(v) AS c FROM eval_offset_input GROUP BY date_trunc('second', now()) |
+------------------------------------------------------------------------------------------------------------------------+
INSERT INTO eval_offset_input VALUES
(now(), 'a', 1.0),
(now(), 'b', 2.0);
Affected Rows: 2
-- SQLNESS SLEEP 5s
-- Prove the offset phase: every row the offset flow wrote has an odd
-- second-of-minute (1 + 2k is always odd). This is deterministic regardless
-- of the exact wall-clock alignment of the evaluations.
SELECT
count(*) > 0 AS offset_flow_ran,
bool_and(date_part('second', ts)::BIGINT % 2 = 1) AS offset_flow_at_odd_seconds
FROM eval_offset_phase_sink;
+-----------------+----------------------------+
| offset_flow_ran | offset_flow_at_odd_seconds |
+-----------------+----------------------------+
| true | true |
+-----------------+----------------------------+
CREATE FLOW invalid_eval_offset_no_interval
SINK TO invalid_eval_offset_sink
EVAL OFFSET '1s'
AS SELECT 1;
Error: 1004(InvalidArguments), Invalid interval provided: EVAL OFFSET requires EVAL INTERVAL to be specified
CREATE FLOW invalid_eval_offset_too_large
SINK TO invalid_eval_offset_sink
EVAL INTERVAL '1s'
EVAL OFFSET '2s'
AS SELECT 1;
Error: 1004(InvalidArguments), Invalid interval provided: EVAL OFFSET must be in range [0, EVAL INTERVAL), got 2 seconds with EVAL INTERVAL 1 seconds
CREATE FLOW invalid_eval_offset_fractional
SINK TO invalid_eval_offset_sink
EVAL INTERVAL '1s'
EVAL OFFSET '0.5s'
AS SELECT 1;
Error: 1004(InvalidArguments), Invalid interval provided: EVAL OFFSET must be a whole number of seconds
CREATE FLOW invalid_eval_offset_fractional_interval
SINK TO invalid_eval_offset_sink
EVAL INTERVAL '1.5s'
EVAL OFFSET '1s'
AS SELECT 1;
Error: 1004(InvalidArguments), Invalid interval provided: EVAL INTERVAL must be a whole number of seconds when EVAL OFFSET is specified
DROP FLOW eval_offset_phase_flow;
Affected Rows: 0
DROP TABLE eval_offset_phase_sink;
Affected Rows: 0
DROP TABLE eval_offset_input;
Affected Rows: 0
@@ -0,0 +1,63 @@
-- EVAL OFFSET: fixed UTC epoch phase (offset + k * interval), deterministic.
CREATE TABLE eval_offset_input (
ts TIMESTAMP(3) TIME INDEX,
series STRING,
v DOUBLE,
PRIMARY KEY(series)
);
-- Scheduled at odd seconds (1 + 2k): phase anchored to the Unix epoch.
CREATE FLOW eval_offset_phase_flow
SINK TO eval_offset_phase_sink
EVAL INTERVAL '2s'
EVAL OFFSET '1s'
AS
SELECT date_trunc('second', now()) AS ts, count(v) AS c
FROM eval_offset_input
GROUP BY date_trunc('second', now());
SHOW CREATE FLOW eval_offset_phase_flow;
SELECT flow_definition FROM information_schema.flows WHERE flow_name = 'eval_offset_phase_flow';
INSERT INTO eval_offset_input VALUES
(now(), 'a', 1.0),
(now(), 'b', 2.0);
-- SQLNESS SLEEP 5s
-- Prove the offset phase: every row the offset flow wrote has an odd
-- second-of-minute (1 + 2k is always odd). This is deterministic regardless
-- of the exact wall-clock alignment of the evaluations.
SELECT
count(*) > 0 AS offset_flow_ran,
bool_and(date_part('second', ts)::BIGINT % 2 = 1) AS offset_flow_at_odd_seconds
FROM eval_offset_phase_sink;
CREATE FLOW invalid_eval_offset_no_interval
SINK TO invalid_eval_offset_sink
EVAL OFFSET '1s'
AS SELECT 1;
CREATE FLOW invalid_eval_offset_too_large
SINK TO invalid_eval_offset_sink
EVAL INTERVAL '1s'
EVAL OFFSET '2s'
AS SELECT 1;
CREATE FLOW invalid_eval_offset_fractional
SINK TO invalid_eval_offset_sink
EVAL INTERVAL '1s'
EVAL OFFSET '0.5s'
AS SELECT 1;
CREATE FLOW invalid_eval_offset_fractional_interval
SINK TO invalid_eval_offset_sink
EVAL INTERVAL '1.5s'
EVAL OFFSET '1s'
AS SELECT 1;
DROP FLOW eval_offset_phase_flow;
DROP TABLE eval_offset_phase_sink;
DROP TABLE eval_offset_input;
@@ -0,0 +1,8 @@
name = "flow_eval_offset_persistence"
reason = "Verify a non-zero EVAL OFFSET created by a v1.3+ old stage persists in flow schedule metadata and remains visible after a distributed full restart on a v1.3+ new stage."
introduced_by = "#8878 thread r3911676153"
topologies = ["distributed"]
from_range = [">=v1.3.0"]
to_range = [">=v1.3.0"]
features = ["flow", "table"]
owner = "flow"
@@ -0,0 +1,15 @@
CREATE TABLE compat_eval_offset_input (
ts TIMESTAMP(3) TIME INDEX,
v DOUBLE
);
CREATE FLOW compat_eval_offset_flow
SINK TO compat_eval_offset_sink
EVAL INTERVAL '10s'
EVAL OFFSET '3s'
AS
SELECT
date_trunc('second', now()) AS ts,
count(v) AS value_count
FROM compat_eval_offset_input
GROUP BY date_trunc('second', now());
@@ -0,0 +1,11 @@
SHOW CREATE FLOW compat_eval_offset_flow;
+-------------------------+-----------------------------------------------------------------------------------------------------------------------------------------+
| Flow | Create Flow |
+-------------------------+-----------------------------------------------------------------------------------------------------------------------------------------+
| compat_eval_offset_flow | CREATE FLOW IF NOT EXISTS compat_eval_offset_flow |
| | SINK TO flow_eval_offset_persistence.compat_eval_offset_sink |
| | EVAL INTERVAL '10 s' |
| | EVAL OFFSET '3 s' |
| | AS SELECT date_trunc('second', now()) AS ts, count(v) AS value_count FROM compat_eval_offset_input GROUP BY date_trunc('second', now()) |
+-------------------------+-----------------------------------------------------------------------------------------------------------------------------------------+
@@ -0,0 +1 @@
SHOW CREATE FLOW compat_eval_offset_flow;