feat(flow): support eval schedule offsets (#8878)

* feat(flow): support eval schedule offsets

Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com>

* fix(flow): remove redundant schedule assertion

Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com>

* refactor(flow): trim eval offset compatibility scope

Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com>

* test(flow): trim eval offset edge coverage

Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com>

* docs(flow): trim eval offset comment noise

Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com>

* fix(flow): address eval offset review feedback

Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com>

* test(compat): cover Flow eval offset persistence

Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com>

---------

Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com>
This commit is contained in:
discord9
2026-09-04 02:39:59 +00:00
committed by GitHub
parent 84bd993131
commit ad7b0ace64
22 changed files with 1702 additions and 151 deletions
+137 -36
View File
@@ -21,6 +21,7 @@ use api::v1::ExpireAfter;
use api::v1::flow::flow_request::Body as PbFlowRequest;
use api::v1::flow::{CreateRequest, FlowRequest, FlowRequestHeader};
use async_trait::async_trait;
use chrono::{DateTime, Utc};
use common_catalog::format_full_flow_name;
use common_procedure::error::{FromJsonSnafu, ToJsonSnafu};
use common_procedure::{
@@ -32,7 +33,7 @@ use common_telemetry::tracing_context::TracingContext;
use futures::future::join_all;
use itertools::Itertools;
use serde::{Deserialize, Serialize};
use snafu::{ResultExt, ensure};
use snafu::{OptionExt, ResultExt, ensure};
use strum::AsRefStr;
use table::metadata::TableId;
use table::table_name::TableName;
@@ -222,7 +223,7 @@ impl CreateFlowProcedure {
.prev_flow_info_value
.as_ref()
.map(|v| v.get_inner_ref()),
);
)?;
self.data.state = if self.data.is_pending() {
self.data.peers.clear();
@@ -465,6 +466,14 @@ pub fn get_flow_type_from_options(flow_task: &CreateFlowTask) -> Result<FlowType
/// The flow option key for creating pending flow metadata when source tables do not exist.
pub const DEFER_ON_MISSING_SOURCE_KEY: &str = "defer_on_missing_source";
/// Internal transient key used to pass the typed `EVAL OFFSET` (whole seconds)
/// from the operator to meta through `CreateFlowExpr.flow_options`. Inserted by
/// the operator only after user option validation; parsed and stripped by
/// `CreateFlowTask::try_from`. Must never be accepted as a user-provided option
/// and must never be persisted into `FlowInfoValue.options` or be visible in
/// user runtime options / SHOW CREATE.
pub const INTERNAL_EVAL_OFFSET_KEY: &str = "__greptime_internal_eval_offset_secs";
/// Internal transient key used to pass the serialized `FlowScheduleConfig` from
/// meta to flownode through `CreateRequest.flow_options`. This key must never
/// be accepted as a user-provided option and must never be persisted into
@@ -515,6 +524,34 @@ pub fn validate_flow_options(flow_task: &CreateFlowTask) -> Result<()> {
.fail();
}
for key in [INTERNAL_EVAL_OFFSET_KEY, INTERNAL_EVAL_SCHEDULE_KEY] {
if flow_task.flow_options.contains_key(key) {
return UnexpectedSnafu {
err_msg: format!("flow option '{key}' is reserved for internal use"),
}
.fail();
}
}
// EVAL OFFSET semantics: only legal with EVAL INTERVAL and must be in
// `[0, eval_interval_secs)`. Never modulo-normalized.
if let Some(offset_secs) = flow_task.eval_offset_secs {
let Some(eval_interval_secs) = flow_task.eval_interval_secs else {
return UnexpectedSnafu {
err_msg: "EVAL OFFSET requires EVAL INTERVAL to be specified".to_string(),
}
.fail();
};
if !(0..eval_interval_secs).contains(&offset_secs) {
return UnexpectedSnafu {
err_msg: format!(
"EVAL OFFSET must be in range [0, EVAL INTERVAL), got {offset_secs} seconds with EVAL INTERVAL {eval_interval_secs} seconds"
),
}
.fail();
}
}
for key in flow_task.flow_options.keys() {
match key.as_str() {
DEFER_ON_MISSING_SOURCE_KEY
@@ -538,12 +575,16 @@ pub fn validate_flow_options(flow_task: &CreateFlowTask) -> Result<()> {
/// Computes the ceiling of `time` to the next schedule boundary aligned with `anchor + k * interval`.
/// All values are Unix timestamps in seconds.
fn ceil_to_boundary(time: i64, anchor: i64, interval: i64) -> i64 {
///
/// Fallible: if the next boundary after `time` does not fit in `i64`, an
/// explicit error is returned instead of clamping to a non-phase value such as
/// `i64::MAX`.
pub(crate) fn ceil_to_boundary(time: i64, anchor: i64, interval: i64) -> Result<i64> {
if interval <= 0 {
return time;
return Ok(time);
}
if time <= anchor {
return anchor;
return Ok(anchor);
}
let diff = i128::from(time) - i128::from(anchor);
@@ -551,7 +592,41 @@ fn ceil_to_boundary(time: i64, anchor: i64, interval: i64) -> i64 {
let k = (diff + interval - 1) / interval;
let boundary = i128::from(anchor) + k * interval;
boundary.clamp(i128::from(i64::MIN), i128::from(i64::MAX)) as i64
i64::try_from(boundary).map_err(|_| {
UnexpectedSnafu {
err_msg: format!(
"Cannot align time {time} to the next `anchor + k * interval` boundary (anchor={anchor}, interval={interval}): result {boundary} does not fit in i64"
),
}
.build()
})
}
/// Rounds a `Utc` instant up to the next whole second (Unix seconds), with
/// nanosecond precision.
///
/// `timestamp()` / `timestamp_millis()` floor the sub-second fraction; using
/// them unchanged could produce a `start_secs` in the past relative to the
/// exact prepare instant, which would make the very first evaluation due before
/// the flow finished being prepared. Uses checked arithmetic so an instant at
/// the very end of the `i64` second range yields an explicit error instead of
/// wrapping.
pub(crate) fn ceil_to_whole_sec(now: DateTime<Utc>) -> Result<i64> {
ceil_whole_sec_from_parts(now.timestamp(), now.timestamp_subsec_nanos() != 0)
}
/// Pure ceiling computation factored out of [`ceil_to_whole_sec`] so the
/// overflow path is testable: `chrono::DateTime<Utc>` cannot represent
/// instants near `i64::MAX` seconds, but the arithmetic below guards it anyway.
pub(crate) fn ceil_whole_sec_from_parts(secs: i64, has_fraction: bool) -> Result<i64> {
if !has_fraction {
return Ok(secs);
}
secs.checked_add(1).context(error::UnexpectedSnafu {
err_msg: format!(
"Cannot round instant at second {secs} up to the next whole second: timestamp overflow"
),
})
}
/// Returns the effective typed schedule config for flow metadata.
@@ -561,26 +636,31 @@ fn ceil_to_boundary(time: i64, anchor: i64, interval: i64) -> i64 {
/// and defaults. This avoids recovery-time wall-clock drift.
pub fn effective_eval_schedule_from_flow_info(
flow_info: &FlowInfoValue,
) -> Option<FlowScheduleConfig> {
) -> Result<Option<FlowScheduleConfig>> {
if let Some(schedule) = &flow_info.eval_schedule {
return Some(schedule.clone());
return Ok(Some(schedule.clone()));
}
let eval_interval_secs = flow_info.eval_interval_secs?;
let Some(eval_interval_secs) = flow_info.eval_interval_secs else {
return Ok(None);
};
if eval_interval_secs <= 0 {
return None;
return Ok(None);
}
// Round the created_time up to the next whole second (same helper as new
// flow resolution) before aligning to the epoch-anchored boundary.
let created_ceil = ceil_to_whole_sec(flow_info.created_time)?;
let start_secs = ceil_to_boundary(
flow_info.created_time.timestamp(),
created_ceil,
FlowScheduleConfig::DEFAULT_ANCHOR_SECS,
eval_interval_secs,
);
)?;
Some(FlowScheduleConfig::default_with_start(
Ok(Some(FlowScheduleConfig::default_with_start(
start_secs,
eval_interval_secs,
))
)))
}
/// Resolve `FlowScheduleConfig` into `task.eval_schedule`.
@@ -591,58 +671,77 @@ pub fn effective_eval_schedule_from_flow_info(
/// The function is idempotent: if `task.eval_schedule` is already `Some`,
/// it returns immediately (important for procedure retry / dump-restore).
///
/// Schedule configuration is NOT read from `task.flow_options` (those keys are
/// no longer user-facing options). For new flows, pure defaults are computed.
/// For OR REPLACE, the previous typed config is preserved when interval+anchor
/// are unchanged; otherwise defaults are recomputed.
/// The schedule phase (anchor) is the `EVAL OFFSET` value: boundaries are
/// `offset + k * interval` (Unix epoch seconds), independent of timezone/DST.
/// An omitted offset means zero. It is NOT anchored to create time and does
/// not drift. Schedule configuration is NOT read from `task.flow_options`
/// (those keys are no longer user-facing options).
///
/// For OR REPLACE, the previous typed config is preserved when interval+offset
/// are unchanged; otherwise the schedule is recomputed.
///
/// Fallible: if the next phase boundary cannot fit in `i64` (extremely far
/// future), an explicit error is returned and flow creation fails instead of
/// silently clamping to a non-phase timestamp.
pub(crate) fn resolve_schedule_defaults_into_task(
task: &mut CreateFlowTask,
prev_flow_info: Option<&FlowInfoValue>,
) {
) -> Result<()> {
// Idempotent: if already computed, skip recomputation.
if task.eval_schedule.is_some() {
return;
return Ok(());
}
let Some(eval_interval_secs) = task.eval_interval_secs else {
return;
return Ok(());
};
if eval_interval_secs <= 0 {
return;
return Ok(());
}
let anchor_secs = FlowScheduleConfig::DEFAULT_ANCHOR_SECS;
let anchor_secs = task.eval_offset_secs.unwrap_or(0);
// Defense: `validate_flow_options` (called before this in `on_prepare`)
// already rejects out-of-range offsets; guard here to never schedule on a
// garbage anchor.
if !(0..eval_interval_secs).contains(&anchor_secs) {
return Ok(());
}
// For OR REPLACE: if interval+anchor unchanged, preserve the entire
// existing typed config so start / policy / limits are stable.
if task.or_replace
&& let Some(prev) = prev_flow_info
&& let Some(old_sched) = effective_eval_schedule_from_flow_info(prev)
&& let Some(old_sched) = effective_eval_schedule_from_flow_info(prev)?
{
let old_interval = prev.eval_interval_secs.unwrap_or(0);
if old_interval == eval_interval_secs && old_sched.anchor_secs == anchor_secs {
task.eval_schedule = Some(old_sched);
return;
return Ok(());
}
}
// --- Compute start ---
let start_secs =
// New flow, or OR REPLACE with changed interval/anchor: start at the next
// aligned boundary after prepare time. The value is written into
// task.eval_schedule once so procedure retry does not recompute it.
ceil_to_boundary(chrono::Utc::now().timestamp(), anchor_secs, eval_interval_secs);
// New flow, or OR REPLACE with changed interval/offset: start at the next
// aligned boundary strictly after the exact prepare instant, rounded up to
// the next whole second so the first boundary is never in the past. The
// value is written into task.eval_schedule once so procedure retry does not
// recompute it.
let prepare_secs = ceil_to_whole_sec(chrono::Utc::now())?;
let start_secs = ceil_to_boundary(prepare_secs, anchor_secs, eval_interval_secs)?;
task.eval_schedule = Some(FlowScheduleConfig::default_with_start(
task.eval_schedule = Some(FlowScheduleConfig::with_anchor(
anchor_secs,
start_secs,
eval_interval_secs,
));
Ok(())
}
fn user_runtime_flow_options(options: &HashMap<String, String>) -> HashMap<String, String> {
let mut options = options.clone();
options.remove(DEFER_ON_MISSING_SOURCE_KEY);
options.remove(INTERNAL_EVAL_SCHEDULE_KEY);
options.remove(INTERNAL_EVAL_OFFSET_KEY);
options
}
@@ -772,14 +871,16 @@ impl From<&CreateFlowData> for (FlowInfoValue, Vec<(FlowPartitionId, FlowRouteVa
let sql = value.task.sql.clone();
let eval_schedule = value.task.eval_schedule.clone();
// Start with a clean options map. The transient schedule payload is
// only for the meta→flownode CreateRequest boundary and must not be
// persisted in FlowInfoValue.options.
// Start with a clean options map. The transient schedule/offset payloads
// are only for the meta→flownode / frontend→meta boundaries and must not
// be persisted in FlowInfoValue.options.
let mut options: HashMap<String, String> = value
.task
.flow_options
.iter()
.filter(|(k, _)| k.as_str() != INTERNAL_EVAL_SCHEDULE_KEY)
.filter(|(k, _)| {
k.as_str() != INTERNAL_EVAL_SCHEDULE_KEY && k.as_str() != INTERNAL_EVAL_OFFSET_KEY
})
.map(|(k, v)| (k.clone(), v.clone()))
.collect();
+314 -12
View File
@@ -25,9 +25,9 @@ use table::table_name::TableName;
use crate::ddl::DdlContext;
use crate::ddl::create_flow::{
CreateFlowData, CreateFlowProcedure, CreateFlowState, DEFER_ON_MISSING_SOURCE_KEY,
FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY, FlowType, defer_on_missing_source,
effective_eval_schedule_from_flow_info, resolve_schedule_defaults_into_task,
validate_flow_options,
FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY, FlowType, ceil_to_whole_sec,
defer_on_missing_source, effective_eval_schedule_from_flow_info,
resolve_schedule_defaults_into_task, validate_flow_options,
};
use crate::ddl::test_util::create_table::test_create_table_task;
use crate::ddl::test_util::flownode_handler::NaiveFlownodeHandler;
@@ -67,6 +67,7 @@ pub(crate) fn test_create_flow_task(
create_if_not_exists,
expire_after: Some(300),
eval_interval_secs: None,
eval_offset_secs: None,
comment: "".to_string(),
sql: "select 1".to_string(),
flow_options: Default::default(),
@@ -333,7 +334,6 @@ fn test_validate_flow_options_rejects_schedule_and_internal_keys_as_unknown() {
"eval_interval_missed_tick_policy",
"eval_interval_catchup_max_runs",
"eval_interval_catchup_max_lag",
"__greptime_internal_eval_schedule",
] {
let mut task = test_create_flow_task(
"my_flow",
@@ -354,6 +354,78 @@ fn test_validate_flow_options_rejects_schedule_and_internal_keys_as_unknown() {
}
}
#[test]
fn test_validate_flow_options_rejects_internal_transport_keys() {
for key in [
"__greptime_internal_eval_schedule",
"__greptime_internal_eval_offset_secs",
] {
let mut task = test_create_flow_task(
"my_flow",
vec![],
TableName::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, "my_sink_table"),
false,
);
task.eval_interval_secs = Some(300);
task.flow_options
.insert(key.to_string(), "value".to_string());
let err = validate_flow_options(&task).unwrap_err();
assert!(
err.to_string()
.contains(&format!("flow option '{key}' is reserved for internal use")),
"unexpected error for {key}: {err}"
);
}
}
#[test]
fn test_validate_flow_options_rejects_invalid_eval_offset() {
let mut task = test_create_flow_task(
"my_flow",
vec![],
TableName::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, "my_sink_table"),
false,
);
task.eval_interval_secs = None;
task.eval_offset_secs = Some(60);
let err = validate_flow_options(&task).unwrap_err();
assert!(
err.to_string()
.contains("EVAL OFFSET requires EVAL INTERVAL"),
"unexpected error: {err}"
);
for offset_secs in [-1, 300, 301] {
let mut task = test_create_flow_task(
"my_flow",
vec![],
TableName::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, "my_sink_table"),
false,
);
task.eval_interval_secs = Some(300);
task.eval_offset_secs = Some(offset_secs);
let err = validate_flow_options(&task).unwrap_err();
assert!(
err.to_string()
.contains("EVAL OFFSET must be in range [0, EVAL INTERVAL)"),
"unexpected error for offset {offset_secs}: {err}"
);
}
for offset_secs in [0, 1, 299] {
let mut task = test_create_flow_task(
"my_flow",
vec![],
TableName::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, "my_sink_table"),
false,
);
task.eval_interval_secs = Some(300);
task.eval_offset_secs = Some(offset_secs);
validate_flow_options(&task).unwrap();
}
}
#[test]
fn test_validate_flow_options_rejects_non_positive_eval_interval() {
for secs in [0, -1] {
@@ -381,7 +453,7 @@ fn test_resolved_schedule_defaults_in_create_request() {
task.eval_interval_secs = Some(300);
// Simulate on_prepare: resolve defaults into task.
resolve_schedule_defaults_into_task(&mut task, None);
resolve_schedule_defaults_into_task(&mut task, None).unwrap();
// Verify typed schedule config is populated.
let sched = task.eval_schedule.as_ref().unwrap();
@@ -410,7 +482,7 @@ fn test_resolved_schedule_defaults_in_create_request() {
// Idempotent: second call does not overwrite.
let start_before = sched.start_secs;
resolve_schedule_defaults_into_task(&mut task, None);
resolve_schedule_defaults_into_task(&mut task, None).unwrap();
assert_eq!(
task.eval_schedule.as_ref().unwrap().start_secs,
start_before
@@ -483,6 +555,221 @@ fn test_resolved_schedule_defaults_in_create_request() {
}
}
#[test]
fn test_resolve_schedule_with_eval_offset_uses_epoch_phase() {
// 1h interval + 120s offset must yield the `:02` phase: anchor 120 and a
// start on `120 + k * 3600`, and no earlier boundary than the prepare ceil.
let prepare_ceil = ceil_to_whole_sec(chrono::Utc::now()).unwrap();
let mut task = test_create_flow_task(
"my_flow",
vec![],
TableName::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, "my_sink_table"),
false,
);
task.eval_interval_secs = Some(3600);
task.eval_offset_secs = Some(120);
resolve_schedule_defaults_into_task(&mut task, None).unwrap();
let sched = task.eval_schedule.as_ref().unwrap();
assert_eq!(sched.anchor_secs, 120);
assert!(sched.start_secs >= prepare_ceil);
assert_eq!((sched.start_secs - 120) % 3600, 0);
// The offset key must not leak into user runtime options / persisted info.
let flow_context = FlowQueryContext {
catalog: DEFAULT_CATALOG_NAME.to_string(),
schema: DEFAULT_SCHEMA_NAME.to_string(),
timezone: "UTC".to_string(),
extensions: HashMap::new(),
channel: 0,
snapshot_seqs: HashMap::new(),
sst_min_sequences: HashMap::new(),
};
let data = CreateFlowData {
state: CreateFlowState::CreateFlows,
task,
flow_id: Some(1024),
peers: vec![],
source_table_ids: vec![],
unresolved_source_table_names: vec![],
flow_context: flow_context.clone(),
prev_flow_info_value: None,
did_replace: false,
flow_type: Some(FlowType::Batching),
};
let request: CreateRequest = (&data).into();
assert!(
!request
.flow_options
.contains_key("__greptime_internal_eval_offset_secs"),
"CreateRequest must not contain internal eval offset key"
);
let (flow_info, _) = <(FlowInfoValue, Vec<(_, _)>)>::from(&data);
assert_eq!(flow_info.eval_schedule.as_ref().unwrap().anchor_secs, 120);
assert!(
!flow_info
.options
.contains_key("__greptime_internal_eval_offset_secs"),
"FlowInfoValue.options must not contain internal eval offset key"
);
}
#[test]
fn test_resolve_schedule_or_replace_preserves_and_recomputes() {
let sink = TableName::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, "my_sink_table");
let mut task = test_create_flow_task("my_flow", vec![], sink.clone(), false);
task.or_replace = false;
task.eval_interval_secs = Some(3600);
task.eval_offset_secs = Some(120);
resolve_schedule_defaults_into_task(&mut task, None).unwrap();
let original_sched = task.eval_schedule.clone().unwrap();
assert_eq!(original_sched.anchor_secs, 120);
let prev_info = flow_info_for_schedule_test(
Some(original_sched.clone()),
Some(3600),
HashMap::new(),
1_700_000_000,
);
let mut replace_same = test_create_flow_task("my_flow", vec![], sink.clone(), false);
replace_same.or_replace = true;
replace_same.eval_interval_secs = Some(3600);
replace_same.eval_offset_secs = Some(120);
resolve_schedule_defaults_into_task(&mut replace_same, Some(&prev_info)).unwrap();
assert_eq!(replace_same.eval_schedule, Some(original_sched.clone()));
let mut replace_changed = test_create_flow_task("my_flow", vec![], sink.clone(), false);
replace_changed.or_replace = true;
replace_changed.eval_interval_secs = Some(3600);
replace_changed.eval_offset_secs = Some(180);
resolve_schedule_defaults_into_task(&mut replace_changed, Some(&prev_info)).unwrap();
let recomputed = replace_changed.eval_schedule.as_ref().unwrap();
assert_eq!(recomputed.anchor_secs, 180);
assert_eq!((recomputed.start_secs - 180) % 3600, 0);
assert_ne!(recomputed.start_secs, original_sched.start_secs);
let mut replace_interval = test_create_flow_task("my_flow", vec![], sink.clone(), false);
replace_interval.or_replace = true;
replace_interval.eval_interval_secs = Some(1800);
replace_interval.eval_offset_secs = Some(120);
resolve_schedule_defaults_into_task(&mut replace_interval, Some(&prev_info)).unwrap();
assert_eq!(
replace_interval.eval_schedule.as_ref().unwrap().anchor_secs,
120
);
let mut replace_omitted = test_create_flow_task("my_flow", vec![], sink, false);
replace_omitted.or_replace = true;
replace_omitted.eval_interval_secs = Some(3600);
replace_omitted.eval_offset_secs = None;
resolve_schedule_defaults_into_task(&mut replace_omitted, Some(&prev_info)).unwrap();
let reset = replace_omitted.eval_schedule.as_ref().unwrap();
assert_eq!(reset.anchor_secs, 0);
assert_eq!(reset.start_secs % 3600, 0);
}
#[test]
fn test_create_flow_task_serde_defaults_for_old_data() {
// Old serialized CreateFlowTask data without `eval_offset_secs` must
// deserialize with `eval_offset_secs: None`.
let json = r#"{
"catalog_name": "catalog",
"flow_name": "flow",
"source_table_names": [],
"sink_table_name": {"catalog_name": "catalog", "schema_name": "schema", "table_name": "sink"},
"or_replace": false,
"create_if_not_exists": false,
"expire_after": null,
"eval_interval_secs": 300,
"comment": "",
"sql": "select 1",
"flow_options": {}
}"#;
let task: CreateFlowTask = serde_json::from_str(json).unwrap();
assert_eq!(task.eval_offset_secs, None);
assert_eq!(task.eval_interval_secs, Some(300));
let encoded = serde_json::to_string(&task).unwrap();
let decoded: CreateFlowTask = serde_json::from_str(&encoded).unwrap();
assert_eq!(decoded.eval_offset_secs, None);
// A task with an offset round-trips through JSON (procedure dump/restore).
let mut task = test_create_flow_task(
"my_flow",
vec![],
TableName::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, "my_sink_table"),
false,
);
task.eval_interval_secs = Some(3600);
task.eval_offset_secs = Some(120);
let encoded = serde_json::to_string(&task).unwrap();
let decoded: CreateFlowTask = serde_json::from_str(&encoded).unwrap();
assert_eq!(decoded.eval_offset_secs, Some(120));
assert_eq!(decoded.eval_interval_secs, Some(3600));
}
#[test]
fn test_create_flow_task_proto_roundtrip_preserves_offset_key() {
// Path A: the operator inserts the trusted offset key into flow_options
// (as produced by `to_create_flow_task_expr`); the proto round-trip
// (frontend -> metasrv DDL task) must preserve it.
let mut task = test_create_flow_task(
"my_flow",
vec![],
TableName::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, "my_sink_table"),
false,
);
task.eval_interval_secs = Some(3600);
task.flow_options.insert(
"__greptime_internal_eval_offset_secs".to_string(),
"120".to_string(),
);
let pb: api::v1::meta::CreateFlowTask = task.clone().into();
assert_eq!(
pb.create_flow
.as_ref()
.unwrap()
.flow_options
.get("__greptime_internal_eval_offset_secs"),
Some(&"120".to_string())
);
let parsed = CreateFlowTask::try_from(pb).unwrap();
assert_eq!(parsed.eval_offset_secs, Some(120));
assert!(
!parsed
.flow_options
.contains_key("__greptime_internal_eval_offset_secs")
);
assert_eq!(parsed.eval_interval_secs, Some(3600));
// Path B: a typed task carrying `eval_offset_secs` (e.g. after procedure
// dump/restore) re-inserts the key when converted back to proto.
let mut typed = test_create_flow_task(
"my_flow",
vec![],
TableName::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, "my_sink_table"),
false,
);
typed.eval_interval_secs = Some(3600);
typed.eval_offset_secs = Some(120);
let pb: api::v1::meta::CreateFlowTask = typed.clone().into();
assert_eq!(
pb.create_flow
.as_ref()
.unwrap()
.flow_options
.get("__greptime_internal_eval_offset_secs"),
Some(&"120".to_string())
);
let reparsed = CreateFlowTask::try_from(pb).unwrap();
assert_eq!(reparsed.eval_offset_secs, Some(120));
assert_eq!(reparsed.flow_options, typed.flow_options);
}
#[test]
fn test_effective_eval_schedule_prefers_typed_config() {
let typed = FlowScheduleConfig {
@@ -496,7 +783,9 @@ fn test_effective_eval_schedule_prefers_typed_config() {
let flow_info =
flow_info_for_schedule_test(Some(typed.clone()), Some(60), unrelated_options, 1);
let effective = effective_eval_schedule_from_flow_info(&flow_info).unwrap();
let effective = effective_eval_schedule_from_flow_info(&flow_info)
.unwrap()
.unwrap();
assert_eq!(effective, typed);
}
@@ -504,7 +793,9 @@ fn test_effective_eval_schedule_prefers_typed_config() {
fn test_effective_eval_schedule_uses_created_time_default() {
let flow_info = flow_info_for_schedule_test(None, Some(60), HashMap::new(), 61);
let effective = effective_eval_schedule_from_flow_info(&flow_info).unwrap();
let effective = effective_eval_schedule_from_flow_info(&flow_info)
.unwrap()
.unwrap();
assert_eq!(effective.anchor_secs, 0);
assert_eq!(effective.start_secs, 120);
assert_eq!(
@@ -518,7 +809,11 @@ fn test_effective_eval_schedule_uses_created_time_default() {
#[test]
fn test_effective_eval_schedule_none_without_eval_interval() {
let flow_info = flow_info_for_schedule_test(None, None, HashMap::new(), 61);
assert!(effective_eval_schedule_from_flow_info(&flow_info).is_none());
assert!(
effective_eval_schedule_from_flow_info(&flow_info)
.unwrap()
.is_none()
);
}
#[test]
@@ -526,8 +821,12 @@ fn test_effective_eval_schedule_deterministic_on_old_metadata() {
// Old metadata: no typed eval_schedule, but has eval_interval_secs and
// a fixed created_time. Repeated calls must produce the same config.
let flow_info = flow_info_for_schedule_test(None, Some(60), HashMap::new(), 61);
let first = effective_eval_schedule_from_flow_info(&flow_info).unwrap();
let second = effective_eval_schedule_from_flow_info(&flow_info).unwrap();
let first = effective_eval_schedule_from_flow_info(&flow_info)
.unwrap()
.unwrap();
let second = effective_eval_schedule_from_flow_info(&flow_info)
.unwrap()
.unwrap();
assert_eq!(first, second);
// Sanity: the derived values are based on created_time=61 + interval=60.
assert_eq!(first.anchor_secs, 0);
@@ -556,7 +855,9 @@ fn test_old_flow_info_json_without_eval_schedule_deserializes() {
let decoded: FlowInfoValue = serde_json::from_value(json).unwrap();
assert!(decoded.eval_schedule.is_none());
let effective = effective_eval_schedule_from_flow_info(&decoded).unwrap();
let effective = effective_eval_schedule_from_flow_info(&decoded)
.unwrap()
.unwrap();
assert_eq!(effective.anchor_secs, 0);
assert_eq!(effective.start_secs, 120);
assert_eq!(
@@ -974,6 +1275,7 @@ fn create_test_flow_task_for_serialization() -> CreateFlowTask {
create_if_not_exists: false,
expire_after: None,
eval_interval_secs: None,
eval_offset_secs: None,
comment: "test comment".to_string(),
sql: "SELECT * FROM source_table".to_string(),
flow_options: HashMap::new(),
+9 -1
View File
@@ -172,8 +172,16 @@ impl FlowScheduleConfig {
}
pub fn default_with_start(start_secs: i64, eval_interval_secs: i64) -> Self {
Self::with_anchor(Self::DEFAULT_ANCHOR_SECS, start_secs, eval_interval_secs)
}
/// Builds a schedule anchored at `anchor_secs` (the `EVAL OFFSET` phase,
/// in Unix epoch seconds) with `start_secs` as the first scheduled time.
/// Callers must ensure `0 <= anchor_secs < eval_interval_secs` and that
/// `start_secs` lies on an `anchor_secs + k * eval_interval_secs` boundary.
pub fn with_anchor(anchor_secs: i64, start_secs: i64, eval_interval_secs: i64) -> Self {
Self {
anchor_secs: Self::DEFAULT_ANCHOR_SECS,
anchor_secs,
start_secs,
missed_tick_policy: FlowMissedTickPolicy::BoundedCatchUp,
catchup_max_runs: Self::DEFAULT_CATCHUP_MAX_RUNS,
+35 -2
View File
@@ -1296,6 +1296,13 @@ pub struct CreateFlowTask {
/// Duration in seconds. Data older than this duration will not be used.
pub expire_after: Option<i64>,
pub eval_interval_secs: Option<i64>,
/// Phase offset of the evaluation schedule within `eval_interval_secs`,
/// in seconds. Must be in `[0, eval_interval_secs)`. `None` means a zero
/// offset (epoch-anchored schedule).
/// Transported through the transient option map (no proto field), see
/// `INTERNAL_EVAL_OFFSET_KEY`.
#[serde(default)]
pub eval_offset_secs: Option<i64>,
pub comment: String,
pub sql: String,
pub flow_options: HashMap<String, String>,
@@ -1321,11 +1328,27 @@ impl TryFrom<PbCreateFlowTask> for CreateFlowTask {
eval_interval,
comment,
sql,
flow_options,
mut flow_options,
} = pb.create_flow.context(error::InvalidProtoMsgSnafu {
err_msg: "expected create_flow",
})?;
// Parse and strip the trusted transient offset key inserted by the
// operator after user option validation. It must never persist in
// user-visible options.
let eval_offset_secs =
match flow_options.remove(crate::ddl::create_flow::INTERNAL_EVAL_OFFSET_KEY) {
Some(value) => Some(value.parse::<i64>().map_err(|_| {
error::UnexpectedSnafu {
err_msg: format!(
"Invalid internal eval offset payload '{value}': expected whole seconds"
),
}
.build()
})?),
None => None,
};
Ok(CreateFlowTask {
catalog_name,
flow_name,
@@ -1339,6 +1362,7 @@ impl TryFrom<PbCreateFlowTask> for CreateFlowTask {
create_if_not_exists,
expire_after: expire_after.map(|e| e.value),
eval_interval_secs: eval_interval.map(|e| e.seconds),
eval_offset_secs,
comment,
sql,
flow_options,
@@ -1358,12 +1382,21 @@ impl From<CreateFlowTask> for PbCreateFlowTask {
create_if_not_exists,
expire_after,
eval_interval_secs: eval_interval,
eval_offset_secs,
comment,
sql,
flow_options,
mut flow_options,
..
}: CreateFlowTask,
) -> Self {
// Re-insert the transient offset key so the proto round-trip (e.g. DDL
// task submission between frontend and metasrv) preserves the offset.
if let Some(offset_secs) = eval_offset_secs {
flow_options.insert(
crate::ddl::create_flow::INTERNAL_EVAL_OFFSET_KEY.to_string(),
offset_secs.to_string(),
);
}
PbCreateFlowTask {
create_flow: Some(CreateFlowExpr {
catalog_name,