From 6fa375021f6cf5b57981a1904058be3bd122aa78 Mon Sep 17 00:00:00 2001 From: discord9 Date: Fri, 18 Sep 2026 04:35:29 +0000 Subject: [PATCH] feat(flow): expose extension-owned batching execution hooks (#9171) * fix: preserve structured query errors through distributed execution Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> * feat: use exact sequence ranges for capable incremental flow sources Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> * test: update SQL expectations for preserved query error codes Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> * style: format exact sequence recovery tests Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> * feat: merge aggregate states in incremental flows Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> * test: exercise dispatched exact delta failure recovery Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> * test: record flushed exact sequence flow results Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> * test: pass query engine to state merge execution tests Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> * feat(flow): expose extension-owned batching execution hooks Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> * fix(flow): correct extension matcher borrowing and regression assertions Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> * test: prove executed empty exact deltas keep incremental mode Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> * test: pass query engine to empty exact delta regression Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> * test: cover aggregate state merge through the full flow path Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> * test: pass query engine to flow batching task regression calls Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> --------- Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> --- merge-main.log | 10 + src/common/base/src/readable_size.rs | 14 +- src/common/base/src/secrets.rs | 14 + src/common/meta/src/ddl_manager.rs | 281 +++++++++- src/flow/src/batching_mode.rs | 9 +- .../src/batching_mode/batching_execution.rs | 54 ++ src/flow/src/batching_mode/engine.rs | 383 +++++++++++++- src/flow/src/batching_mode/frontend_client.rs | 2 +- src/flow/src/batching_mode/state.rs | 14 + src/flow/src/batching_mode/task.rs | 342 ++++++++---- src/flow/src/batching_mode/task/inc.rs | 3 + src/flow/src/batching_mode/task/test.rs | 431 +++++++++++++++- src/flow/src/batching_mode/utils.rs | 464 +++++++++++++---- src/flow/src/batching_mode/utils/test.rs | 485 +++++++++++++++++- src/flow/src/lib.rs | 12 +- src/flow/src/server.rs | 5 +- src/operator/src/statement/ddl.rs | 66 ++- src/servers/src/http/test_helpers.rs | 14 + src/servers/src/repeated_field.rs | 26 +- .../flow/flow_incremental_state_merge.result | 348 +++++++++++++ .../flow/flow_incremental_state_merge.sql | 232 +++++++++ 21 files changed, 2944 insertions(+), 265 deletions(-) create mode 100644 merge-main.log create mode 100644 src/flow/src/batching_mode/batching_execution.rs create mode 100644 tests/cases/standalone/common/flow/flow_incremental_state_merge.result create mode 100644 tests/cases/standalone/common/flow/flow_incremental_state_merge.sql diff --git a/merge-main.log b/merge-main.log new file mode 100644 index 00000000000..ec2af961861 --- /dev/null +++ b/merge-main.log @@ -0,0 +1,10 @@ +Auto-merging src/flow/src/batching_mode/frontend_client.rs +Auto-merging src/flow/src/lib.rs +CONFLICT (content): Merge conflict in src/flow/src/lib.rs +Auto-merging src/flow/src/server.rs +CONFLICT (content): Merge conflict in src/flow/src/server.rs +Auto-merging src/operator/src/statement/ddl.rs +Recorded preimage for 'src/flow/src/lib.rs' +Recorded preimage for 'src/flow/src/server.rs' +Automatic merge failed; fix conflicts and then commit the result. +exit=1 diff --git a/src/common/base/src/readable_size.rs b/src/common/base/src/readable_size.rs index da5eae9bd97..9c095b2fac0 100644 --- a/src/common/base/src/readable_size.rs +++ b/src/common/base/src/readable_size.rs @@ -1,4 +1,16 @@ -// Copyright (c) 2017-present, PingCAP, Inc. Licensed under Apache-2.0. +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. // This file is copied from https://github.com/tikv/raft-engine/blob/0.3.0/src/util.rs diff --git a/src/common/base/src/secrets.rs b/src/common/base/src/secrets.rs index 55f569fe23c..297f5d0836e 100644 --- a/src/common/base/src/secrets.rs +++ b/src/common/base/src/secrets.rs @@ -1,3 +1,17 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + // This file is copied from https://github.com/iqlusioninc/crates/blob/f98d4ccf/secrecy/src/lib.rs //! [`SecretBox`] wrapper type for more carefully handling secret values diff --git a/src/common/meta/src/ddl_manager.rs b/src/common/meta/src/ddl_manager.rs index 51b0a90f304..eddb4148767 100644 --- a/src/common/meta/src/ddl_manager.rs +++ b/src/common/meta/src/ddl_manager.rs @@ -38,7 +38,10 @@ use crate::ddl::alter_logical_tables::AlterLogicalTablesProcedure; use crate::ddl::alter_table::{AlterTableProcedure, RegionRouteChanged, only_sets_skip_wal}; use crate::ddl::comment_on::CommentOnProcedure; use crate::ddl::create_database::{CreateDatabaseMetadataCommitterRef, CreateDatabaseProcedure}; -use crate::ddl::create_flow::CreateFlowProcedure; +use crate::ddl::create_flow::{ + CreateFlowProcedure, FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY, + FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_SEQUENCE_RANGE, +}; use crate::ddl::create_logical_tables::CreateLogicalTablesProcedure; use crate::ddl::create_table::CreateTableProcedure; use crate::ddl::create_view::CreateViewProcedure; @@ -110,6 +113,10 @@ pub struct DdlManager { repartition_procedure_factory: RepartitionProcedureFactoryRef, #[cfg(feature = "enterprise")] trigger_ddl_manager: Option, + #[cfg(feature = "enterprise")] + create_flow_handler: Option, + #[cfg(feature = "enterprise")] + drop_flow_handler: Option, } /// This trait is responsible for handling DDL tasks about triggers. e.g., @@ -141,6 +148,39 @@ pub trait TriggerDdlManager: Send + Sync { #[cfg(feature = "enterprise")] pub type TriggerDdlManagerRef = Arc; +/// This trait is responsible for handling custom/extension CREATE FLOW tasks. +#[cfg(feature = "enterprise")] +#[async_trait::async_trait] +pub trait CreateFlowHandler: Send + Sync { + async fn create_flow( + &self, + create_flow_task: CreateFlowTask, + procedure_manager: ProcedureManagerRef, + ddl_context: DdlContext, + query_context: QueryContext, + procedure_context: ProcedureContext, + ) -> Result; +} + +#[cfg(feature = "enterprise")] +pub type CreateFlowHandlerRef = Arc; + +/// Hook for classifying and handling DROP FLOW requests. +#[cfg(feature = "enterprise")] +#[async_trait::async_trait] +pub trait DropFlowHandler: Send + Sync { + async fn drop_flow( + &self, + drop_flow_task: DropFlowTask, + procedure_manager: ProcedureManagerRef, + ddl_context: DdlContext, + procedure_context: ProcedureContext, + ) -> Result; +} + +#[cfg(feature = "enterprise")] +pub type DropFlowHandlerRef = Arc; + macro_rules! procedure_loader_entry { ($procedure:ident) => { ( @@ -235,15 +275,31 @@ impl DdlManager { repartition_procedure_factory, #[cfg(feature = "enterprise")] trigger_ddl_manager: None, + #[cfg(feature = "enterprise")] + create_flow_handler: None, + #[cfg(feature = "enterprise")] + drop_flow_handler: None, } } + #[cfg(feature = "enterprise")] + pub fn with_drop_flow_handler(mut self, drop_flow_handler: DropFlowHandlerRef) -> Self { + self.drop_flow_handler = Some(drop_flow_handler); + self + } + #[cfg(feature = "enterprise")] pub fn with_trigger_ddl_manager(mut self, trigger_ddl_manager: TriggerDdlManagerRef) -> Self { self.trigger_ddl_manager = Some(trigger_ddl_manager); self } + #[cfg(feature = "enterprise")] + pub fn with_create_flow_handler(mut self, create_flow_handler: CreateFlowHandlerRef) -> Self { + self.create_flow_handler = Some(create_flow_handler); + self + } + pub fn with_create_database_metadata_committer( mut self, committer: CreateDatabaseMetadataCommitterRef, @@ -965,6 +1021,17 @@ impl DdlManager { .await } DropFlow(drop_flow_task) => { + #[cfg(feature = "enterprise")] + if let Some(handler) = self.drop_flow_handler.as_ref() { + return handler + .drop_flow( + drop_flow_task, + self.procedure_manager.clone(), + self.ddl_context.clone(), + procedure_context, + ) + .await; + } handle_drop_flow_task(self, drop_flow_task, procedure_context).await } CreateView(create_view_task) => { @@ -1365,6 +1432,32 @@ async fn handle_create_flow_task( query_context: QueryContext, procedure_context: ProcedureContext, ) -> Result { + if create_flow_task + .flow_options + .get(FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY) + .is_some_and(|value| value == FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_SEQUENCE_RANGE) + { + return error::UnexpectedSnafu { + err_msg: format!( + "reserved flow option value for {FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY} is internal" + ), + } + .fail(); + } + + #[cfg(feature = "enterprise")] + if let Some(handler) = ddl_manager.create_flow_handler.as_ref() { + return handler + .create_flow( + create_flow_task, + ddl_manager.procedure_manager.clone(), + ddl_manager.ddl_context.clone(), + query_context, + procedure_context, + ) + .await; + } + let (id, output) = ddl_manager .submit_create_flow_task(create_flow_task.clone(), query_context, procedure_context) .await?; @@ -1538,6 +1631,11 @@ mod tests { use crate::ddl::create_database::{ AtomicCreateOutcome, CreateDatabaseMetadataCommitter, CreateDatabaseProcedure, }; + #[cfg(feature = "enterprise")] + use crate::ddl::create_flow::{ + CreateFlowProcedure, FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY, + FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_SEQUENCE_RANGE, + }; use crate::ddl::create_table::CreateTableProcedure; use crate::ddl::drop_table::DropTableProcedure; use crate::ddl::flow_meta::FlowMetadataAllocator; @@ -1555,11 +1653,13 @@ mod tests { use crate::region_registry::LeaderRegionRegistry; #[cfg(feature = "enterprise")] use crate::rpc::ddl::trigger::{CreateTriggerTask, DropTriggerTask}; + #[cfg(feature = "enterprise")] + use crate::rpc::ddl::{ + CreateFlowTask, DdlTask, QueryContext, SubmitDdlTaskRequest, SubmitDdlTaskResponse, + }; use crate::rpc::ddl::{CreatorGrantIntent, UndropTableTask}; #[cfg(not(feature = "enterprise"))] use crate::rpc::ddl::{DdlTask, PurgeDroppedTableTask, QueryContext, SubmitDdlTaskRequest}; - #[cfg(feature = "enterprise")] - use crate::rpc::ddl::{DdlTask, QueryContext, SubmitDdlTaskRequest}; use crate::sequence::SequenceBuilder; use crate::state_store::KvStateStore; use crate::test_util::{MockDatanodeManager, new_ddl_context}; @@ -1626,6 +1726,60 @@ mod tests { } } + #[cfg(feature = "enterprise")] + #[derive(Default)] + struct RecordingCreateFlowHandler { + tasks: Mutex>, + contexts: Mutex>, + response: Mutex>, + fail: bool, + } + + #[cfg(feature = "enterprise")] + #[async_trait::async_trait] + impl super::CreateFlowHandler for RecordingCreateFlowHandler { + async fn create_flow( + &self, + create_flow_task: CreateFlowTask, + _procedure_manager: ProcedureManagerRef, + _ddl_context: DdlContext, + query_context: QueryContext, + procedure_context: ProcedureContext, + ) -> crate::error::Result { + self.tasks.lock().unwrap().push(create_flow_task); + self.contexts + .lock() + .unwrap() + .push((query_context, procedure_context)); + if self.fail { + return crate::error::UnsupportedSnafu { + operation: "test create flow handler", + } + .fail(); + } + Ok(self.response.lock().unwrap().take().unwrap_or_default()) + } + } + + #[cfg(feature = "enterprise")] + fn test_create_flow_task() -> CreateFlowTask { + CreateFlowTask { + catalog_name: "greptime".to_string(), + flow_name: "test_flow".to_string(), + source_table_names: vec![], + sink_table_name: TableName::new("greptime", "public", "sink"), + or_replace: false, + create_if_not_exists: false, + expire_after: None, + eval_interval_secs: None, + eval_offset_secs: None, + comment: String::new(), + sql: "select 1".to_string(), + flow_options: Default::default(), + eval_schedule: None, + } + } + #[cfg(feature = "enterprise")] #[derive(Default)] struct RecordingTriggerDdlManager { @@ -1805,6 +1959,127 @@ mod tests { ) } + #[cfg(feature = "enterprise")] + #[tokio::test] + async fn test_reserved_sequence_range_is_rejected_before_custom_handler() { + let handler = Arc::new(RecordingCreateFlowHandler::default()); + let ddl_manager = + build_soft_drop_test_ddl_manager().with_create_flow_handler(handler.clone()); + let mut task = test_create_flow_task(); + task.flow_options.insert( + FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY.to_string(), + FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_SEQUENCE_RANGE.to_string(), + ); + + let result = ddl_manager + .submit_ddl_task( + ExecutorContext { + query_context: Some(QueryContext::default()), + ..Default::default() + }, + SubmitDdlTaskRequest::new(DdlTask::new_create_flow(task)), + ) + .await; + + assert!(result.is_err()); + assert!(handler.tasks.lock().unwrap().is_empty()); + } + + #[cfg(feature = "enterprise")] + #[tokio::test] + async fn test_create_flow_handler_dispatches_without_procedure() { + let response = SubmitDdlTaskResponse { + key: b"custom-handler".to_vec(), + ..Default::default() + }; + let handler = Arc::new(RecordingCreateFlowHandler { + response: Mutex::new(Some(response)), + ..Default::default() + }); + let ddl_manager = + build_soft_drop_test_ddl_manager().with_create_flow_handler(handler.clone()); + let procedure_context = ProcedureContext { + actor: Some("test-user".to_string()), + event_context: Some( + PersistentEventContext::new(TriggerReason::Manual).with_protocol("mysql"), + ), + }; + let query_context = QueryContext { + channel: Channel::Mysql as u8, + ..Default::default() + }; + let actual = ddl_manager + .submit_ddl_task( + ExecutorContext { + query_context: Some(query_context.clone()), + actor: procedure_context.actor.clone(), + event_input: Some(ProcedureEventInput::new(TriggerReason::Manual)), + ..Default::default() + }, + SubmitDdlTaskRequest::new(DdlTask::new_create_flow(test_create_flow_task())), + ) + .await + .unwrap(); + assert_eq!(actual.key, b"custom-handler"); + assert_eq!(handler.tasks.lock().unwrap().len(), 1); + assert_eq!( + handler.contexts.lock().unwrap().as_slice(), + &[(query_context, procedure_context)] + ); + } + + #[cfg(feature = "enterprise")] + #[tokio::test] + async fn test_create_flow_handler_error_does_not_fallback() { + let handler = Arc::new(RecordingCreateFlowHandler { + fail: true, + ..Default::default() + }); + let ddl_manager = + build_soft_drop_test_ddl_manager().with_create_flow_handler(handler.clone()); + let err = ddl_manager + .submit_ddl_task( + ExecutorContext { + query_context: Some(QueryContext::default()), + ..Default::default() + }, + SubmitDdlTaskRequest::new(DdlTask::new_create_flow(test_create_flow_task())), + ) + .await + .unwrap_err(); + assert!(err.to_string().contains("test create flow handler")); + assert_eq!(handler.tasks.lock().unwrap().len(), 1); + } + + #[cfg(feature = "enterprise")] + #[tokio::test] + async fn test_create_flow_without_handler_uses_ordinary_procedure() { + let ddl_manager = build_soft_drop_test_ddl_manager(); + ddl_manager.procedure_manager.start().await.unwrap(); + let err = ddl_manager + .submit_ddl_task( + ExecutorContext { + query_context: Some(QueryContext::default()), + ..Default::default() + }, + SubmitDdlTaskRequest::new(DdlTask::new_create_flow(test_create_flow_task())), + ) + .await + .unwrap_err(); + assert!(!err.to_string().contains("test create flow handler")); + assert_eq!( + ddl_manager + .procedure_manager + .list_procedures() + .await + .unwrap() + .iter() + .filter(|procedure| procedure.type_name == CreateFlowProcedure::TYPE_NAME) + .count(), + 1 + ); + } + #[cfg(feature = "enterprise")] #[tokio::test] async fn test_trigger_ddl_forwards_procedure_context() { diff --git a/src/flow/src/batching_mode.rs b/src/flow/src/batching_mode.rs index 3981446721b..920f847083e 100644 --- a/src/flow/src/batching_mode.rs +++ b/src/flow/src/batching_mode.rs @@ -20,14 +20,15 @@ use common_grpc::channel_manager::ClientTlsOption; use serde::{Deserialize, Serialize}; use session::ReadPreference; -mod checkpoint; +pub(crate) mod batching_execution; +pub(crate) mod checkpoint; pub(crate) mod engine; mod eval_schedule; pub(crate) mod frontend_client; -mod state; +pub(crate) mod state; mod table_creator; -mod task; -mod time_window; +pub(crate) mod task; +pub(crate) mod time_window; pub(crate) mod utils; #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] diff --git a/src/flow/src/batching_mode/batching_execution.rs b/src/flow/src/batching_mode/batching_execution.rs new file mode 100644 index 00000000000..0a699a227be --- /dev/null +++ b/src/flow/src/batching_mode/batching_execution.rs @@ -0,0 +1,54 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Optional execution collaborator for batching tasks. + +use std::sync::Arc; + +use query::QueryEngineRef; +use table::TableRef; + +use crate::Result; +use crate::batching_mode::frontend_client::FrontendClient; +use crate::batching_mode::task::{BatchingExecutionGuard, BatchingTask, ExecuteOnceOutcome}; + +#[async_trait::async_trait] +pub trait BatchingExecution: Send + Sync + 'static { + /// Execute one round while retaining the guard through all task-state updates. + /// An implementation that continues after caller cancellation must retain the + /// guard with that work and make it stoppable through [`Self::stop`]. + async fn execute_once( + self: Arc, + guard: BatchingExecutionGuard, + task: &BatchingTask, + engine: &QueryEngineRef, + frontend: &Arc, + max_window_cnt: Option, + ) -> ExecuteOnceOutcome; + + /// Retire this execution instance, rejecting new work and requesting that any + /// retained local work stop. This is not an acknowledgement of remote quiescence. + fn stop(&self) {} +} + +#[async_trait::async_trait] +pub trait BatchingExecutionFactory: Send + Sync + 'static { + async fn create( + &self, + task: &BatchingTask, + sink: TableRef, + engine: &QueryEngineRef, + frontend: &Arc, + ) -> Result>>; +} diff --git a/src/flow/src/batching_mode/engine.rs b/src/flow/src/batching_mode/engine.rs index 3532b38d366..7206e83e97c 100644 --- a/src/flow/src/batching_mode/engine.rs +++ b/src/flow/src/batching_mode/engine.rs @@ -75,6 +75,7 @@ pub struct BatchingEngine { /// Batching mode options for control how batching mode query works /// pub(crate) batch_opts: Arc, + execution_factory: Option>, } #[derive(Default)] @@ -130,6 +131,26 @@ impl BatchingEngine { table_meta: TableMetadataManagerRef, catalog_manager: CatalogManagerRef, batch_opts: BatchingModeOptions, + ) -> Self { + Self::new_with_execution( + frontend_client, + query_engine, + flow_metadata_manager, + table_meta, + catalog_manager, + batch_opts, + None, + ) + } + + pub fn new_with_execution( + frontend_client: Arc, + query_engine: QueryEngineRef, + flow_metadata_manager: FlowMetadataManagerRef, + table_meta: TableMetadataManagerRef, + catalog_manager: CatalogManagerRef, + batch_opts: BatchingModeOptions, + execution_factory: Option>, ) -> Self { Self { runtime: Default::default(), @@ -139,6 +160,7 @@ impl BatchingEngine { catalog_manager, query_engine, batch_opts: Arc::new(batch_opts), + execution_factory, } } @@ -726,15 +748,22 @@ impl BatchingEngine { ); } - let task_inner = task.clone(); let engine = self.query_engine.clone(); let frontend = self.frontend_client.clone(); - // Create sink table if needed, then validate an existing/created sink schema before - // spawning the background task. This catches user-created sink schema mismatches at - // CREATE FLOW time instead of surfacing them later in the execution loop. - task.check_or_create_sink_table(&engine, &frontend).await?; - task.validate_sink_table_schema(&engine).await?; + // Create the sink before asking an optional extension to own execution. + // The default path retains the OSS schema validation. + let table = task.check_or_create_sink_table(&engine, &frontend).await?; + let execution = if let Some(factory) = &self.execution_factory { + factory.create(&task, table, &engine, &frontend).await? + } else { + None + }; + if execution.is_none() { + task.validate_sink_table_schema(&engine).await?; + } + let task = task.with_execution(execution); + let task_inner = task.clone(); let (start_tx, start_rx) = oneshot::channel(); @@ -1018,6 +1047,8 @@ fn abort_flow_task(flow_id: FlowId, task: Option, action: &str) -> return false; }; + task.stop_execution(); + if let Some(handle) = task.state.write().unwrap().task_handle.take() { handle.abort(); debug!("Aborted {action} flow task {flow_id}"); @@ -1067,17 +1098,23 @@ impl FlowEngine for BatchingEngine { #[cfg(test)] mod tests { use api::v1::flow::{DirtyWindowRequest, TimeRange}; - use catalog::memory::new_memory_catalog_manager; + use catalog::RegisterTableRequest; + use catalog::memory::{MemoryCatalogManager, new_memory_catalog_manager}; use common_meta::key::TableMetadataManager; use common_meta::key::flow::FlowMetadataManager; use common_meta::key::table_route::TableRouteValue; use common_meta::key::test_utils::new_test_table_info_with_name; use common_meta::kv_backend::memory::MemoryKvBackend; + use common_recordbatch::RecordBatch; use common_time::timestamp::TimeUnit; + use datatypes::schema::{ColumnSchema, Schema}; + use datatypes::vectors::{TimestampMillisecondVector, UInt32Vector, VectorRef}; use query::options::QueryOptions; use session::context::QueryContext; + use tokio::sync::Notify; use super::*; + use crate::ExecuteOnceOutcome; use crate::test_utils::create_test_query_engine; struct DropNotify(Option>); @@ -1110,6 +1147,317 @@ mod tests { ) } + async fn new_test_engine_with_execution( + execution_factory: Option>, + ) -> BatchingEngine { + let kv_backend = Arc::new(MemoryKvBackend::new()); + let table_meta = Arc::new(TableMetadataManager::new(kv_backend.clone())); + table_meta.init().await.unwrap(); + let flow_meta = Arc::new(FlowMetadataManager::new(kv_backend)); + let query_engine = create_test_query_engine(); + let catalog_manager = query_engine.engine_state().catalog_manager().clone(); + let (frontend_client, _handler) = + FrontendClient::from_empty_grpc_handler(QueryOptions::default()); + + let engine = BatchingEngine::new_with_execution( + Arc::new(frontend_client), + query_engine, + flow_meta, + table_meta, + catalog_manager, + BatchingModeOptions::default(), + execution_factory, + ); + engine + .table_meta + .create_table_metadata( + new_test_table_info_with_name(1, "numbers_with_ts"), + TableRouteValue::physical(vec![]), + HashMap::new(), + ) + .await + .unwrap(); + engine + } + + fn register_sink_with_schema(engine: &BatchingEngine, name: &str) { + let schema = Arc::new(Schema::new(vec![ + ColumnSchema::new("number", ConcreteDataType::uint32_datatype(), false), + ColumnSchema::new( + "ts", + ConcreteDataType::timestamp_millisecond_datatype(), + false, + ) + .with_time_index(true), + ])); + let recordbatch = RecordBatch::new( + schema, + vec![ + Arc::new(UInt32Vector::from_slice([1_u32])) as VectorRef, + Arc::new(TimestampMillisecondVector::from_slice([0_i64])) as VectorRef, + ], + ) + .unwrap(); + engine + .catalog_manager + .as_any() + .downcast_ref::() + .unwrap() + .register_table_sync(RegisterTableRequest { + catalog: "greptime".to_string(), + schema: "public".to_string(), + table_name: name.to_string(), + table_id: 9000, + table: table::test_util::MemTable::table(name, recordbatch), + }) + .unwrap(); + } + + fn register_number_only_sink(engine: &BatchingEngine, name: &str) { + let schema = Arc::new(Schema::new(vec![ColumnSchema::new( + "number", + ConcreteDataType::uint32_datatype(), + false, + )])); + let recordbatch = RecordBatch::new( + schema, + vec![Arc::new(UInt32Vector::from_slice([1_u32])) as VectorRef], + ) + .unwrap(); + engine + .catalog_manager + .as_any() + .downcast_ref::() + .unwrap() + .register_table_sync(RegisterTableRequest { + catalog: "greptime".to_string(), + schema: "public".to_string(), + table_name: name.to_string(), + table_id: 9001, + table: table::test_util::MemTable::table(name, recordbatch), + }) + .unwrap(); + } + + fn flow_create_args(flow_id: FlowId, sink: &str) -> CreateFlowArgs { + CreateFlowArgs { + flow_id, + sink_table_name: [ + "greptime".to_string(), + "public".to_string(), + sink.to_string(), + ], + source_table_ids: vec![1], + create_if_not_exists: false, + or_replace: false, + expire_after: None, + eval_interval: Some(10), + comment: None, + sql: "SELECT number, ts FROM numbers_with_ts".to_string(), + flow_options: HashMap::new(), + query_ctx: Some(QueryContext::arc().as_ref().clone()), + eval_schedule: None, + } + } + + struct TestExecution { + manual_calls: std::sync::atomic::AtomicUsize, + stops: std::sync::atomic::AtomicUsize, + } + + #[async_trait::async_trait] + impl crate::BatchingExecution for TestExecution { + async fn execute_once( + self: Arc, + _guard: crate::BatchingExecutionGuard, + task: &BatchingTask, + _engine: &QueryEngineRef, + _frontend: &Arc, + _max_window_cnt: Option, + ) -> ExecuteOnceOutcome { + if task + .state + .read() + .unwrap() + .query_ctx + .extension(query::options::FLOW_SCHEDULED_TIME_MILLIS) + .is_none() + { + self.manual_calls + .fetch_add(1, std::sync::atomic::Ordering::SeqCst); + } + ExecuteOnceOutcome { + new_query: None, + result: Ok(None), + } + } + + fn stop(&self) { + self.stops.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + } + } + + type TestExecutionResult = crate::Result>>; + + struct TestExecutionFactory { + entered: Option>, + release: Option>, + result: std::sync::Mutex>, + } + + #[async_trait::async_trait] + impl crate::BatchingExecutionFactory for TestExecutionFactory { + async fn create( + &self, + _task: &BatchingTask, + _sink: table::TableRef, + _engine: &QueryEngineRef, + _frontend: &Arc, + ) -> crate::Result>> { + if let Some(entered) = &self.entered { + entered.notify_one(); + } + if let Some(release) = &self.release { + release.notified().await; + } + self.result + .lock() + .unwrap() + .take() + .expect("execution factory should only be called once") + } + } + + #[tokio::test] + async fn test_execution_factory_finishes_before_task_publication() { + const GATE_TIMEOUT: Duration = Duration::from_secs(1); + + let execution = Arc::new(TestExecution { + manual_calls: Default::default(), + stops: Default::default(), + }); + let entered = Arc::new(Notify::new()); + let release = Arc::new(Notify::new()); + let factory = Arc::new(TestExecutionFactory { + entered: Some(entered.clone()), + release: Some(release.clone()), + result: std::sync::Mutex::new(Some(Ok(Some(execution.clone())))), + }); + let engine = Arc::new(new_test_engine_with_execution(Some(factory)).await); + register_sink_with_schema(&engine, "factory_sink"); + + let entered_wait = entered.notified(); + let mut args = flow_create_args(6, "factory_sink"); + args.eval_interval = Some(86_400); + let mut create = tokio::spawn({ + let engine = engine.clone(); + async move { engine.create_flow_inner(args).await } + }); + if tokio::time::timeout(GATE_TIMEOUT, entered_wait) + .await + .is_err() + { + release.notify_one(); + if tokio::time::timeout(GATE_TIMEOUT, &mut create) + .await + .is_err() + { + create.abort(); + } + panic!("execution factory should be entered"); + } + let unpublished = { + let runtime = engine.runtime.read().await; + !runtime.tasks.contains_key(&6) && !runtime.shutdown_txs.contains_key(&6) + }; + + release.notify_one(); + assert!(unpublished); + let created = tokio::time::timeout(GATE_TIMEOUT, &mut create) + .await + .expect("flow creation should finish after factory release"); + created.unwrap().unwrap(); + assert!(engine.flow_exist_inner(6).await); + let calls_before_flush = execution + .manual_calls + .load(std::sync::atomic::Ordering::SeqCst); + assert_eq!(engine.flush_flow_inner(6).await.unwrap(), 0); + assert_eq!( + execution + .manual_calls + .load(std::sync::atomic::Ordering::SeqCst), + calls_before_flush + 1 + ); + engine.remove_flow_inner(6).await.unwrap(); + } + + #[tokio::test] + async fn test_execution_factory_result_controls_sink_validation_and_publication() { + let no_factory_engine = new_test_engine_with_execution(None).await; + register_number_only_sink(&no_factory_engine, "no_factory_sink"); + assert!( + no_factory_engine + .create_flow_inner(flow_create_args(7, "no_factory_sink")) + .await + .is_err() + ); + assert!(!no_factory_engine.flow_exist_inner(7).await); + + let declined_engine = + new_test_engine_with_execution(Some(Arc::new(TestExecutionFactory { + entered: None, + release: None, + result: std::sync::Mutex::new(Some(Ok(None))), + }))) + .await; + register_number_only_sink(&declined_engine, "declined_factory_sink"); + assert!( + declined_engine + .create_flow_inner(flow_create_args(8, "declined_factory_sink")) + .await + .is_err() + ); + assert!(!declined_engine.flow_exist_inner(8).await); + + let accepted_engine = + new_test_engine_with_execution(Some(Arc::new(TestExecutionFactory { + entered: None, + release: None, + result: std::sync::Mutex::new(Some(Ok(Some(Arc::new(TestExecution { + manual_calls: Default::default(), + stops: Default::default(), + }))))), + }))) + .await; + register_number_only_sink(&accepted_engine, "accepted_factory_sink"); + accepted_engine + .create_flow_inner(flow_create_args(9, "accepted_factory_sink")) + .await + .unwrap(); + assert!(accepted_engine.flow_exist_inner(9).await); + accepted_engine.remove_flow_inner(9).await.unwrap(); + + let error_engine = new_test_engine_with_execution(Some(Arc::new(TestExecutionFactory { + entered: None, + release: None, + result: std::sync::Mutex::new(Some( + UnexpectedSnafu { + reason: "test execution factory failure".to_string(), + } + .fail(), + )), + }))) + .await; + register_sink_with_schema(&error_engine, "error_factory_sink"); + assert!( + error_engine + .create_flow_inner(flow_create_args(10, "error_factory_sink")) + .await + .is_err() + ); + assert!(!error_engine.flow_exist_inner(10).await); + } + #[tokio::test] async fn test_flow_option_overrides_incremental_read_switch() { let engine = new_test_engine().await; @@ -1583,11 +1931,30 @@ GROUP BY l.number, time_window } #[tokio::test] - async fn test_abort_flow_task_aborts_handle() { + async fn test_abort_flow_task_stops_execution_without_loop_handle() { let (task, _shutdown_tx) = new_test_task(42).await; + let execution = Arc::new(TestExecution { + manual_calls: Default::default(), + stops: Default::default(), + }); + let task = task.with_execution(Some(execution.clone())); + + assert!(!abort_flow_task(42, Some(task), "test")); + assert_eq!(execution.stops.load(std::sync::atomic::Ordering::SeqCst), 1); + } + + #[tokio::test] + async fn test_abort_flow_task_stops_execution_before_aborting_handle() { + let (task, _shutdown_tx) = new_test_task(42).await; + let execution = Arc::new(TestExecution { + manual_calls: Default::default(), + stops: Default::default(), + }); + let task = task.with_execution(Some(execution.clone())); let drop_rx = install_abort_observed_handle(&task).await; assert!(abort_flow_task(42, Some(task), "test")); + assert_eq!(execution.stops.load(std::sync::atomic::Ordering::SeqCst), 1); tokio::time::timeout(Duration::from_secs(1), drop_rx) .await diff --git a/src/flow/src/batching_mode/frontend_client.rs b/src/flow/src/batching_mode/frontend_client.rs index 0207837bdc7..efec4cdea63 100644 --- a/src/flow/src/batching_mode/frontend_client.rs +++ b/src/flow/src/batching_mode/frontend_client.rs @@ -749,7 +749,7 @@ fn wrap_standalone_output_with_terminal_metrics( /// Describe a peer of frontend #[derive(Debug, Default, Clone)] -pub(crate) enum PeerDesc { +pub enum PeerDesc { /// The query failed before a frontend peer was selected. #[default] Unknown, diff --git a/src/flow/src/batching_mode/state.rs b/src/flow/src/batching_mode/state.rs index 59c4f0fe10f..b9710a25a0d 100644 --- a/src/flow/src/batching_mode/state.rs +++ b/src/flow/src/batching_mode/state.rs @@ -131,6 +131,11 @@ impl TaskState { &self.checkpoints } + /// Read the live dirty-window queue without exposing TaskState internals to execution owners. + pub fn dirty_time_windows(&self) -> &DirtyTimeWindows { + &self.dirty_time_windows + } + /// Returns the in-progress fenced repair, if the task is repairing dirty /// windows under a frozen full-snapshot high watermark. pub fn pending_fenced_repair(&self) -> Option<&FencedRepair> { @@ -546,6 +551,15 @@ impl DirtyTimeWindows { } } + /// Detach all dirty windows while retaining this queue's configured limits. + pub(crate) fn detach(&mut self) -> Self { + Self { + windows: std::mem::take(&mut self.windows), + max_filter_num_per_query: self.max_filter_num_per_query, + time_window_merge_threshold: self.time_window_merge_threshold, + } + } + /// Clean all dirty time windows, useful when can't found time window expr pub fn clean(&mut self) { self.windows.clear(); diff --git a/src/flow/src/batching_mode/task.rs b/src/flow/src/batching_mode/task.rs index 95b989851f7..c0ee6d1c1b5 100644 --- a/src/flow/src/batching_mode/task.rs +++ b/src/flow/src/batching_mode/task.rs @@ -18,6 +18,7 @@ use std::time::{Duration, SystemTime, UNIX_EPOCH}; use api::v1::{CreateTableExpr, TableName}; use catalog::CatalogManagerRef; +use client::OutputWithMetrics; use common_error::ext::BoxedError; use common_query::logical_plan::breakup_insert_plan; use common_telemetry::tracing::warn; @@ -27,7 +28,7 @@ use datafusion::datasource::DefaultTableSource; use datafusion::sql::unparser::expr_to_sql; use datafusion_common::tree_node::{Transformed, TreeNode}; use datafusion_common::utils::quote_identifier; -use datafusion_common::{DFSchemaRef, TableReference}; +use datafusion_common::{DFSchemaRef, ScalarValue, TableReference}; use datafusion_expr::{DmlStatement, LogicalPlan, WriteOp, col, lit}; use datatypes::schema::Schema; use query::QueryEngineRef; @@ -38,9 +39,10 @@ use snafu::{OptionExt, ResultExt}; use sql::parsers::utils::is_tql; use store_api::mito_engine_options::MERGE_MODE_KEY; use substrait::{DFLogicalSubstraitConvertor, SubstraitPlan}; +use table::TableRef; use table::table::adapter::DfTableProviderAdapter; use tokio::sync::oneshot::error::TryRecvError; -use tokio::sync::{Mutex, oneshot}; +use tokio::sync::{Mutex, OwnedMutexGuard, oneshot}; use tokio::time::Instant; use crate::batching_mode::BatchingModeOptions; @@ -53,8 +55,8 @@ use crate::batching_mode::state::{ use crate::batching_mode::table_creator::{QueryType, create_table_with_expr}; use crate::batching_mode::time_window::TimeWindowExpr; use crate::batching_mode::utils::{ - AddFilterRewriter, ColumnMatcherRewriter, df_plan_to_sql, gen_plan_with_matching_schema, - get_table_info_df_schema, sql_to_df_plan, + AddFilterRewriter, ColumnMatcherRewriter, df_plan_to_sql, + gen_plan_with_matching_schema_and_values, get_table_info_df_schema, sql_to_df_plan, }; use crate::df_optimizer::apply_df_optimizer; use crate::error::{ @@ -199,6 +201,36 @@ fn format_insert_target_columns(plan: &LogicalPlan) -> String { .join(", ") } +/// Owns a whole serialized execution round. It may be moved to an execution +/// collaborator, so cancellation of the caller cannot release the round early. +pub struct BatchingExecutionGuard { + _lock: OwnedMutexGuard<()>, + restore: Option<(Arc>, QueryContextRef)>, +} + +impl BatchingExecutionGuard { + fn new(lock: OwnedMutexGuard<()>) -> Self { + Self { + _lock: lock, + restore: None, + } + } + + fn restore_query_context(&mut self, state: Arc>, old_ctx: QueryContextRef) { + self.restore = Some((state, old_ctx)); + } +} + +impl Drop for BatchingExecutionGuard { + fn drop(&mut self) { + if let Some((state, old_ctx)) = self.restore.take() + && let Ok(mut state) = state.write() + { + state.query_ctx = old_ctx; + } + } +} + #[derive(Clone)] pub struct BatchingTask { pub config: Arc, @@ -207,6 +239,7 @@ pub struct BatchingTask { /// window restoration for this flow. Without this, a manual flush and the /// background loop can process the same checkpoint range concurrently. execution_lock: Arc>, + execution: Option>, } /// Arguments for creating batching task @@ -281,14 +314,14 @@ pub enum DirtyRestore { Unscoped(DirtyTimeWindows), } -struct ExecuteOnceOutcome { - new_query: Option, +pub struct ExecuteOnceOutcome { + pub new_query: Option, /// Execution result of the generated insert plan. /// /// `Ok(Some((affected_rows, elapsed)))` means a query was executed. /// `Ok(None)` means no query was generated because there was no dirty signal. /// `Err(_)` means plan generation or execution failed. - result: Result, Error>, + pub result: Result, Error>, } impl BatchingTask { @@ -345,9 +378,24 @@ impl BatchingTask { }), state: Arc::new(RwLock::new(state)), execution_lock: Arc::new(Mutex::new(())), + execution: None, }) } + pub(crate) fn with_execution( + mut self, + execution: Option>, + ) -> Self { + self.execution = execution; + self + } + + pub(crate) fn stop_execution(&self) { + if let Some(execution) = &self.execution { + execution.stop(); + } + } + pub fn last_execution_time_millis(&self) -> Option { self.state.read().unwrap().last_execution_time_millis() } @@ -403,12 +451,12 @@ impl BatchingTask { Ok(()) } - /// Create sink table if not exists + /// Create the sink table if needed, then return it. pub async fn check_or_create_sink_table( &self, engine: &QueryEngineRef, frontend_client: &Arc, - ) -> Result, Error> { + ) -> Result { if !self.is_table_exist(&self.config.sink_table_name).await? { let create_table = self.gen_create_table_expr(engine.clone()).await?; info!( @@ -421,16 +469,32 @@ impl BatchingTask { self.config.sink_table_name.join(".") ); } - - Ok(None) + let (table, _) = get_table_info_df_schema( + self.config.catalog_manager.clone(), + self.config.sink_table_name.clone(), + ) + .await?; + Ok(table) } - /// Validates that the sink table schema can accept this flow's output. - /// - /// This is a dry-run of the same schema matching logic used by insert-plan - /// generation, but without adding dirty-window filters or executing the query. It is used - /// during CREATE FLOW to catch existing sink table mismatches early. + /// Returns a forked snapshot of the task query context for extension-owned planning. + pub fn query_context_snapshot(&self) -> QueryContextRef { + let query_ctx = self.state.read().unwrap().query_ctx.clone(); + Arc::new(query_ctx.fork()) + } + + /// Validates that the sink table schema can accept this flow's ordinary output. pub async fn validate_sink_table_schema(&self, engine: &QueryEngineRef) -> Result<(), Error> { + self.validate_sink_table_schema_with_values(engine, &BTreeMap::new()) + .await + } + + /// Validates the flow output plus extension-owned typed sink values without consuming dirty work. + pub async fn validate_sink_table_schema_with_values( + &self, + engine: &QueryEngineRef, + values: &BTreeMap, + ) -> Result<(), Error> { let (table, _) = get_table_info_df_schema( self.config.catalog_manager.clone(), self.config.sink_table_name.clone(), @@ -441,15 +505,15 @@ impl BatchingTask { let merge_mode_last_non_null = is_merge_mode_last_non_null(&table_meta.options.extra_options); let primary_key_indices = table_meta.primary_key_indices.clone(); - let query_ctx = self.state.read().unwrap().query_ctx.clone(); - gen_plan_with_matching_schema( + gen_plan_with_matching_schema_and_values( &self.config.query, - query_ctx, + self.query_context_snapshot(), engine.clone(), table_meta.schema.clone(), &primary_key_indices, merge_mode_last_non_null, + values, ) .await .map(|_| ()) @@ -484,13 +548,30 @@ impl BatchingTask { frontend_client: &Arc, max_window_cnt: Option, ) -> ExecuteOnceOutcome { - let _execution_guard = self.execution_lock.lock().await; - self.execute_once_unlocked(engine, frontend_client, max_window_cnt) + let guard = BatchingExecutionGuard::new(self.execution_lock.clone().lock_owned().await); + self.execute_once_with_guard(guard, engine, frontend_client, max_window_cnt) .await } - /// Executes one flow evaluation. Caller must hold `execution_lock`. - async fn execute_once_unlocked( + async fn execute_once_with_guard( + &self, + guard: BatchingExecutionGuard, + engine: &QueryEngineRef, + frontend_client: &Arc, + max_window_cnt: Option, + ) -> ExecuteOnceOutcome { + if let Some(execution) = &self.execution { + return execution + .clone() + .execute_once(guard, self, engine, frontend_client, max_window_cnt) + .await; + } + self.execute_once_default_unlocked(engine, frontend_client, max_window_cnt) + .await + } + + /// Executes the default flow evaluation. Caller owns the execution guard. + async fn execute_once_default_unlocked( &self, engine: &QueryEngineRef, frontend_client: &Arc, @@ -510,6 +591,7 @@ impl BatchingTask { debug!("Generate new query: {}", new_query.plan); let res = self .execute_logical_plan_unlocked( + engine, frontend_client, &new_query.plan, &new_query.dirty_restore, @@ -532,11 +614,24 @@ impl BatchingTask { } } - /// Generates the insert plan. Caller must reach this through the serialized path. + /// Generates the ordinary insert plan. Caller must reach this through the serialized path. async fn gen_insert_plan_unlocked( &self, engine: &QueryEngineRef, max_window_cnt: Option, + ) -> Result, Error> { + self.gen_insert_plan_with_values_unlocked(engine, max_window_cnt, &BTreeMap::new(), false) + .await + } + + /// Generates the insert plan with extension-owned typed sink values. Caller must own the + /// serialized execution guard. + pub async fn gen_insert_plan_with_values_unlocked( + &self, + engine: &QueryEngineRef, + max_window_cnt: Option, + values: &BTreeMap, + force_full_snapshot: bool, ) -> Result, Error> { let (table, df_schema) = get_table_info_df_schema( self.config.catalog_manager.clone(), @@ -550,12 +645,14 @@ impl BatchingTask { let primary_key_indices = table_meta.primary_key_indices.clone(); let new_query = self - .gen_query_with_time_window( + .gen_query_with_time_window_with_values( engine.clone(), &table.table_info().meta.schema, &primary_key_indices, merge_mode_last_non_null, max_window_cnt, + values, + force_full_snapshot, ) .await?; @@ -637,14 +734,55 @@ impl BatchingTask { Ok(()) } - /// Executes the insert plan. Caller must reach this through the serialized path. + /// Applies checkpoint state updates around the raw insert-plan execution. async fn execute_logical_plan_unlocked( &self, + engine: &QueryEngineRef, frontend_client: &Arc, plan: &LogicalPlan, dirty_restore: &DirtyRestore, coverage: &QueryCoverage, ) -> Result, Error> { + let flow_id = self.config.flow_id; + let Some((res, elapsed)) = self + .execute_plan_unlocked(engine, frontend_client, plan, dirty_restore, coverage) + .await? + else { + return Ok(None); + }; + + if let Err(err) = &res { + let decision = { + let mut state = self.state.write().unwrap(); + let reason = Self::query_failure_reason(err, coverage); + Self::apply_query_failure_to_state(&mut state, elapsed, coverage, reason) + }; + if let Some(decision) = decision { + Self::record_checkpoint_decision(flow_id, decision); + } + } + + let res = res?; + let (affected_rows, _) = res.output.extract_rows_and_cost(); + let decision = { + let mut state = self.state.write().unwrap(); + Self::apply_query_result_to_state(&mut state, &res, elapsed, coverage) + }; + Self::record_checkpoint_decision(flow_id, decision); + Ok(Some((affected_rows, elapsed))) + } + + /// Executes a fully constructed insert plan without checkpoint-state mutation. + /// Callers must own the serialized execution guard. Its unsafe-incremental safety fallback + /// still restores the supplied dirty work before returning `Ok(None)`. + pub async fn execute_plan_unlocked( + &self, + engine: &QueryEngineRef, + frontend_client: &Arc, + plan: &LogicalPlan, + dirty_restore: &DirtyRestore, + coverage: &QueryCoverage, + ) -> Result, Duration)>, Error> { let instant = Instant::now(); let flow_id = self.config.flow_id; @@ -676,7 +814,7 @@ impl BatchingTask { // For incremental-mode SQL queries, attempt to rewrite the delta aggregate // plan into a safe delta-LEFT-JOIN-sink form before deciding on extensions. let incremental_plan = if coverage.is_incremental_delta() { - self.prepare_plan_for_incremental(&plan).await? + self.prepare_plan_for_incremental(engine, &plan).await? } else { None }; @@ -805,18 +943,13 @@ impl BatchingTask { .map(ToString::to_string) .unwrap_or_else(|| PeerDesc::default().to_string()); if let Err(err) = &res { + // This raw helper deliberately leaves checkpoint transitions to its caller. The + // default wrapper performs the single OSS transition; extension owners can persist + // their durable marker before updating in-memory state. warn!( "Failed to execute Flow {flow_id} on frontend {peer_label}, result: {err:?}, elapsed: {:?} with query: {}", elapsed, &plan ); - let decision = { - let mut state = self.state.write().unwrap(); - let reason = Self::query_failure_reason(err, coverage); - Self::apply_query_failure_to_state(&mut state, elapsed, coverage, reason) - }; - if let Some(decision) = decision { - Self::record_checkpoint_decision(flow_id, decision); - } } // record slow query @@ -831,29 +964,27 @@ impl BatchingTask { .observe(elapsed.as_secs_f64()); } - let res = res?; - let (affected_rows, _) = res.output.extract_rows_and_cost(); - debug!( - "Flow {flow_id} executed, affected_rows: {affected_rows:?}, elapsed: {:?}, watermark: {:?}", - elapsed, - res.region_watermark_map() - ); - METRIC_FLOW_ROWS - .with_label_values(&[format!("{}-out-batching", flow_id).as_str()]) - .inc_by(affected_rows as _); - let decision = { - let mut state = self.state.write().unwrap(); - Self::apply_query_result_to_state(&mut state, &res, elapsed, coverage) - }; - Self::record_checkpoint_decision(flow_id, decision); - - Ok(Some((affected_rows, elapsed))) + match res { + Ok(res) => { + let (affected_rows, _) = res.output.extract_rows_and_cost(); + debug!( + "Flow {flow_id} executed, affected_rows: {affected_rows:?}, elapsed: {:?}, watermark: {:?}", + elapsed, + res.region_watermark_map() + ); + METRIC_FLOW_ROWS + .with_label_values(&[format!("{}-out-batching", flow_id).as_str()]) + .inc_by(affected_rows as _); + Ok(Some((Ok(res), elapsed))) + } + Err(err) => Ok(Some((Err(err), elapsed))), + } } /// Restore dirty windows consumed by a failed query so they are retried on /// the next execution. /// - fn restore_dirty_windows(&self, dirty_restore: &DirtyRestore) { + pub fn restore_dirty_windows(&self, dirty_restore: &DirtyRestore) { match dirty_restore { DirtyRestore::Scoped(filter) => self.restore_scoped_dirty_windows(filter), DirtyRestore::Unscoped(dirty_windows) => self @@ -931,19 +1062,21 @@ impl BatchingTask { sink_table_schema: Arc, primary_key_indices: &[usize], allow_partial: bool, + values: &BTreeMap, dirty_windows_to_restore: DirtyTimeWindows, retention_filter: Option<(&str, Timestamp, &'static str)>, coverage: QueryCoverage, ) -> Result { let mut plan = self.restore_unscoped_dirty_windows_on_err( &dirty_windows_to_restore, - gen_plan_with_matching_schema( + gen_plan_with_matching_schema_and_values( &self.config.query, query_ctx, engine, sink_table_schema, primary_key_indices, allow_partial, + values, ) .await, )?; @@ -986,6 +1119,7 @@ impl BatchingTask { sink_table_schema: Arc, primary_key_indices: &[usize], allow_partial: bool, + values: &BTreeMap, retention_filter: Option<(&str, Timestamp, &'static str)>, coverage: QueryCoverage, ) -> Result, Error> { @@ -1001,6 +1135,7 @@ impl BatchingTask { sink_table_schema, primary_key_indices, allow_partial, + values, dirty_windows_to_restore, retention_filter, coverage, @@ -1225,7 +1360,7 @@ impl BatchingTask { .inc(); // Dirty-window restoration is handled by the // existing `handle_executed_query_failure` inside - // `execute_once_unlocked`. + // default execution. } } } @@ -1333,23 +1468,7 @@ impl BatchingTask { frontend_client: &Arc, scheduled_time_secs: i64, ) -> ExecuteOnceOutcome { - let _execution_guard = self.execution_lock.lock().await; - - struct QueryContextRestoreGuard { - state: Arc>, - old_ctx: Option, - } - - impl Drop for QueryContextRestoreGuard { - fn drop(&mut self) { - let Some(old_ctx) = self.old_ctx.take() else { - return; - }; - if let Ok(mut state) = self.state.write() { - state.query_ctx = old_ctx; - } - } - } + let mut guard = BatchingExecutionGuard::new(self.execution_lock.clone().lock_owned().await); // Convert to milliseconds before touching the task state so an // unrepresentable scheduled time fails as an explicit error without @@ -1377,21 +1496,12 @@ impl BatchingTask { state.query_ctx = Arc::new(new_ctx); old }; - let restore_guard = QueryContextRestoreGuard { - state: self.state.clone(), - old_ctx: Some(old_ctx), - }; + guard.restore_query_context(self.state.clone(), old_ctx); - let outcome = self - .execute_once_unlocked(engine, frontend_client, None) - .await; - - // Restore while still holding `execution_lock` so no future manual - // flush can observe the temporary scheduled time. The guard also - // restores during unwind/cancellation. - drop(restore_guard); - - outcome + // A collaborator may retain the guard in an owned child task. Its Drop + // restores the scheduled context before releasing serialization. + self.execute_once_with_guard(guard, engine, frontend_client, None) + .await } /// Generate the create table SQL @@ -1419,8 +1529,7 @@ impl BatchingTask { && matches!(self.config.query_type, QueryType::Sql) } - /// Generate the next plan and classify its coverage so checkpoint handling - /// knows whether it is full-query, scoped repair, fenced repair, or delta. + /// Generate the next ordinary plan and classify its coverage. async fn gen_query_with_time_window( &self, engine: QueryEngineRef, @@ -1428,6 +1537,30 @@ impl BatchingTask { primary_key_indices: &[usize], allow_partial: bool, max_window_cnt: Option, + ) -> Result, Error> { + self.gen_query_with_time_window_with_values( + engine, + sink_table_schema, + primary_key_indices, + allow_partial, + max_window_cnt, + &BTreeMap::new(), + false, + ) + .await + } + + /// Generate the next plan with extension-owned sink values and an optional forced full scan. + #[allow(clippy::too_many_arguments)] + async fn gen_query_with_time_window_with_values( + &self, + engine: QueryEngineRef, + sink_table_schema: &Arc, + primary_key_indices: &[usize], + allow_partial: bool, + max_window_cnt: Option, + values: &BTreeMap, + force_full_snapshot: bool, ) -> Result, Error> { let query_ctx = self.state.read().unwrap().query_ctx.clone(); let start = SystemTime::now(); @@ -1478,6 +1611,7 @@ impl BatchingTask { sink_table_schema.clone(), primary_key_indices, allow_partial, + values, dirty_windows_to_restore, None, QueryCoverage::UnfilteredFull, @@ -1514,6 +1648,34 @@ impl BatchingTask { ), })?; + if force_full_snapshot { + // Durable execution cannot trust an uncertain prior checkpoint. Re-run an + // unfiltered full snapshot, but retain the normal expiry predicate and the consumed + // dirty signal so every planning/execution failure remains reversible. + let retention_filter = self.config.expire_after.map(|_| { + ( + col_name.as_str(), + expire_lower_bound, + "forced full snapshot", + ) + }); + let (_, dirty_windows_to_restore) = self.drain_dirty_windows_signal(); + return self + .gen_unfiltered_plan_info( + engine, + query_ctx, + sink_table_schema.clone(), + primary_key_indices, + allow_partial, + values, + dirty_windows_to_restore, + retention_filter, + QueryCoverage::UnfilteredFull, + ) + .await + .map(Some); + } + if self.should_use_unfiltered_incremental_delta() { // In incremental mode, source correctness is defined by the // per-region sequence range `(checkpoint, scan-open snapshot]`, not @@ -1535,6 +1697,7 @@ impl BatchingTask { sink_table_schema.clone(), primary_key_indices, allow_partial, + values, retention_filter, QueryCoverage::IncrementalDelta, ) @@ -1580,10 +1743,11 @@ impl BatchingTask { ); let mut add_filter = AddFilterRewriter::new(expr.expr.clone()); - let mut add_auto_column = ColumnMatcherRewriter::new( + let mut add_auto_column = ColumnMatcherRewriter::new_with_values( sink_table_schema.clone(), primary_key_indices.to_vec(), allow_partial, + values.clone(), ); let plan = self.restore_scoped_dirty_windows_on_err( diff --git a/src/flow/src/batching_mode/task/inc.rs b/src/flow/src/batching_mode/task/inc.rs index 8c090d9814e..a980e1b9814 100644 --- a/src/flow/src/batching_mode/task/inc.rs +++ b/src/flow/src/batching_mode/task/inc.rs @@ -18,6 +18,7 @@ use common_error::ext::BoxedError; use common_telemetry::debug; use common_telemetry::tracing::warn; use datafusion_expr::{DmlStatement, LogicalPlan}; +use query::QueryEngineRef; use query::options::{ FLOW_INCREMENTAL_AFTER_SEQS, FLOW_INCREMENTAL_MODE, FLOW_INCREMENTAL_MODE_MEMTABLE_ONLY, FLOW_INCREMENTAL_MODE_SEQUENCE_RANGE, FLOW_SINK_TABLE_ID, @@ -119,6 +120,7 @@ impl BatchingTask { /// incremental safe without a rewrite, so they return `Some(original_plan)`. pub(super) async fn prepare_plan_for_incremental( &self, + engine: &QueryEngineRef, plan: &LogicalPlan, ) -> Result, Error> { let is_incremental_sql = { @@ -219,6 +221,7 @@ impl BatchingTask { let rewritten_inner = match rewrite_incremental_aggregate_with_sink_merge( &inner_plan, &analysis, + engine, sink_table, &self.config.sink_table_name, None, diff --git a/src/flow/src/batching_mode/task/test.rs b/src/flow/src/batching_mode/task/test.rs index db90cfbdcb3..7a56c70823d 100644 --- a/src/flow/src/batching_mode/task/test.rs +++ b/src/flow/src/batching_mode/task/test.rs @@ -15,6 +15,7 @@ use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; +use std::task::Poll; use catalog::memory::MemoryCatalogManager; use catalog::{DeregisterTableRequest, RegisterTableRequest}; @@ -59,6 +60,329 @@ fn incremental_batch_opts() -> Arc { }) } +struct CountingExecution { + calls: std::sync::atomic::AtomicUsize, + active: std::sync::atomic::AtomicUsize, + max_active: std::sync::atomic::AtomicUsize, +} + +#[async_trait::async_trait] +impl crate::BatchingExecution for CountingExecution { + async fn execute_once( + self: Arc, + _guard: BatchingExecutionGuard, + _task: &BatchingTask, + _engine: &QueryEngineRef, + _frontend: &Arc, + _max_window_cnt: Option, + ) -> ExecuteOnceOutcome { + self.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + let active = self + .active + .fetch_add(1, std::sync::atomic::Ordering::SeqCst) + + 1; + self.max_active + .fetch_max(active, std::sync::atomic::Ordering::SeqCst); + tokio::task::yield_now().await; + self.active + .fetch_sub(1, std::sync::atomic::Ordering::SeqCst); + ExecuteOnceOutcome { + new_query: None, + result: Ok(None), + } + } +} + +struct RetainingExecution { + calls: std::sync::atomic::AtomicUsize, + started: Arc, + release: Arc, + finished: Arc, +} + +#[async_trait::async_trait] +impl crate::BatchingExecution for RetainingExecution { + async fn execute_once( + self: Arc, + guard: BatchingExecutionGuard, + _task: &BatchingTask, + _engine: &QueryEngineRef, + _frontend: &Arc, + _max_window_cnt: Option, + ) -> ExecuteOnceOutcome { + if self.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) != 0 { + return ExecuteOnceOutcome { + new_query: None, + result: Ok(None), + }; + } + + let started = self.started.clone(); + let release = self.release.clone(); + let finished = self.finished.clone(); + let child = tokio::spawn(async move { + started.notify_one(); + release.notified().await; + drop(guard); + finished.notify_one(); + ExecuteOnceOutcome { + new_query: None, + result: Ok(None), + } + }); + match child.await { + Ok(outcome) => outcome, + Err(err) => ExecuteOnceOutcome { + new_query: None, + result: Err(Error::Unexpected { + reason: format!("retaining test child failed: {err}"), + location: snafu::location!(), + }), + }, + } + } +} + +#[tokio::test] +async fn test_execution_delegate_dispatch_is_serialized() { + let TestTaskParts { + task, query_engine, .. + } = new_test_task_engine_and_plan_with_query("SELECT number, ts FROM numbers_with_ts", "sink") + .await; + let execution = Arc::new(CountingExecution { + calls: Default::default(), + active: Default::default(), + max_active: Default::default(), + }); + let task = task.with_execution(Some(execution.clone())); + let (frontend, _handler) = FrontendClient::from_empty_grpc_handler(QueryOptions::default()); + let frontend = Arc::new(frontend); + + let first = task.execute_once_serialized(&query_engine, &frontend, None); + let second = task.execute_once_serialized(&query_engine, &frontend, None); + let (first, second) = tokio::join!(first, second); + assert_eq!(first.unwrap(), None); + assert_eq!(second.unwrap(), None); + assert_eq!(execution.calls.load(std::sync::atomic::Ordering::SeqCst), 2); + assert_eq!( + execution + .max_active + .load(std::sync::atomic::Ordering::SeqCst), + 1, + "the existing execution_lock must span delegate execution" + ); +} + +#[tokio::test] +async fn test_delegate_guard_survives_caller_cancellation_until_child_finishes() { + let TestTaskParts { + task, query_engine, .. + } = new_test_task_engine_and_plan_with_query("SELECT number, ts FROM numbers_with_ts", "sink") + .await; + let execution = Arc::new(RetainingExecution { + calls: Default::default(), + started: Arc::new(tokio::sync::Notify::new()), + release: Arc::new(tokio::sync::Notify::new()), + finished: Arc::new(tokio::sync::Notify::new()), + }); + let task = task.with_execution(Some(execution.clone())); + let (frontend, _handler) = FrontendClient::from_empty_grpc_handler(QueryOptions::default()); + let frontend = Arc::new(frontend); + + let first_task = task.clone(); + let first_engine = query_engine.clone(); + let first_frontend = frontend.clone(); + let first = tokio::spawn(async move { + first_task + .execute_once_serialized(&first_engine, &first_frontend, None) + .await + }); + tokio::time::timeout(Duration::from_secs(1), execution.started.notified()) + .await + .expect("delegate child did not retain the guard"); + first.abort(); + assert!( + first + .await + .expect_err("caller cancellation should abort") + .is_cancelled() + ); + + let second = task.execute_once_serialized(&query_engine, &frontend, None); + futures::pin_mut!(second); + assert!( + matches!(futures::poll!(second.as_mut()), Poll::Pending), + "the next round must remain pending while the retained guard is held" + ); + assert_eq!( + execution.calls.load(std::sync::atomic::Ordering::SeqCst), + 1, + "the pending waiter must not enter the collaborator" + ); + execution.release.notify_one(); + tokio::time::timeout(Duration::from_secs(1), execution.finished.notified()) + .await + .expect("delegate child did not release its guard"); + assert_eq!( + tokio::time::timeout(Duration::from_secs(1), second) + .await + .expect("next round should proceed after child release") + .unwrap(), + None + ); + assert_eq!(execution.calls.load(std::sync::atomic::Ordering::SeqCst), 2); +} + +#[tokio::test] +async fn test_scheduled_context_is_retained_until_delegate_child_releases_guard() { + let TestTaskParts { + task, query_engine, .. + } = new_test_task_engine_and_plan_with_query("SELECT number, ts FROM numbers_with_ts", "sink") + .await; + let execution = Arc::new(RetainingExecution { + calls: Default::default(), + started: Arc::new(tokio::sync::Notify::new()), + release: Arc::new(tokio::sync::Notify::new()), + finished: Arc::new(tokio::sync::Notify::new()), + }); + let task = task.with_execution(Some(execution.clone())); + let (frontend, _handler) = FrontendClient::from_empty_grpc_handler(QueryOptions::default()); + let frontend = Arc::new(frontend); + let scheduled = 1_700_000_000; + + let task_to_run = task.clone(); + let engine_to_run = query_engine.clone(); + let frontend_to_run = frontend.clone(); + let execution_call = tokio::spawn(async move { + task_to_run + .execute_once_serialized_at_scheduled_time(&engine_to_run, &frontend_to_run, scheduled) + .await + }); + tokio::time::timeout(Duration::from_secs(1), execution.started.notified()) + .await + .expect("scheduled delegate child did not start"); + assert_eq!( + task.state + .read() + .unwrap() + .query_ctx + .extension(FLOW_SCHEDULED_TIME_MILLIS), + Some("1700000000000"), + "scheduled context restored before the delegate child released the guard" + ); + execution_call.abort(); + match execution_call.await { + Err(error) => assert!(error.is_cancelled()), + Ok(_) => panic!("scheduled caller cancellation should abort"), + } + assert_eq!( + task.state + .read() + .unwrap() + .query_ctx + .extension(FLOW_SCHEDULED_TIME_MILLIS), + Some("1700000000000"), + "scheduled context restored after caller cancellation but before child release" + ); + + execution.release.notify_one(); + tokio::time::timeout(Duration::from_secs(1), execution.finished.notified()) + .await + .expect("scheduled delegate child did not release its guard"); + assert_eq!( + task.state + .read() + .unwrap() + .query_ctx + .extension(FLOW_SCHEDULED_TIME_MILLIS), + None, + "scheduled context was not restored when child released guard" + ); +} + +struct BlockingDefaultExecutionHandler { + entered: std::sync::Mutex>>, + dropped: std::sync::Mutex>>, +} + +struct DropAck(Option>); + +impl Drop for DropAck { + fn drop(&mut self) { + if let Some(dropped) = self.0.take() { + let _ = dropped.send(()); + } + } +} + +#[async_trait::async_trait] +impl crate::batching_mode::frontend_client::GrpcQueryHandlerWithBoxedError + for BlockingDefaultExecutionHandler +{ + async fn do_query( + &self, + _query: api::v1::greptime_request::Request, + _ctx: QueryContextRef, + ) -> std::result::Result { + let _ack = DropAck(self.dropped.lock().unwrap().take()); + if let Some(entered) = self.entered.lock().unwrap().take() { + let _ = entered.send(()); + } + std::future::pending().await + } +} + +#[tokio::test] +async fn test_default_execution_remains_inline_and_cancellable() { + let query = "SELECT number, date_bin(INTERVAL '5 second', ts) AS time_window \ + FROM numbers_with_ts GROUP BY time_window, number"; + let TestTaskParts { + task, query_engine, .. + } = new_time_window_test_task_with_query(query).await; + register_twe_sink(&query_engine, "missing_sink", 9200); + task.mark_all_windows_as_dirty().unwrap(); + + let (entered_tx, entered_rx) = tokio::sync::oneshot::channel(); + let (dropped_tx, dropped_rx) = tokio::sync::oneshot::channel(); + let handler: Arc = + Arc::new(BlockingDefaultExecutionHandler { + entered: std::sync::Mutex::new(Some(entered_tx)), + dropped: std::sync::Mutex::new(Some(dropped_tx)), + }); + let frontend = Arc::new(FrontendClient::from_grpc_handler( + Arc::downgrade(&handler), + QueryOptions::default(), + )); + + let task_to_cancel = task.clone(); + let engine_to_cancel = query_engine.clone(); + let frontend_to_cancel = frontend.clone(); + let caller = tokio::spawn(async move { + task_to_cancel + .execute_once_serialized(&engine_to_cancel, &frontend_to_cancel, None) + .await + }); + tokio::time::timeout(Duration::from_secs(1), entered_rx) + .await + .expect("default execution did not dispatch a frontend query") + .expect("default execution handler entry notification dropped"); + caller.abort(); + match caller.await { + Err(error) => assert!(error.is_cancelled()), + Ok(_) => panic!("default caller cancellation should abort"), + } + tokio::time::timeout(Duration::from_secs(1), dropped_rx) + .await + .expect("cancelling default execution did not drop the active frontend future") + .expect("default execution drop acknowledgement was not sent"); + tokio::time::timeout( + Duration::from_secs(1), + task.execution_lock.clone().lock_owned(), + ) + .await + .expect("default execution must not leave an owned child holding the lock"); +} + async fn new_test_task_and_plan_with_missing_sink() -> (BatchingTask, LogicalPlan) { new_test_task_engine_and_plan_with_query( "SELECT number, ts FROM numbers_with_ts", @@ -2072,6 +2396,62 @@ async fn test_full_snapshot_seeding_applies_expire_after_retention_filter() { ); } +#[tokio::test] +async fn test_force_full_snapshot_retains_expiration_and_reversible_dirty_capture() { + let TestTaskParts { + mut task, + query_engine, + .. + } = new_time_window_test_task_with_query( + "SELECT max(number) AS number, date_bin(INTERVAL '5 second', ts) AS time_window FROM numbers_with_ts GROUP BY time_window", + ) + .await; + { + let mut state = task.state.write().unwrap(); + state.advance_checkpoints(HashMap::from([(1_u64, 10_u64)])); + state + .dirty_time_windows + .add_window(Timestamp::new_second(0), Some(Timestamp::new_second(5))); + } + register_twe_sink(&query_engine, "missing_sink", 9201); + Arc::get_mut(&mut task.config) + .expect("test task config should be uniquely owned") + .expire_after = Some(expire_after_for_retention_filter_test()); + + let plan = task + .gen_insert_plan_with_values_unlocked(&query_engine, Some(1), &BTreeMap::new(), true) + .await + .unwrap() + .unwrap(); + assert!(matches!(plan.coverage, QueryCoverage::UnfilteredFull)); + assert!( + plan.plan + .to_string() + .contains("Filter: ts >= TimestampMillisecond(") + ); + assert!(task.state.read().unwrap().dirty_time_windows.is_empty()); + + task.restore_dirty_windows(&plan.dirty_restore); + assert_eq!(task.state.read().unwrap().dirty_time_windows.len(), 1); + + task.state.write().unwrap().dirty_time_windows.clean(); + let no_dirty_plan = task + .gen_insert_plan_with_values_unlocked(&query_engine, Some(1), &BTreeMap::new(), true) + .await + .unwrap() + .expect("forced full snapshot must bypass the dirty-window notification gate"); + assert!(matches!( + no_dirty_plan.coverage, + QueryCoverage::UnfilteredFull + )); + assert!( + no_dirty_plan + .plan + .to_string() + .contains("Filter: ts >= TimestampMillisecond(") + ); +} + #[tokio::test] async fn test_incremental_plan_does_not_add_dirty_window_filter() { let TestTaskParts { @@ -2189,19 +2569,38 @@ async fn test_exact_required_executed_failure_selects_full_snapshot_repair() { Arc::downgrade(&handler), QueryOptions::default(), )); - task.execute_logical_plan_unlocked( - &frontend_client, - &plan_info.plan, - &plan_info.dirty_restore, - &plan_info.coverage, - ) - .await - .expect_err("the dispatched exact delta must fail through the injected frontend handler"); + let raw = task + .execute_plan_unlocked( + &query_engine, + &frontend_client, + &plan_info.plan, + &plan_info.dirty_restore, + &plan_info.coverage, + ) + .await + .expect("raw executor should return the dispatched execution result") + .expect("raw executor should dispatch the exact delta"); + assert!( + raw.0.is_err(), + "the dispatched exact delta must fail through the injected frontend handler" + ); assert!( handler_invoked.load(Ordering::SeqCst), "the injected frontend handler must receive the exact delta" ); - task.handle_executed_query_failure(Some(&plan_info)); + // The raw executor is a delegate-safe primitive: it logs failures but does not change + // checkpoint state. Restore the consumed work, then let the default wrapper make its one + // failure transition and restore it again. + { + let state = task.state.read().unwrap(); + assert_eq!(state.checkpoint_mode(), CheckpointMode::Incremental); + assert_eq!(state.checkpoints(), &BTreeMap::from([(1_u64, 10_u64)])); + } + task.restore_dirty_windows(&plan_info.dirty_restore); + let outcome = task + .execute_once_default_unlocked(&query_engine, &frontend_client, Some(1)) + .await; + assert!(outcome.result.is_err()); { let state = task.state.read().unwrap(); @@ -2459,6 +2858,7 @@ async fn test_exact_required_unsupported_plan_keeps_exact_retry_state() { for _ in 0..2 { let err = task .execute_logical_plan_unlocked( + &query_engine, &frontend_client, &dml_plan, &DirtyRestore::Unscoped(dirty_range(10, 15)), @@ -2544,7 +2944,10 @@ async fn test_prepare_plan_for_incremental_disables_on_non_aggregate() { CheckpointMode::Incremental ); - let incremental_plan = task.prepare_plan_for_incremental(&dml_plan).await.unwrap(); + let incremental_plan = task + .prepare_plan_for_incremental(&query_engine, &dml_plan) + .await + .unwrap(); assert!(incremental_plan.is_none()); let state = task.state.read().unwrap(); assert!(state.is_incremental_disabled()); @@ -2621,6 +3024,7 @@ async fn test_unsafe_incremental_plan_skip_restores_dirty_without_query() { let result = task .execute_logical_plan_unlocked( + &query_engine, &Arc::new(frontend_client), &dml_plan, &dirty_restore, @@ -2707,7 +3111,7 @@ async fn test_prepare_plan_for_incremental_group_by_without_merge_columns_uses_o .advance_checkpoints(HashMap::from([(1_u64, 10_u64)])); let incremental_plan = task - .prepare_plan_for_incremental(&dml_plan) + .prepare_plan_for_incremental(&query_engine, &dml_plan) .await .unwrap() .expect("plain GROUP BY is incremental-safe without a rewrite"); @@ -2752,7 +3156,10 @@ async fn test_auto_created_sql_aggregate_sink_reaches_incremental_safe() { .write() .unwrap() .advance_checkpoints(HashMap::from([(1_u64, 10_u64)])); - let incremental_plan = task.prepare_plan_for_incremental(&dml_plan).await.unwrap(); + let incremental_plan = task + .prepare_plan_for_incremental(&query_engine, &dml_plan) + .await + .unwrap(); let incremental_safe = incremental_plan.is_some(); assert!(incremental_safe); diff --git a/src/flow/src/batching_mode/utils.rs b/src/flow/src/batching_mode/utils.rs index cb8e7e50634..87bb0925061 100644 --- a/src/flow/src/batching_mode/utils.rs +++ b/src/flow/src/batching_mode/utils.rs @@ -14,7 +14,7 @@ //! some utils for helping with batching mode -use std::collections::{BTreeSet, HashMap, HashSet}; +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; use std::sync::Arc; use catalog::CatalogManagerRef; @@ -66,6 +66,9 @@ mod test; /// `max(numbers_with_ts.number)`. #[derive(Debug, Clone, PartialEq, Eq)] pub struct IncrementalAggregateMergeColumn { + /// Delta-plan field containing the aggregate result/state column. Repeated + /// projections intentionally share this field while using distinct outputs. + pub input_field_name: String, /// Final output/sink field name for the aggregate result/state column. /// pub output_field_name: String, @@ -73,16 +76,17 @@ pub struct IncrementalAggregateMergeColumn { } impl IncrementalAggregateMergeColumn { - /// Create a new merge column. + /// Create a new merge column whose delta and output fields have the same name. pub fn new(output_field_name: String, merge_op: IncrementalAggregateMergeOp) -> Self { Self { + input_field_name: output_field_name.clone(), output_field_name, merge_op, } } } -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq)] pub enum IncrementalAggregateMergeOp { Sum, Min, @@ -92,6 +96,10 @@ pub enum IncrementalAggregateMergeOp { BitAnd, BitOr, BitXor, + StateDeltaMerge { + function_name: &'static str, + params: Vec, + }, } /// Analysis result for an incremental aggregate plan. @@ -113,24 +121,6 @@ pub struct IncrementalAggregateAnalysis { pub unsupported_exprs: Vec, } -/// Recursively find all `Expr::Column` names inside an expression tree. -/// Only recurses into wrappers that are merge-transparent. -/// Non-transparent wrappers (e.g., `ScalarFunction`, `Negative`, `Cast`) are -/// intentionally not recursed into since their merge semantics would be -/// incorrect. -/// -/// `Cast`/`TryCast` are intentionally opaque: merging already-casted aggregate -/// outputs is not generally equivalent to casting the final merged aggregate. -fn find_column_names(expr: &Expr, names: &mut Vec) { - match expr { - Expr::Column(col) => { - names.push(col.name.clone()); - } - Expr::Alias(alias) => find_column_names(&alias.expr, names), - _ => {} - } -} - fn unqualified_col(name: impl Into) -> Expr { Expr::Column(Column::from_name(name.into())) } @@ -230,8 +220,8 @@ fn check_input_plan_shape(plan: &LogicalPlan) -> Result<(), String> { #[derive(Debug, Default)] struct OutputProjectionInfo { has_top_level_projection: bool, - output_aliases: HashMap, - duplicate_aggregate_aliases: BTreeSet, + /// Aggregate expression name and projected output field, in projection order. + aggregate_outputs: Vec<(String, String)>, literal_columns: HashSet, output_field_names: Vec, } @@ -265,46 +255,34 @@ fn collect_output_projection_info(plan: &LogicalPlan) -> OutputProjectionInfo { ..Default::default() }; - let mut output_aliases = HashMap::new(); if let LogicalPlan::Projection(projection) = plan { for expr in &projection.expr { match expr { Expr::Alias(alias) => { - // Alias resolution has three cases: - // - 0 Column refs (e.g., literal `42 AS lit`): record literal output - // - 1 Column ref: record the mapping (e.g., `sum(x) AS total`) - // - >1 Column refs (e.g., `COALESCE(sum(x), sum(y))`): - // skip — ambiguous merge semantics + // Only a direct aggregate output column has the same + // merge semantics as the original resolver. In particular, + // do not mine aggregate columns through CAST/TryCast or + // other output wrappers. let alias_name = alias.name.clone(); - let mut col_names = Vec::new(); - find_column_names(&alias.expr, &mut col_names); - match col_names.len() { - 0 if is_passthrough_output_column(&alias_name, alias.expr.as_ref()) => { - projection_info.literal_columns.insert(alias_name); - } - 1 => { - if let Some(col_name) = col_names.into_iter().next() { - if let Some(existing_alias) = output_aliases.get(&col_name) { - if existing_alias != &alias_name { - projection_info.duplicate_aggregate_aliases.insert(format!( - "same aggregate output {col_name} is used by multiple aliases: {existing_alias}, {alias_name}" - )); - } - } else { - output_aliases.insert(col_name, alias_name); - } - } - } - _ => {} + if let Expr::Column(column) = alias.expr.as_ref() { + projection_info + .aggregate_outputs + .push((column.name.clone(), alias_name)); + } else if let Expr::Alias(inner_alias) = alias.expr.as_ref() + && inner_alias.name.eq_ignore_ascii_case("count(*)") + && let Expr::Column(column) = inner_alias.expr.as_ref() + { + projection_info + .aggregate_outputs + .push((column.name.clone(), alias_name)); + } else if is_passthrough_output_column(&alias_name, alias.expr.as_ref()) { + projection_info.literal_columns.insert(alias_name); } - - // If >1 column references detected (e.g., COALESCE(sum(x), sum(y))), - // intentionally skip alias mapping — the merge semantics are ambiguous. } Expr::Column(col) => { - output_aliases - .entry(col.name.clone()) - .or_insert(col.name.clone()); + projection_info + .aggregate_outputs + .push((col.name.clone(), col.name.clone())); } Expr::Literal(_, _) => { projection_info @@ -326,7 +304,6 @@ fn collect_output_projection_info(plan: &LogicalPlan) -> OutputProjectionInfo { .insert(AUTO_CREATED_PLACEHOLDER_TS_COL.to_string()); } - projection_info.output_aliases = output_aliases; projection_info } @@ -362,6 +339,12 @@ fn merge_op_for_aggregate_expr(aggr_expr: &Expr) -> Result Ok(IncrementalAggregateMergeOp::Sum), "min" => Ok(IncrementalAggregateMergeOp::Min), @@ -371,28 +354,52 @@ fn merge_op_for_aggregate_expr(aggr_expr: &Expr) -> Result Ok(IncrementalAggregateMergeOp::BitAnd), "bit_or" => Ok(IncrementalAggregateMergeOp::BitOr), "bit_xor" => Ok(IncrementalAggregateMergeOp::BitXor), + // Preserve state-family parameters; value coercion is handled by the aggregate. + "hll" if aggr_func.params.args.len() == 1 => state_delta_merge("__hll_delta_merge", vec![]), + "stddev_pop_state" if aggr_func.params.args.len() == 1 => { + state_delta_merge("__stddev_pop_state_delta_merge", vec![]) + } + "uddsketch_state" if aggr_func.params.args.len() == 3 => { + let [bucket_size, error_rate, _] = aggr_func.params.args.as_slice() else { + unreachable!(); + }; + if !matches!(bucket_size, Expr::Literal(ScalarValue::Int64(Some(_)), _)) + || !matches!(error_rate, Expr::Literal(ScalarValue::Float64(Some(_)), _)) + { + return Err(aggr_expr.to_string()); + } + state_delta_merge( + "__uddsketch_state_delta_merge", + vec![bucket_size.clone(), error_rate.clone()], + ) + } _ => Err(aggr_expr.to_string()), } } -fn resolve_aggregate_output_field_name( +fn resolve_aggregate_output_fields( aggr_expr: &Expr, projection_info: &OutputProjectionInfo, output_field_name_set: &HashSet, -) -> Option { +) -> Vec<(String, String)> { // qualified_name() returns (Option, String) where the second // element is the unqualified column/alias name. This relies on // DataFusion's internal naming convention: aggregate expressions - // emit a column named after the aggregate itself (e.g. "SUM(x)"), - // which matches what the projection aliases reference. + // emit a column named after the aggregate itself (e.g. "SUM(x)"). + // Keep every matching projection occurrence because DataFusion can share + // one aggregate input field for identical expressions. let raw_name = aggr_expr.qualified_name().1; - if let Some(alias) = projection_info.output_aliases.get(&raw_name) { - Some(alias.clone()) - } else if !projection_info.has_top_level_projection && output_field_name_set.contains(&raw_name) - { - Some(raw_name) + if projection_info.has_top_level_projection { + projection_info + .aggregate_outputs + .iter() + .filter(|(input_name, _)| input_name == &raw_name) + .cloned() + .collect() + } else if output_field_name_set.contains(&raw_name) { + vec![(raw_name.clone(), raw_name)] } else { - None + vec![] } } @@ -478,7 +485,6 @@ pub fn analyze_incremental_aggregate_plan( .map(|name| format!("duplicate output field name: {name}")) .collect::>(); unsupported_exprs.push(reason); - unsupported_exprs.extend(projection_info.duplicate_aggregate_aliases.iter().cloned()); return Ok(Some(IncrementalAggregateAnalysis { group_key_names, merge_columns: vec![], @@ -512,7 +518,6 @@ pub fn analyze_incremental_aggregate_plan( aggregate, &group_key_names, )); - unsupported_exprs.extend(projection_info.duplicate_aggregate_aliases.iter().cloned()); for aggr_expr in aggr_exprs { let merge_op = match merge_op_for_aggregate_expr(&aggr_expr) { Ok(merge_op) => merge_op, @@ -521,18 +526,37 @@ pub fn analyze_incremental_aggregate_plan( continue; } }; - let Some(output_field_name) = resolve_aggregate_output_field_name( - &aggr_expr, - &projection_info, - &output_field_name_set, - ) else { + let aggregate_outputs = + resolve_aggregate_output_fields(&aggr_expr, &projection_info, &output_field_name_set); + if aggregate_outputs.is_empty() { unsupported_exprs.push(aggr_expr.to_string()); continue; + } + let Some((_, input_field_name)) = aggregate_outputs.first() else { + continue; }; - merge_columns.push(IncrementalAggregateMergeColumn::new( - output_field_name, - merge_op, - )); + let input_field_name = input_field_name.clone(); + for (_, output_field_name) in aggregate_outputs { + merge_columns.push(IncrementalAggregateMergeColumn { + input_field_name: input_field_name.clone(), + output_field_name, + merge_op: merge_op.clone(), + }); + } + } + if projection_info.has_top_level_projection { + let output_positions = projection_info + .output_field_names + .iter() + .enumerate() + .map(|(position, name)| (name.as_str(), position)) + .collect::>(); + merge_columns.sort_by_key(|column| { + output_positions + .get(column.output_field_name.as_str()) + .copied() + .unwrap_or(usize::MAX) + }); } unsupported_exprs.extend( find_uncovered_output_fields(&projection_info, &group_key_names, &merge_columns) @@ -591,6 +615,7 @@ pub fn analyze_incremental_aggregate_plan( pub async fn rewrite_incremental_aggregate_with_sink_merge( delta_plan: &LogicalPlan, analysis: &IncrementalAggregateAnalysis, + engine: &QueryEngineRef, sink_table: TableRef, sink_table_name: &TableName, sink_dirty_filter: Option, @@ -625,6 +650,12 @@ pub async fn rewrite_incremental_aggregate_with_sink_merge( let delta_alias = "__flow_delta"; let sink_alias = "__flow_sink"; + let state_merge = analysis.merge_columns.iter().any(|column| { + matches!( + &column.merge_op, + IncrementalAggregateMergeOp::StateDeltaMerge { .. } + ) + }); let mut selected_columns = analysis.group_key_names.clone(); selected_columns.extend( analysis @@ -632,8 +663,18 @@ pub async fn rewrite_incremental_aggregate_with_sink_merge( .iter() .map(|c| c.output_field_name.clone()), ); - let mut delta_selected_columns = selected_columns.clone(); + let mut selected_column_names = HashSet::new(); + selected_columns.retain(|name| selected_column_names.insert(name.clone())); + let mut delta_selected_columns = analysis.group_key_names.clone(); + delta_selected_columns.extend( + analysis + .merge_columns + .iter() + .map(|c| c.input_field_name.clone()), + ); delta_selected_columns.extend(analysis.literal_columns.iter().cloned()); + let mut delta_selected_column_names = HashSet::new(); + delta_selected_columns.retain(|name| delta_selected_column_names.insert(name.clone())); let delta_selected_exprs = delta_selected_columns .iter() @@ -745,20 +786,25 @@ pub async fn rewrite_incremental_aggregate_with_sink_merge( .map(|c| (&c.output_field_name, c)) .collect::>(); - let mut projection_exprs = Vec::with_capacity(analysis.output_field_names.len()); + let mut ordinary_exprs = Vec::with_capacity(analysis.output_field_names.len()); + let mut state_aggr_exprs = Vec::new(); for output_field_name in &analysis.output_field_names { if group_key_names.contains(output_field_name) || literal_columns.contains(output_field_name) { - projection_exprs.push( - qualified_col(delta_alias, output_field_name.clone()).alias(output_field_name), - ); + let expr = + qualified_col(delta_alias, output_field_name.clone()).alias(output_field_name); + ordinary_exprs.push(expr); } else if let Some(merge_col) = merge_columns.get(output_field_name) { - projection_exprs.push(build_left_join_merge_expr( - delta_alias, - sink_alias, - merge_col, - )?); + if matches!( + &merge_col.merge_op, + IncrementalAggregateMergeOp::StateDeltaMerge { .. } + ) { + state_aggr_exprs.push(build_state_delta_merge_expr(engine, merge_col)?); + } else { + let expr = build_left_join_merge_expr(delta_alias, sink_alias, merge_col)?; + ordinary_exprs.push(expr); + } } else { return InvalidQuerySnafu { reason: format!( @@ -769,15 +815,88 @@ pub async fn rewrite_incremental_aggregate_with_sink_merge( } } - LogicalPlanBuilder::from(joined) - .project(projection_exprs) - .with_context(|_| DatafusionSnafu { - context: "Failed to build projection merge plan for incremental sink merge".to_string(), - })? - .build() - .with_context(|_| DatafusionSnafu { - context: "Failed to finalize incremental aggregate sink merge plan".to_string(), + if state_merge { + let aggregated = LogicalPlanBuilder::from(joined) + .aggregate(ordinary_exprs, state_aggr_exprs) + .with_context(|_| DatafusionSnafu { + context: "Failed to aggregate state delta merge plan".to_string(), + })? + .build() + .with_context(|_| DatafusionSnafu { + context: "Failed to build state delta merge plan".to_string(), + })?; + let output_exprs = analysis + .output_field_names + .iter() + .cloned() + .map(unqualified_col) + .collect::>(); + LogicalPlanBuilder::from(aggregated) + .project(output_exprs) + .with_context(|_| DatafusionSnafu { + context: "Failed to project state delta merge plan".to_string(), + })? + .build() + .with_context(|_| DatafusionSnafu { + context: "Failed to finalize incremental aggregate sink merge plan".to_string(), + }) + } else { + LogicalPlanBuilder::from(joined) + .project(ordinary_exprs) + .with_context(|_| DatafusionSnafu { + context: "Failed to build projection merge plan for incremental sink merge" + .to_string(), + })? + .build() + .with_context(|_| DatafusionSnafu { + context: "Failed to finalize incremental aggregate sink merge plan".to_string(), + }) + } +} + +fn build_state_delta_merge_expr( + engine: &QueryEngineRef, + merge_col: &IncrementalAggregateMergeColumn, +) -> Result { + let (function_name, params) = match &merge_col.merge_op { + IncrementalAggregateMergeOp::StateDeltaMerge { + function_name, + params, + } => (*function_name, params.clone()), + _ => { + return InvalidQuerySnafu { + reason: "non-state aggregate passed to state delta merge".to_string(), + } + .fail(); + } + }; + let Some(udaf) = engine + .engine_state() + .aggr_function(function_name) + .or_else(|| { + engine + .engine_state() + .session_state() + .aggregate_functions() + .get(function_name) + .map(|udaf| udaf.as_ref().clone()) }) + else { + return InvalidQuerySnafu { + reason: format!("Aggregate function {function_name} is not registered"), + } + .fail(); + }; + let mut args = params; + args.push(qualified_col( + "__flow_delta", + merge_col.input_field_name.clone(), + )); + args.push(qualified_col( + "__flow_sink", + merge_col.output_field_name.clone(), + )); + Ok(udaf.call(args).alias(merge_col.output_field_name.clone())) } fn build_left_join_merge_expr( @@ -785,9 +904,9 @@ fn build_left_join_merge_expr( sink_alias: &str, merge_col: &IncrementalAggregateMergeColumn, ) -> Result { - let left = qualified_col(delta_alias, merge_col.output_field_name.clone()); + let left = qualified_col(delta_alias, merge_col.input_field_name.clone()); let right = qualified_col(sink_alias, merge_col.output_field_name.clone()); - let merged = match merge_col.merge_op { + let merged = match merge_col.merge_op.clone() { IncrementalAggregateMergeOp::Sum => when(is_null(left.clone()), right.clone()) .when(is_null(right.clone()), left.clone()) .otherwise(binary_expr(left.clone(), Operator::Plus, right.clone())) @@ -836,6 +955,12 @@ fn build_left_join_merge_expr( .with_context(|_| DatafusionSnafu { context: "Failed to build BIT_XOR merge expression".to_string(), })?, + IncrementalAggregateMergeOp::StateDeltaMerge { .. } => { + return InvalidQuerySnafu { + reason: "state aggregate must be built with its delta UDAF".to_string(), + } + .fail(); + } }; Ok(merged.alias(merge_col.output_field_name.clone())) } @@ -948,13 +1073,36 @@ pub(crate) async fn gen_plan_with_matching_schema( sink_table_schema: SchemaRef, primary_key_indices: &[usize], allow_partial: bool, +) -> Result { + gen_plan_with_matching_schema_and_values( + sql, + query_ctx, + engine, + sink_table_schema, + primary_key_indices, + allow_partial, + &BTreeMap::new(), + ) + .await +} + +/// Generate a schema-matching plan with extension-owned typed sink values. +pub(crate) async fn gen_plan_with_matching_schema_and_values( + sql: &str, + query_ctx: QueryContextRef, + engine: QueryEngineRef, + sink_table_schema: SchemaRef, + primary_key_indices: &[usize], + allow_partial: bool, + values: &BTreeMap, ) -> Result { let plan = sql_to_df_plan(query_ctx.clone(), engine.clone(), sql, false).await?; - let mut add_auto_column = ColumnMatcherRewriter::new( + let mut add_auto_column = ColumnMatcherRewriter::new_with_values( sink_table_schema, primary_key_indices.to_vec(), allow_partial, + values.clone(), ); let plan = plan .clone() @@ -1090,7 +1238,10 @@ impl TreeNodeVisitor<'_> for FindGroupByFinalName { /// it also give existing columns alias to column in sink table if needed #[derive(Debug)] pub struct ColumnMatcherRewriter { + /// Schema used to match the ordinary flow output. It excludes injected values. pub schema: SchemaRef, + sink_schema: SchemaRef, + values: BTreeMap, pub is_rewritten: bool, pub primary_key_indices: Vec, pub allow_partial: bool, @@ -1099,21 +1250,127 @@ pub struct ColumnMatcherRewriter { impl ColumnMatcherRewriter { pub fn new(schema: SchemaRef, primary_key_indices: Vec, allow_partial: bool) -> Self { Self { - schema, + schema: schema.clone(), + sink_schema: schema, + values: BTreeMap::new(), is_rewritten: false, primary_key_indices, allow_partial, } } + /// Match ordinary output against the sink schema excluding typed values, then inject those + /// values in physical sink-column order. + pub fn new_with_values( + sink_schema: SchemaRef, + primary_key_indices: Vec, + allow_partial: bool, + values: BTreeMap, + ) -> Self { + let value_names = values.keys().collect::>(); + let ordinary_columns = sink_schema + .column_schemas() + .iter() + .filter(|column| !value_names.contains(&column.name)) + .cloned() + .collect(); + let ordinary_schema = Arc::new(datatypes::schema::Schema::new(ordinary_columns)); + let ordinary_primary_key_indices = primary_key_indices + .into_iter() + .filter_map(|index| { + let name = &sink_schema.column_schemas()[index].name; + ordinary_schema + .column_schemas() + .iter() + .position(|column| column.name == *name) + }) + .collect(); + Self { + schema: ordinary_schema, + sink_schema, + values, + is_rewritten: false, + primary_key_indices: ordinary_primary_key_indices, + allow_partial, + } + } + + fn validate_values(&self, output_exprs: &[Expr]) -> DfResult<()> { + let output_names = output_exprs + .iter() + .map(|expr| expr.qualified_name().1) + .collect::>(); + let duplicated = duplicate_names(&output_names); + if !duplicated.is_empty() { + return Err(DataFusionError::Plan(format!( + "Flow output schema contains duplicate column(s) {:?}. {}", + duplicated, + format_flow_sink_schema_mismatch(output_exprs, self.sink_schema.as_ref()) + ))); + } + for (name, value) in &self.values { + let Some(column) = self + .sink_schema + .column_schemas() + .iter() + .find(|column| column.name == *name) + else { + return Err(DataFusionError::Plan(format!( + "Injected flow value column '{name}' is not found in sink schema" + ))); + }; + if output_names.iter().any(|output| output == name) { + return Err(DataFusionError::Plan(format!( + "Injected flow value column '{name}' collides with flow output" + ))); + } + let value_type = ConcreteDataType::from_arrow_type(&value.data_type()); + if value_type != column.data_type { + return Err(DataFusionError::Plan(format!( + "Injected flow value column '{name}' has type {:?}, but sink column has type {:?}", + value_type, column.data_type + ))); + } + } + Ok(()) + } + + fn inject_values_in_physical_order(&self, exprs: Vec) -> DfResult> { + if self.values.is_empty() { + return Ok(exprs); + } + let mut ordinary = exprs + .into_iter() + .map(|expr| (expr.qualified_name().1, expr)) + .collect::>(); + self.sink_schema + .column_schemas() + .iter() + .map(|column| { + if let Some(value) = self.values.get(&column.name) { + Ok(Expr::Literal(value.clone(), None).alias(column.name.clone())) + } else { + ordinary.remove(&column.name).ok_or_else(|| { + DataFusionError::Plan(format!( + "Flow output is missing sink column '{}' after schema matching", + column.name + )) + }) + } + }) + .collect() + } + /// modify the exprs in place so that it matches the schema and some auto columns are added fn modify_project_exprs( &mut self, mut exprs: Vec, input_schema: &DFSchema, ) -> DfResult> { + self.validate_values(&exprs)?; if self.allow_partial { - return self.modify_project_exprs_with_partial(exprs); + let exprs = self.modify_project_exprs_with_partial(exprs)?; + return self.inject_values_in_physical_order(exprs); } let original_exprs = exprs.clone(); @@ -1182,7 +1439,12 @@ impl ColumnMatcherRewriter { ))); } - self.match_extra_output_columns(exprs, input_schema, &original_exprs, &all_names) + self.inject_values_in_physical_order(self.match_extra_output_columns( + exprs, + input_schema, + &original_exprs, + &all_names, + )?) } /// Match flow output columns whose names are not in the sink schema by the same position only. diff --git a/src/flow/src/batching_mode/utils/test.rs b/src/flow/src/batching_mode/utils/test.rs index 1c34caeedd7..7b3099818be 100644 --- a/src/flow/src/batching_mode/utils/test.rs +++ b/src/flow/src/batching_mode/utils/test.rs @@ -12,13 +12,19 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::collections::BTreeMap; use std::sync::Arc; use catalog::RegisterTableRequest; -use common_recordbatch::RecordBatch; +use common_query::OutputData; +use common_recordbatch::recordbatch::merge_record_batches; +use common_recordbatch::{RecordBatch, util}; use common_time::Timestamp; +use datafusion_common::ScalarValue; use datafusion_common::tree_node::TreeNode as _; use datafusion_expr::{GroupingSet, TableScanBuilder}; +use datatypes::arrow::array::{Array, AsArray}; +use datatypes::arrow::datatypes::{Float64Type, Int64Type, UInt64Type}; use datatypes::prelude::{ConcreteDataType, MutableVector, Scalar, ScalarVectorBuilder, VectorRef}; use datatypes::schema::{ColumnSchema, Schema}; use datatypes::timestamp::TimestampMillisecond; @@ -739,6 +745,108 @@ async fn test_gen_plan_with_matching_schema_allows_null_positional_alias() { assert!(sql.contains("NULL AS label"), "{sql}"); } +#[tokio::test] +async fn test_gen_plan_with_matching_schema_and_values_injects_physical_sink_order() { + let sink_schema = Arc::new(Schema::new(vec![ + ColumnSchema::new("number", ConcreteDataType::uint32_datatype(), true), + ColumnSchema::new("marker", ConcreteDataType::uint8_datatype(), true), + ColumnSchema::new( + "ts", + ConcreteDataType::timestamp_millisecond_datatype(), + false, + ) + .with_time_index(true), + ColumnSchema::new("epoch", ConcreteDataType::uint64_datatype(), true), + ])); + let plan = gen_plan_with_matching_schema_and_values( + "SELECT number, ts FROM numbers_with_ts", + QueryContext::arc(), + create_test_query_engine(), + sink_schema, + &[], + false, + &BTreeMap::from([ + ("epoch".to_string(), ScalarValue::UInt64(Some(7))), + ("marker".to_string(), ScalarValue::UInt8(None)), + ]), + ) + .await + .unwrap(); + let names = plan + .schema() + .fields() + .iter() + .map(|field| field.name().clone()) + .collect::>(); + assert_eq!(names, vec!["number", "marker", "ts", "epoch"]); +} + +#[tokio::test] +async fn test_gen_plan_with_matching_schema_and_values_rejects_unknown_collision_and_type() { + let sink_schema = Arc::new(Schema::new(vec![ + ColumnSchema::new("number", ConcreteDataType::uint32_datatype(), true), + ColumnSchema::new( + "ts", + ConcreteDataType::timestamp_millisecond_datatype(), + false, + ) + .with_time_index(true), + ColumnSchema::new("marker", ConcreteDataType::uint8_datatype(), true), + ])); + for (values, expected) in [ + ( + BTreeMap::from([("unknown".to_string(), ScalarValue::UInt8(None))]), + "is not found in sink schema", + ), + ( + BTreeMap::from([("number".to_string(), ScalarValue::UInt32(None))]), + "collides with flow output", + ), + ( + BTreeMap::from([("marker".to_string(), ScalarValue::UInt64(None))]), + "has type UInt64", + ), + ] { + let err = gen_plan_with_matching_schema_and_values( + "SELECT number, ts FROM numbers_with_ts", + QueryContext::arc(), + create_test_query_engine(), + sink_schema.clone(), + &[], + false, + &values, + ) + .await + .unwrap_err() + .to_string(); + assert!(err.contains(expected), "{err}"); + } +} + +#[tokio::test] +async fn test_gen_plan_with_matching_schema_and_values_rejects_duplicate_output() { + let sink_schema = Arc::new(Schema::new(vec![ + ColumnSchema::new("number", ConcreteDataType::uint32_datatype(), true), + ColumnSchema::new("marker", ConcreteDataType::uint8_datatype(), true), + ])); + let err = gen_plan_with_matching_schema_and_values( + "SELECT number, number AS number FROM numbers_with_ts", + QueryContext::arc(), + create_test_query_engine(), + sink_schema, + &[], + false, + &BTreeMap::from([("marker".to_string(), ScalarValue::UInt8(None))]), + ) + .await + .unwrap_err(); + let err = format!("{err:?}"); + assert!( + err.contains("Projections require unique expression names"), + "DataFusion should reject duplicate output aliases during query planning: {err}" + ); +} + #[tokio::test] async fn test_gen_plan_with_matching_schema_accepts_matching_flow_schema() { let query_engine = create_test_query_engine(); @@ -1104,13 +1212,16 @@ async fn test_rewrite_incremental_aggregate_allows_alias_wrapped_scan() { let query_engine = create_test_query_engine(); let ctx = QueryContext::arc(); let sql = "SELECT max(n.number) AS number, n.ts FROM numbers_with_ts AS n GROUP BY n.ts"; - let plan = sql_to_df_plan(ctx, query_engine, sql, false).await.unwrap(); + let plan = sql_to_df_plan(ctx, query_engine.clone(), sql, false) + .await + .unwrap(); let analysis = analyze_incremental_aggregate_plan(&plan).unwrap().unwrap(); assert!(analysis.unsupported_exprs.is_empty()); let rewritten = rewrite_incremental_aggregate_with_sink_merge( &plan, &analysis, + &query_engine, single_row_u32_table("alias_wrapped_sink", vec!["ts", "number"]), &[ "greptime".to_string(), @@ -1362,6 +1473,7 @@ async fn test_analyze_incremental_aggregate_plan_allows_literal_outputs() { let rewritten = rewrite_incremental_aggregate_with_sink_merge( &plan, &analysis, + &query_engine, sink_table.clone(), &sink_table_name, None, @@ -1435,7 +1547,9 @@ async fn test_rewrite_incremental_aggregate_preserves_non_identifier_aliases() { let query_engine = create_test_query_engine(); let ctx = QueryContext::arc(); let sql = "SELECT max(number) AS \"max value\", number, 42 AS \"literal value\" FROM numbers_with_ts GROUP BY number"; - let plan = sql_to_df_plan(ctx, query_engine, sql, false).await.unwrap(); + let plan = sql_to_df_plan(ctx, query_engine.clone(), sql, false) + .await + .unwrap(); let analysis = analyze_incremental_aggregate_plan(&plan).unwrap().unwrap(); assert!(analysis.unsupported_exprs.is_empty()); assert_eq!( @@ -1447,6 +1561,7 @@ async fn test_rewrite_incremental_aggregate_preserves_non_identifier_aliases() { let rewritten = rewrite_incremental_aggregate_with_sink_merge( &plan, &analysis, + &query_engine, sink_table, &[ "greptime".to_string(), @@ -1520,24 +1635,34 @@ async fn test_datafusion_rejects_duplicate_output_names() { } #[tokio::test] -async fn test_analyze_incremental_aggregate_plan_rejects_same_aggregate_multiple_aliases() { - let query_engine = create_test_query_engine(); - let ctx = QueryContext::arc(); - let sql = "SELECT sum(number) AS a, sum(number) AS b, ts FROM numbers_with_ts GROUP BY ts"; - let plan = sql_to_df_plan(ctx, query_engine, sql, false).await.unwrap(); +async fn test_analyze_incremental_aggregate_plan_supports_same_aggregate_multiple_aliases() { + let analysis = analyze_test_sql( + "SELECT sum(number) AS a, sum(number) AS b, ts FROM numbers_with_ts GROUP BY ts", + ) + .await; - let analysis = analyze_incremental_aggregate_plan(&plan).unwrap().unwrap(); + assert!(analysis.unsupported_exprs.is_empty()); + assert_eq!(analysis.merge_columns.len(), 2); + assert_eq!(analysis.merge_columns[0].input_field_name, "a"); + assert_eq!(analysis.merge_columns[1].input_field_name, "a"); assert!( analysis - .unsupported_exprs + .merge_columns .iter() - .any(|expr| expr.contains("same aggregate output") - && expr.contains("a") - && expr.contains("b")), - "same aggregate with multiple aliases should be unsupported until explicit reproduction is implemented: {:?}", - analysis.unsupported_exprs + .all(|column| { column.merge_op == IncrementalAggregateMergeOp::Sum }) + ); + assert!( + analysis + .merge_columns + .iter() + .any(|column| column.output_field_name == "a") + ); + assert!( + analysis + .merge_columns + .iter() + .any(|column| column.output_field_name == "b") ); - assert!(analysis.merge_columns.is_empty()); } #[test] @@ -1581,6 +1706,322 @@ async fn test_analyze_incremental_aggregate_plan_rejects_avg() { assert!(!analysis.unsupported_exprs.is_empty()); } +#[tokio::test] +async fn test_rewrite_incremental_aggregate_merges_populated_mixed_state_families() { + let query_engine = create_test_query_engine(); + let old_sql = "SELECT hll(CAST(number AS VARCHAR)) AS hll_a, hll(CAST(number AS VARCHAR)) AS hll_b, uddsketch_state(128, 0.000001, number) AS percentile_a, uddsketch_state(256, 0.02, number) AS percentile_b, stddev_pop_state(number) AS stddev_state, sum(number) AS total, CASE WHEN number <= 5 THEN CAST(NULL AS BIGINT) ELSE 2 END AS grp FROM numbers_with_ts WHERE number <= 3 OR number = 6 GROUP BY grp"; + let new_sql = "SELECT hll(CAST(number AS VARCHAR)) AS hll_a, hll(CAST(number AS VARCHAR)) AS hll_b, uddsketch_state(128, 0.000001, number) AS percentile_a, uddsketch_state(256, 0.02, number) AS percentile_b, stddev_pop_state(number) AS stddev_state, sum(number) AS total, CASE WHEN number <= 5 THEN CAST(NULL AS BIGINT) WHEN number <= 8 THEN 2 ELSE 3 END AS grp FROM numbers_with_ts WHERE number >= 4 AND number != 6 GROUP BY grp"; + let old_plan = sql_to_df_plan(QueryContext::arc(), query_engine.clone(), old_sql, false) + .await + .unwrap(); + let old_output = query_engine + .execute(old_plan, QueryContext::arc()) + .await + .unwrap(); + let OutputData::Stream(old_stream) = old_output.data else { + panic!("expected old aggregate execution to be a stream"); + }; + let old_batches = util::collect(old_stream).await.unwrap(); + let old_schema = old_batches.first().unwrap().schema.clone(); + let old_batch = merge_record_batches(old_schema, &old_batches).unwrap(); + assert_eq!( + old_batch.num_rows(), + 2, + "old state must have one row per group" + ); + let empty_state_sink_batch = old_batch.clone(); + let sink_table = MemTable::table("state_merge_sink", old_batch); + let sink_table_name = [ + "greptime".to_string(), + "public".to_string(), + "state_merge_sink".to_string(), + ]; + + let new_plan = sql_to_df_plan(QueryContext::arc(), query_engine.clone(), new_sql, false) + .await + .unwrap(); + let analysis = analyze_incremental_aggregate_plan(&new_plan) + .unwrap() + .unwrap(); + assert!(analysis.unsupported_exprs.is_empty()); + let rewritten = rewrite_incremental_aggregate_with_sink_merge( + &new_plan, + &analysis, + &query_engine, + sink_table, + &sink_table_name, + None, + ) + .await + .unwrap(); + assert_eq!( + analysis.output_field_names, + vec![ + "hll_a", + "hll_b", + "percentile_a", + "percentile_b", + "stddev_state", + "total", + "grp", + ], + "repeated HLL aliases must preserve output order" + ); + + let output = query_engine + .execute(rewritten, QueryContext::arc()) + .await + .unwrap(); + let OutputData::Stream(stream) = output.data else { + panic!("expected rewritten plan to execute as a stream"); + }; + let batches = util::collect(stream).await.unwrap(); + let merged_schema = batches.first().unwrap().schema.clone(); + let merged_batch = merge_record_batches(merged_schema, &batches).unwrap(); + assert_eq!( + merged_batch.num_rows(), + 3, + "rewrite must produce one row per group" + ); + let merged_table = MemTable::table("merged_states", merged_batch); + query_engine + .engine_state() + .catalog_manager() + .as_any() + .downcast_ref::() + .unwrap() + .register_table_sync(RegisterTableRequest { + catalog: "greptime".to_string(), + schema: "public".to_string(), + table_name: "merged_states".to_string(), + table_id: 4097, + table: merged_table, + }) + .unwrap(); + let checks = "SELECT grp, sum(total) AS total, hll_count(hll_merge(hll_a)) AS hll_a, hll_count(hll_merge(hll_b)) AS hll_b, stddev_pop_calc(stddev_pop_merge(stddev_state)) AS stddev, uddsketch_calc(0.5, uddsketch_merge(128, 0.000001, percentile_a)) AS p50_a, uddsketch_calc(0.5, uddsketch_merge(256, 0.02, percentile_b)) AS p50_b FROM merged_states GROUP BY grp ORDER BY grp NULLS FIRST"; + let checks_plan = sql_to_df_plan(QueryContext::arc(), query_engine.clone(), checks, false) + .await + .unwrap(); + let checks_output = query_engine + .execute(checks_plan, QueryContext::arc()) + .await + .unwrap(); + let OutputData::Stream(checks_stream) = checks_output.data else { + panic!("expected state check execution to be a stream"); + }; + let checks_batches = util::collect(checks_stream).await.unwrap(); + let checks_schema = checks_batches.first().unwrap().schema.clone(); + let checks = merge_record_batches(checks_schema, &checks_batches).unwrap(); + assert_eq!(checks.num_rows(), 3); + assert_eq!( + checks + .schema + .column_schemas() + .iter() + .map(|column| column.name.as_str()) + .collect::>(), + vec!["grp", "total", "hll_a", "hll_b", "stddev", "p50_a", "p50_b"] + ); + let group = checks.column(0).as_primitive::(); + let total = checks.column(1).as_primitive::(); + let hll_a = checks.column(2).as_primitive::(); + let hll_b = checks.column(3).as_primitive::(); + let stddev = checks.column(4).as_primitive::(); + let p50_a = checks.column(5).as_primitive::(); + let p50_b = checks.column(6).as_primitive::(); + assert_eq!(group.null_count(), 1); + assert_eq!(total.null_count(), 0); + assert_eq!(hll_a.null_count(), 0); + assert_eq!(hll_b.null_count(), 0); + assert_eq!(stddev.null_count(), 0); + assert_eq!(p50_a.null_count(), 0); + assert_eq!(p50_b.null_count(), 0); + for (index, expected) in [ + (None, 15_u64, 5_u64, 2_f64.sqrt(), 3_f64, 3_f64), + ( + Some(2_i64), + 21_u64, + 3_u64, + (2_f64 / 3.0).sqrt(), + 7_f64, + 7_f64, + ), + (Some(3_i64), 19_u64, 2_u64, 0.5_f64, 10_f64, 10_f64), + ] + .into_iter() + .enumerate() + { + assert_eq!( + (!group.is_null(index)).then(|| group.value(index)), + expected.0 + ); + assert_eq!(total.value(index), expected.1); + assert_eq!(hll_a.value(index), expected.2); + assert_eq!(hll_b.value(index), expected.2); + assert!((stddev.value(index) - expected.3).abs() < 1e-12); + // UDDSketch's relative-error bounds are 1e-6 and 2e-2 respectively. + assert!((p50_a.value(index) - expected.4).abs() <= expected.4 * 0.000001); + assert!((p50_b.value(index) - expected.5).abs() <= expected.5 * 0.02); + } + + let empty_state_plan = sql_to_df_plan( + QueryContext::arc(), + query_engine.clone(), + "SELECT hll(CAST(NULL AS VARCHAR)) AS hll_a, hll(CAST(NULL AS VARCHAR)) AS hll_b, uddsketch_state(128, 0.000001, CAST(NULL AS DOUBLE)) AS percentile_a, uddsketch_state(256, 0.02, CAST(NULL AS DOUBLE)) AS percentile_b, stddev_pop_state(CAST(NULL AS DOUBLE)) AS stddev_state, sum(number) AS total, CASE WHEN number <= 5 THEN CAST(NULL AS BIGINT) WHEN number <= 8 THEN 2 ELSE 3 END AS grp FROM numbers_with_ts WHERE number = 10 GROUP BY grp", + false, + ) + .await + .unwrap(); + let empty_state_analysis = analyze_incremental_aggregate_plan(&empty_state_plan) + .unwrap() + .unwrap(); + assert!(empty_state_analysis.unsupported_exprs.is_empty()); + let empty_state_rewritten = rewrite_incremental_aggregate_with_sink_merge( + &empty_state_plan, + &empty_state_analysis, + &query_engine, + MemTable::table("empty_state_sink", empty_state_sink_batch), + &[ + "greptime".to_string(), + "public".to_string(), + "empty_state_sink".to_string(), + ], + None, + ) + .await + .unwrap(); + let empty_state_output = query_engine + .execute(empty_state_rewritten, QueryContext::arc()) + .await + .unwrap(); + let OutputData::Stream(empty_state_stream) = empty_state_output.data else { + panic!("expected empty state merge to execute as a stream"); + }; + let empty_state_batches = util::collect(empty_state_stream).await.unwrap(); + let empty_state_schema = empty_state_batches.first().unwrap().schema.clone(); + let empty_state_batch = merge_record_batches(empty_state_schema, &empty_state_batches).unwrap(); + assert_eq!(empty_state_batch.num_rows(), 1); + let group_index = empty_state_batch + .schema + .column_index_by_name("grp") + .unwrap(); + let total_index = empty_state_batch + .schema + .column_index_by_name("total") + .unwrap(); + let group = empty_state_batch + .column(group_index) + .as_primitive::(); + let total = empty_state_batch + .column(total_index) + .as_primitive::(); + assert!(!group.is_null(0)); + assert_eq!(group.value(0), 3); + assert!(!total.is_null(0)); + assert_eq!(total.value(0), 10); + let empty_state_table = MemTable::table("empty_states", empty_state_batch); + query_engine + .engine_state() + .catalog_manager() + .as_any() + .downcast_ref::() + .unwrap() + .register_table_sync(RegisterTableRequest { + catalog: "greptime".to_string(), + schema: "public".to_string(), + table_name: "empty_states".to_string(), + table_id: 4098, + table: empty_state_table, + }) + .unwrap(); + let empty_checks_plan = sql_to_df_plan( + QueryContext::arc(), + query_engine.clone(), + "SELECT hll_count(hll_merge(hll_a)) AS hll, stddev_pop_calc(stddev_pop_merge(stddev_state)) AS stddev, uddsketch_calc(0.5, uddsketch_merge(128, 0.000001, percentile_a)) AS p50_a, uddsketch_calc(0.5, uddsketch_merge(256, 0.02, percentile_b)) AS p50_b FROM empty_states", + false, + ) + .await + .unwrap(); + let empty_checks_output = query_engine + .execute(empty_checks_plan, QueryContext::arc()) + .await + .unwrap(); + let OutputData::Stream(empty_checks_stream) = empty_checks_output.data else { + panic!("expected empty state check to execute as a stream"); + }; + let empty_checks_batches = util::collect(empty_checks_stream).await.unwrap(); + let empty_checks_schema = empty_checks_batches.first().unwrap().schema.clone(); + let empty_checks = merge_record_batches(empty_checks_schema, &empty_checks_batches).unwrap(); + assert_eq!(empty_checks.num_rows(), 1); + assert_eq!( + empty_checks.column(0).as_primitive::().value(0), + 0 + ); + assert!( + empty_checks + .column(1) + .as_primitive::() + .is_null(0) + ); + assert!( + empty_checks + .column(2) + .as_primitive::() + .is_null(0) + ); + assert!( + empty_checks + .column(3) + .as_primitive::() + .is_null(0) + ); +} + +#[tokio::test] +async fn test_analyze_incremental_aggregate_plan_state_producer_metadata_and_rejections() { + let analysis = analyze_test_sql( + "SELECT hll(CAST(number AS VARCHAR)) AS hll_state, \ + stddev_pop_state(number) AS stddev_state, \ + uddsketch_state(128, 0.000001, number) AS percentile_state, ts \ + FROM numbers_with_ts GROUP BY ts", + ) + .await; + assert!(analysis.unsupported_exprs.is_empty()); + let percentile = analysis + .merge_columns + .iter() + .find(|column| column.output_field_name == "percentile_state") + .unwrap(); + let IncrementalAggregateMergeOp::StateDeltaMerge { + function_name, + params, + } = &percentile.merge_op + else { + panic!("expected UDDSketch state delta merge"); + }; + assert_eq!(*function_name, "__uddsketch_state_delta_merge"); + assert_eq!(params.len(), 2); + assert!(matches!( + params[0], + Expr::Literal(ScalarValue::Int64(Some(128)), _) + )); + assert!(matches!( + params[1], + Expr::Literal(ScalarValue::Float64(Some(rate)), _) if rate == 0.000001 + )); + + for sql in [ + "SELECT uddsketch_state(CAST(number AS BIGINT), 0.01, number) AS state, ts FROM numbers_with_ts GROUP BY ts", + "SELECT uddsketch_state(128, NULL, number) AS state, ts FROM numbers_with_ts GROUP BY ts", + "SELECT uddsketch_state(NULL, 0.01, number) AS state, ts FROM numbers_with_ts GROUP BY ts", + "SELECT avg(number) AS state, ts FROM numbers_with_ts GROUP BY ts", + "SELECT stddev_pop(number) AS state, ts FROM numbers_with_ts GROUP BY ts", + ] { + let analysis = analyze_test_sql(sql).await; + assert!(!analysis.unsupported_exprs.is_empty(), "must reject {sql}"); + } +} + #[tokio::test] async fn test_analyze_incremental_aggregate_plan_rejects_distinct() { let query_engine = create_test_query_engine(); @@ -1638,6 +2079,7 @@ async fn test_rewrite_incremental_aggregate_with_left_join() { let rewritten = rewrite_incremental_aggregate_with_sink_merge( &plan, &analysis, + &query_engine, sink_table.clone(), &sink_table_name, None, @@ -1699,6 +2141,7 @@ async fn test_rewrite_incremental_aggregate_filters_sink_dirty_time_window() { let rewritten = rewrite_incremental_aggregate_with_sink_merge( &plan, &analysis, + &query_engine, sink_table.clone(), &sink_table_name, Some(sink_filter.clone()), @@ -1749,7 +2192,9 @@ async fn test_rewrite_incremental_aggregate_rejects_empty_group_keys() { let query_engine = create_test_query_engine(); let ctx = QueryContext::arc(); let sql = "SELECT max(number) AS number FROM numbers_with_ts"; - let plan = sql_to_df_plan(ctx, query_engine, sql, false).await.unwrap(); + let plan = sql_to_df_plan(ctx, query_engine.clone(), sql, false) + .await + .unwrap(); let analysis = IncrementalAggregateAnalysis { group_key_names: vec![], merge_columns: vec![IncrementalAggregateMergeColumn::new( @@ -1770,6 +2215,7 @@ async fn test_rewrite_incremental_aggregate_rejects_empty_group_keys() { let err = rewrite_incremental_aggregate_with_sink_merge( &plan, &analysis, + &query_engine, sink_table, &sink_table_name, None, @@ -1788,7 +2234,9 @@ async fn test_rewrite_incremental_aggregate_preserves_raw_aggregate_field_name() let query_engine = create_test_query_engine(); let ctx = QueryContext::arc(); let sql = "SELECT max(number), number FROM numbers_with_ts GROUP BY number"; - let plan = sql_to_df_plan(ctx, query_engine, sql, false).await.unwrap(); + let plan = sql_to_df_plan(ctx, query_engine.clone(), sql, false) + .await + .unwrap(); let analysis = analyze_incremental_aggregate_plan(&plan).unwrap().unwrap(); assert!(analysis.unsupported_exprs.is_empty()); @@ -1802,6 +2250,7 @@ async fn test_rewrite_incremental_aggregate_preserves_raw_aggregate_field_name() let rewritten = rewrite_incremental_aggregate_with_sink_merge( &plan, &analysis, + &query_engine, sink_table.clone(), &sink_table_name, None, diff --git a/src/flow/src/lib.rs b/src/flow/src/lib.rs index acba63f51d3..e9ef90e09d6 100644 --- a/src/flow/src/lib.rs +++ b/src/flow/src/lib.rs @@ -38,7 +38,17 @@ mod test_utils; pub use adapter::flownode_impl::FlowDualEngineRef; pub use adapter::{FlowConfig, FlowStreamingEngineRef, StreamingEngine}; -pub use batching_mode::frontend_client::{FrontendClient, GrpcQueryHandlerWithBoxedError}; +pub use batching_mode::BatchingModeOptions; +pub use batching_mode::batching_execution::{BatchingExecution, BatchingExecutionFactory}; +pub use batching_mode::frontend_client::{ + FrontendClient, GrpcQueryHandlerWithBoxedError, PeerDesc, +}; +pub use batching_mode::task::{ + BatchingExecutionGuard, BatchingTask, DirtyRestore, ExecuteOnceOutcome, PlanInfo, + QueryCoverage, TaskArgs, +}; +pub use batching_mode::time_window::{TimeWindowExpr, find_time_window_expr}; +pub use batching_mode::utils::sql_to_df_plan; pub(crate) use engine::{CreateFlowArgs, FlowId, TableName}; pub use error::{Error, Result}; pub use server::{FlownodeBuilder, FlownodeInstance, FlownodeServer, FlownodeServiceBuilder}; diff --git a/src/flow/src/server.rs b/src/flow/src/server.rs index bf23d868e65..63dc8aefa0a 100644 --- a/src/flow/src/server.rs +++ b/src/flow/src/server.rs @@ -47,7 +47,7 @@ use crate::error::{ use crate::heartbeat::HeartbeatTask; use crate::metrics::{METRIC_FLOW_PROCESSING_TIME, METRIC_FLOW_ROWS}; use crate::utils::{SizeReportSender, StateReportHandler}; -use crate::{Error, FlownodeOptions, FrontendClient, StreamingEngine}; +use crate::{BatchingExecutionFactory, Error, FlownodeOptions, FrontendClient, StreamingEngine}; pub const FLOW_NODE_SERVER_NAME: &str = "FLOW_NODE_SERVER"; /// wrapping flow node manager to avoid orphan rule with Arc<...> @@ -333,13 +333,14 @@ impl FlownodeBuilder { context: "Failed to build query engine", })?; let manager = Arc::new(self.build_manager(query_engine_factory.query_engine())); - let batching = Arc::new(BatchingEngine::new( + let batching = Arc::new(BatchingEngine::new_with_execution( self.frontend_client.clone(), query_engine_factory.query_engine(), self.flow_metadata_manager.clone(), self.table_meta.clone(), self.catalog_manager.clone(), self.opts.flow.batching_mode.clone(), + self.plugins.get::>(), )); let dual = Arc::new(FlowDualEngine::new( manager.clone(), diff --git a/src/operator/src/statement/ddl.rs b/src/operator/src/statement/ddl.rs index 9121d1ef382..2f020c28e68 100644 --- a/src/operator/src/statement/ddl.rs +++ b/src/operator/src/statement/ddl.rs @@ -128,6 +128,7 @@ struct DdlSubmitOptions { timeout: Duration, } +#[cfg(not(feature = "enterprise"))] const ALLOWED_FLOW_OPTIONS: [&str; 2] = [ DEFER_ON_MISSING_SOURCE_KEY, FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY, @@ -172,6 +173,7 @@ fn parse_ddl_options(options: &OptionMap) -> Result { Ok(DdlSubmitOptions { wait, timeout }) } +#[cfg(not(feature = "enterprise"))] fn supported_flow_options() -> String { ALLOWED_FLOW_OPTIONS.join(", ") } @@ -221,6 +223,9 @@ fn validate_and_normalize_flow_options( DEFER_ON_MISSING_SOURCE_KEY | FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY => { normalize_flow_bool_option(&key, &value)? } + #[cfg(feature = "enterprise")] + _ => value, + #[cfg(not(feature = "enterprise"))] _ => { return InvalidSqlSnafu { err_msg: format!( @@ -3284,6 +3289,7 @@ mod test { ); } + #[cfg(not(feature = "enterprise"))] #[test] fn test_validate_and_normalize_flow_options_unknown_option() { let err = validate_and_normalize_flow_options( @@ -3333,7 +3339,7 @@ mod test { } #[test] - fn test_validate_and_normalize_flow_options_rejects_redacted_invalid_input() { + fn test_parser_flow_option_access_key_id() { let sql = r" CREATE FLOW task_6 SINK TO schema_1.table_1 @@ -3351,11 +3357,19 @@ SELECT max(c1), min(c2) FROM schema_2.table_2;"; }; let expr = expr_helper::to_create_flow_task_expr(create_flow, &QueryContext::arc()).unwrap(); - let err = validate_and_normalize_flow_options(expr.flow_options, None).unwrap_err(); - assert!( - err.to_string() - .contains("unknown flow option 'access_key_id'") + #[cfg(not(feature = "enterprise"))] + { + let err = validate_and_normalize_flow_options(expr.flow_options, None).unwrap_err(); + assert!( + err.to_string() + .contains("unknown flow option 'access_key_id'") + ); + } + #[cfg(feature = "enterprise")] + assert_eq!( + validate_and_normalize_flow_options(expr.flow_options, None).unwrap(), + HashMap::from([("access_key_id".to_string(), "['true']".to_string())]) ); } @@ -3448,7 +3462,18 @@ SELECT max(c1), min(c2) FROM schema_2.table_2;"; } #[test] - fn test_schedule_and_internal_keys_rejected_as_unknown_options() { + fn test_internal_schedule_key_rejected() { + let err = validate_and_normalize_flow_options( + HashMap::from([(INTERNAL_EVAL_SCHEDULE_KEY.to_string(), "value".to_string())]), + Some(300), + ) + .unwrap_err(); + assert!(err.to_string().contains("reserved for internal use")); + } + + #[cfg(not(feature = "enterprise"))] + #[test] + fn test_schedule_keys_rejected_as_unknown_options() { for key in [ "eval_interval_anchor", "eval_interval_start", @@ -3461,15 +3486,38 @@ SELECT max(c1), min(c2) FROM schema_2.table_2;"; Some(300), ) .unwrap_err(); - assert!( err.to_string() - .contains(&format!("unknown flow option '{key}'")), - "unexpected error for {key}: {err}" + .contains(&format!("unknown flow option '{key}'")) ); } } + #[cfg(feature = "enterprise")] + #[test] + fn test_schedule_keys_preserved() { + let options = HashMap::from([ + ("eval_interval_anchor".to_string(), "anchor".to_string()), + ("eval_interval_start".to_string(), "start".to_string()), + ( + "eval_interval_missed_tick_policy".to_string(), + "missed".to_string(), + ), + ( + "eval_interval_catchup_max_runs".to_string(), + "runs".to_string(), + ), + ( + "eval_interval_catchup_max_lag".to_string(), + "lag".to_string(), + ), + ]); + assert_eq!( + validate_and_normalize_flow_options(options.clone(), Some(300)).unwrap(), + options + ); + } + #[test] fn test_internal_transport_keys_rejected_as_reserved() { for key in [ diff --git a/src/servers/src/http/test_helpers.rs b/src/servers/src/http/test_helpers.rs index 09296dc10d3..1b59e71691b 100644 --- a/src/servers/src/http/test_helpers.rs +++ b/src/servers/src/http/test_helpers.rs @@ -1,3 +1,17 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + // This file is copied from https://github.com/tokio-rs/axum/blob/axum-v0.6.20/axum/src/test_helpers/test_client.rs //! Axum Test Client diff --git a/src/servers/src/repeated_field.rs b/src/servers/src/repeated_field.rs index 63dd83f94aa..f83d16d086a 100644 --- a/src/servers/src/repeated_field.rs +++ b/src/servers/src/repeated_field.rs @@ -1,22 +1,16 @@ -// Copyright (c) 2019 Stepan Koltsov +// Copyright 2023 Greptime Team // -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at // -// The above copyright notice and this permission notice shall be included in all -// copies or substantial portions of the Software. +// http://www.apache.org/licenses/LICENSE-2.0 // -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, -// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE -// OR OTHER DEALINGS IN THE SOFTWARE. +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. // The Clear trait is copied from https://github.com/stepancheg/rust-protobuf/blob/v2.28.0/protobuf/src/clear.rs // The RepeatedField struct is copied from https://github.com/stepancheg/rust-protobuf/blob/v2.28.0/protobuf/src/repeated.rs diff --git a/tests/cases/standalone/common/flow/flow_incremental_state_merge.result b/tests/cases/standalone/common/flow/flow_incremental_state_merge.result new file mode 100644 index 00000000000..9a9b23dfef3 --- /dev/null +++ b/tests/cases/standalone/common/flow/flow_incremental_state_merge.result @@ -0,0 +1,348 @@ +-- Incremental aggregate state merge over a partitioned source (two regions). +-- A WHERE-filtered flow mixes scalar aggregates (sum, count) with aggregate +-- state columns (hll, stddev_pop_state, uddsketch_state); each later delta in +-- the same time window must merge its state with the state already persisted +-- in the sink. Covers two nonempty deltas across both regions, one round whose +-- delta is empty because the flow's predicate drops the newly inserted row, +-- and a final nonempty delta checked against a direct source aggregation. +CREATE TABLE flow_incr_state_merge_input ( + host_id INT, + user_id BIGINT, + n INT, + v DOUBLE, + ts TIMESTAMP TIME INDEX, + PRIMARY KEY(host_id, user_id) +) +PARTITION ON COLUMNS (host_id) ( + host_id < 3, + host_id >= 3 +) +WITH ( + append_mode = 'true' +); + +Affected Rows: 0 + +CREATE FLOW flow_incr_state_merge SINK TO flow_incr_state_merge_sink +WITH (experimental_enable_incremental_read = 'true') +AS +SELECT + sum(n) AS total, + count(n) AS row_count, + hll(user_id) AS user_hll, + stddev_pop_state(v) AS v_stddev_state, + uddsketch_state(128, 0.01, v) AS v_sketch_state, + date_bin(INTERVAL '1 minute', ts, '2024-01-01 00:00:00') AS time_window +FROM + flow_incr_state_merge_input +WHERE + v > 0 +GROUP BY + time_window; + +Affected Rows: 0 + +-- ==== Phase 1: initial rows in the same time window, one per region ==== +INSERT INTO flow_incr_state_merge_input VALUES + (1, 1, 1, 1.0, '2024-01-01 00:00:00'), + (3, 2, 2, 2.0, '2024-01-01 00:00:10'); + +Affected Rows: 2 + +-- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED | +ADMIN FLUSH_FLOW('flow_incr_state_merge'); + ++-------------------------------------------+ +| ADMIN FLUSH_FLOW('flow_incr_state_merge') | ++-------------------------------------------+ +| FLOW_FLUSHED | ++-------------------------------------------+ + +-- The first round seeds the sink with a full-snapshot aggregate state. +SELECT + total, + row_count, + time_window +FROM + flow_incr_state_merge_sink +ORDER BY + time_window; + ++-------+-----------+---------------------+ +| total | row_count | time_window | ++-------+-----------+---------------------+ +| 3 | 2 | 2024-01-01T00:00:00 | ++-------+-----------+---------------------+ + +-- Move the checkpointed source rows and the seeded sink state into SST, so +-- the next round has to merge a new delta with persisted state. +ADMIN FLUSH_TABLE('flow_incr_state_merge_input'); + ++--------------------------------------------------+ +| ADMIN FLUSH_TABLE('flow_incr_state_merge_input') | ++--------------------------------------------------+ +| 0 | ++--------------------------------------------------+ + +ADMIN FLUSH_TABLE('flow_incr_state_merge_sink'); + ++-------------------------------------------------+ +| ADMIN FLUSH_TABLE('flow_incr_state_merge_sink') | ++-------------------------------------------------+ +| 0 | ++-------------------------------------------------+ + +-- ==== Phase 2: first nonempty delta, same window, both regions ==== +INSERT INTO flow_incr_state_merge_input VALUES + (2, 3, 3, 3.0, '2024-01-01 00:00:20'), + (4, 4, 4, 4.0, '2024-01-01 00:00:40'); + +Affected Rows: 2 + +-- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED | +ADMIN FLUSH_FLOW('flow_incr_state_merge'); + ++-------------------------------------------+ +| ADMIN FLUSH_FLOW('flow_incr_state_merge') | ++-------------------------------------------+ +| FLOW_FLUSHED | ++-------------------------------------------+ + +-- Scalars accumulate; each state column must merge with the persisted state. +SELECT + total, + row_count, + hll_count(hll_merge(user_hll)) AS users, + ROUND(stddev_pop_calc(stddev_pop_merge(v_stddev_state)), 6) AS v_stddev, + ROUND(uddsketch_calc(0.5, uddsketch_merge(128, 0.01, v_sketch_state)), 4) AS v_p50, + time_window +FROM + flow_incr_state_merge_sink +GROUP BY + total, + row_count, + time_window +ORDER BY + time_window; + ++-------+-----------+-------+----------+--------+---------------------+ +| total | row_count | users | v_stddev | v_p50 | time_window | ++-------+-----------+-------+----------+--------+---------------------+ +| 10 | 4 | 4 | 1.118034 | 2.9742 | 2024-01-01T00:00:00 | ++-------+-----------+-------+----------+--------+---------------------+ + +ADMIN FLUSH_TABLE('flow_incr_state_merge_input'); + ++--------------------------------------------------+ +| ADMIN FLUSH_TABLE('flow_incr_state_merge_input') | ++--------------------------------------------------+ +| 0 | ++--------------------------------------------------+ + +ADMIN FLUSH_TABLE('flow_incr_state_merge_sink'); + ++-------------------------------------------------+ +| ADMIN FLUSH_TABLE('flow_incr_state_merge_sink') | ++-------------------------------------------------+ +| 0 | ++-------------------------------------------------+ + +-- ==== Phase 3: filtered-out row must run one round with an empty delta ==== +-- Record the flow's last execution time before the excluded insert: the +-- comparison below proves the empty delta was actually evaluated. +CREATE TABLE flow_incr_state_merge_last_exec ( + marker STRING, + last_exec TIMESTAMP TIME INDEX +) WITH ( + append_mode = 'true' +); + +Affected Rows: 0 + +-- The background loop runs on its own interval; keep waiting until it has +-- executed at least once so information_schema.flows.last_execution_time is +-- non-null before snapshotting it. +-- SQLNESS SLEEP 8s +INSERT INTO flow_incr_state_merge_last_exec (marker, last_exec) +SELECT + 'before_empty_delta', + last_execution_time +FROM + information_schema.flows +WHERE + flow_name = 'flow_incr_state_merge'; + +Affected Rows: 1 + +-- The row's timestamp is inside the existing window, so the source write still +-- dirties that window; the flow's WHERE predicate then drops the row from the +-- delta, leaving the delta empty for the window. +INSERT INTO flow_incr_state_merge_input VALUES + (1, 9, 9, -1.0, '2024-01-01 00:00:30'); + +Affected Rows: 1 + +-- SQLNESS SLEEP 6s +-- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED | +ADMIN FLUSH_FLOW('flow_incr_state_merge'); + ++-------------------------------------------+ +| ADMIN FLUSH_FLOW('flow_incr_state_merge') | ++-------------------------------------------+ +| FLOW_FLUSHED | ++-------------------------------------------+ + +-- An empty delta must leave the persisted sink states untouched. +SELECT + total, + row_count, + hll_count(hll_merge(user_hll)) AS users, + ROUND(stddev_pop_calc(stddev_pop_merge(v_stddev_state)), 6) AS v_stddev, + ROUND(uddsketch_calc(0.5, uddsketch_merge(128, 0.01, v_sketch_state)), 4) AS v_p50, + time_window +FROM + flow_incr_state_merge_sink +GROUP BY + total, + row_count, + time_window +ORDER BY + time_window; + ++-------+-----------+-------+----------+--------+---------------------+ +| total | row_count | users | v_stddev | v_p50 | time_window | ++-------+-----------+-------+----------+--------+---------------------+ +| 10 | 4 | 4 | 1.118034 | 2.9742 | 2024-01-01T00:00:00 | ++-------+-----------+-------+----------+--------+---------------------+ + +-- last_execution_time must have advanced past the mark taken above, proving +-- the dirty window triggered a new round even though its delta was empty. A +-- NULL comparison means the mark never advanced: fail explicitly instead of +-- silently treating it as "unchanged". +INSERT INTO flow_incr_state_merge_last_exec (marker, last_exec) +SELECT + 'after_empty_delta', + last_execution_time +FROM + information_schema.flows +WHERE + flow_name = 'flow_incr_state_merge'; + +Affected Rows: 1 + +SELECT + ( + SELECT + last_exec + FROM + flow_incr_state_merge_last_exec + WHERE + marker = 'after_empty_delta' + ) > ( + SELECT + last_exec + FROM + flow_incr_state_merge_last_exec + WHERE + marker = 'before_empty_delta' + ) AS empty_delta_executed; + ++----------------------+ +| empty_delta_executed | ++----------------------+ +| true | ++----------------------+ + +-- Flush the input before the final delta so each phase's delta is isolated in +-- its own round; the background loop may have already consumed the last +-- insert, which is fine — the values below only depend on the merge outcome. +ADMIN FLUSH_TABLE('flow_incr_state_merge_input'); + ++--------------------------------------------------+ +| ADMIN FLUSH_TABLE('flow_incr_state_merge_input') | ++--------------------------------------------------+ +| 0 | ++--------------------------------------------------+ + +ADMIN FLUSH_TABLE('flow_incr_state_merge_sink'); + ++-------------------------------------------------+ +| ADMIN FLUSH_TABLE('flow_incr_state_merge_sink') | ++-------------------------------------------------+ +| 0 | ++-------------------------------------------------+ + +-- ==== Phase 4: final nonempty delta, same window, both regions ==== +INSERT INTO flow_incr_state_merge_input VALUES + (1, 5, 5, 5.0, '2024-01-01 00:00:05'), + (3, 6, 6, 6.0, '2024-01-01 00:00:55'); + +Affected Rows: 2 + +-- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED | +ADMIN FLUSH_FLOW('flow_incr_state_merge'); + ++-------------------------------------------+ +| ADMIN FLUSH_FLOW('flow_incr_state_merge') | ++-------------------------------------------+ +| FLOW_FLUSHED | ++-------------------------------------------+ + +-- Merge the final delta with the persisted states and compare the merged +-- result against a direct aggregation over the accepted source rows. +SELECT + merged.total = direct.total AS total_matches, + merged.row_count = direct.row_count AS count_matches, + merged.users = direct.users AS hll_matches, + abs(merged.v_stddev - direct.v_stddev) < 1e-9 AS stddev_matches, + ROUND(merged.v_p50, 4) = ROUND(direct.v_p50, 4) AS p50_matches +FROM + ( + SELECT + total, + row_count, + hll_count(hll_merge(user_hll)) AS users, + stddev_pop_calc(stddev_pop_merge(v_stddev_state)) AS v_stddev, + uddsketch_calc(0.5, uddsketch_merge(128, 0.01, v_sketch_state)) AS v_p50 + FROM + flow_incr_state_merge_sink + GROUP BY + total, + row_count + ) AS merged, + ( + SELECT + sum(n) AS total, + count(n) AS row_count, + hll_count(hll(user_id)) AS users, + stddev_pop(v) AS v_stddev, + uddsketch_calc(0.5, uddsketch_state(128, 0.01, v)) AS v_p50 + FROM + flow_incr_state_merge_input + WHERE + v > 0 + ) AS direct; + ++---------------+---------------+-------------+----------------+-------------+ +| total_matches | count_matches | hll_matches | stddev_matches | p50_matches | ++---------------+---------------+-------------+----------------+-------------+ +| true | true | true | true | true | ++---------------+---------------+-------------+----------------+-------------+ + +-- Clean up +DROP FLOW flow_incr_state_merge; + +Affected Rows: 0 + +DROP TABLE flow_incr_state_merge_input; + +Affected Rows: 0 + +DROP TABLE flow_incr_state_merge_sink; + +Affected Rows: 0 + +DROP TABLE flow_incr_state_merge_last_exec; + +Affected Rows: 0 + diff --git a/tests/cases/standalone/common/flow/flow_incremental_state_merge.sql b/tests/cases/standalone/common/flow/flow_incremental_state_merge.sql new file mode 100644 index 00000000000..a48b39e9fd2 --- /dev/null +++ b/tests/cases/standalone/common/flow/flow_incremental_state_merge.sql @@ -0,0 +1,232 @@ +-- Incremental aggregate state merge over a partitioned source (two regions). +-- A WHERE-filtered flow mixes scalar aggregates (sum, count) with aggregate +-- state columns (hll, stddev_pop_state, uddsketch_state); each later delta in +-- the same time window must merge its state with the state already persisted +-- in the sink. Covers two nonempty deltas across both regions, one round whose +-- delta is empty because the flow's predicate drops the newly inserted row, +-- and a final nonempty delta checked against a direct source aggregation. +CREATE TABLE flow_incr_state_merge_input ( + host_id INT, + user_id BIGINT, + n INT, + v DOUBLE, + ts TIMESTAMP TIME INDEX, + PRIMARY KEY(host_id, user_id) +) +PARTITION ON COLUMNS (host_id) ( + host_id < 3, + host_id >= 3 +) +WITH ( + append_mode = 'true' +); + +CREATE FLOW flow_incr_state_merge SINK TO flow_incr_state_merge_sink +WITH (experimental_enable_incremental_read = 'true') +AS +SELECT + sum(n) AS total, + count(n) AS row_count, + hll(user_id) AS user_hll, + stddev_pop_state(v) AS v_stddev_state, + uddsketch_state(128, 0.01, v) AS v_sketch_state, + date_bin(INTERVAL '1 minute', ts, '2024-01-01 00:00:00') AS time_window +FROM + flow_incr_state_merge_input +WHERE + v > 0 +GROUP BY + time_window; + +-- ==== Phase 1: initial rows in the same time window, one per region ==== +INSERT INTO flow_incr_state_merge_input VALUES + (1, 1, 1, 1.0, '2024-01-01 00:00:00'), + (3, 2, 2, 2.0, '2024-01-01 00:00:10'); + +-- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED | +ADMIN FLUSH_FLOW('flow_incr_state_merge'); + +-- The first round seeds the sink with a full-snapshot aggregate state. +SELECT + total, + row_count, + time_window +FROM + flow_incr_state_merge_sink +ORDER BY + time_window; + +-- Move the checkpointed source rows and the seeded sink state into SST, so +-- the next round has to merge a new delta with persisted state. +ADMIN FLUSH_TABLE('flow_incr_state_merge_input'); + +ADMIN FLUSH_TABLE('flow_incr_state_merge_sink'); + +-- ==== Phase 2: first nonempty delta, same window, both regions ==== +INSERT INTO flow_incr_state_merge_input VALUES + (2, 3, 3, 3.0, '2024-01-01 00:00:20'), + (4, 4, 4, 4.0, '2024-01-01 00:00:40'); + +-- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED | +ADMIN FLUSH_FLOW('flow_incr_state_merge'); + +-- Scalars accumulate; each state column must merge with the persisted state. +SELECT + total, + row_count, + hll_count(hll_merge(user_hll)) AS users, + ROUND(stddev_pop_calc(stddev_pop_merge(v_stddev_state)), 6) AS v_stddev, + ROUND(uddsketch_calc(0.5, uddsketch_merge(128, 0.01, v_sketch_state)), 4) AS v_p50, + time_window +FROM + flow_incr_state_merge_sink +GROUP BY + total, + row_count, + time_window +ORDER BY + time_window; + +ADMIN FLUSH_TABLE('flow_incr_state_merge_input'); + +ADMIN FLUSH_TABLE('flow_incr_state_merge_sink'); + +-- ==== Phase 3: filtered-out row must run one round with an empty delta ==== +-- Record the flow's last execution time before the excluded insert: the +-- comparison below proves the empty delta was actually evaluated. +CREATE TABLE flow_incr_state_merge_last_exec ( + marker STRING, + last_exec TIMESTAMP TIME INDEX +) WITH ( + append_mode = 'true' +); + +-- The background loop runs on its own interval; keep waiting until it has +-- executed at least once so information_schema.flows.last_execution_time is +-- non-null before snapshotting it. +-- SQLNESS SLEEP 8s +INSERT INTO flow_incr_state_merge_last_exec (marker, last_exec) +SELECT + 'before_empty_delta', + last_execution_time +FROM + information_schema.flows +WHERE + flow_name = 'flow_incr_state_merge'; + +-- The row's timestamp is inside the existing window, so the source write still +-- dirties that window; the flow's WHERE predicate then drops the row from the +-- delta, leaving the delta empty for the window. +INSERT INTO flow_incr_state_merge_input VALUES + (1, 9, 9, -1.0, '2024-01-01 00:00:30'); + +-- SQLNESS SLEEP 6s +-- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED | +ADMIN FLUSH_FLOW('flow_incr_state_merge'); + +-- An empty delta must leave the persisted sink states untouched. +SELECT + total, + row_count, + hll_count(hll_merge(user_hll)) AS users, + ROUND(stddev_pop_calc(stddev_pop_merge(v_stddev_state)), 6) AS v_stddev, + ROUND(uddsketch_calc(0.5, uddsketch_merge(128, 0.01, v_sketch_state)), 4) AS v_p50, + time_window +FROM + flow_incr_state_merge_sink +GROUP BY + total, + row_count, + time_window +ORDER BY + time_window; + +-- last_execution_time must have advanced past the mark taken above, proving +-- the dirty window triggered a new round even though its delta was empty. A +-- NULL comparison means the mark never advanced: fail explicitly instead of +-- silently treating it as "unchanged". +INSERT INTO flow_incr_state_merge_last_exec (marker, last_exec) +SELECT + 'after_empty_delta', + last_execution_time +FROM + information_schema.flows +WHERE + flow_name = 'flow_incr_state_merge'; + +SELECT + ( + SELECT + last_exec + FROM + flow_incr_state_merge_last_exec + WHERE + marker = 'after_empty_delta' + ) > ( + SELECT + last_exec + FROM + flow_incr_state_merge_last_exec + WHERE + marker = 'before_empty_delta' + ) AS empty_delta_executed; + +-- Flush the input before the final delta so each phase's delta is isolated in +-- its own round; the background loop may have already consumed the last +-- insert, which is fine — the values below only depend on the merge outcome. + +ADMIN FLUSH_TABLE('flow_incr_state_merge_input'); + +ADMIN FLUSH_TABLE('flow_incr_state_merge_sink'); + +-- ==== Phase 4: final nonempty delta, same window, both regions ==== +INSERT INTO flow_incr_state_merge_input VALUES + (1, 5, 5, 5.0, '2024-01-01 00:00:05'), + (3, 6, 6, 6.0, '2024-01-01 00:00:55'); + +-- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED | +ADMIN FLUSH_FLOW('flow_incr_state_merge'); + +-- Merge the final delta with the persisted states and compare the merged +-- result against a direct aggregation over the accepted source rows. +SELECT + merged.total = direct.total AS total_matches, + merged.row_count = direct.row_count AS count_matches, + merged.users = direct.users AS hll_matches, + abs(merged.v_stddev - direct.v_stddev) < 1e-9 AS stddev_matches, + ROUND(merged.v_p50, 4) = ROUND(direct.v_p50, 4) AS p50_matches +FROM + ( + SELECT + total, + row_count, + hll_count(hll_merge(user_hll)) AS users, + stddev_pop_calc(stddev_pop_merge(v_stddev_state)) AS v_stddev, + uddsketch_calc(0.5, uddsketch_merge(128, 0.01, v_sketch_state)) AS v_p50 + FROM + flow_incr_state_merge_sink + GROUP BY + total, + row_count + ) AS merged, + ( + SELECT + sum(n) AS total, + count(n) AS row_count, + hll_count(hll(user_id)) AS users, + stddev_pop(v) AS v_stddev, + uddsketch_calc(0.5, uddsketch_state(128, 0.01, v)) AS v_p50 + FROM + flow_incr_state_merge_input + WHERE + v > 0 + ) AS direct; + +-- Clean up +DROP FLOW flow_incr_state_merge; + +DROP TABLE flow_incr_state_merge_input; + +DROP TABLE flow_incr_state_merge_sink; + +DROP TABLE flow_incr_state_merge_last_exec;