feat(flow): expose extension-owned batching execution hooks (#9171)

* fix: preserve structured query errors through distributed execution

Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com>

* feat: use exact sequence ranges for capable incremental flow sources

Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com>

* test: update SQL expectations for preserved query error codes

Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com>

* style: format exact sequence recovery tests

Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com>

* feat: merge aggregate states in incremental flows

Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com>

* test: exercise dispatched exact delta failure recovery

Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com>

* test: record flushed exact sequence flow results

Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com>

* test: pass query engine to state merge execution tests

Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com>

* feat(flow): expose extension-owned batching execution hooks

Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com>

* fix(flow): correct extension matcher borrowing and regression assertions

Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com>

* test: prove executed empty exact deltas keep incremental mode

Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com>

* test: pass query engine to empty exact delta regression

Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com>

* test: cover aggregate state merge through the full flow path

Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com>

* test: pass query engine to flow batching task regression calls

Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com>

---------

Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com>
This commit is contained in:
discord9
2026-09-18 04:35:29 +00:00
committed by GitHub
parent 20d87cff48
commit 6fa375021f
21 changed files with 2944 additions and 265 deletions
+10
View File
@@ -0,0 +1,10 @@
Auto-merging src/flow/src/batching_mode/frontend_client.rs
Auto-merging src/flow/src/lib.rs
CONFLICT (content): Merge conflict in src/flow/src/lib.rs
Auto-merging src/flow/src/server.rs
CONFLICT (content): Merge conflict in src/flow/src/server.rs
Auto-merging src/operator/src/statement/ddl.rs
Recorded preimage for 'src/flow/src/lib.rs'
Recorded preimage for 'src/flow/src/server.rs'
Automatic merge failed; fix conflicts and then commit the result.
exit=1
+13 -1
View File
@@ -1,4 +1,16 @@
// Copyright (c) 2017-present, PingCAP, Inc. Licensed under Apache-2.0.
// Copyright 2023 Greptime Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// This file is copied from https://github.com/tikv/raft-engine/blob/0.3.0/src/util.rs
+14
View File
@@ -1,3 +1,17 @@
// Copyright 2023 Greptime Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// This file is copied from https://github.com/iqlusioninc/crates/blob/f98d4ccf/secrecy/src/lib.rs
//! [`SecretBox`] wrapper type for more carefully handling secret values
+278 -3
View File
@@ -38,7 +38,10 @@ use crate::ddl::alter_logical_tables::AlterLogicalTablesProcedure;
use crate::ddl::alter_table::{AlterTableProcedure, RegionRouteChanged, only_sets_skip_wal};
use crate::ddl::comment_on::CommentOnProcedure;
use crate::ddl::create_database::{CreateDatabaseMetadataCommitterRef, CreateDatabaseProcedure};
use crate::ddl::create_flow::CreateFlowProcedure;
use crate::ddl::create_flow::{
CreateFlowProcedure, FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY,
FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_SEQUENCE_RANGE,
};
use crate::ddl::create_logical_tables::CreateLogicalTablesProcedure;
use crate::ddl::create_table::CreateTableProcedure;
use crate::ddl::create_view::CreateViewProcedure;
@@ -110,6 +113,10 @@ pub struct DdlManager {
repartition_procedure_factory: RepartitionProcedureFactoryRef,
#[cfg(feature = "enterprise")]
trigger_ddl_manager: Option<TriggerDdlManagerRef>,
#[cfg(feature = "enterprise")]
create_flow_handler: Option<CreateFlowHandlerRef>,
#[cfg(feature = "enterprise")]
drop_flow_handler: Option<DropFlowHandlerRef>,
}
/// This trait is responsible for handling DDL tasks about triggers. e.g.,
@@ -141,6 +148,39 @@ pub trait TriggerDdlManager: Send + Sync {
#[cfg(feature = "enterprise")]
pub type TriggerDdlManagerRef = Arc<dyn TriggerDdlManager>;
/// This trait is responsible for handling custom/extension CREATE FLOW tasks.
#[cfg(feature = "enterprise")]
#[async_trait::async_trait]
pub trait CreateFlowHandler: Send + Sync {
async fn create_flow(
&self,
create_flow_task: CreateFlowTask,
procedure_manager: ProcedureManagerRef,
ddl_context: DdlContext,
query_context: QueryContext,
procedure_context: ProcedureContext,
) -> Result<SubmitDdlTaskResponse>;
}
#[cfg(feature = "enterprise")]
pub type CreateFlowHandlerRef = Arc<dyn CreateFlowHandler>;
/// Hook for classifying and handling DROP FLOW requests.
#[cfg(feature = "enterprise")]
#[async_trait::async_trait]
pub trait DropFlowHandler: Send + Sync {
async fn drop_flow(
&self,
drop_flow_task: DropFlowTask,
procedure_manager: ProcedureManagerRef,
ddl_context: DdlContext,
procedure_context: ProcedureContext,
) -> Result<SubmitDdlTaskResponse>;
}
#[cfg(feature = "enterprise")]
pub type DropFlowHandlerRef = Arc<dyn DropFlowHandler>;
macro_rules! procedure_loader_entry {
($procedure:ident) => {
(
@@ -235,15 +275,31 @@ impl DdlManager {
repartition_procedure_factory,
#[cfg(feature = "enterprise")]
trigger_ddl_manager: None,
#[cfg(feature = "enterprise")]
create_flow_handler: None,
#[cfg(feature = "enterprise")]
drop_flow_handler: None,
}
}
#[cfg(feature = "enterprise")]
pub fn with_drop_flow_handler(mut self, drop_flow_handler: DropFlowHandlerRef) -> Self {
self.drop_flow_handler = Some(drop_flow_handler);
self
}
#[cfg(feature = "enterprise")]
pub fn with_trigger_ddl_manager(mut self, trigger_ddl_manager: TriggerDdlManagerRef) -> Self {
self.trigger_ddl_manager = Some(trigger_ddl_manager);
self
}
#[cfg(feature = "enterprise")]
pub fn with_create_flow_handler(mut self, create_flow_handler: CreateFlowHandlerRef) -> Self {
self.create_flow_handler = Some(create_flow_handler);
self
}
pub fn with_create_database_metadata_committer(
mut self,
committer: CreateDatabaseMetadataCommitterRef,
@@ -965,6 +1021,17 @@ impl DdlManager {
.await
}
DropFlow(drop_flow_task) => {
#[cfg(feature = "enterprise")]
if let Some(handler) = self.drop_flow_handler.as_ref() {
return handler
.drop_flow(
drop_flow_task,
self.procedure_manager.clone(),
self.ddl_context.clone(),
procedure_context,
)
.await;
}
handle_drop_flow_task(self, drop_flow_task, procedure_context).await
}
CreateView(create_view_task) => {
@@ -1365,6 +1432,32 @@ async fn handle_create_flow_task(
query_context: QueryContext,
procedure_context: ProcedureContext,
) -> Result<SubmitDdlTaskResponse> {
if create_flow_task
.flow_options
.get(FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY)
.is_some_and(|value| value == FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_SEQUENCE_RANGE)
{
return error::UnexpectedSnafu {
err_msg: format!(
"reserved flow option value for {FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY} is internal"
),
}
.fail();
}
#[cfg(feature = "enterprise")]
if let Some(handler) = ddl_manager.create_flow_handler.as_ref() {
return handler
.create_flow(
create_flow_task,
ddl_manager.procedure_manager.clone(),
ddl_manager.ddl_context.clone(),
query_context,
procedure_context,
)
.await;
}
let (id, output) = ddl_manager
.submit_create_flow_task(create_flow_task.clone(), query_context, procedure_context)
.await?;
@@ -1538,6 +1631,11 @@ mod tests {
use crate::ddl::create_database::{
AtomicCreateOutcome, CreateDatabaseMetadataCommitter, CreateDatabaseProcedure,
};
#[cfg(feature = "enterprise")]
use crate::ddl::create_flow::{
CreateFlowProcedure, FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY,
FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_SEQUENCE_RANGE,
};
use crate::ddl::create_table::CreateTableProcedure;
use crate::ddl::drop_table::DropTableProcedure;
use crate::ddl::flow_meta::FlowMetadataAllocator;
@@ -1555,11 +1653,13 @@ mod tests {
use crate::region_registry::LeaderRegionRegistry;
#[cfg(feature = "enterprise")]
use crate::rpc::ddl::trigger::{CreateTriggerTask, DropTriggerTask};
#[cfg(feature = "enterprise")]
use crate::rpc::ddl::{
CreateFlowTask, DdlTask, QueryContext, SubmitDdlTaskRequest, SubmitDdlTaskResponse,
};
use crate::rpc::ddl::{CreatorGrantIntent, UndropTableTask};
#[cfg(not(feature = "enterprise"))]
use crate::rpc::ddl::{DdlTask, PurgeDroppedTableTask, QueryContext, SubmitDdlTaskRequest};
#[cfg(feature = "enterprise")]
use crate::rpc::ddl::{DdlTask, QueryContext, SubmitDdlTaskRequest};
use crate::sequence::SequenceBuilder;
use crate::state_store::KvStateStore;
use crate::test_util::{MockDatanodeManager, new_ddl_context};
@@ -1626,6 +1726,60 @@ mod tests {
}
}
#[cfg(feature = "enterprise")]
#[derive(Default)]
struct RecordingCreateFlowHandler {
tasks: Mutex<Vec<CreateFlowTask>>,
contexts: Mutex<Vec<(QueryContext, ProcedureContext)>>,
response: Mutex<Option<SubmitDdlTaskResponse>>,
fail: bool,
}
#[cfg(feature = "enterprise")]
#[async_trait::async_trait]
impl super::CreateFlowHandler for RecordingCreateFlowHandler {
async fn create_flow(
&self,
create_flow_task: CreateFlowTask,
_procedure_manager: ProcedureManagerRef,
_ddl_context: DdlContext,
query_context: QueryContext,
procedure_context: ProcedureContext,
) -> crate::error::Result<SubmitDdlTaskResponse> {
self.tasks.lock().unwrap().push(create_flow_task);
self.contexts
.lock()
.unwrap()
.push((query_context, procedure_context));
if self.fail {
return crate::error::UnsupportedSnafu {
operation: "test create flow handler",
}
.fail();
}
Ok(self.response.lock().unwrap().take().unwrap_or_default())
}
}
#[cfg(feature = "enterprise")]
fn test_create_flow_task() -> CreateFlowTask {
CreateFlowTask {
catalog_name: "greptime".to_string(),
flow_name: "test_flow".to_string(),
source_table_names: vec![],
sink_table_name: TableName::new("greptime", "public", "sink"),
or_replace: false,
create_if_not_exists: false,
expire_after: None,
eval_interval_secs: None,
eval_offset_secs: None,
comment: String::new(),
sql: "select 1".to_string(),
flow_options: Default::default(),
eval_schedule: None,
}
}
#[cfg(feature = "enterprise")]
#[derive(Default)]
struct RecordingTriggerDdlManager {
@@ -1805,6 +1959,127 @@ mod tests {
)
}
#[cfg(feature = "enterprise")]
#[tokio::test]
async fn test_reserved_sequence_range_is_rejected_before_custom_handler() {
let handler = Arc::new(RecordingCreateFlowHandler::default());
let ddl_manager =
build_soft_drop_test_ddl_manager().with_create_flow_handler(handler.clone());
let mut task = test_create_flow_task();
task.flow_options.insert(
FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY.to_string(),
FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_SEQUENCE_RANGE.to_string(),
);
let result = ddl_manager
.submit_ddl_task(
ExecutorContext {
query_context: Some(QueryContext::default()),
..Default::default()
},
SubmitDdlTaskRequest::new(DdlTask::new_create_flow(task)),
)
.await;
assert!(result.is_err());
assert!(handler.tasks.lock().unwrap().is_empty());
}
#[cfg(feature = "enterprise")]
#[tokio::test]
async fn test_create_flow_handler_dispatches_without_procedure() {
let response = SubmitDdlTaskResponse {
key: b"custom-handler".to_vec(),
..Default::default()
};
let handler = Arc::new(RecordingCreateFlowHandler {
response: Mutex::new(Some(response)),
..Default::default()
});
let ddl_manager =
build_soft_drop_test_ddl_manager().with_create_flow_handler(handler.clone());
let procedure_context = ProcedureContext {
actor: Some("test-user".to_string()),
event_context: Some(
PersistentEventContext::new(TriggerReason::Manual).with_protocol("mysql"),
),
};
let query_context = QueryContext {
channel: Channel::Mysql as u8,
..Default::default()
};
let actual = ddl_manager
.submit_ddl_task(
ExecutorContext {
query_context: Some(query_context.clone()),
actor: procedure_context.actor.clone(),
event_input: Some(ProcedureEventInput::new(TriggerReason::Manual)),
..Default::default()
},
SubmitDdlTaskRequest::new(DdlTask::new_create_flow(test_create_flow_task())),
)
.await
.unwrap();
assert_eq!(actual.key, b"custom-handler");
assert_eq!(handler.tasks.lock().unwrap().len(), 1);
assert_eq!(
handler.contexts.lock().unwrap().as_slice(),
&[(query_context, procedure_context)]
);
}
#[cfg(feature = "enterprise")]
#[tokio::test]
async fn test_create_flow_handler_error_does_not_fallback() {
let handler = Arc::new(RecordingCreateFlowHandler {
fail: true,
..Default::default()
});
let ddl_manager =
build_soft_drop_test_ddl_manager().with_create_flow_handler(handler.clone());
let err = ddl_manager
.submit_ddl_task(
ExecutorContext {
query_context: Some(QueryContext::default()),
..Default::default()
},
SubmitDdlTaskRequest::new(DdlTask::new_create_flow(test_create_flow_task())),
)
.await
.unwrap_err();
assert!(err.to_string().contains("test create flow handler"));
assert_eq!(handler.tasks.lock().unwrap().len(), 1);
}
#[cfg(feature = "enterprise")]
#[tokio::test]
async fn test_create_flow_without_handler_uses_ordinary_procedure() {
let ddl_manager = build_soft_drop_test_ddl_manager();
ddl_manager.procedure_manager.start().await.unwrap();
let err = ddl_manager
.submit_ddl_task(
ExecutorContext {
query_context: Some(QueryContext::default()),
..Default::default()
},
SubmitDdlTaskRequest::new(DdlTask::new_create_flow(test_create_flow_task())),
)
.await
.unwrap_err();
assert!(!err.to_string().contains("test create flow handler"));
assert_eq!(
ddl_manager
.procedure_manager
.list_procedures()
.await
.unwrap()
.iter()
.filter(|procedure| procedure.type_name == CreateFlowProcedure::TYPE_NAME)
.count(),
1
);
}
#[cfg(feature = "enterprise")]
#[tokio::test]
async fn test_trigger_ddl_forwards_procedure_context() {
+5 -4
View File
@@ -20,14 +20,15 @@ use common_grpc::channel_manager::ClientTlsOption;
use serde::{Deserialize, Serialize};
use session::ReadPreference;
mod checkpoint;
pub(crate) mod batching_execution;
pub(crate) mod checkpoint;
pub(crate) mod engine;
mod eval_schedule;
pub(crate) mod frontend_client;
mod state;
pub(crate) mod state;
mod table_creator;
mod task;
mod time_window;
pub(crate) mod task;
pub(crate) mod time_window;
pub(crate) mod utils;
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
@@ -0,0 +1,54 @@
// Copyright 2023 Greptime Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Optional execution collaborator for batching tasks.
use std::sync::Arc;
use query::QueryEngineRef;
use table::TableRef;
use crate::Result;
use crate::batching_mode::frontend_client::FrontendClient;
use crate::batching_mode::task::{BatchingExecutionGuard, BatchingTask, ExecuteOnceOutcome};
#[async_trait::async_trait]
pub trait BatchingExecution: Send + Sync + 'static {
/// Execute one round while retaining the guard through all task-state updates.
/// An implementation that continues after caller cancellation must retain the
/// guard with that work and make it stoppable through [`Self::stop`].
async fn execute_once(
self: Arc<Self>,
guard: BatchingExecutionGuard,
task: &BatchingTask,
engine: &QueryEngineRef,
frontend: &Arc<FrontendClient>,
max_window_cnt: Option<usize>,
) -> ExecuteOnceOutcome;
/// Retire this execution instance, rejecting new work and requesting that any
/// retained local work stop. This is not an acknowledgement of remote quiescence.
fn stop(&self) {}
}
#[async_trait::async_trait]
pub trait BatchingExecutionFactory: Send + Sync + 'static {
async fn create(
&self,
task: &BatchingTask,
sink: TableRef,
engine: &QueryEngineRef,
frontend: &Arc<FrontendClient>,
) -> Result<Option<Arc<dyn BatchingExecution>>>;
}
+375 -8
View File
@@ -75,6 +75,7 @@ pub struct BatchingEngine {
/// Batching mode options for control how batching mode query works
///
pub(crate) batch_opts: Arc<BatchingModeOptions>,
execution_factory: Option<Arc<dyn crate::BatchingExecutionFactory>>,
}
#[derive(Default)]
@@ -130,6 +131,26 @@ impl BatchingEngine {
table_meta: TableMetadataManagerRef,
catalog_manager: CatalogManagerRef,
batch_opts: BatchingModeOptions,
) -> Self {
Self::new_with_execution(
frontend_client,
query_engine,
flow_metadata_manager,
table_meta,
catalog_manager,
batch_opts,
None,
)
}
pub fn new_with_execution(
frontend_client: Arc<FrontendClient>,
query_engine: QueryEngineRef,
flow_metadata_manager: FlowMetadataManagerRef,
table_meta: TableMetadataManagerRef,
catalog_manager: CatalogManagerRef,
batch_opts: BatchingModeOptions,
execution_factory: Option<Arc<dyn crate::BatchingExecutionFactory>>,
) -> Self {
Self {
runtime: Default::default(),
@@ -139,6 +160,7 @@ impl BatchingEngine {
catalog_manager,
query_engine,
batch_opts: Arc::new(batch_opts),
execution_factory,
}
}
@@ -726,15 +748,22 @@ impl BatchingEngine {
);
}
let task_inner = task.clone();
let engine = self.query_engine.clone();
let frontend = self.frontend_client.clone();
// Create sink table if needed, then validate an existing/created sink schema before
// spawning the background task. This catches user-created sink schema mismatches at
// CREATE FLOW time instead of surfacing them later in the execution loop.
task.check_or_create_sink_table(&engine, &frontend).await?;
task.validate_sink_table_schema(&engine).await?;
// Create the sink before asking an optional extension to own execution.
// The default path retains the OSS schema validation.
let table = task.check_or_create_sink_table(&engine, &frontend).await?;
let execution = if let Some(factory) = &self.execution_factory {
factory.create(&task, table, &engine, &frontend).await?
} else {
None
};
if execution.is_none() {
task.validate_sink_table_schema(&engine).await?;
}
let task = task.with_execution(execution);
let task_inner = task.clone();
let (start_tx, start_rx) = oneshot::channel();
@@ -1018,6 +1047,8 @@ fn abort_flow_task(flow_id: FlowId, task: Option<BatchingTask>, action: &str) ->
return false;
};
task.stop_execution();
if let Some(handle) = task.state.write().unwrap().task_handle.take() {
handle.abort();
debug!("Aborted {action} flow task {flow_id}");
@@ -1067,17 +1098,23 @@ impl FlowEngine for BatchingEngine {
#[cfg(test)]
mod tests {
use api::v1::flow::{DirtyWindowRequest, TimeRange};
use catalog::memory::new_memory_catalog_manager;
use catalog::RegisterTableRequest;
use catalog::memory::{MemoryCatalogManager, new_memory_catalog_manager};
use common_meta::key::TableMetadataManager;
use common_meta::key::flow::FlowMetadataManager;
use common_meta::key::table_route::TableRouteValue;
use common_meta::key::test_utils::new_test_table_info_with_name;
use common_meta::kv_backend::memory::MemoryKvBackend;
use common_recordbatch::RecordBatch;
use common_time::timestamp::TimeUnit;
use datatypes::schema::{ColumnSchema, Schema};
use datatypes::vectors::{TimestampMillisecondVector, UInt32Vector, VectorRef};
use query::options::QueryOptions;
use session::context::QueryContext;
use tokio::sync::Notify;
use super::*;
use crate::ExecuteOnceOutcome;
use crate::test_utils::create_test_query_engine;
struct DropNotify(Option<oneshot::Sender<()>>);
@@ -1110,6 +1147,317 @@ mod tests {
)
}
async fn new_test_engine_with_execution(
execution_factory: Option<Arc<dyn crate::BatchingExecutionFactory>>,
) -> BatchingEngine {
let kv_backend = Arc::new(MemoryKvBackend::new());
let table_meta = Arc::new(TableMetadataManager::new(kv_backend.clone()));
table_meta.init().await.unwrap();
let flow_meta = Arc::new(FlowMetadataManager::new(kv_backend));
let query_engine = create_test_query_engine();
let catalog_manager = query_engine.engine_state().catalog_manager().clone();
let (frontend_client, _handler) =
FrontendClient::from_empty_grpc_handler(QueryOptions::default());
let engine = BatchingEngine::new_with_execution(
Arc::new(frontend_client),
query_engine,
flow_meta,
table_meta,
catalog_manager,
BatchingModeOptions::default(),
execution_factory,
);
engine
.table_meta
.create_table_metadata(
new_test_table_info_with_name(1, "numbers_with_ts"),
TableRouteValue::physical(vec![]),
HashMap::new(),
)
.await
.unwrap();
engine
}
fn register_sink_with_schema(engine: &BatchingEngine, name: &str) {
let schema = Arc::new(Schema::new(vec![
ColumnSchema::new("number", ConcreteDataType::uint32_datatype(), false),
ColumnSchema::new(
"ts",
ConcreteDataType::timestamp_millisecond_datatype(),
false,
)
.with_time_index(true),
]));
let recordbatch = RecordBatch::new(
schema,
vec![
Arc::new(UInt32Vector::from_slice([1_u32])) as VectorRef,
Arc::new(TimestampMillisecondVector::from_slice([0_i64])) as VectorRef,
],
)
.unwrap();
engine
.catalog_manager
.as_any()
.downcast_ref::<MemoryCatalogManager>()
.unwrap()
.register_table_sync(RegisterTableRequest {
catalog: "greptime".to_string(),
schema: "public".to_string(),
table_name: name.to_string(),
table_id: 9000,
table: table::test_util::MemTable::table(name, recordbatch),
})
.unwrap();
}
fn register_number_only_sink(engine: &BatchingEngine, name: &str) {
let schema = Arc::new(Schema::new(vec![ColumnSchema::new(
"number",
ConcreteDataType::uint32_datatype(),
false,
)]));
let recordbatch = RecordBatch::new(
schema,
vec![Arc::new(UInt32Vector::from_slice([1_u32])) as VectorRef],
)
.unwrap();
engine
.catalog_manager
.as_any()
.downcast_ref::<MemoryCatalogManager>()
.unwrap()
.register_table_sync(RegisterTableRequest {
catalog: "greptime".to_string(),
schema: "public".to_string(),
table_name: name.to_string(),
table_id: 9001,
table: table::test_util::MemTable::table(name, recordbatch),
})
.unwrap();
}
fn flow_create_args(flow_id: FlowId, sink: &str) -> CreateFlowArgs {
CreateFlowArgs {
flow_id,
sink_table_name: [
"greptime".to_string(),
"public".to_string(),
sink.to_string(),
],
source_table_ids: vec![1],
create_if_not_exists: false,
or_replace: false,
expire_after: None,
eval_interval: Some(10),
comment: None,
sql: "SELECT number, ts FROM numbers_with_ts".to_string(),
flow_options: HashMap::new(),
query_ctx: Some(QueryContext::arc().as_ref().clone()),
eval_schedule: None,
}
}
struct TestExecution {
manual_calls: std::sync::atomic::AtomicUsize,
stops: std::sync::atomic::AtomicUsize,
}
#[async_trait::async_trait]
impl crate::BatchingExecution for TestExecution {
async fn execute_once(
self: Arc<Self>,
_guard: crate::BatchingExecutionGuard,
task: &BatchingTask,
_engine: &QueryEngineRef,
_frontend: &Arc<FrontendClient>,
_max_window_cnt: Option<usize>,
) -> ExecuteOnceOutcome {
if task
.state
.read()
.unwrap()
.query_ctx
.extension(query::options::FLOW_SCHEDULED_TIME_MILLIS)
.is_none()
{
self.manual_calls
.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
}
ExecuteOnceOutcome {
new_query: None,
result: Ok(None),
}
}
fn stop(&self) {
self.stops.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
}
}
type TestExecutionResult = crate::Result<Option<Arc<dyn crate::BatchingExecution>>>;
struct TestExecutionFactory {
entered: Option<Arc<Notify>>,
release: Option<Arc<Notify>>,
result: std::sync::Mutex<Option<TestExecutionResult>>,
}
#[async_trait::async_trait]
impl crate::BatchingExecutionFactory for TestExecutionFactory {
async fn create(
&self,
_task: &BatchingTask,
_sink: table::TableRef,
_engine: &QueryEngineRef,
_frontend: &Arc<FrontendClient>,
) -> crate::Result<Option<Arc<dyn crate::BatchingExecution>>> {
if let Some(entered) = &self.entered {
entered.notify_one();
}
if let Some(release) = &self.release {
release.notified().await;
}
self.result
.lock()
.unwrap()
.take()
.expect("execution factory should only be called once")
}
}
#[tokio::test]
async fn test_execution_factory_finishes_before_task_publication() {
const GATE_TIMEOUT: Duration = Duration::from_secs(1);
let execution = Arc::new(TestExecution {
manual_calls: Default::default(),
stops: Default::default(),
});
let entered = Arc::new(Notify::new());
let release = Arc::new(Notify::new());
let factory = Arc::new(TestExecutionFactory {
entered: Some(entered.clone()),
release: Some(release.clone()),
result: std::sync::Mutex::new(Some(Ok(Some(execution.clone())))),
});
let engine = Arc::new(new_test_engine_with_execution(Some(factory)).await);
register_sink_with_schema(&engine, "factory_sink");
let entered_wait = entered.notified();
let mut args = flow_create_args(6, "factory_sink");
args.eval_interval = Some(86_400);
let mut create = tokio::spawn({
let engine = engine.clone();
async move { engine.create_flow_inner(args).await }
});
if tokio::time::timeout(GATE_TIMEOUT, entered_wait)
.await
.is_err()
{
release.notify_one();
if tokio::time::timeout(GATE_TIMEOUT, &mut create)
.await
.is_err()
{
create.abort();
}
panic!("execution factory should be entered");
}
let unpublished = {
let runtime = engine.runtime.read().await;
!runtime.tasks.contains_key(&6) && !runtime.shutdown_txs.contains_key(&6)
};
release.notify_one();
assert!(unpublished);
let created = tokio::time::timeout(GATE_TIMEOUT, &mut create)
.await
.expect("flow creation should finish after factory release");
created.unwrap().unwrap();
assert!(engine.flow_exist_inner(6).await);
let calls_before_flush = execution
.manual_calls
.load(std::sync::atomic::Ordering::SeqCst);
assert_eq!(engine.flush_flow_inner(6).await.unwrap(), 0);
assert_eq!(
execution
.manual_calls
.load(std::sync::atomic::Ordering::SeqCst),
calls_before_flush + 1
);
engine.remove_flow_inner(6).await.unwrap();
}
#[tokio::test]
async fn test_execution_factory_result_controls_sink_validation_and_publication() {
let no_factory_engine = new_test_engine_with_execution(None).await;
register_number_only_sink(&no_factory_engine, "no_factory_sink");
assert!(
no_factory_engine
.create_flow_inner(flow_create_args(7, "no_factory_sink"))
.await
.is_err()
);
assert!(!no_factory_engine.flow_exist_inner(7).await);
let declined_engine =
new_test_engine_with_execution(Some(Arc::new(TestExecutionFactory {
entered: None,
release: None,
result: std::sync::Mutex::new(Some(Ok(None))),
})))
.await;
register_number_only_sink(&declined_engine, "declined_factory_sink");
assert!(
declined_engine
.create_flow_inner(flow_create_args(8, "declined_factory_sink"))
.await
.is_err()
);
assert!(!declined_engine.flow_exist_inner(8).await);
let accepted_engine =
new_test_engine_with_execution(Some(Arc::new(TestExecutionFactory {
entered: None,
release: None,
result: std::sync::Mutex::new(Some(Ok(Some(Arc::new(TestExecution {
manual_calls: Default::default(),
stops: Default::default(),
}))))),
})))
.await;
register_number_only_sink(&accepted_engine, "accepted_factory_sink");
accepted_engine
.create_flow_inner(flow_create_args(9, "accepted_factory_sink"))
.await
.unwrap();
assert!(accepted_engine.flow_exist_inner(9).await);
accepted_engine.remove_flow_inner(9).await.unwrap();
let error_engine = new_test_engine_with_execution(Some(Arc::new(TestExecutionFactory {
entered: None,
release: None,
result: std::sync::Mutex::new(Some(
UnexpectedSnafu {
reason: "test execution factory failure".to_string(),
}
.fail(),
)),
})))
.await;
register_sink_with_schema(&error_engine, "error_factory_sink");
assert!(
error_engine
.create_flow_inner(flow_create_args(10, "error_factory_sink"))
.await
.is_err()
);
assert!(!error_engine.flow_exist_inner(10).await);
}
#[tokio::test]
async fn test_flow_option_overrides_incremental_read_switch() {
let engine = new_test_engine().await;
@@ -1583,11 +1931,30 @@ GROUP BY l.number, time_window
}
#[tokio::test]
async fn test_abort_flow_task_aborts_handle() {
async fn test_abort_flow_task_stops_execution_without_loop_handle() {
let (task, _shutdown_tx) = new_test_task(42).await;
let execution = Arc::new(TestExecution {
manual_calls: Default::default(),
stops: Default::default(),
});
let task = task.with_execution(Some(execution.clone()));
assert!(!abort_flow_task(42, Some(task), "test"));
assert_eq!(execution.stops.load(std::sync::atomic::Ordering::SeqCst), 1);
}
#[tokio::test]
async fn test_abort_flow_task_stops_execution_before_aborting_handle() {
let (task, _shutdown_tx) = new_test_task(42).await;
let execution = Arc::new(TestExecution {
manual_calls: Default::default(),
stops: Default::default(),
});
let task = task.with_execution(Some(execution.clone()));
let drop_rx = install_abort_observed_handle(&task).await;
assert!(abort_flow_task(42, Some(task), "test"));
assert_eq!(execution.stops.load(std::sync::atomic::Ordering::SeqCst), 1);
tokio::time::timeout(Duration::from_secs(1), drop_rx)
.await
@@ -749,7 +749,7 @@ fn wrap_standalone_output_with_terminal_metrics(
/// Describe a peer of frontend
#[derive(Debug, Default, Clone)]
pub(crate) enum PeerDesc {
pub enum PeerDesc {
/// The query failed before a frontend peer was selected.
#[default]
Unknown,
+14
View File
@@ -131,6 +131,11 @@ impl TaskState {
&self.checkpoints
}
/// Read the live dirty-window queue without exposing TaskState internals to execution owners.
pub fn dirty_time_windows(&self) -> &DirtyTimeWindows {
&self.dirty_time_windows
}
/// Returns the in-progress fenced repair, if the task is repairing dirty
/// windows under a frozen full-snapshot high watermark.
pub fn pending_fenced_repair(&self) -> Option<&FencedRepair> {
@@ -546,6 +551,15 @@ impl DirtyTimeWindows {
}
}
/// Detach all dirty windows while retaining this queue's configured limits.
pub(crate) fn detach(&mut self) -> Self {
Self {
windows: std::mem::take(&mut self.windows),
max_filter_num_per_query: self.max_filter_num_per_query,
time_window_merge_threshold: self.time_window_merge_threshold,
}
}
/// Clean all dirty time windows, useful when can't found time window expr
pub fn clean(&mut self) {
self.windows.clear();
+253 -89
View File
@@ -18,6 +18,7 @@ use std::time::{Duration, SystemTime, UNIX_EPOCH};
use api::v1::{CreateTableExpr, TableName};
use catalog::CatalogManagerRef;
use client::OutputWithMetrics;
use common_error::ext::BoxedError;
use common_query::logical_plan::breakup_insert_plan;
use common_telemetry::tracing::warn;
@@ -27,7 +28,7 @@ use datafusion::datasource::DefaultTableSource;
use datafusion::sql::unparser::expr_to_sql;
use datafusion_common::tree_node::{Transformed, TreeNode};
use datafusion_common::utils::quote_identifier;
use datafusion_common::{DFSchemaRef, TableReference};
use datafusion_common::{DFSchemaRef, ScalarValue, TableReference};
use datafusion_expr::{DmlStatement, LogicalPlan, WriteOp, col, lit};
use datatypes::schema::Schema;
use query::QueryEngineRef;
@@ -38,9 +39,10 @@ use snafu::{OptionExt, ResultExt};
use sql::parsers::utils::is_tql;
use store_api::mito_engine_options::MERGE_MODE_KEY;
use substrait::{DFLogicalSubstraitConvertor, SubstraitPlan};
use table::TableRef;
use table::table::adapter::DfTableProviderAdapter;
use tokio::sync::oneshot::error::TryRecvError;
use tokio::sync::{Mutex, oneshot};
use tokio::sync::{Mutex, OwnedMutexGuard, oneshot};
use tokio::time::Instant;
use crate::batching_mode::BatchingModeOptions;
@@ -53,8 +55,8 @@ use crate::batching_mode::state::{
use crate::batching_mode::table_creator::{QueryType, create_table_with_expr};
use crate::batching_mode::time_window::TimeWindowExpr;
use crate::batching_mode::utils::{
AddFilterRewriter, ColumnMatcherRewriter, df_plan_to_sql, gen_plan_with_matching_schema,
get_table_info_df_schema, sql_to_df_plan,
AddFilterRewriter, ColumnMatcherRewriter, df_plan_to_sql,
gen_plan_with_matching_schema_and_values, get_table_info_df_schema, sql_to_df_plan,
};
use crate::df_optimizer::apply_df_optimizer;
use crate::error::{
@@ -199,6 +201,36 @@ fn format_insert_target_columns(plan: &LogicalPlan) -> String {
.join(", ")
}
/// Owns a whole serialized execution round. It may be moved to an execution
/// collaborator, so cancellation of the caller cannot release the round early.
pub struct BatchingExecutionGuard {
_lock: OwnedMutexGuard<()>,
restore: Option<(Arc<RwLock<TaskState>>, QueryContextRef)>,
}
impl BatchingExecutionGuard {
fn new(lock: OwnedMutexGuard<()>) -> Self {
Self {
_lock: lock,
restore: None,
}
}
fn restore_query_context(&mut self, state: Arc<RwLock<TaskState>>, old_ctx: QueryContextRef) {
self.restore = Some((state, old_ctx));
}
}
impl Drop for BatchingExecutionGuard {
fn drop(&mut self) {
if let Some((state, old_ctx)) = self.restore.take()
&& let Ok(mut state) = state.write()
{
state.query_ctx = old_ctx;
}
}
}
#[derive(Clone)]
pub struct BatchingTask {
pub config: Arc<TaskConfig>,
@@ -207,6 +239,7 @@ pub struct BatchingTask {
/// window restoration for this flow. Without this, a manual flush and the
/// background loop can process the same checkpoint range concurrently.
execution_lock: Arc<Mutex<()>>,
execution: Option<Arc<dyn crate::BatchingExecution>>,
}
/// Arguments for creating batching task
@@ -281,14 +314,14 @@ pub enum DirtyRestore {
Unscoped(DirtyTimeWindows),
}
struct ExecuteOnceOutcome {
new_query: Option<PlanInfo>,
pub struct ExecuteOnceOutcome {
pub new_query: Option<PlanInfo>,
/// Execution result of the generated insert plan.
///
/// `Ok(Some((affected_rows, elapsed)))` means a query was executed.
/// `Ok(None)` means no query was generated because there was no dirty signal.
/// `Err(_)` means plan generation or execution failed.
result: Result<Option<(usize, Duration)>, Error>,
pub result: Result<Option<(usize, Duration)>, Error>,
}
impl BatchingTask {
@@ -345,9 +378,24 @@ impl BatchingTask {
}),
state: Arc::new(RwLock::new(state)),
execution_lock: Arc::new(Mutex::new(())),
execution: None,
})
}
pub(crate) fn with_execution(
mut self,
execution: Option<Arc<dyn crate::BatchingExecution>>,
) -> Self {
self.execution = execution;
self
}
pub(crate) fn stop_execution(&self) {
if let Some(execution) = &self.execution {
execution.stop();
}
}
pub fn last_execution_time_millis(&self) -> Option<i64> {
self.state.read().unwrap().last_execution_time_millis()
}
@@ -403,12 +451,12 @@ impl BatchingTask {
Ok(())
}
/// Create sink table if not exists
/// Create the sink table if needed, then return it.
pub async fn check_or_create_sink_table(
&self,
engine: &QueryEngineRef,
frontend_client: &Arc<FrontendClient>,
) -> Result<Option<(usize, Duration)>, Error> {
) -> Result<TableRef, Error> {
if !self.is_table_exist(&self.config.sink_table_name).await? {
let create_table = self.gen_create_table_expr(engine.clone()).await?;
info!(
@@ -421,16 +469,32 @@ impl BatchingTask {
self.config.sink_table_name.join(".")
);
}
Ok(None)
let (table, _) = get_table_info_df_schema(
self.config.catalog_manager.clone(),
self.config.sink_table_name.clone(),
)
.await?;
Ok(table)
}
/// Validates that the sink table schema can accept this flow's output.
///
/// This is a dry-run of the same schema matching logic used by insert-plan
/// generation, but without adding dirty-window filters or executing the query. It is used
/// during CREATE FLOW to catch existing sink table mismatches early.
/// Returns a forked snapshot of the task query context for extension-owned planning.
pub fn query_context_snapshot(&self) -> QueryContextRef {
let query_ctx = self.state.read().unwrap().query_ctx.clone();
Arc::new(query_ctx.fork())
}
/// Validates that the sink table schema can accept this flow's ordinary output.
pub async fn validate_sink_table_schema(&self, engine: &QueryEngineRef) -> Result<(), Error> {
self.validate_sink_table_schema_with_values(engine, &BTreeMap::new())
.await
}
/// Validates the flow output plus extension-owned typed sink values without consuming dirty work.
pub async fn validate_sink_table_schema_with_values(
&self,
engine: &QueryEngineRef,
values: &BTreeMap<String, ScalarValue>,
) -> Result<(), Error> {
let (table, _) = get_table_info_df_schema(
self.config.catalog_manager.clone(),
self.config.sink_table_name.clone(),
@@ -441,15 +505,15 @@ impl BatchingTask {
let merge_mode_last_non_null =
is_merge_mode_last_non_null(&table_meta.options.extra_options);
let primary_key_indices = table_meta.primary_key_indices.clone();
let query_ctx = self.state.read().unwrap().query_ctx.clone();
gen_plan_with_matching_schema(
gen_plan_with_matching_schema_and_values(
&self.config.query,
query_ctx,
self.query_context_snapshot(),
engine.clone(),
table_meta.schema.clone(),
&primary_key_indices,
merge_mode_last_non_null,
values,
)
.await
.map(|_| ())
@@ -484,13 +548,30 @@ impl BatchingTask {
frontend_client: &Arc<FrontendClient>,
max_window_cnt: Option<usize>,
) -> ExecuteOnceOutcome {
let _execution_guard = self.execution_lock.lock().await;
self.execute_once_unlocked(engine, frontend_client, max_window_cnt)
let guard = BatchingExecutionGuard::new(self.execution_lock.clone().lock_owned().await);
self.execute_once_with_guard(guard, engine, frontend_client, max_window_cnt)
.await
}
/// Executes one flow evaluation. Caller must hold `execution_lock`.
async fn execute_once_unlocked(
async fn execute_once_with_guard(
&self,
guard: BatchingExecutionGuard,
engine: &QueryEngineRef,
frontend_client: &Arc<FrontendClient>,
max_window_cnt: Option<usize>,
) -> ExecuteOnceOutcome {
if let Some(execution) = &self.execution {
return execution
.clone()
.execute_once(guard, self, engine, frontend_client, max_window_cnt)
.await;
}
self.execute_once_default_unlocked(engine, frontend_client, max_window_cnt)
.await
}
/// Executes the default flow evaluation. Caller owns the execution guard.
async fn execute_once_default_unlocked(
&self,
engine: &QueryEngineRef,
frontend_client: &Arc<FrontendClient>,
@@ -510,6 +591,7 @@ impl BatchingTask {
debug!("Generate new query: {}", new_query.plan);
let res = self
.execute_logical_plan_unlocked(
engine,
frontend_client,
&new_query.plan,
&new_query.dirty_restore,
@@ -532,11 +614,24 @@ impl BatchingTask {
}
}
/// Generates the insert plan. Caller must reach this through the serialized path.
/// Generates the ordinary insert plan. Caller must reach this through the serialized path.
async fn gen_insert_plan_unlocked(
&self,
engine: &QueryEngineRef,
max_window_cnt: Option<usize>,
) -> Result<Option<PlanInfo>, Error> {
self.gen_insert_plan_with_values_unlocked(engine, max_window_cnt, &BTreeMap::new(), false)
.await
}
/// Generates the insert plan with extension-owned typed sink values. Caller must own the
/// serialized execution guard.
pub async fn gen_insert_plan_with_values_unlocked(
&self,
engine: &QueryEngineRef,
max_window_cnt: Option<usize>,
values: &BTreeMap<String, ScalarValue>,
force_full_snapshot: bool,
) -> Result<Option<PlanInfo>, Error> {
let (table, df_schema) = get_table_info_df_schema(
self.config.catalog_manager.clone(),
@@ -550,12 +645,14 @@ impl BatchingTask {
let primary_key_indices = table_meta.primary_key_indices.clone();
let new_query = self
.gen_query_with_time_window(
.gen_query_with_time_window_with_values(
engine.clone(),
&table.table_info().meta.schema,
&primary_key_indices,
merge_mode_last_non_null,
max_window_cnt,
values,
force_full_snapshot,
)
.await?;
@@ -637,14 +734,55 @@ impl BatchingTask {
Ok(())
}
/// Executes the insert plan. Caller must reach this through the serialized path.
/// Applies checkpoint state updates around the raw insert-plan execution.
async fn execute_logical_plan_unlocked(
&self,
engine: &QueryEngineRef,
frontend_client: &Arc<FrontendClient>,
plan: &LogicalPlan,
dirty_restore: &DirtyRestore,
coverage: &QueryCoverage,
) -> Result<Option<(usize, Duration)>, Error> {
let flow_id = self.config.flow_id;
let Some((res, elapsed)) = self
.execute_plan_unlocked(engine, frontend_client, plan, dirty_restore, coverage)
.await?
else {
return Ok(None);
};
if let Err(err) = &res {
let decision = {
let mut state = self.state.write().unwrap();
let reason = Self::query_failure_reason(err, coverage);
Self::apply_query_failure_to_state(&mut state, elapsed, coverage, reason)
};
if let Some(decision) = decision {
Self::record_checkpoint_decision(flow_id, decision);
}
}
let res = res?;
let (affected_rows, _) = res.output.extract_rows_and_cost();
let decision = {
let mut state = self.state.write().unwrap();
Self::apply_query_result_to_state(&mut state, &res, elapsed, coverage)
};
Self::record_checkpoint_decision(flow_id, decision);
Ok(Some((affected_rows, elapsed)))
}
/// Executes a fully constructed insert plan without checkpoint-state mutation.
/// Callers must own the serialized execution guard. Its unsafe-incremental safety fallback
/// still restores the supplied dirty work before returning `Ok(None)`.
pub async fn execute_plan_unlocked(
&self,
engine: &QueryEngineRef,
frontend_client: &Arc<FrontendClient>,
plan: &LogicalPlan,
dirty_restore: &DirtyRestore,
coverage: &QueryCoverage,
) -> Result<Option<(Result<OutputWithMetrics, Error>, Duration)>, Error> {
let instant = Instant::now();
let flow_id = self.config.flow_id;
@@ -676,7 +814,7 @@ impl BatchingTask {
// For incremental-mode SQL queries, attempt to rewrite the delta aggregate
// plan into a safe delta-LEFT-JOIN-sink form before deciding on extensions.
let incremental_plan = if coverage.is_incremental_delta() {
self.prepare_plan_for_incremental(&plan).await?
self.prepare_plan_for_incremental(engine, &plan).await?
} else {
None
};
@@ -805,18 +943,13 @@ impl BatchingTask {
.map(ToString::to_string)
.unwrap_or_else(|| PeerDesc::default().to_string());
if let Err(err) = &res {
// This raw helper deliberately leaves checkpoint transitions to its caller. The
// default wrapper performs the single OSS transition; extension owners can persist
// their durable marker before updating in-memory state.
warn!(
"Failed to execute Flow {flow_id} on frontend {peer_label}, result: {err:?}, elapsed: {:?} with query: {}",
elapsed, &plan
);
let decision = {
let mut state = self.state.write().unwrap();
let reason = Self::query_failure_reason(err, coverage);
Self::apply_query_failure_to_state(&mut state, elapsed, coverage, reason)
};
if let Some(decision) = decision {
Self::record_checkpoint_decision(flow_id, decision);
}
}
// record slow query
@@ -831,29 +964,27 @@ impl BatchingTask {
.observe(elapsed.as_secs_f64());
}
let res = res?;
let (affected_rows, _) = res.output.extract_rows_and_cost();
debug!(
"Flow {flow_id} executed, affected_rows: {affected_rows:?}, elapsed: {:?}, watermark: {:?}",
elapsed,
res.region_watermark_map()
);
METRIC_FLOW_ROWS
.with_label_values(&[format!("{}-out-batching", flow_id).as_str()])
.inc_by(affected_rows as _);
let decision = {
let mut state = self.state.write().unwrap();
Self::apply_query_result_to_state(&mut state, &res, elapsed, coverage)
};
Self::record_checkpoint_decision(flow_id, decision);
Ok(Some((affected_rows, elapsed)))
match res {
Ok(res) => {
let (affected_rows, _) = res.output.extract_rows_and_cost();
debug!(
"Flow {flow_id} executed, affected_rows: {affected_rows:?}, elapsed: {:?}, watermark: {:?}",
elapsed,
res.region_watermark_map()
);
METRIC_FLOW_ROWS
.with_label_values(&[format!("{}-out-batching", flow_id).as_str()])
.inc_by(affected_rows as _);
Ok(Some((Ok(res), elapsed)))
}
Err(err) => Ok(Some((Err(err), elapsed))),
}
}
/// Restore dirty windows consumed by a failed query so they are retried on
/// the next execution.
///
fn restore_dirty_windows(&self, dirty_restore: &DirtyRestore) {
pub fn restore_dirty_windows(&self, dirty_restore: &DirtyRestore) {
match dirty_restore {
DirtyRestore::Scoped(filter) => self.restore_scoped_dirty_windows(filter),
DirtyRestore::Unscoped(dirty_windows) => self
@@ -931,19 +1062,21 @@ impl BatchingTask {
sink_table_schema: Arc<Schema>,
primary_key_indices: &[usize],
allow_partial: bool,
values: &BTreeMap<String, ScalarValue>,
dirty_windows_to_restore: DirtyTimeWindows,
retention_filter: Option<(&str, Timestamp, &'static str)>,
coverage: QueryCoverage,
) -> Result<PlanInfo, Error> {
let mut plan = self.restore_unscoped_dirty_windows_on_err(
&dirty_windows_to_restore,
gen_plan_with_matching_schema(
gen_plan_with_matching_schema_and_values(
&self.config.query,
query_ctx,
engine,
sink_table_schema,
primary_key_indices,
allow_partial,
values,
)
.await,
)?;
@@ -986,6 +1119,7 @@ impl BatchingTask {
sink_table_schema: Arc<Schema>,
primary_key_indices: &[usize],
allow_partial: bool,
values: &BTreeMap<String, ScalarValue>,
retention_filter: Option<(&str, Timestamp, &'static str)>,
coverage: QueryCoverage,
) -> Result<Option<PlanInfo>, Error> {
@@ -1001,6 +1135,7 @@ impl BatchingTask {
sink_table_schema,
primary_key_indices,
allow_partial,
values,
dirty_windows_to_restore,
retention_filter,
coverage,
@@ -1225,7 +1360,7 @@ impl BatchingTask {
.inc();
// Dirty-window restoration is handled by the
// existing `handle_executed_query_failure` inside
// `execute_once_unlocked`.
// default execution.
}
}
}
@@ -1333,23 +1468,7 @@ impl BatchingTask {
frontend_client: &Arc<FrontendClient>,
scheduled_time_secs: i64,
) -> ExecuteOnceOutcome {
let _execution_guard = self.execution_lock.lock().await;
struct QueryContextRestoreGuard {
state: Arc<RwLock<TaskState>>,
old_ctx: Option<QueryContextRef>,
}
impl Drop for QueryContextRestoreGuard {
fn drop(&mut self) {
let Some(old_ctx) = self.old_ctx.take() else {
return;
};
if let Ok(mut state) = self.state.write() {
state.query_ctx = old_ctx;
}
}
}
let mut guard = BatchingExecutionGuard::new(self.execution_lock.clone().lock_owned().await);
// Convert to milliseconds before touching the task state so an
// unrepresentable scheduled time fails as an explicit error without
@@ -1377,21 +1496,12 @@ impl BatchingTask {
state.query_ctx = Arc::new(new_ctx);
old
};
let restore_guard = QueryContextRestoreGuard {
state: self.state.clone(),
old_ctx: Some(old_ctx),
};
guard.restore_query_context(self.state.clone(), old_ctx);
let outcome = self
.execute_once_unlocked(engine, frontend_client, None)
.await;
// Restore while still holding `execution_lock` so no future manual
// flush can observe the temporary scheduled time. The guard also
// restores during unwind/cancellation.
drop(restore_guard);
outcome
// A collaborator may retain the guard in an owned child task. Its Drop
// restores the scheduled context before releasing serialization.
self.execute_once_with_guard(guard, engine, frontend_client, None)
.await
}
/// Generate the create table SQL
@@ -1419,8 +1529,7 @@ impl BatchingTask {
&& matches!(self.config.query_type, QueryType::Sql)
}
/// Generate the next plan and classify its coverage so checkpoint handling
/// knows whether it is full-query, scoped repair, fenced repair, or delta.
/// Generate the next ordinary plan and classify its coverage.
async fn gen_query_with_time_window(
&self,
engine: QueryEngineRef,
@@ -1428,6 +1537,30 @@ impl BatchingTask {
primary_key_indices: &[usize],
allow_partial: bool,
max_window_cnt: Option<usize>,
) -> Result<Option<PlanInfo>, Error> {
self.gen_query_with_time_window_with_values(
engine,
sink_table_schema,
primary_key_indices,
allow_partial,
max_window_cnt,
&BTreeMap::new(),
false,
)
.await
}
/// Generate the next plan with extension-owned sink values and an optional forced full scan.
#[allow(clippy::too_many_arguments)]
async fn gen_query_with_time_window_with_values(
&self,
engine: QueryEngineRef,
sink_table_schema: &Arc<Schema>,
primary_key_indices: &[usize],
allow_partial: bool,
max_window_cnt: Option<usize>,
values: &BTreeMap<String, ScalarValue>,
force_full_snapshot: bool,
) -> Result<Option<PlanInfo>, Error> {
let query_ctx = self.state.read().unwrap().query_ctx.clone();
let start = SystemTime::now();
@@ -1478,6 +1611,7 @@ impl BatchingTask {
sink_table_schema.clone(),
primary_key_indices,
allow_partial,
values,
dirty_windows_to_restore,
None,
QueryCoverage::UnfilteredFull,
@@ -1514,6 +1648,34 @@ impl BatchingTask {
),
})?;
if force_full_snapshot {
// Durable execution cannot trust an uncertain prior checkpoint. Re-run an
// unfiltered full snapshot, but retain the normal expiry predicate and the consumed
// dirty signal so every planning/execution failure remains reversible.
let retention_filter = self.config.expire_after.map(|_| {
(
col_name.as_str(),
expire_lower_bound,
"forced full snapshot",
)
});
let (_, dirty_windows_to_restore) = self.drain_dirty_windows_signal();
return self
.gen_unfiltered_plan_info(
engine,
query_ctx,
sink_table_schema.clone(),
primary_key_indices,
allow_partial,
values,
dirty_windows_to_restore,
retention_filter,
QueryCoverage::UnfilteredFull,
)
.await
.map(Some);
}
if self.should_use_unfiltered_incremental_delta() {
// In incremental mode, source correctness is defined by the
// per-region sequence range `(checkpoint, scan-open snapshot]`, not
@@ -1535,6 +1697,7 @@ impl BatchingTask {
sink_table_schema.clone(),
primary_key_indices,
allow_partial,
values,
retention_filter,
QueryCoverage::IncrementalDelta,
)
@@ -1580,10 +1743,11 @@ impl BatchingTask {
);
let mut add_filter = AddFilterRewriter::new(expr.expr.clone());
let mut add_auto_column = ColumnMatcherRewriter::new(
let mut add_auto_column = ColumnMatcherRewriter::new_with_values(
sink_table_schema.clone(),
primary_key_indices.to_vec(),
allow_partial,
values.clone(),
);
let plan = self.restore_scoped_dirty_windows_on_err(
+3
View File
@@ -18,6 +18,7 @@ use common_error::ext::BoxedError;
use common_telemetry::debug;
use common_telemetry::tracing::warn;
use datafusion_expr::{DmlStatement, LogicalPlan};
use query::QueryEngineRef;
use query::options::{
FLOW_INCREMENTAL_AFTER_SEQS, FLOW_INCREMENTAL_MODE, FLOW_INCREMENTAL_MODE_MEMTABLE_ONLY,
FLOW_INCREMENTAL_MODE_SEQUENCE_RANGE, FLOW_SINK_TABLE_ID,
@@ -119,6 +120,7 @@ impl BatchingTask {
/// incremental safe without a rewrite, so they return `Some(original_plan)`.
pub(super) async fn prepare_plan_for_incremental(
&self,
engine: &QueryEngineRef,
plan: &LogicalPlan,
) -> Result<Option<LogicalPlan>, Error> {
let is_incremental_sql = {
@@ -219,6 +221,7 @@ impl BatchingTask {
let rewritten_inner = match rewrite_incremental_aggregate_with_sink_merge(
&inner_plan,
&analysis,
engine,
sink_table,
&self.config.sink_table_name,
None,
+419 -12
View File
@@ -15,6 +15,7 @@
use std::collections::{BTreeMap, BTreeSet, HashMap};
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use std::task::Poll;
use catalog::memory::MemoryCatalogManager;
use catalog::{DeregisterTableRequest, RegisterTableRequest};
@@ -59,6 +60,329 @@ fn incremental_batch_opts() -> Arc<BatchingModeOptions> {
})
}
struct CountingExecution {
calls: std::sync::atomic::AtomicUsize,
active: std::sync::atomic::AtomicUsize,
max_active: std::sync::atomic::AtomicUsize,
}
#[async_trait::async_trait]
impl crate::BatchingExecution for CountingExecution {
async fn execute_once(
self: Arc<Self>,
_guard: BatchingExecutionGuard,
_task: &BatchingTask,
_engine: &QueryEngineRef,
_frontend: &Arc<FrontendClient>,
_max_window_cnt: Option<usize>,
) -> ExecuteOnceOutcome {
self.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
let active = self
.active
.fetch_add(1, std::sync::atomic::Ordering::SeqCst)
+ 1;
self.max_active
.fetch_max(active, std::sync::atomic::Ordering::SeqCst);
tokio::task::yield_now().await;
self.active
.fetch_sub(1, std::sync::atomic::Ordering::SeqCst);
ExecuteOnceOutcome {
new_query: None,
result: Ok(None),
}
}
}
struct RetainingExecution {
calls: std::sync::atomic::AtomicUsize,
started: Arc<tokio::sync::Notify>,
release: Arc<tokio::sync::Notify>,
finished: Arc<tokio::sync::Notify>,
}
#[async_trait::async_trait]
impl crate::BatchingExecution for RetainingExecution {
async fn execute_once(
self: Arc<Self>,
guard: BatchingExecutionGuard,
_task: &BatchingTask,
_engine: &QueryEngineRef,
_frontend: &Arc<FrontendClient>,
_max_window_cnt: Option<usize>,
) -> ExecuteOnceOutcome {
if self.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) != 0 {
return ExecuteOnceOutcome {
new_query: None,
result: Ok(None),
};
}
let started = self.started.clone();
let release = self.release.clone();
let finished = self.finished.clone();
let child = tokio::spawn(async move {
started.notify_one();
release.notified().await;
drop(guard);
finished.notify_one();
ExecuteOnceOutcome {
new_query: None,
result: Ok(None),
}
});
match child.await {
Ok(outcome) => outcome,
Err(err) => ExecuteOnceOutcome {
new_query: None,
result: Err(Error::Unexpected {
reason: format!("retaining test child failed: {err}"),
location: snafu::location!(),
}),
},
}
}
}
#[tokio::test]
async fn test_execution_delegate_dispatch_is_serialized() {
let TestTaskParts {
task, query_engine, ..
} = new_test_task_engine_and_plan_with_query("SELECT number, ts FROM numbers_with_ts", "sink")
.await;
let execution = Arc::new(CountingExecution {
calls: Default::default(),
active: Default::default(),
max_active: Default::default(),
});
let task = task.with_execution(Some(execution.clone()));
let (frontend, _handler) = FrontendClient::from_empty_grpc_handler(QueryOptions::default());
let frontend = Arc::new(frontend);
let first = task.execute_once_serialized(&query_engine, &frontend, None);
let second = task.execute_once_serialized(&query_engine, &frontend, None);
let (first, second) = tokio::join!(first, second);
assert_eq!(first.unwrap(), None);
assert_eq!(second.unwrap(), None);
assert_eq!(execution.calls.load(std::sync::atomic::Ordering::SeqCst), 2);
assert_eq!(
execution
.max_active
.load(std::sync::atomic::Ordering::SeqCst),
1,
"the existing execution_lock must span delegate execution"
);
}
#[tokio::test]
async fn test_delegate_guard_survives_caller_cancellation_until_child_finishes() {
let TestTaskParts {
task, query_engine, ..
} = new_test_task_engine_and_plan_with_query("SELECT number, ts FROM numbers_with_ts", "sink")
.await;
let execution = Arc::new(RetainingExecution {
calls: Default::default(),
started: Arc::new(tokio::sync::Notify::new()),
release: Arc::new(tokio::sync::Notify::new()),
finished: Arc::new(tokio::sync::Notify::new()),
});
let task = task.with_execution(Some(execution.clone()));
let (frontend, _handler) = FrontendClient::from_empty_grpc_handler(QueryOptions::default());
let frontend = Arc::new(frontend);
let first_task = task.clone();
let first_engine = query_engine.clone();
let first_frontend = frontend.clone();
let first = tokio::spawn(async move {
first_task
.execute_once_serialized(&first_engine, &first_frontend, None)
.await
});
tokio::time::timeout(Duration::from_secs(1), execution.started.notified())
.await
.expect("delegate child did not retain the guard");
first.abort();
assert!(
first
.await
.expect_err("caller cancellation should abort")
.is_cancelled()
);
let second = task.execute_once_serialized(&query_engine, &frontend, None);
futures::pin_mut!(second);
assert!(
matches!(futures::poll!(second.as_mut()), Poll::Pending),
"the next round must remain pending while the retained guard is held"
);
assert_eq!(
execution.calls.load(std::sync::atomic::Ordering::SeqCst),
1,
"the pending waiter must not enter the collaborator"
);
execution.release.notify_one();
tokio::time::timeout(Duration::from_secs(1), execution.finished.notified())
.await
.expect("delegate child did not release its guard");
assert_eq!(
tokio::time::timeout(Duration::from_secs(1), second)
.await
.expect("next round should proceed after child release")
.unwrap(),
None
);
assert_eq!(execution.calls.load(std::sync::atomic::Ordering::SeqCst), 2);
}
#[tokio::test]
async fn test_scheduled_context_is_retained_until_delegate_child_releases_guard() {
let TestTaskParts {
task, query_engine, ..
} = new_test_task_engine_and_plan_with_query("SELECT number, ts FROM numbers_with_ts", "sink")
.await;
let execution = Arc::new(RetainingExecution {
calls: Default::default(),
started: Arc::new(tokio::sync::Notify::new()),
release: Arc::new(tokio::sync::Notify::new()),
finished: Arc::new(tokio::sync::Notify::new()),
});
let task = task.with_execution(Some(execution.clone()));
let (frontend, _handler) = FrontendClient::from_empty_grpc_handler(QueryOptions::default());
let frontend = Arc::new(frontend);
let scheduled = 1_700_000_000;
let task_to_run = task.clone();
let engine_to_run = query_engine.clone();
let frontend_to_run = frontend.clone();
let execution_call = tokio::spawn(async move {
task_to_run
.execute_once_serialized_at_scheduled_time(&engine_to_run, &frontend_to_run, scheduled)
.await
});
tokio::time::timeout(Duration::from_secs(1), execution.started.notified())
.await
.expect("scheduled delegate child did not start");
assert_eq!(
task.state
.read()
.unwrap()
.query_ctx
.extension(FLOW_SCHEDULED_TIME_MILLIS),
Some("1700000000000"),
"scheduled context restored before the delegate child released the guard"
);
execution_call.abort();
match execution_call.await {
Err(error) => assert!(error.is_cancelled()),
Ok(_) => panic!("scheduled caller cancellation should abort"),
}
assert_eq!(
task.state
.read()
.unwrap()
.query_ctx
.extension(FLOW_SCHEDULED_TIME_MILLIS),
Some("1700000000000"),
"scheduled context restored after caller cancellation but before child release"
);
execution.release.notify_one();
tokio::time::timeout(Duration::from_secs(1), execution.finished.notified())
.await
.expect("scheduled delegate child did not release its guard");
assert_eq!(
task.state
.read()
.unwrap()
.query_ctx
.extension(FLOW_SCHEDULED_TIME_MILLIS),
None,
"scheduled context was not restored when child released guard"
);
}
struct BlockingDefaultExecutionHandler {
entered: std::sync::Mutex<Option<tokio::sync::oneshot::Sender<()>>>,
dropped: std::sync::Mutex<Option<tokio::sync::oneshot::Sender<()>>>,
}
struct DropAck(Option<tokio::sync::oneshot::Sender<()>>);
impl Drop for DropAck {
fn drop(&mut self) {
if let Some(dropped) = self.0.take() {
let _ = dropped.send(());
}
}
}
#[async_trait::async_trait]
impl crate::batching_mode::frontend_client::GrpcQueryHandlerWithBoxedError
for BlockingDefaultExecutionHandler
{
async fn do_query(
&self,
_query: api::v1::greptime_request::Request,
_ctx: QueryContextRef,
) -> std::result::Result<Output, BoxedError> {
let _ack = DropAck(self.dropped.lock().unwrap().take());
if let Some(entered) = self.entered.lock().unwrap().take() {
let _ = entered.send(());
}
std::future::pending().await
}
}
#[tokio::test]
async fn test_default_execution_remains_inline_and_cancellable() {
let query = "SELECT number, date_bin(INTERVAL '5 second', ts) AS time_window \
FROM numbers_with_ts GROUP BY time_window, number";
let TestTaskParts {
task, query_engine, ..
} = new_time_window_test_task_with_query(query).await;
register_twe_sink(&query_engine, "missing_sink", 9200);
task.mark_all_windows_as_dirty().unwrap();
let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
let (dropped_tx, dropped_rx) = tokio::sync::oneshot::channel();
let handler: Arc<dyn crate::batching_mode::frontend_client::GrpcQueryHandlerWithBoxedError> =
Arc::new(BlockingDefaultExecutionHandler {
entered: std::sync::Mutex::new(Some(entered_tx)),
dropped: std::sync::Mutex::new(Some(dropped_tx)),
});
let frontend = Arc::new(FrontendClient::from_grpc_handler(
Arc::downgrade(&handler),
QueryOptions::default(),
));
let task_to_cancel = task.clone();
let engine_to_cancel = query_engine.clone();
let frontend_to_cancel = frontend.clone();
let caller = tokio::spawn(async move {
task_to_cancel
.execute_once_serialized(&engine_to_cancel, &frontend_to_cancel, None)
.await
});
tokio::time::timeout(Duration::from_secs(1), entered_rx)
.await
.expect("default execution did not dispatch a frontend query")
.expect("default execution handler entry notification dropped");
caller.abort();
match caller.await {
Err(error) => assert!(error.is_cancelled()),
Ok(_) => panic!("default caller cancellation should abort"),
}
tokio::time::timeout(Duration::from_secs(1), dropped_rx)
.await
.expect("cancelling default execution did not drop the active frontend future")
.expect("default execution drop acknowledgement was not sent");
tokio::time::timeout(
Duration::from_secs(1),
task.execution_lock.clone().lock_owned(),
)
.await
.expect("default execution must not leave an owned child holding the lock");
}
async fn new_test_task_and_plan_with_missing_sink() -> (BatchingTask, LogicalPlan) {
new_test_task_engine_and_plan_with_query(
"SELECT number, ts FROM numbers_with_ts",
@@ -2072,6 +2396,62 @@ async fn test_full_snapshot_seeding_applies_expire_after_retention_filter() {
);
}
#[tokio::test]
async fn test_force_full_snapshot_retains_expiration_and_reversible_dirty_capture() {
let TestTaskParts {
mut task,
query_engine,
..
} = new_time_window_test_task_with_query(
"SELECT max(number) AS number, date_bin(INTERVAL '5 second', ts) AS time_window FROM numbers_with_ts GROUP BY time_window",
)
.await;
{
let mut state = task.state.write().unwrap();
state.advance_checkpoints(HashMap::from([(1_u64, 10_u64)]));
state
.dirty_time_windows
.add_window(Timestamp::new_second(0), Some(Timestamp::new_second(5)));
}
register_twe_sink(&query_engine, "missing_sink", 9201);
Arc::get_mut(&mut task.config)
.expect("test task config should be uniquely owned")
.expire_after = Some(expire_after_for_retention_filter_test());
let plan = task
.gen_insert_plan_with_values_unlocked(&query_engine, Some(1), &BTreeMap::new(), true)
.await
.unwrap()
.unwrap();
assert!(matches!(plan.coverage, QueryCoverage::UnfilteredFull));
assert!(
plan.plan
.to_string()
.contains("Filter: ts >= TimestampMillisecond(")
);
assert!(task.state.read().unwrap().dirty_time_windows.is_empty());
task.restore_dirty_windows(&plan.dirty_restore);
assert_eq!(task.state.read().unwrap().dirty_time_windows.len(), 1);
task.state.write().unwrap().dirty_time_windows.clean();
let no_dirty_plan = task
.gen_insert_plan_with_values_unlocked(&query_engine, Some(1), &BTreeMap::new(), true)
.await
.unwrap()
.expect("forced full snapshot must bypass the dirty-window notification gate");
assert!(matches!(
no_dirty_plan.coverage,
QueryCoverage::UnfilteredFull
));
assert!(
no_dirty_plan
.plan
.to_string()
.contains("Filter: ts >= TimestampMillisecond(")
);
}
#[tokio::test]
async fn test_incremental_plan_does_not_add_dirty_window_filter() {
let TestTaskParts {
@@ -2189,19 +2569,38 @@ async fn test_exact_required_executed_failure_selects_full_snapshot_repair() {
Arc::downgrade(&handler),
QueryOptions::default(),
));
task.execute_logical_plan_unlocked(
&frontend_client,
&plan_info.plan,
&plan_info.dirty_restore,
&plan_info.coverage,
)
.await
.expect_err("the dispatched exact delta must fail through the injected frontend handler");
let raw = task
.execute_plan_unlocked(
&query_engine,
&frontend_client,
&plan_info.plan,
&plan_info.dirty_restore,
&plan_info.coverage,
)
.await
.expect("raw executor should return the dispatched execution result")
.expect("raw executor should dispatch the exact delta");
assert!(
raw.0.is_err(),
"the dispatched exact delta must fail through the injected frontend handler"
);
assert!(
handler_invoked.load(Ordering::SeqCst),
"the injected frontend handler must receive the exact delta"
);
task.handle_executed_query_failure(Some(&plan_info));
// The raw executor is a delegate-safe primitive: it logs failures but does not change
// checkpoint state. Restore the consumed work, then let the default wrapper make its one
// failure transition and restore it again.
{
let state = task.state.read().unwrap();
assert_eq!(state.checkpoint_mode(), CheckpointMode::Incremental);
assert_eq!(state.checkpoints(), &BTreeMap::from([(1_u64, 10_u64)]));
}
task.restore_dirty_windows(&plan_info.dirty_restore);
let outcome = task
.execute_once_default_unlocked(&query_engine, &frontend_client, Some(1))
.await;
assert!(outcome.result.is_err());
{
let state = task.state.read().unwrap();
@@ -2459,6 +2858,7 @@ async fn test_exact_required_unsupported_plan_keeps_exact_retry_state() {
for _ in 0..2 {
let err = task
.execute_logical_plan_unlocked(
&query_engine,
&frontend_client,
&dml_plan,
&DirtyRestore::Unscoped(dirty_range(10, 15)),
@@ -2544,7 +2944,10 @@ async fn test_prepare_plan_for_incremental_disables_on_non_aggregate() {
CheckpointMode::Incremental
);
let incremental_plan = task.prepare_plan_for_incremental(&dml_plan).await.unwrap();
let incremental_plan = task
.prepare_plan_for_incremental(&query_engine, &dml_plan)
.await
.unwrap();
assert!(incremental_plan.is_none());
let state = task.state.read().unwrap();
assert!(state.is_incremental_disabled());
@@ -2621,6 +3024,7 @@ async fn test_unsafe_incremental_plan_skip_restores_dirty_without_query() {
let result = task
.execute_logical_plan_unlocked(
&query_engine,
&Arc::new(frontend_client),
&dml_plan,
&dirty_restore,
@@ -2707,7 +3111,7 @@ async fn test_prepare_plan_for_incremental_group_by_without_merge_columns_uses_o
.advance_checkpoints(HashMap::from([(1_u64, 10_u64)]));
let incremental_plan = task
.prepare_plan_for_incremental(&dml_plan)
.prepare_plan_for_incremental(&query_engine, &dml_plan)
.await
.unwrap()
.expect("plain GROUP BY is incremental-safe without a rewrite");
@@ -2752,7 +3156,10 @@ async fn test_auto_created_sql_aggregate_sink_reaches_incremental_safe() {
.write()
.unwrap()
.advance_checkpoints(HashMap::from([(1_u64, 10_u64)]));
let incremental_plan = task.prepare_plan_for_incremental(&dml_plan).await.unwrap();
let incremental_plan = task
.prepare_plan_for_incremental(&query_engine, &dml_plan)
.await
.unwrap();
let incremental_safe = incremental_plan.is_some();
assert!(incremental_safe);
+363 -101
View File
@@ -14,7 +14,7 @@
//! some utils for helping with batching mode
use std::collections::{BTreeSet, HashMap, HashSet};
use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
use std::sync::Arc;
use catalog::CatalogManagerRef;
@@ -66,6 +66,9 @@ mod test;
/// `max(numbers_with_ts.number)`.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct IncrementalAggregateMergeColumn {
/// Delta-plan field containing the aggregate result/state column. Repeated
/// projections intentionally share this field while using distinct outputs.
pub input_field_name: String,
/// Final output/sink field name for the aggregate result/state column.
///
pub output_field_name: String,
@@ -73,16 +76,17 @@ pub struct IncrementalAggregateMergeColumn {
}
impl IncrementalAggregateMergeColumn {
/// Create a new merge column.
/// Create a new merge column whose delta and output fields have the same name.
pub fn new(output_field_name: String, merge_op: IncrementalAggregateMergeOp) -> Self {
Self {
input_field_name: output_field_name.clone(),
output_field_name,
merge_op,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum IncrementalAggregateMergeOp {
Sum,
Min,
@@ -92,6 +96,10 @@ pub enum IncrementalAggregateMergeOp {
BitAnd,
BitOr,
BitXor,
StateDeltaMerge {
function_name: &'static str,
params: Vec<Expr>,
},
}
/// Analysis result for an incremental aggregate plan.
@@ -113,24 +121,6 @@ pub struct IncrementalAggregateAnalysis {
pub unsupported_exprs: Vec<String>,
}
/// Recursively find all `Expr::Column` names inside an expression tree.
/// Only recurses into wrappers that are merge-transparent.
/// Non-transparent wrappers (e.g., `ScalarFunction`, `Negative`, `Cast`) are
/// intentionally not recursed into since their merge semantics would be
/// incorrect.
///
/// `Cast`/`TryCast` are intentionally opaque: merging already-casted aggregate
/// outputs is not generally equivalent to casting the final merged aggregate.
fn find_column_names(expr: &Expr, names: &mut Vec<String>) {
match expr {
Expr::Column(col) => {
names.push(col.name.clone());
}
Expr::Alias(alias) => find_column_names(&alias.expr, names),
_ => {}
}
}
fn unqualified_col(name: impl Into<String>) -> Expr {
Expr::Column(Column::from_name(name.into()))
}
@@ -230,8 +220,8 @@ fn check_input_plan_shape(plan: &LogicalPlan) -> Result<(), String> {
#[derive(Debug, Default)]
struct OutputProjectionInfo {
has_top_level_projection: bool,
output_aliases: HashMap<String, String>,
duplicate_aggregate_aliases: BTreeSet<String>,
/// Aggregate expression name and projected output field, in projection order.
aggregate_outputs: Vec<(String, String)>,
literal_columns: HashSet<String>,
output_field_names: Vec<String>,
}
@@ -265,46 +255,34 @@ fn collect_output_projection_info(plan: &LogicalPlan) -> OutputProjectionInfo {
..Default::default()
};
let mut output_aliases = HashMap::new();
if let LogicalPlan::Projection(projection) = plan {
for expr in &projection.expr {
match expr {
Expr::Alias(alias) => {
// Alias resolution has three cases:
// - 0 Column refs (e.g., literal `42 AS lit`): record literal output
// - 1 Column ref: record the mapping (e.g., `sum(x) AS total`)
// - >1 Column refs (e.g., `COALESCE(sum(x), sum(y))`):
// skip — ambiguous merge semantics
// Only a direct aggregate output column has the same
// merge semantics as the original resolver. In particular,
// do not mine aggregate columns through CAST/TryCast or
// other output wrappers.
let alias_name = alias.name.clone();
let mut col_names = Vec::new();
find_column_names(&alias.expr, &mut col_names);
match col_names.len() {
0 if is_passthrough_output_column(&alias_name, alias.expr.as_ref()) => {
projection_info.literal_columns.insert(alias_name);
}
1 => {
if let Some(col_name) = col_names.into_iter().next() {
if let Some(existing_alias) = output_aliases.get(&col_name) {
if existing_alias != &alias_name {
projection_info.duplicate_aggregate_aliases.insert(format!(
"same aggregate output {col_name} is used by multiple aliases: {existing_alias}, {alias_name}"
));
}
} else {
output_aliases.insert(col_name, alias_name);
}
}
}
_ => {}
if let Expr::Column(column) = alias.expr.as_ref() {
projection_info
.aggregate_outputs
.push((column.name.clone(), alias_name));
} else if let Expr::Alias(inner_alias) = alias.expr.as_ref()
&& inner_alias.name.eq_ignore_ascii_case("count(*)")
&& let Expr::Column(column) = inner_alias.expr.as_ref()
{
projection_info
.aggregate_outputs
.push((column.name.clone(), alias_name));
} else if is_passthrough_output_column(&alias_name, alias.expr.as_ref()) {
projection_info.literal_columns.insert(alias_name);
}
// If >1 column references detected (e.g., COALESCE(sum(x), sum(y))),
// intentionally skip alias mapping — the merge semantics are ambiguous.
}
Expr::Column(col) => {
output_aliases
.entry(col.name.clone())
.or_insert(col.name.clone());
projection_info
.aggregate_outputs
.push((col.name.clone(), col.name.clone()));
}
Expr::Literal(_, _) => {
projection_info
@@ -326,7 +304,6 @@ fn collect_output_projection_info(plan: &LogicalPlan) -> OutputProjectionInfo {
.insert(AUTO_CREATED_PLACEHOLDER_TS_COL.to_string());
}
projection_info.output_aliases = output_aliases;
projection_info
}
@@ -362,6 +339,12 @@ fn merge_op_for_aggregate_expr(aggr_expr: &Expr) -> Result<IncrementalAggregateM
return Err(format!("unsupported aggregate NULL treatment: {aggr_expr}"));
}
let state_delta_merge = |function_name, params| {
Ok(IncrementalAggregateMergeOp::StateDeltaMerge {
function_name,
params,
})
};
match aggr_func.func.name().to_ascii_lowercase().as_str() {
"sum" | "count" => Ok(IncrementalAggregateMergeOp::Sum),
"min" => Ok(IncrementalAggregateMergeOp::Min),
@@ -371,28 +354,52 @@ fn merge_op_for_aggregate_expr(aggr_expr: &Expr) -> Result<IncrementalAggregateM
"bit_and" => Ok(IncrementalAggregateMergeOp::BitAnd),
"bit_or" => Ok(IncrementalAggregateMergeOp::BitOr),
"bit_xor" => Ok(IncrementalAggregateMergeOp::BitXor),
// Preserve state-family parameters; value coercion is handled by the aggregate.
"hll" if aggr_func.params.args.len() == 1 => state_delta_merge("__hll_delta_merge", vec![]),
"stddev_pop_state" if aggr_func.params.args.len() == 1 => {
state_delta_merge("__stddev_pop_state_delta_merge", vec![])
}
"uddsketch_state" if aggr_func.params.args.len() == 3 => {
let [bucket_size, error_rate, _] = aggr_func.params.args.as_slice() else {
unreachable!();
};
if !matches!(bucket_size, Expr::Literal(ScalarValue::Int64(Some(_)), _))
|| !matches!(error_rate, Expr::Literal(ScalarValue::Float64(Some(_)), _))
{
return Err(aggr_expr.to_string());
}
state_delta_merge(
"__uddsketch_state_delta_merge",
vec![bucket_size.clone(), error_rate.clone()],
)
}
_ => Err(aggr_expr.to_string()),
}
}
fn resolve_aggregate_output_field_name(
fn resolve_aggregate_output_fields(
aggr_expr: &Expr,
projection_info: &OutputProjectionInfo,
output_field_name_set: &HashSet<String>,
) -> Option<String> {
) -> Vec<(String, String)> {
// qualified_name() returns (Option<String>, String) where the second
// element is the unqualified column/alias name. This relies on
// DataFusion's internal naming convention: aggregate expressions
// emit a column named after the aggregate itself (e.g. "SUM(x)"),
// which matches what the projection aliases reference.
// emit a column named after the aggregate itself (e.g. "SUM(x)").
// Keep every matching projection occurrence because DataFusion can share
// one aggregate input field for identical expressions.
let raw_name = aggr_expr.qualified_name().1;
if let Some(alias) = projection_info.output_aliases.get(&raw_name) {
Some(alias.clone())
} else if !projection_info.has_top_level_projection && output_field_name_set.contains(&raw_name)
{
Some(raw_name)
if projection_info.has_top_level_projection {
projection_info
.aggregate_outputs
.iter()
.filter(|(input_name, _)| input_name == &raw_name)
.cloned()
.collect()
} else if output_field_name_set.contains(&raw_name) {
vec![(raw_name.clone(), raw_name)]
} else {
None
vec![]
}
}
@@ -478,7 +485,6 @@ pub fn analyze_incremental_aggregate_plan(
.map(|name| format!("duplicate output field name: {name}"))
.collect::<Vec<_>>();
unsupported_exprs.push(reason);
unsupported_exprs.extend(projection_info.duplicate_aggregate_aliases.iter().cloned());
return Ok(Some(IncrementalAggregateAnalysis {
group_key_names,
merge_columns: vec![],
@@ -512,7 +518,6 @@ pub fn analyze_incremental_aggregate_plan(
aggregate,
&group_key_names,
));
unsupported_exprs.extend(projection_info.duplicate_aggregate_aliases.iter().cloned());
for aggr_expr in aggr_exprs {
let merge_op = match merge_op_for_aggregate_expr(&aggr_expr) {
Ok(merge_op) => merge_op,
@@ -521,18 +526,37 @@ pub fn analyze_incremental_aggregate_plan(
continue;
}
};
let Some(output_field_name) = resolve_aggregate_output_field_name(
&aggr_expr,
&projection_info,
&output_field_name_set,
) else {
let aggregate_outputs =
resolve_aggregate_output_fields(&aggr_expr, &projection_info, &output_field_name_set);
if aggregate_outputs.is_empty() {
unsupported_exprs.push(aggr_expr.to_string());
continue;
}
let Some((_, input_field_name)) = aggregate_outputs.first() else {
continue;
};
merge_columns.push(IncrementalAggregateMergeColumn::new(
output_field_name,
merge_op,
));
let input_field_name = input_field_name.clone();
for (_, output_field_name) in aggregate_outputs {
merge_columns.push(IncrementalAggregateMergeColumn {
input_field_name: input_field_name.clone(),
output_field_name,
merge_op: merge_op.clone(),
});
}
}
if projection_info.has_top_level_projection {
let output_positions = projection_info
.output_field_names
.iter()
.enumerate()
.map(|(position, name)| (name.as_str(), position))
.collect::<HashMap<_, _>>();
merge_columns.sort_by_key(|column| {
output_positions
.get(column.output_field_name.as_str())
.copied()
.unwrap_or(usize::MAX)
});
}
unsupported_exprs.extend(
find_uncovered_output_fields(&projection_info, &group_key_names, &merge_columns)
@@ -591,6 +615,7 @@ pub fn analyze_incremental_aggregate_plan(
pub async fn rewrite_incremental_aggregate_with_sink_merge(
delta_plan: &LogicalPlan,
analysis: &IncrementalAggregateAnalysis,
engine: &QueryEngineRef,
sink_table: TableRef,
sink_table_name: &TableName,
sink_dirty_filter: Option<Expr>,
@@ -625,6 +650,12 @@ pub async fn rewrite_incremental_aggregate_with_sink_merge(
let delta_alias = "__flow_delta";
let sink_alias = "__flow_sink";
let state_merge = analysis.merge_columns.iter().any(|column| {
matches!(
&column.merge_op,
IncrementalAggregateMergeOp::StateDeltaMerge { .. }
)
});
let mut selected_columns = analysis.group_key_names.clone();
selected_columns.extend(
analysis
@@ -632,8 +663,18 @@ pub async fn rewrite_incremental_aggregate_with_sink_merge(
.iter()
.map(|c| c.output_field_name.clone()),
);
let mut delta_selected_columns = selected_columns.clone();
let mut selected_column_names = HashSet::new();
selected_columns.retain(|name| selected_column_names.insert(name.clone()));
let mut delta_selected_columns = analysis.group_key_names.clone();
delta_selected_columns.extend(
analysis
.merge_columns
.iter()
.map(|c| c.input_field_name.clone()),
);
delta_selected_columns.extend(analysis.literal_columns.iter().cloned());
let mut delta_selected_column_names = HashSet::new();
delta_selected_columns.retain(|name| delta_selected_column_names.insert(name.clone()));
let delta_selected_exprs = delta_selected_columns
.iter()
@@ -745,20 +786,25 @@ pub async fn rewrite_incremental_aggregate_with_sink_merge(
.map(|c| (&c.output_field_name, c))
.collect::<HashMap<_, _>>();
let mut projection_exprs = Vec::with_capacity(analysis.output_field_names.len());
let mut ordinary_exprs = Vec::with_capacity(analysis.output_field_names.len());
let mut state_aggr_exprs = Vec::new();
for output_field_name in &analysis.output_field_names {
if group_key_names.contains(output_field_name)
|| literal_columns.contains(output_field_name)
{
projection_exprs.push(
qualified_col(delta_alias, output_field_name.clone()).alias(output_field_name),
);
let expr =
qualified_col(delta_alias, output_field_name.clone()).alias(output_field_name);
ordinary_exprs.push(expr);
} else if let Some(merge_col) = merge_columns.get(output_field_name) {
projection_exprs.push(build_left_join_merge_expr(
delta_alias,
sink_alias,
merge_col,
)?);
if matches!(
&merge_col.merge_op,
IncrementalAggregateMergeOp::StateDeltaMerge { .. }
) {
state_aggr_exprs.push(build_state_delta_merge_expr(engine, merge_col)?);
} else {
let expr = build_left_join_merge_expr(delta_alias, sink_alias, merge_col)?;
ordinary_exprs.push(expr);
}
} else {
return InvalidQuerySnafu {
reason: format!(
@@ -769,15 +815,88 @@ pub async fn rewrite_incremental_aggregate_with_sink_merge(
}
}
LogicalPlanBuilder::from(joined)
.project(projection_exprs)
.with_context(|_| DatafusionSnafu {
context: "Failed to build projection merge plan for incremental sink merge".to_string(),
})?
.build()
.with_context(|_| DatafusionSnafu {
context: "Failed to finalize incremental aggregate sink merge plan".to_string(),
if state_merge {
let aggregated = LogicalPlanBuilder::from(joined)
.aggregate(ordinary_exprs, state_aggr_exprs)
.with_context(|_| DatafusionSnafu {
context: "Failed to aggregate state delta merge plan".to_string(),
})?
.build()
.with_context(|_| DatafusionSnafu {
context: "Failed to build state delta merge plan".to_string(),
})?;
let output_exprs = analysis
.output_field_names
.iter()
.cloned()
.map(unqualified_col)
.collect::<Vec<_>>();
LogicalPlanBuilder::from(aggregated)
.project(output_exprs)
.with_context(|_| DatafusionSnafu {
context: "Failed to project state delta merge plan".to_string(),
})?
.build()
.with_context(|_| DatafusionSnafu {
context: "Failed to finalize incremental aggregate sink merge plan".to_string(),
})
} else {
LogicalPlanBuilder::from(joined)
.project(ordinary_exprs)
.with_context(|_| DatafusionSnafu {
context: "Failed to build projection merge plan for incremental sink merge"
.to_string(),
})?
.build()
.with_context(|_| DatafusionSnafu {
context: "Failed to finalize incremental aggregate sink merge plan".to_string(),
})
}
}
fn build_state_delta_merge_expr(
engine: &QueryEngineRef,
merge_col: &IncrementalAggregateMergeColumn,
) -> Result<Expr, Error> {
let (function_name, params) = match &merge_col.merge_op {
IncrementalAggregateMergeOp::StateDeltaMerge {
function_name,
params,
} => (*function_name, params.clone()),
_ => {
return InvalidQuerySnafu {
reason: "non-state aggregate passed to state delta merge".to_string(),
}
.fail();
}
};
let Some(udaf) = engine
.engine_state()
.aggr_function(function_name)
.or_else(|| {
engine
.engine_state()
.session_state()
.aggregate_functions()
.get(function_name)
.map(|udaf| udaf.as_ref().clone())
})
else {
return InvalidQuerySnafu {
reason: format!("Aggregate function {function_name} is not registered"),
}
.fail();
};
let mut args = params;
args.push(qualified_col(
"__flow_delta",
merge_col.input_field_name.clone(),
));
args.push(qualified_col(
"__flow_sink",
merge_col.output_field_name.clone(),
));
Ok(udaf.call(args).alias(merge_col.output_field_name.clone()))
}
fn build_left_join_merge_expr(
@@ -785,9 +904,9 @@ fn build_left_join_merge_expr(
sink_alias: &str,
merge_col: &IncrementalAggregateMergeColumn,
) -> Result<Expr, Error> {
let left = qualified_col(delta_alias, merge_col.output_field_name.clone());
let left = qualified_col(delta_alias, merge_col.input_field_name.clone());
let right = qualified_col(sink_alias, merge_col.output_field_name.clone());
let merged = match merge_col.merge_op {
let merged = match merge_col.merge_op.clone() {
IncrementalAggregateMergeOp::Sum => when(is_null(left.clone()), right.clone())
.when(is_null(right.clone()), left.clone())
.otherwise(binary_expr(left.clone(), Operator::Plus, right.clone()))
@@ -836,6 +955,12 @@ fn build_left_join_merge_expr(
.with_context(|_| DatafusionSnafu {
context: "Failed to build BIT_XOR merge expression".to_string(),
})?,
IncrementalAggregateMergeOp::StateDeltaMerge { .. } => {
return InvalidQuerySnafu {
reason: "state aggregate must be built with its delta UDAF".to_string(),
}
.fail();
}
};
Ok(merged.alias(merge_col.output_field_name.clone()))
}
@@ -948,13 +1073,36 @@ pub(crate) async fn gen_plan_with_matching_schema(
sink_table_schema: SchemaRef,
primary_key_indices: &[usize],
allow_partial: bool,
) -> Result<LogicalPlan, Error> {
gen_plan_with_matching_schema_and_values(
sql,
query_ctx,
engine,
sink_table_schema,
primary_key_indices,
allow_partial,
&BTreeMap::new(),
)
.await
}
/// Generate a schema-matching plan with extension-owned typed sink values.
pub(crate) async fn gen_plan_with_matching_schema_and_values(
sql: &str,
query_ctx: QueryContextRef,
engine: QueryEngineRef,
sink_table_schema: SchemaRef,
primary_key_indices: &[usize],
allow_partial: bool,
values: &BTreeMap<String, ScalarValue>,
) -> Result<LogicalPlan, Error> {
let plan = sql_to_df_plan(query_ctx.clone(), engine.clone(), sql, false).await?;
let mut add_auto_column = ColumnMatcherRewriter::new(
let mut add_auto_column = ColumnMatcherRewriter::new_with_values(
sink_table_schema,
primary_key_indices.to_vec(),
allow_partial,
values.clone(),
);
let plan = plan
.clone()
@@ -1090,7 +1238,10 @@ impl TreeNodeVisitor<'_> for FindGroupByFinalName {
/// it also give existing columns alias to column in sink table if needed
#[derive(Debug)]
pub struct ColumnMatcherRewriter {
/// Schema used to match the ordinary flow output. It excludes injected values.
pub schema: SchemaRef,
sink_schema: SchemaRef,
values: BTreeMap<String, ScalarValue>,
pub is_rewritten: bool,
pub primary_key_indices: Vec<usize>,
pub allow_partial: bool,
@@ -1099,21 +1250,127 @@ pub struct ColumnMatcherRewriter {
impl ColumnMatcherRewriter {
pub fn new(schema: SchemaRef, primary_key_indices: Vec<usize>, allow_partial: bool) -> Self {
Self {
schema,
schema: schema.clone(),
sink_schema: schema,
values: BTreeMap::new(),
is_rewritten: false,
primary_key_indices,
allow_partial,
}
}
/// Match ordinary output against the sink schema excluding typed values, then inject those
/// values in physical sink-column order.
pub fn new_with_values(
sink_schema: SchemaRef,
primary_key_indices: Vec<usize>,
allow_partial: bool,
values: BTreeMap<String, ScalarValue>,
) -> Self {
let value_names = values.keys().collect::<BTreeSet<_>>();
let ordinary_columns = sink_schema
.column_schemas()
.iter()
.filter(|column| !value_names.contains(&column.name))
.cloned()
.collect();
let ordinary_schema = Arc::new(datatypes::schema::Schema::new(ordinary_columns));
let ordinary_primary_key_indices = primary_key_indices
.into_iter()
.filter_map(|index| {
let name = &sink_schema.column_schemas()[index].name;
ordinary_schema
.column_schemas()
.iter()
.position(|column| column.name == *name)
})
.collect();
Self {
schema: ordinary_schema,
sink_schema,
values,
is_rewritten: false,
primary_key_indices: ordinary_primary_key_indices,
allow_partial,
}
}
fn validate_values(&self, output_exprs: &[Expr]) -> DfResult<()> {
let output_names = output_exprs
.iter()
.map(|expr| expr.qualified_name().1)
.collect::<Vec<_>>();
let duplicated = duplicate_names(&output_names);
if !duplicated.is_empty() {
return Err(DataFusionError::Plan(format!(
"Flow output schema contains duplicate column(s) {:?}. {}",
duplicated,
format_flow_sink_schema_mismatch(output_exprs, self.sink_schema.as_ref())
)));
}
for (name, value) in &self.values {
let Some(column) = self
.sink_schema
.column_schemas()
.iter()
.find(|column| column.name == *name)
else {
return Err(DataFusionError::Plan(format!(
"Injected flow value column '{name}' is not found in sink schema"
)));
};
if output_names.iter().any(|output| output == name) {
return Err(DataFusionError::Plan(format!(
"Injected flow value column '{name}' collides with flow output"
)));
}
let value_type = ConcreteDataType::from_arrow_type(&value.data_type());
if value_type != column.data_type {
return Err(DataFusionError::Plan(format!(
"Injected flow value column '{name}' has type {:?}, but sink column has type {:?}",
value_type, column.data_type
)));
}
}
Ok(())
}
fn inject_values_in_physical_order(&self, exprs: Vec<Expr>) -> DfResult<Vec<Expr>> {
if self.values.is_empty() {
return Ok(exprs);
}
let mut ordinary = exprs
.into_iter()
.map(|expr| (expr.qualified_name().1, expr))
.collect::<HashMap<_, _>>();
self.sink_schema
.column_schemas()
.iter()
.map(|column| {
if let Some(value) = self.values.get(&column.name) {
Ok(Expr::Literal(value.clone(), None).alias(column.name.clone()))
} else {
ordinary.remove(&column.name).ok_or_else(|| {
DataFusionError::Plan(format!(
"Flow output is missing sink column '{}' after schema matching",
column.name
))
})
}
})
.collect()
}
/// modify the exprs in place so that it matches the schema and some auto columns are added
fn modify_project_exprs(
&mut self,
mut exprs: Vec<Expr>,
input_schema: &DFSchema,
) -> DfResult<Vec<Expr>> {
self.validate_values(&exprs)?;
if self.allow_partial {
return self.modify_project_exprs_with_partial(exprs);
let exprs = self.modify_project_exprs_with_partial(exprs)?;
return self.inject_values_in_physical_order(exprs);
}
let original_exprs = exprs.clone();
@@ -1182,7 +1439,12 @@ impl ColumnMatcherRewriter {
)));
}
self.match_extra_output_columns(exprs, input_schema, &original_exprs, &all_names)
self.inject_values_in_physical_order(self.match_extra_output_columns(
exprs,
input_schema,
&original_exprs,
&all_names,
)?)
}
/// Match flow output columns whose names are not in the sink schema by the same position only.
+467 -18
View File
@@ -12,13 +12,19 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use std::collections::BTreeMap;
use std::sync::Arc;
use catalog::RegisterTableRequest;
use common_recordbatch::RecordBatch;
use common_query::OutputData;
use common_recordbatch::recordbatch::merge_record_batches;
use common_recordbatch::{RecordBatch, util};
use common_time::Timestamp;
use datafusion_common::ScalarValue;
use datafusion_common::tree_node::TreeNode as _;
use datafusion_expr::{GroupingSet, TableScanBuilder};
use datatypes::arrow::array::{Array, AsArray};
use datatypes::arrow::datatypes::{Float64Type, Int64Type, UInt64Type};
use datatypes::prelude::{ConcreteDataType, MutableVector, Scalar, ScalarVectorBuilder, VectorRef};
use datatypes::schema::{ColumnSchema, Schema};
use datatypes::timestamp::TimestampMillisecond;
@@ -739,6 +745,108 @@ async fn test_gen_plan_with_matching_schema_allows_null_positional_alias() {
assert!(sql.contains("NULL AS label"), "{sql}");
}
#[tokio::test]
async fn test_gen_plan_with_matching_schema_and_values_injects_physical_sink_order() {
let sink_schema = Arc::new(Schema::new(vec![
ColumnSchema::new("number", ConcreteDataType::uint32_datatype(), true),
ColumnSchema::new("marker", ConcreteDataType::uint8_datatype(), true),
ColumnSchema::new(
"ts",
ConcreteDataType::timestamp_millisecond_datatype(),
false,
)
.with_time_index(true),
ColumnSchema::new("epoch", ConcreteDataType::uint64_datatype(), true),
]));
let plan = gen_plan_with_matching_schema_and_values(
"SELECT number, ts FROM numbers_with_ts",
QueryContext::arc(),
create_test_query_engine(),
sink_schema,
&[],
false,
&BTreeMap::from([
("epoch".to_string(), ScalarValue::UInt64(Some(7))),
("marker".to_string(), ScalarValue::UInt8(None)),
]),
)
.await
.unwrap();
let names = plan
.schema()
.fields()
.iter()
.map(|field| field.name().clone())
.collect::<Vec<_>>();
assert_eq!(names, vec!["number", "marker", "ts", "epoch"]);
}
#[tokio::test]
async fn test_gen_plan_with_matching_schema_and_values_rejects_unknown_collision_and_type() {
let sink_schema = Arc::new(Schema::new(vec![
ColumnSchema::new("number", ConcreteDataType::uint32_datatype(), true),
ColumnSchema::new(
"ts",
ConcreteDataType::timestamp_millisecond_datatype(),
false,
)
.with_time_index(true),
ColumnSchema::new("marker", ConcreteDataType::uint8_datatype(), true),
]));
for (values, expected) in [
(
BTreeMap::from([("unknown".to_string(), ScalarValue::UInt8(None))]),
"is not found in sink schema",
),
(
BTreeMap::from([("number".to_string(), ScalarValue::UInt32(None))]),
"collides with flow output",
),
(
BTreeMap::from([("marker".to_string(), ScalarValue::UInt64(None))]),
"has type UInt64",
),
] {
let err = gen_plan_with_matching_schema_and_values(
"SELECT number, ts FROM numbers_with_ts",
QueryContext::arc(),
create_test_query_engine(),
sink_schema.clone(),
&[],
false,
&values,
)
.await
.unwrap_err()
.to_string();
assert!(err.contains(expected), "{err}");
}
}
#[tokio::test]
async fn test_gen_plan_with_matching_schema_and_values_rejects_duplicate_output() {
let sink_schema = Arc::new(Schema::new(vec![
ColumnSchema::new("number", ConcreteDataType::uint32_datatype(), true),
ColumnSchema::new("marker", ConcreteDataType::uint8_datatype(), true),
]));
let err = gen_plan_with_matching_schema_and_values(
"SELECT number, number AS number FROM numbers_with_ts",
QueryContext::arc(),
create_test_query_engine(),
sink_schema,
&[],
false,
&BTreeMap::from([("marker".to_string(), ScalarValue::UInt8(None))]),
)
.await
.unwrap_err();
let err = format!("{err:?}");
assert!(
err.contains("Projections require unique expression names"),
"DataFusion should reject duplicate output aliases during query planning: {err}"
);
}
#[tokio::test]
async fn test_gen_plan_with_matching_schema_accepts_matching_flow_schema() {
let query_engine = create_test_query_engine();
@@ -1104,13 +1212,16 @@ async fn test_rewrite_incremental_aggregate_allows_alias_wrapped_scan() {
let query_engine = create_test_query_engine();
let ctx = QueryContext::arc();
let sql = "SELECT max(n.number) AS number, n.ts FROM numbers_with_ts AS n GROUP BY n.ts";
let plan = sql_to_df_plan(ctx, query_engine, sql, false).await.unwrap();
let plan = sql_to_df_plan(ctx, query_engine.clone(), sql, false)
.await
.unwrap();
let analysis = analyze_incremental_aggregate_plan(&plan).unwrap().unwrap();
assert!(analysis.unsupported_exprs.is_empty());
let rewritten = rewrite_incremental_aggregate_with_sink_merge(
&plan,
&analysis,
&query_engine,
single_row_u32_table("alias_wrapped_sink", vec!["ts", "number"]),
&[
"greptime".to_string(),
@@ -1362,6 +1473,7 @@ async fn test_analyze_incremental_aggregate_plan_allows_literal_outputs() {
let rewritten = rewrite_incremental_aggregate_with_sink_merge(
&plan,
&analysis,
&query_engine,
sink_table.clone(),
&sink_table_name,
None,
@@ -1435,7 +1547,9 @@ async fn test_rewrite_incremental_aggregate_preserves_non_identifier_aliases() {
let query_engine = create_test_query_engine();
let ctx = QueryContext::arc();
let sql = "SELECT max(number) AS \"max value\", number, 42 AS \"literal value\" FROM numbers_with_ts GROUP BY number";
let plan = sql_to_df_plan(ctx, query_engine, sql, false).await.unwrap();
let plan = sql_to_df_plan(ctx, query_engine.clone(), sql, false)
.await
.unwrap();
let analysis = analyze_incremental_aggregate_plan(&plan).unwrap().unwrap();
assert!(analysis.unsupported_exprs.is_empty());
assert_eq!(
@@ -1447,6 +1561,7 @@ async fn test_rewrite_incremental_aggregate_preserves_non_identifier_aliases() {
let rewritten = rewrite_incremental_aggregate_with_sink_merge(
&plan,
&analysis,
&query_engine,
sink_table,
&[
"greptime".to_string(),
@@ -1520,24 +1635,34 @@ async fn test_datafusion_rejects_duplicate_output_names() {
}
#[tokio::test]
async fn test_analyze_incremental_aggregate_plan_rejects_same_aggregate_multiple_aliases() {
let query_engine = create_test_query_engine();
let ctx = QueryContext::arc();
let sql = "SELECT sum(number) AS a, sum(number) AS b, ts FROM numbers_with_ts GROUP BY ts";
let plan = sql_to_df_plan(ctx, query_engine, sql, false).await.unwrap();
async fn test_analyze_incremental_aggregate_plan_supports_same_aggregate_multiple_aliases() {
let analysis = analyze_test_sql(
"SELECT sum(number) AS a, sum(number) AS b, ts FROM numbers_with_ts GROUP BY ts",
)
.await;
let analysis = analyze_incremental_aggregate_plan(&plan).unwrap().unwrap();
assert!(analysis.unsupported_exprs.is_empty());
assert_eq!(analysis.merge_columns.len(), 2);
assert_eq!(analysis.merge_columns[0].input_field_name, "a");
assert_eq!(analysis.merge_columns[1].input_field_name, "a");
assert!(
analysis
.unsupported_exprs
.merge_columns
.iter()
.any(|expr| expr.contains("same aggregate output")
&& expr.contains("a")
&& expr.contains("b")),
"same aggregate with multiple aliases should be unsupported until explicit reproduction is implemented: {:?}",
analysis.unsupported_exprs
.all(|column| { column.merge_op == IncrementalAggregateMergeOp::Sum })
);
assert!(
analysis
.merge_columns
.iter()
.any(|column| column.output_field_name == "a")
);
assert!(
analysis
.merge_columns
.iter()
.any(|column| column.output_field_name == "b")
);
assert!(analysis.merge_columns.is_empty());
}
#[test]
@@ -1581,6 +1706,322 @@ async fn test_analyze_incremental_aggregate_plan_rejects_avg() {
assert!(!analysis.unsupported_exprs.is_empty());
}
#[tokio::test]
async fn test_rewrite_incremental_aggregate_merges_populated_mixed_state_families() {
let query_engine = create_test_query_engine();
let old_sql = "SELECT hll(CAST(number AS VARCHAR)) AS hll_a, hll(CAST(number AS VARCHAR)) AS hll_b, uddsketch_state(128, 0.000001, number) AS percentile_a, uddsketch_state(256, 0.02, number) AS percentile_b, stddev_pop_state(number) AS stddev_state, sum(number) AS total, CASE WHEN number <= 5 THEN CAST(NULL AS BIGINT) ELSE 2 END AS grp FROM numbers_with_ts WHERE number <= 3 OR number = 6 GROUP BY grp";
let new_sql = "SELECT hll(CAST(number AS VARCHAR)) AS hll_a, hll(CAST(number AS VARCHAR)) AS hll_b, uddsketch_state(128, 0.000001, number) AS percentile_a, uddsketch_state(256, 0.02, number) AS percentile_b, stddev_pop_state(number) AS stddev_state, sum(number) AS total, CASE WHEN number <= 5 THEN CAST(NULL AS BIGINT) WHEN number <= 8 THEN 2 ELSE 3 END AS grp FROM numbers_with_ts WHERE number >= 4 AND number != 6 GROUP BY grp";
let old_plan = sql_to_df_plan(QueryContext::arc(), query_engine.clone(), old_sql, false)
.await
.unwrap();
let old_output = query_engine
.execute(old_plan, QueryContext::arc())
.await
.unwrap();
let OutputData::Stream(old_stream) = old_output.data else {
panic!("expected old aggregate execution to be a stream");
};
let old_batches = util::collect(old_stream).await.unwrap();
let old_schema = old_batches.first().unwrap().schema.clone();
let old_batch = merge_record_batches(old_schema, &old_batches).unwrap();
assert_eq!(
old_batch.num_rows(),
2,
"old state must have one row per group"
);
let empty_state_sink_batch = old_batch.clone();
let sink_table = MemTable::table("state_merge_sink", old_batch);
let sink_table_name = [
"greptime".to_string(),
"public".to_string(),
"state_merge_sink".to_string(),
];
let new_plan = sql_to_df_plan(QueryContext::arc(), query_engine.clone(), new_sql, false)
.await
.unwrap();
let analysis = analyze_incremental_aggregate_plan(&new_plan)
.unwrap()
.unwrap();
assert!(analysis.unsupported_exprs.is_empty());
let rewritten = rewrite_incremental_aggregate_with_sink_merge(
&new_plan,
&analysis,
&query_engine,
sink_table,
&sink_table_name,
None,
)
.await
.unwrap();
assert_eq!(
analysis.output_field_names,
vec![
"hll_a",
"hll_b",
"percentile_a",
"percentile_b",
"stddev_state",
"total",
"grp",
],
"repeated HLL aliases must preserve output order"
);
let output = query_engine
.execute(rewritten, QueryContext::arc())
.await
.unwrap();
let OutputData::Stream(stream) = output.data else {
panic!("expected rewritten plan to execute as a stream");
};
let batches = util::collect(stream).await.unwrap();
let merged_schema = batches.first().unwrap().schema.clone();
let merged_batch = merge_record_batches(merged_schema, &batches).unwrap();
assert_eq!(
merged_batch.num_rows(),
3,
"rewrite must produce one row per group"
);
let merged_table = MemTable::table("merged_states", merged_batch);
query_engine
.engine_state()
.catalog_manager()
.as_any()
.downcast_ref::<catalog::memory::MemoryCatalogManager>()
.unwrap()
.register_table_sync(RegisterTableRequest {
catalog: "greptime".to_string(),
schema: "public".to_string(),
table_name: "merged_states".to_string(),
table_id: 4097,
table: merged_table,
})
.unwrap();
let checks = "SELECT grp, sum(total) AS total, hll_count(hll_merge(hll_a)) AS hll_a, hll_count(hll_merge(hll_b)) AS hll_b, stddev_pop_calc(stddev_pop_merge(stddev_state)) AS stddev, uddsketch_calc(0.5, uddsketch_merge(128, 0.000001, percentile_a)) AS p50_a, uddsketch_calc(0.5, uddsketch_merge(256, 0.02, percentile_b)) AS p50_b FROM merged_states GROUP BY grp ORDER BY grp NULLS FIRST";
let checks_plan = sql_to_df_plan(QueryContext::arc(), query_engine.clone(), checks, false)
.await
.unwrap();
let checks_output = query_engine
.execute(checks_plan, QueryContext::arc())
.await
.unwrap();
let OutputData::Stream(checks_stream) = checks_output.data else {
panic!("expected state check execution to be a stream");
};
let checks_batches = util::collect(checks_stream).await.unwrap();
let checks_schema = checks_batches.first().unwrap().schema.clone();
let checks = merge_record_batches(checks_schema, &checks_batches).unwrap();
assert_eq!(checks.num_rows(), 3);
assert_eq!(
checks
.schema
.column_schemas()
.iter()
.map(|column| column.name.as_str())
.collect::<Vec<_>>(),
vec!["grp", "total", "hll_a", "hll_b", "stddev", "p50_a", "p50_b"]
);
let group = checks.column(0).as_primitive::<Int64Type>();
let total = checks.column(1).as_primitive::<UInt64Type>();
let hll_a = checks.column(2).as_primitive::<UInt64Type>();
let hll_b = checks.column(3).as_primitive::<UInt64Type>();
let stddev = checks.column(4).as_primitive::<Float64Type>();
let p50_a = checks.column(5).as_primitive::<Float64Type>();
let p50_b = checks.column(6).as_primitive::<Float64Type>();
assert_eq!(group.null_count(), 1);
assert_eq!(total.null_count(), 0);
assert_eq!(hll_a.null_count(), 0);
assert_eq!(hll_b.null_count(), 0);
assert_eq!(stddev.null_count(), 0);
assert_eq!(p50_a.null_count(), 0);
assert_eq!(p50_b.null_count(), 0);
for (index, expected) in [
(None, 15_u64, 5_u64, 2_f64.sqrt(), 3_f64, 3_f64),
(
Some(2_i64),
21_u64,
3_u64,
(2_f64 / 3.0).sqrt(),
7_f64,
7_f64,
),
(Some(3_i64), 19_u64, 2_u64, 0.5_f64, 10_f64, 10_f64),
]
.into_iter()
.enumerate()
{
assert_eq!(
(!group.is_null(index)).then(|| group.value(index)),
expected.0
);
assert_eq!(total.value(index), expected.1);
assert_eq!(hll_a.value(index), expected.2);
assert_eq!(hll_b.value(index), expected.2);
assert!((stddev.value(index) - expected.3).abs() < 1e-12);
// UDDSketch's relative-error bounds are 1e-6 and 2e-2 respectively.
assert!((p50_a.value(index) - expected.4).abs() <= expected.4 * 0.000001);
assert!((p50_b.value(index) - expected.5).abs() <= expected.5 * 0.02);
}
let empty_state_plan = sql_to_df_plan(
QueryContext::arc(),
query_engine.clone(),
"SELECT hll(CAST(NULL AS VARCHAR)) AS hll_a, hll(CAST(NULL AS VARCHAR)) AS hll_b, uddsketch_state(128, 0.000001, CAST(NULL AS DOUBLE)) AS percentile_a, uddsketch_state(256, 0.02, CAST(NULL AS DOUBLE)) AS percentile_b, stddev_pop_state(CAST(NULL AS DOUBLE)) AS stddev_state, sum(number) AS total, CASE WHEN number <= 5 THEN CAST(NULL AS BIGINT) WHEN number <= 8 THEN 2 ELSE 3 END AS grp FROM numbers_with_ts WHERE number = 10 GROUP BY grp",
false,
)
.await
.unwrap();
let empty_state_analysis = analyze_incremental_aggregate_plan(&empty_state_plan)
.unwrap()
.unwrap();
assert!(empty_state_analysis.unsupported_exprs.is_empty());
let empty_state_rewritten = rewrite_incremental_aggregate_with_sink_merge(
&empty_state_plan,
&empty_state_analysis,
&query_engine,
MemTable::table("empty_state_sink", empty_state_sink_batch),
&[
"greptime".to_string(),
"public".to_string(),
"empty_state_sink".to_string(),
],
None,
)
.await
.unwrap();
let empty_state_output = query_engine
.execute(empty_state_rewritten, QueryContext::arc())
.await
.unwrap();
let OutputData::Stream(empty_state_stream) = empty_state_output.data else {
panic!("expected empty state merge to execute as a stream");
};
let empty_state_batches = util::collect(empty_state_stream).await.unwrap();
let empty_state_schema = empty_state_batches.first().unwrap().schema.clone();
let empty_state_batch = merge_record_batches(empty_state_schema, &empty_state_batches).unwrap();
assert_eq!(empty_state_batch.num_rows(), 1);
let group_index = empty_state_batch
.schema
.column_index_by_name("grp")
.unwrap();
let total_index = empty_state_batch
.schema
.column_index_by_name("total")
.unwrap();
let group = empty_state_batch
.column(group_index)
.as_primitive::<Int64Type>();
let total = empty_state_batch
.column(total_index)
.as_primitive::<UInt64Type>();
assert!(!group.is_null(0));
assert_eq!(group.value(0), 3);
assert!(!total.is_null(0));
assert_eq!(total.value(0), 10);
let empty_state_table = MemTable::table("empty_states", empty_state_batch);
query_engine
.engine_state()
.catalog_manager()
.as_any()
.downcast_ref::<catalog::memory::MemoryCatalogManager>()
.unwrap()
.register_table_sync(RegisterTableRequest {
catalog: "greptime".to_string(),
schema: "public".to_string(),
table_name: "empty_states".to_string(),
table_id: 4098,
table: empty_state_table,
})
.unwrap();
let empty_checks_plan = sql_to_df_plan(
QueryContext::arc(),
query_engine.clone(),
"SELECT hll_count(hll_merge(hll_a)) AS hll, stddev_pop_calc(stddev_pop_merge(stddev_state)) AS stddev, uddsketch_calc(0.5, uddsketch_merge(128, 0.000001, percentile_a)) AS p50_a, uddsketch_calc(0.5, uddsketch_merge(256, 0.02, percentile_b)) AS p50_b FROM empty_states",
false,
)
.await
.unwrap();
let empty_checks_output = query_engine
.execute(empty_checks_plan, QueryContext::arc())
.await
.unwrap();
let OutputData::Stream(empty_checks_stream) = empty_checks_output.data else {
panic!("expected empty state check to execute as a stream");
};
let empty_checks_batches = util::collect(empty_checks_stream).await.unwrap();
let empty_checks_schema = empty_checks_batches.first().unwrap().schema.clone();
let empty_checks = merge_record_batches(empty_checks_schema, &empty_checks_batches).unwrap();
assert_eq!(empty_checks.num_rows(), 1);
assert_eq!(
empty_checks.column(0).as_primitive::<UInt64Type>().value(0),
0
);
assert!(
empty_checks
.column(1)
.as_primitive::<Float64Type>()
.is_null(0)
);
assert!(
empty_checks
.column(2)
.as_primitive::<Float64Type>()
.is_null(0)
);
assert!(
empty_checks
.column(3)
.as_primitive::<Float64Type>()
.is_null(0)
);
}
#[tokio::test]
async fn test_analyze_incremental_aggregate_plan_state_producer_metadata_and_rejections() {
let analysis = analyze_test_sql(
"SELECT hll(CAST(number AS VARCHAR)) AS hll_state, \
stddev_pop_state(number) AS stddev_state, \
uddsketch_state(128, 0.000001, number) AS percentile_state, ts \
FROM numbers_with_ts GROUP BY ts",
)
.await;
assert!(analysis.unsupported_exprs.is_empty());
let percentile = analysis
.merge_columns
.iter()
.find(|column| column.output_field_name == "percentile_state")
.unwrap();
let IncrementalAggregateMergeOp::StateDeltaMerge {
function_name,
params,
} = &percentile.merge_op
else {
panic!("expected UDDSketch state delta merge");
};
assert_eq!(*function_name, "__uddsketch_state_delta_merge");
assert_eq!(params.len(), 2);
assert!(matches!(
params[0],
Expr::Literal(ScalarValue::Int64(Some(128)), _)
));
assert!(matches!(
params[1],
Expr::Literal(ScalarValue::Float64(Some(rate)), _) if rate == 0.000001
));
for sql in [
"SELECT uddsketch_state(CAST(number AS BIGINT), 0.01, number) AS state, ts FROM numbers_with_ts GROUP BY ts",
"SELECT uddsketch_state(128, NULL, number) AS state, ts FROM numbers_with_ts GROUP BY ts",
"SELECT uddsketch_state(NULL, 0.01, number) AS state, ts FROM numbers_with_ts GROUP BY ts",
"SELECT avg(number) AS state, ts FROM numbers_with_ts GROUP BY ts",
"SELECT stddev_pop(number) AS state, ts FROM numbers_with_ts GROUP BY ts",
] {
let analysis = analyze_test_sql(sql).await;
assert!(!analysis.unsupported_exprs.is_empty(), "must reject {sql}");
}
}
#[tokio::test]
async fn test_analyze_incremental_aggregate_plan_rejects_distinct() {
let query_engine = create_test_query_engine();
@@ -1638,6 +2079,7 @@ async fn test_rewrite_incremental_aggregate_with_left_join() {
let rewritten = rewrite_incremental_aggregate_with_sink_merge(
&plan,
&analysis,
&query_engine,
sink_table.clone(),
&sink_table_name,
None,
@@ -1699,6 +2141,7 @@ async fn test_rewrite_incremental_aggregate_filters_sink_dirty_time_window() {
let rewritten = rewrite_incremental_aggregate_with_sink_merge(
&plan,
&analysis,
&query_engine,
sink_table.clone(),
&sink_table_name,
Some(sink_filter.clone()),
@@ -1749,7 +2192,9 @@ async fn test_rewrite_incremental_aggregate_rejects_empty_group_keys() {
let query_engine = create_test_query_engine();
let ctx = QueryContext::arc();
let sql = "SELECT max(number) AS number FROM numbers_with_ts";
let plan = sql_to_df_plan(ctx, query_engine, sql, false).await.unwrap();
let plan = sql_to_df_plan(ctx, query_engine.clone(), sql, false)
.await
.unwrap();
let analysis = IncrementalAggregateAnalysis {
group_key_names: vec![],
merge_columns: vec![IncrementalAggregateMergeColumn::new(
@@ -1770,6 +2215,7 @@ async fn test_rewrite_incremental_aggregate_rejects_empty_group_keys() {
let err = rewrite_incremental_aggregate_with_sink_merge(
&plan,
&analysis,
&query_engine,
sink_table,
&sink_table_name,
None,
@@ -1788,7 +2234,9 @@ async fn test_rewrite_incremental_aggregate_preserves_raw_aggregate_field_name()
let query_engine = create_test_query_engine();
let ctx = QueryContext::arc();
let sql = "SELECT max(number), number FROM numbers_with_ts GROUP BY number";
let plan = sql_to_df_plan(ctx, query_engine, sql, false).await.unwrap();
let plan = sql_to_df_plan(ctx, query_engine.clone(), sql, false)
.await
.unwrap();
let analysis = analyze_incremental_aggregate_plan(&plan).unwrap().unwrap();
assert!(analysis.unsupported_exprs.is_empty());
@@ -1802,6 +2250,7 @@ async fn test_rewrite_incremental_aggregate_preserves_raw_aggregate_field_name()
let rewritten = rewrite_incremental_aggregate_with_sink_merge(
&plan,
&analysis,
&query_engine,
sink_table.clone(),
&sink_table_name,
None,
+11 -1
View File
@@ -38,7 +38,17 @@ mod test_utils;
pub use adapter::flownode_impl::FlowDualEngineRef;
pub use adapter::{FlowConfig, FlowStreamingEngineRef, StreamingEngine};
pub use batching_mode::frontend_client::{FrontendClient, GrpcQueryHandlerWithBoxedError};
pub use batching_mode::BatchingModeOptions;
pub use batching_mode::batching_execution::{BatchingExecution, BatchingExecutionFactory};
pub use batching_mode::frontend_client::{
FrontendClient, GrpcQueryHandlerWithBoxedError, PeerDesc,
};
pub use batching_mode::task::{
BatchingExecutionGuard, BatchingTask, DirtyRestore, ExecuteOnceOutcome, PlanInfo,
QueryCoverage, TaskArgs,
};
pub use batching_mode::time_window::{TimeWindowExpr, find_time_window_expr};
pub use batching_mode::utils::sql_to_df_plan;
pub(crate) use engine::{CreateFlowArgs, FlowId, TableName};
pub use error::{Error, Result};
pub use server::{FlownodeBuilder, FlownodeInstance, FlownodeServer, FlownodeServiceBuilder};
+3 -2
View File
@@ -47,7 +47,7 @@ use crate::error::{
use crate::heartbeat::HeartbeatTask;
use crate::metrics::{METRIC_FLOW_PROCESSING_TIME, METRIC_FLOW_ROWS};
use crate::utils::{SizeReportSender, StateReportHandler};
use crate::{Error, FlownodeOptions, FrontendClient, StreamingEngine};
use crate::{BatchingExecutionFactory, Error, FlownodeOptions, FrontendClient, StreamingEngine};
pub const FLOW_NODE_SERVER_NAME: &str = "FLOW_NODE_SERVER";
/// wrapping flow node manager to avoid orphan rule with Arc<...>
@@ -333,13 +333,14 @@ impl FlownodeBuilder {
context: "Failed to build query engine",
})?;
let manager = Arc::new(self.build_manager(query_engine_factory.query_engine()));
let batching = Arc::new(BatchingEngine::new(
let batching = Arc::new(BatchingEngine::new_with_execution(
self.frontend_client.clone(),
query_engine_factory.query_engine(),
self.flow_metadata_manager.clone(),
self.table_meta.clone(),
self.catalog_manager.clone(),
self.opts.flow.batching_mode.clone(),
self.plugins.get::<Arc<dyn BatchingExecutionFactory>>(),
));
let dual = Arc::new(FlowDualEngine::new(
manager.clone(),
+57 -9
View File
@@ -128,6 +128,7 @@ struct DdlSubmitOptions {
timeout: Duration,
}
#[cfg(not(feature = "enterprise"))]
const ALLOWED_FLOW_OPTIONS: [&str; 2] = [
DEFER_ON_MISSING_SOURCE_KEY,
FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY,
@@ -172,6 +173,7 @@ fn parse_ddl_options(options: &OptionMap) -> Result<DdlSubmitOptions> {
Ok(DdlSubmitOptions { wait, timeout })
}
#[cfg(not(feature = "enterprise"))]
fn supported_flow_options() -> String {
ALLOWED_FLOW_OPTIONS.join(", ")
}
@@ -221,6 +223,9 @@ fn validate_and_normalize_flow_options(
DEFER_ON_MISSING_SOURCE_KEY | FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY => {
normalize_flow_bool_option(&key, &value)?
}
#[cfg(feature = "enterprise")]
_ => value,
#[cfg(not(feature = "enterprise"))]
_ => {
return InvalidSqlSnafu {
err_msg: format!(
@@ -3284,6 +3289,7 @@ mod test {
);
}
#[cfg(not(feature = "enterprise"))]
#[test]
fn test_validate_and_normalize_flow_options_unknown_option() {
let err = validate_and_normalize_flow_options(
@@ -3333,7 +3339,7 @@ mod test {
}
#[test]
fn test_validate_and_normalize_flow_options_rejects_redacted_invalid_input() {
fn test_parser_flow_option_access_key_id() {
let sql = r"
CREATE FLOW task_6
SINK TO schema_1.table_1
@@ -3351,11 +3357,19 @@ SELECT max(c1), min(c2) FROM schema_2.table_2;";
};
let expr =
expr_helper::to_create_flow_task_expr(create_flow, &QueryContext::arc()).unwrap();
let err = validate_and_normalize_flow_options(expr.flow_options, None).unwrap_err();
assert!(
err.to_string()
.contains("unknown flow option 'access_key_id'")
#[cfg(not(feature = "enterprise"))]
{
let err = validate_and_normalize_flow_options(expr.flow_options, None).unwrap_err();
assert!(
err.to_string()
.contains("unknown flow option 'access_key_id'")
);
}
#[cfg(feature = "enterprise")]
assert_eq!(
validate_and_normalize_flow_options(expr.flow_options, None).unwrap(),
HashMap::from([("access_key_id".to_string(), "['true']".to_string())])
);
}
@@ -3448,7 +3462,18 @@ SELECT max(c1), min(c2) FROM schema_2.table_2;";
}
#[test]
fn test_schedule_and_internal_keys_rejected_as_unknown_options() {
fn test_internal_schedule_key_rejected() {
let err = validate_and_normalize_flow_options(
HashMap::from([(INTERNAL_EVAL_SCHEDULE_KEY.to_string(), "value".to_string())]),
Some(300),
)
.unwrap_err();
assert!(err.to_string().contains("reserved for internal use"));
}
#[cfg(not(feature = "enterprise"))]
#[test]
fn test_schedule_keys_rejected_as_unknown_options() {
for key in [
"eval_interval_anchor",
"eval_interval_start",
@@ -3461,15 +3486,38 @@ SELECT max(c1), min(c2) FROM schema_2.table_2;";
Some(300),
)
.unwrap_err();
assert!(
err.to_string()
.contains(&format!("unknown flow option '{key}'")),
"unexpected error for {key}: {err}"
.contains(&format!("unknown flow option '{key}'"))
);
}
}
#[cfg(feature = "enterprise")]
#[test]
fn test_schedule_keys_preserved() {
let options = HashMap::from([
("eval_interval_anchor".to_string(), "anchor".to_string()),
("eval_interval_start".to_string(), "start".to_string()),
(
"eval_interval_missed_tick_policy".to_string(),
"missed".to_string(),
),
(
"eval_interval_catchup_max_runs".to_string(),
"runs".to_string(),
),
(
"eval_interval_catchup_max_lag".to_string(),
"lag".to_string(),
),
]);
assert_eq!(
validate_and_normalize_flow_options(options.clone(), Some(300)).unwrap(),
options
);
}
#[test]
fn test_internal_transport_keys_rejected_as_reserved() {
for key in [
+14
View File
@@ -1,3 +1,17 @@
// Copyright 2023 Greptime Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// This file is copied from https://github.com/tokio-rs/axum/blob/axum-v0.6.20/axum/src/test_helpers/test_client.rs
//! Axum Test Client
+10 -16
View File
@@ -1,22 +1,16 @@
// Copyright (c) 2019 Stepan Koltsov
// Copyright 2023 Greptime Team
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
// http://www.apache.org/licenses/LICENSE-2.0
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
// OR OTHER DEALINGS IN THE SOFTWARE.
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// The Clear trait is copied from https://github.com/stepancheg/rust-protobuf/blob/v2.28.0/protobuf/src/clear.rs
// The RepeatedField struct is copied from https://github.com/stepancheg/rust-protobuf/blob/v2.28.0/protobuf/src/repeated.rs
@@ -0,0 +1,348 @@
-- Incremental aggregate state merge over a partitioned source (two regions).
-- A WHERE-filtered flow mixes scalar aggregates (sum, count) with aggregate
-- state columns (hll, stddev_pop_state, uddsketch_state); each later delta in
-- the same time window must merge its state with the state already persisted
-- in the sink. Covers two nonempty deltas across both regions, one round whose
-- delta is empty because the flow's predicate drops the newly inserted row,
-- and a final nonempty delta checked against a direct source aggregation.
CREATE TABLE flow_incr_state_merge_input (
host_id INT,
user_id BIGINT,
n INT,
v DOUBLE,
ts TIMESTAMP TIME INDEX,
PRIMARY KEY(host_id, user_id)
)
PARTITION ON COLUMNS (host_id) (
host_id < 3,
host_id >= 3
)
WITH (
append_mode = 'true'
);
Affected Rows: 0
CREATE FLOW flow_incr_state_merge SINK TO flow_incr_state_merge_sink
WITH (experimental_enable_incremental_read = 'true')
AS
SELECT
sum(n) AS total,
count(n) AS row_count,
hll(user_id) AS user_hll,
stddev_pop_state(v) AS v_stddev_state,
uddsketch_state(128, 0.01, v) AS v_sketch_state,
date_bin(INTERVAL '1 minute', ts, '2024-01-01 00:00:00') AS time_window
FROM
flow_incr_state_merge_input
WHERE
v > 0
GROUP BY
time_window;
Affected Rows: 0
-- ==== Phase 1: initial rows in the same time window, one per region ====
INSERT INTO flow_incr_state_merge_input VALUES
(1, 1, 1, 1.0, '2024-01-01 00:00:00'),
(3, 2, 2, 2.0, '2024-01-01 00:00:10');
Affected Rows: 2
-- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED |
ADMIN FLUSH_FLOW('flow_incr_state_merge');
+-------------------------------------------+
| ADMIN FLUSH_FLOW('flow_incr_state_merge') |
+-------------------------------------------+
| FLOW_FLUSHED |
+-------------------------------------------+
-- The first round seeds the sink with a full-snapshot aggregate state.
SELECT
total,
row_count,
time_window
FROM
flow_incr_state_merge_sink
ORDER BY
time_window;
+-------+-----------+---------------------+
| total | row_count | time_window |
+-------+-----------+---------------------+
| 3 | 2 | 2024-01-01T00:00:00 |
+-------+-----------+---------------------+
-- Move the checkpointed source rows and the seeded sink state into SST, so
-- the next round has to merge a new delta with persisted state.
ADMIN FLUSH_TABLE('flow_incr_state_merge_input');
+--------------------------------------------------+
| ADMIN FLUSH_TABLE('flow_incr_state_merge_input') |
+--------------------------------------------------+
| 0 |
+--------------------------------------------------+
ADMIN FLUSH_TABLE('flow_incr_state_merge_sink');
+-------------------------------------------------+
| ADMIN FLUSH_TABLE('flow_incr_state_merge_sink') |
+-------------------------------------------------+
| 0 |
+-------------------------------------------------+
-- ==== Phase 2: first nonempty delta, same window, both regions ====
INSERT INTO flow_incr_state_merge_input VALUES
(2, 3, 3, 3.0, '2024-01-01 00:00:20'),
(4, 4, 4, 4.0, '2024-01-01 00:00:40');
Affected Rows: 2
-- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED |
ADMIN FLUSH_FLOW('flow_incr_state_merge');
+-------------------------------------------+
| ADMIN FLUSH_FLOW('flow_incr_state_merge') |
+-------------------------------------------+
| FLOW_FLUSHED |
+-------------------------------------------+
-- Scalars accumulate; each state column must merge with the persisted state.
SELECT
total,
row_count,
hll_count(hll_merge(user_hll)) AS users,
ROUND(stddev_pop_calc(stddev_pop_merge(v_stddev_state)), 6) AS v_stddev,
ROUND(uddsketch_calc(0.5, uddsketch_merge(128, 0.01, v_sketch_state)), 4) AS v_p50,
time_window
FROM
flow_incr_state_merge_sink
GROUP BY
total,
row_count,
time_window
ORDER BY
time_window;
+-------+-----------+-------+----------+--------+---------------------+
| total | row_count | users | v_stddev | v_p50 | time_window |
+-------+-----------+-------+----------+--------+---------------------+
| 10 | 4 | 4 | 1.118034 | 2.9742 | 2024-01-01T00:00:00 |
+-------+-----------+-------+----------+--------+---------------------+
ADMIN FLUSH_TABLE('flow_incr_state_merge_input');
+--------------------------------------------------+
| ADMIN FLUSH_TABLE('flow_incr_state_merge_input') |
+--------------------------------------------------+
| 0 |
+--------------------------------------------------+
ADMIN FLUSH_TABLE('flow_incr_state_merge_sink');
+-------------------------------------------------+
| ADMIN FLUSH_TABLE('flow_incr_state_merge_sink') |
+-------------------------------------------------+
| 0 |
+-------------------------------------------------+
-- ==== Phase 3: filtered-out row must run one round with an empty delta ====
-- Record the flow's last execution time before the excluded insert: the
-- comparison below proves the empty delta was actually evaluated.
CREATE TABLE flow_incr_state_merge_last_exec (
marker STRING,
last_exec TIMESTAMP TIME INDEX
) WITH (
append_mode = 'true'
);
Affected Rows: 0
-- The background loop runs on its own interval; keep waiting until it has
-- executed at least once so information_schema.flows.last_execution_time is
-- non-null before snapshotting it.
-- SQLNESS SLEEP 8s
INSERT INTO flow_incr_state_merge_last_exec (marker, last_exec)
SELECT
'before_empty_delta',
last_execution_time
FROM
information_schema.flows
WHERE
flow_name = 'flow_incr_state_merge';
Affected Rows: 1
-- The row's timestamp is inside the existing window, so the source write still
-- dirties that window; the flow's WHERE predicate then drops the row from the
-- delta, leaving the delta empty for the window.
INSERT INTO flow_incr_state_merge_input VALUES
(1, 9, 9, -1.0, '2024-01-01 00:00:30');
Affected Rows: 1
-- SQLNESS SLEEP 6s
-- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED |
ADMIN FLUSH_FLOW('flow_incr_state_merge');
+-------------------------------------------+
| ADMIN FLUSH_FLOW('flow_incr_state_merge') |
+-------------------------------------------+
| FLOW_FLUSHED |
+-------------------------------------------+
-- An empty delta must leave the persisted sink states untouched.
SELECT
total,
row_count,
hll_count(hll_merge(user_hll)) AS users,
ROUND(stddev_pop_calc(stddev_pop_merge(v_stddev_state)), 6) AS v_stddev,
ROUND(uddsketch_calc(0.5, uddsketch_merge(128, 0.01, v_sketch_state)), 4) AS v_p50,
time_window
FROM
flow_incr_state_merge_sink
GROUP BY
total,
row_count,
time_window
ORDER BY
time_window;
+-------+-----------+-------+----------+--------+---------------------+
| total | row_count | users | v_stddev | v_p50 | time_window |
+-------+-----------+-------+----------+--------+---------------------+
| 10 | 4 | 4 | 1.118034 | 2.9742 | 2024-01-01T00:00:00 |
+-------+-----------+-------+----------+--------+---------------------+
-- last_execution_time must have advanced past the mark taken above, proving
-- the dirty window triggered a new round even though its delta was empty. A
-- NULL comparison means the mark never advanced: fail explicitly instead of
-- silently treating it as "unchanged".
INSERT INTO flow_incr_state_merge_last_exec (marker, last_exec)
SELECT
'after_empty_delta',
last_execution_time
FROM
information_schema.flows
WHERE
flow_name = 'flow_incr_state_merge';
Affected Rows: 1
SELECT
(
SELECT
last_exec
FROM
flow_incr_state_merge_last_exec
WHERE
marker = 'after_empty_delta'
) > (
SELECT
last_exec
FROM
flow_incr_state_merge_last_exec
WHERE
marker = 'before_empty_delta'
) AS empty_delta_executed;
+----------------------+
| empty_delta_executed |
+----------------------+
| true |
+----------------------+
-- Flush the input before the final delta so each phase's delta is isolated in
-- its own round; the background loop may have already consumed the last
-- insert, which is fine — the values below only depend on the merge outcome.
ADMIN FLUSH_TABLE('flow_incr_state_merge_input');
+--------------------------------------------------+
| ADMIN FLUSH_TABLE('flow_incr_state_merge_input') |
+--------------------------------------------------+
| 0 |
+--------------------------------------------------+
ADMIN FLUSH_TABLE('flow_incr_state_merge_sink');
+-------------------------------------------------+
| ADMIN FLUSH_TABLE('flow_incr_state_merge_sink') |
+-------------------------------------------------+
| 0 |
+-------------------------------------------------+
-- ==== Phase 4: final nonempty delta, same window, both regions ====
INSERT INTO flow_incr_state_merge_input VALUES
(1, 5, 5, 5.0, '2024-01-01 00:00:05'),
(3, 6, 6, 6.0, '2024-01-01 00:00:55');
Affected Rows: 2
-- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED |
ADMIN FLUSH_FLOW('flow_incr_state_merge');
+-------------------------------------------+
| ADMIN FLUSH_FLOW('flow_incr_state_merge') |
+-------------------------------------------+
| FLOW_FLUSHED |
+-------------------------------------------+
-- Merge the final delta with the persisted states and compare the merged
-- result against a direct aggregation over the accepted source rows.
SELECT
merged.total = direct.total AS total_matches,
merged.row_count = direct.row_count AS count_matches,
merged.users = direct.users AS hll_matches,
abs(merged.v_stddev - direct.v_stddev) < 1e-9 AS stddev_matches,
ROUND(merged.v_p50, 4) = ROUND(direct.v_p50, 4) AS p50_matches
FROM
(
SELECT
total,
row_count,
hll_count(hll_merge(user_hll)) AS users,
stddev_pop_calc(stddev_pop_merge(v_stddev_state)) AS v_stddev,
uddsketch_calc(0.5, uddsketch_merge(128, 0.01, v_sketch_state)) AS v_p50
FROM
flow_incr_state_merge_sink
GROUP BY
total,
row_count
) AS merged,
(
SELECT
sum(n) AS total,
count(n) AS row_count,
hll_count(hll(user_id)) AS users,
stddev_pop(v) AS v_stddev,
uddsketch_calc(0.5, uddsketch_state(128, 0.01, v)) AS v_p50
FROM
flow_incr_state_merge_input
WHERE
v > 0
) AS direct;
+---------------+---------------+-------------+----------------+-------------+
| total_matches | count_matches | hll_matches | stddev_matches | p50_matches |
+---------------+---------------+-------------+----------------+-------------+
| true | true | true | true | true |
+---------------+---------------+-------------+----------------+-------------+
-- Clean up
DROP FLOW flow_incr_state_merge;
Affected Rows: 0
DROP TABLE flow_incr_state_merge_input;
Affected Rows: 0
DROP TABLE flow_incr_state_merge_sink;
Affected Rows: 0
DROP TABLE flow_incr_state_merge_last_exec;
Affected Rows: 0
@@ -0,0 +1,232 @@
-- Incremental aggregate state merge over a partitioned source (two regions).
-- A WHERE-filtered flow mixes scalar aggregates (sum, count) with aggregate
-- state columns (hll, stddev_pop_state, uddsketch_state); each later delta in
-- the same time window must merge its state with the state already persisted
-- in the sink. Covers two nonempty deltas across both regions, one round whose
-- delta is empty because the flow's predicate drops the newly inserted row,
-- and a final nonempty delta checked against a direct source aggregation.
CREATE TABLE flow_incr_state_merge_input (
host_id INT,
user_id BIGINT,
n INT,
v DOUBLE,
ts TIMESTAMP TIME INDEX,
PRIMARY KEY(host_id, user_id)
)
PARTITION ON COLUMNS (host_id) (
host_id < 3,
host_id >= 3
)
WITH (
append_mode = 'true'
);
CREATE FLOW flow_incr_state_merge SINK TO flow_incr_state_merge_sink
WITH (experimental_enable_incremental_read = 'true')
AS
SELECT
sum(n) AS total,
count(n) AS row_count,
hll(user_id) AS user_hll,
stddev_pop_state(v) AS v_stddev_state,
uddsketch_state(128, 0.01, v) AS v_sketch_state,
date_bin(INTERVAL '1 minute', ts, '2024-01-01 00:00:00') AS time_window
FROM
flow_incr_state_merge_input
WHERE
v > 0
GROUP BY
time_window;
-- ==== Phase 1: initial rows in the same time window, one per region ====
INSERT INTO flow_incr_state_merge_input VALUES
(1, 1, 1, 1.0, '2024-01-01 00:00:00'),
(3, 2, 2, 2.0, '2024-01-01 00:00:10');
-- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED |
ADMIN FLUSH_FLOW('flow_incr_state_merge');
-- The first round seeds the sink with a full-snapshot aggregate state.
SELECT
total,
row_count,
time_window
FROM
flow_incr_state_merge_sink
ORDER BY
time_window;
-- Move the checkpointed source rows and the seeded sink state into SST, so
-- the next round has to merge a new delta with persisted state.
ADMIN FLUSH_TABLE('flow_incr_state_merge_input');
ADMIN FLUSH_TABLE('flow_incr_state_merge_sink');
-- ==== Phase 2: first nonempty delta, same window, both regions ====
INSERT INTO flow_incr_state_merge_input VALUES
(2, 3, 3, 3.0, '2024-01-01 00:00:20'),
(4, 4, 4, 4.0, '2024-01-01 00:00:40');
-- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED |
ADMIN FLUSH_FLOW('flow_incr_state_merge');
-- Scalars accumulate; each state column must merge with the persisted state.
SELECT
total,
row_count,
hll_count(hll_merge(user_hll)) AS users,
ROUND(stddev_pop_calc(stddev_pop_merge(v_stddev_state)), 6) AS v_stddev,
ROUND(uddsketch_calc(0.5, uddsketch_merge(128, 0.01, v_sketch_state)), 4) AS v_p50,
time_window
FROM
flow_incr_state_merge_sink
GROUP BY
total,
row_count,
time_window
ORDER BY
time_window;
ADMIN FLUSH_TABLE('flow_incr_state_merge_input');
ADMIN FLUSH_TABLE('flow_incr_state_merge_sink');
-- ==== Phase 3: filtered-out row must run one round with an empty delta ====
-- Record the flow's last execution time before the excluded insert: the
-- comparison below proves the empty delta was actually evaluated.
CREATE TABLE flow_incr_state_merge_last_exec (
marker STRING,
last_exec TIMESTAMP TIME INDEX
) WITH (
append_mode = 'true'
);
-- The background loop runs on its own interval; keep waiting until it has
-- executed at least once so information_schema.flows.last_execution_time is
-- non-null before snapshotting it.
-- SQLNESS SLEEP 8s
INSERT INTO flow_incr_state_merge_last_exec (marker, last_exec)
SELECT
'before_empty_delta',
last_execution_time
FROM
information_schema.flows
WHERE
flow_name = 'flow_incr_state_merge';
-- The row's timestamp is inside the existing window, so the source write still
-- dirties that window; the flow's WHERE predicate then drops the row from the
-- delta, leaving the delta empty for the window.
INSERT INTO flow_incr_state_merge_input VALUES
(1, 9, 9, -1.0, '2024-01-01 00:00:30');
-- SQLNESS SLEEP 6s
-- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED |
ADMIN FLUSH_FLOW('flow_incr_state_merge');
-- An empty delta must leave the persisted sink states untouched.
SELECT
total,
row_count,
hll_count(hll_merge(user_hll)) AS users,
ROUND(stddev_pop_calc(stddev_pop_merge(v_stddev_state)), 6) AS v_stddev,
ROUND(uddsketch_calc(0.5, uddsketch_merge(128, 0.01, v_sketch_state)), 4) AS v_p50,
time_window
FROM
flow_incr_state_merge_sink
GROUP BY
total,
row_count,
time_window
ORDER BY
time_window;
-- last_execution_time must have advanced past the mark taken above, proving
-- the dirty window triggered a new round even though its delta was empty. A
-- NULL comparison means the mark never advanced: fail explicitly instead of
-- silently treating it as "unchanged".
INSERT INTO flow_incr_state_merge_last_exec (marker, last_exec)
SELECT
'after_empty_delta',
last_execution_time
FROM
information_schema.flows
WHERE
flow_name = 'flow_incr_state_merge';
SELECT
(
SELECT
last_exec
FROM
flow_incr_state_merge_last_exec
WHERE
marker = 'after_empty_delta'
) > (
SELECT
last_exec
FROM
flow_incr_state_merge_last_exec
WHERE
marker = 'before_empty_delta'
) AS empty_delta_executed;
-- Flush the input before the final delta so each phase's delta is isolated in
-- its own round; the background loop may have already consumed the last
-- insert, which is fine — the values below only depend on the merge outcome.
ADMIN FLUSH_TABLE('flow_incr_state_merge_input');
ADMIN FLUSH_TABLE('flow_incr_state_merge_sink');
-- ==== Phase 4: final nonempty delta, same window, both regions ====
INSERT INTO flow_incr_state_merge_input VALUES
(1, 5, 5, 5.0, '2024-01-01 00:00:05'),
(3, 6, 6, 6.0, '2024-01-01 00:00:55');
-- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED |
ADMIN FLUSH_FLOW('flow_incr_state_merge');
-- Merge the final delta with the persisted states and compare the merged
-- result against a direct aggregation over the accepted source rows.
SELECT
merged.total = direct.total AS total_matches,
merged.row_count = direct.row_count AS count_matches,
merged.users = direct.users AS hll_matches,
abs(merged.v_stddev - direct.v_stddev) < 1e-9 AS stddev_matches,
ROUND(merged.v_p50, 4) = ROUND(direct.v_p50, 4) AS p50_matches
FROM
(
SELECT
total,
row_count,
hll_count(hll_merge(user_hll)) AS users,
stddev_pop_calc(stddev_pop_merge(v_stddev_state)) AS v_stddev,
uddsketch_calc(0.5, uddsketch_merge(128, 0.01, v_sketch_state)) AS v_p50
FROM
flow_incr_state_merge_sink
GROUP BY
total,
row_count
) AS merged,
(
SELECT
sum(n) AS total,
count(n) AS row_count,
hll_count(hll(user_id)) AS users,
stddev_pop(v) AS v_stddev,
uddsketch_calc(0.5, uddsketch_state(128, 0.01, v)) AS v_p50
FROM
flow_incr_state_merge_input
WHERE
v > 0
) AS direct;
-- Clean up
DROP FLOW flow_incr_state_merge;
DROP TABLE flow_incr_state_merge_input;
DROP TABLE flow_incr_state_merge_sink;
DROP TABLE flow_incr_state_merge_last_exec;