mirror of
https://github.com/GreptimeTeam/greptimedb.git
synced 2026-11-18 00:34:11 +00:00
feat(flow): expose extension-owned batching execution hooks (#9171)
* fix: preserve structured query errors through distributed execution Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> * feat: use exact sequence ranges for capable incremental flow sources Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> * test: update SQL expectations for preserved query error codes Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> * style: format exact sequence recovery tests Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> * feat: merge aggregate states in incremental flows Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> * test: exercise dispatched exact delta failure recovery Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> * test: record flushed exact sequence flow results Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> * test: pass query engine to state merge execution tests Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> * feat(flow): expose extension-owned batching execution hooks Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> * fix(flow): correct extension matcher borrowing and regression assertions Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> * test: prove executed empty exact deltas keep incremental mode Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> * test: pass query engine to empty exact delta regression Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> * test: cover aggregate state merge through the full flow path Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> * test: pass query engine to flow batching task regression calls Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> --------- Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com>
This commit is contained in:
@@ -0,0 +1,10 @@
|
||||
Auto-merging src/flow/src/batching_mode/frontend_client.rs
|
||||
Auto-merging src/flow/src/lib.rs
|
||||
CONFLICT (content): Merge conflict in src/flow/src/lib.rs
|
||||
Auto-merging src/flow/src/server.rs
|
||||
CONFLICT (content): Merge conflict in src/flow/src/server.rs
|
||||
Auto-merging src/operator/src/statement/ddl.rs
|
||||
Recorded preimage for 'src/flow/src/lib.rs'
|
||||
Recorded preimage for 'src/flow/src/server.rs'
|
||||
Automatic merge failed; fix conflicts and then commit the result.
|
||||
exit=1
|
||||
@@ -1,4 +1,16 @@
|
||||
// Copyright (c) 2017-present, PingCAP, Inc. Licensed under Apache-2.0.
|
||||
// Copyright 2023 Greptime Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// This file is copied from https://github.com/tikv/raft-engine/blob/0.3.0/src/util.rs
|
||||
|
||||
|
||||
@@ -1,3 +1,17 @@
|
||||
// Copyright 2023 Greptime Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// This file is copied from https://github.com/iqlusioninc/crates/blob/f98d4ccf/secrecy/src/lib.rs
|
||||
|
||||
//! [`SecretBox`] wrapper type for more carefully handling secret values
|
||||
|
||||
@@ -38,7 +38,10 @@ use crate::ddl::alter_logical_tables::AlterLogicalTablesProcedure;
|
||||
use crate::ddl::alter_table::{AlterTableProcedure, RegionRouteChanged, only_sets_skip_wal};
|
||||
use crate::ddl::comment_on::CommentOnProcedure;
|
||||
use crate::ddl::create_database::{CreateDatabaseMetadataCommitterRef, CreateDatabaseProcedure};
|
||||
use crate::ddl::create_flow::CreateFlowProcedure;
|
||||
use crate::ddl::create_flow::{
|
||||
CreateFlowProcedure, FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY,
|
||||
FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_SEQUENCE_RANGE,
|
||||
};
|
||||
use crate::ddl::create_logical_tables::CreateLogicalTablesProcedure;
|
||||
use crate::ddl::create_table::CreateTableProcedure;
|
||||
use crate::ddl::create_view::CreateViewProcedure;
|
||||
@@ -110,6 +113,10 @@ pub struct DdlManager {
|
||||
repartition_procedure_factory: RepartitionProcedureFactoryRef,
|
||||
#[cfg(feature = "enterprise")]
|
||||
trigger_ddl_manager: Option<TriggerDdlManagerRef>,
|
||||
#[cfg(feature = "enterprise")]
|
||||
create_flow_handler: Option<CreateFlowHandlerRef>,
|
||||
#[cfg(feature = "enterprise")]
|
||||
drop_flow_handler: Option<DropFlowHandlerRef>,
|
||||
}
|
||||
|
||||
/// This trait is responsible for handling DDL tasks about triggers. e.g.,
|
||||
@@ -141,6 +148,39 @@ pub trait TriggerDdlManager: Send + Sync {
|
||||
#[cfg(feature = "enterprise")]
|
||||
pub type TriggerDdlManagerRef = Arc<dyn TriggerDdlManager>;
|
||||
|
||||
/// This trait is responsible for handling custom/extension CREATE FLOW tasks.
|
||||
#[cfg(feature = "enterprise")]
|
||||
#[async_trait::async_trait]
|
||||
pub trait CreateFlowHandler: Send + Sync {
|
||||
async fn create_flow(
|
||||
&self,
|
||||
create_flow_task: CreateFlowTask,
|
||||
procedure_manager: ProcedureManagerRef,
|
||||
ddl_context: DdlContext,
|
||||
query_context: QueryContext,
|
||||
procedure_context: ProcedureContext,
|
||||
) -> Result<SubmitDdlTaskResponse>;
|
||||
}
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
pub type CreateFlowHandlerRef = Arc<dyn CreateFlowHandler>;
|
||||
|
||||
/// Hook for classifying and handling DROP FLOW requests.
|
||||
#[cfg(feature = "enterprise")]
|
||||
#[async_trait::async_trait]
|
||||
pub trait DropFlowHandler: Send + Sync {
|
||||
async fn drop_flow(
|
||||
&self,
|
||||
drop_flow_task: DropFlowTask,
|
||||
procedure_manager: ProcedureManagerRef,
|
||||
ddl_context: DdlContext,
|
||||
procedure_context: ProcedureContext,
|
||||
) -> Result<SubmitDdlTaskResponse>;
|
||||
}
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
pub type DropFlowHandlerRef = Arc<dyn DropFlowHandler>;
|
||||
|
||||
macro_rules! procedure_loader_entry {
|
||||
($procedure:ident) => {
|
||||
(
|
||||
@@ -235,15 +275,31 @@ impl DdlManager {
|
||||
repartition_procedure_factory,
|
||||
#[cfg(feature = "enterprise")]
|
||||
trigger_ddl_manager: None,
|
||||
#[cfg(feature = "enterprise")]
|
||||
create_flow_handler: None,
|
||||
#[cfg(feature = "enterprise")]
|
||||
drop_flow_handler: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
pub fn with_drop_flow_handler(mut self, drop_flow_handler: DropFlowHandlerRef) -> Self {
|
||||
self.drop_flow_handler = Some(drop_flow_handler);
|
||||
self
|
||||
}
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
pub fn with_trigger_ddl_manager(mut self, trigger_ddl_manager: TriggerDdlManagerRef) -> Self {
|
||||
self.trigger_ddl_manager = Some(trigger_ddl_manager);
|
||||
self
|
||||
}
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
pub fn with_create_flow_handler(mut self, create_flow_handler: CreateFlowHandlerRef) -> Self {
|
||||
self.create_flow_handler = Some(create_flow_handler);
|
||||
self
|
||||
}
|
||||
|
||||
pub fn with_create_database_metadata_committer(
|
||||
mut self,
|
||||
committer: CreateDatabaseMetadataCommitterRef,
|
||||
@@ -965,6 +1021,17 @@ impl DdlManager {
|
||||
.await
|
||||
}
|
||||
DropFlow(drop_flow_task) => {
|
||||
#[cfg(feature = "enterprise")]
|
||||
if let Some(handler) = self.drop_flow_handler.as_ref() {
|
||||
return handler
|
||||
.drop_flow(
|
||||
drop_flow_task,
|
||||
self.procedure_manager.clone(),
|
||||
self.ddl_context.clone(),
|
||||
procedure_context,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
handle_drop_flow_task(self, drop_flow_task, procedure_context).await
|
||||
}
|
||||
CreateView(create_view_task) => {
|
||||
@@ -1365,6 +1432,32 @@ async fn handle_create_flow_task(
|
||||
query_context: QueryContext,
|
||||
procedure_context: ProcedureContext,
|
||||
) -> Result<SubmitDdlTaskResponse> {
|
||||
if create_flow_task
|
||||
.flow_options
|
||||
.get(FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY)
|
||||
.is_some_and(|value| value == FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_SEQUENCE_RANGE)
|
||||
{
|
||||
return error::UnexpectedSnafu {
|
||||
err_msg: format!(
|
||||
"reserved flow option value for {FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY} is internal"
|
||||
),
|
||||
}
|
||||
.fail();
|
||||
}
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
if let Some(handler) = ddl_manager.create_flow_handler.as_ref() {
|
||||
return handler
|
||||
.create_flow(
|
||||
create_flow_task,
|
||||
ddl_manager.procedure_manager.clone(),
|
||||
ddl_manager.ddl_context.clone(),
|
||||
query_context,
|
||||
procedure_context,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
let (id, output) = ddl_manager
|
||||
.submit_create_flow_task(create_flow_task.clone(), query_context, procedure_context)
|
||||
.await?;
|
||||
@@ -1538,6 +1631,11 @@ mod tests {
|
||||
use crate::ddl::create_database::{
|
||||
AtomicCreateOutcome, CreateDatabaseMetadataCommitter, CreateDatabaseProcedure,
|
||||
};
|
||||
#[cfg(feature = "enterprise")]
|
||||
use crate::ddl::create_flow::{
|
||||
CreateFlowProcedure, FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY,
|
||||
FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_SEQUENCE_RANGE,
|
||||
};
|
||||
use crate::ddl::create_table::CreateTableProcedure;
|
||||
use crate::ddl::drop_table::DropTableProcedure;
|
||||
use crate::ddl::flow_meta::FlowMetadataAllocator;
|
||||
@@ -1555,11 +1653,13 @@ mod tests {
|
||||
use crate::region_registry::LeaderRegionRegistry;
|
||||
#[cfg(feature = "enterprise")]
|
||||
use crate::rpc::ddl::trigger::{CreateTriggerTask, DropTriggerTask};
|
||||
#[cfg(feature = "enterprise")]
|
||||
use crate::rpc::ddl::{
|
||||
CreateFlowTask, DdlTask, QueryContext, SubmitDdlTaskRequest, SubmitDdlTaskResponse,
|
||||
};
|
||||
use crate::rpc::ddl::{CreatorGrantIntent, UndropTableTask};
|
||||
#[cfg(not(feature = "enterprise"))]
|
||||
use crate::rpc::ddl::{DdlTask, PurgeDroppedTableTask, QueryContext, SubmitDdlTaskRequest};
|
||||
#[cfg(feature = "enterprise")]
|
||||
use crate::rpc::ddl::{DdlTask, QueryContext, SubmitDdlTaskRequest};
|
||||
use crate::sequence::SequenceBuilder;
|
||||
use crate::state_store::KvStateStore;
|
||||
use crate::test_util::{MockDatanodeManager, new_ddl_context};
|
||||
@@ -1626,6 +1726,60 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
#[derive(Default)]
|
||||
struct RecordingCreateFlowHandler {
|
||||
tasks: Mutex<Vec<CreateFlowTask>>,
|
||||
contexts: Mutex<Vec<(QueryContext, ProcedureContext)>>,
|
||||
response: Mutex<Option<SubmitDdlTaskResponse>>,
|
||||
fail: bool,
|
||||
}
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
#[async_trait::async_trait]
|
||||
impl super::CreateFlowHandler for RecordingCreateFlowHandler {
|
||||
async fn create_flow(
|
||||
&self,
|
||||
create_flow_task: CreateFlowTask,
|
||||
_procedure_manager: ProcedureManagerRef,
|
||||
_ddl_context: DdlContext,
|
||||
query_context: QueryContext,
|
||||
procedure_context: ProcedureContext,
|
||||
) -> crate::error::Result<SubmitDdlTaskResponse> {
|
||||
self.tasks.lock().unwrap().push(create_flow_task);
|
||||
self.contexts
|
||||
.lock()
|
||||
.unwrap()
|
||||
.push((query_context, procedure_context));
|
||||
if self.fail {
|
||||
return crate::error::UnsupportedSnafu {
|
||||
operation: "test create flow handler",
|
||||
}
|
||||
.fail();
|
||||
}
|
||||
Ok(self.response.lock().unwrap().take().unwrap_or_default())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
fn test_create_flow_task() -> CreateFlowTask {
|
||||
CreateFlowTask {
|
||||
catalog_name: "greptime".to_string(),
|
||||
flow_name: "test_flow".to_string(),
|
||||
source_table_names: vec![],
|
||||
sink_table_name: TableName::new("greptime", "public", "sink"),
|
||||
or_replace: false,
|
||||
create_if_not_exists: false,
|
||||
expire_after: None,
|
||||
eval_interval_secs: None,
|
||||
eval_offset_secs: None,
|
||||
comment: String::new(),
|
||||
sql: "select 1".to_string(),
|
||||
flow_options: Default::default(),
|
||||
eval_schedule: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
#[derive(Default)]
|
||||
struct RecordingTriggerDdlManager {
|
||||
@@ -1805,6 +1959,127 @@ mod tests {
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
#[tokio::test]
|
||||
async fn test_reserved_sequence_range_is_rejected_before_custom_handler() {
|
||||
let handler = Arc::new(RecordingCreateFlowHandler::default());
|
||||
let ddl_manager =
|
||||
build_soft_drop_test_ddl_manager().with_create_flow_handler(handler.clone());
|
||||
let mut task = test_create_flow_task();
|
||||
task.flow_options.insert(
|
||||
FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY.to_string(),
|
||||
FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_SEQUENCE_RANGE.to_string(),
|
||||
);
|
||||
|
||||
let result = ddl_manager
|
||||
.submit_ddl_task(
|
||||
ExecutorContext {
|
||||
query_context: Some(QueryContext::default()),
|
||||
..Default::default()
|
||||
},
|
||||
SubmitDdlTaskRequest::new(DdlTask::new_create_flow(task)),
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(result.is_err());
|
||||
assert!(handler.tasks.lock().unwrap().is_empty());
|
||||
}
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
#[tokio::test]
|
||||
async fn test_create_flow_handler_dispatches_without_procedure() {
|
||||
let response = SubmitDdlTaskResponse {
|
||||
key: b"custom-handler".to_vec(),
|
||||
..Default::default()
|
||||
};
|
||||
let handler = Arc::new(RecordingCreateFlowHandler {
|
||||
response: Mutex::new(Some(response)),
|
||||
..Default::default()
|
||||
});
|
||||
let ddl_manager =
|
||||
build_soft_drop_test_ddl_manager().with_create_flow_handler(handler.clone());
|
||||
let procedure_context = ProcedureContext {
|
||||
actor: Some("test-user".to_string()),
|
||||
event_context: Some(
|
||||
PersistentEventContext::new(TriggerReason::Manual).with_protocol("mysql"),
|
||||
),
|
||||
};
|
||||
let query_context = QueryContext {
|
||||
channel: Channel::Mysql as u8,
|
||||
..Default::default()
|
||||
};
|
||||
let actual = ddl_manager
|
||||
.submit_ddl_task(
|
||||
ExecutorContext {
|
||||
query_context: Some(query_context.clone()),
|
||||
actor: procedure_context.actor.clone(),
|
||||
event_input: Some(ProcedureEventInput::new(TriggerReason::Manual)),
|
||||
..Default::default()
|
||||
},
|
||||
SubmitDdlTaskRequest::new(DdlTask::new_create_flow(test_create_flow_task())),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(actual.key, b"custom-handler");
|
||||
assert_eq!(handler.tasks.lock().unwrap().len(), 1);
|
||||
assert_eq!(
|
||||
handler.contexts.lock().unwrap().as_slice(),
|
||||
&[(query_context, procedure_context)]
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
#[tokio::test]
|
||||
async fn test_create_flow_handler_error_does_not_fallback() {
|
||||
let handler = Arc::new(RecordingCreateFlowHandler {
|
||||
fail: true,
|
||||
..Default::default()
|
||||
});
|
||||
let ddl_manager =
|
||||
build_soft_drop_test_ddl_manager().with_create_flow_handler(handler.clone());
|
||||
let err = ddl_manager
|
||||
.submit_ddl_task(
|
||||
ExecutorContext {
|
||||
query_context: Some(QueryContext::default()),
|
||||
..Default::default()
|
||||
},
|
||||
SubmitDdlTaskRequest::new(DdlTask::new_create_flow(test_create_flow_task())),
|
||||
)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(err.to_string().contains("test create flow handler"));
|
||||
assert_eq!(handler.tasks.lock().unwrap().len(), 1);
|
||||
}
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
#[tokio::test]
|
||||
async fn test_create_flow_without_handler_uses_ordinary_procedure() {
|
||||
let ddl_manager = build_soft_drop_test_ddl_manager();
|
||||
ddl_manager.procedure_manager.start().await.unwrap();
|
||||
let err = ddl_manager
|
||||
.submit_ddl_task(
|
||||
ExecutorContext {
|
||||
query_context: Some(QueryContext::default()),
|
||||
..Default::default()
|
||||
},
|
||||
SubmitDdlTaskRequest::new(DdlTask::new_create_flow(test_create_flow_task())),
|
||||
)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(!err.to_string().contains("test create flow handler"));
|
||||
assert_eq!(
|
||||
ddl_manager
|
||||
.procedure_manager
|
||||
.list_procedures()
|
||||
.await
|
||||
.unwrap()
|
||||
.iter()
|
||||
.filter(|procedure| procedure.type_name == CreateFlowProcedure::TYPE_NAME)
|
||||
.count(),
|
||||
1
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
#[tokio::test]
|
||||
async fn test_trigger_ddl_forwards_procedure_context() {
|
||||
|
||||
@@ -20,14 +20,15 @@ use common_grpc::channel_manager::ClientTlsOption;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use session::ReadPreference;
|
||||
|
||||
mod checkpoint;
|
||||
pub(crate) mod batching_execution;
|
||||
pub(crate) mod checkpoint;
|
||||
pub(crate) mod engine;
|
||||
mod eval_schedule;
|
||||
pub(crate) mod frontend_client;
|
||||
mod state;
|
||||
pub(crate) mod state;
|
||||
mod table_creator;
|
||||
mod task;
|
||||
mod time_window;
|
||||
pub(crate) mod task;
|
||||
pub(crate) mod time_window;
|
||||
pub(crate) mod utils;
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
// Copyright 2023 Greptime Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Optional execution collaborator for batching tasks.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use query::QueryEngineRef;
|
||||
use table::TableRef;
|
||||
|
||||
use crate::Result;
|
||||
use crate::batching_mode::frontend_client::FrontendClient;
|
||||
use crate::batching_mode::task::{BatchingExecutionGuard, BatchingTask, ExecuteOnceOutcome};
|
||||
|
||||
#[async_trait::async_trait]
|
||||
pub trait BatchingExecution: Send + Sync + 'static {
|
||||
/// Execute one round while retaining the guard through all task-state updates.
|
||||
/// An implementation that continues after caller cancellation must retain the
|
||||
/// guard with that work and make it stoppable through [`Self::stop`].
|
||||
async fn execute_once(
|
||||
self: Arc<Self>,
|
||||
guard: BatchingExecutionGuard,
|
||||
task: &BatchingTask,
|
||||
engine: &QueryEngineRef,
|
||||
frontend: &Arc<FrontendClient>,
|
||||
max_window_cnt: Option<usize>,
|
||||
) -> ExecuteOnceOutcome;
|
||||
|
||||
/// Retire this execution instance, rejecting new work and requesting that any
|
||||
/// retained local work stop. This is not an acknowledgement of remote quiescence.
|
||||
fn stop(&self) {}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
pub trait BatchingExecutionFactory: Send + Sync + 'static {
|
||||
async fn create(
|
||||
&self,
|
||||
task: &BatchingTask,
|
||||
sink: TableRef,
|
||||
engine: &QueryEngineRef,
|
||||
frontend: &Arc<FrontendClient>,
|
||||
) -> Result<Option<Arc<dyn BatchingExecution>>>;
|
||||
}
|
||||
@@ -75,6 +75,7 @@ pub struct BatchingEngine {
|
||||
/// Batching mode options for control how batching mode query works
|
||||
///
|
||||
pub(crate) batch_opts: Arc<BatchingModeOptions>,
|
||||
execution_factory: Option<Arc<dyn crate::BatchingExecutionFactory>>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
@@ -130,6 +131,26 @@ impl BatchingEngine {
|
||||
table_meta: TableMetadataManagerRef,
|
||||
catalog_manager: CatalogManagerRef,
|
||||
batch_opts: BatchingModeOptions,
|
||||
) -> Self {
|
||||
Self::new_with_execution(
|
||||
frontend_client,
|
||||
query_engine,
|
||||
flow_metadata_manager,
|
||||
table_meta,
|
||||
catalog_manager,
|
||||
batch_opts,
|
||||
None,
|
||||
)
|
||||
}
|
||||
|
||||
pub fn new_with_execution(
|
||||
frontend_client: Arc<FrontendClient>,
|
||||
query_engine: QueryEngineRef,
|
||||
flow_metadata_manager: FlowMetadataManagerRef,
|
||||
table_meta: TableMetadataManagerRef,
|
||||
catalog_manager: CatalogManagerRef,
|
||||
batch_opts: BatchingModeOptions,
|
||||
execution_factory: Option<Arc<dyn crate::BatchingExecutionFactory>>,
|
||||
) -> Self {
|
||||
Self {
|
||||
runtime: Default::default(),
|
||||
@@ -139,6 +160,7 @@ impl BatchingEngine {
|
||||
catalog_manager,
|
||||
query_engine,
|
||||
batch_opts: Arc::new(batch_opts),
|
||||
execution_factory,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -726,15 +748,22 @@ impl BatchingEngine {
|
||||
);
|
||||
}
|
||||
|
||||
let task_inner = task.clone();
|
||||
let engine = self.query_engine.clone();
|
||||
let frontend = self.frontend_client.clone();
|
||||
|
||||
// Create sink table if needed, then validate an existing/created sink schema before
|
||||
// spawning the background task. This catches user-created sink schema mismatches at
|
||||
// CREATE FLOW time instead of surfacing them later in the execution loop.
|
||||
task.check_or_create_sink_table(&engine, &frontend).await?;
|
||||
task.validate_sink_table_schema(&engine).await?;
|
||||
// Create the sink before asking an optional extension to own execution.
|
||||
// The default path retains the OSS schema validation.
|
||||
let table = task.check_or_create_sink_table(&engine, &frontend).await?;
|
||||
let execution = if let Some(factory) = &self.execution_factory {
|
||||
factory.create(&task, table, &engine, &frontend).await?
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if execution.is_none() {
|
||||
task.validate_sink_table_schema(&engine).await?;
|
||||
}
|
||||
let task = task.with_execution(execution);
|
||||
let task_inner = task.clone();
|
||||
|
||||
let (start_tx, start_rx) = oneshot::channel();
|
||||
|
||||
@@ -1018,6 +1047,8 @@ fn abort_flow_task(flow_id: FlowId, task: Option<BatchingTask>, action: &str) ->
|
||||
return false;
|
||||
};
|
||||
|
||||
task.stop_execution();
|
||||
|
||||
if let Some(handle) = task.state.write().unwrap().task_handle.take() {
|
||||
handle.abort();
|
||||
debug!("Aborted {action} flow task {flow_id}");
|
||||
@@ -1067,17 +1098,23 @@ impl FlowEngine for BatchingEngine {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use api::v1::flow::{DirtyWindowRequest, TimeRange};
|
||||
use catalog::memory::new_memory_catalog_manager;
|
||||
use catalog::RegisterTableRequest;
|
||||
use catalog::memory::{MemoryCatalogManager, new_memory_catalog_manager};
|
||||
use common_meta::key::TableMetadataManager;
|
||||
use common_meta::key::flow::FlowMetadataManager;
|
||||
use common_meta::key::table_route::TableRouteValue;
|
||||
use common_meta::key::test_utils::new_test_table_info_with_name;
|
||||
use common_meta::kv_backend::memory::MemoryKvBackend;
|
||||
use common_recordbatch::RecordBatch;
|
||||
use common_time::timestamp::TimeUnit;
|
||||
use datatypes::schema::{ColumnSchema, Schema};
|
||||
use datatypes::vectors::{TimestampMillisecondVector, UInt32Vector, VectorRef};
|
||||
use query::options::QueryOptions;
|
||||
use session::context::QueryContext;
|
||||
use tokio::sync::Notify;
|
||||
|
||||
use super::*;
|
||||
use crate::ExecuteOnceOutcome;
|
||||
use crate::test_utils::create_test_query_engine;
|
||||
|
||||
struct DropNotify(Option<oneshot::Sender<()>>);
|
||||
@@ -1110,6 +1147,317 @@ mod tests {
|
||||
)
|
||||
}
|
||||
|
||||
async fn new_test_engine_with_execution(
|
||||
execution_factory: Option<Arc<dyn crate::BatchingExecutionFactory>>,
|
||||
) -> BatchingEngine {
|
||||
let kv_backend = Arc::new(MemoryKvBackend::new());
|
||||
let table_meta = Arc::new(TableMetadataManager::new(kv_backend.clone()));
|
||||
table_meta.init().await.unwrap();
|
||||
let flow_meta = Arc::new(FlowMetadataManager::new(kv_backend));
|
||||
let query_engine = create_test_query_engine();
|
||||
let catalog_manager = query_engine.engine_state().catalog_manager().clone();
|
||||
let (frontend_client, _handler) =
|
||||
FrontendClient::from_empty_grpc_handler(QueryOptions::default());
|
||||
|
||||
let engine = BatchingEngine::new_with_execution(
|
||||
Arc::new(frontend_client),
|
||||
query_engine,
|
||||
flow_meta,
|
||||
table_meta,
|
||||
catalog_manager,
|
||||
BatchingModeOptions::default(),
|
||||
execution_factory,
|
||||
);
|
||||
engine
|
||||
.table_meta
|
||||
.create_table_metadata(
|
||||
new_test_table_info_with_name(1, "numbers_with_ts"),
|
||||
TableRouteValue::physical(vec![]),
|
||||
HashMap::new(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
engine
|
||||
}
|
||||
|
||||
fn register_sink_with_schema(engine: &BatchingEngine, name: &str) {
|
||||
let schema = Arc::new(Schema::new(vec![
|
||||
ColumnSchema::new("number", ConcreteDataType::uint32_datatype(), false),
|
||||
ColumnSchema::new(
|
||||
"ts",
|
||||
ConcreteDataType::timestamp_millisecond_datatype(),
|
||||
false,
|
||||
)
|
||||
.with_time_index(true),
|
||||
]));
|
||||
let recordbatch = RecordBatch::new(
|
||||
schema,
|
||||
vec![
|
||||
Arc::new(UInt32Vector::from_slice([1_u32])) as VectorRef,
|
||||
Arc::new(TimestampMillisecondVector::from_slice([0_i64])) as VectorRef,
|
||||
],
|
||||
)
|
||||
.unwrap();
|
||||
engine
|
||||
.catalog_manager
|
||||
.as_any()
|
||||
.downcast_ref::<MemoryCatalogManager>()
|
||||
.unwrap()
|
||||
.register_table_sync(RegisterTableRequest {
|
||||
catalog: "greptime".to_string(),
|
||||
schema: "public".to_string(),
|
||||
table_name: name.to_string(),
|
||||
table_id: 9000,
|
||||
table: table::test_util::MemTable::table(name, recordbatch),
|
||||
})
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
fn register_number_only_sink(engine: &BatchingEngine, name: &str) {
|
||||
let schema = Arc::new(Schema::new(vec![ColumnSchema::new(
|
||||
"number",
|
||||
ConcreteDataType::uint32_datatype(),
|
||||
false,
|
||||
)]));
|
||||
let recordbatch = RecordBatch::new(
|
||||
schema,
|
||||
vec![Arc::new(UInt32Vector::from_slice([1_u32])) as VectorRef],
|
||||
)
|
||||
.unwrap();
|
||||
engine
|
||||
.catalog_manager
|
||||
.as_any()
|
||||
.downcast_ref::<MemoryCatalogManager>()
|
||||
.unwrap()
|
||||
.register_table_sync(RegisterTableRequest {
|
||||
catalog: "greptime".to_string(),
|
||||
schema: "public".to_string(),
|
||||
table_name: name.to_string(),
|
||||
table_id: 9001,
|
||||
table: table::test_util::MemTable::table(name, recordbatch),
|
||||
})
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
fn flow_create_args(flow_id: FlowId, sink: &str) -> CreateFlowArgs {
|
||||
CreateFlowArgs {
|
||||
flow_id,
|
||||
sink_table_name: [
|
||||
"greptime".to_string(),
|
||||
"public".to_string(),
|
||||
sink.to_string(),
|
||||
],
|
||||
source_table_ids: vec![1],
|
||||
create_if_not_exists: false,
|
||||
or_replace: false,
|
||||
expire_after: None,
|
||||
eval_interval: Some(10),
|
||||
comment: None,
|
||||
sql: "SELECT number, ts FROM numbers_with_ts".to_string(),
|
||||
flow_options: HashMap::new(),
|
||||
query_ctx: Some(QueryContext::arc().as_ref().clone()),
|
||||
eval_schedule: None,
|
||||
}
|
||||
}
|
||||
|
||||
struct TestExecution {
|
||||
manual_calls: std::sync::atomic::AtomicUsize,
|
||||
stops: std::sync::atomic::AtomicUsize,
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl crate::BatchingExecution for TestExecution {
|
||||
async fn execute_once(
|
||||
self: Arc<Self>,
|
||||
_guard: crate::BatchingExecutionGuard,
|
||||
task: &BatchingTask,
|
||||
_engine: &QueryEngineRef,
|
||||
_frontend: &Arc<FrontendClient>,
|
||||
_max_window_cnt: Option<usize>,
|
||||
) -> ExecuteOnceOutcome {
|
||||
if task
|
||||
.state
|
||||
.read()
|
||||
.unwrap()
|
||||
.query_ctx
|
||||
.extension(query::options::FLOW_SCHEDULED_TIME_MILLIS)
|
||||
.is_none()
|
||||
{
|
||||
self.manual_calls
|
||||
.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
|
||||
}
|
||||
ExecuteOnceOutcome {
|
||||
new_query: None,
|
||||
result: Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
fn stop(&self) {
|
||||
self.stops.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
|
||||
}
|
||||
}
|
||||
|
||||
type TestExecutionResult = crate::Result<Option<Arc<dyn crate::BatchingExecution>>>;
|
||||
|
||||
struct TestExecutionFactory {
|
||||
entered: Option<Arc<Notify>>,
|
||||
release: Option<Arc<Notify>>,
|
||||
result: std::sync::Mutex<Option<TestExecutionResult>>,
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl crate::BatchingExecutionFactory for TestExecutionFactory {
|
||||
async fn create(
|
||||
&self,
|
||||
_task: &BatchingTask,
|
||||
_sink: table::TableRef,
|
||||
_engine: &QueryEngineRef,
|
||||
_frontend: &Arc<FrontendClient>,
|
||||
) -> crate::Result<Option<Arc<dyn crate::BatchingExecution>>> {
|
||||
if let Some(entered) = &self.entered {
|
||||
entered.notify_one();
|
||||
}
|
||||
if let Some(release) = &self.release {
|
||||
release.notified().await;
|
||||
}
|
||||
self.result
|
||||
.lock()
|
||||
.unwrap()
|
||||
.take()
|
||||
.expect("execution factory should only be called once")
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_execution_factory_finishes_before_task_publication() {
|
||||
const GATE_TIMEOUT: Duration = Duration::from_secs(1);
|
||||
|
||||
let execution = Arc::new(TestExecution {
|
||||
manual_calls: Default::default(),
|
||||
stops: Default::default(),
|
||||
});
|
||||
let entered = Arc::new(Notify::new());
|
||||
let release = Arc::new(Notify::new());
|
||||
let factory = Arc::new(TestExecutionFactory {
|
||||
entered: Some(entered.clone()),
|
||||
release: Some(release.clone()),
|
||||
result: std::sync::Mutex::new(Some(Ok(Some(execution.clone())))),
|
||||
});
|
||||
let engine = Arc::new(new_test_engine_with_execution(Some(factory)).await);
|
||||
register_sink_with_schema(&engine, "factory_sink");
|
||||
|
||||
let entered_wait = entered.notified();
|
||||
let mut args = flow_create_args(6, "factory_sink");
|
||||
args.eval_interval = Some(86_400);
|
||||
let mut create = tokio::spawn({
|
||||
let engine = engine.clone();
|
||||
async move { engine.create_flow_inner(args).await }
|
||||
});
|
||||
if tokio::time::timeout(GATE_TIMEOUT, entered_wait)
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
release.notify_one();
|
||||
if tokio::time::timeout(GATE_TIMEOUT, &mut create)
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
create.abort();
|
||||
}
|
||||
panic!("execution factory should be entered");
|
||||
}
|
||||
let unpublished = {
|
||||
let runtime = engine.runtime.read().await;
|
||||
!runtime.tasks.contains_key(&6) && !runtime.shutdown_txs.contains_key(&6)
|
||||
};
|
||||
|
||||
release.notify_one();
|
||||
assert!(unpublished);
|
||||
let created = tokio::time::timeout(GATE_TIMEOUT, &mut create)
|
||||
.await
|
||||
.expect("flow creation should finish after factory release");
|
||||
created.unwrap().unwrap();
|
||||
assert!(engine.flow_exist_inner(6).await);
|
||||
let calls_before_flush = execution
|
||||
.manual_calls
|
||||
.load(std::sync::atomic::Ordering::SeqCst);
|
||||
assert_eq!(engine.flush_flow_inner(6).await.unwrap(), 0);
|
||||
assert_eq!(
|
||||
execution
|
||||
.manual_calls
|
||||
.load(std::sync::atomic::Ordering::SeqCst),
|
||||
calls_before_flush + 1
|
||||
);
|
||||
engine.remove_flow_inner(6).await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_execution_factory_result_controls_sink_validation_and_publication() {
|
||||
let no_factory_engine = new_test_engine_with_execution(None).await;
|
||||
register_number_only_sink(&no_factory_engine, "no_factory_sink");
|
||||
assert!(
|
||||
no_factory_engine
|
||||
.create_flow_inner(flow_create_args(7, "no_factory_sink"))
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert!(!no_factory_engine.flow_exist_inner(7).await);
|
||||
|
||||
let declined_engine =
|
||||
new_test_engine_with_execution(Some(Arc::new(TestExecutionFactory {
|
||||
entered: None,
|
||||
release: None,
|
||||
result: std::sync::Mutex::new(Some(Ok(None))),
|
||||
})))
|
||||
.await;
|
||||
register_number_only_sink(&declined_engine, "declined_factory_sink");
|
||||
assert!(
|
||||
declined_engine
|
||||
.create_flow_inner(flow_create_args(8, "declined_factory_sink"))
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert!(!declined_engine.flow_exist_inner(8).await);
|
||||
|
||||
let accepted_engine =
|
||||
new_test_engine_with_execution(Some(Arc::new(TestExecutionFactory {
|
||||
entered: None,
|
||||
release: None,
|
||||
result: std::sync::Mutex::new(Some(Ok(Some(Arc::new(TestExecution {
|
||||
manual_calls: Default::default(),
|
||||
stops: Default::default(),
|
||||
}))))),
|
||||
})))
|
||||
.await;
|
||||
register_number_only_sink(&accepted_engine, "accepted_factory_sink");
|
||||
accepted_engine
|
||||
.create_flow_inner(flow_create_args(9, "accepted_factory_sink"))
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(accepted_engine.flow_exist_inner(9).await);
|
||||
accepted_engine.remove_flow_inner(9).await.unwrap();
|
||||
|
||||
let error_engine = new_test_engine_with_execution(Some(Arc::new(TestExecutionFactory {
|
||||
entered: None,
|
||||
release: None,
|
||||
result: std::sync::Mutex::new(Some(
|
||||
UnexpectedSnafu {
|
||||
reason: "test execution factory failure".to_string(),
|
||||
}
|
||||
.fail(),
|
||||
)),
|
||||
})))
|
||||
.await;
|
||||
register_sink_with_schema(&error_engine, "error_factory_sink");
|
||||
assert!(
|
||||
error_engine
|
||||
.create_flow_inner(flow_create_args(10, "error_factory_sink"))
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert!(!error_engine.flow_exist_inner(10).await);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_flow_option_overrides_incremental_read_switch() {
|
||||
let engine = new_test_engine().await;
|
||||
@@ -1583,11 +1931,30 @@ GROUP BY l.number, time_window
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_abort_flow_task_aborts_handle() {
|
||||
async fn test_abort_flow_task_stops_execution_without_loop_handle() {
|
||||
let (task, _shutdown_tx) = new_test_task(42).await;
|
||||
let execution = Arc::new(TestExecution {
|
||||
manual_calls: Default::default(),
|
||||
stops: Default::default(),
|
||||
});
|
||||
let task = task.with_execution(Some(execution.clone()));
|
||||
|
||||
assert!(!abort_flow_task(42, Some(task), "test"));
|
||||
assert_eq!(execution.stops.load(std::sync::atomic::Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_abort_flow_task_stops_execution_before_aborting_handle() {
|
||||
let (task, _shutdown_tx) = new_test_task(42).await;
|
||||
let execution = Arc::new(TestExecution {
|
||||
manual_calls: Default::default(),
|
||||
stops: Default::default(),
|
||||
});
|
||||
let task = task.with_execution(Some(execution.clone()));
|
||||
let drop_rx = install_abort_observed_handle(&task).await;
|
||||
|
||||
assert!(abort_flow_task(42, Some(task), "test"));
|
||||
assert_eq!(execution.stops.load(std::sync::atomic::Ordering::SeqCst), 1);
|
||||
|
||||
tokio::time::timeout(Duration::from_secs(1), drop_rx)
|
||||
.await
|
||||
|
||||
@@ -749,7 +749,7 @@ fn wrap_standalone_output_with_terminal_metrics(
|
||||
|
||||
/// Describe a peer of frontend
|
||||
#[derive(Debug, Default, Clone)]
|
||||
pub(crate) enum PeerDesc {
|
||||
pub enum PeerDesc {
|
||||
/// The query failed before a frontend peer was selected.
|
||||
#[default]
|
||||
Unknown,
|
||||
|
||||
@@ -131,6 +131,11 @@ impl TaskState {
|
||||
&self.checkpoints
|
||||
}
|
||||
|
||||
/// Read the live dirty-window queue without exposing TaskState internals to execution owners.
|
||||
pub fn dirty_time_windows(&self) -> &DirtyTimeWindows {
|
||||
&self.dirty_time_windows
|
||||
}
|
||||
|
||||
/// Returns the in-progress fenced repair, if the task is repairing dirty
|
||||
/// windows under a frozen full-snapshot high watermark.
|
||||
pub fn pending_fenced_repair(&self) -> Option<&FencedRepair> {
|
||||
@@ -546,6 +551,15 @@ impl DirtyTimeWindows {
|
||||
}
|
||||
}
|
||||
|
||||
/// Detach all dirty windows while retaining this queue's configured limits.
|
||||
pub(crate) fn detach(&mut self) -> Self {
|
||||
Self {
|
||||
windows: std::mem::take(&mut self.windows),
|
||||
max_filter_num_per_query: self.max_filter_num_per_query,
|
||||
time_window_merge_threshold: self.time_window_merge_threshold,
|
||||
}
|
||||
}
|
||||
|
||||
/// Clean all dirty time windows, useful when can't found time window expr
|
||||
pub fn clean(&mut self) {
|
||||
self.windows.clear();
|
||||
|
||||
@@ -18,6 +18,7 @@ use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||||
|
||||
use api::v1::{CreateTableExpr, TableName};
|
||||
use catalog::CatalogManagerRef;
|
||||
use client::OutputWithMetrics;
|
||||
use common_error::ext::BoxedError;
|
||||
use common_query::logical_plan::breakup_insert_plan;
|
||||
use common_telemetry::tracing::warn;
|
||||
@@ -27,7 +28,7 @@ use datafusion::datasource::DefaultTableSource;
|
||||
use datafusion::sql::unparser::expr_to_sql;
|
||||
use datafusion_common::tree_node::{Transformed, TreeNode};
|
||||
use datafusion_common::utils::quote_identifier;
|
||||
use datafusion_common::{DFSchemaRef, TableReference};
|
||||
use datafusion_common::{DFSchemaRef, ScalarValue, TableReference};
|
||||
use datafusion_expr::{DmlStatement, LogicalPlan, WriteOp, col, lit};
|
||||
use datatypes::schema::Schema;
|
||||
use query::QueryEngineRef;
|
||||
@@ -38,9 +39,10 @@ use snafu::{OptionExt, ResultExt};
|
||||
use sql::parsers::utils::is_tql;
|
||||
use store_api::mito_engine_options::MERGE_MODE_KEY;
|
||||
use substrait::{DFLogicalSubstraitConvertor, SubstraitPlan};
|
||||
use table::TableRef;
|
||||
use table::table::adapter::DfTableProviderAdapter;
|
||||
use tokio::sync::oneshot::error::TryRecvError;
|
||||
use tokio::sync::{Mutex, oneshot};
|
||||
use tokio::sync::{Mutex, OwnedMutexGuard, oneshot};
|
||||
use tokio::time::Instant;
|
||||
|
||||
use crate::batching_mode::BatchingModeOptions;
|
||||
@@ -53,8 +55,8 @@ use crate::batching_mode::state::{
|
||||
use crate::batching_mode::table_creator::{QueryType, create_table_with_expr};
|
||||
use crate::batching_mode::time_window::TimeWindowExpr;
|
||||
use crate::batching_mode::utils::{
|
||||
AddFilterRewriter, ColumnMatcherRewriter, df_plan_to_sql, gen_plan_with_matching_schema,
|
||||
get_table_info_df_schema, sql_to_df_plan,
|
||||
AddFilterRewriter, ColumnMatcherRewriter, df_plan_to_sql,
|
||||
gen_plan_with_matching_schema_and_values, get_table_info_df_schema, sql_to_df_plan,
|
||||
};
|
||||
use crate::df_optimizer::apply_df_optimizer;
|
||||
use crate::error::{
|
||||
@@ -199,6 +201,36 @@ fn format_insert_target_columns(plan: &LogicalPlan) -> String {
|
||||
.join(", ")
|
||||
}
|
||||
|
||||
/// Owns a whole serialized execution round. It may be moved to an execution
|
||||
/// collaborator, so cancellation of the caller cannot release the round early.
|
||||
pub struct BatchingExecutionGuard {
|
||||
_lock: OwnedMutexGuard<()>,
|
||||
restore: Option<(Arc<RwLock<TaskState>>, QueryContextRef)>,
|
||||
}
|
||||
|
||||
impl BatchingExecutionGuard {
|
||||
fn new(lock: OwnedMutexGuard<()>) -> Self {
|
||||
Self {
|
||||
_lock: lock,
|
||||
restore: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn restore_query_context(&mut self, state: Arc<RwLock<TaskState>>, old_ctx: QueryContextRef) {
|
||||
self.restore = Some((state, old_ctx));
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for BatchingExecutionGuard {
|
||||
fn drop(&mut self) {
|
||||
if let Some((state, old_ctx)) = self.restore.take()
|
||||
&& let Ok(mut state) = state.write()
|
||||
{
|
||||
state.query_ctx = old_ctx;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct BatchingTask {
|
||||
pub config: Arc<TaskConfig>,
|
||||
@@ -207,6 +239,7 @@ pub struct BatchingTask {
|
||||
/// window restoration for this flow. Without this, a manual flush and the
|
||||
/// background loop can process the same checkpoint range concurrently.
|
||||
execution_lock: Arc<Mutex<()>>,
|
||||
execution: Option<Arc<dyn crate::BatchingExecution>>,
|
||||
}
|
||||
|
||||
/// Arguments for creating batching task
|
||||
@@ -281,14 +314,14 @@ pub enum DirtyRestore {
|
||||
Unscoped(DirtyTimeWindows),
|
||||
}
|
||||
|
||||
struct ExecuteOnceOutcome {
|
||||
new_query: Option<PlanInfo>,
|
||||
pub struct ExecuteOnceOutcome {
|
||||
pub new_query: Option<PlanInfo>,
|
||||
/// Execution result of the generated insert plan.
|
||||
///
|
||||
/// `Ok(Some((affected_rows, elapsed)))` means a query was executed.
|
||||
/// `Ok(None)` means no query was generated because there was no dirty signal.
|
||||
/// `Err(_)` means plan generation or execution failed.
|
||||
result: Result<Option<(usize, Duration)>, Error>,
|
||||
pub result: Result<Option<(usize, Duration)>, Error>,
|
||||
}
|
||||
|
||||
impl BatchingTask {
|
||||
@@ -345,9 +378,24 @@ impl BatchingTask {
|
||||
}),
|
||||
state: Arc::new(RwLock::new(state)),
|
||||
execution_lock: Arc::new(Mutex::new(())),
|
||||
execution: None,
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn with_execution(
|
||||
mut self,
|
||||
execution: Option<Arc<dyn crate::BatchingExecution>>,
|
||||
) -> Self {
|
||||
self.execution = execution;
|
||||
self
|
||||
}
|
||||
|
||||
pub(crate) fn stop_execution(&self) {
|
||||
if let Some(execution) = &self.execution {
|
||||
execution.stop();
|
||||
}
|
||||
}
|
||||
|
||||
pub fn last_execution_time_millis(&self) -> Option<i64> {
|
||||
self.state.read().unwrap().last_execution_time_millis()
|
||||
}
|
||||
@@ -403,12 +451,12 @@ impl BatchingTask {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Create sink table if not exists
|
||||
/// Create the sink table if needed, then return it.
|
||||
pub async fn check_or_create_sink_table(
|
||||
&self,
|
||||
engine: &QueryEngineRef,
|
||||
frontend_client: &Arc<FrontendClient>,
|
||||
) -> Result<Option<(usize, Duration)>, Error> {
|
||||
) -> Result<TableRef, Error> {
|
||||
if !self.is_table_exist(&self.config.sink_table_name).await? {
|
||||
let create_table = self.gen_create_table_expr(engine.clone()).await?;
|
||||
info!(
|
||||
@@ -421,16 +469,32 @@ impl BatchingTask {
|
||||
self.config.sink_table_name.join(".")
|
||||
);
|
||||
}
|
||||
|
||||
Ok(None)
|
||||
let (table, _) = get_table_info_df_schema(
|
||||
self.config.catalog_manager.clone(),
|
||||
self.config.sink_table_name.clone(),
|
||||
)
|
||||
.await?;
|
||||
Ok(table)
|
||||
}
|
||||
|
||||
/// Validates that the sink table schema can accept this flow's output.
|
||||
///
|
||||
/// This is a dry-run of the same schema matching logic used by insert-plan
|
||||
/// generation, but without adding dirty-window filters or executing the query. It is used
|
||||
/// during CREATE FLOW to catch existing sink table mismatches early.
|
||||
/// Returns a forked snapshot of the task query context for extension-owned planning.
|
||||
pub fn query_context_snapshot(&self) -> QueryContextRef {
|
||||
let query_ctx = self.state.read().unwrap().query_ctx.clone();
|
||||
Arc::new(query_ctx.fork())
|
||||
}
|
||||
|
||||
/// Validates that the sink table schema can accept this flow's ordinary output.
|
||||
pub async fn validate_sink_table_schema(&self, engine: &QueryEngineRef) -> Result<(), Error> {
|
||||
self.validate_sink_table_schema_with_values(engine, &BTreeMap::new())
|
||||
.await
|
||||
}
|
||||
|
||||
/// Validates the flow output plus extension-owned typed sink values without consuming dirty work.
|
||||
pub async fn validate_sink_table_schema_with_values(
|
||||
&self,
|
||||
engine: &QueryEngineRef,
|
||||
values: &BTreeMap<String, ScalarValue>,
|
||||
) -> Result<(), Error> {
|
||||
let (table, _) = get_table_info_df_schema(
|
||||
self.config.catalog_manager.clone(),
|
||||
self.config.sink_table_name.clone(),
|
||||
@@ -441,15 +505,15 @@ impl BatchingTask {
|
||||
let merge_mode_last_non_null =
|
||||
is_merge_mode_last_non_null(&table_meta.options.extra_options);
|
||||
let primary_key_indices = table_meta.primary_key_indices.clone();
|
||||
let query_ctx = self.state.read().unwrap().query_ctx.clone();
|
||||
|
||||
gen_plan_with_matching_schema(
|
||||
gen_plan_with_matching_schema_and_values(
|
||||
&self.config.query,
|
||||
query_ctx,
|
||||
self.query_context_snapshot(),
|
||||
engine.clone(),
|
||||
table_meta.schema.clone(),
|
||||
&primary_key_indices,
|
||||
merge_mode_last_non_null,
|
||||
values,
|
||||
)
|
||||
.await
|
||||
.map(|_| ())
|
||||
@@ -484,13 +548,30 @@ impl BatchingTask {
|
||||
frontend_client: &Arc<FrontendClient>,
|
||||
max_window_cnt: Option<usize>,
|
||||
) -> ExecuteOnceOutcome {
|
||||
let _execution_guard = self.execution_lock.lock().await;
|
||||
self.execute_once_unlocked(engine, frontend_client, max_window_cnt)
|
||||
let guard = BatchingExecutionGuard::new(self.execution_lock.clone().lock_owned().await);
|
||||
self.execute_once_with_guard(guard, engine, frontend_client, max_window_cnt)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Executes one flow evaluation. Caller must hold `execution_lock`.
|
||||
async fn execute_once_unlocked(
|
||||
async fn execute_once_with_guard(
|
||||
&self,
|
||||
guard: BatchingExecutionGuard,
|
||||
engine: &QueryEngineRef,
|
||||
frontend_client: &Arc<FrontendClient>,
|
||||
max_window_cnt: Option<usize>,
|
||||
) -> ExecuteOnceOutcome {
|
||||
if let Some(execution) = &self.execution {
|
||||
return execution
|
||||
.clone()
|
||||
.execute_once(guard, self, engine, frontend_client, max_window_cnt)
|
||||
.await;
|
||||
}
|
||||
self.execute_once_default_unlocked(engine, frontend_client, max_window_cnt)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Executes the default flow evaluation. Caller owns the execution guard.
|
||||
async fn execute_once_default_unlocked(
|
||||
&self,
|
||||
engine: &QueryEngineRef,
|
||||
frontend_client: &Arc<FrontendClient>,
|
||||
@@ -510,6 +591,7 @@ impl BatchingTask {
|
||||
debug!("Generate new query: {}", new_query.plan);
|
||||
let res = self
|
||||
.execute_logical_plan_unlocked(
|
||||
engine,
|
||||
frontend_client,
|
||||
&new_query.plan,
|
||||
&new_query.dirty_restore,
|
||||
@@ -532,11 +614,24 @@ impl BatchingTask {
|
||||
}
|
||||
}
|
||||
|
||||
/// Generates the insert plan. Caller must reach this through the serialized path.
|
||||
/// Generates the ordinary insert plan. Caller must reach this through the serialized path.
|
||||
async fn gen_insert_plan_unlocked(
|
||||
&self,
|
||||
engine: &QueryEngineRef,
|
||||
max_window_cnt: Option<usize>,
|
||||
) -> Result<Option<PlanInfo>, Error> {
|
||||
self.gen_insert_plan_with_values_unlocked(engine, max_window_cnt, &BTreeMap::new(), false)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Generates the insert plan with extension-owned typed sink values. Caller must own the
|
||||
/// serialized execution guard.
|
||||
pub async fn gen_insert_plan_with_values_unlocked(
|
||||
&self,
|
||||
engine: &QueryEngineRef,
|
||||
max_window_cnt: Option<usize>,
|
||||
values: &BTreeMap<String, ScalarValue>,
|
||||
force_full_snapshot: bool,
|
||||
) -> Result<Option<PlanInfo>, Error> {
|
||||
let (table, df_schema) = get_table_info_df_schema(
|
||||
self.config.catalog_manager.clone(),
|
||||
@@ -550,12 +645,14 @@ impl BatchingTask {
|
||||
let primary_key_indices = table_meta.primary_key_indices.clone();
|
||||
|
||||
let new_query = self
|
||||
.gen_query_with_time_window(
|
||||
.gen_query_with_time_window_with_values(
|
||||
engine.clone(),
|
||||
&table.table_info().meta.schema,
|
||||
&primary_key_indices,
|
||||
merge_mode_last_non_null,
|
||||
max_window_cnt,
|
||||
values,
|
||||
force_full_snapshot,
|
||||
)
|
||||
.await?;
|
||||
|
||||
@@ -637,14 +734,55 @@ impl BatchingTask {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Executes the insert plan. Caller must reach this through the serialized path.
|
||||
/// Applies checkpoint state updates around the raw insert-plan execution.
|
||||
async fn execute_logical_plan_unlocked(
|
||||
&self,
|
||||
engine: &QueryEngineRef,
|
||||
frontend_client: &Arc<FrontendClient>,
|
||||
plan: &LogicalPlan,
|
||||
dirty_restore: &DirtyRestore,
|
||||
coverage: &QueryCoverage,
|
||||
) -> Result<Option<(usize, Duration)>, Error> {
|
||||
let flow_id = self.config.flow_id;
|
||||
let Some((res, elapsed)) = self
|
||||
.execute_plan_unlocked(engine, frontend_client, plan, dirty_restore, coverage)
|
||||
.await?
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
if let Err(err) = &res {
|
||||
let decision = {
|
||||
let mut state = self.state.write().unwrap();
|
||||
let reason = Self::query_failure_reason(err, coverage);
|
||||
Self::apply_query_failure_to_state(&mut state, elapsed, coverage, reason)
|
||||
};
|
||||
if let Some(decision) = decision {
|
||||
Self::record_checkpoint_decision(flow_id, decision);
|
||||
}
|
||||
}
|
||||
|
||||
let res = res?;
|
||||
let (affected_rows, _) = res.output.extract_rows_and_cost();
|
||||
let decision = {
|
||||
let mut state = self.state.write().unwrap();
|
||||
Self::apply_query_result_to_state(&mut state, &res, elapsed, coverage)
|
||||
};
|
||||
Self::record_checkpoint_decision(flow_id, decision);
|
||||
Ok(Some((affected_rows, elapsed)))
|
||||
}
|
||||
|
||||
/// Executes a fully constructed insert plan without checkpoint-state mutation.
|
||||
/// Callers must own the serialized execution guard. Its unsafe-incremental safety fallback
|
||||
/// still restores the supplied dirty work before returning `Ok(None)`.
|
||||
pub async fn execute_plan_unlocked(
|
||||
&self,
|
||||
engine: &QueryEngineRef,
|
||||
frontend_client: &Arc<FrontendClient>,
|
||||
plan: &LogicalPlan,
|
||||
dirty_restore: &DirtyRestore,
|
||||
coverage: &QueryCoverage,
|
||||
) -> Result<Option<(Result<OutputWithMetrics, Error>, Duration)>, Error> {
|
||||
let instant = Instant::now();
|
||||
let flow_id = self.config.flow_id;
|
||||
|
||||
@@ -676,7 +814,7 @@ impl BatchingTask {
|
||||
// For incremental-mode SQL queries, attempt to rewrite the delta aggregate
|
||||
// plan into a safe delta-LEFT-JOIN-sink form before deciding on extensions.
|
||||
let incremental_plan = if coverage.is_incremental_delta() {
|
||||
self.prepare_plan_for_incremental(&plan).await?
|
||||
self.prepare_plan_for_incremental(engine, &plan).await?
|
||||
} else {
|
||||
None
|
||||
};
|
||||
@@ -805,18 +943,13 @@ impl BatchingTask {
|
||||
.map(ToString::to_string)
|
||||
.unwrap_or_else(|| PeerDesc::default().to_string());
|
||||
if let Err(err) = &res {
|
||||
// This raw helper deliberately leaves checkpoint transitions to its caller. The
|
||||
// default wrapper performs the single OSS transition; extension owners can persist
|
||||
// their durable marker before updating in-memory state.
|
||||
warn!(
|
||||
"Failed to execute Flow {flow_id} on frontend {peer_label}, result: {err:?}, elapsed: {:?} with query: {}",
|
||||
elapsed, &plan
|
||||
);
|
||||
let decision = {
|
||||
let mut state = self.state.write().unwrap();
|
||||
let reason = Self::query_failure_reason(err, coverage);
|
||||
Self::apply_query_failure_to_state(&mut state, elapsed, coverage, reason)
|
||||
};
|
||||
if let Some(decision) = decision {
|
||||
Self::record_checkpoint_decision(flow_id, decision);
|
||||
}
|
||||
}
|
||||
|
||||
// record slow query
|
||||
@@ -831,29 +964,27 @@ impl BatchingTask {
|
||||
.observe(elapsed.as_secs_f64());
|
||||
}
|
||||
|
||||
let res = res?;
|
||||
let (affected_rows, _) = res.output.extract_rows_and_cost();
|
||||
debug!(
|
||||
"Flow {flow_id} executed, affected_rows: {affected_rows:?}, elapsed: {:?}, watermark: {:?}",
|
||||
elapsed,
|
||||
res.region_watermark_map()
|
||||
);
|
||||
METRIC_FLOW_ROWS
|
||||
.with_label_values(&[format!("{}-out-batching", flow_id).as_str()])
|
||||
.inc_by(affected_rows as _);
|
||||
let decision = {
|
||||
let mut state = self.state.write().unwrap();
|
||||
Self::apply_query_result_to_state(&mut state, &res, elapsed, coverage)
|
||||
};
|
||||
Self::record_checkpoint_decision(flow_id, decision);
|
||||
|
||||
Ok(Some((affected_rows, elapsed)))
|
||||
match res {
|
||||
Ok(res) => {
|
||||
let (affected_rows, _) = res.output.extract_rows_and_cost();
|
||||
debug!(
|
||||
"Flow {flow_id} executed, affected_rows: {affected_rows:?}, elapsed: {:?}, watermark: {:?}",
|
||||
elapsed,
|
||||
res.region_watermark_map()
|
||||
);
|
||||
METRIC_FLOW_ROWS
|
||||
.with_label_values(&[format!("{}-out-batching", flow_id).as_str()])
|
||||
.inc_by(affected_rows as _);
|
||||
Ok(Some((Ok(res), elapsed)))
|
||||
}
|
||||
Err(err) => Ok(Some((Err(err), elapsed))),
|
||||
}
|
||||
}
|
||||
|
||||
/// Restore dirty windows consumed by a failed query so they are retried on
|
||||
/// the next execution.
|
||||
///
|
||||
fn restore_dirty_windows(&self, dirty_restore: &DirtyRestore) {
|
||||
pub fn restore_dirty_windows(&self, dirty_restore: &DirtyRestore) {
|
||||
match dirty_restore {
|
||||
DirtyRestore::Scoped(filter) => self.restore_scoped_dirty_windows(filter),
|
||||
DirtyRestore::Unscoped(dirty_windows) => self
|
||||
@@ -931,19 +1062,21 @@ impl BatchingTask {
|
||||
sink_table_schema: Arc<Schema>,
|
||||
primary_key_indices: &[usize],
|
||||
allow_partial: bool,
|
||||
values: &BTreeMap<String, ScalarValue>,
|
||||
dirty_windows_to_restore: DirtyTimeWindows,
|
||||
retention_filter: Option<(&str, Timestamp, &'static str)>,
|
||||
coverage: QueryCoverage,
|
||||
) -> Result<PlanInfo, Error> {
|
||||
let mut plan = self.restore_unscoped_dirty_windows_on_err(
|
||||
&dirty_windows_to_restore,
|
||||
gen_plan_with_matching_schema(
|
||||
gen_plan_with_matching_schema_and_values(
|
||||
&self.config.query,
|
||||
query_ctx,
|
||||
engine,
|
||||
sink_table_schema,
|
||||
primary_key_indices,
|
||||
allow_partial,
|
||||
values,
|
||||
)
|
||||
.await,
|
||||
)?;
|
||||
@@ -986,6 +1119,7 @@ impl BatchingTask {
|
||||
sink_table_schema: Arc<Schema>,
|
||||
primary_key_indices: &[usize],
|
||||
allow_partial: bool,
|
||||
values: &BTreeMap<String, ScalarValue>,
|
||||
retention_filter: Option<(&str, Timestamp, &'static str)>,
|
||||
coverage: QueryCoverage,
|
||||
) -> Result<Option<PlanInfo>, Error> {
|
||||
@@ -1001,6 +1135,7 @@ impl BatchingTask {
|
||||
sink_table_schema,
|
||||
primary_key_indices,
|
||||
allow_partial,
|
||||
values,
|
||||
dirty_windows_to_restore,
|
||||
retention_filter,
|
||||
coverage,
|
||||
@@ -1225,7 +1360,7 @@ impl BatchingTask {
|
||||
.inc();
|
||||
// Dirty-window restoration is handled by the
|
||||
// existing `handle_executed_query_failure` inside
|
||||
// `execute_once_unlocked`.
|
||||
// default execution.
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1333,23 +1468,7 @@ impl BatchingTask {
|
||||
frontend_client: &Arc<FrontendClient>,
|
||||
scheduled_time_secs: i64,
|
||||
) -> ExecuteOnceOutcome {
|
||||
let _execution_guard = self.execution_lock.lock().await;
|
||||
|
||||
struct QueryContextRestoreGuard {
|
||||
state: Arc<RwLock<TaskState>>,
|
||||
old_ctx: Option<QueryContextRef>,
|
||||
}
|
||||
|
||||
impl Drop for QueryContextRestoreGuard {
|
||||
fn drop(&mut self) {
|
||||
let Some(old_ctx) = self.old_ctx.take() else {
|
||||
return;
|
||||
};
|
||||
if let Ok(mut state) = self.state.write() {
|
||||
state.query_ctx = old_ctx;
|
||||
}
|
||||
}
|
||||
}
|
||||
let mut guard = BatchingExecutionGuard::new(self.execution_lock.clone().lock_owned().await);
|
||||
|
||||
// Convert to milliseconds before touching the task state so an
|
||||
// unrepresentable scheduled time fails as an explicit error without
|
||||
@@ -1377,21 +1496,12 @@ impl BatchingTask {
|
||||
state.query_ctx = Arc::new(new_ctx);
|
||||
old
|
||||
};
|
||||
let restore_guard = QueryContextRestoreGuard {
|
||||
state: self.state.clone(),
|
||||
old_ctx: Some(old_ctx),
|
||||
};
|
||||
guard.restore_query_context(self.state.clone(), old_ctx);
|
||||
|
||||
let outcome = self
|
||||
.execute_once_unlocked(engine, frontend_client, None)
|
||||
.await;
|
||||
|
||||
// Restore while still holding `execution_lock` so no future manual
|
||||
// flush can observe the temporary scheduled time. The guard also
|
||||
// restores during unwind/cancellation.
|
||||
drop(restore_guard);
|
||||
|
||||
outcome
|
||||
// A collaborator may retain the guard in an owned child task. Its Drop
|
||||
// restores the scheduled context before releasing serialization.
|
||||
self.execute_once_with_guard(guard, engine, frontend_client, None)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Generate the create table SQL
|
||||
@@ -1419,8 +1529,7 @@ impl BatchingTask {
|
||||
&& matches!(self.config.query_type, QueryType::Sql)
|
||||
}
|
||||
|
||||
/// Generate the next plan and classify its coverage so checkpoint handling
|
||||
/// knows whether it is full-query, scoped repair, fenced repair, or delta.
|
||||
/// Generate the next ordinary plan and classify its coverage.
|
||||
async fn gen_query_with_time_window(
|
||||
&self,
|
||||
engine: QueryEngineRef,
|
||||
@@ -1428,6 +1537,30 @@ impl BatchingTask {
|
||||
primary_key_indices: &[usize],
|
||||
allow_partial: bool,
|
||||
max_window_cnt: Option<usize>,
|
||||
) -> Result<Option<PlanInfo>, Error> {
|
||||
self.gen_query_with_time_window_with_values(
|
||||
engine,
|
||||
sink_table_schema,
|
||||
primary_key_indices,
|
||||
allow_partial,
|
||||
max_window_cnt,
|
||||
&BTreeMap::new(),
|
||||
false,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Generate the next plan with extension-owned sink values and an optional forced full scan.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
async fn gen_query_with_time_window_with_values(
|
||||
&self,
|
||||
engine: QueryEngineRef,
|
||||
sink_table_schema: &Arc<Schema>,
|
||||
primary_key_indices: &[usize],
|
||||
allow_partial: bool,
|
||||
max_window_cnt: Option<usize>,
|
||||
values: &BTreeMap<String, ScalarValue>,
|
||||
force_full_snapshot: bool,
|
||||
) -> Result<Option<PlanInfo>, Error> {
|
||||
let query_ctx = self.state.read().unwrap().query_ctx.clone();
|
||||
let start = SystemTime::now();
|
||||
@@ -1478,6 +1611,7 @@ impl BatchingTask {
|
||||
sink_table_schema.clone(),
|
||||
primary_key_indices,
|
||||
allow_partial,
|
||||
values,
|
||||
dirty_windows_to_restore,
|
||||
None,
|
||||
QueryCoverage::UnfilteredFull,
|
||||
@@ -1514,6 +1648,34 @@ impl BatchingTask {
|
||||
),
|
||||
})?;
|
||||
|
||||
if force_full_snapshot {
|
||||
// Durable execution cannot trust an uncertain prior checkpoint. Re-run an
|
||||
// unfiltered full snapshot, but retain the normal expiry predicate and the consumed
|
||||
// dirty signal so every planning/execution failure remains reversible.
|
||||
let retention_filter = self.config.expire_after.map(|_| {
|
||||
(
|
||||
col_name.as_str(),
|
||||
expire_lower_bound,
|
||||
"forced full snapshot",
|
||||
)
|
||||
});
|
||||
let (_, dirty_windows_to_restore) = self.drain_dirty_windows_signal();
|
||||
return self
|
||||
.gen_unfiltered_plan_info(
|
||||
engine,
|
||||
query_ctx,
|
||||
sink_table_schema.clone(),
|
||||
primary_key_indices,
|
||||
allow_partial,
|
||||
values,
|
||||
dirty_windows_to_restore,
|
||||
retention_filter,
|
||||
QueryCoverage::UnfilteredFull,
|
||||
)
|
||||
.await
|
||||
.map(Some);
|
||||
}
|
||||
|
||||
if self.should_use_unfiltered_incremental_delta() {
|
||||
// In incremental mode, source correctness is defined by the
|
||||
// per-region sequence range `(checkpoint, scan-open snapshot]`, not
|
||||
@@ -1535,6 +1697,7 @@ impl BatchingTask {
|
||||
sink_table_schema.clone(),
|
||||
primary_key_indices,
|
||||
allow_partial,
|
||||
values,
|
||||
retention_filter,
|
||||
QueryCoverage::IncrementalDelta,
|
||||
)
|
||||
@@ -1580,10 +1743,11 @@ impl BatchingTask {
|
||||
);
|
||||
|
||||
let mut add_filter = AddFilterRewriter::new(expr.expr.clone());
|
||||
let mut add_auto_column = ColumnMatcherRewriter::new(
|
||||
let mut add_auto_column = ColumnMatcherRewriter::new_with_values(
|
||||
sink_table_schema.clone(),
|
||||
primary_key_indices.to_vec(),
|
||||
allow_partial,
|
||||
values.clone(),
|
||||
);
|
||||
|
||||
let plan = self.restore_scoped_dirty_windows_on_err(
|
||||
|
||||
@@ -18,6 +18,7 @@ use common_error::ext::BoxedError;
|
||||
use common_telemetry::debug;
|
||||
use common_telemetry::tracing::warn;
|
||||
use datafusion_expr::{DmlStatement, LogicalPlan};
|
||||
use query::QueryEngineRef;
|
||||
use query::options::{
|
||||
FLOW_INCREMENTAL_AFTER_SEQS, FLOW_INCREMENTAL_MODE, FLOW_INCREMENTAL_MODE_MEMTABLE_ONLY,
|
||||
FLOW_INCREMENTAL_MODE_SEQUENCE_RANGE, FLOW_SINK_TABLE_ID,
|
||||
@@ -119,6 +120,7 @@ impl BatchingTask {
|
||||
/// incremental safe without a rewrite, so they return `Some(original_plan)`.
|
||||
pub(super) async fn prepare_plan_for_incremental(
|
||||
&self,
|
||||
engine: &QueryEngineRef,
|
||||
plan: &LogicalPlan,
|
||||
) -> Result<Option<LogicalPlan>, Error> {
|
||||
let is_incremental_sql = {
|
||||
@@ -219,6 +221,7 @@ impl BatchingTask {
|
||||
let rewritten_inner = match rewrite_incremental_aggregate_with_sink_merge(
|
||||
&inner_plan,
|
||||
&analysis,
|
||||
engine,
|
||||
sink_table,
|
||||
&self.config.sink_table_name,
|
||||
None,
|
||||
|
||||
@@ -15,6 +15,7 @@
|
||||
use std::collections::{BTreeMap, BTreeSet, HashMap};
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::task::Poll;
|
||||
|
||||
use catalog::memory::MemoryCatalogManager;
|
||||
use catalog::{DeregisterTableRequest, RegisterTableRequest};
|
||||
@@ -59,6 +60,329 @@ fn incremental_batch_opts() -> Arc<BatchingModeOptions> {
|
||||
})
|
||||
}
|
||||
|
||||
struct CountingExecution {
|
||||
calls: std::sync::atomic::AtomicUsize,
|
||||
active: std::sync::atomic::AtomicUsize,
|
||||
max_active: std::sync::atomic::AtomicUsize,
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl crate::BatchingExecution for CountingExecution {
|
||||
async fn execute_once(
|
||||
self: Arc<Self>,
|
||||
_guard: BatchingExecutionGuard,
|
||||
_task: &BatchingTask,
|
||||
_engine: &QueryEngineRef,
|
||||
_frontend: &Arc<FrontendClient>,
|
||||
_max_window_cnt: Option<usize>,
|
||||
) -> ExecuteOnceOutcome {
|
||||
self.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
|
||||
let active = self
|
||||
.active
|
||||
.fetch_add(1, std::sync::atomic::Ordering::SeqCst)
|
||||
+ 1;
|
||||
self.max_active
|
||||
.fetch_max(active, std::sync::atomic::Ordering::SeqCst);
|
||||
tokio::task::yield_now().await;
|
||||
self.active
|
||||
.fetch_sub(1, std::sync::atomic::Ordering::SeqCst);
|
||||
ExecuteOnceOutcome {
|
||||
new_query: None,
|
||||
result: Ok(None),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct RetainingExecution {
|
||||
calls: std::sync::atomic::AtomicUsize,
|
||||
started: Arc<tokio::sync::Notify>,
|
||||
release: Arc<tokio::sync::Notify>,
|
||||
finished: Arc<tokio::sync::Notify>,
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl crate::BatchingExecution for RetainingExecution {
|
||||
async fn execute_once(
|
||||
self: Arc<Self>,
|
||||
guard: BatchingExecutionGuard,
|
||||
_task: &BatchingTask,
|
||||
_engine: &QueryEngineRef,
|
||||
_frontend: &Arc<FrontendClient>,
|
||||
_max_window_cnt: Option<usize>,
|
||||
) -> ExecuteOnceOutcome {
|
||||
if self.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) != 0 {
|
||||
return ExecuteOnceOutcome {
|
||||
new_query: None,
|
||||
result: Ok(None),
|
||||
};
|
||||
}
|
||||
|
||||
let started = self.started.clone();
|
||||
let release = self.release.clone();
|
||||
let finished = self.finished.clone();
|
||||
let child = tokio::spawn(async move {
|
||||
started.notify_one();
|
||||
release.notified().await;
|
||||
drop(guard);
|
||||
finished.notify_one();
|
||||
ExecuteOnceOutcome {
|
||||
new_query: None,
|
||||
result: Ok(None),
|
||||
}
|
||||
});
|
||||
match child.await {
|
||||
Ok(outcome) => outcome,
|
||||
Err(err) => ExecuteOnceOutcome {
|
||||
new_query: None,
|
||||
result: Err(Error::Unexpected {
|
||||
reason: format!("retaining test child failed: {err}"),
|
||||
location: snafu::location!(),
|
||||
}),
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_execution_delegate_dispatch_is_serialized() {
|
||||
let TestTaskParts {
|
||||
task, query_engine, ..
|
||||
} = new_test_task_engine_and_plan_with_query("SELECT number, ts FROM numbers_with_ts", "sink")
|
||||
.await;
|
||||
let execution = Arc::new(CountingExecution {
|
||||
calls: Default::default(),
|
||||
active: Default::default(),
|
||||
max_active: Default::default(),
|
||||
});
|
||||
let task = task.with_execution(Some(execution.clone()));
|
||||
let (frontend, _handler) = FrontendClient::from_empty_grpc_handler(QueryOptions::default());
|
||||
let frontend = Arc::new(frontend);
|
||||
|
||||
let first = task.execute_once_serialized(&query_engine, &frontend, None);
|
||||
let second = task.execute_once_serialized(&query_engine, &frontend, None);
|
||||
let (first, second) = tokio::join!(first, second);
|
||||
assert_eq!(first.unwrap(), None);
|
||||
assert_eq!(second.unwrap(), None);
|
||||
assert_eq!(execution.calls.load(std::sync::atomic::Ordering::SeqCst), 2);
|
||||
assert_eq!(
|
||||
execution
|
||||
.max_active
|
||||
.load(std::sync::atomic::Ordering::SeqCst),
|
||||
1,
|
||||
"the existing execution_lock must span delegate execution"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_delegate_guard_survives_caller_cancellation_until_child_finishes() {
|
||||
let TestTaskParts {
|
||||
task, query_engine, ..
|
||||
} = new_test_task_engine_and_plan_with_query("SELECT number, ts FROM numbers_with_ts", "sink")
|
||||
.await;
|
||||
let execution = Arc::new(RetainingExecution {
|
||||
calls: Default::default(),
|
||||
started: Arc::new(tokio::sync::Notify::new()),
|
||||
release: Arc::new(tokio::sync::Notify::new()),
|
||||
finished: Arc::new(tokio::sync::Notify::new()),
|
||||
});
|
||||
let task = task.with_execution(Some(execution.clone()));
|
||||
let (frontend, _handler) = FrontendClient::from_empty_grpc_handler(QueryOptions::default());
|
||||
let frontend = Arc::new(frontend);
|
||||
|
||||
let first_task = task.clone();
|
||||
let first_engine = query_engine.clone();
|
||||
let first_frontend = frontend.clone();
|
||||
let first = tokio::spawn(async move {
|
||||
first_task
|
||||
.execute_once_serialized(&first_engine, &first_frontend, None)
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(1), execution.started.notified())
|
||||
.await
|
||||
.expect("delegate child did not retain the guard");
|
||||
first.abort();
|
||||
assert!(
|
||||
first
|
||||
.await
|
||||
.expect_err("caller cancellation should abort")
|
||||
.is_cancelled()
|
||||
);
|
||||
|
||||
let second = task.execute_once_serialized(&query_engine, &frontend, None);
|
||||
futures::pin_mut!(second);
|
||||
assert!(
|
||||
matches!(futures::poll!(second.as_mut()), Poll::Pending),
|
||||
"the next round must remain pending while the retained guard is held"
|
||||
);
|
||||
assert_eq!(
|
||||
execution.calls.load(std::sync::atomic::Ordering::SeqCst),
|
||||
1,
|
||||
"the pending waiter must not enter the collaborator"
|
||||
);
|
||||
execution.release.notify_one();
|
||||
tokio::time::timeout(Duration::from_secs(1), execution.finished.notified())
|
||||
.await
|
||||
.expect("delegate child did not release its guard");
|
||||
assert_eq!(
|
||||
tokio::time::timeout(Duration::from_secs(1), second)
|
||||
.await
|
||||
.expect("next round should proceed after child release")
|
||||
.unwrap(),
|
||||
None
|
||||
);
|
||||
assert_eq!(execution.calls.load(std::sync::atomic::Ordering::SeqCst), 2);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_scheduled_context_is_retained_until_delegate_child_releases_guard() {
|
||||
let TestTaskParts {
|
||||
task, query_engine, ..
|
||||
} = new_test_task_engine_and_plan_with_query("SELECT number, ts FROM numbers_with_ts", "sink")
|
||||
.await;
|
||||
let execution = Arc::new(RetainingExecution {
|
||||
calls: Default::default(),
|
||||
started: Arc::new(tokio::sync::Notify::new()),
|
||||
release: Arc::new(tokio::sync::Notify::new()),
|
||||
finished: Arc::new(tokio::sync::Notify::new()),
|
||||
});
|
||||
let task = task.with_execution(Some(execution.clone()));
|
||||
let (frontend, _handler) = FrontendClient::from_empty_grpc_handler(QueryOptions::default());
|
||||
let frontend = Arc::new(frontend);
|
||||
let scheduled = 1_700_000_000;
|
||||
|
||||
let task_to_run = task.clone();
|
||||
let engine_to_run = query_engine.clone();
|
||||
let frontend_to_run = frontend.clone();
|
||||
let execution_call = tokio::spawn(async move {
|
||||
task_to_run
|
||||
.execute_once_serialized_at_scheduled_time(&engine_to_run, &frontend_to_run, scheduled)
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(1), execution.started.notified())
|
||||
.await
|
||||
.expect("scheduled delegate child did not start");
|
||||
assert_eq!(
|
||||
task.state
|
||||
.read()
|
||||
.unwrap()
|
||||
.query_ctx
|
||||
.extension(FLOW_SCHEDULED_TIME_MILLIS),
|
||||
Some("1700000000000"),
|
||||
"scheduled context restored before the delegate child released the guard"
|
||||
);
|
||||
execution_call.abort();
|
||||
match execution_call.await {
|
||||
Err(error) => assert!(error.is_cancelled()),
|
||||
Ok(_) => panic!("scheduled caller cancellation should abort"),
|
||||
}
|
||||
assert_eq!(
|
||||
task.state
|
||||
.read()
|
||||
.unwrap()
|
||||
.query_ctx
|
||||
.extension(FLOW_SCHEDULED_TIME_MILLIS),
|
||||
Some("1700000000000"),
|
||||
"scheduled context restored after caller cancellation but before child release"
|
||||
);
|
||||
|
||||
execution.release.notify_one();
|
||||
tokio::time::timeout(Duration::from_secs(1), execution.finished.notified())
|
||||
.await
|
||||
.expect("scheduled delegate child did not release its guard");
|
||||
assert_eq!(
|
||||
task.state
|
||||
.read()
|
||||
.unwrap()
|
||||
.query_ctx
|
||||
.extension(FLOW_SCHEDULED_TIME_MILLIS),
|
||||
None,
|
||||
"scheduled context was not restored when child released guard"
|
||||
);
|
||||
}
|
||||
|
||||
struct BlockingDefaultExecutionHandler {
|
||||
entered: std::sync::Mutex<Option<tokio::sync::oneshot::Sender<()>>>,
|
||||
dropped: std::sync::Mutex<Option<tokio::sync::oneshot::Sender<()>>>,
|
||||
}
|
||||
|
||||
struct DropAck(Option<tokio::sync::oneshot::Sender<()>>);
|
||||
|
||||
impl Drop for DropAck {
|
||||
fn drop(&mut self) {
|
||||
if let Some(dropped) = self.0.take() {
|
||||
let _ = dropped.send(());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl crate::batching_mode::frontend_client::GrpcQueryHandlerWithBoxedError
|
||||
for BlockingDefaultExecutionHandler
|
||||
{
|
||||
async fn do_query(
|
||||
&self,
|
||||
_query: api::v1::greptime_request::Request,
|
||||
_ctx: QueryContextRef,
|
||||
) -> std::result::Result<Output, BoxedError> {
|
||||
let _ack = DropAck(self.dropped.lock().unwrap().take());
|
||||
if let Some(entered) = self.entered.lock().unwrap().take() {
|
||||
let _ = entered.send(());
|
||||
}
|
||||
std::future::pending().await
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_default_execution_remains_inline_and_cancellable() {
|
||||
let query = "SELECT number, date_bin(INTERVAL '5 second', ts) AS time_window \
|
||||
FROM numbers_with_ts GROUP BY time_window, number";
|
||||
let TestTaskParts {
|
||||
task, query_engine, ..
|
||||
} = new_time_window_test_task_with_query(query).await;
|
||||
register_twe_sink(&query_engine, "missing_sink", 9200);
|
||||
task.mark_all_windows_as_dirty().unwrap();
|
||||
|
||||
let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
|
||||
let (dropped_tx, dropped_rx) = tokio::sync::oneshot::channel();
|
||||
let handler: Arc<dyn crate::batching_mode::frontend_client::GrpcQueryHandlerWithBoxedError> =
|
||||
Arc::new(BlockingDefaultExecutionHandler {
|
||||
entered: std::sync::Mutex::new(Some(entered_tx)),
|
||||
dropped: std::sync::Mutex::new(Some(dropped_tx)),
|
||||
});
|
||||
let frontend = Arc::new(FrontendClient::from_grpc_handler(
|
||||
Arc::downgrade(&handler),
|
||||
QueryOptions::default(),
|
||||
));
|
||||
|
||||
let task_to_cancel = task.clone();
|
||||
let engine_to_cancel = query_engine.clone();
|
||||
let frontend_to_cancel = frontend.clone();
|
||||
let caller = tokio::spawn(async move {
|
||||
task_to_cancel
|
||||
.execute_once_serialized(&engine_to_cancel, &frontend_to_cancel, None)
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(1), entered_rx)
|
||||
.await
|
||||
.expect("default execution did not dispatch a frontend query")
|
||||
.expect("default execution handler entry notification dropped");
|
||||
caller.abort();
|
||||
match caller.await {
|
||||
Err(error) => assert!(error.is_cancelled()),
|
||||
Ok(_) => panic!("default caller cancellation should abort"),
|
||||
}
|
||||
tokio::time::timeout(Duration::from_secs(1), dropped_rx)
|
||||
.await
|
||||
.expect("cancelling default execution did not drop the active frontend future")
|
||||
.expect("default execution drop acknowledgement was not sent");
|
||||
tokio::time::timeout(
|
||||
Duration::from_secs(1),
|
||||
task.execution_lock.clone().lock_owned(),
|
||||
)
|
||||
.await
|
||||
.expect("default execution must not leave an owned child holding the lock");
|
||||
}
|
||||
|
||||
async fn new_test_task_and_plan_with_missing_sink() -> (BatchingTask, LogicalPlan) {
|
||||
new_test_task_engine_and_plan_with_query(
|
||||
"SELECT number, ts FROM numbers_with_ts",
|
||||
@@ -2072,6 +2396,62 @@ async fn test_full_snapshot_seeding_applies_expire_after_retention_filter() {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_force_full_snapshot_retains_expiration_and_reversible_dirty_capture() {
|
||||
let TestTaskParts {
|
||||
mut task,
|
||||
query_engine,
|
||||
..
|
||||
} = new_time_window_test_task_with_query(
|
||||
"SELECT max(number) AS number, date_bin(INTERVAL '5 second', ts) AS time_window FROM numbers_with_ts GROUP BY time_window",
|
||||
)
|
||||
.await;
|
||||
{
|
||||
let mut state = task.state.write().unwrap();
|
||||
state.advance_checkpoints(HashMap::from([(1_u64, 10_u64)]));
|
||||
state
|
||||
.dirty_time_windows
|
||||
.add_window(Timestamp::new_second(0), Some(Timestamp::new_second(5)));
|
||||
}
|
||||
register_twe_sink(&query_engine, "missing_sink", 9201);
|
||||
Arc::get_mut(&mut task.config)
|
||||
.expect("test task config should be uniquely owned")
|
||||
.expire_after = Some(expire_after_for_retention_filter_test());
|
||||
|
||||
let plan = task
|
||||
.gen_insert_plan_with_values_unlocked(&query_engine, Some(1), &BTreeMap::new(), true)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert!(matches!(plan.coverage, QueryCoverage::UnfilteredFull));
|
||||
assert!(
|
||||
plan.plan
|
||||
.to_string()
|
||||
.contains("Filter: ts >= TimestampMillisecond(")
|
||||
);
|
||||
assert!(task.state.read().unwrap().dirty_time_windows.is_empty());
|
||||
|
||||
task.restore_dirty_windows(&plan.dirty_restore);
|
||||
assert_eq!(task.state.read().unwrap().dirty_time_windows.len(), 1);
|
||||
|
||||
task.state.write().unwrap().dirty_time_windows.clean();
|
||||
let no_dirty_plan = task
|
||||
.gen_insert_plan_with_values_unlocked(&query_engine, Some(1), &BTreeMap::new(), true)
|
||||
.await
|
||||
.unwrap()
|
||||
.expect("forced full snapshot must bypass the dirty-window notification gate");
|
||||
assert!(matches!(
|
||||
no_dirty_plan.coverage,
|
||||
QueryCoverage::UnfilteredFull
|
||||
));
|
||||
assert!(
|
||||
no_dirty_plan
|
||||
.plan
|
||||
.to_string()
|
||||
.contains("Filter: ts >= TimestampMillisecond(")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_incremental_plan_does_not_add_dirty_window_filter() {
|
||||
let TestTaskParts {
|
||||
@@ -2189,19 +2569,38 @@ async fn test_exact_required_executed_failure_selects_full_snapshot_repair() {
|
||||
Arc::downgrade(&handler),
|
||||
QueryOptions::default(),
|
||||
));
|
||||
task.execute_logical_plan_unlocked(
|
||||
&frontend_client,
|
||||
&plan_info.plan,
|
||||
&plan_info.dirty_restore,
|
||||
&plan_info.coverage,
|
||||
)
|
||||
.await
|
||||
.expect_err("the dispatched exact delta must fail through the injected frontend handler");
|
||||
let raw = task
|
||||
.execute_plan_unlocked(
|
||||
&query_engine,
|
||||
&frontend_client,
|
||||
&plan_info.plan,
|
||||
&plan_info.dirty_restore,
|
||||
&plan_info.coverage,
|
||||
)
|
||||
.await
|
||||
.expect("raw executor should return the dispatched execution result")
|
||||
.expect("raw executor should dispatch the exact delta");
|
||||
assert!(
|
||||
raw.0.is_err(),
|
||||
"the dispatched exact delta must fail through the injected frontend handler"
|
||||
);
|
||||
assert!(
|
||||
handler_invoked.load(Ordering::SeqCst),
|
||||
"the injected frontend handler must receive the exact delta"
|
||||
);
|
||||
task.handle_executed_query_failure(Some(&plan_info));
|
||||
// The raw executor is a delegate-safe primitive: it logs failures but does not change
|
||||
// checkpoint state. Restore the consumed work, then let the default wrapper make its one
|
||||
// failure transition and restore it again.
|
||||
{
|
||||
let state = task.state.read().unwrap();
|
||||
assert_eq!(state.checkpoint_mode(), CheckpointMode::Incremental);
|
||||
assert_eq!(state.checkpoints(), &BTreeMap::from([(1_u64, 10_u64)]));
|
||||
}
|
||||
task.restore_dirty_windows(&plan_info.dirty_restore);
|
||||
let outcome = task
|
||||
.execute_once_default_unlocked(&query_engine, &frontend_client, Some(1))
|
||||
.await;
|
||||
assert!(outcome.result.is_err());
|
||||
|
||||
{
|
||||
let state = task.state.read().unwrap();
|
||||
@@ -2459,6 +2858,7 @@ async fn test_exact_required_unsupported_plan_keeps_exact_retry_state() {
|
||||
for _ in 0..2 {
|
||||
let err = task
|
||||
.execute_logical_plan_unlocked(
|
||||
&query_engine,
|
||||
&frontend_client,
|
||||
&dml_plan,
|
||||
&DirtyRestore::Unscoped(dirty_range(10, 15)),
|
||||
@@ -2544,7 +2944,10 @@ async fn test_prepare_plan_for_incremental_disables_on_non_aggregate() {
|
||||
CheckpointMode::Incremental
|
||||
);
|
||||
|
||||
let incremental_plan = task.prepare_plan_for_incremental(&dml_plan).await.unwrap();
|
||||
let incremental_plan = task
|
||||
.prepare_plan_for_incremental(&query_engine, &dml_plan)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(incremental_plan.is_none());
|
||||
let state = task.state.read().unwrap();
|
||||
assert!(state.is_incremental_disabled());
|
||||
@@ -2621,6 +3024,7 @@ async fn test_unsafe_incremental_plan_skip_restores_dirty_without_query() {
|
||||
|
||||
let result = task
|
||||
.execute_logical_plan_unlocked(
|
||||
&query_engine,
|
||||
&Arc::new(frontend_client),
|
||||
&dml_plan,
|
||||
&dirty_restore,
|
||||
@@ -2707,7 +3111,7 @@ async fn test_prepare_plan_for_incremental_group_by_without_merge_columns_uses_o
|
||||
.advance_checkpoints(HashMap::from([(1_u64, 10_u64)]));
|
||||
|
||||
let incremental_plan = task
|
||||
.prepare_plan_for_incremental(&dml_plan)
|
||||
.prepare_plan_for_incremental(&query_engine, &dml_plan)
|
||||
.await
|
||||
.unwrap()
|
||||
.expect("plain GROUP BY is incremental-safe without a rewrite");
|
||||
@@ -2752,7 +3156,10 @@ async fn test_auto_created_sql_aggregate_sink_reaches_incremental_safe() {
|
||||
.write()
|
||||
.unwrap()
|
||||
.advance_checkpoints(HashMap::from([(1_u64, 10_u64)]));
|
||||
let incremental_plan = task.prepare_plan_for_incremental(&dml_plan).await.unwrap();
|
||||
let incremental_plan = task
|
||||
.prepare_plan_for_incremental(&query_engine, &dml_plan)
|
||||
.await
|
||||
.unwrap();
|
||||
let incremental_safe = incremental_plan.is_some();
|
||||
|
||||
assert!(incremental_safe);
|
||||
|
||||
+363
-101
@@ -14,7 +14,7 @@
|
||||
|
||||
//! some utils for helping with batching mode
|
||||
|
||||
use std::collections::{BTreeSet, HashMap, HashSet};
|
||||
use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
|
||||
use std::sync::Arc;
|
||||
|
||||
use catalog::CatalogManagerRef;
|
||||
@@ -66,6 +66,9 @@ mod test;
|
||||
/// `max(numbers_with_ts.number)`.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct IncrementalAggregateMergeColumn {
|
||||
/// Delta-plan field containing the aggregate result/state column. Repeated
|
||||
/// projections intentionally share this field while using distinct outputs.
|
||||
pub input_field_name: String,
|
||||
/// Final output/sink field name for the aggregate result/state column.
|
||||
///
|
||||
pub output_field_name: String,
|
||||
@@ -73,16 +76,17 @@ pub struct IncrementalAggregateMergeColumn {
|
||||
}
|
||||
|
||||
impl IncrementalAggregateMergeColumn {
|
||||
/// Create a new merge column.
|
||||
/// Create a new merge column whose delta and output fields have the same name.
|
||||
pub fn new(output_field_name: String, merge_op: IncrementalAggregateMergeOp) -> Self {
|
||||
Self {
|
||||
input_field_name: output_field_name.clone(),
|
||||
output_field_name,
|
||||
merge_op,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum IncrementalAggregateMergeOp {
|
||||
Sum,
|
||||
Min,
|
||||
@@ -92,6 +96,10 @@ pub enum IncrementalAggregateMergeOp {
|
||||
BitAnd,
|
||||
BitOr,
|
||||
BitXor,
|
||||
StateDeltaMerge {
|
||||
function_name: &'static str,
|
||||
params: Vec<Expr>,
|
||||
},
|
||||
}
|
||||
|
||||
/// Analysis result for an incremental aggregate plan.
|
||||
@@ -113,24 +121,6 @@ pub struct IncrementalAggregateAnalysis {
|
||||
pub unsupported_exprs: Vec<String>,
|
||||
}
|
||||
|
||||
/// Recursively find all `Expr::Column` names inside an expression tree.
|
||||
/// Only recurses into wrappers that are merge-transparent.
|
||||
/// Non-transparent wrappers (e.g., `ScalarFunction`, `Negative`, `Cast`) are
|
||||
/// intentionally not recursed into since their merge semantics would be
|
||||
/// incorrect.
|
||||
///
|
||||
/// `Cast`/`TryCast` are intentionally opaque: merging already-casted aggregate
|
||||
/// outputs is not generally equivalent to casting the final merged aggregate.
|
||||
fn find_column_names(expr: &Expr, names: &mut Vec<String>) {
|
||||
match expr {
|
||||
Expr::Column(col) => {
|
||||
names.push(col.name.clone());
|
||||
}
|
||||
Expr::Alias(alias) => find_column_names(&alias.expr, names),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
fn unqualified_col(name: impl Into<String>) -> Expr {
|
||||
Expr::Column(Column::from_name(name.into()))
|
||||
}
|
||||
@@ -230,8 +220,8 @@ fn check_input_plan_shape(plan: &LogicalPlan) -> Result<(), String> {
|
||||
#[derive(Debug, Default)]
|
||||
struct OutputProjectionInfo {
|
||||
has_top_level_projection: bool,
|
||||
output_aliases: HashMap<String, String>,
|
||||
duplicate_aggregate_aliases: BTreeSet<String>,
|
||||
/// Aggregate expression name and projected output field, in projection order.
|
||||
aggregate_outputs: Vec<(String, String)>,
|
||||
literal_columns: HashSet<String>,
|
||||
output_field_names: Vec<String>,
|
||||
}
|
||||
@@ -265,46 +255,34 @@ fn collect_output_projection_info(plan: &LogicalPlan) -> OutputProjectionInfo {
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let mut output_aliases = HashMap::new();
|
||||
if let LogicalPlan::Projection(projection) = plan {
|
||||
for expr in &projection.expr {
|
||||
match expr {
|
||||
Expr::Alias(alias) => {
|
||||
// Alias resolution has three cases:
|
||||
// - 0 Column refs (e.g., literal `42 AS lit`): record literal output
|
||||
// - 1 Column ref: record the mapping (e.g., `sum(x) AS total`)
|
||||
// - >1 Column refs (e.g., `COALESCE(sum(x), sum(y))`):
|
||||
// skip — ambiguous merge semantics
|
||||
// Only a direct aggregate output column has the same
|
||||
// merge semantics as the original resolver. In particular,
|
||||
// do not mine aggregate columns through CAST/TryCast or
|
||||
// other output wrappers.
|
||||
let alias_name = alias.name.clone();
|
||||
let mut col_names = Vec::new();
|
||||
find_column_names(&alias.expr, &mut col_names);
|
||||
match col_names.len() {
|
||||
0 if is_passthrough_output_column(&alias_name, alias.expr.as_ref()) => {
|
||||
projection_info.literal_columns.insert(alias_name);
|
||||
}
|
||||
1 => {
|
||||
if let Some(col_name) = col_names.into_iter().next() {
|
||||
if let Some(existing_alias) = output_aliases.get(&col_name) {
|
||||
if existing_alias != &alias_name {
|
||||
projection_info.duplicate_aggregate_aliases.insert(format!(
|
||||
"same aggregate output {col_name} is used by multiple aliases: {existing_alias}, {alias_name}"
|
||||
));
|
||||
}
|
||||
} else {
|
||||
output_aliases.insert(col_name, alias_name);
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
if let Expr::Column(column) = alias.expr.as_ref() {
|
||||
projection_info
|
||||
.aggregate_outputs
|
||||
.push((column.name.clone(), alias_name));
|
||||
} else if let Expr::Alias(inner_alias) = alias.expr.as_ref()
|
||||
&& inner_alias.name.eq_ignore_ascii_case("count(*)")
|
||||
&& let Expr::Column(column) = inner_alias.expr.as_ref()
|
||||
{
|
||||
projection_info
|
||||
.aggregate_outputs
|
||||
.push((column.name.clone(), alias_name));
|
||||
} else if is_passthrough_output_column(&alias_name, alias.expr.as_ref()) {
|
||||
projection_info.literal_columns.insert(alias_name);
|
||||
}
|
||||
|
||||
// If >1 column references detected (e.g., COALESCE(sum(x), sum(y))),
|
||||
// intentionally skip alias mapping — the merge semantics are ambiguous.
|
||||
}
|
||||
Expr::Column(col) => {
|
||||
output_aliases
|
||||
.entry(col.name.clone())
|
||||
.or_insert(col.name.clone());
|
||||
projection_info
|
||||
.aggregate_outputs
|
||||
.push((col.name.clone(), col.name.clone()));
|
||||
}
|
||||
Expr::Literal(_, _) => {
|
||||
projection_info
|
||||
@@ -326,7 +304,6 @@ fn collect_output_projection_info(plan: &LogicalPlan) -> OutputProjectionInfo {
|
||||
.insert(AUTO_CREATED_PLACEHOLDER_TS_COL.to_string());
|
||||
}
|
||||
|
||||
projection_info.output_aliases = output_aliases;
|
||||
projection_info
|
||||
}
|
||||
|
||||
@@ -362,6 +339,12 @@ fn merge_op_for_aggregate_expr(aggr_expr: &Expr) -> Result<IncrementalAggregateM
|
||||
return Err(format!("unsupported aggregate NULL treatment: {aggr_expr}"));
|
||||
}
|
||||
|
||||
let state_delta_merge = |function_name, params| {
|
||||
Ok(IncrementalAggregateMergeOp::StateDeltaMerge {
|
||||
function_name,
|
||||
params,
|
||||
})
|
||||
};
|
||||
match aggr_func.func.name().to_ascii_lowercase().as_str() {
|
||||
"sum" | "count" => Ok(IncrementalAggregateMergeOp::Sum),
|
||||
"min" => Ok(IncrementalAggregateMergeOp::Min),
|
||||
@@ -371,28 +354,52 @@ fn merge_op_for_aggregate_expr(aggr_expr: &Expr) -> Result<IncrementalAggregateM
|
||||
"bit_and" => Ok(IncrementalAggregateMergeOp::BitAnd),
|
||||
"bit_or" => Ok(IncrementalAggregateMergeOp::BitOr),
|
||||
"bit_xor" => Ok(IncrementalAggregateMergeOp::BitXor),
|
||||
// Preserve state-family parameters; value coercion is handled by the aggregate.
|
||||
"hll" if aggr_func.params.args.len() == 1 => state_delta_merge("__hll_delta_merge", vec![]),
|
||||
"stddev_pop_state" if aggr_func.params.args.len() == 1 => {
|
||||
state_delta_merge("__stddev_pop_state_delta_merge", vec![])
|
||||
}
|
||||
"uddsketch_state" if aggr_func.params.args.len() == 3 => {
|
||||
let [bucket_size, error_rate, _] = aggr_func.params.args.as_slice() else {
|
||||
unreachable!();
|
||||
};
|
||||
if !matches!(bucket_size, Expr::Literal(ScalarValue::Int64(Some(_)), _))
|
||||
|| !matches!(error_rate, Expr::Literal(ScalarValue::Float64(Some(_)), _))
|
||||
{
|
||||
return Err(aggr_expr.to_string());
|
||||
}
|
||||
state_delta_merge(
|
||||
"__uddsketch_state_delta_merge",
|
||||
vec![bucket_size.clone(), error_rate.clone()],
|
||||
)
|
||||
}
|
||||
_ => Err(aggr_expr.to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
fn resolve_aggregate_output_field_name(
|
||||
fn resolve_aggregate_output_fields(
|
||||
aggr_expr: &Expr,
|
||||
projection_info: &OutputProjectionInfo,
|
||||
output_field_name_set: &HashSet<String>,
|
||||
) -> Option<String> {
|
||||
) -> Vec<(String, String)> {
|
||||
// qualified_name() returns (Option<String>, String) where the second
|
||||
// element is the unqualified column/alias name. This relies on
|
||||
// DataFusion's internal naming convention: aggregate expressions
|
||||
// emit a column named after the aggregate itself (e.g. "SUM(x)"),
|
||||
// which matches what the projection aliases reference.
|
||||
// emit a column named after the aggregate itself (e.g. "SUM(x)").
|
||||
// Keep every matching projection occurrence because DataFusion can share
|
||||
// one aggregate input field for identical expressions.
|
||||
let raw_name = aggr_expr.qualified_name().1;
|
||||
if let Some(alias) = projection_info.output_aliases.get(&raw_name) {
|
||||
Some(alias.clone())
|
||||
} else if !projection_info.has_top_level_projection && output_field_name_set.contains(&raw_name)
|
||||
{
|
||||
Some(raw_name)
|
||||
if projection_info.has_top_level_projection {
|
||||
projection_info
|
||||
.aggregate_outputs
|
||||
.iter()
|
||||
.filter(|(input_name, _)| input_name == &raw_name)
|
||||
.cloned()
|
||||
.collect()
|
||||
} else if output_field_name_set.contains(&raw_name) {
|
||||
vec![(raw_name.clone(), raw_name)]
|
||||
} else {
|
||||
None
|
||||
vec![]
|
||||
}
|
||||
}
|
||||
|
||||
@@ -478,7 +485,6 @@ pub fn analyze_incremental_aggregate_plan(
|
||||
.map(|name| format!("duplicate output field name: {name}"))
|
||||
.collect::<Vec<_>>();
|
||||
unsupported_exprs.push(reason);
|
||||
unsupported_exprs.extend(projection_info.duplicate_aggregate_aliases.iter().cloned());
|
||||
return Ok(Some(IncrementalAggregateAnalysis {
|
||||
group_key_names,
|
||||
merge_columns: vec![],
|
||||
@@ -512,7 +518,6 @@ pub fn analyze_incremental_aggregate_plan(
|
||||
aggregate,
|
||||
&group_key_names,
|
||||
));
|
||||
unsupported_exprs.extend(projection_info.duplicate_aggregate_aliases.iter().cloned());
|
||||
for aggr_expr in aggr_exprs {
|
||||
let merge_op = match merge_op_for_aggregate_expr(&aggr_expr) {
|
||||
Ok(merge_op) => merge_op,
|
||||
@@ -521,18 +526,37 @@ pub fn analyze_incremental_aggregate_plan(
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let Some(output_field_name) = resolve_aggregate_output_field_name(
|
||||
&aggr_expr,
|
||||
&projection_info,
|
||||
&output_field_name_set,
|
||||
) else {
|
||||
let aggregate_outputs =
|
||||
resolve_aggregate_output_fields(&aggr_expr, &projection_info, &output_field_name_set);
|
||||
if aggregate_outputs.is_empty() {
|
||||
unsupported_exprs.push(aggr_expr.to_string());
|
||||
continue;
|
||||
}
|
||||
let Some((_, input_field_name)) = aggregate_outputs.first() else {
|
||||
continue;
|
||||
};
|
||||
merge_columns.push(IncrementalAggregateMergeColumn::new(
|
||||
output_field_name,
|
||||
merge_op,
|
||||
));
|
||||
let input_field_name = input_field_name.clone();
|
||||
for (_, output_field_name) in aggregate_outputs {
|
||||
merge_columns.push(IncrementalAggregateMergeColumn {
|
||||
input_field_name: input_field_name.clone(),
|
||||
output_field_name,
|
||||
merge_op: merge_op.clone(),
|
||||
});
|
||||
}
|
||||
}
|
||||
if projection_info.has_top_level_projection {
|
||||
let output_positions = projection_info
|
||||
.output_field_names
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(position, name)| (name.as_str(), position))
|
||||
.collect::<HashMap<_, _>>();
|
||||
merge_columns.sort_by_key(|column| {
|
||||
output_positions
|
||||
.get(column.output_field_name.as_str())
|
||||
.copied()
|
||||
.unwrap_or(usize::MAX)
|
||||
});
|
||||
}
|
||||
unsupported_exprs.extend(
|
||||
find_uncovered_output_fields(&projection_info, &group_key_names, &merge_columns)
|
||||
@@ -591,6 +615,7 @@ pub fn analyze_incremental_aggregate_plan(
|
||||
pub async fn rewrite_incremental_aggregate_with_sink_merge(
|
||||
delta_plan: &LogicalPlan,
|
||||
analysis: &IncrementalAggregateAnalysis,
|
||||
engine: &QueryEngineRef,
|
||||
sink_table: TableRef,
|
||||
sink_table_name: &TableName,
|
||||
sink_dirty_filter: Option<Expr>,
|
||||
@@ -625,6 +650,12 @@ pub async fn rewrite_incremental_aggregate_with_sink_merge(
|
||||
let delta_alias = "__flow_delta";
|
||||
let sink_alias = "__flow_sink";
|
||||
|
||||
let state_merge = analysis.merge_columns.iter().any(|column| {
|
||||
matches!(
|
||||
&column.merge_op,
|
||||
IncrementalAggregateMergeOp::StateDeltaMerge { .. }
|
||||
)
|
||||
});
|
||||
let mut selected_columns = analysis.group_key_names.clone();
|
||||
selected_columns.extend(
|
||||
analysis
|
||||
@@ -632,8 +663,18 @@ pub async fn rewrite_incremental_aggregate_with_sink_merge(
|
||||
.iter()
|
||||
.map(|c| c.output_field_name.clone()),
|
||||
);
|
||||
let mut delta_selected_columns = selected_columns.clone();
|
||||
let mut selected_column_names = HashSet::new();
|
||||
selected_columns.retain(|name| selected_column_names.insert(name.clone()));
|
||||
let mut delta_selected_columns = analysis.group_key_names.clone();
|
||||
delta_selected_columns.extend(
|
||||
analysis
|
||||
.merge_columns
|
||||
.iter()
|
||||
.map(|c| c.input_field_name.clone()),
|
||||
);
|
||||
delta_selected_columns.extend(analysis.literal_columns.iter().cloned());
|
||||
let mut delta_selected_column_names = HashSet::new();
|
||||
delta_selected_columns.retain(|name| delta_selected_column_names.insert(name.clone()));
|
||||
|
||||
let delta_selected_exprs = delta_selected_columns
|
||||
.iter()
|
||||
@@ -745,20 +786,25 @@ pub async fn rewrite_incremental_aggregate_with_sink_merge(
|
||||
.map(|c| (&c.output_field_name, c))
|
||||
.collect::<HashMap<_, _>>();
|
||||
|
||||
let mut projection_exprs = Vec::with_capacity(analysis.output_field_names.len());
|
||||
let mut ordinary_exprs = Vec::with_capacity(analysis.output_field_names.len());
|
||||
let mut state_aggr_exprs = Vec::new();
|
||||
for output_field_name in &analysis.output_field_names {
|
||||
if group_key_names.contains(output_field_name)
|
||||
|| literal_columns.contains(output_field_name)
|
||||
{
|
||||
projection_exprs.push(
|
||||
qualified_col(delta_alias, output_field_name.clone()).alias(output_field_name),
|
||||
);
|
||||
let expr =
|
||||
qualified_col(delta_alias, output_field_name.clone()).alias(output_field_name);
|
||||
ordinary_exprs.push(expr);
|
||||
} else if let Some(merge_col) = merge_columns.get(output_field_name) {
|
||||
projection_exprs.push(build_left_join_merge_expr(
|
||||
delta_alias,
|
||||
sink_alias,
|
||||
merge_col,
|
||||
)?);
|
||||
if matches!(
|
||||
&merge_col.merge_op,
|
||||
IncrementalAggregateMergeOp::StateDeltaMerge { .. }
|
||||
) {
|
||||
state_aggr_exprs.push(build_state_delta_merge_expr(engine, merge_col)?);
|
||||
} else {
|
||||
let expr = build_left_join_merge_expr(delta_alias, sink_alias, merge_col)?;
|
||||
ordinary_exprs.push(expr);
|
||||
}
|
||||
} else {
|
||||
return InvalidQuerySnafu {
|
||||
reason: format!(
|
||||
@@ -769,15 +815,88 @@ pub async fn rewrite_incremental_aggregate_with_sink_merge(
|
||||
}
|
||||
}
|
||||
|
||||
LogicalPlanBuilder::from(joined)
|
||||
.project(projection_exprs)
|
||||
.with_context(|_| DatafusionSnafu {
|
||||
context: "Failed to build projection merge plan for incremental sink merge".to_string(),
|
||||
})?
|
||||
.build()
|
||||
.with_context(|_| DatafusionSnafu {
|
||||
context: "Failed to finalize incremental aggregate sink merge plan".to_string(),
|
||||
if state_merge {
|
||||
let aggregated = LogicalPlanBuilder::from(joined)
|
||||
.aggregate(ordinary_exprs, state_aggr_exprs)
|
||||
.with_context(|_| DatafusionSnafu {
|
||||
context: "Failed to aggregate state delta merge plan".to_string(),
|
||||
})?
|
||||
.build()
|
||||
.with_context(|_| DatafusionSnafu {
|
||||
context: "Failed to build state delta merge plan".to_string(),
|
||||
})?;
|
||||
let output_exprs = analysis
|
||||
.output_field_names
|
||||
.iter()
|
||||
.cloned()
|
||||
.map(unqualified_col)
|
||||
.collect::<Vec<_>>();
|
||||
LogicalPlanBuilder::from(aggregated)
|
||||
.project(output_exprs)
|
||||
.with_context(|_| DatafusionSnafu {
|
||||
context: "Failed to project state delta merge plan".to_string(),
|
||||
})?
|
||||
.build()
|
||||
.with_context(|_| DatafusionSnafu {
|
||||
context: "Failed to finalize incremental aggregate sink merge plan".to_string(),
|
||||
})
|
||||
} else {
|
||||
LogicalPlanBuilder::from(joined)
|
||||
.project(ordinary_exprs)
|
||||
.with_context(|_| DatafusionSnafu {
|
||||
context: "Failed to build projection merge plan for incremental sink merge"
|
||||
.to_string(),
|
||||
})?
|
||||
.build()
|
||||
.with_context(|_| DatafusionSnafu {
|
||||
context: "Failed to finalize incremental aggregate sink merge plan".to_string(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn build_state_delta_merge_expr(
|
||||
engine: &QueryEngineRef,
|
||||
merge_col: &IncrementalAggregateMergeColumn,
|
||||
) -> Result<Expr, Error> {
|
||||
let (function_name, params) = match &merge_col.merge_op {
|
||||
IncrementalAggregateMergeOp::StateDeltaMerge {
|
||||
function_name,
|
||||
params,
|
||||
} => (*function_name, params.clone()),
|
||||
_ => {
|
||||
return InvalidQuerySnafu {
|
||||
reason: "non-state aggregate passed to state delta merge".to_string(),
|
||||
}
|
||||
.fail();
|
||||
}
|
||||
};
|
||||
let Some(udaf) = engine
|
||||
.engine_state()
|
||||
.aggr_function(function_name)
|
||||
.or_else(|| {
|
||||
engine
|
||||
.engine_state()
|
||||
.session_state()
|
||||
.aggregate_functions()
|
||||
.get(function_name)
|
||||
.map(|udaf| udaf.as_ref().clone())
|
||||
})
|
||||
else {
|
||||
return InvalidQuerySnafu {
|
||||
reason: format!("Aggregate function {function_name} is not registered"),
|
||||
}
|
||||
.fail();
|
||||
};
|
||||
let mut args = params;
|
||||
args.push(qualified_col(
|
||||
"__flow_delta",
|
||||
merge_col.input_field_name.clone(),
|
||||
));
|
||||
args.push(qualified_col(
|
||||
"__flow_sink",
|
||||
merge_col.output_field_name.clone(),
|
||||
));
|
||||
Ok(udaf.call(args).alias(merge_col.output_field_name.clone()))
|
||||
}
|
||||
|
||||
fn build_left_join_merge_expr(
|
||||
@@ -785,9 +904,9 @@ fn build_left_join_merge_expr(
|
||||
sink_alias: &str,
|
||||
merge_col: &IncrementalAggregateMergeColumn,
|
||||
) -> Result<Expr, Error> {
|
||||
let left = qualified_col(delta_alias, merge_col.output_field_name.clone());
|
||||
let left = qualified_col(delta_alias, merge_col.input_field_name.clone());
|
||||
let right = qualified_col(sink_alias, merge_col.output_field_name.clone());
|
||||
let merged = match merge_col.merge_op {
|
||||
let merged = match merge_col.merge_op.clone() {
|
||||
IncrementalAggregateMergeOp::Sum => when(is_null(left.clone()), right.clone())
|
||||
.when(is_null(right.clone()), left.clone())
|
||||
.otherwise(binary_expr(left.clone(), Operator::Plus, right.clone()))
|
||||
@@ -836,6 +955,12 @@ fn build_left_join_merge_expr(
|
||||
.with_context(|_| DatafusionSnafu {
|
||||
context: "Failed to build BIT_XOR merge expression".to_string(),
|
||||
})?,
|
||||
IncrementalAggregateMergeOp::StateDeltaMerge { .. } => {
|
||||
return InvalidQuerySnafu {
|
||||
reason: "state aggregate must be built with its delta UDAF".to_string(),
|
||||
}
|
||||
.fail();
|
||||
}
|
||||
};
|
||||
Ok(merged.alias(merge_col.output_field_name.clone()))
|
||||
}
|
||||
@@ -948,13 +1073,36 @@ pub(crate) async fn gen_plan_with_matching_schema(
|
||||
sink_table_schema: SchemaRef,
|
||||
primary_key_indices: &[usize],
|
||||
allow_partial: bool,
|
||||
) -> Result<LogicalPlan, Error> {
|
||||
gen_plan_with_matching_schema_and_values(
|
||||
sql,
|
||||
query_ctx,
|
||||
engine,
|
||||
sink_table_schema,
|
||||
primary_key_indices,
|
||||
allow_partial,
|
||||
&BTreeMap::new(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Generate a schema-matching plan with extension-owned typed sink values.
|
||||
pub(crate) async fn gen_plan_with_matching_schema_and_values(
|
||||
sql: &str,
|
||||
query_ctx: QueryContextRef,
|
||||
engine: QueryEngineRef,
|
||||
sink_table_schema: SchemaRef,
|
||||
primary_key_indices: &[usize],
|
||||
allow_partial: bool,
|
||||
values: &BTreeMap<String, ScalarValue>,
|
||||
) -> Result<LogicalPlan, Error> {
|
||||
let plan = sql_to_df_plan(query_ctx.clone(), engine.clone(), sql, false).await?;
|
||||
|
||||
let mut add_auto_column = ColumnMatcherRewriter::new(
|
||||
let mut add_auto_column = ColumnMatcherRewriter::new_with_values(
|
||||
sink_table_schema,
|
||||
primary_key_indices.to_vec(),
|
||||
allow_partial,
|
||||
values.clone(),
|
||||
);
|
||||
let plan = plan
|
||||
.clone()
|
||||
@@ -1090,7 +1238,10 @@ impl TreeNodeVisitor<'_> for FindGroupByFinalName {
|
||||
/// it also give existing columns alias to column in sink table if needed
|
||||
#[derive(Debug)]
|
||||
pub struct ColumnMatcherRewriter {
|
||||
/// Schema used to match the ordinary flow output. It excludes injected values.
|
||||
pub schema: SchemaRef,
|
||||
sink_schema: SchemaRef,
|
||||
values: BTreeMap<String, ScalarValue>,
|
||||
pub is_rewritten: bool,
|
||||
pub primary_key_indices: Vec<usize>,
|
||||
pub allow_partial: bool,
|
||||
@@ -1099,21 +1250,127 @@ pub struct ColumnMatcherRewriter {
|
||||
impl ColumnMatcherRewriter {
|
||||
pub fn new(schema: SchemaRef, primary_key_indices: Vec<usize>, allow_partial: bool) -> Self {
|
||||
Self {
|
||||
schema,
|
||||
schema: schema.clone(),
|
||||
sink_schema: schema,
|
||||
values: BTreeMap::new(),
|
||||
is_rewritten: false,
|
||||
primary_key_indices,
|
||||
allow_partial,
|
||||
}
|
||||
}
|
||||
|
||||
/// Match ordinary output against the sink schema excluding typed values, then inject those
|
||||
/// values in physical sink-column order.
|
||||
pub fn new_with_values(
|
||||
sink_schema: SchemaRef,
|
||||
primary_key_indices: Vec<usize>,
|
||||
allow_partial: bool,
|
||||
values: BTreeMap<String, ScalarValue>,
|
||||
) -> Self {
|
||||
let value_names = values.keys().collect::<BTreeSet<_>>();
|
||||
let ordinary_columns = sink_schema
|
||||
.column_schemas()
|
||||
.iter()
|
||||
.filter(|column| !value_names.contains(&column.name))
|
||||
.cloned()
|
||||
.collect();
|
||||
let ordinary_schema = Arc::new(datatypes::schema::Schema::new(ordinary_columns));
|
||||
let ordinary_primary_key_indices = primary_key_indices
|
||||
.into_iter()
|
||||
.filter_map(|index| {
|
||||
let name = &sink_schema.column_schemas()[index].name;
|
||||
ordinary_schema
|
||||
.column_schemas()
|
||||
.iter()
|
||||
.position(|column| column.name == *name)
|
||||
})
|
||||
.collect();
|
||||
Self {
|
||||
schema: ordinary_schema,
|
||||
sink_schema,
|
||||
values,
|
||||
is_rewritten: false,
|
||||
primary_key_indices: ordinary_primary_key_indices,
|
||||
allow_partial,
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_values(&self, output_exprs: &[Expr]) -> DfResult<()> {
|
||||
let output_names = output_exprs
|
||||
.iter()
|
||||
.map(|expr| expr.qualified_name().1)
|
||||
.collect::<Vec<_>>();
|
||||
let duplicated = duplicate_names(&output_names);
|
||||
if !duplicated.is_empty() {
|
||||
return Err(DataFusionError::Plan(format!(
|
||||
"Flow output schema contains duplicate column(s) {:?}. {}",
|
||||
duplicated,
|
||||
format_flow_sink_schema_mismatch(output_exprs, self.sink_schema.as_ref())
|
||||
)));
|
||||
}
|
||||
for (name, value) in &self.values {
|
||||
let Some(column) = self
|
||||
.sink_schema
|
||||
.column_schemas()
|
||||
.iter()
|
||||
.find(|column| column.name == *name)
|
||||
else {
|
||||
return Err(DataFusionError::Plan(format!(
|
||||
"Injected flow value column '{name}' is not found in sink schema"
|
||||
)));
|
||||
};
|
||||
if output_names.iter().any(|output| output == name) {
|
||||
return Err(DataFusionError::Plan(format!(
|
||||
"Injected flow value column '{name}' collides with flow output"
|
||||
)));
|
||||
}
|
||||
let value_type = ConcreteDataType::from_arrow_type(&value.data_type());
|
||||
if value_type != column.data_type {
|
||||
return Err(DataFusionError::Plan(format!(
|
||||
"Injected flow value column '{name}' has type {:?}, but sink column has type {:?}",
|
||||
value_type, column.data_type
|
||||
)));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn inject_values_in_physical_order(&self, exprs: Vec<Expr>) -> DfResult<Vec<Expr>> {
|
||||
if self.values.is_empty() {
|
||||
return Ok(exprs);
|
||||
}
|
||||
let mut ordinary = exprs
|
||||
.into_iter()
|
||||
.map(|expr| (expr.qualified_name().1, expr))
|
||||
.collect::<HashMap<_, _>>();
|
||||
self.sink_schema
|
||||
.column_schemas()
|
||||
.iter()
|
||||
.map(|column| {
|
||||
if let Some(value) = self.values.get(&column.name) {
|
||||
Ok(Expr::Literal(value.clone(), None).alias(column.name.clone()))
|
||||
} else {
|
||||
ordinary.remove(&column.name).ok_or_else(|| {
|
||||
DataFusionError::Plan(format!(
|
||||
"Flow output is missing sink column '{}' after schema matching",
|
||||
column.name
|
||||
))
|
||||
})
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// modify the exprs in place so that it matches the schema and some auto columns are added
|
||||
fn modify_project_exprs(
|
||||
&mut self,
|
||||
mut exprs: Vec<Expr>,
|
||||
input_schema: &DFSchema,
|
||||
) -> DfResult<Vec<Expr>> {
|
||||
self.validate_values(&exprs)?;
|
||||
if self.allow_partial {
|
||||
return self.modify_project_exprs_with_partial(exprs);
|
||||
let exprs = self.modify_project_exprs_with_partial(exprs)?;
|
||||
return self.inject_values_in_physical_order(exprs);
|
||||
}
|
||||
|
||||
let original_exprs = exprs.clone();
|
||||
@@ -1182,7 +1439,12 @@ impl ColumnMatcherRewriter {
|
||||
)));
|
||||
}
|
||||
|
||||
self.match_extra_output_columns(exprs, input_schema, &original_exprs, &all_names)
|
||||
self.inject_values_in_physical_order(self.match_extra_output_columns(
|
||||
exprs,
|
||||
input_schema,
|
||||
&original_exprs,
|
||||
&all_names,
|
||||
)?)
|
||||
}
|
||||
|
||||
/// Match flow output columns whose names are not in the sink schema by the same position only.
|
||||
|
||||
@@ -12,13 +12,19 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
use catalog::RegisterTableRequest;
|
||||
use common_recordbatch::RecordBatch;
|
||||
use common_query::OutputData;
|
||||
use common_recordbatch::recordbatch::merge_record_batches;
|
||||
use common_recordbatch::{RecordBatch, util};
|
||||
use common_time::Timestamp;
|
||||
use datafusion_common::ScalarValue;
|
||||
use datafusion_common::tree_node::TreeNode as _;
|
||||
use datafusion_expr::{GroupingSet, TableScanBuilder};
|
||||
use datatypes::arrow::array::{Array, AsArray};
|
||||
use datatypes::arrow::datatypes::{Float64Type, Int64Type, UInt64Type};
|
||||
use datatypes::prelude::{ConcreteDataType, MutableVector, Scalar, ScalarVectorBuilder, VectorRef};
|
||||
use datatypes::schema::{ColumnSchema, Schema};
|
||||
use datatypes::timestamp::TimestampMillisecond;
|
||||
@@ -739,6 +745,108 @@ async fn test_gen_plan_with_matching_schema_allows_null_positional_alias() {
|
||||
assert!(sql.contains("NULL AS label"), "{sql}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_gen_plan_with_matching_schema_and_values_injects_physical_sink_order() {
|
||||
let sink_schema = Arc::new(Schema::new(vec![
|
||||
ColumnSchema::new("number", ConcreteDataType::uint32_datatype(), true),
|
||||
ColumnSchema::new("marker", ConcreteDataType::uint8_datatype(), true),
|
||||
ColumnSchema::new(
|
||||
"ts",
|
||||
ConcreteDataType::timestamp_millisecond_datatype(),
|
||||
false,
|
||||
)
|
||||
.with_time_index(true),
|
||||
ColumnSchema::new("epoch", ConcreteDataType::uint64_datatype(), true),
|
||||
]));
|
||||
let plan = gen_plan_with_matching_schema_and_values(
|
||||
"SELECT number, ts FROM numbers_with_ts",
|
||||
QueryContext::arc(),
|
||||
create_test_query_engine(),
|
||||
sink_schema,
|
||||
&[],
|
||||
false,
|
||||
&BTreeMap::from([
|
||||
("epoch".to_string(), ScalarValue::UInt64(Some(7))),
|
||||
("marker".to_string(), ScalarValue::UInt8(None)),
|
||||
]),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let names = plan
|
||||
.schema()
|
||||
.fields()
|
||||
.iter()
|
||||
.map(|field| field.name().clone())
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(names, vec!["number", "marker", "ts", "epoch"]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_gen_plan_with_matching_schema_and_values_rejects_unknown_collision_and_type() {
|
||||
let sink_schema = Arc::new(Schema::new(vec![
|
||||
ColumnSchema::new("number", ConcreteDataType::uint32_datatype(), true),
|
||||
ColumnSchema::new(
|
||||
"ts",
|
||||
ConcreteDataType::timestamp_millisecond_datatype(),
|
||||
false,
|
||||
)
|
||||
.with_time_index(true),
|
||||
ColumnSchema::new("marker", ConcreteDataType::uint8_datatype(), true),
|
||||
]));
|
||||
for (values, expected) in [
|
||||
(
|
||||
BTreeMap::from([("unknown".to_string(), ScalarValue::UInt8(None))]),
|
||||
"is not found in sink schema",
|
||||
),
|
||||
(
|
||||
BTreeMap::from([("number".to_string(), ScalarValue::UInt32(None))]),
|
||||
"collides with flow output",
|
||||
),
|
||||
(
|
||||
BTreeMap::from([("marker".to_string(), ScalarValue::UInt64(None))]),
|
||||
"has type UInt64",
|
||||
),
|
||||
] {
|
||||
let err = gen_plan_with_matching_schema_and_values(
|
||||
"SELECT number, ts FROM numbers_with_ts",
|
||||
QueryContext::arc(),
|
||||
create_test_query_engine(),
|
||||
sink_schema.clone(),
|
||||
&[],
|
||||
false,
|
||||
&values,
|
||||
)
|
||||
.await
|
||||
.unwrap_err()
|
||||
.to_string();
|
||||
assert!(err.contains(expected), "{err}");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_gen_plan_with_matching_schema_and_values_rejects_duplicate_output() {
|
||||
let sink_schema = Arc::new(Schema::new(vec![
|
||||
ColumnSchema::new("number", ConcreteDataType::uint32_datatype(), true),
|
||||
ColumnSchema::new("marker", ConcreteDataType::uint8_datatype(), true),
|
||||
]));
|
||||
let err = gen_plan_with_matching_schema_and_values(
|
||||
"SELECT number, number AS number FROM numbers_with_ts",
|
||||
QueryContext::arc(),
|
||||
create_test_query_engine(),
|
||||
sink_schema,
|
||||
&[],
|
||||
false,
|
||||
&BTreeMap::from([("marker".to_string(), ScalarValue::UInt8(None))]),
|
||||
)
|
||||
.await
|
||||
.unwrap_err();
|
||||
let err = format!("{err:?}");
|
||||
assert!(
|
||||
err.contains("Projections require unique expression names"),
|
||||
"DataFusion should reject duplicate output aliases during query planning: {err}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_gen_plan_with_matching_schema_accepts_matching_flow_schema() {
|
||||
let query_engine = create_test_query_engine();
|
||||
@@ -1104,13 +1212,16 @@ async fn test_rewrite_incremental_aggregate_allows_alias_wrapped_scan() {
|
||||
let query_engine = create_test_query_engine();
|
||||
let ctx = QueryContext::arc();
|
||||
let sql = "SELECT max(n.number) AS number, n.ts FROM numbers_with_ts AS n GROUP BY n.ts";
|
||||
let plan = sql_to_df_plan(ctx, query_engine, sql, false).await.unwrap();
|
||||
let plan = sql_to_df_plan(ctx, query_engine.clone(), sql, false)
|
||||
.await
|
||||
.unwrap();
|
||||
let analysis = analyze_incremental_aggregate_plan(&plan).unwrap().unwrap();
|
||||
assert!(analysis.unsupported_exprs.is_empty());
|
||||
|
||||
let rewritten = rewrite_incremental_aggregate_with_sink_merge(
|
||||
&plan,
|
||||
&analysis,
|
||||
&query_engine,
|
||||
single_row_u32_table("alias_wrapped_sink", vec!["ts", "number"]),
|
||||
&[
|
||||
"greptime".to_string(),
|
||||
@@ -1362,6 +1473,7 @@ async fn test_analyze_incremental_aggregate_plan_allows_literal_outputs() {
|
||||
let rewritten = rewrite_incremental_aggregate_with_sink_merge(
|
||||
&plan,
|
||||
&analysis,
|
||||
&query_engine,
|
||||
sink_table.clone(),
|
||||
&sink_table_name,
|
||||
None,
|
||||
@@ -1435,7 +1547,9 @@ async fn test_rewrite_incremental_aggregate_preserves_non_identifier_aliases() {
|
||||
let query_engine = create_test_query_engine();
|
||||
let ctx = QueryContext::arc();
|
||||
let sql = "SELECT max(number) AS \"max value\", number, 42 AS \"literal value\" FROM numbers_with_ts GROUP BY number";
|
||||
let plan = sql_to_df_plan(ctx, query_engine, sql, false).await.unwrap();
|
||||
let plan = sql_to_df_plan(ctx, query_engine.clone(), sql, false)
|
||||
.await
|
||||
.unwrap();
|
||||
let analysis = analyze_incremental_aggregate_plan(&plan).unwrap().unwrap();
|
||||
assert!(analysis.unsupported_exprs.is_empty());
|
||||
assert_eq!(
|
||||
@@ -1447,6 +1561,7 @@ async fn test_rewrite_incremental_aggregate_preserves_non_identifier_aliases() {
|
||||
let rewritten = rewrite_incremental_aggregate_with_sink_merge(
|
||||
&plan,
|
||||
&analysis,
|
||||
&query_engine,
|
||||
sink_table,
|
||||
&[
|
||||
"greptime".to_string(),
|
||||
@@ -1520,24 +1635,34 @@ async fn test_datafusion_rejects_duplicate_output_names() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_analyze_incremental_aggregate_plan_rejects_same_aggregate_multiple_aliases() {
|
||||
let query_engine = create_test_query_engine();
|
||||
let ctx = QueryContext::arc();
|
||||
let sql = "SELECT sum(number) AS a, sum(number) AS b, ts FROM numbers_with_ts GROUP BY ts";
|
||||
let plan = sql_to_df_plan(ctx, query_engine, sql, false).await.unwrap();
|
||||
async fn test_analyze_incremental_aggregate_plan_supports_same_aggregate_multiple_aliases() {
|
||||
let analysis = analyze_test_sql(
|
||||
"SELECT sum(number) AS a, sum(number) AS b, ts FROM numbers_with_ts GROUP BY ts",
|
||||
)
|
||||
.await;
|
||||
|
||||
let analysis = analyze_incremental_aggregate_plan(&plan).unwrap().unwrap();
|
||||
assert!(analysis.unsupported_exprs.is_empty());
|
||||
assert_eq!(analysis.merge_columns.len(), 2);
|
||||
assert_eq!(analysis.merge_columns[0].input_field_name, "a");
|
||||
assert_eq!(analysis.merge_columns[1].input_field_name, "a");
|
||||
assert!(
|
||||
analysis
|
||||
.unsupported_exprs
|
||||
.merge_columns
|
||||
.iter()
|
||||
.any(|expr| expr.contains("same aggregate output")
|
||||
&& expr.contains("a")
|
||||
&& expr.contains("b")),
|
||||
"same aggregate with multiple aliases should be unsupported until explicit reproduction is implemented: {:?}",
|
||||
analysis.unsupported_exprs
|
||||
.all(|column| { column.merge_op == IncrementalAggregateMergeOp::Sum })
|
||||
);
|
||||
assert!(
|
||||
analysis
|
||||
.merge_columns
|
||||
.iter()
|
||||
.any(|column| column.output_field_name == "a")
|
||||
);
|
||||
assert!(
|
||||
analysis
|
||||
.merge_columns
|
||||
.iter()
|
||||
.any(|column| column.output_field_name == "b")
|
||||
);
|
||||
assert!(analysis.merge_columns.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1581,6 +1706,322 @@ async fn test_analyze_incremental_aggregate_plan_rejects_avg() {
|
||||
assert!(!analysis.unsupported_exprs.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_rewrite_incremental_aggregate_merges_populated_mixed_state_families() {
|
||||
let query_engine = create_test_query_engine();
|
||||
let old_sql = "SELECT hll(CAST(number AS VARCHAR)) AS hll_a, hll(CAST(number AS VARCHAR)) AS hll_b, uddsketch_state(128, 0.000001, number) AS percentile_a, uddsketch_state(256, 0.02, number) AS percentile_b, stddev_pop_state(number) AS stddev_state, sum(number) AS total, CASE WHEN number <= 5 THEN CAST(NULL AS BIGINT) ELSE 2 END AS grp FROM numbers_with_ts WHERE number <= 3 OR number = 6 GROUP BY grp";
|
||||
let new_sql = "SELECT hll(CAST(number AS VARCHAR)) AS hll_a, hll(CAST(number AS VARCHAR)) AS hll_b, uddsketch_state(128, 0.000001, number) AS percentile_a, uddsketch_state(256, 0.02, number) AS percentile_b, stddev_pop_state(number) AS stddev_state, sum(number) AS total, CASE WHEN number <= 5 THEN CAST(NULL AS BIGINT) WHEN number <= 8 THEN 2 ELSE 3 END AS grp FROM numbers_with_ts WHERE number >= 4 AND number != 6 GROUP BY grp";
|
||||
let old_plan = sql_to_df_plan(QueryContext::arc(), query_engine.clone(), old_sql, false)
|
||||
.await
|
||||
.unwrap();
|
||||
let old_output = query_engine
|
||||
.execute(old_plan, QueryContext::arc())
|
||||
.await
|
||||
.unwrap();
|
||||
let OutputData::Stream(old_stream) = old_output.data else {
|
||||
panic!("expected old aggregate execution to be a stream");
|
||||
};
|
||||
let old_batches = util::collect(old_stream).await.unwrap();
|
||||
let old_schema = old_batches.first().unwrap().schema.clone();
|
||||
let old_batch = merge_record_batches(old_schema, &old_batches).unwrap();
|
||||
assert_eq!(
|
||||
old_batch.num_rows(),
|
||||
2,
|
||||
"old state must have one row per group"
|
||||
);
|
||||
let empty_state_sink_batch = old_batch.clone();
|
||||
let sink_table = MemTable::table("state_merge_sink", old_batch);
|
||||
let sink_table_name = [
|
||||
"greptime".to_string(),
|
||||
"public".to_string(),
|
||||
"state_merge_sink".to_string(),
|
||||
];
|
||||
|
||||
let new_plan = sql_to_df_plan(QueryContext::arc(), query_engine.clone(), new_sql, false)
|
||||
.await
|
||||
.unwrap();
|
||||
let analysis = analyze_incremental_aggregate_plan(&new_plan)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert!(analysis.unsupported_exprs.is_empty());
|
||||
let rewritten = rewrite_incremental_aggregate_with_sink_merge(
|
||||
&new_plan,
|
||||
&analysis,
|
||||
&query_engine,
|
||||
sink_table,
|
||||
&sink_table_name,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
analysis.output_field_names,
|
||||
vec![
|
||||
"hll_a",
|
||||
"hll_b",
|
||||
"percentile_a",
|
||||
"percentile_b",
|
||||
"stddev_state",
|
||||
"total",
|
||||
"grp",
|
||||
],
|
||||
"repeated HLL aliases must preserve output order"
|
||||
);
|
||||
|
||||
let output = query_engine
|
||||
.execute(rewritten, QueryContext::arc())
|
||||
.await
|
||||
.unwrap();
|
||||
let OutputData::Stream(stream) = output.data else {
|
||||
panic!("expected rewritten plan to execute as a stream");
|
||||
};
|
||||
let batches = util::collect(stream).await.unwrap();
|
||||
let merged_schema = batches.first().unwrap().schema.clone();
|
||||
let merged_batch = merge_record_batches(merged_schema, &batches).unwrap();
|
||||
assert_eq!(
|
||||
merged_batch.num_rows(),
|
||||
3,
|
||||
"rewrite must produce one row per group"
|
||||
);
|
||||
let merged_table = MemTable::table("merged_states", merged_batch);
|
||||
query_engine
|
||||
.engine_state()
|
||||
.catalog_manager()
|
||||
.as_any()
|
||||
.downcast_ref::<catalog::memory::MemoryCatalogManager>()
|
||||
.unwrap()
|
||||
.register_table_sync(RegisterTableRequest {
|
||||
catalog: "greptime".to_string(),
|
||||
schema: "public".to_string(),
|
||||
table_name: "merged_states".to_string(),
|
||||
table_id: 4097,
|
||||
table: merged_table,
|
||||
})
|
||||
.unwrap();
|
||||
let checks = "SELECT grp, sum(total) AS total, hll_count(hll_merge(hll_a)) AS hll_a, hll_count(hll_merge(hll_b)) AS hll_b, stddev_pop_calc(stddev_pop_merge(stddev_state)) AS stddev, uddsketch_calc(0.5, uddsketch_merge(128, 0.000001, percentile_a)) AS p50_a, uddsketch_calc(0.5, uddsketch_merge(256, 0.02, percentile_b)) AS p50_b FROM merged_states GROUP BY grp ORDER BY grp NULLS FIRST";
|
||||
let checks_plan = sql_to_df_plan(QueryContext::arc(), query_engine.clone(), checks, false)
|
||||
.await
|
||||
.unwrap();
|
||||
let checks_output = query_engine
|
||||
.execute(checks_plan, QueryContext::arc())
|
||||
.await
|
||||
.unwrap();
|
||||
let OutputData::Stream(checks_stream) = checks_output.data else {
|
||||
panic!("expected state check execution to be a stream");
|
||||
};
|
||||
let checks_batches = util::collect(checks_stream).await.unwrap();
|
||||
let checks_schema = checks_batches.first().unwrap().schema.clone();
|
||||
let checks = merge_record_batches(checks_schema, &checks_batches).unwrap();
|
||||
assert_eq!(checks.num_rows(), 3);
|
||||
assert_eq!(
|
||||
checks
|
||||
.schema
|
||||
.column_schemas()
|
||||
.iter()
|
||||
.map(|column| column.name.as_str())
|
||||
.collect::<Vec<_>>(),
|
||||
vec!["grp", "total", "hll_a", "hll_b", "stddev", "p50_a", "p50_b"]
|
||||
);
|
||||
let group = checks.column(0).as_primitive::<Int64Type>();
|
||||
let total = checks.column(1).as_primitive::<UInt64Type>();
|
||||
let hll_a = checks.column(2).as_primitive::<UInt64Type>();
|
||||
let hll_b = checks.column(3).as_primitive::<UInt64Type>();
|
||||
let stddev = checks.column(4).as_primitive::<Float64Type>();
|
||||
let p50_a = checks.column(5).as_primitive::<Float64Type>();
|
||||
let p50_b = checks.column(6).as_primitive::<Float64Type>();
|
||||
assert_eq!(group.null_count(), 1);
|
||||
assert_eq!(total.null_count(), 0);
|
||||
assert_eq!(hll_a.null_count(), 0);
|
||||
assert_eq!(hll_b.null_count(), 0);
|
||||
assert_eq!(stddev.null_count(), 0);
|
||||
assert_eq!(p50_a.null_count(), 0);
|
||||
assert_eq!(p50_b.null_count(), 0);
|
||||
for (index, expected) in [
|
||||
(None, 15_u64, 5_u64, 2_f64.sqrt(), 3_f64, 3_f64),
|
||||
(
|
||||
Some(2_i64),
|
||||
21_u64,
|
||||
3_u64,
|
||||
(2_f64 / 3.0).sqrt(),
|
||||
7_f64,
|
||||
7_f64,
|
||||
),
|
||||
(Some(3_i64), 19_u64, 2_u64, 0.5_f64, 10_f64, 10_f64),
|
||||
]
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
{
|
||||
assert_eq!(
|
||||
(!group.is_null(index)).then(|| group.value(index)),
|
||||
expected.0
|
||||
);
|
||||
assert_eq!(total.value(index), expected.1);
|
||||
assert_eq!(hll_a.value(index), expected.2);
|
||||
assert_eq!(hll_b.value(index), expected.2);
|
||||
assert!((stddev.value(index) - expected.3).abs() < 1e-12);
|
||||
// UDDSketch's relative-error bounds are 1e-6 and 2e-2 respectively.
|
||||
assert!((p50_a.value(index) - expected.4).abs() <= expected.4 * 0.000001);
|
||||
assert!((p50_b.value(index) - expected.5).abs() <= expected.5 * 0.02);
|
||||
}
|
||||
|
||||
let empty_state_plan = sql_to_df_plan(
|
||||
QueryContext::arc(),
|
||||
query_engine.clone(),
|
||||
"SELECT hll(CAST(NULL AS VARCHAR)) AS hll_a, hll(CAST(NULL AS VARCHAR)) AS hll_b, uddsketch_state(128, 0.000001, CAST(NULL AS DOUBLE)) AS percentile_a, uddsketch_state(256, 0.02, CAST(NULL AS DOUBLE)) AS percentile_b, stddev_pop_state(CAST(NULL AS DOUBLE)) AS stddev_state, sum(number) AS total, CASE WHEN number <= 5 THEN CAST(NULL AS BIGINT) WHEN number <= 8 THEN 2 ELSE 3 END AS grp FROM numbers_with_ts WHERE number = 10 GROUP BY grp",
|
||||
false,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let empty_state_analysis = analyze_incremental_aggregate_plan(&empty_state_plan)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert!(empty_state_analysis.unsupported_exprs.is_empty());
|
||||
let empty_state_rewritten = rewrite_incremental_aggregate_with_sink_merge(
|
||||
&empty_state_plan,
|
||||
&empty_state_analysis,
|
||||
&query_engine,
|
||||
MemTable::table("empty_state_sink", empty_state_sink_batch),
|
||||
&[
|
||||
"greptime".to_string(),
|
||||
"public".to_string(),
|
||||
"empty_state_sink".to_string(),
|
||||
],
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let empty_state_output = query_engine
|
||||
.execute(empty_state_rewritten, QueryContext::arc())
|
||||
.await
|
||||
.unwrap();
|
||||
let OutputData::Stream(empty_state_stream) = empty_state_output.data else {
|
||||
panic!("expected empty state merge to execute as a stream");
|
||||
};
|
||||
let empty_state_batches = util::collect(empty_state_stream).await.unwrap();
|
||||
let empty_state_schema = empty_state_batches.first().unwrap().schema.clone();
|
||||
let empty_state_batch = merge_record_batches(empty_state_schema, &empty_state_batches).unwrap();
|
||||
assert_eq!(empty_state_batch.num_rows(), 1);
|
||||
let group_index = empty_state_batch
|
||||
.schema
|
||||
.column_index_by_name("grp")
|
||||
.unwrap();
|
||||
let total_index = empty_state_batch
|
||||
.schema
|
||||
.column_index_by_name("total")
|
||||
.unwrap();
|
||||
let group = empty_state_batch
|
||||
.column(group_index)
|
||||
.as_primitive::<Int64Type>();
|
||||
let total = empty_state_batch
|
||||
.column(total_index)
|
||||
.as_primitive::<UInt64Type>();
|
||||
assert!(!group.is_null(0));
|
||||
assert_eq!(group.value(0), 3);
|
||||
assert!(!total.is_null(0));
|
||||
assert_eq!(total.value(0), 10);
|
||||
let empty_state_table = MemTable::table("empty_states", empty_state_batch);
|
||||
query_engine
|
||||
.engine_state()
|
||||
.catalog_manager()
|
||||
.as_any()
|
||||
.downcast_ref::<catalog::memory::MemoryCatalogManager>()
|
||||
.unwrap()
|
||||
.register_table_sync(RegisterTableRequest {
|
||||
catalog: "greptime".to_string(),
|
||||
schema: "public".to_string(),
|
||||
table_name: "empty_states".to_string(),
|
||||
table_id: 4098,
|
||||
table: empty_state_table,
|
||||
})
|
||||
.unwrap();
|
||||
let empty_checks_plan = sql_to_df_plan(
|
||||
QueryContext::arc(),
|
||||
query_engine.clone(),
|
||||
"SELECT hll_count(hll_merge(hll_a)) AS hll, stddev_pop_calc(stddev_pop_merge(stddev_state)) AS stddev, uddsketch_calc(0.5, uddsketch_merge(128, 0.000001, percentile_a)) AS p50_a, uddsketch_calc(0.5, uddsketch_merge(256, 0.02, percentile_b)) AS p50_b FROM empty_states",
|
||||
false,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let empty_checks_output = query_engine
|
||||
.execute(empty_checks_plan, QueryContext::arc())
|
||||
.await
|
||||
.unwrap();
|
||||
let OutputData::Stream(empty_checks_stream) = empty_checks_output.data else {
|
||||
panic!("expected empty state check to execute as a stream");
|
||||
};
|
||||
let empty_checks_batches = util::collect(empty_checks_stream).await.unwrap();
|
||||
let empty_checks_schema = empty_checks_batches.first().unwrap().schema.clone();
|
||||
let empty_checks = merge_record_batches(empty_checks_schema, &empty_checks_batches).unwrap();
|
||||
assert_eq!(empty_checks.num_rows(), 1);
|
||||
assert_eq!(
|
||||
empty_checks.column(0).as_primitive::<UInt64Type>().value(0),
|
||||
0
|
||||
);
|
||||
assert!(
|
||||
empty_checks
|
||||
.column(1)
|
||||
.as_primitive::<Float64Type>()
|
||||
.is_null(0)
|
||||
);
|
||||
assert!(
|
||||
empty_checks
|
||||
.column(2)
|
||||
.as_primitive::<Float64Type>()
|
||||
.is_null(0)
|
||||
);
|
||||
assert!(
|
||||
empty_checks
|
||||
.column(3)
|
||||
.as_primitive::<Float64Type>()
|
||||
.is_null(0)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_analyze_incremental_aggregate_plan_state_producer_metadata_and_rejections() {
|
||||
let analysis = analyze_test_sql(
|
||||
"SELECT hll(CAST(number AS VARCHAR)) AS hll_state, \
|
||||
stddev_pop_state(number) AS stddev_state, \
|
||||
uddsketch_state(128, 0.000001, number) AS percentile_state, ts \
|
||||
FROM numbers_with_ts GROUP BY ts",
|
||||
)
|
||||
.await;
|
||||
assert!(analysis.unsupported_exprs.is_empty());
|
||||
let percentile = analysis
|
||||
.merge_columns
|
||||
.iter()
|
||||
.find(|column| column.output_field_name == "percentile_state")
|
||||
.unwrap();
|
||||
let IncrementalAggregateMergeOp::StateDeltaMerge {
|
||||
function_name,
|
||||
params,
|
||||
} = &percentile.merge_op
|
||||
else {
|
||||
panic!("expected UDDSketch state delta merge");
|
||||
};
|
||||
assert_eq!(*function_name, "__uddsketch_state_delta_merge");
|
||||
assert_eq!(params.len(), 2);
|
||||
assert!(matches!(
|
||||
params[0],
|
||||
Expr::Literal(ScalarValue::Int64(Some(128)), _)
|
||||
));
|
||||
assert!(matches!(
|
||||
params[1],
|
||||
Expr::Literal(ScalarValue::Float64(Some(rate)), _) if rate == 0.000001
|
||||
));
|
||||
|
||||
for sql in [
|
||||
"SELECT uddsketch_state(CAST(number AS BIGINT), 0.01, number) AS state, ts FROM numbers_with_ts GROUP BY ts",
|
||||
"SELECT uddsketch_state(128, NULL, number) AS state, ts FROM numbers_with_ts GROUP BY ts",
|
||||
"SELECT uddsketch_state(NULL, 0.01, number) AS state, ts FROM numbers_with_ts GROUP BY ts",
|
||||
"SELECT avg(number) AS state, ts FROM numbers_with_ts GROUP BY ts",
|
||||
"SELECT stddev_pop(number) AS state, ts FROM numbers_with_ts GROUP BY ts",
|
||||
] {
|
||||
let analysis = analyze_test_sql(sql).await;
|
||||
assert!(!analysis.unsupported_exprs.is_empty(), "must reject {sql}");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_analyze_incremental_aggregate_plan_rejects_distinct() {
|
||||
let query_engine = create_test_query_engine();
|
||||
@@ -1638,6 +2079,7 @@ async fn test_rewrite_incremental_aggregate_with_left_join() {
|
||||
let rewritten = rewrite_incremental_aggregate_with_sink_merge(
|
||||
&plan,
|
||||
&analysis,
|
||||
&query_engine,
|
||||
sink_table.clone(),
|
||||
&sink_table_name,
|
||||
None,
|
||||
@@ -1699,6 +2141,7 @@ async fn test_rewrite_incremental_aggregate_filters_sink_dirty_time_window() {
|
||||
let rewritten = rewrite_incremental_aggregate_with_sink_merge(
|
||||
&plan,
|
||||
&analysis,
|
||||
&query_engine,
|
||||
sink_table.clone(),
|
||||
&sink_table_name,
|
||||
Some(sink_filter.clone()),
|
||||
@@ -1749,7 +2192,9 @@ async fn test_rewrite_incremental_aggregate_rejects_empty_group_keys() {
|
||||
let query_engine = create_test_query_engine();
|
||||
let ctx = QueryContext::arc();
|
||||
let sql = "SELECT max(number) AS number FROM numbers_with_ts";
|
||||
let plan = sql_to_df_plan(ctx, query_engine, sql, false).await.unwrap();
|
||||
let plan = sql_to_df_plan(ctx, query_engine.clone(), sql, false)
|
||||
.await
|
||||
.unwrap();
|
||||
let analysis = IncrementalAggregateAnalysis {
|
||||
group_key_names: vec![],
|
||||
merge_columns: vec![IncrementalAggregateMergeColumn::new(
|
||||
@@ -1770,6 +2215,7 @@ async fn test_rewrite_incremental_aggregate_rejects_empty_group_keys() {
|
||||
let err = rewrite_incremental_aggregate_with_sink_merge(
|
||||
&plan,
|
||||
&analysis,
|
||||
&query_engine,
|
||||
sink_table,
|
||||
&sink_table_name,
|
||||
None,
|
||||
@@ -1788,7 +2234,9 @@ async fn test_rewrite_incremental_aggregate_preserves_raw_aggregate_field_name()
|
||||
let query_engine = create_test_query_engine();
|
||||
let ctx = QueryContext::arc();
|
||||
let sql = "SELECT max(number), number FROM numbers_with_ts GROUP BY number";
|
||||
let plan = sql_to_df_plan(ctx, query_engine, sql, false).await.unwrap();
|
||||
let plan = sql_to_df_plan(ctx, query_engine.clone(), sql, false)
|
||||
.await
|
||||
.unwrap();
|
||||
let analysis = analyze_incremental_aggregate_plan(&plan).unwrap().unwrap();
|
||||
assert!(analysis.unsupported_exprs.is_empty());
|
||||
|
||||
@@ -1802,6 +2250,7 @@ async fn test_rewrite_incremental_aggregate_preserves_raw_aggregate_field_name()
|
||||
let rewritten = rewrite_incremental_aggregate_with_sink_merge(
|
||||
&plan,
|
||||
&analysis,
|
||||
&query_engine,
|
||||
sink_table.clone(),
|
||||
&sink_table_name,
|
||||
None,
|
||||
|
||||
+11
-1
@@ -38,7 +38,17 @@ mod test_utils;
|
||||
|
||||
pub use adapter::flownode_impl::FlowDualEngineRef;
|
||||
pub use adapter::{FlowConfig, FlowStreamingEngineRef, StreamingEngine};
|
||||
pub use batching_mode::frontend_client::{FrontendClient, GrpcQueryHandlerWithBoxedError};
|
||||
pub use batching_mode::BatchingModeOptions;
|
||||
pub use batching_mode::batching_execution::{BatchingExecution, BatchingExecutionFactory};
|
||||
pub use batching_mode::frontend_client::{
|
||||
FrontendClient, GrpcQueryHandlerWithBoxedError, PeerDesc,
|
||||
};
|
||||
pub use batching_mode::task::{
|
||||
BatchingExecutionGuard, BatchingTask, DirtyRestore, ExecuteOnceOutcome, PlanInfo,
|
||||
QueryCoverage, TaskArgs,
|
||||
};
|
||||
pub use batching_mode::time_window::{TimeWindowExpr, find_time_window_expr};
|
||||
pub use batching_mode::utils::sql_to_df_plan;
|
||||
pub(crate) use engine::{CreateFlowArgs, FlowId, TableName};
|
||||
pub use error::{Error, Result};
|
||||
pub use server::{FlownodeBuilder, FlownodeInstance, FlownodeServer, FlownodeServiceBuilder};
|
||||
|
||||
@@ -47,7 +47,7 @@ use crate::error::{
|
||||
use crate::heartbeat::HeartbeatTask;
|
||||
use crate::metrics::{METRIC_FLOW_PROCESSING_TIME, METRIC_FLOW_ROWS};
|
||||
use crate::utils::{SizeReportSender, StateReportHandler};
|
||||
use crate::{Error, FlownodeOptions, FrontendClient, StreamingEngine};
|
||||
use crate::{BatchingExecutionFactory, Error, FlownodeOptions, FrontendClient, StreamingEngine};
|
||||
|
||||
pub const FLOW_NODE_SERVER_NAME: &str = "FLOW_NODE_SERVER";
|
||||
/// wrapping flow node manager to avoid orphan rule with Arc<...>
|
||||
@@ -333,13 +333,14 @@ impl FlownodeBuilder {
|
||||
context: "Failed to build query engine",
|
||||
})?;
|
||||
let manager = Arc::new(self.build_manager(query_engine_factory.query_engine()));
|
||||
let batching = Arc::new(BatchingEngine::new(
|
||||
let batching = Arc::new(BatchingEngine::new_with_execution(
|
||||
self.frontend_client.clone(),
|
||||
query_engine_factory.query_engine(),
|
||||
self.flow_metadata_manager.clone(),
|
||||
self.table_meta.clone(),
|
||||
self.catalog_manager.clone(),
|
||||
self.opts.flow.batching_mode.clone(),
|
||||
self.plugins.get::<Arc<dyn BatchingExecutionFactory>>(),
|
||||
));
|
||||
let dual = Arc::new(FlowDualEngine::new(
|
||||
manager.clone(),
|
||||
|
||||
@@ -128,6 +128,7 @@ struct DdlSubmitOptions {
|
||||
timeout: Duration,
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "enterprise"))]
|
||||
const ALLOWED_FLOW_OPTIONS: [&str; 2] = [
|
||||
DEFER_ON_MISSING_SOURCE_KEY,
|
||||
FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY,
|
||||
@@ -172,6 +173,7 @@ fn parse_ddl_options(options: &OptionMap) -> Result<DdlSubmitOptions> {
|
||||
Ok(DdlSubmitOptions { wait, timeout })
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "enterprise"))]
|
||||
fn supported_flow_options() -> String {
|
||||
ALLOWED_FLOW_OPTIONS.join(", ")
|
||||
}
|
||||
@@ -221,6 +223,9 @@ fn validate_and_normalize_flow_options(
|
||||
DEFER_ON_MISSING_SOURCE_KEY | FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY => {
|
||||
normalize_flow_bool_option(&key, &value)?
|
||||
}
|
||||
#[cfg(feature = "enterprise")]
|
||||
_ => value,
|
||||
#[cfg(not(feature = "enterprise"))]
|
||||
_ => {
|
||||
return InvalidSqlSnafu {
|
||||
err_msg: format!(
|
||||
@@ -3284,6 +3289,7 @@ mod test {
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "enterprise"))]
|
||||
#[test]
|
||||
fn test_validate_and_normalize_flow_options_unknown_option() {
|
||||
let err = validate_and_normalize_flow_options(
|
||||
@@ -3333,7 +3339,7 @@ mod test {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_validate_and_normalize_flow_options_rejects_redacted_invalid_input() {
|
||||
fn test_parser_flow_option_access_key_id() {
|
||||
let sql = r"
|
||||
CREATE FLOW task_6
|
||||
SINK TO schema_1.table_1
|
||||
@@ -3351,11 +3357,19 @@ SELECT max(c1), min(c2) FROM schema_2.table_2;";
|
||||
};
|
||||
let expr =
|
||||
expr_helper::to_create_flow_task_expr(create_flow, &QueryContext::arc()).unwrap();
|
||||
let err = validate_and_normalize_flow_options(expr.flow_options, None).unwrap_err();
|
||||
|
||||
assert!(
|
||||
err.to_string()
|
||||
.contains("unknown flow option 'access_key_id'")
|
||||
#[cfg(not(feature = "enterprise"))]
|
||||
{
|
||||
let err = validate_and_normalize_flow_options(expr.flow_options, None).unwrap_err();
|
||||
assert!(
|
||||
err.to_string()
|
||||
.contains("unknown flow option 'access_key_id'")
|
||||
);
|
||||
}
|
||||
#[cfg(feature = "enterprise")]
|
||||
assert_eq!(
|
||||
validate_and_normalize_flow_options(expr.flow_options, None).unwrap(),
|
||||
HashMap::from([("access_key_id".to_string(), "['true']".to_string())])
|
||||
);
|
||||
}
|
||||
|
||||
@@ -3448,7 +3462,18 @@ SELECT max(c1), min(c2) FROM schema_2.table_2;";
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_schedule_and_internal_keys_rejected_as_unknown_options() {
|
||||
fn test_internal_schedule_key_rejected() {
|
||||
let err = validate_and_normalize_flow_options(
|
||||
HashMap::from([(INTERNAL_EVAL_SCHEDULE_KEY.to_string(), "value".to_string())]),
|
||||
Some(300),
|
||||
)
|
||||
.unwrap_err();
|
||||
assert!(err.to_string().contains("reserved for internal use"));
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "enterprise"))]
|
||||
#[test]
|
||||
fn test_schedule_keys_rejected_as_unknown_options() {
|
||||
for key in [
|
||||
"eval_interval_anchor",
|
||||
"eval_interval_start",
|
||||
@@ -3461,15 +3486,38 @@ SELECT max(c1), min(c2) FROM schema_2.table_2;";
|
||||
Some(300),
|
||||
)
|
||||
.unwrap_err();
|
||||
|
||||
assert!(
|
||||
err.to_string()
|
||||
.contains(&format!("unknown flow option '{key}'")),
|
||||
"unexpected error for {key}: {err}"
|
||||
.contains(&format!("unknown flow option '{key}'"))
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
#[test]
|
||||
fn test_schedule_keys_preserved() {
|
||||
let options = HashMap::from([
|
||||
("eval_interval_anchor".to_string(), "anchor".to_string()),
|
||||
("eval_interval_start".to_string(), "start".to_string()),
|
||||
(
|
||||
"eval_interval_missed_tick_policy".to_string(),
|
||||
"missed".to_string(),
|
||||
),
|
||||
(
|
||||
"eval_interval_catchup_max_runs".to_string(),
|
||||
"runs".to_string(),
|
||||
),
|
||||
(
|
||||
"eval_interval_catchup_max_lag".to_string(),
|
||||
"lag".to_string(),
|
||||
),
|
||||
]);
|
||||
assert_eq!(
|
||||
validate_and_normalize_flow_options(options.clone(), Some(300)).unwrap(),
|
||||
options
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_internal_transport_keys_rejected_as_reserved() {
|
||||
for key in [
|
||||
|
||||
@@ -1,3 +1,17 @@
|
||||
// Copyright 2023 Greptime Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// This file is copied from https://github.com/tokio-rs/axum/blob/axum-v0.6.20/axum/src/test_helpers/test_client.rs
|
||||
|
||||
//! Axum Test Client
|
||||
|
||||
@@ -1,22 +1,16 @@
|
||||
// Copyright (c) 2019 Stepan Koltsov
|
||||
// Copyright 2023 Greptime Team
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
// of this software and associated documentation files (the "Software"), to deal
|
||||
// in the Software without restriction, including without limitation the rights
|
||||
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
// copies of the Software, and to permit persons to whom the Software is
|
||||
// furnished to do so, subject to the following conditions:
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in all
|
||||
// copies or substantial portions of the Software.
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
|
||||
// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
|
||||
// OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// The Clear trait is copied from https://github.com/stepancheg/rust-protobuf/blob/v2.28.0/protobuf/src/clear.rs
|
||||
// The RepeatedField struct is copied from https://github.com/stepancheg/rust-protobuf/blob/v2.28.0/protobuf/src/repeated.rs
|
||||
|
||||
@@ -0,0 +1,348 @@
|
||||
-- Incremental aggregate state merge over a partitioned source (two regions).
|
||||
-- A WHERE-filtered flow mixes scalar aggregates (sum, count) with aggregate
|
||||
-- state columns (hll, stddev_pop_state, uddsketch_state); each later delta in
|
||||
-- the same time window must merge its state with the state already persisted
|
||||
-- in the sink. Covers two nonempty deltas across both regions, one round whose
|
||||
-- delta is empty because the flow's predicate drops the newly inserted row,
|
||||
-- and a final nonempty delta checked against a direct source aggregation.
|
||||
CREATE TABLE flow_incr_state_merge_input (
|
||||
host_id INT,
|
||||
user_id BIGINT,
|
||||
n INT,
|
||||
v DOUBLE,
|
||||
ts TIMESTAMP TIME INDEX,
|
||||
PRIMARY KEY(host_id, user_id)
|
||||
)
|
||||
PARTITION ON COLUMNS (host_id) (
|
||||
host_id < 3,
|
||||
host_id >= 3
|
||||
)
|
||||
WITH (
|
||||
append_mode = 'true'
|
||||
);
|
||||
|
||||
Affected Rows: 0
|
||||
|
||||
CREATE FLOW flow_incr_state_merge SINK TO flow_incr_state_merge_sink
|
||||
WITH (experimental_enable_incremental_read = 'true')
|
||||
AS
|
||||
SELECT
|
||||
sum(n) AS total,
|
||||
count(n) AS row_count,
|
||||
hll(user_id) AS user_hll,
|
||||
stddev_pop_state(v) AS v_stddev_state,
|
||||
uddsketch_state(128, 0.01, v) AS v_sketch_state,
|
||||
date_bin(INTERVAL '1 minute', ts, '2024-01-01 00:00:00') AS time_window
|
||||
FROM
|
||||
flow_incr_state_merge_input
|
||||
WHERE
|
||||
v > 0
|
||||
GROUP BY
|
||||
time_window;
|
||||
|
||||
Affected Rows: 0
|
||||
|
||||
-- ==== Phase 1: initial rows in the same time window, one per region ====
|
||||
INSERT INTO flow_incr_state_merge_input VALUES
|
||||
(1, 1, 1, 1.0, '2024-01-01 00:00:00'),
|
||||
(3, 2, 2, 2.0, '2024-01-01 00:00:10');
|
||||
|
||||
Affected Rows: 2
|
||||
|
||||
-- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED |
|
||||
ADMIN FLUSH_FLOW('flow_incr_state_merge');
|
||||
|
||||
+-------------------------------------------+
|
||||
| ADMIN FLUSH_FLOW('flow_incr_state_merge') |
|
||||
+-------------------------------------------+
|
||||
| FLOW_FLUSHED |
|
||||
+-------------------------------------------+
|
||||
|
||||
-- The first round seeds the sink with a full-snapshot aggregate state.
|
||||
SELECT
|
||||
total,
|
||||
row_count,
|
||||
time_window
|
||||
FROM
|
||||
flow_incr_state_merge_sink
|
||||
ORDER BY
|
||||
time_window;
|
||||
|
||||
+-------+-----------+---------------------+
|
||||
| total | row_count | time_window |
|
||||
+-------+-----------+---------------------+
|
||||
| 3 | 2 | 2024-01-01T00:00:00 |
|
||||
+-------+-----------+---------------------+
|
||||
|
||||
-- Move the checkpointed source rows and the seeded sink state into SST, so
|
||||
-- the next round has to merge a new delta with persisted state.
|
||||
ADMIN FLUSH_TABLE('flow_incr_state_merge_input');
|
||||
|
||||
+--------------------------------------------------+
|
||||
| ADMIN FLUSH_TABLE('flow_incr_state_merge_input') |
|
||||
+--------------------------------------------------+
|
||||
| 0 |
|
||||
+--------------------------------------------------+
|
||||
|
||||
ADMIN FLUSH_TABLE('flow_incr_state_merge_sink');
|
||||
|
||||
+-------------------------------------------------+
|
||||
| ADMIN FLUSH_TABLE('flow_incr_state_merge_sink') |
|
||||
+-------------------------------------------------+
|
||||
| 0 |
|
||||
+-------------------------------------------------+
|
||||
|
||||
-- ==== Phase 2: first nonempty delta, same window, both regions ====
|
||||
INSERT INTO flow_incr_state_merge_input VALUES
|
||||
(2, 3, 3, 3.0, '2024-01-01 00:00:20'),
|
||||
(4, 4, 4, 4.0, '2024-01-01 00:00:40');
|
||||
|
||||
Affected Rows: 2
|
||||
|
||||
-- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED |
|
||||
ADMIN FLUSH_FLOW('flow_incr_state_merge');
|
||||
|
||||
+-------------------------------------------+
|
||||
| ADMIN FLUSH_FLOW('flow_incr_state_merge') |
|
||||
+-------------------------------------------+
|
||||
| FLOW_FLUSHED |
|
||||
+-------------------------------------------+
|
||||
|
||||
-- Scalars accumulate; each state column must merge with the persisted state.
|
||||
SELECT
|
||||
total,
|
||||
row_count,
|
||||
hll_count(hll_merge(user_hll)) AS users,
|
||||
ROUND(stddev_pop_calc(stddev_pop_merge(v_stddev_state)), 6) AS v_stddev,
|
||||
ROUND(uddsketch_calc(0.5, uddsketch_merge(128, 0.01, v_sketch_state)), 4) AS v_p50,
|
||||
time_window
|
||||
FROM
|
||||
flow_incr_state_merge_sink
|
||||
GROUP BY
|
||||
total,
|
||||
row_count,
|
||||
time_window
|
||||
ORDER BY
|
||||
time_window;
|
||||
|
||||
+-------+-----------+-------+----------+--------+---------------------+
|
||||
| total | row_count | users | v_stddev | v_p50 | time_window |
|
||||
+-------+-----------+-------+----------+--------+---------------------+
|
||||
| 10 | 4 | 4 | 1.118034 | 2.9742 | 2024-01-01T00:00:00 |
|
||||
+-------+-----------+-------+----------+--------+---------------------+
|
||||
|
||||
ADMIN FLUSH_TABLE('flow_incr_state_merge_input');
|
||||
|
||||
+--------------------------------------------------+
|
||||
| ADMIN FLUSH_TABLE('flow_incr_state_merge_input') |
|
||||
+--------------------------------------------------+
|
||||
| 0 |
|
||||
+--------------------------------------------------+
|
||||
|
||||
ADMIN FLUSH_TABLE('flow_incr_state_merge_sink');
|
||||
|
||||
+-------------------------------------------------+
|
||||
| ADMIN FLUSH_TABLE('flow_incr_state_merge_sink') |
|
||||
+-------------------------------------------------+
|
||||
| 0 |
|
||||
+-------------------------------------------------+
|
||||
|
||||
-- ==== Phase 3: filtered-out row must run one round with an empty delta ====
|
||||
-- Record the flow's last execution time before the excluded insert: the
|
||||
-- comparison below proves the empty delta was actually evaluated.
|
||||
CREATE TABLE flow_incr_state_merge_last_exec (
|
||||
marker STRING,
|
||||
last_exec TIMESTAMP TIME INDEX
|
||||
) WITH (
|
||||
append_mode = 'true'
|
||||
);
|
||||
|
||||
Affected Rows: 0
|
||||
|
||||
-- The background loop runs on its own interval; keep waiting until it has
|
||||
-- executed at least once so information_schema.flows.last_execution_time is
|
||||
-- non-null before snapshotting it.
|
||||
-- SQLNESS SLEEP 8s
|
||||
INSERT INTO flow_incr_state_merge_last_exec (marker, last_exec)
|
||||
SELECT
|
||||
'before_empty_delta',
|
||||
last_execution_time
|
||||
FROM
|
||||
information_schema.flows
|
||||
WHERE
|
||||
flow_name = 'flow_incr_state_merge';
|
||||
|
||||
Affected Rows: 1
|
||||
|
||||
-- The row's timestamp is inside the existing window, so the source write still
|
||||
-- dirties that window; the flow's WHERE predicate then drops the row from the
|
||||
-- delta, leaving the delta empty for the window.
|
||||
INSERT INTO flow_incr_state_merge_input VALUES
|
||||
(1, 9, 9, -1.0, '2024-01-01 00:00:30');
|
||||
|
||||
Affected Rows: 1
|
||||
|
||||
-- SQLNESS SLEEP 6s
|
||||
-- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED |
|
||||
ADMIN FLUSH_FLOW('flow_incr_state_merge');
|
||||
|
||||
+-------------------------------------------+
|
||||
| ADMIN FLUSH_FLOW('flow_incr_state_merge') |
|
||||
+-------------------------------------------+
|
||||
| FLOW_FLUSHED |
|
||||
+-------------------------------------------+
|
||||
|
||||
-- An empty delta must leave the persisted sink states untouched.
|
||||
SELECT
|
||||
total,
|
||||
row_count,
|
||||
hll_count(hll_merge(user_hll)) AS users,
|
||||
ROUND(stddev_pop_calc(stddev_pop_merge(v_stddev_state)), 6) AS v_stddev,
|
||||
ROUND(uddsketch_calc(0.5, uddsketch_merge(128, 0.01, v_sketch_state)), 4) AS v_p50,
|
||||
time_window
|
||||
FROM
|
||||
flow_incr_state_merge_sink
|
||||
GROUP BY
|
||||
total,
|
||||
row_count,
|
||||
time_window
|
||||
ORDER BY
|
||||
time_window;
|
||||
|
||||
+-------+-----------+-------+----------+--------+---------------------+
|
||||
| total | row_count | users | v_stddev | v_p50 | time_window |
|
||||
+-------+-----------+-------+----------+--------+---------------------+
|
||||
| 10 | 4 | 4 | 1.118034 | 2.9742 | 2024-01-01T00:00:00 |
|
||||
+-------+-----------+-------+----------+--------+---------------------+
|
||||
|
||||
-- last_execution_time must have advanced past the mark taken above, proving
|
||||
-- the dirty window triggered a new round even though its delta was empty. A
|
||||
-- NULL comparison means the mark never advanced: fail explicitly instead of
|
||||
-- silently treating it as "unchanged".
|
||||
INSERT INTO flow_incr_state_merge_last_exec (marker, last_exec)
|
||||
SELECT
|
||||
'after_empty_delta',
|
||||
last_execution_time
|
||||
FROM
|
||||
information_schema.flows
|
||||
WHERE
|
||||
flow_name = 'flow_incr_state_merge';
|
||||
|
||||
Affected Rows: 1
|
||||
|
||||
SELECT
|
||||
(
|
||||
SELECT
|
||||
last_exec
|
||||
FROM
|
||||
flow_incr_state_merge_last_exec
|
||||
WHERE
|
||||
marker = 'after_empty_delta'
|
||||
) > (
|
||||
SELECT
|
||||
last_exec
|
||||
FROM
|
||||
flow_incr_state_merge_last_exec
|
||||
WHERE
|
||||
marker = 'before_empty_delta'
|
||||
) AS empty_delta_executed;
|
||||
|
||||
+----------------------+
|
||||
| empty_delta_executed |
|
||||
+----------------------+
|
||||
| true |
|
||||
+----------------------+
|
||||
|
||||
-- Flush the input before the final delta so each phase's delta is isolated in
|
||||
-- its own round; the background loop may have already consumed the last
|
||||
-- insert, which is fine — the values below only depend on the merge outcome.
|
||||
ADMIN FLUSH_TABLE('flow_incr_state_merge_input');
|
||||
|
||||
+--------------------------------------------------+
|
||||
| ADMIN FLUSH_TABLE('flow_incr_state_merge_input') |
|
||||
+--------------------------------------------------+
|
||||
| 0 |
|
||||
+--------------------------------------------------+
|
||||
|
||||
ADMIN FLUSH_TABLE('flow_incr_state_merge_sink');
|
||||
|
||||
+-------------------------------------------------+
|
||||
| ADMIN FLUSH_TABLE('flow_incr_state_merge_sink') |
|
||||
+-------------------------------------------------+
|
||||
| 0 |
|
||||
+-------------------------------------------------+
|
||||
|
||||
-- ==== Phase 4: final nonempty delta, same window, both regions ====
|
||||
INSERT INTO flow_incr_state_merge_input VALUES
|
||||
(1, 5, 5, 5.0, '2024-01-01 00:00:05'),
|
||||
(3, 6, 6, 6.0, '2024-01-01 00:00:55');
|
||||
|
||||
Affected Rows: 2
|
||||
|
||||
-- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED |
|
||||
ADMIN FLUSH_FLOW('flow_incr_state_merge');
|
||||
|
||||
+-------------------------------------------+
|
||||
| ADMIN FLUSH_FLOW('flow_incr_state_merge') |
|
||||
+-------------------------------------------+
|
||||
| FLOW_FLUSHED |
|
||||
+-------------------------------------------+
|
||||
|
||||
-- Merge the final delta with the persisted states and compare the merged
|
||||
-- result against a direct aggregation over the accepted source rows.
|
||||
SELECT
|
||||
merged.total = direct.total AS total_matches,
|
||||
merged.row_count = direct.row_count AS count_matches,
|
||||
merged.users = direct.users AS hll_matches,
|
||||
abs(merged.v_stddev - direct.v_stddev) < 1e-9 AS stddev_matches,
|
||||
ROUND(merged.v_p50, 4) = ROUND(direct.v_p50, 4) AS p50_matches
|
||||
FROM
|
||||
(
|
||||
SELECT
|
||||
total,
|
||||
row_count,
|
||||
hll_count(hll_merge(user_hll)) AS users,
|
||||
stddev_pop_calc(stddev_pop_merge(v_stddev_state)) AS v_stddev,
|
||||
uddsketch_calc(0.5, uddsketch_merge(128, 0.01, v_sketch_state)) AS v_p50
|
||||
FROM
|
||||
flow_incr_state_merge_sink
|
||||
GROUP BY
|
||||
total,
|
||||
row_count
|
||||
) AS merged,
|
||||
(
|
||||
SELECT
|
||||
sum(n) AS total,
|
||||
count(n) AS row_count,
|
||||
hll_count(hll(user_id)) AS users,
|
||||
stddev_pop(v) AS v_stddev,
|
||||
uddsketch_calc(0.5, uddsketch_state(128, 0.01, v)) AS v_p50
|
||||
FROM
|
||||
flow_incr_state_merge_input
|
||||
WHERE
|
||||
v > 0
|
||||
) AS direct;
|
||||
|
||||
+---------------+---------------+-------------+----------------+-------------+
|
||||
| total_matches | count_matches | hll_matches | stddev_matches | p50_matches |
|
||||
+---------------+---------------+-------------+----------------+-------------+
|
||||
| true | true | true | true | true |
|
||||
+---------------+---------------+-------------+----------------+-------------+
|
||||
|
||||
-- Clean up
|
||||
DROP FLOW flow_incr_state_merge;
|
||||
|
||||
Affected Rows: 0
|
||||
|
||||
DROP TABLE flow_incr_state_merge_input;
|
||||
|
||||
Affected Rows: 0
|
||||
|
||||
DROP TABLE flow_incr_state_merge_sink;
|
||||
|
||||
Affected Rows: 0
|
||||
|
||||
DROP TABLE flow_incr_state_merge_last_exec;
|
||||
|
||||
Affected Rows: 0
|
||||
|
||||
@@ -0,0 +1,232 @@
|
||||
-- Incremental aggregate state merge over a partitioned source (two regions).
|
||||
-- A WHERE-filtered flow mixes scalar aggregates (sum, count) with aggregate
|
||||
-- state columns (hll, stddev_pop_state, uddsketch_state); each later delta in
|
||||
-- the same time window must merge its state with the state already persisted
|
||||
-- in the sink. Covers two nonempty deltas across both regions, one round whose
|
||||
-- delta is empty because the flow's predicate drops the newly inserted row,
|
||||
-- and a final nonempty delta checked against a direct source aggregation.
|
||||
CREATE TABLE flow_incr_state_merge_input (
|
||||
host_id INT,
|
||||
user_id BIGINT,
|
||||
n INT,
|
||||
v DOUBLE,
|
||||
ts TIMESTAMP TIME INDEX,
|
||||
PRIMARY KEY(host_id, user_id)
|
||||
)
|
||||
PARTITION ON COLUMNS (host_id) (
|
||||
host_id < 3,
|
||||
host_id >= 3
|
||||
)
|
||||
WITH (
|
||||
append_mode = 'true'
|
||||
);
|
||||
|
||||
CREATE FLOW flow_incr_state_merge SINK TO flow_incr_state_merge_sink
|
||||
WITH (experimental_enable_incremental_read = 'true')
|
||||
AS
|
||||
SELECT
|
||||
sum(n) AS total,
|
||||
count(n) AS row_count,
|
||||
hll(user_id) AS user_hll,
|
||||
stddev_pop_state(v) AS v_stddev_state,
|
||||
uddsketch_state(128, 0.01, v) AS v_sketch_state,
|
||||
date_bin(INTERVAL '1 minute', ts, '2024-01-01 00:00:00') AS time_window
|
||||
FROM
|
||||
flow_incr_state_merge_input
|
||||
WHERE
|
||||
v > 0
|
||||
GROUP BY
|
||||
time_window;
|
||||
|
||||
-- ==== Phase 1: initial rows in the same time window, one per region ====
|
||||
INSERT INTO flow_incr_state_merge_input VALUES
|
||||
(1, 1, 1, 1.0, '2024-01-01 00:00:00'),
|
||||
(3, 2, 2, 2.0, '2024-01-01 00:00:10');
|
||||
|
||||
-- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED |
|
||||
ADMIN FLUSH_FLOW('flow_incr_state_merge');
|
||||
|
||||
-- The first round seeds the sink with a full-snapshot aggregate state.
|
||||
SELECT
|
||||
total,
|
||||
row_count,
|
||||
time_window
|
||||
FROM
|
||||
flow_incr_state_merge_sink
|
||||
ORDER BY
|
||||
time_window;
|
||||
|
||||
-- Move the checkpointed source rows and the seeded sink state into SST, so
|
||||
-- the next round has to merge a new delta with persisted state.
|
||||
ADMIN FLUSH_TABLE('flow_incr_state_merge_input');
|
||||
|
||||
ADMIN FLUSH_TABLE('flow_incr_state_merge_sink');
|
||||
|
||||
-- ==== Phase 2: first nonempty delta, same window, both regions ====
|
||||
INSERT INTO flow_incr_state_merge_input VALUES
|
||||
(2, 3, 3, 3.0, '2024-01-01 00:00:20'),
|
||||
(4, 4, 4, 4.0, '2024-01-01 00:00:40');
|
||||
|
||||
-- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED |
|
||||
ADMIN FLUSH_FLOW('flow_incr_state_merge');
|
||||
|
||||
-- Scalars accumulate; each state column must merge with the persisted state.
|
||||
SELECT
|
||||
total,
|
||||
row_count,
|
||||
hll_count(hll_merge(user_hll)) AS users,
|
||||
ROUND(stddev_pop_calc(stddev_pop_merge(v_stddev_state)), 6) AS v_stddev,
|
||||
ROUND(uddsketch_calc(0.5, uddsketch_merge(128, 0.01, v_sketch_state)), 4) AS v_p50,
|
||||
time_window
|
||||
FROM
|
||||
flow_incr_state_merge_sink
|
||||
GROUP BY
|
||||
total,
|
||||
row_count,
|
||||
time_window
|
||||
ORDER BY
|
||||
time_window;
|
||||
|
||||
ADMIN FLUSH_TABLE('flow_incr_state_merge_input');
|
||||
|
||||
ADMIN FLUSH_TABLE('flow_incr_state_merge_sink');
|
||||
|
||||
-- ==== Phase 3: filtered-out row must run one round with an empty delta ====
|
||||
-- Record the flow's last execution time before the excluded insert: the
|
||||
-- comparison below proves the empty delta was actually evaluated.
|
||||
CREATE TABLE flow_incr_state_merge_last_exec (
|
||||
marker STRING,
|
||||
last_exec TIMESTAMP TIME INDEX
|
||||
) WITH (
|
||||
append_mode = 'true'
|
||||
);
|
||||
|
||||
-- The background loop runs on its own interval; keep waiting until it has
|
||||
-- executed at least once so information_schema.flows.last_execution_time is
|
||||
-- non-null before snapshotting it.
|
||||
-- SQLNESS SLEEP 8s
|
||||
INSERT INTO flow_incr_state_merge_last_exec (marker, last_exec)
|
||||
SELECT
|
||||
'before_empty_delta',
|
||||
last_execution_time
|
||||
FROM
|
||||
information_schema.flows
|
||||
WHERE
|
||||
flow_name = 'flow_incr_state_merge';
|
||||
|
||||
-- The row's timestamp is inside the existing window, so the source write still
|
||||
-- dirties that window; the flow's WHERE predicate then drops the row from the
|
||||
-- delta, leaving the delta empty for the window.
|
||||
INSERT INTO flow_incr_state_merge_input VALUES
|
||||
(1, 9, 9, -1.0, '2024-01-01 00:00:30');
|
||||
|
||||
-- SQLNESS SLEEP 6s
|
||||
-- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED |
|
||||
ADMIN FLUSH_FLOW('flow_incr_state_merge');
|
||||
|
||||
-- An empty delta must leave the persisted sink states untouched.
|
||||
SELECT
|
||||
total,
|
||||
row_count,
|
||||
hll_count(hll_merge(user_hll)) AS users,
|
||||
ROUND(stddev_pop_calc(stddev_pop_merge(v_stddev_state)), 6) AS v_stddev,
|
||||
ROUND(uddsketch_calc(0.5, uddsketch_merge(128, 0.01, v_sketch_state)), 4) AS v_p50,
|
||||
time_window
|
||||
FROM
|
||||
flow_incr_state_merge_sink
|
||||
GROUP BY
|
||||
total,
|
||||
row_count,
|
||||
time_window
|
||||
ORDER BY
|
||||
time_window;
|
||||
|
||||
-- last_execution_time must have advanced past the mark taken above, proving
|
||||
-- the dirty window triggered a new round even though its delta was empty. A
|
||||
-- NULL comparison means the mark never advanced: fail explicitly instead of
|
||||
-- silently treating it as "unchanged".
|
||||
INSERT INTO flow_incr_state_merge_last_exec (marker, last_exec)
|
||||
SELECT
|
||||
'after_empty_delta',
|
||||
last_execution_time
|
||||
FROM
|
||||
information_schema.flows
|
||||
WHERE
|
||||
flow_name = 'flow_incr_state_merge';
|
||||
|
||||
SELECT
|
||||
(
|
||||
SELECT
|
||||
last_exec
|
||||
FROM
|
||||
flow_incr_state_merge_last_exec
|
||||
WHERE
|
||||
marker = 'after_empty_delta'
|
||||
) > (
|
||||
SELECT
|
||||
last_exec
|
||||
FROM
|
||||
flow_incr_state_merge_last_exec
|
||||
WHERE
|
||||
marker = 'before_empty_delta'
|
||||
) AS empty_delta_executed;
|
||||
|
||||
-- Flush the input before the final delta so each phase's delta is isolated in
|
||||
-- its own round; the background loop may have already consumed the last
|
||||
-- insert, which is fine — the values below only depend on the merge outcome.
|
||||
|
||||
ADMIN FLUSH_TABLE('flow_incr_state_merge_input');
|
||||
|
||||
ADMIN FLUSH_TABLE('flow_incr_state_merge_sink');
|
||||
|
||||
-- ==== Phase 4: final nonempty delta, same window, both regions ====
|
||||
INSERT INTO flow_incr_state_merge_input VALUES
|
||||
(1, 5, 5, 5.0, '2024-01-01 00:00:05'),
|
||||
(3, 6, 6, 6.0, '2024-01-01 00:00:55');
|
||||
|
||||
-- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED |
|
||||
ADMIN FLUSH_FLOW('flow_incr_state_merge');
|
||||
|
||||
-- Merge the final delta with the persisted states and compare the merged
|
||||
-- result against a direct aggregation over the accepted source rows.
|
||||
SELECT
|
||||
merged.total = direct.total AS total_matches,
|
||||
merged.row_count = direct.row_count AS count_matches,
|
||||
merged.users = direct.users AS hll_matches,
|
||||
abs(merged.v_stddev - direct.v_stddev) < 1e-9 AS stddev_matches,
|
||||
ROUND(merged.v_p50, 4) = ROUND(direct.v_p50, 4) AS p50_matches
|
||||
FROM
|
||||
(
|
||||
SELECT
|
||||
total,
|
||||
row_count,
|
||||
hll_count(hll_merge(user_hll)) AS users,
|
||||
stddev_pop_calc(stddev_pop_merge(v_stddev_state)) AS v_stddev,
|
||||
uddsketch_calc(0.5, uddsketch_merge(128, 0.01, v_sketch_state)) AS v_p50
|
||||
FROM
|
||||
flow_incr_state_merge_sink
|
||||
GROUP BY
|
||||
total,
|
||||
row_count
|
||||
) AS merged,
|
||||
(
|
||||
SELECT
|
||||
sum(n) AS total,
|
||||
count(n) AS row_count,
|
||||
hll_count(hll(user_id)) AS users,
|
||||
stddev_pop(v) AS v_stddev,
|
||||
uddsketch_calc(0.5, uddsketch_state(128, 0.01, v)) AS v_p50
|
||||
FROM
|
||||
flow_incr_state_merge_input
|
||||
WHERE
|
||||
v > 0
|
||||
) AS direct;
|
||||
|
||||
-- Clean up
|
||||
DROP FLOW flow_incr_state_merge;
|
||||
|
||||
DROP TABLE flow_incr_state_merge_input;
|
||||
|
||||
DROP TABLE flow_incr_state_merge_sink;
|
||||
|
||||
DROP TABLE flow_incr_state_merge_last_exec;
|
||||
Reference in New Issue
Block a user