diff --git a/Cargo.lock b/Cargo.lock index 12ef98ef5a..d2c45a28fe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2420,6 +2420,7 @@ dependencies = [ "serde_json", "snafu 0.8.6", "store-api", + "strum 0.27.1", "tokio", "tokio-util", "toml 0.8.23", diff --git a/src/common/event-recorder/Cargo.toml b/src/common/event-recorder/Cargo.toml index 7990cd3d18..9a033d7009 100644 --- a/src/common/event-recorder/Cargo.toml +++ b/src/common/event-recorder/Cargo.toml @@ -23,6 +23,7 @@ serde.workspace = true serde_json.workspace = true snafu.workspace = true store-api.workspace = true +strum.workspace = true tokio.workspace = true tokio-util.workspace = true diff --git a/src/common/event-recorder/src/context.rs b/src/common/event-recorder/src/context.rs new file mode 100644 index 0000000000..601276967e --- /dev/null +++ b/src/common/event-recorder/src/context.rs @@ -0,0 +1,104 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use serde::{Deserialize, Serialize}; +use strum::{AsRefStr, EnumString}; + +/// Stable context recorded for a procedure event. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct PersistentEventContext { + pub reason: TriggerReason, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub protocol: Option, + #[serde(default, skip_serializing_if = "serde_json::Map::is_empty")] + pub extensions: serde_json::Map, +} + +impl PersistentEventContext { + /// Creates an event context with no additional extensions. + pub fn new(reason: TriggerReason) -> Self { + Self { + reason, + protocol: None, + extensions: Default::default(), + } + } + + /// Adds the protocol that originated the operation. + pub fn with_protocol(mut self, protocol: impl Into) -> Self { + self.protocol = Some(protocol.into()); + self + } +} + +impl Default for PersistentEventContext { + fn default() -> Self { + Self::new(TriggerReason::default()) + } +} + +/// Stable classification of a procedure trigger. +#[derive( + Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize, AsRefStr, EnumString, +)] +#[serde(rename_all = "snake_case")] +#[strum(serialize_all = "snake_case")] +pub enum TriggerReason { + Manual, + AutoCreate, + AutoAlter, + AutoRepartition, + AutoRebalance, + RegionFailover, + ScheduledGc, + #[default] + #[serde(other)] + Unknown, +} + +impl TriggerReason { + pub fn from_extension(value: &str) -> Self { + value.parse().unwrap_or_default() + } +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + #[test] + fn test_event_context_serialization() { + let context = PersistentEventContext::new(TriggerReason::Manual).with_protocol("mysql"); + + assert_eq!( + json!({ + "reason": "manual", + "protocol": "mysql", + }), + serde_json::to_value(context).unwrap() + ); + assert_eq!( + json!({ "reason": "manual" }), + serde_json::to_value(PersistentEventContext::new(TriggerReason::Manual)).unwrap() + ); + } + + #[test] + fn test_trigger_reason_deserializes_unknown_value() { + let reason: TriggerReason = serde_json::from_str("\"future_reason\"").unwrap(); + assert_eq!(TriggerReason::Unknown, reason); + } +} diff --git a/src/common/event-recorder/src/lib.rs b/src/common/event-recorder/src/lib.rs index 3e62482498..5b9cb4a7b9 100644 --- a/src/common/event-recorder/src/lib.rs +++ b/src/common/event-recorder/src/lib.rs @@ -14,6 +14,7 @@ #![feature(duration_constructors)] +pub mod context; pub mod error; pub mod event_table; pub mod recorder; @@ -21,4 +22,5 @@ pub mod recorder; #[cfg(any(test, feature = "testing"))] pub mod testing; +pub use context::{PersistentEventContext, TriggerReason}; pub use recorder::*; diff --git a/src/common/meta/src/ddl/alter_database.rs b/src/common/meta/src/ddl/alter_database.rs index f934421302..ac24ff77e1 100644 --- a/src/common/meta/src/ddl/alter_database.rs +++ b/src/common/meta/src/ddl/alter_database.rs @@ -15,7 +15,7 @@ use async_trait::async_trait; use common_procedure::error::{FromJsonSnafu, Result as ProcedureResult, ToJsonSnafu}; use common_procedure::{ - Context as ProcedureContext, EventRuntimeContext, EventTrigger, LockKey, Procedure, Status, + Context as ProcedureContext, EventContext, EventTrigger, LockKey, Procedure, Status, }; use common_telemetry::tracing::info; use serde::{Deserialize, Serialize}; @@ -32,7 +32,7 @@ use crate::key::DeserializedValueWithBytes; use crate::key::schema_name::{SchemaName, SchemaNameKey, SchemaNameValue}; use crate::lock_key::{CatalogLock, SchemaLock}; use crate::rpc::ddl::UnsetDatabaseOption::{self}; -use crate::rpc::ddl::{AlterDatabaseKind, AlterDatabaseTask, EventContext, SetDatabaseOption}; +use crate::rpc::ddl::{AlterDatabaseKind, AlterDatabaseTask, SetDatabaseOption}; pub struct AlterDatabaseProcedure { pub context: DdlContext, @@ -73,14 +73,10 @@ fn build_new_schema_value( impl AlterDatabaseProcedure { pub const TYPE_NAME: &'static str = "metasrv-procedure::AlterDatabase"; - pub fn new( - task: AlterDatabaseTask, - event_context: EventContext, - context: DdlContext, - ) -> Result { + pub fn new(task: AlterDatabaseTask, context: DdlContext) -> Result { Ok(Self { context, - data: AlterDatabaseData::new(task, event_context)?, + data: AlterDatabaseData::new(task)?, }) } @@ -180,10 +176,7 @@ impl Procedure for AlterDatabaseProcedure { LockKey::new(lock_key) } - fn event( - &self, - ctx: &EventRuntimeContext<'_>, - ) -> Option> { + fn event(&self, ctx: &EventContext<'_>) -> Option> { if !ctx.event_type_filter.allows(ALTER_DATABASE_EVENT_TYPE) { return None; } @@ -193,7 +186,6 @@ impl Procedure for AlterDatabaseProcedure { self.data.catalog(), self.data.schema(), &self.data.kind, - self.data.event_context.clone(), ) } else { DatabaseDdlEvent::alter_lifecycle(self.data.catalog(), self.data.schema()) @@ -217,19 +209,16 @@ pub struct AlterDatabaseData { catalog_name: String, schema_name: String, schema_value: Option>, - #[serde(default)] - event_context: EventContext, } impl AlterDatabaseData { - pub fn new(task: AlterDatabaseTask, event_context: EventContext) -> Result { + pub fn new(task: AlterDatabaseTask) -> Result { Ok(Self { state: AlterDatabaseState::Prepare, kind: AlterDatabaseKind::try_from(task.alter_expr.kind.unwrap())?, catalog_name: task.alter_expr.catalog_name, schema_name: task.alter_expr.schema_name, schema_value: None, - event_context, }) } diff --git a/src/common/meta/src/ddl/alter_logical_tables.rs b/src/common/meta/src/ddl/alter_logical_tables.rs index f2f1543093..9207f91057 100644 --- a/src/common/meta/src/ddl/alter_logical_tables.rs +++ b/src/common/meta/src/ddl/alter_logical_tables.rs @@ -20,7 +20,7 @@ use api::region::RegionResponse; use async_trait::async_trait; use common_catalog::format_full_table_name; use common_procedure::error::{FromJsonSnafu, Result as ProcedureResult, ToJsonSnafu}; -use common_procedure::{Context, EventRuntimeContext, EventTrigger, LockKey, Procedure, Status}; +use common_procedure::{Context, EventContext, EventTrigger, LockKey, Procedure, Status}; use common_telemetry::{debug, error, info, warn}; pub use executor::make_alter_region_request; use serde::{Deserialize, Serialize}; @@ -47,7 +47,7 @@ use crate::key::table_info::TableInfoValue; use crate::key::table_route::PhysicalTableRouteValue; use crate::lock_key::{CatalogLock, SchemaLock, TableLock}; use crate::metrics; -use crate::rpc::ddl::{AlterTableTask, EventContext}; +use crate::rpc::ddl::AlterTableTask; use crate::rpc::router::RegionRoute; pub struct AlterLogicalTablesProcedure { @@ -88,7 +88,6 @@ impl AlterLogicalTablesProcedure { pub fn new( tasks: Vec, physical_table_id: TableId, - event_context: EventContext, context: DdlContext, ) -> Self { Self { @@ -101,7 +100,6 @@ impl AlterLogicalTablesProcedure { physical_table_info: None, physical_columns: vec![], table_cache_keys_to_invalidate: vec![], - event_context, }, physical_table_route: None, } @@ -322,10 +320,7 @@ impl Procedure for AlterLogicalTablesProcedure { LockKey::new(lock_key) } - fn event( - &self, - ctx: &EventRuntimeContext<'_>, - ) -> Option> { + fn event(&self, ctx: &EventContext<'_>) -> Option> { if !ctx .event_type_filter .allows(TableDdlEventType::AlterLogicalTables.as_str()) @@ -358,7 +353,6 @@ impl Procedure for AlterLogicalTablesProcedure { locators, self.data.tasks.len(), kinds, - self.data.event_context.clone(), ))) } } @@ -375,8 +369,6 @@ pub struct AlterTablesData { physical_table_info: Option>, physical_columns: Vec, table_cache_keys_to_invalidate: Vec, - #[serde(default)] - event_context: EventContext, } impl AlterTablesData { diff --git a/src/common/meta/src/ddl/alter_table.rs b/src/common/meta/src/ddl/alter_table.rs index 21975fab71..63342a7bbb 100644 --- a/src/common/meta/src/ddl/alter_table.rs +++ b/src/common/meta/src/ddl/alter_table.rs @@ -27,7 +27,7 @@ use common_catalog::consts::{METRIC_ENGINE, MITO_ENGINE}; use common_error::ext::BoxedError; use common_procedure::error::{FromJsonSnafu, Result as ProcedureResult, ToJsonSnafu}; use common_procedure::{ - Context as ProcedureContext, ContextProvider, Error as ProcedureError, EventRuntimeContext, + Context as ProcedureContext, ContextProvider, Error as ProcedureError, EventContext, EventTrigger, LockKey, PoisonKey, PoisonKeys, Procedure, ProcedureId, Status, StringKey, }; use common_telemetry::{error, info, warn}; @@ -58,7 +58,7 @@ use crate::key::{DeserializedValueWithBytes, RegionDistribution}; use crate::lock_key::{CatalogLock, RegionLock, SchemaLock, TableLock, TableNameLock}; use crate::metrics; use crate::poison_key::table_poison_key; -use crate::rpc::ddl::{AlterTableTask, EventContext}; +use crate::rpc::ddl::AlterTableTask; use crate::rpc::router::{RegionRoute, find_leaders, region_distribution}; /// The alter table procedure @@ -97,24 +97,18 @@ fn build_executor_from_alter_expr(alter_data: &AlterTableData) -> AlterTableExec impl AlterTableProcedure { pub const TYPE_NAME: &'static str = "metasrv-procedure::AlterTable"; - pub fn new( - table_id: TableId, - task: AlterTableTask, - event_context: EventContext, - context: DdlContext, - ) -> Result { - Self::new_with_region_locks(table_id, task, event_context, vec![], context) + pub fn new(table_id: TableId, task: AlterTableTask, context: DdlContext) -> Result { + Self::new_with_region_locks(table_id, task, vec![], context) } pub(crate) fn new_with_region_locks( table_id: TableId, task: AlterTableTask, - event_context: EventContext, region_locks: Vec, context: DdlContext, ) -> Result { task.validate()?; - let data = AlterTableData::new(task, table_id, event_context, region_locks); + let data = AlterTableData::new(task, table_id, region_locks); let executor = build_executor_from_alter_expr(&data); Ok(Self { context, @@ -569,10 +563,7 @@ impl Procedure for AlterTableProcedure { PoisonKeys::new(vec![self.table_poison_key()]) } - fn event( - &self, - ctx: &EventRuntimeContext<'_>, - ) -> Option> { + fn event(&self, ctx: &EventContext<'_>) -> Option> { if !ctx .event_type_filter .allows(TableDdlEventType::AlterTable.as_str()) @@ -591,7 +582,7 @@ impl Procedure for AlterTableProcedure { .kind .as_ref() .and_then(alter_table_kind_name); - TableDdlEvent::alter_table_submitted(locator, kind, self.data.event_context.clone()) + TableDdlEvent::alter_table_submitted(locator, kind) } _ => TableDdlEvent::lifecycle(TableDdlEventType::AlterTable, [locator]), }; @@ -619,8 +610,6 @@ pub struct AlterTableData { task: AlterTableTask, table_id: TableId, #[serde(default)] - event_context: EventContext, - #[serde(default)] column_metadatas: Vec, /// Table info value before alteration. table_info_value: Option>, @@ -632,17 +621,11 @@ pub struct AlterTableData { } impl AlterTableData { - pub fn new( - task: AlterTableTask, - table_id: TableId, - event_context: EventContext, - region_locks: Vec, - ) -> Self { + pub fn new(task: AlterTableTask, table_id: TableId, region_locks: Vec) -> Self { Self { state: AlterTableState::Prepare, task, table_id, - event_context, column_metadatas: vec![], table_info_value: None, region_distribution: None, diff --git a/src/common/meta/src/ddl/alter_table/region_request.rs b/src/common/meta/src/ddl/alter_table/region_request.rs index ef8d1487c7..73edd4c361 100644 --- a/src/common/meta/src/ddl/alter_table/region_request.rs +++ b/src/common/meta/src/ddl/alter_table/region_request.rs @@ -144,7 +144,7 @@ mod tests { }; use crate::key::table_route::TableRouteValue; use crate::peer::Peer; - use crate::rpc::ddl::{AlterTableTask, EventContext}; + use crate::rpc::ddl::AlterTableTask; use crate::rpc::router::{Region, RegionRoute}; use crate::test_util::{MockDatanodeManager, new_ddl_context}; @@ -240,8 +240,7 @@ mod tests { }, }; - let mut procedure = - AlterTableProcedure::new(table_id, task, EventContext::default(), ddl_context).unwrap(); + let mut procedure = AlterTableProcedure::new(table_id, task, ddl_context).unwrap(); procedure.on_prepare().await.unwrap(); let alter_kind = procedure.make_region_alter_kind().unwrap(); let Some(Body::Alter(alter_region_request)) = @@ -297,8 +296,7 @@ mod tests { }, }; - let mut procedure = - AlterTableProcedure::new(table_id, task, EventContext::default(), ddl_context).unwrap(); + let mut procedure = AlterTableProcedure::new(table_id, task, ddl_context).unwrap(); procedure.on_prepare().await.unwrap(); let alter_kind = procedure.make_region_alter_kind().unwrap(); let Some(Body::Alter(alter_region_request)) = diff --git a/src/common/meta/src/ddl/create_database.rs b/src/common/meta/src/ddl/create_database.rs index b48e700a51..82d467888a 100644 --- a/src/common/meta/src/ddl/create_database.rs +++ b/src/common/meta/src/ddl/create_database.rs @@ -19,8 +19,8 @@ use async_trait::async_trait; use common_error::ext::{BoxedError, ErrorExt}; use common_procedure::error::{FromJsonSnafu, Result as ProcedureResult, ToJsonSnafu}; use common_procedure::{ - Context as ProcedureContext, EventRuntimeContext, EventTrigger, LockKey, Procedure, - ProcedureId, Status, + Context as ProcedureContext, EventContext, EventTrigger, LockKey, Procedure, ProcedureId, + Status, }; use serde::{Deserialize, Serialize}; use serde_with::{DefaultOnNull, serde_as}; @@ -34,7 +34,7 @@ use crate::error::{self, Result}; use crate::instruction::{CacheIdent, UserCacheIdent}; use crate::key::schema_name::{SchemaNameKey, SchemaNameValue}; use crate::lock_key::{CatalogLock, SchemaLock}; -use crate::rpc::ddl::{CreatorGrantIntent, EventContext}; +use crate::rpc::ddl::CreatorGrantIntent; /// Describes the creator-access result of an atomic create. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -93,7 +93,6 @@ impl CreateDatabaseProcedure { create_if_not_exists: bool, options: HashMap, creator: Option, - event_context: EventContext, context: DdlContext, ) -> Self { Self { @@ -105,7 +104,6 @@ impl CreateDatabaseProcedure { create_if_not_exists, options, creator, - event_context, }, } } @@ -280,10 +278,7 @@ impl Procedure for CreateDatabaseProcedure { LockKey::new(lock_key) } - fn event( - &self, - ctx: &EventRuntimeContext<'_>, - ) -> Option> { + fn event(&self, ctx: &EventContext<'_>) -> Option> { if !ctx.event_type_filter.allows(CREATE_DATABASE_EVENT_TYPE) { return None; } @@ -294,7 +289,6 @@ impl Procedure for CreateDatabaseProcedure { &self.data.schema, self.data.create_if_not_exists, &self.data.options, - self.data.event_context.clone(), ) } else { DatabaseDdlEvent::create_lifecycle(&self.data.catalog, &self.data.schema) @@ -327,8 +321,6 @@ pub struct CreateDatabaseData { /// Authenticated creator whose access is ensured, absent for legacy schema-only requests. #[serde(default)] pub creator: Option, - #[serde(default)] - pub event_context: EventContext, } #[cfg(test)] @@ -423,7 +415,6 @@ mod tests { username: "alice".to_string(), created_at_ns: 1, }), - EventContext::default(), context, ); (procedure, committer) @@ -583,7 +574,6 @@ mod tests { false, HashMap::new(), None, - EventContext::default(), context, ); procedure.data.state = CreateDatabaseState::CreateMetadata; @@ -619,7 +609,6 @@ mod tests { username: "alice".to_string(), created_at_ns: 1, }), - EventContext::default(), context, ); procedure.data.state = CreateDatabaseState::CreateMetadata; diff --git a/src/common/meta/src/ddl/create_flow.rs b/src/common/meta/src/ddl/create_flow.rs index 3ef4dadbc6..a2f306306d 100644 --- a/src/common/meta/src/ddl/create_flow.rs +++ b/src/common/meta/src/ddl/create_flow.rs @@ -24,8 +24,8 @@ use async_trait::async_trait; use common_catalog::format_full_flow_name; use common_procedure::error::{FromJsonSnafu, ToJsonSnafu}; use common_procedure::{ - Context as ProcedureContext, EventRuntimeContext, EventTrigger, LockKey, Procedure, - ProcedureState, Result as ProcedureResult, Status, + Context as ProcedureContext, EventContext, EventTrigger, LockKey, Procedure, ProcedureState, + Result as ProcedureResult, Status, }; use common_telemetry::info; use common_telemetry::tracing_context::TracingContext; @@ -50,7 +50,7 @@ use crate::key::{DeserializedValueWithBytes, FlowId, FlowPartitionId}; use crate::lock_key::{CatalogLock, FlowNameLock}; use crate::metrics; use crate::peer::Peer; -use crate::rpc::ddl::{CreateFlowTask, EventContext, FlowQueryContext, QueryContext}; +use crate::rpc::ddl::{CreateFlowTask, FlowQueryContext, QueryContext}; /// The procedure of flow creation. pub struct CreateFlowProcedure { @@ -62,12 +62,7 @@ impl CreateFlowProcedure { pub const TYPE_NAME: &'static str = "metasrv-procedure::CreateFlow"; /// Returns a new [CreateFlowProcedure]. - pub fn new( - task: CreateFlowTask, - query_context: QueryContext, - event_context: EventContext, - context: DdlContext, - ) -> Self { + pub fn new(task: CreateFlowTask, query_context: QueryContext, context: DdlContext) -> Self { Self { context, data: CreateFlowData { @@ -81,7 +76,6 @@ impl CreateFlowProcedure { prev_flow_info_value: None, did_replace: false, flow_type: None, - event_context, }, } } @@ -410,10 +404,7 @@ impl Procedure for CreateFlowProcedure { ]) } - fn event( - &self, - ctx: &EventRuntimeContext<'_>, - ) -> Option> { + fn event(&self, ctx: &EventContext<'_>) -> Option> { if !ctx.event_type_filter.allows(CREATE_FLOW_EVENT_TYPE) { return None; } @@ -428,7 +419,6 @@ impl Procedure for CreateFlowProcedure { expire_after: self.data.task.expire_after, eval_interval_secs: self.data.task.eval_interval_secs, }, - self.data.event_context.clone(), ), EventTrigger::Succeeded => { let flow_id = match ctx.lifecycle_state { @@ -718,8 +708,6 @@ pub struct CreateFlowData { #[serde(default)] pub(crate) did_replace: bool, pub(crate) flow_type: Option, - #[serde(default)] - pub(crate) event_context: EventContext, } impl CreateFlowData { diff --git a/src/common/meta/src/ddl/create_logical_tables.rs b/src/common/meta/src/ddl/create_logical_tables.rs index 1c9efb865e..9b231f3c53 100644 --- a/src/common/meta/src/ddl/create_logical_tables.rs +++ b/src/common/meta/src/ddl/create_logical_tables.rs @@ -24,8 +24,8 @@ use common_catalog::consts::METRIC_ENGINE; use common_event_recorder::Event; use common_procedure::error::{FromJsonSnafu, Result as ProcedureResult, ToJsonSnafu}; use common_procedure::{ - Context as ProcedureContext, EventRuntimeContext, EventTrigger, LockKey, Procedure, - ProcedureState, Status, + Context as ProcedureContext, EventContext, EventTrigger, LockKey, Procedure, ProcedureState, + Status, }; use common_telemetry::{debug, error, warn}; use futures::future; @@ -48,7 +48,7 @@ use crate::error::Result; use crate::key::table_route::TableRouteValue; use crate::lock_key::{CatalogLock, SchemaLock, TableLock, TableNameLock}; use crate::metrics; -use crate::rpc::ddl::{CreateTableTask, EventContext}; +use crate::rpc::ddl::CreateTableTask; use crate::rpc::router::{RegionRoute, find_leaders}; pub struct CreateLogicalTablesProcedure { @@ -62,7 +62,6 @@ impl CreateLogicalTablesProcedure { pub fn new( tasks: Vec, physical_table_id: TableId, - event_context: EventContext, context: DdlContext, ) -> Self { Self { @@ -75,7 +74,6 @@ impl CreateLogicalTablesProcedure { physical_region_numbers: vec![], physical_columns: vec![], physical_partition_columns: vec![], - event_context, }, } } @@ -273,7 +271,7 @@ impl Procedure for CreateLogicalTablesProcedure { LockKey::new(lock_key) } - fn event(&self, ctx: &EventRuntimeContext<'_>) -> Option> { + fn event(&self, ctx: &EventContext<'_>) -> Option> { if !ctx .event_type_filter .allows(TableDdlEventType::CreateLogicalTables.as_str()) @@ -284,7 +282,6 @@ impl Procedure for CreateLogicalTablesProcedure { EventTrigger::Submitted => TableDdlEvent::create_logical_tables_submitted( self.event_locators(), self.data.tasks.len(), - self.data.event_context.clone(), ), EventTrigger::Succeeded => match ctx.lifecycle_state { ProcedureState::Done { @@ -329,8 +326,6 @@ pub struct CreateTablesData { physical_region_numbers: Vec, physical_columns: Vec, physical_partition_columns: Vec, - #[serde(default)] - event_context: EventContext, } impl CreateTablesData { diff --git a/src/common/meta/src/ddl/create_table.rs b/src/common/meta/src/ddl/create_table.rs index c63d7f7a34..ce3e1aa060 100644 --- a/src/common/meta/src/ddl/create_table.rs +++ b/src/common/meta/src/ddl/create_table.rs @@ -23,8 +23,8 @@ use common_procedure::error::{ }; use common_procedure::local::DynamicKeyLockGuard; use common_procedure::{ - Context as ProcedureContext, EventRuntimeContext, EventTrigger, LockKey, Procedure, - ProcedureId, ProcedureState, Status, + Context as ProcedureContext, EventContext, EventTrigger, LockKey, Procedure, ProcedureId, + ProcedureState, Status, }; use common_telemetry::info; use serde::{Deserialize, Serialize}; @@ -47,7 +47,7 @@ use crate::lock_key::{CatalogLock, SchemaLock, TableNameLock}; use crate::metrics; use crate::peer::PeerAllocContext; use crate::region_keeper::OperatingRegionGuard; -use crate::rpc::ddl::{CreateTableTask, EventContext, QueryContext}; +use crate::rpc::ddl::{CreateTableTask, QueryContext}; use crate::rpc::router::{RegionRoute, operating_leader_region_roles}; use crate::wal_provider::{ RegionWalOptions, acquire_remote_wal_read_locks, optional_region_wal_options_serde, @@ -84,17 +84,20 @@ fn build_executor_from_create_table_data( impl CreateTableProcedure { pub const TYPE_NAME: &'static str = "metasrv-procedure::CreateTable"; - pub fn new( + pub fn new(task: CreateTableTask, context: DdlContext) -> Result { + Self::new_with_query_context(task, QueryContext::default(), context) + } + + pub fn new_with_query_context( task: CreateTableTask, query_context: QueryContext, - event_context: EventContext, context: DdlContext, ) -> Result { let executor = build_executor_from_create_table_data(&task.create_table)?; Ok(Self { context, - data: CreateTableData::new_with_query_context(task, event_context, query_context), + data: CreateTableData::new(task, query_context), opening_regions: vec![], executor, remote_wal_lock_guards: vec![], @@ -396,10 +399,7 @@ impl Procedure for CreateTableProcedure { ]) } - fn event( - &self, - ctx: &EventRuntimeContext<'_>, - ) -> Option> { + fn event(&self, ctx: &EventContext<'_>) -> Option> { if !ctx .event_type_filter .allows(TableDdlEventType::CreateTable.as_str()) @@ -415,7 +415,6 @@ impl Procedure for CreateTableProcedure { locator, create_table.create_if_not_exists, &create_table.engine, - self.data.event_context.clone(), ) } EventTrigger::Succeeded => match ctx.lifecycle_state { @@ -456,8 +455,6 @@ pub struct CreateTableData { #[serde(default)] pub query_context: QueryContext, #[serde(default)] - pub event_context: EventContext, - #[serde(default)] pub column_metadatas: Vec, /// None stands for not allocated yet. pub(crate) table_route: Option, @@ -468,21 +465,12 @@ pub struct CreateTableData { } impl CreateTableData { - pub fn new(task: CreateTableTask, event_context: EventContext) -> Self { - Self::new_with_query_context(task, event_context, QueryContext::default()) - } - - pub fn new_with_query_context( - task: CreateTableTask, - event_context: EventContext, - query_context: QueryContext, - ) -> Self { + pub fn new(task: CreateTableTask, query_context: QueryContext) -> Self { CreateTableData { state: CreateTableState::Prepare, column_metadatas: vec![], task, query_context, - event_context, table_route: None, region_wal_options: None, } diff --git a/src/common/meta/src/ddl/create_view.rs b/src/common/meta/src/ddl/create_view.rs index a7175854b7..666ef19efd 100644 --- a/src/common/meta/src/ddl/create_view.rs +++ b/src/common/meta/src/ddl/create_view.rs @@ -16,8 +16,8 @@ use async_trait::async_trait; use common_event_recorder::Event; use common_procedure::error::{FromJsonSnafu, Result as ProcedureResult, ToJsonSnafu}; use common_procedure::{ - Context as ProcedureContext, EventRuntimeContext, EventTrigger, LockKey, Procedure, - ProcedureState, Status, + Context as ProcedureContext, EventContext, EventTrigger, LockKey, Procedure, ProcedureState, + Status, }; use common_telemetry::info; use serde::{Deserialize, Serialize}; @@ -35,7 +35,7 @@ use crate::instruction::CacheIdent; use crate::key::table_name::TableNameKey; use crate::lock_key::{CatalogLock, SchemaLock, TableNameLock}; use crate::metrics; -use crate::rpc::ddl::{CreateViewTask, EventContext}; +use crate::rpc::ddl::CreateViewTask; // The procedure to execute `[CreateViewTask]`. pub struct CreateViewProcedure { @@ -46,14 +46,13 @@ pub struct CreateViewProcedure { impl CreateViewProcedure { pub const TYPE_NAME: &'static str = "metasrv-procedure::CreateView"; - pub fn new(task: CreateViewTask, event_context: EventContext, context: DdlContext) -> Self { + pub fn new(task: CreateViewTask, context: DdlContext) -> Self { Self { context, data: CreateViewData { state: CreateViewState::Prepare, task, need_update: false, - event_context, }, } } @@ -272,7 +271,7 @@ impl Procedure for CreateViewProcedure { ]) } - fn event(&self, ctx: &EventRuntimeContext<'_>) -> Option> { + fn event(&self, ctx: &EventContext<'_>) -> Option> { if !ctx.event_type_filter.allows(CREATE_VIEW_EVENT_TYPE) { return None; } @@ -290,7 +289,6 @@ impl Procedure for CreateViewProcedure { referenced_table_count: self.data.task.table_names().len(), column_count: self.data.task.columns().len(), }, - self.data.event_context.clone(), ) } EventTrigger::Succeeded => match ctx.lifecycle_state { @@ -344,8 +342,6 @@ pub struct CreateViewData { pub task: CreateViewTask, /// Whether to update the view info. pub need_update: bool, - #[serde(default)] - pub event_context: EventContext, } impl CreateViewData { diff --git a/src/common/meta/src/ddl/drop_database.rs b/src/common/meta/src/ddl/drop_database.rs index 7772e966cb..5bbd229ec5 100644 --- a/src/common/meta/src/ddl/drop_database.rs +++ b/src/common/meta/src/ddl/drop_database.rs @@ -23,7 +23,7 @@ use std::fmt::Debug; use common_error::ext::BoxedError; use common_procedure::error::{ExternalSnafu, FromJsonSnafu, ToJsonSnafu}; use common_procedure::{ - Context as ProcedureContext, EventRuntimeContext, EventTrigger, LockKey, Procedure, + Context as ProcedureContext, EventContext, EventTrigger, LockKey, Procedure, Result as ProcedureResult, Status, }; use futures::stream::BoxStream; @@ -38,13 +38,11 @@ use crate::ddl::utils::map_to_procedure_error; use crate::error::Result; use crate::key::table_name::TableNameValue; use crate::lock_key::{CatalogLock, SchemaLock}; -use crate::rpc::ddl::EventContext; pub struct DropDatabaseProcedure { /// The context of procedure runtime. runtime_context: DdlContext, context: DropDatabaseContext, - event_context: EventContext, state: Box, } @@ -87,16 +85,9 @@ pub(crate) trait State: Send + Debug { impl DropDatabaseProcedure { pub const TYPE_NAME: &'static str = "metasrv-procedure::DropDatabase"; - pub fn new( - catalog: String, - schema: String, - drop_if_exists: bool, - event_context: EventContext, - context: DdlContext, - ) -> Self { + pub fn new(catalog: String, schema: String, drop_if_exists: bool, context: DdlContext) -> Self { Self { runtime_context: context, - event_context, context: DropDatabaseContext { catalog, schema, @@ -113,13 +104,11 @@ impl DropDatabaseProcedure { catalog, schema, drop_if_exists, - event_context, state, } = serde_json::from_str(json).context(FromJsonSnafu)?; Ok(Self { runtime_context, - event_context, context: DropDatabaseContext { catalog, schema, @@ -170,7 +159,6 @@ impl Procedure for DropDatabaseProcedure { catalog: &self.context.catalog, schema: &self.context.schema, drop_if_exists: self.context.drop_if_exists, - event_context: &self.event_context, state: self.state.as_ref(), }; @@ -186,10 +174,7 @@ impl Procedure for DropDatabaseProcedure { LockKey::new(lock_key) } - fn event( - &self, - ctx: &EventRuntimeContext<'_>, - ) -> Option> { + fn event(&self, ctx: &EventContext<'_>) -> Option> { if !ctx.event_type_filter.allows(DROP_DATABASE_EVENT_TYPE) { return None; } @@ -199,7 +184,6 @@ impl Procedure for DropDatabaseProcedure { &self.context.catalog, &self.context.schema, self.context.drop_if_exists, - self.event_context.clone(), ) } else { DatabaseDdlEvent::drop_lifecycle(&self.context.catalog, &self.context.schema) @@ -215,7 +199,6 @@ struct DropDatabaseData<'a> { // The schema name schema: &'a str, drop_if_exists: bool, - event_context: &'a EventContext, state: &'a dyn State, } @@ -226,7 +209,5 @@ struct DropDatabaseOwnedData { // The schema name schema: String, drop_if_exists: bool, - #[serde(default)] - event_context: EventContext, state: Box, } diff --git a/src/common/meta/src/ddl/drop_flow.rs b/src/common/meta/src/ddl/drop_flow.rs index 99c18ebc13..783fb7796e 100644 --- a/src/common/meta/src/ddl/drop_flow.rs +++ b/src/common/meta/src/ddl/drop_flow.rs @@ -21,7 +21,7 @@ use common_error::ext::ErrorExt; use common_error::status_code::StatusCode; use common_procedure::error::{FromJsonSnafu, ToJsonSnafu}; use common_procedure::{ - Context as ProcedureContext, EventRuntimeContext, EventTrigger, LockKey, Procedure, + Context as ProcedureContext, EventContext, EventTrigger, LockKey, Procedure, Result as ProcedureResult, Status, }; use common_telemetry::info; @@ -41,7 +41,7 @@ use crate::key::flow::flow_info::FlowInfoValue; use crate::key::flow::flow_route::FlowRouteValue; use crate::lock_key::{CatalogLock, FlowLock}; use crate::metrics; -use crate::rpc::ddl::{DropFlowTask, EventContext}; +use crate::rpc::ddl::DropFlowTask; /// The procedure for dropping a flow. pub struct DropFlowProcedure { @@ -54,7 +54,7 @@ pub struct DropFlowProcedure { impl DropFlowProcedure { pub const TYPE_NAME: &'static str = "metasrv-procedure::DropFlow"; - pub fn new(task: DropFlowTask, event_context: EventContext, context: DdlContext) -> Self { + pub fn new(task: DropFlowTask, context: DdlContext) -> Self { Self { context, data: DropFlowData { @@ -62,7 +62,6 @@ impl DropFlowProcedure { task, flow_info_value: None, flow_route_values: vec![], - event_context, }, } } @@ -223,10 +222,7 @@ impl Procedure for DropFlowProcedure { LockKey::new(lock_key) } - fn event( - &self, - ctx: &EventRuntimeContext<'_>, - ) -> Option> { + fn event(&self, ctx: &EventContext<'_>) -> Option> { if !ctx.event_type_filter.allows(DROP_FLOW_EVENT_TYPE) { return None; } @@ -237,7 +233,6 @@ impl Procedure for DropFlowProcedure { &self.data.task.flow_name, self.data.task.flow_id, self.data.task.drop_if_exists, - self.data.event_context.clone(), ), _ => FlowDdlEvent::drop_lifecycle( &self.data.task.catalog_name, @@ -257,8 +252,6 @@ pub(crate) struct DropFlowData { task: DropFlowTask, pub(crate) flow_info_value: Option, pub(crate) flow_route_values: Vec, - #[serde(default)] - pub(crate) event_context: EventContext, } /// The state of drop flow diff --git a/src/common/meta/src/ddl/drop_table.rs b/src/common/meta/src/ddl/drop_table.rs index 5ba23874e2..5db98a5e00 100644 --- a/src/common/meta/src/ddl/drop_table.rs +++ b/src/common/meta/src/ddl/drop_table.rs @@ -22,8 +22,8 @@ use common_error::ext::BoxedError; use common_event_recorder::Event; use common_procedure::error::{ExternalSnafu, FromJsonSnafu, ToJsonSnafu}; use common_procedure::{ - Context as ProcedureContext, Error as ProcedureError, EventRuntimeContext, EventTrigger, - LockKey, Procedure, Result as ProcedureResult, Status, + Context as ProcedureContext, Error as ProcedureError, EventContext, EventTrigger, LockKey, + Procedure, Result as ProcedureResult, Status, }; use common_telemetry::info; use common_telemetry::tracing::warn; @@ -48,7 +48,7 @@ use crate::key::table_route::TableRouteValue; use crate::lock_key::{CatalogLock, SchemaLock, TableLock, TableNameLock}; use crate::metrics; use crate::region_keeper::OperatingRegionGuard; -use crate::rpc::ddl::{DropTableTask, EventContext}; +use crate::rpc::ddl::DropTableTask; use crate::rpc::router::{RegionRoute, operating_leader_region_roles}; #[cfg(feature = "enterprise")] @@ -74,12 +74,11 @@ pub struct DropTableProcedure { impl DropTableProcedure { pub const TYPE_NAME: &'static str = "metasrv-procedure::DropTable"; - pub fn new(task: DropTableTask, context: DdlContext, event_context: EventContext) -> Self { + pub fn new(task: DropTableTask, context: DdlContext) -> Self { let data = DropTableData::new( task, cfg!(feature = "enterprise") && context.soft_drop_enabled, context.soft_drop_retention, - event_context, ); let executor = data.build_executor(); Self { @@ -414,7 +413,7 @@ impl Procedure for DropTableProcedure { LockKey::new(lock_key) } - fn event(&self, ctx: &EventRuntimeContext<'_>) -> Option> { + fn event(&self, ctx: &EventContext<'_>) -> Option> { if !ctx .event_type_filter .allows(TableDdlEventType::DropTable.as_str()) @@ -425,11 +424,9 @@ impl Procedure for DropTableProcedure { let locator = TableDdlLocator::new(&task.catalog, &task.schema, &task.table) .with_table_id(task.table_id); let event = match &ctx.trigger { - EventTrigger::Submitted => TableDdlEvent::drop_table_submitted( - locator, - task.drop_if_exists, - self.data.event_context.clone(), - ), + EventTrigger::Submitted => { + TableDdlEvent::drop_table_submitted(locator, task.drop_if_exists) + } _ => TableDdlEvent::lifecycle(TableDdlEventType::DropTable, [locator]), }; @@ -483,8 +480,6 @@ pub struct DropTableData { pub soft_drop_retention_millis: Option, #[serde(default)] pub drop_generation: Option, - #[serde(default)] - pub event_context: EventContext, } impl DropTableData { @@ -492,7 +487,6 @@ impl DropTableData { task: DropTableTask, soft_drop_enabled: bool, soft_drop_retention: Option, - event_context: EventContext, ) -> Self { Self { state: DropTableState::Prepare, @@ -506,7 +500,6 @@ impl DropTableData { retention_expires_at: None, soft_drop_retention_millis: soft_drop_retention_millis(soft_drop_retention), drop_generation: None, - event_context, } } diff --git a/src/common/meta/src/ddl/drop_view.rs b/src/common/meta/src/ddl/drop_view.rs index ece1d2b06b..32d843fbd4 100644 --- a/src/common/meta/src/ddl/drop_view.rs +++ b/src/common/meta/src/ddl/drop_view.rs @@ -16,7 +16,7 @@ use async_trait::async_trait; use common_event_recorder::Event; use common_procedure::error::{FromJsonSnafu, ToJsonSnafu}; use common_procedure::{ - Context as ProcedureContext, EventRuntimeContext, EventTrigger, LockKey, Procedure, + Context as ProcedureContext, EventContext, EventTrigger, LockKey, Procedure, Result as ProcedureResult, Status, }; use common_telemetry::info; @@ -35,7 +35,7 @@ use crate::instruction::CacheIdent; use crate::key::table_name::TableNameKey; use crate::lock_key::{CatalogLock, SchemaLock, TableLock}; use crate::metrics; -use crate::rpc::ddl::{DropViewTask, EventContext}; +use crate::rpc::ddl::DropViewTask; /// The procedure for dropping a view. pub struct DropViewProcedure { @@ -48,13 +48,12 @@ pub struct DropViewProcedure { impl DropViewProcedure { pub const TYPE_NAME: &'static str = "metasrv-procedure::DropView"; - pub fn new(task: DropViewTask, event_context: EventContext, context: DdlContext) -> Self { + pub fn new(task: DropViewTask, context: DdlContext) -> Self { Self { context, data: DropViewData { state: DropViewState::Prepare, task, - event_context, }, } } @@ -214,7 +213,7 @@ impl Procedure for DropViewProcedure { LockKey::new(lock_key) } - fn event(&self, ctx: &EventRuntimeContext<'_>) -> Option> { + fn event(&self, ctx: &EventContext<'_>) -> Option> { if !ctx.event_type_filter.allows(DROP_VIEW_EVENT_TYPE) { return None; } @@ -228,7 +227,6 @@ impl Procedure for DropViewProcedure { table_ref.table, self.data.view_id(), self.data.task.drop_if_exists, - self.data.event_context.clone(), ) } _ => { @@ -251,8 +249,6 @@ impl Procedure for DropViewProcedure { pub(crate) struct DropViewData { state: DropViewState, task: DropViewTask, - #[serde(default)] - event_context: EventContext, } impl DropViewData { diff --git a/src/common/meta/src/ddl/event/database.rs b/src/common/meta/src/ddl/event/database.rs index b24611799c..cab42020f9 100644 --- a/src/common/meta/src/ddl/event/database.rs +++ b/src/common/meta/src/ddl/event/database.rs @@ -19,14 +19,13 @@ use api::v1::{ColumnSchema, Row}; use common_event_recorder::Event; use common_event_recorder::error::{Result, SerializeEventSnafu}; use common_event_recorder::event_table::{ - CATALOG_NAME_COLUMN as EVENT_TABLE_CATALOG_NAME_COLUMN, EVENT_CONTEXT_COLUMN, - SCHEMA_NAME_COLUMN as EVENT_TABLE_SCHEMA_NAME_COLUMN, column_schemas, nullable_json, - nullable_string, + CATALOG_NAME_COLUMN as EVENT_TABLE_CATALOG_NAME_COLUMN, + SCHEMA_NAME_COLUMN as EVENT_TABLE_SCHEMA_NAME_COLUMN, column_schemas, nullable_string, }; use serde::Serialize; use snafu::ResultExt; -use crate::rpc::ddl::{AlterDatabaseKind, EventContext, SetDatabaseOption, UnsetDatabaseOption}; +use crate::rpc::ddl::{AlterDatabaseKind, SetDatabaseOption, UnsetDatabaseOption}; pub(crate) const CREATE_DATABASE_EVENT_TYPE: &str = "create_database"; pub(crate) const ALTER_DATABASE_EVENT_TYPE: &str = "alter_database"; @@ -40,7 +39,6 @@ pub(crate) struct DatabaseDdlEvent { catalog_name: Option, schema_name: Option, payload: Option, - event_context: Option, } #[derive(Debug, Serialize)] @@ -90,7 +88,6 @@ impl DatabaseDdlEvent { schema_name: &str, create_if_not_exists: bool, options: &HashMap, - event_context: EventContext, ) -> Self { let mut options = options.iter().collect::>(); options.sort_unstable_by_key(|(left, _)| *left); @@ -110,7 +107,6 @@ impl DatabaseDdlEvent { create_if_not_exists, options, }), - event_context, ) } @@ -118,7 +114,6 @@ impl DatabaseDdlEvent { catalog_name: &str, schema_name: &str, kind: &AlterDatabaseKind, - event_context: EventContext, ) -> Self { let intent = match kind { AlterDatabaseKind::SetDatabaseOptions(options) => AlterDatabaseIntent::Set { @@ -157,7 +152,6 @@ impl DatabaseDdlEvent { version: PAYLOAD_VERSION, intent, }), - event_context, ) } @@ -165,7 +159,6 @@ impl DatabaseDdlEvent { catalog_name: &str, schema_name: &str, drop_if_exists: bool, - event_context: EventContext, ) -> Self { Self::submitted( DROP_DATABASE_EVENT_TYPE, @@ -175,7 +168,6 @@ impl DatabaseDdlEvent { version: PAYLOAD_VERSION, drop_if_exists, }), - event_context, ) } @@ -196,14 +188,12 @@ impl DatabaseDdlEvent { catalog_name: &str, schema_name: &str, payload: DatabaseDdlPayload, - event_context: EventContext, ) -> Self { Self { event_type, catalog_name: Some(catalog_name.to_string()), schema_name: Some(schema_name.to_string()), payload: Some(payload), - event_context: Some(event_context), } } @@ -213,7 +203,6 @@ impl DatabaseDdlEvent { catalog_name: Some(catalog_name.to_string()), schema_name: Some(schema_name.to_string()), payload: None, - event_context: None, } } } @@ -234,22 +223,14 @@ impl Event for DatabaseDdlEvent { column_schemas([ &EVENT_TABLE_CATALOG_NAME_COLUMN, &EVENT_TABLE_SCHEMA_NAME_COLUMN, - &EVENT_CONTEXT_COLUMN, ]) } fn extra_rows(&self) -> Result> { - let event_context = self - .event_context - .as_ref() - .map(serde_json::to_value) - .transpose() - .context(SerializeEventSnafu)?; Ok(vec![Row { values: vec![ nullable_string(self.catalog_name.as_deref()), nullable_string(self.schema_name.as_deref()), - nullable_json(event_context.as_ref()), ], }]) } diff --git a/src/common/meta/src/ddl/event/flow.rs b/src/common/meta/src/ddl/event/flow.rs index 7d0d6cb539..0a9ec2dd24 100644 --- a/src/common/meta/src/ddl/event/flow.rs +++ b/src/common/meta/src/ddl/event/flow.rs @@ -19,14 +19,12 @@ use api::v1::{ColumnSchema, Row}; use common_event_recorder::Event; use common_event_recorder::error::{Result, SerializeEventSnafu}; use common_event_recorder::event_table::{ - CATALOG_NAME_COLUMN, EVENT_CONTEXT_COLUMN, FLOW_ID_COLUMN, FLOW_NAME_COLUMN, column_schemas, - nullable_json, nullable_string, nullable_value, + CATALOG_NAME_COLUMN, FLOW_ID_COLUMN, FLOW_NAME_COLUMN, column_schemas, nullable_string, + nullable_value, }; use serde::Serialize; use snafu::ResultExt; -use crate::rpc::ddl::EventContext; - pub(crate) const CREATE_FLOW_EVENT_TYPE: &str = "create_flow"; pub(crate) const DROP_FLOW_EVENT_TYPE: &str = "drop_flow"; @@ -71,15 +69,14 @@ pub(crate) struct FlowDdlEvent { flow_name: Option, flow_id: Option, payload: Option, - event_context: Option, } impl FlowDdlEvent { + /// Builds the bounded event emitted when creating a Flow is submitted. pub(crate) fn create_submitted( catalog_name: &str, flow_name: &str, intent: CreateFlowEventIntent, - event_context: EventContext, ) -> Self { Self { event_type: CREATE_FLOW_EVENT_TYPE, @@ -93,16 +90,15 @@ impl FlowDdlEvent { expire_after: intent.expire_after, eval_interval_secs: intent.eval_interval_secs, })), - event_context: Some(event_context), } } + /// Builds the bounded event emitted when dropping a Flow is submitted. pub(crate) fn drop_submitted( catalog_name: &str, flow_name: &str, flow_id: u32, drop_if_exists: bool, - event_context: EventContext, ) -> Self { Self { event_type: DROP_FLOW_EVENT_TYPE, @@ -113,7 +109,6 @@ impl FlowDdlEvent { version: PAYLOAD_VERSION, drop_if_exists, })), - event_context: Some(event_context), } } @@ -149,7 +144,6 @@ impl FlowDdlEvent { flow_name: Some(flow_name.to_string()), flow_id: None, payload: None, - event_context: None, } } } @@ -167,27 +161,15 @@ impl Event for FlowDdlEvent { } fn extra_schema(&self) -> Vec { - column_schemas([ - &CATALOG_NAME_COLUMN, - &FLOW_NAME_COLUMN, - &FLOW_ID_COLUMN, - &EVENT_CONTEXT_COLUMN, - ]) + column_schemas([&CATALOG_NAME_COLUMN, &FLOW_NAME_COLUMN, &FLOW_ID_COLUMN]) } fn extra_rows(&self) -> Result> { - let event_context = self - .event_context - .as_ref() - .map(serde_json::to_value) - .transpose() - .context(SerializeEventSnafu)?; Ok(vec![Row { values: vec![ nullable_string(self.catalog_name.as_deref()), nullable_string(self.flow_name.as_deref()), nullable_value(self.flow_id.map(ValueData::U32Value)), - nullable_json(event_context.as_ref()), ], }]) } diff --git a/src/common/meta/src/ddl/event/table.rs b/src/common/meta/src/ddl/event/table.rs index 9ae6f53076..670f3c7c1d 100644 --- a/src/common/meta/src/ddl/event/table.rs +++ b/src/common/meta/src/ddl/event/table.rs @@ -21,17 +21,14 @@ use api::v1::{ColumnSchema, Row}; use common_event_recorder::Event; use common_event_recorder::error::{Result, SerializeEventSnafu}; use common_event_recorder::event_table::{ - CATALOG_NAME_COLUMN, EVENT_CONTEXT_COLUMN, PHYSICAL_TABLE_ID_COLUMN, SCHEMA_NAME_COLUMN, - TABLE_ID_COLUMN, TABLE_NAME_COLUMN, column_schemas, nullable_json, nullable_string, - nullable_value, + CATALOG_NAME_COLUMN, PHYSICAL_TABLE_ID_COLUMN, SCHEMA_NAME_COLUMN, TABLE_ID_COLUMN, + TABLE_NAME_COLUMN, column_schemas, nullable_string, nullable_value, }; use serde::Serialize; use serde_json::Value as JsonValue; use snafu::ResultExt; use store_api::storage::TableId; -use crate::rpc::ddl::EventContext; - /// Current version of table DDL event payloads. pub(crate) const TABLE_DDL_PAYLOAD_VERSION: u8 = 1; @@ -215,7 +212,6 @@ pub(crate) struct TableDdlEvent { event_type: TableDdlEventType, locators: Vec, payload: Option, - event_context: Option, } impl TableDdlEvent { @@ -224,7 +220,6 @@ impl TableDdlEvent { locator: TableDdlLocator, create_if_not_exists: bool, engine: &str, - event_context: EventContext, ) -> Self { Self::submitted( TableDdlEventType::CreateTable, @@ -234,7 +229,6 @@ impl TableDdlEvent { create_if_not_exists, engine: engine.to_string(), }), - Some(event_context), ) } @@ -242,7 +236,6 @@ impl TableDdlEvent { pub(crate) fn create_logical_tables_submitted( locators: impl IntoIterator, table_count: usize, - event_context: EventContext, ) -> Self { Self::submitted( TableDdlEventType::CreateLogicalTables, @@ -251,7 +244,6 @@ impl TableDdlEvent { version: TABLE_DDL_PAYLOAD_VERSION, table_count, }), - Some(event_context), ) } @@ -259,7 +251,6 @@ impl TableDdlEvent { pub(crate) fn alter_table_submitted( locator: TableDdlLocator, kind: Option<&'static str>, - event_context: EventContext, ) -> Self { Self::submitted( TableDdlEventType::AlterTable, @@ -268,7 +259,6 @@ impl TableDdlEvent { version: TABLE_DDL_PAYLOAD_VERSION, kind, }), - Some(event_context), ) } @@ -277,7 +267,6 @@ impl TableDdlEvent { locators: impl IntoIterator, table_count: usize, kinds: impl IntoIterator, - event_context: EventContext, ) -> Self { let kinds = kinds .into_iter() @@ -292,16 +281,11 @@ impl TableDdlEvent { table_count, kinds, }), - Some(event_context), ) } /// Builds the bounded event emitted when dropping a table is submitted. - pub(crate) fn drop_table_submitted( - locator: TableDdlLocator, - drop_if_exists: bool, - event_context: EventContext, - ) -> Self { + pub(crate) fn drop_table_submitted(locator: TableDdlLocator, drop_if_exists: bool) -> Self { Self::submitted( TableDdlEventType::DropTable, [locator], @@ -309,39 +293,30 @@ impl TableDdlEvent { version: TABLE_DDL_PAYLOAD_VERSION, drop_if_exists, }), - Some(event_context), ) } /// Builds the bounded event emitted when restoring a dropped table is submitted. #[cfg(feature = "enterprise")] - pub(crate) fn undrop_table_submitted( - locator: TableDdlLocator, - event_context: EventContext, - ) -> Self { + pub(crate) fn undrop_table_submitted(locator: TableDdlLocator) -> Self { Self::submitted( TableDdlEventType::UndropTable, [locator], TableDdlPayload::UndropTable(UndropTablePayload { version: TABLE_DDL_PAYLOAD_VERSION, }), - Some(event_context), ) } /// Builds the bounded event emitted when purging a dropped table is submitted. #[cfg(feature = "enterprise")] - pub(crate) fn purge_dropped_table_submitted( - locator: TableDdlLocator, - event_context: EventContext, - ) -> Self { + pub(crate) fn purge_dropped_table_submitted(locator: TableDdlLocator) -> Self { Self::submitted( TableDdlEventType::PurgeDroppedTable, [locator], TableDdlPayload::PurgeDroppedTable(PurgeDroppedTablePayload { version: TABLE_DDL_PAYLOAD_VERSION, }), - Some(event_context), ) } @@ -349,7 +324,6 @@ impl TableDdlEvent { pub(crate) fn truncate_table_submitted( locator: TableDdlLocator, time_range_count: usize, - event_context: EventContext, ) -> Self { Self::submitted( TableDdlEventType::TruncateTable, @@ -358,7 +332,6 @@ impl TableDdlEvent { version: TABLE_DDL_PAYLOAD_VERSION, time_range_count, }), - Some(event_context), ) } @@ -371,7 +344,6 @@ impl TableDdlEvent { event_type, locators: locators.into_iter().collect(), payload: None, - event_context: None, } } @@ -394,13 +366,11 @@ impl TableDdlEvent { event_type: TableDdlEventType, locators: impl IntoIterator, payload: TableDdlPayload, - event_context: Option, ) -> Self { Self { event_type, locators: locators.into_iter().collect(), payload: Some(payload), - event_context, } } @@ -413,7 +383,7 @@ impl TableDdlEvent { ]) } - fn locator_row(&self, locator: &TableDdlLocator) -> Result { + fn locator_row(&self, locator: &TableDdlLocator) -> Row { let mut values = vec![ nullable_string(locator.catalog_name.as_deref()), nullable_string(locator.schema_name.as_deref()), @@ -423,14 +393,7 @@ impl TableDdlEvent { if self.event_type.has_physical_table_id() { values.push(nullable_table_id(locator.physical_table_id)); } - let event_context = self - .event_context - .as_ref() - .map(serde_json::to_value) - .transpose() - .context(SerializeEventSnafu)?; - values.push(nullable_json(event_context.as_ref())); - Ok(Row { values }) + Row { values } } } @@ -451,15 +414,15 @@ impl Event for TableDdlEvent { if self.event_type.has_physical_table_id() { schema.push(PHYSICAL_TABLE_ID_COLUMN.column_schema()); } - schema.push(EVENT_CONTEXT_COLUMN.column_schema()); schema } fn extra_rows(&self) -> Result> { - self.locators + Ok(self + .locators .iter() .map(|locator| self.locator_row(locator)) - .collect() + .collect()) } fn as_any(&self) -> &dyn Any { diff --git a/src/common/meta/src/ddl/event/view.rs b/src/common/meta/src/ddl/event/view.rs index c1d5846fcf..cdce595408 100644 --- a/src/common/meta/src/ddl/event/view.rs +++ b/src/common/meta/src/ddl/event/view.rs @@ -19,14 +19,12 @@ use api::v1::{ColumnSchema, Row}; use common_event_recorder::Event; use common_event_recorder::error::{Result, SerializeEventSnafu}; use common_event_recorder::event_table::{ - CATALOG_NAME_COLUMN, EVENT_CONTEXT_COLUMN, SCHEMA_NAME_COLUMN, VIEW_ID_COLUMN, - VIEW_NAME_COLUMN, column_schemas, nullable_json, nullable_string, nullable_value, + CATALOG_NAME_COLUMN, SCHEMA_NAME_COLUMN, VIEW_ID_COLUMN, VIEW_NAME_COLUMN, column_schemas, + nullable_string, nullable_value, }; use serde::Serialize; use snafu::ResultExt; -use crate::rpc::ddl::EventContext; - pub(crate) const CREATE_VIEW_EVENT_TYPE: &str = "create_view"; pub(crate) const DROP_VIEW_EVENT_TYPE: &str = "drop_view"; @@ -64,7 +62,6 @@ pub(crate) struct ViewDdlEvent { view_name: Option, view_id: Option, payload: Option, - event_context: Option, } #[derive(Debug, Serialize)] @@ -75,12 +72,12 @@ enum ViewDdlPayload { } impl ViewDdlEvent { + /// Builds the bounded event emitted when creating a View is submitted. pub(crate) fn create_submitted( catalog_name: &str, schema_name: &str, view_name: &str, intent: CreateViewEventIntent, - event_context: EventContext, ) -> Self { Self::submitted( CREATE_VIEW_EVENT_TYPE, @@ -95,17 +92,16 @@ impl ViewDdlEvent { referenced_table_count: intent.referenced_table_count, column_count: intent.column_count, }), - event_context, ) } + /// Builds the bounded event emitted when dropping a View is submitted. pub(crate) fn drop_submitted( catalog_name: &str, schema_name: &str, view_name: &str, view_id: u32, drop_if_exists: bool, - event_context: EventContext, ) -> Self { Self::submitted( DROP_VIEW_EVENT_TYPE, @@ -117,7 +113,6 @@ impl ViewDdlEvent { version: PAYLOAD_VERSION, drop_if_exists, }), - event_context, ) } @@ -162,7 +157,6 @@ impl ViewDdlEvent { view_name: &str, view_id: Option, payload: ViewDdlPayload, - event_context: EventContext, ) -> Self { Self { event_type, @@ -171,7 +165,6 @@ impl ViewDdlEvent { view_name: Some(view_name.to_string()), view_id, payload: Some(payload), - event_context: Some(event_context), } } @@ -188,7 +181,6 @@ impl ViewDdlEvent { view_name: Some(view_name.to_string()), view_id: None, payload: None, - event_context: None, } } @@ -206,7 +198,6 @@ impl ViewDdlEvent { view_name: Some(view_name.to_string()), view_id: Some(view_id), payload: None, - event_context: None, } } } @@ -229,24 +220,16 @@ impl Event for ViewDdlEvent { &SCHEMA_NAME_COLUMN, &VIEW_NAME_COLUMN, &VIEW_ID_COLUMN, - &EVENT_CONTEXT_COLUMN, ]) } fn extra_rows(&self) -> Result> { - let event_context = self - .event_context - .as_ref() - .map(serde_json::to_value) - .transpose() - .context(SerializeEventSnafu)?; Ok(vec![Row { values: vec![ nullable_string(self.catalog_name.as_deref()), nullable_string(self.schema_name.as_deref()), nullable_string(self.view_name.as_deref()), nullable_value(self.view_id.map(ValueData::U32Value)), - nullable_json(event_context.as_ref()), ], }]) } diff --git a/src/common/meta/src/ddl/purge_dropped_table.rs b/src/common/meta/src/ddl/purge_dropped_table.rs index c81aa043dc..cdafe40de8 100644 --- a/src/common/meta/src/ddl/purge_dropped_table.rs +++ b/src/common/meta/src/ddl/purge_dropped_table.rs @@ -14,7 +14,7 @@ use std::collections::HashMap; use async_trait::async_trait; use common_procedure::error::{FromJsonSnafu, ToJsonSnafu}; use common_procedure::{ - Context as ProcedureContext, EventRuntimeContext, EventTrigger, LockKey, Procedure, + Context as ProcedureContext, EventContext, EventTrigger, LockKey, Procedure, Result as ProcedureResult, Status, }; use common_telemetry::info; @@ -38,7 +38,7 @@ use crate::error::{self, Result}; use crate::key::DroppedTableMetadata; use crate::key::table_route::TableRouteValue; use crate::lock_key::TableLock; -use crate::rpc::ddl::{EventContext, PurgeDroppedTableTask}; +use crate::rpc::ddl::PurgeDroppedTableTask; use crate::rpc::router::RegionRoute; pub struct PurgeDroppedTableProcedure { @@ -50,25 +50,17 @@ impl PurgeDroppedTableProcedure { pub const TYPE_NAME: &'static str = "metasrv-procedure::PurgeDroppedTable"; pub const EXPIRED_TYPE_NAME: &'static str = "metasrv-procedure::PurgeExpiredDroppedTable"; - pub fn new( - task: PurgeDroppedTableTask, - context: DdlContext, - event_context: EventContext, - ) -> Self { + pub fn new(task: PurgeDroppedTableTask, context: DdlContext) -> Self { Self { context, - data: PurgeDroppedTableData::new(task, event_context), + data: PurgeDroppedTableData::new(task), } } - pub fn new_if_expired( - task: PurgeDroppedTableTask, - context: DdlContext, - event_context: EventContext, - ) -> Self { + pub fn new_if_expired(task: PurgeDroppedTableTask, context: DdlContext) -> Self { Self { context, - data: PurgeDroppedTableData::new_if_expired(task, event_context), + data: PurgeDroppedTableData::new_if_expired(task), } } @@ -278,10 +270,7 @@ impl Procedure for PurgeDroppedTableProcedure { LockKey::new(vec![TableLock::Write(self.data.task.table_id).into()]) } - fn event( - &self, - ctx: &EventRuntimeContext<'_>, - ) -> Option> { + fn event(&self, ctx: &EventContext<'_>) -> Option> { if !ctx .event_type_filter .allows(TableDdlEventType::PurgeDroppedTable.as_str()) @@ -290,7 +279,7 @@ impl Procedure for PurgeDroppedTableProcedure { } let locator = self.event_locator(); let event = if ctx.trigger == EventTrigger::Submitted { - TableDdlEvent::purge_dropped_table_submitted(locator, self.data.event_context.clone()) + TableDdlEvent::purge_dropped_table_submitted(locator) } else { TableDdlEvent::lifecycle(TableDdlEventType::PurgeDroppedTable, [locator]) }; @@ -316,12 +305,10 @@ pub struct PurgeDroppedTableData { drop_generation_loaded: bool, #[serde(default)] purging_claimed: bool, - #[serde(default)] - event_context: EventContext, } impl PurgeDroppedTableData { - fn new(task: PurgeDroppedTableTask, event_context: EventContext) -> Self { + fn new(task: PurgeDroppedTableTask) -> Self { Self { state: PurgeDroppedTableState::Prepare, task, @@ -334,14 +321,13 @@ impl PurgeDroppedTableData { drop_generation: None, drop_generation_loaded: false, purging_claimed: false, - event_context, } } - fn new_if_expired(task: PurgeDroppedTableTask, event_context: EventContext) -> Self { + fn new_if_expired(task: PurgeDroppedTableTask) -> Self { Self { check_expired: true, - ..Self::new(task, event_context) + ..Self::new(task) } } diff --git a/src/common/meta/src/ddl/test_util.rs b/src/common/meta/src/ddl/test_util.rs index 3686ceb33d..7d8cdd895b 100644 --- a/src/common/meta/src/ddl/test_util.rs +++ b/src/common/meta/src/ddl/test_util.rs @@ -45,7 +45,7 @@ use crate::key::node_address::{NodeAddressKey, NodeAddressValue}; use crate::key::table_route::TableRouteValue; use crate::key::{MetadataKey, MetadataValue}; use crate::peer::Peer; -use crate::rpc::ddl::{CreateTableTask, EventContext}; +use crate::rpc::ddl::CreateTableTask; use crate::rpc::store::PutRequest; pub async fn create_physical_table_metadata( @@ -104,12 +104,7 @@ pub async fn create_logical_table( table_name: &str, ) -> TableId { let tasks = vec![test_create_logical_table_task(table_name)]; - let mut procedure = CreateLogicalTablesProcedure::new( - tasks, - physical_table_id, - EventContext::default(), - ddl_context, - ); + let mut procedure = CreateLogicalTablesProcedure::new(tasks, physical_table_id, ddl_context); let status = procedure.on_prepare().await.unwrap(); assert_matches!( status, diff --git a/src/common/meta/src/ddl/tests/alter_logical_tables.rs b/src/common/meta/src/ddl/tests/alter_logical_tables.rs index 0966219938..351c0fd96e 100644 --- a/src/common/meta/src/ddl/tests/alter_logical_tables.rs +++ b/src/common/meta/src/ddl/tests/alter_logical_tables.rs @@ -43,7 +43,7 @@ use crate::error::Error::{AlterLogicalTablesInvalidArguments, TableNotFound}; use crate::error::Result; use crate::key::table_name::TableNameKey; use crate::key::table_route::{PhysicalTableRouteValue, TableRouteValue}; -use crate::rpc::ddl::{AlterTableTask, EventContext}; +use crate::rpc::ddl::AlterTableTask; use crate::rpc::router::{Region, RegionRoute}; use crate::test_util::{MockDatanodeManager, new_ddl_context}; @@ -162,12 +162,7 @@ async fn test_on_prepare_check_schema() { ), ]; let physical_table_id = 1024u32; - let mut procedure = AlterLogicalTablesProcedure::new( - tasks, - physical_table_id, - EventContext::default(), - ddl_context, - ); + let mut procedure = AlterLogicalTablesProcedure::new(tasks, physical_table_id, ddl_context); let err = procedure.on_prepare().await.unwrap_err(); assert_matches!(err, AlterLogicalTablesInvalidArguments { .. }); } @@ -182,12 +177,7 @@ async fn test_on_prepare_check_alter_kind() { "new_table1", )]; let physical_table_id = 1024u32; - let mut procedure = AlterLogicalTablesProcedure::new( - tasks, - physical_table_id, - EventContext::default(), - ddl_context, - ); + let mut procedure = AlterLogicalTablesProcedure::new(tasks, physical_table_id, ddl_context); let err = procedure.on_prepare().await.unwrap_err(); assert_matches!(err, AlterLogicalTablesInvalidArguments { .. }); } @@ -207,8 +197,7 @@ async fn test_on_prepare_different_physical_table() { make_alter_logical_table_add_column_task(None, "table2", vec!["column2".to_string()]), ]; - let mut procedure = - AlterLogicalTablesProcedure::new(tasks, phy1_id, EventContext::default(), ddl_context); + let mut procedure = AlterLogicalTablesProcedure::new(tasks, phy1_id, ddl_context); let err = procedure.on_prepare().await.unwrap_err(); assert_matches!(err, AlterLogicalTablesInvalidArguments { .. }); } @@ -229,8 +218,7 @@ async fn test_on_prepare_logical_table_not_exists() { make_alter_logical_table_add_column_task(None, "table2", vec!["column2".to_string()]), ]; - let mut procedure = - AlterLogicalTablesProcedure::new(tasks, phy_id, EventContext::default(), ddl_context); + let mut procedure = AlterLogicalTablesProcedure::new(tasks, phy_id, ddl_context); let err = procedure.on_prepare().await.unwrap_err(); assert_matches!(err, TableNotFound { .. }); } @@ -253,8 +241,7 @@ async fn test_on_prepare() { make_alter_logical_table_add_column_task(None, "table3", vec!["column3".to_string()]), ]; - let mut procedure = - AlterLogicalTablesProcedure::new(tasks, phy_id, EventContext::default(), ddl_context); + let mut procedure = AlterLogicalTablesProcedure::new(tasks, phy_id, ddl_context); let result = procedure.on_prepare().await; assert_matches!( result, @@ -290,12 +277,7 @@ async fn test_on_update_metadata() { make_alter_logical_table_add_column_task(None, "table3", vec!["new_col".to_string()]), ]; - let mut procedure = AlterLogicalTablesProcedure::new( - tasks, - phy_id, - EventContext::default(), - ddl_context.clone(), - ); + let mut procedure = AlterLogicalTablesProcedure::new(tasks, phy_id, ddl_context.clone()); let mut status = procedure.on_prepare().await.unwrap(); assert_matches!( status, @@ -308,6 +290,7 @@ async fn test_on_update_metadata() { let ctx = common_procedure::Context { procedure_id: ProcedureId::random(), provider: Arc::new(MockContextProvider::default()), + event_context: None, }; // on_submit_alter_region_requests status = procedure.execute(&ctx).await.unwrap(); @@ -379,12 +362,7 @@ async fn test_on_part_duplicate_alter_request() { make_alter_logical_table_add_column_task(None, "table2", vec!["col_0".to_string()]), ]; - let mut procedure = AlterLogicalTablesProcedure::new( - tasks, - phy_id, - EventContext::default(), - ddl_context.clone(), - ); + let mut procedure = AlterLogicalTablesProcedure::new(tasks, phy_id, ddl_context.clone()); let mut status = procedure.on_prepare().await.unwrap(); assert_matches!( status, @@ -397,6 +375,7 @@ async fn test_on_part_duplicate_alter_request() { let ctx = common_procedure::Context { procedure_id: ProcedureId::random(), provider: Arc::new(MockContextProvider::default()), + event_context: None, }; // on_submit_alter_region_requests status = procedure.execute(&ctx).await.unwrap(); @@ -469,12 +448,7 @@ async fn test_on_part_duplicate_alter_request() { ), ]; - let mut procedure = AlterLogicalTablesProcedure::new( - tasks, - phy_id, - EventContext::default(), - ddl_context.clone(), - ); + let mut procedure = AlterLogicalTablesProcedure::new(tasks, phy_id, ddl_context.clone()); let mut status = procedure.on_prepare().await.unwrap(); assert_matches!( status, @@ -487,6 +461,7 @@ async fn test_on_part_duplicate_alter_request() { let ctx = common_procedure::Context { procedure_id: ProcedureId::random(), provider: Arc::new(MockContextProvider::default()), + event_context: None, }; // on_submit_alter_region_requests status = procedure.execute(&ctx).await.unwrap(); @@ -645,8 +620,7 @@ async fn test_on_submit_alter_region_request() { make_alter_logical_table_add_column_task(None, "table2", vec!["mew_col".to_string()]), ]; - let mut procedure = - AlterLogicalTablesProcedure::new(tasks, phy_id, EventContext::default(), ddl_context); + let mut procedure = AlterLogicalTablesProcedure::new(tasks, phy_id, ddl_context); procedure.on_prepare().await.unwrap(); procedure.on_submit_alter_region_requests().await.unwrap(); let mut results = Vec::new(); diff --git a/src/common/meta/src/ddl/tests/alter_table.rs b/src/common/meta/src/ddl/tests/alter_table.rs index b2943a0ab8..b35666e52e 100644 --- a/src/common/meta/src/ddl/tests/alter_table.rs +++ b/src/common/meta/src/ddl/tests/alter_table.rs @@ -59,7 +59,7 @@ use crate::lock_key::RegionLock; use crate::node_manager::NodeManagerRef; use crate::peer::Peer; use crate::poison_key::table_poison_key; -use crate::rpc::ddl::{AlterTableTask, EventContext}; +use crate::rpc::ddl::AlterTableTask; use crate::rpc::router::{Region, RegionRoute}; use crate::test_util::{MockDatanodeManager, new_ddl_context}; @@ -121,8 +121,7 @@ async fn test_on_prepare_table_exists_err() { .unwrap(); let task = test_rename_alter_table_task("non-exists", "foo"); - let mut procedure = - AlterTableProcedure::new(1024, task, EventContext::default(), ddl_context).unwrap(); + let mut procedure = AlterTableProcedure::new(1024, task, ddl_context).unwrap(); let err = procedure.on_prepare().await.unwrap_err(); assert_matches!(err.status_code(), StatusCode::TableAlreadyExists); } @@ -132,8 +131,7 @@ async fn test_on_prepare_table_not_exists_err() { let node_manager = Arc::new(MockDatanodeManager::new(())); let ddl_context = new_ddl_context(node_manager); let task = test_rename_alter_table_task("non-exists", "foo"); - let mut procedure = - AlterTableProcedure::new(1024, task, EventContext::default(), ddl_context).unwrap(); + let mut procedure = AlterTableProcedure::new(1024, task, ddl_context).unwrap(); let err = procedure.on_prepare().await.unwrap_err(); assert_matches!(err.status_code(), StatusCode::TableNotFound); } @@ -247,13 +245,8 @@ async fn test_on_submit_alter_request() { let alter_table_task = test_alter_table_task(table_name); let procedure_id = ProcedureId::random(); let provider = Arc::new(MockContextProvider::default()); - let mut procedure = AlterTableProcedure::new( - table_id, - alter_table_task, - EventContext::default(), - ddl_context.clone(), - ) - .unwrap(); + let mut procedure = + AlterTableProcedure::new(table_id, alter_table_task, ddl_context.clone()).unwrap(); procedure.on_prepare().await.unwrap(); procedure .submit_alter_region_requests(procedure_id, provider.as_ref()) @@ -306,13 +299,8 @@ async fn test_on_submit_alter_request_without_sync_request() { let alter_table_task = test_alter_table_task(table_name); let procedure_id = ProcedureId::random(); let provider = Arc::new(MockContextProvider::default()); - let mut procedure = AlterTableProcedure::new( - table_id, - alter_table_task, - EventContext::default(), - ddl_context.clone(), - ) - .unwrap(); + let mut procedure = + AlterTableProcedure::new(table_id, alter_table_task, ddl_context.clone()).unwrap(); procedure.on_prepare().await.unwrap(); procedure .submit_alter_region_requests(procedure_id, provider.as_ref()) @@ -369,13 +357,7 @@ async fn test_on_submit_alter_request_with_outdated_request() { }; let procedure_id = ProcedureId::random(); let provider = Arc::new(MockContextProvider::default()); - let mut procedure = AlterTableProcedure::new( - table_id, - alter_table_task, - EventContext::default(), - ddl_context, - ) - .unwrap(); + let mut procedure = AlterTableProcedure::new(table_id, alter_table_task, ddl_context).unwrap(); procedure.on_prepare().await.unwrap(); let err = procedure .submit_alter_region_requests(procedure_id, provider.as_ref()) @@ -404,9 +386,7 @@ async fn test_on_update_metadata_rename() { .unwrap(); let task = test_rename_alter_table_task(table_name, new_table_name); - let mut procedure = - AlterTableProcedure::new(table_id, task, EventContext::default(), ddl_context.clone()) - .unwrap(); + let mut procedure = AlterTableProcedure::new(table_id, task, ddl_context.clone()).unwrap(); procedure.on_prepare().await.unwrap(); procedure.on_update_metadata().await.unwrap(); @@ -486,9 +466,7 @@ async fn test_on_update_metadata_add_columns() { }; let procedure_id = ProcedureId::random(); let provider = Arc::new(MockContextProvider::default()); - let mut procedure = - AlterTableProcedure::new(table_id, task, EventContext::default(), ddl_context.clone()) - .unwrap(); + let mut procedure = AlterTableProcedure::new(table_id, task, ddl_context.clone()).unwrap(); procedure.on_prepare().await.unwrap(); procedure .submit_alter_region_requests(procedure_id, provider.as_ref()) @@ -585,9 +563,7 @@ async fn test_on_update_table_options() { }; let procedure_id = ProcedureId::random(); let provider = Arc::new(MockContextProvider::default()); - let mut procedure = - AlterTableProcedure::new(table_id, task, EventContext::default(), ddl_context.clone()) - .unwrap(); + let mut procedure = AlterTableProcedure::new(table_id, task, ddl_context.clone()).unwrap(); procedure.on_prepare().await.unwrap(); procedure .submit_alter_region_requests(procedure_id, provider.as_ref()) @@ -658,7 +634,6 @@ async fn test_skip_wal_rejects_mixed_table_options() { })), }, }, - EventContext::default(), ddl_context, ) .unwrap(); @@ -687,13 +662,8 @@ async fn test_skip_wal_rejects_logical_table() { }, }; - let mut procedure = AlterTableProcedure::new( - logical_table_id, - task, - EventContext::default(), - ddl_context.clone(), - ) - .unwrap(); + let mut procedure = + AlterTableProcedure::new(logical_table_id, task, ddl_context.clone()).unwrap(); let error = procedure.on_prepare().await.unwrap_err(); assert_matches!(error, Error::Unsupported { .. }); @@ -739,9 +709,7 @@ async fn test_skip_wal_rejects_file_engine_table() { }, }; - let mut procedure = - AlterTableProcedure::new(table_id, task, EventContext::default(), ddl_context.clone()) - .unwrap(); + let mut procedure = AlterTableProcedure::new(table_id, task, ddl_context.clone()).unwrap(); let error = procedure.on_prepare().await.unwrap_err(); assert_matches!(error, Error::Unsupported { .. }); @@ -778,7 +746,6 @@ fn test_skip_wal_holds_region_locks() { let procedure = AlterTableProcedure::new_with_region_locks( table_id, task, - EventContext::default(), region_ids.clone(), context.clone(), ) @@ -830,7 +797,6 @@ async fn test_skip_wal_detects_region_route_change() { let mut procedure = AlterTableProcedure::new_with_region_locks( table_id, alter_task, - EventContext::default(), stale_region_locks, ddl_context.clone(), ) @@ -950,7 +916,6 @@ async fn test_skip_wal_updates_metadata_before_all_replicas() { let mut procedure = AlterTableProcedure::new_with_region_locks( table_id, alter_task, - EventContext::default(), region_locks, ddl_context.clone(), ) @@ -1032,13 +997,8 @@ async fn prepare_alter_table_procedure( }, }; let procedure_id = ProcedureId::random(); - let mut procedure = AlterTableProcedure::new( - table_id, - alter_table_task, - EventContext::default(), - ddl_context.clone(), - ) - .unwrap(); + let mut procedure = + AlterTableProcedure::new(table_id, alter_table_task, ddl_context.clone()).unwrap(); procedure.on_prepare().await.unwrap(); (procedure, procedure_id) } diff --git a/src/common/meta/src/ddl/tests/create_flow.rs b/src/common/meta/src/ddl/tests/create_flow.rs index 280a2633e3..b526856bda 100644 --- a/src/common/meta/src/ddl/tests/create_flow.rs +++ b/src/common/meta/src/ddl/tests/create_flow.rs @@ -37,7 +37,7 @@ use crate::key::flow::flow_info::{ FlowInfoValue, FlowMissedTickPolicy, FlowScheduleConfig, FlowStatus, }; use crate::key::table_route::TableRouteValue; -use crate::rpc::ddl::{CreateFlowTask, EventContext, FlowQueryContext, QueryContext}; +use crate::rpc::ddl::{CreateFlowTask, FlowQueryContext, QueryContext}; use crate::test_util::{MockFlownodeManager, new_ddl_context}; pub(crate) fn test_query_context() -> QueryContext { @@ -120,8 +120,7 @@ async fn test_create_flow_source_table_not_found() { let node_manager = Arc::new(MockFlownodeManager::new(NaiveFlownodeHandler)); let ddl_context = new_ddl_context(node_manager); let query_ctx = test_query_context(); - let mut procedure = - CreateFlowProcedure::new(task, query_ctx, EventContext::default(), ddl_context); + let mut procedure = CreateFlowProcedure::new(task, query_ctx, ddl_context); let err = procedure.on_prepare().await.unwrap_err(); assert_matches!(err, error::Error::Unsupported { .. }); assert!( @@ -144,12 +143,7 @@ async fn test_create_pending_flow_source_table_not_found_with_defer() { let node_manager = Arc::new(MockFlownodeManager::new(NaiveFlownodeHandler)); let ddl_context = new_ddl_context(node_manager); let query_ctx = test_query_context(); - let mut procedure = CreateFlowProcedure::new( - task, - query_ctx, - EventContext::default(), - ddl_context.clone(), - ); + let mut procedure = CreateFlowProcedure::new(task, query_ctx, ddl_context.clone()); let status = procedure.on_prepare().await.unwrap(); assert_matches!(status, Status::Executing { persist: true, .. }); assert_eq!(procedure.data.unresolved_source_table_names.len(), 1); @@ -189,8 +183,7 @@ async fn test_create_pending_flow_source_table_not_found_with_defer_false() { let node_manager = Arc::new(MockFlownodeManager::new(NaiveFlownodeHandler)); let ddl_context = new_ddl_context(node_manager); let query_ctx = test_query_context(); - let mut procedure = - CreateFlowProcedure::new(task, query_ctx, EventContext::default(), ddl_context); + let mut procedure = CreateFlowProcedure::new(task, query_ctx, ddl_context); let err = procedure.on_prepare().await.unwrap_err(); assert_matches!(err, error::Error::Unsupported { .. }); assert!( @@ -240,12 +233,7 @@ async fn test_create_pending_flow_records_partial_source_resolution() { ); enable_defer_on_missing_source(&mut task); let query_ctx = test_query_context(); - let mut procedure = CreateFlowProcedure::new( - task, - query_ctx, - EventContext::default(), - ddl_context.clone(), - ); + let mut procedure = CreateFlowProcedure::new(task, query_ctx, ddl_context.clone()); let status = procedure.on_prepare().await.unwrap(); assert_matches!(status, Status::Executing { persist: true, .. }); assert_eq!(procedure.data.source_table_ids, vec![existing_table_id]); @@ -440,7 +428,6 @@ fn test_resolved_schedule_defaults_in_create_request() { // Construct CreateFlowData and verify CreateRequest carries internal key. let data = CreateFlowData { - event_context: EventContext::default(), state: CreateFlowState::CreateFlows, task, flow_id: Some(1024), @@ -454,7 +441,6 @@ fn test_resolved_schedule_defaults_in_create_request() { }; let data2 = CreateFlowData { - event_context: EventContext::default(), state: CreateFlowState::CreateMetadata, task: data.task.clone(), flow_id: Some(1024), @@ -594,8 +580,7 @@ async fn test_create_flow_rejects_unknown_option_in_meta_task() { let node_manager = Arc::new(MockFlownodeManager::new(NaiveFlownodeHandler)); let ddl_context = new_ddl_context(node_manager); let query_ctx = test_query_context(); - let mut procedure = - CreateFlowProcedure::new(task, query_ctx, EventContext::default(), ddl_context); + let mut procedure = CreateFlowProcedure::new(task, query_ctx, ddl_context); let err = procedure.on_prepare().await.unwrap_err(); assert_matches!(err, error::Error::Unexpected { .. }); @@ -616,7 +601,6 @@ fn test_create_request_strips_defer_on_missing_source_runtime_option() { enable_defer_on_missing_source(&mut task); let data = CreateFlowData { - event_context: EventContext::default(), state: CreateFlowState::CreateFlows, task, flow_id: Some(1024), @@ -666,12 +650,7 @@ pub(crate) async fn create_test_flow( false, ); let query_ctx = test_query_context(); - let mut procedure = CreateFlowProcedure::new( - task.clone(), - query_ctx, - EventContext::default(), - ddl_context.clone(), - ); + let mut procedure = CreateFlowProcedure::new(task.clone(), query_ctx, ddl_context.clone()); let output = execute_procedure_until_done(&mut procedure).await.unwrap(); let flow_id = output.downcast_ref::().unwrap(); @@ -692,12 +671,7 @@ pub(crate) async fn create_test_pending_flow( ); enable_defer_on_missing_source(&mut task); let query_ctx = test_query_context(); - let mut procedure = CreateFlowProcedure::new( - task, - query_ctx, - EventContext::default(), - ddl_context.clone(), - ); + let mut procedure = CreateFlowProcedure::new(task, query_ctx, ddl_context.clone()); let output = execute_procedure_until_done(&mut procedure).await.unwrap(); let flow_id = output.downcast_ref::().unwrap(); @@ -744,12 +718,7 @@ async fn test_create_flow() { true, ); let query_ctx = test_query_context(); - let mut procedure = CreateFlowProcedure::new( - task.clone(), - query_ctx, - EventContext::default(), - ddl_context.clone(), - ); + let mut procedure = CreateFlowProcedure::new(task.clone(), query_ctx, ddl_context.clone()); let output = execute_procedure_until_done(&mut procedure).await.unwrap(); let flow_id = output.downcast_ref::().unwrap(); assert_eq!(*flow_id, 1024); @@ -757,12 +726,7 @@ async fn test_create_flow() { // Creates again let task = test_create_flow_task("my_flow", source_table_names, sink_table_name, false); let query_ctx = test_query_context(); - let mut procedure = CreateFlowProcedure::new( - task.clone(), - query_ctx, - EventContext::default(), - ddl_context, - ); + let mut procedure = CreateFlowProcedure::new(task.clone(), query_ctx, ddl_context); let err = procedure.on_prepare().await.unwrap_err(); assert_matches!(err, error::Error::FlowAlreadyExists { .. }); } @@ -819,12 +783,7 @@ async fn test_replace_pending_flow_with_active_flow_is_unsupported() { ); replace_task.or_replace = true; let query_ctx = test_query_context(); - let mut procedure = CreateFlowProcedure::new( - replace_task, - query_ctx, - EventContext::default(), - ddl_context.clone(), - ); + let mut procedure = CreateFlowProcedure::new(replace_task, query_ctx, ddl_context.clone()); let err = procedure.on_prepare().await.unwrap_err(); assert_matches!(err, error::Error::Unsupported { .. }); assert!( @@ -882,12 +841,7 @@ async fn test_replace_active_flow_with_pending_flow_is_unsupported() { enable_defer_on_missing_source(&mut replace_task); replace_task.or_replace = true; let query_ctx = test_query_context(); - let mut procedure = CreateFlowProcedure::new( - replace_task, - query_ctx, - EventContext::default(), - ddl_context.clone(), - ); + let mut procedure = CreateFlowProcedure::new(replace_task, query_ctx, ddl_context.clone()); let err = procedure.on_prepare().await.unwrap_err(); assert_matches!(err, error::Error::Unsupported { .. }); assert!( @@ -947,12 +901,7 @@ async fn test_replace_pending_flow_with_pending_flow_updates_metadata() { enable_defer_on_missing_source(&mut replace_task); replace_task.or_replace = true; let query_ctx = test_query_context(); - let mut procedure = CreateFlowProcedure::new( - replace_task, - query_ctx, - EventContext::default(), - ddl_context.clone(), - ); + let mut procedure = CreateFlowProcedure::new(replace_task, query_ctx, ddl_context.clone()); let output = execute_procedure_until_done(&mut procedure).await.unwrap(); let replaced_flow_id = *output.downcast_ref::().unwrap(); assert_eq!(replaced_flow_id, original_flow_id); @@ -1004,8 +953,7 @@ async fn test_create_flow_same_source_and_sink_table() { // Try to create a flow with same source and sink table - should fail let task = test_create_flow_task("my_flow", source_table_names, sink_table_name, false); let query_ctx = test_query_context(); - let mut procedure = - CreateFlowProcedure::new(task, query_ctx, EventContext::default(), ddl_context); + let mut procedure = CreateFlowProcedure::new(task, query_ctx, ddl_context); let err = procedure.on_prepare().await.unwrap_err(); assert_matches!(err, error::Error::Unsupported { .. }); @@ -1038,7 +986,7 @@ fn test_create_flow_lock_key_does_not_lock_sink_table_name() { let task = create_test_flow_task_for_serialization(); let query_ctx = test_query_context(); let ddl_context = new_ddl_context(Arc::new(MockFlownodeManager::new(NaiveFlownodeHandler))); - let procedure = CreateFlowProcedure::new(task, query_ctx, EventContext::default(), ddl_context); + let procedure = CreateFlowProcedure::new(task, query_ctx, ddl_context); let lock_keys = procedure.lock_key().get_keys(); @@ -1113,7 +1061,6 @@ fn test_create_flow_data_new_format_serialization() { }; let data = CreateFlowData { - event_context: EventContext::default(), state: CreateFlowState::Prepare, task: create_test_flow_task_for_serialization(), flow_id: None, @@ -1182,7 +1129,7 @@ fn test_create_flow_procedure_strips_scheduled_time_extension() { .extensions .insert("flow.other".to_string(), "kept".to_string()); - let procedure = CreateFlowProcedure::new(task, query_ctx, EventContext::default(), ddl_context); + let procedure = CreateFlowProcedure::new(task, query_ctx, ddl_context); assert!( !procedure @@ -1215,7 +1162,6 @@ fn test_flow_info_conversion_with_flow_context() { }; let data = CreateFlowData { - event_context: EventContext::default(), state: CreateFlowState::CreateMetadata, task: create_test_flow_task_for_serialization(), flow_id: Some(123), @@ -1258,7 +1204,6 @@ fn test_flow_info_conversion_strips_scheduled_time_extension() { }; let data = CreateFlowData { - event_context: EventContext::default(), state: CreateFlowState::CreateMetadata, task: create_test_flow_task_for_serialization(), flow_id: Some(123), diff --git a/src/common/meta/src/ddl/tests/create_logical_tables.rs b/src/common/meta/src/ddl/tests/create_logical_tables.rs index d1cc2420e0..89ec486ee0 100644 --- a/src/common/meta/src/ddl/tests/create_logical_tables.rs +++ b/src/common/meta/src/ddl/tests/create_logical_tables.rs @@ -39,7 +39,6 @@ use crate::ddl::test_util::{ }; use crate::error::{Error, Result}; use crate::key::table_route::{PhysicalTableRouteValue, TableRouteValue}; -use crate::rpc::ddl::EventContext; use crate::rpc::router::{Region, RegionRoute}; use crate::test_util::{MockDatanodeManager, new_ddl_context}; @@ -97,12 +96,7 @@ async fn test_on_prepare_physical_table_not_found() { let ddl_context = new_ddl_context(node_manager); let tasks = vec![test_create_logical_table_task("foo")]; let physical_table_id = 1024u32; - let mut procedure = CreateLogicalTablesProcedure::new( - tasks, - physical_table_id, - EventContext::default(), - ddl_context, - ); + let mut procedure = CreateLogicalTablesProcedure::new(tasks, physical_table_id, ddl_context); let err = procedure.on_prepare().await.unwrap_err(); assert_matches!(err, Error::TableRouteNotFound { .. }); } @@ -132,12 +126,7 @@ async fn test_on_prepare() { // The create logical table procedure. let tasks = vec![test_create_logical_table_task("foo")]; let physical_table_id = table_id; - let mut procedure = CreateLogicalTablesProcedure::new( - tasks, - physical_table_id, - EventContext::default(), - ddl_context, - ); + let mut procedure = CreateLogicalTablesProcedure::new(tasks, physical_table_id, ddl_context); let status = procedure.on_prepare().await.unwrap(); assert_matches!( status, @@ -183,12 +172,8 @@ async fn test_on_prepare_logical_table_exists_err() { .unwrap(); // The create logical table procedure. let physical_table_id = table_id; - let mut procedure = CreateLogicalTablesProcedure::new( - vec![task], - physical_table_id, - EventContext::default(), - ddl_context, - ); + let mut procedure = + CreateLogicalTablesProcedure::new(vec![task], physical_table_id, ddl_context); let err = procedure.on_prepare().await.unwrap_err(); assert_matches!(err, Error::TableAlreadyExists { .. }); assert_eq!(err.status_code(), StatusCode::TableAlreadyExists); @@ -231,12 +216,8 @@ async fn test_on_prepare_with_create_if_table_exists() { let physical_table_id = table_id; // Sets `create_if_not_exists` task.create_table.create_if_not_exists = true; - let mut procedure = CreateLogicalTablesProcedure::new( - vec![task], - physical_table_id, - EventContext::default(), - ddl_context, - ); + let mut procedure = + CreateLogicalTablesProcedure::new(vec![task], physical_table_id, ddl_context); let status = procedure.on_prepare().await.unwrap(); let output = status.downcast_output_ref::>().unwrap(); assert_eq!(*output, vec![8192]); @@ -283,7 +264,6 @@ async fn test_on_prepare_part_logical_tables_exist() { let mut procedure = CreateLogicalTablesProcedure::new( vec![task, non_exist_task], physical_table_id, - EventContext::default(), ddl_context, ); let status = procedure.on_prepare().await.unwrap(); @@ -331,7 +311,6 @@ async fn test_on_create_metadata() { let mut procedure = CreateLogicalTablesProcedure::new( vec![task, yet_another_task], physical_table_id, - EventContext::default(), ddl_context.clone(), ); let status = procedure.on_prepare().await.unwrap(); @@ -345,6 +324,7 @@ async fn test_on_create_metadata() { let ctx = ProcedureContext { procedure_id: ProcedureId::random(), provider: Arc::new(MockContextProvider::default()), + event_context: None, }; procedure.execute(&ctx).await.unwrap(); // Triggers procedure to create table metadata @@ -425,7 +405,6 @@ async fn test_on_create_metadata_part_logical_tables_exist() { let mut procedure = CreateLogicalTablesProcedure::new( vec![task, non_exist_task], physical_table_id, - EventContext::default(), ddl_context.clone(), ); let status = procedure.on_prepare().await.unwrap(); @@ -439,6 +418,7 @@ async fn test_on_create_metadata_part_logical_tables_exist() { let ctx = ProcedureContext { procedure_id: ProcedureId::random(), provider: Arc::new(MockContextProvider::default()), + event_context: None, }; procedure.execute(&ctx).await.unwrap(); // Triggers procedure to create table metadata @@ -498,7 +478,6 @@ async fn test_on_create_metadata_err() { let mut procedure = CreateLogicalTablesProcedure::new( vec![task.clone(), yet_another_task], physical_table_id, - EventContext::default(), ddl_context.clone(), ); let status = procedure.on_prepare().await.unwrap(); @@ -512,6 +491,7 @@ async fn test_on_create_metadata_err() { let ctx = ProcedureContext { procedure_id: ProcedureId::random(), provider: Arc::new(MockContextProvider::default()), + event_context: None, }; procedure.execute(&ctx).await.unwrap(); // Creates logical table metadata(different with the task) @@ -559,7 +539,6 @@ async fn test_on_submit_create_request() { let mut procedure = CreateLogicalTablesProcedure::new( vec![task, yet_another_task], physical_table_id, - EventContext::default(), ddl_context, ); procedure.on_prepare().await.unwrap(); diff --git a/src/common/meta/src/ddl/tests/create_table.rs b/src/common/meta/src/ddl/tests/create_table.rs index 2eee317dc1..c97e61926c 100644 --- a/src/common/meta/src/ddl/tests/create_table.rs +++ b/src/common/meta/src/ddl/tests/create_table.rs @@ -48,7 +48,7 @@ use crate::ddl::test_util::{assert_column_name, get_raw_table_info, put_datanode use crate::error::{Error, Result}; use crate::key::table_route::TableRouteValue; use crate::kv_backend::memory::MemoryKvBackend; -use crate::rpc::ddl::{CreateTableTask, EventContext, QueryContext}; +use crate::rpc::ddl::CreateTableTask; use crate::test_util::{MockDatanodeManager, new_ddl_context, new_ddl_context_with_kv_backend}; fn create_request_handler(_peer: Peer, request: RegionRequest) -> Result { @@ -189,13 +189,7 @@ async fn test_on_prepare_table_exists_err() { ) .await .unwrap(); - let mut procedure = CreateTableProcedure::new( - task, - QueryContext::default(), - EventContext::default(), - ddl_context, - ) - .unwrap(); + let mut procedure = CreateTableProcedure::new(task, ddl_context).unwrap(); let err = procedure.on_prepare().await.unwrap_err(); assert_matches!(err, Error::TableAlreadyExists { .. }); assert_eq!(err.status_code(), StatusCode::TableAlreadyExists); @@ -218,13 +212,7 @@ async fn test_on_prepare_with_create_if_table_exists() { ) .await .unwrap(); - let mut procedure = CreateTableProcedure::new( - task, - QueryContext::default(), - EventContext::default(), - ddl_context, - ) - .unwrap(); + let mut procedure = CreateTableProcedure::new(task, ddl_context).unwrap(); let status = procedure.on_prepare().await.unwrap(); assert_matches!(status, Status::Done { output: Some(..) }); let table_id = *status.downcast_output_ref::().unwrap(); @@ -237,13 +225,7 @@ async fn test_on_prepare_without_create_if_table_exists() { let ddl_context = new_ddl_context(node_manager); let mut task = test_create_table_task("foo"); task.create_table.create_if_not_exists = true; - let mut procedure = CreateTableProcedure::new( - task, - QueryContext::default(), - EventContext::default(), - ddl_context, - ) - .unwrap(); + let mut procedure = CreateTableProcedure::new(task, ddl_context).unwrap(); let status = procedure.on_prepare().await.unwrap(); assert_matches!( status, @@ -262,17 +244,12 @@ async fn test_on_datanode_create_regions_should_retry() { let ddl_context = new_ddl_context(node_manager); let task = test_create_table_task("foo"); assert!(!task.create_table.create_if_not_exists); - let mut procedure = CreateTableProcedure::new( - task, - QueryContext::default(), - EventContext::default(), - ddl_context, - ) - .unwrap(); + let mut procedure = CreateTableProcedure::new(task, ddl_context).unwrap(); procedure.on_prepare().await.unwrap(); let ctx = ProcedureContext { procedure_id: ProcedureId::random(), provider: Arc::new(MockContextProvider::default()), + event_context: None, }; let error = procedure.execute(&ctx).await.unwrap_err(); assert!(error.is_retry_later()); @@ -285,17 +262,12 @@ async fn test_on_datanode_create_regions_should_not_retry() { let ddl_context = new_ddl_context(node_manager); let task = test_create_table_task("foo"); assert!(!task.create_table.create_if_not_exists); - let mut procedure = CreateTableProcedure::new( - task, - QueryContext::default(), - EventContext::default(), - ddl_context, - ) - .unwrap(); + let mut procedure = CreateTableProcedure::new(task, ddl_context).unwrap(); procedure.on_prepare().await.unwrap(); let ctx = ProcedureContext { procedure_id: ProcedureId::random(), provider: Arc::new(MockContextProvider::default()), + event_context: None, }; let error = procedure.execute(&ctx).await.unwrap_err(); assert!(!error.is_retry_later()); @@ -308,13 +280,7 @@ async fn test_on_datanode_create_regions_remaps_addresses_when_retrying() { let node_manager = Arc::new(MockDatanodeManager::new(datanode_handler)); let ddl_context = new_ddl_context(node_manager); let task = test_create_table_task("foo"); - let mut procedure = CreateTableProcedure::new( - task, - QueryContext::default(), - EventContext::default(), - ddl_context.clone(), - ) - .unwrap(); + let mut procedure = CreateTableProcedure::new(task, ddl_context.clone()).unwrap(); procedure.on_prepare().await.unwrap(); let table_route = procedure.data.table_route.as_mut().unwrap(); @@ -335,17 +301,12 @@ async fn test_on_create_metadata_error() { let ddl_context = new_ddl_context(node_manager); let task = test_create_table_task("foo"); assert!(!task.create_table.create_if_not_exists); - let mut procedure = CreateTableProcedure::new( - task.clone(), - QueryContext::default(), - EventContext::default(), - ddl_context.clone(), - ) - .unwrap(); + let mut procedure = CreateTableProcedure::new(task.clone(), ddl_context.clone()).unwrap(); procedure.on_prepare().await.unwrap(); let ctx = ProcedureContext { procedure_id: ProcedureId::random(), provider: Arc::new(MockContextProvider::default()), + event_context: None, }; procedure.execute(&ctx).await.unwrap(); let mut task = task; @@ -374,17 +335,12 @@ async fn test_on_create_metadata() { let ddl_context = new_ddl_context(node_manager); let task = test_create_table_task("foo"); assert!(!task.create_table.create_if_not_exists); - let mut procedure = CreateTableProcedure::new( - task, - QueryContext::default(), - EventContext::default(), - ddl_context.clone(), - ) - .unwrap(); + let mut procedure = CreateTableProcedure::new(task, ddl_context.clone()).unwrap(); procedure.on_prepare().await.unwrap(); let ctx = ProcedureContext { procedure_id: ProcedureId::random(), provider: Arc::new(MockContextProvider::default()), + event_context: None, }; procedure.execute(&ctx).await.unwrap(); // Triggers procedure to create table metadata @@ -408,13 +364,7 @@ async fn test_memory_region_keeper_guard_dropped_on_procedure_done() { let ddl_context = new_ddl_context_with_kv_backend(node_manager, kv_backend); let task = test_create_table_task("foo"); - let mut procedure = CreateTableProcedure::new( - task, - QueryContext::default(), - EventContext::default(), - ddl_context.clone(), - ) - .unwrap(); + let mut procedure = CreateTableProcedure::new(task, ddl_context.clone()).unwrap(); execute_procedure_until(&mut procedure, |p| { p.data.state == CreateTableState::CreateMetadata diff --git a/src/common/meta/src/ddl/tests/create_view.rs b/src/common/meta/src/ddl/tests/create_view.rs index 618249fd1c..b73977c79f 100644 --- a/src/common/meta/src/ddl/tests/create_view.rs +++ b/src/common/meta/src/ddl/tests/create_view.rs @@ -29,7 +29,7 @@ use crate::ddl::create_view::CreateViewProcedure; use crate::ddl::test_util::datanode_handler::NaiveDatanodeHandler; use crate::ddl::tests::create_table::test_create_table_task; use crate::error::Error; -use crate::rpc::ddl::{CreateViewTask, EventContext, QueryContext}; +use crate::rpc::ddl::CreateViewTask; use crate::test_util::{MockDatanodeManager, new_ddl_context}; pub(crate) fn test_table_names() -> HashSet { @@ -112,7 +112,7 @@ async fn test_on_prepare_view_exists_err() { ) .await .unwrap(); - let mut procedure = CreateViewProcedure::new(task, EventContext::default(), ddl_context); + let mut procedure = CreateViewProcedure::new(task, ddl_context); let err = procedure.on_prepare().await.unwrap_err(); assert_matches!(err, Error::ViewAlreadyExists { .. }); assert_eq!(err.status_code(), StatusCode::TableAlreadyExists); @@ -138,7 +138,7 @@ async fn test_on_prepare_with_create_if_view_exists() { ) .await .unwrap(); - let mut procedure = CreateViewProcedure::new(task, EventContext::default(), ddl_context); + let mut procedure = CreateViewProcedure::new(task, ddl_context); let status = procedure.on_prepare().await.unwrap(); assert_matches!(status, Status::Done { output: Some(..) }); let table_id = *status.downcast_output_ref::().unwrap(); @@ -151,7 +151,7 @@ async fn test_on_prepare_without_create_if_table_exists() { let ddl_context = new_ddl_context(node_manager); let mut task = test_create_view_task("foo"); task.create_view.create_if_not_exists = true; - let mut procedure = CreateViewProcedure::new(task, EventContext::default(), ddl_context); + let mut procedure = CreateViewProcedure::new(task, ddl_context); let status = procedure.on_prepare().await.unwrap(); assert_matches!( status, @@ -170,11 +170,12 @@ async fn test_on_create_metadata() { let ddl_context = new_ddl_context(node_manager); let task = test_create_view_task("foo"); assert!(!task.create_view.create_if_not_exists); - let mut procedure = CreateViewProcedure::new(task, EventContext::default(), ddl_context); + let mut procedure = CreateViewProcedure::new(task, ddl_context); procedure.on_prepare().await.unwrap(); let ctx = ProcedureContext { procedure_id: ProcedureId::random(), provider: Arc::new(MockContextProvider::default()), + event_context: None, }; // Triggers procedure to create view metadata let status = procedure.execute(&ctx).await.unwrap(); @@ -189,12 +190,12 @@ async fn test_replace_view_metadata() { let ddl_context = new_ddl_context(node_manager.clone()); let task = test_create_view_task("foo"); assert!(!task.create_view.create_if_not_exists); - let mut procedure = - CreateViewProcedure::new(task.clone(), EventContext::default(), ddl_context.clone()); + let mut procedure = CreateViewProcedure::new(task.clone(), ddl_context.clone()); procedure.on_prepare().await.unwrap(); let ctx = ProcedureContext { procedure_id: ProcedureId::random(), provider: Arc::new(MockContextProvider::default()), + event_context: None, }; // Triggers procedure to create view metadata let status = procedure.execute(&ctx).await.unwrap(); @@ -215,8 +216,7 @@ async fn test_replace_view_metadata() { let mut task = test_create_view_task("foo"); // The view already exists, prepare should fail { - let mut procedure = - CreateViewProcedure::new(task.clone(), EventContext::default(), ddl_context.clone()); + let mut procedure = CreateViewProcedure::new(task.clone(), ddl_context.clone()); let err = procedure.on_prepare().await.unwrap_err(); assert_matches!(err, Error::ViewAlreadyExists { .. }); assert_eq!(err.status_code(), StatusCode::TableAlreadyExists); @@ -227,12 +227,12 @@ async fn test_replace_view_metadata() { task.create_view.logical_plan = vec![4, 5, 6]; task.create_view.definition = "new_definition".to_string(); - let mut procedure = - CreateViewProcedure::new(task, EventContext::default(), ddl_context.clone()); + let mut procedure = CreateViewProcedure::new(task, ddl_context.clone()); procedure.on_prepare().await.unwrap(); let ctx = ProcedureContext { procedure_id: ProcedureId::random(), provider: Arc::new(MockContextProvider::default()), + event_context: None, }; // Triggers procedure to replace view metadata, but the view_id is unchanged. let status = procedure.execute(&ctx).await.unwrap(); @@ -262,17 +262,12 @@ async fn test_replace_table() { { // Create a `foo` table. let task = test_create_table_task("foo"); - let mut procedure = CreateTableProcedure::new( - task, - QueryContext::default(), - EventContext::default(), - ddl_context.clone(), - ) - .unwrap(); + let mut procedure = CreateTableProcedure::new(task, ddl_context.clone()).unwrap(); procedure.on_prepare().await.unwrap(); let ctx = ProcedureContext { procedure_id: ProcedureId::random(), provider: Arc::new(MockContextProvider::default()), + event_context: None, }; procedure.execute(&ctx).await.unwrap(); procedure.execute(&ctx).await.unwrap(); @@ -281,8 +276,7 @@ async fn test_replace_table() { // Try to replace a view named `foo` too. let mut task = test_create_view_task("foo"); task.create_view.or_replace = true; - let mut procedure = - CreateViewProcedure::new(task.clone(), EventContext::default(), ddl_context.clone()); + let mut procedure = CreateViewProcedure::new(task.clone(), ddl_context.clone()); let err = procedure.on_prepare().await.unwrap_err(); assert_matches!(err, Error::TableAlreadyExists { .. }); assert_eq!(err.status_code(), StatusCode::TableAlreadyExists); diff --git a/src/common/meta/src/ddl/tests/drop_database.rs b/src/common/meta/src/ddl/tests/drop_database.rs index 955625e4bb..05a70962bf 100644 --- a/src/common/meta/src/ddl/tests/drop_database.rs +++ b/src/common/meta/src/ddl/tests/drop_database.rs @@ -26,7 +26,6 @@ use crate::ddl::drop_database::executor::DropDatabaseExecutor; use crate::ddl::test_util::datanode_handler::{NaiveDatanodeHandler, RetryErrorDatanodeHandler}; use crate::ddl::test_util::{create_logical_table, create_physical_table}; use crate::key::schema_name::SchemaNameKey; -use crate::rpc::ddl::EventContext; use crate::test_util::{MockDatanodeManager, new_ddl_context}; #[tokio::test] @@ -55,13 +54,13 @@ async fn test_drop_database_with_logical_tables() { DEFAULT_CATALOG_NAME.to_string(), DEFAULT_SCHEMA_NAME.to_string(), false, - EventContext::default(), ddl_context.clone(), ); let ctx = ProcedureContext { procedure_id: ProcedureId::random(), provider: Arc::new(MockContextProvider::default()), + event_context: None, }; while !procedure.execute(&ctx).await.unwrap().is_done() { @@ -104,13 +103,13 @@ async fn test_drop_database_retryable_error() { DEFAULT_CATALOG_NAME.to_string(), DEFAULT_SCHEMA_NAME.to_string(), false, - EventContext::default(), ddl_context.clone(), ); let ctx = ProcedureContext { procedure_id: ProcedureId::random(), provider: Arc::new(MockContextProvider::default()), + event_context: None, }; loop { @@ -149,7 +148,6 @@ async fn test_drop_database_recover() { DEFAULT_CATALOG_NAME.to_string(), DEFAULT_SCHEMA_NAME.to_string(), false, - EventContext::default(), ddl_context.clone(), ); let num_operating_regions = 1; diff --git a/src/common/meta/src/ddl/tests/drop_flow.rs b/src/common/meta/src/ddl/tests/drop_flow.rs index 0976aa572f..6d36fc93ba 100644 --- a/src/common/meta/src/ddl/tests/drop_flow.rs +++ b/src/common/meta/src/ddl/tests/drop_flow.rs @@ -26,7 +26,7 @@ use crate::ddl::test_util::flownode_handler::NaiveFlownodeHandler; use crate::ddl::tests::create_flow::{create_test_flow, create_test_pending_flow}; use crate::error; use crate::key::table_route::TableRouteValue; -use crate::rpc::ddl::{DropFlowTask, EventContext}; +use crate::rpc::ddl::DropFlowTask; use crate::test_util::{MockFlownodeManager, new_ddl_context}; pub(crate) fn test_drop_flow_task( @@ -48,7 +48,7 @@ async fn test_drop_flow_not_found() { let node_manager = Arc::new(MockFlownodeManager::new(NaiveFlownodeHandler)); let ddl_context = new_ddl_context(node_manager); let task = test_drop_flow_task("my_flow", flow_id, false); - let mut procedure = DropFlowProcedure::new(task, EventContext::default(), ddl_context); + let mut procedure = DropFlowProcedure::new(task, ddl_context); let err = procedure.on_prepare().await.unwrap_err(); assert_matches!(err, error::Error::FlowNotFound { .. }); } @@ -81,17 +81,17 @@ async fn test_drop_flow() { create_test_flow(&ddl_context, "my_flow", source_table_names, sink_table_name).await; // Drops the flows let task = test_drop_flow_task("my_flow", flow_id, false); - let mut procedure = DropFlowProcedure::new(task, EventContext::default(), ddl_context.clone()); + let mut procedure = DropFlowProcedure::new(task, ddl_context.clone()); execute_procedure_until_done(&mut procedure).await; // Drops if not exists let task = test_drop_flow_task("my_flow", flow_id, true); - let mut procedure = DropFlowProcedure::new(task, EventContext::default(), ddl_context.clone()); + let mut procedure = DropFlowProcedure::new(task, ddl_context.clone()); execute_procedure_until_done(&mut procedure).await; // Drops again let task = test_drop_flow_task("my_flow", flow_id, false); - let mut procedure = DropFlowProcedure::new(task, EventContext::default(), ddl_context); + let mut procedure = DropFlowProcedure::new(task, ddl_context); let err = procedure.on_prepare().await.unwrap_err(); assert_matches!(err, error::Error::FlowNotFound { .. }); } @@ -129,11 +129,11 @@ async fn test_drop_pending_flow_without_routes() { assert!(flow_info.flownode_ids().is_empty()); let task = test_drop_flow_task("drop_pending_flow", flow_id, false); - let mut procedure = DropFlowProcedure::new(task, EventContext::default(), ddl_context.clone()); + let mut procedure = DropFlowProcedure::new(task, ddl_context.clone()); execute_procedure_until_done(&mut procedure).await; let task = test_drop_flow_task("drop_pending_flow", flow_id, false); - let mut procedure = DropFlowProcedure::new(task, EventContext::default(), ddl_context); + let mut procedure = DropFlowProcedure::new(task, ddl_context); let err = procedure.on_prepare().await.unwrap_err(); assert_matches!(err, error::Error::FlowNotFound { .. }); } diff --git a/src/common/meta/src/ddl/tests/drop_table.rs b/src/common/meta/src/ddl/tests/drop_table.rs index 8b16191467..23dd92d16b 100644 --- a/src/common/meta/src/ddl/tests/drop_table.rs +++ b/src/common/meta/src/ddl/tests/drop_table.rs @@ -65,9 +65,9 @@ use crate::key::table_route::TableRouteValue; use crate::kv_backend::KvBackend; use crate::kv_backend::memory::MemoryKvBackend; use crate::peer::Peer; -use crate::rpc::ddl::{DropTableTask, EventContext}; +use crate::rpc::ddl::DropTableTask; #[cfg(feature = "enterprise")] -use crate::rpc::ddl::{PurgeDroppedTableTask, QueryContext, UndropTableTask}; +use crate::rpc::ddl::{PurgeDroppedTableTask, UndropTableTask}; use crate::rpc::router::{Region, RegionRoute}; #[cfg(feature = "enterprise")] use crate::rpc::store::{BatchDeleteRequest, PutRequest}; @@ -128,11 +128,7 @@ async fn undrop_at_restore_metadata( table_id: TableId, ) -> UndropTableProcedure { create_dropped_table(context, table_id, Some(1), Some(i64::MAX)).await; - let mut undrop = UndropTableProcedure::new( - new_undrop_table_task(table_id), - context.clone(), - EventContext::default(), - ); + let mut undrop = UndropTableProcedure::new(new_undrop_table_task(table_id), context.clone()); let procedure_context = new_test_procedure_context(); undrop.execute(&procedure_context).await.unwrap(); undrop.execute(&procedure_context).await.unwrap(); @@ -161,7 +157,6 @@ fn test_old_drop_table_json_defaults_to_hard_drop() { let procedure = DropTableProcedure::new( new_drop_table_task("foo", 1024, false), runtime_context.clone(), - EventContext::default(), ); let mut old_data: serde_json::Value = serde_json::from_str(&procedure.dump().unwrap()).unwrap(); old_data @@ -169,7 +164,6 @@ fn test_old_drop_table_json_defaults_to_hard_drop() { .unwrap() .remove("soft_drop_enabled"); old_data.as_object_mut().unwrap().remove("dropped_at"); - old_data.as_object_mut().unwrap().remove("event_context"); let recovered = DropTableProcedure::from_json(&old_data.to_string(), runtime_context).unwrap(); let recovered_data: serde_json::Value = @@ -177,20 +171,12 @@ fn test_old_drop_table_json_defaults_to_hard_drop() { assert_eq!(recovered_data["soft_drop_enabled"], false); assert_eq!(recovered_data["dropped_at"], serde_json::Value::Null); - assert_eq!( - recovered_data["event_context"], - serde_json::json!({"reason": "unknown"}) - ); } #[test] fn test_drop_table_lock_key_includes_table_name() { let context = new_ddl_context(Arc::new(MockDatanodeManager::new(()))); - let procedure = DropTableProcedure::new( - new_drop_table_task("foo", 1024, false), - context, - EventContext::default(), - ); + let procedure = DropTableProcedure::new(new_drop_table_task("foo", 1024, false), context); let keys = procedure .lock_key() @@ -219,11 +205,7 @@ fn test_new_drop_table_procedure_disables_soft_drop_without_enterprise() { let mut context = new_ddl_context(Arc::new(MockDatanodeManager::new(()))); context.soft_drop_enabled = true; - let procedure = DropTableProcedure::new( - new_drop_table_task("foo", 1024, false), - context, - EventContext::default(), - ); + let procedure = DropTableProcedure::new(new_drop_table_task("foo", 1024, false), context); assert!(!procedure.data.soft_drop_enabled); } @@ -234,11 +216,7 @@ fn test_new_drop_table_procedure_preserves_soft_drop_with_enterprise() { let mut context = new_ddl_context(Arc::new(MockDatanodeManager::new(()))); context.soft_drop_enabled = true; - let procedure = DropTableProcedure::new( - new_drop_table_task("foo", 1024, false), - context, - EventContext::default(), - ); + let procedure = DropTableProcedure::new(new_drop_table_task("foo", 1024, false), context); assert!(procedure.data.soft_drop_enabled); } @@ -273,7 +251,6 @@ async fn test_recovered_soft_drop_preserves_persisted_mode() { let procedure = DropTableProcedure::new( new_drop_table_task("foo", table_id, false), soft_context.clone(), - EventContext::default(), ); let hard_context = new_ddl_context_with_kv_backend(node_manager, kv_backend); @@ -313,13 +290,9 @@ async fn test_legacy_soft_drop_prepare_preserves_persisted_mode() { .unwrap(); let mut persisted: serde_json::Value = serde_json::from_str( - &DropTableProcedure::new( - new_drop_table_task("foo", table_id, false), - context.clone(), - EventContext::default(), - ) - .dump() - .unwrap(), + &DropTableProcedure::new(new_drop_table_task("foo", table_id, false), context.clone()) + .dump() + .unwrap(), ) .unwrap(); persisted["soft_drop_enabled"] = true.into(); @@ -349,11 +322,7 @@ async fn test_disabled_soft_drop_creates_hard_drop_procedure() { context.soft_drop_enabled = false; context.soft_drop_retention = Some(Duration::from_millis(100)); - let procedure = DropTableProcedure::new( - new_drop_table_task("foo", 1024, false), - context, - EventContext::default(), - ); + let procedure = DropTableProcedure::new(new_drop_table_task("foo", 1024, false), context); assert!(!procedure.data.soft_drop_enabled); assert_eq!(None, procedure.data.dropped_at); @@ -387,7 +356,6 @@ async fn test_recovered_hard_drop_ignores_soft_runtime_context() { let procedure = DropTableProcedure::new( new_drop_table_task("foo", table_id, false), hard_context.clone(), - EventContext::default(), ); let mut soft_context = new_ddl_context_with_kv_backend(node_manager, kv_backend); @@ -429,7 +397,7 @@ async fn test_on_prepare_table_not_exists_err() { .unwrap(); let task = new_drop_table_task("bar", table_id, false); - let mut procedure = DropTableProcedure::new(task, ddl_context, EventContext::default()); + let mut procedure = DropTableProcedure::new(task, ddl_context); assert_eq!(procedure.data.dropped_at, None); assert_eq!(procedure.data.retention_expires_at, None); let err = procedure.on_prepare().await.unwrap_err(); @@ -459,7 +427,6 @@ async fn test_soft_drop_prepare_assigns_stable_deadline_and_recovers_it() { let mut procedure = DropTableProcedure::new( new_drop_table_task("foo", table_id, false), ddl_context.clone(), - EventContext::default(), ); procedure.on_prepare().await.unwrap(); @@ -496,7 +463,6 @@ async fn test_soft_drop_prepare_rejects_deadline_overflow_before_metadata_delete let mut procedure = DropTableProcedure::new( new_drop_table_task("foo", table_id, false), ddl_context.clone(), - EventContext::default(), ); let error = procedure.on_prepare().await.unwrap_err(); @@ -536,13 +502,13 @@ async fn test_on_prepare_table() { let task = new_drop_table_task("bar", table_id, true); // Drop if exists - let mut procedure = DropTableProcedure::new(task, ddl_context.clone(), EventContext::default()); + let mut procedure = DropTableProcedure::new(task, ddl_context.clone()); procedure.on_prepare().await.unwrap(); assert!(!procedure.rollback_supported()); let task = new_drop_table_task(table_name, table_id, false); // Drop table - let mut procedure = DropTableProcedure::new(task, ddl_context, EventContext::default()); + let mut procedure = DropTableProcedure::new(task, ddl_context); procedure.on_prepare().await.unwrap(); } @@ -593,7 +559,7 @@ async fn test_on_datanode_drop_regions() { let task = new_drop_table_task(table_name, table_id, false); // Drop table - let mut procedure = DropTableProcedure::new(task, ddl_context, EventContext::default()); + let mut procedure = DropTableProcedure::new(task, ddl_context); procedure.on_prepare().await.unwrap(); procedure.on_datanode_drop_regions(false).await.unwrap(); @@ -662,7 +628,7 @@ async fn test_on_datanode_drop_regions_remaps_addresses_when_retrying() { .unwrap(); let task = new_drop_table_task(table_name, table_id, false); - let mut procedure = DropTableProcedure::new(task, ddl_context.clone(), EventContext::default()); + let mut procedure = DropTableProcedure::new(task, ddl_context.clone()); procedure.on_prepare().await.unwrap(); put_datanode_address(&ddl_context, 1, "new-leader").await; @@ -721,7 +687,7 @@ async fn test_soft_drop_closes_regions_and_keeps_tombstone() { .unwrap(); let task = new_drop_table_task(table_name, table_id, false); - let mut procedure = DropTableProcedure::new(task, ddl_context.clone(), EventContext::default()); + let mut procedure = DropTableProcedure::new(task, ddl_context.clone()); execute_procedure_until_done(&mut procedure).await; @@ -835,7 +801,7 @@ async fn test_soft_drop_keeps_metadata_live_until_regions_close() { .unwrap(); let task = new_drop_table_task(table_name, table_id, false); - let mut procedure = DropTableProcedure::new(task, ddl_context.clone(), EventContext::default()); + let mut procedure = DropTableProcedure::new(task, ddl_context.clone()); let ctx = new_test_procedure_context(); procedure.execute(&ctx).await.unwrap(); put_datanode_address(&ddl_context, 1, "new-leader").await; @@ -922,7 +888,6 @@ async fn test_soft_drop_timestamp_is_stable_across_retry_and_recovery() { let procedure = DropTableProcedure::new( new_drop_table_task("foo", table_id, false), ddl_context.clone(), - EventContext::default(), ); assert_eq!(procedure.data.dropped_at, None); let mut procedure = procedure; @@ -993,7 +958,7 @@ async fn test_hard_drop_keeps_delete_tombstone_flow() { .unwrap(); let task = new_drop_table_task(table_name, table_id, false); - let mut procedure = DropTableProcedure::new(task, ddl_context.clone(), EventContext::default()); + let mut procedure = DropTableProcedure::new(task, ddl_context.clone()); execute_procedure_until(&mut procedure, |p| { p.data.state == DropTableState::DeleteTombstone @@ -1036,20 +1001,13 @@ async fn test_create_table_succeeds_while_tombstone_exists() { .unwrap(); let drop_task = new_drop_table_task(table_name, dropped_table_id, false); - let mut drop_procedure = - DropTableProcedure::new(drop_task, ddl_context.clone(), EventContext::default()); + let mut drop_procedure = DropTableProcedure::new(drop_task, ddl_context.clone()); execute_procedure_until_done(&mut drop_procedure).await; let mut create_task = test_create_table_task(table_name, 1025); create_task.create_table.table_id = None; create_task.table_info.ident.table_id = 0; - let mut create_procedure = CreateTableProcedure::new( - create_task, - QueryContext::default(), - EventContext::default(), - ddl_context.clone(), - ) - .unwrap(); + let mut create_procedure = CreateTableProcedure::new(create_task, ddl_context.clone()).unwrap(); execute_procedure_until_done(&mut create_procedure).await; let live_table = ddl_context @@ -1094,8 +1052,7 @@ async fn test_hard_drop_recreated_table_fails_when_soft_tombstone_exists() { .unwrap(); let drop_task = new_drop_table_task(table_name, original_table_id, false); - let mut drop_procedure = - DropTableProcedure::new(drop_task, ddl_context.clone(), EventContext::default()); + let mut drop_procedure = DropTableProcedure::new(drop_task, ddl_context.clone()); execute_procedure_until_done(&mut drop_procedure).await; ddl_context.soft_drop_enabled = false; @@ -1112,7 +1069,6 @@ async fn test_hard_drop_recreated_table_fails_when_soft_tombstone_exists() { let mut procedure = DropTableProcedure::new( new_drop_table_task(table_name, recreated_table_id, false), ddl_context, - EventContext::default(), ); let err = procedure.on_prepare().await.unwrap_err(); @@ -1142,8 +1098,7 @@ async fn test_hard_drop_recreated_table_ignores_previous_orphan_tombstone() { .unwrap(); let drop_task = new_drop_table_task(table_name, original_table_id, false); - let mut drop_procedure = - DropTableProcedure::new(drop_task, ddl_context.clone(), EventContext::default()); + let mut drop_procedure = DropTableProcedure::new(drop_task, ddl_context.clone()); execute_procedure_until(&mut drop_procedure, |p| { p.data.state == DropTableState::DeleteTombstone }) @@ -1162,7 +1117,6 @@ async fn test_hard_drop_recreated_table_ignores_previous_orphan_tombstone() { let mut procedure = DropTableProcedure::new( new_drop_table_task(table_name, recreated_table_id, false), ddl_context, - EventContext::default(), ); procedure.on_prepare().await.unwrap(); @@ -1202,7 +1156,6 @@ async fn test_undrop_table_restores_metadata_and_reopens_regions() { let mut drop_procedure = DropTableProcedure::new( new_drop_table_task(table_name, table_id, false), ddl_context.clone(), - EventContext::default(), ); execute_procedure_until_done(&mut drop_procedure).await; assert!( @@ -1219,11 +1172,8 @@ async fn test_undrop_table_restores_metadata_and_reopens_regions() { while rx.try_recv().is_ok() {} - let mut procedure = UndropTableProcedure::new( - new_undrop_table_task(table_id), - ddl_context.clone(), - EventContext::default(), - ); + let mut procedure = + UndropTableProcedure::new(new_undrop_table_task(table_id), ddl_context.clone()); execute_procedure_until_done(&mut procedure).await; let live_table = ddl_context @@ -1314,16 +1264,12 @@ async fn test_undrop_table_opens_regions_before_restoring_live_metadata() { let mut drop_procedure = DropTableProcedure::new( new_drop_table_task(table_name, table_id, false), ddl_context.clone(), - EventContext::default(), ); execute_procedure_until_done(&mut drop_procedure).await; while rx.try_recv().is_ok() {} - let mut procedure = UndropTableProcedure::new( - new_undrop_table_task(table_id), - ddl_context.clone(), - EventContext::default(), - ); + let mut procedure = + UndropTableProcedure::new(new_undrop_table_task(table_id), ddl_context.clone()); let ctx = new_test_procedure_context(); procedure.execute(&ctx).await.unwrap(); procedure.execute(&ctx).await.unwrap(); @@ -1387,17 +1333,13 @@ async fn test_undrop_logical_table_skips_datanode_open() { let mut drop_procedure = DropTableProcedure::new( new_drop_table_task(table_name, logical_table_id, false), ddl_context.clone(), - EventContext::default(), ); execute_procedure_until_done(&mut drop_procedure).await; while rx.try_recv().is_ok() {} - let mut procedure = UndropTableProcedure::new( - new_undrop_table_task(logical_table_id), - ddl_context.clone(), - EventContext::default(), - ); + let mut procedure = + UndropTableProcedure::new(new_undrop_table_task(logical_table_id), ddl_context.clone()); execute_procedure_until_done(&mut procedure).await; let live_table = ddl_context @@ -1428,7 +1370,6 @@ async fn test_soft_drop_metric_logical_table_falls_back_to_hard_drop() { let mut procedure = DropTableProcedure::new( new_drop_table_task("foo", logical_table_id, false), ddl_context.clone(), - EventContext::default(), ); procedure.on_prepare().await.unwrap(); assert!(!procedure.data.soft_drop_enabled); @@ -1472,7 +1413,6 @@ async fn test_soft_drop_metric_physical_table_remains_enabled() { let mut procedure = DropTableProcedure::new( new_drop_table_task(table_name, table_id, false), ddl_context.clone(), - EventContext::default(), ); execute_procedure_until_done(&mut procedure).await; @@ -1517,7 +1457,6 @@ async fn test_soft_drop_file_engine_table_falls_back_to_hard_drop() { let mut procedure = DropTableProcedure::new( new_drop_table_task(table_name, table_id, false), ddl_context.clone(), - EventContext::default(), ); procedure.on_prepare().await.unwrap(); assert!(!procedure.data.soft_drop_enabled); @@ -1554,7 +1493,6 @@ async fn test_file_engine_fallback_is_resolved_before_tombstone_conflict() { let mut procedure = DropTableProcedure::new( new_drop_table_task(table_name, recreated_table_id, false), ddl_context, - EventContext::default(), ); procedure.on_prepare().await.unwrap(); @@ -1570,11 +1508,8 @@ async fn test_undrop_metric_logical_table_fails() { let logical_table_id = create_metric_logical_table_tombstone(&ddl_context, physical_table_id, "foo").await; - let mut procedure = UndropTableProcedure::new( - new_undrop_table_task(logical_table_id), - ddl_context, - EventContext::default(), - ); + let mut procedure = + UndropTableProcedure::new(new_undrop_table_task(logical_table_id), ddl_context); let err = procedure.on_prepare().await.unwrap_err(); assert_eq!(err.status_code(), StatusCode::Unsupported); @@ -1592,7 +1527,6 @@ async fn test_purge_metric_logical_table_fails() { let mut procedure = PurgeDroppedTableProcedure::new( new_purge_dropped_table_task(logical_table_id), ddl_context, - EventContext::default(), ); let err = procedure .execute(&new_test_procedure_context()) @@ -1624,7 +1558,6 @@ async fn test_undrop_table_fails_when_live_name_exists() { let mut drop_procedure = DropTableProcedure::new( new_drop_table_task(table_name, dropped_table_id, false), ddl_context.clone(), - EventContext::default(), ); execute_procedure_until_done(&mut drop_procedure).await; ddl_context @@ -1637,11 +1570,8 @@ async fn test_undrop_table_fails_when_live_name_exists() { .await .unwrap(); - let mut procedure = UndropTableProcedure::new( - new_undrop_table_task(dropped_table_id), - ddl_context, - EventContext::default(), - ); + let mut procedure = + UndropTableProcedure::new(new_undrop_table_task(dropped_table_id), ddl_context); let err = procedure.on_prepare().await.unwrap_err(); assert_matches!(err, Error::TableAlreadyExists { .. }); @@ -1669,15 +1599,11 @@ async fn test_undrop_table_fails_when_live_name_is_created_after_prepare() { let mut drop_procedure = DropTableProcedure::new( new_drop_table_task(table_name, dropped_table_id, false), ddl_context.clone(), - EventContext::default(), ); execute_procedure_until_done(&mut drop_procedure).await; - let mut procedure = UndropTableProcedure::new( - new_undrop_table_task(dropped_table_id), - ddl_context.clone(), - EventContext::default(), - ); + let mut procedure = + UndropTableProcedure::new(new_undrop_table_task(dropped_table_id), ddl_context.clone()); procedure.on_prepare().await.unwrap(); ddl_context .table_metadata_manager @@ -1741,17 +1667,13 @@ async fn test_undrop_table_closes_opened_regions_when_restore_metadata_races_wit let mut drop_procedure = DropTableProcedure::new( new_drop_table_task(table_name, dropped_table_id, false), ddl_context.clone(), - EventContext::default(), ); execute_procedure_until_done(&mut drop_procedure).await; while rx.try_recv().is_ok() {} detector_controller.clear().await; - let mut procedure = UndropTableProcedure::new( - new_undrop_table_task(dropped_table_id), - ddl_context.clone(), - EventContext::default(), - ); + let mut procedure = + UndropTableProcedure::new(new_undrop_table_task(dropped_table_id), ddl_context.clone()); let ctx = new_test_procedure_context(); procedure.execute(&ctx).await.unwrap(); procedure.execute(&ctx).await.unwrap(); @@ -1845,7 +1767,6 @@ async fn test_undrop_table_lock_key_includes_original_table_name_before_prepare( let mut drop_procedure = DropTableProcedure::new( new_drop_table_task(table_name, table_id, false), ddl_context.clone(), - EventContext::default(), ); execute_procedure_until_done(&mut drop_procedure).await; @@ -1860,7 +1781,6 @@ async fn test_undrop_table_lock_key_includes_original_table_name_before_prepare( new_undrop_table_task(table_id), ddl_context, Some(original_table_name), - EventContext::default(), ); let keys = procedure @@ -1928,12 +1848,10 @@ async fn test_undrop_rejects_stale_id_after_name_tombstone_is_consumed() { new_undrop_table_task(original_table_id), ddl_context.clone(), Some(original_table_name.clone()), - EventContext::default(), ); let mut hard_drop = DropTableProcedure::new( new_drop_table_task(table_name, recreated_table_id, false), ddl_context.clone(), - EventContext::default(), ); execute_procedure_until_done(&mut hard_drop).await; assert!( @@ -1988,15 +1906,11 @@ async fn test_undrop_table_replayed_restore_metadata_is_idempotent() { let mut drop_procedure = DropTableProcedure::new( new_drop_table_task(table_name, table_id, false), ddl_context.clone(), - EventContext::default(), ); execute_procedure_until_done(&mut drop_procedure).await; - let mut procedure = UndropTableProcedure::new( - new_undrop_table_task(table_id), - ddl_context.clone(), - EventContext::default(), - ); + let mut procedure = + UndropTableProcedure::new(new_undrop_table_task(table_id), ddl_context.clone()); let ctx = new_test_procedure_context(); procedure.execute(&ctx).await.unwrap(); procedure.execute(&ctx).await.unwrap(); @@ -2100,7 +2014,6 @@ async fn test_purge_dropped_table_cleans_regions_offline_and_deletes_tombstone() let mut drop_procedure = DropTableProcedure::new( new_drop_table_task(table_name, table_id, false), ddl_context.clone(), - EventContext::default(), ); execute_procedure_until_done(&mut drop_procedure).await; while rx.try_recv().is_ok() {} @@ -2109,7 +2022,6 @@ async fn test_purge_dropped_table_cleans_regions_offline_and_deletes_tombstone() let mut procedure = PurgeDroppedTableProcedure::new( new_purge_dropped_table_task(table_id), ddl_context.clone(), - EventContext::default(), ); execute_procedure_until_done(&mut procedure).await; @@ -2188,7 +2100,6 @@ async fn test_automatic_purge_rechecks_unexpired_tombstone() { let procedure = PurgeDroppedTableProcedure::new( new_purge_dropped_table_task(table_id), ddl_context.clone(), - EventContext::default(), ); let mut data: serde_json::Value = serde_json::from_str(&procedure.dump().unwrap()).unwrap(); data["check_expired"] = true.into(); @@ -2245,7 +2156,6 @@ async fn test_recovered_automatic_purge_rechecks_unexpired_tombstone() { let procedure = PurgeDroppedTableProcedure::new( new_purge_dropped_table_task(table_id), ddl_context.clone(), - EventContext::default(), ); let mut data: serde_json::Value = serde_json::from_str(&procedure.dump().unwrap()).unwrap(); data["check_expired"] = true.into(); @@ -2323,7 +2233,6 @@ async fn test_recovered_automatic_purge_obeys_runtime_disable() { let procedure = PurgeDroppedTableProcedure::new_if_expired( new_purge_dropped_table_task(table_id), ddl_context.clone(), - EventContext::default(), ); let mut recovered = PurgeDroppedTableProcedure::from_json(&procedure.dump().unwrap(), ddl_context.clone()) @@ -2355,7 +2264,6 @@ async fn test_recovered_automatic_purge_finishes_after_cleanup_is_claimed() { let mut procedure = PurgeDroppedTableProcedure::new_if_expired( new_purge_dropped_table_task(table_id), enabled_context.clone(), - EventContext::default(), ); procedure .execute(&new_test_procedure_context()) @@ -2419,7 +2327,6 @@ async fn test_recovered_automatic_purge_ignores_foreign_generation_claim() { let mut procedure = PurgeDroppedTableProcedure::new_if_expired( new_purge_dropped_table_task(table_id), ddl_context.clone(), - EventContext::default(), ); procedure .execute(&new_test_procedure_context()) @@ -2462,11 +2369,8 @@ async fn test_undrop_rejects_table_claimed_for_purge() { .await .unwrap(); - let mut procedure = UndropTableProcedure::new( - new_undrop_table_task(table_id), - ddl_context.clone(), - EventContext::default(), - ); + let mut procedure = + UndropTableProcedure::new(new_undrop_table_task(table_id), ddl_context.clone()); let error = procedure.on_prepare().await.unwrap_err(); assert_eq!(StatusCode::TableNotFound, error.status_code()); @@ -2488,11 +2392,8 @@ async fn test_recovered_undrop_rejects_tombstone_deleted_by_purge() { let table_id = 1024; create_dropped_table(&ddl_context, table_id, Some(1), Some(i64::MAX)).await; - let mut undrop = UndropTableProcedure::new( - new_undrop_table_task(table_id), - ddl_context.clone(), - EventContext::default(), - ); + let mut undrop = + UndropTableProcedure::new(new_undrop_table_task(table_id), ddl_context.clone()); undrop.on_prepare().await.unwrap(); let open_regions_state = undrop.dump().unwrap(); undrop.execute(&new_test_procedure_context()).await.unwrap(); @@ -2501,7 +2402,6 @@ async fn test_recovered_undrop_rejects_tombstone_deleted_by_purge() { let mut purge = PurgeDroppedTableProcedure::new( new_purge_dropped_table_task(table_id), ddl_context.clone(), - EventContext::default(), ); execute_procedure_until_done(&mut purge).await; @@ -2533,11 +2433,8 @@ async fn test_recovered_undrop_rejects_replacement_generation() { .await .unwrap(); - let mut undrop = UndropTableProcedure::new( - new_undrop_table_task(table_id), - ddl_context.clone(), - EventContext::default(), - ); + let mut undrop = + UndropTableProcedure::new(new_undrop_table_task(table_id), ddl_context.clone()); undrop.on_prepare().await.unwrap(); let persisted = undrop.dump().unwrap(); kv_backend @@ -2603,11 +2500,8 @@ async fn test_legacy_unmarked_tombstone_can_be_undropped_and_purged() { assert_eq!(dropped_tables.len(), 1); assert_eq!(dropped_tables[0].dropped_at, None); - let mut undrop = UndropTableProcedure::new( - new_undrop_table_task(table_id), - ddl_context.clone(), - EventContext::default(), - ); + let mut undrop = + UndropTableProcedure::new(new_undrop_table_task(table_id), ddl_context.clone()); execute_procedure_until_done(&mut undrop).await; ddl_context @@ -2624,7 +2518,6 @@ async fn test_legacy_unmarked_tombstone_can_be_undropped_and_purged() { let mut purge = PurgeDroppedTableProcedure::new( new_purge_dropped_table_task(table_id), ddl_context.clone(), - EventContext::default(), ); execute_procedure_until_done(&mut purge).await; assert!( @@ -2667,7 +2560,6 @@ async fn test_purge_dropped_table_by_id_selects_tombstone_when_live_table_exists let mut drop_procedure = DropTableProcedure::new( new_drop_table_task(table_name, dropped_table_id, false), ddl_context.clone(), - EventContext::default(), ); execute_procedure_until_done(&mut drop_procedure).await; ddl_context @@ -2684,7 +2576,6 @@ async fn test_purge_dropped_table_by_id_selects_tombstone_when_live_table_exists let mut procedure = PurgeDroppedTableProcedure::new( new_purge_dropped_table_task(dropped_table_id), ddl_context.clone(), - EventContext::default(), ); execute_procedure_until_done(&mut procedure).await; @@ -2735,12 +2626,8 @@ async fn test_on_rollback() { let physical_table_id = table_id; // Creates the logical table metadata. let task = test_create_logical_table_task("foo"); - let mut procedure = CreateLogicalTablesProcedure::new( - vec![task], - physical_table_id, - EventContext::default(), - ddl_context.clone(), - ); + let mut procedure = + CreateLogicalTablesProcedure::new(vec![task], physical_table_id, ddl_context.clone()); procedure.on_prepare().await.unwrap(); let ctx = new_test_procedure_context(); procedure.execute(&ctx).await.unwrap(); @@ -2753,8 +2640,7 @@ async fn test_on_rollback() { // Drops the physical table { let task = new_drop_table_task("phy_table", physical_table_id, false); - let mut procedure = - DropTableProcedure::new(task, ddl_context.clone(), EventContext::default()); + let mut procedure = DropTableProcedure::new(task, ddl_context.clone()); procedure.on_prepare().await.unwrap(); assert!(procedure.rollback_supported()); procedure.on_delete_metadata().await.unwrap(); @@ -2765,8 +2651,7 @@ async fn test_on_rollback() { // The physical table can be dropped again after the metadata rollback. let retry_task = new_drop_table_task("phy_table", physical_table_id, false); - let mut retry = - DropTableProcedure::new(retry_task, ddl_context.clone(), EventContext::default()); + let mut retry = DropTableProcedure::new(retry_task, ddl_context.clone()); retry.on_prepare().await.unwrap(); retry.on_delete_metadata().await.unwrap(); retry.rollback(&ctx).await.unwrap(); @@ -2782,7 +2667,7 @@ async fn test_on_rollback() { // Drops the logical table ddl_context.soft_drop_enabled = false; let task = new_drop_table_task("foo", table_ids[0], false); - let mut procedure = DropTableProcedure::new(task, ddl_context.clone(), EventContext::default()); + let mut procedure = DropTableProcedure::new(task, ddl_context.clone()); procedure.on_prepare().await.unwrap(); assert!(!procedure.rollback_supported()); } @@ -2879,8 +2764,7 @@ async fn test_memory_region_keeper_guard_dropped_on_procedure_done() { let logical_table_id = create_logical_table(ddl_context.clone(), physical_table_id, "s").await; let inner_test = |task: DropTableTask| async { - let mut procedure = - DropTableProcedure::new(task, ddl_context.clone(), EventContext::default()); + let mut procedure = DropTableProcedure::new(task, ddl_context.clone()); execute_procedure_until(&mut procedure, |p| { p.data.state == DropTableState::InvalidateTableCache }) @@ -2932,8 +2816,7 @@ async fn test_from_json() { let physical_table_id = create_physical_table(&ddl_context, "t").await; let task = new_drop_table_task("t", physical_table_id, false); - let mut procedure = - DropTableProcedure::new(task, ddl_context.clone(), EventContext::default()); + let mut procedure = DropTableProcedure::new(task, ddl_context.clone()); execute_procedure_until(&mut procedure, |p| p.data.state == state).await; let data = procedure.dump().unwrap(); assert_eq!( @@ -2962,7 +2845,7 @@ async fn test_from_json() { let physical_table_id = create_physical_table(&ddl_context, "t").await; let task = new_drop_table_task("t", physical_table_id, false); - let mut procedure = DropTableProcedure::new(task, ddl_context.clone(), EventContext::default()); + let mut procedure = DropTableProcedure::new(task, ddl_context.clone()); execute_procedure_until_done(&mut procedure).await; let data = procedure.dump().unwrap(); assert_eq!( diff --git a/src/common/meta/src/ddl/tests/drop_view.rs b/src/common/meta/src/ddl/tests/drop_view.rs index 09b2a450c5..412c5a78f0 100644 --- a/src/common/meta/src/ddl/tests/drop_view.rs +++ b/src/common/meta/src/ddl/tests/drop_view.rs @@ -24,7 +24,7 @@ use crate::ddl::drop_view::{DropViewProcedure, DropViewState}; use crate::ddl::test_util::create_table::test_create_table_task; use crate::ddl::tests::create_view::{test_create_view_task, test_table_names}; use crate::key::table_route::TableRouteValue; -use crate::rpc::ddl::{DropViewTask, EventContext}; +use crate::rpc::ddl::DropViewTask; use crate::test_util::{MockDatanodeManager, new_ddl_context}; pub(crate) fn new_drop_view_task( @@ -63,7 +63,7 @@ async fn test_on_prepare_view_not_exists_err() { .unwrap(); let task = new_drop_view_task("bar", view_id, false); - let mut procedure = DropViewProcedure::new(task, EventContext::default(), ddl_context); + let mut procedure = DropViewProcedure::new(task, ddl_context); let err = procedure.on_prepare().await.unwrap_err(); assert_eq!(err.status_code(), StatusCode::TableNotFound); } @@ -87,7 +87,7 @@ async fn test_on_prepare_not_view_err() { .unwrap(); let task = new_drop_view_task(view_name, view_id, false); - let mut procedure = DropViewProcedure::new(task, EventContext::default(), ddl_context); + let mut procedure = DropViewProcedure::new(task, ddl_context); // It's not a view, expect error let err = procedure.on_prepare().await.unwrap_err(); assert_eq!(err.status_code(), StatusCode::InvalidArguments); @@ -117,12 +117,12 @@ async fn test_on_prepare_success() { let task = new_drop_view_task("bar", view_id, true); // Drop if exists - let mut procedure = DropViewProcedure::new(task, EventContext::default(), ddl_context.clone()); + let mut procedure = DropViewProcedure::new(task, ddl_context.clone()); procedure.on_prepare().await.unwrap(); let task = new_drop_view_task(view_name, view_id, false); // Prepare success - let mut procedure = DropViewProcedure::new(task, EventContext::default(), ddl_context); + let mut procedure = DropViewProcedure::new(task, ddl_context); procedure.on_prepare().await.unwrap(); assert_eq!(DropViewState::DeleteMetadata, procedure.state()); } @@ -161,7 +161,7 @@ async fn test_drop_view_success() { let task = new_drop_view_task(view_name, view_id, false); // Prepare success - let mut procedure = DropViewProcedure::new(task, EventContext::default(), ddl_context.clone()); + let mut procedure = DropViewProcedure::new(task, ddl_context.clone()); execute_procedure_until_done(&mut procedure).await; assert_eq!(DropViewState::InvalidateViewCache, procedure.state()); @@ -178,7 +178,7 @@ async fn test_drop_view_success() { // Drop again let task = new_drop_view_task(view_name, view_id, false); - let mut procedure = DropViewProcedure::new(task, EventContext::default(), ddl_context); + let mut procedure = DropViewProcedure::new(task, ddl_context); let err = procedure.on_prepare().await.unwrap_err(); assert_eq!(err.status_code(), StatusCode::TableNotFound); } diff --git a/src/common/meta/src/ddl/tests/event/database.rs b/src/common/meta/src/ddl/tests/event/database.rs index 29cb48f7b8..8649982ed1 100644 --- a/src/common/meta/src/ddl/tests/event/database.rs +++ b/src/common/meta/src/ddl/tests/event/database.rs @@ -25,12 +25,12 @@ use common_event_recorder::event_table::{ PROCEDURE_ID_COLUMN as EVENT_TABLE_PROCEDURE_ID_COLUMN, PROCEDURE_STATE_COLUMN as EVENT_TABLE_PROCEDURE_STATE_COLUMN, PROCEDURE_TRIGGER_COLUMN as EVENT_TABLE_PROCEDURE_TRIGGER_COLUMN, - SCHEMA_NAME_COLUMN as EVENT_TABLE_SCHEMA_NAME_COLUMN, + SCHEMA_NAME_COLUMN as EVENT_TABLE_SCHEMA_NAME_COLUMN, jsonb_value, }; use common_event_recorder::testing::assert_event_contract; use common_procedure::{EventTrigger, ProcedureEvent, ProcedureId, ProcedureState}; -use super::test_util::{assert_event_filter, default_event_context_value, procedure_trigger_value}; +use super::test_util::assert_event_filter; use crate::ddl::alter_database::AlterDatabaseProcedure; use crate::ddl::create_database::CreateDatabaseProcedure; use crate::ddl::drop_database::DropDatabaseProcedure; @@ -39,7 +39,7 @@ use crate::ddl::event::database::{ DatabaseDdlEvent, }; use crate::rpc::ddl::{ - AlterDatabaseKind, EventContext, SetDatabaseOption, SetDatabaseOptions, UnsetDatabaseOption, + AlterDatabaseKind, SetDatabaseOption, SetDatabaseOptions, UnsetDatabaseOption, UnsetDatabaseOptions, }; use crate::test_util::{MockDatanodeManager, new_ddl_context}; @@ -50,13 +50,7 @@ fn test_create_database_submitted_event_contract() { ("password".to_string(), "do-not-record".to_string()), ("compaction.type".to_string(), "twcs".to_string()), ]); - let event = DatabaseDdlEvent::create_submitted( - "greptime", - "metrics", - true, - &options, - EventContext::default(), - ); + let event = DatabaseDdlEvent::create_submitted("greptime", "metrics", true, &options); assert_event_locator( &event, @@ -93,7 +87,6 @@ fn test_alter_database_set_and_unset_event_contracts() { SetDatabaseOption::Other("secret_token".to_string(), "hidden".to_string()), SetDatabaseOption::Ttl(std::time::Duration::from_secs(3600).into()), ])), - EventContext::default(), ); assert_event_locator( &set, @@ -120,7 +113,6 @@ fn test_alter_database_set_and_unset_event_contracts() { UnsetDatabaseOption::Ttl, UnsetDatabaseOption::Other("compaction.type".to_string()), ])), - EventContext::default(), ); assert_event_locator( &unset, @@ -140,8 +132,7 @@ fn test_alter_database_set_and_unset_event_contracts() { #[test] fn test_drop_database_submitted_event_contract() { - let event = - DatabaseDdlEvent::drop_submitted("greptime", "metrics", true, EventContext::default()); + let event = DatabaseDdlEvent::drop_submitted("greptime", "metrics", true); assert_event_locator( &event, @@ -177,14 +168,7 @@ fn test_database_lifecycle_events_preserve_locator_and_null_intent() { assert_eq!( DatabaseDdlEvent::create_lifecycle("greptime", "metrics").extra_schema(), - DatabaseDdlEvent::create_submitted( - "c", - "s", - false, - &HashMap::new(), - EventContext::default(), - ) - .extra_schema() + DatabaseDdlEvent::create_submitted("c", "s", false, &HashMap::new()).extra_schema() ); } @@ -198,7 +182,6 @@ fn test_database_events_preserve_procedure_envelope_contract() { "metrics", false, &HashMap::new(), - EventContext::default(), )), ProcedureState::Running, EventTrigger::Submitted, @@ -237,7 +220,6 @@ fn test_create_database_event_filter() { false, HashMap::new(), None, - EventContext::default(), new_ddl_context(Arc::new(MockDatanodeManager::new(()))), ); @@ -258,7 +240,6 @@ fn test_alter_database_event_filter() { )), }, }, - EventContext::default(), new_ddl_context(Arc::new(MockDatanodeManager::new(()))), ) .unwrap(); @@ -272,7 +253,6 @@ fn test_drop_database_event_filter() { "greptime".to_string(), "metrics".to_string(), false, - EventContext::default(), new_ddl_context(Arc::new(MockDatanodeManager::new(()))), ); @@ -285,14 +265,12 @@ fn assert_event_locator( catalog_name: Option<&str>, schema_name: Option<&str>, ) { - let has_event_context = !event.json_payload().unwrap().is_null(); assert_event_contract( event, event_type, &[ EVENT_TABLE_CATALOG_NAME_COLUMN.column_schema(), EVENT_TABLE_SCHEMA_NAME_COLUMN.column_schema(), - EVENT_CONTEXT_COLUMN.column_schema(), ], &[Row { values: vec![ @@ -302,11 +280,6 @@ fn assert_event_locator( Value { value_data: schema_name.map(|value| ValueData::StringValue(value.to_string())), }, - if has_event_context { - default_event_context_value() - } else { - Value { value_data: None } - }, ], }], ); @@ -320,7 +293,6 @@ fn assert_procedure_event_contract( catalog_name: Option<&str>, schema_name: Option<&str>, ) { - let has_event_context = procedure_trigger == "Submitted"; assert_event_contract( event, event_type, @@ -346,18 +318,14 @@ fn assert_procedure_event_contract( Value { value_data: Some(ValueData::StringValue(String::new())), }, - procedure_trigger_value(procedure_trigger), + jsonb_value(&serde_json::json!({"type": procedure_trigger})), Value { value_data: catalog_name.map(|value| ValueData::StringValue(value.to_string())), }, Value { value_data: schema_name.map(|value| ValueData::StringValue(value.to_string())), }, - if has_event_context { - default_event_context_value() - } else { - Value { value_data: None } - }, + Value { value_data: None }, ], }], ); diff --git a/src/common/meta/src/ddl/tests/event/flow.rs b/src/common/meta/src/ddl/tests/event/flow.rs index 39f24b74e5..aae3613a08 100644 --- a/src/common/meta/src/ddl/tests/event/flow.rs +++ b/src/common/meta/src/ddl/tests/event/flow.rs @@ -21,12 +21,13 @@ use common_event_recorder::Event; use common_event_recorder::event_table::{ CATALOG_NAME_COLUMN, EVENT_CONTEXT_COLUMN, FLOW_ID_COLUMN, FLOW_NAME_COLUMN, PROCEDURE_ERROR_COLUMN, PROCEDURE_ID_COLUMN, PROCEDURE_STATE_COLUMN, PROCEDURE_TRIGGER_COLUMN, + jsonb_value, }; use common_event_recorder::testing::assert_event_contract; use common_procedure::{EventTrigger, ProcedureEvent, ProcedureId, ProcedureState}; use table::table_name::TableName; -use super::test_util::{assert_event_filter, default_event_context_value, procedure_trigger_value}; +use super::test_util::assert_event_filter; use crate::ddl::create_flow::CreateFlowProcedure; use crate::ddl::drop_flow::DropFlowProcedure; use crate::ddl::event::flow::{ @@ -35,7 +36,6 @@ use crate::ddl::event::flow::{ use crate::ddl::test_util::flownode_handler::NaiveFlownodeHandler; use crate::ddl::tests::create_flow::{test_create_flow_task, test_query_context}; use crate::ddl::tests::drop_flow::test_drop_flow_task; -use crate::rpc::ddl::EventContext; use crate::test_util::{MockFlownodeManager, new_ddl_context}; #[test] @@ -49,7 +49,6 @@ fn test_flow_submitted_event_contracts() { expire_after: Some(300), eval_interval_secs: Some(60), }, - EventContext::default(), ); assert_event_contract( &create, @@ -60,7 +59,6 @@ fn test_flow_submitted_event_contracts() { ValueData::StringValue("greptime".to_string()).into(), ValueData::StringValue("metrics".to_string()).into(), Value { value_data: None }, - default_event_context_value(), ], }], ); @@ -75,8 +73,7 @@ fn test_flow_submitted_event_contracts() { }) ); - let drop = - FlowDdlEvent::drop_submitted("greptime", "metrics", 42, true, EventContext::default()); + let drop = FlowDdlEvent::drop_submitted("greptime", "metrics", 42, true); assert_event_contract( &drop, DROP_FLOW_EVENT_TYPE, @@ -86,7 +83,6 @@ fn test_flow_submitted_event_contracts() { ValueData::StringValue("greptime".to_string()).into(), ValueData::StringValue("metrics".to_string()).into(), ValueData::U32Value(42).into(), - default_event_context_value(), ], }], ); @@ -125,7 +121,6 @@ fn test_flow_lifecycle_events_have_fixed_schema_and_null_intent() { } else { Value { value_data: None } }, - Value { value_data: None }, ], }], ); @@ -142,7 +137,6 @@ fn test_flow_lifecycle_events_have_fixed_schema_and_null_intent() { ValueData::StringValue("greptime".to_string()).into(), ValueData::StringValue("metrics".to_string()).into(), ValueData::U32Value(42).into(), - Value { value_data: None }, ], }], ); @@ -162,7 +156,6 @@ fn test_flow_events_preserve_procedure_envelope_contract() { expire_after: None, eval_interval_secs: None, }, - EventContext::default(), )), ProcedureState::Running, EventTrigger::Submitted, @@ -212,7 +205,6 @@ fn test_create_flow_event_filter() { false, ), test_query_context(), - EventContext::default(), new_ddl_context(Arc::new(MockFlownodeManager::new(NaiveFlownodeHandler))), ); assert_event_filter(&procedure, CREATE_FLOW_EVENT_TYPE); @@ -222,7 +214,6 @@ fn test_create_flow_event_filter() { fn test_drop_flow_event_filter() { let procedure = DropFlowProcedure::new( test_drop_flow_task("flow", 42, false), - EventContext::default(), new_ddl_context(Arc::new(MockFlownodeManager::new(NaiveFlownodeHandler))), ); assert_event_filter(&procedure, DROP_FLOW_EVENT_TYPE); @@ -233,7 +224,6 @@ fn flow_schema() -> Vec { CATALOG_NAME_COLUMN.column_schema(), FLOW_NAME_COLUMN.column_schema(), FLOW_ID_COLUMN.column_schema(), - EVENT_CONTEXT_COLUMN.column_schema(), ] } @@ -257,6 +247,7 @@ fn assert_procedure_event_contract( PROCEDURE_TRIGGER_COLUMN.column_schema(), ]; schema.extend(flow_schema()); + schema.push(EVENT_CONTEXT_COLUMN.column_schema()); assert_event_contract( event, event_type, @@ -266,7 +257,7 @@ fn assert_procedure_event_contract( ValueData::StringValue(event.procedure_id.to_string()).into(), ValueData::StringValue(state.to_string()).into(), ValueData::StringValue(String::new()).into(), - procedure_trigger_value(trigger), + jsonb_value(&serde_json::json!({"type": trigger})), optional_string(locator.catalog_name), optional_string(locator.flow_name), locator @@ -274,11 +265,7 @@ fn assert_procedure_event_contract( .map(ValueData::U32Value) .map(Into::into) .unwrap_or(Value { value_data: None }), - if trigger == "Submitted" { - default_event_context_value() - } else { - Value { value_data: None } - }, + Value { value_data: None }, ], }], ); diff --git a/src/common/meta/src/ddl/tests/event/table.rs b/src/common/meta/src/ddl/tests/event/table.rs index 3326cb0680..24182fd19a 100644 --- a/src/common/meta/src/ddl/tests/event/table.rs +++ b/src/common/meta/src/ddl/tests/event/table.rs @@ -19,13 +19,13 @@ use api::v1::value::ValueData; use api::v1::{ColumnDataType, Repartition, SemanticType, Value}; use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME}; use common_event_recorder::event_table::{ - CATALOG_NAME_COLUMN, EVENT_CONTEXT_COLUMN, PHYSICAL_TABLE_ID_COLUMN, SCHEMA_NAME_COLUMN, - TABLE_ID_COLUMN, TABLE_NAME_COLUMN, jsonb_value, + CATALOG_NAME_COLUMN, PHYSICAL_TABLE_ID_COLUMN, SCHEMA_NAME_COLUMN, TABLE_ID_COLUMN, + TABLE_NAME_COLUMN, }; use common_event_recorder::{Event, EventTypeFilter}; use common_procedure::{ - ChildSubmissionOutcome, EventRuntimeContext, EventTrigger, Procedure, ProcedureId, - ProcedureState, RetryPhase, + ChildSubmissionOutcome, EventContext, EventTrigger, Procedure, ProcedureId, ProcedureState, + RetryPhase, }; use common_time::Timestamp; use serde_json::{Value as JsonValue, json}; @@ -52,9 +52,9 @@ use crate::ddl::truncate_table::TruncateTableProcedure; use crate::ddl::undrop_table::UndropTableProcedure; use crate::key::DeserializedValueWithBytes; use crate::key::table_info::TableInfoValue; -use crate::rpc::ddl::{DropTableTask, EventContext, QueryContext, TruncateTableTask}; +use crate::rpc::ddl::{DropTableTask, TruncateTableTask}; #[cfg(feature = "enterprise")] -use crate::rpc::ddl::{PurgeDroppedTableTask, TriggerReason, UndropTableTask}; +use crate::rpc::ddl::{PurgeDroppedTableTask, UndropTableTask}; use crate::test_util::{MockDatanodeManager, new_ddl_context}; struct EventCase { @@ -175,11 +175,7 @@ fn later_lifecycle_events_are_uniform() { .extra_rows() .unwrap() .into_iter() - .map(|mut row| { - // Lifecycle events retain locators but not submitted event context. - *row.values.last_mut().unwrap() = Value::default(); - row.values - }) + .map(|row| row.values) .collect::>(); for trigger in &triggers { @@ -205,13 +201,7 @@ fn later_lifecycle_events_are_uniform() { fn create_success_events_keep_allocated_ids() { let mut task = test_create_table_task("create_success"); task.table_info.ident.table_id = 7; - let create_table = CreateTableProcedure::new( - task, - QueryContext::default(), - EventContext::default(), - test_context(), - ) - .unwrap(); + let create_table = CreateTableProcedure::new(task, test_context()).unwrap(); let state = ProcedureState::Done { output: Some(Arc::new(42_u32)), }; @@ -230,7 +220,6 @@ fn create_success_events_keep_allocated_ids() { test_create_logical_table_task("bar"), ], 1024, - EventContext::default(), test_context(), ); let state = ProcedureState::Done { @@ -269,14 +258,13 @@ fn event_cases() -> Vec { TableDdlLocator::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, "create"), true, "mito2", - EventContext::default(), ), payload: json!({ "version": TABLE_DDL_PAYLOAD_VERSION, "create_if_not_exists": true, "engine": "mito2", }), - rows: vec![submitted_table_locator_values(Some("create"), None)], + rows: vec![table_locator_values(Some("create"), None)], }, EventCase { event_type: TableDdlEventType::CreateLogicalTables, @@ -288,15 +276,14 @@ fn event_cases() -> Vec { .with_physical_table_id(10), ], 2, - EventContext::default(), ), payload: json!({ "version": TABLE_DDL_PAYLOAD_VERSION, "table_count": 2, }), rows: vec![ - submitted_logical_locator_values("logical1", None, 10), - submitted_logical_locator_values("logical2", None, 10), + logical_locator_values("logical1", None, 10), + logical_locator_values("logical2", None, 10), ], }, EventCase { @@ -305,13 +292,12 @@ fn event_cases() -> Vec { TableDdlLocator::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, "alter") .with_table_id(11), Some("drop_columns"), - EventContext::default(), ), payload: json!({ "version": TABLE_DDL_PAYLOAD_VERSION, "kind": "drop_columns", }), - rows: vec![submitted_table_locator_values(Some("alter"), Some(11))], + rows: vec![table_locator_values(Some("alter"), Some(11))], }, EventCase { event_type: TableDdlEventType::AlterLogicalTables, @@ -324,7 +310,6 @@ fn event_cases() -> Vec { ], 2, ["rename_table", "add_columns", "add_columns"], - EventContext::default(), ), payload: json!({ "version": TABLE_DDL_PAYLOAD_VERSION, @@ -332,8 +317,8 @@ fn event_cases() -> Vec { "kinds": ["add_columns", "rename_table"], }), rows: vec![ - submitted_logical_locator_values("logical1", None, 10), - submitted_logical_locator_values("logical2", None, 10), + logical_locator_values("logical1", None, 10), + logical_locator_values("logical2", None, 10), ], }, EventCase { @@ -342,33 +327,26 @@ fn event_cases() -> Vec { TableDdlLocator::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, "drop") .with_table_id(12), true, - EventContext::default(), ), payload: json!({ "version": TABLE_DDL_PAYLOAD_VERSION, "drop_if_exists": true, }), - rows: vec![submitted_table_locator_values(Some("drop"), Some(12))], + rows: vec![table_locator_values(Some("drop"), Some(12))], }, #[cfg(feature = "enterprise")] EventCase { event_type: TableDdlEventType::UndropTable, - event: TableDdlEvent::undrop_table_submitted( - TableDdlLocator::from_table_id(13), - EventContext::default(), - ), + event: TableDdlEvent::undrop_table_submitted(TableDdlLocator::from_table_id(13)), payload: json!({"version": TABLE_DDL_PAYLOAD_VERSION}), - rows: vec![submitted_table_locator_values(None, Some(13))], + rows: vec![table_locator_values(None, Some(13))], }, #[cfg(feature = "enterprise")] EventCase { event_type: TableDdlEventType::PurgeDroppedTable, - event: TableDdlEvent::purge_dropped_table_submitted( - TableDdlLocator::from_table_id(14), - EventContext::default(), - ), + event: TableDdlEvent::purge_dropped_table_submitted(TableDdlLocator::from_table_id(14)), payload: json!({"version": TABLE_DDL_PAYLOAD_VERSION}), - rows: vec![submitted_table_locator_values(None, Some(14))], + rows: vec![table_locator_values(None, Some(14))], }, EventCase { event_type: TableDdlEventType::TruncateTable, @@ -376,41 +354,29 @@ fn event_cases() -> Vec { TableDdlLocator::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, "truncate") .with_table_id(15), 4, - EventContext::default(), ), payload: json!({ "version": TABLE_DDL_PAYLOAD_VERSION, "time_range_count": 4, }), - rows: vec![submitted_table_locator_values(Some("truncate"), Some(15))], + rows: vec![table_locator_values(Some("truncate"), Some(15))], }, ] } fn procedure_cases() -> Vec { - let create_table = CreateTableProcedure::new( - test_create_table_task("create"), - QueryContext::default(), - EventContext::default(), - test_context(), - ) - .unwrap(); + let create_table = + CreateTableProcedure::new(test_create_table_task("create"), test_context()).unwrap(); let create_logical_tables = CreateLogicalTablesProcedure::new( vec![ test_create_logical_table_task("logical1"), test_create_logical_table_task("logical2"), ], 41, - EventContext::default(), test_context(), ); - let alter_table = AlterTableProcedure::new( - 42, - test_alter_table_task("alter"), - EventContext::default(), - test_context(), - ) - .unwrap(); + let alter_table = + AlterTableProcedure::new(42, test_alter_table_task("alter"), test_context()).unwrap(); let alter_logical_tables = AlterLogicalTablesProcedure::new( vec![ make_alter_logical_table_add_column_task( @@ -425,7 +391,6 @@ fn procedure_cases() -> Vec { ), ], 43, - EventContext::default(), test_context(), ); let drop_table = DropTableProcedure::new( @@ -437,20 +402,12 @@ fn procedure_cases() -> Vec { drop_if_exists: true, }, test_context(), - EventContext::default(), ); #[cfg(feature = "enterprise")] - let undrop_table = UndropTableProcedure::new( - UndropTableTask { table_id: 45 }, - test_context(), - EventContext::default(), - ); + let undrop_table = UndropTableProcedure::new(UndropTableTask { table_id: 45 }, test_context()); #[cfg(feature = "enterprise")] - let purge_dropped_table = PurgeDroppedTableProcedure::new_if_expired( - PurgeDroppedTableTask { table_id: 46 }, - test_context(), - EventContext::new(TriggerReason::ScheduledGc), - ); + let purge_dropped_table = + PurgeDroppedTableProcedure::new(PurgeDroppedTableTask { table_id: 46 }, test_context()); let truncate_table = truncate_procedure(TruncateTableTask { catalog: DEFAULT_CATALOG_NAME.to_string(), schema: DEFAULT_SCHEMA_NAME.to_string(), @@ -471,7 +428,7 @@ fn procedure_cases() -> Vec { "create_if_not_exists": false, "engine": "mito2", }), - rows: vec![submitted_table_locator_values(Some("create"), None)], + rows: vec![table_locator_values(Some("create"), None)], }, ProcedureCase { procedure: Box::new(create_logical_tables), @@ -481,8 +438,8 @@ fn procedure_cases() -> Vec { "table_count": 2, }), rows: vec![ - submitted_logical_locator_values("logical1", None, 41), - submitted_logical_locator_values("logical2", None, 41), + logical_locator_values("logical1", None, 41), + logical_locator_values("logical2", None, 41), ], }, ProcedureCase { @@ -492,7 +449,7 @@ fn procedure_cases() -> Vec { "version": TABLE_DDL_PAYLOAD_VERSION, "kind": "drop_columns", }), - rows: vec![submitted_table_locator_values(Some("alter"), Some(42))], + rows: vec![table_locator_values(Some("alter"), Some(42))], }, ProcedureCase { procedure: Box::new(alter_logical_tables), @@ -503,8 +460,8 @@ fn procedure_cases() -> Vec { "kinds": ["add_columns"], }), rows: vec![ - submitted_logical_locator_values("logical1", None, 43), - submitted_logical_locator_values("logical2", None, 43), + logical_locator_values("logical1", None, 43), + logical_locator_values("logical2", None, 43), ], }, ProcedureCase { @@ -514,25 +471,21 @@ fn procedure_cases() -> Vec { "version": TABLE_DDL_PAYLOAD_VERSION, "drop_if_exists": true, }), - rows: vec![submitted_table_locator_values(Some("drop"), Some(44))], + rows: vec![table_locator_values(Some("drop"), Some(44))], }, #[cfg(feature = "enterprise")] ProcedureCase { procedure: Box::new(undrop_table), event_type: "undrop_table", payload: json!({"version": TABLE_DDL_PAYLOAD_VERSION}), - rows: vec![submitted_table_locator_values(None, Some(45))], + rows: vec![table_locator_values(None, Some(45))], }, #[cfg(feature = "enterprise")] ProcedureCase { procedure: Box::new(purge_dropped_table), event_type: "purge_dropped_table", payload: json!({"version": TABLE_DDL_PAYLOAD_VERSION}), - rows: vec![submitted_table_locator_values_with_event_context( - None, - Some(46), - EventContext::new(TriggerReason::ScheduledGc), - )], + rows: vec![table_locator_values(None, Some(46))], }, ProcedureCase { procedure: Box::new(truncate_table), @@ -541,7 +494,7 @@ fn procedure_cases() -> Vec { "version": TABLE_DDL_PAYLOAD_VERSION, "time_range_count": 1, }), - rows: vec![submitted_table_locator_values(Some("truncate"), Some(47))], + rows: vec![table_locator_values(Some("truncate"), Some(47))], }, ] } @@ -562,7 +515,6 @@ fn expected_schema(event_type: TableDdlEventType) -> Vec<(&'static str, i32)> { ColumnDataType::Uint32 as i32, )); } - schema.push((EVENT_CONTEXT_COLUMN.name(), ColumnDataType::Binary as i32)); schema } @@ -575,14 +527,12 @@ fn table_locator_values(table_name: Option<&str>, table_id: Option) -> Vec< } else { (Value::default(), Value::default()) }; - let mut values = vec![ + vec![ catalog_name, schema_name, table_name.map(string_value).unwrap_or_default(), table_id.map(table_id_value).unwrap_or_default(), - ]; - values.push(Value::default()); - values + ] } fn logical_locator_values( @@ -591,44 +541,10 @@ fn logical_locator_values( physical_table_id: u32, ) -> Vec { let mut values = table_locator_values(Some(table_name), table_id); - let event_context = values.pop().unwrap(); values.push(table_id_value(physical_table_id)); - values.push(event_context); values } -fn submitted_table_locator_values(table_name: Option<&str>, table_id: Option) -> Vec { - submitted_table_locator_values_with_event_context(table_name, table_id, EventContext::default()) -} - -fn submitted_table_locator_values_with_event_context( - table_name: Option<&str>, - table_id: Option, - event_context: EventContext, -) -> Vec { - let mut values = table_locator_values(table_name, table_id); - *values.last_mut().unwrap() = event_context_value_for(event_context); - values -} - -fn submitted_logical_locator_values( - table_name: &str, - table_id: Option, - physical_table_id: u32, -) -> Vec { - let mut values = logical_locator_values(table_name, table_id, physical_table_id); - *values.last_mut().unwrap() = event_context_value(); - values -} - -fn event_context_value() -> Value { - event_context_value_for(EventContext::default()) -} - -fn event_context_value_for(event_context: EventContext) -> Value { - jsonb_value(&serde_json::to_value(event_context).unwrap()) -} - fn string_value(value: &str) -> Value { ValueData::StringValue(value.to_string()).into() } @@ -643,7 +559,6 @@ fn truncate_procedure(task: TruncateTableTask) -> TruncateTableProcedure { task, DeserializedValueWithBytes::from_inner(TableInfoValue::new(table_info)), test_context(), - EventContext::default(), ) } @@ -661,11 +576,12 @@ fn event_for_state( lifecycle_state: &ProcedureState, ) -> Box { procedure - .event(&EventRuntimeContext { + .event(&EventContext { procedure_id: ProcedureId::random(), lifecycle_state, trigger, event_type_filter: Arc::new(EventTypeFilter::All), + event_context: None, }) .unwrap() } diff --git a/src/common/meta/src/ddl/tests/event/test_util.rs b/src/common/meta/src/ddl/tests/event/test_util.rs index 281b5deb48..e20d294180 100644 --- a/src/common/meta/src/ddl/tests/event/test_util.rs +++ b/src/common/meta/src/ddl/tests/event/test_util.rs @@ -15,32 +15,21 @@ use std::collections::HashSet; use std::sync::Arc; -use api::v1::Value; use common_event_recorder::EventTypeFilter; -use common_event_recorder::event_table::jsonb_value; -use common_procedure::{EventRuntimeContext, EventTrigger, Procedure, ProcedureId, ProcedureState}; - -use crate::rpc::ddl::EventContext; - -pub(crate) fn default_event_context_value() -> Value { - jsonb_value(&serde_json::to_value(EventContext::default()).unwrap()) -} - -pub(crate) fn procedure_trigger_value(trigger: &str) -> Value { - jsonb_value(&serde_json::json!({"type": trigger})) -} +use common_procedure::{EventContext, EventTrigger, Procedure, ProcedureId, ProcedureState}; pub(crate) fn assert_event_filter(procedure: &dyn Procedure, event_type: &str) { let state = ProcedureState::Running; - let runtime_context = |event_type_filter| EventRuntimeContext { + let event_context = |event_type_filter| EventContext { procedure_id: ProcedureId::random(), lifecycle_state: &state, trigger: EventTrigger::Submitted, event_type_filter: Arc::new(event_type_filter), + event_context: None, }; let allowed = procedure - .event(&runtime_context(EventTypeFilter::Only(HashSet::from([ + .event(&event_context(EventTypeFilter::Only(HashSet::from([ event_type.to_string(), ])))) .unwrap(); @@ -49,7 +38,7 @@ pub(crate) fn assert_event_filter(procedure: &dyn Procedure, event_type: &str) { for denied in [HashSet::from(["other_event".to_string()]), HashSet::new()] { assert!( procedure - .event(&runtime_context(EventTypeFilter::Only(denied))) + .event(&event_context(EventTypeFilter::Only(denied))) .is_none() ); } diff --git a/src/common/meta/src/ddl/tests/event/view.rs b/src/common/meta/src/ddl/tests/event/view.rs index cabe7418ff..5cb5a6bb7d 100644 --- a/src/common/meta/src/ddl/tests/event/view.rs +++ b/src/common/meta/src/ddl/tests/event/view.rs @@ -19,16 +19,16 @@ use api::v1::{ColumnSchema, Row, Value}; use common_event_recorder::event_table::{ CATALOG_NAME_COLUMN, EVENT_CONTEXT_COLUMN, PROCEDURE_ERROR_COLUMN, PROCEDURE_ID_COLUMN, PROCEDURE_STATE_COLUMN, PROCEDURE_TRIGGER_COLUMN, SCHEMA_NAME_COLUMN, VIEW_ID_COLUMN, - VIEW_NAME_COLUMN, + VIEW_NAME_COLUMN, jsonb_value, }; use common_event_recorder::testing::assert_event_contract; use common_event_recorder::{Event, EventTypeFilter}; use common_procedure::{ - ChildSubmissionOutcome, EventRuntimeContext, EventTrigger, Output, Procedure, ProcedureEvent, + ChildSubmissionOutcome, EventContext, EventTrigger, Output, Procedure, ProcedureEvent, ProcedureId, ProcedureState, RetryPhase, }; -use super::test_util::{assert_event_filter, default_event_context_value, procedure_trigger_value}; +use super::test_util::assert_event_filter; use crate::ddl::create_view::CreateViewProcedure; use crate::ddl::drop_view::DropViewProcedure; use crate::ddl::event::view::{ @@ -36,7 +36,6 @@ use crate::ddl::event::view::{ }; use crate::ddl::tests::create_view::test_create_view_task; use crate::ddl::tests::drop_view::new_drop_view_task; -use crate::rpc::ddl::EventContext; use crate::test_util::{MockDatanodeManager, new_ddl_context}; #[test] @@ -44,7 +43,7 @@ fn test_view_submitted_event_contracts() { let mut task = test_create_view_task("v_metrics"); task.create_view.or_replace = true; task.create_view.create_if_not_exists = true; - let create = CreateViewProcedure::new(task, EventContext::default(), test_context()); + let create = CreateViewProcedure::new(task, test_context()); let event = event_for(&create, EventTrigger::Submitted); assert_view_event_contract( @@ -72,11 +71,7 @@ fn test_view_submitted_event_contracts() { assert!(!payload.contains(omitted)); } - let drop = DropViewProcedure::new( - new_drop_view_task("view_name", 42, true), - EventContext::default(), - test_context(), - ); + let drop = DropViewProcedure::new(new_drop_view_task("view_name", 42, true), test_context()); let event = event_for(&drop, EventTrigger::Submitted); assert_view_event_contract( @@ -140,16 +135,8 @@ fn test_view_lifecycle_event_contracts() { #[test] fn test_view_procedures_preserve_lifecycle_locators() { - let create = CreateViewProcedure::new( - test_create_view_task("view_name"), - EventContext::default(), - test_context(), - ); - let drop = DropViewProcedure::new( - new_drop_view_task("view_name", 42, false), - EventContext::default(), - test_context(), - ); + let create = CreateViewProcedure::new(test_create_view_task("view_name"), test_context()); + let drop = DropViewProcedure::new(new_drop_view_task("view_name", 42, false), test_context()); let triggers = [ EventTrigger::Recovered, EventTrigger::ChildSubmitted { @@ -199,11 +186,7 @@ fn test_view_procedures_preserve_lifecycle_locators() { #[test] fn test_create_view_succeeded_output_mapping() { - let procedure = CreateViewProcedure::new( - test_create_view_task("view_name"), - EventContext::default(), - test_context(), - ); + let procedure = CreateViewProcedure::new(test_create_view_task("view_name"), test_context()); let state = ProcedureState::Done { output: Some(Arc::new(84_u32)), }; @@ -240,21 +223,14 @@ fn test_create_view_succeeded_output_mapping() { #[test] fn test_create_view_event_filter() { - let procedure = CreateViewProcedure::new( - test_create_view_task("view_name"), - EventContext::default(), - test_context(), - ); + let procedure = CreateViewProcedure::new(test_create_view_task("view_name"), test_context()); assert_event_filter(&procedure, CREATE_VIEW_EVENT_TYPE); } #[test] fn test_drop_view_event_filter() { - let procedure = DropViewProcedure::new( - new_drop_view_task("view_name", 42, false), - EventContext::default(), - test_context(), - ); + let procedure = + DropViewProcedure::new(new_drop_view_task("view_name", 42, false), test_context()); assert_event_filter(&procedure, DROP_VIEW_EVENT_TYPE); } @@ -273,7 +249,6 @@ fn test_view_event_procedure_envelope_contract() { referenced_table_count: 1, column_count: 1, }, - EventContext::default(), )), ProcedureState::Running, EventTrigger::Submitted, @@ -344,17 +319,11 @@ fn view_schema() -> Vec { SCHEMA_NAME_COLUMN.column_schema(), VIEW_NAME_COLUMN.column_schema(), VIEW_ID_COLUMN.column_schema(), - EVENT_CONTEXT_COLUMN.column_schema(), ] } fn assert_view_event_contract(event: &dyn Event, event_type: &str, locator: ViewEventLocator<'_>) { - let mut values = locator.values(); - values.push(if event.json_payload().unwrap().is_null() { - Value { value_data: None } - } else { - default_event_context_value() - }); + let values = locator.values(); assert_event_contract(event, event_type, &view_schema(), &[Row { values }]); } @@ -372,19 +341,16 @@ fn assert_procedure_event_contract( PROCEDURE_TRIGGER_COLUMN.column_schema(), ]; schema.extend(view_schema()); + schema.push(EVENT_CONTEXT_COLUMN.column_schema()); let mut values = vec![ ValueData::StringValue(event.procedure_id.to_string()).into(), ValueData::StringValue(state.to_string()).into(), ValueData::StringValue(String::new()).into(), - procedure_trigger_value(trigger), + jsonb_value(&serde_json::json!({"type": trigger})), ]; values.extend(locator.values()); - values.push(if trigger == "Submitted" { - default_event_context_value() - } else { - Value { value_data: None } - }); + values.push(Value { value_data: None }); assert_event_contract(event, event_type, &schema, &[Row { values }]); } @@ -403,11 +369,12 @@ fn event_for_state( lifecycle_state: &ProcedureState, ) -> Box { procedure - .event(&EventRuntimeContext { + .event(&EventContext { procedure_id: ProcedureId::random(), lifecycle_state, trigger, event_type_filter: Arc::new(EventTypeFilter::All), + event_context: None, }) .unwrap() } diff --git a/src/common/meta/src/ddl/truncate_table.rs b/src/common/meta/src/ddl/truncate_table.rs index 6a561b62af..67d7436198 100644 --- a/src/common/meta/src/ddl/truncate_table.rs +++ b/src/common/meta/src/ddl/truncate_table.rs @@ -20,7 +20,7 @@ use api::v1::region::{ use async_trait::async_trait; use common_procedure::error::{FromJsonSnafu, ToJsonSnafu}; use common_procedure::{ - Context as ProcedureContext, EventRuntimeContext, EventTrigger, LockKey, Procedure, + Context as ProcedureContext, EventContext, EventTrigger, LockKey, Procedure, Result as ProcedureResult, Status, }; use common_telemetry::debug; @@ -43,7 +43,7 @@ use crate::key::table_info::TableInfoValue; use crate::key::table_name::TableNameKey; use crate::lock_key::{CatalogLock, SchemaLock, TableLock}; use crate::metrics; -use crate::rpc::ddl::{EventContext, TruncateTableTask}; +use crate::rpc::ddl::TruncateTableTask; use crate::rpc::router::{find_leader_regions, find_leaders}; pub struct TruncateTableProcedure { @@ -89,10 +89,7 @@ impl Procedure for TruncateTableProcedure { LockKey::new(lock_key) } - fn event( - &self, - ctx: &EventRuntimeContext<'_>, - ) -> Option> { + fn event(&self, ctx: &EventContext<'_>) -> Option> { if !ctx .event_type_filter .allows(TableDdlEventType::TruncateTable.as_str()) @@ -103,11 +100,9 @@ impl Procedure for TruncateTableProcedure { let locator = TableDdlLocator::new(&task.catalog, &task.schema, &task.table) .with_table_id(task.table_id); let event = match &ctx.trigger { - EventTrigger::Submitted => TableDdlEvent::truncate_table_submitted( - locator, - task.time_ranges.len(), - self.data.event_context.clone(), - ), + EventTrigger::Submitted => { + TableDdlEvent::truncate_table_submitted(locator, task.time_ranges.len()) + } _ => TableDdlEvent::lifecycle(TableDdlEventType::TruncateTable, [locator]), }; @@ -122,11 +117,10 @@ impl TruncateTableProcedure { task: TruncateTableTask, table_info_value: DeserializedValueWithBytes, context: DdlContext, - event_context: EventContext, ) -> Self { Self { context, - data: TruncateTableData::new(task, table_info_value, event_context), + data: TruncateTableData::new(task, table_info_value), } } @@ -233,21 +227,17 @@ pub struct TruncateTableData { state: TruncateTableState, task: TruncateTableTask, table_info_value: DeserializedValueWithBytes, - #[serde(default)] - event_context: EventContext, } impl TruncateTableData { pub fn new( task: TruncateTableTask, table_info_value: DeserializedValueWithBytes, - event_context: EventContext, ) -> Self { Self { state: TruncateTableState::Prepare, task, table_info_value, - event_context, } } diff --git a/src/common/meta/src/ddl/undrop_table.rs b/src/common/meta/src/ddl/undrop_table.rs index 3295b1163e..73a361ffd0 100644 --- a/src/common/meta/src/ddl/undrop_table.rs +++ b/src/common/meta/src/ddl/undrop_table.rs @@ -17,7 +17,7 @@ use api::v1::region::{ use async_trait::async_trait; use common_procedure::error::{FromJsonSnafu, ToJsonSnafu}; use common_procedure::{ - Context as ProcedureContext, EventRuntimeContext, EventTrigger, LockKey, Procedure, + Context as ProcedureContext, EventContext, EventTrigger, LockKey, Procedure, Result as ProcedureResult, Status, }; use common_telemetry::tracing_context::TracingContext; @@ -43,7 +43,7 @@ use crate::instruction::CacheIdent; use crate::key::table_name::TableNameKey; use crate::key::table_route::TableRouteValue; use crate::lock_key::{CatalogLock, SchemaLock, TableLock, TableNameLock}; -use crate::rpc::ddl::{EventContext, UndropTableTask}; +use crate::rpc::ddl::UndropTableTask; use crate::rpc::router::{ RegionRoute, find_follower_regions, find_followers, find_leader_regions, find_leaders, }; @@ -56,17 +56,16 @@ pub struct UndropTableProcedure { impl UndropTableProcedure { pub const TYPE_NAME: &'static str = "metasrv-procedure::UndropTable"; - pub fn new(task: UndropTableTask, context: DdlContext, event_context: EventContext) -> Self { - Self::new_with_original_table_name(task, context, None, event_context) + pub fn new(task: UndropTableTask, context: DdlContext) -> Self { + Self::new_with_original_table_name(task, context, None) } pub(crate) fn new_with_original_table_name( task: UndropTableTask, context: DdlContext, table_name: Option, - event_context: EventContext, ) -> Self { - let mut data = UndropTableData::new(task, event_context); + let mut data = UndropTableData::new(task); data.table_name = table_name; Self { context, data } } @@ -444,10 +443,7 @@ impl Procedure for UndropTableProcedure { LockKey::new(lock_key) } - fn event( - &self, - ctx: &EventRuntimeContext<'_>, - ) -> Option> { + fn event(&self, ctx: &EventContext<'_>) -> Option> { if !ctx .event_type_filter .allows(TableDdlEventType::UndropTable.as_str()) @@ -456,9 +452,7 @@ impl Procedure for UndropTableProcedure { } let locator = self.event_locator(); let event = match &ctx.trigger { - EventTrigger::Submitted => { - TableDdlEvent::undrop_table_submitted(locator, self.data.event_context.clone()) - } + EventTrigger::Submitted => TableDdlEvent::undrop_table_submitted(locator), _ => TableDdlEvent::lifecycle(TableDdlEventType::UndropTable, [locator]), }; @@ -565,12 +559,10 @@ pub struct UndropTableData { drop_generation: Option, #[serde(default)] tombstone_identity_loaded: bool, - #[serde(default)] - event_context: EventContext, } impl UndropTableData { - fn new(task: UndropTableTask, event_context: EventContext) -> Self { + fn new(task: UndropTableTask) -> Self { Self { state: UndropTableState::Prepare, task, @@ -582,7 +574,6 @@ impl UndropTableData { retention_expires_at: None, drop_generation: None, tombstone_identity_loaded: false, - event_context, } } @@ -647,11 +638,7 @@ mod tests { #[test] fn test_map_restore_metadata_error_without_table_name() { let context = new_ddl_context(Arc::new(MockDatanodeManager::new(()))); - let procedure = UndropTableProcedure::new( - UndropTableTask { table_id: 42 }, - context, - EventContext::default(), - ); + let procedure = UndropTableProcedure::new(UndropTableTask { table_id: 42 }, context); let err = procedure.map_restore_metadata_error( error::TombstoneTargetAlreadyExistsSnafu { @@ -666,11 +653,8 @@ mod tests { #[test] fn test_recovered_legacy_snapshot_reloads_tombstone_identity() { let context = new_ddl_context(Arc::new(MockDatanodeManager::new(()))); - let mut procedure = UndropTableProcedure::new( - UndropTableTask { table_id: 42 }, - context.clone(), - EventContext::default(), - ); + let mut procedure = + UndropTableProcedure::new(UndropTableTask { table_id: 42 }, context.clone()); procedure.data.state = UndropTableState::OpenRegions; let mut data: serde_json::Value = serde_json::from_str(&procedure.dump().unwrap()).unwrap(); let data = data.as_object_mut().unwrap(); @@ -695,11 +679,7 @@ mod tests { let table_id = 1024; let region_id = RegionId::new(table_id, 1); - let mut procedure = UndropTableProcedure::new( - UndropTableTask { table_id }, - context, - EventContext::default(), - ); + let mut procedure = UndropTableProcedure::new(UndropTableTask { table_id }, context); procedure.data.table_route_value = Some(TableRouteValue::physical(vec![RegionRoute { region: Region::new_test(region_id), leader_peer: Some(Peer::empty(1)), @@ -738,11 +718,7 @@ mod tests { let table_id = 1024; let region_id = RegionId::new(table_id, 1); - let mut procedure = UndropTableProcedure::new( - UndropTableTask { table_id }, - context, - EventContext::default(), - ); + let mut procedure = UndropTableProcedure::new(UndropTableTask { table_id }, context); procedure.data.table_name = Some(TableName::new( DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, diff --git a/src/common/meta/src/ddl_manager.rs b/src/common/meta/src/ddl_manager.rs index 137a16793d..2eed4628ac 100644 --- a/src/common/meta/src/ddl_manager.rs +++ b/src/common/meta/src/ddl_manager.rs @@ -78,8 +78,8 @@ use crate::rpc::ddl::trigger::DropTriggerTask; use crate::rpc::ddl::{ AlterDatabaseTask, AlterTableTask, CommentOnTask, CreateDatabaseTask, CreateFlowTask, CreateTableTask, CreateViewTask, DropDatabaseTask, DropFlowTask, DropTableTask, DropViewTask, - EventContext, PurgeDroppedTableTask, QueryContext, SubmitDdlTaskRequest, SubmitDdlTaskResponse, - TruncateTableTask, UndropTableTask, + PersistentEventContext, PurgeDroppedTableTask, QueryContext, SubmitDdlTaskRequest, + SubmitDdlTaskResponse, TruncateTableTask, UndropTableTask, }; const MAX_REGION_ROUTE_CHANGE_RETRIES: usize = 3; @@ -188,7 +188,6 @@ pub trait RepartitionProcedureFactory: Send + Sync { source: RepartitionSource, to_exprs: Vec, timeout: Option, - event_context: EventContext, ) -> std::result::Result; fn register_loaders( @@ -342,7 +341,7 @@ impl DdlManager { repartition: Repartition, wait: bool, timeout: Duration, - event_context: EventContext, + event_context: PersistentEventContext, ) -> Result<(ProcedureId, Option)> { let context = self.create_context(); @@ -378,14 +377,14 @@ impl DdlManager { source, into_partition_exprs, Some(timeout), - event_context, ) .context(CreateRepartitionProcedureSnafu)?; self.repartition_procedure_factory .ensure_gc_requirement() .await .context(PersistRepartitionGcRequirementSnafu)?; - let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure)); + let procedure_with_id = + ProcedureWithId::with_random_id(Box::new(procedure)).with_event_context(event_context); if wait { self.execute_procedure_and_wait(procedure_with_id).await } else { @@ -401,7 +400,7 @@ impl DdlManager { &self, table_id: TableId, alter_table_task: AlterTableTask, - event_context: EventContext, + event_context: PersistentEventContext, ddl_options: DdlOptions, ) -> Result<(ProcedureId, Option)> { // make alter_table_task mutable so we can call .take() on its field @@ -456,12 +455,12 @@ impl DdlManager { let procedure = AlterTableProcedure::new_with_region_locks( table_id, alter_table_task.clone(), - event_context.clone(), region_ids_to_lock, context, )?; - let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure)); + let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure)) + .with_event_context(event_context.clone()); let result = self.execute_procedure_and_wait(procedure_with_id).await?; if result .1 @@ -490,14 +489,18 @@ impl DdlManager { &self, create_table_task: CreateTableTask, query_context: QueryContext, - event_context: EventContext, + event_context: PersistentEventContext, ) -> Result<(ProcedureId, Option)> { let context = self.create_context(); - let procedure = - CreateTableProcedure::new(create_table_task, query_context, event_context, context)?; + let procedure = CreateTableProcedure::new_with_query_context( + create_table_task, + query_context, + context, + )?; - let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure)); + let procedure_with_id = + ProcedureWithId::with_random_id(Box::new(procedure)).with_event_context(event_context); self.execute_procedure_and_wait(procedure_with_id).await } @@ -507,13 +510,14 @@ impl DdlManager { pub async fn submit_create_view_task( &self, create_view_task: CreateViewTask, - event_context: EventContext, + event_context: PersistentEventContext, ) -> Result<(ProcedureId, Option)> { let context = self.create_context(); - let procedure = CreateViewProcedure::new(create_view_task, event_context, context); + let procedure = CreateViewProcedure::new(create_view_task, context); - let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure)); + let procedure_with_id = + ProcedureWithId::with_random_id(Box::new(procedure)).with_event_context(event_context); self.execute_procedure_and_wait(procedure_with_id).await } @@ -524,18 +528,15 @@ impl DdlManager { &self, create_table_tasks: Vec, physical_table_id: TableId, - event_context: EventContext, + event_context: PersistentEventContext, ) -> Result<(ProcedureId, Option)> { let context = self.create_context(); - let procedure = CreateLogicalTablesProcedure::new( - create_table_tasks, - physical_table_id, - event_context, - context, - ); + let procedure = + CreateLogicalTablesProcedure::new(create_table_tasks, physical_table_id, context); - let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure)); + let procedure_with_id = + ProcedureWithId::with_random_id(Box::new(procedure)).with_event_context(event_context); self.execute_procedure_and_wait(procedure_with_id).await } @@ -546,18 +547,15 @@ impl DdlManager { &self, alter_table_tasks: Vec, physical_table_id: TableId, - event_context: EventContext, + event_context: PersistentEventContext, ) -> Result<(ProcedureId, Option)> { let context = self.create_context(); - let procedure = AlterLogicalTablesProcedure::new( - alter_table_tasks, - physical_table_id, - event_context, - context, - ); + let procedure = + AlterLogicalTablesProcedure::new(alter_table_tasks, physical_table_id, context); - let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure)); + let procedure_with_id = + ProcedureWithId::with_random_id(Box::new(procedure)).with_event_context(event_context); self.execute_procedure_and_wait(procedure_with_id).await } @@ -567,13 +565,14 @@ impl DdlManager { pub async fn submit_drop_table_task( &self, drop_table_task: DropTableTask, - event_context: EventContext, + event_context: PersistentEventContext, ) -> Result<(ProcedureId, Option)> { let context = self.create_context(); - let procedure = DropTableProcedure::new(drop_table_task, context, event_context); + let procedure = DropTableProcedure::new(drop_table_task, context); - let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure)); + let procedure_with_id = + ProcedureWithId::with_random_id(Box::new(procedure)).with_event_context(event_context); self.execute_procedure_and_wait(procedure_with_id).await } @@ -584,7 +583,7 @@ impl DdlManager { pub async fn submit_undrop_table_task( &self, undrop_table_task: UndropTableTask, - event_context: EventContext, + event_context: PersistentEventContext, ) -> Result<(ProcedureId, Option)> { #[cfg(not(feature = "enterprise"))] { @@ -610,9 +609,9 @@ impl DdlManager { undrop_table_task, context, Some(original_table_name), - event_context, ); - let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure)); + let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure)) + .with_event_context(event_context); self.execute_procedure_and_wait(procedure_with_id).await } @@ -624,7 +623,7 @@ impl DdlManager { pub async fn submit_purge_dropped_table_task( &self, purge_dropped_table_task: PurgeDroppedTableTask, - event_context: EventContext, + event_context: PersistentEventContext, ) -> Result<(ProcedureId, Option)> { #[cfg(not(feature = "enterprise"))] { @@ -638,9 +637,9 @@ impl DdlManager { #[cfg(feature = "enterprise")] { let context = self.create_context(); - let procedure = - PurgeDroppedTableProcedure::new(purge_dropped_table_task, context, event_context); - let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure)); + let procedure = PurgeDroppedTableProcedure::new(purge_dropped_table_task, context); + let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure)) + .with_event_context(event_context); self.execute_procedure_and_wait(procedure_with_id).await } @@ -666,12 +665,11 @@ impl DdlManager { #[cfg(feature = "enterprise")] { let context = self.create_context(); - let procedure = PurgeDroppedTableProcedure::new_if_expired( - purge_dropped_table_task, - context, - EventContext::new(TriggerReason::ScheduledGc), - ); - let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure)); + let event_context = PersistentEventContext::new(TriggerReason::ScheduledGc); + let procedure = + PurgeDroppedTableProcedure::new_if_expired(purge_dropped_table_task, context); + let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure)) + .with_event_context(event_context); self.execute_procedure_and_wait(procedure_with_id).await } @@ -688,7 +686,7 @@ impl DdlManager { options, creator, }: CreateDatabaseTask, - event_context: EventContext, + event_context: PersistentEventContext, ) -> Result<(ProcedureId, Option)> { let context = self.create_context(); let procedure = CreateDatabaseProcedure::new( @@ -697,10 +695,10 @@ impl DdlManager { create_if_not_exists, options, creator, - event_context, context, ); - let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure)); + let procedure_with_id = + ProcedureWithId::with_random_id(Box::new(procedure)).with_event_context(event_context); self.execute_procedure_and_wait(procedure_with_id).await } @@ -714,12 +712,12 @@ impl DdlManager { schema, drop_if_exists, }: DropDatabaseTask, - event_context: EventContext, + event_context: PersistentEventContext, ) -> Result<(ProcedureId, Option)> { let context = self.create_context(); - let procedure = - DropDatabaseProcedure::new(catalog, schema, drop_if_exists, event_context, context); - let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure)); + let procedure = DropDatabaseProcedure::new(catalog, schema, drop_if_exists, context); + let procedure_with_id = + ProcedureWithId::with_random_id(Box::new(procedure)).with_event_context(event_context); self.execute_procedure_and_wait(procedure_with_id).await } @@ -727,11 +725,12 @@ impl DdlManager { pub async fn submit_alter_database( &self, alter_database_task: AlterDatabaseTask, - event_context: EventContext, + event_context: PersistentEventContext, ) -> Result<(ProcedureId, Option)> { let context = self.create_context(); - let procedure = AlterDatabaseProcedure::new(alter_database_task, event_context, context)?; - let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure)); + let procedure = AlterDatabaseProcedure::new(alter_database_task, context)?; + let procedure_with_id = + ProcedureWithId::with_random_id(Box::new(procedure)).with_event_context(event_context); self.execute_procedure_and_wait(procedure_with_id).await } @@ -742,12 +741,12 @@ impl DdlManager { &self, create_flow: CreateFlowTask, query_context: QueryContext, - event_context: EventContext, + event_context: PersistentEventContext, ) -> Result<(ProcedureId, Option)> { let context = self.create_context(); - let procedure = - CreateFlowProcedure::new(create_flow, query_context, event_context, context); - let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure)); + let procedure = CreateFlowProcedure::new(create_flow, query_context, context); + let procedure_with_id = + ProcedureWithId::with_random_id(Box::new(procedure)).with_event_context(event_context); self.execute_procedure_and_wait(procedure_with_id).await } @@ -757,11 +756,12 @@ impl DdlManager { pub async fn submit_drop_flow_task( &self, drop_flow: DropFlowTask, - event_context: EventContext, + event_context: PersistentEventContext, ) -> Result<(ProcedureId, Option)> { let context = self.create_context(); - let procedure = DropFlowProcedure::new(drop_flow, event_context, context); - let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure)); + let procedure = DropFlowProcedure::new(drop_flow, context); + let procedure_with_id = + ProcedureWithId::with_random_id(Box::new(procedure)).with_event_context(event_context); self.execute_procedure_and_wait(procedure_with_id).await } @@ -771,11 +771,12 @@ impl DdlManager { pub async fn submit_drop_view_task( &self, drop_view: DropViewTask, - event_context: EventContext, + event_context: PersistentEventContext, ) -> Result<(ProcedureId, Option)> { let context = self.create_context(); - let procedure = DropViewProcedure::new(drop_view, event_context, context); - let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure)); + let procedure = DropViewProcedure::new(drop_view, context); + let procedure_with_id = + ProcedureWithId::with_random_id(Box::new(procedure)).with_event_context(event_context); self.execute_procedure_and_wait(procedure_with_id).await } @@ -786,17 +787,13 @@ impl DdlManager { &self, truncate_table_task: TruncateTableTask, table_info_value: DeserializedValueWithBytes, - event_context: EventContext, + event_context: PersistentEventContext, ) -> Result<(ProcedureId, Option)> { let context = self.create_context(); - let procedure = TruncateTableProcedure::new( - truncate_table_task, - table_info_value, - context, - event_context, - ); + let procedure = TruncateTableProcedure::new(truncate_table_task, table_info_value, context); - let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure)); + let procedure_with_id = + ProcedureWithId::with_random_id(Box::new(procedure)).with_event_context(event_context); self.execute_procedure_and_wait(procedure_with_id).await } @@ -806,6 +803,7 @@ impl DdlManager { pub async fn submit_comment_on_task( &self, mut comment_on_task: CommentOnTask, + event_context: PersistentEventContext, ) -> Result<(ProcedureId, Option)> { let context = self.create_context(); comment_on_task @@ -815,7 +813,8 @@ impl DdlManager { ) .await?; let procedure = CommentOnProcedure::new(comment_on_task, context); - let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure)); + let procedure_with_id = + ProcedureWithId::with_random_id(Box::new(procedure)).with_event_context(event_context); self.execute_procedure_and_wait(procedure_with_id).await } @@ -869,7 +868,10 @@ impl DdlManager { timeout, task, } = request; - let event_context = EventContext::from_query_context(&query_context); + let event_context = ctx + .event_context + .clone() + .unwrap_or_else(|| crate::rpc::ddl::event_context_from_query_context(&query_context)); let ddl_options = DdlOptions { wait, timeout }; async move { debug!("Submitting Ddl task: {:?}", task); @@ -924,7 +926,9 @@ impl DdlManager { DropView(drop_view_task) => { handle_drop_view_task(self, drop_view_task, event_context).await } - CommentOn(comment_on_task) => handle_comment_on_task(self, comment_on_task).await, + CommentOn(comment_on_task) => { + handle_comment_on_task(self, comment_on_task, event_context).await + } #[cfg(feature = "enterprise")] CreateTrigger(create_trigger_task) => { handle_create_trigger_task(self, create_trigger_task, query_context).await @@ -943,7 +947,7 @@ impl DdlManager { async fn handle_truncate_table_task( ddl_manager: &DdlManager, truncate_table_task: TruncateTableTask, - event_context: EventContext, + event_context: PersistentEventContext, ) -> Result { let table_id = truncate_table_task.table_id; let table_metadata_manager = &ddl_manager.table_metadata_manager(); @@ -983,7 +987,7 @@ async fn handle_alter_table_task( ddl_manager: &DdlManager, alter_table_task: AlterTableTask, ddl_options: DdlOptions, - event_context: EventContext, + event_context: PersistentEventContext, ) -> Result { let table_ref = alter_table_task.table_ref(); @@ -1030,7 +1034,7 @@ async fn handle_alter_table_task( async fn handle_drop_table_task( ddl_manager: &DdlManager, drop_table_task: DropTableTask, - event_context: EventContext, + event_context: PersistentEventContext, ) -> Result { let table_id = drop_table_task.table_id; let (id, _) = ddl_manager @@ -1048,7 +1052,7 @@ async fn handle_drop_table_task( async fn handle_undrop_table_task( ddl_manager: &DdlManager, undrop_table_task: UndropTableTask, - event_context: EventContext, + event_context: PersistentEventContext, ) -> Result { let table_id = undrop_table_task.table_id; let (id, _) = ddl_manager @@ -1066,7 +1070,7 @@ async fn handle_undrop_table_task( async fn handle_purge_dropped_table_task( ddl_manager: &DdlManager, purge_dropped_table_task: PurgeDroppedTableTask, - event_context: EventContext, + event_context: PersistentEventContext, ) -> Result { let (id, _) = ddl_manager .submit_purge_dropped_table_task(purge_dropped_table_task, event_context) @@ -1084,7 +1088,7 @@ async fn handle_create_table_task( ddl_manager: &DdlManager, create_table_task: CreateTableTask, query_context: QueryContext, - event_context: EventContext, + event_context: PersistentEventContext, ) -> Result { let (id, output) = ddl_manager .submit_create_table_task(create_table_task, query_context, event_context) @@ -1110,7 +1114,7 @@ async fn handle_create_table_task( async fn handle_create_logical_table_tasks( ddl_manager: &DdlManager, create_table_tasks: Vec, - event_context: EventContext, + event_context: PersistentEventContext, ) -> Result { ensure!( !create_table_tasks.is_empty(), @@ -1155,7 +1159,7 @@ async fn handle_create_logical_table_tasks( async fn handle_create_database_task( ddl_manager: &DdlManager, create_database_task: CreateDatabaseTask, - event_context: EventContext, + event_context: PersistentEventContext, ) -> Result { let catalog = create_database_task.catalog.clone(); let schema = create_database_task.schema.clone(); @@ -1178,7 +1182,7 @@ async fn handle_create_database_task( async fn handle_drop_database_task( ddl_manager: &DdlManager, drop_database_task: DropDatabaseTask, - event_context: EventContext, + event_context: PersistentEventContext, ) -> Result { let (id, _) = ddl_manager .submit_drop_database(drop_database_task.clone(), event_context) @@ -1199,7 +1203,7 @@ async fn handle_drop_database_task( async fn handle_alter_database_task( ddl_manager: &DdlManager, alter_database_task: AlterDatabaseTask, - event_context: EventContext, + event_context: PersistentEventContext, ) -> Result { let (id, _) = ddl_manager .submit_alter_database(alter_database_task.clone(), event_context) @@ -1221,7 +1225,7 @@ async fn handle_alter_database_task( async fn handle_drop_flow_task( ddl_manager: &DdlManager, drop_flow_task: DropFlowTask, - event_context: EventContext, + event_context: PersistentEventContext, ) -> Result { let (id, _) = ddl_manager .submit_drop_flow_task(drop_flow_task.clone(), event_context) @@ -1266,7 +1270,7 @@ async fn handle_drop_trigger_task( async fn handle_drop_view_task( ddl_manager: &DdlManager, drop_view_task: DropViewTask, - event_context: EventContext, + event_context: PersistentEventContext, ) -> Result { let (id, _) = ddl_manager .submit_drop_view_task(drop_view_task.clone(), event_context) @@ -1289,7 +1293,7 @@ async fn handle_create_flow_task( ddl_manager: &DdlManager, create_flow_task: CreateFlowTask, query_context: QueryContext, - event_context: EventContext, + event_context: PersistentEventContext, ) -> Result { let (id, output) = ddl_manager .submit_create_flow_task(create_flow_task.clone(), query_context, event_context) @@ -1349,7 +1353,7 @@ async fn handle_create_trigger_task( async fn handle_alter_logical_table_tasks( ddl_manager: &DdlManager, alter_table_tasks: Vec, - event_context: EventContext, + event_context: PersistentEventContext, ) -> Result { ensure!( !alter_table_tasks.is_empty(), @@ -1388,7 +1392,7 @@ async fn handle_alter_logical_table_tasks( async fn handle_create_view_task( ddl_manager: &DdlManager, create_view_task: CreateViewTask, - event_context: EventContext, + event_context: PersistentEventContext, ) -> Result { let (id, output) = ddl_manager .submit_create_view_task(create_view_task, event_context) @@ -1414,9 +1418,10 @@ async fn handle_create_view_task( async fn handle_comment_on_task( ddl_manager: &DdlManager, comment_on_task: CommentOnTask, + event_context: PersistentEventContext, ) -> Result { let (id, _) = ddl_manager - .submit_comment_on_task(comment_on_task.clone()) + .submit_comment_on_task(comment_on_task.clone(), event_context) .await?; let procedure_id = id.to_string(); @@ -1469,7 +1474,7 @@ mod tests { use crate::procedure_executor::ExecutorContext; use crate::region_keeper::MemoryRegionKeeper; use crate::region_registry::LeaderRegionRegistry; - use crate::rpc::ddl::{CreatorGrantIntent, EventContext, UndropTableTask}; + use crate::rpc::ddl::{CreatorGrantIntent, PersistentEventContext, UndropTableTask}; #[cfg(not(feature = "enterprise"))] use crate::rpc::ddl::{DdlTask, PurgeDroppedTableTask, QueryContext, SubmitDdlTaskRequest}; use crate::sequence::SequenceBuilder; @@ -1506,7 +1511,6 @@ mod tests { _source: RepartitionSource, _to_exprs: Vec, _timeout: Option, - _event_context: EventContext, ) -> std::result::Result { unimplemented!() } @@ -1680,7 +1684,10 @@ mod tests { let ddl_manager = build_soft_drop_test_ddl_manager(); let err = ddl_manager - .submit_undrop_table_task(UndropTableTask { table_id: 1024 }, EventContext::default()) + .submit_undrop_table_task( + UndropTableTask { table_id: 1024 }, + PersistentEventContext::default(), + ) .await .unwrap_err(); @@ -1694,7 +1701,10 @@ mod tests { let ddl_manager = build_soft_drop_test_ddl_manager(); let err = ddl_manager - .submit_undrop_table_task(UndropTableTask { table_id: 1024 }, EventContext::default()) + .submit_undrop_table_task( + UndropTableTask { table_id: 1024 }, + PersistentEventContext::default(), + ) .await .unwrap_err(); assert!(matches!(err, crate::error::Error::Unsupported { .. })); @@ -1702,7 +1712,7 @@ mod tests { let err = ddl_manager .submit_purge_dropped_table_task( PurgeDroppedTableTask { table_id: 1024 }, - EventContext::default(), + PersistentEventContext::default(), ) .await .unwrap_err(); diff --git a/src/common/meta/src/procedure_executor.rs b/src/common/meta/src/procedure_executor.rs index c0165dd80f..b812aeb6aa 100644 --- a/src/common/meta/src/procedure_executor.rs +++ b/src/common/meta/src/procedure_executor.rs @@ -15,6 +15,7 @@ use std::sync::Arc; use api::v1::meta::{ProcedureDetailResponse, ReconcileRequest, ReconcileResponse}; +use common_event_recorder::PersistentEventContext; use common_procedure::{ProcedureId, ProcedureManagerRef}; use common_telemetry::tracing_context::W3cTrace; use snafu::{OptionExt, ResultExt}; @@ -30,9 +31,11 @@ use crate::rpc::procedure::{ }; /// The context of procedure executor. -#[derive(Debug, Default)] +#[derive(Debug, Clone, Default)] pub struct ExecutorContext { pub tracing_context: Option, + pub actor: Option, + pub event_context: Option, } /// The procedure executor that accepts ddl, region migration task etc. diff --git a/src/common/meta/src/rpc/ddl.rs b/src/common/meta/src/rpc/ddl.rs index 9cb75aa29c..a7c6030fb0 100644 --- a/src/common/meta/src/rpc/ddl.rs +++ b/src/common/meta/src/rpc/ddl.rs @@ -43,14 +43,14 @@ use api::v1::{ use base64::Engine as _; use base64::engine::general_purpose; use common_catalog::{format_full_flow_name, format_full_table_name}; +use common_base::protocol::Channel; use common_error::ext::BoxedError; -use common_session::channel_protocol; +pub use common_event_recorder::{PersistentEventContext, TriggerReason}; use common_time::{DatabaseTimeToLive, Timestamp}; use prost::Message; use serde::{Deserialize, Serialize}; use serde_with::{DefaultOnNull, serde_as}; use snafu::{OptionExt, ResultExt}; -use strum::{AsRefStr, EnumString}; use table::metadata::{TableId, TableInfo}; use table::requests::validate_database_option; use table::table_name::TableName; @@ -1668,83 +1668,21 @@ pub struct QueryContext { pub sst_min_sequences: HashMap, } -impl QueryContext { - /// Returns the protocol name represented by the wire channel value. - pub fn channel_protocol(&self) -> Option<&'static str> { - channel_protocol(self.channel) +/// Builds event context from frontend query context metadata. +pub fn event_context_from_query_context(query_context: &QueryContext) -> PersistentEventContext { + let reason = query_context + .extensions + .get(TRIGGER_REASON_EXTENSION_KEY) + .map(|reason| TriggerReason::from_extension(reason)) + .unwrap_or_default(); + let channel = Channel::from(u32::from(query_context.channel)); + let context = PersistentEventContext::new(reason); + if channel == Channel::Unknown { + context + } else { + context.with_protocol(channel.as_ref()) } } - -/// The stable context recorded for a procedure event. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub struct EventContext { - pub reason: TriggerReason, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub protocol: Option, - #[serde(default, skip_serializing_if = "serde_json::Map::is_empty")] - pub extensions: serde_json::Map, -} - -impl EventContext { - /// Creates an event context with no additional extensions. - pub fn new(reason: TriggerReason) -> Self { - Self { - reason, - protocol: None, - extensions: Default::default(), - } - } - - /// Adds the protocol that originated the operation. - pub fn with_protocol(mut self, protocol: impl Into) -> Self { - self.protocol = Some(protocol.into()); - self - } - - /// Builds an event context from frontend query context metadata. - pub fn from_query_context(query_context: &QueryContext) -> Self { - let reason = query_context - .extensions - .get(TRIGGER_REASON_EXTENSION_KEY) - .map(|reason| TriggerReason::from_extension(reason)) - .unwrap_or_default(); - let context = Self::new(reason); - if let Some(protocol) = query_context.channel_protocol() { - context.with_protocol(protocol) - } else { - context - } - } -} - -impl Default for EventContext { - fn default() -> Self { - Self::new(TriggerReason::default()) - } -} - -/// The stable classification of a procedure trigger. -#[derive( - Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize, AsRefStr, EnumString, -)] -#[serde(rename_all = "snake_case")] -#[strum(serialize_all = "snake_case")] -pub enum TriggerReason { - Manual, - AutoCreate, - AutoAlter, - AutoRepartition, - AutoRebalance, - RegionFailover, - ScheduledGc, - #[default] - #[serde(other)] - Unknown, -} - -impl TriggerReason { - pub fn from_extension(value: &str) -> Self { - value.parse().unwrap_or_default() } } @@ -2329,7 +2267,7 @@ mod tests { #[test] fn test_event_context_serialization() { - let context = EventContext::new(TriggerReason::Manual).with_protocol("mysql"); + let context = PersistentEventContext::new(TriggerReason::Manual).with_protocol("mysql"); assert_eq!( serde_json::json!({ @@ -2341,7 +2279,7 @@ mod tests { assert_eq!( serde_json::json!({ "reason": "manual" }), - serde_json::to_value(EventContext::new(TriggerReason::Manual)).unwrap() + serde_json::to_value(PersistentEventContext::new(TriggerReason::Manual)).unwrap() ); } @@ -2354,7 +2292,7 @@ mod tests { ); query_context.channel = 4; - let event_context = EventContext::from_query_context(&query_context); + let event_context = event_context_from_query_context(&query_context); assert_eq!(event_context.reason, TriggerReason::AutoCreate); assert_eq!(event_context.protocol.as_deref(), Some("prometheus")); diff --git a/src/common/procedure-test/src/lib.rs b/src/common/procedure-test/src/lib.rs index f6d816381d..2d342ed91a 100644 --- a/src/common/procedure-test/src/lib.rs +++ b/src/common/procedure-test/src/lib.rs @@ -84,6 +84,7 @@ pub async fn execute_procedure_until_done(procedure: &mut dyn Procedure) -> Opti let ctx = Context { procedure_id: ProcedureId::random(), provider: Arc::new(MockContextProvider::default()), + event_context: None, }; loop { @@ -110,6 +111,7 @@ pub async fn execute_procedure_once( let ctx = Context { procedure_id, provider: Arc::new(provider), + event_context: None, }; match procedure.execute(&ctx).await.unwrap() { @@ -137,6 +139,7 @@ pub async fn execute_until_suspended_or_done( let ctx = Context { procedure_id, provider: Arc::new(provider), + event_context: None, }; loop { @@ -155,6 +158,7 @@ pub fn new_test_procedure_context() -> Context { Context { procedure_id: ProcedureId::random(), provider: Arc::new(MockContextProvider::default()), + event_context: None, } } diff --git a/src/common/procedure/src/event.rs b/src/common/procedure/src/event.rs index c3100b34cd..a4a6c290bf 100644 --- a/src/common/procedure/src/event.rs +++ b/src/common/procedure/src/event.rs @@ -19,13 +19,13 @@ use api::v1::{ColumnSchema, Row}; use common_event_recorder::Event; use common_event_recorder::error::{Result, SerializeEventSnafu}; use common_event_recorder::event_table::{ - PROCEDURE_ERROR_COLUMN, PROCEDURE_ID_COLUMN, PROCEDURE_STATE_COLUMN, PROCEDURE_TRIGGER_COLUMN, - jsonb_value, procedure_event_column_schemas, + EVENT_CONTEXT_COLUMN, PROCEDURE_ERROR_COLUMN, PROCEDURE_ID_COLUMN, PROCEDURE_STATE_COLUMN, + PROCEDURE_TRIGGER_COLUMN, jsonb_value, nullable_json, procedure_event_column_schemas, }; use common_time::timestamp::{TimeUnit, Timestamp}; use snafu::ResultExt; -use crate::{EventTrigger, ProcedureId, ProcedureState}; +use crate::{EventTrigger, ProcedureContext, ProcedureId, ProcedureState}; pub const EVENTS_TABLE_PROCEDURE_ID_COLUMN_NAME: &str = PROCEDURE_ID_COLUMN.name(); pub const EVENTS_TABLE_PROCEDURE_STATE_COLUMN_NAME: &str = PROCEDURE_STATE_COLUMN.name(); @@ -43,6 +43,8 @@ pub struct ProcedureEvent { pub state: ProcedureState, /// The lifecycle trigger that caused the event to be emitted. pub trigger: EventTrigger, + /// Context associated with the root submission. + pub context: ProcedureContext, /// The event emitted by the procedure. It's generated by [Procedure::event]. pub internal_event: Box, } @@ -53,6 +55,22 @@ impl ProcedureEvent { internal_event: Box, state: ProcedureState, trigger: EventTrigger, + ) -> Self { + Self::new_with_context( + procedure_id, + internal_event, + state, + trigger, + ProcedureContext::default(), + ) + } + + pub fn new_with_context( + procedure_id: ProcedureId, + internal_event: Box, + state: ProcedureState, + trigger: EventTrigger, + context: ProcedureContext, ) -> Self { Self { procedure_id, @@ -60,6 +78,7 @@ impl ProcedureEvent { timestamp: Timestamp::current_time(TimeUnit::Nanosecond), state, trigger, + context, } } } @@ -79,7 +98,9 @@ impl Event for ProcedureEvent { fn extra_schema(&self) -> Vec { let mut schema = procedure_event_column_schemas(); - schema.append(&mut self.internal_event.extra_schema()); + let mut internal_schema = self.internal_event.extra_schema(); + schema.append(&mut internal_schema); + schema.push(EVENT_CONTEXT_COLUMN.column_schema()); schema } @@ -97,9 +118,15 @@ impl Event for ProcedureEvent { _ => String::new(), }; let trigger = serde_json::to_value(&self.trigger).context(SerializeEventSnafu)?; + let event_context = matches!(self.trigger, EventTrigger::Submitted) + .then_some(self.context.event_context.as_ref()) + .flatten() + .map(serde_json::to_value) + .transpose() + .context(SerializeEventSnafu)?; for internal_event_extra_row in internal_event_extra_rows.iter_mut() { - let mut values = Vec::with_capacity(4 + internal_event_extra_row.values.len()); + let mut values = Vec::with_capacity(5 + internal_event_extra_row.values.len()); values.extend([ ValueData::StringValue(procedure_id.clone()).into(), ValueData::StringValue(state.clone()).into(), @@ -107,6 +134,7 @@ impl Event for ProcedureEvent { jsonb_value(&trigger), ]); values.append(&mut internal_event_extra_row.values); + values.push(nullable_json(event_context.as_ref())); rows.push(Row { values }); } @@ -126,13 +154,15 @@ mod tests { use api::v1::{ColumnDataType, ColumnSchema, Row, SemanticType, Value}; use common_error::mock::MockError; use common_error::status_code::StatusCode; - use common_event_recorder::Event; - use common_event_recorder::event_table::{PROCEDURE_TRIGGER_COLUMN, jsonb_value}; + use common_event_recorder::event_table::{ + EVENT_CONTEXT_COLUMN, PROCEDURE_TRIGGER_COLUMN, jsonb_value, + }; + use common_event_recorder::{Event, PersistentEventContext, TriggerReason}; use serde_json::json; use crate::{ - ChildSubmissionOutcome, Error, EventTrigger, ProcedureEvent, ProcedureId, ProcedureState, - RetryPhase, + ChildSubmissionOutcome, Error, EventTrigger, ProcedureContext, ProcedureEvent, ProcedureId, + ProcedureState, RetryPhase, }; #[derive(Debug)] @@ -192,6 +222,7 @@ mod tests { ValueData::StringValue(String::new()).into(), jsonb_value(&json!({"type": "Submitted"})), ValueData::StringValue("test_event1".to_string()).into(), + Value { value_data: None }, ], }, Row { @@ -201,6 +232,7 @@ mod tests { ValueData::StringValue(String::new()).into(), jsonb_value(&json!({"type": "Submitted"})), ValueData::StringValue("test_event2".to_string()).into(), + Value { value_data: None }, ], }, Row { @@ -210,6 +242,7 @@ mod tests { ValueData::StringValue(String::new()).into(), jsonb_value(&json!({"type": "Submitted"})), Value { value_data: None }, + Value { value_data: None }, ], }, ] @@ -280,10 +313,41 @@ mod tests { semantic_type: SemanticType::Field.into(), ..Default::default() }, + EVENT_CONTEXT_COLUMN.column_schema(), ] ); } + #[test] + fn procedure_event_only_records_context_on_submission() { + let context = ProcedureContext::from_event_context(PersistentEventContext::new( + TriggerReason::AutoRepartition, + )); + let submitted = ProcedureEvent::new_with_context( + ProcedureId::random(), + Box::new(TestEvent {}), + ProcedureState::Running, + EventTrigger::Submitted, + context.clone(), + ); + let succeeded = ProcedureEvent::new_with_context( + ProcedureId::random(), + Box::new(TestEvent {}), + ProcedureState::Done { output: None }, + EventTrigger::Succeeded, + context, + ); + + assert_eq!( + submitted.extra_rows().unwrap()[0].values[5], + jsonb_value(&json!({"reason": "auto_repartition"})) + ); + assert_eq!( + succeeded.extra_rows().unwrap()[0].values[5], + Value { value_data: None } + ); + } + #[test] fn test_event_trigger_serialization() { let procedure_id = ProcedureId::parse_str("00000000-0000-0000-0000-000000000001").unwrap(); diff --git a/src/common/procedure/src/lib.rs b/src/common/procedure/src/lib.rs index 1d7a200ca5..a21e5d1c47 100644 --- a/src/common/procedure/src/lib.rs +++ b/src/common/procedure/src/lib.rs @@ -30,8 +30,8 @@ pub use crate::error::{Error, Result}; pub use crate::event::ProcedureEvent; pub use crate::procedure::{ BoxedProcedure, BoxedProcedureLoader, ChildSubmissionOutcome, Context, ContextProvider, - ContextProviderRef, EventRuntimeContext, EventTrigger, LockKey, Output, ParseIdError, - PoisonKey, PoisonKeys, Procedure, ProcedureId, ProcedureInfo, ProcedureManager, + ContextProviderRef, EventContext, EventTrigger, LockKey, Output, ParseIdError, PoisonKey, + PoisonKeys, Procedure, ProcedureContext, ProcedureId, ProcedureInfo, ProcedureManager, ProcedureManagerRef, ProcedureState, ProcedureWithId, RetryPhase, Status, StringKey, }; pub use crate::watcher::Watcher; diff --git a/src/common/procedure/src/local.rs b/src/common/procedure/src/local.rs index 59eaee2d78..8fdd316758 100644 --- a/src/common/procedure/src/local.rs +++ b/src/common/procedure/src/local.rs @@ -43,8 +43,8 @@ use crate::rwlock::{KeyRwLock, OwnedKeyRwLockGuard}; use crate::store::poison_store::PoisonStoreRef; use crate::store::{ProcedureMessage, ProcedureMessages, ProcedureStore, StateStoreRef}; use crate::{ - BoxedProcedure, ContextProvider, EventTrigger, LockKey, PoisonKey, ProcedureId, - ProcedureManager, ProcedureState, ProcedureWithId, StringKey, Watcher, + BoxedProcedure, ContextProvider, EventTrigger, LockKey, PoisonKey, ProcedureContext, + ProcedureId, ProcedureManager, ProcedureState, ProcedureWithId, StringKey, Watcher, }; /// The expired time of a procedure's metadata. @@ -81,6 +81,8 @@ pub(crate) struct ProcedureMeta { type_name: String, /// Parent procedure id. parent_id: Option, + /// Context associated with the root submission. + context: ProcedureContext, /// Notify to wait for subprocedures. child_notify: Notify, /// Lock required by this procedure. @@ -105,6 +107,7 @@ impl ProcedureMeta { id: ProcedureId, procedure_state: ProcedureState, parent_id: Option, + context: ProcedureContext, lock_key: LockKey, poison_keys: PoisonKeys, type_name: &str, @@ -113,6 +116,7 @@ impl ProcedureMeta { ProcedureMeta { id, parent_id, + context, child_notify: Notify::new(), lock_key, poison_keys, @@ -716,6 +720,7 @@ impl LocalManager { procedure_state: ProcedureState, step: u32, procedure: BoxedProcedure, + context: ProcedureContext, origin: RootSubmissionOrigin, ) -> Result { ensure!(self.manager_ctx.running(), ManagerNotStartSnafu); @@ -724,6 +729,7 @@ impl LocalManager { procedure_id, procedure_state, None, + context, procedure.lock_key(), procedure.poison_keys(), procedure.type_name(), @@ -837,6 +843,7 @@ impl LocalManager { procedure_state, loaded_procedure.step, loaded_procedure.procedure, + message.context.clone(), RootSubmissionOrigin::Recovery, ) { error!(e; "Failed to recover procedure {}", procedure_id); @@ -964,6 +971,7 @@ impl ProcedureManager for LocalManager { ProcedureState::Running, 0, procedure.procedure, + procedure.context, RootSubmissionOrigin::Fresh, ) } @@ -1021,6 +1029,7 @@ pub(crate) mod test_util { ProcedureId::random(), ProcedureState::Running, None, + ProcedureContext::default(), LockKey::default(), PoisonKeys::default(), "ProcedureAdapter", @@ -1043,7 +1052,10 @@ mod tests { use common_error::mock::MockError; use common_error::status_code::StatusCode; - use common_event_recorder::{Event, EventRecorder, EventTypeFilter, EventTypeFilterRef}; + use common_event_recorder::{ + Event, EventRecorder, EventTypeFilter, EventTypeFilterRef, PersistentEventContext, + TriggerReason, + }; use common_test_util::temp_dir::create_temp_dir; use tokio::sync::oneshot; use tokio::time::{sleep, timeout}; @@ -1053,8 +1065,8 @@ mod tests { use crate::store::state_store::ObjectStateStore; use crate::test_util::InMemoryPoisonStore; use crate::{ - ChildSubmissionOutcome, Context, EventRuntimeContext, EventTrigger, Procedure, - ProcedureEvent, Status, + ChildSubmissionOutcome, Context, EventContext, EventTrigger, Procedure, ProcedureEvent, + Status, }; fn new_test_manager_context() -> ManagerContext { @@ -1103,6 +1115,13 @@ mod tests { }) .collect() } + + fn procedure_context(&self, procedure_id: ProcedureId) -> Option { + self.events.lock().unwrap().iter().find_map(|event| { + let event = event.as_any().downcast_ref::().unwrap(); + (event.procedure_id == procedure_id).then(|| event.context.clone()) + }) + } } impl EventRecorder for CapturingEventRecorder { @@ -1296,7 +1315,7 @@ mod tests { self.poison_keys.clone() } - fn event(&self, _ctx: &EventRuntimeContext<'_>) -> Option> { + fn event(&self, _ctx: &EventContext<'_>) -> Option> { Some(Box::new(TestProcedureEvent)) } } @@ -1341,6 +1360,7 @@ mod tests { vec![ProcedureWithId { id: self.child_id, procedure: Box::new(ProcedureToLoad::new("child")), + context: ProcedureContext::default(), }], false, )) @@ -1354,7 +1374,7 @@ mod tests { LockKey::default() } - fn event(&self, _ctx: &EventRuntimeContext<'_>) -> Option> { + fn event(&self, _ctx: &EventContext<'_>) -> Option> { Some(Box::new(TestProcedureEvent)) } } @@ -1384,6 +1404,7 @@ mod tests { vec![ProcedureWithId { id: self.child_id, procedure: Box::new(ProcedureToLoad::new("child")), + context: ProcedureContext::default(), }], false, )) @@ -1397,7 +1418,7 @@ mod tests { LockKey::default() } - fn event(&self, _ctx: &EventRuntimeContext<'_>) -> Option> { + fn event(&self, _ctx: &EventContext<'_>) -> Option> { Some(Box::new(TestProcedureEvent)) } } @@ -1424,7 +1445,7 @@ mod tests { LockKey::default() } - fn event(&self, ctx: &EventRuntimeContext<'_>) -> Option> { + fn event(&self, ctx: &EventContext<'_>) -> Option> { *self.captured_filter.lock().unwrap() = Some(ctx.event_type_filter.clone()); Some(Box::new(TestProcedureEvent)) } @@ -1458,6 +1479,7 @@ mod tests { procedure: Box::new(FilterCapturingProcedure { captured_filter: captured_filter.clone(), }), + context: ProcedureContext::default(), }) .await .unwrap(); @@ -1499,6 +1521,7 @@ mod tests { .submit(ProcedureWithId { id: ProcedureId::random(), procedure: Box::new(ProcedureToLoad::new("fresh submission")), + context: ProcedureContext::default(), }) .await .unwrap(); @@ -1533,6 +1556,7 @@ mod tests { ProcedureState::Running, 0, Box::new(ProcedureToLoad::new("submit root without runner")), + ProcedureContext::default(), RootSubmissionOrigin::Fresh, ) }); @@ -1584,6 +1608,9 @@ mod tests { child_id, child_submitted: false, }), + context: ProcedureContext::from_event_context(PersistentEventContext::new( + TriggerReason::AutoRepartition, + )), }) .await .unwrap(); @@ -1604,6 +1631,15 @@ mod tests { }, ))); assert!(procedure_events.contains(&(child_id, EventTrigger::Submitted))); + assert_eq!( + event_recorder + .procedure_context(child_id) + .unwrap() + .event_context + .unwrap() + .reason, + TriggerReason::AutoRepartition + ); } #[tokio::test] @@ -1637,6 +1673,7 @@ mod tests { proceed: Some(proceed_rx), child_submitted: false, }), + context: ProcedureContext::default(), }) .await .unwrap(); @@ -1711,12 +1748,15 @@ mod tests { let procedure = ProcedureToLoad::new("recovered submission"); let procedure_id = ProcedureId::random(); ProcedureStore::from_object_store(object_store) - .store_procedure( + .store_procedure_with_context( procedure_id, 0, procedure.type_name().to_string(), procedure.dump().unwrap(), None, + ProcedureContext::from_event_context(PersistentEventContext::new( + TriggerReason::AutoRebalance, + )), ) .await .unwrap(); @@ -1732,6 +1772,15 @@ mod tests { ); wait_for_trigger(&event_recorder, EventTrigger::Recovered).await; assert!(!event_recorder.triggers().contains(&EventTrigger::Submitted)); + assert_eq!( + event_recorder + .procedure_context(procedure_id) + .unwrap() + .event_context + .unwrap() + .reason, + TriggerReason::AutoRebalance + ); } #[tokio::test] @@ -1813,6 +1862,7 @@ mod tests { .submit(ProcedureWithId { id: procedure_id, procedure: Box::new(procedure), + context: ProcedureContext::default(), }) .await .unwrap(); @@ -1965,6 +2015,7 @@ mod tests { .submit(ProcedureWithId { id: procedure_id, procedure: Box::new(procedure), + context: ProcedureContext::default(), }) .await .is_ok() @@ -1986,6 +2037,7 @@ mod tests { .submit(ProcedureWithId { id: procedure_id, procedure: Box::new(ProcedureToLoad::new("submit")), + context: ProcedureContext::default(), }) .await .unwrap_err(); @@ -2054,6 +2106,7 @@ mod tests { .submit(ProcedureWithId { id: procedure_id, procedure: Box::new(procedure), + context: ProcedureContext::default(), }) .await .unwrap() @@ -2094,6 +2147,7 @@ mod tests { .submit(ProcedureWithId { id: procedure_id, procedure: Box::new(procedure), + context: ProcedureContext::default(), }) .await .unwrap_err(), @@ -2126,6 +2180,7 @@ mod tests { .submit(ProcedureWithId { id: procedure_id, procedure: Box::new(procedure), + context: ProcedureContext::default(), }) .await .is_ok() @@ -2164,6 +2219,7 @@ mod tests { .submit(ProcedureWithId { id: procedure_id, procedure: Box::new(procedure), + context: ProcedureContext::default(), }) .await .is_ok() @@ -2193,6 +2249,7 @@ mod tests { .submit(ProcedureWithId { id: procedure_id, procedure: Box::new(procedure), + context: ProcedureContext::default(), }) .await .is_ok() @@ -2217,6 +2274,7 @@ mod tests { .submit(ProcedureWithId { id: procedure_id, procedure: Box::new(procedure), + context: ProcedureContext::default(), }) .await .is_ok() @@ -2265,6 +2323,7 @@ mod tests { .submit(ProcedureWithId { id: procedure_id, procedure: Box::new(procedure), + context: ProcedureContext::default(), }) .await .unwrap_err(); @@ -2285,6 +2344,7 @@ mod tests { .submit(ProcedureWithId { id: procedure_id, procedure: Box::new(procedure), + context: ProcedureContext::default(), }) .await .is_ok() diff --git a/src/common/procedure/src/local/runner.rs b/src/common/procedure/src/local/runner.rs index a460b7376b..30b775dbd7 100644 --- a/src/common/procedure/src/local/runner.rs +++ b/src/common/procedure/src/local/runner.rs @@ -34,8 +34,8 @@ use crate::procedure::{Output, StringKey}; use crate::rwlock::OwnedKeyRwLockGuard; use crate::store::{ProcedureMessage, ProcedureStore}; use crate::{ - BoxedProcedure, ChildSubmissionOutcome, Context, Error, EventRuntimeContext, EventTrigger, - Procedure, ProcedureId, ProcedureState, ProcedureWithId, RetryPhase, Status, + BoxedProcedure, ChildSubmissionOutcome, Context, Error, EventContext, EventTrigger, Procedure, + ProcedureId, ProcedureState, ProcedureWithId, RetryPhase, Status, }; /// A guard to cleanup procedure state. @@ -227,6 +227,7 @@ impl Runner { let ctx = Context { procedure_id: self.meta.id, provider: self.manager_ctx.clone(), + event_context: self.meta.context.event_context.clone(), }; self.rolling_back = false; @@ -508,6 +509,7 @@ impl Runner { procedure_id, procedure_state, Some(self.meta.id), + self.meta.context.clone(), procedure.lock_key(), procedure.poison_keys(), procedure.type_name(), @@ -662,12 +664,13 @@ impl Runner { let data = self.procedure.dump()?; self.store - .store_procedure( + .store_procedure_with_context( self.meta.id, self.step, type_name, data, self.meta.parent_id, + self.meta.context.clone(), ) .await .map_err(|e| { @@ -708,6 +711,7 @@ impl Runner { parent_id: self.meta.parent_id, step: self.step, error: Some(error), + context: self.meta.context.clone(), }; self.store .rollback_procedure(self.meta.id, message) @@ -776,15 +780,22 @@ impl Runner { pub(crate) fn build_event(&self, trigger: EventTrigger) -> Option { let recorder = self.event_recorder.as_ref()?; let state = self.meta.state(); - let context = EventRuntimeContext { + let context = EventContext { procedure_id: self.meta.id, lifecycle_state: &state, trigger: trigger.clone(), event_type_filter: recorder.event_type_filter(), + event_context: self.meta.context.event_context.as_ref(), }; - self.procedure - .event(&context) - .map(|event| ProcedureEvent::new(self.meta.id, event, state, trigger)) + self.procedure.event(&context).map(|event| { + ProcedureEvent::new_with_context( + self.meta.id, + event, + state, + trigger, + self.meta.context.clone(), + ) + }) } /// Builds and dispatches an event from the live procedure. Delivery is best effort and is @@ -876,6 +887,7 @@ mod tests { Context { procedure_id, provider, + event_context: None, } } @@ -914,6 +926,7 @@ mod tests { Context { procedure_id, provider: Arc::new(MockProvider), + event_context: None, } } @@ -1100,6 +1113,7 @@ mod tests { ProcedureWithId { id: procedure_id, procedure: Box::new(child), + context: Default::default(), } } @@ -1480,6 +1494,7 @@ mod tests { subprocedures: vec![ProcedureWithId { id: child_id, procedure: Box::new(fail), + context: Default::default(), }], persist: true, }) @@ -1672,6 +1687,7 @@ mod tests { subprocedures: vec![ProcedureWithId { id: child_id, procedure: Box::new(fail), + context: Default::default(), }], persist: true, }) @@ -2192,6 +2208,7 @@ mod tests { subprocedures: vec![ProcedureWithId { id: child_id, procedure: Box::new(child), + context: Default::default(), }], persist: false, }) @@ -2248,6 +2265,7 @@ mod tests { subprocedures: vec![ProcedureWithId { id: child_id, procedure: Box::new(child), + context: Default::default(), }], persist: false, }) diff --git a/src/common/procedure/src/procedure.rs b/src/common/procedure/src/procedure.rs index a8af03e0da..cd11aea921 100644 --- a/src/common/procedure/src/procedure.rs +++ b/src/common/procedure/src/procedure.rs @@ -19,7 +19,7 @@ use std::str::FromStr; use std::sync::Arc; use async_trait::async_trait; -use common_event_recorder::{Event, EventTypeFilterRef}; +use common_event_recorder::{Event, EventTypeFilterRef, PersistentEventContext}; use serde::{Deserialize, Serialize}; use smallvec::{SmallVec, smallvec}; use snafu::{ResultExt, Snafu}; @@ -32,6 +32,26 @@ use crate::watcher::Watcher; pub type Output = Arc; +/// Context attached to a procedure submission and inherited by its children. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct ProcedureContext { + /// Context describing why and how the procedure was submitted. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub event_context: Option, +} + +impl ProcedureContext { + pub fn from_event_context(event_context: PersistentEventContext) -> Self { + Self { + event_context: Some(event_context), + } + } + + pub fn is_empty(&self) -> bool { + self.event_context.is_none() + } +} + /// Procedure execution status. #[derive(Debug)] pub enum Status { @@ -175,6 +195,8 @@ pub struct Context { pub procedure_id: ProcedureId, /// [ProcedureManager] context provider. pub provider: ContextProviderRef, + /// Event context inherited from the root submission. + pub event_context: Option, } impl Context { @@ -237,13 +259,13 @@ pub trait Procedure: Send { /// [`Event::extra_schema`] values. The event recorder batches events by type /// and rejects incompatible schemas; use a distinct event type for a /// different schema. - fn event(&self, _ctx: &EventRuntimeContext<'_>) -> Option> { + fn event(&self, _ctx: &EventContext<'_>) -> Option> { None } } /// Framework-owned context supplied when a procedure builds a lifecycle event. -pub struct EventRuntimeContext<'a> { +pub struct EventContext<'a> { /// Id of the procedure associated with the event. pub procedure_id: ProcedureId, /// Current framework state of the procedure. @@ -252,6 +274,8 @@ pub struct EventRuntimeContext<'a> { pub trigger: EventTrigger, /// Event types retained by the configured recorder. pub event_type_filter: EventTypeFilterRef, + /// Event context inherited from the root submission. + pub event_context: Option<&'a PersistentEventContext>, } /// Lifecycle action that causes the framework to invoke [`Procedure::event`]. @@ -336,7 +360,7 @@ impl Procedure for Box { (**self).poison_keys() } - fn event(&self, ctx: &EventRuntimeContext<'_>) -> Option> { + fn event(&self, ctx: &EventContext<'_>) -> Option> { (**self).event(ctx) } } @@ -459,6 +483,8 @@ pub struct ProcedureWithId { /// Id of the procedure. pub id: ProcedureId, pub procedure: BoxedProcedure, + /// Context associated with this procedure submission. + pub context: ProcedureContext, } impl ProcedureWithId { @@ -468,8 +494,15 @@ impl ProcedureWithId { ProcedureWithId { id: ProcedureId::random(), procedure, + context: ProcedureContext::default(), } } + + /// Attaches event context to this procedure submission. + pub fn with_event_context(mut self, event_context: PersistentEventContext) -> Self { + self.context.event_context = Some(event_context); + self + } } impl fmt::Debug for ProcedureWithId { @@ -721,11 +754,12 @@ mod tests { #[test] fn test_default_procedure_event_hook() { let state = ProcedureState::Running; - let context = EventRuntimeContext { + let context = EventContext { procedure_id: ProcedureId::random(), lifecycle_state: &state, trigger: EventTrigger::Succeeded, event_type_filter: Arc::new(common_event_recorder::EventTypeFilter::All), + event_context: None, }; assert!(DefaultEventProcedure.event(&context).is_none()); diff --git a/src/common/procedure/src/store.rs b/src/common/procedure/src/store.rs index 906682ed6e..4867046f8d 100644 --- a/src/common/procedure/src/store.rs +++ b/src/common/procedure/src/store.rs @@ -20,9 +20,9 @@ use futures::TryStreamExt; use serde::{Deserialize, Serialize}; use snafu::ResultExt; -use crate::ProcedureId; use crate::error::{Result, ToJsonSnafu}; pub(crate) use crate::store::state_store::StateStoreRef; +use crate::{ProcedureContext, ProcedureId}; pub mod poison_store; pub mod state_store; @@ -55,6 +55,9 @@ pub struct ProcedureMessage { /// Errors raised during the procedure. #[serde(default, skip_serializing_if = "Option::is_none")] pub error: Option, + /// Context associated with the root submission. + #[serde(default, skip_serializing_if = "ProcedureContext::is_empty")] + pub context: ProcedureContext, } /// A collection of all procedures' messages. @@ -88,6 +91,7 @@ impl ProcedureStore { } /// Dump the `procedure` to the storage. + #[cfg(test)] pub(crate) async fn store_procedure( &self, procedure_id: ProcedureId, @@ -95,6 +99,26 @@ impl ProcedureStore { type_name: String, data: String, parent_id: Option, + ) -> Result<()> { + self.store_procedure_with_context( + procedure_id, + step, + type_name, + data, + parent_id, + ProcedureContext::default(), + ) + .await + } + + pub(crate) async fn store_procedure_with_context( + &self, + procedure_id: ProcedureId, + step: u32, + type_name: String, + data: String, + parent_id: Option, + context: ProcedureContext, ) -> Result<()> { let message = ProcedureMessage { type_name, @@ -102,6 +126,7 @@ impl ProcedureStore { parent_id, step, error: None, + context, }; let key = ParsedKey { prefix: &self.proc_path, @@ -356,6 +381,7 @@ mod tests { } use async_trait::async_trait; + use common_event_recorder::{PersistentEventContext, TriggerReason}; use common_test_util::temp_dir::{TempDir, create_temp_dir}; use object_store::services::Fs as Builder; @@ -462,6 +488,7 @@ mod tests { parent_id: None, step: 4, error: None, + context: ProcedureContext::default(), }; let json = serde_json::to_string(&message).unwrap(); @@ -477,6 +504,21 @@ mod tests { json, r#"{"type_name":"TestMessage","data":"no parent id","parent_id":"9f805a1f-05f7-490c-9f91-bd56e3cc54c1","step":4}"# ); + + message.context = ProcedureContext::from_event_context(PersistentEventContext::new( + TriggerReason::AutoRebalance, + )); + let json = serde_json::to_string(&message).unwrap(); + assert_eq!( + json, + r#"{"type_name":"TestMessage","data":"no parent id","parent_id":"9f805a1f-05f7-490c-9f91-bd56e3cc54c1","step":4,"context":{"event_context":{"reason":"auto_rebalance"}}}"# + ); + + let legacy: ProcedureMessage = serde_json::from_str( + r#"{"type_name":"TestMessage","data":"legacy","parent_id":null,"step":1}"#, + ) + .unwrap(); + assert_eq!(legacy.context, ProcedureContext::default()); } struct MockProcedure { @@ -541,6 +583,7 @@ mod tests { parent_id: None, step: 0, error: None, + context: ProcedureContext::default(), }; assert_eq!(expect, *msg); } @@ -589,6 +632,7 @@ mod tests { parent_id: None, step: 1, error: None, + context: ProcedureContext::default(), }; store .rollback_procedure(procedure_id, message) diff --git a/src/common/procedure/src/watcher.rs b/src/common/procedure/src/watcher.rs index 4ecd75ded0..4717fcd001 100644 --- a/src/common/procedure/src/watcher.rs +++ b/src/common/procedure/src/watcher.rs @@ -124,6 +124,7 @@ mod tests { .submit(ProcedureWithId { id: procedure_id, procedure: Box::new(MockProcedure { error: true }), + context: Default::default(), }) .await .unwrap(); diff --git a/src/meta-srv/src/event/gc.rs b/src/meta-srv/src/event/gc.rs index c467bc665c..3575ac20c2 100644 --- a/src/meta-srv/src/event/gc.rs +++ b/src/meta-srv/src/event/gc.rs @@ -21,10 +21,9 @@ use api::v1::{ColumnSchema, Row, Value}; use common_event_recorder::Event; use common_event_recorder::error::{Result, SerializeEventSnafu}; use common_event_recorder::event_table::{ - EVENT_CONTEXT_COLUMN, GC_REPORT_COLUMN, REGION_ID_COLUMN, REGION_NUMBER_COLUMN, - TABLE_ID_COLUMN, column_schemas, nullable_json, + GC_REPORT_COLUMN, REGION_ID_COLUMN, REGION_NUMBER_COLUMN, TABLE_ID_COLUMN, column_schemas, + nullable_json, }; -use common_meta::rpc::ddl::EventContext; use serde::Serialize; use snafu::ResultExt; use store_api::storage::{GcReport, IndexVersion, RegionId}; @@ -67,7 +66,6 @@ struct BatchGcRegionRow { #[derive(Debug)] pub(crate) struct BatchGcEvent { payload: Option, - event_context: Option, // None emits a procedure-level lifecycle row with null Region dimensions. regions: Option>, } @@ -77,7 +75,6 @@ impl BatchGcEvent { regions: &[RegionId], full_file_listing: bool, timeout: Duration, - event_context: Option, ) -> Self { Self { payload: Some(BatchGcPayload { @@ -86,7 +83,6 @@ impl BatchGcEvent { full_file_listing, timeout, }), - event_context, regions: None, } } @@ -111,7 +107,6 @@ impl BatchGcEvent { (!regions.is_empty()).then_some(Self { payload: None, - event_context: None, regions: Some(regions), }) } @@ -136,22 +131,8 @@ impl Event for BatchGcEvent { } fn extra_rows(&self) -> Result> { - let event_context = self - .event_context - .as_ref() - .map(serde_json::to_value) - .transpose() - .context(SerializeEventSnafu)?; let Some(regions) = &self.regions else { - return Ok(vec![Row { - values: vec![ - Value { value_data: None }, - Value { value_data: None }, - Value { value_data: None }, - Value { value_data: None }, - nullable_json(event_context.as_ref()), - ], - }]); + return Ok(vec![null_row()]); }; regions @@ -164,7 +145,6 @@ impl Event for BatchGcEvent { ValueData::U32Value(region.region_id.table_id()).into(), ValueData::U32Value(region.region_id.region_number()).into(), nullable_json(Some(&report)), - nullable_json(None), ], }) }) @@ -182,11 +162,9 @@ fn schema() -> Vec { &TABLE_ID_COLUMN, ®ION_NUMBER_COLUMN, &GC_REPORT_COLUMN, - &EVENT_CONTEXT_COLUMN, ]) } -#[cfg(test)] fn null_row() -> Row { Row { values: (0..schema().len()) @@ -233,17 +211,17 @@ mod tests { use std::sync::Arc; use api::v1::ColumnSchema; - use common_event_recorder::EventTypeFilter; use common_event_recorder::event_table::{ - PROCEDURE_ERROR_COLUMN, PROCEDURE_ID_COLUMN, PROCEDURE_STATE_COLUMN, + EVENT_CONTEXT_COLUMN, PROCEDURE_ERROR_COLUMN, PROCEDURE_ID_COLUMN, PROCEDURE_STATE_COLUMN, PROCEDURE_TRIGGER_COLUMN, jsonb_value, }; use common_event_recorder::testing::assert_event_contract; + use common_event_recorder::{EventTypeFilter, PersistentEventContext, TriggerReason}; use common_meta::key::TableMetadataManager; use common_meta::kv_backend::memory::MemoryKvBackend; use common_meta::sequence::SequenceBuilder; use common_procedure::{ - EventRuntimeContext, EventTrigger, Procedure, ProcedureEvent, ProcedureId, ProcedureState, + EventContext, EventTrigger, Procedure, ProcedureEvent, ProcedureId, ProcedureState, RetryPhase, }; use store_api::storage::FileId; @@ -256,12 +234,8 @@ mod tests { fn test_batch_gc_lifecycle_event_contract() { let first = RegionId::new(1024, 1); let second = RegionId::new(1024, 2); - let event = BatchGcEvent::with_config( - &[second, first, second], - true, - Duration::from_secs(10), - None, - ); + let event = + BatchGcEvent::with_config(&[second, first, second], true, Duration::from_secs(10)); assert_event_contract(&event, BATCH_GC_EVENT_TYPE, &schema(), &[null_row()]); assert_eq!( @@ -356,22 +330,35 @@ mod tests { fn test_batch_gc_event_filter() { let procedure = batch_gc_procedure(); let running = ProcedureState::Running; - let runtime_context = |trigger, lifecycle_state, event_type_filter| EventRuntimeContext { + let manual_context = PersistentEventContext::new(TriggerReason::Manual); + let event_context = |trigger, lifecycle_state, event_type_filter| EventContext { procedure_id: ProcedureId::random(), lifecycle_state, trigger, event_type_filter: Arc::new(event_type_filter), + event_context: None, }; assert!( procedure - .event(&runtime_context( + .event(&event_context( EventTrigger::Submitted, &running, EventTypeFilter::All, )) .is_none() ); + assert!( + procedure + .event(&EventContext { + procedure_id: ProcedureId::random(), + lifecycle_state: &running, + trigger: EventTrigger::Submitted, + event_type_filter: Arc::new(EventTypeFilter::All), + event_context: Some(&manual_context), + }) + .is_some() + ); let report = GcReport { processed_regions: HashSet::from([RegionId::new(1024, 1)]), @@ -382,7 +369,7 @@ mod tests { }; assert!( procedure - .event(&runtime_context( + .event(&event_context( EventTrigger::Succeeded, &done, EventTypeFilter::All, @@ -392,7 +379,7 @@ mod tests { assert!( procedure - .event(&runtime_context( + .event(&event_context( EventTrigger::Recovered, &running, EventTypeFilter::All, @@ -401,7 +388,7 @@ mod tests { ); let retrying = procedure - .event(&runtime_context( + .event(&event_context( EventTrigger::Retrying { phase: RetryPhase::Execute, attempt: 1, @@ -423,7 +410,7 @@ mod tests { assert!( procedure - .event(&runtime_context( + .event(&event_context( EventTrigger::Recovered, &running, EventTypeFilter::Only(HashSet::from(["another_event".to_string()])), @@ -432,7 +419,7 @@ mod tests { ); assert!( procedure - .event(&runtime_context( + .event(&event_context( EventTrigger::Recovered, &running, EventTypeFilter::Only(HashSet::new()), @@ -443,7 +430,7 @@ mod tests { let missing = ProcedureState::Done { output: None }; assert!( procedure - .event(&runtime_context( + .event(&event_context( EventTrigger::Succeeded, &missing, EventTypeFilter::All, @@ -455,7 +442,7 @@ mod tests { }; assert!( procedure - .event(&runtime_context( + .event(&event_context( EventTrigger::Succeeded, &wrong, EventTypeFilter::All, @@ -468,11 +455,12 @@ mod tests { fn test_batch_gc_terminal_event_with_report() { let mut procedure = batch_gc_procedure(); let running = ProcedureState::Running; - let runtime_context = |trigger| EventRuntimeContext { + let event_context = |trigger| EventContext { procedure_id: ProcedureId::random(), lifecycle_state: &running, trigger, event_type_filter: Arc::new(EventTypeFilter::All), + event_context: None, }; let report = GcReport { need_retry_regions: HashSet::from([RegionId::new(1024, 2)]), @@ -481,7 +469,7 @@ mod tests { procedure.set_gc_report_for_test(report); let retrying = procedure - .event(&runtime_context(EventTrigger::Retrying { + .event(&event_context(EventTrigger::Retrying { phase: RetryPhase::Execute, attempt: 2, })) @@ -498,7 +486,7 @@ mod tests { assert_eq!(retrying.extra_rows().unwrap(), vec![null_row()]); for trigger in [EventTrigger::Failed, EventTrigger::Poisoned] { - let event = procedure.event(&runtime_context(trigger)).unwrap(); + let event = procedure.event(&event_context(trigger)).unwrap(); assert_eq!(event.json_payload().unwrap(), serde_json::Value::Null); assert_eq!( event.extra_rows().unwrap(), @@ -513,7 +501,7 @@ mod tests { let procedure = batch_gc_procedure(); let failed_without_report = procedure - .event(&runtime_context(EventTrigger::Failed)) + .event(&event_context(EventTrigger::Failed)) .unwrap(); assert_eq!( failed_without_report.json_payload().unwrap(), @@ -548,6 +536,7 @@ mod tests { ); let mut event_schema = procedure_schema(); event_schema.extend(schema()); + event_schema.push(EVENT_CONTEXT_COLUMN.column_schema()); let mut values = vec![ ValueData::StringValue(procedure_id.to_string()).into(), ValueData::StringValue("Done".to_string()).into(), @@ -564,6 +553,7 @@ mod tests { ) .values, ); + values.push(Value { value_data: None }); assert_event_contract( &event, @@ -589,7 +579,6 @@ mod tests { ValueData::U32Value(region_id.table_id()).into(), ValueData::U32Value(region_id.region_number()).into(), nullable_json(report.as_ref()), - nullable_json(None), ], } } @@ -607,7 +596,6 @@ mod tests { true, Duration::from_secs(10), HashMap::new(), - EventContext::default(), ) } } diff --git a/src/meta-srv/src/event/region_migration.rs b/src/meta-srv/src/event/region_migration.rs index 65b89df251..cfbe114077 100644 --- a/src/meta-srv/src/event/region_migration.rs +++ b/src/meta-srv/src/event/region_migration.rs @@ -17,14 +17,13 @@ use std::time::Duration; use api::v1::value::ValueData; use api::v1::{ColumnSchema, Row}; -use common_event_recorder::Event; use common_event_recorder::error::{Result, SerializeEventSnafu}; use common_event_recorder::event_table::{ - EVENT_CONTEXT_COLUMN, REGION_ID_COLUMN, REGION_MIGRATION_DST_NODE_ID_COLUMN, - REGION_MIGRATION_DST_PEER_ADDR_COLUMN, REGION_MIGRATION_SRC_NODE_ID_COLUMN, - REGION_MIGRATION_SRC_PEER_ADDR_COLUMN, REGION_MIGRATION_TRIGGER_REASON_COLUMN, - REGION_NUMBER_COLUMN, TABLE_ID_COLUMN, column_schemas, nullable_json, + REGION_ID_COLUMN, REGION_MIGRATION_DST_NODE_ID_COLUMN, REGION_MIGRATION_DST_PEER_ADDR_COLUMN, + REGION_MIGRATION_SRC_NODE_ID_COLUMN, REGION_MIGRATION_SRC_PEER_ADDR_COLUMN, + REGION_MIGRATION_TRIGGER_REASON_COLUMN, REGION_NUMBER_COLUMN, TABLE_ID_COLUMN, column_schemas, }; +use common_event_recorder::{Event, TriggerReason}; use serde::Serialize; use snafu::ResultExt; use store_api::storage::RegionId; @@ -50,7 +49,6 @@ pub(crate) struct RegionMigrationEvent { dst_peer_addr: String, // The timeout of the region migration. timeout: Duration, - event_context: Option, } #[derive(Debug, Serialize)] @@ -60,16 +58,15 @@ struct Payload { } impl RegionMigrationEvent { - pub fn from_persistent_ctx(ctx: &PersistentContext, include_event_context: bool) -> Self { + pub fn from_persistent_ctx(ctx: &PersistentContext, trigger_reason: TriggerReason) -> Self { Self { region_ids: ctx.region_ids.clone(), - trigger_reason: ctx.trigger_reason, + trigger_reason: RegionMigrationTriggerReason::from_trigger_reason(trigger_reason), src_node_id: ctx.from_peer.id, src_peer_addr: ctx.from_peer.addr.clone(), dst_node_id: ctx.to_peer.id, dst_peer_addr: ctx.to_peer.addr.clone(), timeout: ctx.timeout, - event_context: include_event_context.then(|| ctx.event_context.clone()), } } } @@ -89,7 +86,6 @@ impl Event for RegionMigrationEvent { ®ION_MIGRATION_SRC_PEER_ADDR_COLUMN, ®ION_MIGRATION_DST_NODE_ID_COLUMN, ®ION_MIGRATION_DST_PEER_ADDR_COLUMN, - &EVENT_CONTEXT_COLUMN, ]) } @@ -106,12 +102,6 @@ impl Event for RegionMigrationEvent { ValueData::StringValue(self.src_peer_addr.clone()).into(), ValueData::U64Value(self.dst_node_id).into(), ValueData::StringValue(self.dst_peer_addr.clone()).into(), - self.event_context - .as_ref() - .map(serde_json::to_value) - .transpose() - .context(SerializeEventSnafu) - .map(|value| nullable_json(value.as_ref()))?, ], }); } diff --git a/src/meta-srv/src/event/repartition.rs b/src/meta-srv/src/event/repartition.rs index 958bc4e11d..1faf3f1821 100644 --- a/src/meta-srv/src/event/repartition.rs +++ b/src/meta-srv/src/event/repartition.rs @@ -21,11 +21,11 @@ use api::v1::{ColumnSchema, Row}; use common_event_recorder::Event; use common_event_recorder::error::{Result, SerializeEventSnafu}; use common_event_recorder::event_table::{ - CATALOG_NAME_COLUMN, EVENT_CONTEXT_COLUMN, PARENT_PROCEDURE_ID_COLUMN, - REPARTITION_GROUP_ID_COLUMN, SCHEMA_NAME_COLUMN, SOURCE_PARTITION_EXPR_COLUMN, - SOURCE_REGION_ID_COLUMN, SOURCE_REGION_NUMBER_COLUMN, TABLE_ID_COLUMN, TABLE_NAME_COLUMN, - TARGET_PARTITION_EXPR_COLUMN, TARGET_REGION_ID_COLUMN, TARGET_REGION_NUMBER_COLUMN, - column_schemas, nullable_json, nullable_string, nullable_value, + CATALOG_NAME_COLUMN, PARENT_PROCEDURE_ID_COLUMN, REPARTITION_GROUP_ID_COLUMN, + SCHEMA_NAME_COLUMN, SOURCE_PARTITION_EXPR_COLUMN, SOURCE_REGION_ID_COLUMN, + SOURCE_REGION_NUMBER_COLUMN, TABLE_ID_COLUMN, TABLE_NAME_COLUMN, TARGET_PARTITION_EXPR_COLUMN, + TARGET_REGION_ID_COLUMN, TARGET_REGION_NUMBER_COLUMN, column_schemas, nullable_string, + nullable_value, }; use serde::Serialize; use snafu::ResultExt; @@ -61,7 +61,6 @@ pub(crate) struct RepartitionEvent { table_name: Option, table_id: Option, payload: Option, - event_context: Option, } impl RepartitionEvent { @@ -91,7 +90,6 @@ impl RepartitionEvent { target_partition_columns: intent.target_partition_columns().map(ToOwned::to_owned), timeout: persistent_ctx.timeout, }), - event_context: Some(persistent_ctx.event_context.clone()), } } @@ -102,17 +100,10 @@ impl RepartitionEvent { table_name: Some(persistent_ctx.table_name.clone()), table_id: Some(persistent_ctx.table_id), payload: None, - event_context: None, } } fn schema() -> Vec { - let mut schema = Self::base_schema(); - schema.push(EVENT_CONTEXT_COLUMN.column_schema()); - schema - } - - fn base_schema() -> Vec { column_schemas([ &CATALOG_NAME_COLUMN, &SCHEMA_NAME_COLUMN, @@ -141,19 +132,12 @@ impl Event for RepartitionEvent { } fn extra_rows(&self) -> Result> { - let event_context = self - .event_context - .as_ref() - .map(serde_json::to_value) - .transpose() - .context(SerializeEventSnafu)?; Ok(vec![Row { values: vec![ nullable_string(self.catalog_name.as_deref()), nullable_string(self.schema_name.as_deref()), nullable_string(self.table_name.as_deref()), nullable_value(self.table_id.map(ValueData::U32Value)), - nullable_json(event_context.as_ref()), ], }]) } @@ -323,7 +307,7 @@ impl RepartitionGroupEvent { } fn schema() -> Vec { - let mut schema = RepartitionEvent::base_schema(); + let mut schema = RepartitionEvent::schema(); schema.extend(column_schemas([ &PARENT_PROCEDURE_ID_COLUMN, &REPARTITION_GROUP_ID_COLUMN, @@ -377,14 +361,13 @@ mod tests { use std::time::Duration; use api::v1::value::ValueData; - use api::v1::{ColumnSchema, Row}; + use api::v1::{ColumnSchema, Row, Value}; use common_event_recorder::Event; use common_event_recorder::event_table::{ - PROCEDURE_ERROR_COLUMN, PROCEDURE_ID_COLUMN, PROCEDURE_STATE_COLUMN, + EVENT_CONTEXT_COLUMN, PROCEDURE_ERROR_COLUMN, PROCEDURE_ID_COLUMN, PROCEDURE_STATE_COLUMN, PROCEDURE_TRIGGER_COLUMN, jsonb_value, }; use common_event_recorder::testing::assert_event_contract; - use common_meta::rpc::ddl::EventContext; use common_procedure::{EventTrigger, ProcedureEvent, ProcedureId, ProcedureState}; use table::table_name::TableName; use uuid::Uuid; @@ -402,7 +385,6 @@ mod tests { TableName::new("greptime", "public", "repartition_events"), 1024, Some(Duration::from_secs(30)), - EventContext::default(), ) } @@ -438,9 +420,6 @@ mod tests { ValueData::StringValue("public".to_string()).into(), ValueData::StringValue("repartition_events".to_string()).into(), ValueData::U32Value(1024).into(), - jsonb_value(&serde_json::json!({ - "reason": "unknown", - })), ], }], ); @@ -654,7 +633,6 @@ mod tests { ValueData::StringValue("public".to_string()).into(), ValueData::StringValue("repartition_events".to_string()).into(), ValueData::U32Value(1024).into(), - Default::default(), ], }], ); @@ -690,6 +668,7 @@ mod tests { ); let mut schema = procedure_schema(); schema.extend(parent_schema()); + schema.push(EVENT_CONTEXT_COLUMN.column_schema()); assert_event_contract( &event, @@ -705,9 +684,7 @@ mod tests { ValueData::StringValue("public".to_string()).into(), ValueData::StringValue("repartition_events".to_string()).into(), ValueData::U32Value(1024).into(), - jsonb_value(&serde_json::json!({ - "reason": "unknown", - })), + Value { value_data: None }, ], }], ); @@ -719,12 +696,11 @@ mod tests { &SCHEMA_NAME_COLUMN, &TABLE_NAME_COLUMN, &TABLE_ID_COLUMN, - &EVENT_CONTEXT_COLUMN, ]) } fn group_schema() -> Vec { - let mut schema = RepartitionEvent::base_schema(); + let mut schema = parent_schema(); schema.extend(column_schemas([ &PARENT_PROCEDURE_ID_COLUMN, &REPARTITION_GROUP_ID_COLUMN, diff --git a/src/meta-srv/src/gc/ctx.rs b/src/meta-srv/src/gc/ctx.rs index 41027a6539..c9cc804986 100644 --- a/src/meta-srv/src/gc/ctx.rs +++ b/src/meta-srv/src/gc/ctx.rs @@ -29,7 +29,7 @@ use common_meta::key::DroppedTableName; use common_meta::key::TableMetadataManagerRef; use common_meta::key::table_repart::TableRepartValue; use common_meta::key::table_route::PhysicalTableRouteValue; -use common_meta::rpc::ddl::EventContext; +use common_meta::rpc::ddl::PersistentEventContext; #[cfg(feature = "enterprise")] use common_meta::rpc::ddl::PurgeDroppedTableTask; use common_procedure::{ProcedureManagerRef, ProcedureWithId, watcher}; @@ -68,7 +68,7 @@ pub(crate) trait SchedulerCtx: Send + Sync { full_file_listing: bool, timeout: Duration, region_routes_override: Region2Peers, - event_context: EventContext, + event_context: PersistentEventContext, ) -> Result; #[cfg(feature = "enterprise")] @@ -256,7 +256,7 @@ impl SchedulerCtx for DefaultGcSchedulerCtx { full_file_listing: bool, timeout: Duration, region_routes_override: Region2Peers, - event_context: EventContext, + event_context: PersistentEventContext, ) -> Result { self.gc_regions_inner( region_ids, @@ -310,7 +310,7 @@ impl DefaultGcSchedulerCtx { full_file_listing: bool, timeout: Duration, region_routes_override: Region2Peers, - event_context: EventContext, + event_context: PersistentEventContext, ) -> Result { debug!( "Sending GC instruction for {} regions (full_file_listing: {})", @@ -326,9 +326,9 @@ impl DefaultGcSchedulerCtx { full_file_listing, timeout, region_routes_override, - event_context, ); - let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure)); + let procedure_with_id = + ProcedureWithId::with_random_id(Box::new(procedure)).with_event_context(event_context); let id = procedure_with_id.id; diff --git a/src/meta-srv/src/gc/handler.rs b/src/meta-srv/src/gc/handler.rs index 4532e9a8ca..16b29a39c7 100644 --- a/src/meta-srv/src/gc/handler.rs +++ b/src/meta-srv/src/gc/handler.rs @@ -18,7 +18,7 @@ use std::time::Instant; use common_catalog::consts::MITO_ENGINE; use common_meta::datanode::{RegionManifestInfo, RegionStat}; use common_meta::peer::Peer; -use common_meta::rpc::ddl::EventContext; +use common_meta::rpc::ddl::PersistentEventContext; use common_telemetry::tracing::Instrument as _; use common_telemetry::{debug, error, info, warn}; use futures::StreamExt; @@ -37,7 +37,10 @@ use crate::gc::tracker::RegionGcInfo; use crate::metrics::METRIC_META_GC_CANDIDATE_REGIONS; impl GcScheduler { - pub(crate) async fn trigger_gc(&self, event_context: EventContext) -> Result { + pub(crate) async fn trigger_gc( + &self, + event_context: PersistentEventContext, + ) -> Result { let start_time = Instant::now(); info!("Starting GC cycle"); @@ -212,7 +215,7 @@ impl GcScheduler { datanode_to_candidates: HashMap>, force_full_listing_by_peer: HashMap>, region_routes_override_by_peer: HashMap, - event_context: EventContext, + event_context: PersistentEventContext, ) -> GcJobReport { let mut per_datanode_reports = HashMap::new(); let mut failed_datanodes: HashMap<_, Vec<_>> = HashMap::new(); @@ -283,7 +286,7 @@ impl GcScheduler { candidates: Vec<(TableId, GcCandidate)>, force_full_listing: HashSet, region_routes_override: Region2Peers, - event_context: EventContext, + event_context: PersistentEventContext, ) -> Result { info!( "Starting GC for datanode {} with {} candidate regions", diff --git a/src/meta-srv/src/gc/mock.rs b/src/meta-srv/src/gc/mock.rs index a941476ab4..94749dfb46 100644 --- a/src/meta-srv/src/gc/mock.rs +++ b/src/meta-srv/src/gc/mock.rs @@ -30,7 +30,7 @@ use common_meta::datanode::{RegionManifestInfo, RegionStat}; use common_meta::key::table_repart::TableRepartValue; use common_meta::key::table_route::PhysicalTableRouteValue; use common_meta::peer::Peer; -use common_meta::rpc::ddl::EventContext; +use common_meta::rpc::ddl::PersistentEventContext; use common_meta::rpc::router::{Region, RegionRoute}; use common_telemetry::debug; use ordered_float::OrderedFloat; @@ -75,7 +75,7 @@ pub struct MockSchedulerCtx { pub candidates: Arc>>>>, pub get_table_to_region_stats_calls: Arc>, pub gc_regions_calls: Arc>, - pub gc_event_contexts: Arc>>, + pub gc_event_contexts: Arc>>, // Error injection fields for testing pub get_table_to_region_stats_error: Arc>>, pub get_table_route_error: Arc>>, @@ -214,7 +214,7 @@ impl SchedulerCtx for MockSchedulerCtx { _full_file_listing: bool, _timeout: Duration, _region_routes_override: Region2Peers, - event_context: EventContext, + event_context: PersistentEventContext, ) -> Result { *self.gc_regions_calls.lock().unwrap() += 1; self.gc_event_contexts.lock().unwrap().push(event_context); diff --git a/src/meta-srv/src/gc/mock/basic.rs b/src/meta-srv/src/gc/mock/basic.rs index 45d6491056..ed051cc839 100644 --- a/src/meta-srv/src/gc/mock/basic.rs +++ b/src/meta-srv/src/gc/mock/basic.rs @@ -17,7 +17,7 @@ use std::sync::{Arc, Mutex}; use std::time::Instant; use common_meta::peer::Peer; -use common_meta::rpc::ddl::{EventContext, TriggerReason}; +use common_meta::rpc::ddl::{PersistentEventContext, TriggerReason}; use common_telemetry::init_default_ut_logging; use store_api::region_engine::RegionRole; use store_api::storage::{FileId, FileRefsManifest, GcReport, RegionId}; @@ -36,7 +36,7 @@ async fn test_parallel_process_datanodes_empty() { HashMap::new(), HashMap::new(), HashMap::new(), - EventContext::default(), + PersistentEventContext::default(), ) .await; @@ -105,7 +105,7 @@ async fn test_parallel_process_datanodes_with_candidates() { datanode_to_candidates, HashMap::new(), HashMap::new(), - EventContext::default(), + PersistentEventContext::default(), ) .await; @@ -189,7 +189,7 @@ async fn test_handle_tick() { assert_eq!(*ctx.gc_regions_calls.lock().unwrap(), 1); assert_eq!( ctx.gc_event_contexts.lock().unwrap().as_slice(), - &[EventContext::new(TriggerReason::ScheduledGc)] + &[PersistentEventContext::new(TriggerReason::ScheduledGc)] ); let tracker = scheduler.region_gc_tracker.lock().await; @@ -249,6 +249,6 @@ async fn test_handle_manual_gc_without_regions_records_manual_event_context() { assert_eq!( ctx.gc_event_contexts.lock().unwrap().as_slice(), - &[EventContext::new(TriggerReason::Manual)] + &[PersistentEventContext::new(TriggerReason::Manual)] ); } diff --git a/src/meta-srv/src/gc/mock/concurrent.rs b/src/meta-srv/src/gc/mock/concurrent.rs index 7dca378b71..5bce9a1be8 100644 --- a/src/meta-srv/src/gc/mock/concurrent.rs +++ b/src/meta-srv/src/gc/mock/concurrent.rs @@ -18,7 +18,7 @@ use std::time::{Duration, Instant}; use common_meta::key::table_route::PhysicalTableRouteValue; use common_meta::peer::Peer; -use common_meta::rpc::ddl::EventContext; +use common_meta::rpc::ddl::PersistentEventContext; use common_meta::rpc::router::{Region, RegionRoute}; use common_telemetry::{info, init_default_ut_logging}; use store_api::region_engine::RegionRole; @@ -102,7 +102,7 @@ async fn test_concurrent_table_processing_limits() { datanode_to_candidates, HashMap::new(), HashMap::new(), - EventContext::default(), + PersistentEventContext::default(), ) .await; @@ -190,7 +190,7 @@ async fn test_datanode_processes_tables_with_partial_gc_failures() { datanode_to_candidates, HashMap::new(), HashMap::new(), - EventContext::default(), + PersistentEventContext::default(), ) .await; @@ -296,7 +296,7 @@ async fn test_region_gc_concurrency_limit() { candidates.into_iter().map(|c| (table_id, c)).collect(), HashSet::new(), HashMap::new(), - EventContext::default(), + PersistentEventContext::default(), ) .await .unwrap(); @@ -413,7 +413,7 @@ async fn test_region_gc_concurrency_with_partial_failures() { datanode_to_candidates, HashMap::new(), HashMap::new(), - EventContext::default(), + PersistentEventContext::default(), ) .await; @@ -557,7 +557,7 @@ async fn test_region_gc_concurrency_with_retryable_errors() { datanode_to_candidates, HashMap::new(), HashMap::new(), - EventContext::default(), + PersistentEventContext::default(), ) .await; diff --git a/src/meta-srv/src/gc/mock/full_list.rs b/src/meta-srv/src/gc/mock/full_list.rs index a5d5012c6d..8193a9094e 100644 --- a/src/meta-srv/src/gc/mock/full_list.rs +++ b/src/meta-srv/src/gc/mock/full_list.rs @@ -17,7 +17,7 @@ use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant}; use common_meta::peer::Peer; -use common_meta::rpc::ddl::EventContext; +use common_meta::rpc::ddl::PersistentEventContext; use common_telemetry::init_default_ut_logging; use store_api::region_engine::RegionRole; use store_api::storage::{FileId, FileRefsManifest, GcReport, RegionId}; @@ -83,7 +83,7 @@ async fn test_full_file_listing_first_time_gc() { vec![(table_id, mock_candidate(region_id))], HashSet::new(), HashMap::new(), - EventContext::default(), + PersistentEventContext::default(), ) .await .unwrap(); @@ -157,7 +157,7 @@ async fn test_full_file_listing_interval_enforcement() { vec![(table_id, mock_candidate(region_id))], HashSet::new(), HashMap::new(), - EventContext::default(), + PersistentEventContext::default(), ) .await .unwrap(); @@ -184,7 +184,7 @@ async fn test_full_file_listing_interval_enforcement() { vec![(table_id, mock_candidate(region_id))], HashSet::new(), HashMap::new(), - EventContext::default(), + PersistentEventContext::default(), ) .await .unwrap(); @@ -260,7 +260,7 @@ async fn test_full_file_listing_no_interval_passed() { vec![(table_id, mock_candidate(region_id))], HashSet::new(), HashMap::new(), - EventContext::default(), + PersistentEventContext::default(), ) .await .unwrap(); @@ -284,7 +284,7 @@ async fn test_full_file_listing_no_interval_passed() { vec![(table_id, mock_candidate(region_id))], HashSet::new(), HashMap::new(), - EventContext::default(), + PersistentEventContext::default(), ) .await .unwrap(); diff --git a/src/meta-srv/src/gc/mock/misc.rs b/src/meta-srv/src/gc/mock/misc.rs index 9b3c35cbe9..575a5c5fc0 100644 --- a/src/meta-srv/src/gc/mock/misc.rs +++ b/src/meta-srv/src/gc/mock/misc.rs @@ -17,7 +17,7 @@ use std::sync::{Arc, Mutex}; use std::time::Instant; use common_meta::peer::Peer; -use common_meta::rpc::ddl::EventContext; +use common_meta::rpc::ddl::PersistentEventContext; use common_telemetry::init_default_ut_logging; use store_api::storage::{FileRefsManifest, GcReport, RegionId}; @@ -76,7 +76,7 @@ async fn test_empty_file_refs_manifest() { datanode_to_candidates, HashMap::new(), HashMap::new(), - EventContext::default(), + PersistentEventContext::default(), ) .await; @@ -169,7 +169,7 @@ async fn test_multiple_regions_per_table() { datanode_to_candidates, HashMap::new(), HashMap::new(), - EventContext::default(), + PersistentEventContext::default(), ) .await; diff --git a/src/meta-srv/src/gc/procedure.rs b/src/meta-srv/src/gc/procedure.rs index d745e88e2c..f9f597577a 100644 --- a/src/meta-srv/src/gc/procedure.rs +++ b/src/meta-srv/src/gc/procedure.rs @@ -24,11 +24,11 @@ use common_meta::key::table_repart::TableRepartValue; use common_meta::key::table_route::PhysicalTableRouteValue; use common_meta::lock_key::{RegionLock, TableLock}; use common_meta::peer::Peer; -use common_meta::rpc::ddl::{EventContext, TriggerReason}; +use common_meta::rpc::ddl::TriggerReason; use common_procedure::error::ToJsonSnafu; use common_procedure::{ - Context as ProcedureContext, Error as ProcedureError, EventRuntimeContext, EventTrigger, - LockKey, Procedure, ProcedureState, Result as ProcedureResult, Status, + Context as ProcedureContext, Error as ProcedureError, EventContext, EventTrigger, LockKey, + Procedure, ProcedureState, Result as ProcedureResult, Status, }; use common_telemetry::tracing::Instrument as _; use common_telemetry::tracing_context::TracingContext; @@ -200,8 +200,6 @@ pub struct BatchGcData { /// mailbox timeout duration timeout: Duration, gc_report: Option, - #[serde(default)] - event_context: EventContext, } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] @@ -219,7 +217,6 @@ pub enum State { impl BatchGcProcedure { pub const TYPE_NAME: &'static str = "metasrv-procedure::BatchGcProcedure"; - #[allow(clippy::too_many_arguments)] pub fn new( mailbox: MailboxRef, table_metadata_manager: TableMetadataManagerRef, @@ -228,7 +225,6 @@ impl BatchGcProcedure { full_file_listing: bool, timeout: Duration, region_routes_override: Region2Peers, - event_context: EventContext, ) -> Self { Self { mailbox, @@ -244,7 +240,6 @@ impl BatchGcProcedure { related_regions: HashMap::new(), file_refs: FileRefsManifest::default(), gc_report: None, - event_context, }, } } @@ -275,7 +270,6 @@ impl BatchGcProcedure { related_regions: HashMap::new(), file_refs, gc_report: Some(GcReport::default()), - event_context: EventContext::default(), }, } } @@ -1072,23 +1066,21 @@ impl Procedure for BatchGcProcedure { LockKey::new(lock_key) } - fn event( - &self, - ctx: &EventRuntimeContext<'_>, - ) -> Option> { + fn event(&self, ctx: &EventContext<'_>) -> Option> { if !ctx.event_type_filter.allows(BATCH_GC_EVENT_TYPE) { return None; } let event = match &ctx.trigger { // Keep scheduled GC low-noise; record submitted manual requests for auditability. - EventTrigger::Submitted => (self.data.event_context.reason == TriggerReason::Manual) + EventTrigger::Submitted => ctx + .event_context + .is_some_and(|context| context.reason == TriggerReason::Manual) .then(|| { BatchGcEvent::with_config( &self.data.regions, self.data.full_file_listing, self.data.timeout, - Some(self.data.event_context.clone()), ) })?, EventTrigger::Recovered | EventTrigger::ChildSubmitted { .. } => return None, @@ -1106,7 +1098,6 @@ impl Procedure for BatchGcProcedure { &self.data.regions, self.data.full_file_listing, self.data.timeout, - None, ), EventTrigger::Failed | EventTrigger::Poisoned => self .data @@ -1118,7 +1109,6 @@ impl Procedure for BatchGcProcedure { &self.data.regions, self.data.full_file_listing, self.data.timeout, - None, ) }), }; @@ -1211,16 +1201,6 @@ mod tests { assert!(report.need_retry_regions.contains(®ion_id)); } - #[test] - fn test_batch_gc_data_defaults_missing_event_context() { - let mut data = serde_json::to_value(&batch_gc_procedure().data).unwrap(); - data.as_object_mut().unwrap().remove("event_context"); - - let recovered: BatchGcData = serde_json::from_value(data).unwrap(); - - assert_eq!(recovered.event_context, EventContext::default()); - } - #[tokio::test] async fn test_send_gc_instructions_preserves_partial_report() { let first_region = RegionId::new(1024, 1); @@ -1255,7 +1235,6 @@ mod tests { true, Duration::from_secs(10), HashMap::new(), - EventContext::default(), ); procedure.data.region_routes = HashMap::from([ (first_region, (first_peer, vec![])), @@ -1296,7 +1275,6 @@ mod tests { true, Duration::from_secs(10), HashMap::new(), - EventContext::default(), ) } } diff --git a/src/meta-srv/src/gc/scheduler.rs b/src/meta-srv/src/gc/scheduler.rs index 92b362ad94..b89c0d06be 100644 --- a/src/meta-srv/src/gc/scheduler.rs +++ b/src/meta-srv/src/gc/scheduler.rs @@ -18,7 +18,7 @@ use std::time::{Duration, Instant}; use common_meta::DatanodeId; use common_meta::key::runtime_switch::RuntimeSwitchManagerRef; -use common_meta::rpc::ddl::{EventContext, TriggerReason}; +use common_meta::rpc::ddl::{PersistentEventContext, TriggerReason}; use common_telemetry::tracing::Instrument as _; use common_telemetry::{error, info}; use snafu::ResultExt; @@ -218,7 +218,7 @@ impl GcScheduler { } let span = common_telemetry::tracing::info_span!("meta_gc_handle_tick"); let report = self - .trigger_gc(EventContext::new(TriggerReason::ScheduledGc)) + .trigger_gc(PersistentEventContext::new(TriggerReason::ScheduledGc)) .instrument(span) .await?; @@ -305,7 +305,7 @@ impl GcScheduler { // No specific regions, use default tick behavior let Some(regions) = region_ids else { let report = self - .trigger_gc(EventContext::new(TriggerReason::Manual)) + .trigger_gc(PersistentEventContext::new(TriggerReason::Manual)) .await?; info!("Finished manual gc request"); return Ok(report); @@ -353,7 +353,7 @@ impl GcScheduler { full_listing, gc_timeout, Region2Peers::new(), - EventContext::new(TriggerReason::Manual), + PersistentEventContext::new(TriggerReason::Manual), ) .await?; combined_report.merge(report); @@ -367,7 +367,7 @@ impl GcScheduler { true, gc_timeout, dropped_routes_override, - EventContext::new(TriggerReason::Manual), + PersistentEventContext::new(TriggerReason::Manual), ) .await?; combined_report.merge(report); @@ -490,7 +490,7 @@ mod tests { _full_file_listing: bool, _timeout: Duration, _region_routes_override: Region2Peers, - _event_context: EventContext, + _event_context: PersistentEventContext, ) -> Result { self.gc_regions_calls.fetch_add(1, Ordering::Relaxed); panic!("gc_regions should not be called in maintenance mode") @@ -564,7 +564,7 @@ mod tests { _full_file_listing: bool, _timeout: Duration, _region_routes_override: Region2Peers, - _event_context: EventContext, + _event_context: PersistentEventContext, ) -> Result { Ok(GcReport::default()) } @@ -898,7 +898,7 @@ mod tests { _full_file_listing: bool, _timeout: Duration, _region_routes_override: Region2Peers, - _event_context: EventContext, + _event_context: PersistentEventContext, ) -> Result { crate::error::UnexpectedSnafu { violated: "mock gc failure".to_string(), diff --git a/src/meta-srv/src/procedure/region_migration.rs b/src/meta-srv/src/procedure/region_migration.rs index 25fc7d6a30..ad1553c92d 100644 --- a/src/meta-srv/src/procedure/region_migration.rs +++ b/src/meta-srv/src/procedure/region_migration.rs @@ -45,13 +45,12 @@ use common_meta::kv_backend::{KvBackendRef, ResettableKvBackendRef}; use common_meta::lock_key::{CatalogLock, RegionLock, SchemaLock, TableLock}; use common_meta::peer::Peer; use common_meta::region_keeper::{MemoryRegionKeeperRef, OperatingRegionGuard}; -use common_meta::rpc::ddl::{EventContext, TriggerReason}; +use common_meta::rpc::ddl::{PersistentEventContext, TriggerReason}; use common_procedure::error::{ Error as ProcedureError, FromJsonSnafu, Result as ProcedureResult, ToJsonSnafu, }; use common_procedure::{ - Context as ProcedureContext, EventRuntimeContext, EventTrigger, LockKey, Procedure, Status, - StringKey, + Context as ProcedureContext, EventContext, LockKey, Procedure, Status, StringKey, }; use common_telemetry::{debug, error, info}; use manager::RegionMigrationProcedureGuard; @@ -123,11 +122,9 @@ pub struct PersistentContext { /// The timeout for downgrading leader region and upgrading candidate region operations. #[serde(with = "humantime_serde", default = "default_timeout")] pub(crate) timeout: Duration, - /// The trigger reason of region migration. + /// The trigger reason persisted for compatibility with versions without procedure context. #[serde(default)] - pub(crate) trigger_reason: RegionMigrationTriggerReason, - #[serde(default)] - pub(crate) event_context: EventContext, + trigger_reason: RegionMigrationTriggerReason, } impl PersistentContext { @@ -137,7 +134,7 @@ impl PersistentContext { to_peer: Peer, region_ids: Vec, timeout: Duration, - event_context: EventContext, + trigger_reason: RegionMigrationTriggerReason, ) -> Self { #[allow(deprecated)] Self { @@ -148,17 +145,7 @@ impl PersistentContext { to_peer, region_ids, timeout, - trigger_reason: RegionMigrationTriggerReason::from_trigger_reason(event_context.reason), - event_context, - } - } - - fn normalize_trigger_reasons(&mut self) { - if self.trigger_reason != RegionMigrationTriggerReason::Unknown { - self.event_context.reason = self.trigger_reason.to_trigger_reason(); - } else if self.event_context.reason != TriggerReason::Unknown { - self.trigger_reason = - RegionMigrationTriggerReason::from_trigger_reason(self.event_context.reason); + trigger_reason, } } } @@ -173,7 +160,7 @@ impl RegionMigrationTriggerReason { } } - fn from_trigger_reason(reason: TriggerReason) -> Self { + pub(crate) fn from_trigger_reason(reason: TriggerReason) -> Self { match reason { TriggerReason::Manual => Self::Manual, TriggerReason::AutoRebalance => Self::AutoRebalance, @@ -457,6 +444,14 @@ pub struct Context { } impl Context { + pub(crate) fn trigger_reason( + &self, + event_context: Option<&PersistentEventContext>, + ) -> RegionMigrationTriggerReason { + event_context + .map(|ctx| RegionMigrationTriggerReason::from_trigger_reason(ctx.reason)) + .unwrap_or(self.persistent_ctx.trigger_reason) + } /// Returns the next operation's timeout. pub fn next_operation_timeout(&self) -> Option { self.persistent_ctx @@ -852,10 +847,9 @@ impl RegionMigrationProcedure { tracker: RegionMigrationProcedureTracker, ) -> ProcedureResult { let RegionMigrationDataOwned { - mut persistent_ctx, + persistent_ctx, state, } = serde_json::from_str(json).context(FromJsonSnafu)?; - persistent_ctx.normalize_trigger_reasons(); let guards = persistent_ctx .region_ids .iter() @@ -999,14 +993,16 @@ impl Procedure for RegionMigrationProcedure { LockKey::new(self.context.persistent_ctx.lock_key()) } - fn event(&self, ctx: &EventRuntimeContext<'_>) -> Option> { + fn event(&self, ctx: &EventContext<'_>) -> Option> { if !ctx.event_type_filter.allows(REGION_MIGRATION_EVENT_TYPE) { return None; } Some(Box::new(RegionMigrationEvent::from_persistent_ctx( &self.context.persistent_ctx, - matches!(ctx.trigger, EventTrigger::Submitted), + self.context + .trigger_reason(ctx.event_context) + .to_trigger_reason(), ))) } } @@ -1063,7 +1059,7 @@ mod tests { let procedure = RegionMigrationProcedure::new(persistent_context, context, vec![]); let serialized = procedure.dump().unwrap(); - let expected = r#"{"persistent_ctx":{"catalog_and_schema":[["greptime","public"]],"from_peer":{"id":1,"addr":""},"to_peer":{"id":2,"addr":""},"region_ids":[4398046511105],"timeout":"10s","trigger_reason":"Unknown","event_context":{"reason":"unknown"}},"state":{"region_migration_state":"RegionMigrationStart"}}"#; + let expected = r#"{"persistent_ctx":{"catalog_and_schema":[["greptime","public"]],"from_peer":{"id":1,"addr":""},"to_peer":{"id":2,"addr":""},"region_ids":[4398046511105],"timeout":"10s","trigger_reason":"Unknown"},"state":{"region_migration_state":"RegionMigrationStart"}}"#; assert_eq!(expected, serialized); } @@ -1087,11 +1083,12 @@ mod tests { for trigger in triggers { let event = procedure - .event(&EventRuntimeContext { + .event(&EventContext { procedure_id: common_procedure::ProcedureId::random(), lifecycle_state: &state, trigger, event_type_filter: Arc::new(common_event_recorder::EventTypeFilter::All), + event_context: None, }) .unwrap(); assert_eq!(event.event_type(), "region_migration"); @@ -1111,61 +1108,98 @@ mod tests { to_peer: Peer::empty(2), region_ids: vec![RegionId::new(1024, 1)], timeout: Duration::from_secs(10), - trigger_reason: RegionMigrationTriggerReason::default(), - event_context: EventContext::default(), + trigger_reason: RegionMigrationTriggerReason::Unknown, }; // NOTES: Changes it will break backward compatibility. let serialized = r#"{"catalog":"greptime","schema":"public","from_peer":{"id":1,"addr":""},"to_peer":{"id":2,"addr":""},"region_id":4398046511105}"#; let deserialized: PersistentContext = serde_json::from_str(serialized).unwrap(); assert_eq!(persistent_ctx, deserialized); - assert_eq!(deserialized.event_context, EventContext::default()); } #[test] - fn test_new_persistent_context_keeps_event_context() { - let mut event_context = - EventContext::new(TriggerReason::AutoRebalance).with_protocol("internal"); - event_context - .extensions - .insert("scheduler".into(), serde_json::json!("load_based")); - - let context = PersistentContext::new( - vec![("greptime".into(), "public".into())], - Peer::empty(1), - Peer::empty(2), - vec![RegionId::new(1024, 1)], - Duration::from_secs(10), - event_context.clone(), + fn test_legacy_trigger_reason_survives_recovery_and_repersistence() { + let serialized = r#"{"persistent_ctx":{"catalog":"greptime","schema":"public","from_peer":{"id":1,"addr":""},"to_peer":{"id":2,"addr":""},"region_id":4398046511105,"trigger_reason":"Failover"},"state":{"region_migration_state":"RegionMigrationStart"}}"#; + let env = TestingEnv::new(); + let procedure = RegionMigrationProcedure::from_json( + serialized, + env.context_factory(), + RegionMigrationProcedureTracker::default(), + ) + .unwrap(); + assert_eq!( + procedure.context.persistent_ctx.trigger_reason, + RegionMigrationTriggerReason::Failover + ); + let repersisted = procedure.dump().unwrap(); + assert_eq!( + serde_json::from_str::(&repersisted).unwrap()["persistent_ctx"]["trigger_reason"], + "Failover" ); + let recovered = RegionMigrationProcedure::from_json( + &repersisted, + env.context_factory(), + RegionMigrationProcedureTracker::default(), + ) + .unwrap(); assert_eq!( - context.trigger_reason, + recovered.context.trigger_reason(None), + RegionMigrationTriggerReason::Failover + ); + let event_context = PersistentEventContext::new(TriggerReason::AutoRebalance); + assert_eq!( + recovered.context.trigger_reason(Some(&event_context)), RegionMigrationTriggerReason::AutoRebalance ); - assert_eq!(context.event_context, event_context); + + let state = common_procedure::ProcedureState::Running; + let event = recovered + .event(&EventContext { + procedure_id: common_procedure::ProcedureId::random(), + lifecycle_state: &state, + trigger: common_procedure::EventTrigger::Recovered, + event_type_filter: Arc::new(common_event_recorder::EventTypeFilter::All), + event_context: None, + }) + .unwrap(); + assert_eq!( + event.extra_rows().unwrap()[0].values[3].value_data, + Some(api::v1::value::ValueData::StringValue( + "Failover".to_string() + )) + ); } #[test] - fn test_normalize_trigger_reasons() { - let mut context = new_persistent_context(); - context.trigger_reason = RegionMigrationTriggerReason::Manual; - context.event_context = EventContext::default(); - context.normalize_trigger_reasons(); - assert_eq!(context.event_context.reason, TriggerReason::Manual); + fn test_migration_reason_uses_event_context() { + let env = TestingEnv::new(); + let context = env.context_factory().new_context(new_persistent_context()); + let event_context = PersistentEventContext::new(TriggerReason::RegionFailover); - context.trigger_reason = RegionMigrationTriggerReason::Unknown; - context.event_context = EventContext::new(TriggerReason::RegionFailover); - context.normalize_trigger_reasons(); assert_eq!( - context.trigger_reason, + context.trigger_reason(Some(&event_context)), RegionMigrationTriggerReason::Failover ); - context.trigger_reason = RegionMigrationTriggerReason::Manual; - context.event_context = EventContext::new(TriggerReason::RegionFailover); - context.normalize_trigger_reasons(); - assert_eq!(context.event_context.reason, TriggerReason::Manual); + let procedure = + RegionMigrationProcedure::new(new_persistent_context(), env.context_factory(), vec![]); + let state = common_procedure::ProcedureState::Running; + let event = procedure + .event(&EventContext { + procedure_id: common_procedure::ProcedureId::random(), + lifecycle_state: &state, + trigger: common_procedure::EventTrigger::Submitted, + event_type_filter: Arc::new(common_event_recorder::EventTypeFilter::All), + event_context: Some(&event_context), + }) + .unwrap(); + assert_eq!( + event.extra_rows().unwrap()[0].values[3].value_data, + Some(api::v1::value::ValueData::StringValue( + "Failover".to_string() + )) + ); } #[derive(Debug, Serialize, Deserialize, Default)] diff --git a/src/meta-srv/src/procedure/region_migration/close_downgraded_region.rs b/src/meta-srv/src/procedure/region_migration/close_downgraded_region.rs index 5cdcad4503..29cd1804d0 100644 --- a/src/meta-srv/src/procedure/region_migration/close_downgraded_region.rs +++ b/src/meta-srv/src/procedure/region_migration/close_downgraded_region.rs @@ -40,7 +40,7 @@ impl State for CloseDowngradedRegion { async fn next( &mut self, ctx: &mut Context, - _procedure_ctx: &ProcedureContext, + procedure_ctx: &ProcedureContext, ) -> Result<(Box, Status)> { if let Err(err) = self.close_downgraded_leader_region(ctx).await { let downgrade_leader_datanode = &ctx.persistent_ctx.from_peer; @@ -52,7 +52,7 @@ impl State for CloseDowngradedRegion { ctx.persistent_ctx.region_ids, ctx.persistent_ctx.from_peer, ctx.persistent_ctx.to_peer, - ctx.persistent_ctx.trigger_reason, + ctx.trigger_reason(procedure_ctx.event_context.as_ref()), ctx.volatile_ctx.metrics, ); Ok((Box::new(RegionMigrationEnd), Status::done())) diff --git a/src/meta-srv/src/procedure/region_migration/downgrade_leader_region.rs b/src/meta-srv/src/procedure/region_migration/downgrade_leader_region.rs index c58c1f8cd1..97c3a8e2f3 100644 --- a/src/meta-srv/src/procedure/region_migration/downgrade_leader_region.rs +++ b/src/meta-srv/src/procedure/region_migration/downgrade_leader_region.rs @@ -382,7 +382,6 @@ mod tests { use common_meta::key::table_route::TableRouteValue; use common_meta::key::test_utils::new_test_table_info; use common_meta::peer::Peer; - use common_meta::rpc::ddl::{EventContext, TriggerReason}; use common_meta::rpc::router::{Region, RegionRoute}; use common_meta::wal_provider::RegionWalOptions; use store_api::storage::RegionId; @@ -391,7 +390,9 @@ mod tests { use super::*; use crate::error::Error; use crate::procedure::region_migration::test_util::{TestingEnv, new_procedure_context}; - use crate::procedure::region_migration::{ContextFactory, PersistentContext}; + use crate::procedure::region_migration::{ + ContextFactory, PersistentContext, RegionMigrationTriggerReason, + }; use crate::procedure::test_util::{ new_close_region_reply, new_downgrade_region_reply, send_mock_reply, }; @@ -403,7 +404,7 @@ mod tests { Peer::empty(2), vec![RegionId::new(1024, 1)], Duration::from_millis(1000), - EventContext::new(TriggerReason::Manual), + RegionMigrationTriggerReason::Unknown, ) } diff --git a/src/meta-srv/src/procedure/region_migration/manager.rs b/src/meta-srv/src/procedure/region_migration/manager.rs index 3de79735e3..e1d719d974 100644 --- a/src/meta-srv/src/procedure/region_migration/manager.rs +++ b/src/meta-srv/src/procedure/region_migration/manager.rs @@ -21,7 +21,7 @@ use std::time::Duration; use common_meta::key::table_info::TableInfoValue; use common_meta::key::table_route::TableRouteValue; use common_meta::peer::Peer; -use common_meta::rpc::ddl::EventContext; +use common_meta::rpc::ddl::PersistentEventContext; use common_meta::rpc::router::RegionRoute; use common_procedure::{ProcedureId, ProcedureManagerRef, ProcedureWithId, watcher}; use common_telemetry::{error, info, warn}; @@ -479,6 +479,7 @@ impl RegionMigrationManager { procedure_guards: Vec, catalog_and_schema: Vec<(String, String)>, ) -> Result { + let event_context = PersistentEventContext::new(task.trigger_reason.to_trigger_reason()); let procedure = RegionMigrationProcedure::new( PersistentContext::new( catalog_and_schema, @@ -486,12 +487,13 @@ impl RegionMigrationManager { task.to_peer.clone(), task.region_ids.clone(), task.timeout, - EventContext::new(task.trigger_reason.to_trigger_reason()), + task.trigger_reason, ), self.context_factory.clone(), procedure_guards, ); - let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure)); + let procedure_with_id = + ProcedureWithId::with_random_id(Box::new(procedure)).with_event_context(event_context); let procedure_id = procedure_with_id.id; info!("Starting region migration procedure {procedure_id} for {task}"); let procedure_manager = self.procedure_manager.clone(); @@ -575,6 +577,7 @@ impl RegionMigrationManager { timeout, trigger_reason, } = task.clone(); + let event_context = PersistentEventContext::new(trigger_reason.to_trigger_reason()); let procedure = RegionMigrationProcedure::new( PersistentContext::new( vec![(catalog_name, schema_name)], @@ -582,12 +585,13 @@ impl RegionMigrationManager { to_peer, vec![region_id], timeout, - EventContext::new(trigger_reason.to_trigger_reason()), + trigger_reason, ), self.context_factory.clone(), vec![guard], ); - let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure)); + let procedure_with_id = + ProcedureWithId::with_random_id(Box::new(procedure)).with_event_context(event_context); let procedure_id = procedure_with_id.id; info!("Starting region migration procedure {procedure_id} for {task}"); let procedure_manager = self.procedure_manager.clone(); diff --git a/src/meta-srv/src/procedure/region_migration/migration_abort.rs b/src/meta-srv/src/procedure/region_migration/migration_abort.rs index f3ad8052de..981c2a2025 100644 --- a/src/meta-srv/src/procedure/region_migration/migration_abort.rs +++ b/src/meta-srv/src/procedure/region_migration/migration_abort.rs @@ -41,7 +41,7 @@ impl State for RegionMigrationAbort { async fn next( &mut self, ctx: &mut Context, - _procedure_ctx: &ProcedureContext, + procedure_ctx: &ProcedureContext, ) -> Result<(Box, Status)> { warn!( "Region migration is aborted: {}, regions: {:?}, from_peer: {}, to_peer: {}, trigger_reason: {}, {}", @@ -49,7 +49,7 @@ impl State for RegionMigrationAbort { ctx.persistent_ctx.region_ids, ctx.persistent_ctx.from_peer, ctx.persistent_ctx.to_peer, - ctx.persistent_ctx.trigger_reason, + ctx.trigger_reason(procedure_ctx.event_context.as_ref()), ctx.volatile_ctx.metrics, ); error::MigrationAbortSnafu { diff --git a/src/meta-srv/src/procedure/region_migration/open_candidate_region.rs b/src/meta-srv/src/procedure/region_migration/open_candidate_region.rs index 51d7054c7c..0cb1131e54 100644 --- a/src/meta-srv/src/procedure/region_migration/open_candidate_region.rs +++ b/src/meta-srv/src/procedure/region_migration/open_candidate_region.rs @@ -47,9 +47,12 @@ impl State for OpenCandidateRegion { async fn next( &mut self, ctx: &mut Context, - _procedure_ctx: &ProcedureContext, + procedure_ctx: &ProcedureContext, ) -> Result<(Box, Status)> { - let instruction = self.build_open_region_instruction(ctx).await?; + let trigger_reason = ctx.trigger_reason(procedure_ctx.event_context.as_ref()); + let instruction = self + .build_open_region_instruction(ctx, trigger_reason) + .await?; let now = Instant::now(); self.open_candidate_region(ctx, instruction).await?; ctx.update_open_candidate_region_elapsed(now); @@ -67,11 +70,15 @@ impl OpenCandidateRegion { /// /// Abort(non-retry): /// - Datanode Table is not found. - async fn build_open_region_instruction(&self, ctx: &mut Context) -> Result { + async fn build_open_region_instruction( + &self, + ctx: &mut Context, + trigger_reason: RegionMigrationTriggerReason, + ) -> Result { let region_ids = ctx.persistent_ctx.region_ids.clone(); let from_peer_id = ctx.persistent_ctx.from_peer.id; let to_peer_id = ctx.persistent_ctx.to_peer.id; - let reason = match ctx.persistent_ctx.trigger_reason { + let reason = match trigger_reason { RegionMigrationTriggerReason::Failover => OpenRegionReason::RegionFailover, _ => OpenRegionReason::RegionMigration, }; @@ -280,7 +287,7 @@ mod tests { let mut ctx = env.context_factory().new_context(persistent_context); let err = state - .build_open_region_instruction(&mut ctx) + .build_open_region_instruction(&mut ctx, RegionMigrationTriggerReason::Unknown) .await .unwrap_err(); @@ -291,7 +298,7 @@ mod tests { #[tokio::test] async fn test_build_open_region_instruction_reason() { let state = OpenCandidateRegion; - let mut persistent_context = new_persistent_context(); + let persistent_context = new_persistent_context(); let from_peer_id = persistent_context.from_peer.id; let region_id = persistent_context.region_ids[0]; let env = TestingEnv::new(); @@ -314,7 +321,10 @@ mod tests { let mut ctx = env .context_factory() .new_context(persistent_context.clone()); - let instruction = state.build_open_region_instruction(&mut ctx).await.unwrap(); + let instruction = state + .build_open_region_instruction(&mut ctx, RegionMigrationTriggerReason::Unknown) + .await + .unwrap(); let open_regions = instruction.into_open_regions().unwrap(); assert_eq!( Some(OpenRegionReason::RegionMigration), @@ -325,9 +335,11 @@ mod tests { open_regions[0].requirements ); - persistent_context.trigger_reason = RegionMigrationTriggerReason::Failover; let mut ctx = env.context_factory().new_context(persistent_context); - let instruction = state.build_open_region_instruction(&mut ctx).await.unwrap(); + let instruction = state + .build_open_region_instruction(&mut ctx, RegionMigrationTriggerReason::Failover) + .await + .unwrap(); let open_regions = instruction.into_open_regions().unwrap(); assert_eq!( Some(OpenRegionReason::RegionFailover), diff --git a/src/meta-srv/src/procedure/region_migration/test_util.rs b/src/meta-srv/src/procedure/region_migration/test_util.rs index 7034260864..44180dc6f8 100644 --- a/src/meta-srv/src/procedure/region_migration/test_util.rs +++ b/src/meta-srv/src/procedure/region_migration/test_util.rs @@ -27,7 +27,6 @@ use common_meta::kv_backend::KvBackendRef; use common_meta::kv_backend::memory::MemoryKvBackend; use common_meta::peer::Peer; use common_meta::region_keeper::{MemoryRegionKeeper, MemoryRegionKeeperRef}; -use common_meta::rpc::ddl::EventContext; use common_meta::rpc::router::RegionRoute; use common_meta::sequence::SequenceBuilder; use common_meta::state_store::KvStateStore; @@ -53,7 +52,8 @@ use crate::procedure::region_migration::open_candidate_region::OpenCandidateRegi use crate::procedure::region_migration::update_metadata::UpdateMetadata; use crate::procedure::region_migration::upgrade_candidate_region::UpgradeCandidateRegion; use crate::procedure::region_migration::{ - Context, ContextFactory, DefaultContextFactory, PersistentContext, State, VolatileContext, + Context, ContextFactory, DefaultContextFactory, PersistentContext, + RegionMigrationTriggerReason, State, VolatileContext, }; use crate::procedure::test_util::{MailboxContext, send_mock_reply}; use crate::service::mailbox::Channel; @@ -157,6 +157,7 @@ impl TestingEnv { ProcedureContext { procedure_id: ProcedureId::random(), provider: Arc::new(MockContextProvider::default()), + event_context: None, } } @@ -190,7 +191,7 @@ pub fn new_persistent_context(from: u64, to: u64, region_id: RegionId) -> Persis Peer::empty(to), vec![region_id], Duration::from_secs(10), - EventContext::default(), + RegionMigrationTriggerReason::Unknown, ) } @@ -577,5 +578,6 @@ pub fn new_procedure_context() -> ProcedureContext { ProcedureContext { procedure_id: ProcedureId::random(), provider: Arc::new(MockContextProvider::default()), + event_context: None, } } diff --git a/src/meta-srv/src/procedure/region_migration/upgrade_candidate_region.rs b/src/meta-srv/src/procedure/region_migration/upgrade_candidate_region.rs index 16d0270354..86e1c35c10 100644 --- a/src/meta-srv/src/procedure/region_migration/upgrade_candidate_region.rs +++ b/src/meta-srv/src/procedure/region_migration/upgrade_candidate_region.rs @@ -356,7 +356,6 @@ mod tests { use common_meta::key::topic_name::TopicNameKey; use common_meta::key::topic_region::{ReplayCheckpoint, TopicRegionKey, TopicRegionValue}; use common_meta::peer::Peer; - use common_meta::rpc::ddl::{EventContext, TriggerReason}; use common_meta::rpc::router::{Region, RegionRoute}; use common_meta::wal_provider::RegionWalOptions; use common_wal::options::KafkaWalOptions; @@ -365,7 +364,9 @@ mod tests { use super::*; use crate::error::Error; use crate::procedure::region_migration::test_util::{TestingEnv, new_procedure_context}; - use crate::procedure::region_migration::{ContextFactory, PersistentContext}; + use crate::procedure::region_migration::{ + ContextFactory, PersistentContext, RegionMigrationTriggerReason, + }; use crate::procedure::test_util::{ new_close_region_reply, new_upgrade_region_reply, send_mock_reply, }; @@ -377,7 +378,7 @@ mod tests { Peer::empty(2), vec![RegionId::new(1024, 1)], Duration::from_millis(1000), - EventContext::new(TriggerReason::Manual), + RegionMigrationTriggerReason::Unknown, ) } diff --git a/src/meta-srv/src/procedure/repartition.rs b/src/meta-srv/src/procedure/repartition.rs index d8b585b28e..ee8b3ee749 100644 --- a/src/meta-srv/src/procedure/repartition.rs +++ b/src/meta-srv/src/procedure/repartition.rs @@ -44,12 +44,11 @@ use common_meta::lock_key::{CatalogLock, SchemaLock, TableLock, TableNameLock}; use common_meta::node_manager::NodeManagerRef; use common_meta::region_keeper::{MemoryRegionKeeperRef, OperatingRegionGuard}; use common_meta::region_registry::LeaderRegionRegistryRef; -use common_meta::rpc::ddl::EventContext; use common_meta::rpc::router::{RegionRoute, operating_leader_region_roles}; use common_meta::wal_provider::RegionWalOptions; use common_procedure::error::{FromJsonSnafu, ToJsonSnafu}; use common_procedure::{ - BoxedProcedure, Context as ProcedureContext, Error as ProcedureError, EventRuntimeContext, + BoxedProcedure, Context as ProcedureContext, Error as ProcedureError, EventContext, EventTrigger, LockKey, Procedure, ProcedureManagerRef, Result as ProcedureResult, Status, StringKey, }; @@ -104,8 +103,6 @@ pub struct PersistentContext { #[serde(default)] /// Records table-level partition metadata updated by this repartition. pub partition_metadata_update: Option, - #[serde(default)] - pub event_context: EventContext, } fn default_timeout() -> Duration { @@ -113,7 +110,7 @@ fn default_timeout() -> Duration { } impl PersistentContext { - /// Creates a new [PersistentContext] with the table, timeout and event context. + /// Creates a new [PersistentContext] with the given table name, table id and timeout. /// /// If the timeout is not provided, the default timeout will be used. pub fn new( @@ -124,7 +121,6 @@ impl PersistentContext { }: TableName, table_id: TableId, timeout: Option, - event_context: EventContext, ) -> Self { Self { catalog_name, @@ -136,7 +132,6 @@ impl PersistentContext { unknown_procedures: vec![], timeout: timeout.unwrap_or_else(default_timeout), partition_metadata_update: None, - event_context, } } @@ -797,10 +792,7 @@ impl Procedure for RepartitionProcedure { LockKey::new(self.context.persistent_ctx.lock_key()) } - fn event( - &self, - ctx: &EventRuntimeContext<'_>, - ) -> Option> { + fn event(&self, ctx: &EventContext<'_>) -> Option> { if !ctx.event_type_filter.allows(REPARTITION_EVENT_TYPE) { return None; } @@ -869,7 +861,6 @@ impl RepartitionProcedureFactory for GcDisabledRepartitionProcedureFactory { _source: RepartitionSource, _to_exprs: Vec, _timeout: Option, - _event_context: EventContext, ) -> std::result::Result { Err(BoxedError::new( error::InvalidArgumentsSnafu { @@ -908,9 +899,8 @@ impl RepartitionProcedureFactory for DefaultRepartitionProcedureFactory { source: RepartitionSource, to_exprs: Vec, timeout: Option, - event_context: EventContext, ) -> std::result::Result { - let persistent_ctx = PersistentContext::new(table_name, table_id, timeout, event_context); + let persistent_ctx = PersistentContext::new(table_name, table_id, timeout); let from = match source { RepartitionSource::Partitioned { exprs, @@ -1143,7 +1133,6 @@ mod tests { TableName::new("test_catalog", "test_schema", "test_table"), table_id, None, - EventContext::default(), ); Context::new( @@ -1175,7 +1164,6 @@ mod tests { }, vec![], None, - EventContext::default(), ) .err() .expect("GC-disabled factory must reject repartition"); @@ -1277,53 +1265,58 @@ mod tests { let all = Arc::new(EventTypeFilter::All); let submitted = procedure - .event(&EventRuntimeContext { + .event(&EventContext { procedure_id: ProcedureId::random(), lifecycle_state: &state, trigger: EventTrigger::Submitted, event_type_filter: all.clone(), + event_context: None, }) .unwrap(); assert_eq!(submitted.event_type(), REPARTITION_EVENT_TYPE); assert_ne!(submitted.json_payload().unwrap(), serde_json::Value::Null); let allowed = procedure - .event(&EventRuntimeContext { + .event(&EventContext { procedure_id: ProcedureId::random(), lifecycle_state: &state, trigger: EventTrigger::Submitted, event_type_filter: Arc::new(EventTypeFilter::Only(HashSet::from([ REPARTITION_EVENT_TYPE.to_string(), ]))), + event_context: None, }) .unwrap(); assert_eq!(allowed.event_type(), REPARTITION_EVENT_TYPE); let succeeded = procedure - .event(&EventRuntimeContext { + .event(&EventContext { procedure_id: ProcedureId::random(), lifecycle_state: &state, trigger: EventTrigger::Succeeded, event_type_filter: all, + event_context: None, }) .unwrap(); assert_eq!(succeeded.json_payload().unwrap(), serde_json::Value::Null); - let filtered = procedure.event(&EventRuntimeContext { + let filtered = procedure.event(&EventContext { procedure_id: ProcedureId::random(), lifecycle_state: &state, trigger: EventTrigger::Submitted, event_type_filter: Arc::new(EventTypeFilter::Only(HashSet::from([ "another_event".to_string() ]))), + event_context: None, }); assert!(filtered.is_none()); - let empty = procedure.event(&EventRuntimeContext { + let empty = procedure.event(&EventContext { procedure_id: ProcedureId::random(), lifecycle_state: &state, trigger: EventTrigger::Submitted, event_type_filter: Arc::new(EventTypeFilter::Only(HashSet::new())), + event_context: None, }); assert!(empty.is_none()); } @@ -1382,7 +1375,7 @@ mod tests { } #[test] - fn test_persistent_context_serde_defaults_for_new_fields() { + fn test_persistent_context_partition_metadata_update_serde_default() { let json = r#"{ "catalog_name":"test_catalog", "schema_name":"test_schema", @@ -1394,10 +1387,7 @@ mod tests { let persistent_ctx: PersistentContext = serde_json::from_str(json).unwrap(); - assert!(persistent_ctx.failed_procedures.is_empty()); - assert!(persistent_ctx.unknown_procedures.is_empty()); assert!(persistent_ctx.partition_metadata_update.is_none()); - assert_eq!(persistent_ctx.event_context, EventContext::default()); } #[tokio::test] @@ -1474,7 +1464,6 @@ mod tests { TableName::new("test_catalog", "test_schema", "test_table"), table_id, None, - EventContext::default(), ); persistent_ctx.plans = vec![with_rollback_metadata( test_plan(table_id), @@ -1541,7 +1530,6 @@ mod tests { TableName::new("test_catalog", "test_schema", "test_table"), table_id, None, - EventContext::default(), ); persistent_ctx.plans = vec![test_plan(table_id)]; let context = Context::new( @@ -1595,7 +1583,6 @@ mod tests { TableName::new("test_catalog", "test_schema", "test_table"), table_id, None, - EventContext::default(), ); let failed_plan = test_plan(table_id); let failed_plan = with_rollback_metadata( @@ -1729,7 +1716,6 @@ mod tests { TableName::new("test_catalog", "test_schema", "test_table"), table_id, None, - EventContext::default(), ); persistent_ctx.plans = vec![failed_plan, succeeded_plan.clone()]; persistent_ctx.failed_procedures = vec![ProcedureMeta { @@ -1839,7 +1825,6 @@ mod tests { TableName::new("test_catalog", "test_schema", "test_table"), table_id, None, - EventContext::default(), ); persistent_ctx.plans = vec![plan.clone()]; persistent_ctx.unknown_procedures = vec![ProcedureMeta { @@ -1901,7 +1886,6 @@ mod tests { TableName::new("test_catalog", "test_schema", "test_table"), table_id, None, - EventContext::default(), ); persistent_ctx.plans = vec![with_rollback_metadata( test_plan(table_id), @@ -2030,7 +2014,6 @@ mod tests { TableName::new("test_catalog", "test_schema", "test_table"), table_id, None, - EventContext::default(), ); persistent_ctx.plans = vec![failed_merge_plan, succeeded_split_plan.clone()]; persistent_ctx.failed_procedures = vec![ProcedureMeta { diff --git a/src/meta-srv/src/procedure/repartition/allocate_region.rs b/src/meta-srv/src/procedure/repartition/allocate_region.rs index aa36e95be2..857ac94670 100644 --- a/src/meta-srv/src/procedure/repartition/allocate_region.rs +++ b/src/meta-srv/src/procedure/repartition/allocate_region.rs @@ -777,6 +777,7 @@ mod tests { concurrent_region_route, region_wal_options: test_region_wal_options(&[1, 2]), }), + event_context: None, }; let mut state = ExecutePlan; diff --git a/src/meta-srv/src/procedure/repartition/collect.rs b/src/meta-srv/src/procedure/repartition/collect.rs index 56d9a007a1..b71c3d6650 100644 --- a/src/meta-srv/src/procedure/repartition/collect.rs +++ b/src/meta-srv/src/procedure/repartition/collect.rs @@ -136,7 +136,6 @@ mod tests { use common_error::mock::MockError; use common_error::status_code::StatusCode; - use common_meta::rpc::ddl::EventContext; use common_meta::test_util::MockDatanodeManager; use common_procedure::{ Context as ProcedureContext, ContextProvider, Error as ProcedureError, ProcedureId, @@ -194,7 +193,6 @@ mod tests { table::table_name::TableName::new("test_catalog", "test_schema", "test_table"), 1024, None, - EventContext::default(), ); let mut ctx = crate::procedure::repartition::Context::new( &ddl_ctx, @@ -229,7 +227,6 @@ mod tests { table::table_name::TableName::new("test_catalog", "test_schema", "test_table"), 1024, None, - EventContext::default(), ); let mut ctx = crate::procedure::repartition::Context::new( &ddl_ctx, @@ -249,6 +246,7 @@ mod tests { receiver: rx, inner: MockContextProvider::default(), }), + event_context: None, }; let mut state = Collect { inflight_procedures: vec![ProcedureMeta { diff --git a/src/meta-srv/src/procedure/repartition/gc_requirement.rs b/src/meta-srv/src/procedure/repartition/gc_requirement.rs index 11e82d294a..1ae24e4031 100644 --- a/src/meta-srv/src/procedure/repartition/gc_requirement.rs +++ b/src/meta-srv/src/procedure/repartition/gc_requirement.rs @@ -244,6 +244,7 @@ mod tests { block_after_persist: true, persisted_tx: Some(persisted_tx), }), + context: Default::default(), }) .await .unwrap(); diff --git a/src/meta-srv/src/procedure/repartition/group.rs b/src/meta-srv/src/procedure/repartition/group.rs index a3c38b5955..63e05bcb15 100644 --- a/src/meta-srv/src/procedure/repartition/group.rs +++ b/src/meta-srv/src/procedure/repartition/group.rs @@ -38,8 +38,8 @@ use common_meta::peer::Peer; use common_meta::rpc::router::RegionRoute; use common_procedure::error::{FromJsonSnafu, ToJsonSnafu}; use common_procedure::{ - Context as ProcedureContext, Error as ProcedureError, EventRuntimeContext, EventTrigger, - LockKey, Procedure, ProcedureId, Result as ProcedureResult, Status, StringKey, + Context as ProcedureContext, Error as ProcedureError, EventContext, EventTrigger, LockKey, + Procedure, ProcedureId, Result as ProcedureResult, Status, StringKey, }; use common_telemetry::{error, info}; use serde::{Deserialize, Serialize}; @@ -262,10 +262,7 @@ impl Procedure for RepartitionGroupProcedure { LockKey::new(self.context.persistent_ctx.lock_key()) } - fn event( - &self, - ctx: &EventRuntimeContext<'_>, - ) -> Option> { + fn event(&self, ctx: &EventContext<'_>) -> Option> { if !ctx.event_type_filter.allows(REPARTITION_GROUP_EVENT_TYPE) { return None; } @@ -634,9 +631,7 @@ mod tests { use common_event_recorder::EventTypeFilter; use common_meta::key::TableMetadataManager; use common_meta::kv_backend::test_util::MockKvBackendBuilder; - use common_procedure::{ - EventRuntimeContext, EventTrigger, Procedure, ProcedureId, ProcedureState, - }; + use common_procedure::{EventContext, EventTrigger, Procedure, ProcedureId, ProcedureState}; use crate::error::Error; use crate::event::repartition::REPARTITION_GROUP_EVENT_TYPE; @@ -713,15 +708,16 @@ mod tests { context: env.create_context(new_persistent_context(1024, vec![], vec![])), }; let state = ProcedureState::Running; - let runtime_context = |event_type_filter| EventRuntimeContext { + let event_context = |event_type_filter| EventContext { procedure_id: ProcedureId::random(), lifecycle_state: &state, trigger: EventTrigger::Submitted, event_type_filter: Arc::new(event_type_filter), + event_context: None, }; let allowed = procedure - .event(&runtime_context(EventTypeFilter::Only(HashSet::from([ + .event(&event_context(EventTypeFilter::Only(HashSet::from([ REPARTITION_GROUP_EVENT_TYPE.to_string(), ])))) .unwrap(); @@ -729,14 +725,14 @@ mod tests { assert!( procedure - .event(&runtime_context(EventTypeFilter::Only(HashSet::from([ + .event(&event_context(EventTypeFilter::Only(HashSet::from([ "another_event".to_string(), ])))) .is_none() ); assert!( procedure - .event(&runtime_context(EventTypeFilter::Only(HashSet::new()))) + .event(&event_context(EventTypeFilter::Only(HashSet::new()))) .is_none() ); } diff --git a/src/meta-srv/src/procedure/repartition/test_util.rs b/src/meta-srv/src/procedure/repartition/test_util.rs index a6f2330b83..2ce494ee7c 100644 --- a/src/meta-srv/src/procedure/repartition/test_util.rs +++ b/src/meta-srv/src/procedure/repartition/test_util.rs @@ -24,7 +24,6 @@ use common_meta::kv_backend::KvBackendRef; use common_meta::kv_backend::memory::MemoryKvBackend; use common_meta::node_manager::NodeManagerRef; use common_meta::peer::Peer; -use common_meta::rpc::ddl::EventContext; use common_meta::rpc::router::{Region, RegionRoute}; use common_meta::sequence::SequenceBuilder; use common_meta::test_util::new_ddl_context_with_kv_backend; @@ -103,6 +102,7 @@ impl TestingEnv { ProcedureContext { procedure_id: ProcedureId::random(), provider: Arc::new(MockContextProvider::default()), + event_context: None, } } @@ -225,7 +225,6 @@ pub fn new_parent_context( TableName::new("test_catalog", "test_schema", "test_table"), table_id, None, - EventContext::default(), ); ParentContext::new( @@ -266,6 +265,7 @@ pub fn procedure_context_with_receivers( receivers, inner: MockContextProvider::default(), }), + event_context: None, } } diff --git a/src/meta-srv/src/procedure/tests.rs b/src/meta-srv/src/procedure/tests.rs index dd855c66aa..4f70dcca95 100644 --- a/src/meta-srv/src/procedure/tests.rs +++ b/src/meta-srv/src/procedure/tests.rs @@ -29,7 +29,7 @@ use common_meta::ddl::test_util::create_table::{ }; use common_meta::key::table_route::{PhysicalTableRouteValue, TableRouteValue}; use common_meta::node_manager::NodeManagerRef; -use common_meta::rpc::ddl::{CreateTableTask, EventContext, QueryContext}; +use common_meta::rpc::ddl::CreateTableTask; use common_meta::rpc::router::{RegionRoute, find_leaders}; use common_procedure::Status; use store_api::storage::RegionId; @@ -91,8 +91,6 @@ fn create_table_task(table_name: Option<&str>) -> CreateTableTask { fn test_region_request_builder() { let mut procedure = CreateTableProcedure::new( create_table_task(None), - QueryContext::default(), - EventContext::default(), test_data::new_ddl_context(Arc::new(NodeClients::default())), ) .unwrap(); @@ -190,8 +188,6 @@ async fn test_on_datanode_create_regions() { let mut procedure = CreateTableProcedure::new( create_table_task(None), - QueryContext::default(), - EventContext::default(), test_data::new_ddl_context(node_manager), ) .unwrap(); @@ -264,12 +260,8 @@ async fn test_on_datanode_create_logical_regions() { .unwrap() .0; let _ = kv_backend.txn(physical_route_txn).await.unwrap(); - let mut procedure = CreateLogicalTablesProcedure::new( - vec![task1, task2, task3], - physical_table_id, - EventContext::default(), - ctx, - ); + let mut procedure = + CreateLogicalTablesProcedure::new(vec![task1, task2, task3], physical_table_id, ctx); let expected_created_regions = Arc::new(Mutex::new(HashMap::from([(1, 3), (2, 3), (3, 3)]))); diff --git a/src/meta-srv/src/procedure/wal_prune.rs b/src/meta-srv/src/procedure/wal_prune.rs index 128fc58c44..5aca85a51a 100644 --- a/src/meta-srv/src/procedure/wal_prune.rs +++ b/src/meta-srv/src/procedure/wal_prune.rs @@ -25,8 +25,8 @@ use common_meta::lock_key::RemoteWalLock; use common_meta::region_registry::LeaderRegionRegistryRef; use common_procedure::error::ToJsonSnafu; use common_procedure::{ - Context as ProcedureContext, Error as ProcedureError, EventRuntimeContext, EventTrigger, - LockKey, Procedure, ProcedureState, Result as ProcedureResult, Status, StringKey, + Context as ProcedureContext, Error as ProcedureError, EventContext, EventTrigger, LockKey, + Procedure, ProcedureState, Result as ProcedureResult, Status, StringKey, }; use common_telemetry::{info, warn}; use manager::{WalPruneProcedureGuard, WalPruneProcedureTracker}; @@ -213,10 +213,7 @@ impl Procedure for WalPruneProcedure { LockKey::new(vec![lock_key]) } - fn event( - &self, - ctx: &EventRuntimeContext<'_>, - ) -> Option> { + fn event(&self, ctx: &EventContext<'_>) -> Option> { if !ctx.event_type_filter.allows(WAL_PRUNE_EVENT_TYPE) { return None; } @@ -506,11 +503,12 @@ mod tests { let mut procedure = WalPruneProcedure::new(context, None, "test_topic".to_string(), 42, false); let running = ProcedureState::Running; - let runtime_context = |trigger, lifecycle_state, event_type_filter| EventRuntimeContext { + let runtime_context = |trigger, lifecycle_state, event_type_filter| EventContext { procedure_id: ProcedureId::random(), lifecycle_state, trigger, event_type_filter: Arc::new(event_type_filter), + event_context: None, }; for trigger in [EventTrigger::Submitted, EventTrigger::Recovered] { diff --git a/src/meta-srv/src/service/procedure.rs b/src/meta-srv/src/service/procedure.rs index b20d7f6477..1b5c4c42db 100644 --- a/src/meta-srv/src/service/procedure.rs +++ b/src/meta-srv/src/service/procedure.rs @@ -107,6 +107,7 @@ impl procedure_service_server::ProcedureService for Metasrv { .submit_ddl_task( &ExecutorContext { tracing_context: Some(header.tracing_context), + ..Default::default() }, SubmitDdlTaskRequest { query_context, diff --git a/src/standalone/src/procedure.rs b/src/standalone/src/procedure.rs index 85973e9f64..ec63b11703 100644 --- a/src/standalone/src/procedure.rs +++ b/src/standalone/src/procedure.rs @@ -20,7 +20,6 @@ use common_meta::ddl::DdlContext; use common_meta::ddl_manager::{RepartitionProcedureFactory, RepartitionSource}; use common_meta::key::runtime_switch::RuntimeSwitchManager; use common_meta::kv_backend::KvBackendRef; -use common_meta::rpc::ddl::EventContext; use common_meta::state_store::KvStateStore; use common_procedure::local::{EventRecorderHandle, LocalManager, ManagerConfig}; use common_procedure::options::ProcedureConfig; @@ -74,7 +73,6 @@ impl RepartitionProcedureFactory for StandaloneRepartitionProcedureFactory { _source: RepartitionSource, _to_exprs: Vec, _timeout: Option, - _event_context: EventContext, ) -> std::result::Result { Err(BoxedError::new(NoSupportRepartitionProcedureSnafu.build())) } diff --git a/tests-integration/src/tests/gc.rs b/tests-integration/src/tests/gc.rs index 5bc20e2e2c..badaef5f7a 100644 --- a/tests-integration/src/tests/gc.rs +++ b/tests-integration/src/tests/gc.rs @@ -17,7 +17,6 @@ use std::sync::Arc; use std::time::Duration; use common_meta::key::TableMetadataManagerRef; -use common_meta::rpc::ddl::EventContext; use common_procedure::ProcedureWithId; use common_telemetry::info; use common_test_util::recordbatch::check_output_stream; @@ -214,7 +213,6 @@ async fn test_gc_basic(store_type: &StorageType) { false, // full_file_listing Duration::from_secs(10), // timeout Default::default(), - EventContext::default(), ); // Submit the procedure to the procedure manager diff --git a/tests-integration/tests/gc_event.rs b/tests-integration/tests/gc_event.rs index b62ba1d364..9ecedc7777 100644 --- a/tests-integration/tests/gc_event.rs +++ b/tests-integration/tests/gc_event.rs @@ -15,7 +15,7 @@ use std::sync::Arc; use std::time::Duration; -use common_meta::rpc::ddl::{EventContext, TriggerReason}; +use common_meta::rpc::ddl::{PersistentEventContext, TriggerReason}; use common_procedure::{ProcedureId, ProcedureWithId, watcher}; use common_test_util::temp_dir::create_temp_dir; use meta_srv::gc::{BatchGcProcedure, GcSchedulerOptions}; @@ -129,7 +129,6 @@ async fn test_batch_gc_event() { false, Duration::from_secs(10), Default::default(), - EventContext::new(TriggerReason::Manual), ); let procedure_id = ProcedureId::parse_str("00000000-0000-0000-0000-00000000bac0").unwrap(); let mut watcher = cluster @@ -138,6 +137,9 @@ async fn test_batch_gc_event() { .submit(ProcedureWithId { id: procedure_id, procedure: Box::new(procedure), + context: common_procedure::ProcedureContext::from_event_context( + PersistentEventContext::new(TriggerReason::Manual), + ), }) .await .unwrap(); diff --git a/tests-integration/tests/repartition.rs b/tests-integration/tests/repartition.rs index 3dac860066..1990cc2c43 100644 --- a/tests-integration/tests/repartition.rs +++ b/tests-integration/tests/repartition.rs @@ -19,7 +19,6 @@ use std::time::Duration; use client::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME}; use common_error::root_source; use common_meta::key::table_name::TableNameKey; -use common_meta::rpc::ddl::EventContext; use common_procedure::{ProcedureWithId, watcher}; use common_query::Output; use common_telemetry::info; @@ -817,7 +816,6 @@ async fn trigger_table_gc(metasrv: &Arc, table_name: &str) { false, // full_file_listing Duration::from_secs(10), // timeout Default::default(), - EventContext::default(), ); // Submit the procedure to the procedure manager diff --git a/tests-integration/tests/table_ddl_event.rs b/tests-integration/tests/table_ddl_event.rs index 5c23e101de..5194640880 100644 --- a/tests-integration/tests/table_ddl_event.rs +++ b/tests-integration/tests/table_ddl_event.rs @@ -377,7 +377,7 @@ async fn test_table_ddl_procedure_events() { .ddl_manager() .submit_undrop_table_task( common_meta::rpc::ddl::UndropTableTask { table_id }, - common_meta::rpc::ddl::EventContext::new( + common_meta::rpc::ddl::PersistentEventContext::new( common_meta::rpc::ddl::TriggerReason::Manual, ), ) @@ -419,7 +419,7 @@ async fn test_table_ddl_procedure_events() { .ddl_manager() .submit_purge_dropped_table_task( common_meta::rpc::ddl::PurgeDroppedTableTask { table_id }, - common_meta::rpc::ddl::EventContext::new( + common_meta::rpc::ddl::PersistentEventContext::new( common_meta::rpc::ddl::TriggerReason::Manual, ), ) diff --git a/tests/conf/frontend-test.toml.template b/tests/conf/frontend-test.toml.template index 25d44ff6e4..0042716bd8 100644 --- a/tests/conf/frontend-test.toml.template +++ b/tests/conf/frontend-test.toml.template @@ -1,3 +1,6 @@ [grpc] bind_addr = "{addrs.grpc_addr}" server_addr = "{addrs.grpc_addr}" + +[event_recorder] +event_types = []