From 408929ffd8bad5aa75547274ab62e667f90a449c Mon Sep 17 00:00:00 2001 From: Weny Xu Date: Fri, 7 Aug 2026 20:37:58 +0800 Subject: [PATCH] feat(event): add event context to procedure events (#8734) * feat(event): add trigger context to procedure events Signed-off-by: WenyXu * test(event): fix trigger context event contracts Signed-off-by: WenyXu * fix(event): preserve trigger context origins Signed-off-by: WenyXu * fix(event): preserve lifecycle trigger contexts Signed-off-by: WenyXu * fix(meta): gate enterprise trigger imports Signed-off-by: WenyXu * test(event): assert trigger contexts exactly Signed-off-by: WenyXu * fix(test): restore migration test literals Signed-off-by: WenyXu * feat(event): add trigger context to non-table ddl Signed-off-by: WenyXu * refactor(event): simplify trigger context encoding Signed-off-by: WenyXu * fix(event): preserve auto alter trigger reason Signed-off-by: WenyXu * refactor(event): require explicit trigger context Signed-off-by: WenyXu * refactor(event): derive trigger context at ddl boundary Signed-off-by: WenyXu * refactor(event): propagate DDL trigger reasons Signed-off-by: WenyXu * fix(meta): resolve alter table rebase conflict Signed-off-by: WenyXu * test(meta): fix alter table trigger context setup Signed-off-by: WenyXu * refactor(meta): pass trigger context to region migration Signed-off-by: WenyXu * test(event): fix unknown trigger context assertions Signed-off-by: WenyXu * fix(meta): centralize trigger context protocol mapping Signed-off-by: WenyXu * test(event): fix migration trigger context assertion Signed-off-by: WenyXu * refactor(procedure): rename event runtime context Signed-off-by: WenyXu * refactor(event): rename trigger context Signed-off-by: WenyXu * test(event): fix migration event context assertion Signed-off-by: WenyXu --------- Signed-off-by: WenyXu (cherry picked from commit 1f1c9270a806658d2c3d9700c8cd1445d687f89f) --- Cargo.lock | 2 +- src/common/event-recorder/src/event_table.rs | 3 + src/common/meta/Cargo.toml | 1 + src/common/meta/src/ddl/alter_database.rs | 23 +- .../meta/src/ddl/alter_logical_tables.rs | 14 +- src/common/meta/src/ddl/alter_table.rs | 33 +- .../src/ddl/alter_table/region_request.rs | 8 +- src/common/meta/src/ddl/create_database.rs | 19 +- src/common/meta/src/ddl/create_flow.rs | 22 +- .../meta/src/ddl/create_logical_tables.rs | 18 +- src/common/meta/src/ddl/create_table.rs | 34 +- src/common/meta/src/ddl/create_view.rs | 14 +- src/common/meta/src/ddl/drop_database.rs | 25 +- src/common/meta/src/ddl/drop_flow.rs | 15 +- src/common/meta/src/ddl/drop_table.rs | 21 +- src/common/meta/src/ddl/drop_view.rs | 12 +- src/common/meta/src/ddl/event/database.rs | 25 +- src/common/meta/src/ddl/event/flow.rs | 28 +- src/common/meta/src/ddl/event/table.rs | 59 ++- src/common/meta/src/ddl/event/view.rs | 25 +- .../meta/src/ddl/purge_dropped_table.rs | 39 +- src/common/meta/src/ddl/test_util.rs | 9 +- .../src/ddl/tests/alter_logical_tables.rs | 49 ++- src/common/meta/src/ddl/tests/alter_table.rs | 72 +++- src/common/meta/src/ddl/tests/create_flow.rs | 87 ++++- .../src/ddl/tests/create_logical_tables.rs | 36 +- src/common/meta/src/ddl/tests/create_table.rs | 74 +++- src/common/meta/src/ddl/tests/create_view.rs | 30 +- .../meta/src/ddl/tests/drop_database.rs | 4 + src/common/meta/src/ddl/tests/drop_flow.rs | 14 +- src/common/meta/src/ddl/tests/drop_table.rs | 234 +++++++++--- src/common/meta/src/ddl/tests/drop_view.rs | 14 +- .../meta/src/ddl/tests/event/database.rs | 48 ++- src/common/meta/src/ddl/tests/event/flow.rs | 26 +- src/common/meta/src/ddl/tests/event/table.rs | 159 ++++++-- .../meta/src/ddl/tests/event/test_util.rs | 20 +- src/common/meta/src/ddl/tests/event/view.rs | 60 ++- src/common/meta/src/ddl/truncate_table.rs | 22 +- src/common/meta/src/ddl/undrop_table.rs | 48 ++- src/common/meta/src/ddl_manager.rs | 203 +++++++---- src/common/meta/src/rpc/ddl.rs | 131 ++++++- src/common/procedure/src/lib.rs | 6 +- src/common/procedure/src/local.rs | 12 +- src/common/procedure/src/local/runner.rs | 6 +- src/common/procedure/src/procedure.rs | 8 +- src/common/session/src/lib.rs | 49 ++- src/flow/src/adapter/util.rs | 3 +- src/frontend/src/instance/dashboard.rs | 8 +- src/frontend/src/instance/grpc.rs | 10 +- .../src/instance/otlp/trace_ingest.rs | 3 +- src/frontend/src/instance/prom_store.rs | 9 +- src/meta-srv/Cargo.toml | 1 - src/meta-srv/src/event/gc.rs | 66 +++- src/meta-srv/src/event/region_migration.rs | 18 +- src/meta-srv/src/event/repartition.rs | 39 +- src/meta-srv/src/gc/ctx.rs | 6 + src/meta-srv/src/gc/handler.rs | 10 +- src/meta-srv/src/gc/mock.rs | 4 + src/meta-srv/src/gc/mock/basic.rs | 72 +++- src/meta-srv/src/gc/mock/concurrent.rs | 30 +- src/meta-srv/src/gc/mock/full_list.rs | 6 + src/meta-srv/src/gc/mock/misc.rs | 15 +- src/meta-srv/src/gc/procedure.rs | 45 ++- src/meta-srv/src/gc/scheduler.rs | 22 +- .../src/procedure/region_migration.rs | 98 ++++- .../downgrade_leader_region.rs | 4 +- .../src/procedure/region_migration/manager.rs | 5 +- .../procedure/region_migration/test_util.rs | 7 +- .../upgrade_candidate_region.rs | 4 +- src/meta-srv/src/procedure/repartition.rs | 42 ++- .../src/procedure/repartition/collect.rs | 3 + .../src/procedure/repartition/group.rs | 21 +- .../src/procedure/repartition/test_util.rs | 2 + src/meta-srv/src/procedure/tests.rs | 14 +- src/meta-srv/src/procedure/wal_prune.rs | 25 +- src/meta-srv/src/service/procedure.rs | 1 - src/operator/src/insert.rs | 23 +- src/operator/src/procedure.rs | 19 +- src/operator/src/statement/comment.rs | 22 +- src/operator/src/statement/ddl.rs | 121 ++++-- src/operator/src/utils.rs | 43 ++- src/pipeline/src/manager/pipeline_operator.rs | 3 +- src/session/src/context.rs | 19 +- src/standalone/src/procedure.rs | 2 + tests-integration/src/tests/gc.rs | 2 + tests-integration/tests/database_ddl_event.rs | 11 +- tests-integration/tests/flow_ddl_event.rs | 18 + tests-integration/tests/gc_event.rs | 18 +- tests-integration/tests/http.rs | 42 ++- tests-integration/tests/region_migration.rs | 16 +- tests-integration/tests/repartition.rs | 2 + tests-integration/tests/table_ddl_event.rs | 344 +++++++++++++++++- tests-integration/tests/view_ddl_event.rs | 18 + 93 files changed, 2556 insertions(+), 621 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d46e63db64d..12ef98ef5ac 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2645,6 +2645,7 @@ dependencies = [ "common-query", "common-recordbatch", "common-runtime", + "common-session", "common-telemetry", "common-test-util", "common-time", @@ -8404,7 +8405,6 @@ dependencies = [ "serde", "serde_json", "servers", - "session", "snafu 0.8.6", "sqlx", "store-api", diff --git a/src/common/event-recorder/src/event_table.rs b/src/common/event-recorder/src/event_table.rs index 40d6625b5d9..0ec38c8eabc 100644 --- a/src/common/event-recorder/src/event_table.rs +++ b/src/common/event-recorder/src/event_table.rs @@ -79,6 +79,9 @@ pub const TYPE_COLUMN: EventTableColumn = /// The canonical event payload column. pub const PAYLOAD_COLUMN: EventTableColumn = EventTableColumn::json_binary("payload", ColumnDataType::Binary, SemanticType::Field); +/// The optional context describing why an event was triggered. +pub const EVENT_CONTEXT_COLUMN: EventTableColumn = + EventTableColumn::json_binary("event_context", ColumnDataType::Binary, SemanticType::Field); /// The canonical event timestamp column. pub const TIMESTAMP_COLUMN: EventTableColumn = EventTableColumn::new( "timestamp", diff --git a/src/common/meta/Cargo.toml b/src/common/meta/Cargo.toml index 8e0cbbdf354..43714b7d599 100644 --- a/src/common/meta/Cargo.toml +++ b/src/common/meta/Cargo.toml @@ -41,6 +41,7 @@ common-procedure.workspace = true common-query.workspace = true common-recordbatch.workspace = true common-runtime.workspace = true +common-session.workspace = true common-telemetry.workspace = true common-time.workspace = true common-wal.workspace = true diff --git a/src/common/meta/src/ddl/alter_database.rs b/src/common/meta/src/ddl/alter_database.rs index dfa42f4009c..7e16275da3c 100644 --- a/src/common/meta/src/ddl/alter_database.rs +++ b/src/common/meta/src/ddl/alter_database.rs @@ -15,7 +15,7 @@ use async_trait::async_trait; use common_procedure::error::{FromJsonSnafu, Result as ProcedureResult, ToJsonSnafu}; use common_procedure::{ - Context as ProcedureContext, EventContext, EventTrigger, LockKey, Procedure, Status, + Context as ProcedureContext, EventRuntimeContext, EventTrigger, LockKey, Procedure, Status, }; use common_telemetry::tracing::info; use serde::{Deserialize, Serialize}; @@ -32,7 +32,7 @@ use crate::key::DeserializedValueWithBytes; use crate::key::schema_name::{SchemaName, SchemaNameKey, SchemaNameValue}; use crate::lock_key::{CatalogLock, SchemaLock}; use crate::rpc::ddl::UnsetDatabaseOption::{self}; -use crate::rpc::ddl::{AlterDatabaseKind, AlterDatabaseTask, SetDatabaseOption}; +use crate::rpc::ddl::{AlterDatabaseKind, AlterDatabaseTask, EventContext, SetDatabaseOption}; pub struct AlterDatabaseProcedure { pub context: DdlContext, @@ -73,10 +73,14 @@ fn build_new_schema_value( impl AlterDatabaseProcedure { pub const TYPE_NAME: &'static str = "metasrv-procedure::AlterDatabase"; - pub fn new(task: AlterDatabaseTask, context: DdlContext) -> Result { + pub fn new( + task: AlterDatabaseTask, + event_context: EventContext, + context: DdlContext, + ) -> Result { Ok(Self { context, - data: AlterDatabaseData::new(task)?, + data: AlterDatabaseData::new(task, event_context)?, }) } @@ -176,7 +180,10 @@ impl Procedure for AlterDatabaseProcedure { LockKey::new(lock_key) } - fn event(&self, ctx: &EventContext<'_>) -> Option> { + fn event( + &self, + ctx: &EventRuntimeContext<'_>, + ) -> Option> { if !ctx.event_type_filter.allows(ALTER_DATABASE_EVENT_TYPE) { return None; } @@ -186,6 +193,7 @@ impl Procedure for AlterDatabaseProcedure { self.data.catalog(), self.data.schema(), &self.data.kind, + self.data.event_context.clone(), ) } else { DatabaseDdlEvent::alter_lifecycle() @@ -209,16 +217,19 @@ pub struct AlterDatabaseData { catalog_name: String, schema_name: String, schema_value: Option>, + #[serde(default)] + event_context: EventContext, } impl AlterDatabaseData { - pub fn new(task: AlterDatabaseTask) -> Result { + pub fn new(task: AlterDatabaseTask, event_context: EventContext) -> Result { Ok(Self { state: AlterDatabaseState::Prepare, kind: AlterDatabaseKind::try_from(task.alter_expr.kind.unwrap())?, catalog_name: task.alter_expr.catalog_name, schema_name: task.alter_expr.schema_name, schema_value: None, + event_context, }) } diff --git a/src/common/meta/src/ddl/alter_logical_tables.rs b/src/common/meta/src/ddl/alter_logical_tables.rs index 683248965ff..816db0a6afb 100644 --- a/src/common/meta/src/ddl/alter_logical_tables.rs +++ b/src/common/meta/src/ddl/alter_logical_tables.rs @@ -20,7 +20,7 @@ use api::region::RegionResponse; use async_trait::async_trait; use common_catalog::format_full_table_name; use common_procedure::error::{FromJsonSnafu, Result as ProcedureResult, ToJsonSnafu}; -use common_procedure::{Context, EventContext, EventTrigger, LockKey, Procedure, Status}; +use common_procedure::{Context, EventRuntimeContext, EventTrigger, LockKey, Procedure, Status}; use common_telemetry::{debug, error, info, warn}; pub use executor::make_alter_region_request; use serde::{Deserialize, Serialize}; @@ -47,7 +47,7 @@ use crate::key::table_info::TableInfoValue; use crate::key::table_route::PhysicalTableRouteValue; use crate::lock_key::{CatalogLock, SchemaLock, TableLock}; use crate::metrics; -use crate::rpc::ddl::AlterTableTask; +use crate::rpc::ddl::{AlterTableTask, EventContext}; use crate::rpc::router::RegionRoute; pub struct AlterLogicalTablesProcedure { @@ -88,6 +88,7 @@ impl AlterLogicalTablesProcedure { pub fn new( tasks: Vec, physical_table_id: TableId, + event_context: EventContext, context: DdlContext, ) -> Self { Self { @@ -100,6 +101,7 @@ impl AlterLogicalTablesProcedure { physical_table_info: None, physical_columns: vec![], table_cache_keys_to_invalidate: vec![], + event_context, }, physical_table_route: None, } @@ -320,7 +322,10 @@ impl Procedure for AlterLogicalTablesProcedure { LockKey::new(lock_key) } - fn event(&self, ctx: &EventContext<'_>) -> Option> { + fn event( + &self, + ctx: &EventRuntimeContext<'_>, + ) -> Option> { if !ctx .event_type_filter .allows(TableDdlEventType::AlterLogicalTables.as_str()) @@ -348,6 +353,7 @@ impl Procedure for AlterLogicalTablesProcedure { locators, self.data.tasks.len(), kinds, + self.data.event_context.clone(), ))) } } @@ -364,6 +370,8 @@ pub struct AlterTablesData { physical_table_info: Option>, physical_columns: Vec, table_cache_keys_to_invalidate: Vec, + #[serde(default)] + event_context: EventContext, } impl AlterTablesData { diff --git a/src/common/meta/src/ddl/alter_table.rs b/src/common/meta/src/ddl/alter_table.rs index a19b49b016e..acc02c4d1a8 100644 --- a/src/common/meta/src/ddl/alter_table.rs +++ b/src/common/meta/src/ddl/alter_table.rs @@ -27,7 +27,7 @@ use common_catalog::consts::{METRIC_ENGINE, MITO_ENGINE}; use common_error::ext::BoxedError; use common_procedure::error::{FromJsonSnafu, Result as ProcedureResult, ToJsonSnafu}; use common_procedure::{ - Context as ProcedureContext, ContextProvider, Error as ProcedureError, EventContext, + Context as ProcedureContext, ContextProvider, Error as ProcedureError, EventRuntimeContext, EventTrigger, LockKey, PoisonKey, PoisonKeys, Procedure, ProcedureId, Status, StringKey, }; use common_telemetry::{error, info, warn}; @@ -58,7 +58,7 @@ use crate::key::{DeserializedValueWithBytes, RegionDistribution}; use crate::lock_key::{CatalogLock, RegionLock, SchemaLock, TableLock, TableNameLock}; use crate::metrics; use crate::poison_key::table_poison_key; -use crate::rpc::ddl::AlterTableTask; +use crate::rpc::ddl::{AlterTableTask, EventContext}; use crate::rpc::router::{RegionRoute, find_leaders, region_distribution}; /// The alter table procedure @@ -97,18 +97,24 @@ fn build_executor_from_alter_expr(alter_data: &AlterTableData) -> AlterTableExec impl AlterTableProcedure { pub const TYPE_NAME: &'static str = "metasrv-procedure::AlterTable"; - pub fn new(table_id: TableId, task: AlterTableTask, context: DdlContext) -> Result { - Self::new_with_region_locks(table_id, task, vec![], context) + pub fn new( + table_id: TableId, + task: AlterTableTask, + event_context: EventContext, + context: DdlContext, + ) -> Result { + Self::new_with_region_locks(table_id, task, event_context, vec![], context) } pub(crate) fn new_with_region_locks( table_id: TableId, task: AlterTableTask, + event_context: EventContext, region_locks: Vec, context: DdlContext, ) -> Result { task.validate()?; - let data = AlterTableData::new(task, table_id, region_locks); + let data = AlterTableData::new(task, table_id, event_context, region_locks); let executor = build_executor_from_alter_expr(&data); Ok(Self { context, @@ -563,7 +569,10 @@ impl Procedure for AlterTableProcedure { PoisonKeys::new(vec![self.table_poison_key()]) } - fn event(&self, ctx: &EventContext<'_>) -> Option> { + fn event( + &self, + ctx: &EventRuntimeContext<'_>, + ) -> Option> { if !ctx .event_type_filter .allows(TableDdlEventType::AlterTable.as_str()) @@ -583,7 +592,7 @@ impl Procedure for AlterTableProcedure { .kind .as_ref() .and_then(alter_table_kind_name); - TableDdlEvent::alter_table_submitted(locator, kind) + TableDdlEvent::alter_table_submitted(locator, kind, self.data.event_context.clone()) } _ => TableDdlEvent::lifecycle(TableDdlEventType::AlterTable), }; @@ -611,6 +620,8 @@ pub struct AlterTableData { task: AlterTableTask, table_id: TableId, #[serde(default)] + event_context: EventContext, + #[serde(default)] column_metadatas: Vec, /// Table info value before alteration. table_info_value: Option>, @@ -622,11 +633,17 @@ pub struct AlterTableData { } impl AlterTableData { - pub fn new(task: AlterTableTask, table_id: TableId, region_locks: Vec) -> Self { + pub fn new( + task: AlterTableTask, + table_id: TableId, + event_context: EventContext, + region_locks: Vec, + ) -> Self { Self { state: AlterTableState::Prepare, task, table_id, + event_context, column_metadatas: vec![], table_info_value: None, region_distribution: None, diff --git a/src/common/meta/src/ddl/alter_table/region_request.rs b/src/common/meta/src/ddl/alter_table/region_request.rs index 73edd4c3615..ef8d1487c70 100644 --- a/src/common/meta/src/ddl/alter_table/region_request.rs +++ b/src/common/meta/src/ddl/alter_table/region_request.rs @@ -144,7 +144,7 @@ mod tests { }; use crate::key::table_route::TableRouteValue; use crate::peer::Peer; - use crate::rpc::ddl::AlterTableTask; + use crate::rpc::ddl::{AlterTableTask, EventContext}; use crate::rpc::router::{Region, RegionRoute}; use crate::test_util::{MockDatanodeManager, new_ddl_context}; @@ -240,7 +240,8 @@ mod tests { }, }; - let mut procedure = AlterTableProcedure::new(table_id, task, ddl_context).unwrap(); + let mut procedure = + AlterTableProcedure::new(table_id, task, EventContext::default(), ddl_context).unwrap(); procedure.on_prepare().await.unwrap(); let alter_kind = procedure.make_region_alter_kind().unwrap(); let Some(Body::Alter(alter_region_request)) = @@ -296,7 +297,8 @@ mod tests { }, }; - let mut procedure = AlterTableProcedure::new(table_id, task, ddl_context).unwrap(); + let mut procedure = + AlterTableProcedure::new(table_id, task, EventContext::default(), ddl_context).unwrap(); procedure.on_prepare().await.unwrap(); let alter_kind = procedure.make_region_alter_kind().unwrap(); let Some(Body::Alter(alter_region_request)) = diff --git a/src/common/meta/src/ddl/create_database.rs b/src/common/meta/src/ddl/create_database.rs index b5c50235c63..8f9c5bee1a5 100644 --- a/src/common/meta/src/ddl/create_database.rs +++ b/src/common/meta/src/ddl/create_database.rs @@ -19,8 +19,8 @@ use async_trait::async_trait; use common_error::ext::{BoxedError, ErrorExt}; use common_procedure::error::{FromJsonSnafu, Result as ProcedureResult, ToJsonSnafu}; use common_procedure::{ - Context as ProcedureContext, EventContext, EventTrigger, LockKey, Procedure, ProcedureId, - Status, + Context as ProcedureContext, EventRuntimeContext, EventTrigger, LockKey, Procedure, + ProcedureId, Status, }; use serde::{Deserialize, Serialize}; use serde_with::{DefaultOnNull, serde_as}; @@ -34,7 +34,7 @@ use crate::error::{self, Result}; use crate::instruction::{CacheIdent, UserCacheIdent}; use crate::key::schema_name::{SchemaNameKey, SchemaNameValue}; use crate::lock_key::{CatalogLock, SchemaLock}; -use crate::rpc::ddl::CreatorGrantIntent; +use crate::rpc::ddl::{CreatorGrantIntent, EventContext}; /// Describes the creator-access result of an atomic create. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -93,6 +93,7 @@ impl CreateDatabaseProcedure { create_if_not_exists: bool, options: HashMap, creator: Option, + event_context: EventContext, context: DdlContext, ) -> Self { Self { @@ -104,6 +105,7 @@ impl CreateDatabaseProcedure { create_if_not_exists, options, creator, + event_context, }, } } @@ -278,7 +280,10 @@ impl Procedure for CreateDatabaseProcedure { LockKey::new(lock_key) } - fn event(&self, ctx: &EventContext<'_>) -> Option> { + fn event( + &self, + ctx: &EventRuntimeContext<'_>, + ) -> Option> { if !ctx.event_type_filter.allows(CREATE_DATABASE_EVENT_TYPE) { return None; } @@ -289,6 +294,7 @@ impl Procedure for CreateDatabaseProcedure { &self.data.schema, self.data.create_if_not_exists, &self.data.options, + self.data.event_context.clone(), ) } else { DatabaseDdlEvent::create_lifecycle() @@ -321,6 +327,8 @@ pub struct CreateDatabaseData { /// Authenticated creator whose access is ensured, absent for legacy schema-only requests. #[serde(default)] pub creator: Option, + #[serde(default)] + pub event_context: EventContext, } #[cfg(test)] @@ -415,6 +423,7 @@ mod tests { username: "alice".to_string(), created_at_ns: 1, }), + EventContext::default(), context, ); (procedure, committer) @@ -574,6 +583,7 @@ mod tests { false, HashMap::new(), None, + EventContext::default(), context, ); procedure.data.state = CreateDatabaseState::CreateMetadata; @@ -609,6 +619,7 @@ mod tests { username: "alice".to_string(), created_at_ns: 1, }), + EventContext::default(), context, ); procedure.data.state = CreateDatabaseState::CreateMetadata; diff --git a/src/common/meta/src/ddl/create_flow.rs b/src/common/meta/src/ddl/create_flow.rs index 489e4f760ce..66083f5949c 100644 --- a/src/common/meta/src/ddl/create_flow.rs +++ b/src/common/meta/src/ddl/create_flow.rs @@ -24,8 +24,8 @@ use async_trait::async_trait; use common_catalog::format_full_flow_name; use common_procedure::error::{FromJsonSnafu, ToJsonSnafu}; use common_procedure::{ - Context as ProcedureContext, EventContext, EventTrigger, LockKey, Procedure, ProcedureState, - Result as ProcedureResult, Status, + Context as ProcedureContext, EventRuntimeContext, EventTrigger, LockKey, Procedure, + ProcedureState, Result as ProcedureResult, Status, }; use common_telemetry::info; use common_telemetry::tracing_context::TracingContext; @@ -50,7 +50,7 @@ use crate::key::{DeserializedValueWithBytes, FlowId, FlowPartitionId}; use crate::lock_key::{CatalogLock, FlowNameLock}; use crate::metrics; use crate::peer::Peer; -use crate::rpc::ddl::{CreateFlowTask, FlowQueryContext, QueryContext}; +use crate::rpc::ddl::{CreateFlowTask, EventContext, FlowQueryContext, QueryContext}; /// The procedure of flow creation. pub struct CreateFlowProcedure { @@ -62,7 +62,12 @@ impl CreateFlowProcedure { pub const TYPE_NAME: &'static str = "metasrv-procedure::CreateFlow"; /// Returns a new [CreateFlowProcedure]. - pub fn new(task: CreateFlowTask, query_context: QueryContext, context: DdlContext) -> Self { + pub fn new( + task: CreateFlowTask, + query_context: QueryContext, + event_context: EventContext, + context: DdlContext, + ) -> Self { Self { context, data: CreateFlowData { @@ -76,6 +81,7 @@ impl CreateFlowProcedure { prev_flow_info_value: None, did_replace: false, flow_type: None, + event_context, }, } } @@ -404,7 +410,10 @@ impl Procedure for CreateFlowProcedure { ]) } - fn event(&self, ctx: &EventContext<'_>) -> Option> { + fn event( + &self, + ctx: &EventRuntimeContext<'_>, + ) -> Option> { if !ctx.event_type_filter.allows(CREATE_FLOW_EVENT_TYPE) { return None; } @@ -419,6 +428,7 @@ impl Procedure for CreateFlowProcedure { expire_after: self.data.task.expire_after, eval_interval_secs: self.data.task.eval_interval_secs, }, + self.data.event_context.clone(), ), EventTrigger::Succeeded => { let flow_id = match ctx.lifecycle_state { @@ -701,6 +711,8 @@ pub struct CreateFlowData { #[serde(default)] pub(crate) did_replace: bool, pub(crate) flow_type: Option, + #[serde(default)] + pub(crate) event_context: EventContext, } impl CreateFlowData { diff --git a/src/common/meta/src/ddl/create_logical_tables.rs b/src/common/meta/src/ddl/create_logical_tables.rs index 08f6cb855c1..8b93ecfcc0f 100644 --- a/src/common/meta/src/ddl/create_logical_tables.rs +++ b/src/common/meta/src/ddl/create_logical_tables.rs @@ -24,8 +24,8 @@ use common_catalog::consts::METRIC_ENGINE; use common_event_recorder::Event; use common_procedure::error::{FromJsonSnafu, Result as ProcedureResult, ToJsonSnafu}; use common_procedure::{ - Context as ProcedureContext, EventContext, EventTrigger, LockKey, Procedure, ProcedureState, - Status, + Context as ProcedureContext, EventRuntimeContext, EventTrigger, LockKey, Procedure, + ProcedureState, Status, }; use common_telemetry::{debug, error, warn}; use futures::future; @@ -48,7 +48,7 @@ use crate::error::Result; use crate::key::table_route::TableRouteValue; use crate::lock_key::{CatalogLock, SchemaLock, TableLock, TableNameLock}; use crate::metrics; -use crate::rpc::ddl::CreateTableTask; +use crate::rpc::ddl::{CreateTableTask, EventContext}; use crate::rpc::router::{RegionRoute, find_leaders}; pub struct CreateLogicalTablesProcedure { @@ -62,6 +62,7 @@ impl CreateLogicalTablesProcedure { pub fn new( tasks: Vec, physical_table_id: TableId, + event_context: EventContext, context: DdlContext, ) -> Self { Self { @@ -74,6 +75,7 @@ impl CreateLogicalTablesProcedure { physical_region_numbers: vec![], physical_columns: vec![], physical_partition_columns: vec![], + event_context, }, } } @@ -258,7 +260,7 @@ impl Procedure for CreateLogicalTablesProcedure { LockKey::new(lock_key) } - fn event(&self, ctx: &EventContext<'_>) -> Option> { + fn event(&self, ctx: &EventRuntimeContext<'_>) -> Option> { if !ctx .event_type_filter .allows(TableDdlEventType::CreateLogicalTables.as_str()) @@ -275,7 +277,11 @@ impl Procedure for CreateLogicalTablesProcedure { ) .with_physical_table_id(self.data.physical_table_id) }); - TableDdlEvent::create_logical_tables_submitted(locators, self.data.tasks.len()) + TableDdlEvent::create_logical_tables_submitted( + locators, + self.data.tasks.len(), + self.data.event_context.clone(), + ) } EventTrigger::Succeeded => match ctx.lifecycle_state { ProcedureState::Done { @@ -321,6 +327,8 @@ pub struct CreateTablesData { physical_region_numbers: Vec, physical_columns: Vec, physical_partition_columns: Vec, + #[serde(default)] + event_context: EventContext, } impl CreateTablesData { diff --git a/src/common/meta/src/ddl/create_table.rs b/src/common/meta/src/ddl/create_table.rs index 298fa8e56a7..c1690416a2e 100644 --- a/src/common/meta/src/ddl/create_table.rs +++ b/src/common/meta/src/ddl/create_table.rs @@ -23,8 +23,8 @@ use common_procedure::error::{ }; use common_procedure::local::DynamicKeyLockGuard; use common_procedure::{ - Context as ProcedureContext, EventContext, EventTrigger, LockKey, Procedure, ProcedureId, - ProcedureState, Status, + Context as ProcedureContext, EventRuntimeContext, EventTrigger, LockKey, Procedure, + ProcedureId, ProcedureState, Status, }; use common_telemetry::info; use serde::{Deserialize, Serialize}; @@ -47,7 +47,7 @@ use crate::lock_key::{CatalogLock, SchemaLock, TableNameLock}; use crate::metrics; use crate::peer::PeerAllocContext; use crate::region_keeper::OperatingRegionGuard; -use crate::rpc::ddl::{CreateTableTask, QueryContext}; +use crate::rpc::ddl::{CreateTableTask, EventContext, QueryContext}; use crate::rpc::router::{RegionRoute, operating_leader_region_roles}; use crate::wal_provider::{ RegionWalOptions, acquire_remote_wal_read_locks, optional_region_wal_options_serde, @@ -84,20 +84,17 @@ fn build_executor_from_create_table_data( impl CreateTableProcedure { pub const TYPE_NAME: &'static str = "metasrv-procedure::CreateTable"; - pub fn new(task: CreateTableTask, context: DdlContext) -> Result { - Self::new_with_query_context(task, QueryContext::default(), context) - } - - pub fn new_with_query_context( + pub fn new( task: CreateTableTask, query_context: QueryContext, + event_context: EventContext, context: DdlContext, ) -> Result { let executor = build_executor_from_create_table_data(&task.create_table)?; Ok(Self { context, - data: CreateTableData::new(task, query_context), + data: CreateTableData::new_with_query_context(task, event_context, query_context), opening_regions: vec![], executor, remote_wal_lock_guards: vec![], @@ -399,7 +396,10 @@ impl Procedure for CreateTableProcedure { ]) } - fn event(&self, ctx: &EventContext<'_>) -> Option> { + fn event( + &self, + ctx: &EventRuntimeContext<'_>, + ) -> Option> { if !ctx .event_type_filter .allows(TableDdlEventType::CreateTable.as_str()) @@ -416,6 +416,7 @@ impl Procedure for CreateTableProcedure { locator, create_table.create_if_not_exists, &create_table.engine, + self.data.event_context.clone(), ) } EventTrigger::Succeeded => match ctx.lifecycle_state { @@ -452,6 +453,8 @@ pub struct CreateTableData { #[serde(default)] pub query_context: QueryContext, #[serde(default)] + pub event_context: EventContext, + #[serde(default)] pub column_metadatas: Vec, /// None stands for not allocated yet. pub(crate) table_route: Option, @@ -462,12 +465,21 @@ pub struct CreateTableData { } impl CreateTableData { - pub fn new(task: CreateTableTask, query_context: QueryContext) -> Self { + pub fn new(task: CreateTableTask, event_context: EventContext) -> Self { + Self::new_with_query_context(task, event_context, QueryContext::default()) + } + + pub fn new_with_query_context( + task: CreateTableTask, + event_context: EventContext, + query_context: QueryContext, + ) -> Self { CreateTableData { state: CreateTableState::Prepare, column_metadatas: vec![], task, query_context, + event_context, table_route: None, region_wal_options: None, } diff --git a/src/common/meta/src/ddl/create_view.rs b/src/common/meta/src/ddl/create_view.rs index 1a6b85d55aa..0cb622774f0 100644 --- a/src/common/meta/src/ddl/create_view.rs +++ b/src/common/meta/src/ddl/create_view.rs @@ -16,8 +16,8 @@ use async_trait::async_trait; use common_event_recorder::Event; use common_procedure::error::{FromJsonSnafu, Result as ProcedureResult, ToJsonSnafu}; use common_procedure::{ - Context as ProcedureContext, EventContext, EventTrigger, LockKey, Procedure, ProcedureState, - Status, + Context as ProcedureContext, EventRuntimeContext, EventTrigger, LockKey, Procedure, + ProcedureState, Status, }; use common_telemetry::info; use serde::{Deserialize, Serialize}; @@ -35,7 +35,7 @@ use crate::instruction::CacheIdent; use crate::key::table_name::TableNameKey; use crate::lock_key::{CatalogLock, SchemaLock, TableNameLock}; use crate::metrics; -use crate::rpc::ddl::CreateViewTask; +use crate::rpc::ddl::{CreateViewTask, EventContext}; // The procedure to execute `[CreateViewTask]`. pub struct CreateViewProcedure { @@ -46,13 +46,14 @@ pub struct CreateViewProcedure { impl CreateViewProcedure { pub const TYPE_NAME: &'static str = "metasrv-procedure::CreateView"; - pub fn new(task: CreateViewTask, context: DdlContext) -> Self { + pub fn new(task: CreateViewTask, event_context: EventContext, context: DdlContext) -> Self { Self { context, data: CreateViewData { state: CreateViewState::Prepare, task, need_update: false, + event_context, }, } } @@ -271,7 +272,7 @@ impl Procedure for CreateViewProcedure { ]) } - fn event(&self, ctx: &EventContext<'_>) -> Option> { + fn event(&self, ctx: &EventRuntimeContext<'_>) -> Option> { if !ctx.event_type_filter.allows(CREATE_VIEW_EVENT_TYPE) { return None; } @@ -289,6 +290,7 @@ impl Procedure for CreateViewProcedure { referenced_table_count: self.data.task.table_names().len(), column_count: self.data.task.columns().len(), }, + self.data.event_context.clone(), ) } EventTrigger::Succeeded => match ctx.lifecycle_state { @@ -321,6 +323,8 @@ pub struct CreateViewData { pub task: CreateViewTask, /// Whether to update the view info. pub need_update: bool, + #[serde(default)] + pub event_context: EventContext, } impl CreateViewData { diff --git a/src/common/meta/src/ddl/drop_database.rs b/src/common/meta/src/ddl/drop_database.rs index 6b6a6852daa..d84a5e6dc86 100644 --- a/src/common/meta/src/ddl/drop_database.rs +++ b/src/common/meta/src/ddl/drop_database.rs @@ -23,7 +23,7 @@ use std::fmt::Debug; use common_error::ext::BoxedError; use common_procedure::error::{ExternalSnafu, FromJsonSnafu, ToJsonSnafu}; use common_procedure::{ - Context as ProcedureContext, EventContext, EventTrigger, LockKey, Procedure, + Context as ProcedureContext, EventRuntimeContext, EventTrigger, LockKey, Procedure, Result as ProcedureResult, Status, }; use futures::stream::BoxStream; @@ -38,11 +38,13 @@ use crate::ddl::utils::map_to_procedure_error; use crate::error::Result; use crate::key::table_name::TableNameValue; use crate::lock_key::{CatalogLock, SchemaLock}; +use crate::rpc::ddl::EventContext; pub struct DropDatabaseProcedure { /// The context of procedure runtime. runtime_context: DdlContext, context: DropDatabaseContext, + event_context: EventContext, state: Box, } @@ -85,9 +87,16 @@ pub(crate) trait State: Send + Debug { impl DropDatabaseProcedure { pub const TYPE_NAME: &'static str = "metasrv-procedure::DropDatabase"; - pub fn new(catalog: String, schema: String, drop_if_exists: bool, context: DdlContext) -> Self { + pub fn new( + catalog: String, + schema: String, + drop_if_exists: bool, + event_context: EventContext, + context: DdlContext, + ) -> Self { Self { runtime_context: context, + event_context, context: DropDatabaseContext { catalog, schema, @@ -104,11 +113,13 @@ impl DropDatabaseProcedure { catalog, schema, drop_if_exists, + event_context, state, } = serde_json::from_str(json).context(FromJsonSnafu)?; Ok(Self { runtime_context, + event_context, context: DropDatabaseContext { catalog, schema, @@ -159,6 +170,7 @@ impl Procedure for DropDatabaseProcedure { catalog: &self.context.catalog, schema: &self.context.schema, drop_if_exists: self.context.drop_if_exists, + event_context: &self.event_context, state: self.state.as_ref(), }; @@ -174,7 +186,10 @@ impl Procedure for DropDatabaseProcedure { LockKey::new(lock_key) } - fn event(&self, ctx: &EventContext<'_>) -> Option> { + fn event( + &self, + ctx: &EventRuntimeContext<'_>, + ) -> Option> { if !ctx.event_type_filter.allows(DROP_DATABASE_EVENT_TYPE) { return None; } @@ -184,6 +199,7 @@ impl Procedure for DropDatabaseProcedure { &self.context.catalog, &self.context.schema, self.context.drop_if_exists, + self.event_context.clone(), ) } else { DatabaseDdlEvent::drop_lifecycle() @@ -199,6 +215,7 @@ struct DropDatabaseData<'a> { // The schema name schema: &'a str, drop_if_exists: bool, + event_context: &'a EventContext, state: &'a dyn State, } @@ -209,5 +226,7 @@ struct DropDatabaseOwnedData { // The schema name schema: String, drop_if_exists: bool, + #[serde(default)] + event_context: EventContext, state: Box, } diff --git a/src/common/meta/src/ddl/drop_flow.rs b/src/common/meta/src/ddl/drop_flow.rs index 1725b125508..badded386ad 100644 --- a/src/common/meta/src/ddl/drop_flow.rs +++ b/src/common/meta/src/ddl/drop_flow.rs @@ -21,7 +21,7 @@ use common_error::ext::ErrorExt; use common_error::status_code::StatusCode; use common_procedure::error::{FromJsonSnafu, ToJsonSnafu}; use common_procedure::{ - Context as ProcedureContext, EventContext, EventTrigger, LockKey, Procedure, + Context as ProcedureContext, EventRuntimeContext, EventTrigger, LockKey, Procedure, Result as ProcedureResult, Status, }; use common_telemetry::info; @@ -41,7 +41,7 @@ use crate::key::flow::flow_info::FlowInfoValue; use crate::key::flow::flow_route::FlowRouteValue; use crate::lock_key::{CatalogLock, FlowLock}; use crate::metrics; -use crate::rpc::ddl::DropFlowTask; +use crate::rpc::ddl::{DropFlowTask, EventContext}; /// The procedure for dropping a flow. pub struct DropFlowProcedure { @@ -54,7 +54,7 @@ pub struct DropFlowProcedure { impl DropFlowProcedure { pub const TYPE_NAME: &'static str = "metasrv-procedure::DropFlow"; - pub fn new(task: DropFlowTask, context: DdlContext) -> Self { + pub fn new(task: DropFlowTask, event_context: EventContext, context: DdlContext) -> Self { Self { context, data: DropFlowData { @@ -62,6 +62,7 @@ impl DropFlowProcedure { task, flow_info_value: None, flow_route_values: vec![], + event_context, }, } } @@ -222,7 +223,10 @@ impl Procedure for DropFlowProcedure { LockKey::new(lock_key) } - fn event(&self, ctx: &EventContext<'_>) -> Option> { + fn event( + &self, + ctx: &EventRuntimeContext<'_>, + ) -> Option> { if !ctx.event_type_filter.allows(DROP_FLOW_EVENT_TYPE) { return None; } @@ -233,6 +237,7 @@ impl Procedure for DropFlowProcedure { &self.data.task.flow_name, self.data.task.flow_id, self.data.task.drop_if_exists, + self.data.event_context.clone(), ), _ => FlowDdlEvent::drop_lifecycle(), }; @@ -248,6 +253,8 @@ pub(crate) struct DropFlowData { task: DropFlowTask, pub(crate) flow_info_value: Option, pub(crate) flow_route_values: Vec, + #[serde(default)] + pub(crate) event_context: EventContext, } /// The state of drop flow diff --git a/src/common/meta/src/ddl/drop_table.rs b/src/common/meta/src/ddl/drop_table.rs index d03f8166b45..01101a1508b 100644 --- a/src/common/meta/src/ddl/drop_table.rs +++ b/src/common/meta/src/ddl/drop_table.rs @@ -22,8 +22,8 @@ use common_error::ext::BoxedError; use common_event_recorder::Event; use common_procedure::error::{ExternalSnafu, FromJsonSnafu, ToJsonSnafu}; use common_procedure::{ - Context as ProcedureContext, Error as ProcedureError, EventContext, EventTrigger, LockKey, - Procedure, Result as ProcedureResult, Status, + Context as ProcedureContext, Error as ProcedureError, EventRuntimeContext, EventTrigger, + LockKey, Procedure, Result as ProcedureResult, Status, }; use common_telemetry::info; use common_telemetry::tracing::warn; @@ -48,7 +48,7 @@ use crate::key::table_route::TableRouteValue; use crate::lock_key::{CatalogLock, SchemaLock, TableLock, TableNameLock}; use crate::metrics; use crate::region_keeper::OperatingRegionGuard; -use crate::rpc::ddl::DropTableTask; +use crate::rpc::ddl::{DropTableTask, EventContext}; use crate::rpc::router::{RegionRoute, operating_leader_region_roles}; #[cfg(feature = "enterprise")] @@ -74,11 +74,12 @@ pub struct DropTableProcedure { impl DropTableProcedure { pub const TYPE_NAME: &'static str = "metasrv-procedure::DropTable"; - pub fn new(task: DropTableTask, context: DdlContext) -> Self { + pub fn new(task: DropTableTask, context: DdlContext, event_context: EventContext) -> Self { let data = DropTableData::new( task, cfg!(feature = "enterprise") && context.soft_drop_enabled, context.soft_drop_retention, + event_context, ); let executor = data.build_executor(); Self { @@ -413,7 +414,7 @@ impl Procedure for DropTableProcedure { LockKey::new(lock_key) } - fn event(&self, ctx: &EventContext<'_>) -> Option> { + fn event(&self, ctx: &EventRuntimeContext<'_>) -> Option> { if !ctx .event_type_filter .allows(TableDdlEventType::DropTable.as_str()) @@ -425,7 +426,11 @@ impl Procedure for DropTableProcedure { let task = &self.data.task; let locator = TableDdlLocator::new(&task.catalog, &task.schema, &task.table) .with_table_id(task.table_id); - TableDdlEvent::drop_table_submitted(locator, task.drop_if_exists) + TableDdlEvent::drop_table_submitted( + locator, + task.drop_if_exists, + self.data.event_context.clone(), + ) } _ => TableDdlEvent::lifecycle(TableDdlEventType::DropTable), }; @@ -480,6 +485,8 @@ pub struct DropTableData { pub soft_drop_retention_millis: Option, #[serde(default)] pub drop_generation: Option, + #[serde(default)] + pub event_context: EventContext, } impl DropTableData { @@ -487,6 +494,7 @@ impl DropTableData { task: DropTableTask, soft_drop_enabled: bool, soft_drop_retention: Option, + event_context: EventContext, ) -> Self { Self { state: DropTableState::Prepare, @@ -500,6 +508,7 @@ impl DropTableData { retention_expires_at: None, soft_drop_retention_millis: soft_drop_retention_millis(soft_drop_retention), drop_generation: None, + event_context, } } diff --git a/src/common/meta/src/ddl/drop_view.rs b/src/common/meta/src/ddl/drop_view.rs index 3804df6862c..53a62e76159 100644 --- a/src/common/meta/src/ddl/drop_view.rs +++ b/src/common/meta/src/ddl/drop_view.rs @@ -16,7 +16,7 @@ use async_trait::async_trait; use common_event_recorder::Event; use common_procedure::error::{FromJsonSnafu, ToJsonSnafu}; use common_procedure::{ - Context as ProcedureContext, EventContext, EventTrigger, LockKey, Procedure, + Context as ProcedureContext, EventRuntimeContext, EventTrigger, LockKey, Procedure, Result as ProcedureResult, Status, }; use common_telemetry::info; @@ -35,7 +35,7 @@ use crate::instruction::CacheIdent; use crate::key::table_name::TableNameKey; use crate::lock_key::{CatalogLock, SchemaLock, TableLock}; use crate::metrics; -use crate::rpc::ddl::DropViewTask; +use crate::rpc::ddl::{DropViewTask, EventContext}; /// The procedure for dropping a view. pub struct DropViewProcedure { @@ -48,12 +48,13 @@ pub struct DropViewProcedure { impl DropViewProcedure { pub const TYPE_NAME: &'static str = "metasrv-procedure::DropView"; - pub fn new(task: DropViewTask, context: DdlContext) -> Self { + pub fn new(task: DropViewTask, event_context: EventContext, context: DdlContext) -> Self { Self { context, data: DropViewData { state: DropViewState::Prepare, task, + event_context, }, } } @@ -213,7 +214,7 @@ impl Procedure for DropViewProcedure { LockKey::new(lock_key) } - fn event(&self, ctx: &EventContext<'_>) -> Option> { + fn event(&self, ctx: &EventRuntimeContext<'_>) -> Option> { if !ctx.event_type_filter.allows(DROP_VIEW_EVENT_TYPE) { return None; } @@ -227,6 +228,7 @@ impl Procedure for DropViewProcedure { table_ref.table, self.data.view_id(), self.data.task.drop_if_exists, + self.data.event_context.clone(), ) } _ => ViewDdlEvent::drop_lifecycle(), @@ -241,6 +243,8 @@ impl Procedure for DropViewProcedure { pub(crate) struct DropViewData { state: DropViewState, task: DropViewTask, + #[serde(default)] + event_context: EventContext, } impl DropViewData { diff --git a/src/common/meta/src/ddl/event/database.rs b/src/common/meta/src/ddl/event/database.rs index a3db0ebab28..9d70ea35a5e 100644 --- a/src/common/meta/src/ddl/event/database.rs +++ b/src/common/meta/src/ddl/event/database.rs @@ -19,13 +19,14 @@ use api::v1::{ColumnSchema, Row}; use common_event_recorder::Event; use common_event_recorder::error::{Result, SerializeEventSnafu}; use common_event_recorder::event_table::{ - CATALOG_NAME_COLUMN as EVENT_TABLE_CATALOG_NAME_COLUMN, - SCHEMA_NAME_COLUMN as EVENT_TABLE_SCHEMA_NAME_COLUMN, column_schemas, nullable_string, + CATALOG_NAME_COLUMN as EVENT_TABLE_CATALOG_NAME_COLUMN, EVENT_CONTEXT_COLUMN, + SCHEMA_NAME_COLUMN as EVENT_TABLE_SCHEMA_NAME_COLUMN, column_schemas, nullable_json, + nullable_string, }; use serde::Serialize; use snafu::ResultExt; -use crate::rpc::ddl::{AlterDatabaseKind, SetDatabaseOption, UnsetDatabaseOption}; +use crate::rpc::ddl::{AlterDatabaseKind, EventContext, SetDatabaseOption, UnsetDatabaseOption}; pub(crate) const CREATE_DATABASE_EVENT_TYPE: &str = "create_database"; pub(crate) const ALTER_DATABASE_EVENT_TYPE: &str = "alter_database"; @@ -39,6 +40,7 @@ pub(crate) struct DatabaseDdlEvent { catalog_name: Option, schema_name: Option, payload: Option, + event_context: Option, } #[derive(Debug, Serialize)] @@ -88,6 +90,7 @@ impl DatabaseDdlEvent { schema_name: &str, create_if_not_exists: bool, options: &HashMap, + event_context: EventContext, ) -> Self { let mut options = options.iter().collect::>(); options.sort_unstable_by_key(|(left, _)| *left); @@ -107,6 +110,7 @@ impl DatabaseDdlEvent { create_if_not_exists, options, }), + event_context, ) } @@ -114,6 +118,7 @@ impl DatabaseDdlEvent { catalog_name: &str, schema_name: &str, kind: &AlterDatabaseKind, + event_context: EventContext, ) -> Self { let intent = match kind { AlterDatabaseKind::SetDatabaseOptions(options) => AlterDatabaseIntent::Set { @@ -152,6 +157,7 @@ impl DatabaseDdlEvent { version: PAYLOAD_VERSION, intent, }), + event_context, ) } @@ -159,6 +165,7 @@ impl DatabaseDdlEvent { catalog_name: &str, schema_name: &str, drop_if_exists: bool, + event_context: EventContext, ) -> Self { Self::submitted( DROP_DATABASE_EVENT_TYPE, @@ -168,6 +175,7 @@ impl DatabaseDdlEvent { version: PAYLOAD_VERSION, drop_if_exists, }), + event_context, ) } @@ -188,12 +196,14 @@ impl DatabaseDdlEvent { catalog_name: &str, schema_name: &str, payload: DatabaseDdlPayload, + event_context: EventContext, ) -> Self { Self { event_type, catalog_name: Some(catalog_name.to_string()), schema_name: Some(schema_name.to_string()), payload: Some(payload), + event_context: Some(event_context), } } @@ -203,6 +213,7 @@ impl DatabaseDdlEvent { catalog_name: None, schema_name: None, payload: None, + event_context: None, } } } @@ -223,14 +234,22 @@ impl Event for DatabaseDdlEvent { column_schemas([ &EVENT_TABLE_CATALOG_NAME_COLUMN, &EVENT_TABLE_SCHEMA_NAME_COLUMN, + &EVENT_CONTEXT_COLUMN, ]) } fn extra_rows(&self) -> Result> { + let event_context = self + .event_context + .as_ref() + .map(serde_json::to_value) + .transpose() + .context(SerializeEventSnafu)?; Ok(vec![Row { values: vec![ nullable_string(self.catalog_name.as_deref()), nullable_string(self.schema_name.as_deref()), + nullable_json(event_context.as_ref()), ], }]) } diff --git a/src/common/meta/src/ddl/event/flow.rs b/src/common/meta/src/ddl/event/flow.rs index 68ee51a4fc6..8a52486ddfe 100644 --- a/src/common/meta/src/ddl/event/flow.rs +++ b/src/common/meta/src/ddl/event/flow.rs @@ -19,12 +19,14 @@ use api::v1::{ColumnSchema, Row}; use common_event_recorder::Event; use common_event_recorder::error::{Result, SerializeEventSnafu}; use common_event_recorder::event_table::{ - CATALOG_NAME_COLUMN, FLOW_ID_COLUMN, FLOW_NAME_COLUMN, column_schemas, nullable_string, - nullable_value, + CATALOG_NAME_COLUMN, EVENT_CONTEXT_COLUMN, FLOW_ID_COLUMN, FLOW_NAME_COLUMN, column_schemas, + nullable_json, nullable_string, nullable_value, }; use serde::Serialize; use snafu::ResultExt; +use crate::rpc::ddl::EventContext; + pub(crate) const CREATE_FLOW_EVENT_TYPE: &str = "create_flow"; pub(crate) const DROP_FLOW_EVENT_TYPE: &str = "drop_flow"; @@ -69,14 +71,15 @@ pub(crate) struct FlowDdlEvent { flow_name: Option, flow_id: Option, payload: Option, + event_context: Option, } impl FlowDdlEvent { - /// Builds the bounded event emitted when creating a Flow is submitted. pub(crate) fn create_submitted( catalog_name: &str, flow_name: &str, intent: CreateFlowEventIntent, + event_context: EventContext, ) -> Self { Self { event_type: CREATE_FLOW_EVENT_TYPE, @@ -90,15 +93,16 @@ impl FlowDdlEvent { expire_after: intent.expire_after, eval_interval_secs: intent.eval_interval_secs, })), + event_context: Some(event_context), } } - /// Builds the bounded event emitted when dropping a Flow is submitted. pub(crate) fn drop_submitted( catalog_name: &str, flow_name: &str, flow_id: u32, drop_if_exists: bool, + event_context: EventContext, ) -> Self { Self { event_type: DROP_FLOW_EVENT_TYPE, @@ -109,6 +113,7 @@ impl FlowDdlEvent { version: PAYLOAD_VERSION, drop_if_exists, })), + event_context: Some(event_context), } } @@ -137,6 +142,7 @@ impl FlowDdlEvent { flow_name: None, flow_id: None, payload: None, + event_context: None, } } } @@ -154,15 +160,27 @@ impl Event for FlowDdlEvent { } fn extra_schema(&self) -> Vec { - column_schemas([&CATALOG_NAME_COLUMN, &FLOW_NAME_COLUMN, &FLOW_ID_COLUMN]) + column_schemas([ + &CATALOG_NAME_COLUMN, + &FLOW_NAME_COLUMN, + &FLOW_ID_COLUMN, + &EVENT_CONTEXT_COLUMN, + ]) } fn extra_rows(&self) -> Result> { + let event_context = self + .event_context + .as_ref() + .map(serde_json::to_value) + .transpose() + .context(SerializeEventSnafu)?; Ok(vec![Row { values: vec![ nullable_string(self.catalog_name.as_deref()), nullable_string(self.flow_name.as_deref()), nullable_value(self.flow_id.map(ValueData::U32Value)), + nullable_json(event_context.as_ref()), ], }]) } diff --git a/src/common/meta/src/ddl/event/table.rs b/src/common/meta/src/ddl/event/table.rs index f4b25b141b8..8608b3151a6 100644 --- a/src/common/meta/src/ddl/event/table.rs +++ b/src/common/meta/src/ddl/event/table.rs @@ -21,14 +21,17 @@ use api::v1::{ColumnSchema, Row}; use common_event_recorder::Event; use common_event_recorder::error::{Result, SerializeEventSnafu}; use common_event_recorder::event_table::{ - CATALOG_NAME_COLUMN, PHYSICAL_TABLE_ID_COLUMN, SCHEMA_NAME_COLUMN, TABLE_ID_COLUMN, - TABLE_NAME_COLUMN, column_schemas, nullable_string, nullable_value, + CATALOG_NAME_COLUMN, EVENT_CONTEXT_COLUMN, PHYSICAL_TABLE_ID_COLUMN, SCHEMA_NAME_COLUMN, + TABLE_ID_COLUMN, TABLE_NAME_COLUMN, column_schemas, nullable_json, nullable_string, + nullable_value, }; use serde::Serialize; use serde_json::Value as JsonValue; use snafu::ResultExt; use store_api::storage::TableId; +use crate::rpc::ddl::EventContext; + /// Current version of table DDL event payloads. pub(crate) const TABLE_DDL_PAYLOAD_VERSION: u8 = 1; @@ -211,6 +214,7 @@ pub(crate) struct TableDdlEvent { event_type: TableDdlEventType, locators: Vec, payload: Option, + event_context: Option, } impl TableDdlEvent { @@ -219,6 +223,7 @@ impl TableDdlEvent { locator: TableDdlLocator, create_if_not_exists: bool, engine: &str, + event_context: EventContext, ) -> Self { Self::submitted( TableDdlEventType::CreateTable, @@ -228,6 +233,7 @@ impl TableDdlEvent { create_if_not_exists, engine: engine.to_string(), }), + Some(event_context), ) } @@ -235,6 +241,7 @@ impl TableDdlEvent { pub(crate) fn create_logical_tables_submitted( locators: impl IntoIterator, table_count: usize, + event_context: EventContext, ) -> Self { Self::submitted( TableDdlEventType::CreateLogicalTables, @@ -243,6 +250,7 @@ impl TableDdlEvent { version: TABLE_DDL_PAYLOAD_VERSION, table_count, }), + Some(event_context), ) } @@ -250,6 +258,7 @@ impl TableDdlEvent { pub(crate) fn alter_table_submitted( locator: TableDdlLocator, kind: Option<&'static str>, + event_context: EventContext, ) -> Self { Self::submitted( TableDdlEventType::AlterTable, @@ -258,6 +267,7 @@ impl TableDdlEvent { version: TABLE_DDL_PAYLOAD_VERSION, kind, }), + Some(event_context), ) } @@ -266,6 +276,7 @@ impl TableDdlEvent { locators: impl IntoIterator, table_count: usize, kinds: impl IntoIterator, + event_context: EventContext, ) -> Self { let kinds = kinds .into_iter() @@ -280,11 +291,16 @@ impl TableDdlEvent { table_count, kinds, }), + Some(event_context), ) } /// Builds the bounded event emitted when dropping a table is submitted. - pub(crate) fn drop_table_submitted(locator: TableDdlLocator, drop_if_exists: bool) -> Self { + pub(crate) fn drop_table_submitted( + locator: TableDdlLocator, + drop_if_exists: bool, + event_context: EventContext, + ) -> Self { Self::submitted( TableDdlEventType::DropTable, [locator], @@ -292,30 +308,39 @@ impl TableDdlEvent { version: TABLE_DDL_PAYLOAD_VERSION, drop_if_exists, }), + Some(event_context), ) } /// Builds the bounded event emitted when restoring a dropped table is submitted. #[cfg(feature = "enterprise")] - pub(crate) fn undrop_table_submitted(locator: TableDdlLocator) -> Self { + pub(crate) fn undrop_table_submitted( + locator: TableDdlLocator, + event_context: EventContext, + ) -> Self { Self::submitted( TableDdlEventType::UndropTable, [locator], TableDdlPayload::UndropTable(UndropTablePayload { version: TABLE_DDL_PAYLOAD_VERSION, }), + Some(event_context), ) } /// Builds the bounded event emitted when purging a dropped table is submitted. #[cfg(feature = "enterprise")] - pub(crate) fn purge_dropped_table_submitted(locator: TableDdlLocator) -> Self { + pub(crate) fn purge_dropped_table_submitted( + locator: TableDdlLocator, + event_context: EventContext, + ) -> Self { Self::submitted( TableDdlEventType::PurgeDroppedTable, [locator], TableDdlPayload::PurgeDroppedTable(PurgeDroppedTablePayload { version: TABLE_DDL_PAYLOAD_VERSION, }), + Some(event_context), ) } @@ -323,6 +348,7 @@ impl TableDdlEvent { pub(crate) fn truncate_table_submitted( locator: TableDdlLocator, time_range_count: usize, + event_context: EventContext, ) -> Self { Self::submitted( TableDdlEventType::TruncateTable, @@ -331,6 +357,7 @@ impl TableDdlEvent { version: TABLE_DDL_PAYLOAD_VERSION, time_range_count, }), + Some(event_context), ) } @@ -340,6 +367,7 @@ impl TableDdlEvent { event_type, locators: vec![TableDdlLocator::default()], payload: None, + event_context: None, } } @@ -349,6 +377,7 @@ impl TableDdlEvent { event_type: TableDdlEventType::CreateTable, locators: vec![TableDdlLocator::from_table_id(table_id)], payload: None, + event_context: None, } } @@ -360,6 +389,7 @@ impl TableDdlEvent { event_type: TableDdlEventType::CreateLogicalTables, locators: locators.into_iter().collect(), payload: None, + event_context: None, } } @@ -367,11 +397,13 @@ impl TableDdlEvent { event_type: TableDdlEventType, locators: impl IntoIterator, payload: TableDdlPayload, + event_context: Option, ) -> Self { Self { event_type, locators: locators.into_iter().collect(), payload: Some(payload), + event_context, } } @@ -384,7 +416,7 @@ impl TableDdlEvent { ]) } - fn locator_row(&self, locator: &TableDdlLocator) -> Row { + fn locator_row(&self, locator: &TableDdlLocator) -> Result { let mut values = vec![ nullable_string(locator.catalog_name.as_deref()), nullable_string(locator.schema_name.as_deref()), @@ -394,7 +426,14 @@ impl TableDdlEvent { if self.event_type.has_physical_table_id() { values.push(nullable_table_id(locator.physical_table_id)); } - Row { values } + let event_context = self + .event_context + .as_ref() + .map(serde_json::to_value) + .transpose() + .context(SerializeEventSnafu)?; + values.push(nullable_json(event_context.as_ref())); + Ok(Row { values }) } } @@ -415,15 +454,15 @@ impl Event for TableDdlEvent { if self.event_type.has_physical_table_id() { schema.push(PHYSICAL_TABLE_ID_COLUMN.column_schema()); } + schema.push(EVENT_CONTEXT_COLUMN.column_schema()); schema } fn extra_rows(&self) -> Result> { - Ok(self - .locators + self.locators .iter() .map(|locator| self.locator_row(locator)) - .collect()) + .collect() } fn as_any(&self) -> &dyn Any { diff --git a/src/common/meta/src/ddl/event/view.rs b/src/common/meta/src/ddl/event/view.rs index 651805f46d1..d6dcf2c5f17 100644 --- a/src/common/meta/src/ddl/event/view.rs +++ b/src/common/meta/src/ddl/event/view.rs @@ -19,12 +19,14 @@ use api::v1::{ColumnSchema, Row}; use common_event_recorder::Event; use common_event_recorder::error::{Result, SerializeEventSnafu}; use common_event_recorder::event_table::{ - CATALOG_NAME_COLUMN, SCHEMA_NAME_COLUMN, VIEW_ID_COLUMN, VIEW_NAME_COLUMN, column_schemas, - nullable_string, nullable_value, + CATALOG_NAME_COLUMN, EVENT_CONTEXT_COLUMN, SCHEMA_NAME_COLUMN, VIEW_ID_COLUMN, + VIEW_NAME_COLUMN, column_schemas, nullable_json, nullable_string, nullable_value, }; use serde::Serialize; use snafu::ResultExt; +use crate::rpc::ddl::EventContext; + pub(crate) const CREATE_VIEW_EVENT_TYPE: &str = "create_view"; pub(crate) const DROP_VIEW_EVENT_TYPE: &str = "drop_view"; @@ -62,6 +64,7 @@ pub(crate) struct ViewDdlEvent { view_name: Option, view_id: Option, payload: Option, + event_context: Option, } #[derive(Debug, Serialize)] @@ -72,12 +75,12 @@ enum ViewDdlPayload { } impl ViewDdlEvent { - /// Builds the bounded event emitted when creating a View is submitted. pub(crate) fn create_submitted( catalog_name: &str, schema_name: &str, view_name: &str, intent: CreateViewEventIntent, + event_context: EventContext, ) -> Self { Self::submitted( CREATE_VIEW_EVENT_TYPE, @@ -92,16 +95,17 @@ impl ViewDdlEvent { referenced_table_count: intent.referenced_table_count, column_count: intent.column_count, }), + event_context, ) } - /// Builds the bounded event emitted when dropping a View is submitted. pub(crate) fn drop_submitted( catalog_name: &str, schema_name: &str, view_name: &str, view_id: u32, drop_if_exists: bool, + event_context: EventContext, ) -> Self { Self::submitted( DROP_VIEW_EVENT_TYPE, @@ -113,6 +117,7 @@ impl ViewDdlEvent { version: PAYLOAD_VERSION, drop_if_exists, }), + event_context, ) } @@ -138,6 +143,7 @@ impl ViewDdlEvent { view_name: &str, view_id: Option, payload: ViewDdlPayload, + event_context: EventContext, ) -> Self { Self { event_type, @@ -146,6 +152,7 @@ impl ViewDdlEvent { view_name: Some(view_name.to_string()), view_id, payload: Some(payload), + event_context: Some(event_context), } } @@ -157,6 +164,7 @@ impl ViewDdlEvent { view_name: None, view_id: None, payload: None, + event_context: None, } } @@ -168,6 +176,7 @@ impl ViewDdlEvent { view_name: None, view_id: Some(view_id), payload: None, + event_context: None, } } } @@ -190,16 +199,24 @@ impl Event for ViewDdlEvent { &SCHEMA_NAME_COLUMN, &VIEW_NAME_COLUMN, &VIEW_ID_COLUMN, + &EVENT_CONTEXT_COLUMN, ]) } fn extra_rows(&self) -> Result> { + let event_context = self + .event_context + .as_ref() + .map(serde_json::to_value) + .transpose() + .context(SerializeEventSnafu)?; Ok(vec![Row { values: vec![ nullable_string(self.catalog_name.as_deref()), nullable_string(self.schema_name.as_deref()), nullable_string(self.view_name.as_deref()), nullable_value(self.view_id.map(ValueData::U32Value)), + nullable_json(event_context.as_ref()), ], }]) } diff --git a/src/common/meta/src/ddl/purge_dropped_table.rs b/src/common/meta/src/ddl/purge_dropped_table.rs index 18c161a297f..111b79cabe8 100644 --- a/src/common/meta/src/ddl/purge_dropped_table.rs +++ b/src/common/meta/src/ddl/purge_dropped_table.rs @@ -14,7 +14,7 @@ use std::collections::HashMap; use async_trait::async_trait; use common_procedure::error::{FromJsonSnafu, ToJsonSnafu}; use common_procedure::{ - Context as ProcedureContext, EventContext, EventTrigger, LockKey, Procedure, + Context as ProcedureContext, EventRuntimeContext, EventTrigger, LockKey, Procedure, Result as ProcedureResult, Status, }; use common_telemetry::info; @@ -38,7 +38,7 @@ use crate::error::{self, Result}; use crate::key::DroppedTableMetadata; use crate::key::table_route::TableRouteValue; use crate::lock_key::TableLock; -use crate::rpc::ddl::PurgeDroppedTableTask; +use crate::rpc::ddl::{EventContext, PurgeDroppedTableTask}; use crate::rpc::router::RegionRoute; pub struct PurgeDroppedTableProcedure { @@ -50,17 +50,25 @@ impl PurgeDroppedTableProcedure { pub const TYPE_NAME: &'static str = "metasrv-procedure::PurgeDroppedTable"; pub const EXPIRED_TYPE_NAME: &'static str = "metasrv-procedure::PurgeExpiredDroppedTable"; - pub fn new(task: PurgeDroppedTableTask, context: DdlContext) -> Self { + pub fn new( + task: PurgeDroppedTableTask, + context: DdlContext, + event_context: EventContext, + ) -> Self { Self { context, - data: PurgeDroppedTableData::new(task), + data: PurgeDroppedTableData::new(task, event_context), } } - pub fn new_if_expired(task: PurgeDroppedTableTask, context: DdlContext) -> Self { + pub fn new_if_expired( + task: PurgeDroppedTableTask, + context: DdlContext, + event_context: EventContext, + ) -> Self { Self { context, - data: PurgeDroppedTableData::new_if_expired(task), + data: PurgeDroppedTableData::new_if_expired(task, event_context), } } @@ -256,7 +264,10 @@ impl Procedure for PurgeDroppedTableProcedure { LockKey::new(vec![TableLock::Write(self.data.task.table_id).into()]) } - fn event(&self, ctx: &EventContext<'_>) -> Option> { + fn event( + &self, + ctx: &EventRuntimeContext<'_>, + ) -> Option> { if !ctx .event_type_filter .allows(TableDdlEventType::PurgeDroppedTable.as_str()) @@ -266,7 +277,10 @@ impl Procedure for PurgeDroppedTableProcedure { let event = match &ctx.trigger { EventTrigger::Submitted => { let locator = TableDdlLocator::from_table_id(self.data.task.table_id); - TableDdlEvent::purge_dropped_table_submitted(locator) + TableDdlEvent::purge_dropped_table_submitted( + locator, + self.data.event_context.clone(), + ) } _ => TableDdlEvent::lifecycle(TableDdlEventType::PurgeDroppedTable), }; @@ -293,10 +307,12 @@ pub struct PurgeDroppedTableData { drop_generation_loaded: bool, #[serde(default)] purging_claimed: bool, + #[serde(default)] + event_context: EventContext, } impl PurgeDroppedTableData { - fn new(task: PurgeDroppedTableTask) -> Self { + fn new(task: PurgeDroppedTableTask, event_context: EventContext) -> Self { Self { state: PurgeDroppedTableState::Prepare, task, @@ -309,13 +325,14 @@ impl PurgeDroppedTableData { drop_generation: None, drop_generation_loaded: false, purging_claimed: false, + event_context, } } - fn new_if_expired(task: PurgeDroppedTableTask) -> Self { + fn new_if_expired(task: PurgeDroppedTableTask, event_context: EventContext) -> Self { Self { check_expired: true, - ..Self::new(task) + ..Self::new(task, event_context) } } diff --git a/src/common/meta/src/ddl/test_util.rs b/src/common/meta/src/ddl/test_util.rs index 7d8cdd895b2..3686ceb33d2 100644 --- a/src/common/meta/src/ddl/test_util.rs +++ b/src/common/meta/src/ddl/test_util.rs @@ -45,7 +45,7 @@ use crate::key::node_address::{NodeAddressKey, NodeAddressValue}; use crate::key::table_route::TableRouteValue; use crate::key::{MetadataKey, MetadataValue}; use crate::peer::Peer; -use crate::rpc::ddl::CreateTableTask; +use crate::rpc::ddl::{CreateTableTask, EventContext}; use crate::rpc::store::PutRequest; pub async fn create_physical_table_metadata( @@ -104,7 +104,12 @@ pub async fn create_logical_table( table_name: &str, ) -> TableId { let tasks = vec![test_create_logical_table_task(table_name)]; - let mut procedure = CreateLogicalTablesProcedure::new(tasks, physical_table_id, ddl_context); + let mut procedure = CreateLogicalTablesProcedure::new( + tasks, + physical_table_id, + EventContext::default(), + ddl_context, + ); let status = procedure.on_prepare().await.unwrap(); assert_matches!( status, diff --git a/src/common/meta/src/ddl/tests/alter_logical_tables.rs b/src/common/meta/src/ddl/tests/alter_logical_tables.rs index bef06cd8391..09662199389 100644 --- a/src/common/meta/src/ddl/tests/alter_logical_tables.rs +++ b/src/common/meta/src/ddl/tests/alter_logical_tables.rs @@ -43,7 +43,7 @@ use crate::error::Error::{AlterLogicalTablesInvalidArguments, TableNotFound}; use crate::error::Result; use crate::key::table_name::TableNameKey; use crate::key::table_route::{PhysicalTableRouteValue, TableRouteValue}; -use crate::rpc::ddl::AlterTableTask; +use crate::rpc::ddl::{AlterTableTask, EventContext}; use crate::rpc::router::{Region, RegionRoute}; use crate::test_util::{MockDatanodeManager, new_ddl_context}; @@ -162,7 +162,12 @@ async fn test_on_prepare_check_schema() { ), ]; let physical_table_id = 1024u32; - let mut procedure = AlterLogicalTablesProcedure::new(tasks, physical_table_id, ddl_context); + let mut procedure = AlterLogicalTablesProcedure::new( + tasks, + physical_table_id, + EventContext::default(), + ddl_context, + ); let err = procedure.on_prepare().await.unwrap_err(); assert_matches!(err, AlterLogicalTablesInvalidArguments { .. }); } @@ -177,7 +182,12 @@ async fn test_on_prepare_check_alter_kind() { "new_table1", )]; let physical_table_id = 1024u32; - let mut procedure = AlterLogicalTablesProcedure::new(tasks, physical_table_id, ddl_context); + let mut procedure = AlterLogicalTablesProcedure::new( + tasks, + physical_table_id, + EventContext::default(), + ddl_context, + ); let err = procedure.on_prepare().await.unwrap_err(); assert_matches!(err, AlterLogicalTablesInvalidArguments { .. }); } @@ -197,7 +207,8 @@ async fn test_on_prepare_different_physical_table() { make_alter_logical_table_add_column_task(None, "table2", vec!["column2".to_string()]), ]; - let mut procedure = AlterLogicalTablesProcedure::new(tasks, phy1_id, ddl_context); + let mut procedure = + AlterLogicalTablesProcedure::new(tasks, phy1_id, EventContext::default(), ddl_context); let err = procedure.on_prepare().await.unwrap_err(); assert_matches!(err, AlterLogicalTablesInvalidArguments { .. }); } @@ -218,7 +229,8 @@ async fn test_on_prepare_logical_table_not_exists() { make_alter_logical_table_add_column_task(None, "table2", vec!["column2".to_string()]), ]; - let mut procedure = AlterLogicalTablesProcedure::new(tasks, phy_id, ddl_context); + let mut procedure = + AlterLogicalTablesProcedure::new(tasks, phy_id, EventContext::default(), ddl_context); let err = procedure.on_prepare().await.unwrap_err(); assert_matches!(err, TableNotFound { .. }); } @@ -241,7 +253,8 @@ async fn test_on_prepare() { make_alter_logical_table_add_column_task(None, "table3", vec!["column3".to_string()]), ]; - let mut procedure = AlterLogicalTablesProcedure::new(tasks, phy_id, ddl_context); + let mut procedure = + AlterLogicalTablesProcedure::new(tasks, phy_id, EventContext::default(), ddl_context); let result = procedure.on_prepare().await; assert_matches!( result, @@ -277,7 +290,12 @@ async fn test_on_update_metadata() { make_alter_logical_table_add_column_task(None, "table3", vec!["new_col".to_string()]), ]; - let mut procedure = AlterLogicalTablesProcedure::new(tasks, phy_id, ddl_context.clone()); + let mut procedure = AlterLogicalTablesProcedure::new( + tasks, + phy_id, + EventContext::default(), + ddl_context.clone(), + ); let mut status = procedure.on_prepare().await.unwrap(); assert_matches!( status, @@ -361,7 +379,12 @@ async fn test_on_part_duplicate_alter_request() { make_alter_logical_table_add_column_task(None, "table2", vec!["col_0".to_string()]), ]; - let mut procedure = AlterLogicalTablesProcedure::new(tasks, phy_id, ddl_context.clone()); + let mut procedure = AlterLogicalTablesProcedure::new( + tasks, + phy_id, + EventContext::default(), + ddl_context.clone(), + ); let mut status = procedure.on_prepare().await.unwrap(); assert_matches!( status, @@ -446,7 +469,12 @@ async fn test_on_part_duplicate_alter_request() { ), ]; - let mut procedure = AlterLogicalTablesProcedure::new(tasks, phy_id, ddl_context.clone()); + let mut procedure = AlterLogicalTablesProcedure::new( + tasks, + phy_id, + EventContext::default(), + ddl_context.clone(), + ); let mut status = procedure.on_prepare().await.unwrap(); assert_matches!( status, @@ -617,7 +645,8 @@ async fn test_on_submit_alter_region_request() { make_alter_logical_table_add_column_task(None, "table2", vec!["mew_col".to_string()]), ]; - let mut procedure = AlterLogicalTablesProcedure::new(tasks, phy_id, ddl_context); + let mut procedure = + AlterLogicalTablesProcedure::new(tasks, phy_id, EventContext::default(), ddl_context); procedure.on_prepare().await.unwrap(); procedure.on_submit_alter_region_requests().await.unwrap(); let mut results = Vec::new(); diff --git a/src/common/meta/src/ddl/tests/alter_table.rs b/src/common/meta/src/ddl/tests/alter_table.rs index b35666e52e1..b2943a0ab8d 100644 --- a/src/common/meta/src/ddl/tests/alter_table.rs +++ b/src/common/meta/src/ddl/tests/alter_table.rs @@ -59,7 +59,7 @@ use crate::lock_key::RegionLock; use crate::node_manager::NodeManagerRef; use crate::peer::Peer; use crate::poison_key::table_poison_key; -use crate::rpc::ddl::AlterTableTask; +use crate::rpc::ddl::{AlterTableTask, EventContext}; use crate::rpc::router::{Region, RegionRoute}; use crate::test_util::{MockDatanodeManager, new_ddl_context}; @@ -121,7 +121,8 @@ async fn test_on_prepare_table_exists_err() { .unwrap(); let task = test_rename_alter_table_task("non-exists", "foo"); - let mut procedure = AlterTableProcedure::new(1024, task, ddl_context).unwrap(); + let mut procedure = + AlterTableProcedure::new(1024, task, EventContext::default(), ddl_context).unwrap(); let err = procedure.on_prepare().await.unwrap_err(); assert_matches!(err.status_code(), StatusCode::TableAlreadyExists); } @@ -131,7 +132,8 @@ async fn test_on_prepare_table_not_exists_err() { let node_manager = Arc::new(MockDatanodeManager::new(())); let ddl_context = new_ddl_context(node_manager); let task = test_rename_alter_table_task("non-exists", "foo"); - let mut procedure = AlterTableProcedure::new(1024, task, ddl_context).unwrap(); + let mut procedure = + AlterTableProcedure::new(1024, task, EventContext::default(), ddl_context).unwrap(); let err = procedure.on_prepare().await.unwrap_err(); assert_matches!(err.status_code(), StatusCode::TableNotFound); } @@ -245,8 +247,13 @@ async fn test_on_submit_alter_request() { let alter_table_task = test_alter_table_task(table_name); let procedure_id = ProcedureId::random(); let provider = Arc::new(MockContextProvider::default()); - let mut procedure = - AlterTableProcedure::new(table_id, alter_table_task, ddl_context.clone()).unwrap(); + let mut procedure = AlterTableProcedure::new( + table_id, + alter_table_task, + EventContext::default(), + ddl_context.clone(), + ) + .unwrap(); procedure.on_prepare().await.unwrap(); procedure .submit_alter_region_requests(procedure_id, provider.as_ref()) @@ -299,8 +306,13 @@ async fn test_on_submit_alter_request_without_sync_request() { let alter_table_task = test_alter_table_task(table_name); let procedure_id = ProcedureId::random(); let provider = Arc::new(MockContextProvider::default()); - let mut procedure = - AlterTableProcedure::new(table_id, alter_table_task, ddl_context.clone()).unwrap(); + let mut procedure = AlterTableProcedure::new( + table_id, + alter_table_task, + EventContext::default(), + ddl_context.clone(), + ) + .unwrap(); procedure.on_prepare().await.unwrap(); procedure .submit_alter_region_requests(procedure_id, provider.as_ref()) @@ -357,7 +369,13 @@ async fn test_on_submit_alter_request_with_outdated_request() { }; let procedure_id = ProcedureId::random(); let provider = Arc::new(MockContextProvider::default()); - let mut procedure = AlterTableProcedure::new(table_id, alter_table_task, ddl_context).unwrap(); + let mut procedure = AlterTableProcedure::new( + table_id, + alter_table_task, + EventContext::default(), + ddl_context, + ) + .unwrap(); procedure.on_prepare().await.unwrap(); let err = procedure .submit_alter_region_requests(procedure_id, provider.as_ref()) @@ -386,7 +404,9 @@ async fn test_on_update_metadata_rename() { .unwrap(); let task = test_rename_alter_table_task(table_name, new_table_name); - let mut procedure = AlterTableProcedure::new(table_id, task, ddl_context.clone()).unwrap(); + let mut procedure = + AlterTableProcedure::new(table_id, task, EventContext::default(), ddl_context.clone()) + .unwrap(); procedure.on_prepare().await.unwrap(); procedure.on_update_metadata().await.unwrap(); @@ -466,7 +486,9 @@ async fn test_on_update_metadata_add_columns() { }; let procedure_id = ProcedureId::random(); let provider = Arc::new(MockContextProvider::default()); - let mut procedure = AlterTableProcedure::new(table_id, task, ddl_context.clone()).unwrap(); + let mut procedure = + AlterTableProcedure::new(table_id, task, EventContext::default(), ddl_context.clone()) + .unwrap(); procedure.on_prepare().await.unwrap(); procedure .submit_alter_region_requests(procedure_id, provider.as_ref()) @@ -563,7 +585,9 @@ async fn test_on_update_table_options() { }; let procedure_id = ProcedureId::random(); let provider = Arc::new(MockContextProvider::default()); - let mut procedure = AlterTableProcedure::new(table_id, task, ddl_context.clone()).unwrap(); + let mut procedure = + AlterTableProcedure::new(table_id, task, EventContext::default(), ddl_context.clone()) + .unwrap(); procedure.on_prepare().await.unwrap(); procedure .submit_alter_region_requests(procedure_id, provider.as_ref()) @@ -634,6 +658,7 @@ async fn test_skip_wal_rejects_mixed_table_options() { })), }, }, + EventContext::default(), ddl_context, ) .unwrap(); @@ -662,8 +687,13 @@ async fn test_skip_wal_rejects_logical_table() { }, }; - let mut procedure = - AlterTableProcedure::new(logical_table_id, task, ddl_context.clone()).unwrap(); + let mut procedure = AlterTableProcedure::new( + logical_table_id, + task, + EventContext::default(), + ddl_context.clone(), + ) + .unwrap(); let error = procedure.on_prepare().await.unwrap_err(); assert_matches!(error, Error::Unsupported { .. }); @@ -709,7 +739,9 @@ async fn test_skip_wal_rejects_file_engine_table() { }, }; - let mut procedure = AlterTableProcedure::new(table_id, task, ddl_context.clone()).unwrap(); + let mut procedure = + AlterTableProcedure::new(table_id, task, EventContext::default(), ddl_context.clone()) + .unwrap(); let error = procedure.on_prepare().await.unwrap_err(); assert_matches!(error, Error::Unsupported { .. }); @@ -746,6 +778,7 @@ fn test_skip_wal_holds_region_locks() { let procedure = AlterTableProcedure::new_with_region_locks( table_id, task, + EventContext::default(), region_ids.clone(), context.clone(), ) @@ -797,6 +830,7 @@ async fn test_skip_wal_detects_region_route_change() { let mut procedure = AlterTableProcedure::new_with_region_locks( table_id, alter_task, + EventContext::default(), stale_region_locks, ddl_context.clone(), ) @@ -916,6 +950,7 @@ async fn test_skip_wal_updates_metadata_before_all_replicas() { let mut procedure = AlterTableProcedure::new_with_region_locks( table_id, alter_task, + EventContext::default(), region_locks, ddl_context.clone(), ) @@ -997,8 +1032,13 @@ async fn prepare_alter_table_procedure( }, }; let procedure_id = ProcedureId::random(); - let mut procedure = - AlterTableProcedure::new(table_id, alter_table_task, ddl_context.clone()).unwrap(); + let mut procedure = AlterTableProcedure::new( + table_id, + alter_table_task, + EventContext::default(), + ddl_context.clone(), + ) + .unwrap(); procedure.on_prepare().await.unwrap(); (procedure, procedure_id) } diff --git a/src/common/meta/src/ddl/tests/create_flow.rs b/src/common/meta/src/ddl/tests/create_flow.rs index b526856bdac..280a2633e32 100644 --- a/src/common/meta/src/ddl/tests/create_flow.rs +++ b/src/common/meta/src/ddl/tests/create_flow.rs @@ -37,7 +37,7 @@ use crate::key::flow::flow_info::{ FlowInfoValue, FlowMissedTickPolicy, FlowScheduleConfig, FlowStatus, }; use crate::key::table_route::TableRouteValue; -use crate::rpc::ddl::{CreateFlowTask, FlowQueryContext, QueryContext}; +use crate::rpc::ddl::{CreateFlowTask, EventContext, FlowQueryContext, QueryContext}; use crate::test_util::{MockFlownodeManager, new_ddl_context}; pub(crate) fn test_query_context() -> QueryContext { @@ -120,7 +120,8 @@ async fn test_create_flow_source_table_not_found() { let node_manager = Arc::new(MockFlownodeManager::new(NaiveFlownodeHandler)); let ddl_context = new_ddl_context(node_manager); let query_ctx = test_query_context(); - let mut procedure = CreateFlowProcedure::new(task, query_ctx, ddl_context); + let mut procedure = + CreateFlowProcedure::new(task, query_ctx, EventContext::default(), ddl_context); let err = procedure.on_prepare().await.unwrap_err(); assert_matches!(err, error::Error::Unsupported { .. }); assert!( @@ -143,7 +144,12 @@ async fn test_create_pending_flow_source_table_not_found_with_defer() { let node_manager = Arc::new(MockFlownodeManager::new(NaiveFlownodeHandler)); let ddl_context = new_ddl_context(node_manager); let query_ctx = test_query_context(); - let mut procedure = CreateFlowProcedure::new(task, query_ctx, ddl_context.clone()); + let mut procedure = CreateFlowProcedure::new( + task, + query_ctx, + EventContext::default(), + ddl_context.clone(), + ); let status = procedure.on_prepare().await.unwrap(); assert_matches!(status, Status::Executing { persist: true, .. }); assert_eq!(procedure.data.unresolved_source_table_names.len(), 1); @@ -183,7 +189,8 @@ async fn test_create_pending_flow_source_table_not_found_with_defer_false() { let node_manager = Arc::new(MockFlownodeManager::new(NaiveFlownodeHandler)); let ddl_context = new_ddl_context(node_manager); let query_ctx = test_query_context(); - let mut procedure = CreateFlowProcedure::new(task, query_ctx, ddl_context); + let mut procedure = + CreateFlowProcedure::new(task, query_ctx, EventContext::default(), ddl_context); let err = procedure.on_prepare().await.unwrap_err(); assert_matches!(err, error::Error::Unsupported { .. }); assert!( @@ -233,7 +240,12 @@ async fn test_create_pending_flow_records_partial_source_resolution() { ); enable_defer_on_missing_source(&mut task); let query_ctx = test_query_context(); - let mut procedure = CreateFlowProcedure::new(task, query_ctx, ddl_context.clone()); + let mut procedure = CreateFlowProcedure::new( + task, + query_ctx, + EventContext::default(), + ddl_context.clone(), + ); let status = procedure.on_prepare().await.unwrap(); assert_matches!(status, Status::Executing { persist: true, .. }); assert_eq!(procedure.data.source_table_ids, vec![existing_table_id]); @@ -428,6 +440,7 @@ fn test_resolved_schedule_defaults_in_create_request() { // Construct CreateFlowData and verify CreateRequest carries internal key. let data = CreateFlowData { + event_context: EventContext::default(), state: CreateFlowState::CreateFlows, task, flow_id: Some(1024), @@ -441,6 +454,7 @@ fn test_resolved_schedule_defaults_in_create_request() { }; let data2 = CreateFlowData { + event_context: EventContext::default(), state: CreateFlowState::CreateMetadata, task: data.task.clone(), flow_id: Some(1024), @@ -580,7 +594,8 @@ async fn test_create_flow_rejects_unknown_option_in_meta_task() { let node_manager = Arc::new(MockFlownodeManager::new(NaiveFlownodeHandler)); let ddl_context = new_ddl_context(node_manager); let query_ctx = test_query_context(); - let mut procedure = CreateFlowProcedure::new(task, query_ctx, ddl_context); + let mut procedure = + CreateFlowProcedure::new(task, query_ctx, EventContext::default(), ddl_context); let err = procedure.on_prepare().await.unwrap_err(); assert_matches!(err, error::Error::Unexpected { .. }); @@ -601,6 +616,7 @@ fn test_create_request_strips_defer_on_missing_source_runtime_option() { enable_defer_on_missing_source(&mut task); let data = CreateFlowData { + event_context: EventContext::default(), state: CreateFlowState::CreateFlows, task, flow_id: Some(1024), @@ -650,7 +666,12 @@ pub(crate) async fn create_test_flow( false, ); let query_ctx = test_query_context(); - let mut procedure = CreateFlowProcedure::new(task.clone(), query_ctx, ddl_context.clone()); + let mut procedure = CreateFlowProcedure::new( + task.clone(), + query_ctx, + EventContext::default(), + ddl_context.clone(), + ); let output = execute_procedure_until_done(&mut procedure).await.unwrap(); let flow_id = output.downcast_ref::().unwrap(); @@ -671,7 +692,12 @@ pub(crate) async fn create_test_pending_flow( ); enable_defer_on_missing_source(&mut task); let query_ctx = test_query_context(); - let mut procedure = CreateFlowProcedure::new(task, query_ctx, ddl_context.clone()); + let mut procedure = CreateFlowProcedure::new( + task, + query_ctx, + EventContext::default(), + ddl_context.clone(), + ); let output = execute_procedure_until_done(&mut procedure).await.unwrap(); let flow_id = output.downcast_ref::().unwrap(); @@ -718,7 +744,12 @@ async fn test_create_flow() { true, ); let query_ctx = test_query_context(); - let mut procedure = CreateFlowProcedure::new(task.clone(), query_ctx, ddl_context.clone()); + let mut procedure = CreateFlowProcedure::new( + task.clone(), + query_ctx, + EventContext::default(), + ddl_context.clone(), + ); let output = execute_procedure_until_done(&mut procedure).await.unwrap(); let flow_id = output.downcast_ref::().unwrap(); assert_eq!(*flow_id, 1024); @@ -726,7 +757,12 @@ async fn test_create_flow() { // Creates again let task = test_create_flow_task("my_flow", source_table_names, sink_table_name, false); let query_ctx = test_query_context(); - let mut procedure = CreateFlowProcedure::new(task.clone(), query_ctx, ddl_context); + let mut procedure = CreateFlowProcedure::new( + task.clone(), + query_ctx, + EventContext::default(), + ddl_context, + ); let err = procedure.on_prepare().await.unwrap_err(); assert_matches!(err, error::Error::FlowAlreadyExists { .. }); } @@ -783,7 +819,12 @@ async fn test_replace_pending_flow_with_active_flow_is_unsupported() { ); replace_task.or_replace = true; let query_ctx = test_query_context(); - let mut procedure = CreateFlowProcedure::new(replace_task, query_ctx, ddl_context.clone()); + let mut procedure = CreateFlowProcedure::new( + replace_task, + query_ctx, + EventContext::default(), + ddl_context.clone(), + ); let err = procedure.on_prepare().await.unwrap_err(); assert_matches!(err, error::Error::Unsupported { .. }); assert!( @@ -841,7 +882,12 @@ async fn test_replace_active_flow_with_pending_flow_is_unsupported() { enable_defer_on_missing_source(&mut replace_task); replace_task.or_replace = true; let query_ctx = test_query_context(); - let mut procedure = CreateFlowProcedure::new(replace_task, query_ctx, ddl_context.clone()); + let mut procedure = CreateFlowProcedure::new( + replace_task, + query_ctx, + EventContext::default(), + ddl_context.clone(), + ); let err = procedure.on_prepare().await.unwrap_err(); assert_matches!(err, error::Error::Unsupported { .. }); assert!( @@ -901,7 +947,12 @@ async fn test_replace_pending_flow_with_pending_flow_updates_metadata() { enable_defer_on_missing_source(&mut replace_task); replace_task.or_replace = true; let query_ctx = test_query_context(); - let mut procedure = CreateFlowProcedure::new(replace_task, query_ctx, ddl_context.clone()); + let mut procedure = CreateFlowProcedure::new( + replace_task, + query_ctx, + EventContext::default(), + ddl_context.clone(), + ); let output = execute_procedure_until_done(&mut procedure).await.unwrap(); let replaced_flow_id = *output.downcast_ref::().unwrap(); assert_eq!(replaced_flow_id, original_flow_id); @@ -953,7 +1004,8 @@ async fn test_create_flow_same_source_and_sink_table() { // Try to create a flow with same source and sink table - should fail let task = test_create_flow_task("my_flow", source_table_names, sink_table_name, false); let query_ctx = test_query_context(); - let mut procedure = CreateFlowProcedure::new(task, query_ctx, ddl_context); + let mut procedure = + CreateFlowProcedure::new(task, query_ctx, EventContext::default(), ddl_context); let err = procedure.on_prepare().await.unwrap_err(); assert_matches!(err, error::Error::Unsupported { .. }); @@ -986,7 +1038,7 @@ fn test_create_flow_lock_key_does_not_lock_sink_table_name() { let task = create_test_flow_task_for_serialization(); let query_ctx = test_query_context(); let ddl_context = new_ddl_context(Arc::new(MockFlownodeManager::new(NaiveFlownodeHandler))); - let procedure = CreateFlowProcedure::new(task, query_ctx, ddl_context); + let procedure = CreateFlowProcedure::new(task, query_ctx, EventContext::default(), ddl_context); let lock_keys = procedure.lock_key().get_keys(); @@ -1061,6 +1113,7 @@ fn test_create_flow_data_new_format_serialization() { }; let data = CreateFlowData { + event_context: EventContext::default(), state: CreateFlowState::Prepare, task: create_test_flow_task_for_serialization(), flow_id: None, @@ -1129,7 +1182,7 @@ fn test_create_flow_procedure_strips_scheduled_time_extension() { .extensions .insert("flow.other".to_string(), "kept".to_string()); - let procedure = CreateFlowProcedure::new(task, query_ctx, ddl_context); + let procedure = CreateFlowProcedure::new(task, query_ctx, EventContext::default(), ddl_context); assert!( !procedure @@ -1162,6 +1215,7 @@ fn test_flow_info_conversion_with_flow_context() { }; let data = CreateFlowData { + event_context: EventContext::default(), state: CreateFlowState::CreateMetadata, task: create_test_flow_task_for_serialization(), flow_id: Some(123), @@ -1204,6 +1258,7 @@ fn test_flow_info_conversion_strips_scheduled_time_extension() { }; let data = CreateFlowData { + event_context: EventContext::default(), state: CreateFlowState::CreateMetadata, task: create_test_flow_task_for_serialization(), flow_id: Some(123), diff --git a/src/common/meta/src/ddl/tests/create_logical_tables.rs b/src/common/meta/src/ddl/tests/create_logical_tables.rs index e2927e8df65..d1cc2420e04 100644 --- a/src/common/meta/src/ddl/tests/create_logical_tables.rs +++ b/src/common/meta/src/ddl/tests/create_logical_tables.rs @@ -39,6 +39,7 @@ use crate::ddl::test_util::{ }; use crate::error::{Error, Result}; use crate::key::table_route::{PhysicalTableRouteValue, TableRouteValue}; +use crate::rpc::ddl::EventContext; use crate::rpc::router::{Region, RegionRoute}; use crate::test_util::{MockDatanodeManager, new_ddl_context}; @@ -96,7 +97,12 @@ async fn test_on_prepare_physical_table_not_found() { let ddl_context = new_ddl_context(node_manager); let tasks = vec![test_create_logical_table_task("foo")]; let physical_table_id = 1024u32; - let mut procedure = CreateLogicalTablesProcedure::new(tasks, physical_table_id, ddl_context); + let mut procedure = CreateLogicalTablesProcedure::new( + tasks, + physical_table_id, + EventContext::default(), + ddl_context, + ); let err = procedure.on_prepare().await.unwrap_err(); assert_matches!(err, Error::TableRouteNotFound { .. }); } @@ -126,7 +132,12 @@ async fn test_on_prepare() { // The create logical table procedure. let tasks = vec![test_create_logical_table_task("foo")]; let physical_table_id = table_id; - let mut procedure = CreateLogicalTablesProcedure::new(tasks, physical_table_id, ddl_context); + let mut procedure = CreateLogicalTablesProcedure::new( + tasks, + physical_table_id, + EventContext::default(), + ddl_context, + ); let status = procedure.on_prepare().await.unwrap(); assert_matches!( status, @@ -172,8 +183,12 @@ async fn test_on_prepare_logical_table_exists_err() { .unwrap(); // The create logical table procedure. let physical_table_id = table_id; - let mut procedure = - CreateLogicalTablesProcedure::new(vec![task], physical_table_id, ddl_context); + let mut procedure = CreateLogicalTablesProcedure::new( + vec![task], + physical_table_id, + EventContext::default(), + ddl_context, + ); let err = procedure.on_prepare().await.unwrap_err(); assert_matches!(err, Error::TableAlreadyExists { .. }); assert_eq!(err.status_code(), StatusCode::TableAlreadyExists); @@ -216,8 +231,12 @@ async fn test_on_prepare_with_create_if_table_exists() { let physical_table_id = table_id; // Sets `create_if_not_exists` task.create_table.create_if_not_exists = true; - let mut procedure = - CreateLogicalTablesProcedure::new(vec![task], physical_table_id, ddl_context); + let mut procedure = CreateLogicalTablesProcedure::new( + vec![task], + physical_table_id, + EventContext::default(), + ddl_context, + ); let status = procedure.on_prepare().await.unwrap(); let output = status.downcast_output_ref::>().unwrap(); assert_eq!(*output, vec![8192]); @@ -264,6 +283,7 @@ async fn test_on_prepare_part_logical_tables_exist() { let mut procedure = CreateLogicalTablesProcedure::new( vec![task, non_exist_task], physical_table_id, + EventContext::default(), ddl_context, ); let status = procedure.on_prepare().await.unwrap(); @@ -311,6 +331,7 @@ async fn test_on_create_metadata() { let mut procedure = CreateLogicalTablesProcedure::new( vec![task, yet_another_task], physical_table_id, + EventContext::default(), ddl_context.clone(), ); let status = procedure.on_prepare().await.unwrap(); @@ -404,6 +425,7 @@ async fn test_on_create_metadata_part_logical_tables_exist() { let mut procedure = CreateLogicalTablesProcedure::new( vec![task, non_exist_task], physical_table_id, + EventContext::default(), ddl_context.clone(), ); let status = procedure.on_prepare().await.unwrap(); @@ -476,6 +498,7 @@ async fn test_on_create_metadata_err() { let mut procedure = CreateLogicalTablesProcedure::new( vec![task.clone(), yet_another_task], physical_table_id, + EventContext::default(), ddl_context.clone(), ); let status = procedure.on_prepare().await.unwrap(); @@ -536,6 +559,7 @@ async fn test_on_submit_create_request() { let mut procedure = CreateLogicalTablesProcedure::new( vec![task, yet_another_task], physical_table_id, + EventContext::default(), ddl_context, ); procedure.on_prepare().await.unwrap(); diff --git a/src/common/meta/src/ddl/tests/create_table.rs b/src/common/meta/src/ddl/tests/create_table.rs index db94a12f84f..2eee317dc14 100644 --- a/src/common/meta/src/ddl/tests/create_table.rs +++ b/src/common/meta/src/ddl/tests/create_table.rs @@ -48,7 +48,7 @@ use crate::ddl::test_util::{assert_column_name, get_raw_table_info, put_datanode use crate::error::{Error, Result}; use crate::key::table_route::TableRouteValue; use crate::kv_backend::memory::MemoryKvBackend; -use crate::rpc::ddl::CreateTableTask; +use crate::rpc::ddl::{CreateTableTask, EventContext, QueryContext}; use crate::test_util::{MockDatanodeManager, new_ddl_context, new_ddl_context_with_kv_backend}; fn create_request_handler(_peer: Peer, request: RegionRequest) -> Result { @@ -189,7 +189,13 @@ async fn test_on_prepare_table_exists_err() { ) .await .unwrap(); - let mut procedure = CreateTableProcedure::new(task, ddl_context).unwrap(); + let mut procedure = CreateTableProcedure::new( + task, + QueryContext::default(), + EventContext::default(), + ddl_context, + ) + .unwrap(); let err = procedure.on_prepare().await.unwrap_err(); assert_matches!(err, Error::TableAlreadyExists { .. }); assert_eq!(err.status_code(), StatusCode::TableAlreadyExists); @@ -212,7 +218,13 @@ async fn test_on_prepare_with_create_if_table_exists() { ) .await .unwrap(); - let mut procedure = CreateTableProcedure::new(task, ddl_context).unwrap(); + let mut procedure = CreateTableProcedure::new( + task, + QueryContext::default(), + EventContext::default(), + ddl_context, + ) + .unwrap(); let status = procedure.on_prepare().await.unwrap(); assert_matches!(status, Status::Done { output: Some(..) }); let table_id = *status.downcast_output_ref::().unwrap(); @@ -225,7 +237,13 @@ async fn test_on_prepare_without_create_if_table_exists() { let ddl_context = new_ddl_context(node_manager); let mut task = test_create_table_task("foo"); task.create_table.create_if_not_exists = true; - let mut procedure = CreateTableProcedure::new(task, ddl_context).unwrap(); + let mut procedure = CreateTableProcedure::new( + task, + QueryContext::default(), + EventContext::default(), + ddl_context, + ) + .unwrap(); let status = procedure.on_prepare().await.unwrap(); assert_matches!( status, @@ -244,7 +262,13 @@ async fn test_on_datanode_create_regions_should_retry() { let ddl_context = new_ddl_context(node_manager); let task = test_create_table_task("foo"); assert!(!task.create_table.create_if_not_exists); - let mut procedure = CreateTableProcedure::new(task, ddl_context).unwrap(); + let mut procedure = CreateTableProcedure::new( + task, + QueryContext::default(), + EventContext::default(), + ddl_context, + ) + .unwrap(); procedure.on_prepare().await.unwrap(); let ctx = ProcedureContext { procedure_id: ProcedureId::random(), @@ -261,7 +285,13 @@ async fn test_on_datanode_create_regions_should_not_retry() { let ddl_context = new_ddl_context(node_manager); let task = test_create_table_task("foo"); assert!(!task.create_table.create_if_not_exists); - let mut procedure = CreateTableProcedure::new(task, ddl_context).unwrap(); + let mut procedure = CreateTableProcedure::new( + task, + QueryContext::default(), + EventContext::default(), + ddl_context, + ) + .unwrap(); procedure.on_prepare().await.unwrap(); let ctx = ProcedureContext { procedure_id: ProcedureId::random(), @@ -278,7 +308,13 @@ async fn test_on_datanode_create_regions_remaps_addresses_when_retrying() { let node_manager = Arc::new(MockDatanodeManager::new(datanode_handler)); let ddl_context = new_ddl_context(node_manager); let task = test_create_table_task("foo"); - let mut procedure = CreateTableProcedure::new(task, ddl_context.clone()).unwrap(); + let mut procedure = CreateTableProcedure::new( + task, + QueryContext::default(), + EventContext::default(), + ddl_context.clone(), + ) + .unwrap(); procedure.on_prepare().await.unwrap(); let table_route = procedure.data.table_route.as_mut().unwrap(); @@ -299,7 +335,13 @@ async fn test_on_create_metadata_error() { let ddl_context = new_ddl_context(node_manager); let task = test_create_table_task("foo"); assert!(!task.create_table.create_if_not_exists); - let mut procedure = CreateTableProcedure::new(task.clone(), ddl_context.clone()).unwrap(); + let mut procedure = CreateTableProcedure::new( + task.clone(), + QueryContext::default(), + EventContext::default(), + ddl_context.clone(), + ) + .unwrap(); procedure.on_prepare().await.unwrap(); let ctx = ProcedureContext { procedure_id: ProcedureId::random(), @@ -332,7 +374,13 @@ async fn test_on_create_metadata() { let ddl_context = new_ddl_context(node_manager); let task = test_create_table_task("foo"); assert!(!task.create_table.create_if_not_exists); - let mut procedure = CreateTableProcedure::new(task, ddl_context.clone()).unwrap(); + let mut procedure = CreateTableProcedure::new( + task, + QueryContext::default(), + EventContext::default(), + ddl_context.clone(), + ) + .unwrap(); procedure.on_prepare().await.unwrap(); let ctx = ProcedureContext { procedure_id: ProcedureId::random(), @@ -360,7 +408,13 @@ async fn test_memory_region_keeper_guard_dropped_on_procedure_done() { let ddl_context = new_ddl_context_with_kv_backend(node_manager, kv_backend); let task = test_create_table_task("foo"); - let mut procedure = CreateTableProcedure::new(task, ddl_context.clone()).unwrap(); + let mut procedure = CreateTableProcedure::new( + task, + QueryContext::default(), + EventContext::default(), + ddl_context.clone(), + ) + .unwrap(); execute_procedure_until(&mut procedure, |p| { p.data.state == CreateTableState::CreateMetadata diff --git a/src/common/meta/src/ddl/tests/create_view.rs b/src/common/meta/src/ddl/tests/create_view.rs index cc98bb1baed..618249fd1cd 100644 --- a/src/common/meta/src/ddl/tests/create_view.rs +++ b/src/common/meta/src/ddl/tests/create_view.rs @@ -29,7 +29,7 @@ use crate::ddl::create_view::CreateViewProcedure; use crate::ddl::test_util::datanode_handler::NaiveDatanodeHandler; use crate::ddl::tests::create_table::test_create_table_task; use crate::error::Error; -use crate::rpc::ddl::CreateViewTask; +use crate::rpc::ddl::{CreateViewTask, EventContext, QueryContext}; use crate::test_util::{MockDatanodeManager, new_ddl_context}; pub(crate) fn test_table_names() -> HashSet { @@ -112,7 +112,7 @@ async fn test_on_prepare_view_exists_err() { ) .await .unwrap(); - let mut procedure = CreateViewProcedure::new(task, ddl_context); + let mut procedure = CreateViewProcedure::new(task, EventContext::default(), ddl_context); let err = procedure.on_prepare().await.unwrap_err(); assert_matches!(err, Error::ViewAlreadyExists { .. }); assert_eq!(err.status_code(), StatusCode::TableAlreadyExists); @@ -138,7 +138,7 @@ async fn test_on_prepare_with_create_if_view_exists() { ) .await .unwrap(); - let mut procedure = CreateViewProcedure::new(task, ddl_context); + let mut procedure = CreateViewProcedure::new(task, EventContext::default(), ddl_context); let status = procedure.on_prepare().await.unwrap(); assert_matches!(status, Status::Done { output: Some(..) }); let table_id = *status.downcast_output_ref::().unwrap(); @@ -151,7 +151,7 @@ async fn test_on_prepare_without_create_if_table_exists() { let ddl_context = new_ddl_context(node_manager); let mut task = test_create_view_task("foo"); task.create_view.create_if_not_exists = true; - let mut procedure = CreateViewProcedure::new(task, ddl_context); + let mut procedure = CreateViewProcedure::new(task, EventContext::default(), ddl_context); let status = procedure.on_prepare().await.unwrap(); assert_matches!( status, @@ -170,7 +170,7 @@ async fn test_on_create_metadata() { let ddl_context = new_ddl_context(node_manager); let task = test_create_view_task("foo"); assert!(!task.create_view.create_if_not_exists); - let mut procedure = CreateViewProcedure::new(task, ddl_context); + let mut procedure = CreateViewProcedure::new(task, EventContext::default(), ddl_context); procedure.on_prepare().await.unwrap(); let ctx = ProcedureContext { procedure_id: ProcedureId::random(), @@ -189,7 +189,8 @@ async fn test_replace_view_metadata() { let ddl_context = new_ddl_context(node_manager.clone()); let task = test_create_view_task("foo"); assert!(!task.create_view.create_if_not_exists); - let mut procedure = CreateViewProcedure::new(task.clone(), ddl_context.clone()); + let mut procedure = + CreateViewProcedure::new(task.clone(), EventContext::default(), ddl_context.clone()); procedure.on_prepare().await.unwrap(); let ctx = ProcedureContext { procedure_id: ProcedureId::random(), @@ -214,7 +215,8 @@ async fn test_replace_view_metadata() { let mut task = test_create_view_task("foo"); // The view already exists, prepare should fail { - let mut procedure = CreateViewProcedure::new(task.clone(), ddl_context.clone()); + let mut procedure = + CreateViewProcedure::new(task.clone(), EventContext::default(), ddl_context.clone()); let err = procedure.on_prepare().await.unwrap_err(); assert_matches!(err, Error::ViewAlreadyExists { .. }); assert_eq!(err.status_code(), StatusCode::TableAlreadyExists); @@ -225,7 +227,8 @@ async fn test_replace_view_metadata() { task.create_view.logical_plan = vec![4, 5, 6]; task.create_view.definition = "new_definition".to_string(); - let mut procedure = CreateViewProcedure::new(task, ddl_context.clone()); + let mut procedure = + CreateViewProcedure::new(task, EventContext::default(), ddl_context.clone()); procedure.on_prepare().await.unwrap(); let ctx = ProcedureContext { procedure_id: ProcedureId::random(), @@ -259,7 +262,13 @@ async fn test_replace_table() { { // Create a `foo` table. let task = test_create_table_task("foo"); - let mut procedure = CreateTableProcedure::new(task, ddl_context.clone()).unwrap(); + let mut procedure = CreateTableProcedure::new( + task, + QueryContext::default(), + EventContext::default(), + ddl_context.clone(), + ) + .unwrap(); procedure.on_prepare().await.unwrap(); let ctx = ProcedureContext { procedure_id: ProcedureId::random(), @@ -272,7 +281,8 @@ async fn test_replace_table() { // Try to replace a view named `foo` too. let mut task = test_create_view_task("foo"); task.create_view.or_replace = true; - let mut procedure = CreateViewProcedure::new(task.clone(), ddl_context.clone()); + let mut procedure = + CreateViewProcedure::new(task.clone(), EventContext::default(), ddl_context.clone()); let err = procedure.on_prepare().await.unwrap_err(); assert_matches!(err, Error::TableAlreadyExists { .. }); assert_eq!(err.status_code(), StatusCode::TableAlreadyExists); diff --git a/src/common/meta/src/ddl/tests/drop_database.rs b/src/common/meta/src/ddl/tests/drop_database.rs index 81cc29a51ba..955625e4bb3 100644 --- a/src/common/meta/src/ddl/tests/drop_database.rs +++ b/src/common/meta/src/ddl/tests/drop_database.rs @@ -26,6 +26,7 @@ use crate::ddl::drop_database::executor::DropDatabaseExecutor; use crate::ddl::test_util::datanode_handler::{NaiveDatanodeHandler, RetryErrorDatanodeHandler}; use crate::ddl::test_util::{create_logical_table, create_physical_table}; use crate::key::schema_name::SchemaNameKey; +use crate::rpc::ddl::EventContext; use crate::test_util::{MockDatanodeManager, new_ddl_context}; #[tokio::test] @@ -54,6 +55,7 @@ async fn test_drop_database_with_logical_tables() { DEFAULT_CATALOG_NAME.to_string(), DEFAULT_SCHEMA_NAME.to_string(), false, + EventContext::default(), ddl_context.clone(), ); @@ -102,6 +104,7 @@ async fn test_drop_database_retryable_error() { DEFAULT_CATALOG_NAME.to_string(), DEFAULT_SCHEMA_NAME.to_string(), false, + EventContext::default(), ddl_context.clone(), ); @@ -146,6 +149,7 @@ async fn test_drop_database_recover() { DEFAULT_CATALOG_NAME.to_string(), DEFAULT_SCHEMA_NAME.to_string(), false, + EventContext::default(), ddl_context.clone(), ); let num_operating_regions = 1; diff --git a/src/common/meta/src/ddl/tests/drop_flow.rs b/src/common/meta/src/ddl/tests/drop_flow.rs index 6d36fc93ba3..0976aa572f7 100644 --- a/src/common/meta/src/ddl/tests/drop_flow.rs +++ b/src/common/meta/src/ddl/tests/drop_flow.rs @@ -26,7 +26,7 @@ use crate::ddl::test_util::flownode_handler::NaiveFlownodeHandler; use crate::ddl::tests::create_flow::{create_test_flow, create_test_pending_flow}; use crate::error; use crate::key::table_route::TableRouteValue; -use crate::rpc::ddl::DropFlowTask; +use crate::rpc::ddl::{DropFlowTask, EventContext}; use crate::test_util::{MockFlownodeManager, new_ddl_context}; pub(crate) fn test_drop_flow_task( @@ -48,7 +48,7 @@ async fn test_drop_flow_not_found() { let node_manager = Arc::new(MockFlownodeManager::new(NaiveFlownodeHandler)); let ddl_context = new_ddl_context(node_manager); let task = test_drop_flow_task("my_flow", flow_id, false); - let mut procedure = DropFlowProcedure::new(task, ddl_context); + let mut procedure = DropFlowProcedure::new(task, EventContext::default(), ddl_context); let err = procedure.on_prepare().await.unwrap_err(); assert_matches!(err, error::Error::FlowNotFound { .. }); } @@ -81,17 +81,17 @@ async fn test_drop_flow() { create_test_flow(&ddl_context, "my_flow", source_table_names, sink_table_name).await; // Drops the flows let task = test_drop_flow_task("my_flow", flow_id, false); - let mut procedure = DropFlowProcedure::new(task, ddl_context.clone()); + let mut procedure = DropFlowProcedure::new(task, EventContext::default(), ddl_context.clone()); execute_procedure_until_done(&mut procedure).await; // Drops if not exists let task = test_drop_flow_task("my_flow", flow_id, true); - let mut procedure = DropFlowProcedure::new(task, ddl_context.clone()); + let mut procedure = DropFlowProcedure::new(task, EventContext::default(), ddl_context.clone()); execute_procedure_until_done(&mut procedure).await; // Drops again let task = test_drop_flow_task("my_flow", flow_id, false); - let mut procedure = DropFlowProcedure::new(task, ddl_context); + let mut procedure = DropFlowProcedure::new(task, EventContext::default(), ddl_context); let err = procedure.on_prepare().await.unwrap_err(); assert_matches!(err, error::Error::FlowNotFound { .. }); } @@ -129,11 +129,11 @@ async fn test_drop_pending_flow_without_routes() { assert!(flow_info.flownode_ids().is_empty()); let task = test_drop_flow_task("drop_pending_flow", flow_id, false); - let mut procedure = DropFlowProcedure::new(task, ddl_context.clone()); + let mut procedure = DropFlowProcedure::new(task, EventContext::default(), ddl_context.clone()); execute_procedure_until_done(&mut procedure).await; let task = test_drop_flow_task("drop_pending_flow", flow_id, false); - let mut procedure = DropFlowProcedure::new(task, ddl_context); + let mut procedure = DropFlowProcedure::new(task, EventContext::default(), ddl_context); let err = procedure.on_prepare().await.unwrap_err(); assert_matches!(err, error::Error::FlowNotFound { .. }); } diff --git a/src/common/meta/src/ddl/tests/drop_table.rs b/src/common/meta/src/ddl/tests/drop_table.rs index d2afb73a3df..8b161914671 100644 --- a/src/common/meta/src/ddl/tests/drop_table.rs +++ b/src/common/meta/src/ddl/tests/drop_table.rs @@ -65,9 +65,9 @@ use crate::key::table_route::TableRouteValue; use crate::kv_backend::KvBackend; use crate::kv_backend::memory::MemoryKvBackend; use crate::peer::Peer; -use crate::rpc::ddl::DropTableTask; +use crate::rpc::ddl::{DropTableTask, EventContext}; #[cfg(feature = "enterprise")] -use crate::rpc::ddl::{PurgeDroppedTableTask, UndropTableTask}; +use crate::rpc::ddl::{PurgeDroppedTableTask, QueryContext, UndropTableTask}; use crate::rpc::router::{Region, RegionRoute}; #[cfg(feature = "enterprise")] use crate::rpc::store::{BatchDeleteRequest, PutRequest}; @@ -128,7 +128,11 @@ async fn undrop_at_restore_metadata( table_id: TableId, ) -> UndropTableProcedure { create_dropped_table(context, table_id, Some(1), Some(i64::MAX)).await; - let mut undrop = UndropTableProcedure::new(new_undrop_table_task(table_id), context.clone()); + let mut undrop = UndropTableProcedure::new( + new_undrop_table_task(table_id), + context.clone(), + EventContext::default(), + ); let procedure_context = new_test_procedure_context(); undrop.execute(&procedure_context).await.unwrap(); undrop.execute(&procedure_context).await.unwrap(); @@ -157,6 +161,7 @@ fn test_old_drop_table_json_defaults_to_hard_drop() { let procedure = DropTableProcedure::new( new_drop_table_task("foo", 1024, false), runtime_context.clone(), + EventContext::default(), ); let mut old_data: serde_json::Value = serde_json::from_str(&procedure.dump().unwrap()).unwrap(); old_data @@ -164,6 +169,7 @@ fn test_old_drop_table_json_defaults_to_hard_drop() { .unwrap() .remove("soft_drop_enabled"); old_data.as_object_mut().unwrap().remove("dropped_at"); + old_data.as_object_mut().unwrap().remove("event_context"); let recovered = DropTableProcedure::from_json(&old_data.to_string(), runtime_context).unwrap(); let recovered_data: serde_json::Value = @@ -171,12 +177,20 @@ fn test_old_drop_table_json_defaults_to_hard_drop() { assert_eq!(recovered_data["soft_drop_enabled"], false); assert_eq!(recovered_data["dropped_at"], serde_json::Value::Null); + assert_eq!( + recovered_data["event_context"], + serde_json::json!({"reason": "unknown"}) + ); } #[test] fn test_drop_table_lock_key_includes_table_name() { let context = new_ddl_context(Arc::new(MockDatanodeManager::new(()))); - let procedure = DropTableProcedure::new(new_drop_table_task("foo", 1024, false), context); + let procedure = DropTableProcedure::new( + new_drop_table_task("foo", 1024, false), + context, + EventContext::default(), + ); let keys = procedure .lock_key() @@ -205,7 +219,11 @@ fn test_new_drop_table_procedure_disables_soft_drop_without_enterprise() { let mut context = new_ddl_context(Arc::new(MockDatanodeManager::new(()))); context.soft_drop_enabled = true; - let procedure = DropTableProcedure::new(new_drop_table_task("foo", 1024, false), context); + let procedure = DropTableProcedure::new( + new_drop_table_task("foo", 1024, false), + context, + EventContext::default(), + ); assert!(!procedure.data.soft_drop_enabled); } @@ -216,7 +234,11 @@ fn test_new_drop_table_procedure_preserves_soft_drop_with_enterprise() { let mut context = new_ddl_context(Arc::new(MockDatanodeManager::new(()))); context.soft_drop_enabled = true; - let procedure = DropTableProcedure::new(new_drop_table_task("foo", 1024, false), context); + let procedure = DropTableProcedure::new( + new_drop_table_task("foo", 1024, false), + context, + EventContext::default(), + ); assert!(procedure.data.soft_drop_enabled); } @@ -251,6 +273,7 @@ async fn test_recovered_soft_drop_preserves_persisted_mode() { let procedure = DropTableProcedure::new( new_drop_table_task("foo", table_id, false), soft_context.clone(), + EventContext::default(), ); let hard_context = new_ddl_context_with_kv_backend(node_manager, kv_backend); @@ -290,9 +313,13 @@ async fn test_legacy_soft_drop_prepare_preserves_persisted_mode() { .unwrap(); let mut persisted: serde_json::Value = serde_json::from_str( - &DropTableProcedure::new(new_drop_table_task("foo", table_id, false), context.clone()) - .dump() - .unwrap(), + &DropTableProcedure::new( + new_drop_table_task("foo", table_id, false), + context.clone(), + EventContext::default(), + ) + .dump() + .unwrap(), ) .unwrap(); persisted["soft_drop_enabled"] = true.into(); @@ -322,7 +349,11 @@ async fn test_disabled_soft_drop_creates_hard_drop_procedure() { context.soft_drop_enabled = false; context.soft_drop_retention = Some(Duration::from_millis(100)); - let procedure = DropTableProcedure::new(new_drop_table_task("foo", 1024, false), context); + let procedure = DropTableProcedure::new( + new_drop_table_task("foo", 1024, false), + context, + EventContext::default(), + ); assert!(!procedure.data.soft_drop_enabled); assert_eq!(None, procedure.data.dropped_at); @@ -356,6 +387,7 @@ async fn test_recovered_hard_drop_ignores_soft_runtime_context() { let procedure = DropTableProcedure::new( new_drop_table_task("foo", table_id, false), hard_context.clone(), + EventContext::default(), ); let mut soft_context = new_ddl_context_with_kv_backend(node_manager, kv_backend); @@ -397,7 +429,7 @@ async fn test_on_prepare_table_not_exists_err() { .unwrap(); let task = new_drop_table_task("bar", table_id, false); - let mut procedure = DropTableProcedure::new(task, ddl_context); + let mut procedure = DropTableProcedure::new(task, ddl_context, EventContext::default()); assert_eq!(procedure.data.dropped_at, None); assert_eq!(procedure.data.retention_expires_at, None); let err = procedure.on_prepare().await.unwrap_err(); @@ -427,6 +459,7 @@ async fn test_soft_drop_prepare_assigns_stable_deadline_and_recovers_it() { let mut procedure = DropTableProcedure::new( new_drop_table_task("foo", table_id, false), ddl_context.clone(), + EventContext::default(), ); procedure.on_prepare().await.unwrap(); @@ -463,6 +496,7 @@ async fn test_soft_drop_prepare_rejects_deadline_overflow_before_metadata_delete let mut procedure = DropTableProcedure::new( new_drop_table_task("foo", table_id, false), ddl_context.clone(), + EventContext::default(), ); let error = procedure.on_prepare().await.unwrap_err(); @@ -502,13 +536,13 @@ async fn test_on_prepare_table() { let task = new_drop_table_task("bar", table_id, true); // Drop if exists - let mut procedure = DropTableProcedure::new(task, ddl_context.clone()); + let mut procedure = DropTableProcedure::new(task, ddl_context.clone(), EventContext::default()); procedure.on_prepare().await.unwrap(); assert!(!procedure.rollback_supported()); let task = new_drop_table_task(table_name, table_id, false); // Drop table - let mut procedure = DropTableProcedure::new(task, ddl_context); + let mut procedure = DropTableProcedure::new(task, ddl_context, EventContext::default()); procedure.on_prepare().await.unwrap(); } @@ -559,7 +593,7 @@ async fn test_on_datanode_drop_regions() { let task = new_drop_table_task(table_name, table_id, false); // Drop table - let mut procedure = DropTableProcedure::new(task, ddl_context); + let mut procedure = DropTableProcedure::new(task, ddl_context, EventContext::default()); procedure.on_prepare().await.unwrap(); procedure.on_datanode_drop_regions(false).await.unwrap(); @@ -628,7 +662,7 @@ async fn test_on_datanode_drop_regions_remaps_addresses_when_retrying() { .unwrap(); let task = new_drop_table_task(table_name, table_id, false); - let mut procedure = DropTableProcedure::new(task, ddl_context.clone()); + let mut procedure = DropTableProcedure::new(task, ddl_context.clone(), EventContext::default()); procedure.on_prepare().await.unwrap(); put_datanode_address(&ddl_context, 1, "new-leader").await; @@ -687,7 +721,7 @@ async fn test_soft_drop_closes_regions_and_keeps_tombstone() { .unwrap(); let task = new_drop_table_task(table_name, table_id, false); - let mut procedure = DropTableProcedure::new(task, ddl_context.clone()); + let mut procedure = DropTableProcedure::new(task, ddl_context.clone(), EventContext::default()); execute_procedure_until_done(&mut procedure).await; @@ -801,7 +835,7 @@ async fn test_soft_drop_keeps_metadata_live_until_regions_close() { .unwrap(); let task = new_drop_table_task(table_name, table_id, false); - let mut procedure = DropTableProcedure::new(task, ddl_context.clone()); + let mut procedure = DropTableProcedure::new(task, ddl_context.clone(), EventContext::default()); let ctx = new_test_procedure_context(); procedure.execute(&ctx).await.unwrap(); put_datanode_address(&ddl_context, 1, "new-leader").await; @@ -888,6 +922,7 @@ async fn test_soft_drop_timestamp_is_stable_across_retry_and_recovery() { let procedure = DropTableProcedure::new( new_drop_table_task("foo", table_id, false), ddl_context.clone(), + EventContext::default(), ); assert_eq!(procedure.data.dropped_at, None); let mut procedure = procedure; @@ -958,7 +993,7 @@ async fn test_hard_drop_keeps_delete_tombstone_flow() { .unwrap(); let task = new_drop_table_task(table_name, table_id, false); - let mut procedure = DropTableProcedure::new(task, ddl_context.clone()); + let mut procedure = DropTableProcedure::new(task, ddl_context.clone(), EventContext::default()); execute_procedure_until(&mut procedure, |p| { p.data.state == DropTableState::DeleteTombstone @@ -1001,13 +1036,20 @@ async fn test_create_table_succeeds_while_tombstone_exists() { .unwrap(); let drop_task = new_drop_table_task(table_name, dropped_table_id, false); - let mut drop_procedure = DropTableProcedure::new(drop_task, ddl_context.clone()); + let mut drop_procedure = + DropTableProcedure::new(drop_task, ddl_context.clone(), EventContext::default()); execute_procedure_until_done(&mut drop_procedure).await; let mut create_task = test_create_table_task(table_name, 1025); create_task.create_table.table_id = None; create_task.table_info.ident.table_id = 0; - let mut create_procedure = CreateTableProcedure::new(create_task, ddl_context.clone()).unwrap(); + let mut create_procedure = CreateTableProcedure::new( + create_task, + QueryContext::default(), + EventContext::default(), + ddl_context.clone(), + ) + .unwrap(); execute_procedure_until_done(&mut create_procedure).await; let live_table = ddl_context @@ -1052,7 +1094,8 @@ async fn test_hard_drop_recreated_table_fails_when_soft_tombstone_exists() { .unwrap(); let drop_task = new_drop_table_task(table_name, original_table_id, false); - let mut drop_procedure = DropTableProcedure::new(drop_task, ddl_context.clone()); + let mut drop_procedure = + DropTableProcedure::new(drop_task, ddl_context.clone(), EventContext::default()); execute_procedure_until_done(&mut drop_procedure).await; ddl_context.soft_drop_enabled = false; @@ -1069,6 +1112,7 @@ async fn test_hard_drop_recreated_table_fails_when_soft_tombstone_exists() { let mut procedure = DropTableProcedure::new( new_drop_table_task(table_name, recreated_table_id, false), ddl_context, + EventContext::default(), ); let err = procedure.on_prepare().await.unwrap_err(); @@ -1098,7 +1142,8 @@ async fn test_hard_drop_recreated_table_ignores_previous_orphan_tombstone() { .unwrap(); let drop_task = new_drop_table_task(table_name, original_table_id, false); - let mut drop_procedure = DropTableProcedure::new(drop_task, ddl_context.clone()); + let mut drop_procedure = + DropTableProcedure::new(drop_task, ddl_context.clone(), EventContext::default()); execute_procedure_until(&mut drop_procedure, |p| { p.data.state == DropTableState::DeleteTombstone }) @@ -1117,6 +1162,7 @@ async fn test_hard_drop_recreated_table_ignores_previous_orphan_tombstone() { let mut procedure = DropTableProcedure::new( new_drop_table_task(table_name, recreated_table_id, false), ddl_context, + EventContext::default(), ); procedure.on_prepare().await.unwrap(); @@ -1156,6 +1202,7 @@ async fn test_undrop_table_restores_metadata_and_reopens_regions() { let mut drop_procedure = DropTableProcedure::new( new_drop_table_task(table_name, table_id, false), ddl_context.clone(), + EventContext::default(), ); execute_procedure_until_done(&mut drop_procedure).await; assert!( @@ -1172,8 +1219,11 @@ async fn test_undrop_table_restores_metadata_and_reopens_regions() { while rx.try_recv().is_ok() {} - let mut procedure = - UndropTableProcedure::new(new_undrop_table_task(table_id), ddl_context.clone()); + let mut procedure = UndropTableProcedure::new( + new_undrop_table_task(table_id), + ddl_context.clone(), + EventContext::default(), + ); execute_procedure_until_done(&mut procedure).await; let live_table = ddl_context @@ -1264,12 +1314,16 @@ async fn test_undrop_table_opens_regions_before_restoring_live_metadata() { let mut drop_procedure = DropTableProcedure::new( new_drop_table_task(table_name, table_id, false), ddl_context.clone(), + EventContext::default(), ); execute_procedure_until_done(&mut drop_procedure).await; while rx.try_recv().is_ok() {} - let mut procedure = - UndropTableProcedure::new(new_undrop_table_task(table_id), ddl_context.clone()); + let mut procedure = UndropTableProcedure::new( + new_undrop_table_task(table_id), + ddl_context.clone(), + EventContext::default(), + ); let ctx = new_test_procedure_context(); procedure.execute(&ctx).await.unwrap(); procedure.execute(&ctx).await.unwrap(); @@ -1333,13 +1387,17 @@ async fn test_undrop_logical_table_skips_datanode_open() { let mut drop_procedure = DropTableProcedure::new( new_drop_table_task(table_name, logical_table_id, false), ddl_context.clone(), + EventContext::default(), ); execute_procedure_until_done(&mut drop_procedure).await; while rx.try_recv().is_ok() {} - let mut procedure = - UndropTableProcedure::new(new_undrop_table_task(logical_table_id), ddl_context.clone()); + let mut procedure = UndropTableProcedure::new( + new_undrop_table_task(logical_table_id), + ddl_context.clone(), + EventContext::default(), + ); execute_procedure_until_done(&mut procedure).await; let live_table = ddl_context @@ -1370,6 +1428,7 @@ async fn test_soft_drop_metric_logical_table_falls_back_to_hard_drop() { let mut procedure = DropTableProcedure::new( new_drop_table_task("foo", logical_table_id, false), ddl_context.clone(), + EventContext::default(), ); procedure.on_prepare().await.unwrap(); assert!(!procedure.data.soft_drop_enabled); @@ -1413,6 +1472,7 @@ async fn test_soft_drop_metric_physical_table_remains_enabled() { let mut procedure = DropTableProcedure::new( new_drop_table_task(table_name, table_id, false), ddl_context.clone(), + EventContext::default(), ); execute_procedure_until_done(&mut procedure).await; @@ -1457,6 +1517,7 @@ async fn test_soft_drop_file_engine_table_falls_back_to_hard_drop() { let mut procedure = DropTableProcedure::new( new_drop_table_task(table_name, table_id, false), ddl_context.clone(), + EventContext::default(), ); procedure.on_prepare().await.unwrap(); assert!(!procedure.data.soft_drop_enabled); @@ -1493,6 +1554,7 @@ async fn test_file_engine_fallback_is_resolved_before_tombstone_conflict() { let mut procedure = DropTableProcedure::new( new_drop_table_task(table_name, recreated_table_id, false), ddl_context, + EventContext::default(), ); procedure.on_prepare().await.unwrap(); @@ -1508,8 +1570,11 @@ async fn test_undrop_metric_logical_table_fails() { let logical_table_id = create_metric_logical_table_tombstone(&ddl_context, physical_table_id, "foo").await; - let mut procedure = - UndropTableProcedure::new(new_undrop_table_task(logical_table_id), ddl_context); + let mut procedure = UndropTableProcedure::new( + new_undrop_table_task(logical_table_id), + ddl_context, + EventContext::default(), + ); let err = procedure.on_prepare().await.unwrap_err(); assert_eq!(err.status_code(), StatusCode::Unsupported); @@ -1527,6 +1592,7 @@ async fn test_purge_metric_logical_table_fails() { let mut procedure = PurgeDroppedTableProcedure::new( new_purge_dropped_table_task(logical_table_id), ddl_context, + EventContext::default(), ); let err = procedure .execute(&new_test_procedure_context()) @@ -1558,6 +1624,7 @@ async fn test_undrop_table_fails_when_live_name_exists() { let mut drop_procedure = DropTableProcedure::new( new_drop_table_task(table_name, dropped_table_id, false), ddl_context.clone(), + EventContext::default(), ); execute_procedure_until_done(&mut drop_procedure).await; ddl_context @@ -1570,8 +1637,11 @@ async fn test_undrop_table_fails_when_live_name_exists() { .await .unwrap(); - let mut procedure = - UndropTableProcedure::new(new_undrop_table_task(dropped_table_id), ddl_context); + let mut procedure = UndropTableProcedure::new( + new_undrop_table_task(dropped_table_id), + ddl_context, + EventContext::default(), + ); let err = procedure.on_prepare().await.unwrap_err(); assert_matches!(err, Error::TableAlreadyExists { .. }); @@ -1599,11 +1669,15 @@ async fn test_undrop_table_fails_when_live_name_is_created_after_prepare() { let mut drop_procedure = DropTableProcedure::new( new_drop_table_task(table_name, dropped_table_id, false), ddl_context.clone(), + EventContext::default(), ); execute_procedure_until_done(&mut drop_procedure).await; - let mut procedure = - UndropTableProcedure::new(new_undrop_table_task(dropped_table_id), ddl_context.clone()); + let mut procedure = UndropTableProcedure::new( + new_undrop_table_task(dropped_table_id), + ddl_context.clone(), + EventContext::default(), + ); procedure.on_prepare().await.unwrap(); ddl_context .table_metadata_manager @@ -1667,13 +1741,17 @@ async fn test_undrop_table_closes_opened_regions_when_restore_metadata_races_wit let mut drop_procedure = DropTableProcedure::new( new_drop_table_task(table_name, dropped_table_id, false), ddl_context.clone(), + EventContext::default(), ); execute_procedure_until_done(&mut drop_procedure).await; while rx.try_recv().is_ok() {} detector_controller.clear().await; - let mut procedure = - UndropTableProcedure::new(new_undrop_table_task(dropped_table_id), ddl_context.clone()); + let mut procedure = UndropTableProcedure::new( + new_undrop_table_task(dropped_table_id), + ddl_context.clone(), + EventContext::default(), + ); let ctx = new_test_procedure_context(); procedure.execute(&ctx).await.unwrap(); procedure.execute(&ctx).await.unwrap(); @@ -1767,6 +1845,7 @@ async fn test_undrop_table_lock_key_includes_original_table_name_before_prepare( let mut drop_procedure = DropTableProcedure::new( new_drop_table_task(table_name, table_id, false), ddl_context.clone(), + EventContext::default(), ); execute_procedure_until_done(&mut drop_procedure).await; @@ -1781,6 +1860,7 @@ async fn test_undrop_table_lock_key_includes_original_table_name_before_prepare( new_undrop_table_task(table_id), ddl_context, Some(original_table_name), + EventContext::default(), ); let keys = procedure @@ -1848,10 +1928,12 @@ async fn test_undrop_rejects_stale_id_after_name_tombstone_is_consumed() { new_undrop_table_task(original_table_id), ddl_context.clone(), Some(original_table_name.clone()), + EventContext::default(), ); let mut hard_drop = DropTableProcedure::new( new_drop_table_task(table_name, recreated_table_id, false), ddl_context.clone(), + EventContext::default(), ); execute_procedure_until_done(&mut hard_drop).await; assert!( @@ -1906,11 +1988,15 @@ async fn test_undrop_table_replayed_restore_metadata_is_idempotent() { let mut drop_procedure = DropTableProcedure::new( new_drop_table_task(table_name, table_id, false), ddl_context.clone(), + EventContext::default(), ); execute_procedure_until_done(&mut drop_procedure).await; - let mut procedure = - UndropTableProcedure::new(new_undrop_table_task(table_id), ddl_context.clone()); + let mut procedure = UndropTableProcedure::new( + new_undrop_table_task(table_id), + ddl_context.clone(), + EventContext::default(), + ); let ctx = new_test_procedure_context(); procedure.execute(&ctx).await.unwrap(); procedure.execute(&ctx).await.unwrap(); @@ -2014,6 +2100,7 @@ async fn test_purge_dropped_table_cleans_regions_offline_and_deletes_tombstone() let mut drop_procedure = DropTableProcedure::new( new_drop_table_task(table_name, table_id, false), ddl_context.clone(), + EventContext::default(), ); execute_procedure_until_done(&mut drop_procedure).await; while rx.try_recv().is_ok() {} @@ -2022,6 +2109,7 @@ async fn test_purge_dropped_table_cleans_regions_offline_and_deletes_tombstone() let mut procedure = PurgeDroppedTableProcedure::new( new_purge_dropped_table_task(table_id), ddl_context.clone(), + EventContext::default(), ); execute_procedure_until_done(&mut procedure).await; @@ -2100,6 +2188,7 @@ async fn test_automatic_purge_rechecks_unexpired_tombstone() { let procedure = PurgeDroppedTableProcedure::new( new_purge_dropped_table_task(table_id), ddl_context.clone(), + EventContext::default(), ); let mut data: serde_json::Value = serde_json::from_str(&procedure.dump().unwrap()).unwrap(); data["check_expired"] = true.into(); @@ -2156,6 +2245,7 @@ async fn test_recovered_automatic_purge_rechecks_unexpired_tombstone() { let procedure = PurgeDroppedTableProcedure::new( new_purge_dropped_table_task(table_id), ddl_context.clone(), + EventContext::default(), ); let mut data: serde_json::Value = serde_json::from_str(&procedure.dump().unwrap()).unwrap(); data["check_expired"] = true.into(); @@ -2233,6 +2323,7 @@ async fn test_recovered_automatic_purge_obeys_runtime_disable() { let procedure = PurgeDroppedTableProcedure::new_if_expired( new_purge_dropped_table_task(table_id), ddl_context.clone(), + EventContext::default(), ); let mut recovered = PurgeDroppedTableProcedure::from_json(&procedure.dump().unwrap(), ddl_context.clone()) @@ -2264,6 +2355,7 @@ async fn test_recovered_automatic_purge_finishes_after_cleanup_is_claimed() { let mut procedure = PurgeDroppedTableProcedure::new_if_expired( new_purge_dropped_table_task(table_id), enabled_context.clone(), + EventContext::default(), ); procedure .execute(&new_test_procedure_context()) @@ -2327,6 +2419,7 @@ async fn test_recovered_automatic_purge_ignores_foreign_generation_claim() { let mut procedure = PurgeDroppedTableProcedure::new_if_expired( new_purge_dropped_table_task(table_id), ddl_context.clone(), + EventContext::default(), ); procedure .execute(&new_test_procedure_context()) @@ -2369,8 +2462,11 @@ async fn test_undrop_rejects_table_claimed_for_purge() { .await .unwrap(); - let mut procedure = - UndropTableProcedure::new(new_undrop_table_task(table_id), ddl_context.clone()); + let mut procedure = UndropTableProcedure::new( + new_undrop_table_task(table_id), + ddl_context.clone(), + EventContext::default(), + ); let error = procedure.on_prepare().await.unwrap_err(); assert_eq!(StatusCode::TableNotFound, error.status_code()); @@ -2392,8 +2488,11 @@ async fn test_recovered_undrop_rejects_tombstone_deleted_by_purge() { let table_id = 1024; create_dropped_table(&ddl_context, table_id, Some(1), Some(i64::MAX)).await; - let mut undrop = - UndropTableProcedure::new(new_undrop_table_task(table_id), ddl_context.clone()); + let mut undrop = UndropTableProcedure::new( + new_undrop_table_task(table_id), + ddl_context.clone(), + EventContext::default(), + ); undrop.on_prepare().await.unwrap(); let open_regions_state = undrop.dump().unwrap(); undrop.execute(&new_test_procedure_context()).await.unwrap(); @@ -2402,6 +2501,7 @@ async fn test_recovered_undrop_rejects_tombstone_deleted_by_purge() { let mut purge = PurgeDroppedTableProcedure::new( new_purge_dropped_table_task(table_id), ddl_context.clone(), + EventContext::default(), ); execute_procedure_until_done(&mut purge).await; @@ -2433,8 +2533,11 @@ async fn test_recovered_undrop_rejects_replacement_generation() { .await .unwrap(); - let mut undrop = - UndropTableProcedure::new(new_undrop_table_task(table_id), ddl_context.clone()); + let mut undrop = UndropTableProcedure::new( + new_undrop_table_task(table_id), + ddl_context.clone(), + EventContext::default(), + ); undrop.on_prepare().await.unwrap(); let persisted = undrop.dump().unwrap(); kv_backend @@ -2500,8 +2603,11 @@ async fn test_legacy_unmarked_tombstone_can_be_undropped_and_purged() { assert_eq!(dropped_tables.len(), 1); assert_eq!(dropped_tables[0].dropped_at, None); - let mut undrop = - UndropTableProcedure::new(new_undrop_table_task(table_id), ddl_context.clone()); + let mut undrop = UndropTableProcedure::new( + new_undrop_table_task(table_id), + ddl_context.clone(), + EventContext::default(), + ); execute_procedure_until_done(&mut undrop).await; ddl_context @@ -2518,6 +2624,7 @@ async fn test_legacy_unmarked_tombstone_can_be_undropped_and_purged() { let mut purge = PurgeDroppedTableProcedure::new( new_purge_dropped_table_task(table_id), ddl_context.clone(), + EventContext::default(), ); execute_procedure_until_done(&mut purge).await; assert!( @@ -2560,6 +2667,7 @@ async fn test_purge_dropped_table_by_id_selects_tombstone_when_live_table_exists let mut drop_procedure = DropTableProcedure::new( new_drop_table_task(table_name, dropped_table_id, false), ddl_context.clone(), + EventContext::default(), ); execute_procedure_until_done(&mut drop_procedure).await; ddl_context @@ -2576,6 +2684,7 @@ async fn test_purge_dropped_table_by_id_selects_tombstone_when_live_table_exists let mut procedure = PurgeDroppedTableProcedure::new( new_purge_dropped_table_task(dropped_table_id), ddl_context.clone(), + EventContext::default(), ); execute_procedure_until_done(&mut procedure).await; @@ -2626,8 +2735,12 @@ async fn test_on_rollback() { let physical_table_id = table_id; // Creates the logical table metadata. let task = test_create_logical_table_task("foo"); - let mut procedure = - CreateLogicalTablesProcedure::new(vec![task], physical_table_id, ddl_context.clone()); + let mut procedure = CreateLogicalTablesProcedure::new( + vec![task], + physical_table_id, + EventContext::default(), + ddl_context.clone(), + ); procedure.on_prepare().await.unwrap(); let ctx = new_test_procedure_context(); procedure.execute(&ctx).await.unwrap(); @@ -2640,12 +2753,25 @@ async fn test_on_rollback() { // Drops the physical table { let task = new_drop_table_task("phy_table", physical_table_id, false); - let mut procedure = DropTableProcedure::new(task, ddl_context.clone()); + let mut procedure = + DropTableProcedure::new(task, ddl_context.clone(), EventContext::default()); procedure.on_prepare().await.unwrap(); assert!(procedure.rollback_supported()); procedure.on_delete_metadata().await.unwrap(); assert!(procedure.rollback_supported()); procedure.rollback(&ctx).await.unwrap(); + assert!(procedure.dropping_regions.is_empty()); + assert_eq!(ddl_context.memory_region_keeper.len(), 0); + + // The physical table can be dropped again after the metadata rollback. + let retry_task = new_drop_table_task("phy_table", physical_table_id, false); + let mut retry = + DropTableProcedure::new(retry_task, ddl_context.clone(), EventContext::default()); + retry.on_prepare().await.unwrap(); + retry.on_delete_metadata().await.unwrap(); + retry.rollback(&ctx).await.unwrap(); + assert!(retry.dropping_regions.is_empty()); + assert_eq!(ddl_context.memory_region_keeper.len(), 0); // Rollback again assert!(procedure.rollback_supported()); procedure.rollback(&ctx).await.unwrap(); @@ -2656,7 +2782,7 @@ async fn test_on_rollback() { // Drops the logical table ddl_context.soft_drop_enabled = false; let task = new_drop_table_task("foo", table_ids[0], false); - let mut procedure = DropTableProcedure::new(task, ddl_context.clone()); + let mut procedure = DropTableProcedure::new(task, ddl_context.clone(), EventContext::default()); procedure.on_prepare().await.unwrap(); assert!(!procedure.rollback_supported()); } @@ -2753,7 +2879,8 @@ async fn test_memory_region_keeper_guard_dropped_on_procedure_done() { let logical_table_id = create_logical_table(ddl_context.clone(), physical_table_id, "s").await; let inner_test = |task: DropTableTask| async { - let mut procedure = DropTableProcedure::new(task, ddl_context.clone()); + let mut procedure = + DropTableProcedure::new(task, ddl_context.clone(), EventContext::default()); execute_procedure_until(&mut procedure, |p| { p.data.state == DropTableState::InvalidateTableCache }) @@ -2805,7 +2932,8 @@ async fn test_from_json() { let physical_table_id = create_physical_table(&ddl_context, "t").await; let task = new_drop_table_task("t", physical_table_id, false); - let mut procedure = DropTableProcedure::new(task, ddl_context.clone()); + let mut procedure = + DropTableProcedure::new(task, ddl_context.clone(), EventContext::default()); execute_procedure_until(&mut procedure, |p| p.data.state == state).await; let data = procedure.dump().unwrap(); assert_eq!( @@ -2834,7 +2962,7 @@ async fn test_from_json() { let physical_table_id = create_physical_table(&ddl_context, "t").await; let task = new_drop_table_task("t", physical_table_id, false); - let mut procedure = DropTableProcedure::new(task, ddl_context.clone()); + let mut procedure = DropTableProcedure::new(task, ddl_context.clone(), EventContext::default()); execute_procedure_until_done(&mut procedure).await; let data = procedure.dump().unwrap(); assert_eq!( diff --git a/src/common/meta/src/ddl/tests/drop_view.rs b/src/common/meta/src/ddl/tests/drop_view.rs index 412c5a78f0b..09b2a450c52 100644 --- a/src/common/meta/src/ddl/tests/drop_view.rs +++ b/src/common/meta/src/ddl/tests/drop_view.rs @@ -24,7 +24,7 @@ use crate::ddl::drop_view::{DropViewProcedure, DropViewState}; use crate::ddl::test_util::create_table::test_create_table_task; use crate::ddl::tests::create_view::{test_create_view_task, test_table_names}; use crate::key::table_route::TableRouteValue; -use crate::rpc::ddl::DropViewTask; +use crate::rpc::ddl::{DropViewTask, EventContext}; use crate::test_util::{MockDatanodeManager, new_ddl_context}; pub(crate) fn new_drop_view_task( @@ -63,7 +63,7 @@ async fn test_on_prepare_view_not_exists_err() { .unwrap(); let task = new_drop_view_task("bar", view_id, false); - let mut procedure = DropViewProcedure::new(task, ddl_context); + let mut procedure = DropViewProcedure::new(task, EventContext::default(), ddl_context); let err = procedure.on_prepare().await.unwrap_err(); assert_eq!(err.status_code(), StatusCode::TableNotFound); } @@ -87,7 +87,7 @@ async fn test_on_prepare_not_view_err() { .unwrap(); let task = new_drop_view_task(view_name, view_id, false); - let mut procedure = DropViewProcedure::new(task, ddl_context); + let mut procedure = DropViewProcedure::new(task, EventContext::default(), ddl_context); // It's not a view, expect error let err = procedure.on_prepare().await.unwrap_err(); assert_eq!(err.status_code(), StatusCode::InvalidArguments); @@ -117,12 +117,12 @@ async fn test_on_prepare_success() { let task = new_drop_view_task("bar", view_id, true); // Drop if exists - let mut procedure = DropViewProcedure::new(task, ddl_context.clone()); + let mut procedure = DropViewProcedure::new(task, EventContext::default(), ddl_context.clone()); procedure.on_prepare().await.unwrap(); let task = new_drop_view_task(view_name, view_id, false); // Prepare success - let mut procedure = DropViewProcedure::new(task, ddl_context); + let mut procedure = DropViewProcedure::new(task, EventContext::default(), ddl_context); procedure.on_prepare().await.unwrap(); assert_eq!(DropViewState::DeleteMetadata, procedure.state()); } @@ -161,7 +161,7 @@ async fn test_drop_view_success() { let task = new_drop_view_task(view_name, view_id, false); // Prepare success - let mut procedure = DropViewProcedure::new(task, ddl_context.clone()); + let mut procedure = DropViewProcedure::new(task, EventContext::default(), ddl_context.clone()); execute_procedure_until_done(&mut procedure).await; assert_eq!(DropViewState::InvalidateViewCache, procedure.state()); @@ -178,7 +178,7 @@ async fn test_drop_view_success() { // Drop again let task = new_drop_view_task(view_name, view_id, false); - let mut procedure = DropViewProcedure::new(task, ddl_context); + let mut procedure = DropViewProcedure::new(task, EventContext::default(), ddl_context); let err = procedure.on_prepare().await.unwrap_err(); assert_eq!(err.status_code(), StatusCode::TableNotFound); } diff --git a/src/common/meta/src/ddl/tests/event/database.rs b/src/common/meta/src/ddl/tests/event/database.rs index 39c001e9ffd..8aa85c2fbbd 100644 --- a/src/common/meta/src/ddl/tests/event/database.rs +++ b/src/common/meta/src/ddl/tests/event/database.rs @@ -20,17 +20,17 @@ use api::v1::value::ValueData; use api::v1::{AlterDatabaseExpr, Row, SetDatabaseOptions as PbSetDatabaseOptions, Value}; use common_event_recorder::Event; use common_event_recorder::event_table::{ - CATALOG_NAME_COLUMN as EVENT_TABLE_CATALOG_NAME_COLUMN, + CATALOG_NAME_COLUMN as EVENT_TABLE_CATALOG_NAME_COLUMN, EVENT_CONTEXT_COLUMN, PROCEDURE_ERROR_COLUMN as EVENT_TABLE_PROCEDURE_ERROR_COLUMN, PROCEDURE_ID_COLUMN as EVENT_TABLE_PROCEDURE_ID_COLUMN, PROCEDURE_STATE_COLUMN as EVENT_TABLE_PROCEDURE_STATE_COLUMN, PROCEDURE_TRIGGER_COLUMN as EVENT_TABLE_PROCEDURE_TRIGGER_COLUMN, - SCHEMA_NAME_COLUMN as EVENT_TABLE_SCHEMA_NAME_COLUMN, jsonb_value, + SCHEMA_NAME_COLUMN as EVENT_TABLE_SCHEMA_NAME_COLUMN, }; use common_event_recorder::testing::assert_event_contract; use common_procedure::{EventTrigger, ProcedureEvent, ProcedureId, ProcedureState}; -use super::test_util::assert_event_filter; +use super::test_util::{assert_event_filter, default_event_context_value, procedure_trigger_value}; use crate::ddl::alter_database::AlterDatabaseProcedure; use crate::ddl::create_database::CreateDatabaseProcedure; use crate::ddl::drop_database::DropDatabaseProcedure; @@ -39,7 +39,7 @@ use crate::ddl::event::database::{ DatabaseDdlEvent, }; use crate::rpc::ddl::{ - AlterDatabaseKind, SetDatabaseOption, SetDatabaseOptions, UnsetDatabaseOption, + AlterDatabaseKind, EventContext, SetDatabaseOption, SetDatabaseOptions, UnsetDatabaseOption, UnsetDatabaseOptions, }; use crate::test_util::{MockDatanodeManager, new_ddl_context}; @@ -50,7 +50,13 @@ fn test_create_database_submitted_event_contract() { ("password".to_string(), "do-not-record".to_string()), ("compaction.type".to_string(), "twcs".to_string()), ]); - let event = DatabaseDdlEvent::create_submitted("greptime", "metrics", true, &options); + let event = DatabaseDdlEvent::create_submitted( + "greptime", + "metrics", + true, + &options, + EventContext::default(), + ); assert_event_locator( &event, @@ -87,6 +93,7 @@ fn test_alter_database_set_and_unset_event_contracts() { SetDatabaseOption::Other("secret_token".to_string(), "hidden".to_string()), SetDatabaseOption::Ttl(std::time::Duration::from_secs(3600).into()), ])), + EventContext::default(), ); assert_event_locator( &set, @@ -113,6 +120,7 @@ fn test_alter_database_set_and_unset_event_contracts() { UnsetDatabaseOption::Ttl, UnsetDatabaseOption::Other("compaction.type".to_string()), ])), + EventContext::default(), ); assert_event_locator( &unset, @@ -132,7 +140,8 @@ fn test_alter_database_set_and_unset_event_contracts() { #[test] fn test_drop_database_submitted_event_contract() { - let event = DatabaseDdlEvent::drop_submitted("greptime", "metrics", true); + let event = + DatabaseDdlEvent::drop_submitted("greptime", "metrics", true, EventContext::default()); assert_event_locator( &event, @@ -165,7 +174,14 @@ fn test_database_lifecycle_events_have_fixed_schema_and_null_intent() { assert_eq!( DatabaseDdlEvent::create_lifecycle().extra_schema(), - DatabaseDdlEvent::create_submitted("c", "s", false, &HashMap::new()).extra_schema() + DatabaseDdlEvent::create_submitted( + "c", + "s", + false, + &HashMap::new(), + EventContext::default() + ) + .extra_schema() ); } @@ -179,6 +195,7 @@ fn test_database_events_preserve_procedure_envelope_contract() { "metrics", false, &HashMap::new(), + EventContext::default(), )), ProcedureState::Running, EventTrigger::Submitted, @@ -217,6 +234,7 @@ fn test_create_database_event_filter() { false, HashMap::new(), None, + EventContext::default(), new_ddl_context(Arc::new(MockDatanodeManager::new(()))), ); @@ -237,6 +255,7 @@ fn test_alter_database_event_filter() { )), }, }, + EventContext::default(), new_ddl_context(Arc::new(MockDatanodeManager::new(()))), ) .unwrap(); @@ -250,6 +269,7 @@ fn test_drop_database_event_filter() { "greptime".to_string(), "metrics".to_string(), false, + EventContext::default(), new_ddl_context(Arc::new(MockDatanodeManager::new(()))), ); @@ -268,6 +288,7 @@ fn assert_event_locator( &[ EVENT_TABLE_CATALOG_NAME_COLUMN.column_schema(), EVENT_TABLE_SCHEMA_NAME_COLUMN.column_schema(), + EVENT_CONTEXT_COLUMN.column_schema(), ], &[Row { values: vec![ @@ -277,6 +298,11 @@ fn assert_event_locator( Value { value_data: schema_name.map(|value| ValueData::StringValue(value.to_string())), }, + if catalog_name.is_some() { + default_event_context_value() + } else { + Value { value_data: None } + }, ], }], ); @@ -300,6 +326,7 @@ fn assert_procedure_event_contract( EVENT_TABLE_PROCEDURE_TRIGGER_COLUMN.column_schema(), EVENT_TABLE_CATALOG_NAME_COLUMN.column_schema(), EVENT_TABLE_SCHEMA_NAME_COLUMN.column_schema(), + EVENT_CONTEXT_COLUMN.column_schema(), ], &[Row { values: vec![ @@ -314,13 +341,18 @@ fn assert_procedure_event_contract( Value { value_data: Some(ValueData::StringValue(String::new())), }, - jsonb_value(&serde_json::json!({"type": procedure_trigger})), + procedure_trigger_value(procedure_trigger), Value { value_data: catalog_name.map(|value| ValueData::StringValue(value.to_string())), }, Value { value_data: schema_name.map(|value| ValueData::StringValue(value.to_string())), }, + if catalog_name.is_some() { + default_event_context_value() + } else { + Value { value_data: None } + }, ], }], ); diff --git a/src/common/meta/src/ddl/tests/event/flow.rs b/src/common/meta/src/ddl/tests/event/flow.rs index db3d5e2ce11..b2758197917 100644 --- a/src/common/meta/src/ddl/tests/event/flow.rs +++ b/src/common/meta/src/ddl/tests/event/flow.rs @@ -19,14 +19,14 @@ use api::v1::{ColumnSchema, Row, Value}; use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME}; use common_event_recorder::Event; use common_event_recorder::event_table::{ - CATALOG_NAME_COLUMN, FLOW_ID_COLUMN, FLOW_NAME_COLUMN, PROCEDURE_ERROR_COLUMN, - PROCEDURE_ID_COLUMN, PROCEDURE_STATE_COLUMN, PROCEDURE_TRIGGER_COLUMN, jsonb_value, + CATALOG_NAME_COLUMN, EVENT_CONTEXT_COLUMN, FLOW_ID_COLUMN, FLOW_NAME_COLUMN, + PROCEDURE_ERROR_COLUMN, PROCEDURE_ID_COLUMN, PROCEDURE_STATE_COLUMN, PROCEDURE_TRIGGER_COLUMN, }; use common_event_recorder::testing::assert_event_contract; use common_procedure::{EventTrigger, ProcedureEvent, ProcedureId, ProcedureState}; use table::table_name::TableName; -use super::test_util::assert_event_filter; +use super::test_util::{assert_event_filter, default_event_context_value, procedure_trigger_value}; use crate::ddl::create_flow::CreateFlowProcedure; use crate::ddl::drop_flow::DropFlowProcedure; use crate::ddl::event::flow::{ @@ -35,6 +35,7 @@ use crate::ddl::event::flow::{ use crate::ddl::test_util::flownode_handler::NaiveFlownodeHandler; use crate::ddl::tests::create_flow::{test_create_flow_task, test_query_context}; use crate::ddl::tests::drop_flow::test_drop_flow_task; +use crate::rpc::ddl::EventContext; use crate::test_util::{MockFlownodeManager, new_ddl_context}; #[test] @@ -48,6 +49,7 @@ fn test_flow_submitted_event_contracts() { expire_after: Some(300), eval_interval_secs: Some(60), }, + EventContext::default(), ); assert_event_contract( &create, @@ -58,6 +60,7 @@ fn test_flow_submitted_event_contracts() { ValueData::StringValue("greptime".to_string()).into(), ValueData::StringValue("metrics".to_string()).into(), Value { value_data: None }, + default_event_context_value(), ], }], ); @@ -72,7 +75,8 @@ fn test_flow_submitted_event_contracts() { }) ); - let drop = FlowDdlEvent::drop_submitted("greptime", "metrics", 42, true); + let drop = + FlowDdlEvent::drop_submitted("greptime", "metrics", 42, true, EventContext::default()); assert_event_contract( &drop, DROP_FLOW_EVENT_TYPE, @@ -82,6 +86,7 @@ fn test_flow_submitted_event_contracts() { ValueData::StringValue("greptime".to_string()).into(), ValueData::StringValue("metrics".to_string()).into(), ValueData::U32Value(42).into(), + default_event_context_value(), ], }], ); @@ -107,6 +112,7 @@ fn test_flow_lifecycle_events_have_fixed_schema_and_null_intent() { Value { value_data: None }, Value { value_data: None }, Value { value_data: None }, + Value { value_data: None }, ], }], ); @@ -123,6 +129,7 @@ fn test_flow_lifecycle_events_have_fixed_schema_and_null_intent() { Value { value_data: None }, Value { value_data: None }, ValueData::U32Value(42).into(), + Value { value_data: None }, ], }], ); @@ -142,6 +149,7 @@ fn test_flow_events_preserve_procedure_envelope_contract() { expire_after: None, eval_interval_secs: None, }, + EventContext::default(), )), ProcedureState::Running, EventTrigger::Submitted, @@ -187,6 +195,7 @@ fn test_create_flow_event_filter() { false, ), test_query_context(), + EventContext::default(), new_ddl_context(Arc::new(MockFlownodeManager::new(NaiveFlownodeHandler))), ); assert_event_filter(&procedure, CREATE_FLOW_EVENT_TYPE); @@ -196,6 +205,7 @@ fn test_create_flow_event_filter() { fn test_drop_flow_event_filter() { let procedure = DropFlowProcedure::new( test_drop_flow_task("flow", 42, false), + EventContext::default(), new_ddl_context(Arc::new(MockFlownodeManager::new(NaiveFlownodeHandler))), ); assert_event_filter(&procedure, DROP_FLOW_EVENT_TYPE); @@ -206,6 +216,7 @@ fn flow_schema() -> Vec { CATALOG_NAME_COLUMN.column_schema(), FLOW_NAME_COLUMN.column_schema(), FLOW_ID_COLUMN.column_schema(), + EVENT_CONTEXT_COLUMN.column_schema(), ] } @@ -238,7 +249,7 @@ fn assert_procedure_event_contract( ValueData::StringValue(event.procedure_id.to_string()).into(), ValueData::StringValue(state.to_string()).into(), ValueData::StringValue(String::new()).into(), - jsonb_value(&serde_json::json!({"type": trigger})), + procedure_trigger_value(trigger), optional_string(locator.catalog_name), optional_string(locator.flow_name), locator @@ -246,6 +257,11 @@ fn assert_procedure_event_contract( .map(ValueData::U32Value) .map(Into::into) .unwrap_or(Value { value_data: None }), + if locator.catalog_name.is_some() { + default_event_context_value() + } else { + Value { value_data: None } + }, ], }], ); diff --git a/src/common/meta/src/ddl/tests/event/table.rs b/src/common/meta/src/ddl/tests/event/table.rs index c0a731a20e6..90e7d83b3f2 100644 --- a/src/common/meta/src/ddl/tests/event/table.rs +++ b/src/common/meta/src/ddl/tests/event/table.rs @@ -19,13 +19,13 @@ use api::v1::value::ValueData; use api::v1::{ColumnDataType, Repartition, SemanticType, Value}; use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME}; use common_event_recorder::event_table::{ - CATALOG_NAME_COLUMN, PHYSICAL_TABLE_ID_COLUMN, SCHEMA_NAME_COLUMN, TABLE_ID_COLUMN, - TABLE_NAME_COLUMN, + CATALOG_NAME_COLUMN, EVENT_CONTEXT_COLUMN, PHYSICAL_TABLE_ID_COLUMN, SCHEMA_NAME_COLUMN, + TABLE_ID_COLUMN, TABLE_NAME_COLUMN, jsonb_value, }; use common_event_recorder::{Event, EventTypeFilter}; use common_procedure::{ - ChildSubmissionOutcome, EventContext, EventTrigger, Procedure, ProcedureId, ProcedureState, - RetryPhase, + ChildSubmissionOutcome, EventRuntimeContext, EventTrigger, Procedure, ProcedureId, + ProcedureState, RetryPhase, }; use common_time::Timestamp; use serde_json::{Value as JsonValue, json}; @@ -52,9 +52,9 @@ use crate::ddl::truncate_table::TruncateTableProcedure; use crate::ddl::undrop_table::UndropTableProcedure; use crate::key::DeserializedValueWithBytes; use crate::key::table_info::TableInfoValue; -use crate::rpc::ddl::{DropTableTask, TruncateTableTask}; +use crate::rpc::ddl::{DropTableTask, EventContext, QueryContext, TruncateTableTask}; #[cfg(feature = "enterprise")] -use crate::rpc::ddl::{PurgeDroppedTableTask, UndropTableTask}; +use crate::rpc::ddl::{PurgeDroppedTableTask, TriggerReason, UndropTableTask}; use crate::test_util::{MockDatanodeManager, new_ddl_context}; struct EventCase { @@ -177,7 +177,13 @@ fn later_lifecycle_events_are_uniform() { fn create_success_events_keep_allocated_ids() { let mut task = test_create_table_task("create_success"); task.table_info.ident.table_id = 7; - let create_table = CreateTableProcedure::new(task, test_context()).unwrap(); + let create_table = CreateTableProcedure::new( + task, + QueryContext::default(), + EventContext::default(), + test_context(), + ) + .unwrap(); let state = ProcedureState::Done { output: Some(Arc::new(42_u32)), }; @@ -196,6 +202,7 @@ fn create_success_events_keep_allocated_ids() { test_create_logical_table_task("bar"), ], 1024, + EventContext::default(), test_context(), ); let state = ProcedureState::Done { @@ -234,13 +241,14 @@ fn event_cases() -> Vec { TableDdlLocator::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, "create"), true, "mito2", + EventContext::default(), ), payload: json!({ "version": TABLE_DDL_PAYLOAD_VERSION, "create_if_not_exists": true, "engine": "mito2", }), - rows: vec![table_locator_values(Some("create"), None)], + rows: vec![submitted_table_locator_values(Some("create"), None)], }, EventCase { event_type: TableDdlEventType::CreateLogicalTables, @@ -252,14 +260,15 @@ fn event_cases() -> Vec { .with_physical_table_id(10), ], 2, + EventContext::default(), ), payload: json!({ "version": TABLE_DDL_PAYLOAD_VERSION, "table_count": 2, }), rows: vec![ - logical_locator_values("logical1", None, 10), - logical_locator_values("logical2", None, 10), + submitted_logical_locator_values("logical1", None, 10), + submitted_logical_locator_values("logical2", None, 10), ], }, EventCase { @@ -268,12 +277,13 @@ fn event_cases() -> Vec { TableDdlLocator::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, "alter") .with_table_id(11), Some("drop_columns"), + EventContext::default(), ), payload: json!({ "version": TABLE_DDL_PAYLOAD_VERSION, "kind": "drop_columns", }), - rows: vec![table_locator_values(Some("alter"), Some(11))], + rows: vec![submitted_table_locator_values(Some("alter"), Some(11))], }, EventCase { event_type: TableDdlEventType::AlterLogicalTables, @@ -286,6 +296,7 @@ fn event_cases() -> Vec { ], 2, ["rename_table", "add_columns", "add_columns"], + EventContext::default(), ), payload: json!({ "version": TABLE_DDL_PAYLOAD_VERSION, @@ -293,8 +304,8 @@ fn event_cases() -> Vec { "kinds": ["add_columns", "rename_table"], }), rows: vec![ - logical_locator_values("logical1", None, 10), - logical_locator_values("logical2", None, 10), + submitted_logical_locator_values("logical1", None, 10), + submitted_logical_locator_values("logical2", None, 10), ], }, EventCase { @@ -303,26 +314,33 @@ fn event_cases() -> Vec { TableDdlLocator::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, "drop") .with_table_id(12), true, + EventContext::default(), ), payload: json!({ "version": TABLE_DDL_PAYLOAD_VERSION, "drop_if_exists": true, }), - rows: vec![table_locator_values(Some("drop"), Some(12))], + rows: vec![submitted_table_locator_values(Some("drop"), Some(12))], }, #[cfg(feature = "enterprise")] EventCase { event_type: TableDdlEventType::UndropTable, - event: TableDdlEvent::undrop_table_submitted(TableDdlLocator::from_table_id(13)), + event: TableDdlEvent::undrop_table_submitted( + TableDdlLocator::from_table_id(13), + EventContext::default(), + ), payload: json!({"version": TABLE_DDL_PAYLOAD_VERSION}), - rows: vec![table_locator_values(None, Some(13))], + rows: vec![submitted_table_locator_values(None, Some(13))], }, #[cfg(feature = "enterprise")] EventCase { event_type: TableDdlEventType::PurgeDroppedTable, - event: TableDdlEvent::purge_dropped_table_submitted(TableDdlLocator::from_table_id(14)), + event: TableDdlEvent::purge_dropped_table_submitted( + TableDdlLocator::from_table_id(14), + EventContext::default(), + ), payload: json!({"version": TABLE_DDL_PAYLOAD_VERSION}), - rows: vec![table_locator_values(None, Some(14))], + rows: vec![submitted_table_locator_values(None, Some(14))], }, EventCase { event_type: TableDdlEventType::TruncateTable, @@ -330,29 +348,41 @@ fn event_cases() -> Vec { TableDdlLocator::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, "truncate") .with_table_id(15), 4, + EventContext::default(), ), payload: json!({ "version": TABLE_DDL_PAYLOAD_VERSION, "time_range_count": 4, }), - rows: vec![table_locator_values(Some("truncate"), Some(15))], + rows: vec![submitted_table_locator_values(Some("truncate"), Some(15))], }, ] } fn procedure_cases() -> Vec { - let create_table = - CreateTableProcedure::new(test_create_table_task("create"), test_context()).unwrap(); + let create_table = CreateTableProcedure::new( + test_create_table_task("create"), + QueryContext::default(), + EventContext::default(), + test_context(), + ) + .unwrap(); let create_logical_tables = CreateLogicalTablesProcedure::new( vec![ test_create_logical_table_task("logical1"), test_create_logical_table_task("logical2"), ], 41, + EventContext::default(), test_context(), ); - let alter_table = - AlterTableProcedure::new(42, test_alter_table_task("alter"), test_context()).unwrap(); + let alter_table = AlterTableProcedure::new( + 42, + test_alter_table_task("alter"), + EventContext::default(), + test_context(), + ) + .unwrap(); let alter_logical_tables = AlterLogicalTablesProcedure::new( vec![ make_alter_logical_table_add_column_task( @@ -367,6 +397,7 @@ fn procedure_cases() -> Vec { ), ], 43, + EventContext::default(), test_context(), ); let drop_table = DropTableProcedure::new( @@ -378,12 +409,20 @@ fn procedure_cases() -> Vec { drop_if_exists: true, }, test_context(), + EventContext::default(), ); #[cfg(feature = "enterprise")] - let undrop_table = UndropTableProcedure::new(UndropTableTask { table_id: 45 }, test_context()); + let undrop_table = UndropTableProcedure::new( + UndropTableTask { table_id: 45 }, + test_context(), + EventContext::default(), + ); #[cfg(feature = "enterprise")] - let purge_dropped_table = - PurgeDroppedTableProcedure::new(PurgeDroppedTableTask { table_id: 46 }, test_context()); + let purge_dropped_table = PurgeDroppedTableProcedure::new_if_expired( + PurgeDroppedTableTask { table_id: 46 }, + test_context(), + EventContext::new(TriggerReason::ScheduledGc), + ); let truncate_table = truncate_procedure(TruncateTableTask { catalog: DEFAULT_CATALOG_NAME.to_string(), schema: DEFAULT_SCHEMA_NAME.to_string(), @@ -404,7 +443,7 @@ fn procedure_cases() -> Vec { "create_if_not_exists": false, "engine": "mito2", }), - rows: vec![table_locator_values(Some("create"), None)], + rows: vec![submitted_table_locator_values(Some("create"), None)], }, ProcedureCase { procedure: Box::new(create_logical_tables), @@ -414,8 +453,8 @@ fn procedure_cases() -> Vec { "table_count": 2, }), rows: vec![ - logical_locator_values("logical1", None, 41), - logical_locator_values("logical2", None, 41), + submitted_logical_locator_values("logical1", None, 41), + submitted_logical_locator_values("logical2", None, 41), ], }, ProcedureCase { @@ -425,7 +464,7 @@ fn procedure_cases() -> Vec { "version": TABLE_DDL_PAYLOAD_VERSION, "kind": "drop_columns", }), - rows: vec![table_locator_values(Some("alter"), Some(42))], + rows: vec![submitted_table_locator_values(Some("alter"), Some(42))], }, ProcedureCase { procedure: Box::new(alter_logical_tables), @@ -436,8 +475,8 @@ fn procedure_cases() -> Vec { "kinds": ["add_columns"], }), rows: vec![ - logical_locator_values("logical1", None, 43), - logical_locator_values("logical2", None, 43), + submitted_logical_locator_values("logical1", None, 43), + submitted_logical_locator_values("logical2", None, 43), ], }, ProcedureCase { @@ -447,21 +486,25 @@ fn procedure_cases() -> Vec { "version": TABLE_DDL_PAYLOAD_VERSION, "drop_if_exists": true, }), - rows: vec![table_locator_values(Some("drop"), Some(44))], + rows: vec![submitted_table_locator_values(Some("drop"), Some(44))], }, #[cfg(feature = "enterprise")] ProcedureCase { procedure: Box::new(undrop_table), event_type: "undrop_table", payload: json!({"version": TABLE_DDL_PAYLOAD_VERSION}), - rows: vec![table_locator_values(None, Some(45))], + rows: vec![submitted_table_locator_values(None, Some(45))], }, #[cfg(feature = "enterprise")] ProcedureCase { procedure: Box::new(purge_dropped_table), event_type: "purge_dropped_table", payload: json!({"version": TABLE_DDL_PAYLOAD_VERSION}), - rows: vec![table_locator_values(None, Some(46))], + rows: vec![submitted_table_locator_values_with_event_context( + None, + Some(46), + EventContext::new(TriggerReason::ScheduledGc), + )], }, ProcedureCase { procedure: Box::new(truncate_table), @@ -470,7 +513,7 @@ fn procedure_cases() -> Vec { "version": TABLE_DDL_PAYLOAD_VERSION, "time_range_count": 1, }), - rows: vec![table_locator_values(Some("truncate"), Some(47))], + rows: vec![submitted_table_locator_values(Some("truncate"), Some(47))], }, ] } @@ -491,6 +534,7 @@ fn expected_schema(event_type: TableDdlEventType) -> Vec<(&'static str, i32)> { ColumnDataType::Uint32 as i32, )); } + schema.push((EVENT_CONTEXT_COLUMN.name(), ColumnDataType::Binary as i32)); schema } @@ -503,12 +547,14 @@ fn table_locator_values(table_name: Option<&str>, table_id: Option) -> Vec< } else { (Value::default(), Value::default()) }; - vec![ + let mut values = vec![ catalog_name, schema_name, table_name.map(string_value).unwrap_or_default(), table_id.map(table_id_value).unwrap_or_default(), - ] + ]; + values.push(Value::default()); + values } fn logical_locator_values( @@ -517,10 +563,44 @@ fn logical_locator_values( physical_table_id: u32, ) -> Vec { let mut values = table_locator_values(Some(table_name), table_id); + let event_context = values.pop().unwrap(); values.push(table_id_value(physical_table_id)); + values.push(event_context); values } +fn submitted_table_locator_values(table_name: Option<&str>, table_id: Option) -> Vec { + submitted_table_locator_values_with_event_context(table_name, table_id, EventContext::default()) +} + +fn submitted_table_locator_values_with_event_context( + table_name: Option<&str>, + table_id: Option, + event_context: EventContext, +) -> Vec { + let mut values = table_locator_values(table_name, table_id); + *values.last_mut().unwrap() = event_context_value_for(event_context); + values +} + +fn submitted_logical_locator_values( + table_name: &str, + table_id: Option, + physical_table_id: u32, +) -> Vec { + let mut values = logical_locator_values(table_name, table_id, physical_table_id); + *values.last_mut().unwrap() = event_context_value(); + values +} + +fn event_context_value() -> Value { + event_context_value_for(EventContext::default()) +} + +fn event_context_value_for(event_context: EventContext) -> Value { + jsonb_value(&serde_json::to_value(event_context).unwrap()) +} + fn string_value(value: &str) -> Value { ValueData::StringValue(value.to_string()).into() } @@ -535,6 +615,7 @@ fn truncate_procedure(task: TruncateTableTask) -> TruncateTableProcedure { task, DeserializedValueWithBytes::from_inner(TableInfoValue::new(table_info)), test_context(), + EventContext::default(), ) } @@ -552,7 +633,7 @@ fn event_for_state( lifecycle_state: &ProcedureState, ) -> Box { procedure - .event(&EventContext { + .event(&EventRuntimeContext { procedure_id: ProcedureId::random(), lifecycle_state, trigger, diff --git a/src/common/meta/src/ddl/tests/event/test_util.rs b/src/common/meta/src/ddl/tests/event/test_util.rs index 884fa3ec4d4..281b5deb488 100644 --- a/src/common/meta/src/ddl/tests/event/test_util.rs +++ b/src/common/meta/src/ddl/tests/event/test_util.rs @@ -15,12 +15,24 @@ use std::collections::HashSet; use std::sync::Arc; +use api::v1::Value; use common_event_recorder::EventTypeFilter; -use common_procedure::{EventContext, EventTrigger, Procedure, ProcedureId, ProcedureState}; +use common_event_recorder::event_table::jsonb_value; +use common_procedure::{EventRuntimeContext, EventTrigger, Procedure, ProcedureId, ProcedureState}; + +use crate::rpc::ddl::EventContext; + +pub(crate) fn default_event_context_value() -> Value { + jsonb_value(&serde_json::to_value(EventContext::default()).unwrap()) +} + +pub(crate) fn procedure_trigger_value(trigger: &str) -> Value { + jsonb_value(&serde_json::json!({"type": trigger})) +} pub(crate) fn assert_event_filter(procedure: &dyn Procedure, event_type: &str) { let state = ProcedureState::Running; - let event_context = |event_type_filter| EventContext { + let runtime_context = |event_type_filter| EventRuntimeContext { procedure_id: ProcedureId::random(), lifecycle_state: &state, trigger: EventTrigger::Submitted, @@ -28,7 +40,7 @@ pub(crate) fn assert_event_filter(procedure: &dyn Procedure, event_type: &str) { }; let allowed = procedure - .event(&event_context(EventTypeFilter::Only(HashSet::from([ + .event(&runtime_context(EventTypeFilter::Only(HashSet::from([ event_type.to_string(), ])))) .unwrap(); @@ -37,7 +49,7 @@ pub(crate) fn assert_event_filter(procedure: &dyn Procedure, event_type: &str) { for denied in [HashSet::from(["other_event".to_string()]), HashSet::new()] { assert!( procedure - .event(&event_context(EventTypeFilter::Only(denied))) + .event(&runtime_context(EventTypeFilter::Only(denied))) .is_none() ); } diff --git a/src/common/meta/src/ddl/tests/event/view.rs b/src/common/meta/src/ddl/tests/event/view.rs index 024aa023c87..10947e45ee8 100644 --- a/src/common/meta/src/ddl/tests/event/view.rs +++ b/src/common/meta/src/ddl/tests/event/view.rs @@ -17,17 +17,18 @@ use std::sync::Arc; use api::v1::value::ValueData; use api::v1::{ColumnSchema, Row, Value}; use common_event_recorder::event_table::{ - CATALOG_NAME_COLUMN, PROCEDURE_ERROR_COLUMN, PROCEDURE_ID_COLUMN, PROCEDURE_STATE_COLUMN, - PROCEDURE_TRIGGER_COLUMN, SCHEMA_NAME_COLUMN, VIEW_ID_COLUMN, VIEW_NAME_COLUMN, jsonb_value, + CATALOG_NAME_COLUMN, EVENT_CONTEXT_COLUMN, PROCEDURE_ERROR_COLUMN, PROCEDURE_ID_COLUMN, + PROCEDURE_STATE_COLUMN, PROCEDURE_TRIGGER_COLUMN, SCHEMA_NAME_COLUMN, VIEW_ID_COLUMN, + VIEW_NAME_COLUMN, }; use common_event_recorder::testing::assert_event_contract; use common_event_recorder::{Event, EventTypeFilter}; use common_procedure::{ - ChildSubmissionOutcome, EventContext, EventTrigger, Output, Procedure, ProcedureEvent, + ChildSubmissionOutcome, EventRuntimeContext, EventTrigger, Output, Procedure, ProcedureEvent, ProcedureId, ProcedureState, RetryPhase, }; -use super::test_util::assert_event_filter; +use super::test_util::{assert_event_filter, default_event_context_value, procedure_trigger_value}; use crate::ddl::create_view::CreateViewProcedure; use crate::ddl::drop_view::DropViewProcedure; use crate::ddl::event::view::{ @@ -35,6 +36,7 @@ use crate::ddl::event::view::{ }; use crate::ddl::tests::create_view::test_create_view_task; use crate::ddl::tests::drop_view::new_drop_view_task; +use crate::rpc::ddl::EventContext; use crate::test_util::{MockDatanodeManager, new_ddl_context}; #[test] @@ -42,7 +44,7 @@ fn test_view_submitted_event_contracts() { let mut task = test_create_view_task("v_metrics"); task.create_view.or_replace = true; task.create_view.create_if_not_exists = true; - let create = CreateViewProcedure::new(task, test_context()); + let create = CreateViewProcedure::new(task, EventContext::default(), test_context()); let event = event_for(&create, EventTrigger::Submitted); assert_view_event_contract( @@ -70,7 +72,11 @@ fn test_view_submitted_event_contracts() { assert!(!payload.contains(omitted)); } - let drop = DropViewProcedure::new(new_drop_view_task("view_name", 42, true), test_context()); + let drop = DropViewProcedure::new( + new_drop_view_task("view_name", 42, true), + EventContext::default(), + test_context(), + ); let event = event_for(&drop, EventTrigger::Submitted); assert_view_event_contract( @@ -116,8 +122,16 @@ fn test_view_lifecycle_event_contracts() { #[test] fn test_view_procedures_emit_lightweight_lifecycle_events() { - let create = CreateViewProcedure::new(test_create_view_task("view_name"), test_context()); - let drop = DropViewProcedure::new(new_drop_view_task("view_name", 42, false), test_context()); + let create = CreateViewProcedure::new( + test_create_view_task("view_name"), + EventContext::default(), + test_context(), + ); + let drop = DropViewProcedure::new( + new_drop_view_task("view_name", 42, false), + EventContext::default(), + test_context(), + ); let triggers = [ EventTrigger::Recovered, EventTrigger::ChildSubmitted { @@ -149,7 +163,11 @@ fn test_view_procedures_emit_lightweight_lifecycle_events() { #[test] fn test_create_view_succeeded_output_mapping() { - let procedure = CreateViewProcedure::new(test_create_view_task("view_name"), test_context()); + let procedure = CreateViewProcedure::new( + test_create_view_task("view_name"), + EventContext::default(), + test_context(), + ); let state = ProcedureState::Done { output: Some(Arc::new(84_u32)), }; @@ -175,14 +193,21 @@ fn test_create_view_succeeded_output_mapping() { #[test] fn test_create_view_event_filter() { - let procedure = CreateViewProcedure::new(test_create_view_task("view_name"), test_context()); + let procedure = CreateViewProcedure::new( + test_create_view_task("view_name"), + EventContext::default(), + test_context(), + ); assert_event_filter(&procedure, CREATE_VIEW_EVENT_TYPE); } #[test] fn test_drop_view_event_filter() { - let procedure = - DropViewProcedure::new(new_drop_view_task("view_name", 42, false), test_context()); + let procedure = DropViewProcedure::new( + new_drop_view_task("view_name", 42, false), + EventContext::default(), + test_context(), + ); assert_event_filter(&procedure, DROP_VIEW_EVENT_TYPE); } @@ -201,6 +226,7 @@ fn test_view_event_procedure_envelope_contract() { referenced_table_count: 1, column_count: 1, }, + EventContext::default(), )), ProcedureState::Running, EventTrigger::Submitted, @@ -254,6 +280,11 @@ impl ViewEventLocator<'_> { .map(ValueData::U32Value) .map(Into::into) .unwrap_or_default(), + if self.catalog_name.is_some() { + default_event_context_value() + } else { + Value { value_data: None } + }, ] } } @@ -264,6 +295,7 @@ fn view_schema() -> Vec { SCHEMA_NAME_COLUMN.column_schema(), VIEW_NAME_COLUMN.column_schema(), VIEW_ID_COLUMN.column_schema(), + EVENT_CONTEXT_COLUMN.column_schema(), ] } @@ -302,7 +334,7 @@ fn assert_procedure_event_contract( ValueData::StringValue(event.procedure_id.to_string()).into(), ValueData::StringValue(state.to_string()).into(), ValueData::StringValue(String::new()).into(), - jsonb_value(&serde_json::json!({"type": trigger})), + procedure_trigger_value(trigger), ]; values.extend(locator.values()); @@ -323,7 +355,7 @@ fn event_for_state( lifecycle_state: &ProcedureState, ) -> Box { procedure - .event(&EventContext { + .event(&EventRuntimeContext { procedure_id: ProcedureId::random(), lifecycle_state, trigger, diff --git a/src/common/meta/src/ddl/truncate_table.rs b/src/common/meta/src/ddl/truncate_table.rs index 454b7e2de45..9d0ed09e8c8 100644 --- a/src/common/meta/src/ddl/truncate_table.rs +++ b/src/common/meta/src/ddl/truncate_table.rs @@ -20,7 +20,7 @@ use api::v1::region::{ use async_trait::async_trait; use common_procedure::error::{FromJsonSnafu, ToJsonSnafu}; use common_procedure::{ - Context as ProcedureContext, EventContext, EventTrigger, LockKey, Procedure, + Context as ProcedureContext, EventRuntimeContext, EventTrigger, LockKey, Procedure, Result as ProcedureResult, Status, }; use common_telemetry::debug; @@ -43,7 +43,7 @@ use crate::key::table_info::TableInfoValue; use crate::key::table_name::TableNameKey; use crate::lock_key::{CatalogLock, SchemaLock, TableLock}; use crate::metrics; -use crate::rpc::ddl::TruncateTableTask; +use crate::rpc::ddl::{EventContext, TruncateTableTask}; use crate::rpc::router::{find_leader_regions, find_leaders}; pub struct TruncateTableProcedure { @@ -89,7 +89,10 @@ impl Procedure for TruncateTableProcedure { LockKey::new(lock_key) } - fn event(&self, ctx: &EventContext<'_>) -> Option> { + fn event( + &self, + ctx: &EventRuntimeContext<'_>, + ) -> Option> { if !ctx .event_type_filter .allows(TableDdlEventType::TruncateTable.as_str()) @@ -101,7 +104,11 @@ impl Procedure for TruncateTableProcedure { let task = &self.data.task; let locator = TableDdlLocator::new(&task.catalog, &task.schema, &task.table) .with_table_id(task.table_id); - TableDdlEvent::truncate_table_submitted(locator, task.time_ranges.len()) + TableDdlEvent::truncate_table_submitted( + locator, + task.time_ranges.len(), + self.data.event_context.clone(), + ) } _ => TableDdlEvent::lifecycle(TableDdlEventType::TruncateTable), }; @@ -117,10 +124,11 @@ impl TruncateTableProcedure { task: TruncateTableTask, table_info_value: DeserializedValueWithBytes, context: DdlContext, + event_context: EventContext, ) -> Self { Self { context, - data: TruncateTableData::new(task, table_info_value), + data: TruncateTableData::new(task, table_info_value, event_context), } } @@ -227,17 +235,21 @@ pub struct TruncateTableData { state: TruncateTableState, task: TruncateTableTask, table_info_value: DeserializedValueWithBytes, + #[serde(default)] + event_context: EventContext, } impl TruncateTableData { pub fn new( task: TruncateTableTask, table_info_value: DeserializedValueWithBytes, + event_context: EventContext, ) -> Self { Self { state: TruncateTableState::Prepare, task, table_info_value, + event_context, } } diff --git a/src/common/meta/src/ddl/undrop_table.rs b/src/common/meta/src/ddl/undrop_table.rs index 816ba3cd9b3..635d15ee6a5 100644 --- a/src/common/meta/src/ddl/undrop_table.rs +++ b/src/common/meta/src/ddl/undrop_table.rs @@ -17,7 +17,7 @@ use api::v1::region::{ use async_trait::async_trait; use common_procedure::error::{FromJsonSnafu, ToJsonSnafu}; use common_procedure::{ - Context as ProcedureContext, EventContext, EventTrigger, LockKey, Procedure, + Context as ProcedureContext, EventRuntimeContext, EventTrigger, LockKey, Procedure, Result as ProcedureResult, Status, }; use common_telemetry::tracing_context::TracingContext; @@ -43,7 +43,7 @@ use crate::instruction::CacheIdent; use crate::key::table_name::TableNameKey; use crate::key::table_route::TableRouteValue; use crate::lock_key::{CatalogLock, SchemaLock, TableLock, TableNameLock}; -use crate::rpc::ddl::UndropTableTask; +use crate::rpc::ddl::{EventContext, UndropTableTask}; use crate::rpc::router::{ RegionRoute, find_follower_regions, find_followers, find_leader_regions, find_leaders, }; @@ -56,16 +56,17 @@ pub struct UndropTableProcedure { impl UndropTableProcedure { pub const TYPE_NAME: &'static str = "metasrv-procedure::UndropTable"; - pub fn new(task: UndropTableTask, context: DdlContext) -> Self { - Self::new_with_original_table_name(task, context, None) + pub fn new(task: UndropTableTask, context: DdlContext, event_context: EventContext) -> Self { + Self::new_with_original_table_name(task, context, None, event_context) } pub(crate) fn new_with_original_table_name( task: UndropTableTask, context: DdlContext, table_name: Option, + event_context: EventContext, ) -> Self { - let mut data = UndropTableData::new(task); + let mut data = UndropTableData::new(task, event_context); data.table_name = table_name; Self { context, data } } @@ -426,7 +427,10 @@ impl Procedure for UndropTableProcedure { LockKey::new(lock_key) } - fn event(&self, ctx: &EventContext<'_>) -> Option> { + fn event( + &self, + ctx: &EventRuntimeContext<'_>, + ) -> Option> { if !ctx .event_type_filter .allows(TableDdlEventType::UndropTable.as_str()) @@ -448,7 +452,7 @@ impl Procedure for UndropTableProcedure { }) .unwrap_or_default() .with_table_id(self.data.task.table_id); - TableDdlEvent::undrop_table_submitted(locator) + TableDdlEvent::undrop_table_submitted(locator, self.data.event_context.clone()) } _ => TableDdlEvent::lifecycle(TableDdlEventType::UndropTable), }; @@ -556,10 +560,12 @@ pub struct UndropTableData { drop_generation: Option, #[serde(default)] tombstone_identity_loaded: bool, + #[serde(default)] + event_context: EventContext, } impl UndropTableData { - fn new(task: UndropTableTask) -> Self { + fn new(task: UndropTableTask, event_context: EventContext) -> Self { Self { state: UndropTableState::Prepare, task, @@ -571,6 +577,7 @@ impl UndropTableData { retention_expires_at: None, drop_generation: None, tombstone_identity_loaded: false, + event_context, } } @@ -635,7 +642,11 @@ mod tests { #[test] fn test_map_restore_metadata_error_without_table_name() { let context = new_ddl_context(Arc::new(MockDatanodeManager::new(()))); - let procedure = UndropTableProcedure::new(UndropTableTask { table_id: 42 }, context); + let procedure = UndropTableProcedure::new( + UndropTableTask { table_id: 42 }, + context, + EventContext::default(), + ); let err = procedure.map_restore_metadata_error( error::TombstoneTargetAlreadyExistsSnafu { @@ -650,8 +661,11 @@ mod tests { #[test] fn test_recovered_legacy_snapshot_reloads_tombstone_identity() { let context = new_ddl_context(Arc::new(MockDatanodeManager::new(()))); - let mut procedure = - UndropTableProcedure::new(UndropTableTask { table_id: 42 }, context.clone()); + let mut procedure = UndropTableProcedure::new( + UndropTableTask { table_id: 42 }, + context.clone(), + EventContext::default(), + ); procedure.data.state = UndropTableState::OpenRegions; let mut data: serde_json::Value = serde_json::from_str(&procedure.dump().unwrap()).unwrap(); let data = data.as_object_mut().unwrap(); @@ -676,7 +690,11 @@ mod tests { let table_id = 1024; let region_id = RegionId::new(table_id, 1); - let mut procedure = UndropTableProcedure::new(UndropTableTask { table_id }, context); + let mut procedure = UndropTableProcedure::new( + UndropTableTask { table_id }, + context, + EventContext::default(), + ); procedure.data.table_route_value = Some(TableRouteValue::physical(vec![RegionRoute { region: Region::new_test(region_id), leader_peer: Some(Peer::empty(1)), @@ -715,7 +733,11 @@ mod tests { let table_id = 1024; let region_id = RegionId::new(table_id, 1); - let mut procedure = UndropTableProcedure::new(UndropTableTask { table_id }, context); + let mut procedure = UndropTableProcedure::new( + UndropTableTask { table_id }, + context, + EventContext::default(), + ); procedure.data.table_name = Some(TableName::new( DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, diff --git a/src/common/meta/src/ddl_manager.rs b/src/common/meta/src/ddl_manager.rs index e333b3ccb7f..137a16793d7 100644 --- a/src/common/meta/src/ddl_manager.rs +++ b/src/common/meta/src/ddl_manager.rs @@ -70,13 +70,15 @@ use crate::rpc::ddl::DdlTask::{ DropTable, DropView, PurgeDroppedTable, TruncateTable, UndropTable, }; #[cfg(feature = "enterprise")] +use crate::rpc::ddl::TriggerReason; +#[cfg(feature = "enterprise")] use crate::rpc::ddl::trigger::CreateTriggerTask; #[cfg(feature = "enterprise")] use crate::rpc::ddl::trigger::DropTriggerTask; use crate::rpc::ddl::{ AlterDatabaseTask, AlterTableTask, CommentOnTask, CreateDatabaseTask, CreateFlowTask, CreateTableTask, CreateViewTask, DropDatabaseTask, DropFlowTask, DropTableTask, DropViewTask, - PurgeDroppedTableTask, QueryContext, SubmitDdlTaskRequest, SubmitDdlTaskResponse, + EventContext, PurgeDroppedTableTask, QueryContext, SubmitDdlTaskRequest, SubmitDdlTaskResponse, TruncateTableTask, UndropTableTask, }; @@ -177,6 +179,7 @@ pub enum RepartitionSource { #[async_trait::async_trait] pub trait RepartitionProcedureFactory: Send + Sync { + #[allow(clippy::too_many_arguments)] fn create( &self, ddl_ctx: &DdlContext, @@ -185,6 +188,7 @@ pub trait RepartitionProcedureFactory: Send + Sync { source: RepartitionSource, to_exprs: Vec, timeout: Option, + event_context: EventContext, ) -> std::result::Result; fn register_loaders( @@ -338,6 +342,7 @@ impl DdlManager { repartition: Repartition, wait: bool, timeout: Duration, + event_context: EventContext, ) -> Result<(ProcedureId, Option)> { let context = self.create_context(); @@ -373,6 +378,7 @@ impl DdlManager { source, into_partition_exprs, Some(timeout), + event_context, ) .context(CreateRepartitionProcedureSnafu)?; self.repartition_procedure_factory @@ -395,6 +401,7 @@ impl DdlManager { &self, table_id: TableId, alter_table_task: AlterTableTask, + event_context: EventContext, ddl_options: DdlOptions, ) -> Result<(ProcedureId, Option)> { // make alter_table_task mutable so we can call .take() on its field @@ -415,6 +422,7 @@ impl DdlManager { repartition, ddl_options.wait, ddl_options.timeout, + event_context, ) .await; } @@ -448,6 +456,7 @@ impl DdlManager { let procedure = AlterTableProcedure::new_with_region_locks( table_id, alter_table_task.clone(), + event_context.clone(), region_ids_to_lock, context, )?; @@ -481,14 +490,12 @@ impl DdlManager { &self, create_table_task: CreateTableTask, query_context: QueryContext, + event_context: EventContext, ) -> Result<(ProcedureId, Option)> { let context = self.create_context(); - let procedure = CreateTableProcedure::new_with_query_context( - create_table_task, - query_context, - context, - )?; + let procedure = + CreateTableProcedure::new(create_table_task, query_context, event_context, context)?; let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure)); @@ -500,10 +507,11 @@ impl DdlManager { pub async fn submit_create_view_task( &self, create_view_task: CreateViewTask, + event_context: EventContext, ) -> Result<(ProcedureId, Option)> { let context = self.create_context(); - let procedure = CreateViewProcedure::new(create_view_task, context); + let procedure = CreateViewProcedure::new(create_view_task, event_context, context); let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure)); @@ -516,11 +524,16 @@ impl DdlManager { &self, create_table_tasks: Vec, physical_table_id: TableId, + event_context: EventContext, ) -> Result<(ProcedureId, Option)> { let context = self.create_context(); - let procedure = - CreateLogicalTablesProcedure::new(create_table_tasks, physical_table_id, context); + let procedure = CreateLogicalTablesProcedure::new( + create_table_tasks, + physical_table_id, + event_context, + context, + ); let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure)); @@ -533,11 +546,16 @@ impl DdlManager { &self, alter_table_tasks: Vec, physical_table_id: TableId, + event_context: EventContext, ) -> Result<(ProcedureId, Option)> { let context = self.create_context(); - let procedure = - AlterLogicalTablesProcedure::new(alter_table_tasks, physical_table_id, context); + let procedure = AlterLogicalTablesProcedure::new( + alter_table_tasks, + physical_table_id, + event_context, + context, + ); let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure)); @@ -549,10 +567,11 @@ impl DdlManager { pub async fn submit_drop_table_task( &self, drop_table_task: DropTableTask, + event_context: EventContext, ) -> Result<(ProcedureId, Option)> { let context = self.create_context(); - let procedure = DropTableProcedure::new(drop_table_task, context); + let procedure = DropTableProcedure::new(drop_table_task, context, event_context); let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure)); @@ -565,6 +584,7 @@ impl DdlManager { pub async fn submit_undrop_table_task( &self, undrop_table_task: UndropTableTask, + event_context: EventContext, ) -> Result<(ProcedureId, Option)> { #[cfg(not(feature = "enterprise"))] { @@ -590,6 +610,7 @@ impl DdlManager { undrop_table_task, context, Some(original_table_name), + event_context, ); let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure)); @@ -603,6 +624,7 @@ impl DdlManager { pub async fn submit_purge_dropped_table_task( &self, purge_dropped_table_task: PurgeDroppedTableTask, + event_context: EventContext, ) -> Result<(ProcedureId, Option)> { #[cfg(not(feature = "enterprise"))] { @@ -616,7 +638,8 @@ impl DdlManager { #[cfg(feature = "enterprise")] { let context = self.create_context(); - let procedure = PurgeDroppedTableProcedure::new(purge_dropped_table_task, context); + let procedure = + PurgeDroppedTableProcedure::new(purge_dropped_table_task, context, event_context); let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure)); self.execute_procedure_and_wait(procedure_with_id).await @@ -643,8 +666,11 @@ impl DdlManager { #[cfg(feature = "enterprise")] { let context = self.create_context(); - let procedure = - PurgeDroppedTableProcedure::new_if_expired(purge_dropped_table_task, context); + let procedure = PurgeDroppedTableProcedure::new_if_expired( + purge_dropped_table_task, + context, + EventContext::new(TriggerReason::ScheduledGc), + ); let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure)); self.execute_procedure_and_wait(procedure_with_id).await @@ -662,6 +688,7 @@ impl DdlManager { options, creator, }: CreateDatabaseTask, + event_context: EventContext, ) -> Result<(ProcedureId, Option)> { let context = self.create_context(); let procedure = CreateDatabaseProcedure::new( @@ -670,6 +697,7 @@ impl DdlManager { create_if_not_exists, options, creator, + event_context, context, ); let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure)); @@ -686,9 +714,11 @@ impl DdlManager { schema, drop_if_exists, }: DropDatabaseTask, + event_context: EventContext, ) -> Result<(ProcedureId, Option)> { let context = self.create_context(); - let procedure = DropDatabaseProcedure::new(catalog, schema, drop_if_exists, context); + let procedure = + DropDatabaseProcedure::new(catalog, schema, drop_if_exists, event_context, context); let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure)); self.execute_procedure_and_wait(procedure_with_id).await @@ -697,9 +727,10 @@ impl DdlManager { pub async fn submit_alter_database( &self, alter_database_task: AlterDatabaseTask, + event_context: EventContext, ) -> Result<(ProcedureId, Option)> { let context = self.create_context(); - let procedure = AlterDatabaseProcedure::new(alter_database_task, context)?; + let procedure = AlterDatabaseProcedure::new(alter_database_task, event_context, context)?; let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure)); self.execute_procedure_and_wait(procedure_with_id).await @@ -711,9 +742,11 @@ impl DdlManager { &self, create_flow: CreateFlowTask, query_context: QueryContext, + event_context: EventContext, ) -> Result<(ProcedureId, Option)> { let context = self.create_context(); - let procedure = CreateFlowProcedure::new(create_flow, query_context, context); + let procedure = + CreateFlowProcedure::new(create_flow, query_context, event_context, context); let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure)); self.execute_procedure_and_wait(procedure_with_id).await @@ -724,9 +757,10 @@ impl DdlManager { pub async fn submit_drop_flow_task( &self, drop_flow: DropFlowTask, + event_context: EventContext, ) -> Result<(ProcedureId, Option)> { let context = self.create_context(); - let procedure = DropFlowProcedure::new(drop_flow, context); + let procedure = DropFlowProcedure::new(drop_flow, event_context, context); let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure)); self.execute_procedure_and_wait(procedure_with_id).await @@ -737,9 +771,10 @@ impl DdlManager { pub async fn submit_drop_view_task( &self, drop_view: DropViewTask, + event_context: EventContext, ) -> Result<(ProcedureId, Option)> { let context = self.create_context(); - let procedure = DropViewProcedure::new(drop_view, context); + let procedure = DropViewProcedure::new(drop_view, event_context, context); let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure)); self.execute_procedure_and_wait(procedure_with_id).await @@ -751,9 +786,15 @@ impl DdlManager { &self, truncate_table_task: TruncateTableTask, table_info_value: DeserializedValueWithBytes, + event_context: EventContext, ) -> Result<(ProcedureId, Option)> { let context = self.create_context(); - let procedure = TruncateTableProcedure::new(truncate_table_task, table_info_value, context); + let procedure = TruncateTableProcedure::new( + truncate_table_task, + table_info_value, + context, + event_context, + ); let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure)); @@ -822,62 +863,75 @@ impl DdlManager { .map(TracingContext::from_w3c) .unwrap_or_else(TracingContext::from_current_span) .attach(tracing::info_span!("DdlManager::submit_ddl_task")); - let ddl_options = DdlOptions { - wait: request.wait, - timeout: request.timeout, - }; + let SubmitDdlTaskRequest { + query_context, + wait, + timeout, + task, + } = request; + let event_context = EventContext::from_query_context(&query_context); + let ddl_options = DdlOptions { wait, timeout }; async move { - debug!("Submitting Ddl task: {:?}", request.task); - match request.task { + debug!("Submitting Ddl task: {:?}", task); + match task { CreateTable(create_table_task) => { - handle_create_table_task(self, create_table_task, request.query_context).await + handle_create_table_task(self, create_table_task, query_context, event_context) + .await + } + DropTable(drop_table_task) => { + handle_drop_table_task(self, drop_table_task, event_context).await } - DropTable(drop_table_task) => handle_drop_table_task(self, drop_table_task).await, UndropTable(undrop_table_task) => { - handle_undrop_table_task(self, undrop_table_task).await + handle_undrop_table_task(self, undrop_table_task, event_context).await } PurgeDroppedTable(purge_dropped_table_task) => { - handle_purge_dropped_table_task(self, purge_dropped_table_task).await + handle_purge_dropped_table_task(self, purge_dropped_table_task, event_context) + .await } AlterTable(alter_table_task) => { - handle_alter_table_task(self, alter_table_task, ddl_options).await + handle_alter_table_task(self, alter_table_task, ddl_options, event_context) + .await } TruncateTable(truncate_table_task) => { - handle_truncate_table_task(self, truncate_table_task).await + handle_truncate_table_task(self, truncate_table_task, event_context).await } CreateLogicalTables(create_table_tasks) => { - handle_create_logical_table_tasks(self, create_table_tasks).await + handle_create_logical_table_tasks(self, create_table_tasks, event_context).await } AlterLogicalTables(alter_table_tasks) => { - handle_alter_logical_table_tasks(self, alter_table_tasks).await + handle_alter_logical_table_tasks(self, alter_table_tasks, event_context).await } DropLogicalTables(_) => todo!(), CreateDatabase(create_database_task) => { - handle_create_database_task(self, create_database_task).await + handle_create_database_task(self, create_database_task, event_context).await } DropDatabase(drop_database_task) => { - handle_drop_database_task(self, drop_database_task).await + handle_drop_database_task(self, drop_database_task, event_context).await } AlterDatabase(alter_database_task) => { - handle_alter_database_task(self, alter_database_task).await + handle_alter_database_task(self, alter_database_task, event_context).await } CreateFlow(create_flow_task) => { - handle_create_flow_task(self, create_flow_task, request.query_context).await + handle_create_flow_task(self, create_flow_task, query_context, event_context) + .await + } + DropFlow(drop_flow_task) => { + handle_drop_flow_task(self, drop_flow_task, event_context).await } - DropFlow(drop_flow_task) => handle_drop_flow_task(self, drop_flow_task).await, CreateView(create_view_task) => { - handle_create_view_task(self, create_view_task).await + handle_create_view_task(self, create_view_task, event_context).await + } + DropView(drop_view_task) => { + handle_drop_view_task(self, drop_view_task, event_context).await } - DropView(drop_view_task) => handle_drop_view_task(self, drop_view_task).await, CommentOn(comment_on_task) => handle_comment_on_task(self, comment_on_task).await, #[cfg(feature = "enterprise")] CreateTrigger(create_trigger_task) => { - handle_create_trigger_task(self, create_trigger_task, request.query_context) - .await + handle_create_trigger_task(self, create_trigger_task, query_context).await } #[cfg(feature = "enterprise")] DropTrigger(drop_trigger_task) => { - handle_drop_trigger_task(self, drop_trigger_task, request.query_context).await + handle_drop_trigger_task(self, drop_trigger_task, query_context).await } } } @@ -889,6 +943,7 @@ impl DdlManager { async fn handle_truncate_table_task( ddl_manager: &DdlManager, truncate_table_task: TruncateTableTask, + event_context: EventContext, ) -> Result { let table_id = truncate_table_task.table_id; let table_metadata_manager = &ddl_manager.table_metadata_manager(); @@ -913,7 +968,7 @@ async fn handle_truncate_table_task( ); let (id, _) = ddl_manager - .submit_truncate_table_task(truncate_table_task, table_info_value) + .submit_truncate_table_task(truncate_table_task, table_info_value, event_context) .await?; info!("Table: {table_id} is truncated via procedure_id {id:?}"); @@ -928,6 +983,7 @@ async fn handle_alter_table_task( ddl_manager: &DdlManager, alter_table_task: AlterTableTask, ddl_options: DdlOptions, + event_context: EventContext, ) -> Result { let table_ref = alter_table_task.table_ref(); @@ -960,7 +1016,7 @@ async fn handle_alter_table_task( ); let (id, _) = ddl_manager - .submit_alter_table_task(table_id, alter_table_task, ddl_options) + .submit_alter_table_task(table_id, alter_table_task, event_context, ddl_options) .await?; info!("Table: {table_id} is altered via procedure_id {id:?}"); @@ -974,9 +1030,12 @@ async fn handle_alter_table_task( async fn handle_drop_table_task( ddl_manager: &DdlManager, drop_table_task: DropTableTask, + event_context: EventContext, ) -> Result { let table_id = drop_table_task.table_id; - let (id, _) = ddl_manager.submit_drop_table_task(drop_table_task).await?; + let (id, _) = ddl_manager + .submit_drop_table_task(drop_table_task, event_context) + .await?; info!("Table: {table_id} is dropped via procedure_id {id:?}"); @@ -989,10 +1048,11 @@ async fn handle_drop_table_task( async fn handle_undrop_table_task( ddl_manager: &DdlManager, undrop_table_task: UndropTableTask, + event_context: EventContext, ) -> Result { let table_id = undrop_table_task.table_id; let (id, _) = ddl_manager - .submit_undrop_table_task(undrop_table_task) + .submit_undrop_table_task(undrop_table_task, event_context) .await?; info!("Table: {table_id} is undropped via procedure_id {id:?}"); @@ -1006,9 +1066,10 @@ async fn handle_undrop_table_task( async fn handle_purge_dropped_table_task( ddl_manager: &DdlManager, purge_dropped_table_task: PurgeDroppedTableTask, + event_context: EventContext, ) -> Result { let (id, _) = ddl_manager - .submit_purge_dropped_table_task(purge_dropped_table_task) + .submit_purge_dropped_table_task(purge_dropped_table_task, event_context) .await?; info!("Dropped table is purged via procedure_id {id:?}"); @@ -1023,9 +1084,10 @@ async fn handle_create_table_task( ddl_manager: &DdlManager, create_table_task: CreateTableTask, query_context: QueryContext, + event_context: EventContext, ) -> Result { let (id, output) = ddl_manager - .submit_create_table_task(create_table_task, query_context) + .submit_create_table_task(create_table_task, query_context, event_context) .await?; let procedure_id = id.to_string(); @@ -1048,6 +1110,7 @@ async fn handle_create_table_task( async fn handle_create_logical_table_tasks( ddl_manager: &DdlManager, create_table_tasks: Vec, + event_context: EventContext, ) -> Result { ensure!( !create_table_tasks.is_empty(), @@ -1063,7 +1126,7 @@ async fn handle_create_logical_table_tasks( let num_logical_tables = create_table_tasks.len(); let (id, output) = ddl_manager - .submit_create_logical_table_tasks(create_table_tasks, physical_table_id) + .submit_create_logical_table_tasks(create_table_tasks, physical_table_id, event_context) .await?; info!( @@ -1092,11 +1155,12 @@ async fn handle_create_logical_table_tasks( async fn handle_create_database_task( ddl_manager: &DdlManager, create_database_task: CreateDatabaseTask, + event_context: EventContext, ) -> Result { let catalog = create_database_task.catalog.clone(); let schema = create_database_task.schema.clone(); let (id, _) = ddl_manager - .submit_create_database(create_database_task) + .submit_create_database(create_database_task, event_context) .await?; let procedure_id = id.to_string(); @@ -1114,9 +1178,10 @@ async fn handle_create_database_task( async fn handle_drop_database_task( ddl_manager: &DdlManager, drop_database_task: DropDatabaseTask, + event_context: EventContext, ) -> Result { let (id, _) = ddl_manager - .submit_drop_database(drop_database_task.clone()) + .submit_drop_database(drop_database_task.clone(), event_context) .await?; let procedure_id = id.to_string(); @@ -1134,9 +1199,10 @@ async fn handle_drop_database_task( async fn handle_alter_database_task( ddl_manager: &DdlManager, alter_database_task: AlterDatabaseTask, + event_context: EventContext, ) -> Result { let (id, _) = ddl_manager - .submit_alter_database(alter_database_task.clone()) + .submit_alter_database(alter_database_task.clone(), event_context) .await?; let procedure_id = id.to_string(); @@ -1155,9 +1221,10 @@ async fn handle_alter_database_task( async fn handle_drop_flow_task( ddl_manager: &DdlManager, drop_flow_task: DropFlowTask, + event_context: EventContext, ) -> Result { let (id, _) = ddl_manager - .submit_drop_flow_task(drop_flow_task.clone()) + .submit_drop_flow_task(drop_flow_task.clone(), event_context) .await?; let procedure_id = id.to_string(); @@ -1199,9 +1266,10 @@ async fn handle_drop_trigger_task( async fn handle_drop_view_task( ddl_manager: &DdlManager, drop_view_task: DropViewTask, + event_context: EventContext, ) -> Result { let (id, _) = ddl_manager - .submit_drop_view_task(drop_view_task.clone()) + .submit_drop_view_task(drop_view_task.clone(), event_context) .await?; let procedure_id = id.to_string(); @@ -1221,9 +1289,10 @@ async fn handle_create_flow_task( ddl_manager: &DdlManager, create_flow_task: CreateFlowTask, query_context: QueryContext, + event_context: EventContext, ) -> Result { let (id, output) = ddl_manager - .submit_create_flow_task(create_flow_task.clone(), query_context) + .submit_create_flow_task(create_flow_task.clone(), query_context, event_context) .await?; let procedure_id = id.to_string(); @@ -1280,6 +1349,7 @@ async fn handle_create_trigger_task( async fn handle_alter_logical_table_tasks( ddl_manager: &DdlManager, alter_table_tasks: Vec, + event_context: EventContext, ) -> Result { ensure!( !alter_table_tasks.is_empty(), @@ -1299,7 +1369,7 @@ async fn handle_alter_logical_table_tasks( let num_logical_tables = alter_table_tasks.len(); let (id, _) = ddl_manager - .submit_alter_logical_table_tasks(alter_table_tasks, physical_table_id) + .submit_alter_logical_table_tasks(alter_table_tasks, physical_table_id, event_context) .await?; info!( @@ -1318,9 +1388,10 @@ async fn handle_alter_logical_table_tasks( async fn handle_create_view_task( ddl_manager: &DdlManager, create_view_task: CreateViewTask, + event_context: EventContext, ) -> Result { let (id, output) = ddl_manager - .submit_create_view_task(create_view_task) + .submit_create_view_task(create_view_task, event_context) .await?; let procedure_id = id.to_string(); @@ -1398,7 +1469,7 @@ mod tests { use crate::procedure_executor::ExecutorContext; use crate::region_keeper::MemoryRegionKeeper; use crate::region_registry::LeaderRegionRegistry; - use crate::rpc::ddl::{CreatorGrantIntent, UndropTableTask}; + use crate::rpc::ddl::{CreatorGrantIntent, EventContext, UndropTableTask}; #[cfg(not(feature = "enterprise"))] use crate::rpc::ddl::{DdlTask, PurgeDroppedTableTask, QueryContext, SubmitDdlTaskRequest}; use crate::sequence::SequenceBuilder; @@ -1435,6 +1506,7 @@ mod tests { _source: RepartitionSource, _to_exprs: Vec, _timeout: Option, + _event_context: EventContext, ) -> std::result::Result { unimplemented!() } @@ -1608,7 +1680,7 @@ mod tests { let ddl_manager = build_soft_drop_test_ddl_manager(); let err = ddl_manager - .submit_undrop_table_task(UndropTableTask { table_id: 1024 }) + .submit_undrop_table_task(UndropTableTask { table_id: 1024 }, EventContext::default()) .await .unwrap_err(); @@ -1622,13 +1694,16 @@ mod tests { let ddl_manager = build_soft_drop_test_ddl_manager(); let err = ddl_manager - .submit_undrop_table_task(UndropTableTask { table_id: 1024 }) + .submit_undrop_table_task(UndropTableTask { table_id: 1024 }, EventContext::default()) .await .unwrap_err(); assert!(matches!(err, crate::error::Error::Unsupported { .. })); let err = ddl_manager - .submit_purge_dropped_table_task(PurgeDroppedTableTask { table_id: 1024 }) + .submit_purge_dropped_table_task( + PurgeDroppedTableTask { table_id: 1024 }, + EventContext::default(), + ) .await .unwrap_err(); assert!(matches!(err, crate::error::Error::Unsupported { .. })); diff --git a/src/common/meta/src/rpc/ddl.rs b/src/common/meta/src/rpc/ddl.rs index 0b61d2b2a5e..9cb75aa29c0 100644 --- a/src/common/meta/src/rpc/ddl.rs +++ b/src/common/meta/src/rpc/ddl.rs @@ -44,11 +44,13 @@ use base64::Engine as _; use base64::engine::general_purpose; use common_catalog::{format_full_flow_name, format_full_table_name}; use common_error::ext::BoxedError; +use common_session::channel_protocol; use common_time::{DatabaseTimeToLive, Timestamp}; use prost::Message; use serde::{Deserialize, Serialize}; use serde_with::{DefaultOnNull, serde_as}; use snafu::{OptionExt, ResultExt}; +use strum::{AsRefStr, EnumString}; use table::metadata::{TableId, TableInfo}; use table::requests::validate_database_option; use table::table_name::TableName; @@ -66,7 +68,8 @@ use crate::key::table_name::{TableNameKey, TableNameManager}; /// Reserved query-context extension key for the frontend peer address that submitted a DDL request. pub const ORIGIN_FRONTEND_ADDR_EXTENSION_KEY: &str = "__greptime_origin_frontend.addr"; - +/// Reserved query-context extension key for the trigger reason supplied by frontend. +pub const TRIGGER_REASON_EXTENSION_KEY: &str = "__greptime_event.trigger_reason"; /// Reserved query-context extension key for the authenticated database creator. pub const CREATE_DATABASE_CREATOR_EXTENSION_KEY: &str = "__greptime_create_database.creator"; /// Internal gRPC metadata key for the authenticated database creator. @@ -1665,6 +1668,86 @@ pub struct QueryContext { pub sst_min_sequences: HashMap, } +impl QueryContext { + /// Returns the protocol name represented by the wire channel value. + pub fn channel_protocol(&self) -> Option<&'static str> { + channel_protocol(self.channel) + } +} + +/// The stable context recorded for a procedure event. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct EventContext { + pub reason: TriggerReason, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub protocol: Option, + #[serde(default, skip_serializing_if = "serde_json::Map::is_empty")] + pub extensions: serde_json::Map, +} + +impl EventContext { + /// Creates an event context with no additional extensions. + pub fn new(reason: TriggerReason) -> Self { + Self { + reason, + protocol: None, + extensions: Default::default(), + } + } + + /// Adds the protocol that originated the operation. + pub fn with_protocol(mut self, protocol: impl Into) -> Self { + self.protocol = Some(protocol.into()); + self + } + + /// Builds an event context from frontend query context metadata. + pub fn from_query_context(query_context: &QueryContext) -> Self { + let reason = query_context + .extensions + .get(TRIGGER_REASON_EXTENSION_KEY) + .map(|reason| TriggerReason::from_extension(reason)) + .unwrap_or_default(); + let context = Self::new(reason); + if let Some(protocol) = query_context.channel_protocol() { + context.with_protocol(protocol) + } else { + context + } + } +} + +impl Default for EventContext { + fn default() -> Self { + Self::new(TriggerReason::default()) + } +} + +/// The stable classification of a procedure trigger. +#[derive( + Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize, AsRefStr, EnumString, +)] +#[serde(rename_all = "snake_case")] +#[strum(serialize_all = "snake_case")] +pub enum TriggerReason { + Manual, + AutoCreate, + AutoAlter, + AutoRepartition, + AutoRebalance, + RegionFailover, + ScheduledGc, + #[default] + #[serde(other)] + Unknown, +} + +impl TriggerReason { + pub fn from_extension(value: &str) -> Self { + value.parse().unwrap_or_default() + } +} + impl QueryContext { /// Get the current catalog pub fn current_catalog(&self) -> &str { @@ -2237,4 +2320,50 @@ mod tests { assert_eq!(pb_roundtrip.channel, pb.channel); assert_eq!(pb_roundtrip.snapshot_seqs, pb.snapshot_seqs); } + + #[test] + fn test_trigger_reason_deserializes_unknown_value() { + let reason: TriggerReason = serde_json::from_str("\"future_reason\"").unwrap(); + assert_eq!(TriggerReason::Unknown, reason); + } + + #[test] + fn test_event_context_serialization() { + let context = EventContext::new(TriggerReason::Manual).with_protocol("mysql"); + + assert_eq!( + serde_json::json!({ + "reason": "manual", + "protocol": "mysql", + }), + serde_json::to_value(context).unwrap() + ); + + assert_eq!( + serde_json::json!({ "reason": "manual" }), + serde_json::to_value(EventContext::new(TriggerReason::Manual)).unwrap() + ); + } + + #[test] + fn test_event_context_from_query_context_preserves_extensions() { + let mut query_context = QueryContext::default(); + query_context.extensions.insert( + TRIGGER_REASON_EXTENSION_KEY.to_string(), + TriggerReason::AutoCreate.as_ref().to_string(), + ); + query_context.channel = 4; + + let event_context = EventContext::from_query_context(&query_context); + + assert_eq!(event_context.reason, TriggerReason::AutoCreate); + assert_eq!(event_context.protocol.as_deref(), Some("prometheus")); + assert_eq!( + query_context + .extensions + .get(TRIGGER_REASON_EXTENSION_KEY) + .map(String::as_str), + Some("auto_create") + ); + } } diff --git a/src/common/procedure/src/lib.rs b/src/common/procedure/src/lib.rs index a1525ad242c..1d7a200ca58 100644 --- a/src/common/procedure/src/lib.rs +++ b/src/common/procedure/src/lib.rs @@ -30,8 +30,8 @@ pub use crate::error::{Error, Result}; pub use crate::event::ProcedureEvent; pub use crate::procedure::{ BoxedProcedure, BoxedProcedureLoader, ChildSubmissionOutcome, Context, ContextProvider, - ContextProviderRef, EventContext, EventTrigger, LockKey, Output, ParseIdError, PoisonKey, - PoisonKeys, Procedure, ProcedureId, ProcedureInfo, ProcedureManager, ProcedureManagerRef, - ProcedureState, ProcedureWithId, RetryPhase, Status, StringKey, + ContextProviderRef, EventRuntimeContext, EventTrigger, LockKey, Output, ParseIdError, + PoisonKey, PoisonKeys, Procedure, ProcedureId, ProcedureInfo, ProcedureManager, + ProcedureManagerRef, ProcedureState, ProcedureWithId, RetryPhase, Status, StringKey, }; pub use crate::watcher::Watcher; diff --git a/src/common/procedure/src/local.rs b/src/common/procedure/src/local.rs index fbcbc131b8d..59eaee2d789 100644 --- a/src/common/procedure/src/local.rs +++ b/src/common/procedure/src/local.rs @@ -1053,8 +1053,8 @@ mod tests { use crate::store::state_store::ObjectStateStore; use crate::test_util::InMemoryPoisonStore; use crate::{ - ChildSubmissionOutcome, Context, EventContext, EventTrigger, Procedure, ProcedureEvent, - Status, + ChildSubmissionOutcome, Context, EventRuntimeContext, EventTrigger, Procedure, + ProcedureEvent, Status, }; fn new_test_manager_context() -> ManagerContext { @@ -1296,7 +1296,7 @@ mod tests { self.poison_keys.clone() } - fn event(&self, _ctx: &EventContext<'_>) -> Option> { + fn event(&self, _ctx: &EventRuntimeContext<'_>) -> Option> { Some(Box::new(TestProcedureEvent)) } } @@ -1354,7 +1354,7 @@ mod tests { LockKey::default() } - fn event(&self, _ctx: &EventContext<'_>) -> Option> { + fn event(&self, _ctx: &EventRuntimeContext<'_>) -> Option> { Some(Box::new(TestProcedureEvent)) } } @@ -1397,7 +1397,7 @@ mod tests { LockKey::default() } - fn event(&self, _ctx: &EventContext<'_>) -> Option> { + fn event(&self, _ctx: &EventRuntimeContext<'_>) -> Option> { Some(Box::new(TestProcedureEvent)) } } @@ -1424,7 +1424,7 @@ mod tests { LockKey::default() } - fn event(&self, ctx: &EventContext<'_>) -> Option> { + fn event(&self, ctx: &EventRuntimeContext<'_>) -> Option> { *self.captured_filter.lock().unwrap() = Some(ctx.event_type_filter.clone()); Some(Box::new(TestProcedureEvent)) } diff --git a/src/common/procedure/src/local/runner.rs b/src/common/procedure/src/local/runner.rs index eea95f711b7..a460b7376bf 100644 --- a/src/common/procedure/src/local/runner.rs +++ b/src/common/procedure/src/local/runner.rs @@ -34,8 +34,8 @@ use crate::procedure::{Output, StringKey}; use crate::rwlock::OwnedKeyRwLockGuard; use crate::store::{ProcedureMessage, ProcedureStore}; use crate::{ - BoxedProcedure, ChildSubmissionOutcome, Context, Error, EventContext, EventTrigger, Procedure, - ProcedureId, ProcedureState, ProcedureWithId, RetryPhase, Status, + BoxedProcedure, ChildSubmissionOutcome, Context, Error, EventRuntimeContext, EventTrigger, + Procedure, ProcedureId, ProcedureState, ProcedureWithId, RetryPhase, Status, }; /// A guard to cleanup procedure state. @@ -776,7 +776,7 @@ impl Runner { pub(crate) fn build_event(&self, trigger: EventTrigger) -> Option { let recorder = self.event_recorder.as_ref()?; let state = self.meta.state(); - let context = EventContext { + let context = EventRuntimeContext { procedure_id: self.meta.id, lifecycle_state: &state, trigger: trigger.clone(), diff --git a/src/common/procedure/src/procedure.rs b/src/common/procedure/src/procedure.rs index c37e1b58bcc..a8af03e0da0 100644 --- a/src/common/procedure/src/procedure.rs +++ b/src/common/procedure/src/procedure.rs @@ -237,13 +237,13 @@ pub trait Procedure: Send { /// [`Event::extra_schema`] values. The event recorder batches events by type /// and rejects incompatible schemas; use a distinct event type for a /// different schema. - fn event(&self, _ctx: &EventContext<'_>) -> Option> { + fn event(&self, _ctx: &EventRuntimeContext<'_>) -> Option> { None } } /// Framework-owned context supplied when a procedure builds a lifecycle event. -pub struct EventContext<'a> { +pub struct EventRuntimeContext<'a> { /// Id of the procedure associated with the event. pub procedure_id: ProcedureId, /// Current framework state of the procedure. @@ -336,7 +336,7 @@ impl Procedure for Box { (**self).poison_keys() } - fn event(&self, ctx: &EventContext<'_>) -> Option> { + fn event(&self, ctx: &EventRuntimeContext<'_>) -> Option> { (**self).event(ctx) } } @@ -721,7 +721,7 @@ mod tests { #[test] fn test_default_procedure_event_hook() { let state = ProcedureState::Running; - let context = EventContext { + let context = EventRuntimeContext { procedure_id: ProcedureId::random(), lifecycle_state: &state, trigger: EventTrigger::Succeeded, diff --git a/src/common/session/src/lib.rs b/src/common/session/src/lib.rs index f09baf92866..418425df15e 100644 --- a/src/common/session/src/lib.rs +++ b/src/common/session/src/lib.rs @@ -15,6 +15,27 @@ use serde::{Deserialize, Serialize}; use strum::{AsRefStr, Display, EnumString}; +/// Returns the protocol name represented by a wire query channel value. +pub fn channel_protocol(channel: u8) -> Option<&'static str> { + match channel { + 1 => Some("mysql"), + 2 => Some("postgres"), + 3 => Some("httpsql"), + 4 => Some("prometheus"), + 5 => Some("otlp"), + 6 => Some("grpc"), + 7 => Some("influx"), + 8 => Some("opentsdb"), + 9 => Some("loki"), + 10 => Some("elasticsearch"), + 11 => Some("jaeger"), + 12 => Some("log"), + 13 => Some("promql"), + 14 => Some("splunk"), + _ => None, + } +} + /// Defines the read preference for frontend route operations, /// determining whether to read from the region leader or follower. #[derive( @@ -31,7 +52,33 @@ pub enum ReadPreference { mod tests { use std::str::FromStr; - use crate::ReadPreference; + use crate::{ReadPreference, channel_protocol}; + + #[test] + fn test_channel_protocol() { + let expected = [ + (1, "mysql"), + (2, "postgres"), + (3, "httpsql"), + (4, "prometheus"), + (5, "otlp"), + (6, "grpc"), + (7, "influx"), + (8, "opentsdb"), + (9, "loki"), + (10, "elasticsearch"), + (11, "jaeger"), + (12, "log"), + (13, "promql"), + (14, "splunk"), + ]; + + for (channel, protocol) in expected { + assert_eq!(Some(protocol), channel_protocol(channel)); + } + assert_eq!(None, channel_protocol(0)); + assert_eq!(None, channel_protocol(15)); + } #[test] fn test_read_preference() { diff --git a/src/flow/src/adapter/util.rs b/src/flow/src/adapter/util.rs index 80b3f5ae199..b9e73e296c3 100644 --- a/src/flow/src/adapter/util.rs +++ b/src/flow/src/adapter/util.rs @@ -21,6 +21,7 @@ use api::v1::column_def::options_from_column_schema; use api::v1::{ColumnDataType, ColumnDataTypeExtension, CreateTableExpr, SemanticType}; use common_error::ext::BoxedError; use common_meta::key::table_info::TableInfoValue; +use common_meta::rpc::ddl::TriggerReason; use datatypes::prelude::ConcreteDataType; use datatypes::schema::{ColumnDefaultConstraint, ColumnSchema}; use itertools::Itertools; @@ -131,7 +132,7 @@ impl StreamingEngine { .build(), ); stmt_exec - .create_table_inner(&mut create_table, None, ctx) + .create_table_inner(&mut create_table, None, ctx, TriggerReason::AutoCreate) .await .map_err(BoxedError::new) .context(ExternalSnafu)?; diff --git a/src/frontend/src/instance/dashboard.rs b/src/frontend/src/instance/dashboard.rs index 068e3e102c1..33a9083cc8e 100644 --- a/src/frontend/src/instance/dashboard.rs +++ b/src/frontend/src/instance/dashboard.rs @@ -24,6 +24,7 @@ use async_trait::async_trait; use auth::{DASHBOARD_DELETE, DASHBOARD_QUERY, DASHBOARD_SAVE, PermissionReq}; use common_catalog::consts::{DEFAULT_PRIVATE_SCHEMA_NAME, default_engine}; use common_error::ext::BoxedError; +use common_meta::rpc::ddl::TriggerReason; use common_query::OutputData; use common_recordbatch::util as record_util; use common_telemetry::info; @@ -165,7 +166,12 @@ impl Instance { }; self.statement_executor - .create_table_inner(&mut create_table_expr, None, ctx.clone()) + .create_table_inner( + &mut create_table_expr, + None, + ctx.clone(), + TriggerReason::AutoCreate, + ) .await .map_err(BoxedError::new) .context(ExecuteQuerySnafu)?; diff --git a/src/frontend/src/instance/grpc.rs b/src/frontend/src/instance/grpc.rs index 0327809239f..7d4f0ee7959 100644 --- a/src/frontend/src/instance/grpc.rs +++ b/src/frontend/src/instance/grpc.rs @@ -31,6 +31,7 @@ use auth::{ }; use common_error::ext::BoxedError; use common_grpc::flight::do_put::DoPutResponse; +use common_meta::rpc::ddl::TriggerReason; use common_query::Output; use common_query::logical_plan::add_insert_to_logical_plan; use common_telemetry::tracing::{self}; @@ -187,7 +188,12 @@ impl GrpcQueryHandler for Instance { operator::expr_helper::validate_create_expr(&expr)?; let _ = self .statement_executor - .create_table_inner(&mut expr, None, ctx.clone()) + .create_table_inner( + &mut expr, + None, + ctx.clone(), + TriggerReason::Manual, + ) .await?; Output::new_with_affected_rows(0) } @@ -200,7 +206,7 @@ impl GrpcQueryHandler for Instance { } DdlExpr::AlterTable(expr) => { self.statement_executor - .alter_table_inner(expr, ctx.clone()) + .alter_table_inner(expr, ctx.clone(), TriggerReason::Manual) .await? } DdlExpr::CreateDatabase(expr) => { diff --git a/src/frontend/src/instance/otlp/trace_ingest.rs b/src/frontend/src/instance/otlp/trace_ingest.rs index 0866189b571..7b4be126d9c 100644 --- a/src/frontend/src/instance/otlp/trace_ingest.rs +++ b/src/frontend/src/instance/otlp/trace_ingest.rs @@ -24,6 +24,7 @@ use api::v1::{ use client::Output; use common_error::ext::{BoxedError, ErrorExt}; use common_error::status_code::StatusCode; +use common_meta::rpc::ddl::TriggerReason; use common_telemetry::warn; use datatypes::prelude::ConcreteDataType; use opentelemetry_proto::tonic::collector::trace::v1::ExportTraceServiceRequest; @@ -1678,7 +1679,7 @@ impl Instance { if let Err(err) = self .statement_executor - .alter_table_inner(alter_expr, ctx.clone()) + .alter_table_inner(alter_expr, ctx.clone(), TriggerReason::AutoAlter) .await { let table = self diff --git a/src/frontend/src/instance/prom_store.rs b/src/frontend/src/instance/prom_store.rs index 33765d4445c..1bfb5f56437 100644 --- a/src/frontend/src/instance/prom_store.rs +++ b/src/frontend/src/instance/prom_store.rs @@ -30,6 +30,7 @@ use auth::{ use client::OutputData; use common_catalog::{format_full_table_name, parse_optional_catalog_and_schema_from_db_string}; use common_error::ext::BoxedError; +use common_meta::rpc::ddl::TriggerReason; use common_query::Output; use common_query::prelude::{GREPTIME_PHYSICAL_TABLE, greptime_value}; use common_recordbatch::RecordBatches; @@ -342,7 +343,7 @@ impl PendingRowsSchemaAlterer for Instance { AutoCreateTableType::Logical(_) => { // Use the batch API for logical tables. self.statement_executor - .create_logical_tables(&create_exprs, ctx) + .create_logical_tables(&create_exprs, ctx, TriggerReason::AutoCreate) .await .map_err(BoxedError::new) .context(error::ExecuteGrpcQuerySnafu)?; @@ -353,7 +354,7 @@ impl PendingRowsSchemaAlterer for Instance { expr.table_options .insert(SST_FORMAT_KEY.to_string(), "flat".to_string()); self.statement_executor - .create_table_inner(&mut expr, None, ctx.clone()) + .create_table_inner(&mut expr, None, ctx.clone(), TriggerReason::AutoCreate) .await .map_err(BoxedError::new) .context(error::ExecuteGrpcQuerySnafu)?; @@ -420,7 +421,7 @@ impl PendingRowsSchemaAlterer for Instance { } self.statement_executor - .alter_logical_tables(alter_exprs, ctx) + .alter_logical_tables(alter_exprs, ctx, TriggerReason::AutoAlter) .await .map_err(BoxedError::new) .context(error::ExecuteGrpcQuerySnafu)?; @@ -686,7 +687,7 @@ impl Instance { fill_metric_physical_table_options(&mut create_table_expr.table_options); self.statement_executor - .create_table_inner(&mut create_table_expr, None, ctx) + .create_table_inner(&mut create_table_expr, None, ctx, TriggerReason::AutoCreate) .await .map_err(BoxedError::new) .context(error::ExecuteGrpcQuerySnafu)?; diff --git a/src/meta-srv/Cargo.toml b/src/meta-srv/Cargo.toml index 7e8347e83bf..e6585505db4 100644 --- a/src/meta-srv/Cargo.toml +++ b/src/meta-srv/Cargo.toml @@ -100,6 +100,5 @@ common-event-recorder = { workspace = true, features = ["testing"] } common-meta = { workspace = true, features = ["testing"] } common-procedure-test.workspace = true common-wal = { workspace = true, features = ["testing"] } -session.workspace = true tracing = "0.1" tracing-subscriber.workspace = true diff --git a/src/meta-srv/src/event/gc.rs b/src/meta-srv/src/event/gc.rs index dd417ca77fb..c467bc665c6 100644 --- a/src/meta-srv/src/event/gc.rs +++ b/src/meta-srv/src/event/gc.rs @@ -21,9 +21,10 @@ use api::v1::{ColumnSchema, Row, Value}; use common_event_recorder::Event; use common_event_recorder::error::{Result, SerializeEventSnafu}; use common_event_recorder::event_table::{ - GC_REPORT_COLUMN, REGION_ID_COLUMN, REGION_NUMBER_COLUMN, TABLE_ID_COLUMN, column_schemas, - nullable_json, + EVENT_CONTEXT_COLUMN, GC_REPORT_COLUMN, REGION_ID_COLUMN, REGION_NUMBER_COLUMN, + TABLE_ID_COLUMN, column_schemas, nullable_json, }; +use common_meta::rpc::ddl::EventContext; use serde::Serialize; use snafu::ResultExt; use store_api::storage::{GcReport, IndexVersion, RegionId}; @@ -66,6 +67,7 @@ struct BatchGcRegionRow { #[derive(Debug)] pub(crate) struct BatchGcEvent { payload: Option, + event_context: Option, // None emits a procedure-level lifecycle row with null Region dimensions. regions: Option>, } @@ -75,6 +77,7 @@ impl BatchGcEvent { regions: &[RegionId], full_file_listing: bool, timeout: Duration, + event_context: Option, ) -> Self { Self { payload: Some(BatchGcPayload { @@ -83,6 +86,7 @@ impl BatchGcEvent { full_file_listing, timeout, }), + event_context, regions: None, } } @@ -107,6 +111,7 @@ impl BatchGcEvent { (!regions.is_empty()).then_some(Self { payload: None, + event_context: None, regions: Some(regions), }) } @@ -131,8 +136,22 @@ impl Event for BatchGcEvent { } fn extra_rows(&self) -> Result> { + let event_context = self + .event_context + .as_ref() + .map(serde_json::to_value) + .transpose() + .context(SerializeEventSnafu)?; let Some(regions) = &self.regions else { - return Ok(vec![null_row()]); + return Ok(vec![Row { + values: vec![ + Value { value_data: None }, + Value { value_data: None }, + Value { value_data: None }, + Value { value_data: None }, + nullable_json(event_context.as_ref()), + ], + }]); }; regions @@ -145,6 +164,7 @@ impl Event for BatchGcEvent { ValueData::U32Value(region.region_id.table_id()).into(), ValueData::U32Value(region.region_id.region_number()).into(), nullable_json(Some(&report)), + nullable_json(None), ], }) }) @@ -162,9 +182,11 @@ fn schema() -> Vec { &TABLE_ID_COLUMN, ®ION_NUMBER_COLUMN, &GC_REPORT_COLUMN, + &EVENT_CONTEXT_COLUMN, ]) } +#[cfg(test)] fn null_row() -> Row { Row { values: (0..schema().len()) @@ -221,7 +243,7 @@ mod tests { use common_meta::kv_backend::memory::MemoryKvBackend; use common_meta::sequence::SequenceBuilder; use common_procedure::{ - EventContext, EventTrigger, Procedure, ProcedureEvent, ProcedureId, ProcedureState, + EventRuntimeContext, EventTrigger, Procedure, ProcedureEvent, ProcedureId, ProcedureState, RetryPhase, }; use store_api::storage::FileId; @@ -234,8 +256,12 @@ mod tests { fn test_batch_gc_lifecycle_event_contract() { let first = RegionId::new(1024, 1); let second = RegionId::new(1024, 2); - let event = - BatchGcEvent::with_config(&[second, first, second], true, Duration::from_secs(10)); + let event = BatchGcEvent::with_config( + &[second, first, second], + true, + Duration::from_secs(10), + None, + ); assert_event_contract(&event, BATCH_GC_EVENT_TYPE, &schema(), &[null_row()]); assert_eq!( @@ -330,7 +356,7 @@ mod tests { fn test_batch_gc_event_filter() { let procedure = batch_gc_procedure(); let running = ProcedureState::Running; - let event_context = |trigger, lifecycle_state, event_type_filter| EventContext { + let runtime_context = |trigger, lifecycle_state, event_type_filter| EventRuntimeContext { procedure_id: ProcedureId::random(), lifecycle_state, trigger, @@ -339,7 +365,7 @@ mod tests { assert!( procedure - .event(&event_context( + .event(&runtime_context( EventTrigger::Submitted, &running, EventTypeFilter::All, @@ -356,7 +382,7 @@ mod tests { }; assert!( procedure - .event(&event_context( + .event(&runtime_context( EventTrigger::Succeeded, &done, EventTypeFilter::All, @@ -366,7 +392,7 @@ mod tests { assert!( procedure - .event(&event_context( + .event(&runtime_context( EventTrigger::Recovered, &running, EventTypeFilter::All, @@ -375,7 +401,7 @@ mod tests { ); let retrying = procedure - .event(&event_context( + .event(&runtime_context( EventTrigger::Retrying { phase: RetryPhase::Execute, attempt: 1, @@ -397,7 +423,7 @@ mod tests { assert!( procedure - .event(&event_context( + .event(&runtime_context( EventTrigger::Recovered, &running, EventTypeFilter::Only(HashSet::from(["another_event".to_string()])), @@ -406,7 +432,7 @@ mod tests { ); assert!( procedure - .event(&event_context( + .event(&runtime_context( EventTrigger::Recovered, &running, EventTypeFilter::Only(HashSet::new()), @@ -417,7 +443,7 @@ mod tests { let missing = ProcedureState::Done { output: None }; assert!( procedure - .event(&event_context( + .event(&runtime_context( EventTrigger::Succeeded, &missing, EventTypeFilter::All, @@ -429,7 +455,7 @@ mod tests { }; assert!( procedure - .event(&event_context( + .event(&runtime_context( EventTrigger::Succeeded, &wrong, EventTypeFilter::All, @@ -442,7 +468,7 @@ mod tests { fn test_batch_gc_terminal_event_with_report() { let mut procedure = batch_gc_procedure(); let running = ProcedureState::Running; - let event_context = |trigger| EventContext { + let runtime_context = |trigger| EventRuntimeContext { procedure_id: ProcedureId::random(), lifecycle_state: &running, trigger, @@ -455,7 +481,7 @@ mod tests { procedure.set_gc_report_for_test(report); let retrying = procedure - .event(&event_context(EventTrigger::Retrying { + .event(&runtime_context(EventTrigger::Retrying { phase: RetryPhase::Execute, attempt: 2, })) @@ -472,7 +498,7 @@ mod tests { assert_eq!(retrying.extra_rows().unwrap(), vec![null_row()]); for trigger in [EventTrigger::Failed, EventTrigger::Poisoned] { - let event = procedure.event(&event_context(trigger)).unwrap(); + let event = procedure.event(&runtime_context(trigger)).unwrap(); assert_eq!(event.json_payload().unwrap(), serde_json::Value::Null); assert_eq!( event.extra_rows().unwrap(), @@ -487,7 +513,7 @@ mod tests { let procedure = batch_gc_procedure(); let failed_without_report = procedure - .event(&event_context(EventTrigger::Failed)) + .event(&runtime_context(EventTrigger::Failed)) .unwrap(); assert_eq!( failed_without_report.json_payload().unwrap(), @@ -563,6 +589,7 @@ mod tests { ValueData::U32Value(region_id.table_id()).into(), ValueData::U32Value(region_id.region_number()).into(), nullable_json(report.as_ref()), + nullable_json(None), ], } } @@ -580,6 +607,7 @@ mod tests { true, Duration::from_secs(10), HashMap::new(), + EventContext::default(), ) } } diff --git a/src/meta-srv/src/event/region_migration.rs b/src/meta-srv/src/event/region_migration.rs index f7b311c13ba..65b89df2515 100644 --- a/src/meta-srv/src/event/region_migration.rs +++ b/src/meta-srv/src/event/region_migration.rs @@ -20,9 +20,10 @@ use api::v1::{ColumnSchema, Row}; use common_event_recorder::Event; use common_event_recorder::error::{Result, SerializeEventSnafu}; use common_event_recorder::event_table::{ - REGION_ID_COLUMN, REGION_MIGRATION_DST_NODE_ID_COLUMN, REGION_MIGRATION_DST_PEER_ADDR_COLUMN, - REGION_MIGRATION_SRC_NODE_ID_COLUMN, REGION_MIGRATION_SRC_PEER_ADDR_COLUMN, - REGION_MIGRATION_TRIGGER_REASON_COLUMN, REGION_NUMBER_COLUMN, TABLE_ID_COLUMN, column_schemas, + EVENT_CONTEXT_COLUMN, REGION_ID_COLUMN, REGION_MIGRATION_DST_NODE_ID_COLUMN, + REGION_MIGRATION_DST_PEER_ADDR_COLUMN, REGION_MIGRATION_SRC_NODE_ID_COLUMN, + REGION_MIGRATION_SRC_PEER_ADDR_COLUMN, REGION_MIGRATION_TRIGGER_REASON_COLUMN, + REGION_NUMBER_COLUMN, TABLE_ID_COLUMN, column_schemas, nullable_json, }; use serde::Serialize; use snafu::ResultExt; @@ -49,6 +50,7 @@ pub(crate) struct RegionMigrationEvent { dst_peer_addr: String, // The timeout of the region migration. timeout: Duration, + event_context: Option, } #[derive(Debug, Serialize)] @@ -58,7 +60,7 @@ struct Payload { } impl RegionMigrationEvent { - pub fn from_persistent_ctx(ctx: &PersistentContext) -> Self { + pub fn from_persistent_ctx(ctx: &PersistentContext, include_event_context: bool) -> Self { Self { region_ids: ctx.region_ids.clone(), trigger_reason: ctx.trigger_reason, @@ -67,6 +69,7 @@ impl RegionMigrationEvent { dst_node_id: ctx.to_peer.id, dst_peer_addr: ctx.to_peer.addr.clone(), timeout: ctx.timeout, + event_context: include_event_context.then(|| ctx.event_context.clone()), } } } @@ -86,6 +89,7 @@ impl Event for RegionMigrationEvent { ®ION_MIGRATION_SRC_PEER_ADDR_COLUMN, ®ION_MIGRATION_DST_NODE_ID_COLUMN, ®ION_MIGRATION_DST_PEER_ADDR_COLUMN, + &EVENT_CONTEXT_COLUMN, ]) } @@ -102,6 +106,12 @@ impl Event for RegionMigrationEvent { ValueData::StringValue(self.src_peer_addr.clone()).into(), ValueData::U64Value(self.dst_node_id).into(), ValueData::StringValue(self.dst_peer_addr.clone()).into(), + self.event_context + .as_ref() + .map(serde_json::to_value) + .transpose() + .context(SerializeEventSnafu) + .map(|value| nullable_json(value.as_ref()))?, ], }); } diff --git a/src/meta-srv/src/event/repartition.rs b/src/meta-srv/src/event/repartition.rs index d266b581f82..ad47fbf9d42 100644 --- a/src/meta-srv/src/event/repartition.rs +++ b/src/meta-srv/src/event/repartition.rs @@ -21,11 +21,11 @@ use api::v1::{ColumnSchema, Row}; use common_event_recorder::Event; use common_event_recorder::error::{Result, SerializeEventSnafu}; use common_event_recorder::event_table::{ - CATALOG_NAME_COLUMN, PARENT_PROCEDURE_ID_COLUMN, REPARTITION_GROUP_ID_COLUMN, - SCHEMA_NAME_COLUMN, SOURCE_PARTITION_EXPR_COLUMN, SOURCE_REGION_ID_COLUMN, - SOURCE_REGION_NUMBER_COLUMN, TABLE_ID_COLUMN, TABLE_NAME_COLUMN, TARGET_PARTITION_EXPR_COLUMN, - TARGET_REGION_ID_COLUMN, TARGET_REGION_NUMBER_COLUMN, column_schemas, nullable_string, - nullable_value, + CATALOG_NAME_COLUMN, EVENT_CONTEXT_COLUMN, PARENT_PROCEDURE_ID_COLUMN, + REPARTITION_GROUP_ID_COLUMN, SCHEMA_NAME_COLUMN, SOURCE_PARTITION_EXPR_COLUMN, + SOURCE_REGION_ID_COLUMN, SOURCE_REGION_NUMBER_COLUMN, TABLE_ID_COLUMN, TABLE_NAME_COLUMN, + TARGET_PARTITION_EXPR_COLUMN, TARGET_REGION_ID_COLUMN, TARGET_REGION_NUMBER_COLUMN, + column_schemas, nullable_json, nullable_string, nullable_value, }; use serde::Serialize; use snafu::ResultExt; @@ -61,6 +61,7 @@ pub(crate) struct RepartitionEvent { table_name: Option, table_id: Option, payload: Option, + event_context: Option, } impl RepartitionEvent { @@ -90,6 +91,7 @@ impl RepartitionEvent { target_partition_columns: intent.target_partition_columns().map(ToOwned::to_owned), timeout: persistent_ctx.timeout, }), + event_context: Some(persistent_ctx.event_context.clone()), } } @@ -100,10 +102,17 @@ impl RepartitionEvent { table_name: None, table_id: None, payload: None, + event_context: None, } } fn schema() -> Vec { + let mut schema = Self::base_schema(); + schema.push(EVENT_CONTEXT_COLUMN.column_schema()); + schema + } + + fn base_schema() -> Vec { column_schemas([ &CATALOG_NAME_COLUMN, &SCHEMA_NAME_COLUMN, @@ -132,12 +141,19 @@ impl Event for RepartitionEvent { } fn extra_rows(&self) -> Result> { + let event_context = self + .event_context + .as_ref() + .map(serde_json::to_value) + .transpose() + .context(SerializeEventSnafu)?; Ok(vec![Row { values: vec![ nullable_string(self.catalog_name.as_deref()), nullable_string(self.schema_name.as_deref()), nullable_string(self.table_name.as_deref()), nullable_value(self.table_id.map(ValueData::U32Value)), + nullable_json(event_context.as_ref()), ], }]) } @@ -307,7 +323,7 @@ impl RepartitionGroupEvent { } fn schema() -> Vec { - let mut schema = RepartitionEvent::schema(); + let mut schema = RepartitionEvent::base_schema(); schema.extend(column_schemas([ &PARENT_PROCEDURE_ID_COLUMN, &REPARTITION_GROUP_ID_COLUMN, @@ -368,6 +384,7 @@ mod tests { PROCEDURE_TRIGGER_COLUMN, jsonb_value, }; use common_event_recorder::testing::assert_event_contract; + use common_meta::rpc::ddl::EventContext; use common_procedure::{EventTrigger, ProcedureEvent, ProcedureId, ProcedureState}; use table::table_name::TableName; use uuid::Uuid; @@ -385,6 +402,7 @@ mod tests { TableName::new("greptime", "public", "repartition_events"), 1024, Some(Duration::from_secs(30)), + EventContext::default(), ) } @@ -420,6 +438,9 @@ mod tests { ValueData::StringValue("public".to_string()).into(), ValueData::StringValue("repartition_events".to_string()).into(), ValueData::U32Value(1024).into(), + jsonb_value(&serde_json::json!({ + "reason": "unknown", + })), ], }], ); @@ -674,6 +695,9 @@ mod tests { ValueData::StringValue("public".to_string()).into(), ValueData::StringValue("repartition_events".to_string()).into(), ValueData::U32Value(1024).into(), + jsonb_value(&serde_json::json!({ + "reason": "unknown", + })), ], }], ); @@ -685,11 +709,12 @@ mod tests { &SCHEMA_NAME_COLUMN, &TABLE_NAME_COLUMN, &TABLE_ID_COLUMN, + &EVENT_CONTEXT_COLUMN, ]) } fn group_schema() -> Vec { - let mut schema = parent_schema(); + let mut schema = RepartitionEvent::base_schema(); schema.extend(column_schemas([ &PARENT_PROCEDURE_ID_COLUMN, &REPARTITION_GROUP_ID_COLUMN, diff --git a/src/meta-srv/src/gc/ctx.rs b/src/meta-srv/src/gc/ctx.rs index 79eea6cca33..41027a65390 100644 --- a/src/meta-srv/src/gc/ctx.rs +++ b/src/meta-srv/src/gc/ctx.rs @@ -29,6 +29,7 @@ use common_meta::key::DroppedTableName; use common_meta::key::TableMetadataManagerRef; use common_meta::key::table_repart::TableRepartValue; use common_meta::key::table_route::PhysicalTableRouteValue; +use common_meta::rpc::ddl::EventContext; #[cfg(feature = "enterprise")] use common_meta::rpc::ddl::PurgeDroppedTableTask; use common_procedure::{ProcedureManagerRef, ProcedureWithId, watcher}; @@ -67,6 +68,7 @@ pub(crate) trait SchedulerCtx: Send + Sync { full_file_listing: bool, timeout: Duration, region_routes_override: Region2Peers, + event_context: EventContext, ) -> Result; #[cfg(feature = "enterprise")] @@ -254,12 +256,14 @@ impl SchedulerCtx for DefaultGcSchedulerCtx { full_file_listing: bool, timeout: Duration, region_routes_override: Region2Peers, + event_context: EventContext, ) -> Result { self.gc_regions_inner( region_ids, full_file_listing, timeout, region_routes_override, + event_context, ) .await } @@ -306,6 +310,7 @@ impl DefaultGcSchedulerCtx { full_file_listing: bool, timeout: Duration, region_routes_override: Region2Peers, + event_context: EventContext, ) -> Result { debug!( "Sending GC instruction for {} regions (full_file_listing: {})", @@ -321,6 +326,7 @@ impl DefaultGcSchedulerCtx { full_file_listing, timeout, region_routes_override, + event_context, ); let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure)); diff --git a/src/meta-srv/src/gc/handler.rs b/src/meta-srv/src/gc/handler.rs index 37a71580fa7..4532e9a8ca3 100644 --- a/src/meta-srv/src/gc/handler.rs +++ b/src/meta-srv/src/gc/handler.rs @@ -18,6 +18,7 @@ use std::time::Instant; use common_catalog::consts::MITO_ENGINE; use common_meta::datanode::{RegionManifestInfo, RegionStat}; use common_meta::peer::Peer; +use common_meta::rpc::ddl::EventContext; use common_telemetry::tracing::Instrument as _; use common_telemetry::{debug, error, info, warn}; use futures::StreamExt; @@ -36,7 +37,7 @@ use crate::gc::tracker::RegionGcInfo; use crate::metrics::METRIC_META_GC_CANDIDATE_REGIONS; impl GcScheduler { - pub(crate) async fn trigger_gc(&self) -> Result { + pub(crate) async fn trigger_gc(&self, event_context: EventContext) -> Result { let start_time = Instant::now(); info!("Starting GC cycle"); @@ -107,6 +108,7 @@ impl GcScheduler { datanode_to_candidates, dropped_assignment.force_full_listing, dropped_assignment.region_routes_override, + event_context, ) .instrument(common_telemetry::tracing::info_span!( "meta_gc_dispatch_to_datanodes" @@ -210,6 +212,7 @@ impl GcScheduler { datanode_to_candidates: HashMap>, force_full_listing_by_peer: HashMap>, region_routes_override_by_peer: HashMap, + event_context: EventContext, ) -> GcJobReport { let mut per_datanode_reports = HashMap::new(); let mut failed_datanodes: HashMap<_, Vec<_>> = HashMap::new(); @@ -231,6 +234,7 @@ impl GcScheduler { .get(&peer) .cloned() .unwrap_or_default(); + let event_context = event_context.clone(); async move { ( peer, @@ -240,6 +244,7 @@ impl GcScheduler { candidates, force_full_listing, region_routes_override, + event_context, ) .await, ) @@ -278,6 +283,7 @@ impl GcScheduler { candidates: Vec<(TableId, GcCandidate)>, force_full_listing: HashSet, region_routes_override: Region2Peers, + event_context: EventContext, ) -> Result { info!( "Starting GC for datanode {} with {} candidate regions", @@ -327,6 +333,7 @@ impl GcScheduler { false, self.config.mailbox_timeout, region_routes_override.clone(), + event_context.clone(), ) .instrument(common_telemetry::tracing::info_span!( "meta_gc_call_datanode", @@ -359,6 +366,7 @@ impl GcScheduler { true, self.config.mailbox_timeout, region_routes_override, + event_context, ) .instrument(common_telemetry::tracing::info_span!( "meta_gc_call_datanode", diff --git a/src/meta-srv/src/gc/mock.rs b/src/meta-srv/src/gc/mock.rs index 1d007dc30bc..a941476ab44 100644 --- a/src/meta-srv/src/gc/mock.rs +++ b/src/meta-srv/src/gc/mock.rs @@ -30,6 +30,7 @@ use common_meta::datanode::{RegionManifestInfo, RegionStat}; use common_meta::key::table_repart::TableRepartValue; use common_meta::key::table_route::PhysicalTableRouteValue; use common_meta::peer::Peer; +use common_meta::rpc::ddl::EventContext; use common_meta::rpc::router::{Region, RegionRoute}; use common_telemetry::debug; use ordered_float::OrderedFloat; @@ -74,6 +75,7 @@ pub struct MockSchedulerCtx { pub candidates: Arc>>>>, pub get_table_to_region_stats_calls: Arc>, pub gc_regions_calls: Arc>, + pub gc_event_contexts: Arc>>, // Error injection fields for testing pub get_table_to_region_stats_error: Arc>>, pub get_table_route_error: Arc>>, @@ -212,8 +214,10 @@ impl SchedulerCtx for MockSchedulerCtx { _full_file_listing: bool, _timeout: Duration, _region_routes_override: Region2Peers, + event_context: EventContext, ) -> Result { *self.gc_regions_calls.lock().unwrap() += 1; + self.gc_event_contexts.lock().unwrap().push(event_context); // Check per-region error injection first (for any region) for ®ion_id in region_ids { diff --git a/src/meta-srv/src/gc/mock/basic.rs b/src/meta-srv/src/gc/mock/basic.rs index 87278abcbf9..45d64910561 100644 --- a/src/meta-srv/src/gc/mock/basic.rs +++ b/src/meta-srv/src/gc/mock/basic.rs @@ -17,6 +17,7 @@ use std::sync::{Arc, Mutex}; use std::time::Instant; use common_meta::peer::Peer; +use common_meta::rpc::ddl::{EventContext, TriggerReason}; use common_telemetry::init_default_ut_logging; use store_api::region_engine::RegionRole; use store_api::storage::{FileId, FileRefsManifest, GcReport, RegionId}; @@ -31,7 +32,12 @@ async fn test_parallel_process_datanodes_empty() { let env = TestEnv::new(); let report = env .scheduler - .parallel_process_datanodes(HashMap::new(), HashMap::new(), HashMap::new()) + .parallel_process_datanodes( + HashMap::new(), + HashMap::new(), + HashMap::new(), + EventContext::default(), + ) .await; match report { @@ -95,7 +101,12 @@ async fn test_parallel_process_datanodes_with_candidates() { )]); let report = scheduler - .parallel_process_datanodes(datanode_to_candidates, HashMap::new(), HashMap::new()) + .parallel_process_datanodes( + datanode_to_candidates, + HashMap::new(), + HashMap::new(), + EventContext::default(), + ) .await; match report { @@ -176,6 +187,10 @@ async fn test_handle_tick() { assert_eq!(*ctx.get_table_to_region_stats_calls.lock().unwrap(), 1); assert_eq!(*ctx.gc_regions_calls.lock().unwrap(), 1); + assert_eq!( + ctx.gc_event_contexts.lock().unwrap().as_slice(), + &[EventContext::new(TriggerReason::ScheduledGc)] + ); let tracker = scheduler.region_gc_tracker.lock().await; assert!( @@ -184,3 +199,56 @@ async fn test_handle_tick() { tracker ); } + +#[tokio::test] +async fn test_handle_manual_gc_without_regions_records_manual_event_context() { + init_default_ut_logging(); + + let table_id = 1; + let region_id = RegionId::new(table_id, 1); + let peer = Peer::new(1, ""); + let candidates = HashMap::from([(table_id, vec![new_candidate(region_id, 1.0)])]); + let file_refs = FileRefsManifest { + manifest_version: HashMap::from([(region_id, 1)]), + ..Default::default() + }; + let ctx = Arc::new( + MockSchedulerCtx { + table_to_region_stats: Arc::new(Mutex::new(Some(HashMap::from([( + table_id, + vec![mock_region_stat( + region_id, + RegionRole::Leader, + TEST_REGION_SIZE_200MB, + 10, + )], + )])))), + gc_reports: Arc::new(Mutex::new(HashMap::from([( + region_id, + GcReport::default(), + )]))), + candidates: Arc::new(Mutex::new(Some(candidates))), + file_refs: Arc::new(Mutex::new(Some(file_refs))), + ..Default::default() + } + .with_table_routes(HashMap::from([( + table_id, + (table_id, vec![(region_id, peer)]), + )])), + ); + let scheduler = GcScheduler { + ctx: ctx.clone(), + runtime_switch_manager: crate::gc::scheduler::new_test_runtime_switch_manager(), + receiver: GcScheduler::channel().1, + config: GcSchedulerOptions::default(), + region_gc_tracker: Arc::new(tokio::sync::Mutex::new(HashMap::new())), + last_tracker_cleanup: Arc::new(tokio::sync::Mutex::new(Instant::now())), + }; + + scheduler.handle_manual_gc(None, None, None).await.unwrap(); + + assert_eq!( + ctx.gc_event_contexts.lock().unwrap().as_slice(), + &[EventContext::new(TriggerReason::Manual)] + ); +} diff --git a/src/meta-srv/src/gc/mock/concurrent.rs b/src/meta-srv/src/gc/mock/concurrent.rs index ddaf8128431..7dca378b71d 100644 --- a/src/meta-srv/src/gc/mock/concurrent.rs +++ b/src/meta-srv/src/gc/mock/concurrent.rs @@ -18,6 +18,7 @@ use std::time::{Duration, Instant}; use common_meta::key::table_route::PhysicalTableRouteValue; use common_meta::peer::Peer; +use common_meta::rpc::ddl::EventContext; use common_meta::rpc::router::{Region, RegionRoute}; use common_telemetry::{info, init_default_ut_logging}; use store_api::region_engine::RegionRole; @@ -97,7 +98,12 @@ async fn test_concurrent_table_processing_limits() { )]); let report = scheduler - .parallel_process_datanodes(datanode_to_candidates, HashMap::new(), HashMap::new()) + .parallel_process_datanodes( + datanode_to_candidates, + HashMap::new(), + HashMap::new(), + EventContext::default(), + ) .await; // Should process all datanodes @@ -180,7 +186,12 @@ async fn test_datanode_processes_tables_with_partial_gc_failures() { )]); let report = scheduler - .parallel_process_datanodes(datanode_to_candidates, HashMap::new(), HashMap::new()) + .parallel_process_datanodes( + datanode_to_candidates, + HashMap::new(), + HashMap::new(), + EventContext::default(), + ) .await; // Should have one datanode with mixed results @@ -285,6 +296,7 @@ async fn test_region_gc_concurrency_limit() { candidates.into_iter().map(|c| (table_id, c)).collect(), HashSet::new(), HashMap::new(), + EventContext::default(), ) .await .unwrap(); @@ -397,7 +409,12 @@ async fn test_region_gc_concurrency_with_partial_failures() { )]); let report = scheduler - .parallel_process_datanodes(datanode_to_candidates, HashMap::new(), HashMap::new()) + .parallel_process_datanodes( + datanode_to_candidates, + HashMap::new(), + HashMap::new(), + EventContext::default(), + ) .await; let report = match &report { @@ -536,7 +553,12 @@ async fn test_region_gc_concurrency_with_retryable_errors() { candidates.into_iter().map(|c| (table_id, c)).collect(), )]); let report = scheduler - .parallel_process_datanodes(datanode_to_candidates, HashMap::new(), HashMap::new()) + .parallel_process_datanodes( + datanode_to_candidates, + HashMap::new(), + HashMap::new(), + EventContext::default(), + ) .await; let report = match &report { diff --git a/src/meta-srv/src/gc/mock/full_list.rs b/src/meta-srv/src/gc/mock/full_list.rs index 847c0e73e40..a5d5012c6d3 100644 --- a/src/meta-srv/src/gc/mock/full_list.rs +++ b/src/meta-srv/src/gc/mock/full_list.rs @@ -17,6 +17,7 @@ use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant}; use common_meta::peer::Peer; +use common_meta::rpc::ddl::EventContext; use common_telemetry::init_default_ut_logging; use store_api::region_engine::RegionRole; use store_api::storage::{FileId, FileRefsManifest, GcReport, RegionId}; @@ -82,6 +83,7 @@ async fn test_full_file_listing_first_time_gc() { vec![(table_id, mock_candidate(region_id))], HashSet::new(), HashMap::new(), + EventContext::default(), ) .await .unwrap(); @@ -155,6 +157,7 @@ async fn test_full_file_listing_interval_enforcement() { vec![(table_id, mock_candidate(region_id))], HashSet::new(), HashMap::new(), + EventContext::default(), ) .await .unwrap(); @@ -181,6 +184,7 @@ async fn test_full_file_listing_interval_enforcement() { vec![(table_id, mock_candidate(region_id))], HashSet::new(), HashMap::new(), + EventContext::default(), ) .await .unwrap(); @@ -256,6 +260,7 @@ async fn test_full_file_listing_no_interval_passed() { vec![(table_id, mock_candidate(region_id))], HashSet::new(), HashMap::new(), + EventContext::default(), ) .await .unwrap(); @@ -279,6 +284,7 @@ async fn test_full_file_listing_no_interval_passed() { vec![(table_id, mock_candidate(region_id))], HashSet::new(), HashMap::new(), + EventContext::default(), ) .await .unwrap(); diff --git a/src/meta-srv/src/gc/mock/misc.rs b/src/meta-srv/src/gc/mock/misc.rs index c8b2b43155e..9b3c35cbe9a 100644 --- a/src/meta-srv/src/gc/mock/misc.rs +++ b/src/meta-srv/src/gc/mock/misc.rs @@ -17,6 +17,7 @@ use std::sync::{Arc, Mutex}; use std::time::Instant; use common_meta::peer::Peer; +use common_meta::rpc::ddl::EventContext; use common_telemetry::init_default_ut_logging; use store_api::storage::{FileRefsManifest, GcReport, RegionId}; @@ -71,7 +72,12 @@ async fn test_empty_file_refs_manifest() { )]); let report = scheduler - .parallel_process_datanodes(datanode_to_candidates, HashMap::new(), HashMap::new()) + .parallel_process_datanodes( + datanode_to_candidates, + HashMap::new(), + HashMap::new(), + EventContext::default(), + ) .await; match report { @@ -159,7 +165,12 @@ async fn test_multiple_regions_per_table() { )]); let report = scheduler - .parallel_process_datanodes(datanode_to_candidates, HashMap::new(), HashMap::new()) + .parallel_process_datanodes( + datanode_to_candidates, + HashMap::new(), + HashMap::new(), + EventContext::default(), + ) .await; match report { diff --git a/src/meta-srv/src/gc/procedure.rs b/src/meta-srv/src/gc/procedure.rs index 311a14e4a98..d745e88e2c5 100644 --- a/src/meta-srv/src/gc/procedure.rs +++ b/src/meta-srv/src/gc/procedure.rs @@ -24,10 +24,11 @@ use common_meta::key::table_repart::TableRepartValue; use common_meta::key::table_route::PhysicalTableRouteValue; use common_meta::lock_key::{RegionLock, TableLock}; use common_meta::peer::Peer; +use common_meta::rpc::ddl::{EventContext, TriggerReason}; use common_procedure::error::ToJsonSnafu; use common_procedure::{ - Context as ProcedureContext, Error as ProcedureError, EventContext, EventTrigger, LockKey, - Procedure, ProcedureState, Result as ProcedureResult, Status, + Context as ProcedureContext, Error as ProcedureError, EventRuntimeContext, EventTrigger, + LockKey, Procedure, ProcedureState, Result as ProcedureResult, Status, }; use common_telemetry::tracing::Instrument as _; use common_telemetry::tracing_context::TracingContext; @@ -199,6 +200,8 @@ pub struct BatchGcData { /// mailbox timeout duration timeout: Duration, gc_report: Option, + #[serde(default)] + event_context: EventContext, } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] @@ -216,6 +219,7 @@ pub enum State { impl BatchGcProcedure { pub const TYPE_NAME: &'static str = "metasrv-procedure::BatchGcProcedure"; + #[allow(clippy::too_many_arguments)] pub fn new( mailbox: MailboxRef, table_metadata_manager: TableMetadataManagerRef, @@ -224,6 +228,7 @@ impl BatchGcProcedure { full_file_listing: bool, timeout: Duration, region_routes_override: Region2Peers, + event_context: EventContext, ) -> Self { Self { mailbox, @@ -239,6 +244,7 @@ impl BatchGcProcedure { related_regions: HashMap::new(), file_refs: FileRefsManifest::default(), gc_report: None, + event_context, }, } } @@ -269,6 +275,7 @@ impl BatchGcProcedure { related_regions: HashMap::new(), file_refs, gc_report: Some(GcReport::default()), + event_context: EventContext::default(), }, } } @@ -1065,16 +1072,26 @@ impl Procedure for BatchGcProcedure { LockKey::new(lock_key) } - fn event(&self, ctx: &EventContext<'_>) -> Option> { + fn event( + &self, + ctx: &EventRuntimeContext<'_>, + ) -> Option> { if !ctx.event_type_filter.allows(BATCH_GC_EVENT_TYPE) { return None; } let event = match &ctx.trigger { - // Keep normal GC low-noise; only terminal events with meaningful reports are recorded. - EventTrigger::Submitted - | EventTrigger::Recovered - | EventTrigger::ChildSubmitted { .. } => return None, + // Keep scheduled GC low-noise; record submitted manual requests for auditability. + EventTrigger::Submitted => (self.data.event_context.reason == TriggerReason::Manual) + .then(|| { + BatchGcEvent::with_config( + &self.data.regions, + self.data.full_file_listing, + self.data.timeout, + Some(self.data.event_context.clone()), + ) + })?, + EventTrigger::Recovered | EventTrigger::ChildSubmitted { .. } => return None, EventTrigger::Succeeded => { let ProcedureState::Done { output: Some(output), @@ -1089,6 +1106,7 @@ impl Procedure for BatchGcProcedure { &self.data.regions, self.data.full_file_listing, self.data.timeout, + None, ), EventTrigger::Failed | EventTrigger::Poisoned => self .data @@ -1100,6 +1118,7 @@ impl Procedure for BatchGcProcedure { &self.data.regions, self.data.full_file_listing, self.data.timeout, + None, ) }), }; @@ -1192,6 +1211,16 @@ mod tests { assert!(report.need_retry_regions.contains(®ion_id)); } + #[test] + fn test_batch_gc_data_defaults_missing_event_context() { + let mut data = serde_json::to_value(&batch_gc_procedure().data).unwrap(); + data.as_object_mut().unwrap().remove("event_context"); + + let recovered: BatchGcData = serde_json::from_value(data).unwrap(); + + assert_eq!(recovered.event_context, EventContext::default()); + } + #[tokio::test] async fn test_send_gc_instructions_preserves_partial_report() { let first_region = RegionId::new(1024, 1); @@ -1226,6 +1255,7 @@ mod tests { true, Duration::from_secs(10), HashMap::new(), + EventContext::default(), ); procedure.data.region_routes = HashMap::from([ (first_region, (first_peer, vec![])), @@ -1266,6 +1296,7 @@ mod tests { true, Duration::from_secs(10), HashMap::new(), + EventContext::default(), ) } } diff --git a/src/meta-srv/src/gc/scheduler.rs b/src/meta-srv/src/gc/scheduler.rs index c637d152526..92b362ad947 100644 --- a/src/meta-srv/src/gc/scheduler.rs +++ b/src/meta-srv/src/gc/scheduler.rs @@ -18,6 +18,7 @@ use std::time::{Duration, Instant}; use common_meta::DatanodeId; use common_meta::key::runtime_switch::RuntimeSwitchManagerRef; +use common_meta::rpc::ddl::{EventContext, TriggerReason}; use common_telemetry::tracing::Instrument as _; use common_telemetry::{error, info}; use snafu::ResultExt; @@ -216,7 +217,10 @@ impl GcScheduler { .await; } let span = common_telemetry::tracing::info_span!("meta_gc_handle_tick"); - let report = self.trigger_gc().instrument(span).await?; + let report = self + .trigger_gc(EventContext::new(TriggerReason::ScheduledGc)) + .instrument(span) + .await?; // Periodically clean up stale tracker entries self.cleanup_tracker_if_needed().await?; @@ -300,7 +304,9 @@ impl GcScheduler { // No specific regions, use default tick behavior let Some(regions) = region_ids else { - let report = self.trigger_gc().await?; + let report = self + .trigger_gc(EventContext::new(TriggerReason::Manual)) + .await?; info!("Finished manual gc request"); return Ok(report); }; @@ -347,6 +353,7 @@ impl GcScheduler { full_listing, gc_timeout, Region2Peers::new(), + EventContext::new(TriggerReason::Manual), ) .await?; combined_report.merge(report); @@ -355,7 +362,13 @@ impl GcScheduler { if !dropped_regions.is_empty() { let report = self .ctx - .gc_regions(&dropped_regions, true, gc_timeout, dropped_routes_override) + .gc_regions( + &dropped_regions, + true, + gc_timeout, + dropped_routes_override, + EventContext::new(TriggerReason::Manual), + ) .await?; combined_report.merge(report); } @@ -477,6 +490,7 @@ mod tests { _full_file_listing: bool, _timeout: Duration, _region_routes_override: Region2Peers, + _event_context: EventContext, ) -> Result { self.gc_regions_calls.fetch_add(1, Ordering::Relaxed); panic!("gc_regions should not be called in maintenance mode") @@ -550,6 +564,7 @@ mod tests { _full_file_listing: bool, _timeout: Duration, _region_routes_override: Region2Peers, + _event_context: EventContext, ) -> Result { Ok(GcReport::default()) } @@ -883,6 +898,7 @@ mod tests { _full_file_listing: bool, _timeout: Duration, _region_routes_override: Region2Peers, + _event_context: EventContext, ) -> Result { crate::error::UnexpectedSnafu { violated: "mock gc failure".to_string(), diff --git a/src/meta-srv/src/procedure/region_migration.rs b/src/meta-srv/src/procedure/region_migration.rs index 23a48381ebd..25fc7d6a309 100644 --- a/src/meta-srv/src/procedure/region_migration.rs +++ b/src/meta-srv/src/procedure/region_migration.rs @@ -45,11 +45,13 @@ use common_meta::kv_backend::{KvBackendRef, ResettableKvBackendRef}; use common_meta::lock_key::{CatalogLock, RegionLock, SchemaLock, TableLock}; use common_meta::peer::Peer; use common_meta::region_keeper::{MemoryRegionKeeperRef, OperatingRegionGuard}; +use common_meta::rpc::ddl::{EventContext, TriggerReason}; use common_procedure::error::{ Error as ProcedureError, FromJsonSnafu, Result as ProcedureResult, ToJsonSnafu, }; use common_procedure::{ - Context as ProcedureContext, EventContext, LockKey, Procedure, Status, StringKey, + Context as ProcedureContext, EventRuntimeContext, EventTrigger, LockKey, Procedure, Status, + StringKey, }; use common_telemetry::{debug, error, info}; use manager::RegionMigrationProcedureGuard; @@ -124,6 +126,8 @@ pub struct PersistentContext { /// The trigger reason of region migration. #[serde(default)] pub(crate) trigger_reason: RegionMigrationTriggerReason, + #[serde(default)] + pub(crate) event_context: EventContext, } impl PersistentContext { @@ -133,7 +137,7 @@ impl PersistentContext { to_peer: Peer, region_ids: Vec, timeout: Duration, - trigger_reason: RegionMigrationTriggerReason, + event_context: EventContext, ) -> Self { #[allow(deprecated)] Self { @@ -144,7 +148,37 @@ impl PersistentContext { to_peer, region_ids, timeout, - trigger_reason, + trigger_reason: RegionMigrationTriggerReason::from_trigger_reason(event_context.reason), + event_context, + } + } + + fn normalize_trigger_reasons(&mut self) { + if self.trigger_reason != RegionMigrationTriggerReason::Unknown { + self.event_context.reason = self.trigger_reason.to_trigger_reason(); + } else if self.event_context.reason != TriggerReason::Unknown { + self.trigger_reason = + RegionMigrationTriggerReason::from_trigger_reason(self.event_context.reason); + } + } +} + +impl RegionMigrationTriggerReason { + fn to_trigger_reason(self) -> TriggerReason { + match self { + Self::Manual => TriggerReason::Manual, + Self::AutoRebalance => TriggerReason::AutoRebalance, + Self::Failover => TriggerReason::RegionFailover, + Self::Unknown => TriggerReason::Unknown, + } + } + + fn from_trigger_reason(reason: TriggerReason) -> Self { + match reason { + TriggerReason::Manual => Self::Manual, + TriggerReason::AutoRebalance => Self::AutoRebalance, + TriggerReason::RegionFailover => Self::Failover, + _ => Self::Unknown, } } } @@ -818,9 +852,10 @@ impl RegionMigrationProcedure { tracker: RegionMigrationProcedureTracker, ) -> ProcedureResult { let RegionMigrationDataOwned { - persistent_ctx, + mut persistent_ctx, state, } = serde_json::from_str(json).context(FromJsonSnafu)?; + persistent_ctx.normalize_trigger_reasons(); let guards = persistent_ctx .region_ids .iter() @@ -964,13 +999,14 @@ impl Procedure for RegionMigrationProcedure { LockKey::new(self.context.persistent_ctx.lock_key()) } - fn event(&self, ctx: &EventContext<'_>) -> Option> { + fn event(&self, ctx: &EventRuntimeContext<'_>) -> Option> { if !ctx.event_type_filter.allows(REGION_MIGRATION_EVENT_TYPE) { return None; } Some(Box::new(RegionMigrationEvent::from_persistent_ctx( &self.context.persistent_ctx, + matches!(ctx.trigger, EventTrigger::Submitted), ))) } } @@ -1027,7 +1063,7 @@ mod tests { let procedure = RegionMigrationProcedure::new(persistent_context, context, vec![]); let serialized = procedure.dump().unwrap(); - let expected = r#"{"persistent_ctx":{"catalog_and_schema":[["greptime","public"]],"from_peer":{"id":1,"addr":""},"to_peer":{"id":2,"addr":""},"region_ids":[4398046511105],"timeout":"10s","trigger_reason":"Unknown"},"state":{"region_migration_state":"RegionMigrationStart"}}"#; + let expected = r#"{"persistent_ctx":{"catalog_and_schema":[["greptime","public"]],"from_peer":{"id":1,"addr":""},"to_peer":{"id":2,"addr":""},"region_ids":[4398046511105],"timeout":"10s","trigger_reason":"Unknown","event_context":{"reason":"unknown"}},"state":{"region_migration_state":"RegionMigrationStart"}}"#; assert_eq!(expected, serialized); } @@ -1051,7 +1087,7 @@ mod tests { for trigger in triggers { let event = procedure - .event(&EventContext { + .event(&EventRuntimeContext { procedure_id: common_procedure::ProcedureId::random(), lifecycle_state: &state, trigger, @@ -1076,12 +1112,60 @@ mod tests { region_ids: vec![RegionId::new(1024, 1)], timeout: Duration::from_secs(10), trigger_reason: RegionMigrationTriggerReason::default(), + event_context: EventContext::default(), }; // NOTES: Changes it will break backward compatibility. let serialized = r#"{"catalog":"greptime","schema":"public","from_peer":{"id":1,"addr":""},"to_peer":{"id":2,"addr":""},"region_id":4398046511105}"#; let deserialized: PersistentContext = serde_json::from_str(serialized).unwrap(); assert_eq!(persistent_ctx, deserialized); + assert_eq!(deserialized.event_context, EventContext::default()); + } + + #[test] + fn test_new_persistent_context_keeps_event_context() { + let mut event_context = + EventContext::new(TriggerReason::AutoRebalance).with_protocol("internal"); + event_context + .extensions + .insert("scheduler".into(), serde_json::json!("load_based")); + + let context = PersistentContext::new( + vec![("greptime".into(), "public".into())], + Peer::empty(1), + Peer::empty(2), + vec![RegionId::new(1024, 1)], + Duration::from_secs(10), + event_context.clone(), + ); + + assert_eq!( + context.trigger_reason, + RegionMigrationTriggerReason::AutoRebalance + ); + assert_eq!(context.event_context, event_context); + } + + #[test] + fn test_normalize_trigger_reasons() { + let mut context = new_persistent_context(); + context.trigger_reason = RegionMigrationTriggerReason::Manual; + context.event_context = EventContext::default(); + context.normalize_trigger_reasons(); + assert_eq!(context.event_context.reason, TriggerReason::Manual); + + context.trigger_reason = RegionMigrationTriggerReason::Unknown; + context.event_context = EventContext::new(TriggerReason::RegionFailover); + context.normalize_trigger_reasons(); + assert_eq!( + context.trigger_reason, + RegionMigrationTriggerReason::Failover + ); + + context.trigger_reason = RegionMigrationTriggerReason::Manual; + context.event_context = EventContext::new(TriggerReason::RegionFailover); + context.normalize_trigger_reasons(); + assert_eq!(context.event_context.reason, TriggerReason::Manual); } #[derive(Debug, Serialize, Deserialize, Default)] diff --git a/src/meta-srv/src/procedure/region_migration/downgrade_leader_region.rs b/src/meta-srv/src/procedure/region_migration/downgrade_leader_region.rs index 1aee76b89f0..c58c1f8cd1f 100644 --- a/src/meta-srv/src/procedure/region_migration/downgrade_leader_region.rs +++ b/src/meta-srv/src/procedure/region_migration/downgrade_leader_region.rs @@ -382,6 +382,7 @@ mod tests { use common_meta::key::table_route::TableRouteValue; use common_meta::key::test_utils::new_test_table_info; use common_meta::peer::Peer; + use common_meta::rpc::ddl::{EventContext, TriggerReason}; use common_meta::rpc::router::{Region, RegionRoute}; use common_meta::wal_provider::RegionWalOptions; use store_api::storage::RegionId; @@ -389,7 +390,6 @@ mod tests { use super::*; use crate::error::Error; - use crate::procedure::region_migration::manager::RegionMigrationTriggerReason; use crate::procedure::region_migration::test_util::{TestingEnv, new_procedure_context}; use crate::procedure::region_migration::{ContextFactory, PersistentContext}; use crate::procedure::test_util::{ @@ -403,7 +403,7 @@ mod tests { Peer::empty(2), vec![RegionId::new(1024, 1)], Duration::from_millis(1000), - RegionMigrationTriggerReason::Manual, + EventContext::new(TriggerReason::Manual), ) } diff --git a/src/meta-srv/src/procedure/region_migration/manager.rs b/src/meta-srv/src/procedure/region_migration/manager.rs index d18458b08f4..3de79735e3b 100644 --- a/src/meta-srv/src/procedure/region_migration/manager.rs +++ b/src/meta-srv/src/procedure/region_migration/manager.rs @@ -21,6 +21,7 @@ use std::time::Duration; use common_meta::key::table_info::TableInfoValue; use common_meta::key::table_route::TableRouteValue; use common_meta::peer::Peer; +use common_meta::rpc::ddl::EventContext; use common_meta::rpc::router::RegionRoute; use common_procedure::{ProcedureId, ProcedureManagerRef, ProcedureWithId, watcher}; use common_telemetry::{error, info, warn}; @@ -485,7 +486,7 @@ impl RegionMigrationManager { task.to_peer.clone(), task.region_ids.clone(), task.timeout, - task.trigger_reason, + EventContext::new(task.trigger_reason.to_trigger_reason()), ), self.context_factory.clone(), procedure_guards, @@ -581,7 +582,7 @@ impl RegionMigrationManager { to_peer, vec![region_id], timeout, - trigger_reason, + EventContext::new(trigger_reason.to_trigger_reason()), ), self.context_factory.clone(), vec![guard], diff --git a/src/meta-srv/src/procedure/region_migration/test_util.rs b/src/meta-srv/src/procedure/region_migration/test_util.rs index 269357bc0ac..70342608646 100644 --- a/src/meta-srv/src/procedure/region_migration/test_util.rs +++ b/src/meta-srv/src/procedure/region_migration/test_util.rs @@ -27,6 +27,7 @@ use common_meta::kv_backend::KvBackendRef; use common_meta::kv_backend::memory::MemoryKvBackend; use common_meta::peer::Peer; use common_meta::region_keeper::{MemoryRegionKeeper, MemoryRegionKeeperRef}; +use common_meta::rpc::ddl::EventContext; use common_meta::rpc::router::RegionRoute; use common_meta::sequence::SequenceBuilder; use common_meta::state_store::KvStateStore; @@ -45,9 +46,7 @@ use crate::metasrv::MetasrvInfo; use crate::procedure::region_migration::close_downgraded_region::CloseDowngradedRegion; use crate::procedure::region_migration::downgrade_leader_region::DowngradeLeaderRegion; use crate::procedure::region_migration::flush_leader_region::PreFlushRegion; -use crate::procedure::region_migration::manager::{ - RegionMigrationProcedureTracker, RegionMigrationTriggerReason, -}; +use crate::procedure::region_migration::manager::RegionMigrationProcedureTracker; use crate::procedure::region_migration::migration_abort::RegionMigrationAbort; use crate::procedure::region_migration::migration_end::RegionMigrationEnd; use crate::procedure::region_migration::open_candidate_region::OpenCandidateRegion; @@ -191,7 +190,7 @@ pub fn new_persistent_context(from: u64, to: u64, region_id: RegionId) -> Persis Peer::empty(to), vec![region_id], Duration::from_secs(10), - RegionMigrationTriggerReason::default(), + EventContext::default(), ) } diff --git a/src/meta-srv/src/procedure/region_migration/upgrade_candidate_region.rs b/src/meta-srv/src/procedure/region_migration/upgrade_candidate_region.rs index debe72f4632..16d0270354b 100644 --- a/src/meta-srv/src/procedure/region_migration/upgrade_candidate_region.rs +++ b/src/meta-srv/src/procedure/region_migration/upgrade_candidate_region.rs @@ -356,6 +356,7 @@ mod tests { use common_meta::key::topic_name::TopicNameKey; use common_meta::key::topic_region::{ReplayCheckpoint, TopicRegionKey, TopicRegionValue}; use common_meta::peer::Peer; + use common_meta::rpc::ddl::{EventContext, TriggerReason}; use common_meta::rpc::router::{Region, RegionRoute}; use common_meta::wal_provider::RegionWalOptions; use common_wal::options::KafkaWalOptions; @@ -363,7 +364,6 @@ mod tests { use super::*; use crate::error::Error; - use crate::procedure::region_migration::manager::RegionMigrationTriggerReason; use crate::procedure::region_migration::test_util::{TestingEnv, new_procedure_context}; use crate::procedure::region_migration::{ContextFactory, PersistentContext}; use crate::procedure::test_util::{ @@ -377,7 +377,7 @@ mod tests { Peer::empty(2), vec![RegionId::new(1024, 1)], Duration::from_millis(1000), - RegionMigrationTriggerReason::Manual, + EventContext::new(TriggerReason::Manual), ) } diff --git a/src/meta-srv/src/procedure/repartition.rs b/src/meta-srv/src/procedure/repartition.rs index f8452d196fb..62f9f26aabd 100644 --- a/src/meta-srv/src/procedure/repartition.rs +++ b/src/meta-srv/src/procedure/repartition.rs @@ -44,11 +44,12 @@ use common_meta::lock_key::{CatalogLock, SchemaLock, TableLock, TableNameLock}; use common_meta::node_manager::NodeManagerRef; use common_meta::region_keeper::{MemoryRegionKeeperRef, OperatingRegionGuard}; use common_meta::region_registry::LeaderRegionRegistryRef; +use common_meta::rpc::ddl::EventContext; use common_meta::rpc::router::{RegionRoute, operating_leader_region_roles}; use common_meta::wal_provider::RegionWalOptions; use common_procedure::error::{FromJsonSnafu, ToJsonSnafu}; use common_procedure::{ - BoxedProcedure, Context as ProcedureContext, Error as ProcedureError, EventContext, + BoxedProcedure, Context as ProcedureContext, Error as ProcedureError, EventRuntimeContext, EventTrigger, LockKey, Procedure, ProcedureManagerRef, Result as ProcedureResult, Status, StringKey, }; @@ -103,6 +104,8 @@ pub struct PersistentContext { #[serde(default)] /// Records table-level partition metadata updated by this repartition. pub partition_metadata_update: Option, + #[serde(default)] + pub event_context: EventContext, } fn default_timeout() -> Duration { @@ -110,7 +113,7 @@ fn default_timeout() -> Duration { } impl PersistentContext { - /// Creates a new [PersistentContext] with the given table name, table id and timeout. + /// Creates a new [PersistentContext] with the table, timeout and event context. /// /// If the timeout is not provided, the default timeout will be used. pub fn new( @@ -121,6 +124,7 @@ impl PersistentContext { }: TableName, table_id: TableId, timeout: Option, + event_context: EventContext, ) -> Self { Self { catalog_name, @@ -132,6 +136,7 @@ impl PersistentContext { unknown_procedures: vec![], timeout: timeout.unwrap_or_else(default_timeout), partition_metadata_update: None, + event_context, } } @@ -792,7 +797,10 @@ impl Procedure for RepartitionProcedure { LockKey::new(self.context.persistent_ctx.lock_key()) } - fn event(&self, ctx: &EventContext<'_>) -> Option> { + fn event( + &self, + ctx: &EventRuntimeContext<'_>, + ) -> Option> { if !ctx.event_type_filter.allows(REPARTITION_EVENT_TYPE) { return None; } @@ -861,6 +869,7 @@ impl RepartitionProcedureFactory for GcDisabledRepartitionProcedureFactory { _source: RepartitionSource, _to_exprs: Vec, _timeout: Option, + _event_context: EventContext, ) -> std::result::Result { Err(BoxedError::new( error::InvalidArgumentsSnafu { @@ -899,8 +908,9 @@ impl RepartitionProcedureFactory for DefaultRepartitionProcedureFactory { source: RepartitionSource, to_exprs: Vec, timeout: Option, + event_context: EventContext, ) -> std::result::Result { - let persistent_ctx = PersistentContext::new(table_name, table_id, timeout); + let persistent_ctx = PersistentContext::new(table_name, table_id, timeout, event_context); let from = match source { RepartitionSource::Partitioned { exprs, @@ -1133,6 +1143,7 @@ mod tests { TableName::new("test_catalog", "test_schema", "test_table"), table_id, None, + EventContext::default(), ); Context::new( @@ -1164,6 +1175,7 @@ mod tests { }, vec![], None, + EventContext::default(), ) .err() .expect("GC-disabled factory must reject repartition"); @@ -1265,7 +1277,7 @@ mod tests { let all = Arc::new(EventTypeFilter::All); let submitted = procedure - .event(&EventContext { + .event(&EventRuntimeContext { procedure_id: ProcedureId::random(), lifecycle_state: &state, trigger: EventTrigger::Submitted, @@ -1276,7 +1288,7 @@ mod tests { assert_ne!(submitted.json_payload().unwrap(), serde_json::Value::Null); let allowed = procedure - .event(&EventContext { + .event(&EventRuntimeContext { procedure_id: ProcedureId::random(), lifecycle_state: &state, trigger: EventTrigger::Submitted, @@ -1288,7 +1300,7 @@ mod tests { assert_eq!(allowed.event_type(), REPARTITION_EVENT_TYPE); let succeeded = procedure - .event(&EventContext { + .event(&EventRuntimeContext { procedure_id: ProcedureId::random(), lifecycle_state: &state, trigger: EventTrigger::Succeeded, @@ -1297,7 +1309,7 @@ mod tests { .unwrap(); assert_eq!(succeeded.json_payload().unwrap(), serde_json::Value::Null); - let filtered = procedure.event(&EventContext { + let filtered = procedure.event(&EventRuntimeContext { procedure_id: ProcedureId::random(), lifecycle_state: &state, trigger: EventTrigger::Submitted, @@ -1307,7 +1319,7 @@ mod tests { }); assert!(filtered.is_none()); - let empty = procedure.event(&EventContext { + let empty = procedure.event(&EventRuntimeContext { procedure_id: ProcedureId::random(), lifecycle_state: &state, trigger: EventTrigger::Submitted, @@ -1370,7 +1382,7 @@ mod tests { } #[test] - fn test_persistent_context_partition_metadata_update_serde_default() { + fn test_persistent_context_serde_defaults_for_new_fields() { let json = r#"{ "catalog_name":"test_catalog", "schema_name":"test_schema", @@ -1382,7 +1394,10 @@ mod tests { let persistent_ctx: PersistentContext = serde_json::from_str(json).unwrap(); + assert!(persistent_ctx.failed_procedures.is_empty()); + assert!(persistent_ctx.unknown_procedures.is_empty()); assert!(persistent_ctx.partition_metadata_update.is_none()); + assert_eq!(persistent_ctx.event_context, EventContext::default()); } #[tokio::test] @@ -1459,6 +1474,7 @@ mod tests { TableName::new("test_catalog", "test_schema", "test_table"), table_id, None, + EventContext::default(), ); persistent_ctx.plans = vec![with_rollback_metadata( test_plan(table_id), @@ -1525,6 +1541,7 @@ mod tests { TableName::new("test_catalog", "test_schema", "test_table"), table_id, None, + EventContext::default(), ); persistent_ctx.plans = vec![test_plan(table_id)]; let context = Context::new( @@ -1578,6 +1595,7 @@ mod tests { TableName::new("test_catalog", "test_schema", "test_table"), table_id, None, + EventContext::default(), ); let failed_plan = test_plan(table_id); let failed_plan = with_rollback_metadata( @@ -1711,6 +1729,7 @@ mod tests { TableName::new("test_catalog", "test_schema", "test_table"), table_id, None, + EventContext::default(), ); persistent_ctx.plans = vec![failed_plan, succeeded_plan.clone()]; persistent_ctx.failed_procedures = vec![ProcedureMeta { @@ -1820,6 +1839,7 @@ mod tests { TableName::new("test_catalog", "test_schema", "test_table"), table_id, None, + EventContext::default(), ); persistent_ctx.plans = vec![plan.clone()]; persistent_ctx.unknown_procedures = vec![ProcedureMeta { @@ -1881,6 +1901,7 @@ mod tests { TableName::new("test_catalog", "test_schema", "test_table"), table_id, None, + EventContext::default(), ); persistent_ctx.plans = vec![with_rollback_metadata( test_plan(table_id), @@ -2009,6 +2030,7 @@ mod tests { TableName::new("test_catalog", "test_schema", "test_table"), table_id, None, + EventContext::default(), ); persistent_ctx.plans = vec![failed_merge_plan, succeeded_split_plan.clone()]; persistent_ctx.failed_procedures = vec![ProcedureMeta { diff --git a/src/meta-srv/src/procedure/repartition/collect.rs b/src/meta-srv/src/procedure/repartition/collect.rs index 1a6d0c62574..56d9a007a16 100644 --- a/src/meta-srv/src/procedure/repartition/collect.rs +++ b/src/meta-srv/src/procedure/repartition/collect.rs @@ -136,6 +136,7 @@ mod tests { use common_error::mock::MockError; use common_error::status_code::StatusCode; + use common_meta::rpc::ddl::EventContext; use common_meta::test_util::MockDatanodeManager; use common_procedure::{ Context as ProcedureContext, ContextProvider, Error as ProcedureError, ProcedureId, @@ -193,6 +194,7 @@ mod tests { table::table_name::TableName::new("test_catalog", "test_schema", "test_table"), 1024, None, + EventContext::default(), ); let mut ctx = crate::procedure::repartition::Context::new( &ddl_ctx, @@ -227,6 +229,7 @@ mod tests { table::table_name::TableName::new("test_catalog", "test_schema", "test_table"), 1024, None, + EventContext::default(), ); let mut ctx = crate::procedure::repartition::Context::new( &ddl_ctx, diff --git a/src/meta-srv/src/procedure/repartition/group.rs b/src/meta-srv/src/procedure/repartition/group.rs index e8005a21471..700c2fdcd1d 100644 --- a/src/meta-srv/src/procedure/repartition/group.rs +++ b/src/meta-srv/src/procedure/repartition/group.rs @@ -38,8 +38,8 @@ use common_meta::peer::Peer; use common_meta::rpc::router::RegionRoute; use common_procedure::error::{FromJsonSnafu, ToJsonSnafu}; use common_procedure::{ - Context as ProcedureContext, Error as ProcedureError, EventContext, EventTrigger, LockKey, - Procedure, ProcedureId, Result as ProcedureResult, Status, StringKey, + Context as ProcedureContext, Error as ProcedureError, EventRuntimeContext, EventTrigger, + LockKey, Procedure, ProcedureId, Result as ProcedureResult, Status, StringKey, }; use common_telemetry::{error, info}; use serde::{Deserialize, Serialize}; @@ -262,7 +262,10 @@ impl Procedure for RepartitionGroupProcedure { LockKey::new(self.context.persistent_ctx.lock_key()) } - fn event(&self, ctx: &EventContext<'_>) -> Option> { + fn event( + &self, + ctx: &EventRuntimeContext<'_>, + ) -> Option> { if !ctx.event_type_filter.allows(REPARTITION_GROUP_EVENT_TYPE) { return None; } @@ -631,7 +634,9 @@ mod tests { use common_event_recorder::EventTypeFilter; use common_meta::key::TableMetadataManager; use common_meta::kv_backend::test_util::MockKvBackendBuilder; - use common_procedure::{EventContext, EventTrigger, Procedure, ProcedureId, ProcedureState}; + use common_procedure::{ + EventRuntimeContext, EventTrigger, Procedure, ProcedureId, ProcedureState, + }; use crate::error::Error; use crate::event::repartition::REPARTITION_GROUP_EVENT_TYPE; @@ -708,7 +713,7 @@ mod tests { context: env.create_context(new_persistent_context(1024, vec![], vec![])), }; let state = ProcedureState::Running; - let event_context = |event_type_filter| EventContext { + let runtime_context = |event_type_filter| EventRuntimeContext { procedure_id: ProcedureId::random(), lifecycle_state: &state, trigger: EventTrigger::Submitted, @@ -716,7 +721,7 @@ mod tests { }; let allowed = procedure - .event(&event_context(EventTypeFilter::Only(HashSet::from([ + .event(&runtime_context(EventTypeFilter::Only(HashSet::from([ REPARTITION_GROUP_EVENT_TYPE.to_string(), ])))) .unwrap(); @@ -724,14 +729,14 @@ mod tests { assert!( procedure - .event(&event_context(EventTypeFilter::Only(HashSet::from([ + .event(&runtime_context(EventTypeFilter::Only(HashSet::from([ "another_event".to_string(), ])))) .is_none() ); assert!( procedure - .event(&event_context(EventTypeFilter::Only(HashSet::new()))) + .event(&runtime_context(EventTypeFilter::Only(HashSet::new()))) .is_none() ); } diff --git a/src/meta-srv/src/procedure/repartition/test_util.rs b/src/meta-srv/src/procedure/repartition/test_util.rs index 74289cdff9d..a6f2330b83a 100644 --- a/src/meta-srv/src/procedure/repartition/test_util.rs +++ b/src/meta-srv/src/procedure/repartition/test_util.rs @@ -24,6 +24,7 @@ use common_meta::kv_backend::KvBackendRef; use common_meta::kv_backend::memory::MemoryKvBackend; use common_meta::node_manager::NodeManagerRef; use common_meta::peer::Peer; +use common_meta::rpc::ddl::EventContext; use common_meta::rpc::router::{Region, RegionRoute}; use common_meta::sequence::SequenceBuilder; use common_meta::test_util::new_ddl_context_with_kv_backend; @@ -224,6 +225,7 @@ pub fn new_parent_context( TableName::new("test_catalog", "test_schema", "test_table"), table_id, None, + EventContext::default(), ); ParentContext::new( diff --git a/src/meta-srv/src/procedure/tests.rs b/src/meta-srv/src/procedure/tests.rs index 4f70dcca95c..dd855c66aa0 100644 --- a/src/meta-srv/src/procedure/tests.rs +++ b/src/meta-srv/src/procedure/tests.rs @@ -29,7 +29,7 @@ use common_meta::ddl::test_util::create_table::{ }; use common_meta::key::table_route::{PhysicalTableRouteValue, TableRouteValue}; use common_meta::node_manager::NodeManagerRef; -use common_meta::rpc::ddl::CreateTableTask; +use common_meta::rpc::ddl::{CreateTableTask, EventContext, QueryContext}; use common_meta::rpc::router::{RegionRoute, find_leaders}; use common_procedure::Status; use store_api::storage::RegionId; @@ -91,6 +91,8 @@ fn create_table_task(table_name: Option<&str>) -> CreateTableTask { fn test_region_request_builder() { let mut procedure = CreateTableProcedure::new( create_table_task(None), + QueryContext::default(), + EventContext::default(), test_data::new_ddl_context(Arc::new(NodeClients::default())), ) .unwrap(); @@ -188,6 +190,8 @@ async fn test_on_datanode_create_regions() { let mut procedure = CreateTableProcedure::new( create_table_task(None), + QueryContext::default(), + EventContext::default(), test_data::new_ddl_context(node_manager), ) .unwrap(); @@ -260,8 +264,12 @@ async fn test_on_datanode_create_logical_regions() { .unwrap() .0; let _ = kv_backend.txn(physical_route_txn).await.unwrap(); - let mut procedure = - CreateLogicalTablesProcedure::new(vec![task1, task2, task3], physical_table_id, ctx); + let mut procedure = CreateLogicalTablesProcedure::new( + vec![task1, task2, task3], + physical_table_id, + EventContext::default(), + ctx, + ); let expected_created_regions = Arc::new(Mutex::new(HashMap::from([(1, 3), (2, 3), (3, 3)]))); diff --git a/src/meta-srv/src/procedure/wal_prune.rs b/src/meta-srv/src/procedure/wal_prune.rs index b5b556a778d..128fc58c44d 100644 --- a/src/meta-srv/src/procedure/wal_prune.rs +++ b/src/meta-srv/src/procedure/wal_prune.rs @@ -25,8 +25,8 @@ use common_meta::lock_key::RemoteWalLock; use common_meta::region_registry::LeaderRegionRegistryRef; use common_procedure::error::ToJsonSnafu; use common_procedure::{ - Context as ProcedureContext, Error as ProcedureError, EventContext, EventTrigger, LockKey, - Procedure, ProcedureState, Result as ProcedureResult, Status, StringKey, + Context as ProcedureContext, Error as ProcedureError, EventRuntimeContext, EventTrigger, + LockKey, Procedure, ProcedureState, Result as ProcedureResult, Status, StringKey, }; use common_telemetry::{info, warn}; use manager::{WalPruneProcedureGuard, WalPruneProcedureTracker}; @@ -213,7 +213,10 @@ impl Procedure for WalPruneProcedure { LockKey::new(vec![lock_key]) } - fn event(&self, ctx: &EventContext<'_>) -> Option> { + fn event( + &self, + ctx: &EventRuntimeContext<'_>, + ) -> Option> { if !ctx.event_type_filter.allows(WAL_PRUNE_EVENT_TYPE) { return None; } @@ -503,7 +506,7 @@ mod tests { let mut procedure = WalPruneProcedure::new(context, None, "test_topic".to_string(), 42, false); let running = ProcedureState::Running; - let event_context = |trigger, lifecycle_state, event_type_filter| EventContext { + let runtime_context = |trigger, lifecycle_state, event_type_filter| EventRuntimeContext { procedure_id: ProcedureId::random(), lifecycle_state, trigger, @@ -513,13 +516,13 @@ mod tests { for trigger in [EventTrigger::Submitted, EventTrigger::Recovered] { assert!( procedure - .event(&event_context(trigger, &running, EventTypeFilter::All)) + .event(&runtime_context(trigger, &running, EventTypeFilter::All)) .is_none() ); } let retrying_event = procedure - .event(&event_context( + .event(&runtime_context( EventTrigger::Retrying { phase: common_procedure::RetryPhase::Execute, attempt: 1, @@ -537,7 +540,7 @@ mod tests { procedure.observed_latest_offset = Some(100); for trigger in [EventTrigger::Failed, EventTrigger::Poisoned] { let event = procedure - .event(&event_context(trigger, &running, EventTypeFilter::All)) + .event(&runtime_context(trigger, &running, EventTypeFilter::All)) .unwrap(); assert_eq!( event.extra_rows().unwrap()[0].values[2].value_data, @@ -548,7 +551,7 @@ mod tests { let done_without_output = ProcedureState::Done { output: None }; assert!( procedure - .event(&event_context( + .event(&runtime_context( EventTrigger::Succeeded, &done_without_output, EventTypeFilter::All, @@ -560,7 +563,7 @@ mod tests { }; assert!( procedure - .event(&event_context( + .event(&runtime_context( EventTrigger::Succeeded, &done_with_wrong_output, EventTypeFilter::All, @@ -572,7 +575,7 @@ mod tests { output: Some(Arc::new(WalPruneOutcome) as Output), }; let event = procedure - .event(&event_context( + .event(&runtime_context( EventTrigger::Succeeded, &done, EventTypeFilter::All, @@ -589,7 +592,7 @@ mod tests { assert!( procedure - .event(&event_context( + .event(&runtime_context( EventTrigger::Succeeded, &done, EventTypeFilter::Only(HashSet::new()), diff --git a/src/meta-srv/src/service/procedure.rs b/src/meta-srv/src/service/procedure.rs index 87ec36def3e..b20d7f64775 100644 --- a/src/meta-srv/src/service/procedure.rs +++ b/src/meta-srv/src/service/procedure.rs @@ -102,7 +102,6 @@ impl procedure_service_server::ProcedureService for Metasrv { .try_into() .context(error::ConvertProtoDataSnafu)?; restore_create_database_creator(&metadata, header.role, &mut task, &mut query_context)?; - let resp = self .ddl_manager() .submit_ddl_task( diff --git a/src/operator/src/insert.rs b/src/operator/src/insert.rs index 74ecb6b3cd8..1834b4f976a 100644 --- a/src/operator/src/insert.rs +++ b/src/operator/src/insert.rs @@ -36,6 +36,7 @@ use common_grpc_expr::util::ColumnExpr; use common_meta::cache::TableFlownodeSetCacheRef; use common_meta::node_manager::{AffectedRows, NodeManagerRef}; use common_meta::peer::Peer; +use common_meta::rpc::ddl::TriggerReason; use common_query::Output; use common_query::native_histogram::{is_native_histogram_value_type, native_histogram_value_type}; use common_query::prelude::{greptime_timestamp, greptime_value}; @@ -656,7 +657,7 @@ impl Inserter { if !alter_tables.is_empty() { // Alter logical tables in batch. statement_executor - .alter_logical_tables(alter_tables, ctx.clone()) + .alter_logical_tables(alter_tables, ctx.clone(), TriggerReason::AutoAlter) .await?; } } @@ -677,7 +678,7 @@ impl Inserter { } for alter_expr in alter_tables.into_iter() { statement_executor - .alter_table_inner(alter_expr, ctx.clone()) + .alter_table_inner(alter_expr, ctx.clone(), TriggerReason::AutoAlter) .await?; } } @@ -786,7 +787,7 @@ impl Inserter { } for alter_expr in alter_tables.into_iter() { statement_executor - .alter_table_inner(alter_expr, ctx.clone()) + .alter_table_inner(alter_expr, ctx.clone(), TriggerReason::AutoAlter) .await?; } } @@ -872,7 +873,12 @@ impl Inserter { // create physical table let res = statement_executor - .create_table_inner(create_table_expr, None, ctx.clone()) + .create_table_inner( + create_table_expr, + None, + ctx.clone(), + TriggerReason::AutoCreate, + ) .await; match res { @@ -1060,7 +1066,12 @@ impl Inserter { info!("Table `{table_ref}` does not exist, try creating table"); } let res = statement_executor - .create_table_inner(&mut create_table_expr, partitions, ctx.clone()) + .create_table_inner( + &mut create_table_expr, + partitions, + ctx.clone(), + TriggerReason::AutoCreate, + ) .await; let table_ref = TableReference::full( @@ -1091,7 +1102,7 @@ impl Inserter { statement_executor: &StatementExecutor, ) -> Result> { let res = statement_executor - .create_logical_tables(&create_table_exprs, ctx.clone()) + .create_logical_tables(&create_table_exprs, ctx.clone(), TriggerReason::AutoCreate) .await; match res { diff --git a/src/operator/src/procedure.rs b/src/operator/src/procedure.rs index cd2a0842f1a..1a849afaf7a 100644 --- a/src/operator/src/procedure.rs +++ b/src/operator/src/procedure.rs @@ -19,7 +19,7 @@ use common_error::ext::BoxedError; use common_function::handlers::ProcedureServiceHandler; use common_meta::key::TableMetadataManagerRef; use common_meta::procedure_executor::{ExecutorContext, ProcedureExecutorRef}; -use common_meta::rpc::ddl::{DdlTask, SubmitDdlTaskRequest}; +use common_meta::rpc::ddl::{DdlTask, SubmitDdlTaskRequest, TriggerReason}; use common_meta::rpc::procedure::{ GcRegionsRequest as MetaGcRegionsRequest, GcResponse as MetaGcResponse, GcTableRequest as MetaGcTableRequest, ManageRegionFollowerRequest, MigrateRegionRequest, @@ -31,7 +31,7 @@ use snafu::ResultExt; use table::table_name::TableName; use crate::error; -use crate::utils::to_meta_query_context; +use crate::utils::to_meta_query_context_with_trigger_reason; /// The operator for procedures which implements [`ProcedureServiceHandler`]. #[derive(Clone)] @@ -76,8 +76,10 @@ impl ProcedureServiceHandler for ProcedureServiceOperator { }) .map_err(BoxedError::new) .context(query_error::ProcedureServiceSnafu)?; + let meta_query_context = + to_meta_query_context_with_trigger_reason(query_ctx, TriggerReason::Manual); let request = SubmitDdlTaskRequest::new( - to_meta_query_context(query_ctx), + meta_query_context, DdlTask::new_purge_dropped_table(dropped.table_id), ); self.procedure_executor @@ -161,7 +163,9 @@ mod tests { use common_meta::key::test_utils::new_test_table_info_with_name; use common_meta::kv_backend::memory::MemoryKvBackend; use common_meta::procedure_executor::{ProcedureExecutor, ProcedureExecutorRef}; - use common_meta::rpc::ddl::{DdlTask, SubmitDdlTaskRequest, SubmitDdlTaskResponse}; + use common_meta::rpc::ddl::{ + DdlTask, SubmitDdlTaskRequest, SubmitDdlTaskResponse, TRIGGER_REASON_EXTENSION_KEY, + }; use common_meta::rpc::procedure::{MigrateRegionResponse, ProcedureStateResponse}; use session::context::QueryContextBuilder; use table::table_name::TableName; @@ -277,6 +281,13 @@ mod tests { ); assert_eq!(requests[0].query_context.current_catalog, "catalog"); assert_eq!(requests[0].query_context.current_schema, "schema"); + assert_eq!( + Some(&TriggerReason::Manual.as_ref().to_string()), + requests[0] + .query_context + .extensions + .get(TRIGGER_REASON_EXTENSION_KEY) + ); } #[tokio::test] diff --git a/src/operator/src/statement/comment.rs b/src/operator/src/statement/comment.rs index dc3eff0b8d8..824287fc80b 100644 --- a/src/operator/src/statement/comment.rs +++ b/src/operator/src/statement/comment.rs @@ -16,7 +16,9 @@ use api::v1::CommentOnExpr; use common_error::ext::BoxedError; use common_meta::cache_invalidator::Context; use common_meta::procedure_executor::ExecutorContext; -use common_meta::rpc::ddl::{CommentObjectType, CommentOnTask, DdlTask, SubmitDdlTaskRequest}; +use common_meta::rpc::ddl::{ + CommentObjectType, CommentOnTask, DdlTask, SubmitDdlTaskRequest, TriggerReason, +}; use common_query::Output; use session::context::QueryContextRef; use session::table_name::table_idents_to_full_name; @@ -28,7 +30,7 @@ use crate::error::{ self, ExecuteDdlSnafu, ExternalSnafu, InvalidSqlSnafu, Result, TableMetadataManagerSnafu, }; use crate::statement::StatementExecutor; -use crate::utils::to_meta_query_context; +use crate::utils::to_meta_query_context_with_trigger_reason; impl StatementExecutor { /// Adds a comment to a database object (table, column, or flow). @@ -52,10 +54,10 @@ impl StatementExecutor { .context(TableMetadataManagerSnafu)?; let cache_idents = comment_on_task.cache_idents(); - let request = SubmitDdlTaskRequest::new( - to_meta_query_context(query_ctx), - DdlTask::new_comment_on(comment_on_task), - ); + let meta_query_context = + to_meta_query_context_with_trigger_reason(query_ctx, TriggerReason::Manual); + let request = + SubmitDdlTaskRequest::new(meta_query_context, DdlTask::new_comment_on(comment_on_task)); self.procedure_executor .submit_ddl_task(&ExecutorContext::default(), request) @@ -86,10 +88,10 @@ impl StatementExecutor { .context(TableMetadataManagerSnafu)?; let cache_idents = comment_on_task.cache_idents(); - let request = SubmitDdlTaskRequest::new( - to_meta_query_context(query_ctx), - DdlTask::new_comment_on(comment_on_task), - ); + let meta_query_context = + to_meta_query_context_with_trigger_reason(query_ctx, TriggerReason::Manual); + let request = + SubmitDdlTaskRequest::new(meta_query_context, DdlTask::new_comment_on(comment_on_task)); self.procedure_executor .submit_ddl_task(&ExecutorContext::default(), request) diff --git a/src/operator/src/statement/ddl.rs b/src/operator/src/statement/ddl.rs index 8f09566c52e..9ecff4249fd 100644 --- a/src/operator/src/statement/ddl.rs +++ b/src/operator/src/statement/ddl.rs @@ -53,7 +53,7 @@ use common_meta::rpc::ddl::trigger::CreateTriggerTask; use common_meta::rpc::ddl::trigger::DropTriggerTask; use common_meta::rpc::ddl::{ CreateFlowTask, CreatorGrantIntent, DdlTask, DropFlowTask, DropViewTask, SubmitDdlTaskRequest, - SubmitDdlTaskResponse, + SubmitDdlTaskResponse, TriggerReason, }; use common_query::Output; use common_recordbatch::{RecordBatch, RecordBatches}; @@ -116,7 +116,10 @@ use crate::error::{ use crate::expr_helper::{self, RepartitionRequest, RepartitionSource}; use crate::statement::StatementExecutor; use crate::statement::show::create_partitions_stmt; -use crate::utils::{to_meta_query_context, to_meta_query_context_with_origin_frontend}; +use crate::utils::{ + to_meta_query_context_with_origin_frontend_and_trigger_reason, + to_meta_query_context_with_trigger_reason, +}; #[derive(Debug, Clone, Copy)] struct DdlSubmitOptions { @@ -300,7 +303,7 @@ impl StatementExecutor { } } - self.create_table_inner(create_expr, stmt.partitions, ctx) + self.create_table_inner(create_expr, stmt.partitions, ctx, TriggerReason::Manual) .await } @@ -357,7 +360,8 @@ impl StatementExecutor { ); let create_expr = &mut expr_helper::create_to_expr(&create_stmt, &ctx)?; - self.create_table_inner(create_expr, partitions, ctx).await + self.create_table_inner(create_expr, partitions, ctx, TriggerReason::Manual) + .await } #[tracing::instrument(skip_all)] @@ -369,7 +373,8 @@ impl StatementExecutor { let create_expr = &mut expr_helper::create_external_expr(create_expr, &ctx, &self.local_file_access) .await?; - self.create_table_inner(create_expr, None, ctx).await + self.create_table_inner(create_expr, None, ctx, TriggerReason::Manual) + .await } #[tracing::instrument(skip_all)] @@ -378,6 +383,7 @@ impl StatementExecutor { create_table: &mut CreateTableExpr, partitions: Option, query_ctx: QueryContextRef, + trigger_reason: TriggerReason, ) -> Result { ensure!( !is_readonly_schema(&create_table.schema_name), @@ -398,16 +404,20 @@ impl StatementExecutor { .await?; } // Create logical tables - self.create_logical_tables(std::slice::from_ref(create_table), query_ctx) - .await? - .into_iter() - .next() - .context(error::UnexpectedSnafu { - violated: "expected to create logical tables", - }) + self.create_logical_tables( + std::slice::from_ref(create_table), + query_ctx, + trigger_reason, + ) + .await? + .into_iter() + .next() + .context(error::UnexpectedSnafu { + violated: "expected to create logical tables", + }) } else { // Create other normal table - self.create_non_logic_table(create_table, partitions, query_ctx) + self.create_non_logic_table(create_table, partitions, query_ctx, trigger_reason) .await } } @@ -418,6 +428,7 @@ impl StatementExecutor { create_table: &mut CreateTableExpr, partitions: Option, query_ctx: QueryContextRef, + trigger_reason: TriggerReason, ) -> Result { let _timer = crate::metrics::DIST_CREATE_TABLE.start_timer(); @@ -486,6 +497,7 @@ impl StatementExecutor { partitions, table_info.clone(), query_ctx, + trigger_reason, ) .await?; @@ -513,6 +525,7 @@ impl StatementExecutor { &self, create_table_exprs: &[CreateTableExpr], query_context: QueryContextRef, + trigger_reason: TriggerReason, ) -> Result> { let _timer = crate::metrics::DIST_CREATE_TABLES.start_timer(); ensure!( @@ -543,7 +556,7 @@ impl StatementExecutor { .collect::>(); let resp = self - .create_logical_tables_procedure(tables_data, query_context.clone()) + .create_logical_tables_procedure(tables_data, query_context.clone(), trigger_reason) .await?; let table_ids = resp.table_ids; @@ -706,7 +719,7 @@ impl StatementExecutor { .context(error::InvalidExprSnafu)?; let request = SubmitDdlTaskRequest::new( - to_meta_query_context(query_context), + to_meta_query_context_with_trigger_reason(query_context, TriggerReason::Manual), DdlTask::new_create_trigger(task), ); @@ -770,7 +783,7 @@ impl StatementExecutor { }) .context(error::InvalidExprSnafu)?; let request = SubmitDdlTaskRequest::new( - to_meta_query_context(query_context), + to_meta_query_context_with_trigger_reason(query_context, TriggerReason::Manual), DdlTask::new_create_flow(task), ); @@ -1083,7 +1096,7 @@ impl StatementExecutor { }; let request = SubmitDdlTaskRequest::new( - to_meta_query_context(ctx), + to_meta_query_context_with_trigger_reason(ctx, TriggerReason::Manual), DdlTask::new_create_view(expr, view_info.clone()), ); @@ -1169,7 +1182,7 @@ impl StatementExecutor { query_context: QueryContextRef, ) -> Result { let request = SubmitDdlTaskRequest::new( - to_meta_query_context(query_context), + to_meta_query_context_with_trigger_reason(query_context, TriggerReason::Manual), DdlTask::new_drop_flow(expr), ); @@ -1204,7 +1217,7 @@ impl StatementExecutor { query_context: QueryContextRef, ) -> Result { let request = SubmitDdlTaskRequest::new( - to_meta_query_context(query_context), + to_meta_query_context_with_trigger_reason(query_context, TriggerReason::Manual), DdlTask::new_drop_trigger(expr), ); @@ -1272,7 +1285,7 @@ impl StatementExecutor { query_context: QueryContextRef, ) -> Result { let request = SubmitDdlTaskRequest::new( - to_meta_query_context(query_context), + to_meta_query_context_with_trigger_reason(query_context, TriggerReason::Manual), DdlTask::new_drop_view(expr), ); @@ -1287,6 +1300,7 @@ impl StatementExecutor { &self, alter_table_exprs: Vec, query_context: QueryContextRef, + trigger_reason: TriggerReason, ) -> Result { let _timer = crate::metrics::DIST_ALTER_TABLES.start_timer(); ensure!( @@ -1332,7 +1346,8 @@ impl StatementExecutor { // Submit procedure for each physical table let mut handles = Vec::with_capacity(groups.len()); for (_physical_table_id, exprs) in groups { - let fut = self.alter_logical_tables_procedure(exprs, query_context.clone()); + let fut = + self.alter_logical_tables_procedure(exprs, query_context.clone(), trigger_reason); handles.push(fut); } let _results = futures::future::try_join_all(handles).await?; @@ -1514,7 +1529,8 @@ impl StatementExecutor { } let expr = expr_helper::to_alter_table_expr(alter_table, &query_context)?; - self.alter_table_inner(expr, query_context).await + self.alter_table_inner(expr, query_context, TriggerReason::Manual) + .await } #[tracing::instrument(skip_all)] @@ -1752,8 +1768,10 @@ impl StatementExecutor { source: Some(source), ..Default::default() }; + let meta_query_context = + to_meta_query_context_with_trigger_reason(query_context.clone(), TriggerReason::Manual); let mut req = SubmitDdlTaskRequest::new( - to_meta_query_context(query_context.clone()), + meta_query_context, DdlTask::new_alter_table(AlterTableExpr { catalog_name: request.catalog_name.clone(), schema_name: request.schema_name.clone(), @@ -1808,6 +1826,7 @@ impl StatementExecutor { &self, expr: AlterTableExpr, query_context: QueryContextRef, + trigger_reason: TriggerReason, ) -> Result { ensure!( !is_readonly_schema(&expr.schema_name), @@ -1862,12 +1881,12 @@ impl StatementExecutor { .await .context(TableMetadataManagerSnafu)?; + let meta_query_context = + to_meta_query_context_with_trigger_reason(query_context, trigger_reason); + let (req, invalidate_keys) = if physical_table_id == table_id { // This is physical table - let req = SubmitDdlTaskRequest::new( - to_meta_query_context(query_context), - DdlTask::new_alter_table(expr), - ); + let req = SubmitDdlTaskRequest::new(meta_query_context, DdlTask::new_alter_table(expr)); let invalidate_keys = vec![ CacheIdent::TableId(table_id), @@ -1878,7 +1897,7 @@ impl StatementExecutor { } else { // This is logical table let req = SubmitDdlTaskRequest::new( - to_meta_query_context(query_context), + meta_query_context, DdlTask::new_alter_logical_tables(vec![expr]), ); @@ -1992,6 +2011,7 @@ impl StatementExecutor { partitions: Vec, table_info: TableInfo, query_context: QueryContextRef, + trigger_reason: TriggerReason, ) -> Result { let partitions = partitions .into_iter() @@ -1999,7 +2019,11 @@ impl StatementExecutor { .collect::>>()?; let request = SubmitDdlTaskRequest::new( - to_meta_query_context_with_origin_frontend(query_context, &self.origin_frontend_addr), + to_meta_query_context_with_origin_frontend_and_trigger_reason( + query_context, + &self.origin_frontend_addr, + trigger_reason, + ), DdlTask::new_create_table(create_table, partitions, table_info), ); @@ -2013,9 +2037,14 @@ impl StatementExecutor { &self, tables_data: Vec<(CreateTableExpr, TableInfo)>, query_context: QueryContextRef, + trigger_reason: TriggerReason, ) -> Result { let request = SubmitDdlTaskRequest::new( - to_meta_query_context_with_origin_frontend(query_context, &self.origin_frontend_addr), + to_meta_query_context_with_origin_frontend_and_trigger_reason( + query_context, + &self.origin_frontend_addr, + trigger_reason, + ), DdlTask::new_create_logical_tables(tables_data), ); @@ -2029,9 +2058,12 @@ impl StatementExecutor { &self, tables_data: Vec, query_context: QueryContextRef, + trigger_reason: TriggerReason, ) -> Result { + let meta_query_context = + to_meta_query_context_with_trigger_reason(query_context, trigger_reason); let request = SubmitDdlTaskRequest::new( - to_meta_query_context(query_context), + meta_query_context, DdlTask::new_alter_logical_tables(tables_data), ); @@ -2049,7 +2081,7 @@ impl StatementExecutor { query_context: QueryContextRef, ) -> Result { let request = SubmitDdlTaskRequest::new( - to_meta_query_context(query_context), + to_meta_query_context_with_trigger_reason(query_context, TriggerReason::Manual), DdlTask::new_drop_table( table_name.catalog_name.clone(), table_name.schema_name.clone(), @@ -2073,7 +2105,7 @@ impl StatementExecutor { query_context: QueryContextRef, ) -> Result { let request = SubmitDdlTaskRequest::new( - to_meta_query_context(query_context), + to_meta_query_context_with_trigger_reason(query_context, TriggerReason::Manual), DdlTask::new_drop_database(catalog, schema, drop_if_exists), ); @@ -2089,7 +2121,7 @@ impl StatementExecutor { query_context: QueryContextRef, ) -> Result { let request = SubmitDdlTaskRequest::new( - to_meta_query_context(query_context), + to_meta_query_context_with_trigger_reason(query_context, TriggerReason::Manual), DdlTask::new_alter_database(alter_expr), ); @@ -2106,8 +2138,10 @@ impl StatementExecutor { time_ranges: Vec<(Timestamp, Timestamp)>, query_context: QueryContextRef, ) -> Result { + let meta_query_context = + to_meta_query_context_with_trigger_reason(query_context, TriggerReason::Manual); let request = SubmitDdlTaskRequest::new( - to_meta_query_context(query_context), + meta_query_context, DdlTask::new_truncate_table( table_name.catalog_name.clone(), table_name.schema_name.clone(), @@ -2199,7 +2233,7 @@ impl StatementExecutor { creator: Option, ) -> Result { let request = SubmitDdlTaskRequest::new( - to_meta_query_context(query_context), + to_meta_query_context_with_trigger_reason(query_context, TriggerReason::Manual), DdlTask::new_create_database(catalog, database, create_if_not_exists, options, creator), ); @@ -2734,8 +2768,10 @@ async fn execute_undrop_table( table_name: table_name.to_string(), })?; + let meta_query_context = + to_meta_query_context_with_trigger_reason(query_context, TriggerReason::Manual); let request = SubmitDdlTaskRequest::new( - to_meta_query_context(query_context), + meta_query_context, DdlTask::new_undrop_table(dropped.table_id), ); procedure_executor @@ -2787,7 +2823,9 @@ mod test { ExecutorContext, ProcedureExecutor, ProcedureExecutorRef, }; #[cfg(feature = "enterprise")] - use common_meta::rpc::ddl::{DdlTask, SubmitDdlTaskRequest, SubmitDdlTaskResponse}; + use common_meta::rpc::ddl::{ + DdlTask, SubmitDdlTaskRequest, SubmitDdlTaskResponse, TRIGGER_REASON_EXTENSION_KEY, + }; #[cfg(feature = "enterprise")] use common_meta::rpc::procedure::{ MigrateRegionRequest, MigrateRegionResponse, ProcedureStateResponse, @@ -2932,6 +2970,13 @@ mod test { let requests = procedure.requests.lock().unwrap(); assert!(matches!(&requests[0].task, DdlTask::UndropTable(task) if task.table_id == 42)); + assert_eq!( + requests[0] + .query_context + .extensions + .get(TRIGGER_REASON_EXTENSION_KEY), + Some(&TriggerReason::Manual.as_ref().to_string()) + ); assert_eq!( cache.invalidations.lock().unwrap()[0], vec![CacheIdent::TableId(42), CacheIdent::TableName(name)] diff --git a/src/operator/src/utils.rs b/src/operator/src/utils.rs index fbb760c4fca..03649713b0b 100644 --- a/src/operator/src/utils.rs +++ b/src/operator/src/utils.rs @@ -14,6 +14,9 @@ use std::sync::{Arc, RwLock}; +use common_meta::rpc::ddl::{ + ORIGIN_FRONTEND_ADDR_EXTENSION_KEY, TRIGGER_REASON_EXTENSION_KEY, TriggerReason, +}; use common_time::Timezone; use session::context::{QueryContextBuilder, QueryContextRef}; use snafu::ResultExt; @@ -34,13 +37,28 @@ pub fn to_meta_query_context( } } -pub fn to_meta_query_context_with_origin_frontend( +/// Converts a session query context and records the DDL trigger reason. +pub fn to_meta_query_context_with_trigger_reason( query_context: QueryContextRef, - origin_frontend_addr: &str, + trigger_reason: TriggerReason, ) -> common_meta::rpc::ddl::QueryContext { let mut meta_query_context = to_meta_query_context(query_context); meta_query_context.extensions.insert( - common_meta::rpc::ddl::ORIGIN_FRONTEND_ADDR_EXTENSION_KEY.to_string(), + TRIGGER_REASON_EXTENSION_KEY.to_string(), + trigger_reason.as_ref().to_string(), + ); + meta_query_context +} + +pub fn to_meta_query_context_with_origin_frontend_and_trigger_reason( + query_context: QueryContextRef, + origin_frontend_addr: &str, + trigger_reason: TriggerReason, +) -> common_meta::rpc::ddl::QueryContext { + let mut meta_query_context = + to_meta_query_context_with_trigger_reason(query_context, trigger_reason); + meta_query_context.extensions.insert( + ORIGIN_FRONTEND_ADDR_EXTENSION_KEY.to_string(), origin_frontend_addr.to_string(), ); meta_query_context @@ -69,12 +87,14 @@ mod tests { use std::collections::HashMap; use std::sync::{Arc, RwLock}; - use common_meta::rpc::ddl::ORIGIN_FRONTEND_ADDR_EXTENSION_KEY; + use common_meta::rpc::ddl::{ + ORIGIN_FRONTEND_ADDR_EXTENSION_KEY, TRIGGER_REASON_EXTENSION_KEY, TriggerReason, + }; use common_time::Timezone; use session::context::QueryContextBuilder; use super::{ - to_meta_query_context, to_meta_query_context_with_origin_frontend, + to_meta_query_context, to_meta_query_context_with_origin_frontend_and_trigger_reason, try_to_session_query_context, }; @@ -112,7 +132,11 @@ mod tests { .build(), ); - let meta_ctx = to_meta_query_context_with_origin_frontend(session_ctx, "127.0.0.1:4000"); + let meta_ctx = to_meta_query_context_with_origin_frontend_and_trigger_reason( + session_ctx, + "127.0.0.1:4000", + TriggerReason::Manual, + ); assert_eq!( meta_ctx @@ -121,5 +145,12 @@ mod tests { .map(String::as_str), Some("127.0.0.1:4000") ); + assert_eq!( + meta_ctx + .extensions + .get(TRIGGER_REASON_EXTENSION_KEY) + .map(String::as_str), + Some("manual") + ); } } diff --git a/src/pipeline/src/manager/pipeline_operator.rs b/src/pipeline/src/manager/pipeline_operator.rs index 6c4256db69b..4f60ce4e40f 100644 --- a/src/pipeline/src/manager/pipeline_operator.rs +++ b/src/pipeline/src/manager/pipeline_operator.rs @@ -19,6 +19,7 @@ use std::time::Instant; use api::v1::CreateTableExpr; use catalog::{CatalogManagerRef, RegisterSystemTableRequest}; use common_catalog::consts::{DEFAULT_PRIVATE_SCHEMA_NAME, default_engine}; +use common_meta::rpc::ddl::TriggerReason; use common_telemetry::info; use common_time::FOREVER; use datatypes::timestamp::TimestampNanosecond; @@ -131,7 +132,7 @@ impl PipelineOperator { // create table self.statement_executor - .create_table_inner(&mut expr, None, ctx.clone()) + .create_table_inner(&mut expr, None, ctx.clone(), TriggerReason::AutoCreate) .await .context(CreateTableSnafu)?; diff --git a/src/session/src/context.rs b/src/session/src/context.rs index c554c0955f0..4780523e4b7 100644 --- a/src/session/src/context.rs +++ b/src/session/src/context.rs @@ -618,6 +618,7 @@ impl ConnInfo { } } +/// The protocol mapping for these wire values lives in [`common_session::channel_protocol`]. #[derive(Debug, PartialEq, Default, Clone, Copy)] #[repr(u8)] pub enum Channel { @@ -680,23 +681,7 @@ impl Display for Channel { impl AsRef for Channel { fn as_ref(&self) -> &str { - match self { - Channel::Mysql => "mysql", - Channel::Postgres => "postgres", - Channel::HttpSql => "httpsql", - Channel::Prometheus => "prometheus", - Channel::Otlp => "otlp", - Channel::Grpc => "grpc", - Channel::Influx => "influx", - Channel::Opentsdb => "opentsdb", - Channel::Loki => "loki", - Channel::Elasticsearch => "elasticsearch", - Channel::Jaeger => "jaeger", - Channel::Log => "log", - Channel::Promql => "promql", - Channel::Splunk => "splunk", - Channel::Unknown => "unknown", - } + common_session::channel_protocol(*self as u8).unwrap_or("unknown") } } diff --git a/src/standalone/src/procedure.rs b/src/standalone/src/procedure.rs index ec63b11703b..85973e9f643 100644 --- a/src/standalone/src/procedure.rs +++ b/src/standalone/src/procedure.rs @@ -20,6 +20,7 @@ use common_meta::ddl::DdlContext; use common_meta::ddl_manager::{RepartitionProcedureFactory, RepartitionSource}; use common_meta::key::runtime_switch::RuntimeSwitchManager; use common_meta::kv_backend::KvBackendRef; +use common_meta::rpc::ddl::EventContext; use common_meta::state_store::KvStateStore; use common_procedure::local::{EventRecorderHandle, LocalManager, ManagerConfig}; use common_procedure::options::ProcedureConfig; @@ -73,6 +74,7 @@ impl RepartitionProcedureFactory for StandaloneRepartitionProcedureFactory { _source: RepartitionSource, _to_exprs: Vec, _timeout: Option, + _event_context: EventContext, ) -> std::result::Result { Err(BoxedError::new(NoSupportRepartitionProcedureSnafu.build())) } diff --git a/tests-integration/src/tests/gc.rs b/tests-integration/src/tests/gc.rs index badaef5f7ac..5bc20e2e2cd 100644 --- a/tests-integration/src/tests/gc.rs +++ b/tests-integration/src/tests/gc.rs @@ -17,6 +17,7 @@ use std::sync::Arc; use std::time::Duration; use common_meta::key::TableMetadataManagerRef; +use common_meta::rpc::ddl::EventContext; use common_procedure::ProcedureWithId; use common_telemetry::info; use common_test_util::recordbatch::check_output_stream; @@ -213,6 +214,7 @@ async fn test_gc_basic(store_type: &StorageType) { false, // full_file_listing Duration::from_secs(10), // timeout Default::default(), + EventContext::default(), ); // Submit the procedure to the procedure manager diff --git a/tests-integration/tests/database_ddl_event.rs b/tests-integration/tests/database_ddl_event.rs index ab8d3ee0251..79b741b977f 100644 --- a/tests-integration/tests/database_ddl_event.rs +++ b/tests-integration/tests/database_ddl_event.rs @@ -22,7 +22,7 @@ use tests_integration::cluster::GreptimeDbClusterBuilder; use tests_integration::standalone::GreptimeDbStandaloneBuilder; use tests_integration::test_util::{StorageType, get_test_store_config}; -use crate::event_recorder_test_util::assert_single_event; +use crate::event_recorder_test_util::{assert_single_event, find_eventually_string}; const DATABASE_NAME: &str = "database_ddl_events"; @@ -122,6 +122,15 @@ WHERE type = '{event_type}' AND {submitted_payload_predicate}"# ); assert_single_event(instance, &submitted).await; + let actual = find_eventually_string( + instance, + &format!( + "SELECT json_to_string(event_context) AS event_context FROM greptime_private.events WHERE type = '{event_type}' AND procedure_state = 'Running' AND json_path_match(procedure_trigger, '$.type == \"Submitted\"') ORDER BY timestamp DESC LIMIT 1" + ), + "event_context", + ) + .await; + assert_eq!(r#"{"reason":"manual"}"#, actual); let lifecycle = format!( r#"SELECT count(*) AS event_count diff --git a/tests-integration/tests/flow_ddl_event.rs b/tests-integration/tests/flow_ddl_event.rs index 13ca36898aa..004d8382ed2 100644 --- a/tests-integration/tests/flow_ddl_event.rs +++ b/tests-integration/tests/flow_ddl_event.rs @@ -88,6 +88,7 @@ WHERE type = '{CREATE_FLOW_EVENT_TYPE}' ), ) .await; + assert_event_context(instance, CREATE_FLOW_EVENT_TYPE, &procedure_id).await; assert_single_event( instance, &format!( @@ -106,6 +107,22 @@ WHERE type = '{CREATE_FLOW_EVENT_TYPE}' .await; } +async fn assert_event_context( + instance: &Arc, + event_type: &str, + procedure_id: &str, +) { + let actual = find_eventually_string( + instance, + &format!( + "SELECT json_to_string(event_context) AS event_context FROM greptime_private.events WHERE type = '{event_type}' AND procedure_id = '{procedure_id}' AND json_path_match(procedure_trigger, '$.type == \"Submitted\"')" + ), + "event_context", + ) + .await; + assert_eq!(r#"{"reason":"manual"}"#, actual); +} + async fn assert_drop_events(instance: &Arc, flow: &str) { let procedure_id = find_submitted_procedure_id(instance, DROP_FLOW_EVENT_TYPE, flow).await; assert_single_event( @@ -125,6 +142,7 @@ WHERE type = '{DROP_FLOW_EVENT_TYPE}' ), ) .await; + assert_event_context(instance, DROP_FLOW_EVENT_TYPE, &procedure_id).await; assert_single_event( instance, &format!( diff --git a/tests-integration/tests/gc_event.rs b/tests-integration/tests/gc_event.rs index 3cea55bca78..b62ba1d3649 100644 --- a/tests-integration/tests/gc_event.rs +++ b/tests-integration/tests/gc_event.rs @@ -15,6 +15,7 @@ use std::sync::Arc; use std::time::Duration; +use common_meta::rpc::ddl::{EventContext, TriggerReason}; use common_procedure::{ProcedureId, ProcedureWithId, watcher}; use common_test_util::temp_dir::create_temp_dir; use meta_srv::gc::{BatchGcProcedure, GcSchedulerOptions}; @@ -128,6 +129,7 @@ async fn test_batch_gc_event() { false, Duration::from_secs(10), Default::default(), + EventContext::new(TriggerReason::Manual), ); let procedure_id = ProcedureId::parse_str("00000000-0000-0000-0000-00000000bac0").unwrap(); let mut watcher = cluster @@ -142,6 +144,19 @@ async fn test_batch_gc_event() { watcher::wait(&mut watcher).await.unwrap(); assert_sst_count(&cluster, 1).await; + let submitted = format!( + r#"SELECT json_to_string(event_context) AS event_context +FROM greptime_private.events +WHERE type = 'batch_gc' + AND procedure_id = '{procedure_id}' + AND procedure_state = 'Running' + AND json_get_string(procedure_trigger, 'type') = 'Submitted'"#, + ); + assert_eq!( + r#"{"reason":"manual"}"#, + find_eventually_string(instance, &submitted, "event_context").await + ); + let succeeded = format!( r#"SELECT json_to_string(gc_report) AS gc_report FROM greptime_private.events @@ -149,7 +164,8 @@ WHERE type = 'batch_gc' AND procedure_id = '{procedure_id}' AND procedure_state = 'Done' AND json_path_match(procedure_trigger, '$.type == "Succeeded"') - AND json_is_null(payload)"#, + AND json_is_null(payload) + AND event_context IS NULL"#, ); let actual_report: serde_json::Value = serde_json::from_str(&find_eventually_string(instance, &succeeded, "gc_report").await) diff --git a/tests-integration/tests/http.rs b/tests-integration/tests/http.rs index d524c96328a..2545925206c 100644 --- a/tests-integration/tests/http.rs +++ b/tests-integration/tests/http.rs @@ -2566,6 +2566,15 @@ pub async fn test_prometheus_remote_write(store_type: StorageType) { .await; assert_eq!(res.status(), StatusCode::NO_CONTENT); + // The direct Prom remote-write handler creates tables without going through + // the generic inserter. Its submitted events must retain this origin. + wait_for_event_data( + &client, + "SELECT DISTINCT json_to_string(event_context) FROM greptime_private.events WHERE type IN ('create_table', 'create_logical_tables') AND json_get_string(procedure_trigger, 'type') = 'Submitted' AND json_get_string(event_context, 'protocol') = 'prometheus' AND json_get_string(event_context, 'reason') = 'auto_create'", + r#"[["{\"protocol\":\"prometheus\",\"reason\":\"auto_create\"}"]]"#, + ) + .await; + let expected = "[[\"demo\"],[\"demo_metrics\"],[\"demo_metrics_with_nanos\"],[\"greptime_physical_table\"],[\"metric1\"],[\"metric2\"],[\"metric3\"],[\"mito\"],[\"multi_labels\"],[\"numbers\"],[\"phy\"],[\"phy2\"],[\"phy_ns\"]]"; validate_data("prometheus_remote_write", &client, "show tables;", expected).await; @@ -2608,6 +2617,15 @@ pub async fn test_prometheus_remote_write(store_type: StorageType) { assert_eq!(res.status(), StatusCode::NO_CONTENT); + // Adding a label is handled by PendingRowsSchemaAlterer, which is another + // direct DDL path outside the generic inserter. + wait_for_event_data( + &client, + "SELECT DISTINCT json_to_string(event_context) FROM greptime_private.events WHERE type IN ('alter_table', 'alter_logical_tables') AND json_get_string(procedure_trigger, 'type') = 'Submitted' AND json_get_string(event_context, 'protocol') = 'prometheus' AND json_get_string(event_context, 'reason') = 'auto_alter'", + r#"[["{\"protocol\":\"prometheus\",\"reason\":\"auto_alter\"}"]]"#, + ) + .await; + guard.remove_all().await; } @@ -7122,6 +7140,15 @@ pub async fn test_otlp_traces_v1(store_type: StorageType) { let res = send_trace_v1_req(&client, existing_int_table_name, existing_int_req, false).await; assert_eq!(StatusCode::OK, res.status()); + // Trace schema reconciliation widens the existing column through a direct + // alter-table call, so it must identify itself as an OTLP auto alteration. + wait_for_event_data( + &client, + "SELECT DISTINCT json_to_string(event_context) FROM greptime_private.events WHERE type = 'alter_table' AND json_get_string(procedure_trigger, 'type') = 'Submitted' AND json_get_string(event_context, 'protocol') = 'otlp' AND json_get_string(event_context, 'reason') = 'auto_alter'", + r#"[["{\"protocol\":\"otlp\",\"reason\":\"auto_alter\"}"]]"#, + ) + .await; + validate_data( "otlp_traces_v1_existing_int_widens_rows", &client, @@ -9872,7 +9899,20 @@ async fn validate_data(test_name: &str, client: &TestClient, sql: &str, expected } async fn wait_for_data(client: &TestClient, sql: &str, expected: &str) { - tokio::time::timeout(Duration::from_secs(10), async { + wait_for_data_with_timeout(client, sql, expected, Duration::from_secs(10)).await; +} + +async fn wait_for_event_data(client: &TestClient, sql: &str, expected: &str) { + wait_for_data_with_timeout(client, sql, expected, Duration::from_secs(30)).await; +} + +async fn wait_for_data_with_timeout( + client: &TestClient, + sql: &str, + expected: &str, + timeout: Duration, +) { + tokio::time::timeout(timeout, async { let encoded_sql = encode(sql); loop { let res = client diff --git a/tests-integration/tests/region_migration.rs b/tests-integration/tests/region_migration.rs index 2d11966dc73..3d27dabc3f7 100644 --- a/tests-integration/tests/region_migration.rs +++ b/tests-integration/tests/region_migration.rs @@ -1347,6 +1347,10 @@ async fn check_region_migration_events_system_table( Expr::cust("json_get_string(procedure_trigger, 'type')"), Alias::new("procedure_trigger"), ) + .expr_as( + Expr::cust("json_to_string(event_context)"), + Alias::new("event_context"), + ) .from((RegionMigrationEvents::Schema, RegionMigrationEvents::Table)) .and_where(Expr::col(RegionMigrationEvents::EventType).eq(REGION_MIGRATION_EVENT_TYPE)) .and_where(Expr::col(RegionMigrationEvents::ProcedureId).eq(procedure_id)) @@ -1362,11 +1366,11 @@ async fn check_region_migration_events_system_table( .remove(0); let expected = "\ -+---------------------------------+-----------------+-------------------+ -| region_migration_trigger_reason | procedure_state | procedure_trigger | -+---------------------------------+-----------------+-------------------+ -| Manual | Running | Submitted | -| Manual | Done | Succeeded | -+---------------------------------+-----------------+-------------------+"; ++---------------------------------+-----------------+-------------------+---------------------+ +| region_migration_trigger_reason | procedure_state | procedure_trigger | event_context | ++---------------------------------+-----------------+-------------------+---------------------+ +| Manual | Running | Submitted | {\"reason\":\"manual\"} | +| Manual | Done | Succeeded | | ++---------------------------------+-----------------+-------------------+---------------------+"; check_output_stream(result.unwrap().data, expected).await; } diff --git a/tests-integration/tests/repartition.rs b/tests-integration/tests/repartition.rs index 1990cc2c43d..3dac860066b 100644 --- a/tests-integration/tests/repartition.rs +++ b/tests-integration/tests/repartition.rs @@ -19,6 +19,7 @@ use std::time::Duration; use client::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME}; use common_error::root_source; use common_meta::key::table_name::TableNameKey; +use common_meta::rpc::ddl::EventContext; use common_procedure::{ProcedureWithId, watcher}; use common_query::Output; use common_telemetry::info; @@ -816,6 +817,7 @@ async fn trigger_table_gc(metasrv: &Arc, table_name: &str) { false, // full_file_listing Duration::from_secs(10), // timeout Default::default(), + EventContext::default(), ); // Submit the procedure to the procedure manager diff --git a/tests-integration/tests/table_ddl_event.rs b/tests-integration/tests/table_ddl_event.rs index 73ce21e828a..45d87f30971 100644 --- a/tests-integration/tests/table_ddl_event.rs +++ b/tests-integration/tests/table_ddl_event.rs @@ -16,7 +16,15 @@ use std::sync::Arc; #[cfg(feature = "enterprise")] use std::time::Duration; +use api::v1::ddl_request::Expr as DdlExpr; +use api::v1::greptime_request::Request; +use api::v1::value::ValueData; +use api::v1::{ + AddColumn, AddColumns, AlterTableExpr, ColumnDataType, ColumnDef, CreateTableExpr, DdlRequest, + Row, RowInsertRequest, RowInsertRequests, Rows, SemanticType, alter_table_expr, +}; use client::OutputData; +use common_catalog::consts::MITO_ENGINE; use common_event_recorder::event_table::{ CATALOG_NAME_COLUMN, PAYLOAD_COLUMN, PHYSICAL_TABLE_ID_COLUMN, PROCEDURE_ID_COLUMN, PROCEDURE_TRIGGER_COLUMN, SCHEMA_NAME_COLUMN, TABLE_ID_COLUMN, TABLE_NAME_COLUMN, TYPE_COLUMN, @@ -25,8 +33,9 @@ use common_test_util::temp_dir::create_temp_dir; use frontend::instance::Instance; use meta_srv::gc::GcSchedulerOptions; use mito2::gc::GcConfig; +use servers::query_handler::grpc::GrpcQueryHandler; use servers::query_handler::sql::SqlQueryHandler; -use session::context::QueryContext; +use session::context::{Channel, QueryContext}; use tests_integration::cluster::GreptimeDbClusterBuilder; use crate::event_recorder_test_util::{ @@ -37,6 +46,9 @@ const EVENTS_TABLE: &str = "greptime_private.events"; const TABLE: &str = "table_ddl_events"; const PHYSICAL_TABLE: &str = "table_ddl_events_phy"; const LOGICAL_TABLE: &str = "table_ddl_events_logical"; +const AUTO_TABLE: &str = "table_ddl_events_auto"; +const AUTO_INFLUX_TABLE: &str = "table_ddl_events_auto_influx"; +const GRPC_TABLE: &str = "table_ddl_events_grpc"; #[tokio::test(flavor = "multi_thread")] async fn test_table_ddl_procedure_events() { @@ -70,17 +82,142 @@ async fn test_table_ddl_procedure_events() { .await; let frontend = cluster.fe_instance().clone(); + auto_create_table(&frontend, AUTO_TABLE, Channel::Prometheus).await; + let auto_create_procedure_id = + submitted_procedure_id(&frontend, "create_table", AUTO_TABLE).await; + assert_event_context( + &frontend, + "create_table", + &auto_create_procedure_id, + "auto_create", + "prometheus", + ) + .await; + + auto_create_table(&frontend, AUTO_INFLUX_TABLE, Channel::Influx).await; + let auto_influx_create_procedure_id = + submitted_procedure_id(&frontend, "create_table", AUTO_INFLUX_TABLE).await; + assert_event_context( + &frontend, + "create_table", + &auto_influx_create_procedure_id, + "auto_create", + "influx", + ) + .await; + + // gRPC DDL bypasses the SQL statement executor, so it must set its own + // manual trigger reason before submitting the table procedure. + GrpcQueryHandler::do_query( + frontend.as_ref(), + Request::Ddl(DdlRequest { + expr: Some(DdlExpr::CreateTable(CreateTableExpr { + catalog_name: "greptime".to_string(), + schema_name: "public".to_string(), + table_name: GRPC_TABLE.to_string(), + column_defs: vec![ + ColumnDef { + name: "host".to_string(), + data_type: ColumnDataType::String as i32, + is_nullable: true, + semantic_type: SemanticType::Tag as i32, + ..Default::default() + }, + ColumnDef { + name: "ts".to_string(), + data_type: ColumnDataType::TimestampMillisecond as i32, + is_nullable: false, + semantic_type: SemanticType::Timestamp as i32, + ..Default::default() + }, + ], + time_index: "ts".to_string(), + engine: MITO_ENGINE.to_string(), + ..Default::default() + })), + }), + Arc::new(QueryContext::with_channel( + "greptime", + "public", + Channel::Grpc, + )), + ) + .await + .unwrap(); + let grpc_create_procedure_id = + submitted_procedure_id(&frontend, "create_table", GRPC_TABLE).await; + assert_event_context( + &frontend, + "create_table", + &grpc_create_procedure_id, + "manual", + "grpc", + ) + .await; + GrpcQueryHandler::do_query( + frontend.as_ref(), + Request::Ddl(DdlRequest { + expr: Some(DdlExpr::AlterTable(AlterTableExpr { + catalog_name: "greptime".to_string(), + schema_name: "public".to_string(), + table_name: GRPC_TABLE.to_string(), + kind: Some(alter_table_expr::Kind::AddColumns(AddColumns { + add_columns: vec![AddColumn { + column_def: Some(ColumnDef { + name: "value".to_string(), + data_type: ColumnDataType::Float64 as i32, + is_nullable: true, + semantic_type: SemanticType::Field as i32, + ..Default::default() + }), + ..Default::default() + }], + })), + })), + }), + Arc::new(QueryContext::with_channel( + "greptime", + "public", + Channel::Grpc, + )), + ) + .await + .unwrap(); + let grpc_alter_procedure_id = + submitted_procedure_id(&frontend, "alter_table", GRPC_TABLE).await; + assert_event_context( + &frontend, + "alter_table", + &grpc_alter_procedure_id, + "manual", + "grpc", + ) + .await; + // Act / Assert: Create Table retains its rich submitted row and records the // created table ID when it completes. - run_sql( + run_sql_with_context( &frontend, &format!( "CREATE TABLE {TABLE} (host STRING PRIMARY KEY, ts TIMESTAMP TIME INDEX, val DOUBLE)" ), + Arc::new(QueryContext::with_channel( + "greptime", + "public", + Channel::HttpSql, + )), ) .await; let table_id = find_table_id(&frontend, TABLE).await; let create_table_procedure_id = submitted_procedure_id(&frontend, "create_table", TABLE).await; + assert_event_context( + &frontend, + "create_table", + &create_table_procedure_id, + "manual", + "httpsql", + ) + .await; assert_named_submitted_event( &frontend, "create_table", @@ -110,12 +247,25 @@ async fn test_table_ddl_procedure_events() { // Act / Assert: Alter and Truncate have a rich submitted row and lightweight // terminal lifecycle row. - run_sql( + run_sql_with_context( &frontend, &format!("ALTER TABLE {TABLE} ADD COLUMN extra STRING"), + Arc::new(QueryContext::with_channel( + "greptime", + "public", + Channel::HttpSql, + )), ) .await; let alter_table_procedure_id = submitted_procedure_id(&frontend, "alter_table", TABLE).await; + assert_event_context( + &frontend, + "alter_table", + &alter_table_procedure_id, + "manual", + "httpsql", + ) + .await; assert_named_submitted_event( &frontend, "alter_table", @@ -140,9 +290,26 @@ async fn test_table_ddl_procedure_events() { &format!("INSERT INTO {TABLE} VALUES ('a', 0, 1, 'b')"), ) .await; - run_sql(&frontend, &format!("TRUNCATE TABLE {TABLE}")).await; + run_sql_with_context( + &frontend, + &format!("TRUNCATE TABLE {TABLE}"), + Arc::new(QueryContext::with_channel( + "greptime", + "public", + Channel::HttpSql, + )), + ) + .await; let truncate_table_procedure_id = submitted_procedure_id(&frontend, "truncate_table", TABLE).await; + assert_event_context( + &frontend, + "truncate_table", + &truncate_table_procedure_id, + "manual", + "httpsql", + ) + .await; assert_named_submitted_event( &frontend, "truncate_table", @@ -165,8 +332,25 @@ async fn test_table_ddl_procedure_events() { // Act / Assert: Drop records the table name, while Undrop and Purge use the // table ID as their submitted locator. - run_sql(&frontend, &format!("DROP TABLE {TABLE}")).await; + run_sql_with_context( + &frontend, + &format!("DROP TABLE {TABLE}"), + Arc::new(QueryContext::with_channel( + "greptime", + "public", + Channel::HttpSql, + )), + ) + .await; let drop_table_procedure_id = submitted_procedure_id(&frontend, "drop_table", TABLE).await; + assert_event_context( + &frontend, + "drop_table", + &drop_table_procedure_id, + "manual", + "httpsql", + ) + .await; assert_named_submitted_event( &frontend, "drop_table", @@ -191,10 +375,23 @@ async fn test_table_ddl_procedure_events() { let (undrop_table_procedure_id, _) = cluster .metasrv .ddl_manager() - .submit_undrop_table_task(common_meta::rpc::ddl::UndropTableTask { table_id }) + .submit_undrop_table_task( + common_meta::rpc::ddl::UndropTableTask { table_id }, + common_meta::rpc::ddl::EventContext::new( + common_meta::rpc::ddl::TriggerReason::Manual, + ), + ) .await .unwrap(); let undrop_table_procedure_id = undrop_table_procedure_id.to_string(); + assert_event_context( + &frontend, + "undrop_table", + &undrop_table_procedure_id, + "manual", + "unknown", + ) + .await; assert_id_submitted_event( &frontend, "undrop_table", @@ -218,12 +415,23 @@ async fn test_table_ddl_procedure_events() { let (purge_table_procedure_id, _) = cluster .metasrv .ddl_manager() - .submit_purge_dropped_table_task(common_meta::rpc::ddl::PurgeDroppedTableTask { - table_id, - }) + .submit_purge_dropped_table_task( + common_meta::rpc::ddl::PurgeDroppedTableTask { table_id }, + common_meta::rpc::ddl::EventContext::new( + common_meta::rpc::ddl::TriggerReason::Manual, + ), + ) .await .unwrap(); let purge_table_procedure_id = purge_table_procedure_id.to_string(); + assert_event_context( + &frontend, + "purge_dropped_table", + &purge_table_procedure_id, + "manual", + "unknown", + ) + .await; assert_id_submitted_event( &frontend, "purge_dropped_table", @@ -255,16 +463,29 @@ async fn test_table_ddl_procedure_events() { ) .await; let physical_table_id = find_table_id(&frontend, PHYSICAL_TABLE).await; - run_sql( + run_sql_with_context( &frontend, &format!( "CREATE TABLE {LOGICAL_TABLE} (ts TIMESTAMP TIME INDEX, val DOUBLE, host STRING PRIMARY KEY) ENGINE=metric WITH (\"on_physical_table\" = \"{PHYSICAL_TABLE}\")" ), + Arc::new(QueryContext::with_channel( + "greptime", + "public", + Channel::HttpSql, + )), ) .await; let logical_table_id = find_table_id(&frontend, LOGICAL_TABLE).await; let create_logical_tables_procedure_id = submitted_procedure_id(&frontend, "create_logical_tables", LOGICAL_TABLE).await; + assert_event_context( + &frontend, + "create_logical_tables", + &create_logical_tables_procedure_id, + "manual", + "httpsql", + ) + .await; // Assert: logical Create emits one submitted row per logical table and // preserves the logical and physical table IDs on its terminal row. @@ -298,13 +519,26 @@ async fn test_table_ddl_procedure_events() { // Act / Assert: logical Alter preserves the one-row-per-logical-table submitted // contract and emits a lightweight terminal row. - run_sql( + run_sql_with_context( &frontend, &format!("ALTER TABLE {LOGICAL_TABLE} ADD COLUMN rack STRING PRIMARY KEY"), + Arc::new(QueryContext::with_channel( + "greptime", + "public", + Channel::HttpSql, + )), ) .await; let alter_logical_tables_procedure_id = submitted_procedure_id(&frontend, "alter_logical_tables", LOGICAL_TABLE).await; + assert_event_context( + &frontend, + "alter_logical_tables", + &alter_logical_tables_procedure_id, + "manual", + "httpsql", + ) + .await; assert_named_submitted_event( &frontend, "alter_logical_tables", @@ -327,14 +561,98 @@ async fn test_table_ddl_procedure_events() { } async fn run_sql(instance: &Arc, sql: &str) { - let output = instance - .do_query(sql, QueryContext::arc()) + run_sql_with_context(instance, sql, QueryContext::arc()).await; +} + +async fn auto_create_table(instance: &Arc, table_name: &str, channel: Channel) { + instance + .handle_row_inserts( + RowInsertRequests { + inserts: vec![RowInsertRequest { + table_name: table_name.to_string(), + rows: Some(Rows { + schema: vec![ + api::v1::ColumnSchema { + column_name: "host".to_string(), + datatype: ColumnDataType::String as i32, + semantic_type: SemanticType::Tag as i32, + datatype_extension: None, + options: None, + }, + api::v1::ColumnSchema { + column_name: "ts".to_string(), + datatype: ColumnDataType::TimestampMillisecond as i32, + semantic_type: SemanticType::Timestamp as i32, + datatype_extension: None, + options: None, + }, + api::v1::ColumnSchema { + column_name: "val".to_string(), + datatype: ColumnDataType::Float64 as i32, + semantic_type: SemanticType::Field as i32, + datatype_extension: None, + options: None, + }, + ], + rows: vec![Row { + values: vec![ + ValueData::StringValue("host".to_string()).into(), + ValueData::TimestampMillisecondValue(0).into(), + ValueData::F64Value(1.0).into(), + ], + }], + }), + }], + }, + Arc::new(QueryContext::with_channel("greptime", "public", channel)), + false, + false, + ) + .await + .unwrap(); +} + +async fn run_sql_with_context( + instance: &Arc, + sql: &str, + query_context: Arc, +) { + let output = SqlQueryHandler::do_query(instance.as_ref(), sql, query_context) .await .remove(0) .unwrap(); assert!(matches!(output.data, OutputData::AffectedRows(_)), "{sql}"); } +async fn assert_event_context( + instance: &Arc, + event_type: &str, + procedure_id: &str, + reason: &str, + protocol: &str, +) { + let event_filter = format!( + "FROM {EVENTS_TABLE} WHERE {} = '{event_type}' AND {} = '{procedure_id}' AND json_get_string({}, 'type') = 'Submitted'", + TYPE_COLUMN.name(), + PROCEDURE_ID_COLUMN.name(), + PROCEDURE_TRIGGER_COLUMN.name(), + ); + assert_single_event( + instance, + &format!("SELECT count(*) AS event_count {event_filter}"), + ) + .await; + + let actual = find_eventually_string( + instance, + &format!("SELECT json_to_string(event_context) AS event_context {event_filter}"), + "event_context", + ) + .await; + let expected = format!(r#"{{"protocol":"{protocol}","reason":"{reason}"}}"#); + assert_eq!(expected, actual); +} + async fn find_table_id(instance: &Arc, table_name: &str) -> u32 { find_eventually_u32( instance, diff --git a/tests-integration/tests/view_ddl_event.rs b/tests-integration/tests/view_ddl_event.rs index 2f362c256fe..a25b58b684a 100644 --- a/tests-integration/tests/view_ddl_event.rs +++ b/tests-integration/tests/view_ddl_event.rs @@ -124,6 +124,7 @@ WHERE type = '{CREATE_VIEW_EVENT_TYPE}' ), ) .await; + assert_event_context(instance, CREATE_VIEW_EVENT_TYPE, &procedure_id).await; assert_single_event( instance, &format!( @@ -143,6 +144,22 @@ WHERE type = '{CREATE_VIEW_EVENT_TYPE}' .await; } +async fn assert_event_context( + instance: &Arc, + event_type: &str, + procedure_id: &str, +) { + let actual = find_eventually_string( + instance, + &format!( + "SELECT json_to_string(event_context) AS event_context FROM greptime_private.events WHERE type = '{event_type}' AND procedure_id = '{procedure_id}' AND json_path_match(procedure_trigger, '$.type == \"Submitted\"')" + ), + "event_context", + ) + .await; + assert_eq!(r#"{"reason":"manual"}"#, actual); +} + async fn assert_drop_events(instance: &Arc, view: &str) { let procedure_id = find_submitted_procedure_id(instance, DROP_VIEW_EVENT_TYPE, view).await; assert_single_event( @@ -163,6 +180,7 @@ WHERE type = '{DROP_VIEW_EVENT_TYPE}' ), ) .await; + assert_event_context(instance, DROP_VIEW_EVENT_TYPE, &procedure_id).await; assert_single_event( instance, &format!(